|
|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudAgentCompletionEvent;
|
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
|
use App\Models\CloudOrder;
|
|
|
|
|
|
use App\Models\User;
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
* Converges one dispatched create only from operation-bound absence evidence.
|
|
|
|
|
|
*
|
|
|
|
|
|
* A completion receipt proves its own immutable Agent generation. Read-only
|
|
|
|
|
|
* inspection instead uses a durable PHP prepare marker before the Agent fence
|
|
|
|
|
|
* is released, so a database failure after that release can safely resume.
|
|
|
|
|
|
* Neither path ever sends the original create request again.
|
|
|
|
|
|
*/
|
|
|
|
|
|
final class CloudCreateFailureRecoveryService
|
|
|
|
|
|
{
|
|
|
|
|
|
private const ESCROW_KEY = '_create_failure_recovery';
|
|
|
|
|
|
private const ESCROW_VERSION = 1;
|
|
|
|
|
|
private const MARKER_PREFIX = 'prepared:create_not_applied:v1:';
|
|
|
|
|
|
private const INSPECTION_MAX_AGE_SECONDS = 120;
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
* Settle an exact failed completion, or ACK it as non-authoritative audit
|
|
|
|
|
|
* data so a later fresh host inspection may continue recovery.
|
|
|
|
|
|
*/
|
|
|
|
|
|
public static function settleReceipt(
|
|
|
|
|
|
CloudJob $source,
|
|
|
|
|
|
CloudAgentCompletionEvent $sourceEvent
|
|
|
|
|
|
): string {
|
|
|
|
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
|
|
|
|
static function () use ($source, $sourceEvent): string {
|
|
|
|
|
|
[$user, $order, $job] = self::lockLedger($source);
|
|
|
|
|
|
$event = CloudAgentCompletionEvent::where(
|
|
|
|
|
|
'event_id',
|
|
|
|
|
|
(string) $sourceEvent->event_id
|
|
|
|
|
|
)->lockForUpdate()->first();
|
|
|
|
|
|
if ($event === null) {
|
|
|
|
|
|
return 'received';
|
|
|
|
|
|
}
|
|
|
|
|
|
if ((string) $event->disposition !== 'received') {
|
|
|
|
|
|
return (string) $event->disposition;
|
|
|
|
|
|
}
|
|
|
|
|
|
if ((int) $event->host_id !== (int) $job->host_id
|
|
|
|
|
|
|| !hash_equals(
|
|
|
|
|
|
(string) $job->operation_id,
|
|
|
|
|
|
(string) $event->operation_id
|
|
|
|
|
|
)) {
|
|
|
|
|
|
throw new CloudRecoveryOperationIdentityConflict(
|
|
|
|
|
|
'开通失败回执与原任务不匹配',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
CloudOperationCompletionService::assertEventIdentityForJob(
|
|
|
|
|
|
$job,
|
|
|
|
|
|
$event
|
|
|
|
|
|
);
|
|
|
|
|
|
$operation = CloudOperationCompletionService::operation($event);
|
|
|
|
|
|
if (self::settledNotApplied($job, $order, $event)) {
|
|
|
|
|
|
// A fresh inspection may have completed the atomic PHP
|
|
|
|
|
|
// finalizer before the Agent could deliver its immutable
|
|
|
|
|
|
// failure. The receipt no longer decides the ledger, but
|
|
|
|
|
|
// the exact identity and committed audit allow it to be
|
|
|
|
|
|
// ACKed without reopening or rewriting that resolution.
|
|
|
|
|
|
$event->disposition = 'applied';
|
|
|
|
|
|
$event->applied_at = time();
|
|
|
|
|
|
$event->save();
|
|
|
|
|
|
return 'applied';
|
|
|
|
|
|
}
|
|
|
|
|
|
self::assertRecoverableJob($job);
|
|
|
|
|
|
if (!self::receiptProvesAbsence($event, $operation)) {
|
|
|
|
|
|
$event->disposition = 'late';
|
|
|
|
|
|
$event->applied_at = null;
|
|
|
|
|
|
$event->save();
|
|
|
|
|
|
return 'late';
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
self::assertNoSuccessfulReceiptLocked($job);
|
|
|
|
|
|
self::finalizeLocked(
|
|
|
|
|
|
$job,
|
|
|
|
|
|
$user,
|
|
|
|
|
|
$order,
|
|
|
|
|
|
'late_completion_receipt:' . (string) $event->event_id,
|
|
|
|
|
|
$event
|
|
|
|
|
|
);
|
|
|
|
|
|
return 'applied';
|
|
|
|
|
|
},
|
|
|
|
|
|
3
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Recover one lease-owned create without replaying its Agent mutation. */
|
|
|
|
|
|
public static function recover(CloudJob $claimed): void
|
|
|
|
|
|
{
|
|
|
|
|
|
$job = self::claimedJob($claimed);
|
|
|
|
|
|
$prepared = self::preparedEvidence($job);
|
|
|
|
|
|
if ($prepared !== null) {
|
|
|
|
|
|
if (!$prepared['fence_resolution_required']) {
|
|
|
|
|
|
self::finishPrepared($claimed, $prepared, false);
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
try {
|
|
|
|
|
|
self::finishPrepared($claimed, $prepared, null);
|
|
|
|
|
|
return;
|
|
|
|
|
|
} catch (CloudAgentError $error) {
|
|
|
|
|
|
if ($error->agentHttpStatus !== 404) {
|
|
|
|
|
|
throw $error;
|
|
|
|
|
|
}
|
|
|
|
|
|
// No fence verdict was committed before the prior PHP crash.
|
|
|
|
|
|
// A new write requires new, bounded read-only evidence.
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$inspection = CloudAdminOperationRecoveryService::inspectForRecovery(
|
|
|
|
|
|
(string) $job->id
|
|
|
|
|
|
);
|
|
|
|
|
|
$prepared = self::prepareInspection($claimed, $inspection);
|
|
|
|
|
|
self::finishPrepared(
|
|
|
|
|
|
$claimed,
|
|
|
|
|
|
$prepared,
|
|
|
|
|
|
$prepared['fence_resolution_required'] ? $inspection : false
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
* Authenticate the durable PHP intent before an automatic Agent fence read
|
|
|
|
|
|
* or write. This is deliberately create-specific and does not broaden the
|
|
|
|
|
|
* generic recovery allow-list.
|
|
|
|
|
|
*/
|
|
|
|
|
|
public static function assertPreparedForFence(
|
|
|
|
|
|
CloudJob $job,
|
|
|
|
|
|
string $resolution,
|
|
|
|
|
|
string $reason
|
|
|
|
|
|
): void {
|
|
|
|
|
|
if ($resolution !== 'not_applied') {
|
|
|
|
|
|
throw new CloudError('开通任务只支持确认未应用', 409);
|
|
|
|
|
|
}
|
|
|
|
|
|
self::assertRecoverableJob($job);
|
|
|
|
|
|
$prepared = self::preparedEvidence($job);
|
|
|
|
|
|
if ($prepared === null
|
|
|
|
|
|
|| !hash_equals($prepared['reason'], $reason)) {
|
|
|
|
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
|
|
|
|
'开通任务的恢复预提交证据不匹配',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
* @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}
|
|
|
|
|
|
*/
|
|
|
|
|
|
private static function prepareInspection(
|
|
|
|
|
|
CloudJob $claimed,
|
|
|
|
|
|
array $inspection
|
|
|
|
|
|
): array {
|
|
|
|
|
|
$job = self::claimedJob($claimed);
|
|
|
|
|
|
$identity = CloudOperationRecoveryIdentity::expected($job, false);
|
|
|
|
|
|
CloudOperationRecoveryIdentity::assertMatches($identity, [
|
|
|
|
|
|
'action' => $inspection['action'] ?? null,
|
|
|
|
|
|
'resource_id' => $inspection['resource_id'] ?? null,
|
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
|
$inspection['request_fingerprint'] ?? null,
|
|
|
|
|
|
]);
|
|
|
|
|
|
self::assertInspectionProvesAbsence($job, $inspection);
|
|
|
|
|
|
$evidence = self::inspectionEvidence($inspection);
|
|
|
|
|
|
$marker = self::inspectionMarker($evidence);
|
|
|
|
|
|
$reason = self::fenceReason($job, $marker);
|
|
|
|
|
|
|
|
|
|
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
|
|
|
|
static function () use (
|
|
|
|
|
|
$claimed,
|
|
|
|
|
|
$identity,
|
|
|
|
|
|
$evidence,
|
|
|
|
|
|
$marker,
|
|
|
|
|
|
$reason
|
|
|
|
|
|
): array {
|
|
|
|
|
|
[$user, $order, $locked] = self::lockLedger($claimed);
|
|
|
|
|
|
self::assertClaimedJob($locked, $claimed);
|
|
|
|
|
|
self::assertNoSuccessfulReceiptLocked($locked);
|
|
|
|
|
|
|
|
|
|
|
|
$payload = self::payload($locked);
|
|
|
|
|
|
$lockedIdentity = CloudOperationRecoveryIdentity::
|
|
|
|
|
|
expectedFromPayload($locked, $payload, false);
|
|
|
|
|
|
CloudOperationRecoveryIdentity::assertMatches(
|
|
|
|
|
|
$lockedIdentity,
|
|
|
|
|
|
$identity
|
|
|
|
|
|
);
|
|
|
|
|
|
$payload[self::ESCROW_KEY] = [
|
|
|
|
|
|
'version' => self::ESCROW_VERSION,
|
|
|
|
|
|
'marker' => $marker,
|
|
|
|
|
|
'reason' => $reason,
|
|
|
|
|
|
'identity' => CloudOperationRecoveryIdentity::persisted(
|
|
|
|
|
|
$lockedIdentity
|
|
|
|
|
|
),
|
|
|
|
|
|
'inspection' => $evidence,
|
|
|
|
|
|
];
|
|
|
|
|
|
$encoded = json_encode(
|
|
|
|
|
|
$payload,
|
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
|
);
|
|
|
|
|
|
if (strlen($encoded) > 196608) {
|
|
|
|
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
|
|
|
|
'开通任务恢复预提交数据过大',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
$locked->effect_resolution_reason = $marker;
|
|
|
|
|
|
$locked->sealed_payload = CloudSecrets::encrypt(
|
|
|
|
|
|
$payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$locked->updated_at = time();
|
|
|
|
|
|
$locked->save();
|
|
|
|
|
|
|
|
|
|
|
|
return [
|
|
|
|
|
|
'marker' => $marker,
|
|
|
|
|
|
'reason' => $reason,
|
|
|
|
|
|
'identity' => $lockedIdentity,
|
|
|
|
|
|
'inspection' => $evidence,
|
|
|
|
|
|
'fence_resolution_required' => (bool)
|
|
|
|
|
|
$evidence['fence_resolution_required'],
|
|
|
|
|
|
];
|
|
|
|
|
|
},
|
|
|
|
|
|
3
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
* Resolve or verify the Agent fence while holding the canonical financial
|
|
|
|
|
|
* locks, then commit every PHP ledger transition in that same transaction.
|
|
|
|
|
|
*
|
|
|
|
|
|
* @param array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} $prepared
|
|
|
|
|
|
* @param array|false|null $inspection Fresh evidence, false when no fence
|
|
|
|
|
|
* is required, null when resuming an already committed fence resolution.
|
|
|
|
|
|
*/
|
|
|
|
|
|
private static function finishPrepared(
|
|
|
|
|
|
CloudJob $claimed,
|
|
|
|
|
|
array $prepared,
|
|
|
|
|
|
array|false|null $inspection
|
|
|
|
|
|
): void {
|
|
|
|
|
|
(new CloudJob())->getConnection()->transaction(
|
|
|
|
|
|
static function () use ($claimed, $prepared, $inspection): void {
|
|
|
|
|
|
[$user, $order, $job] = self::lockLedger($claimed);
|
|
|
|
|
|
self::assertClaimedJob($job, $claimed);
|
|
|
|
|
|
$lockedPrepared = self::preparedEvidence($job);
|
|
|
|
|
|
if ($lockedPrepared === null
|
|
|
|
|
|
|| !hash_equals(
|
|
|
|
|
|
$prepared['marker'],
|
|
|
|
|
|
$lockedPrepared['marker']
|
|
|
|
|
|
)
|
|
|
|
|
|
|| !hash_equals(
|
|
|
|
|
|
$prepared['reason'],
|
|
|
|
|
|
$lockedPrepared['reason']
|
|
|
|
|
|
)) {
|
|
|
|
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
|
|
|
|
'开通任务的恢复预提交已变化',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
self::assertNoSuccessfulReceiptLocked($job);
|
|
|
|
|
|
|
|
|
|
|
|
if ($inspection === null) {
|
|
|
|
|
|
CloudAdminOperationRecoveryService::
|
|
|
|
|
|
preparedFenceResolutionForRecovery(
|
|
|
|
|
|
(string) $job->id,
|
|
|
|
|
|
'not_applied',
|
|
|
|
|
|
$lockedPrepared['reason']
|
|
|
|
|
|
);
|
|
|
|
|
|
} elseif ($inspection !== false) {
|
|
|
|
|
|
CloudAdminOperationRecoveryService::resolveFenceForRecovery(
|
|
|
|
|
|
(string) $job->id,
|
|
|
|
|
|
'not_applied',
|
|
|
|
|
|
$lockedPrepared['reason'],
|
|
|
|
|
|
$inspection
|
|
|
|
|
|
);
|
|
|
|
|
|
} elseif ($lockedPrepared['fence_resolution_required']) {
|
|
|
|
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
|
|
|
|
'开通任务仍需解除宿主机资源栅栏',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
self::assertNoSuccessfulReceiptLocked($job);
|
|
|
|
|
|
self::finalizeLocked(
|
|
|
|
|
|
$job,
|
|
|
|
|
|
$user,
|
|
|
|
|
|
$order,
|
|
|
|
|
|
'automatic_create_not_applied:'
|
|
|
|
|
|
. substr($lockedPrepared['marker'], -64)
|
|
|
|
|
|
);
|
|
|
|
|
|
},
|
|
|
|
|
|
3
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
* @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}|null
|
|
|
|
|
|
*/
|
|
|
|
|
|
private static function preparedEvidence(CloudJob $job): ?array
|
|
|
|
|
|
{
|
|
|
|
|
|
$marker = (string) ($job->effect_resolution_reason ?? '');
|
|
|
|
|
|
if (preg_match(
|
|
|
|
|
|
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
|
|
|
|
|
|
$marker
|
|
|
|
|
|
) !== 1) {
|
|
|
|
|
|
return null;
|
|
|
|
|
|
}
|
|
|
|
|
|
$payload = self::payload($job);
|
|
|
|
|
|
$value = $payload[self::ESCROW_KEY] ?? null;
|
|
|
|
|
|
$identity = is_array($value) ? ($value['identity'] ?? null) : null;
|
|
|
|
|
|
$inspection = is_array($value) ? ($value['inspection'] ?? null) : null;
|
|
|
|
|
|
if (!is_array($value) || array_is_list($value)
|
|
|
|
|
|
|| array_diff(array_keys($value), [
|
|
|
|
|
|
'version', 'marker', 'reason', 'identity', 'inspection',
|
|
|
|
|
|
]) !== []
|
|
|
|
|
|
|| ($value['version'] ?? null) !== self::ESCROW_VERSION
|
|
|
|
|
|
|| !is_string($value['marker'] ?? null)
|
|
|
|
|
|
|| !hash_equals($marker, $value['marker'])
|
|
|
|
|
|
|| !is_string($value['reason'] ?? null)
|
|
|
|
|
|
|| mb_strlen($value['reason']) < 20
|
|
|
|
|
|
|| mb_strlen($value['reason']) > 500
|
|
|
|
|
|
|| !is_array($identity) || array_is_list($identity)
|
|
|
|
|
|
|| !is_array($inspection) || array_is_list($inspection)) {
|
|
|
|
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
|
|
|
|
'开通任务恢复预提交数据无效',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
$expected = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
|
$job,
|
|
|
|
|
|
$payload,
|
|
|
|
|
|
false
|
|
|
|
|
|
);
|
|
|
|
|
|
CloudOperationRecoveryIdentity::assertMatches($expected, $identity);
|
|
|
|
|
|
self::assertInspectionProvesAbsence($job, $inspection, false);
|
|
|
|
|
|
CloudOperationRecoveryIdentity::assertMatches($expected, [
|
|
|
|
|
|
'action' => $inspection['action'] ?? null,
|
|
|
|
|
|
'resource_id' => $inspection['resource_id'] ?? null,
|
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
|
$inspection['request_fingerprint'] ?? null,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$evidence = self::inspectionEvidence($inspection);
|
|
|
|
|
|
$expectedMarker = self::inspectionMarker($evidence);
|
|
|
|
|
|
if (!hash_equals($expectedMarker, $marker)
|
|
|
|
|
|
|| !hash_equals(self::fenceReason($job, $marker), $value['reason'])) {
|
|
|
|
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
|
|
|
|
'开通任务恢复预提交摘要不匹配',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
return [
|
|
|
|
|
|
'marker' => $marker,
|
|
|
|
|
|
'reason' => $value['reason'],
|
|
|
|
|
|
'identity' => $expected,
|
|
|
|
|
|
'inspection' => $evidence,
|
|
|
|
|
|
'fence_resolution_required' =>
|
|
|
|
|
|
(bool) $evidence['fence_resolution_required'],
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function assertInspectionProvesAbsence(
|
|
|
|
|
|
CloudJob $job,
|
|
|
|
|
|
array $inspection,
|
|
|
|
|
|
bool $requireFresh = true
|
|
|
|
|
|
): void {
|
|
|
|
|
|
$checkedAt = $inspection['checked_at'] ?? null;
|
|
|
|
|
|
$projection = $inspection['effect_projection'] ?? null;
|
|
|
|
|
|
if (($inspection['operation_id'] ?? null)
|
|
|
|
|
|
!== (string) $job->operation_id
|
|
|
|
|
|
|| ($inspection['journal_status'] ?? null) !== 'failed'
|
|
|
|
|
|
|| ($inspection['outcome'] ?? null) !== 'failed'
|
|
|
|
|
|
|| ($inspection['summary_code'] ?? null) !== 'operation_failed'
|
|
|
|
|
|
|| ($inspection['effect_resolution'] ?? null) !== 'not_applied'
|
|
|
|
|
|
|| !is_int($checkedAt)
|
|
|
|
|
|
|| ($requireFresh && ($checkedAt < time()
|
|
|
|
|
|
- self::INSPECTION_MAX_AGE_SECONDS
|
|
|
|
|
|
|| $checkedAt > time() + 5))
|
|
|
|
|
|
|| !is_array($projection) || array_is_list($projection)
|
|
|
|
|
|
|| ($projection['creation_absent'] ?? null) !== true
|
|
|
|
|
|
|| ($projection['managed_artifacts_absent'] ?? null) !== true
|
|
|
|
|
|
|| !in_array(
|
|
|
|
|
|
$inspection['fence_state'] ?? null,
|
|
|
|
|
|
['not_required', 'active'],
|
|
|
|
|
|
true
|
|
|
|
|
|
)
|
|
|
|
|
|
|| !is_bool($inspection['fence_resolution_required'] ?? null)
|
|
|
|
|
|
|| (bool) $inspection['fence_resolution_required']
|
|
|
|
|
|
!== (($inspection['fence_state'] ?? null) === 'active')
|
|
|
|
|
|
|| array_key_exists('completion_delivery_state', $inspection)) {
|
|
|
|
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
|
|
|
|
'宿主机检查没有同时确认虚拟机及面板管理资源均不存在',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
CloudCreateFailureFinalizer::assertResolutionSupported(
|
|
|
|
|
|
$job,
|
|
|
|
|
|
'not_applied',
|
|
|
|
|
|
$inspection
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function receiptProvesAbsence(
|
|
|
|
|
|
CloudAgentCompletionEvent $event,
|
|
|
|
|
|
array $operation
|
|
|
|
|
|
): bool {
|
|
|
|
|
|
$result = $operation['result'] ?? null;
|
|
|
|
|
|
return (string) $event->outcome === 'failed'
|
|
|
|
|
|
&& !(bool) $event->effect_may_have_applied
|
|
|
|
|
|
&& is_array($result) && !array_is_list($result)
|
|
|
|
|
|
&& ($result['creation_absent'] ?? null) === true
|
|
|
|
|
|
&& ($result['managed_artifacts_absent'] ?? null) === true;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** @return array{0:User,1:CloudOrder,2:CloudJob} */
|
|
|
|
|
|
private static function lockLedger(CloudJob $source): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$user = User::where('id', (int) $source->user_id)
|
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
|
$order = CloudOrder::where('id', (int) $source->order_id)
|
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
|
$job = CloudJob::where('id', (int) $source->id)
|
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
|
if ($user === null || $order === null || $job === null
|
|
|
|
|
|
|| (int) $job->user_id !== (int) $user->id
|
|
|
|
|
|
|| (int) $job->order_id !== (int) $order->id
|
|
|
|
|
|
|| (int) $order->user_id !== (int) $user->id) {
|
|
|
|
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
|
|
|
|
'开通任务与资金账本关联不一致',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
return [$user, $order, $job];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function assertRecoverableJob(CloudJob $job): void
|
|
|
|
|
|
{
|
|
|
|
|
|
self::assertDispatchedEnvelope($job);
|
|
|
|
|
|
if ((string) $job->kind !== 'create'
|
|
|
|
|
|
|| $job->order_id === null
|
|
|
|
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|
|
|
|
|| (string) $job->phase !== 'terminal'
|
|
|
|
|
|
|| (string) $job->outcome !== 'failed'
|
|
|
|
|
|
|| (string) $job->status !== 'failed'
|
|
|
|
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|
|
|
|
|| (bool) $job->safe_to_resubmit
|
|
|
|
|
|
|| (string) $job->effect_resolution !== 'unknown') {
|
|
|
|
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
|
|
|
|
'开通任务不处于可自动核对状态',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function settledNotApplied(
|
|
|
|
|
|
CloudJob $job,
|
|
|
|
|
|
CloudOrder $order,
|
|
|
|
|
|
CloudAgentCompletionEvent $event
|
|
|
|
|
|
): bool {
|
|
|
|
|
|
self::assertDispatchedEnvelope($job);
|
|
|
|
|
|
$reason = $job->effect_resolution_reason;
|
|
|
|
|
|
$completionEventId = $job->completion_event_id;
|
|
|
|
|
|
return (string) $event->outcome === 'failed'
|
|
|
|
|
|
&& (string) $job->kind === 'create'
|
|
|
|
|
|
&& $job->order_id !== null
|
|
|
|
|
|
&& (string) $job->execution_mode === 'agent'
|
|
|
|
|
|
&& (string) $job->phase === 'terminal'
|
|
|
|
|
|
&& (string) $job->outcome === 'failed'
|
|
|
|
|
|
&& (string) $job->status === 'failed'
|
|
|
|
|
|
&& (int) $job->dispatch_attempts === 1
|
|
|
|
|
|
&& !(bool) $job->safe_to_resubmit
|
|
|
|
|
|
&& (string) $job->effect_resolution === 'not_applied'
|
|
|
|
|
|
&& (int) ($job->effect_resolved_at ?? 0) > 0
|
|
|
|
|
|
&& is_string($reason)
|
|
|
|
|
|
&& mb_strlen($reason) >= 20
|
|
|
|
|
|
&& mb_strlen($reason) <= 500
|
|
|
|
|
|
&& !(bool) $job->finalization_incident
|
|
|
|
|
|
&& $job->finalization_error_code === null
|
|
|
|
|
|
&& $job->instance_id === null
|
|
|
|
|
|
&& $order->instance_id === null
|
|
|
|
|
|
&& (string) $order->status === 'failed'
|
|
|
|
|
|
&& $order->refunded_at !== null
|
|
|
|
|
|
&& ($completionEventId === null
|
|
|
|
|
|
|| (is_string($completionEventId)
|
|
|
|
|
|
&& hash_equals(
|
|
|
|
|
|
$completionEventId,
|
|
|
|
|
|
(string) $event->event_id
|
|
|
|
|
|
)));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Validate create's immutable generation without broadening policy. */
|
|
|
|
|
|
private static function assertDispatchedEnvelope(CloudJob $job): void
|
|
|
|
|
|
{
|
|
|
|
|
|
CloudDispatchLedger::assertDispatched($job);
|
|
|
|
|
|
if ((string) $job->attempts !== '1'
|
|
|
|
|
|
|| (int) ($job->host_id ?? 0) < 1
|
|
|
|
|
|
|| (int) ($job->dispatch_deadline_at ?? 0) < 1
|
|
|
|
|
|
|| (int) ($job->dispatch_started_at ?? 0) < 1
|
|
|
|
|
|
|| (($job->agent_accepted_at ?? null) !== null
|
|
|
|
|
|
&& (int) $job->agent_accepted_at < 1)
|
|
|
|
|
|
|| (int) ($job->dispatched_at ?? 0) < 1
|
|
|
|
|
|
|| (int) ($job->receipt_deadline_at ?? 0) < 1
|
|
|
|
|
|
|| (int) ($job->terminal_at ?? 0) < 1
|
|
|
|
|
|
|| preg_match(
|
|
|
|
|
|
'/^[a-f0-9]{32}$/D',
|
|
|
|
|
|
(string) ($job->operation_id ?? '')
|
|
|
|
|
|
) !== 1
|
|
|
|
|
|
|| preg_match(
|
|
|
|
|
|
'/^[a-f0-9]{64}$/D',
|
|
|
|
|
|
(string) ($job->request_fingerprint ?? '')
|
|
|
|
|
|
) !== 1
|
|
|
|
|
|
|| preg_match(
|
|
|
|
|
|
'/^[A-Za-z0-9_.-]{16,128}$/D',
|
|
|
|
|
|
(string) ($job->request_idempotency_key ?? '')
|
|
|
|
|
|
) !== 1) {
|
|
|
|
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
|
|
|
|
'开通任务单次派发信封不完整',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function assertClaimedJob(
|
|
|
|
|
|
CloudJob $job,
|
|
|
|
|
|
CloudJob $claimed
|
|
|
|
|
|
): void {
|
|
|
|
|
|
self::assertRecoverableJob($job);
|
|
|
|
|
|
if (!is_string($claimed->lease_token)
|
|
|
|
|
|
|| !is_string($job->lease_token)
|
|
|
|
|
|
|| !hash_equals(
|
|
|
|
|
|
(string) $claimed->lease_token,
|
|
|
|
|
|
(string) $job->lease_token
|
|
|
|
|
|
)
|
|
|
|
|
|
|| (int) $job->lease_until <= time()) {
|
|
|
|
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
|
|
|
|
'开通任务自动核对租约已失效',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function claimedJob(CloudJob $claimed): CloudJob
|
|
|
|
|
|
{
|
|
|
|
|
|
$job = CloudJob::find((int) $claimed->id);
|
|
|
|
|
|
if ($job === null) {
|
|
|
|
|
|
throw new CloudError('任务不存在', 404);
|
|
|
|
|
|
}
|
|
|
|
|
|
self::assertClaimedJob($job, $claimed);
|
|
|
|
|
|
return $job;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function assertNoSuccessfulReceiptLocked(CloudJob $job): void
|
|
|
|
|
|
{
|
|
|
|
|
|
$event = CloudAgentCompletionEvent::where(
|
|
|
|
|
|
'host_id',
|
|
|
|
|
|
(int) $job->host_id
|
|
|
|
|
|
)->where(
|
|
|
|
|
|
'operation_id',
|
|
|
|
|
|
(string) $job->operation_id
|
|
|
|
|
|
)->where('outcome', 'succeeded')->lockForUpdate()->first();
|
|
|
|
|
|
if ($event === null) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
$exact = CloudOperationCompletionService::isExactStoredEvent($event)
|
|
|
|
|
|
|| CloudOperationCompletionService::isExactAppliedEvent($event);
|
|
|
|
|
|
if (!$exact) {
|
|
|
|
|
|
throw new CloudRecoveryOperationIdentityConflict(
|
|
|
|
|
|
'开通成功回执状态异常,不能执行退款',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
$payload = self::payload($job);
|
|
|
|
|
|
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
|
$job,
|
|
|
|
|
|
$payload,
|
|
|
|
|
|
true
|
|
|
|
|
|
);
|
|
|
|
|
|
if (CloudOperationRecoveryIdentity::same(
|
|
|
|
|
|
$identity,
|
|
|
|
|
|
CloudOperationCompletionService::identity($event)
|
|
|
|
|
|
)) {
|
|
|
|
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
|
|
|
|
'已收到匹配的实例创建成功回执,不能按未应用退款',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
throw new CloudRecoveryOperationIdentityConflict(
|
|
|
|
|
|
'开通成功回执身份冲突,不能执行退款',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function finalizeLocked(
|
|
|
|
|
|
CloudJob $job,
|
|
|
|
|
|
User $user,
|
|
|
|
|
|
CloudOrder $order,
|
|
|
|
|
|
string $resolutionReason,
|
|
|
|
|
|
?CloudAgentCompletionEvent $event = null
|
|
|
|
|
|
): void {
|
|
|
|
|
|
$payload = self::payload($job);
|
|
|
|
|
|
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
|
$job,
|
|
|
|
|
|
$payload,
|
|
|
|
|
|
false
|
|
|
|
|
|
);
|
|
|
|
|
|
CloudCreateFailureFinalizer::refundAndReleaseLocked(
|
|
|
|
|
|
$job,
|
|
|
|
|
|
$user,
|
|
|
|
|
|
$order,
|
|
|
|
|
|
'create_effect_not_applied'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
$safePayload = CloudInput::safeResult($payload);
|
|
|
|
|
|
unset($safePayload[self::ESCROW_KEY]);
|
|
|
|
|
|
$safePayload[CloudOperationRecoveryIdentity::payloadKey()] =
|
|
|
|
|
|
CloudOperationRecoveryIdentity::persisted($identity);
|
|
|
|
|
|
$now = time();
|
|
|
|
|
|
$job->effect_resolution = 'not_applied';
|
|
|
|
|
|
$job->effect_resolved_at = $now;
|
|
|
|
|
|
$job->effect_resolved_by = null;
|
|
|
|
|
|
$job->effect_resolution_reason = $resolutionReason;
|
|
|
|
|
|
if ($event !== null) {
|
|
|
|
|
|
$job->completion_event_id = (string) $event->event_id;
|
|
|
|
|
|
}
|
|
|
|
|
|
$job->sealed_payload = CloudSecrets::encrypt(
|
|
|
|
|
|
$safePayload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$job->finalization_incident = 0;
|
|
|
|
|
|
$job->finalization_error_code = null;
|
|
|
|
|
|
$job->read_at = null;
|
|
|
|
|
|
$job->lease_token = null;
|
|
|
|
|
|
$job->lease_until = 0;
|
|
|
|
|
|
$job->available_at = 0;
|
|
|
|
|
|
$job->updated_at = $now;
|
|
|
|
|
|
$job->save();
|
|
|
|
|
|
if ($event !== null) {
|
|
|
|
|
|
$event->disposition = 'applied';
|
|
|
|
|
|
$event->applied_at = $now;
|
|
|
|
|
|
$event->save();
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function payload(CloudJob $job): array
|
|
|
|
|
|
{
|
|
|
|
|
|
try {
|
|
|
|
|
|
$payload = CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $job->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
|
|
|
|
'开通任务恢复数据无法解密',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!is_array($payload) || array_is_list($payload)) {
|
|
|
|
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
|
|
|
|
'开通任务恢复数据结构不完整',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
return $payload;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function fenceReason(CloudJob $job, string $marker): string
|
|
|
|
|
|
{
|
|
|
|
|
|
return '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id='
|
|
|
|
|
|
. (string) $job->operation_id . ';evidence='
|
|
|
|
|
|
. substr($marker, -64);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Hash only stable evidence so a lost fence response can be resumed. */
|
|
|
|
|
|
private static function inspectionMarker(array $evidence): string
|
|
|
|
|
|
{
|
|
|
|
|
|
unset($evidence['checked_at']);
|
|
|
|
|
|
return self::MARKER_PREFIX . hash(
|
|
|
|
|
|
'sha256',
|
|
|
|
|
|
json_encode(
|
|
|
|
|
|
$evidence,
|
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
|
)
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Return the bounded, canonical evidence persisted in encrypted escrow. */
|
|
|
|
|
|
private static function inspectionEvidence(array $inspection): array
|
|
|
|
|
|
{
|
|
|
|
|
|
return [
|
|
|
|
|
|
'operation_id' => $inspection['operation_id'],
|
|
|
|
|
|
'action' => $inspection['action'],
|
|
|
|
|
|
'resource_id' => $inspection['resource_id'],
|
|
|
|
|
|
'request_fingerprint' => $inspection['request_fingerprint'],
|
|
|
|
|
|
'journal_status' => $inspection['journal_status'],
|
|
|
|
|
|
'checked_at' => $inspection['checked_at'],
|
|
|
|
|
|
'outcome' => $inspection['outcome'],
|
|
|
|
|
|
'summary_code' => $inspection['summary_code'],
|
|
|
|
|
|
'error_code' => $inspection['error_code'] ?? '',
|
|
|
|
|
|
'effect_resolution' => $inspection['effect_resolution'],
|
|
|
|
|
|
'fence_state' => $inspection['fence_state'],
|
|
|
|
|
|
'fence_resolution_required' =>
|
|
|
|
|
|
$inspection['fence_resolution_required'],
|
|
|
|
|
|
'effect_projection' => [
|
|
|
|
|
|
'creation_absent' => true,
|
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|