You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudPortableRestoreFinaliz...

1089 lines
47 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudBackupRestore;
use App\Models\CloudBackupRestoreAttempt;
use App\Models\CloudHost;
use App\Models\CloudInstance;
use App\Models\CloudInstanceBackup;
use App\Models\CloudJob;
use App\Models\CloudResourceReservation;
use App\Models\User;
/** Durable evidence and PHP-only settlement for independent backup restores. */
final class CloudPortableRestoreFinalizer
{
public static function owns(CloudJob $job): bool
{
return (string) $job->kind === 'backup_restore'
&& CloudBackupRestoreAttempt::where('job_id', (int) $job->id)
->exists();
}
public static function markDispatched(CloudJob $job): void
{
if (!self::owns($job)) {
return;
}
$changed = CloudBackupRestoreAttempt::where('job_id', (int) $job->id)
->where('operation_id', (string) $job->operation_id)
->where('state', 'queued')
->where('effect_resolution', 'not_applicable')
->where('active_attempt', 1)
->update(['state' => 'awaiting_receipt', 'updated_at' => time()]);
if ($changed === 1) {
return;
}
$attempt = CloudBackupRestoreAttempt::where(
'job_id', (int) $job->id
)->first();
if (!$attempt instanceof CloudBackupRestoreAttempt
|| !hash_equals((string) $attempt->operation_id,
(string) $job->operation_id)
|| !in_array((string) $attempt->state, [
'awaiting_receipt', 'effect_unknown',
'finalization_pending', 'succeeded', 'failed',
], true)) {
throw new CloudError('恢复尝试派发状态不一致', 409);
}
}
/**
* Persist the validated Agent conclusion before the larger Worker
* finalization transaction. A later PHP-only pass can then finish without
* replaying the host mutation.
*/
public static function recordEvidence(CloudJob $snapshot, string $status,
mixed $result, array $payload): void
{
if (!self::owns($snapshot)) {
return;
}
self::assertStatus($status);
$snapshot->getConnection()->transaction(static function () use (
$snapshot, $status, $result, $payload
): void {
$user = User::where('id', (int) $snapshot->user_id)
->lockForUpdate()->first();
$job = CloudJob::where('id', (int) $snapshot->id)
->lockForUpdate()->first();
if (!$user instanceof User || !$job instanceof CloudJob
|| (int) $job->user_id !== (int) $user->id) {
throw new CloudError('恢复任务账户或作业不存在', 409);
}
$context = self::lockContext($job, $payload);
self::recordEvidenceLocked($context, $status, $result);
}, 3);
}
/** Apply one persisted conclusion inside CloudWorkerService's lock. */
public static function finishJobLocked(CloudJob $job, string $status,
mixed $result, array $payload, bool $deferPublish = false): void
{
self::assertStatus($status);
if (!self::owns($job)) {
throw new CloudError('恢复任务不属于尝试账本', 409);
}
if ($job->getConnection()->transactionLevel() < 1) {
throw new \LogicException(
'Portable restore finalization requires a transaction.'
);
}
$context = self::lockContext($job, $payload);
$attempt = $context['attempt'];
if ((string) $attempt->evidence_resolution === 'not_applicable'
|| ((string) $attempt->evidence_resolution === 'unknown'
&& $status !== 'uncertain')) {
self::recordEvidenceLocked($context, $status, $result);
$context = self::lockContext($job, $payload);
$attempt = $context['attempt'];
}
self::assertMatchingEvidence($context, $status, $result);
if ((string) $attempt->state === 'succeeded') {
self::verifySucceeded($context);
return;
}
if ((string) $attempt->state === 'failed') {
self::verifyFailed($context);
return;
}
if ((string) $attempt->state === 'effect_unknown') {
self::verifyUnknown($context);
return;
}
if ((string) $attempt->state !== 'finalization_pending') {
throw new CloudError('恢复尝试没有可结算的持久证据', 409);
}
if ($deferPublish) {
$attempt->last_error_code = 'recovery_prepare_pending';
$attempt->finalization_error_code = 'recovery_prepare_pending';
$attempt->updated_at = time();
$attempt->save();
return;
}
if ($status === 'succeeded') {
self::settleSucceeded($context);
} elseif ($status === 'failed') {
self::settleFailed($context);
} else {
self::verifyUnknown($context);
}
}
/** Publish a prepared projection only after the Agent fence is resolved. */
public static function publishPreparedLocked(CloudJob $job,
string $resolution): void
{
if (!in_array($resolution, ['applied', 'not_applied'], true)) {
throw new \InvalidArgumentException(
'Invalid portable restore prepared resolution.'
);
}
if ($job->getConnection()->transactionLevel() < 1) {
throw new \LogicException(
'Portable restore prepared publication requires a transaction.'
);
}
if (!self::owns($job)) {
throw new CloudRecoveryInstanceStateConflict(
'备份恢复尝试账本不存在', 409
);
}
$context = self::lockContext($job, self::payload($job));
$attempt = $context['attempt'];
if ($resolution === 'applied'
&& (string) $attempt->state === 'succeeded') {
self::verifySucceeded($context);
return;
}
if ($resolution === 'not_applied'
&& (string) $attempt->state === 'failed') {
self::verifyFailed($context);
return;
}
if ((string) $attempt->state !== 'finalization_pending'
|| (string) $attempt->effect_resolution !== $resolution
|| (string) $attempt->evidence_resolution !== $resolution
|| (int) $attempt->active_attempt !== 1
|| $attempt->terminal_at !== null
|| !hash_equals(
'recovery_prepare_pending',
(string) $attempt->finalization_error_code
)) {
throw new CloudRecoveryEvidenceConflict(
'恢复尝试尚未完成匹配的账本预提交', 409
);
}
if ($resolution === 'applied') {
self::settleSucceeded($context);
} else {
self::settleFailed($context);
}
}
/** Retry only PHP finalization from already persisted Agent evidence. */
public static function retryPendingFinalization(int $jobId,
string $leaseToken): CloudJob
{
$snapshot = CloudJob::find($jobId);
if (!$snapshot instanceof CloudJob) {
throw new CloudError('恢复任务不存在', 404);
}
$payload = self::payload($snapshot);
CloudOperationRecoveryIdentity::expected($snapshot);
$snapshot->getConnection()->transaction(static function () use (
$snapshot, $payload, $leaseToken
): void {
$user = User::where('id', (int) $snapshot->user_id)
->lockForUpdate()->first();
$job = CloudJob::where('id', (int) $snapshot->id)
->lockForUpdate()->first();
if (!$user instanceof User || !$job instanceof CloudJob
|| (int) $job->user_id !== (int) $user->id
|| (string) $job->kind !== 'backup_restore'
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !(bool) $job->finalization_incident
|| !is_string($job->lease_token)
|| !hash_equals((string) $job->lease_token, $leaseToken)) {
throw new CloudError('恢复任务不处于本地收尾重试状态', 409);
}
$context = self::lockContext($job, $payload);
$attempt = $context['attempt'];
$resolution = (string) $attempt->effect_resolution;
if ((string) $attempt->state !== 'finalization_pending'
|| !in_array($resolution, ['applied', 'not_applied'], true)
|| (string) $attempt->evidence_resolution !== $resolution
|| (int) $attempt->active_attempt !== 1
|| $attempt->terminal_at !== null
|| hash_equals(
'recovery_prepare_pending',
(string) $attempt->finalization_error_code
)) {
throw new CloudError('恢复尝试不处于可重试的本地收尾状态', 409);
}
if ($resolution === 'applied') {
self::settleSucceeded($context);
} else {
self::settleFailed($context);
}
$now = time();
$job->effect_resolution = $resolution;
$job->effect_resolved_at = $now;
$job->effect_resolved_by = null;
$job->effect_resolution_reason =
'automatic_portable_restore_finalizer';
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->read_at = null;
$job->updated_at = $now;
$job->save();
CloudJobService::reconcileAggregateParentEffectLocked($job, null);
}, 3);
return CloudJob::find($jobId) ?? $snapshot;
}
/** Keep resources fenced when generic Worker finalization itself fails. */
public static function markJobUncertain(CloudJob $snapshot): void
{
if (!self::owns($snapshot)) {
return;
}
try {
$payload = CloudSecrets::decrypt(
(string) $snapshot->sealed_payload, 'job-payload'
);
} catch (\Throwable) {
return;
}
if (!is_array($payload) || array_is_list($payload)) {
return;
}
$snapshot->getConnection()->transaction(static function () use (
$snapshot, $payload
): void {
$user = User::where('id', (int) $snapshot->user_id)
->lockForUpdate()->first();
$job = CloudJob::where('id', (int) $snapshot->id)
->lockForUpdate()->first();
if (!$user instanceof User || !$job instanceof CloudJob) {
return;
}
$context = self::lockContext($job, $payload);
$attempt = $context['attempt'];
if (in_array((string) $attempt->state,
['finalization_pending', 'succeeded', 'failed'], true)) {
$attempt->finalization_error_code =
'local_finalization_failed';
$attempt->updated_at = time();
$attempt->save();
return;
}
self::recordEvidenceLocked($context, 'uncertain', null);
}, 3);
}
private static function assertStatus(string $status): void
{
if (!in_array($status, ['succeeded', 'failed', 'uncertain'], true)) {
throw new \InvalidArgumentException(
'Invalid portable restore finalization status.'
);
}
}
/** @return array<string,mixed> */
private static function payload(CloudJob $job): array
{
try {
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
} catch (\Throwable) {
throw new CloudRecoveryPayloadUnavailable(
'恢复任务加密账本暂时无法读取', 409
);
}
if (!is_array($payload) || array_is_list($payload)) {
throw new CloudRecoveryPayloadUnavailable(
'恢复任务加密账本结构无效', 409
);
}
return $payload;
}
/** @return array<string,mixed> */
private static function lockContext(CloudJob $job, array $payload): array
{
$attemptKey = $payload['attempt_key'] ?? null;
if (!is_string($attemptKey)
|| preg_match('/^[a-f0-9]{32}$/D', $attemptKey) !== 1) {
throw new CloudError('恢复任务缺少尝试身份', 409);
}
// Hints select the fixed lock hierarchy only; every value is repeated
// against the locked rows before it authorizes a mutation.
$attemptHint = CloudBackupRestoreAttempt::where(
'attempt_key', $attemptKey
)->first();
$restoreHint = $attemptHint instanceof CloudBackupRestoreAttempt
? CloudBackupRestore::find((int) $attemptHint->backup_restore_id)
: null;
$backupHint = $restoreHint instanceof CloudBackupRestore
? CloudInstanceBackup::find((int) $restoreHint->backup_id) : null;
if (!$attemptHint instanceof CloudBackupRestoreAttempt
|| !$restoreHint instanceof CloudBackupRestore
|| !$backupHint instanceof CloudInstanceBackup) {
throw new CloudError('恢复任务最终化账本不存在', 409);
}
// Worker owns User -> Job. Continue with Backup -> Host -> Restore ->
// Attempt, then let the allocator own catalog -> reservation -> claims.
$backup = CloudInstanceBackup::where('id', (int) $backupHint->id)
->lockForUpdate()->first();
$host = CloudHost::where('id', (int) $attemptHint->target_host_id)
->lockForUpdate()->first();
$restore = CloudBackupRestore::where('id', (int) $restoreHint->id)
->lockForUpdate()->first();
$attempt = CloudBackupRestoreAttempt::where(
'id', (int) $attemptHint->id
)->where('attempt_key', $attemptKey)->lockForUpdate()->first();
if (!$backup instanceof CloudInstanceBackup
|| !$host instanceof CloudHost
|| !$restore instanceof CloudBackupRestore
|| !$attempt instanceof CloudBackupRestoreAttempt) {
throw new CloudError('恢复任务最终化账本已经变化', 409);
}
try {
$plan = CloudPortableRestoreTargetPlan::fromStored(
(string) $attempt->target_plan_json,
(string) $attempt->target_plan_sha256
);
} catch (\InvalidArgumentException) {
throw new CloudError('恢复任务保存的目标计划损坏', 409);
}
$planValue = $plan->toArray();
$backupEvidence = CloudPortableRestoreBackupEvidence::fromBackup(
$backup
);
if ((int) $attempt->job_id !== (int) $job->id
|| (int) $job->backup_restore_id !== (int) $restore->id
|| (int) $job->backup_id !== (int) $backup->id
|| (int) $job->user_id !== (int) $backup->user_id
|| (int) $job->host_id !== (int) $host->id
|| (string) $job->kind !== 'backup_restore'
|| (string) $job->execution_mode !== 'agent'
|| (int) $attempt->backup_restore_id !== (int) $restore->id
|| (int) $attempt->target_host_id !== (int) $host->id
|| (int) $restore->target_host_id !== (int) $host->id
|| (int) $restore->backup_id !== (int) $backup->id
|| (int) $planValue['target']['host_id'] !== (int) $host->id
|| !hash_equals((string) $job->operation_id,
(string) $attempt->operation_id)
|| !hash_equals((string) $job->request_idempotency_key,
(string) $attempt->attempt_key)
|| !hash_equals((string) $restore->selected_plan_json,
$plan->json())
|| !hash_equals((string) $restore->selected_plan_sha256,
$plan->sha256())
|| CloudCanonicalJson::normalize($planValue['backup'])
!== CloudCanonicalJson::normalize($backupEvidence->backup())
|| CloudCanonicalJson::normalize($planValue['resources'])
!== CloudCanonicalJson::normalize(
$backupEvidence->resources()
)) {
throw new CloudError('恢复任务最终化身份不一致', 409);
}
$reservationHint = CloudResourceReservation::find(
(int) $attempt->reservation_id
);
if (!$reservationHint instanceof CloudResourceReservation) {
throw new CloudError('恢复任务资源预留不存在', 409);
}
if ((string) $reservationHint->state !== 'released') {
CloudPublicNetworkAllocationService::verifyExistingBundle(
(int) $host->id,
(int) $planValue['target']['line_id'],
(int) $attempt->reservation_id,
(string) $attempt->operation_id
);
}
$reservation = CloudResourceReservation::where(
'id', (int) $attempt->reservation_id
)->lockForUpdate()->first();
if (!$reservation instanceof CloudResourceReservation
|| !$attempt->matchesReservation($reservation)
|| (int) $reservation->user_id !== (int) $backup->user_id
|| (int) $reservation->host_id !== (int) $host->id
|| (int) $reservation->resource_pool_id
!== (int) $planValue['target']['resource_pool_id']
|| !hash_equals((string) $reservation->operation_id,
(string) $attempt->operation_id)) {
throw new CloudError('恢复任务资源预留身份不一致', 409);
}
foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $field) {
if ((int) $reservation->{$field}
!== (int) $planValue['resources'][$field]) {
throw new CloudError('恢复任务资源规格已经变化', 409);
}
}
$network = self::frozenNetwork($attempt, $reservation);
self::assertPayload($job, $payload, $backup, $restore, $attempt,
$reservation, $planValue, $network);
return compact('job', 'payload', 'backup', 'restore', 'attempt',
'reservation', 'host', 'plan', 'planValue', 'network');
}
private static function frozenNetwork(CloudBackupRestoreAttempt $attempt,
CloudResourceReservation $reservation
): CloudPublicNetworkSnapshot {
$state = (string) $reservation->state;
if ($state === 'released') {
try {
$decoded = json_decode(
(string) $attempt->target_public_network_json,
true,
64,
JSON_THROW_ON_ERROR
);
$network = CloudPublicNetworkSnapshot::fromArray($decoded);
} catch (\Throwable) {
throw new CloudError('恢复任务冻结双栈网络损坏', 409);
}
if (!hash_equals($network->fingerprint(),
(string) $attempt->target_public_network_sha256)) {
throw new CloudError('恢复任务冻结双栈网络摘要不一致', 409);
}
return $network;
}
$claimSet = CloudPublicNetworkAllocationService::currentClaimSet(
(int) $reservation->id,
$state,
$state === 'committed' ? (int) $reservation->instance_id : null
);
try {
return CloudFrozenPublicNetworkIdentity::fromStored(
(int) $reservation->id,
(int) $attempt->target_host_id,
(string) $attempt->target_uuid,
(int) $attempt->target_network_attachment_id,
(string) $attempt->target_identity_json,
(string) $attempt->target_identity_sha256,
(string) $attempt->target_public_network_json,
(string) $attempt->target_public_network_sha256,
(string) $attempt->target_claim_set_sha256,
$claimSet
)->publicNetwork();
} catch (\InvalidArgumentException) {
throw new CloudError('恢复任务冻结双栈身份损坏', 409);
}
}
private static function assertPayload(CloudJob $job, array $payload,
CloudInstanceBackup $backup, CloudBackupRestore $restore,
CloudBackupRestoreAttempt $attempt,
CloudResourceReservation $reservation, array $plan,
CloudPublicNetworkSnapshot $network): void
{
$body = $payload['body'] ?? null;
try {
$wireNetwork = is_array($body)
? CloudPublicNetworkSnapshot::fromArray(
$body['public_network'] ?? null
) : null;
} catch (\Throwable) {
$wireNetwork = null;
}
if (!is_array($body) || array_is_list($body)
|| !$wireNetwork instanceof CloudPublicNetworkSnapshot
|| ($payload['method'] ?? null) !== 'POST'
|| !hash_equals((string) ($payload['request_target'] ?? ''),
'/v1/vms/' . rawurlencode(
strtolower((string) $attempt->target_uuid)
) . '/restore-backup')
|| !hash_equals((string) ($payload['attempt_key'] ?? ''),
(string) $attempt->attempt_key)
|| (int) ($payload['generation'] ?? 0)
!== (int) $attempt->generation
|| (int) ($payload['reservation_id'] ?? 0)
!== (int) $reservation->id
|| (int) ($payload['owner_id'] ?? 0) !== (int) $backup->user_id
|| !hash_equals((string) ($payload['backup_key'] ?? ''),
(string) $backup->backup_key)
|| !hash_equals((string) ($payload['restore_key'] ?? ''),
(string) $restore->restore_key)
|| !hash_equals((string) ($payload['target_plan_sha256'] ?? ''),
(string) $attempt->target_plan_sha256)
|| !hash_equals(
(string) ($payload['target_identity_sha256'] ?? ''),
(string) $attempt->target_identity_sha256)
|| !hash_equals(
(string) ($payload['target_claim_set_sha256'] ?? ''),
(string) $attempt->target_claim_set_sha256)
|| !hash_equals(
(string) ($payload['target_public_network_sha256'] ?? ''),
$network->fingerprint())
|| !hash_equals((string) ($body['operation_id'] ?? ''),
(string) $job->operation_id)
|| !hash_equals(strtolower((string) ($body['uuid'] ?? '')),
strtolower((string) $attempt->target_uuid))
|| !hash_equals((string) ($body['name'] ?? ''),
(string) $attempt->target_name)
|| (int) ($body['vcpu'] ?? 0)
!== (int) $plan['resources']['cpu_cores']
|| (int) ($body['memory_mib'] ?? 0)
!== (int) $plan['resources']['memory_mib']
|| (int) ($body['disk_gib'] ?? 0)
!== (int) $plan['resources']['disk_gib']
|| (int) ($body['bandwidth_mbps'] ?? 0)
!== (int) $plan['resources']['bandwidth_mbps']
|| !hash_equals($wireNetwork->fingerprint(),
$network->fingerprint())
|| !hash_equals((string) ($body['backup_key'] ?? ''),
(string) $backup->backup_key)
|| (int) ($body['source_size_bytes'] ?? 0)
!== (int) $backup->object_size_bytes
|| !hash_equals((string) ($body['source_sha256'] ?? ''),
(string) $backup->sha256)
|| array_key_exists('source_url', $body)
|| array_key_exists('vpc_port', $body)) {
throw new CloudError('恢复任务冻结信封与账本不一致', 409);
}
}
/** @param array<string,mixed> $context */
private static function recordEvidenceLocked(array $context,
string $status, mixed $result): void
{
$attempt = $context['attempt'];
$evidence = self::evidence($context, $status, $result);
$json = self::encode($evidence);
$digest = hash('sha256', $json);
$resolution = self::resolution($status);
$recoveringUnknown = false;
if ((string) $attempt->evidence_resolution !== 'not_applicable') {
if (hash_equals((string) $attempt->evidence_resolution,
$resolution)
&& hash_equals((string) $attempt->evidence_sha256, $digest)
&& hash_equals((string) $attempt->evidence_json, $json)) {
return;
}
$recoveringUnknown = (string) $attempt->state === 'effect_unknown'
&& (string) $attempt->effect_resolution === 'unknown'
&& (string) $attempt->evidence_resolution === 'unknown'
&& in_array($resolution, ['applied', 'not_applied'], true)
&& (int) $attempt->active_attempt === 1
&& $attempt->terminal_at === null
&& (string) $context['reservation']->state === 'uncertain';
if (!$recoveringUnknown) {
throw new CloudRecoveryEvidenceConflict(
'恢复尝试已有不同的持久效果证据', 409
);
}
}
if (!$recoveringUnknown && (!in_array((string) $attempt->state,
['queued', 'awaiting_receipt'], true)
|| (string) $attempt->effect_resolution !== 'not_applicable'
|| (int) $attempt->active_attempt !== 1
|| $attempt->effect_resolved_at !== null
|| $attempt->terminal_at !== null
|| (string) $context['reservation']->state !== 'held')) {
throw new CloudError('恢复尝试当前不能记录效果证据', 409);
}
$now = time();
$attempt->evidence_resolution = $resolution;
$attempt->evidence_json = $json;
$attempt->evidence_sha256 = $digest;
$attempt->evidence_recorded_at = $now;
$attempt->finalization_error_code = '';
if ($resolution === 'unknown') {
CloudPublicNetworkAllocationService::fenceBundle(
(int) $context['reservation']->id,
(string) $attempt->operation_id
);
$reservation = $context['reservation'];
$reservation->state = 'uncertain';
$reservation->expires_at = null;
$reservation->updated_at = $now;
$reservation->save();
$attempt->state = 'effect_unknown';
$attempt->effect_resolution = 'unknown';
$attempt->last_error_code = 'operation_effect_unknown';
$attempt->effect_resolved_at = null;
$attempt->terminal_at = null;
self::projectParent($context, 'uncertain', $now);
} else {
$attempt->state = 'finalization_pending';
$attempt->effect_resolution = $resolution;
$attempt->last_error_code = '';
$attempt->effect_resolved_at = $now;
$attempt->terminal_at = null;
}
$attempt->updated_at = $now;
$attempt->save();
}
/** @param array<string,mixed> $context */
private static function assertMatchingEvidence(array $context,
string $status, mixed $result): void
{
$attempt = $context['attempt'];
$json = self::encode(self::evidence($context, $status, $result));
if (!hash_equals((string) $attempt->evidence_resolution,
self::resolution($status))
|| !hash_equals((string) $attempt->evidence_json, $json)
|| !hash_equals((string) $attempt->evidence_sha256,
hash('sha256', $json))
|| $attempt->evidence_recorded_at === null) {
throw new CloudRecoveryEvidenceConflict(
'恢复尝试持久证据与最终化结论不一致', 409
);
}
}
/** @param array<string,mixed> $context */
private static function evidence(array $context, string $status,
mixed $result): array
{
if ($status === 'failed') {
if (!is_array($result) || array_is_list($result)
|| ($result['creation_absent'] ?? null) !== true
|| ($result['managed_artifacts_absent'] ?? null) !== true) {
throw new CloudRecoveryEvidenceConflict(
'恢复未生效结论缺少完整资源缺席证据', 409
);
}
return [
'creation_absent' => true,
'managed_artifacts_absent' => true,
];
}
if ($status === 'uncertain') {
return ['effect_may_have_applied' => true];
}
$vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
if (!is_array($result) || array_is_list($result)
|| !is_array($vm) || array_is_list($vm)) {
throw new CloudError('Agent 恢复成功结果结构无效', 502);
}
try {
$network = CloudPublicNetworkSnapshot::fromArray(
$vm['public_network'] ?? null
);
} catch (\Throwable) {
throw new CloudError('Agent 恢复成功结果缺少双栈公网快照', 502);
}
$attempt = $context['attempt'];
$backup = $context['backup'];
$resources = $context['planValue']['resources'];
$compatibility = CloudPortableRestoreCompatibility::supported()
->toArray();
$value = [
'uuid' => strtolower((string) ($vm['uuid'] ?? '')),
'name' => (string) ($vm['name'] ?? ''),
'status' => (string) ($vm['status'] ?? ''),
'vcpu' => (int) ($vm['vcpu'] ?? 0),
'memory_mib' => (int) ($vm['memory_mib'] ?? 0),
'disk_gib' => (int) ($vm['disk_gib'] ?? 0),
'bandwidth_mbps' => (int) ($vm['bandwidth_mbps'] ?? 0),
'public_network' => $network->toArray(),
'machine' => (string) ($vm['machine'] ?? ''),
'firmware' => (string) ($vm['firmware'] ?? ''),
'secure_boot' => $vm['secure_boot'] ?? null,
'backup_key' => (string) ($result['backup_key']
?? $vm['backup_key'] ?? ''),
'sha256' => (string) ($result['sha256'] ?? $vm['sha256'] ?? ''),
];
if (!hash_equals($value['uuid'],
strtolower((string) $attempt->target_uuid))
|| !hash_equals($value['name'], (string) $attempt->target_name)
|| $value['vcpu'] !== (int) $resources['cpu_cores']
|| $value['memory_mib'] !== (int) $resources['memory_mib']
|| $value['disk_gib'] !== (int) $resources['disk_gib']
|| $value['bandwidth_mbps']
!== (int) $resources['bandwidth_mbps']
|| !hash_equals($network->fingerprint(),
$context['network']->fingerprint())
|| !hash_equals($value['backup_key'],
(string) $backup->backup_key)
|| !hash_equals($value['sha256'], (string) $backup->sha256)) {
throw new CloudError(
'Agent 恢复成功结果与冻结备份、规格或双栈网络不一致',
502
);
}
if (!hash_equals(
$compatibility['machine'],
$value['machine']
)
|| !hash_equals(
$compatibility['firmware'],
$value['firmware']
)
|| !is_bool($value['secure_boot'])
|| $value['secure_boot'] !== $compatibility['secure_boot']) {
throw new CloudError(
'Agent 恢复成功结果与冻结平台兼容合同不一致',
502
);
}
return $value;
}
private static function resolution(string $status): string
{
return $status === 'succeeded' ? 'applied'
: ($status === 'failed' ? 'not_applied' : 'unknown');
}
/** @param array<string,mixed> $context */
private static function settleSucceeded(array $context): void
{
$attempt = $context['attempt'];
$reservation = $context['reservation'];
if ((string) $attempt->effect_resolution !== 'applied'
|| (int) $attempt->active_attempt !== 1
|| $attempt->effect_resolved_at === null
|| $attempt->terminal_at !== null
|| !in_array((string) $reservation->state,
['held', 'uncertain'], true)
|| $reservation->instance_id !== null) {
throw new CloudError('恢复成功证据与资源状态不一致', 409);
}
$evidence = self::decodedEvidence($attempt);
$now = time();
$instance = self::instanceForSuccess($context, $evidence, $now);
$committed = CloudPublicNetworkAllocationService::commitBundle(
(int) $reservation->id,
(int) $instance->id,
(string) $attempt->operation_id
);
if (!hash_equals($context['network']->fingerprint(),
$committed->fingerprint())) {
throw new CloudError('恢复成功回执与双栈公网预留不一致', 409);
}
$reservation->state = 'committed';
$reservation->instance_id = (int) $instance->id;
$reservation->expires_at = null;
$reservation->released_at = null;
$reservation->release_reason = '';
$reservation->updated_at = $now;
$reservation->save();
$job = $context['job'];
$job->instance_id = (int) $instance->id;
$job->updated_at = $now;
$job->save();
$restore = $context['restore'];
$restore->target_instance_id = (int) $instance->id;
$restore->state = 'ready';
$restore->error_code = '';
$restore->ready_at = $now;
$restore->updated_at = $now;
$restore->save();
$attempt->state = 'succeeded';
$attempt->active_attempt = null;
$attempt->terminal_at = $now;
$attempt->last_error_code = '';
$attempt->finalization_error_code = '';
$attempt->updated_at = $now;
$attempt->save();
self::recordRuntime($instance, $evidence);
}
/** @param array<string,mixed> $context */
private static function settleFailed(array $context): void
{
$attempt = $context['attempt'];
$reservation = $context['reservation'];
if ((string) $attempt->effect_resolution !== 'not_applied'
|| (int) $attempt->active_attempt !== 1
|| $attempt->effect_resolved_at === null
|| $attempt->terminal_at !== null
|| !in_array((string) $reservation->state,
['held', 'uncertain'], true)
|| $reservation->instance_id !== null
|| CloudInstance::where('host_id',
(int) $attempt->target_host_id)
->where('uuid', (string) $attempt->target_uuid)
->where('status', '<>', 'deleted')->exists()) {
throw new CloudError('恢复未生效证据与资源状态不一致', 409);
}
$now = time();
CloudPublicNetworkAllocationService::releaseBundle(
(int) $reservation->id,
(string) $attempt->operation_id,
'backup_restore_not_applied',
true
);
$reservation->state = 'released';
$reservation->instance_id = null;
$reservation->expires_at = null;
$reservation->released_at = $now;
$reservation->release_reason = 'backup_restore_not_applied';
$reservation->updated_at = $now;
$reservation->save();
self::projectParent($context, 'failed', $now);
$attempt->state = 'failed';
$attempt->active_attempt = null;
$attempt->terminal_at = $now;
$attempt->last_error_code = 'backup_restore_not_applied';
$attempt->finalization_error_code = '';
$attempt->updated_at = $now;
$attempt->save();
}
/** @param array<string,mixed> $context */
private static function projectParent(array $context, string $state,
int $now): void
{
$restore = $context['restore'];
$restore->target_instance_id = null;
$restore->state = $state;
$restore->error_code = $state === 'uncertain'
? 'operation_effect_unknown' : 'backup_restore_not_applied';
$restore->ready_at = null;
$restore->updated_at = $now;
$restore->save();
}
/** @param array<string,mixed> $context */
private static function instanceForSuccess(array $context,
array $evidence, int $now): CloudInstance
{
$attempt = $context['attempt'];
$reservation = $context['reservation'];
$backup = $context['backup'];
$byReservation = CloudInstance::where('reservation_id',
(int) $reservation->id)->lockForUpdate()->first();
$byUuid = CloudInstance::where('host_id',
(int) $attempt->target_host_id)
->where('uuid', (string) $attempt->target_uuid)
->lockForUpdate()->first();
if ($byReservation instanceof CloudInstance
&& $byUuid instanceof CloudInstance
&& (int) $byReservation->id !== (int) $byUuid->id) {
throw new CloudError('恢复成功实例身份发生冲突', 409);
}
$instance = $byReservation ?? $byUuid ?? new CloudInstance();
if ($instance->exists
&& ((int) $instance->host_id !== (int) $attempt->target_host_id
|| (int) $instance->user_id !== (int) $backup->user_id
|| (int) $instance->reservation_id
!== (int) $reservation->id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $attempt->target_uuid)))) {
throw new CloudError('恢复成功实例归属发生冲突', 409);
}
$source = self::decodeSpec((string) $backup->spec_json);
$spec = CloudInput::safeResult($source);
foreach ([
'plan_id', 'offer_id', 'price_version', 'stock_mode',
'validity_days', 'traffic_quota_bytes', 'renewal_price',
'currency', 'public_ip', 'public_ip_cidr', 'public_ipv4',
'public_ipv4_cidr', 'public_ipv6', 'public_ipv6_cidr',
'public_pool_key', 'public_pool_cidr', 'gateway', 'dns_servers',
'resource_pool_id', 'resource_pool_name', 'line_id', 'line_name',
'region_id', 'region', 'country_code', 'vpc_port', 'vpc_port_id',
'private_ip', 'private_mac', 'guest_hostname', 'hostname',
'root_password', 'guest_ssh_public_key', 'sealed_credential',
] as $field) {
unset($spec[$field]);
}
$spec = CloudInstanceMetadata::merge($spec, $evidence);
unset($spec['guest_hostname'], $spec['hostname']);
$resources = $context['planValue']['resources'];
$target = $context['planValue']['target'];
$public = $context['network']->toArray();
$spec = array_replace($spec, [
'managed' => true,
'vcpu' => (int) $resources['cpu_cores'],
'cpu_cores' => (int) $resources['cpu_cores'],
'memory_mib' => (int) $resources['memory_mib'],
'disk_gib' => (int) $resources['disk_gib'],
'bandwidth_mbps' => (int) $resources['bandwidth_mbps'],
'network' => (string) $public['network_selector'],
'public_network' => $public,
'resource_pool_id' => (int) $target['resource_pool_id'],
'resource_pool_name' => (string) $target['resource_pool_name'],
'line_id' => (int) $target['line_id'],
'line_name' => (string) $target['line_name'],
'region_id' => (int) $target['region_id'],
'region' => (string) $target['region_name'],
'country_code' => (string) $target['country_code'],
'restored_from_backup' => (string) $backup->backup_key,
]);
foreach (['root_password', 'guest_hostname', 'hostname',
'guest_ssh_public_key', 'vpc_port', 'vpc_port_id'] as $field) {
unset($spec[$field]);
}
$instance->uuid = strtolower((string) $attempt->target_uuid);
$instance->host_id = (int) $attempt->target_host_id;
$instance->user_id = (int) $backup->user_id;
$instance->order_id = null;
$instance->plan_id = null;
$instance->offer_id = null;
$instance->resource_pool_id = (int) $reservation->resource_pool_id;
$instance->reservation_id = (int) $reservation->id;
$instance->current_period_id = null;
$instance->name = (string) $attempt->target_name;
$instance->status = 'active';
$instance->service_status = 'exempt';
$instance->network_status = 'active';
$instance->expires_at = null;
$instance->renewal_price = null;
$instance->renewal_days = 0;
$instance->renewal_traffic_bytes = 0;
$instance->traffic_quota_bytes = 0;
$instance->traffic_download_bytes = 0;
$instance->traffic_upload_bytes = 0;
$instance->lifetime_download_bytes = 0;
$instance->lifetime_upload_bytes = 0;
$instance->traffic_synced_at = 0;
$instance->suspended_at = null;
$instance->network_suspended_at = null;
$instance->spec_json = self::encode($spec);
$instance->sealed_credential = null;
if (!$instance->exists) {
$instance->created_at = $now;
}
$instance->updated_at = $now;
$instance->save();
return $instance;
}
/** @param array<string,mixed> $context */
private static function verifySucceeded(array $context): void
{
$attempt = $context['attempt'];
$restore = $context['restore'];
$reservation = $context['reservation'];
$instance = $restore->target_instance_id === null ? null
: CloudInstance::where('id', (int) $restore->target_instance_id)
->lockForUpdate()->first();
if (!$instance instanceof CloudInstance
|| (int) $context['job']->instance_id !== (int) $instance->id
|| (string) $restore->state !== 'ready'
|| (string) $attempt->effect_resolution !== 'applied'
|| $attempt->active_attempt !== null
|| $attempt->effect_resolved_at === null
|| $attempt->terminal_at === null
|| (string) $reservation->state !== 'committed'
|| (int) $reservation->instance_id !== (int) $instance->id
|| (int) $instance->reservation_id !== (int) $reservation->id
|| (int) $instance->host_id !== (int) $attempt->target_host_id
|| (int) $instance->user_id !== (int) $context['backup']->user_id
|| $instance->sealed_credential !== null) {
throw new CloudError('已完成恢复的实例或资源账本不一致', 409);
}
CloudPublicNetworkAllocationService::currentClaimSet(
(int) $reservation->id, 'committed', (int) $instance->id
);
}
/** @param array<string,mixed> $context */
private static function verifyFailed(array $context): void
{
$attempt = $context['attempt'];
$restore = $context['restore'];
$reservation = $context['reservation'];
if ((string) $attempt->effect_resolution !== 'not_applied'
|| $attempt->active_attempt !== null
|| $attempt->effect_resolved_at === null
|| $attempt->terminal_at === null
|| (string) $restore->state !== 'failed'
|| $restore->target_instance_id !== null
|| (string) $reservation->state !== 'released'
|| $reservation->instance_id !== null) {
throw new CloudError('已失败恢复的释放账本不一致', 409);
}
CloudReleasedPublicNetworkBundleVerifier::verify(
$attempt, $reservation, (int) $context['backup']->user_id
);
}
/** @param array<string,mixed> $context */
private static function verifyUnknown(array $context): void
{
$attempt = $context['attempt'];
$restore = $context['restore'];
$reservation = $context['reservation'];
if ((string) $attempt->effect_resolution !== 'unknown'
|| (int) $attempt->active_attempt !== 1
|| $attempt->effect_resolved_at !== null
|| $attempt->terminal_at !== null
|| (string) $restore->state !== 'uncertain'
|| $restore->target_instance_id !== null
|| (string) $reservation->state !== 'uncertain'
|| $reservation->instance_id !== null) {
throw new CloudError('效果未知恢复的隔离账本不一致', 409);
}
CloudPublicNetworkAllocationService::currentClaimSet(
(int) $reservation->id, 'uncertain'
);
}
private static function decodedEvidence(
CloudBackupRestoreAttempt $attempt
): array {
try {
$value = json_decode((string) $attempt->evidence_json, true, 64,
JSON_THROW_ON_ERROR);
} catch (\JsonException) {
throw new CloudError('恢复尝试持久证据损坏', 409);
}
if (!is_array($value)
|| !hash_equals(hash('sha256', self::encode($value)),
(string) $attempt->evidence_sha256)) {
throw new CloudError('恢复尝试持久证据摘要不一致', 409);
}
return $value;
}
private static function decodeSpec(string $encoded): array
{
try {
$spec = json_decode($encoded, true, 64, JSON_THROW_ON_ERROR);
} catch (\JsonException) {
throw new CloudError('实例备份规格快照损坏', 409);
}
if (!is_array($spec) || array_is_list($spec)) {
throw new CloudError('实例备份规格快照损坏', 409);
}
return $spec;
}
private static function encode(array $value): string
{
return json_encode(CloudCanonicalJson::normalize($value),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
| JSON_UNESCAPED_LINE_TERMINATORS);
}
private static function recordRuntime(CloudInstance $instance,
array $result): void
{
try {
CloudInstanceRuntimeService::seed((int) $instance->id,
(int) $instance->host_id, (string) $instance->uuid);
$state = $result['status']
?? ($result['state'] ?? ($result['power_state'] ?? null));
if (CloudInstanceRuntimeService::normalizeState($state)
!== 'unknown') {
CloudInstanceRuntimeService::markObserved((int) $instance->id,
(int) $instance->host_id, (string) $instance->uuid,
$state);
}
} catch (\Throwable $error) {
error_log('portable restore runtime cache unavailable: '
. get_class($error));
}
}
}