You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1089 lines
47 KiB
PHP
1089 lines
47 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudBackupRestoreAttempt;
|
|
use App\Models\CloudHost;
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudResourceReservation;
|
|
use App\Models\User;
|
|
|
|
/** Durable evidence and PHP-only settlement for independent backup restores. */
|
|
final class CloudPortableRestoreFinalizer
|
|
{
|
|
public static function owns(CloudJob $job): bool
|
|
{
|
|
return (string) $job->kind === 'backup_restore'
|
|
&& CloudBackupRestoreAttempt::where('job_id', (int) $job->id)
|
|
->exists();
|
|
}
|
|
|
|
public static function markDispatched(CloudJob $job): void
|
|
{
|
|
if (!self::owns($job)) {
|
|
return;
|
|
}
|
|
$changed = CloudBackupRestoreAttempt::where('job_id', (int) $job->id)
|
|
->where('operation_id', (string) $job->operation_id)
|
|
->where('state', 'queued')
|
|
->where('effect_resolution', 'not_applicable')
|
|
->where('active_attempt', 1)
|
|
->update(['state' => 'awaiting_receipt', 'updated_at' => time()]);
|
|
if ($changed === 1) {
|
|
return;
|
|
}
|
|
$attempt = CloudBackupRestoreAttempt::where(
|
|
'job_id', (int) $job->id
|
|
)->first();
|
|
if (!$attempt instanceof CloudBackupRestoreAttempt
|
|
|| !hash_equals((string) $attempt->operation_id,
|
|
(string) $job->operation_id)
|
|
|| !in_array((string) $attempt->state, [
|
|
'awaiting_receipt', 'effect_unknown',
|
|
'finalization_pending', 'succeeded', 'failed',
|
|
], true)) {
|
|
throw new CloudError('恢复尝试派发状态不一致', 409);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Persist the validated Agent conclusion before the larger Worker
|
|
* finalization transaction. A later PHP-only pass can then finish without
|
|
* replaying the host mutation.
|
|
*/
|
|
public static function recordEvidence(CloudJob $snapshot, string $status,
|
|
mixed $result, array $payload): void
|
|
{
|
|
if (!self::owns($snapshot)) {
|
|
return;
|
|
}
|
|
self::assertStatus($status);
|
|
$snapshot->getConnection()->transaction(static function () use (
|
|
$snapshot, $status, $result, $payload
|
|
): void {
|
|
$user = User::where('id', (int) $snapshot->user_id)
|
|
->lockForUpdate()->first();
|
|
$job = CloudJob::where('id', (int) $snapshot->id)
|
|
->lockForUpdate()->first();
|
|
if (!$user instanceof User || !$job instanceof CloudJob
|
|
|| (int) $job->user_id !== (int) $user->id) {
|
|
throw new CloudError('恢复任务账户或作业不存在', 409);
|
|
}
|
|
$context = self::lockContext($job, $payload);
|
|
self::recordEvidenceLocked($context, $status, $result);
|
|
}, 3);
|
|
}
|
|
|
|
/** Apply one persisted conclusion inside CloudWorkerService's lock. */
|
|
public static function finishJobLocked(CloudJob $job, string $status,
|
|
mixed $result, array $payload, bool $deferPublish = false): void
|
|
{
|
|
self::assertStatus($status);
|
|
if (!self::owns($job)) {
|
|
throw new CloudError('恢复任务不属于尝试账本', 409);
|
|
}
|
|
if ($job->getConnection()->transactionLevel() < 1) {
|
|
throw new \LogicException(
|
|
'Portable restore finalization requires a transaction.'
|
|
);
|
|
}
|
|
$context = self::lockContext($job, $payload);
|
|
$attempt = $context['attempt'];
|
|
if ((string) $attempt->evidence_resolution === 'not_applicable'
|
|
|| ((string) $attempt->evidence_resolution === 'unknown'
|
|
&& $status !== 'uncertain')) {
|
|
self::recordEvidenceLocked($context, $status, $result);
|
|
$context = self::lockContext($job, $payload);
|
|
$attempt = $context['attempt'];
|
|
}
|
|
self::assertMatchingEvidence($context, $status, $result);
|
|
if ((string) $attempt->state === 'succeeded') {
|
|
self::verifySucceeded($context);
|
|
return;
|
|
}
|
|
if ((string) $attempt->state === 'failed') {
|
|
self::verifyFailed($context);
|
|
return;
|
|
}
|
|
if ((string) $attempt->state === 'effect_unknown') {
|
|
self::verifyUnknown($context);
|
|
return;
|
|
}
|
|
if ((string) $attempt->state !== 'finalization_pending') {
|
|
throw new CloudError('恢复尝试没有可结算的持久证据', 409);
|
|
}
|
|
if ($deferPublish) {
|
|
$attempt->last_error_code = 'recovery_prepare_pending';
|
|
$attempt->finalization_error_code = 'recovery_prepare_pending';
|
|
$attempt->updated_at = time();
|
|
$attempt->save();
|
|
return;
|
|
}
|
|
if ($status === 'succeeded') {
|
|
self::settleSucceeded($context);
|
|
} elseif ($status === 'failed') {
|
|
self::settleFailed($context);
|
|
} else {
|
|
self::verifyUnknown($context);
|
|
}
|
|
}
|
|
|
|
/** Publish a prepared projection only after the Agent fence is resolved. */
|
|
public static function publishPreparedLocked(CloudJob $job,
|
|
string $resolution): void
|
|
{
|
|
if (!in_array($resolution, ['applied', 'not_applied'], true)) {
|
|
throw new \InvalidArgumentException(
|
|
'Invalid portable restore prepared resolution.'
|
|
);
|
|
}
|
|
if ($job->getConnection()->transactionLevel() < 1) {
|
|
throw new \LogicException(
|
|
'Portable restore prepared publication requires a transaction.'
|
|
);
|
|
}
|
|
if (!self::owns($job)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'备份恢复尝试账本不存在', 409
|
|
);
|
|
}
|
|
$context = self::lockContext($job, self::payload($job));
|
|
$attempt = $context['attempt'];
|
|
if ($resolution === 'applied'
|
|
&& (string) $attempt->state === 'succeeded') {
|
|
self::verifySucceeded($context);
|
|
return;
|
|
}
|
|
if ($resolution === 'not_applied'
|
|
&& (string) $attempt->state === 'failed') {
|
|
self::verifyFailed($context);
|
|
return;
|
|
}
|
|
if ((string) $attempt->state !== 'finalization_pending'
|
|
|| (string) $attempt->effect_resolution !== $resolution
|
|
|| (string) $attempt->evidence_resolution !== $resolution
|
|
|| (int) $attempt->active_attempt !== 1
|
|
|| $attempt->terminal_at !== null
|
|
|| !hash_equals(
|
|
'recovery_prepare_pending',
|
|
(string) $attempt->finalization_error_code
|
|
)) {
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
'恢复尝试尚未完成匹配的账本预提交', 409
|
|
);
|
|
}
|
|
if ($resolution === 'applied') {
|
|
self::settleSucceeded($context);
|
|
} else {
|
|
self::settleFailed($context);
|
|
}
|
|
}
|
|
|
|
/** Retry only PHP finalization from already persisted Agent evidence. */
|
|
public static function retryPendingFinalization(int $jobId,
|
|
string $leaseToken): CloudJob
|
|
{
|
|
$snapshot = CloudJob::find($jobId);
|
|
if (!$snapshot instanceof CloudJob) {
|
|
throw new CloudError('恢复任务不存在', 404);
|
|
}
|
|
$payload = self::payload($snapshot);
|
|
CloudOperationRecoveryIdentity::expected($snapshot);
|
|
$snapshot->getConnection()->transaction(static function () use (
|
|
$snapshot, $payload, $leaseToken
|
|
): void {
|
|
$user = User::where('id', (int) $snapshot->user_id)
|
|
->lockForUpdate()->first();
|
|
$job = CloudJob::where('id', (int) $snapshot->id)
|
|
->lockForUpdate()->first();
|
|
if (!$user instanceof User || !$job instanceof CloudJob
|
|
|| (int) $job->user_id !== (int) $user->id
|
|
|| (string) $job->kind !== 'backup_restore'
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| !(bool) $job->finalization_incident
|
|
|| !is_string($job->lease_token)
|
|
|| !hash_equals((string) $job->lease_token, $leaseToken)) {
|
|
throw new CloudError('恢复任务不处于本地收尾重试状态', 409);
|
|
}
|
|
$context = self::lockContext($job, $payload);
|
|
$attempt = $context['attempt'];
|
|
$resolution = (string) $attempt->effect_resolution;
|
|
if ((string) $attempt->state !== 'finalization_pending'
|
|
|| !in_array($resolution, ['applied', 'not_applied'], true)
|
|
|| (string) $attempt->evidence_resolution !== $resolution
|
|
|| (int) $attempt->active_attempt !== 1
|
|
|| $attempt->terminal_at !== null
|
|
|| hash_equals(
|
|
'recovery_prepare_pending',
|
|
(string) $attempt->finalization_error_code
|
|
)) {
|
|
throw new CloudError('恢复尝试不处于可重试的本地收尾状态', 409);
|
|
}
|
|
if ($resolution === 'applied') {
|
|
self::settleSucceeded($context);
|
|
} else {
|
|
self::settleFailed($context);
|
|
}
|
|
$now = time();
|
|
$job->effect_resolution = $resolution;
|
|
$job->effect_resolved_at = $now;
|
|
$job->effect_resolved_by = null;
|
|
$job->effect_resolution_reason =
|
|
'automatic_portable_restore_finalizer';
|
|
$job->finalization_incident = 0;
|
|
$job->finalization_error_code = null;
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = 0;
|
|
$job->read_at = null;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
CloudJobService::reconcileAggregateParentEffectLocked($job, null);
|
|
}, 3);
|
|
return CloudJob::find($jobId) ?? $snapshot;
|
|
}
|
|
|
|
/** Keep resources fenced when generic Worker finalization itself fails. */
|
|
public static function markJobUncertain(CloudJob $snapshot): void
|
|
{
|
|
if (!self::owns($snapshot)) {
|
|
return;
|
|
}
|
|
try {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $snapshot->sealed_payload, 'job-payload'
|
|
);
|
|
} catch (\Throwable) {
|
|
return;
|
|
}
|
|
if (!is_array($payload) || array_is_list($payload)) {
|
|
return;
|
|
}
|
|
$snapshot->getConnection()->transaction(static function () use (
|
|
$snapshot, $payload
|
|
): void {
|
|
$user = User::where('id', (int) $snapshot->user_id)
|
|
->lockForUpdate()->first();
|
|
$job = CloudJob::where('id', (int) $snapshot->id)
|
|
->lockForUpdate()->first();
|
|
if (!$user instanceof User || !$job instanceof CloudJob) {
|
|
return;
|
|
}
|
|
$context = self::lockContext($job, $payload);
|
|
$attempt = $context['attempt'];
|
|
if (in_array((string) $attempt->state,
|
|
['finalization_pending', 'succeeded', 'failed'], true)) {
|
|
$attempt->finalization_error_code =
|
|
'local_finalization_failed';
|
|
$attempt->updated_at = time();
|
|
$attempt->save();
|
|
return;
|
|
}
|
|
self::recordEvidenceLocked($context, 'uncertain', null);
|
|
}, 3);
|
|
}
|
|
|
|
private static function assertStatus(string $status): void
|
|
{
|
|
if (!in_array($status, ['succeeded', 'failed', 'uncertain'], true)) {
|
|
throw new \InvalidArgumentException(
|
|
'Invalid portable restore finalization status.'
|
|
);
|
|
}
|
|
}
|
|
|
|
/** @return array<string,mixed> */
|
|
private static function payload(CloudJob $job): array
|
|
{
|
|
try {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
} catch (\Throwable) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'恢复任务加密账本暂时无法读取', 409
|
|
);
|
|
}
|
|
if (!is_array($payload) || array_is_list($payload)) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'恢复任务加密账本结构无效', 409
|
|
);
|
|
}
|
|
return $payload;
|
|
}
|
|
|
|
/** @return array<string,mixed> */
|
|
private static function lockContext(CloudJob $job, array $payload): array
|
|
{
|
|
$attemptKey = $payload['attempt_key'] ?? null;
|
|
if (!is_string($attemptKey)
|
|
|| preg_match('/^[a-f0-9]{32}$/D', $attemptKey) !== 1) {
|
|
throw new CloudError('恢复任务缺少尝试身份', 409);
|
|
}
|
|
// Hints select the fixed lock hierarchy only; every value is repeated
|
|
// against the locked rows before it authorizes a mutation.
|
|
$attemptHint = CloudBackupRestoreAttempt::where(
|
|
'attempt_key', $attemptKey
|
|
)->first();
|
|
$restoreHint = $attemptHint instanceof CloudBackupRestoreAttempt
|
|
? CloudBackupRestore::find((int) $attemptHint->backup_restore_id)
|
|
: null;
|
|
$backupHint = $restoreHint instanceof CloudBackupRestore
|
|
? CloudInstanceBackup::find((int) $restoreHint->backup_id) : null;
|
|
if (!$attemptHint instanceof CloudBackupRestoreAttempt
|
|
|| !$restoreHint instanceof CloudBackupRestore
|
|
|| !$backupHint instanceof CloudInstanceBackup) {
|
|
throw new CloudError('恢复任务最终化账本不存在', 409);
|
|
}
|
|
// Worker owns User -> Job. Continue with Backup -> Host -> Restore ->
|
|
// Attempt, then let the allocator own catalog -> reservation -> claims.
|
|
$backup = CloudInstanceBackup::where('id', (int) $backupHint->id)
|
|
->lockForUpdate()->first();
|
|
$host = CloudHost::where('id', (int) $attemptHint->target_host_id)
|
|
->lockForUpdate()->first();
|
|
$restore = CloudBackupRestore::where('id', (int) $restoreHint->id)
|
|
->lockForUpdate()->first();
|
|
$attempt = CloudBackupRestoreAttempt::where(
|
|
'id', (int) $attemptHint->id
|
|
)->where('attempt_key', $attemptKey)->lockForUpdate()->first();
|
|
if (!$backup instanceof CloudInstanceBackup
|
|
|| !$host instanceof CloudHost
|
|
|| !$restore instanceof CloudBackupRestore
|
|
|| !$attempt instanceof CloudBackupRestoreAttempt) {
|
|
throw new CloudError('恢复任务最终化账本已经变化', 409);
|
|
}
|
|
try {
|
|
$plan = CloudPortableRestoreTargetPlan::fromStored(
|
|
(string) $attempt->target_plan_json,
|
|
(string) $attempt->target_plan_sha256
|
|
);
|
|
} catch (\InvalidArgumentException) {
|
|
throw new CloudError('恢复任务保存的目标计划损坏', 409);
|
|
}
|
|
$planValue = $plan->toArray();
|
|
$backupEvidence = CloudPortableRestoreBackupEvidence::fromBackup(
|
|
$backup
|
|
);
|
|
if ((int) $attempt->job_id !== (int) $job->id
|
|
|| (int) $job->backup_restore_id !== (int) $restore->id
|
|
|| (int) $job->backup_id !== (int) $backup->id
|
|
|| (int) $job->user_id !== (int) $backup->user_id
|
|
|| (int) $job->host_id !== (int) $host->id
|
|
|| (string) $job->kind !== 'backup_restore'
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (int) $attempt->backup_restore_id !== (int) $restore->id
|
|
|| (int) $attempt->target_host_id !== (int) $host->id
|
|
|| (int) $restore->target_host_id !== (int) $host->id
|
|
|| (int) $restore->backup_id !== (int) $backup->id
|
|
|| (int) $planValue['target']['host_id'] !== (int) $host->id
|
|
|| !hash_equals((string) $job->operation_id,
|
|
(string) $attempt->operation_id)
|
|
|| !hash_equals((string) $job->request_idempotency_key,
|
|
(string) $attempt->attempt_key)
|
|
|| !hash_equals((string) $restore->selected_plan_json,
|
|
$plan->json())
|
|
|| !hash_equals((string) $restore->selected_plan_sha256,
|
|
$plan->sha256())
|
|
|| CloudCanonicalJson::normalize($planValue['backup'])
|
|
!== CloudCanonicalJson::normalize($backupEvidence->backup())
|
|
|| CloudCanonicalJson::normalize($planValue['resources'])
|
|
!== CloudCanonicalJson::normalize(
|
|
$backupEvidence->resources()
|
|
)) {
|
|
throw new CloudError('恢复任务最终化身份不一致', 409);
|
|
}
|
|
|
|
$reservationHint = CloudResourceReservation::find(
|
|
(int) $attempt->reservation_id
|
|
);
|
|
if (!$reservationHint instanceof CloudResourceReservation) {
|
|
throw new CloudError('恢复任务资源预留不存在', 409);
|
|
}
|
|
if ((string) $reservationHint->state !== 'released') {
|
|
CloudPublicNetworkAllocationService::verifyExistingBundle(
|
|
(int) $host->id,
|
|
(int) $planValue['target']['line_id'],
|
|
(int) $attempt->reservation_id,
|
|
(string) $attempt->operation_id
|
|
);
|
|
}
|
|
$reservation = CloudResourceReservation::where(
|
|
'id', (int) $attempt->reservation_id
|
|
)->lockForUpdate()->first();
|
|
if (!$reservation instanceof CloudResourceReservation
|
|
|| !$attempt->matchesReservation($reservation)
|
|
|| (int) $reservation->user_id !== (int) $backup->user_id
|
|
|| (int) $reservation->host_id !== (int) $host->id
|
|
|| (int) $reservation->resource_pool_id
|
|
!== (int) $planValue['target']['resource_pool_id']
|
|
|| !hash_equals((string) $reservation->operation_id,
|
|
(string) $attempt->operation_id)) {
|
|
throw new CloudError('恢复任务资源预留身份不一致', 409);
|
|
}
|
|
foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $field) {
|
|
if ((int) $reservation->{$field}
|
|
!== (int) $planValue['resources'][$field]) {
|
|
throw new CloudError('恢复任务资源规格已经变化', 409);
|
|
}
|
|
}
|
|
$network = self::frozenNetwork($attempt, $reservation);
|
|
self::assertPayload($job, $payload, $backup, $restore, $attempt,
|
|
$reservation, $planValue, $network);
|
|
return compact('job', 'payload', 'backup', 'restore', 'attempt',
|
|
'reservation', 'host', 'plan', 'planValue', 'network');
|
|
}
|
|
|
|
private static function frozenNetwork(CloudBackupRestoreAttempt $attempt,
|
|
CloudResourceReservation $reservation
|
|
): CloudPublicNetworkSnapshot {
|
|
$state = (string) $reservation->state;
|
|
if ($state === 'released') {
|
|
try {
|
|
$decoded = json_decode(
|
|
(string) $attempt->target_public_network_json,
|
|
true,
|
|
64,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
$network = CloudPublicNetworkSnapshot::fromArray($decoded);
|
|
} catch (\Throwable) {
|
|
throw new CloudError('恢复任务冻结双栈网络损坏', 409);
|
|
}
|
|
if (!hash_equals($network->fingerprint(),
|
|
(string) $attempt->target_public_network_sha256)) {
|
|
throw new CloudError('恢复任务冻结双栈网络摘要不一致', 409);
|
|
}
|
|
return $network;
|
|
}
|
|
$claimSet = CloudPublicNetworkAllocationService::currentClaimSet(
|
|
(int) $reservation->id,
|
|
$state,
|
|
$state === 'committed' ? (int) $reservation->instance_id : null
|
|
);
|
|
try {
|
|
return CloudFrozenPublicNetworkIdentity::fromStored(
|
|
(int) $reservation->id,
|
|
(int) $attempt->target_host_id,
|
|
(string) $attempt->target_uuid,
|
|
(int) $attempt->target_network_attachment_id,
|
|
(string) $attempt->target_identity_json,
|
|
(string) $attempt->target_identity_sha256,
|
|
(string) $attempt->target_public_network_json,
|
|
(string) $attempt->target_public_network_sha256,
|
|
(string) $attempt->target_claim_set_sha256,
|
|
$claimSet
|
|
)->publicNetwork();
|
|
} catch (\InvalidArgumentException) {
|
|
throw new CloudError('恢复任务冻结双栈身份损坏', 409);
|
|
}
|
|
}
|
|
|
|
private static function assertPayload(CloudJob $job, array $payload,
|
|
CloudInstanceBackup $backup, CloudBackupRestore $restore,
|
|
CloudBackupRestoreAttempt $attempt,
|
|
CloudResourceReservation $reservation, array $plan,
|
|
CloudPublicNetworkSnapshot $network): void
|
|
{
|
|
$body = $payload['body'] ?? null;
|
|
try {
|
|
$wireNetwork = is_array($body)
|
|
? CloudPublicNetworkSnapshot::fromArray(
|
|
$body['public_network'] ?? null
|
|
) : null;
|
|
} catch (\Throwable) {
|
|
$wireNetwork = null;
|
|
}
|
|
if (!is_array($body) || array_is_list($body)
|
|
|| !$wireNetwork instanceof CloudPublicNetworkSnapshot
|
|
|| ($payload['method'] ?? null) !== 'POST'
|
|
|| !hash_equals((string) ($payload['request_target'] ?? ''),
|
|
'/v1/vms/' . rawurlencode(
|
|
strtolower((string) $attempt->target_uuid)
|
|
) . '/restore-backup')
|
|
|| !hash_equals((string) ($payload['attempt_key'] ?? ''),
|
|
(string) $attempt->attempt_key)
|
|
|| (int) ($payload['generation'] ?? 0)
|
|
!== (int) $attempt->generation
|
|
|| (int) ($payload['reservation_id'] ?? 0)
|
|
!== (int) $reservation->id
|
|
|| (int) ($payload['owner_id'] ?? 0) !== (int) $backup->user_id
|
|
|| !hash_equals((string) ($payload['backup_key'] ?? ''),
|
|
(string) $backup->backup_key)
|
|
|| !hash_equals((string) ($payload['restore_key'] ?? ''),
|
|
(string) $restore->restore_key)
|
|
|| !hash_equals((string) ($payload['target_plan_sha256'] ?? ''),
|
|
(string) $attempt->target_plan_sha256)
|
|
|| !hash_equals(
|
|
(string) ($payload['target_identity_sha256'] ?? ''),
|
|
(string) $attempt->target_identity_sha256)
|
|
|| !hash_equals(
|
|
(string) ($payload['target_claim_set_sha256'] ?? ''),
|
|
(string) $attempt->target_claim_set_sha256)
|
|
|| !hash_equals(
|
|
(string) ($payload['target_public_network_sha256'] ?? ''),
|
|
$network->fingerprint())
|
|
|| !hash_equals((string) ($body['operation_id'] ?? ''),
|
|
(string) $job->operation_id)
|
|
|| !hash_equals(strtolower((string) ($body['uuid'] ?? '')),
|
|
strtolower((string) $attempt->target_uuid))
|
|
|| !hash_equals((string) ($body['name'] ?? ''),
|
|
(string) $attempt->target_name)
|
|
|| (int) ($body['vcpu'] ?? 0)
|
|
!== (int) $plan['resources']['cpu_cores']
|
|
|| (int) ($body['memory_mib'] ?? 0)
|
|
!== (int) $plan['resources']['memory_mib']
|
|
|| (int) ($body['disk_gib'] ?? 0)
|
|
!== (int) $plan['resources']['disk_gib']
|
|
|| (int) ($body['bandwidth_mbps'] ?? 0)
|
|
!== (int) $plan['resources']['bandwidth_mbps']
|
|
|| !hash_equals($wireNetwork->fingerprint(),
|
|
$network->fingerprint())
|
|
|| !hash_equals((string) ($body['backup_key'] ?? ''),
|
|
(string) $backup->backup_key)
|
|
|| (int) ($body['source_size_bytes'] ?? 0)
|
|
!== (int) $backup->object_size_bytes
|
|
|| !hash_equals((string) ($body['source_sha256'] ?? ''),
|
|
(string) $backup->sha256)
|
|
|| array_key_exists('source_url', $body)
|
|
|| array_key_exists('vpc_port', $body)) {
|
|
throw new CloudError('恢复任务冻结信封与账本不一致', 409);
|
|
}
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function recordEvidenceLocked(array $context,
|
|
string $status, mixed $result): void
|
|
{
|
|
$attempt = $context['attempt'];
|
|
$evidence = self::evidence($context, $status, $result);
|
|
$json = self::encode($evidence);
|
|
$digest = hash('sha256', $json);
|
|
$resolution = self::resolution($status);
|
|
$recoveringUnknown = false;
|
|
if ((string) $attempt->evidence_resolution !== 'not_applicable') {
|
|
if (hash_equals((string) $attempt->evidence_resolution,
|
|
$resolution)
|
|
&& hash_equals((string) $attempt->evidence_sha256, $digest)
|
|
&& hash_equals((string) $attempt->evidence_json, $json)) {
|
|
return;
|
|
}
|
|
$recoveringUnknown = (string) $attempt->state === 'effect_unknown'
|
|
&& (string) $attempt->effect_resolution === 'unknown'
|
|
&& (string) $attempt->evidence_resolution === 'unknown'
|
|
&& in_array($resolution, ['applied', 'not_applied'], true)
|
|
&& (int) $attempt->active_attempt === 1
|
|
&& $attempt->terminal_at === null
|
|
&& (string) $context['reservation']->state === 'uncertain';
|
|
if (!$recoveringUnknown) {
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
'恢复尝试已有不同的持久效果证据', 409
|
|
);
|
|
}
|
|
}
|
|
if (!$recoveringUnknown && (!in_array((string) $attempt->state,
|
|
['queued', 'awaiting_receipt'], true)
|
|
|| (string) $attempt->effect_resolution !== 'not_applicable'
|
|
|| (int) $attempt->active_attempt !== 1
|
|
|| $attempt->effect_resolved_at !== null
|
|
|| $attempt->terminal_at !== null
|
|
|| (string) $context['reservation']->state !== 'held')) {
|
|
throw new CloudError('恢复尝试当前不能记录效果证据', 409);
|
|
}
|
|
$now = time();
|
|
$attempt->evidence_resolution = $resolution;
|
|
$attempt->evidence_json = $json;
|
|
$attempt->evidence_sha256 = $digest;
|
|
$attempt->evidence_recorded_at = $now;
|
|
$attempt->finalization_error_code = '';
|
|
if ($resolution === 'unknown') {
|
|
CloudPublicNetworkAllocationService::fenceBundle(
|
|
(int) $context['reservation']->id,
|
|
(string) $attempt->operation_id
|
|
);
|
|
$reservation = $context['reservation'];
|
|
$reservation->state = 'uncertain';
|
|
$reservation->expires_at = null;
|
|
$reservation->updated_at = $now;
|
|
$reservation->save();
|
|
$attempt->state = 'effect_unknown';
|
|
$attempt->effect_resolution = 'unknown';
|
|
$attempt->last_error_code = 'operation_effect_unknown';
|
|
$attempt->effect_resolved_at = null;
|
|
$attempt->terminal_at = null;
|
|
self::projectParent($context, 'uncertain', $now);
|
|
} else {
|
|
$attempt->state = 'finalization_pending';
|
|
$attempt->effect_resolution = $resolution;
|
|
$attempt->last_error_code = '';
|
|
$attempt->effect_resolved_at = $now;
|
|
$attempt->terminal_at = null;
|
|
}
|
|
$attempt->updated_at = $now;
|
|
$attempt->save();
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function assertMatchingEvidence(array $context,
|
|
string $status, mixed $result): void
|
|
{
|
|
$attempt = $context['attempt'];
|
|
$json = self::encode(self::evidence($context, $status, $result));
|
|
if (!hash_equals((string) $attempt->evidence_resolution,
|
|
self::resolution($status))
|
|
|| !hash_equals((string) $attempt->evidence_json, $json)
|
|
|| !hash_equals((string) $attempt->evidence_sha256,
|
|
hash('sha256', $json))
|
|
|| $attempt->evidence_recorded_at === null) {
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
'恢复尝试持久证据与最终化结论不一致', 409
|
|
);
|
|
}
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function evidence(array $context, string $status,
|
|
mixed $result): array
|
|
{
|
|
if ($status === 'failed') {
|
|
if (!is_array($result) || array_is_list($result)
|
|
|| ($result['creation_absent'] ?? null) !== true
|
|
|| ($result['managed_artifacts_absent'] ?? null) !== true) {
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
'恢复未生效结论缺少完整资源缺席证据', 409
|
|
);
|
|
}
|
|
return [
|
|
'creation_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
];
|
|
}
|
|
if ($status === 'uncertain') {
|
|
return ['effect_may_have_applied' => true];
|
|
}
|
|
$vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
|
|
if (!is_array($result) || array_is_list($result)
|
|
|| !is_array($vm) || array_is_list($vm)) {
|
|
throw new CloudError('Agent 恢复成功结果结构无效', 502);
|
|
}
|
|
try {
|
|
$network = CloudPublicNetworkSnapshot::fromArray(
|
|
$vm['public_network'] ?? null
|
|
);
|
|
} catch (\Throwable) {
|
|
throw new CloudError('Agent 恢复成功结果缺少双栈公网快照', 502);
|
|
}
|
|
$attempt = $context['attempt'];
|
|
$backup = $context['backup'];
|
|
$resources = $context['planValue']['resources'];
|
|
$compatibility = CloudPortableRestoreCompatibility::supported()
|
|
->toArray();
|
|
$value = [
|
|
'uuid' => strtolower((string) ($vm['uuid'] ?? '')),
|
|
'name' => (string) ($vm['name'] ?? ''),
|
|
'status' => (string) ($vm['status'] ?? ''),
|
|
'vcpu' => (int) ($vm['vcpu'] ?? 0),
|
|
'memory_mib' => (int) ($vm['memory_mib'] ?? 0),
|
|
'disk_gib' => (int) ($vm['disk_gib'] ?? 0),
|
|
'bandwidth_mbps' => (int) ($vm['bandwidth_mbps'] ?? 0),
|
|
'public_network' => $network->toArray(),
|
|
'machine' => (string) ($vm['machine'] ?? ''),
|
|
'firmware' => (string) ($vm['firmware'] ?? ''),
|
|
'secure_boot' => $vm['secure_boot'] ?? null,
|
|
'backup_key' => (string) ($result['backup_key']
|
|
?? $vm['backup_key'] ?? ''),
|
|
'sha256' => (string) ($result['sha256'] ?? $vm['sha256'] ?? ''),
|
|
];
|
|
if (!hash_equals($value['uuid'],
|
|
strtolower((string) $attempt->target_uuid))
|
|
|| !hash_equals($value['name'], (string) $attempt->target_name)
|
|
|| $value['vcpu'] !== (int) $resources['cpu_cores']
|
|
|| $value['memory_mib'] !== (int) $resources['memory_mib']
|
|
|| $value['disk_gib'] !== (int) $resources['disk_gib']
|
|
|| $value['bandwidth_mbps']
|
|
!== (int) $resources['bandwidth_mbps']
|
|
|| !hash_equals($network->fingerprint(),
|
|
$context['network']->fingerprint())
|
|
|| !hash_equals($value['backup_key'],
|
|
(string) $backup->backup_key)
|
|
|| !hash_equals($value['sha256'], (string) $backup->sha256)) {
|
|
throw new CloudError(
|
|
'Agent 恢复成功结果与冻结备份、规格或双栈网络不一致',
|
|
502
|
|
);
|
|
}
|
|
if (!hash_equals(
|
|
$compatibility['machine'],
|
|
$value['machine']
|
|
)
|
|
|| !hash_equals(
|
|
$compatibility['firmware'],
|
|
$value['firmware']
|
|
)
|
|
|| !is_bool($value['secure_boot'])
|
|
|| $value['secure_boot'] !== $compatibility['secure_boot']) {
|
|
throw new CloudError(
|
|
'Agent 恢复成功结果与冻结平台兼容合同不一致',
|
|
502
|
|
);
|
|
}
|
|
return $value;
|
|
}
|
|
|
|
private static function resolution(string $status): string
|
|
{
|
|
return $status === 'succeeded' ? 'applied'
|
|
: ($status === 'failed' ? 'not_applied' : 'unknown');
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function settleSucceeded(array $context): void
|
|
{
|
|
$attempt = $context['attempt'];
|
|
$reservation = $context['reservation'];
|
|
if ((string) $attempt->effect_resolution !== 'applied'
|
|
|| (int) $attempt->active_attempt !== 1
|
|
|| $attempt->effect_resolved_at === null
|
|
|| $attempt->terminal_at !== null
|
|
|| !in_array((string) $reservation->state,
|
|
['held', 'uncertain'], true)
|
|
|| $reservation->instance_id !== null) {
|
|
throw new CloudError('恢复成功证据与资源状态不一致', 409);
|
|
}
|
|
$evidence = self::decodedEvidence($attempt);
|
|
$now = time();
|
|
$instance = self::instanceForSuccess($context, $evidence, $now);
|
|
$committed = CloudPublicNetworkAllocationService::commitBundle(
|
|
(int) $reservation->id,
|
|
(int) $instance->id,
|
|
(string) $attempt->operation_id
|
|
);
|
|
if (!hash_equals($context['network']->fingerprint(),
|
|
$committed->fingerprint())) {
|
|
throw new CloudError('恢复成功回执与双栈公网预留不一致', 409);
|
|
}
|
|
$reservation->state = 'committed';
|
|
$reservation->instance_id = (int) $instance->id;
|
|
$reservation->expires_at = null;
|
|
$reservation->released_at = null;
|
|
$reservation->release_reason = '';
|
|
$reservation->updated_at = $now;
|
|
$reservation->save();
|
|
$job = $context['job'];
|
|
$job->instance_id = (int) $instance->id;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
$restore = $context['restore'];
|
|
$restore->target_instance_id = (int) $instance->id;
|
|
$restore->state = 'ready';
|
|
$restore->error_code = '';
|
|
$restore->ready_at = $now;
|
|
$restore->updated_at = $now;
|
|
$restore->save();
|
|
$attempt->state = 'succeeded';
|
|
$attempt->active_attempt = null;
|
|
$attempt->terminal_at = $now;
|
|
$attempt->last_error_code = '';
|
|
$attempt->finalization_error_code = '';
|
|
$attempt->updated_at = $now;
|
|
$attempt->save();
|
|
self::recordRuntime($instance, $evidence);
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function settleFailed(array $context): void
|
|
{
|
|
$attempt = $context['attempt'];
|
|
$reservation = $context['reservation'];
|
|
if ((string) $attempt->effect_resolution !== 'not_applied'
|
|
|| (int) $attempt->active_attempt !== 1
|
|
|| $attempt->effect_resolved_at === null
|
|
|| $attempt->terminal_at !== null
|
|
|| !in_array((string) $reservation->state,
|
|
['held', 'uncertain'], true)
|
|
|| $reservation->instance_id !== null
|
|
|| CloudInstance::where('host_id',
|
|
(int) $attempt->target_host_id)
|
|
->where('uuid', (string) $attempt->target_uuid)
|
|
->where('status', '<>', 'deleted')->exists()) {
|
|
throw new CloudError('恢复未生效证据与资源状态不一致', 409);
|
|
}
|
|
$now = time();
|
|
CloudPublicNetworkAllocationService::releaseBundle(
|
|
(int) $reservation->id,
|
|
(string) $attempt->operation_id,
|
|
'backup_restore_not_applied',
|
|
true
|
|
);
|
|
$reservation->state = 'released';
|
|
$reservation->instance_id = null;
|
|
$reservation->expires_at = null;
|
|
$reservation->released_at = $now;
|
|
$reservation->release_reason = 'backup_restore_not_applied';
|
|
$reservation->updated_at = $now;
|
|
$reservation->save();
|
|
self::projectParent($context, 'failed', $now);
|
|
$attempt->state = 'failed';
|
|
$attempt->active_attempt = null;
|
|
$attempt->terminal_at = $now;
|
|
$attempt->last_error_code = 'backup_restore_not_applied';
|
|
$attempt->finalization_error_code = '';
|
|
$attempt->updated_at = $now;
|
|
$attempt->save();
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function projectParent(array $context, string $state,
|
|
int $now): void
|
|
{
|
|
$restore = $context['restore'];
|
|
$restore->target_instance_id = null;
|
|
$restore->state = $state;
|
|
$restore->error_code = $state === 'uncertain'
|
|
? 'operation_effect_unknown' : 'backup_restore_not_applied';
|
|
$restore->ready_at = null;
|
|
$restore->updated_at = $now;
|
|
$restore->save();
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function instanceForSuccess(array $context,
|
|
array $evidence, int $now): CloudInstance
|
|
{
|
|
$attempt = $context['attempt'];
|
|
$reservation = $context['reservation'];
|
|
$backup = $context['backup'];
|
|
$byReservation = CloudInstance::where('reservation_id',
|
|
(int) $reservation->id)->lockForUpdate()->first();
|
|
$byUuid = CloudInstance::where('host_id',
|
|
(int) $attempt->target_host_id)
|
|
->where('uuid', (string) $attempt->target_uuid)
|
|
->lockForUpdate()->first();
|
|
if ($byReservation instanceof CloudInstance
|
|
&& $byUuid instanceof CloudInstance
|
|
&& (int) $byReservation->id !== (int) $byUuid->id) {
|
|
throw new CloudError('恢复成功实例身份发生冲突', 409);
|
|
}
|
|
$instance = $byReservation ?? $byUuid ?? new CloudInstance();
|
|
if ($instance->exists
|
|
&& ((int) $instance->host_id !== (int) $attempt->target_host_id
|
|
|| (int) $instance->user_id !== (int) $backup->user_id
|
|
|| (int) $instance->reservation_id
|
|
!== (int) $reservation->id
|
|
|| !hash_equals(strtolower((string) $instance->uuid),
|
|
strtolower((string) $attempt->target_uuid)))) {
|
|
throw new CloudError('恢复成功实例归属发生冲突', 409);
|
|
}
|
|
$source = self::decodeSpec((string) $backup->spec_json);
|
|
$spec = CloudInput::safeResult($source);
|
|
foreach ([
|
|
'plan_id', 'offer_id', 'price_version', 'stock_mode',
|
|
'validity_days', 'traffic_quota_bytes', 'renewal_price',
|
|
'currency', 'public_ip', 'public_ip_cidr', 'public_ipv4',
|
|
'public_ipv4_cidr', 'public_ipv6', 'public_ipv6_cidr',
|
|
'public_pool_key', 'public_pool_cidr', 'gateway', 'dns_servers',
|
|
'resource_pool_id', 'resource_pool_name', 'line_id', 'line_name',
|
|
'region_id', 'region', 'country_code', 'vpc_port', 'vpc_port_id',
|
|
'private_ip', 'private_mac', 'guest_hostname', 'hostname',
|
|
'root_password', 'guest_ssh_public_key', 'sealed_credential',
|
|
] as $field) {
|
|
unset($spec[$field]);
|
|
}
|
|
$spec = CloudInstanceMetadata::merge($spec, $evidence);
|
|
unset($spec['guest_hostname'], $spec['hostname']);
|
|
$resources = $context['planValue']['resources'];
|
|
$target = $context['planValue']['target'];
|
|
$public = $context['network']->toArray();
|
|
$spec = array_replace($spec, [
|
|
'managed' => true,
|
|
'vcpu' => (int) $resources['cpu_cores'],
|
|
'cpu_cores' => (int) $resources['cpu_cores'],
|
|
'memory_mib' => (int) $resources['memory_mib'],
|
|
'disk_gib' => (int) $resources['disk_gib'],
|
|
'bandwidth_mbps' => (int) $resources['bandwidth_mbps'],
|
|
'network' => (string) $public['network_selector'],
|
|
'public_network' => $public,
|
|
'resource_pool_id' => (int) $target['resource_pool_id'],
|
|
'resource_pool_name' => (string) $target['resource_pool_name'],
|
|
'line_id' => (int) $target['line_id'],
|
|
'line_name' => (string) $target['line_name'],
|
|
'region_id' => (int) $target['region_id'],
|
|
'region' => (string) $target['region_name'],
|
|
'country_code' => (string) $target['country_code'],
|
|
'restored_from_backup' => (string) $backup->backup_key,
|
|
]);
|
|
foreach (['root_password', 'guest_hostname', 'hostname',
|
|
'guest_ssh_public_key', 'vpc_port', 'vpc_port_id'] as $field) {
|
|
unset($spec[$field]);
|
|
}
|
|
$instance->uuid = strtolower((string) $attempt->target_uuid);
|
|
$instance->host_id = (int) $attempt->target_host_id;
|
|
$instance->user_id = (int) $backup->user_id;
|
|
$instance->order_id = null;
|
|
$instance->plan_id = null;
|
|
$instance->offer_id = null;
|
|
$instance->resource_pool_id = (int) $reservation->resource_pool_id;
|
|
$instance->reservation_id = (int) $reservation->id;
|
|
$instance->current_period_id = null;
|
|
$instance->name = (string) $attempt->target_name;
|
|
$instance->status = 'active';
|
|
$instance->service_status = 'exempt';
|
|
$instance->network_status = 'active';
|
|
$instance->expires_at = null;
|
|
$instance->renewal_price = null;
|
|
$instance->renewal_days = 0;
|
|
$instance->renewal_traffic_bytes = 0;
|
|
$instance->traffic_quota_bytes = 0;
|
|
$instance->traffic_download_bytes = 0;
|
|
$instance->traffic_upload_bytes = 0;
|
|
$instance->lifetime_download_bytes = 0;
|
|
$instance->lifetime_upload_bytes = 0;
|
|
$instance->traffic_synced_at = 0;
|
|
$instance->suspended_at = null;
|
|
$instance->network_suspended_at = null;
|
|
$instance->spec_json = self::encode($spec);
|
|
$instance->sealed_credential = null;
|
|
if (!$instance->exists) {
|
|
$instance->created_at = $now;
|
|
}
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
return $instance;
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function verifySucceeded(array $context): void
|
|
{
|
|
$attempt = $context['attempt'];
|
|
$restore = $context['restore'];
|
|
$reservation = $context['reservation'];
|
|
$instance = $restore->target_instance_id === null ? null
|
|
: CloudInstance::where('id', (int) $restore->target_instance_id)
|
|
->lockForUpdate()->first();
|
|
if (!$instance instanceof CloudInstance
|
|
|| (int) $context['job']->instance_id !== (int) $instance->id
|
|
|| (string) $restore->state !== 'ready'
|
|
|| (string) $attempt->effect_resolution !== 'applied'
|
|
|| $attempt->active_attempt !== null
|
|
|| $attempt->effect_resolved_at === null
|
|
|| $attempt->terminal_at === null
|
|
|| (string) $reservation->state !== 'committed'
|
|
|| (int) $reservation->instance_id !== (int) $instance->id
|
|
|| (int) $instance->reservation_id !== (int) $reservation->id
|
|
|| (int) $instance->host_id !== (int) $attempt->target_host_id
|
|
|| (int) $instance->user_id !== (int) $context['backup']->user_id
|
|
|| $instance->sealed_credential !== null) {
|
|
throw new CloudError('已完成恢复的实例或资源账本不一致', 409);
|
|
}
|
|
CloudPublicNetworkAllocationService::currentClaimSet(
|
|
(int) $reservation->id, 'committed', (int) $instance->id
|
|
);
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function verifyFailed(array $context): void
|
|
{
|
|
$attempt = $context['attempt'];
|
|
$restore = $context['restore'];
|
|
$reservation = $context['reservation'];
|
|
if ((string) $attempt->effect_resolution !== 'not_applied'
|
|
|| $attempt->active_attempt !== null
|
|
|| $attempt->effect_resolved_at === null
|
|
|| $attempt->terminal_at === null
|
|
|| (string) $restore->state !== 'failed'
|
|
|| $restore->target_instance_id !== null
|
|
|| (string) $reservation->state !== 'released'
|
|
|| $reservation->instance_id !== null) {
|
|
throw new CloudError('已失败恢复的释放账本不一致', 409);
|
|
}
|
|
CloudReleasedPublicNetworkBundleVerifier::verify(
|
|
$attempt, $reservation, (int) $context['backup']->user_id
|
|
);
|
|
}
|
|
|
|
/** @param array<string,mixed> $context */
|
|
private static function verifyUnknown(array $context): void
|
|
{
|
|
$attempt = $context['attempt'];
|
|
$restore = $context['restore'];
|
|
$reservation = $context['reservation'];
|
|
if ((string) $attempt->effect_resolution !== 'unknown'
|
|
|| (int) $attempt->active_attempt !== 1
|
|
|| $attempt->effect_resolved_at !== null
|
|
|| $attempt->terminal_at !== null
|
|
|| (string) $restore->state !== 'uncertain'
|
|
|| $restore->target_instance_id !== null
|
|
|| (string) $reservation->state !== 'uncertain'
|
|
|| $reservation->instance_id !== null) {
|
|
throw new CloudError('效果未知恢复的隔离账本不一致', 409);
|
|
}
|
|
CloudPublicNetworkAllocationService::currentClaimSet(
|
|
(int) $reservation->id, 'uncertain'
|
|
);
|
|
}
|
|
|
|
private static function decodedEvidence(
|
|
CloudBackupRestoreAttempt $attempt
|
|
): array {
|
|
try {
|
|
$value = json_decode((string) $attempt->evidence_json, true, 64,
|
|
JSON_THROW_ON_ERROR);
|
|
} catch (\JsonException) {
|
|
throw new CloudError('恢复尝试持久证据损坏', 409);
|
|
}
|
|
if (!is_array($value)
|
|
|| !hash_equals(hash('sha256', self::encode($value)),
|
|
(string) $attempt->evidence_sha256)) {
|
|
throw new CloudError('恢复尝试持久证据摘要不一致', 409);
|
|
}
|
|
return $value;
|
|
}
|
|
|
|
private static function decodeSpec(string $encoded): array
|
|
{
|
|
try {
|
|
$spec = json_decode($encoded, true, 64, JSON_THROW_ON_ERROR);
|
|
} catch (\JsonException) {
|
|
throw new CloudError('实例备份规格快照损坏', 409);
|
|
}
|
|
if (!is_array($spec) || array_is_list($spec)) {
|
|
throw new CloudError('实例备份规格快照损坏', 409);
|
|
}
|
|
return $spec;
|
|
}
|
|
|
|
private static function encode(array $value): string
|
|
{
|
|
return json_encode(CloudCanonicalJson::normalize($value),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
| JSON_UNESCAPED_LINE_TERMINATORS);
|
|
}
|
|
|
|
private static function recordRuntime(CloudInstance $instance,
|
|
array $result): void
|
|
{
|
|
try {
|
|
CloudInstanceRuntimeService::seed((int) $instance->id,
|
|
(int) $instance->host_id, (string) $instance->uuid);
|
|
$state = $result['status']
|
|
?? ($result['state'] ?? ($result['power_state'] ?? null));
|
|
if (CloudInstanceRuntimeService::normalizeState($state)
|
|
!== 'unknown') {
|
|
CloudInstanceRuntimeService::markObserved((int) $instance->id,
|
|
(int) $instance->host_id, (string) $instance->uuid,
|
|
$state);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
error_log('portable restore runtime cache unavailable: '
|
|
. get_class($error));
|
|
}
|
|
}
|
|
}
|