You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2311 lines
97 KiB
PHP
2311 lines
97 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudBackupRestoreAttempt;
|
|
use App\Models\CloudBackupRecovery;
|
|
use App\Models\CloudBackupPeriodEntitlement;
|
|
use App\Models\CloudBackupSubscription;
|
|
use App\Models\CloudDeploymentSnapshot;
|
|
use App\Models\CloudHost;
|
|
use App\Models\CloudHostAgent;
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Models\CloudInstanceMigration;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudResourceReservation;
|
|
|
|
/** PHP-owned catalog and resource ledger for portable system-disk backups. */
|
|
final class CloudBackupService
|
|
{
|
|
public const MAX_PER_INSTANCE = 10;
|
|
|
|
private const ACTIVE_BACKUP_STATES = [
|
|
'creating', 'ready', 'uncertain',
|
|
];
|
|
private const CLEANUP_DEBT_STATES = [
|
|
'deleting', 'delete_uncertain', 'delete_attention',
|
|
];
|
|
private const TRANSFER_URL_SECONDS = 604800;
|
|
private const GIB = 1073741824;
|
|
private const MAX_SCHEDULE_ATTEMPT = 4294967295;
|
|
|
|
/**
|
|
* Global administrator backup catalog. It is intentionally keyed by the
|
|
* backup ledger instead of the source VM lifecycle, so a retained backup
|
|
* remains restorable and deletable after expiry cleanup tombstones the
|
|
* source instance.
|
|
*/
|
|
public static function library(array $actor, array $input): array
|
|
{
|
|
if (($actor['is_admin'] ?? false) !== true) {
|
|
throw new CloudError('只有管理员可以查看虚拟机备份', 403);
|
|
}
|
|
$filters = self::libraryFilters($input);
|
|
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
|
|
$perPage = self::libraryInteger($input['per_page'] ?? 20,
|
|
20, 50, 20);
|
|
if (!in_array($perPage, [20, 50], true)) {
|
|
throw new CloudError('备份库每页条数仅支持 20 或 50', 400);
|
|
}
|
|
$query = CloudInstanceBackup::where('purpose', 'portable')
|
|
->where('state', '<>', 'deleted');
|
|
foreach (['user_id' => 'user_id', 'instance_id' => 'source_instance_id',
|
|
'host_id' => 'source_host_id', 'state' => 'state'] as $key => $column) {
|
|
if ($filters[$key] !== '') {
|
|
$query->where($column, $filters[$key]);
|
|
}
|
|
}
|
|
if ($filters['q'] !== '') {
|
|
$like = '%' . str_replace(['!', '%', '_'], ['!!', '!%', '!_'], $filters['q']) . '%';
|
|
$query->whereRaw("name LIKE ? ESCAPE '!'", [$like]);
|
|
}
|
|
$total = (int) (clone $query)->count('id');
|
|
$totalPages = max(1, (int) ceil($total / $perPage));
|
|
$page = min($page, $totalPages);
|
|
$rows = $query->orderByDesc('id')
|
|
->offset(($page - 1) * $perPage)->limit($perPage)->get();
|
|
|
|
$sourceIds = [];
|
|
$hostIds = [];
|
|
$operationIds = [];
|
|
foreach ($rows as $row) {
|
|
$sourceIds[] = (int) $row->source_instance_id;
|
|
$hostIds[] = (int) $row->source_host_id;
|
|
if (preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) $row->last_operation_id) === 1) {
|
|
$operationIds[] = (string) $row->last_operation_id;
|
|
}
|
|
}
|
|
$sources = [];
|
|
if ($sourceIds !== []) {
|
|
foreach (CloudInstance::whereIn('id', array_values(array_unique(
|
|
$sourceIds
|
|
)))->get(['id', 'name', 'status', 'user_id']) as $source) {
|
|
$sources[(int) $source->id] = $source;
|
|
}
|
|
}
|
|
$hosts = [];
|
|
if ($hostIds !== []) {
|
|
foreach (CloudHost::whereIn('id', array_values(array_unique(
|
|
$hostIds
|
|
)))->get(['id', 'name', 'region']) as $host) {
|
|
$hosts[(int) $host->id] = $host;
|
|
}
|
|
}
|
|
$jobs = [];
|
|
if ($operationIds !== []) {
|
|
foreach (CloudJob::whereIn('operation_id', array_values(array_unique(
|
|
$operationIds
|
|
)))->get() as $job) {
|
|
$jobs[(string) $job->operation_id] = $job;
|
|
}
|
|
}
|
|
|
|
$canManage = ($actor['is_super_admin'] ?? false) === true
|
|
&& ($actor['can_manage'] ?? false) === true;
|
|
$items = [];
|
|
foreach ($rows as $row) {
|
|
$source = $sources[(int) $row->source_instance_id] ?? null;
|
|
$host = $hosts[(int) $row->source_host_id] ?? null;
|
|
$item = self::displayBackup(
|
|
$row,
|
|
$jobs[(string) $row->last_operation_id] ?? null,
|
|
$canManage
|
|
);
|
|
$item['source'] = [
|
|
'instance_id' => (string) $row->source_instance_id,
|
|
'name' => $source === null ? '' : (string) $source->name,
|
|
'status' => $source === null ? 'missing'
|
|
: (string) $source->status,
|
|
'deleted' => $source === null
|
|
|| (string) $source->status === 'deleted',
|
|
'user_id' => (string) $row->user_id,
|
|
'host_id' => (string) $row->source_host_id,
|
|
'host_name' => $host === null ? '' : (string) $host->name,
|
|
'region' => $host === null ? '' : (string) $host->region,
|
|
];
|
|
if ((string) $row->state === 'ready') {
|
|
$projection = self::restoreTargetProjection($row);
|
|
$item['restore_targets'] = $projection['targets'];
|
|
$item['restore_unavailable_reason'] =
|
|
$projection['reason'];
|
|
}
|
|
$items[] = $item;
|
|
}
|
|
$from = $total === 0 ? 0 : (($page - 1) * $perPage) + 1;
|
|
return [
|
|
'items' => $items,
|
|
'filters' => $filters,
|
|
'can_manage' => $canManage,
|
|
'pagination' => [
|
|
'page' => $page,
|
|
'per_page' => $perPage,
|
|
'total' => $total,
|
|
'total_pages' => $totalPages,
|
|
'from' => $from,
|
|
'to' => $from === 0 ? 0 : $from + count($items) - 1,
|
|
],
|
|
];
|
|
}
|
|
|
|
public static function listing(CloudInstance $instance, array $actor): array
|
|
{
|
|
if (($actor['is_admin'] ?? false) !== true) {
|
|
throw new CloudError('只有管理员可以查看虚拟机备份', 403);
|
|
}
|
|
$canManage = ($actor['is_super_admin'] ?? false) === true
|
|
&& ($actor['can_manage'] ?? false) === true;
|
|
if ((string) $instance->status === 'deleted') {
|
|
throw new CloudError('虚拟机不存在', 404);
|
|
}
|
|
$rows = CloudInstanceBackup::where(
|
|
'source_instance_id', (int) $instance->id
|
|
)->where('purpose', 'portable')->where('state', '<>', 'deleted')
|
|
->orderByDesc('id')->get();
|
|
$operationIds = [];
|
|
foreach ($rows as $row) {
|
|
if (preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) $row->last_operation_id) === 1) {
|
|
$operationIds[] = (string) $row->last_operation_id;
|
|
}
|
|
}
|
|
$jobs = [];
|
|
if ($operationIds !== []) {
|
|
foreach (CloudJob::whereIn('operation_id',
|
|
array_values(array_unique($operationIds)))->get() as $job) {
|
|
$jobs[(string) $job->operation_id] = $job;
|
|
}
|
|
}
|
|
$backups = [];
|
|
$targetMap = [];
|
|
$manualActiveCount = 0;
|
|
foreach ($rows as $row) {
|
|
if ((string) $row->source === 'manual'
|
|
&& in_array((string) $row->state,
|
|
self::ACTIVE_BACKUP_STATES, true)) {
|
|
$manualActiveCount++;
|
|
}
|
|
$display = self::displayBackup(
|
|
$row,
|
|
$jobs[(string) $row->last_operation_id] ?? null,
|
|
$canManage
|
|
);
|
|
if ((string) $row->state === 'ready') {
|
|
$projection = self::restoreTargetProjection($row);
|
|
foreach ($projection['targets'] as $normalized) {
|
|
$targetMap[$normalized['target_plan_sha256']] =
|
|
$normalized;
|
|
}
|
|
$display['restore_targets'] = $projection['targets'];
|
|
$display['restore_unavailable_reason'] =
|
|
$projection['reason'];
|
|
}
|
|
$backups[] = $display;
|
|
}
|
|
$active = CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->orderBy('id')->first();
|
|
$spec = self::decodeSpec((string) $instance->spec_json, false);
|
|
$compatibilityFrozen = $spec !== null
|
|
&& is_bool($spec['debian_12_network_fix'] ?? null);
|
|
$supported = $spec !== null
|
|
&& ($spec['managed'] ?? false) === true
|
|
&& ($spec['os_source'] ?? null) === 'cloud_template'
|
|
&& $compatibilityFrozen;
|
|
$storage = CloudObjectStorageBackendRegistry::summary();
|
|
$storageWritable = (int) ($storage['writable_backends'] ?? 0) > 0;
|
|
$migration = CloudInstanceMigration::where(
|
|
'instance_id', (int) $instance->id
|
|
)->orderByDesc('id')->first();
|
|
$migrationTargetCandidates = $supported
|
|
? CloudPlacementService::maintenanceMigrationTargets(
|
|
$instance,
|
|
self::resourcesFromSpec($spec),
|
|
CloudPeerMigrationArtifactService::advisoryAdditionalDiskGib(
|
|
$instance
|
|
)
|
|
) : [];
|
|
$migrationTargets = array_map(
|
|
static fn (array $target): array => [
|
|
'host_id' => (string) ($target['id'] ?? ''),
|
|
'host_name' => (string) ($target['name'] ?? ''),
|
|
'region' => (string) ($target['region'] ?? ''),
|
|
'resource_pool_id' => (string) (
|
|
$target['resource_pool_id'] ?? ''
|
|
),
|
|
'resource_pool' => (string) ($target['resource_pool'] ?? ''),
|
|
'line' => (string) ($target['line'] ?? ''),
|
|
'network_selectors' => array_values(
|
|
is_array($target['network_selectors'] ?? null)
|
|
? $target['network_selectors'] : []
|
|
),
|
|
'available_ip_count' =>
|
|
(int) ($target['available_ip_count'] ?? 0),
|
|
],
|
|
array_values(array_filter(
|
|
$migrationTargetCandidates,
|
|
static fn (array $target): bool =>
|
|
(int) ($target['available_ip_count'] ?? 0) > 0
|
|
))
|
|
);
|
|
$migrationBlocks = $migration !== null
|
|
&& CloudInstanceMigrationWorkflow::blocksInstance(
|
|
(string) $migration->state
|
|
);
|
|
$canOperate = $canManage
|
|
&& $active === null
|
|
&& !$migrationBlocks
|
|
&& (string) $instance->status === 'active'
|
|
&& $supported
|
|
&& $storageWritable;
|
|
$canMigrate = $canManage
|
|
&& $active === null
|
|
&& !($migration !== null
|
|
&& CloudInstanceMigrationWorkflow::blocksNewMigration(
|
|
(string) $migration->state
|
|
))
|
|
&& (string) $instance->status === 'active'
|
|
&& $supported
|
|
&& $migrationTargets !== [];
|
|
$canCreate = $canOperate
|
|
&& $manualActiveCount < self::MAX_PER_INSTANCE;
|
|
$reason = null;
|
|
if (!$canManage) {
|
|
$reason = '当前管理员仅可查看,超级管理员才可以创建或恢复备份';
|
|
} elseif (!$storageWritable) {
|
|
$reason = '请先启用至少一个已通过验证且健康的对象存储后端';
|
|
} elseif ($spec !== null
|
|
&& ($spec['managed'] ?? false) === true
|
|
&& ($spec['os_source'] ?? null) === 'cloud_template'
|
|
&& !$compatibilityFrozen) {
|
|
$reason = '实例缺少创建时冻结的网络兼容快照,不能生成可移植备份';
|
|
} elseif (!$supported) {
|
|
$reason = '仅面板创建的云镜像实例支持可移植备份';
|
|
} elseif ($active !== null) {
|
|
$reason = '实例已有任务,请等待完成';
|
|
} elseif ($migrationBlocks) {
|
|
$reason = '实例已有未完成的维护迁移,请先完成核对或源副本清理';
|
|
} elseif ((string) $instance->status !== 'active') {
|
|
$reason = '只有活动实例可以创建备份';
|
|
} elseif (!$canCreate) {
|
|
$reason = '每台实例最多保留十个有效备份';
|
|
}
|
|
return [
|
|
'instance_id' => (string) $instance->id,
|
|
'backups' => $backups,
|
|
'restore_targets' => array_values($targetMap),
|
|
'migration_targets' => $migrationTargets,
|
|
'maintenance_migration' => $migration === null ? null
|
|
: CloudInstanceMigrationService::display($migration),
|
|
'max_backups' => self::MAX_PER_INSTANCE,
|
|
'can_create' => $canCreate,
|
|
'can_restore' => $canManage && $targetMap !== [],
|
|
'can_migrate' => $canMigrate,
|
|
'create_block_reason' => $reason,
|
|
'active_job' => $active === null ? null : [
|
|
'id' => (string) $active->id,
|
|
'kind' => (string) $active->kind,
|
|
'status' => CloudOperationErrorCatalog::userStatus(
|
|
(string) $active->status
|
|
) ?? 'unknown',
|
|
],
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Delete or reconcile every replica of one backup storage artifact.
|
|
*
|
|
* Per-replica operations own the no-replay and bounded reconciliation
|
|
* contract; the backup row only projects their aggregate cleanup state.
|
|
*/
|
|
public static function delete(CloudInstanceBackup $backup, array $actor,
|
|
?CloudObjectStorageS3Client $client = null,
|
|
?callable $beforeDeleteLocked = null,
|
|
?CloudStorageArtifactDeletionService $artifactDeletion = null): array
|
|
{
|
|
CloudAccess::manage($actor);
|
|
$client ??= new CloudObjectStorageS3Client();
|
|
$now = time();
|
|
$prepared = $backup->getConnection()->transaction(function () use (
|
|
$backup, $beforeDeleteLocked, $now
|
|
): array {
|
|
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
->lockForUpdate()->first();
|
|
if ($locked === null) {
|
|
throw new CloudError('备份不存在', 404);
|
|
}
|
|
$state = (string) $locked->state;
|
|
if ($state === 'deleted') {
|
|
return ['mode' => 'deleted', 'backup' => $locked];
|
|
}
|
|
self::assertBackupDeletionAvailable($locked);
|
|
if ($locked->storage_artifact_id === null) {
|
|
throw new CloudError(
|
|
'备份缺少复制对象存储账本,不能执行删除',
|
|
409
|
|
);
|
|
}
|
|
if ($state === 'delete_attention') {
|
|
return ['mode' => 'attention', 'backup' => $locked];
|
|
}
|
|
if ($locked->storage_artifact_id !== null) {
|
|
if ($state === 'creating') {
|
|
if ($locked->active_job_id !== null
|
|
|| $locked->last_operation_id !== null) {
|
|
throw new CloudError(
|
|
'备份创建已绑定任务,不能按未派发意图取消',
|
|
409
|
|
);
|
|
}
|
|
return [
|
|
'mode' => 'artifact_cancel',
|
|
'artifact_id' =>
|
|
(int) $locked->storage_artifact_id,
|
|
'backup' => $locked,
|
|
];
|
|
}
|
|
if (in_array($state, ['deleting', 'delete_uncertain'], true)) {
|
|
if ((int) $locked->delete_next_check_at > $now) {
|
|
return ['mode' => 'waiting', 'backup' => $locked];
|
|
}
|
|
} elseif (!in_array($state, ['ready', 'failed'], true)) {
|
|
throw new CloudError('当前备份状态不允许删除', 409);
|
|
} else {
|
|
if ($beforeDeleteLocked !== null) {
|
|
$beforeDeleteLocked($locked);
|
|
}
|
|
$locked->state = 'deleting';
|
|
$locked->delete_requested_at = $now;
|
|
$locked->delete_next_check_at = 0;
|
|
$locked->delete_attention_at = null;
|
|
$locked->error_code = '';
|
|
$locked->updated_at = $now;
|
|
$locked->save();
|
|
}
|
|
return [
|
|
'mode' => 'artifact',
|
|
'artifact_id' => (int) $locked->storage_artifact_id,
|
|
'backup' => $locked,
|
|
];
|
|
}
|
|
throw new \LogicException(
|
|
'Replicated backup deletion did not select an artifact mode'
|
|
);
|
|
}, 3);
|
|
|
|
if (in_array($prepared['mode'], ['deleted', 'waiting', 'attention'], true)) {
|
|
return self::deletionResult($prepared['backup']);
|
|
}
|
|
if (in_array($prepared['mode'], [
|
|
'artifact', 'artifact_cancel',
|
|
], true)) {
|
|
$service = $artifactDeletion
|
|
?? new CloudStorageArtifactDeletionService($client);
|
|
$expectedOwner = ['backup_id' => (int) $backup->id];
|
|
if ($prepared['mode'] === 'artifact_cancel') {
|
|
$service->prepareNeverDispatched(
|
|
(int) $prepared['artifact_id'],
|
|
$expectedOwner
|
|
);
|
|
$backup->getConnection()->transaction(function () use (
|
|
$backup,
|
|
$prepared,
|
|
$beforeDeleteLocked
|
|
): void {
|
|
$locked = CloudInstanceBackup::where(
|
|
'id', (int) $backup->id
|
|
)->lockForUpdate()->firstOrFail();
|
|
if ((int) ($locked->storage_artifact_id ?? 0)
|
|
!== (int) $prepared['artifact_id']) {
|
|
throw new CloudError(
|
|
'备份与对象存储删除归属已发生变化',
|
|
409
|
|
);
|
|
}
|
|
if (in_array((string) $locked->state,
|
|
['deleting', 'delete_uncertain'], true)) {
|
|
return;
|
|
}
|
|
if ((string) $locked->state !== 'creating'
|
|
|| $locked->active_job_id !== null
|
|
|| $locked->last_operation_id !== null) {
|
|
throw new CloudError(
|
|
'备份创建状态已变化,不能取消',
|
|
409
|
|
);
|
|
}
|
|
if ($beforeDeleteLocked !== null) {
|
|
$beforeDeleteLocked($locked);
|
|
}
|
|
$now = time();
|
|
$locked->state = 'deleting';
|
|
$locked->delete_requested_at = $now;
|
|
$locked->delete_next_check_at = 0;
|
|
$locked->delete_attention_at = null;
|
|
$locked->error_code = '';
|
|
$locked->updated_at = $now;
|
|
$locked->save();
|
|
}, 3);
|
|
}
|
|
$result = $service->process(
|
|
(int) $prepared['artifact_id'],
|
|
$expectedOwner
|
|
);
|
|
return self::settleArtifactDeletion(
|
|
(int) $backup->id,
|
|
$result
|
|
);
|
|
}
|
|
throw new \LogicException(
|
|
'Replicated backup deletion returned an unsupported mode'
|
|
);
|
|
}
|
|
|
|
/** Sanitized administrator view of remote objects awaiting cleanup. */
|
|
public static function cleanupDebt(array $actor, array $input = []): array
|
|
{
|
|
if (($actor['is_admin'] ?? false) !== true) {
|
|
throw new CloudError('只有管理员可以查看对象清理异常', 403);
|
|
}
|
|
CloudInput::only($input, ['page', 'per_page']);
|
|
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
|
|
$perPage = self::libraryInteger($input['per_page'] ?? 20, 10, 50, 20);
|
|
$query = CloudInstanceBackup::whereIn('state', self::CLEANUP_DEBT_STATES);
|
|
$total = (int) (clone $query)->count('id');
|
|
$bytes = (int) (clone $query)->sum('object_size_bytes');
|
|
$attention = (int) (clone $query)
|
|
->where('state', 'delete_attention')->count('id');
|
|
$totalPages = max(1, (int) ceil($total / $perPage));
|
|
$page = min($page, $totalPages);
|
|
$rows = $query->orderByRaw(
|
|
"CASE WHEN state = 'delete_attention' THEN 0 ELSE 1 END"
|
|
)->orderBy('delete_next_check_at')->orderBy('id')
|
|
->offset(($page - 1) * $perPage)->limit($perPage)->get();
|
|
$items = [];
|
|
foreach ($rows as $row) {
|
|
$items[] = [
|
|
'id' => (string) $row->id,
|
|
'name' => (string) $row->name,
|
|
'source_instance_id' => (string) $row->source_instance_id,
|
|
'user_id' => (string) $row->user_id,
|
|
'state' => (string) $row->state,
|
|
'object_size_bytes' => max(0, (int) $row->object_size_bytes),
|
|
'delete_generation' => max(0, (int) $row->delete_generation),
|
|
'cycle_attempts' => max(0, (int) $row->delete_cycle_attempts),
|
|
'next_check_at' => max(0, (int) $row->delete_next_check_at),
|
|
'attention_at' => $row->delete_attention_at === null ? null
|
|
: max(0, (int) $row->delete_attention_at),
|
|
'error_code' => (string) $row->error_code,
|
|
'can_retry' => (string) $row->state === 'delete_attention'
|
|
&& ($actor['is_super_admin'] ?? false) === true
|
|
&& ($actor['can_manage'] ?? false) === true,
|
|
];
|
|
}
|
|
return [
|
|
'summary' => [
|
|
'count' => $total,
|
|
'bytes' => $bytes,
|
|
'attention_count' => $attention,
|
|
],
|
|
'items' => $items,
|
|
'pagination' => [
|
|
'page' => $page,
|
|
'per_page' => $perPage,
|
|
'total' => $total,
|
|
'total_pages' => $totalPages,
|
|
],
|
|
];
|
|
}
|
|
|
|
/** Begin one new bounded cycle after an administrator has inspected debt. */
|
|
public static function retryCleanup(CloudInstanceBackup $backup, array $actor,
|
|
?CloudObjectStorageS3Client $client = null,
|
|
?CloudStorageArtifactDeletionService $artifactDeletion = null): array
|
|
{
|
|
CloudAccess::manage($actor);
|
|
$client ??= new CloudObjectStorageS3Client();
|
|
$prepared = $backup->getConnection()->transaction(function () use ($backup): array {
|
|
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
->lockForUpdate()->firstOrFail();
|
|
if ((string) $locked->state !== 'delete_attention') {
|
|
throw new CloudError('只有待人工处理的对象可以重新进入清理流程', 409);
|
|
}
|
|
self::assertBackupDeletionAvailable($locked);
|
|
if ($locked->storage_artifact_id === null) {
|
|
throw new CloudError(
|
|
'备份缺少复制对象存储账本,不能继续清理',
|
|
409
|
|
);
|
|
}
|
|
return [
|
|
'artifact_id' => (int) $locked->storage_artifact_id,
|
|
'backup' => $locked,
|
|
];
|
|
}, 3);
|
|
$service = $artifactDeletion
|
|
?? new CloudStorageArtifactDeletionService($client);
|
|
$expectedOwner = ['backup_id' => (int) $backup->id];
|
|
$service->reopenAttention(
|
|
(int) $prepared['artifact_id'],
|
|
$expectedOwner
|
|
);
|
|
return self::settleArtifactDeletion(
|
|
(int) $backup->id,
|
|
$service->process(
|
|
(int) $prepared['artifact_id'],
|
|
$expectedOwner
|
|
)
|
|
);
|
|
}
|
|
|
|
public static function create(CloudInstance $instance, array $actor,
|
|
array $input): array
|
|
{
|
|
return self::createInternal($instance, $actor, $input, null);
|
|
}
|
|
|
|
/** Queue one deterministic paid automatic backup for a scheduled slot. */
|
|
public static function createAutomatic(CloudInstance $instance,
|
|
CloudBackupSubscription $subscription, array $actor,
|
|
int $scheduledFor, int $scheduleAttempt): array
|
|
{
|
|
if ($scheduledFor < 1 || $scheduleAttempt < 1) {
|
|
throw new CloudError('自动备份计划时间无效', 409);
|
|
}
|
|
$zone = new \DateTimeZone(CloudAutomaticBackupService::TIMEZONE);
|
|
$name = 'autobackup-'
|
|
. (new \DateTimeImmutable('@' . $scheduledFor))
|
|
->setTimezone($zone)->format('Ymd-His');
|
|
return self::createInternal($instance, $actor, [
|
|
'name' => $name,
|
|
'consistency_mode' => 'filesystem',
|
|
], [
|
|
'subscription_id' => (int) $subscription->id,
|
|
'scheduled_for' => $scheduledFor,
|
|
'schedule_attempt' => $scheduleAttempt,
|
|
]);
|
|
}
|
|
|
|
private static function createInternal(CloudInstance $instance, array $actor,
|
|
array $input, ?array $automatic): array
|
|
{
|
|
if ($automatic === null) {
|
|
CloudAccess::manage($actor);
|
|
} elseif ((int) ($actor['user_id'] ?? 0)
|
|
!== (int) $instance->user_id) {
|
|
throw new CloudError('无权为该实例创建自动备份', 403);
|
|
}
|
|
CloudInput::only($input, ['name', 'consistency_mode']);
|
|
$name = CloudInput::snapshotName($input['name'] ?? null);
|
|
$mode = CloudInput::snapshotConsistency(
|
|
$input['consistency_mode'] ?? null
|
|
);
|
|
$requestTarget = '/v1/vms/'
|
|
. rawurlencode((string) $instance->uuid) . '/backups';
|
|
$canonicalRequest = [
|
|
'instance_id' => (int) $instance->id,
|
|
'name' => $name,
|
|
'consistency_mode' => $mode,
|
|
];
|
|
if ($automatic !== null) {
|
|
$canonicalRequest['source'] = 'automatic';
|
|
$canonicalRequest['subscription_id'] =
|
|
(int) $automatic['subscription_id'];
|
|
$canonicalRequest['scheduled_for'] =
|
|
(int) $automatic['scheduled_for'];
|
|
$canonicalRequest['schedule_attempt'] =
|
|
(int) $automatic['schedule_attempt'];
|
|
}
|
|
$fingerprint = CloudJobService::standaloneRequestFingerprint(
|
|
(int) $instance->host_id,
|
|
'backup_create',
|
|
'POST',
|
|
$requestTarget,
|
|
$canonicalRequest
|
|
);
|
|
$requestKey = self::createRequestKey($actor);
|
|
$actorId = (int) $actor['user_id'];
|
|
$prior = self::priorJob(
|
|
$actor,
|
|
'backup_create',
|
|
(int) $instance->id,
|
|
null,
|
|
$fingerprint,
|
|
false
|
|
);
|
|
if ($prior !== null) {
|
|
$backup = CloudInstanceBackup::find((int) $prior->backup_id);
|
|
if ($backup === null) {
|
|
throw new CloudError('备份任务目录记录缺失', 409);
|
|
}
|
|
return ['backup' => self::displayBackup($backup, $prior),
|
|
'job' => $prior];
|
|
}
|
|
$reserved = self::findCreateIntent(
|
|
$actorId,
|
|
$requestKey,
|
|
$fingerprint,
|
|
(int) $instance->id,
|
|
false
|
|
);
|
|
if ($reserved instanceof CloudInstanceBackup) {
|
|
return self::finishCreatePreparation($reserved);
|
|
}
|
|
|
|
// Do not allocate a backup row, storage artifact, multipart upload or
|
|
// Agent job while a known transient host state makes backup unsafe.
|
|
// Idempotent replays above still resolve their existing durable intent.
|
|
if ($automatic === null) {
|
|
$eligibility = (new CloudBackupEligibility())->assess(
|
|
$instance,
|
|
$actor
|
|
);
|
|
if (!$eligibility->allowed()) {
|
|
$status = $eligibility->reasonCode
|
|
=== CloudBackupEligibilityDecision::READINESS_UNAVAILABLE
|
|
? 503 : 409;
|
|
throw new CloudError($eligibility->message, $status);
|
|
}
|
|
}
|
|
|
|
try {
|
|
$backup = $instance->getConnection()->transaction(function () use (
|
|
$instance, $actor, $name, $mode, $fingerprint, $automatic,
|
|
$requestKey, $actorId
|
|
): CloudInstanceBackup {
|
|
$locked = CloudInstance::where('id', (int) $instance->id)
|
|
->lockForUpdate()->first();
|
|
if ($locked === null || (string) $locked->status === 'deleted') {
|
|
throw new CloudError('虚拟机不存在', 404);
|
|
}
|
|
$prior = self::priorJob($actor, 'backup_create',
|
|
(int) $locked->id, null, $fingerprint);
|
|
if ($prior !== null) {
|
|
$backup = CloudInstanceBackup::find((int) $prior->backup_id);
|
|
if ($backup === null) {
|
|
throw new CloudError('备份任务目录记录缺失', 409);
|
|
}
|
|
return $backup;
|
|
}
|
|
$existingIntent = self::findCreateIntent(
|
|
$actorId,
|
|
$requestKey,
|
|
$fingerprint,
|
|
(int) $locked->id,
|
|
true
|
|
);
|
|
if ($existingIntent instanceof CloudInstanceBackup) {
|
|
return $existingIntent;
|
|
}
|
|
$automaticSubscription = null;
|
|
if ($automatic !== null) {
|
|
$automaticSubscription = CloudBackupSubscription::where(
|
|
'id', (int) $automatic['subscription_id']
|
|
)->lockForUpdate()->first();
|
|
$scheduledFor = (int) $automatic['scheduled_for'];
|
|
$scheduleAttempt = (int) $automatic['schedule_attempt'];
|
|
$entitled = $automaticSubscription !== null
|
|
&& CloudBackupPeriodEntitlement::where(
|
|
'subscription_id',
|
|
(int) $automaticSubscription->id
|
|
)->where('instance_id', (int) $locked->id)
|
|
->where('user_id', (int) $locked->user_id)
|
|
->where('starts_at', '<=', $scheduledFor)
|
|
->where('ends_at', '>', $scheduledFor)
|
|
->lockForUpdate()->exists();
|
|
if ($automaticSubscription === null
|
|
|| !(bool) $automaticSubscription->enabled
|
|
|| (int) $automaticSubscription->instance_id
|
|
!== (int) $locked->id
|
|
|| (int) $automaticSubscription->user_id
|
|
!== (int) $locked->user_id
|
|
|| !$entitled) {
|
|
throw new CloudError('自动备份订阅与实例账本不一致', 409);
|
|
}
|
|
self::assertAutomaticScheduleAttempt(
|
|
$automaticSubscription,
|
|
$scheduledFor,
|
|
$scheduleAttempt
|
|
);
|
|
$existing = CloudInstanceBackup::where(
|
|
'subscription_id', (int) $automaticSubscription->id
|
|
)->where('scheduled_for',
|
|
$scheduledFor)->where(
|
|
'schedule_attempt',
|
|
$scheduleAttempt
|
|
)->first();
|
|
if ($existing !== null) {
|
|
if ((int) ($existing->create_request_actor_id ?? 0)
|
|
!== $actorId
|
|
|| !is_string(
|
|
$existing->create_request_idempotency_key
|
|
)
|
|
|| !hash_equals(
|
|
(string) $existing->create_request_idempotency_key,
|
|
$requestKey
|
|
)
|
|
|| !is_string($existing->create_request_fingerprint)
|
|
|| !hash_equals(
|
|
(string) $existing->create_request_fingerprint,
|
|
$fingerprint
|
|
)) {
|
|
throw new CloudError(
|
|
'自动备份计划已绑定其他创建请求',
|
|
409
|
|
);
|
|
}
|
|
return $existing;
|
|
}
|
|
}
|
|
if ((string) $locked->status !== 'active') {
|
|
throw new CloudError('只有活动实例可以创建备份', 409);
|
|
}
|
|
if ($automatic !== null
|
|
&& ((string) $locked->service_status !== 'active'
|
|
|| CloudAutomaticBackupService::
|
|
shouldPauseForInstanceExpiry($locked, time()))) {
|
|
throw new CloudError('实例服务当前不可创建自动备份', 409);
|
|
}
|
|
self::assertManagedSource($locked);
|
|
self::assertNoActiveJob($locked);
|
|
$hostAgent = CloudHostAgent::where(
|
|
'host_id', (int) $locked->host_id
|
|
)->first();
|
|
$sourceCompatibility = $hostAgent?->
|
|
testedPortableRestoreCompatibility();
|
|
if ($sourceCompatibility === null) {
|
|
throw new CloudError(
|
|
'源宿主机当前版本尚未通过可移植恢复能力测试',
|
|
409
|
|
);
|
|
}
|
|
$usedQuery = CloudInstanceBackup::where(
|
|
'source_instance_id', (int) $locked->id
|
|
)->where('purpose', 'portable')
|
|
->whereIn('state', self::ACTIVE_BACKUP_STATES);
|
|
if ($automatic === null) {
|
|
$usedQuery->where('source', 'manual');
|
|
} else {
|
|
$usedQuery->where('source', 'automatic')
|
|
->where('subscription_id',
|
|
(int) $automaticSubscription->id)
|
|
->whereNotExists(static function ($query): void {
|
|
$query->selectRaw('1')
|
|
->from('cloud_deployment_snapshots')
|
|
->whereColumn(
|
|
'cloud_deployment_snapshots.backup_id',
|
|
'cloud_instance_backups.id'
|
|
)->where(
|
|
'cloud_deployment_snapshots.state',
|
|
'<>',
|
|
'deleted'
|
|
);
|
|
});
|
|
}
|
|
$used = (int) $usedQuery->count('id');
|
|
if ($automatic === null && $used >= self::MAX_PER_INSTANCE) {
|
|
throw new CloudError('每台实例最多保留十个有效备份', 409);
|
|
}
|
|
if ($automatic !== null
|
|
&& $used >= min(
|
|
CloudAutomaticBackupService::MAX_RETENTION,
|
|
max(1, (int) $automaticSubscription->retention_count)
|
|
) + 1) {
|
|
throw new CloudError('自动备份正在等待旧备份安全清理', 409);
|
|
}
|
|
$spec = self::freezeRestoreCompatibility(
|
|
self::decodeSpec((string) $locked->spec_json)
|
|
);
|
|
$resources = self::resourcesFromSpec($spec);
|
|
$maximumBytes = self::maximumObjectBytes(
|
|
$resources['disk_gib']
|
|
);
|
|
$key = bin2hex(random_bytes(16));
|
|
$objectSuffix = 'vm-backups/' . $key . '/system.qcow2';
|
|
// The owner row must exist before the storage publication can
|
|
// be bound to it. Until placement selects a primary, the
|
|
// suffix is a unique non-secret placeholder for object_key.
|
|
$objectKey = $objectSuffix;
|
|
$now = time();
|
|
$backup = new CloudInstanceBackup();
|
|
$backup->backup_key = $key;
|
|
$backup->storage_artifact_id = null;
|
|
$backup->create_request_actor_id = $actorId;
|
|
$backup->create_request_idempotency_key = $requestKey;
|
|
$backup->create_request_fingerprint = $fingerprint;
|
|
$backup->source_instance_id = (int) $locked->id;
|
|
$backup->source_host_id = (int) $locked->host_id;
|
|
$backup->user_id = (int) $locked->user_id;
|
|
$backup->source_uuid = strtolower((string) $locked->uuid);
|
|
$backup->name = $name;
|
|
$backup->purpose = 'portable';
|
|
$backup->source = $automatic === null
|
|
? 'manual' : 'automatic';
|
|
$backup->subscription_id = $automatic === null ? null
|
|
: (int) $automaticSubscription->id;
|
|
$backup->scheduled_for = $automatic === null ? null
|
|
: (int) $automatic['scheduled_for'];
|
|
if ($automatic !== null) {
|
|
$backup->schedule_attempt =
|
|
(int) $automatic['schedule_attempt'];
|
|
}
|
|
$backup->state = 'creating';
|
|
$backup->consistency = $mode;
|
|
$backup->object_key = $objectKey;
|
|
$backup->object_size_bytes = 0;
|
|
$backup->virtual_size_bytes = 0;
|
|
$backup->sha256 = '';
|
|
$backup->source_portable_restore_contract_json =
|
|
$sourceCompatibility->json();
|
|
$backup->source_portable_restore_contract_sha256 =
|
|
$sourceCompatibility->sha256();
|
|
$backup->spec_json = json_encode(
|
|
CloudInput::safeResult($spec),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
$backup->sealed_credential = $locked->sealed_credential;
|
|
$backup->storage_revision = 0;
|
|
$backup->active_job_id = null;
|
|
$backup->last_operation_id = null;
|
|
$backup->error_code = '';
|
|
$backup->ready_at = null;
|
|
$backup->created_at = $backup->updated_at = $now;
|
|
$backup->save();
|
|
|
|
$publication = CloudStorageArtifactService::
|
|
beginAgentPublication(
|
|
'backup',
|
|
$objectSuffix,
|
|
$maximumBytes,
|
|
'backup-publish-' . hash('sha256', implode('|', [
|
|
(string) $actorId, $requestKey,
|
|
])),
|
|
hash('sha256', 'backup|' . $fingerprint),
|
|
['backup_id' => (int) $backup->id]
|
|
);
|
|
$objectKey = (string) $publication['object_key'];
|
|
$backup->storage_artifact_id =
|
|
(int) $publication['artifact_id'];
|
|
$backup->object_key = $objectKey;
|
|
$backup->updated_at = time();
|
|
$backup->save();
|
|
return $backup;
|
|
}, 3);
|
|
} catch (\Throwable $error) {
|
|
$reserved = self::findCreateIntent(
|
|
$actorId,
|
|
$requestKey,
|
|
$fingerprint,
|
|
(int) $instance->id,
|
|
false
|
|
);
|
|
if (!$reserved instanceof CloudInstanceBackup) {
|
|
throw $error;
|
|
}
|
|
$backup = $reserved;
|
|
}
|
|
return self::finishCreatePreparation($backup);
|
|
}
|
|
|
|
public static function restore(CloudInstanceBackup $backup, array $actor,
|
|
array $input): array
|
|
{
|
|
$result = CloudPortableRestoreRequestService::request(
|
|
$backup,
|
|
$actor,
|
|
$input
|
|
);
|
|
return [
|
|
'restore' => self::displayRestore($result['restore']),
|
|
'job' => $result['job'],
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build the short-lived download capability for a paid create order.
|
|
* The caller must already hold and validate the owner/order row locks;
|
|
* signing is local and therefore does not extend that transaction across
|
|
* a remote object-storage request.
|
|
*
|
|
* @return array{source_url:string}
|
|
*/
|
|
public static function provisioningSource(
|
|
CloudInstanceBackup $backup
|
|
): array {
|
|
if ((string) $backup->purpose !== 'portable'
|
|
|| (string) $backup->state !== 'ready'
|
|
|| (int) $backup->object_size_bytes < 1
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $backup->sha256) !== 1) {
|
|
throw new CloudError('云备份尚未处于可创建实例状态', 409);
|
|
}
|
|
return ['source_url' => self::backupReadUrl($backup)];
|
|
}
|
|
|
|
/** Replace an owner's existing system disk from its own automatic backup. */
|
|
public static function recover(CloudInstance $instance,
|
|
CloudInstanceBackup $backup, array $actor, array $input): array
|
|
{
|
|
if ((int) ($actor['user_id'] ?? 0) !== (int) $instance->user_id) {
|
|
throw new CloudError('无权恢复该实例', 403);
|
|
}
|
|
CloudInput::only($input, ['confirm_restore']);
|
|
if (($input['confirm_restore'] ?? false) !== true) {
|
|
throw new CloudError('请确认恢复会覆盖当前系统盘内容');
|
|
}
|
|
$requestKey = $actor['request_idempotency_key'] ?? null;
|
|
if (!is_string($requestKey) || $requestKey === '') {
|
|
throw new CloudError('恢复云备份必须提供请求幂等标识', 400);
|
|
}
|
|
$target = '/v1/vms/' . rawurlencode((string) $instance->uuid)
|
|
. '/recover-backup';
|
|
$canonical = [
|
|
'instance_id' => (int) $instance->id,
|
|
'backup_id' => (int) $backup->id,
|
|
'confirm_restore' => true,
|
|
];
|
|
$fingerprint = CloudJobService::standaloneRequestFingerprint(
|
|
(int) $instance->host_id,
|
|
'backup_recover',
|
|
'POST',
|
|
$target,
|
|
$canonical
|
|
);
|
|
$prior = self::priorJob($actor, 'backup_recover',
|
|
(int) $instance->id, (int) $backup->id, $fingerprint, false);
|
|
if ($prior !== null) {
|
|
return self::priorRecoveryResult($prior);
|
|
}
|
|
CloudOriginalBackupRestorePolicy::assertEligible($instance, $backup);
|
|
|
|
$sourceUrl = self::backupReadUrl($backup);
|
|
|
|
return $instance->getConnection()->transaction(function () use (
|
|
$instance, $backup, $actor, $target, $canonical, $fingerprint,
|
|
$sourceUrl
|
|
): array {
|
|
$lockedInstance = CloudInstance::where(
|
|
'id', (int) $instance->id
|
|
)->lockForUpdate()->first();
|
|
$lockedBackup = CloudInstanceBackup::where(
|
|
'id', (int) $backup->id
|
|
)->lockForUpdate()->first();
|
|
if ($lockedInstance === null || $lockedBackup === null) {
|
|
throw new CloudError('实例或云备份不存在', 404);
|
|
}
|
|
$prior = self::priorJob($actor, 'backup_recover',
|
|
(int) $lockedInstance->id, (int) $lockedBackup->id,
|
|
$fingerprint);
|
|
if ($prior !== null) {
|
|
return self::priorRecoveryResult($prior);
|
|
}
|
|
CloudOriginalBackupRestorePolicy::assertEligible(
|
|
$lockedInstance, $lockedBackup
|
|
);
|
|
$claimSet = CloudPublicNetworkAllocationService::currentClaimSet(
|
|
(int) ($lockedInstance->reservation_id ?? 0),
|
|
'committed',
|
|
(int) $lockedInstance->id
|
|
);
|
|
$instanceSpec = self::decodeSpec(
|
|
(string) $lockedInstance->spec_json
|
|
);
|
|
try {
|
|
$frozenNetwork = CloudPublicNetworkSnapshot::fromArray(
|
|
$instanceSpec['public_network'] ?? null
|
|
);
|
|
} catch (\Throwable) {
|
|
throw new CloudError(
|
|
'实例缺少有效的双栈公网网络快照,不能恢复',
|
|
409
|
|
);
|
|
}
|
|
if (!hash_equals(
|
|
$claimSet->publicNetwork()->fingerprint(),
|
|
$frozenNetwork->fingerprint()
|
|
)) {
|
|
throw new CloudError(
|
|
'实例公网网络快照与活动地址声明不一致',
|
|
409
|
|
);
|
|
}
|
|
self::assertNoActiveJob($lockedInstance);
|
|
if (CloudBackupRecovery::where(
|
|
'instance_id', (int) $lockedInstance->id
|
|
)->whereIn('state', ['recovering', 'uncertain'])->exists()) {
|
|
throw new CloudError('实例已有恢复任务或未确认效果', 409);
|
|
}
|
|
|
|
$operationId = bin2hex(random_bytes(16));
|
|
$recoveryKey = bin2hex(random_bytes(16));
|
|
$now = time();
|
|
$recovery = new CloudBackupRecovery();
|
|
$recovery->recovery_key = $recoveryKey;
|
|
$recovery->backup_id = (int) $lockedBackup->id;
|
|
$recovery->instance_id = (int) $lockedInstance->id;
|
|
$recovery->job_id = null;
|
|
$recovery->operation_id = $operationId;
|
|
$recovery->state = 'recovering';
|
|
$recovery->error_code = '';
|
|
$recovery->completed_at = null;
|
|
$recovery->created_at = $recovery->updated_at = $now;
|
|
$recovery->save();
|
|
|
|
$body = [
|
|
'backup_key' => (string) $lockedBackup->backup_key,
|
|
'source_url' => $sourceUrl,
|
|
'source_size_bytes' => (int) $lockedBackup->object_size_bytes,
|
|
'source_sha256' => (string) $lockedBackup->sha256,
|
|
];
|
|
$jobActor = $actor;
|
|
$jobActor['can_manage'] = true;
|
|
[$job, $created] = CloudJobService::
|
|
reserveStandaloneAgentMutation(
|
|
$jobActor,
|
|
(int) $lockedInstance->host_id,
|
|
'backup_recover',
|
|
'POST',
|
|
$target,
|
|
$body,
|
|
[
|
|
'owner_id' => (int) $lockedInstance->user_id,
|
|
'can_manage' => true,
|
|
'backup_key' => (string) $lockedBackup->backup_key,
|
|
'recovery_key' => $recoveryKey,
|
|
],
|
|
$canonical,
|
|
$operationId
|
|
);
|
|
if (!$created) {
|
|
throw new CloudError('云备份恢复幂等任务状态冲突', 409);
|
|
}
|
|
$job->instance_id = (int) $lockedInstance->id;
|
|
$job->backup_id = (int) $lockedBackup->id;
|
|
$job->backup_recovery_id = (int) $recovery->id;
|
|
$job->save();
|
|
$recovery->job_id = (int) $job->id;
|
|
$recovery->updated_at = time();
|
|
$recovery->save();
|
|
return [
|
|
'recovery' => self::displayRecovery($recovery),
|
|
'job' => $job,
|
|
];
|
|
}, 3);
|
|
}
|
|
|
|
/** Apply a terminal/running Agent result inside CloudWorkerService's lock. */
|
|
public static function finishJobLocked(CloudJob $job, string $status,
|
|
$result, array $payload, bool $deferPublish = false,
|
|
array $operation = []): void
|
|
{
|
|
if ($job->kind === 'backup_create') {
|
|
self::finishCreateLocked(
|
|
$job,
|
|
$status,
|
|
$result,
|
|
$payload,
|
|
$deferPublish,
|
|
$operation
|
|
);
|
|
} elseif ($job->kind === 'backup_restore') {
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job, $status, $result, $payload, $deferPublish
|
|
);
|
|
} elseif ($job->kind === 'backup_recover') {
|
|
self::finishRecoveryLocked($job, $status, $result, $payload);
|
|
} elseif ($job->kind === 'backup_workspace_cleanup') {
|
|
self::finishWorkspaceCleanupLocked(
|
|
$job, $status, $result, $payload
|
|
);
|
|
}
|
|
}
|
|
|
|
/** Queue one evidence-bound cleanup of a source-host backup workspace. */
|
|
public static function cleanupWorkspace(
|
|
CloudInstanceBackup $backup,
|
|
array $actor
|
|
): CloudJob {
|
|
CloudAccess::manage($actor);
|
|
if (($actor['is_super_admin'] ?? false) !== true) {
|
|
throw new CloudError('只有超级管理员可以清理备份工作区', 403);
|
|
}
|
|
$original = CloudJob::where(
|
|
'operation_id', (string) $backup->last_operation_id
|
|
)->first();
|
|
if ($original === null || (string) $original->kind !== 'backup_create'
|
|
|| (string) $original->phase !== 'terminal'
|
|
|| (string) $original->outcome !== 'succeeded'
|
|
|| (int) $original->backup_id !== (int) $backup->id
|
|
|| (int) $original->host_id !== (int) $backup->source_host_id) {
|
|
throw new CloudError('原备份任务身份不完整,不能清理工作区', 409);
|
|
}
|
|
$inspection = CloudAdminOperationRecoveryService::inspect(
|
|
$actor, (string) $original->id
|
|
);
|
|
$cleanup = is_array($inspection['cleanup'] ?? null)
|
|
? $inspection['cleanup'] : [];
|
|
$planId = ($cleanup['available'] ?? false) === true
|
|
&& is_string($cleanup['plan_id'] ?? null)
|
|
? strtolower((string) $cleanup['plan_id']) : '';
|
|
if ($planId !== ''
|
|
&& preg_match('/^[a-f0-9]{64}$/D', $planId) !== 1) {
|
|
throw new CloudError('Agent 返回的清理计划身份无效', 502);
|
|
}
|
|
|
|
return $backup->getConnection()->transaction(function () use (
|
|
$backup, $actor, $original, $planId
|
|
): CloudJob {
|
|
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
->lockForUpdate()->first();
|
|
if ($locked === null || (string) $locked->state !== 'ready'
|
|
|| (string) $locked->error_code
|
|
!== 'backup_local_cleanup_pending'
|
|
|| !hash_equals(
|
|
(string) $locked->last_operation_id,
|
|
(string) $original->operation_id
|
|
)) {
|
|
throw new CloudError('备份工作区状态已变化,请刷新后重试', 409);
|
|
}
|
|
$operationId = bin2hex(random_bytes(16));
|
|
$target = '/v1/operations/' . rawurlencode($operationId)
|
|
. '/cleanup-backup-workspace';
|
|
$body = [
|
|
'prior_operation_id' => (string) $original->operation_id,
|
|
'cleanup_plan_id' => $planId,
|
|
];
|
|
$canonical = [
|
|
'backup_id' => (int) $locked->id,
|
|
'backup_key' => (string) $locked->backup_key,
|
|
'prior_operation_id' => (string) $original->operation_id,
|
|
'cleanup_plan_id' => $planId,
|
|
];
|
|
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
|
|
$actor,
|
|
(int) $locked->source_host_id,
|
|
'backup_workspace_cleanup',
|
|
'POST',
|
|
$target,
|
|
$body,
|
|
[
|
|
'owner_id' => (int) $locked->user_id,
|
|
'can_manage' => true,
|
|
'backup_key' => (string) $locked->backup_key,
|
|
'prior_job_id' => (int) $original->id,
|
|
'prior_operation_id' => (string) $original->operation_id,
|
|
'cleanup_plan_id' => $planId,
|
|
],
|
|
$canonical,
|
|
$operationId,
|
|
'/v1/backups/' . (int) $locked->id . '/cleanup-workspace'
|
|
);
|
|
if ($created) {
|
|
$job->instance_id = (int) $locked->source_instance_id;
|
|
$job->backup_id = (int) $locked->id;
|
|
$job->save();
|
|
}
|
|
return $job;
|
|
}, 3);
|
|
}
|
|
|
|
private static function finishWorkspaceCleanupLocked(
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $payload
|
|
): void {
|
|
$backup = $job->backup_id === null ? null
|
|
: CloudInstanceBackup::where('id', (int) $job->backup_id)
|
|
->lockForUpdate()->first();
|
|
if ($backup === null
|
|
|| (int) $backup->source_host_id !== (int) $job->host_id
|
|
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|
|
|| !is_string($payload['prior_operation_id'] ?? null)
|
|
|| !hash_equals(
|
|
(string) $backup->last_operation_id,
|
|
(string) $payload['prior_operation_id']
|
|
)
|
|
|| !hash_equals(
|
|
(string) $backup->backup_key,
|
|
(string) ($payload['backup_key'] ?? '')
|
|
)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'备份工作区清理任务与目录账本不一致', 409
|
|
);
|
|
}
|
|
if ($status !== 'succeeded') {
|
|
return;
|
|
}
|
|
if (!is_array($result)
|
|
|| !hash_equals(
|
|
(string) $backup->backup_key,
|
|
(string) ($result['backup_key'] ?? '')
|
|
)
|
|
|| ($result['cleaned'] ?? null) !== true
|
|
|| ($result['workspace_absent'] ?? null) !== true
|
|
|| ($result['metadata_absent'] ?? null) !== true) {
|
|
throw new CloudError('Agent 清理成功结果缺少完整缺席证据', 502);
|
|
}
|
|
if ((string) $backup->error_code
|
|
=== 'backup_local_cleanup_pending') {
|
|
$backup->error_code = '';
|
|
$backup->updated_at = time();
|
|
$backup->save();
|
|
}
|
|
}
|
|
|
|
public static function markJobUncertain(CloudJob $job): void
|
|
{
|
|
if ($job->kind === 'backup_create' && $job->backup_id !== null) {
|
|
CloudInstanceBackup::where('id', (int) $job->backup_id)
|
|
->where('last_operation_id', (string) $job->operation_id)
|
|
->update(['state' => 'uncertain', 'error_code' =>
|
|
'finalization_incident', 'updated_at' => time()]);
|
|
} elseif ($job->kind === 'backup_restore') {
|
|
CloudPortableRestoreFinalizer::markJobUncertain($job);
|
|
} elseif ($job->kind === 'backup_recover'
|
|
&& $job->backup_recovery_id !== null) {
|
|
CloudBackupRecovery::where(
|
|
'id', (int) $job->backup_recovery_id
|
|
)->where('operation_id', (string) $job->operation_id)
|
|
->update(['state' => 'uncertain', 'error_code' =>
|
|
'finalization_incident', 'updated_at' => time()]);
|
|
}
|
|
CloudInstanceMigrationService::markJobUncertain($job);
|
|
}
|
|
|
|
private static function finishCreateLocked(CloudJob $job, string $status,
|
|
$result, array $payload, bool $deferPublish,
|
|
array $operation): void
|
|
{
|
|
$backup = $job->backup_id === null ? null
|
|
: CloudInstanceBackup::where('id', (int) $job->backup_id)
|
|
->lockForUpdate()->first();
|
|
if ($backup === null
|
|
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|
|
|| (int) $backup->source_host_id !== (int) $job->host_id
|
|
|| (int) $backup->user_id !== (int) ($payload['owner_id'] ?? 0)
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($payload['backup_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->last_operation_id,
|
|
(string) $job->operation_id)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'备份任务与面板目录账本不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
$artifactId = filter_var(
|
|
$backup->storage_artifact_id,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
);
|
|
$payloadArtifactId = filter_var(
|
|
$payload['storage_artifact_id'] ?? null,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
);
|
|
$storageOperationId =
|
|
$payload['storage_publication_operation_id'] ?? null;
|
|
if ($artifactId === false
|
|
|| $payloadArtifactId === false
|
|
|| (int) $payloadArtifactId !== (int) $artifactId
|
|
|| !is_string($storageOperationId)
|
|
|| preg_match('/^[a-f0-9]{32}$/D', $storageOperationId) !== 1) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'备份任务与复制对象存储账本不一致',
|
|
409
|
|
);
|
|
}
|
|
$now = time();
|
|
$transientRetryReason = self::automaticTransientRetryReason(
|
|
$backup,
|
|
$job,
|
|
$status,
|
|
$deferPublish,
|
|
$operation
|
|
);
|
|
if ((string) $backup->state === 'deleted'
|
|
&& (string) $backup->error_code
|
|
=== 'automatic_transient_retried') {
|
|
if ($transientRetryReason === null
|
|
|| $backup->sealed_credential !== null
|
|
|| $backup->active_job_id !== null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'自动备份安全重试终态与重复回执不一致',
|
|
409
|
|
);
|
|
}
|
|
CloudStorageArtifactService::retireBlankNotAppliedPublication(
|
|
$storageOperationId,
|
|
(int) $artifactId,
|
|
(int) $backup->id,
|
|
$now
|
|
);
|
|
return;
|
|
}
|
|
if ($status === 'uncertain') {
|
|
CloudStorageArtifactService::failPrimary(
|
|
$storageOperationId,
|
|
(int) $artifactId,
|
|
'unknown',
|
|
'backup_operation_effect_unknown'
|
|
);
|
|
$backup->state = 'uncertain';
|
|
$backup->error_code = 'operation_effect_unknown';
|
|
} elseif ($status === 'failed') {
|
|
CloudStorageArtifactService::failPrimary(
|
|
$storageOperationId,
|
|
(int) $artifactId,
|
|
'not_applied',
|
|
'backup_agent_failed'
|
|
);
|
|
if ($transientRetryReason !== null) {
|
|
CloudStorageArtifactService::
|
|
retireBlankNotAppliedPublication(
|
|
$storageOperationId,
|
|
(int) $artifactId,
|
|
(int) $backup->id,
|
|
$now
|
|
);
|
|
$backup->state = 'deleted';
|
|
$backup->sealed_credential = null;
|
|
$backup->active_job_id = null;
|
|
$backup->error_code = 'automatic_transient_retried';
|
|
$backup->updated_at = $now;
|
|
$backup->save();
|
|
CloudAutomaticBackupService::rearmAfterTransientFailure(
|
|
$backup,
|
|
$transientRetryReason,
|
|
$now
|
|
);
|
|
return;
|
|
}
|
|
$backup->state = 'failed';
|
|
$backup->error_code = 'backup_failed';
|
|
$backup->active_job_id = null;
|
|
} elseif ($status === 'succeeded') {
|
|
try {
|
|
$sourceCompatibility =
|
|
CloudPortableRestoreCompatibility::fromPayload(
|
|
is_array($result)
|
|
? ($result['source_contract'] ?? null) : null
|
|
);
|
|
} catch (\InvalidArgumentException $error) {
|
|
throw new CloudError(
|
|
'Agent 成功结果缺少可信可移植恢复合同', 502
|
|
);
|
|
}
|
|
$size = filter_var($result['size_bytes'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$virtual = filter_var($result['virtual_size_bytes'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$sha = $result['sha256'] ?? null;
|
|
$objectEtag = $result['object_etag'] ?? '';
|
|
$objectVersionId = $result['object_version_id'] ?? '';
|
|
$consistency = $result['consistency'] ?? null;
|
|
$cleanupPending = $result['cleanup_pending'] ?? false;
|
|
$expectedContractSHA =
|
|
$payload['source_portable_restore_contract_sha256'] ?? null;
|
|
if (!is_array($result)
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($result['backup_key'] ?? ''))
|
|
|| $size === false || $virtual === false
|
|
|| !is_string($sha)
|
|
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|
|
|| !is_string($objectEtag) || strlen($objectEtag) > 256
|
|
|| preg_match('/[\x00-\x1f\x7f]/D', $objectEtag) === 1
|
|
|| !is_string($objectVersionId)
|
|
|| strlen($objectVersionId) > 1024
|
|
|| preg_match('/[\x00-\x1f\x7f]/D', $objectVersionId) === 1
|
|
|| ($objectVersionId !== '' && $objectEtag === '')
|
|
|| !in_array($consistency, ['filesystem', 'crash'], true)
|
|
|| !is_bool($cleanupPending)
|
|
|| !is_string($expectedContractSHA)
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
$expectedContractSHA) !== 1
|
|
|| !hash_equals(
|
|
$expectedContractSHA,
|
|
$sourceCompatibility->sha256()
|
|
)
|
|
|| (int) $size > (int) ($payload['maximum_size_bytes'] ?? 0)) {
|
|
throw new CloudError('Agent 成功结果缺少可信备份对象身份', 502);
|
|
}
|
|
$publication = CloudStorageArtifactService::completePrimary(
|
|
$storageOperationId,
|
|
(int) $artifactId,
|
|
[
|
|
'size_bytes' => (int) $size,
|
|
'sha256' => $sha,
|
|
'etag' => $objectEtag,
|
|
'version_id' => $objectVersionId,
|
|
]
|
|
);
|
|
if ((int) ($publication['artifact_id'] ?? 0)
|
|
!== (int) $artifactId
|
|
|| ($publication['outcome'] ?? null) !== 'succeeded') {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'对象存储主副本结算回执与备份目录不一致',
|
|
409
|
|
);
|
|
}
|
|
$backup->state = 'ready';
|
|
$backup->consistency = $consistency;
|
|
$backup->object_size_bytes = (int) $size;
|
|
$backup->virtual_size_bytes = (int) $virtual;
|
|
$backup->sha256 = $sha;
|
|
$backup->object_etag = $objectEtag;
|
|
$backup->object_version_id = $objectVersionId;
|
|
$backup->source_portable_restore_contract_json =
|
|
$sourceCompatibility->json();
|
|
$backup->source_portable_restore_contract_sha256 =
|
|
$sourceCompatibility->sha256();
|
|
$backup->ready_at = $now;
|
|
$backup->active_job_id = null;
|
|
// The remote object is complete and restorable. A host-side
|
|
// snapshot cleanup warning must not discard this sole immutable
|
|
// object identity or trigger another upload.
|
|
$backup->error_code = $cleanupPending
|
|
? 'backup_local_cleanup_pending' : '';
|
|
}
|
|
$backup->updated_at = $now;
|
|
$backup->save();
|
|
if ($status === 'succeeded') {
|
|
$finalized = CloudStoragePublicationFinalizerService::complete(
|
|
'backup',
|
|
(int) $backup->id,
|
|
(int) $artifactId,
|
|
$storageOperationId
|
|
);
|
|
if (($finalized['state'] ?? null) !== 'completed') {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'对象存储发布账本未完成最终化',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
if ((string) $backup->source === 'automatic') {
|
|
$subscription = CloudBackupSubscription::where(
|
|
'id',
|
|
(int) $backup->subscription_id
|
|
)->lockForUpdate()->first();
|
|
if (!$subscription instanceof CloudBackupSubscription
|
|
|| (int) $subscription->instance_id
|
|
!== (int) $backup->source_instance_id
|
|
|| (int) $subscription->user_id !== (int) $backup->user_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'自动备份终态与订阅账本不一致',
|
|
409
|
|
);
|
|
}
|
|
CloudAutomaticBackupService::advanceAfterScheduled(
|
|
$subscription,
|
|
(int) $backup->scheduled_for,
|
|
(int) $backup->id,
|
|
(int) $backup->schedule_attempt
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function automaticTransientRetryReason(
|
|
CloudInstanceBackup $backup,
|
|
CloudJob $job,
|
|
string $status,
|
|
bool $deferPublish,
|
|
array $operation
|
|
): ?string {
|
|
if ((string) $backup->source !== 'automatic'
|
|
|| $status !== 'failed'
|
|
|| $deferPublish
|
|
|| (bool) ($job->finalization_incident ?? false)
|
|
|| ($operation['status'] ?? null) !== 'failed'
|
|
|| ($operation['effect_may_have_applied'] ?? null) !== false
|
|
|| ($operation['safe_to_resubmit'] ?? null) !== true) {
|
|
return null;
|
|
}
|
|
$reason = $operation['cause_code'] ?? null;
|
|
return in_array($reason, [
|
|
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE,
|
|
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE,
|
|
], true) ? $reason : null;
|
|
}
|
|
|
|
private static function finishRecoveryLocked(CloudJob $job,
|
|
string $status, $result, array $payload): void
|
|
{
|
|
$recovery = $job->backup_recovery_id === null ? null
|
|
: CloudBackupRecovery::where(
|
|
'id', (int) $job->backup_recovery_id
|
|
)->lockForUpdate()->first();
|
|
$backup = $job->backup_id === null ? null
|
|
: CloudInstanceBackup::where('id', (int) $job->backup_id)
|
|
->lockForUpdate()->first();
|
|
$instance = $job->instance_id === null ? null
|
|
: CloudInstance::where('id', (int) $job->instance_id)
|
|
->lockForUpdate()->first();
|
|
$body = is_array($payload['body'] ?? null)
|
|
? $payload['body'] : [];
|
|
if ($recovery === null || $backup === null || $instance === null
|
|
|| (int) $recovery->job_id !== (int) $job->id
|
|
|| (int) $recovery->backup_id !== (int) $backup->id
|
|
|| (int) $recovery->instance_id !== (int) $instance->id
|
|
|| !hash_equals((string) $recovery->operation_id,
|
|
(string) $job->operation_id)
|
|
|| !hash_equals((string) $recovery->recovery_key,
|
|
(string) ($payload['recovery_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($payload['backup_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($body['backup_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->sha256,
|
|
(string) ($body['source_sha256'] ?? ''))
|
|
|| (int) $backup->object_size_bytes
|
|
!== (int) ($body['source_size_bytes'] ?? 0)
|
|
|| (int) $instance->user_id
|
|
!== (int) ($payload['owner_id'] ?? 0)
|
|
|| (int) $job->user_id !== (int) $instance->user_id
|
|
|| (int) $job->host_id !== (int) $instance->host_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'云备份恢复任务与实例、备份或恢复账本不一致',
|
|
409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
$now = time();
|
|
if ($status === 'uncertain') {
|
|
$recovery->state = 'uncertain';
|
|
$recovery->error_code = 'operation_effect_unknown';
|
|
$recovery->completed_at = null;
|
|
} elseif ($status === 'failed') {
|
|
$recovery->state = 'failed';
|
|
$recovery->error_code = 'backup_recovery_failed';
|
|
$recovery->completed_at = $now;
|
|
} elseif ($status === 'succeeded') {
|
|
if (!is_array($result)
|
|
|| ($result['restored'] ?? null) !== true
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($result['backup_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->sha256,
|
|
(string) ($result['sha256'] ?? ''))
|
|
|| CloudInstanceRuntimeService::normalizeState(
|
|
$result['power_state'] ?? null
|
|
) === 'unknown') {
|
|
throw new CloudError(
|
|
'Agent 恢复成功结果缺少可信的备份或电源身份',
|
|
502
|
|
);
|
|
}
|
|
$backupSpec = self::decodeSpec((string) $backup->spec_json);
|
|
if (($backupSpec['managed'] ?? null) !== true
|
|
|| ($backupSpec['os_source'] ?? null) !== 'cloud_template'
|
|
|| !is_string($backupSpec['os_name'] ?? null)
|
|
|| trim((string) $backupSpec['os_name']) === ''
|
|
|| ($backupSpec['login_name'] ?? null) !== 'root'
|
|
|| !is_bool($backupSpec['debian_12_network_fix'] ?? null)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'云备份的客户机身份快照无效',
|
|
409
|
|
);
|
|
}
|
|
$instanceSpec = self::decodeSpec((string) $instance->spec_json);
|
|
foreach ([
|
|
'template_id', 'template_revision',
|
|
'template_generation_token', 'os_source', 'os_name',
|
|
'login_name', 'debian_12_network_fix',
|
|
] as $field) {
|
|
if (array_key_exists($field, $backupSpec)) {
|
|
$instanceSpec[$field] = $backupSpec[$field];
|
|
} else {
|
|
unset($instanceSpec[$field]);
|
|
}
|
|
}
|
|
unset($instanceSpec['install_iso']);
|
|
// Passwords and hostnames may have been changed inside the guest
|
|
// without passing through the panel. The restored system disk is
|
|
// authoritative, so the panel must not claim either value is
|
|
// known or cause the Agent's NoCloud seed to inject an old value.
|
|
unset($instanceSpec['guest_hostname'], $instanceSpec['hostname']);
|
|
$instanceSpec['managed'] = true;
|
|
$instanceSpec['restored_from_backup'] =
|
|
(string) $backup->backup_key;
|
|
$instance->spec_json = json_encode(
|
|
CloudInput::safeResult($instanceSpec),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
$instance->sealed_credential = null;
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
$recovery->state = 'succeeded';
|
|
$recovery->error_code = '';
|
|
$recovery->completed_at = $now;
|
|
self::recordRuntime($instance, $result);
|
|
}
|
|
$recovery->updated_at = $now;
|
|
$recovery->save();
|
|
}
|
|
|
|
private static function priorRecoveryResult(CloudJob $job): array
|
|
{
|
|
$recovery = $job->backup_recovery_id === null ? null
|
|
: CloudBackupRecovery::find((int) $job->backup_recovery_id);
|
|
if ($recovery === null) {
|
|
throw new CloudError('云备份恢复任务目录记录缺失', 409);
|
|
}
|
|
return [
|
|
'recovery' => self::displayRecovery($recovery),
|
|
'job' => $job,
|
|
];
|
|
}
|
|
|
|
private static function displayRecovery(CloudBackupRecovery $recovery): array
|
|
{
|
|
return [
|
|
'recovery_key' => (string) $recovery->recovery_key,
|
|
'backup_id' => (string) $recovery->backup_id,
|
|
'instance_id' => (string) $recovery->instance_id,
|
|
'state' => (string) $recovery->state,
|
|
'created_at' => max(0, (int) $recovery->created_at),
|
|
'completed_at' => $recovery->completed_at === null ? null
|
|
: max(0, (int) $recovery->completed_at),
|
|
];
|
|
}
|
|
|
|
private static function assertManagedSource(CloudInstance $instance): void
|
|
{
|
|
$spec = self::decodeSpec((string) $instance->spec_json);
|
|
if (($spec['managed'] ?? false) !== true
|
|
|| ($spec['os_source'] ?? null) !== 'cloud_template') {
|
|
throw new CloudError('仅面板创建的云镜像实例支持可移植备份', 409);
|
|
}
|
|
self::resourcesFromSpec($spec);
|
|
}
|
|
|
|
private static function assertNoActiveJob(CloudInstance $instance): void
|
|
{
|
|
CloudInstanceMigrationService::assertInstanceMutationAvailable(
|
|
$instance
|
|
);
|
|
if (CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->exists()) {
|
|
throw new CloudError('虚拟机已有任务,请等待完成', 409);
|
|
}
|
|
}
|
|
|
|
private static function assertBackupDeletionAvailable(
|
|
CloudInstanceBackup $backup
|
|
): void {
|
|
if ((string) $backup->purpose !== 'portable') {
|
|
throw new CloudError('该备份类型不能由此入口删除', 409);
|
|
}
|
|
if ((string) $backup->state === 'uncertain') {
|
|
throw new CloudError('备份效果尚未确认,不能删除', 409);
|
|
}
|
|
if ($backup->active_job_id !== null) {
|
|
throw new CloudError('备份仍有关联任务,不能删除', 409);
|
|
}
|
|
if (CloudBackupRestore::where('backup_id', (int) $backup->id)
|
|
->whereIn('state', ['restoring', 'uncertain'])->exists()) {
|
|
throw new CloudError('备份仍有恢复任务或未知效果,不能删除', 409);
|
|
}
|
|
self::assertPortableRestoreAttemptsDeletable($backup);
|
|
if (CloudBackupRecovery::where('backup_id', (int) $backup->id)
|
|
->whereIn('state', ['recovering', 'uncertain'])->exists()) {
|
|
throw new CloudError('备份仍有原实例恢复任务或未知效果,不能删除', 409);
|
|
}
|
|
if (CloudJob::where('backup_id', (int) $backup->id)
|
|
->where(static function ($query): void {
|
|
$query->where('phase', '<>', 'terminal')
|
|
->orWhere('effect_resolution', 'unknown')
|
|
->orWhere('finalization_incident', 1);
|
|
})->exists()) {
|
|
throw new CloudError('备份仍有关联任务,不能删除', 409);
|
|
}
|
|
if (CloudDeploymentSnapshot::where('backup_id', (int) $backup->id)
|
|
->where('state', 'ready')->exists()) {
|
|
throw new CloudError('该对象已保存为云快照,请从云快照页面删除', 409);
|
|
}
|
|
}
|
|
|
|
/** Resolve one immutable backup read capability from the replica ledger. */
|
|
private static function backupReadUrl(CloudInstanceBackup $backup): string
|
|
{
|
|
$artifactId = filter_var(
|
|
$backup->storage_artifact_id,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
);
|
|
if ($artifactId === false) {
|
|
throw new CloudError('云备份缺少复制副本账本,不能读取', 409);
|
|
}
|
|
$capability = (new CloudStorageArtifactService())->presignedRead(
|
|
(int) $artifactId,
|
|
self::TRANSFER_URL_SECONDS
|
|
);
|
|
if ((int) ($capability['size_bytes'] ?? 0)
|
|
!== (int) $backup->object_size_bytes) {
|
|
throw new CloudError('云备份副本与备份目录账本不一致,不能读取', 409);
|
|
}
|
|
if (!is_string($capability['sha256'] ?? null)
|
|
|| !hash_equals(
|
|
(string) $backup->sha256,
|
|
(string) $capability['sha256']
|
|
)) {
|
|
throw new CloudError('云备份副本与备份目录账本不一致,不能读取', 409);
|
|
}
|
|
$url = $capability['url'] ?? null;
|
|
if (!is_string($url) || $url === '') {
|
|
throw new CloudError('云备份副本未返回有效读取凭据', 502);
|
|
}
|
|
return $url;
|
|
}
|
|
|
|
private static function assertPortableRestoreAttemptsDeletable(
|
|
CloudInstanceBackup $backup
|
|
): void {
|
|
$restoreIds = CloudBackupRestore::where(
|
|
'backup_id',
|
|
(int) $backup->id
|
|
)
|
|
->pluck('id')->map(static fn ($id): int => (int) $id)->all();
|
|
if ($restoreIds === []) {
|
|
return;
|
|
}
|
|
$attempts = CloudBackupRestoreAttempt::whereIn(
|
|
'backup_restore_id',
|
|
$restoreIds
|
|
)->orderBy('id')->get();
|
|
foreach ($attempts as $attempt) {
|
|
$state = (string) $attempt->state;
|
|
$resolution = (string) $attempt->effect_resolution;
|
|
if ($attempt->active_attempt !== null
|
|
|| in_array($state, [
|
|
'reserved', 'queued', 'awaiting_receipt',
|
|
'effect_unknown', 'finalization_pending',
|
|
], true)) {
|
|
throw new CloudError(
|
|
'备份仍有恢复尝试或未知效果,不能删除',
|
|
409
|
|
);
|
|
}
|
|
if ($state === 'succeeded' && $resolution === 'applied') {
|
|
continue;
|
|
}
|
|
if ($state !== 'failed' || $resolution !== 'not_applied'
|
|
|| $attempt->effect_resolved_at === null
|
|
|| $attempt->terminal_at === null) {
|
|
throw new CloudError('备份恢复尝试终态不明确,不能删除', 409);
|
|
}
|
|
$reservation = CloudResourceReservation::where(
|
|
'id',
|
|
(int) $attempt->reservation_id
|
|
)->where(
|
|
'backup_restore_attempt_key',
|
|
(string) $attempt->attempt_key
|
|
)->first();
|
|
if (!$reservation instanceof CloudResourceReservation
|
|
|| (string) $reservation->state !== 'released'
|
|
|| (int) $reservation->host_id
|
|
!== (int) $attempt->target_host_id
|
|
|| !hash_equals(
|
|
(string) $reservation->operation_id,
|
|
(string) $attempt->operation_id
|
|
)
|
|
|| $reservation->released_at === null) {
|
|
throw new CloudError('备份恢复尝试仍占用资源,不能删除', 409);
|
|
}
|
|
CloudReleasedPublicNetworkBundleVerifier::verify(
|
|
$attempt,
|
|
$reservation,
|
|
(int) $backup->user_id
|
|
);
|
|
}
|
|
}
|
|
|
|
/** @param array<string,mixed> $result */
|
|
private static function settleArtifactDeletion(
|
|
int $backupId,
|
|
array $result
|
|
): array {
|
|
if (($result['deleted'] ?? false) === true) {
|
|
return self::finishDelete($backupId);
|
|
}
|
|
$backup = CloudInstanceBackup::findOrFail($backupId);
|
|
$locked = $backup->getConnection()->transaction(function () use (
|
|
$backupId,
|
|
$result
|
|
): CloudInstanceBackup {
|
|
$row = CloudInstanceBackup::where('id', $backupId)
|
|
->lockForUpdate()->firstOrFail();
|
|
if ((string) $row->state === 'deleted') {
|
|
return $row;
|
|
}
|
|
self::assertBackupDeletionAvailable($row);
|
|
$attention = ($result['requires_attention'] ?? false) === true;
|
|
$row->state = $attention
|
|
? 'delete_attention' : 'delete_uncertain';
|
|
$nextCheck = (int) ($result['next_check_at'] ?? 0);
|
|
$row->delete_next_check_at = $attention ? 0 : max(
|
|
time() + 60,
|
|
$nextCheck
|
|
);
|
|
$row->delete_attention_at = $attention ? time() : null;
|
|
$row->error_code = $attention
|
|
? 'artifact_delete_attention'
|
|
: 'artifact_delete_reconciliation';
|
|
$row->updated_at = time();
|
|
$row->save();
|
|
return $row;
|
|
}, 3);
|
|
return self::deletionResult($locked);
|
|
}
|
|
|
|
private static function finishDelete(int $backupId): array
|
|
{
|
|
$backup = CloudInstanceBackup::findOrFail($backupId);
|
|
return $backup->getConnection()->transaction(function () use (
|
|
$backupId
|
|
): array {
|
|
$locked = CloudInstanceBackup::where('id', $backupId)
|
|
->lockForUpdate()->firstOrFail();
|
|
if ((string) $locked->state !== 'deleted') {
|
|
self::assertBackupDeletionAvailable($locked);
|
|
if (!in_array((string) $locked->state,
|
|
['ready', 'failed', 'deleting', 'delete_uncertain',
|
|
'delete_attention'], true)) {
|
|
throw new CloudError('备份删除状态已发生变化', 409);
|
|
}
|
|
$locked->state = 'deleted';
|
|
$locked->sealed_credential = null;
|
|
$locked->active_job_id = null;
|
|
$locked->delete_presence_confirmations = 0;
|
|
$locked->delete_next_check_at = 0;
|
|
$locked->delete_attention_at = null;
|
|
$locked->error_code = '';
|
|
$locked->updated_at = time();
|
|
$locked->save();
|
|
CloudDeploymentSnapshot::where('backup_id', $backupId)
|
|
->whereIn('state', ['deleting', 'delete_uncertain'])
|
|
->update([
|
|
'state' => 'deleted',
|
|
'deleted_at' => time(),
|
|
'error_code' => '',
|
|
'updated_at' => time(),
|
|
]);
|
|
}
|
|
return self::deletionResult($locked);
|
|
}, 3);
|
|
}
|
|
|
|
private static function deletionResult(CloudInstanceBackup $backup): array
|
|
{
|
|
$state = (string) $backup->state;
|
|
return [
|
|
'backup_id' => (string) $backup->id,
|
|
'state' => $state,
|
|
'deleted' => $state === 'deleted',
|
|
'requires_reconciliation' => in_array($state,
|
|
['deleting', 'delete_uncertain'], true),
|
|
'requires_attention' => $state === 'delete_attention',
|
|
'next_check_at' => max(0, (int) $backup->delete_next_check_at),
|
|
];
|
|
}
|
|
|
|
private static function createRequestKey(array $actor): string
|
|
{
|
|
$actorId = filter_var(
|
|
$actor['user_id'] ?? null,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
);
|
|
$key = $actor['request_idempotency_key'] ?? null;
|
|
if ($actorId === false || !is_string($key)
|
|
|| preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) {
|
|
throw new CloudError('备份创建请求缺少有效幂等标识', 400);
|
|
}
|
|
return $key;
|
|
}
|
|
|
|
/**
|
|
* Reject a stale scheduler generation while the subscription lock is
|
|
* held. A safely retired row is the sole authority for incrementing the
|
|
* generation; every other state must be resumed idempotently.
|
|
*/
|
|
private static function assertAutomaticScheduleAttempt(
|
|
CloudBackupSubscription $subscription,
|
|
int $scheduledFor,
|
|
int $scheduleAttempt
|
|
): void {
|
|
if ($scheduleAttempt < 1) {
|
|
throw new CloudError('自动备份计划重试代次无效', 409);
|
|
}
|
|
$latest = CloudInstanceBackup::where(
|
|
'subscription_id',
|
|
(int) $subscription->id
|
|
)->where('scheduled_for', $scheduledFor)
|
|
->where('source', 'automatic')
|
|
->orderByDesc('schedule_attempt')->orderByDesc('id')->first();
|
|
$expected = 1;
|
|
if ($latest instanceof CloudInstanceBackup) {
|
|
$expected = max(1, (int) $latest->schedule_attempt);
|
|
if ((string) $latest->state === 'deleted') {
|
|
if ((string) $latest->error_code
|
|
!== 'automatic_transient_retried'
|
|
|| $expected >= self::MAX_SCHEDULE_ATTEMPT) {
|
|
throw new CloudError(
|
|
'自动备份计划存在不可重用的已删除代次',
|
|
409
|
|
);
|
|
}
|
|
$expected++;
|
|
}
|
|
}
|
|
if ($scheduleAttempt !== $expected) {
|
|
throw new CloudError('自动备份计划重试代次已变化', 409);
|
|
}
|
|
}
|
|
|
|
private static function findCreateIntent(
|
|
int $actorId,
|
|
string $requestKey,
|
|
string $fingerprint,
|
|
int $instanceId,
|
|
bool $lock
|
|
): ?CloudInstanceBackup {
|
|
$query = CloudInstanceBackup::query()
|
|
->where('create_request_actor_id', $actorId)
|
|
->where('create_request_idempotency_key', $requestKey);
|
|
if ($lock) {
|
|
$query->lockForUpdate();
|
|
}
|
|
$backup = $query->first();
|
|
if (!$backup instanceof CloudInstanceBackup) {
|
|
return null;
|
|
}
|
|
if (!is_string($backup->create_request_fingerprint)
|
|
|| !hash_equals(
|
|
(string) $backup->create_request_fingerprint,
|
|
$fingerprint
|
|
)
|
|
|| (int) $backup->source_instance_id !== $instanceId
|
|
|| (string) $backup->purpose !== 'portable') {
|
|
throw new CloudError(
|
|
'同一请求标识不能用于不同的备份创建操作',
|
|
409
|
|
);
|
|
}
|
|
return $backup;
|
|
}
|
|
|
|
/** @return array{backup:array<string,mixed>,job:CloudJob} */
|
|
private static function finishCreatePreparation(
|
|
CloudInstanceBackup $backup
|
|
): array {
|
|
$job = (new CloudBackupPublicationPreparation())
|
|
->ensureAgentJob((int) $backup->id);
|
|
if (!$job instanceof CloudJob) {
|
|
throw new CloudError(
|
|
'备份上传正在核对对象存储状态,请稍后使用同一请求标识重试',
|
|
503
|
|
);
|
|
}
|
|
$backup->refresh();
|
|
return [
|
|
'backup' => self::displayBackup($backup, $job),
|
|
'job' => $job,
|
|
];
|
|
}
|
|
|
|
private static function priorJob(array $actor, string $kind,
|
|
?int $instanceId, ?int $backupId, string $fingerprint,
|
|
bool $lock = true): ?CloudJob
|
|
{
|
|
$key = $actor['request_idempotency_key'] ?? null;
|
|
if (!is_string($key) || $key === '') {
|
|
return null;
|
|
}
|
|
$query = CloudJob::where('user_id', (int) $actor['user_id'])
|
|
->where('request_idempotency_key', $key);
|
|
if ($lock) {
|
|
$query->lockForUpdate();
|
|
}
|
|
$job = $query->first();
|
|
if ($job === null) {
|
|
return null;
|
|
}
|
|
if ((string) $job->kind !== $kind
|
|
|| !hash_equals((string) $job->request_fingerprint,
|
|
$fingerprint)
|
|
|| ($instanceId !== null
|
|
&& (int) $job->instance_id !== $instanceId)
|
|
|| ($backupId !== null && (int) $job->backup_id !== $backupId)) {
|
|
throw new CloudError('同一请求标识不能用于不同的备份操作', 409);
|
|
}
|
|
return $job;
|
|
}
|
|
|
|
private static function resourcesFromSpec(array $spec): array
|
|
{
|
|
$cpu = filter_var($spec['vcpu'] ?? $spec['cpu_cores'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$memory = filter_var($spec['memory_mib'] ?? $spec['ram_mib'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$disk = filter_var($spec['disk_gib'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$bandwidth = filter_var($spec['bandwidth_mbps'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
if ($cpu === false || $memory === false || $disk === false
|
|
|| $bandwidth === false) {
|
|
throw new CloudError('实例缺少可信的资源规格快照', 409);
|
|
}
|
|
return ['cpu_cores' => (int) $cpu, 'memory_mib' => (int) $memory,
|
|
'disk_gib' => (int) $disk, 'bandwidth_mbps' => (int) $bandwidth];
|
|
}
|
|
|
|
/** @return array<string,mixed> */
|
|
private static function restoreTargetProjection(
|
|
CloudInstanceBackup $backup
|
|
): array
|
|
{
|
|
try {
|
|
$targets = array_map(
|
|
static fn (CloudPortableRestoreTargetPlan $target): array =>
|
|
self::restoreTargetDisplay($target),
|
|
CloudPortableRestoreTargetService::candidates($backup)
|
|
);
|
|
return ['targets' => $targets, 'reason' => null];
|
|
} catch (CloudError $error) {
|
|
return [
|
|
'targets' => [],
|
|
'reason' => $error->getMessage(),
|
|
];
|
|
}
|
|
}
|
|
|
|
/** @return array<string,mixed> */
|
|
private static function restoreTargetDisplay(
|
|
CloudPortableRestoreTargetPlan $plan
|
|
): array
|
|
{
|
|
$value = $plan->toArray();
|
|
$target = $value['target'];
|
|
$network = $value['public_network'];
|
|
$ipv4Available = max(
|
|
0,
|
|
(int) $network['ipv4_pool']['available_units']
|
|
);
|
|
$ipv6Available = max(
|
|
0,
|
|
(int) $network['ipv6_pool']['available_units']
|
|
);
|
|
return [
|
|
'host_id' => (string) $target['host_id'],
|
|
'host_name' => (string) $target['host_name'],
|
|
'region_id' => (string) $target['region_id'],
|
|
'region' => (string) $target['region_name'],
|
|
'country_code' => (string) $target['country_code'],
|
|
'line_id' => (string) $target['line_id'],
|
|
'line' => (string) $target['line_name'],
|
|
'resource_pool_id' => (string) $target['resource_pool_id'],
|
|
'resource_pool' => (string) $target['resource_pool_name'],
|
|
'network_attachment_id' =>
|
|
(string) $network['network_attachment_id'],
|
|
'network_selector' => (string) $network['network_selector'],
|
|
'network_selectors' => [(string) $network['network_selector']],
|
|
'available_ip_count' => min($ipv4Available, $ipv6Available),
|
|
'available_ipv4_count' => $ipv4Available,
|
|
'available_ipv6_count' => $ipv6Available,
|
|
'target_plan_sha256' => $plan->sha256(),
|
|
'is_source_host' => (bool) $target['is_source_host'],
|
|
];
|
|
}
|
|
|
|
/** Freeze template-specific boot compatibility into the backup manifest. */
|
|
private static function freezeRestoreCompatibility(array $spec): array
|
|
{
|
|
if (!is_bool($spec['debian_12_network_fix'] ?? null)) {
|
|
throw new CloudError(
|
|
'实例缺少创建时冻结的网络兼容快照,不能生成可移植备份',
|
|
409
|
|
);
|
|
}
|
|
return $spec;
|
|
}
|
|
|
|
private static function decodeSpec(string $encoded,
|
|
bool $throw = true): ?array
|
|
{
|
|
$spec = json_decode($encoded, true);
|
|
if (!is_array($spec) || array_is_list($spec)) {
|
|
if ($throw) {
|
|
throw new CloudError('实例备份规格快照损坏', 409);
|
|
}
|
|
return null;
|
|
}
|
|
return $spec;
|
|
}
|
|
|
|
private static function maximumObjectBytes(int $diskGib): int
|
|
{
|
|
if ($diskGib < 1 || $diskGib > 2559) {
|
|
throw new CloudError('实例系统盘超出当前备份最大容量', 409);
|
|
}
|
|
return ($diskGib + 1) * self::GIB;
|
|
}
|
|
|
|
private static function displayBackup(CloudInstanceBackup $backup,
|
|
?CloudJob $job, bool $canManage = true): array
|
|
{
|
|
$state = in_array((string) $backup->state,
|
|
['creating', 'ready', 'failed', 'uncertain', 'deleting',
|
|
'delete_uncertain', 'delete_attention', 'deleted'], true)
|
|
? (string) $backup->state : 'uncertain';
|
|
$cancelableCreating = $state === 'creating'
|
|
&& $backup->storage_artifact_id !== null
|
|
&& $backup->active_job_id === null
|
|
&& $backup->last_operation_id === null;
|
|
$canDelete = $canManage && in_array($state,
|
|
['ready', 'failed', 'deleting', 'delete_uncertain'], true);
|
|
$canDelete = $canDelete || ($canManage && $cancelableCreating);
|
|
if ($canDelete) {
|
|
try {
|
|
self::assertBackupDeletionAvailable($backup);
|
|
} catch (CloudError $error) {
|
|
$canDelete = false;
|
|
}
|
|
}
|
|
return [
|
|
'id' => (string) $backup->id,
|
|
'backup_key' => (string) $backup->backup_key,
|
|
'name' => (string) $backup->name,
|
|
'state' => $state,
|
|
'consistency' => in_array((string) $backup->consistency,
|
|
['filesystem', 'crash'], true)
|
|
? (string) $backup->consistency : 'unknown',
|
|
'object_size_bytes' => max(0, (int) $backup->object_size_bytes),
|
|
'virtual_size_bytes' => max(0, (int) $backup->virtual_size_bytes),
|
|
'sha256' => preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $backup->sha256) === 1
|
|
? (string) $backup->sha256 : '',
|
|
'source_host_id' => (string) $backup->source_host_id,
|
|
'created_at' => max(0, (int) $backup->created_at),
|
|
'ready_at' => $backup->ready_at === null ? null
|
|
: max(0, (int) $backup->ready_at),
|
|
'updated_at' => max(0, (int) $backup->updated_at),
|
|
'task_id' => $job === null ? null : (string) $job->id,
|
|
'job_status' => $job === null ? null
|
|
: (CloudOperationErrorCatalog::userStatus(
|
|
(string) $job->status
|
|
) ?? 'unknown'),
|
|
'cleanup_pending' => (string) $backup->error_code
|
|
=== 'backup_local_cleanup_pending',
|
|
'can_cleanup_workspace' => $canManage
|
|
&& (string) $backup->error_code
|
|
=== 'backup_local_cleanup_pending',
|
|
'can_delete' => $canDelete,
|
|
'delete_action' => in_array($state,
|
|
['deleting', 'delete_uncertain'], true)
|
|
? 'reconcile' : 'delete',
|
|
'error' => $state === 'delete_attention'
|
|
? '对象自动清理已达到三次上限,需要管理员检查对象存储后手动恢复清理。'
|
|
: ($state === 'delete_uncertain'
|
|
? '删除结果尚未确认;系统会在两次稳定核对后创建新的删除代次。'
|
|
: null),
|
|
];
|
|
}
|
|
|
|
private static function displayRestore(CloudBackupRestore $restore): array
|
|
{
|
|
return [
|
|
'restore_key' => (string) $restore->restore_key,
|
|
'state' => (string) $restore->state,
|
|
'target_host_id' => (string) $restore->target_host_id,
|
|
'target_instance_id' => $restore->target_instance_id === null
|
|
? null : (string) $restore->target_instance_id,
|
|
'name' => (string) $restore->new_name,
|
|
'created_at' => max(0, (int) $restore->created_at),
|
|
'ready_at' => $restore->ready_at === null ? null
|
|
: max(0, (int) $restore->ready_at),
|
|
];
|
|
}
|
|
|
|
private static function libraryFilters(array $input): array
|
|
{
|
|
CloudInput::only($input, ['page', 'per_page', 'q', 'user_id',
|
|
'instance_id', 'host_id', 'state']);
|
|
foreach ($input as $value) {
|
|
if (!is_string($value) && !is_int($value)) {
|
|
throw new CloudError('备份库筛选参数无效', 400);
|
|
}
|
|
}
|
|
$filters = [];
|
|
foreach (['user_id', 'instance_id', 'host_id'] as $key) {
|
|
$value = (string) ($input[$key] ?? '');
|
|
if ($value !== '' && (preg_match('/^[1-9][0-9]{0,17}$/D', $value) !== 1)) {
|
|
throw new CloudError('备份库 ' . $key . ' 必须为有效的正整数', 400);
|
|
}
|
|
$filters[$key] = $value;
|
|
}
|
|
$state = (string) ($input['state'] ?? '');
|
|
if (!in_array($state, ['', 'creating', 'ready', 'restoring', 'failed',
|
|
'uncertain', 'deleting', 'delete_uncertain', 'delete_attention'], true)) {
|
|
throw new CloudError('备份库状态筛选无效', 400);
|
|
}
|
|
$q = trim((string) ($input['q'] ?? ''));
|
|
if (preg_match('//u', $q) !== 1 || mb_strlen($q, 'UTF-8') > 80
|
|
|| preg_match('/[\x00-\x1f\x7f]/u', $q) !== 0) {
|
|
throw new CloudError('备份名称关键词最多 80 个字符,且不能包含控制字符', 400);
|
|
}
|
|
return $filters + ['state' => $state, 'q' => $q];
|
|
}
|
|
|
|
private static function libraryInteger($value, int $minimum, int $maximum,
|
|
int $default): int
|
|
{
|
|
if ($value === null || $value === '') {
|
|
return $default;
|
|
}
|
|
if (is_int($value)) {
|
|
$parsed = $value;
|
|
} elseif (is_string($value)
|
|
&& preg_match('/^(?:0|[1-9][0-9]{0,9})$/D', $value) === 1) {
|
|
$parsed = (int) $value;
|
|
} else {
|
|
throw new CloudError('备份库分页参数无效', 400);
|
|
}
|
|
if ($parsed < $minimum || $parsed > $maximum) {
|
|
throw new CloudError('备份库分页参数超出允许范围', 400);
|
|
}
|
|
return $parsed;
|
|
}
|
|
|
|
private static function recordRuntime(CloudInstance $instance,
|
|
array $result): void
|
|
{
|
|
try {
|
|
CloudInstanceRuntimeService::seed(
|
|
(int) $instance->id, (int) $instance->host_id,
|
|
(string) $instance->uuid
|
|
);
|
|
$state = $result['status']
|
|
?? ($result['state'] ?? ($result['power_state'] ?? null));
|
|
if (CloudInstanceRuntimeService::normalizeState($state)
|
|
!== 'unknown') {
|
|
CloudInstanceRuntimeService::markObserved(
|
|
(int) $instance->id, (int) $instance->host_id,
|
|
(string) $instance->uuid, $state
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
error_log('cloud backup restore runtime cache unavailable: '
|
|
. get_class($error));
|
|
}
|
|
}
|
|
}
|