You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudBackupService.php

2139 lines
90 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudBackupRestore;
use App\Models\CloudBackupRestoreAttempt;
use App\Models\CloudBackupRecovery;
use App\Models\CloudBackupPeriodEntitlement;
use App\Models\CloudBackupSubscription;
use App\Models\CloudDeploymentSnapshot;
use App\Models\CloudHost;
use App\Models\CloudHostAgent;
use App\Models\CloudInstance;
use App\Models\CloudInstanceBackup;
use App\Models\CloudInstanceMigration;
use App\Models\CloudJob;
use App\Models\CloudResourceReservation;
/** PHP-owned catalog and resource ledger for portable system-disk backups. */
final class CloudBackupService
{
public const MAX_PER_INSTANCE = 10;
private const ACTIVE_BACKUP_STATES = [
'creating', 'ready', 'uncertain',
];
private const CLEANUP_DEBT_STATES = [
'deleting', 'delete_uncertain', 'delete_attention',
];
private const TRANSFER_URL_SECONDS = 604800;
private const GIB = 1073741824;
/**
* Global administrator backup catalog. It is intentionally keyed by the
* backup ledger instead of the source VM lifecycle, so a retained backup
* remains restorable and deletable after expiry cleanup tombstones the
* source instance.
*/
public static function library(array $actor, array $input): array
{
if (($actor['is_admin'] ?? false) !== true) {
throw new CloudError('只有管理员可以查看虚拟机备份', 403);
}
$filters = self::libraryFilters($input);
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
$perPage = self::libraryInteger($input['per_page'] ?? 20,
20, 50, 20);
if (!in_array($perPage, [20, 50], true)) {
throw new CloudError('备份库每页条数仅支持 20 或 50', 400);
}
$query = CloudInstanceBackup::where('purpose', 'portable')
->where('state', '<>', 'deleted');
foreach (['user_id' => 'user_id', 'instance_id' => 'source_instance_id',
'host_id' => 'source_host_id', 'state' => 'state'] as $key => $column) {
if ($filters[$key] !== '') {
$query->where($column, $filters[$key]);
}
}
if ($filters['q'] !== '') {
$like = '%' . str_replace(['!', '%', '_'], ['!!', '!%', '!_'], $filters['q']) . '%';
$query->whereRaw("name LIKE ? ESCAPE '!'", [$like]);
}
$total = (int) (clone $query)->count('id');
$totalPages = max(1, (int) ceil($total / $perPage));
$page = min($page, $totalPages);
$rows = $query->orderByDesc('id')
->offset(($page - 1) * $perPage)->limit($perPage)->get();
$sourceIds = [];
$hostIds = [];
$operationIds = [];
foreach ($rows as $row) {
$sourceIds[] = (int) $row->source_instance_id;
$hostIds[] = (int) $row->source_host_id;
if (preg_match('/^[a-f0-9]{32}$/D',
(string) $row->last_operation_id) === 1) {
$operationIds[] = (string) $row->last_operation_id;
}
}
$sources = [];
if ($sourceIds !== []) {
foreach (CloudInstance::whereIn('id', array_values(array_unique(
$sourceIds
)))->get(['id', 'name', 'status', 'user_id']) as $source) {
$sources[(int) $source->id] = $source;
}
}
$hosts = [];
if ($hostIds !== []) {
foreach (CloudHost::whereIn('id', array_values(array_unique(
$hostIds
)))->get(['id', 'name', 'region']) as $host) {
$hosts[(int) $host->id] = $host;
}
}
$jobs = [];
if ($operationIds !== []) {
foreach (CloudJob::whereIn('operation_id', array_values(array_unique(
$operationIds
)))->get() as $job) {
$jobs[(string) $job->operation_id] = $job;
}
}
$canManage = ($actor['is_super_admin'] ?? false) === true
&& ($actor['can_manage'] ?? false) === true;
$items = [];
foreach ($rows as $row) {
$source = $sources[(int) $row->source_instance_id] ?? null;
$host = $hosts[(int) $row->source_host_id] ?? null;
$item = self::displayBackup(
$row,
$jobs[(string) $row->last_operation_id] ?? null,
$canManage
);
$item['source'] = [
'instance_id' => (string) $row->source_instance_id,
'name' => $source === null ? '' : (string) $source->name,
'status' => $source === null ? 'missing'
: (string) $source->status,
'deleted' => $source === null
|| (string) $source->status === 'deleted',
'user_id' => (string) $row->user_id,
'host_id' => (string) $row->source_host_id,
'host_name' => $host === null ? '' : (string) $host->name,
'region' => $host === null ? '' : (string) $host->region,
];
if ((string) $row->state === 'ready') {
$projection = self::restoreTargetProjection($row);
$item['restore_targets'] = $projection['targets'];
$item['restore_unavailable_reason'] =
$projection['reason'];
}
$items[] = $item;
}
$from = $total === 0 ? 0 : (($page - 1) * $perPage) + 1;
return [
'items' => $items,
'filters' => $filters,
'can_manage' => $canManage,
'pagination' => [
'page' => $page,
'per_page' => $perPage,
'total' => $total,
'total_pages' => $totalPages,
'from' => $from,
'to' => $from === 0 ? 0 : $from + count($items) - 1,
],
];
}
public static function listing(CloudInstance $instance, array $actor): array
{
if (($actor['is_admin'] ?? false) !== true) {
throw new CloudError('只有管理员可以查看虚拟机备份', 403);
}
$canManage = ($actor['is_super_admin'] ?? false) === true
&& ($actor['can_manage'] ?? false) === true;
if ((string) $instance->status === 'deleted') {
throw new CloudError('虚拟机不存在', 404);
}
$rows = CloudInstanceBackup::where(
'source_instance_id', (int) $instance->id
)->where('purpose', 'portable')->where('state', '<>', 'deleted')
->orderByDesc('id')->get();
$operationIds = [];
foreach ($rows as $row) {
if (preg_match('/^[a-f0-9]{32}$/D',
(string) $row->last_operation_id) === 1) {
$operationIds[] = (string) $row->last_operation_id;
}
}
$jobs = [];
if ($operationIds !== []) {
foreach (CloudJob::whereIn('operation_id',
array_values(array_unique($operationIds)))->get() as $job) {
$jobs[(string) $job->operation_id] = $job;
}
}
$backups = [];
$targetMap = [];
$manualActiveCount = 0;
foreach ($rows as $row) {
if ((string) $row->source === 'manual'
&& in_array((string) $row->state,
self::ACTIVE_BACKUP_STATES, true)) {
$manualActiveCount++;
}
$display = self::displayBackup(
$row,
$jobs[(string) $row->last_operation_id] ?? null,
$canManage
);
if ((string) $row->state === 'ready') {
$projection = self::restoreTargetProjection($row);
foreach ($projection['targets'] as $normalized) {
$targetMap[$normalized['target_plan_sha256']] =
$normalized;
}
$display['restore_targets'] = $projection['targets'];
$display['restore_unavailable_reason'] =
$projection['reason'];
}
$backups[] = $display;
}
$active = CloudJobService::blockingQueryForInstance(
(int) $instance->id
)->orderBy('id')->first();
$spec = self::decodeSpec((string) $instance->spec_json, false);
$compatibilityFrozen = $spec !== null
&& is_bool($spec['debian_12_network_fix'] ?? null);
$supported = $spec !== null
&& ($spec['managed'] ?? false) === true
&& ($spec['os_source'] ?? null) === 'cloud_template'
&& $compatibilityFrozen;
$storage = CloudObjectStorageBackendRegistry::summary();
$storageWritable = (int) ($storage['writable_backends'] ?? 0) > 0;
$migration = CloudInstanceMigration::where(
'instance_id', (int) $instance->id
)->orderByDesc('id')->first();
$migrationTargetCandidates = $supported
? CloudPlacementService::maintenanceMigrationTargets(
$instance,
self::resourcesFromSpec($spec),
CloudPeerMigrationArtifactService::advisoryAdditionalDiskGib(
$instance
)
) : [];
$migrationTargets = array_map(
static fn (array $target): array => [
'host_id' => (string) ($target['id'] ?? ''),
'host_name' => (string) ($target['name'] ?? ''),
'region' => (string) ($target['region'] ?? ''),
'resource_pool_id' => (string) (
$target['resource_pool_id'] ?? ''
),
'resource_pool' => (string) ($target['resource_pool'] ?? ''),
'line' => (string) ($target['line'] ?? ''),
'network_selectors' => array_values(
is_array($target['network_selectors'] ?? null)
? $target['network_selectors'] : []
),
'available_ip_count' =>
(int) ($target['available_ip_count'] ?? 0),
],
array_values(array_filter(
$migrationTargetCandidates,
static fn (array $target): bool =>
(int) ($target['available_ip_count'] ?? 0) > 0
))
);
$migrationBlocks = $migration !== null
&& CloudInstanceMigrationWorkflow::blocksInstance(
(string) $migration->state
);
$canOperate = $canManage
&& $active === null
&& !$migrationBlocks
&& (string) $instance->status === 'active'
&& $supported
&& $storageWritable;
$canMigrate = $canManage
&& $active === null
&& !($migration !== null
&& CloudInstanceMigrationWorkflow::blocksNewMigration(
(string) $migration->state
))
&& (string) $instance->status === 'active'
&& $supported
&& $migrationTargets !== [];
$canCreate = $canOperate
&& $manualActiveCount < self::MAX_PER_INSTANCE;
$reason = null;
if (!$canManage) {
$reason = '当前管理员仅可查看,超级管理员才可以创建或恢复备份';
} elseif (!$storageWritable) {
$reason = '请先启用至少一个已通过验证且健康的对象存储后端';
} elseif ($spec !== null
&& ($spec['managed'] ?? false) === true
&& ($spec['os_source'] ?? null) === 'cloud_template'
&& !$compatibilityFrozen) {
$reason = '实例缺少创建时冻结的网络兼容快照,不能生成可移植备份';
} elseif (!$supported) {
$reason = '仅面板创建的云镜像实例支持可移植备份';
} elseif ($active !== null) {
$reason = '实例已有任务,请等待完成';
} elseif ($migrationBlocks) {
$reason = '实例已有未完成的维护迁移,请先完成核对或源副本清理';
} elseif ((string) $instance->status !== 'active') {
$reason = '只有活动实例可以创建备份';
} elseif (!$canCreate) {
$reason = '每台实例最多保留十个有效备份';
}
return [
'instance_id' => (string) $instance->id,
'backups' => $backups,
'restore_targets' => array_values($targetMap),
'migration_targets' => $migrationTargets,
'maintenance_migration' => $migration === null ? null
: CloudInstanceMigrationService::display($migration),
'max_backups' => self::MAX_PER_INSTANCE,
'can_create' => $canCreate,
'can_restore' => $canManage && $targetMap !== [],
'can_migrate' => $canMigrate,
'create_block_reason' => $reason,
'active_job' => $active === null ? null : [
'id' => (string) $active->id,
'kind' => (string) $active->kind,
'status' => CloudOperationErrorCatalog::userStatus(
(string) $active->status
) ?? 'unknown',
],
];
}
/**
* Delete or reconcile every replica of one backup storage artifact.
*
* Per-replica operations own the no-replay and bounded reconciliation
* contract; the backup row only projects their aggregate cleanup state.
*/
public static function delete(CloudInstanceBackup $backup, array $actor,
?CloudObjectStorageS3Client $client = null,
?callable $beforeDeleteLocked = null,
?CloudStorageArtifactDeletionService $artifactDeletion = null): array
{
CloudAccess::manage($actor);
$client ??= new CloudObjectStorageS3Client();
$now = time();
$prepared = $backup->getConnection()->transaction(function () use (
$backup, $beforeDeleteLocked, $now
): array {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->first();
if ($locked === null) {
throw new CloudError('备份不存在', 404);
}
$state = (string) $locked->state;
if ($state === 'deleted') {
return ['mode' => 'deleted', 'backup' => $locked];
}
self::assertBackupDeletionAvailable($locked);
if ($locked->storage_artifact_id === null) {
throw new CloudError(
'备份缺少复制对象存储账本,不能执行删除',
409
);
}
if ($state === 'delete_attention') {
return ['mode' => 'attention', 'backup' => $locked];
}
if ($locked->storage_artifact_id !== null) {
if ($state === 'creating') {
if ($locked->active_job_id !== null
|| $locked->last_operation_id !== null) {
throw new CloudError(
'备份创建已绑定任务,不能按未派发意图取消',
409
);
}
return [
'mode' => 'artifact_cancel',
'artifact_id' =>
(int) $locked->storage_artifact_id,
'backup' => $locked,
];
}
if (in_array($state, ['deleting', 'delete_uncertain'], true)) {
if ((int) $locked->delete_next_check_at > $now) {
return ['mode' => 'waiting', 'backup' => $locked];
}
} elseif (!in_array($state, ['ready', 'failed'], true)) {
throw new CloudError('当前备份状态不允许删除', 409);
} else {
if ($beforeDeleteLocked !== null) {
$beforeDeleteLocked($locked);
}
$locked->state = 'deleting';
$locked->delete_requested_at = $now;
$locked->delete_next_check_at = 0;
$locked->delete_attention_at = null;
$locked->error_code = '';
$locked->updated_at = $now;
$locked->save();
}
return [
'mode' => 'artifact',
'artifact_id' => (int) $locked->storage_artifact_id,
'backup' => $locked,
];
}
throw new \LogicException(
'Replicated backup deletion did not select an artifact mode'
);
}, 3);
if (in_array($prepared['mode'], ['deleted', 'waiting', 'attention'], true)) {
return self::deletionResult($prepared['backup']);
}
if (in_array($prepared['mode'], [
'artifact', 'artifact_cancel',
], true)) {
$service = $artifactDeletion
?? new CloudStorageArtifactDeletionService($client);
$expectedOwner = ['backup_id' => (int) $backup->id];
if ($prepared['mode'] === 'artifact_cancel') {
$service->prepareNeverDispatched(
(int) $prepared['artifact_id'],
$expectedOwner
);
$backup->getConnection()->transaction(function () use (
$backup,
$prepared,
$beforeDeleteLocked
): void {
$locked = CloudInstanceBackup::where(
'id', (int) $backup->id
)->lockForUpdate()->firstOrFail();
if ((int) ($locked->storage_artifact_id ?? 0)
!== (int) $prepared['artifact_id']) {
throw new CloudError(
'备份与对象存储删除归属已发生变化',
409
);
}
if (in_array((string) $locked->state,
['deleting', 'delete_uncertain'], true)) {
return;
}
if ((string) $locked->state !== 'creating'
|| $locked->active_job_id !== null
|| $locked->last_operation_id !== null) {
throw new CloudError(
'备份创建状态已变化,不能取消',
409
);
}
if ($beforeDeleteLocked !== null) {
$beforeDeleteLocked($locked);
}
$now = time();
$locked->state = 'deleting';
$locked->delete_requested_at = $now;
$locked->delete_next_check_at = 0;
$locked->delete_attention_at = null;
$locked->error_code = '';
$locked->updated_at = $now;
$locked->save();
}, 3);
}
$result = $service->process(
(int) $prepared['artifact_id'],
$expectedOwner
);
return self::settleArtifactDeletion(
(int) $backup->id,
$result
);
}
throw new \LogicException(
'Replicated backup deletion returned an unsupported mode'
);
}
/** Sanitized administrator view of remote objects awaiting cleanup. */
public static function cleanupDebt(array $actor, array $input = []): array
{
if (($actor['is_admin'] ?? false) !== true) {
throw new CloudError('只有管理员可以查看对象清理异常', 403);
}
CloudInput::only($input, ['page', 'per_page']);
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
$perPage = self::libraryInteger($input['per_page'] ?? 20, 10, 50, 20);
$query = CloudInstanceBackup::whereIn('state', self::CLEANUP_DEBT_STATES);
$total = (int) (clone $query)->count('id');
$bytes = (int) (clone $query)->sum('object_size_bytes');
$attention = (int) (clone $query)
->where('state', 'delete_attention')->count('id');
$totalPages = max(1, (int) ceil($total / $perPage));
$page = min($page, $totalPages);
$rows = $query->orderByRaw(
"CASE WHEN state = 'delete_attention' THEN 0 ELSE 1 END"
)->orderBy('delete_next_check_at')->orderBy('id')
->offset(($page - 1) * $perPage)->limit($perPage)->get();
$items = [];
foreach ($rows as $row) {
$items[] = [
'id' => (string) $row->id,
'name' => (string) $row->name,
'source_instance_id' => (string) $row->source_instance_id,
'user_id' => (string) $row->user_id,
'state' => (string) $row->state,
'object_size_bytes' => max(0, (int) $row->object_size_bytes),
'delete_generation' => max(0, (int) $row->delete_generation),
'cycle_attempts' => max(0, (int) $row->delete_cycle_attempts),
'next_check_at' => max(0, (int) $row->delete_next_check_at),
'attention_at' => $row->delete_attention_at === null ? null
: max(0, (int) $row->delete_attention_at),
'error_code' => (string) $row->error_code,
'can_retry' => (string) $row->state === 'delete_attention'
&& ($actor['is_super_admin'] ?? false) === true
&& ($actor['can_manage'] ?? false) === true,
];
}
return [
'summary' => [
'count' => $total,
'bytes' => $bytes,
'attention_count' => $attention,
],
'items' => $items,
'pagination' => [
'page' => $page,
'per_page' => $perPage,
'total' => $total,
'total_pages' => $totalPages,
],
];
}
/** Begin one new bounded cycle after an administrator has inspected debt. */
public static function retryCleanup(CloudInstanceBackup $backup, array $actor,
?CloudObjectStorageS3Client $client = null,
?CloudStorageArtifactDeletionService $artifactDeletion = null): array
{
CloudAccess::manage($actor);
$client ??= new CloudObjectStorageS3Client();
$prepared = $backup->getConnection()->transaction(function () use ($backup): array {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->firstOrFail();
if ((string) $locked->state !== 'delete_attention') {
throw new CloudError('只有待人工处理的对象可以重新进入清理流程', 409);
}
self::assertBackupDeletionAvailable($locked);
if ($locked->storage_artifact_id === null) {
throw new CloudError(
'备份缺少复制对象存储账本,不能继续清理',
409
);
}
return [
'artifact_id' => (int) $locked->storage_artifact_id,
'backup' => $locked,
];
}, 3);
$service = $artifactDeletion
?? new CloudStorageArtifactDeletionService($client);
$expectedOwner = ['backup_id' => (int) $backup->id];
$service->reopenAttention(
(int) $prepared['artifact_id'],
$expectedOwner
);
return self::settleArtifactDeletion(
(int) $backup->id,
$service->process(
(int) $prepared['artifact_id'],
$expectedOwner
)
);
}
public static function create(CloudInstance $instance, array $actor,
array $input): array
{
return self::createInternal($instance, $actor, $input, null);
}
/** Queue one deterministic paid automatic backup for a scheduled slot. */
public static function createAutomatic(CloudInstance $instance,
CloudBackupSubscription $subscription, array $actor,
int $scheduledFor): array
{
if ($scheduledFor < 1) {
throw new CloudError('自动备份计划时间无效', 409);
}
$zone = new \DateTimeZone(CloudAutomaticBackupService::TIMEZONE);
$name = 'autobackup-'
. (new \DateTimeImmutable('@' . $scheduledFor))
->setTimezone($zone)->format('Ymd-His');
return self::createInternal($instance, $actor, [
'name' => $name,
'consistency_mode' => 'filesystem',
], [
'subscription_id' => (int) $subscription->id,
'scheduled_for' => $scheduledFor,
]);
}
private static function createInternal(CloudInstance $instance, array $actor,
array $input, ?array $automatic): array
{
if ($automatic === null) {
CloudAccess::manage($actor);
} elseif ((int) ($actor['user_id'] ?? 0)
!== (int) $instance->user_id) {
throw new CloudError('无权为该实例创建自动备份', 403);
}
CloudInput::only($input, ['name', 'consistency_mode']);
$name = CloudInput::snapshotName($input['name'] ?? null);
$mode = CloudInput::snapshotConsistency(
$input['consistency_mode'] ?? null
);
$requestTarget = '/v1/vms/'
. rawurlencode((string) $instance->uuid) . '/backups';
$canonicalRequest = [
'instance_id' => (int) $instance->id,
'name' => $name,
'consistency_mode' => $mode,
];
if ($automatic !== null) {
$canonicalRequest['source'] = 'automatic';
$canonicalRequest['subscription_id'] =
(int) $automatic['subscription_id'];
$canonicalRequest['scheduled_for'] =
(int) $automatic['scheduled_for'];
}
$fingerprint = CloudJobService::standaloneRequestFingerprint(
(int) $instance->host_id,
'backup_create',
'POST',
$requestTarget,
$canonicalRequest
);
$requestKey = self::createRequestKey($actor);
$actorId = (int) $actor['user_id'];
$prior = self::priorJob(
$actor,
'backup_create',
(int) $instance->id,
null,
$fingerprint,
false
);
if ($prior !== null) {
$backup = CloudInstanceBackup::find((int) $prior->backup_id);
if ($backup === null) {
throw new CloudError('备份任务目录记录缺失', 409);
}
return ['backup' => self::displayBackup($backup, $prior),
'job' => $prior];
}
$reserved = self::findCreateIntent(
$actorId,
$requestKey,
$fingerprint,
(int) $instance->id,
false
);
if ($reserved instanceof CloudInstanceBackup) {
return self::finishCreatePreparation($reserved);
}
try {
$backup = $instance->getConnection()->transaction(function () use (
$instance, $actor, $name, $mode, $fingerprint, $automatic,
$requestKey, $actorId
): CloudInstanceBackup {
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->first();
if ($locked === null || (string) $locked->status === 'deleted') {
throw new CloudError('虚拟机不存在', 404);
}
$prior = self::priorJob($actor, 'backup_create',
(int) $locked->id, null, $fingerprint);
if ($prior !== null) {
$backup = CloudInstanceBackup::find((int) $prior->backup_id);
if ($backup === null) {
throw new CloudError('备份任务目录记录缺失', 409);
}
return $backup;
}
$existingIntent = self::findCreateIntent(
$actorId,
$requestKey,
$fingerprint,
(int) $locked->id,
true
);
if ($existingIntent instanceof CloudInstanceBackup) {
return $existingIntent;
}
$automaticSubscription = null;
if ($automatic !== null) {
$automaticSubscription = CloudBackupSubscription::where(
'id', (int) $automatic['subscription_id']
)->lockForUpdate()->first();
$scheduledFor = (int) $automatic['scheduled_for'];
$entitled = $automaticSubscription !== null
&& CloudBackupPeriodEntitlement::where(
'subscription_id',
(int) $automaticSubscription->id
)->where('instance_id', (int) $locked->id)
->where('user_id', (int) $locked->user_id)
->where('starts_at', '<=', $scheduledFor)
->where('ends_at', '>', $scheduledFor)
->lockForUpdate()->exists();
if ($automaticSubscription === null
|| !(bool) $automaticSubscription->enabled
|| (int) $automaticSubscription->instance_id
!== (int) $locked->id
|| (int) $automaticSubscription->user_id
!== (int) $locked->user_id
|| !$entitled) {
throw new CloudError('自动备份订阅与实例账本不一致', 409);
}
$existing = CloudInstanceBackup::where(
'subscription_id', (int) $automaticSubscription->id
)->where('scheduled_for',
$scheduledFor)->first();
if ($existing !== null) {
if ((int) ($existing->create_request_actor_id ?? 0)
!== $actorId
|| !is_string(
$existing->create_request_idempotency_key
)
|| !hash_equals(
(string) $existing->create_request_idempotency_key,
$requestKey
)
|| !is_string($existing->create_request_fingerprint)
|| !hash_equals(
(string) $existing->create_request_fingerprint,
$fingerprint
)) {
throw new CloudError(
'自动备份计划已绑定其他创建请求',
409
);
}
return $existing;
}
}
if ((string) $locked->status !== 'active') {
throw new CloudError('只有活动实例可以创建备份', 409);
}
if ($automatic !== null
&& ((string) $locked->service_status !== 'active'
|| CloudAutomaticBackupService::
shouldPauseForInstanceExpiry($locked, time()))) {
throw new CloudError('实例服务当前不可创建自动备份', 409);
}
self::assertManagedSource($locked);
self::assertNoActiveJob($locked);
$hostAgent = CloudHostAgent::where(
'host_id', (int) $locked->host_id
)->first();
$sourceCompatibility = $hostAgent?->
testedPortableRestoreCompatibility();
if ($sourceCompatibility === null) {
throw new CloudError(
'源宿主机当前版本尚未通过可移植恢复能力测试',
409
);
}
$usedQuery = CloudInstanceBackup::where(
'source_instance_id', (int) $locked->id
)->where('purpose', 'portable')
->whereIn('state', self::ACTIVE_BACKUP_STATES);
if ($automatic === null) {
$usedQuery->where('source', 'manual');
} else {
$usedQuery->where('source', 'automatic')
->where('subscription_id',
(int) $automaticSubscription->id)
->whereNotExists(static function ($query): void {
$query->selectRaw('1')
->from('cloud_deployment_snapshots')
->whereColumn(
'cloud_deployment_snapshots.backup_id',
'cloud_instance_backups.id'
)->where(
'cloud_deployment_snapshots.state',
'<>',
'deleted'
);
});
}
$used = (int) $usedQuery->count('id');
if ($automatic === null && $used >= self::MAX_PER_INSTANCE) {
throw new CloudError('每台实例最多保留十个有效备份', 409);
}
if ($automatic !== null
&& $used >= min(
CloudAutomaticBackupService::MAX_RETENTION,
max(1, (int) $automaticSubscription->retention_count)
) + 1) {
throw new CloudError('自动备份正在等待旧备份安全清理', 409);
}
$spec = self::freezeRestoreCompatibility(
self::decodeSpec((string) $locked->spec_json)
);
$resources = self::resourcesFromSpec($spec);
$maximumBytes = self::maximumObjectBytes(
$resources['disk_gib']
);
$key = bin2hex(random_bytes(16));
$objectSuffix = 'vm-backups/' . $key . '/system.qcow2';
// The owner row must exist before the storage publication can
// be bound to it. Until placement selects a primary, the
// suffix is a unique non-secret placeholder for object_key.
$objectKey = $objectSuffix;
$now = time();
$backup = new CloudInstanceBackup();
$backup->backup_key = $key;
$backup->storage_artifact_id = null;
$backup->create_request_actor_id = $actorId;
$backup->create_request_idempotency_key = $requestKey;
$backup->create_request_fingerprint = $fingerprint;
$backup->source_instance_id = (int) $locked->id;
$backup->source_host_id = (int) $locked->host_id;
$backup->user_id = (int) $locked->user_id;
$backup->source_uuid = strtolower((string) $locked->uuid);
$backup->name = $name;
$backup->purpose = 'portable';
$backup->source = $automatic === null
? 'manual' : 'automatic';
$backup->subscription_id = $automatic === null ? null
: (int) $automaticSubscription->id;
$backup->scheduled_for = $automatic === null ? null
: (int) $automatic['scheduled_for'];
$backup->state = 'creating';
$backup->consistency = $mode;
$backup->object_key = $objectKey;
$backup->object_size_bytes = 0;
$backup->virtual_size_bytes = 0;
$backup->sha256 = '';
$backup->source_portable_restore_contract_json =
$sourceCompatibility->json();
$backup->source_portable_restore_contract_sha256 =
$sourceCompatibility->sha256();
$backup->spec_json = json_encode(
CloudInput::safeResult($spec),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
$backup->sealed_credential = $locked->sealed_credential;
$backup->storage_revision = 0;
$backup->active_job_id = null;
$backup->last_operation_id = null;
$backup->error_code = '';
$backup->ready_at = null;
$backup->created_at = $backup->updated_at = $now;
$backup->save();
$publication = CloudStorageArtifactService::
beginAgentPublication(
'backup',
$objectSuffix,
$maximumBytes,
'backup-publish-' . hash('sha256', implode('|', [
(string) $actorId, $requestKey,
])),
hash('sha256', 'backup|' . $fingerprint),
['backup_id' => (int) $backup->id]
);
$objectKey = (string) $publication['object_key'];
$backup->storage_artifact_id =
(int) $publication['artifact_id'];
$backup->object_key = $objectKey;
$backup->updated_at = time();
$backup->save();
return $backup;
}, 3);
} catch (\Throwable $error) {
$reserved = self::findCreateIntent(
$actorId,
$requestKey,
$fingerprint,
(int) $instance->id,
false
);
if (!$reserved instanceof CloudInstanceBackup) {
throw $error;
}
$backup = $reserved;
}
return self::finishCreatePreparation($backup);
}
public static function restore(CloudInstanceBackup $backup, array $actor,
array $input): array
{
$result = CloudPortableRestoreRequestService::request(
$backup,
$actor,
$input
);
return [
'restore' => self::displayRestore($result['restore']),
'job' => $result['job'],
];
}
/**
* Build the short-lived download capability for a paid create order.
* The caller must already hold and validate the owner/order row locks;
* signing is local and therefore does not extend that transaction across
* a remote object-storage request.
*
* @return array{source_url:string}
*/
public static function provisioningSource(
CloudInstanceBackup $backup
): array {
if ((string) $backup->purpose !== 'portable'
|| (string) $backup->state !== 'ready'
|| (int) $backup->object_size_bytes < 1
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $backup->sha256) !== 1) {
throw new CloudError('云备份尚未处于可创建实例状态', 409);
}
return ['source_url' => self::backupReadUrl($backup)];
}
/** Replace an owner's existing system disk from its own automatic backup. */
public static function recover(CloudInstance $instance,
CloudInstanceBackup $backup, array $actor, array $input): array
{
if ((int) ($actor['user_id'] ?? 0) !== (int) $instance->user_id) {
throw new CloudError('无权恢复该实例', 403);
}
CloudInput::only($input, ['confirm_restore']);
if (($input['confirm_restore'] ?? false) !== true) {
throw new CloudError('请确认恢复会覆盖当前系统盘内容');
}
$requestKey = $actor['request_idempotency_key'] ?? null;
if (!is_string($requestKey) || $requestKey === '') {
throw new CloudError('恢复云备份必须提供请求幂等标识', 400);
}
$target = '/v1/vms/' . rawurlencode((string) $instance->uuid)
. '/recover-backup';
$canonical = [
'instance_id' => (int) $instance->id,
'backup_id' => (int) $backup->id,
'confirm_restore' => true,
];
$fingerprint = CloudJobService::standaloneRequestFingerprint(
(int) $instance->host_id,
'backup_recover',
'POST',
$target,
$canonical
);
$prior = self::priorJob($actor, 'backup_recover',
(int) $instance->id, (int) $backup->id, $fingerprint, false);
if ($prior !== null) {
return self::priorRecoveryResult($prior);
}
CloudOriginalBackupRestorePolicy::assertEligible($instance, $backup);
$sourceUrl = self::backupReadUrl($backup);
return $instance->getConnection()->transaction(function () use (
$instance, $backup, $actor, $target, $canonical, $fingerprint,
$sourceUrl
): array {
$lockedInstance = CloudInstance::where(
'id', (int) $instance->id
)->lockForUpdate()->first();
$lockedBackup = CloudInstanceBackup::where(
'id', (int) $backup->id
)->lockForUpdate()->first();
if ($lockedInstance === null || $lockedBackup === null) {
throw new CloudError('实例或云备份不存在', 404);
}
$prior = self::priorJob($actor, 'backup_recover',
(int) $lockedInstance->id, (int) $lockedBackup->id,
$fingerprint);
if ($prior !== null) {
return self::priorRecoveryResult($prior);
}
CloudOriginalBackupRestorePolicy::assertEligible(
$lockedInstance, $lockedBackup
);
$claimSet = CloudPublicNetworkAllocationService::currentClaimSet(
(int) ($lockedInstance->reservation_id ?? 0),
'committed',
(int) $lockedInstance->id
);
$instanceSpec = self::decodeSpec(
(string) $lockedInstance->spec_json
);
try {
$frozenNetwork = CloudPublicNetworkSnapshot::fromArray(
$instanceSpec['public_network'] ?? null
);
} catch (\Throwable) {
throw new CloudError(
'实例缺少有效的双栈公网网络快照,不能恢复',
409
);
}
if (!hash_equals(
$claimSet->publicNetwork()->fingerprint(),
$frozenNetwork->fingerprint()
)) {
throw new CloudError(
'实例公网网络快照与活动地址声明不一致',
409
);
}
self::assertNoActiveJob($lockedInstance);
if (CloudBackupRecovery::where(
'instance_id', (int) $lockedInstance->id
)->whereIn('state', ['recovering', 'uncertain'])->exists()) {
throw new CloudError('实例已有恢复任务或未确认效果', 409);
}
$operationId = bin2hex(random_bytes(16));
$recoveryKey = bin2hex(random_bytes(16));
$now = time();
$recovery = new CloudBackupRecovery();
$recovery->recovery_key = $recoveryKey;
$recovery->backup_id = (int) $lockedBackup->id;
$recovery->instance_id = (int) $lockedInstance->id;
$recovery->job_id = null;
$recovery->operation_id = $operationId;
$recovery->state = 'recovering';
$recovery->error_code = '';
$recovery->completed_at = null;
$recovery->created_at = $recovery->updated_at = $now;
$recovery->save();
$body = [
'backup_key' => (string) $lockedBackup->backup_key,
'source_url' => $sourceUrl,
'source_size_bytes' => (int) $lockedBackup->object_size_bytes,
'source_sha256' => (string) $lockedBackup->sha256,
];
$jobActor = $actor;
$jobActor['can_manage'] = true;
[$job, $created] = CloudJobService::
reserveStandaloneAgentMutation(
$jobActor,
(int) $lockedInstance->host_id,
'backup_recover',
'POST',
$target,
$body,
[
'owner_id' => (int) $lockedInstance->user_id,
'can_manage' => true,
'backup_key' => (string) $lockedBackup->backup_key,
'recovery_key' => $recoveryKey,
],
$canonical,
$operationId
);
if (!$created) {
throw new CloudError('云备份恢复幂等任务状态冲突', 409);
}
$job->instance_id = (int) $lockedInstance->id;
$job->backup_id = (int) $lockedBackup->id;
$job->backup_recovery_id = (int) $recovery->id;
$job->save();
$recovery->job_id = (int) $job->id;
$recovery->updated_at = time();
$recovery->save();
return [
'recovery' => self::displayRecovery($recovery),
'job' => $job,
];
}, 3);
}
/** Apply a terminal/running Agent result inside CloudWorkerService's lock. */
public static function finishJobLocked(CloudJob $job, string $status,
$result, array $payload, bool $deferPublish = false): void
{
if ($job->kind === 'backup_create') {
self::finishCreateLocked($job, $status, $result, $payload);
} elseif ($job->kind === 'backup_restore') {
CloudPortableRestoreFinalizer::finishJobLocked(
$job, $status, $result, $payload, $deferPublish
);
} elseif ($job->kind === 'backup_recover') {
self::finishRecoveryLocked($job, $status, $result, $payload);
} elseif ($job->kind === 'backup_workspace_cleanup') {
self::finishWorkspaceCleanupLocked(
$job, $status, $result, $payload
);
}
}
/** Queue one evidence-bound cleanup of a source-host backup workspace. */
public static function cleanupWorkspace(
CloudInstanceBackup $backup,
array $actor
): CloudJob {
CloudAccess::manage($actor);
if (($actor['is_super_admin'] ?? false) !== true) {
throw new CloudError('只有超级管理员可以清理备份工作区', 403);
}
$original = CloudJob::where(
'operation_id', (string) $backup->last_operation_id
)->first();
if ($original === null || (string) $original->kind !== 'backup_create'
|| (string) $original->phase !== 'terminal'
|| (string) $original->outcome !== 'succeeded'
|| (int) $original->backup_id !== (int) $backup->id
|| (int) $original->host_id !== (int) $backup->source_host_id) {
throw new CloudError('原备份任务身份不完整,不能清理工作区', 409);
}
$inspection = CloudAdminOperationRecoveryService::inspect(
$actor, (string) $original->id
);
$cleanup = is_array($inspection['cleanup'] ?? null)
? $inspection['cleanup'] : [];
$planId = ($cleanup['available'] ?? false) === true
&& is_string($cleanup['plan_id'] ?? null)
? strtolower((string) $cleanup['plan_id']) : '';
if ($planId !== ''
&& preg_match('/^[a-f0-9]{64}$/D', $planId) !== 1) {
throw new CloudError('Agent 返回的清理计划身份无效', 502);
}
return $backup->getConnection()->transaction(function () use (
$backup, $actor, $original, $planId
): CloudJob {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->first();
if ($locked === null || (string) $locked->state !== 'ready'
|| (string) $locked->error_code
!== 'backup_local_cleanup_pending'
|| !hash_equals(
(string) $locked->last_operation_id,
(string) $original->operation_id
)) {
throw new CloudError('备份工作区状态已变化,请刷新后重试', 409);
}
$operationId = bin2hex(random_bytes(16));
$target = '/v1/operations/' . rawurlencode($operationId)
. '/cleanup-backup-workspace';
$body = [
'prior_operation_id' => (string) $original->operation_id,
'cleanup_plan_id' => $planId,
];
$canonical = [
'backup_id' => (int) $locked->id,
'backup_key' => (string) $locked->backup_key,
'prior_operation_id' => (string) $original->operation_id,
'cleanup_plan_id' => $planId,
];
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
$actor,
(int) $locked->source_host_id,
'backup_workspace_cleanup',
'POST',
$target,
$body,
[
'owner_id' => (int) $locked->user_id,
'can_manage' => true,
'backup_key' => (string) $locked->backup_key,
'prior_job_id' => (int) $original->id,
'prior_operation_id' => (string) $original->operation_id,
'cleanup_plan_id' => $planId,
],
$canonical,
$operationId,
'/v1/backups/' . (int) $locked->id . '/cleanup-workspace'
);
if ($created) {
$job->instance_id = (int) $locked->source_instance_id;
$job->backup_id = (int) $locked->id;
$job->save();
}
return $job;
}, 3);
}
private static function finishWorkspaceCleanupLocked(
CloudJob $job,
string $status,
$result,
array $payload
): void {
$backup = $job->backup_id === null ? null
: CloudInstanceBackup::where('id', (int) $job->backup_id)
->lockForUpdate()->first();
if ($backup === null
|| (int) $backup->source_host_id !== (int) $job->host_id
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|| !is_string($payload['prior_operation_id'] ?? null)
|| !hash_equals(
(string) $backup->last_operation_id,
(string) $payload['prior_operation_id']
)
|| !hash_equals(
(string) $backup->backup_key,
(string) ($payload['backup_key'] ?? '')
)) {
throw new CloudRecoveryInstanceStateConflict(
'备份工作区清理任务与目录账本不一致', 409
);
}
if ($status !== 'succeeded') {
return;
}
if (!is_array($result)
|| !hash_equals(
(string) $backup->backup_key,
(string) ($result['backup_key'] ?? '')
)
|| ($result['cleaned'] ?? null) !== true
|| ($result['workspace_absent'] ?? null) !== true
|| ($result['metadata_absent'] ?? null) !== true) {
throw new CloudError('Agent 清理成功结果缺少完整缺席证据', 502);
}
if ((string) $backup->error_code
=== 'backup_local_cleanup_pending') {
$backup->error_code = '';
$backup->updated_at = time();
$backup->save();
}
}
public static function markJobUncertain(CloudJob $job): void
{
if ($job->kind === 'backup_create' && $job->backup_id !== null) {
CloudInstanceBackup::where('id', (int) $job->backup_id)
->where('last_operation_id', (string) $job->operation_id)
->update(['state' => 'uncertain', 'error_code' =>
'finalization_incident', 'updated_at' => time()]);
} elseif ($job->kind === 'backup_restore') {
CloudPortableRestoreFinalizer::markJobUncertain($job);
} elseif ($job->kind === 'backup_recover'
&& $job->backup_recovery_id !== null) {
CloudBackupRecovery::where(
'id', (int) $job->backup_recovery_id
)->where('operation_id', (string) $job->operation_id)
->update(['state' => 'uncertain', 'error_code' =>
'finalization_incident', 'updated_at' => time()]);
}
CloudInstanceMigrationService::markJobUncertain($job);
}
private static function finishCreateLocked(CloudJob $job, string $status,
$result, array $payload): void
{
$backup = $job->backup_id === null ? null
: CloudInstanceBackup::where('id', (int) $job->backup_id)
->lockForUpdate()->first();
if ($backup === null
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|| (int) $backup->source_host_id !== (int) $job->host_id
|| (int) $backup->user_id !== (int) ($payload['owner_id'] ?? 0)
|| !hash_equals((string) $backup->backup_key,
(string) ($payload['backup_key'] ?? ''))
|| !hash_equals((string) $backup->last_operation_id,
(string) $job->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'备份任务与面板目录账本不一致', 409
);
}
if ($status === 'running') {
return;
}
$artifactId = filter_var(
$backup->storage_artifact_id,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$payloadArtifactId = filter_var(
$payload['storage_artifact_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$storageOperationId =
$payload['storage_publication_operation_id'] ?? null;
if ($artifactId === false
|| $payloadArtifactId === false
|| (int) $payloadArtifactId !== (int) $artifactId
|| !is_string($storageOperationId)
|| preg_match('/^[a-f0-9]{32}$/D', $storageOperationId) !== 1) {
throw new CloudRecoveryInstanceStateConflict(
'备份任务与复制对象存储账本不一致',
409
);
}
$now = time();
if ($status === 'uncertain') {
CloudStorageArtifactService::failPrimary(
$storageOperationId,
(int) $artifactId,
'unknown',
'backup_operation_effect_unknown'
);
$backup->state = 'uncertain';
$backup->error_code = 'operation_effect_unknown';
} elseif ($status === 'failed') {
CloudStorageArtifactService::failPrimary(
$storageOperationId,
(int) $artifactId,
'not_applied',
'backup_agent_failed'
);
$backup->state = 'failed';
$backup->error_code = 'backup_failed';
$backup->active_job_id = null;
} elseif ($status === 'succeeded') {
try {
$sourceCompatibility =
CloudPortableRestoreCompatibility::fromPayload(
is_array($result)
? ($result['source_contract'] ?? null) : null
);
} catch (\InvalidArgumentException $error) {
throw new CloudError(
'Agent 成功结果缺少可信可移植恢复合同', 502
);
}
$size = filter_var($result['size_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$virtual = filter_var($result['virtual_size_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$sha = $result['sha256'] ?? null;
$objectEtag = $result['object_etag'] ?? '';
$objectVersionId = $result['object_version_id'] ?? '';
$consistency = $result['consistency'] ?? null;
$cleanupPending = $result['cleanup_pending'] ?? false;
$expectedContractSHA =
$payload['source_portable_restore_contract_sha256'] ?? null;
if (!is_array($result)
|| !hash_equals((string) $backup->backup_key,
(string) ($result['backup_key'] ?? ''))
|| $size === false || $virtual === false
|| !is_string($sha)
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|| !is_string($objectEtag) || strlen($objectEtag) > 256
|| preg_match('/[\x00-\x1f\x7f]/D', $objectEtag) === 1
|| !is_string($objectVersionId)
|| strlen($objectVersionId) > 1024
|| preg_match('/[\x00-\x1f\x7f]/D', $objectVersionId) === 1
|| ($objectVersionId !== '' && $objectEtag === '')
|| !in_array($consistency, ['filesystem', 'crash'], true)
|| !is_bool($cleanupPending)
|| !is_string($expectedContractSHA)
|| preg_match('/^[a-f0-9]{64}$/D',
$expectedContractSHA) !== 1
|| !hash_equals(
$expectedContractSHA,
$sourceCompatibility->sha256()
)
|| (int) $size > (int) ($payload['maximum_size_bytes'] ?? 0)) {
throw new CloudError('Agent 成功结果缺少可信备份对象身份', 502);
}
$publication = CloudStorageArtifactService::completePrimary(
$storageOperationId,
(int) $artifactId,
[
'size_bytes' => (int) $size,
'sha256' => $sha,
'etag' => $objectEtag,
'version_id' => $objectVersionId,
]
);
if ((int) ($publication['artifact_id'] ?? 0)
!== (int) $artifactId
|| ($publication['outcome'] ?? null) !== 'succeeded') {
throw new CloudRecoveryInstanceStateConflict(
'对象存储主副本结算回执与备份目录不一致',
409
);
}
$backup->state = 'ready';
$backup->consistency = $consistency;
$backup->object_size_bytes = (int) $size;
$backup->virtual_size_bytes = (int) $virtual;
$backup->sha256 = $sha;
$backup->object_etag = $objectEtag;
$backup->object_version_id = $objectVersionId;
$backup->source_portable_restore_contract_json =
$sourceCompatibility->json();
$backup->source_portable_restore_contract_sha256 =
$sourceCompatibility->sha256();
$backup->ready_at = $now;
$backup->active_job_id = null;
// The remote object is complete and restorable. A host-side
// snapshot cleanup warning must not discard this sole immutable
// object identity or trigger another upload.
$backup->error_code = $cleanupPending
? 'backup_local_cleanup_pending' : '';
}
$backup->updated_at = $now;
$backup->save();
if ($status === 'succeeded') {
$finalized = CloudStoragePublicationFinalizerService::complete(
'backup',
(int) $backup->id,
(int) $artifactId,
$storageOperationId
);
if (($finalized['state'] ?? null) !== 'completed') {
throw new CloudRecoveryInstanceStateConflict(
'对象存储发布账本未完成最终化',
409
);
}
}
}
private static function finishRecoveryLocked(CloudJob $job,
string $status, $result, array $payload): void
{
$recovery = $job->backup_recovery_id === null ? null
: CloudBackupRecovery::where(
'id', (int) $job->backup_recovery_id
)->lockForUpdate()->first();
$backup = $job->backup_id === null ? null
: CloudInstanceBackup::where('id', (int) $job->backup_id)
->lockForUpdate()->first();
$instance = $job->instance_id === null ? null
: CloudInstance::where('id', (int) $job->instance_id)
->lockForUpdate()->first();
$body = is_array($payload['body'] ?? null)
? $payload['body'] : [];
if ($recovery === null || $backup === null || $instance === null
|| (int) $recovery->job_id !== (int) $job->id
|| (int) $recovery->backup_id !== (int) $backup->id
|| (int) $recovery->instance_id !== (int) $instance->id
|| !hash_equals((string) $recovery->operation_id,
(string) $job->operation_id)
|| !hash_equals((string) $recovery->recovery_key,
(string) ($payload['recovery_key'] ?? ''))
|| !hash_equals((string) $backup->backup_key,
(string) ($payload['backup_key'] ?? ''))
|| !hash_equals((string) $backup->backup_key,
(string) ($body['backup_key'] ?? ''))
|| !hash_equals((string) $backup->sha256,
(string) ($body['source_sha256'] ?? ''))
|| (int) $backup->object_size_bytes
!== (int) ($body['source_size_bytes'] ?? 0)
|| (int) $instance->user_id
!== (int) ($payload['owner_id'] ?? 0)
|| (int) $job->user_id !== (int) $instance->user_id
|| (int) $job->host_id !== (int) $instance->host_id) {
throw new CloudRecoveryInstanceStateConflict(
'云备份恢复任务与实例、备份或恢复账本不一致',
409
);
}
if ($status === 'running') {
return;
}
$now = time();
if ($status === 'uncertain') {
$recovery->state = 'uncertain';
$recovery->error_code = 'operation_effect_unknown';
$recovery->completed_at = null;
} elseif ($status === 'failed') {
$recovery->state = 'failed';
$recovery->error_code = 'backup_recovery_failed';
$recovery->completed_at = $now;
} elseif ($status === 'succeeded') {
if (!is_array($result)
|| ($result['restored'] ?? null) !== true
|| !hash_equals((string) $backup->backup_key,
(string) ($result['backup_key'] ?? ''))
|| !hash_equals((string) $backup->sha256,
(string) ($result['sha256'] ?? ''))
|| CloudInstanceRuntimeService::normalizeState(
$result['power_state'] ?? null
) === 'unknown') {
throw new CloudError(
'Agent 恢复成功结果缺少可信的备份或电源身份',
502
);
}
$backupSpec = self::decodeSpec((string) $backup->spec_json);
if (($backupSpec['managed'] ?? null) !== true
|| ($backupSpec['os_source'] ?? null) !== 'cloud_template'
|| !is_string($backupSpec['os_name'] ?? null)
|| trim((string) $backupSpec['os_name']) === ''
|| ($backupSpec['login_name'] ?? null) !== 'root'
|| !is_bool($backupSpec['debian_12_network_fix'] ?? null)) {
throw new CloudRecoveryInstanceStateConflict(
'云备份的客户机身份快照无效',
409
);
}
$instanceSpec = self::decodeSpec((string) $instance->spec_json);
foreach ([
'template_id', 'template_revision',
'template_generation_token', 'os_source', 'os_name',
'login_name', 'debian_12_network_fix',
] as $field) {
if (array_key_exists($field, $backupSpec)) {
$instanceSpec[$field] = $backupSpec[$field];
} else {
unset($instanceSpec[$field]);
}
}
unset($instanceSpec['install_iso']);
// Passwords and hostnames may have been changed inside the guest
// without passing through the panel. The restored system disk is
// authoritative, so the panel must not claim either value is
// known or cause the Agent's NoCloud seed to inject an old value.
unset($instanceSpec['guest_hostname'], $instanceSpec['hostname']);
$instanceSpec['managed'] = true;
$instanceSpec['restored_from_backup'] =
(string) $backup->backup_key;
$instance->spec_json = json_encode(
CloudInput::safeResult($instanceSpec),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
$instance->sealed_credential = null;
$instance->updated_at = $now;
$instance->save();
$recovery->state = 'succeeded';
$recovery->error_code = '';
$recovery->completed_at = $now;
self::recordRuntime($instance, $result);
}
$recovery->updated_at = $now;
$recovery->save();
}
private static function priorRecoveryResult(CloudJob $job): array
{
$recovery = $job->backup_recovery_id === null ? null
: CloudBackupRecovery::find((int) $job->backup_recovery_id);
if ($recovery === null) {
throw new CloudError('云备份恢复任务目录记录缺失', 409);
}
return [
'recovery' => self::displayRecovery($recovery),
'job' => $job,
];
}
private static function displayRecovery(CloudBackupRecovery $recovery): array
{
return [
'recovery_key' => (string) $recovery->recovery_key,
'backup_id' => (string) $recovery->backup_id,
'instance_id' => (string) $recovery->instance_id,
'state' => (string) $recovery->state,
'created_at' => max(0, (int) $recovery->created_at),
'completed_at' => $recovery->completed_at === null ? null
: max(0, (int) $recovery->completed_at),
];
}
private static function assertManagedSource(CloudInstance $instance): void
{
$spec = self::decodeSpec((string) $instance->spec_json);
if (($spec['managed'] ?? false) !== true
|| ($spec['os_source'] ?? null) !== 'cloud_template') {
throw new CloudError('仅面板创建的云镜像实例支持可移植备份', 409);
}
self::resourcesFromSpec($spec);
}
private static function assertNoActiveJob(CloudInstance $instance): void
{
CloudInstanceMigrationService::assertInstanceMutationAvailable(
$instance
);
if (CloudJobService::blockingQueryForInstance(
(int) $instance->id
)->exists()) {
throw new CloudError('虚拟机已有任务,请等待完成', 409);
}
}
private static function assertBackupDeletionAvailable(
CloudInstanceBackup $backup
): void {
if ((string) $backup->purpose !== 'portable') {
throw new CloudError('该备份类型不能由此入口删除', 409);
}
if ((string) $backup->state === 'uncertain') {
throw new CloudError('备份效果尚未确认,不能删除', 409);
}
if ($backup->active_job_id !== null) {
throw new CloudError('备份仍有关联任务,不能删除', 409);
}
if (CloudBackupRestore::where('backup_id', (int) $backup->id)
->whereIn('state', ['restoring', 'uncertain'])->exists()) {
throw new CloudError('备份仍有恢复任务或未知效果,不能删除', 409);
}
self::assertPortableRestoreAttemptsDeletable($backup);
if (CloudBackupRecovery::where('backup_id', (int) $backup->id)
->whereIn('state', ['recovering', 'uncertain'])->exists()) {
throw new CloudError('备份仍有原实例恢复任务或未知效果,不能删除', 409);
}
if (CloudJob::where('backup_id', (int) $backup->id)
->where(static function ($query): void {
$query->where('phase', '<>', 'terminal')
->orWhere('effect_resolution', 'unknown')
->orWhere('finalization_incident', 1);
})->exists()) {
throw new CloudError('备份仍有关联任务,不能删除', 409);
}
if (CloudDeploymentSnapshot::where('backup_id', (int) $backup->id)
->where('state', 'ready')->exists()) {
throw new CloudError('该对象已保存为云快照,请从云快照页面删除', 409);
}
}
/** Resolve one immutable backup read capability from the replica ledger. */
private static function backupReadUrl(CloudInstanceBackup $backup): string
{
$artifactId = filter_var(
$backup->storage_artifact_id,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
if ($artifactId === false) {
throw new CloudError('云备份缺少复制副本账本,不能读取', 409);
}
$capability = (new CloudStorageArtifactService())->presignedRead(
(int) $artifactId,
self::TRANSFER_URL_SECONDS
);
if ((int) ($capability['size_bytes'] ?? 0)
!== (int) $backup->object_size_bytes) {
throw new CloudError('云备份副本与备份目录账本不一致,不能读取', 409);
}
if (!is_string($capability['sha256'] ?? null)
|| !hash_equals(
(string) $backup->sha256,
(string) $capability['sha256']
)) {
throw new CloudError('云备份副本与备份目录账本不一致,不能读取', 409);
}
$url = $capability['url'] ?? null;
if (!is_string($url) || $url === '') {
throw new CloudError('云备份副本未返回有效读取凭据', 502);
}
return $url;
}
private static function assertPortableRestoreAttemptsDeletable(
CloudInstanceBackup $backup
): void {
$restoreIds = CloudBackupRestore::where(
'backup_id',
(int) $backup->id
)
->pluck('id')->map(static fn ($id): int => (int) $id)->all();
if ($restoreIds === []) {
return;
}
$attempts = CloudBackupRestoreAttempt::whereIn(
'backup_restore_id',
$restoreIds
)->orderBy('id')->get();
foreach ($attempts as $attempt) {
$state = (string) $attempt->state;
$resolution = (string) $attempt->effect_resolution;
if ($attempt->active_attempt !== null
|| in_array($state, [
'reserved', 'queued', 'awaiting_receipt',
'effect_unknown', 'finalization_pending',
], true)) {
throw new CloudError(
'备份仍有恢复尝试或未知效果,不能删除',
409
);
}
if ($state === 'succeeded' && $resolution === 'applied') {
continue;
}
if ($state !== 'failed' || $resolution !== 'not_applied'
|| $attempt->effect_resolved_at === null
|| $attempt->terminal_at === null) {
throw new CloudError('备份恢复尝试终态不明确,不能删除', 409);
}
$reservation = CloudResourceReservation::where(
'id',
(int) $attempt->reservation_id
)->where(
'backup_restore_attempt_key',
(string) $attempt->attempt_key
)->first();
if (!$reservation instanceof CloudResourceReservation
|| (string) $reservation->state !== 'released'
|| (int) $reservation->host_id
!== (int) $attempt->target_host_id
|| !hash_equals(
(string) $reservation->operation_id,
(string) $attempt->operation_id
)
|| $reservation->released_at === null) {
throw new CloudError('备份恢复尝试仍占用资源,不能删除', 409);
}
CloudReleasedPublicNetworkBundleVerifier::verify(
$attempt,
$reservation,
(int) $backup->user_id
);
}
}
/** @param array<string,mixed> $result */
private static function settleArtifactDeletion(
int $backupId,
array $result
): array {
if (($result['deleted'] ?? false) === true) {
return self::finishDelete($backupId);
}
$backup = CloudInstanceBackup::findOrFail($backupId);
$locked = $backup->getConnection()->transaction(function () use (
$backupId,
$result
): CloudInstanceBackup {
$row = CloudInstanceBackup::where('id', $backupId)
->lockForUpdate()->firstOrFail();
if ((string) $row->state === 'deleted') {
return $row;
}
self::assertBackupDeletionAvailable($row);
$attention = ($result['requires_attention'] ?? false) === true;
$row->state = $attention
? 'delete_attention' : 'delete_uncertain';
$nextCheck = (int) ($result['next_check_at'] ?? 0);
$row->delete_next_check_at = $attention ? 0 : max(
time() + 60,
$nextCheck
);
$row->delete_attention_at = $attention ? time() : null;
$row->error_code = $attention
? 'artifact_delete_attention'
: 'artifact_delete_reconciliation';
$row->updated_at = time();
$row->save();
return $row;
}, 3);
return self::deletionResult($locked);
}
private static function finishDelete(int $backupId): array
{
$backup = CloudInstanceBackup::findOrFail($backupId);
return $backup->getConnection()->transaction(function () use (
$backupId
): array {
$locked = CloudInstanceBackup::where('id', $backupId)
->lockForUpdate()->firstOrFail();
if ((string) $locked->state !== 'deleted') {
self::assertBackupDeletionAvailable($locked);
if (!in_array((string) $locked->state,
['ready', 'failed', 'deleting', 'delete_uncertain',
'delete_attention'], true)) {
throw new CloudError('备份删除状态已发生变化', 409);
}
$locked->state = 'deleted';
$locked->sealed_credential = null;
$locked->active_job_id = null;
$locked->delete_presence_confirmations = 0;
$locked->delete_next_check_at = 0;
$locked->delete_attention_at = null;
$locked->error_code = '';
$locked->updated_at = time();
$locked->save();
CloudDeploymentSnapshot::where('backup_id', $backupId)
->whereIn('state', ['deleting', 'delete_uncertain'])
->update([
'state' => 'deleted',
'deleted_at' => time(),
'error_code' => '',
'updated_at' => time(),
]);
}
return self::deletionResult($locked);
}, 3);
}
private static function deletionResult(CloudInstanceBackup $backup): array
{
$state = (string) $backup->state;
return [
'backup_id' => (string) $backup->id,
'state' => $state,
'deleted' => $state === 'deleted',
'requires_reconciliation' => in_array($state,
['deleting', 'delete_uncertain'], true),
'requires_attention' => $state === 'delete_attention',
'next_check_at' => max(0, (int) $backup->delete_next_check_at),
];
}
private static function createRequestKey(array $actor): string
{
$actorId = filter_var(
$actor['user_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$key = $actor['request_idempotency_key'] ?? null;
if ($actorId === false || !is_string($key)
|| preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) {
throw new CloudError('备份创建请求缺少有效幂等标识', 400);
}
return $key;
}
private static function findCreateIntent(
int $actorId,
string $requestKey,
string $fingerprint,
int $instanceId,
bool $lock
): ?CloudInstanceBackup {
$query = CloudInstanceBackup::query()
->where('create_request_actor_id', $actorId)
->where('create_request_idempotency_key', $requestKey);
if ($lock) {
$query->lockForUpdate();
}
$backup = $query->first();
if (!$backup instanceof CloudInstanceBackup) {
return null;
}
if (!is_string($backup->create_request_fingerprint)
|| !hash_equals(
(string) $backup->create_request_fingerprint,
$fingerprint
)
|| (int) $backup->source_instance_id !== $instanceId
|| (string) $backup->purpose !== 'portable') {
throw new CloudError(
'同一请求标识不能用于不同的备份创建操作',
409
);
}
return $backup;
}
/** @return array{backup:array<string,mixed>,job:CloudJob} */
private static function finishCreatePreparation(
CloudInstanceBackup $backup
): array {
$job = (new CloudBackupPublicationPreparation())
->ensureAgentJob((int) $backup->id);
if (!$job instanceof CloudJob) {
throw new CloudError(
'备份上传正在核对对象存储状态,请稍后使用同一请求标识重试',
503
);
}
$backup->refresh();
return [
'backup' => self::displayBackup($backup, $job),
'job' => $job,
];
}
private static function priorJob(array $actor, string $kind,
?int $instanceId, ?int $backupId, string $fingerprint,
bool $lock = true): ?CloudJob
{
$key = $actor['request_idempotency_key'] ?? null;
if (!is_string($key) || $key === '') {
return null;
}
$query = CloudJob::where('user_id', (int) $actor['user_id'])
->where('request_idempotency_key', $key);
if ($lock) {
$query->lockForUpdate();
}
$job = $query->first();
if ($job === null) {
return null;
}
if ((string) $job->kind !== $kind
|| !hash_equals((string) $job->request_fingerprint,
$fingerprint)
|| ($instanceId !== null
&& (int) $job->instance_id !== $instanceId)
|| ($backupId !== null && (int) $job->backup_id !== $backupId)) {
throw new CloudError('同一请求标识不能用于不同的备份操作', 409);
}
return $job;
}
private static function resourcesFromSpec(array $spec): array
{
$cpu = filter_var($spec['vcpu'] ?? $spec['cpu_cores'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$memory = filter_var($spec['memory_mib'] ?? $spec['ram_mib'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$disk = filter_var($spec['disk_gib'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$bandwidth = filter_var($spec['bandwidth_mbps'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
if ($cpu === false || $memory === false || $disk === false
|| $bandwidth === false) {
throw new CloudError('实例缺少可信的资源规格快照', 409);
}
return ['cpu_cores' => (int) $cpu, 'memory_mib' => (int) $memory,
'disk_gib' => (int) $disk, 'bandwidth_mbps' => (int) $bandwidth];
}
/** @return array<string,mixed> */
private static function restoreTargetProjection(
CloudInstanceBackup $backup
): array
{
try {
$targets = array_map(
static fn (CloudPortableRestoreTargetPlan $target): array =>
self::restoreTargetDisplay($target),
CloudPortableRestoreTargetService::candidates($backup)
);
return ['targets' => $targets, 'reason' => null];
} catch (CloudError $error) {
return [
'targets' => [],
'reason' => $error->getMessage(),
];
}
}
/** @return array<string,mixed> */
private static function restoreTargetDisplay(
CloudPortableRestoreTargetPlan $plan
): array
{
$value = $plan->toArray();
$target = $value['target'];
$network = $value['public_network'];
$ipv4Available = max(
0,
(int) $network['ipv4_pool']['available_units']
);
$ipv6Available = max(
0,
(int) $network['ipv6_pool']['available_units']
);
return [
'host_id' => (string) $target['host_id'],
'host_name' => (string) $target['host_name'],
'region_id' => (string) $target['region_id'],
'region' => (string) $target['region_name'],
'country_code' => (string) $target['country_code'],
'line_id' => (string) $target['line_id'],
'line' => (string) $target['line_name'],
'resource_pool_id' => (string) $target['resource_pool_id'],
'resource_pool' => (string) $target['resource_pool_name'],
'network_attachment_id' =>
(string) $network['network_attachment_id'],
'network_selector' => (string) $network['network_selector'],
'network_selectors' => [(string) $network['network_selector']],
'available_ip_count' => min($ipv4Available, $ipv6Available),
'available_ipv4_count' => $ipv4Available,
'available_ipv6_count' => $ipv6Available,
'target_plan_sha256' => $plan->sha256(),
'is_source_host' => (bool) $target['is_source_host'],
];
}
/** Freeze template-specific boot compatibility into the backup manifest. */
private static function freezeRestoreCompatibility(array $spec): array
{
if (!is_bool($spec['debian_12_network_fix'] ?? null)) {
throw new CloudError(
'实例缺少创建时冻结的网络兼容快照,不能生成可移植备份',
409
);
}
return $spec;
}
private static function decodeSpec(string $encoded,
bool $throw = true): ?array
{
$spec = json_decode($encoded, true);
if (!is_array($spec) || array_is_list($spec)) {
if ($throw) {
throw new CloudError('实例备份规格快照损坏', 409);
}
return null;
}
return $spec;
}
private static function maximumObjectBytes(int $diskGib): int
{
if ($diskGib < 1 || $diskGib > 2559) {
throw new CloudError('实例系统盘超出当前备份最大容量', 409);
}
return ($diskGib + 1) * self::GIB;
}
private static function displayBackup(CloudInstanceBackup $backup,
?CloudJob $job, bool $canManage = true): array
{
$state = in_array((string) $backup->state,
['creating', 'ready', 'failed', 'uncertain', 'deleting',
'delete_uncertain', 'delete_attention', 'deleted'], true)
? (string) $backup->state : 'uncertain';
$cancelableCreating = $state === 'creating'
&& $backup->storage_artifact_id !== null
&& $backup->active_job_id === null
&& $backup->last_operation_id === null;
$canDelete = $canManage && in_array($state,
['ready', 'failed', 'deleting', 'delete_uncertain'], true);
$canDelete = $canDelete || ($canManage && $cancelableCreating);
if ($canDelete) {
try {
self::assertBackupDeletionAvailable($backup);
} catch (CloudError $error) {
$canDelete = false;
}
}
return [
'id' => (string) $backup->id,
'backup_key' => (string) $backup->backup_key,
'name' => (string) $backup->name,
'state' => $state,
'consistency' => in_array((string) $backup->consistency,
['filesystem', 'crash'], true)
? (string) $backup->consistency : 'unknown',
'object_size_bytes' => max(0, (int) $backup->object_size_bytes),
'virtual_size_bytes' => max(0, (int) $backup->virtual_size_bytes),
'sha256' => preg_match('/^[a-f0-9]{64}$/D',
(string) $backup->sha256) === 1
? (string) $backup->sha256 : '',
'source_host_id' => (string) $backup->source_host_id,
'created_at' => max(0, (int) $backup->created_at),
'ready_at' => $backup->ready_at === null ? null
: max(0, (int) $backup->ready_at),
'updated_at' => max(0, (int) $backup->updated_at),
'task_id' => $job === null ? null : (string) $job->id,
'job_status' => $job === null ? null
: (CloudOperationErrorCatalog::userStatus(
(string) $job->status
) ?? 'unknown'),
'cleanup_pending' => (string) $backup->error_code
=== 'backup_local_cleanup_pending',
'can_cleanup_workspace' => $canManage
&& (string) $backup->error_code
=== 'backup_local_cleanup_pending',
'can_delete' => $canDelete,
'delete_action' => in_array($state,
['deleting', 'delete_uncertain'], true)
? 'reconcile' : 'delete',
'error' => $state === 'delete_attention'
? '对象自动清理已达到三次上限,需要管理员检查对象存储后手动恢复清理。'
: ($state === 'delete_uncertain'
? '删除结果尚未确认;系统会在两次稳定核对后创建新的删除代次。'
: null),
];
}
private static function displayRestore(CloudBackupRestore $restore): array
{
return [
'restore_key' => (string) $restore->restore_key,
'state' => (string) $restore->state,
'target_host_id' => (string) $restore->target_host_id,
'target_instance_id' => $restore->target_instance_id === null
? null : (string) $restore->target_instance_id,
'name' => (string) $restore->new_name,
'created_at' => max(0, (int) $restore->created_at),
'ready_at' => $restore->ready_at === null ? null
: max(0, (int) $restore->ready_at),
];
}
private static function libraryFilters(array $input): array
{
CloudInput::only($input, ['page', 'per_page', 'q', 'user_id',
'instance_id', 'host_id', 'state']);
foreach ($input as $value) {
if (!is_string($value) && !is_int($value)) {
throw new CloudError('备份库筛选参数无效', 400);
}
}
$filters = [];
foreach (['user_id', 'instance_id', 'host_id'] as $key) {
$value = (string) ($input[$key] ?? '');
if ($value !== '' && (preg_match('/^[1-9][0-9]{0,17}$/D', $value) !== 1)) {
throw new CloudError('备份库 ' . $key . ' 必须为有效的正整数', 400);
}
$filters[$key] = $value;
}
$state = (string) ($input['state'] ?? '');
if (!in_array($state, ['', 'creating', 'ready', 'restoring', 'failed',
'uncertain', 'deleting', 'delete_uncertain', 'delete_attention'], true)) {
throw new CloudError('备份库状态筛选无效', 400);
}
$q = trim((string) ($input['q'] ?? ''));
if (preg_match('//u', $q) !== 1 || mb_strlen($q, 'UTF-8') > 80
|| preg_match('/[\x00-\x1f\x7f]/u', $q) !== 0) {
throw new CloudError('备份名称关键词最多 80 个字符,且不能包含控制字符', 400);
}
return $filters + ['state' => $state, 'q' => $q];
}
private static function libraryInteger($value, int $minimum, int $maximum,
int $default): int
{
if ($value === null || $value === '') {
return $default;
}
if (is_int($value)) {
$parsed = $value;
} elseif (is_string($value)
&& preg_match('/^(?:0|[1-9][0-9]{0,9})$/D', $value) === 1) {
$parsed = (int) $value;
} else {
throw new CloudError('备份库分页参数无效', 400);
}
if ($parsed < $minimum || $parsed > $maximum) {
throw new CloudError('备份库分页参数超出允许范围', 400);
}
return $parsed;
}
private static function recordRuntime(CloudInstance $instance,
array $result): void
{
try {
CloudInstanceRuntimeService::seed(
(int) $instance->id, (int) $instance->host_id,
(string) $instance->uuid
);
$state = $result['status']
?? ($result['state'] ?? ($result['power_state'] ?? null));
if (CloudInstanceRuntimeService::normalizeState($state)
!== 'unknown') {
CloudInstanceRuntimeService::markObserved(
(int) $instance->id, (int) $instance->host_id,
(string) $instance->uuid, $state
);
}
} catch (\Throwable $error) {
error_log('cloud backup restore runtime cache unavailable: '
. get_class($error));
}
}
}