You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudBackupService.php

2416 lines
106 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudBackupRestore;
use App\Models\CloudBackupRecovery;
use App\Models\CloudBackupPeriodEntitlement;
use App\Models\CloudBackupSubscription;
use App\Models\CloudDeploymentSnapshot;
use App\Models\CloudHost;
use App\Models\CloudInstance;
use App\Models\CloudInstanceBackup;
use App\Models\CloudInstanceMigration;
use App\Models\CloudJob;
/** PHP-owned catalog and resource ledger for portable system-disk backups. */
final class CloudBackupService
{
public const MAX_PER_INSTANCE = 10;
private const STORAGE_PURPOSE = 'object-storage-backup-runtime';
private const ACTIVE_BACKUP_STATES = [
'creating', 'ready', 'uncertain',
];
private const CLEANUP_DEBT_STATES = [
'deleting', 'delete_uncertain', 'delete_attention',
];
private const DELETE_STABLE_PROBES = 2;
private const DELETE_STABLE_SECONDS = 60;
private const DELETE_RETRY_SECONDS = 300;
private const DELETE_MAX_AUTOMATIC_ATTEMPTS = 3;
private const TRANSFER_URL_SECONDS = 604800;
private const MAX_AGENT_BODY_BYTES = 1048576;
// The create transaction opens a remote S3 multipart upload. Database
// deadlock handling must never replay that external side effect.
private const CREATE_TRANSACTION_ATTEMPTS = 1;
private const GIB = 1073741824;
/**
* Global administrator backup catalog. It is intentionally keyed by the
* backup ledger instead of the source VM lifecycle, so a retained backup
* remains restorable and deletable after expiry cleanup tombstones the
* source instance.
*/
public static function library(array $actor, array $input): array
{
if (($actor['is_admin'] ?? false) !== true) {
throw new CloudError('只有管理员可以查看虚拟机备份', 403);
}
$filters = self::libraryFilters($input);
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
$perPage = self::libraryInteger($input['per_page'] ?? 20,
20, 50, 20);
if (!in_array($perPage, [20, 50], true)) {
throw new CloudError('备份库每页条数仅支持 20 或 50', 400);
}
$query = CloudInstanceBackup::where('purpose', 'portable')
->where('state', '<>', 'deleted');
foreach (['user_id' => 'user_id', 'instance_id' => 'source_instance_id',
'host_id' => 'source_host_id', 'state' => 'state'] as $key => $column) {
if ($filters[$key] !== '') {
$query->where($column, $filters[$key]);
}
}
if ($filters['q'] !== '') {
$like = '%' . str_replace(['!', '%', '_'], ['!!', '!%', '!_'], $filters['q']) . '%';
$query->whereRaw("name LIKE ? ESCAPE '!'", [$like]);
}
$total = (int) (clone $query)->count('id');
$totalPages = max(1, (int) ceil($total / $perPage));
$page = min($page, $totalPages);
$rows = $query->orderByDesc('id')
->offset(($page - 1) * $perPage)->limit($perPage)->get();
$sourceIds = [];
$hostIds = [];
$operationIds = [];
foreach ($rows as $row) {
$sourceIds[] = (int) $row->source_instance_id;
$hostIds[] = (int) $row->source_host_id;
if (preg_match('/^[a-f0-9]{32}$/D',
(string) $row->last_operation_id) === 1) {
$operationIds[] = (string) $row->last_operation_id;
}
}
$sources = [];
if ($sourceIds !== []) {
foreach (CloudInstance::whereIn('id', array_values(array_unique(
$sourceIds
)))->get(['id', 'name', 'status', 'user_id']) as $source) {
$sources[(int) $source->id] = $source;
}
}
$hosts = [];
if ($hostIds !== []) {
foreach (CloudHost::whereIn('id', array_values(array_unique(
$hostIds
)))->get(['id', 'name', 'region']) as $host) {
$hosts[(int) $host->id] = $host;
}
}
$jobs = [];
if ($operationIds !== []) {
foreach (CloudJob::whereIn('operation_id', array_values(array_unique(
$operationIds
)))->get() as $job) {
$jobs[(string) $job->operation_id] = $job;
}
}
$canManage = ($actor['is_super_admin'] ?? false) === true
&& ($actor['can_manage'] ?? false) === true;
$items = [];
foreach ($rows as $row) {
$source = $sources[(int) $row->source_instance_id] ?? null;
$host = $hosts[(int) $row->source_host_id] ?? null;
$item = self::displayBackup(
$row,
$jobs[(string) $row->last_operation_id] ?? null,
$canManage
);
$item['source'] = [
'instance_id' => (string) $row->source_instance_id,
'name' => $source === null ? '' : (string) $source->name,
'status' => $source === null ? 'missing'
: (string) $source->status,
'deleted' => $source === null
|| (string) $source->status === 'deleted',
'user_id' => (string) $row->user_id,
'host_id' => (string) $row->source_host_id,
'host_name' => $host === null ? '' : (string) $host->name,
'region' => $host === null ? '' : (string) $host->region,
];
if ((string) $row->state === 'ready') {
$resources = self::resourcesFromSpec(
self::decodeSpec((string) $row->spec_json)
);
$item['restore_targets'] = array_map(
static fn (array $target): array => [
'host_id' => (string) ($target['id'] ?? ''),
'host_name' => (string) ($target['name'] ?? ''),
'region' => (string) ($target['region'] ?? ''),
'resource_pool' => (string) (
$target['resource_pool'] ?? ''
),
'network' => 'bridge:br0',
'available_ip_count' => max(1,
(int) ($target['available_ip_count'] ?? 1)),
'is_source_host' =>
($target['is_source_host'] ?? false) === true,
],
CloudPlacementService::backupRestoreTargets(
(int) $row->source_host_id,
$resources
)
);
}
$items[] = $item;
}
$from = $total === 0 ? 0 : (($page - 1) * $perPage) + 1;
return [
'items' => $items,
'filters' => $filters,
'can_manage' => $canManage,
'pagination' => [
'page' => $page,
'per_page' => $perPage,
'total' => $total,
'total_pages' => $totalPages,
'from' => $from,
'to' => $from === 0 ? 0 : $from + count($items) - 1,
],
];
}
public static function listing(CloudInstance $instance, array $actor): array
{
if (($actor['is_admin'] ?? false) !== true) {
throw new CloudError('只有管理员可以查看虚拟机备份', 403);
}
$canManage = ($actor['is_super_admin'] ?? false) === true
&& ($actor['can_manage'] ?? false) === true;
if ((string) $instance->status === 'deleted') {
throw new CloudError('虚拟机不存在', 404);
}
$rows = CloudInstanceBackup::where(
'source_instance_id', (int) $instance->id
)->where('purpose', 'portable')->where('state', '<>', 'deleted')
->orderByDesc('id')->get();
$operationIds = [];
foreach ($rows as $row) {
if (preg_match('/^[a-f0-9]{32}$/D',
(string) $row->last_operation_id) === 1) {
$operationIds[] = (string) $row->last_operation_id;
}
}
$jobs = [];
if ($operationIds !== []) {
foreach (CloudJob::whereIn('operation_id',
array_values(array_unique($operationIds)))->get() as $job) {
$jobs[(string) $job->operation_id] = $job;
}
}
$backups = [];
$targetMap = [];
$manualActiveCount = 0;
foreach ($rows as $row) {
if ((string) $row->source === 'manual'
&& in_array((string) $row->state,
self::ACTIVE_BACKUP_STATES, true)) {
$manualActiveCount++;
}
$display = self::displayBackup(
$row,
$jobs[(string) $row->last_operation_id] ?? null,
$canManage
);
if ((string) $row->state === 'ready') {
$resources = self::resourcesFromSpec(
self::decodeSpec((string) $row->spec_json)
);
$backupTargets = [];
foreach (CloudPlacementService::backupRestoreTargets(
(int) $row->source_host_id,
$resources
) as $target) {
$normalized = [
'host_id' => (string) ($target['id'] ?? ''),
'host_name' => (string) ($target['name'] ?? ''),
'region' => (string) ($target['region'] ?? ''),
'resource_pool' => (string) (
$target['resource_pool'] ?? ''
),
'network' => 'bridge:br0',
'available_ip_count' => max(
1,
(int) ($target['available_ip_count'] ?? 1)
),
'is_source_host' =>
($target['is_source_host'] ?? false) === true,
];
$backupTargets[] = $normalized;
$targetMap[$normalized['host_id']] = $normalized;
}
$display['restore_targets'] = $backupTargets;
}
$backups[] = $display;
}
$active = CloudJobService::blockingQueryForInstance(
(int) $instance->id
)->orderBy('id')->first();
$spec = self::decodeSpec((string) $instance->spec_json, false);
$compatibilityFrozen = $spec !== null
&& is_bool($spec['debian_12_network_fix'] ?? null);
$supported = $spec !== null
&& ($spec['managed'] ?? false) === true
&& ($spec['os_source'] ?? null) === 'cloud_template'
&& $compatibilityFrozen;
$storage = CloudObjectStorageConfig::view();
$migration = CloudInstanceMigration::where(
'instance_id', (int) $instance->id
)->orderByDesc('id')->first();
$migrationTargets = $supported ? array_map(
static fn (array $target): array => [
'host_id' => (string) ($target['id'] ?? ''),
'host_name' => (string) ($target['name'] ?? ''),
'region' => (string) ($target['region'] ?? ''),
'resource_pool' => (string) ($target['resource_pool'] ?? ''),
'available_ip_count' => max(
1,
(int) ($target['available_ip_count'] ?? 1)
),
],
CloudPlacementService::maintenanceMigrationTargets(
$instance,
self::resourcesFromSpec($spec),
CloudPeerMigrationArtifactService::advisoryAdditionalDiskGib(
$instance
)
)
) : [];
$migrationBlocks = $migration !== null
&& CloudInstanceMigrationWorkflow::blocksInstance(
(string) $migration->state
);
$canOperate = $canManage
&& $active === null
&& !$migrationBlocks
&& (string) $instance->status === 'active'
&& $supported
&& (bool) ($storage['ready'] ?? false);
$canMigrate = $canManage
&& $active === null
&& !($migration !== null
&& CloudInstanceMigrationWorkflow::blocksNewMigration(
(string) $migration->state
))
&& (string) $instance->status === 'active'
&& $supported
&& $migrationTargets !== [];
$canCreate = $canOperate
&& $manualActiveCount < self::MAX_PER_INSTANCE;
$reason = null;
if (!$canManage) {
$reason = '当前管理员仅可查看,超级管理员才可以创建或恢复备份';
} elseif (!(bool) ($storage['ready'] ?? false)) {
$reason = '请先在对象存储页面保存配置并通过连接验证';
} elseif ($spec !== null
&& ($spec['managed'] ?? false) === true
&& ($spec['os_source'] ?? null) === 'cloud_template'
&& !$compatibilityFrozen) {
$reason = '实例缺少创建时冻结的网络兼容快照,不能生成可移植备份';
} elseif (!$supported) {
$reason = '仅面板创建的云镜像实例支持可移植备份';
} elseif ($active !== null) {
$reason = '实例已有任务,请等待完成';
} elseif ($migrationBlocks) {
$reason = '实例已有未完成的维护迁移,请先完成核对或源副本清理';
} elseif ((string) $instance->status !== 'active') {
$reason = '只有活动实例可以创建备份';
} elseif (!$canCreate) {
$reason = '每台实例最多保留十个有效备份';
}
return [
'instance_id' => (string) $instance->id,
'backups' => $backups,
'restore_targets' => array_values($targetMap),
'migration_targets' => $migrationTargets,
'maintenance_migration' => $migration === null ? null
: CloudInstanceMigrationService::display($migration),
'max_backups' => self::MAX_PER_INSTANCE,
'can_create' => $canCreate,
'can_restore' => $canManage && $targetMap !== [],
'can_migrate' => $canMigrate,
'create_block_reason' => $reason,
'active_job' => $active === null ? null : [
'id' => (string) $active->id,
'kind' => (string) $active->kind,
'status' => CloudOperationErrorCatalog::userStatus(
(string) $active->status
) ?? 'unknown',
],
];
}
/**
* Delete or safely reconcile one frozen S3 backup object.
*
* An ambiguous DELETE is never replayed. Two spaced HEAD observations of
* the same immutable identity may authorize a fresh deletion generation;
* at most three generations run automatically in one supervised cycle.
*/
public static function delete(CloudInstanceBackup $backup, array $actor,
?CloudObjectStorageS3Client $client = null,
?callable $beforeDeleteLocked = null): array
{
CloudAccess::manage($actor);
$client ??= new CloudObjectStorageS3Client();
$now = time();
$prepared = $backup->getConnection()->transaction(function () use (
$backup, $beforeDeleteLocked, $now
): array {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->first();
if ($locked === null) {
throw new CloudError('备份不存在', 404);
}
$state = (string) $locked->state;
if ($state === 'deleted') {
return ['mode' => 'deleted', 'backup' => $locked];
}
self::assertBackupDeletionAvailable($locked);
if ($state === 'delete_attention') {
return ['mode' => 'attention', 'backup' => $locked];
}
if (in_array($state, ['deleting', 'delete_uncertain'], true)) {
if ((int) $locked->delete_next_check_at > $now) {
return ['mode' => 'waiting', 'backup' => $locked];
}
return [
'mode' => 'reconcile',
'backup' => $locked,
'runtime' => self::storageRuntime($locked),
'object_key' => (string) $locked->object_key,
'object_etag' => (string) $locked->object_etag,
'object_version_id' => (string) $locked->object_version_id,
];
}
if (!in_array($state, ['ready', 'failed'], true)) {
throw new CloudError('当前备份状态不允许删除', 409);
}
// Decrypt before any remote request. The exact immutable identity
// is captured by HEAD before crossing the deleting boundary.
$runtime = self::storageRuntime($locked);
return [
'mode' => 'identify',
'previous_state' => $state,
'backup' => $locked,
'runtime' => $runtime,
'object_key' => (string) $locked->object_key,
'object_etag' => (string) $locked->object_etag,
'object_version_id' => (string) $locked->object_version_id,
];
}, 3);
if (in_array($prepared['mode'], ['deleted', 'waiting', 'attention'], true)) {
return self::deletionResult($prepared['backup']);
}
$runtime = $prepared['runtime'];
$objectKey = $prepared['object_key'];
if ($prepared['mode'] === 'identify') {
try {
$identity = $client->inspectObject(
$runtime['config'], $runtime['credentials'], $objectKey,
(string) $prepared['object_version_id']
);
} catch (CloudObjectStorageTransferError $error) {
throw new CloudError($error->getMessage(), 502);
}
if ($identity === null) {
return self::finishDelete((int) $backup->id);
}
self::assertObjectIdentity(
$prepared['backup'], $identity,
(string) $prepared['object_etag'] !== ''
);
$prepared = self::claimDeleteGeneration(
(int) $backup->id,
(string) $prepared['previous_state'],
$identity,
time(),
$beforeDeleteLocked
);
if ($prepared['mode'] !== 'delete') {
return self::deletionResult($prepared['backup']);
}
}
if ($prepared['mode'] === 'reconcile') {
try {
$identity = $client->inspectObject(
$runtime['config'], $runtime['credentials'], $objectKey,
(string) $prepared['object_version_id']
);
} catch (CloudObjectStorageTransferError $error) {
self::markDeleteUncertain(
(int) $backup->id, $error->publicCode, time(), true,
self::DELETE_RETRY_SECONDS
);
return self::deletionResult(
CloudInstanceBackup::findOrFail((int) $backup->id)
);
}
if ($identity === null) {
return self::finishDelete((int) $backup->id);
}
try {
self::assertObjectIdentity($prepared['backup'], $identity, true);
} catch (CloudError $error) {
self::markDeleteAttention(
(int) $backup->id, 'object_identity_changed', time()
);
return self::deletionResult(
CloudInstanceBackup::findOrFail((int) $backup->id)
);
}
$prepared = self::recordStableDeleteProbe(
(int) $backup->id, time()
);
if ($prepared['mode'] !== 'delete') {
return self::deletionResult($prepared['backup']);
}
$runtime = $prepared['runtime'];
$objectKey = $prepared['object_key'];
}
try {
$client->deleteObject(
$runtime['config'], $runtime['credentials'], $objectKey,
(string) $prepared['object_version_id']
);
} catch (CloudObjectStorageTransferError $error) {
if ($error->effectMayHaveApplied) {
self::markDeleteUncertain(
(int) $backup->id, $error->publicCode, time(), true
);
return self::deletionResult(
CloudInstanceBackup::findOrFail((int) $backup->id)
);
}
if (isset($prepared['previous_state'])) {
self::restoreAfterRejectedDelete(
(int) $backup->id,
(string) $prepared['previous_state'],
$error->publicCode
);
} else {
self::markDeleteAttention(
(int) $backup->id, $error->publicCode, time()
);
}
throw new CloudError($error->getMessage(), 502);
}
return self::finishDelete((int) $backup->id);
}
/** Sanitized administrator view of remote objects awaiting cleanup. */
public static function cleanupDebt(array $actor, array $input = []): array
{
if (($actor['is_admin'] ?? false) !== true) {
throw new CloudError('只有管理员可以查看对象清理异常', 403);
}
CloudInput::only($input, ['page', 'per_page']);
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
$perPage = self::libraryInteger($input['per_page'] ?? 20, 10, 50, 20);
$query = CloudInstanceBackup::whereIn('state', self::CLEANUP_DEBT_STATES);
$total = (int) (clone $query)->count('id');
$bytes = (int) (clone $query)->sum('object_size_bytes');
$attention = (int) (clone $query)
->where('state', 'delete_attention')->count('id');
$totalPages = max(1, (int) ceil($total / $perPage));
$page = min($page, $totalPages);
$rows = $query->orderByRaw(
"CASE WHEN state = 'delete_attention' THEN 0 ELSE 1 END"
)->orderBy('delete_next_check_at')->orderBy('id')
->offset(($page - 1) * $perPage)->limit($perPage)->get();
$items = [];
foreach ($rows as $row) {
$items[] = [
'id' => (string) $row->id,
'name' => (string) $row->name,
'source_instance_id' => (string) $row->source_instance_id,
'user_id' => (string) $row->user_id,
'state' => (string) $row->state,
'object_size_bytes' => max(0, (int) $row->object_size_bytes),
'delete_generation' => max(0, (int) $row->delete_generation),
'cycle_attempts' => max(0, (int) $row->delete_cycle_attempts),
'next_check_at' => max(0, (int) $row->delete_next_check_at),
'attention_at' => $row->delete_attention_at === null ? null
: max(0, (int) $row->delete_attention_at),
'error_code' => (string) $row->error_code,
'can_retry' => (string) $row->state === 'delete_attention'
&& ($actor['is_super_admin'] ?? false) === true
&& ($actor['can_manage'] ?? false) === true,
];
}
return [
'summary' => [
'count' => $total,
'bytes' => $bytes,
'attention_count' => $attention,
],
'items' => $items,
'pagination' => [
'page' => $page,
'per_page' => $perPage,
'total' => $total,
'total_pages' => $totalPages,
],
];
}
/** Begin one new bounded cycle after an administrator has inspected debt. */
public static function retryCleanup(CloudInstanceBackup $backup, array $actor,
?CloudObjectStorageS3Client $client = null): array
{
CloudAccess::manage($actor);
$client ??= new CloudObjectStorageS3Client();
$prepared = $backup->getConnection()->transaction(function () use ($backup): array {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->firstOrFail();
if ((string) $locked->state !== 'delete_attention') {
throw new CloudError('只有待人工处理的对象可以重新进入清理流程', 409);
}
self::assertBackupDeletionAvailable($locked);
return [
'runtime' => self::storageRuntime($locked),
'object_key' => (string) $locked->object_key,
'object_version_id' => (string) $locked->object_version_id,
'backup' => $locked,
];
}, 3);
try {
$identity = $client->inspectObject(
$prepared['runtime']['config'],
$prepared['runtime']['credentials'],
$prepared['object_key'],
$prepared['object_version_id']
);
} catch (CloudObjectStorageTransferError $error) {
throw new CloudError($error->getMessage(), 502);
}
if ($identity === null) {
return self::finishDelete((int) $backup->id);
}
self::assertObjectIdentity($prepared['backup'], $identity, true);
$now = time();
$locked = $backup->getConnection()->transaction(function () use (
$backup, $now
): CloudInstanceBackup {
$row = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->firstOrFail();
if ((string) $row->state !== 'delete_attention') {
throw new CloudError('对象清理状态已发生变化', 409);
}
$row->state = 'delete_uncertain';
$row->delete_cycle_attempts = 0;
$row->delete_presence_confirmations = 1;
$row->delete_last_checked_at = $now;
$row->delete_next_check_at = $now + self::DELETE_STABLE_SECONDS;
$row->delete_attention_at = null;
$row->error_code = 'manual_retry_awaiting_stable_probe';
$row->updated_at = $now;
$row->save();
return $row;
}, 3);
return self::deletionResult($locked);
}
public static function create(CloudInstance $instance, array $actor,
array $input): array
{
return self::createInternal($instance, $actor, $input, null);
}
/** Queue one deterministic paid automatic backup for a scheduled slot. */
public static function createAutomatic(CloudInstance $instance,
CloudBackupSubscription $subscription, array $actor,
int $scheduledFor): array
{
if ($scheduledFor < 1) {
throw new CloudError('自动备份计划时间无效', 409);
}
$zone = new \DateTimeZone(CloudAutomaticBackupService::TIMEZONE);
$name = 'autobackup-'
. (new \DateTimeImmutable('@' . $scheduledFor))
->setTimezone($zone)->format('Ymd-His');
return self::createInternal($instance, $actor, [
'name' => $name,
'consistency_mode' => 'filesystem',
], [
'subscription_id' => (int) $subscription->id,
'scheduled_for' => $scheduledFor,
]);
}
private static function createInternal(CloudInstance $instance, array $actor,
array $input, ?array $automatic): array
{
if ($automatic === null) {
CloudAccess::manage($actor);
} elseif ((int) ($actor['user_id'] ?? 0)
!== (int) $instance->user_id) {
throw new CloudError('无权为该实例创建自动备份', 403);
}
CloudInput::only($input, ['name', 'consistency_mode']);
$name = CloudInput::snapshotName($input['name'] ?? null);
$mode = CloudInput::snapshotConsistency(
$input['consistency_mode'] ?? null
);
$requestTarget = '/v1/vms/'
. rawurlencode((string) $instance->uuid) . '/backups';
$canonicalRequest = [
'instance_id' => (int) $instance->id,
'name' => $name,
'consistency_mode' => $mode,
];
if ($automatic !== null) {
$canonicalRequest['source'] = 'automatic';
$canonicalRequest['subscription_id'] =
(int) $automatic['subscription_id'];
$canonicalRequest['scheduled_for'] =
(int) $automatic['scheduled_for'];
}
$fingerprint = CloudJobService::standaloneRequestFingerprint(
(int) $instance->host_id,
'backup_create',
'POST',
$requestTarget,
$canonicalRequest
);
$prior = self::priorJob(
$actor,
'backup_create',
(int) $instance->id,
null,
$fingerprint,
false
);
if ($prior !== null) {
$backup = CloudInstanceBackup::find((int) $prior->backup_id);
if ($backup === null) {
throw new CloudError('备份任务目录记录缺失', 409);
}
return ['backup' => self::displayBackup($backup, $prior),
'job' => $prior];
}
$runtime = CloudObjectStorageConfig::runtime(true);
$plan = null;
$objectKey = null;
try {
return $instance->getConnection()->transaction(function () use (
$instance, $actor, $name, $mode, $runtime, $requestTarget,
$canonicalRequest, $fingerprint, $automatic,
&$plan, &$objectKey
): array {
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->first();
if ($locked === null || (string) $locked->status === 'deleted') {
throw new CloudError('虚拟机不存在', 404);
}
$prior = self::priorJob($actor, 'backup_create',
(int) $locked->id, null, $fingerprint);
if ($prior !== null) {
$backup = CloudInstanceBackup::find((int) $prior->backup_id);
if ($backup === null) {
throw new CloudError('备份任务目录记录缺失', 409);
}
return ['backup' => self::displayBackup($backup, $prior),
'job' => $prior];
}
$automaticSubscription = null;
if ($automatic !== null) {
$automaticSubscription = CloudBackupSubscription::where(
'id', (int) $automatic['subscription_id']
)->lockForUpdate()->first();
$scheduledFor = (int) $automatic['scheduled_for'];
$entitled = $automaticSubscription !== null
&& CloudBackupPeriodEntitlement::where(
'subscription_id',
(int) $automaticSubscription->id
)->where('instance_id', (int) $locked->id)
->where('user_id', (int) $locked->user_id)
->where('starts_at', '<=', $scheduledFor)
->where('ends_at', '>', $scheduledFor)
->lockForUpdate()->exists();
if ($automaticSubscription === null
|| !(bool) $automaticSubscription->enabled
|| (int) $automaticSubscription->instance_id
!== (int) $locked->id
|| (int) $automaticSubscription->user_id
!== (int) $locked->user_id
|| !$entitled) {
throw new CloudError('自动备份订阅与实例账本不一致', 409);
}
$existing = CloudInstanceBackup::where(
'subscription_id', (int) $automaticSubscription->id
)->where('scheduled_for',
$scheduledFor)->first();
if ($existing !== null) {
$job = CloudJob::where('kind', 'backup_create')
->where('backup_id', (int) $existing->id)
->orderByDesc('id')->first();
if ($job === null) {
throw new CloudError('自动备份计划账本不完整', 409);
}
CloudAutomaticBackupService::advanceAfterScheduled(
$automaticSubscription,
$scheduledFor,
(int) $existing->id
);
return [
'backup' => self::displayBackup($existing, $job),
'job' => $job,
];
}
}
if ((string) $locked->status !== 'active') {
throw new CloudError('只有活动实例可以创建备份', 409);
}
if ($automatic !== null
&& ((string) $locked->service_status !== 'active'
|| CloudAutomaticBackupService::
shouldPauseForInstanceExpiry($locked, time()))) {
throw new CloudError('实例服务当前不可创建自动备份', 409);
}
self::assertManagedSource($locked);
self::assertNoActiveJob($locked);
$usedQuery = CloudInstanceBackup::where(
'source_instance_id', (int) $locked->id
)->where('purpose', 'portable')
->whereIn('state', self::ACTIVE_BACKUP_STATES);
if ($automatic === null) {
$usedQuery->where('source', 'manual');
} else {
$usedQuery->where('source', 'automatic')
->where('subscription_id',
(int) $automaticSubscription->id)
->whereNotExists(static function ($query): void {
$query->selectRaw('1')
->from('cloud_deployment_snapshots')
->whereColumn(
'cloud_deployment_snapshots.backup_id',
'cloud_instance_backups.id'
)->where(
'cloud_deployment_snapshots.state',
'<>',
'deleted'
);
});
}
$used = (int) $usedQuery->count('id');
if ($automatic === null && $used >= self::MAX_PER_INSTANCE) {
throw new CloudError('每台实例最多保留十个有效备份', 409);
}
if ($automatic !== null
&& $used >= min(
CloudAutomaticBackupService::MAX_RETENTION,
max(1, (int) $automaticSubscription->retention_count)
) + 1) {
throw new CloudError('自动备份正在等待旧备份安全清理', 409);
}
$spec = self::freezeRestoreCompatibility(
self::decodeSpec((string) $locked->spec_json)
);
$resources = self::resourcesFromSpec($spec);
$maximumBytes = self::maximumObjectBytes(
$resources['disk_gib']
);
$key = bin2hex(random_bytes(16));
$prefix = trim((string) $runtime['config']['prefix'], '/');
$objectKey = ($prefix === '' ? '' : $prefix . '/')
. 'vm-backups/' . $key . '/system.qcow2';
$plan = (new CloudObjectStorageS3Client())->beginMultipartPlan(
$runtime['config'], $runtime['credentials'], $objectKey,
$maximumBytes, self::TRANSFER_URL_SECONDS
);
$agentBody = [
'backup_key' => $key,
'name' => $name,
'consistency_mode' => $mode,
'object_upload' => self::publicTransferPlan($plan),
];
$encodedAgentBody = json_encode(
$agentBody,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
if (strlen($encodedAgentBody) > self::MAX_AGENT_BODY_BYTES) {
throw new CloudError(
'对象存储返回的分段上传凭据过大,不能安全派发备份',
409
);
}
$now = time();
$backup = new CloudInstanceBackup();
$backup->backup_key = $key;
$backup->source_instance_id = (int) $locked->id;
$backup->source_host_id = (int) $locked->host_id;
$backup->user_id = (int) $locked->user_id;
$backup->source_uuid = strtolower((string) $locked->uuid);
$backup->name = $name;
$backup->purpose = 'portable';
$backup->source = $automatic === null
? 'manual' : 'automatic';
$backup->subscription_id = $automatic === null ? null
: (int) $automaticSubscription->id;
$backup->scheduled_for = $automatic === null ? null
: (int) $automatic['scheduled_for'];
$backup->state = 'creating';
$backup->consistency = $mode;
$backup->object_key = $objectKey;
$backup->object_size_bytes = 0;
$backup->virtual_size_bytes = 0;
$backup->sha256 = '';
$backup->spec_json = json_encode(
CloudInput::safeResult($spec),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
$backup->sealed_storage_runtime = CloudSecrets::encrypt([
'config' => $runtime['config'],
'credentials' => $runtime['credentials'],
'revision' => (int) $runtime['revision'],
], self::STORAGE_PURPOSE);
$backup->sealed_credential = $locked->sealed_credential;
$backup->storage_revision = (int) $runtime['revision'];
$backup->active_job_id = null;
$backup->last_operation_id = null;
$backup->error_code = '';
$backup->ready_at = null;
$backup->created_at = $backup->updated_at = $now;
$backup->save();
[$job, $created] = CloudJobService::
reserveStandaloneAgentMutation(
$actor,
(int) $locked->host_id,
'backup_create',
'POST',
$requestTarget,
$agentBody,
[
'owner_id' => (int) $locked->user_id,
'can_manage' => true,
'backup_key' => $key,
'maximum_size_bytes' => $maximumBytes,
],
$canonicalRequest
);
if (!$created) {
throw new CloudError('备份幂等任务状态冲突', 409);
}
$job->instance_id = (int) $locked->id;
$job->backup_id = (int) $backup->id;
$job->save();
$backup->active_job_id = (int) $job->id;
$backup->last_operation_id = (string) $job->operation_id;
$backup->updated_at = time();
$backup->save();
if ($automaticSubscription !== null) {
CloudAutomaticBackupService::advanceAfterScheduled(
$automaticSubscription,
(int) $automatic['scheduled_for'],
(int) $backup->id
);
}
return ['backup' => self::displayBackup($backup, $job),
'job' => $job];
}, self::CREATE_TRANSACTION_ATTEMPTS);
} catch (\Throwable $error) {
if (is_array($plan) && is_string($objectKey)) {
try {
(new CloudObjectStorageS3Client())->abortMultipart(
$runtime['config'], $runtime['credentials'], $objectKey,
(string) ($plan['upload_id'] ?? '')
);
} catch (\Throwable $cleanupError) {
error_log('cloud backup multipart cleanup unavailable: '
. get_class($cleanupError));
}
}
throw $error;
}
}
public static function restore(CloudInstanceBackup $backup, array $actor,
array $input): array
{
CloudAccess::manage($actor);
CloudInput::only($input,
['target_host_id', 'name', 'confirm_new_instance']);
$targetHostId = filter_var($input['target_host_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
if ($targetHostId === false) {
throw new CloudError('请选择目标宿主机');
}
$name = CloudInput::name($input['name'] ?? null);
if (($input['confirm_new_instance'] ?? false) !== true) {
throw new CloudError('请确认恢复会创建免计费的独立新实例');
}
$canonicalRequest = [
'backup_id' => (int) $backup->id,
'target_host_id' => (int) $targetHostId,
'name' => $name,
];
$idempotencyTarget = '/v1/backup-restores';
$fingerprint = CloudJobService::standaloneRequestFingerprint(
(int) $targetHostId,
'backup_restore',
'POST',
$idempotencyTarget,
$canonicalRequest
);
$prior = self::priorJob(
$actor,
'backup_restore',
null,
(int) $backup->id,
$fingerprint,
false
);
if ($prior !== null) {
$restore = CloudBackupRestore::find(
(int) $prior->backup_restore_id
);
if ($restore === null) {
throw new CloudError('恢复任务目录记录缺失', 409);
}
return ['restore' => self::displayRestore($restore),
'job' => $prior];
}
if ((string) $backup->purpose !== 'portable') {
throw new CloudError('只有普通可移植备份可以创建独立恢复实例', 409);
}
if ((string) $backup->state !== 'ready'
|| (int) $backup->object_size_bytes < 1
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $backup->sha256) !== 1) {
throw new CloudError('备份尚未处于可恢复状态', 409);
}
$runtime = self::storageRuntime($backup);
$object = (new CloudObjectStorageS3Client())->headObject(
$runtime['config'], $runtime['credentials'],
(string) $backup->object_key
);
if ((int) $object['size_bytes'] !== (int) $backup->object_size_bytes) {
throw new CloudError('备份对象大小与目录记录不一致,不能恢复', 409);
}
$sourceUrl = (new CloudObjectStorageS3Client())->presignedGet(
$runtime['config'], $runtime['credentials'],
(string) $backup->object_key, self::TRANSFER_URL_SECONDS
);
return $backup->getConnection()->transaction(function () use (
$backup, $actor, $targetHostId, $name, $sourceUrl,
$canonicalRequest, $idempotencyTarget, $fingerprint
): array {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->first();
if ($locked === null || (string) $locked->state !== 'ready'
|| (int) $locked->object_size_bytes < 1
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $locked->sha256) !== 1) {
throw new CloudError('备份尚未处于可恢复状态', 409);
}
$prior = self::priorJob($actor, 'backup_restore', null,
(int) $locked->id, $fingerprint);
if ($prior !== null) {
$restore = CloudBackupRestore::find(
(int) $prior->backup_restore_id
);
if ($restore === null) {
throw new CloudError('恢复任务目录记录缺失', 409);
}
return ['restore' => self::displayRestore($restore),
'job' => $prior];
}
if (CloudInstance::where('host_id', (int) $targetHostId)
->where('name', $name)->where('status', '<>', 'deleted')
->exists()) {
throw new CloudError('目标宿主机已有同名实例', 409);
}
$spec = self::decodeSpec((string) $locked->spec_json);
$resources = self::resourcesFromSpec($spec);
$uuid = self::uuidV4();
$operationId = bin2hex(random_bytes(16));
$now = time();
$restore = new CloudBackupRestore();
$restore->restore_key = bin2hex(random_bytes(16));
$restore->backup_id = (int) $locked->id;
$restore->target_host_id = (int) $targetHostId;
$restore->target_instance_id = null;
$restore->reservation_id = null;
$restore->job_id = null;
$restore->operation_id = $operationId;
$restore->new_uuid = $uuid;
$restore->new_name = $name;
$restore->state = 'restoring';
$restore->error_code = '';
$restore->ready_at = null;
$restore->created_at = $restore->updated_at = $now;
$restore->save();
$placement = CloudPlacementService::reserveBackupRestore(
$restore, (int) $locked->user_id, $resources
);
$network = $placement['network'];
$instanceSpec = self::restoredSpec(
$spec, $locked, $network, $resources
);
$instance = new CloudInstance();
$instance->uuid = $uuid;
$instance->host_id = (int) $targetHostId;
$instance->user_id = (int) $locked->user_id;
$instance->order_id = null;
$instance->plan_id = null;
$instance->offer_id = null;
$instance->resource_pool_id = (int) $network['resource_pool_id'];
$instance->reservation_id = (int) $placement['reservation']->id;
$instance->current_period_id = null;
$instance->name = $name;
$instance->status = 'provisioning';
$instance->service_status = 'exempt';
$instance->network_status = 'pending';
$instance->expires_at = null;
$instance->renewal_price = null;
$instance->renewal_days = 0;
$instance->renewal_traffic_bytes = 0;
$instance->traffic_quota_bytes = 0;
$instance->traffic_download_bytes = 0;
$instance->traffic_upload_bytes = 0;
$instance->lifetime_download_bytes = 0;
$instance->lifetime_upload_bytes = 0;
$instance->traffic_synced_at = 0;
$instance->suspended_at = null;
$instance->network_suspended_at = null;
$instance->spec_json = json_encode($instanceSpec,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE);
$instance->sealed_credential = $locked->sealed_credential;
$instance->created_at = $instance->updated_at = $now;
$instance->save();
$reservation = $placement['reservation'];
$reservation->instance_id = (int) $instance->id;
$reservation->updated_at = time();
$reservation->save();
$policy = $spec['platform_network_policy_snapshot'] ?? [];
if (!is_array($policy) || array_is_list($policy)) {
$policy = [];
}
$body = [
'uuid' => $uuid,
'source' => '',
'name' => $name,
'description' => '',
'vcpu' => $resources['cpu_cores'],
'memory_mib' => $resources['memory_mib'],
'disk_gib' => $resources['disk_gib'],
'bandwidth_mbps' => $resources['bandwidth_mbps'],
'network' => 'bridge:br0',
'template_id' => '',
'template_revision' => 0,
'template_generation_token' => '',
'startup_script' => '',
'hostname' => CloudInput::hostname($name),
'guest_ssh_public_key' => '',
'root_password' => '',
'public_pool_key' => (string) $network['public_pool_key'],
'public_pool_cidr' => (string) $network['public_pool_cidr'],
'public_ip_cidr' => (string) $network['public_ip_cidr'],
'gateway' => (string) $network['gateway'],
'dns_servers' => array_values($network['dns_servers']),
'platform_network_policy_snapshot' => $policy,
'debian_12_network_fix' =>
($spec['debian_12_network_fix'] ?? null) === true,
'backup_key' => (string) $locked->backup_key,
'source_url' => $sourceUrl,
'source_size_bytes' => (int) $locked->object_size_bytes,
'source_sha256' => (string) $locked->sha256,
];
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
$actor, (int) $targetHostId, 'backup_restore', 'POST',
'/v1/vms/' . rawurlencode($uuid) . '/restore-backup',
$body,
[
'owner_id' => (int) $locked->user_id,
'can_manage' => true,
'backup_key' => (string) $locked->backup_key,
'restore_key' => (string) $restore->restore_key,
'reservation_id' => (int) $reservation->id,
],
$canonicalRequest,
$operationId,
$idempotencyTarget
);
if (!$created) {
throw new CloudError('恢复幂等任务状态冲突', 409);
}
$job->instance_id = (int) $instance->id;
$job->backup_id = (int) $locked->id;
$job->backup_restore_id = (int) $restore->id;
$job->save();
$restore->target_instance_id = (int) $instance->id;
$restore->reservation_id = (int) $reservation->id;
$restore->job_id = (int) $job->id;
$restore->updated_at = time();
$restore->save();
return ['restore' => self::displayRestore($restore), 'job' => $job];
}, 3);
}
/**
* Build the short-lived download capability for a paid create order.
* The caller must already hold and validate the owner/order row locks;
* signing is local and therefore does not extend that transaction across
* a remote object-storage request.
*
* @return array{source_url:string}
*/
public static function provisioningSource(
CloudInstanceBackup $backup
): array {
if ((string) $backup->purpose !== 'portable'
|| (string) $backup->state !== 'ready'
|| (int) $backup->object_size_bytes < 1
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $backup->sha256) !== 1) {
throw new CloudError('云备份尚未处于可创建实例状态', 409);
}
$runtime = self::storageRuntime($backup);
return [
'source_url' => (new CloudObjectStorageS3Client())->presignedGet(
$runtime['config'],
$runtime['credentials'],
(string) $backup->object_key,
self::TRANSFER_URL_SECONDS
),
];
}
/** Replace an owner's existing system disk from its own automatic backup. */
public static function recover(CloudInstance $instance,
CloudInstanceBackup $backup, array $actor, array $input): array
{
if ((int) ($actor['user_id'] ?? 0) !== (int) $instance->user_id) {
throw new CloudError('无权恢复该实例', 403);
}
CloudInput::only($input, ['confirm_restore']);
if (($input['confirm_restore'] ?? false) !== true) {
throw new CloudError('请确认恢复会覆盖当前系统盘内容');
}
$requestKey = $actor['request_idempotency_key'] ?? null;
if (!is_string($requestKey) || $requestKey === '') {
throw new CloudError('恢复云备份必须提供请求幂等标识', 400);
}
$target = '/v1/vms/' . rawurlencode((string) $instance->uuid)
. '/recover-backup';
$canonical = [
'instance_id' => (int) $instance->id,
'backup_id' => (int) $backup->id,
'confirm_restore' => true,
];
$fingerprint = CloudJobService::standaloneRequestFingerprint(
(int) $instance->host_id,
'backup_recover',
'POST',
$target,
$canonical
);
$prior = self::priorJob($actor, 'backup_recover',
(int) $instance->id, (int) $backup->id, $fingerprint, false);
if ($prior !== null) {
return self::priorRecoveryResult($prior);
}
CloudOriginalBackupRestorePolicy::assertEligible($instance, $backup);
$runtime = self::storageRuntime($backup);
$object = (new CloudObjectStorageS3Client())->headObject(
$runtime['config'], $runtime['credentials'],
(string) $backup->object_key
);
if ((int) $object['size_bytes'] !== (int) $backup->object_size_bytes) {
throw new CloudError('备份对象大小与目录记录不一致,不能恢复', 409);
}
$sourceUrl = (new CloudObjectStorageS3Client())->presignedGet(
$runtime['config'], $runtime['credentials'],
(string) $backup->object_key, self::TRANSFER_URL_SECONDS
);
return $instance->getConnection()->transaction(function () use (
$instance, $backup, $actor, $target, $canonical, $fingerprint,
$sourceUrl
): array {
$lockedInstance = CloudInstance::where(
'id', (int) $instance->id
)->lockForUpdate()->first();
$lockedBackup = CloudInstanceBackup::where(
'id', (int) $backup->id
)->lockForUpdate()->first();
if ($lockedInstance === null || $lockedBackup === null) {
throw new CloudError('实例或云备份不存在', 404);
}
$prior = self::priorJob($actor, 'backup_recover',
(int) $lockedInstance->id, (int) $lockedBackup->id,
$fingerprint);
if ($prior !== null) {
return self::priorRecoveryResult($prior);
}
CloudOriginalBackupRestorePolicy::assertEligible(
$lockedInstance, $lockedBackup
);
self::assertNoActiveJob($lockedInstance);
if (CloudBackupRecovery::where(
'instance_id', (int) $lockedInstance->id
)->whereIn('state', ['recovering', 'uncertain'])->exists()) {
throw new CloudError('实例已有恢复任务或未确认效果', 409);
}
$operationId = bin2hex(random_bytes(16));
$recoveryKey = bin2hex(random_bytes(16));
$now = time();
$recovery = new CloudBackupRecovery();
$recovery->recovery_key = $recoveryKey;
$recovery->backup_id = (int) $lockedBackup->id;
$recovery->instance_id = (int) $lockedInstance->id;
$recovery->job_id = null;
$recovery->operation_id = $operationId;
$recovery->state = 'recovering';
$recovery->error_code = '';
$recovery->completed_at = null;
$recovery->created_at = $recovery->updated_at = $now;
$recovery->save();
$body = [
'backup_key' => (string) $lockedBackup->backup_key,
'source_url' => $sourceUrl,
'source_size_bytes' => (int) $lockedBackup->object_size_bytes,
'source_sha256' => (string) $lockedBackup->sha256,
];
$jobActor = $actor;
$jobActor['can_manage'] = true;
[$job, $created] = CloudJobService::
reserveStandaloneAgentMutation(
$jobActor,
(int) $lockedInstance->host_id,
'backup_recover',
'POST',
$target,
$body,
[
'owner_id' => (int) $lockedInstance->user_id,
'can_manage' => true,
'backup_key' => (string) $lockedBackup->backup_key,
'recovery_key' => $recoveryKey,
],
$canonical,
$operationId
);
if (!$created) {
throw new CloudError('云备份恢复幂等任务状态冲突', 409);
}
$job->instance_id = (int) $lockedInstance->id;
$job->backup_id = (int) $lockedBackup->id;
$job->backup_recovery_id = (int) $recovery->id;
$job->save();
$recovery->job_id = (int) $job->id;
$recovery->updated_at = time();
$recovery->save();
return [
'recovery' => self::displayRecovery($recovery),
'job' => $job,
];
}, 3);
}
/** Apply a terminal/running Agent result inside CloudWorkerService's lock. */
public static function finishJobLocked(CloudJob $job, string $status,
$result, array $payload): void
{
if ($job->kind === 'backup_create') {
self::finishCreateLocked($job, $status, $result, $payload);
} elseif ($job->kind === 'backup_restore') {
self::finishRestoreLocked($job, $status, $result, $payload);
} elseif ($job->kind === 'backup_recover') {
self::finishRecoveryLocked($job, $status, $result, $payload);
} elseif ($job->kind === 'backup_workspace_cleanup') {
self::finishWorkspaceCleanupLocked(
$job, $status, $result, $payload
);
}
}
/** Queue one evidence-bound cleanup of a source-host backup workspace. */
public static function cleanupWorkspace(
CloudInstanceBackup $backup,
array $actor
): CloudJob {
CloudAccess::manage($actor);
if (($actor['is_super_admin'] ?? false) !== true) {
throw new CloudError('只有超级管理员可以清理备份工作区', 403);
}
$original = CloudJob::where(
'operation_id', (string) $backup->last_operation_id
)->first();
if ($original === null || (string) $original->kind !== 'backup_create'
|| (string) $original->phase !== 'terminal'
|| (string) $original->outcome !== 'succeeded'
|| (int) $original->backup_id !== (int) $backup->id
|| (int) $original->host_id !== (int) $backup->source_host_id) {
throw new CloudError('原备份任务身份不完整,不能清理工作区', 409);
}
$inspection = CloudAdminOperationRecoveryService::inspect(
$actor, (string) $original->id
);
$cleanup = is_array($inspection['cleanup'] ?? null)
? $inspection['cleanup'] : [];
$planId = ($cleanup['available'] ?? false) === true
&& is_string($cleanup['plan_id'] ?? null)
? strtolower((string) $cleanup['plan_id']) : '';
if ($planId !== ''
&& preg_match('/^[a-f0-9]{64}$/D', $planId) !== 1) {
throw new CloudError('Agent 返回的清理计划身份无效', 502);
}
return $backup->getConnection()->transaction(function () use (
$backup, $actor, $original, $planId
): CloudJob {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->first();
if ($locked === null || (string) $locked->state !== 'ready'
|| (string) $locked->error_code
!== 'backup_local_cleanup_pending'
|| !hash_equals(
(string) $locked->last_operation_id,
(string) $original->operation_id
)) {
throw new CloudError('备份工作区状态已变化,请刷新后重试', 409);
}
$operationId = bin2hex(random_bytes(16));
$target = '/v1/operations/' . rawurlencode($operationId)
. '/cleanup-backup-workspace';
$body = [
'prior_operation_id' => (string) $original->operation_id,
'cleanup_plan_id' => $planId,
];
$canonical = [
'backup_id' => (int) $locked->id,
'backup_key' => (string) $locked->backup_key,
'prior_operation_id' => (string) $original->operation_id,
'cleanup_plan_id' => $planId,
];
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
$actor,
(int) $locked->source_host_id,
'backup_workspace_cleanup',
'POST',
$target,
$body,
[
'owner_id' => (int) $locked->user_id,
'can_manage' => true,
'backup_key' => (string) $locked->backup_key,
'prior_job_id' => (int) $original->id,
'prior_operation_id' => (string) $original->operation_id,
'cleanup_plan_id' => $planId,
],
$canonical,
$operationId,
'/v1/backups/' . (int) $locked->id . '/cleanup-workspace'
);
if ($created) {
$job->instance_id = (int) $locked->source_instance_id;
$job->backup_id = (int) $locked->id;
$job->save();
}
return $job;
}, 3);
}
private static function finishWorkspaceCleanupLocked(
CloudJob $job,
string $status,
$result,
array $payload
): void {
$backup = $job->backup_id === null ? null
: CloudInstanceBackup::where('id', (int) $job->backup_id)
->lockForUpdate()->first();
if ($backup === null
|| (int) $backup->source_host_id !== (int) $job->host_id
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|| !is_string($payload['prior_operation_id'] ?? null)
|| !hash_equals(
(string) $backup->last_operation_id,
(string) $payload['prior_operation_id']
)
|| !hash_equals(
(string) $backup->backup_key,
(string) ($payload['backup_key'] ?? '')
)) {
throw new CloudRecoveryInstanceStateConflict(
'备份工作区清理任务与目录账本不一致', 409
);
}
if ($status !== 'succeeded') {
return;
}
if (!is_array($result)
|| !hash_equals(
(string) $backup->backup_key,
(string) ($result['backup_key'] ?? '')
)
|| ($result['cleaned'] ?? null) !== true
|| ($result['workspace_absent'] ?? null) !== true
|| ($result['metadata_absent'] ?? null) !== true) {
throw new CloudError('Agent 清理成功结果缺少完整缺席证据', 502);
}
if ((string) $backup->error_code
=== 'backup_local_cleanup_pending') {
$backup->error_code = '';
$backup->updated_at = time();
$backup->save();
}
}
public static function markJobUncertain(CloudJob $job): void
{
if ($job->kind === 'backup_create' && $job->backup_id !== null) {
CloudInstanceBackup::where('id', (int) $job->backup_id)
->where('last_operation_id', (string) $job->operation_id)
->update(['state' => 'uncertain', 'error_code' =>
'finalization_incident', 'updated_at' => time()]);
} elseif ($job->kind === 'backup_restore'
&& $job->backup_restore_id !== null) {
CloudBackupRestore::where('id', (int) $job->backup_restore_id)
->where('operation_id', (string) $job->operation_id)
->update(['state' => 'uncertain', 'error_code' =>
'finalization_incident', 'updated_at' => time()]);
} elseif ($job->kind === 'backup_recover'
&& $job->backup_recovery_id !== null) {
CloudBackupRecovery::where(
'id', (int) $job->backup_recovery_id
)->where('operation_id', (string) $job->operation_id)
->update(['state' => 'uncertain', 'error_code' =>
'finalization_incident', 'updated_at' => time()]);
}
CloudInstanceMigrationService::markJobUncertain($job);
}
private static function finishCreateLocked(CloudJob $job, string $status,
$result, array $payload): void
{
$backup = $job->backup_id === null ? null
: CloudInstanceBackup::where('id', (int) $job->backup_id)
->lockForUpdate()->first();
if ($backup === null
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|| (int) $backup->source_host_id !== (int) $job->host_id
|| (int) $backup->user_id !== (int) ($payload['owner_id'] ?? 0)
|| !hash_equals((string) $backup->backup_key,
(string) ($payload['backup_key'] ?? ''))
|| !hash_equals((string) $backup->last_operation_id,
(string) $job->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'备份任务与面板目录账本不一致', 409
);
}
if ($status === 'running') {
return;
}
$now = time();
if ($status === 'uncertain') {
$backup->state = 'uncertain';
$backup->error_code = 'operation_effect_unknown';
} elseif ($status === 'failed') {
$backup->state = 'failed';
$backup->error_code = 'backup_failed';
$backup->active_job_id = null;
} elseif ($status === 'succeeded') {
$size = filter_var($result['size_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$virtual = filter_var($result['virtual_size_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$sha = $result['sha256'] ?? null;
$objectEtag = $result['object_etag'] ?? '';
$objectVersionId = $result['object_version_id'] ?? '';
$consistency = $result['consistency'] ?? null;
$cleanupPending = $result['cleanup_pending'] ?? false;
if (!is_array($result)
|| !hash_equals((string) $backup->backup_key,
(string) ($result['backup_key'] ?? ''))
|| $size === false || $virtual === false
|| !is_string($sha)
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|| !is_string($objectEtag) || strlen($objectEtag) > 256
|| preg_match('/[\x00-\x1f\x7f]/D', $objectEtag) === 1
|| !is_string($objectVersionId)
|| strlen($objectVersionId) > 1024
|| preg_match('/[\x00-\x1f\x7f]/D', $objectVersionId) === 1
|| ($objectVersionId !== '' && $objectEtag === '')
|| !in_array($consistency, ['filesystem', 'crash'], true)
|| !is_bool($cleanupPending)
|| (int) $size > (int) ($payload['maximum_size_bytes'] ?? 0)) {
throw new CloudError('Agent 成功结果缺少可信备份对象身份', 502);
}
$backup->state = 'ready';
$backup->consistency = $consistency;
$backup->object_size_bytes = (int) $size;
$backup->virtual_size_bytes = (int) $virtual;
$backup->sha256 = $sha;
$backup->object_etag = $objectEtag;
$backup->object_version_id = $objectVersionId;
$backup->ready_at = $now;
$backup->active_job_id = null;
// The remote object is complete and restorable. A host-side
// snapshot cleanup warning must not discard this sole immutable
// object identity or trigger another upload.
$backup->error_code = $cleanupPending
? 'backup_local_cleanup_pending' : '';
}
$backup->updated_at = $now;
$backup->save();
}
private static function finishRestoreLocked(CloudJob $job, string $status,
$result, array $payload): void
{
$restore = $job->backup_restore_id === null ? null
: CloudBackupRestore::where('id', (int) $job->backup_restore_id)
->lockForUpdate()->first();
$backup = $job->backup_id === null ? null
: CloudInstanceBackup::where('id', (int) $job->backup_id)
->lockForUpdate()->first();
$instance = $job->instance_id === null ? null
: CloudInstance::where('id', (int) $job->instance_id)
->lockForUpdate()->first();
if ($restore === null || $backup === null || $instance === null
|| (int) $restore->backup_id !== (int) $backup->id
|| (int) $restore->target_instance_id !== (int) $instance->id
|| (int) $restore->target_host_id !== (int) $job->host_id
|| !hash_equals((string) $restore->operation_id,
(string) $job->operation_id)
|| !hash_equals((string) $restore->restore_key,
(string) ($payload['restore_key'] ?? ''))
|| !hash_equals((string) $backup->backup_key,
(string) ($payload['backup_key'] ?? ''))
|| (int) $instance->user_id !== (int) ($payload['owner_id'] ?? 0)
|| !hash_equals(strtolower((string) $restore->new_uuid),
strtolower((string) $instance->uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'备份恢复任务与实例、资源或目录账本不一致', 409
);
}
if ($status === 'running') {
return;
}
$requestedIp = $payload['body']['public_ip_cidr'] ?? null;
if (!is_string($requestedIp)) {
throw new CloudRecoveryInstanceStateConflict(
'备份恢复任务缺少公网 IP 快照', 409
);
}
$now = time();
if ($status === 'uncertain') {
CloudPlacementService::finishBackupRestore(
$restore, (int) $instance->id, $requestedIp, 'uncertain'
);
$restore->state = 'uncertain';
$restore->error_code = 'operation_effect_unknown';
} elseif ($status === 'failed') {
CloudPlacementService::finishBackupRestore(
$restore, (int) $instance->id, $requestedIp, 'failed'
);
$restore->state = 'failed';
$restore->error_code = 'backup_restore_failed';
$instance->status = 'deleted';
$instance->network_status = 'inactive';
$instance->sealed_credential = null;
$instance->updated_at = $now;
$instance->save();
} elseif ($status === 'succeeded') {
$vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
if (!is_array($result) || !is_array($vm)
|| !hash_equals((string) $backup->backup_key,
(string) ($result['backup_key'] ?? ''))
|| !hash_equals((string) $backup->sha256,
(string) ($result['sha256'] ?? ''))
|| !hash_equals(strtolower((string) $restore->new_uuid),
strtolower((string) ($vm['uuid'] ?? '')))
|| !hash_equals($requestedIp,
(string) ($vm['public_ip_cidr'] ?? ''))
|| !hash_equals((string) ($payload['body']['public_pool_key'] ?? ''),
(string) ($vm['public_pool_key'] ?? ''))) {
throw new CloudError(
'Agent 恢复成功结果与备份、实例或公网 IP 预留不一致',
502
);
}
CloudPlacementService::finishBackupRestore(
$restore, (int) $instance->id, $requestedIp, 'succeeded'
);
$spec = self::decodeSpec((string) $instance->spec_json);
$sourceSpec = self::decodeSpec((string) $backup->spec_json);
$spec = CloudInstanceMetadata::merge($spec, $vm);
foreach (['os_source', 'os_name', 'login_name'] as $field) {
if (isset($sourceSpec[$field])) {
$spec[$field] = $sourceSpec[$field];
}
}
$spec['managed'] = true;
$spec['restored_from_backup'] = (string) $backup->backup_key;
$instance->spec_json = json_encode($spec,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE);
$instance->status = 'active';
$instance->service_status = 'exempt';
$instance->network_status = 'active';
$instance->updated_at = $now;
$instance->save();
self::recordRuntime($instance, $vm);
$restore->state = 'ready';
$restore->error_code = '';
$restore->ready_at = $now;
}
$restore->updated_at = $now;
$restore->save();
}
private static function finishRecoveryLocked(CloudJob $job,
string $status, $result, array $payload): void
{
$recovery = $job->backup_recovery_id === null ? null
: CloudBackupRecovery::where(
'id', (int) $job->backup_recovery_id
)->lockForUpdate()->first();
$backup = $job->backup_id === null ? null
: CloudInstanceBackup::where('id', (int) $job->backup_id)
->lockForUpdate()->first();
$instance = $job->instance_id === null ? null
: CloudInstance::where('id', (int) $job->instance_id)
->lockForUpdate()->first();
$body = is_array($payload['body'] ?? null)
? $payload['body'] : [];
if ($recovery === null || $backup === null || $instance === null
|| (int) $recovery->job_id !== (int) $job->id
|| (int) $recovery->backup_id !== (int) $backup->id
|| (int) $recovery->instance_id !== (int) $instance->id
|| !hash_equals((string) $recovery->operation_id,
(string) $job->operation_id)
|| !hash_equals((string) $recovery->recovery_key,
(string) ($payload['recovery_key'] ?? ''))
|| !hash_equals((string) $backup->backup_key,
(string) ($payload['backup_key'] ?? ''))
|| !hash_equals((string) $backup->backup_key,
(string) ($body['backup_key'] ?? ''))
|| !hash_equals((string) $backup->sha256,
(string) ($body['source_sha256'] ?? ''))
|| (int) $backup->object_size_bytes
!== (int) ($body['source_size_bytes'] ?? 0)
|| (int) $instance->user_id
!== (int) ($payload['owner_id'] ?? 0)
|| (int) $job->user_id !== (int) $instance->user_id
|| (int) $job->host_id !== (int) $instance->host_id) {
throw new CloudRecoveryInstanceStateConflict(
'云备份恢复任务与实例、备份或恢复账本不一致',
409
);
}
if ($status === 'running') {
return;
}
$now = time();
if ($status === 'uncertain') {
$recovery->state = 'uncertain';
$recovery->error_code = 'operation_effect_unknown';
$recovery->completed_at = null;
} elseif ($status === 'failed') {
$recovery->state = 'failed';
$recovery->error_code = 'backup_recovery_failed';
$recovery->completed_at = $now;
} elseif ($status === 'succeeded') {
if (!is_array($result)
|| ($result['restored'] ?? null) !== true
|| !hash_equals((string) $backup->backup_key,
(string) ($result['backup_key'] ?? ''))
|| !hash_equals((string) $backup->sha256,
(string) ($result['sha256'] ?? ''))
|| CloudInstanceRuntimeService::normalizeState(
$result['power_state'] ?? null
) === 'unknown') {
throw new CloudError(
'Agent 恢复成功结果缺少可信的备份或电源身份',
502
);
}
$backupSpec = self::decodeSpec((string) $backup->spec_json);
if (($backupSpec['managed'] ?? null) !== true
|| ($backupSpec['os_source'] ?? null) !== 'cloud_template'
|| !is_string($backupSpec['os_name'] ?? null)
|| trim((string) $backupSpec['os_name']) === ''
|| ($backupSpec['login_name'] ?? null) !== 'root'
|| !is_bool($backupSpec['debian_12_network_fix'] ?? null)) {
throw new CloudRecoveryInstanceStateConflict(
'云备份的客户机身份快照无效',
409
);
}
$instanceSpec = self::decodeSpec((string) $instance->spec_json);
foreach ([
'template_id', 'template_revision',
'template_generation_token', 'os_source', 'os_name',
'login_name', 'debian_12_network_fix',
] as $field) {
if (array_key_exists($field, $backupSpec)) {
$instanceSpec[$field] = $backupSpec[$field];
} else {
unset($instanceSpec[$field]);
}
}
unset($instanceSpec['install_iso']);
// Passwords and hostnames may have been changed inside the guest
// without passing through the panel. The restored system disk is
// authoritative, so the panel must not claim either value is
// known or cause the Agent's NoCloud seed to inject an old value.
unset($instanceSpec['guest_hostname'], $instanceSpec['hostname']);
$instanceSpec['managed'] = true;
$instanceSpec['restored_from_backup'] =
(string) $backup->backup_key;
$instance->spec_json = json_encode(
CloudInput::safeResult($instanceSpec),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
$instance->sealed_credential = null;
$instance->updated_at = $now;
$instance->save();
$recovery->state = 'succeeded';
$recovery->error_code = '';
$recovery->completed_at = $now;
self::recordRuntime($instance, $result);
}
$recovery->updated_at = $now;
$recovery->save();
}
private static function priorRecoveryResult(CloudJob $job): array
{
$recovery = $job->backup_recovery_id === null ? null
: CloudBackupRecovery::find((int) $job->backup_recovery_id);
if ($recovery === null) {
throw new CloudError('云备份恢复任务目录记录缺失', 409);
}
return [
'recovery' => self::displayRecovery($recovery),
'job' => $job,
];
}
private static function displayRecovery(CloudBackupRecovery $recovery): array
{
return [
'recovery_key' => (string) $recovery->recovery_key,
'backup_id' => (string) $recovery->backup_id,
'instance_id' => (string) $recovery->instance_id,
'state' => (string) $recovery->state,
'created_at' => max(0, (int) $recovery->created_at),
'completed_at' => $recovery->completed_at === null ? null
: max(0, (int) $recovery->completed_at),
];
}
private static function assertManagedSource(CloudInstance $instance): void
{
$spec = self::decodeSpec((string) $instance->spec_json);
if (($spec['managed'] ?? false) !== true
|| ($spec['os_source'] ?? null) !== 'cloud_template') {
throw new CloudError('仅面板创建的云镜像实例支持可移植备份', 409);
}
self::resourcesFromSpec($spec);
}
private static function assertNoActiveJob(CloudInstance $instance): void
{
CloudInstanceMigrationService::assertInstanceMutationAvailable(
$instance
);
if (CloudJobService::blockingQueryForInstance(
(int) $instance->id
)->exists()) {
throw new CloudError('虚拟机已有任务,请等待完成', 409);
}
}
private static function assertBackupDeletionAvailable(
CloudInstanceBackup $backup
): void {
if ((string) $backup->purpose !== 'portable') {
throw new CloudError('该备份类型不能由此入口删除', 409);
}
if ((string) $backup->state === 'uncertain') {
throw new CloudError('备份效果尚未确认,不能删除', 409);
}
if ($backup->active_job_id !== null) {
throw new CloudError('备份仍有关联任务,不能删除', 409);
}
if (CloudBackupRestore::where('backup_id', (int) $backup->id)
->whereIn('state', ['restoring', 'uncertain'])->exists()) {
throw new CloudError('备份仍有恢复任务或未知效果,不能删除', 409);
}
if (CloudBackupRecovery::where('backup_id', (int) $backup->id)
->whereIn('state', ['recovering', 'uncertain'])->exists()) {
throw new CloudError('备份仍有原实例恢复任务或未知效果,不能删除', 409);
}
if (CloudJob::where('backup_id', (int) $backup->id)
->where(static function ($query): void {
$query->where('phase', '<>', 'terminal')
->orWhere('effect_resolution', 'unknown')
->orWhere('finalization_incident', 1);
})->exists()) {
throw new CloudError('备份仍有关联任务,不能删除', 409);
}
if (CloudDeploymentSnapshot::where('backup_id', (int) $backup->id)
->where('state', 'ready')->exists()) {
throw new CloudError('该对象已保存为云快照,请从云快照页面删除', 409);
}
}
/** @param array{size_bytes:int,etag:string,version_id:string} $identity */
private static function claimDeleteGeneration(int $backupId,
string $previousState, array $identity, int $now,
?callable $beforeDeleteLocked): array
{
$backup = CloudInstanceBackup::findOrFail($backupId);
return $backup->getConnection()->transaction(function () use (
$backupId, $previousState, $identity, $now, $beforeDeleteLocked
): array {
$locked = CloudInstanceBackup::where('id', $backupId)
->lockForUpdate()->firstOrFail();
if ((string) $locked->state === 'deleted') {
return ['mode' => 'deleted', 'backup' => $locked];
}
if (in_array((string) $locked->state,
self::CLEANUP_DEBT_STATES, true)) {
return ['mode' => 'waiting', 'backup' => $locked];
}
if ((string) $locked->state !== $previousState) {
throw new CloudError('备份删除状态已发生变化', 409);
}
self::assertBackupDeletionAvailable($locked);
if ($beforeDeleteLocked !== null) {
$beforeDeleteLocked($locked);
}
$locked->object_etag = (string) $identity['etag'];
$locked->object_version_id = (string) $identity['version_id'];
$locked->state = 'deleting';
$locked->delete_operation_id = bin2hex(random_bytes(16));
$locked->delete_generation = max(0,
(int) $locked->delete_generation) + 1;
$locked->delete_cycle_attempts = 1;
$locked->delete_presence_confirmations = 0;
$locked->delete_requested_at = $now;
$locked->delete_last_checked_at = null;
$locked->delete_next_check_at = 0;
$locked->delete_attention_at = null;
$locked->error_code = '';
$locked->updated_at = $now;
$locked->save();
return [
'mode' => 'delete',
'previous_state' => $previousState,
'backup' => $locked,
'runtime' => self::storageRuntime($locked),
'object_key' => (string) $locked->object_key,
'object_version_id' => (string) $locked->object_version_id,
];
}, 3);
}
private static function recordStableDeleteProbe(int $backupId,
int $now): array
{
$backup = CloudInstanceBackup::findOrFail($backupId);
return $backup->getConnection()->transaction(function () use (
$backupId, $now
): array {
$locked = CloudInstanceBackup::where('id', $backupId)
->lockForUpdate()->firstOrFail();
if ((string) $locked->state === 'deleted') {
return ['mode' => 'deleted', 'backup' => $locked];
}
if ((string) $locked->state === 'delete_attention') {
return ['mode' => 'attention', 'backup' => $locked];
}
if (!in_array((string) $locked->state,
['deleting', 'delete_uncertain'], true)) {
throw new CloudError('备份删除状态已发生变化', 409);
}
$last = max(0, (int) $locked->delete_last_checked_at);
if ($last > 0 && $now - $last < self::DELETE_STABLE_SECONDS) {
$locked->state = 'delete_uncertain';
$locked->delete_next_check_at = $last
+ self::DELETE_STABLE_SECONDS;
$locked->updated_at = $now;
$locked->save();
return ['mode' => 'waiting', 'backup' => $locked];
}
$confirmations = min(self::DELETE_STABLE_PROBES,
max(0, (int) $locked->delete_presence_confirmations) + 1);
$locked->state = 'delete_uncertain';
$locked->delete_presence_confirmations = $confirmations;
$locked->delete_last_checked_at = $now;
$locked->delete_next_check_at = $now
+ self::DELETE_STABLE_SECONDS;
$locked->error_code = 'object_delete_still_present';
$locked->updated_at = $now;
if ($confirmations < self::DELETE_STABLE_PROBES) {
$locked->save();
return ['mode' => 'waiting', 'backup' => $locked];
}
if ((int) $locked->delete_cycle_attempts
>= self::DELETE_MAX_AUTOMATIC_ATTEMPTS) {
$locked->state = 'delete_attention';
$locked->delete_attention_at = $now;
$locked->delete_next_check_at = 0;
$locked->error_code = 'object_delete_retry_exhausted';
$locked->save();
return ['mode' => 'attention', 'backup' => $locked];
}
$locked->state = 'deleting';
$locked->delete_operation_id = bin2hex(random_bytes(16));
$locked->delete_generation = max(0,
(int) $locked->delete_generation) + 1;
$locked->delete_cycle_attempts = max(0,
(int) $locked->delete_cycle_attempts) + 1;
$locked->delete_presence_confirmations = 0;
$locked->delete_next_check_at = 0;
$locked->error_code = '';
$locked->save();
return [
'mode' => 'delete',
'backup' => $locked,
'runtime' => self::storageRuntime($locked),
'object_key' => (string) $locked->object_key,
'object_version_id' => (string) $locked->object_version_id,
];
}, 3);
}
/** @param array{size_bytes:int,etag:string,version_id:string} $identity */
private static function assertObjectIdentity(CloudInstanceBackup $backup,
array $identity, bool $strict): void
{
if ((int) $identity['size_bytes'] !== (int) $backup->object_size_bytes) {
throw new CloudError('对象存储中的备份大小与目录账本不一致', 409);
}
if (!$strict) {
return;
}
$etag = (string) $backup->object_etag;
if ($etag === '' || !hash_equals($etag, (string) $identity['etag'])) {
throw new CloudError('对象存储中的备份 ETag 已发生变化', 409);
}
$versionId = (string) $backup->object_version_id;
if ($versionId !== ''
&& !hash_equals($versionId, (string) $identity['version_id'])) {
throw new CloudError('对象存储中的备份版本已发生变化', 409);
}
}
private static function markDeleteUncertain(int $backupId,
string $errorCode, int $now, bool $resetEvidence,
int $retryAfter = self::DELETE_STABLE_SECONDS): void
{
$values = [
'state' => 'delete_uncertain',
'delete_next_check_at' => $now + max(1, $retryAfter),
'error_code' => substr($errorCode, 0, 64),
'updated_at' => $now,
];
if ($resetEvidence) {
$values['delete_presence_confirmations'] = 0;
$values['delete_last_checked_at'] = null;
}
CloudInstanceBackup::where('id', $backupId)
->whereIn('state', ['deleting', 'delete_uncertain'])
->update($values);
}
private static function markDeleteAttention(int $backupId,
string $errorCode, int $now): void
{
CloudInstanceBackup::where('id', $backupId)
->whereIn('state', ['deleting', 'delete_uncertain'])
->update([
'state' => 'delete_attention',
'delete_next_check_at' => 0,
'delete_attention_at' => $now,
'error_code' => substr($errorCode, 0, 64),
'updated_at' => $now,
]);
}
private static function restoreAfterRejectedDelete(int $backupId,
string $previousState, string $errorCode): void
{
CloudInstanceBackup::where('id', $backupId)
->where('state', 'deleting')
->update([
'state' => $previousState,
'delete_operation_id' => null,
'delete_cycle_attempts' => 0,
'delete_presence_confirmations' => 0,
'delete_requested_at' => null,
'delete_last_checked_at' => null,
'delete_next_check_at' => 0,
'delete_attention_at' => null,
'error_code' => substr($errorCode, 0, 64),
'updated_at' => time(),
]);
}
private static function finishDelete(int $backupId): array
{
$backup = CloudInstanceBackup::findOrFail($backupId);
return $backup->getConnection()->transaction(function () use (
$backupId
): array {
$locked = CloudInstanceBackup::where('id', $backupId)
->lockForUpdate()->firstOrFail();
if ((string) $locked->state !== 'deleted') {
self::assertBackupDeletionAvailable($locked);
if (!in_array((string) $locked->state,
['ready', 'failed', 'deleting', 'delete_uncertain',
'delete_attention'], true)) {
throw new CloudError('备份删除状态已发生变化', 409);
}
$locked->state = 'deleted';
$locked->sealed_storage_runtime = '';
$locked->sealed_credential = null;
$locked->active_job_id = null;
$locked->delete_presence_confirmations = 0;
$locked->delete_next_check_at = 0;
$locked->delete_attention_at = null;
$locked->error_code = '';
$locked->updated_at = time();
$locked->save();
CloudDeploymentSnapshot::where('backup_id', $backupId)
->whereIn('state', ['deleting', 'delete_uncertain'])
->update([
'state' => 'deleted',
'deleted_at' => time(),
'error_code' => '',
'updated_at' => time(),
]);
}
return self::deletionResult($locked);
}, 3);
}
private static function deletionResult(CloudInstanceBackup $backup): array
{
$state = (string) $backup->state;
return [
'backup_id' => (string) $backup->id,
'state' => $state,
'deleted' => $state === 'deleted',
'requires_reconciliation' => in_array($state,
['deleting', 'delete_uncertain'], true),
'requires_attention' => $state === 'delete_attention',
'next_check_at' => max(0, (int) $backup->delete_next_check_at),
];
}
private static function priorJob(array $actor, string $kind,
?int $instanceId, ?int $backupId, string $fingerprint,
bool $lock = true): ?CloudJob
{
$key = $actor['request_idempotency_key'] ?? null;
if (!is_string($key) || $key === '') {
return null;
}
$query = CloudJob::where('user_id', (int) $actor['user_id'])
->where('request_idempotency_key', $key);
if ($lock) {
$query->lockForUpdate();
}
$job = $query->first();
if ($job === null) {
return null;
}
if ((string) $job->kind !== $kind
|| !hash_equals((string) $job->request_fingerprint,
$fingerprint)
|| ($instanceId !== null
&& (int) $job->instance_id !== $instanceId)
|| ($backupId !== null && (int) $job->backup_id !== $backupId)) {
throw new CloudError('同一请求标识不能用于不同的备份操作', 409);
}
return $job;
}
private static function resourcesFromSpec(array $spec): array
{
$cpu = filter_var($spec['vcpu'] ?? $spec['cpu_cores'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$memory = filter_var($spec['memory_mib'] ?? $spec['ram_mib'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$disk = filter_var($spec['disk_gib'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$bandwidth = filter_var($spec['bandwidth_mbps'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
if ($cpu === false || $memory === false || $disk === false
|| $bandwidth === false) {
throw new CloudError('实例缺少可信的资源规格快照', 409);
}
return ['cpu_cores' => (int) $cpu, 'memory_mib' => (int) $memory,
'disk_gib' => (int) $disk, 'bandwidth_mbps' => (int) $bandwidth];
}
/** Freeze template-specific boot compatibility into the backup manifest. */
private static function freezeRestoreCompatibility(array $spec): array
{
if (!is_bool($spec['debian_12_network_fix'] ?? null)) {
throw new CloudError(
'实例缺少创建时冻结的网络兼容快照,不能生成可移植备份',
409
);
}
return $spec;
}
private static function restoredSpec(array $source,
CloudInstanceBackup $backup, array $network, array $resources): array
{
$spec = CloudInput::safeResult($source);
foreach (['plan_id', 'offer_id', 'price_version', 'stock_mode',
'validity_days', 'traffic_quota_bytes', 'renewal_price',
'currency', 'public_ip', 'public_ip_cidr'] as $field) {
unset($spec[$field]);
}
return array_replace($spec, [
'managed' => false,
'vcpu' => $resources['cpu_cores'],
'cpu_cores' => $resources['cpu_cores'],
'memory_mib' => $resources['memory_mib'],
'disk_gib' => $resources['disk_gib'],
'bandwidth_mbps' => $resources['bandwidth_mbps'],
'network' => 'bridge:br0',
'public_pool_key' => (string) $network['public_pool_key'],
'public_pool_cidr' => (string) $network['public_pool_cidr'],
'public_ip_cidr' => (string) $network['public_ip_cidr'],
'gateway' => (string) $network['gateway'],
'dns_servers' => array_values($network['dns_servers']),
'resource_pool_id' => (int) $network['resource_pool_id'],
'resource_pool_name' => (string) $network['resource_pool_name'],
'line_id' => (int) $network['line_id'],
'line_name' => (string) $network['line_name'],
'region_id' => (int) $network['region_id'],
'region' => (string) $network['region_name'],
'country_code' => (string) $network['country_code'],
'restored_from_backup' => (string) $backup->backup_key,
]);
}
private static function decodeSpec(string $encoded,
bool $throw = true): ?array
{
$spec = json_decode($encoded, true);
if (!is_array($spec) || array_is_list($spec)) {
if ($throw) {
throw new CloudError('实例备份规格快照损坏', 409);
}
return null;
}
return $spec;
}
private static function storageRuntime(CloudInstanceBackup $backup): array
{
$runtime = CloudSecrets::decrypt(
(string) $backup->sealed_storage_runtime,
self::STORAGE_PURPOSE
);
if (!is_array($runtime['config'] ?? null)
|| !is_array($runtime['credentials'] ?? null)
|| (int) ($runtime['revision'] ?? -1)
!== (int) $backup->storage_revision) {
throw new CloudError('备份对象存储快照无效', 503);
}
return $runtime;
}
private static function publicTransferPlan(array $plan): array
{
return [
'part_size_bytes' => (int) $plan['part_size_bytes'],
'maximum_size_bytes' => (int) $plan['maximum_size_bytes'],
'part_urls' => array_values($plan['part_urls']),
'complete_url' => (string) $plan['complete_url'],
'abort_url' => (string) $plan['abort_url'],
'verify_url' => (string) $plan['verify_url'],
];
}
private static function maximumObjectBytes(int $diskGib): int
{
if ($diskGib < 1 || $diskGib > 2559) {
throw new CloudError('实例系统盘超出当前备份最大容量', 409);
}
return ($diskGib + 1) * self::GIB;
}
private static function displayBackup(CloudInstanceBackup $backup,
?CloudJob $job, bool $canManage = true): array
{
$state = in_array((string) $backup->state,
['creating', 'ready', 'failed', 'uncertain', 'deleting',
'delete_uncertain', 'delete_attention', 'deleted'], true)
? (string) $backup->state : 'uncertain';
$canDelete = $canManage && in_array($state,
['ready', 'failed', 'deleting', 'delete_uncertain'], true);
if ($canDelete) {
try {
self::assertBackupDeletionAvailable($backup);
} catch (CloudError $error) {
$canDelete = false;
}
}
return [
'id' => (string) $backup->id,
'backup_key' => (string) $backup->backup_key,
'name' => (string) $backup->name,
'state' => $state,
'consistency' => in_array((string) $backup->consistency,
['filesystem', 'crash'], true)
? (string) $backup->consistency : 'unknown',
'object_size_bytes' => max(0, (int) $backup->object_size_bytes),
'virtual_size_bytes' => max(0, (int) $backup->virtual_size_bytes),
'sha256' => preg_match('/^[a-f0-9]{64}$/D',
(string) $backup->sha256) === 1
? (string) $backup->sha256 : '',
'source_host_id' => (string) $backup->source_host_id,
'created_at' => max(0, (int) $backup->created_at),
'ready_at' => $backup->ready_at === null ? null
: max(0, (int) $backup->ready_at),
'updated_at' => max(0, (int) $backup->updated_at),
'task_id' => $job === null ? null : (string) $job->id,
'job_status' => $job === null ? null
: (CloudOperationErrorCatalog::userStatus(
(string) $job->status
) ?? 'unknown'),
'cleanup_pending' => (string) $backup->error_code
=== 'backup_local_cleanup_pending',
'can_cleanup_workspace' => $canManage
&& (string) $backup->error_code
=== 'backup_local_cleanup_pending',
'can_delete' => $canDelete,
'delete_action' => in_array($state,
['deleting', 'delete_uncertain'], true)
? 'reconcile' : 'delete',
'error' => $state === 'delete_attention'
? '对象自动清理已达到三次上限,需要管理员检查对象存储后手动恢复清理。'
: ($state === 'delete_uncertain'
? '删除结果尚未确认;系统会在两次稳定核对后创建新的删除代次。'
: null),
];
}
private static function displayRestore(CloudBackupRestore $restore): array
{
return [
'restore_key' => (string) $restore->restore_key,
'state' => (string) $restore->state,
'target_host_id' => (string) $restore->target_host_id,
'target_instance_id' => $restore->target_instance_id === null
? null : (string) $restore->target_instance_id,
'name' => (string) $restore->new_name,
'created_at' => max(0, (int) $restore->created_at),
'ready_at' => $restore->ready_at === null ? null
: max(0, (int) $restore->ready_at),
];
}
private static function uuidV4(): string
{
$bytes = random_bytes(16);
$bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40);
$bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80);
$hex = bin2hex($bytes);
return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-'
. substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-'
. substr($hex, 20, 12);
}
private static function libraryFilters(array $input): array
{
CloudInput::only($input, ['page', 'per_page', 'q', 'user_id',
'instance_id', 'host_id', 'state']);
foreach ($input as $value) {
if (!is_string($value) && !is_int($value)) {
throw new CloudError('备份库筛选参数无效', 400);
}
}
$filters = [];
foreach (['user_id', 'instance_id', 'host_id'] as $key) {
$value = (string) ($input[$key] ?? '');
if ($value !== '' && (preg_match('/^[1-9][0-9]{0,17}$/D', $value) !== 1)) {
throw new CloudError('备份库 ' . $key . ' 必须为有效的正整数', 400);
}
$filters[$key] = $value;
}
$state = (string) ($input['state'] ?? '');
if (!in_array($state, ['', 'creating', 'ready', 'restoring', 'failed',
'uncertain', 'deleting', 'delete_uncertain', 'delete_attention'], true)) {
throw new CloudError('备份库状态筛选无效', 400);
}
$q = trim((string) ($input['q'] ?? ''));
if (preg_match('//u', $q) !== 1 || mb_strlen($q, 'UTF-8') > 80
|| preg_match('/[\x00-\x1f\x7f]/u', $q) !== 0) {
throw new CloudError('备份名称关键词最多 80 个字符,且不能包含控制字符', 400);
}
return $filters + ['state' => $state, 'q' => $q];
}
private static function libraryInteger($value, int $minimum, int $maximum,
int $default): int
{
if ($value === null || $value === '') {
return $default;
}
if (is_int($value)) {
$parsed = $value;
} elseif (is_string($value)
&& preg_match('/^(?:0|[1-9][0-9]{0,9})$/D', $value) === 1) {
$parsed = (int) $value;
} else {
throw new CloudError('备份库分页参数无效', 400);
}
if ($parsed < $minimum || $parsed > $maximum) {
throw new CloudError('备份库分页参数超出允许范围', 400);
}
return $parsed;
}
private static function recordRuntime(CloudInstance $instance,
array $result): void
{
try {
CloudInstanceRuntimeService::seed(
(int) $instance->id, (int) $instance->host_id,
(string) $instance->uuid
);
$state = $result['status']
?? ($result['state'] ?? ($result['power_state'] ?? null));
if (CloudInstanceRuntimeService::normalizeState($state)
!== 'unknown') {
CloudInstanceRuntimeService::markObserved(
(int) $instance->id, (int) $instance->host_id,
(string) $instance->uuid, $state
);
}
} catch (\Throwable $error) {
error_log('cloud backup restore runtime cache unavailable: '
. get_class($error));
}
}
}