You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2416 lines
106 KiB
PHP
2416 lines
106 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudBackupRecovery;
|
|
use App\Models\CloudBackupPeriodEntitlement;
|
|
use App\Models\CloudBackupSubscription;
|
|
use App\Models\CloudDeploymentSnapshot;
|
|
use App\Models\CloudHost;
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Models\CloudInstanceMigration;
|
|
use App\Models\CloudJob;
|
|
|
|
/** PHP-owned catalog and resource ledger for portable system-disk backups. */
|
|
final class CloudBackupService
|
|
{
|
|
public const MAX_PER_INSTANCE = 10;
|
|
|
|
private const STORAGE_PURPOSE = 'object-storage-backup-runtime';
|
|
private const ACTIVE_BACKUP_STATES = [
|
|
'creating', 'ready', 'uncertain',
|
|
];
|
|
private const CLEANUP_DEBT_STATES = [
|
|
'deleting', 'delete_uncertain', 'delete_attention',
|
|
];
|
|
private const DELETE_STABLE_PROBES = 2;
|
|
private const DELETE_STABLE_SECONDS = 60;
|
|
private const DELETE_RETRY_SECONDS = 300;
|
|
private const DELETE_MAX_AUTOMATIC_ATTEMPTS = 3;
|
|
private const TRANSFER_URL_SECONDS = 604800;
|
|
private const MAX_AGENT_BODY_BYTES = 1048576;
|
|
// The create transaction opens a remote S3 multipart upload. Database
|
|
// deadlock handling must never replay that external side effect.
|
|
private const CREATE_TRANSACTION_ATTEMPTS = 1;
|
|
private const GIB = 1073741824;
|
|
|
|
/**
|
|
* Global administrator backup catalog. It is intentionally keyed by the
|
|
* backup ledger instead of the source VM lifecycle, so a retained backup
|
|
* remains restorable and deletable after expiry cleanup tombstones the
|
|
* source instance.
|
|
*/
|
|
public static function library(array $actor, array $input): array
|
|
{
|
|
if (($actor['is_admin'] ?? false) !== true) {
|
|
throw new CloudError('只有管理员可以查看虚拟机备份', 403);
|
|
}
|
|
$filters = self::libraryFilters($input);
|
|
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
|
|
$perPage = self::libraryInteger($input['per_page'] ?? 20,
|
|
20, 50, 20);
|
|
if (!in_array($perPage, [20, 50], true)) {
|
|
throw new CloudError('备份库每页条数仅支持 20 或 50', 400);
|
|
}
|
|
$query = CloudInstanceBackup::where('purpose', 'portable')
|
|
->where('state', '<>', 'deleted');
|
|
foreach (['user_id' => 'user_id', 'instance_id' => 'source_instance_id',
|
|
'host_id' => 'source_host_id', 'state' => 'state'] as $key => $column) {
|
|
if ($filters[$key] !== '') {
|
|
$query->where($column, $filters[$key]);
|
|
}
|
|
}
|
|
if ($filters['q'] !== '') {
|
|
$like = '%' . str_replace(['!', '%', '_'], ['!!', '!%', '!_'], $filters['q']) . '%';
|
|
$query->whereRaw("name LIKE ? ESCAPE '!'", [$like]);
|
|
}
|
|
$total = (int) (clone $query)->count('id');
|
|
$totalPages = max(1, (int) ceil($total / $perPage));
|
|
$page = min($page, $totalPages);
|
|
$rows = $query->orderByDesc('id')
|
|
->offset(($page - 1) * $perPage)->limit($perPage)->get();
|
|
|
|
$sourceIds = [];
|
|
$hostIds = [];
|
|
$operationIds = [];
|
|
foreach ($rows as $row) {
|
|
$sourceIds[] = (int) $row->source_instance_id;
|
|
$hostIds[] = (int) $row->source_host_id;
|
|
if (preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) $row->last_operation_id) === 1) {
|
|
$operationIds[] = (string) $row->last_operation_id;
|
|
}
|
|
}
|
|
$sources = [];
|
|
if ($sourceIds !== []) {
|
|
foreach (CloudInstance::whereIn('id', array_values(array_unique(
|
|
$sourceIds
|
|
)))->get(['id', 'name', 'status', 'user_id']) as $source) {
|
|
$sources[(int) $source->id] = $source;
|
|
}
|
|
}
|
|
$hosts = [];
|
|
if ($hostIds !== []) {
|
|
foreach (CloudHost::whereIn('id', array_values(array_unique(
|
|
$hostIds
|
|
)))->get(['id', 'name', 'region']) as $host) {
|
|
$hosts[(int) $host->id] = $host;
|
|
}
|
|
}
|
|
$jobs = [];
|
|
if ($operationIds !== []) {
|
|
foreach (CloudJob::whereIn('operation_id', array_values(array_unique(
|
|
$operationIds
|
|
)))->get() as $job) {
|
|
$jobs[(string) $job->operation_id] = $job;
|
|
}
|
|
}
|
|
|
|
$canManage = ($actor['is_super_admin'] ?? false) === true
|
|
&& ($actor['can_manage'] ?? false) === true;
|
|
$items = [];
|
|
foreach ($rows as $row) {
|
|
$source = $sources[(int) $row->source_instance_id] ?? null;
|
|
$host = $hosts[(int) $row->source_host_id] ?? null;
|
|
$item = self::displayBackup(
|
|
$row,
|
|
$jobs[(string) $row->last_operation_id] ?? null,
|
|
$canManage
|
|
);
|
|
$item['source'] = [
|
|
'instance_id' => (string) $row->source_instance_id,
|
|
'name' => $source === null ? '' : (string) $source->name,
|
|
'status' => $source === null ? 'missing'
|
|
: (string) $source->status,
|
|
'deleted' => $source === null
|
|
|| (string) $source->status === 'deleted',
|
|
'user_id' => (string) $row->user_id,
|
|
'host_id' => (string) $row->source_host_id,
|
|
'host_name' => $host === null ? '' : (string) $host->name,
|
|
'region' => $host === null ? '' : (string) $host->region,
|
|
];
|
|
if ((string) $row->state === 'ready') {
|
|
$resources = self::resourcesFromSpec(
|
|
self::decodeSpec((string) $row->spec_json)
|
|
);
|
|
$item['restore_targets'] = array_map(
|
|
static fn (array $target): array => [
|
|
'host_id' => (string) ($target['id'] ?? ''),
|
|
'host_name' => (string) ($target['name'] ?? ''),
|
|
'region' => (string) ($target['region'] ?? ''),
|
|
'resource_pool' => (string) (
|
|
$target['resource_pool'] ?? ''
|
|
),
|
|
'network' => 'bridge:br0',
|
|
'available_ip_count' => max(1,
|
|
(int) ($target['available_ip_count'] ?? 1)),
|
|
'is_source_host' =>
|
|
($target['is_source_host'] ?? false) === true,
|
|
],
|
|
CloudPlacementService::backupRestoreTargets(
|
|
(int) $row->source_host_id,
|
|
$resources
|
|
)
|
|
);
|
|
}
|
|
$items[] = $item;
|
|
}
|
|
$from = $total === 0 ? 0 : (($page - 1) * $perPage) + 1;
|
|
return [
|
|
'items' => $items,
|
|
'filters' => $filters,
|
|
'can_manage' => $canManage,
|
|
'pagination' => [
|
|
'page' => $page,
|
|
'per_page' => $perPage,
|
|
'total' => $total,
|
|
'total_pages' => $totalPages,
|
|
'from' => $from,
|
|
'to' => $from === 0 ? 0 : $from + count($items) - 1,
|
|
],
|
|
];
|
|
}
|
|
|
|
public static function listing(CloudInstance $instance, array $actor): array
|
|
{
|
|
if (($actor['is_admin'] ?? false) !== true) {
|
|
throw new CloudError('只有管理员可以查看虚拟机备份', 403);
|
|
}
|
|
$canManage = ($actor['is_super_admin'] ?? false) === true
|
|
&& ($actor['can_manage'] ?? false) === true;
|
|
if ((string) $instance->status === 'deleted') {
|
|
throw new CloudError('虚拟机不存在', 404);
|
|
}
|
|
$rows = CloudInstanceBackup::where(
|
|
'source_instance_id', (int) $instance->id
|
|
)->where('purpose', 'portable')->where('state', '<>', 'deleted')
|
|
->orderByDesc('id')->get();
|
|
$operationIds = [];
|
|
foreach ($rows as $row) {
|
|
if (preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) $row->last_operation_id) === 1) {
|
|
$operationIds[] = (string) $row->last_operation_id;
|
|
}
|
|
}
|
|
$jobs = [];
|
|
if ($operationIds !== []) {
|
|
foreach (CloudJob::whereIn('operation_id',
|
|
array_values(array_unique($operationIds)))->get() as $job) {
|
|
$jobs[(string) $job->operation_id] = $job;
|
|
}
|
|
}
|
|
$backups = [];
|
|
$targetMap = [];
|
|
$manualActiveCount = 0;
|
|
foreach ($rows as $row) {
|
|
if ((string) $row->source === 'manual'
|
|
&& in_array((string) $row->state,
|
|
self::ACTIVE_BACKUP_STATES, true)) {
|
|
$manualActiveCount++;
|
|
}
|
|
$display = self::displayBackup(
|
|
$row,
|
|
$jobs[(string) $row->last_operation_id] ?? null,
|
|
$canManage
|
|
);
|
|
if ((string) $row->state === 'ready') {
|
|
$resources = self::resourcesFromSpec(
|
|
self::decodeSpec((string) $row->spec_json)
|
|
);
|
|
$backupTargets = [];
|
|
foreach (CloudPlacementService::backupRestoreTargets(
|
|
(int) $row->source_host_id,
|
|
$resources
|
|
) as $target) {
|
|
$normalized = [
|
|
'host_id' => (string) ($target['id'] ?? ''),
|
|
'host_name' => (string) ($target['name'] ?? ''),
|
|
'region' => (string) ($target['region'] ?? ''),
|
|
'resource_pool' => (string) (
|
|
$target['resource_pool'] ?? ''
|
|
),
|
|
'network' => 'bridge:br0',
|
|
'available_ip_count' => max(
|
|
1,
|
|
(int) ($target['available_ip_count'] ?? 1)
|
|
),
|
|
'is_source_host' =>
|
|
($target['is_source_host'] ?? false) === true,
|
|
];
|
|
$backupTargets[] = $normalized;
|
|
$targetMap[$normalized['host_id']] = $normalized;
|
|
}
|
|
$display['restore_targets'] = $backupTargets;
|
|
}
|
|
$backups[] = $display;
|
|
}
|
|
$active = CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->orderBy('id')->first();
|
|
$spec = self::decodeSpec((string) $instance->spec_json, false);
|
|
$compatibilityFrozen = $spec !== null
|
|
&& is_bool($spec['debian_12_network_fix'] ?? null);
|
|
$supported = $spec !== null
|
|
&& ($spec['managed'] ?? false) === true
|
|
&& ($spec['os_source'] ?? null) === 'cloud_template'
|
|
&& $compatibilityFrozen;
|
|
$storage = CloudObjectStorageConfig::view();
|
|
$migration = CloudInstanceMigration::where(
|
|
'instance_id', (int) $instance->id
|
|
)->orderByDesc('id')->first();
|
|
$migrationTargets = $supported ? array_map(
|
|
static fn (array $target): array => [
|
|
'host_id' => (string) ($target['id'] ?? ''),
|
|
'host_name' => (string) ($target['name'] ?? ''),
|
|
'region' => (string) ($target['region'] ?? ''),
|
|
'resource_pool' => (string) ($target['resource_pool'] ?? ''),
|
|
'available_ip_count' => max(
|
|
1,
|
|
(int) ($target['available_ip_count'] ?? 1)
|
|
),
|
|
],
|
|
CloudPlacementService::maintenanceMigrationTargets(
|
|
$instance,
|
|
self::resourcesFromSpec($spec),
|
|
CloudPeerMigrationArtifactService::advisoryAdditionalDiskGib(
|
|
$instance
|
|
)
|
|
)
|
|
) : [];
|
|
$migrationBlocks = $migration !== null
|
|
&& CloudInstanceMigrationWorkflow::blocksInstance(
|
|
(string) $migration->state
|
|
);
|
|
$canOperate = $canManage
|
|
&& $active === null
|
|
&& !$migrationBlocks
|
|
&& (string) $instance->status === 'active'
|
|
&& $supported
|
|
&& (bool) ($storage['ready'] ?? false);
|
|
$canMigrate = $canManage
|
|
&& $active === null
|
|
&& !($migration !== null
|
|
&& CloudInstanceMigrationWorkflow::blocksNewMigration(
|
|
(string) $migration->state
|
|
))
|
|
&& (string) $instance->status === 'active'
|
|
&& $supported
|
|
&& $migrationTargets !== [];
|
|
$canCreate = $canOperate
|
|
&& $manualActiveCount < self::MAX_PER_INSTANCE;
|
|
$reason = null;
|
|
if (!$canManage) {
|
|
$reason = '当前管理员仅可查看,超级管理员才可以创建或恢复备份';
|
|
} elseif (!(bool) ($storage['ready'] ?? false)) {
|
|
$reason = '请先在对象存储页面保存配置并通过连接验证';
|
|
} elseif ($spec !== null
|
|
&& ($spec['managed'] ?? false) === true
|
|
&& ($spec['os_source'] ?? null) === 'cloud_template'
|
|
&& !$compatibilityFrozen) {
|
|
$reason = '实例缺少创建时冻结的网络兼容快照,不能生成可移植备份';
|
|
} elseif (!$supported) {
|
|
$reason = '仅面板创建的云镜像实例支持可移植备份';
|
|
} elseif ($active !== null) {
|
|
$reason = '实例已有任务,请等待完成';
|
|
} elseif ($migrationBlocks) {
|
|
$reason = '实例已有未完成的维护迁移,请先完成核对或源副本清理';
|
|
} elseif ((string) $instance->status !== 'active') {
|
|
$reason = '只有活动实例可以创建备份';
|
|
} elseif (!$canCreate) {
|
|
$reason = '每台实例最多保留十个有效备份';
|
|
}
|
|
return [
|
|
'instance_id' => (string) $instance->id,
|
|
'backups' => $backups,
|
|
'restore_targets' => array_values($targetMap),
|
|
'migration_targets' => $migrationTargets,
|
|
'maintenance_migration' => $migration === null ? null
|
|
: CloudInstanceMigrationService::display($migration),
|
|
'max_backups' => self::MAX_PER_INSTANCE,
|
|
'can_create' => $canCreate,
|
|
'can_restore' => $canManage && $targetMap !== [],
|
|
'can_migrate' => $canMigrate,
|
|
'create_block_reason' => $reason,
|
|
'active_job' => $active === null ? null : [
|
|
'id' => (string) $active->id,
|
|
'kind' => (string) $active->kind,
|
|
'status' => CloudOperationErrorCatalog::userStatus(
|
|
(string) $active->status
|
|
) ?? 'unknown',
|
|
],
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Delete or safely reconcile one frozen S3 backup object.
|
|
*
|
|
* An ambiguous DELETE is never replayed. Two spaced HEAD observations of
|
|
* the same immutable identity may authorize a fresh deletion generation;
|
|
* at most three generations run automatically in one supervised cycle.
|
|
*/
|
|
public static function delete(CloudInstanceBackup $backup, array $actor,
|
|
?CloudObjectStorageS3Client $client = null,
|
|
?callable $beforeDeleteLocked = null): array
|
|
{
|
|
CloudAccess::manage($actor);
|
|
$client ??= new CloudObjectStorageS3Client();
|
|
$now = time();
|
|
$prepared = $backup->getConnection()->transaction(function () use (
|
|
$backup, $beforeDeleteLocked, $now
|
|
): array {
|
|
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
->lockForUpdate()->first();
|
|
if ($locked === null) {
|
|
throw new CloudError('备份不存在', 404);
|
|
}
|
|
$state = (string) $locked->state;
|
|
if ($state === 'deleted') {
|
|
return ['mode' => 'deleted', 'backup' => $locked];
|
|
}
|
|
self::assertBackupDeletionAvailable($locked);
|
|
if ($state === 'delete_attention') {
|
|
return ['mode' => 'attention', 'backup' => $locked];
|
|
}
|
|
if (in_array($state, ['deleting', 'delete_uncertain'], true)) {
|
|
if ((int) $locked->delete_next_check_at > $now) {
|
|
return ['mode' => 'waiting', 'backup' => $locked];
|
|
}
|
|
return [
|
|
'mode' => 'reconcile',
|
|
'backup' => $locked,
|
|
'runtime' => self::storageRuntime($locked),
|
|
'object_key' => (string) $locked->object_key,
|
|
'object_etag' => (string) $locked->object_etag,
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
];
|
|
}
|
|
if (!in_array($state, ['ready', 'failed'], true)) {
|
|
throw new CloudError('当前备份状态不允许删除', 409);
|
|
}
|
|
// Decrypt before any remote request. The exact immutable identity
|
|
// is captured by HEAD before crossing the deleting boundary.
|
|
$runtime = self::storageRuntime($locked);
|
|
return [
|
|
'mode' => 'identify',
|
|
'previous_state' => $state,
|
|
'backup' => $locked,
|
|
'runtime' => $runtime,
|
|
'object_key' => (string) $locked->object_key,
|
|
'object_etag' => (string) $locked->object_etag,
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
];
|
|
}, 3);
|
|
|
|
if (in_array($prepared['mode'], ['deleted', 'waiting', 'attention'], true)) {
|
|
return self::deletionResult($prepared['backup']);
|
|
}
|
|
$runtime = $prepared['runtime'];
|
|
$objectKey = $prepared['object_key'];
|
|
if ($prepared['mode'] === 'identify') {
|
|
try {
|
|
$identity = $client->inspectObject(
|
|
$runtime['config'], $runtime['credentials'], $objectKey,
|
|
(string) $prepared['object_version_id']
|
|
);
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
throw new CloudError($error->getMessage(), 502);
|
|
}
|
|
if ($identity === null) {
|
|
return self::finishDelete((int) $backup->id);
|
|
}
|
|
self::assertObjectIdentity(
|
|
$prepared['backup'], $identity,
|
|
(string) $prepared['object_etag'] !== ''
|
|
);
|
|
$prepared = self::claimDeleteGeneration(
|
|
(int) $backup->id,
|
|
(string) $prepared['previous_state'],
|
|
$identity,
|
|
time(),
|
|
$beforeDeleteLocked
|
|
);
|
|
if ($prepared['mode'] !== 'delete') {
|
|
return self::deletionResult($prepared['backup']);
|
|
}
|
|
}
|
|
if ($prepared['mode'] === 'reconcile') {
|
|
try {
|
|
$identity = $client->inspectObject(
|
|
$runtime['config'], $runtime['credentials'], $objectKey,
|
|
(string) $prepared['object_version_id']
|
|
);
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
self::markDeleteUncertain(
|
|
(int) $backup->id, $error->publicCode, time(), true,
|
|
self::DELETE_RETRY_SECONDS
|
|
);
|
|
return self::deletionResult(
|
|
CloudInstanceBackup::findOrFail((int) $backup->id)
|
|
);
|
|
}
|
|
if ($identity === null) {
|
|
return self::finishDelete((int) $backup->id);
|
|
}
|
|
try {
|
|
self::assertObjectIdentity($prepared['backup'], $identity, true);
|
|
} catch (CloudError $error) {
|
|
self::markDeleteAttention(
|
|
(int) $backup->id, 'object_identity_changed', time()
|
|
);
|
|
return self::deletionResult(
|
|
CloudInstanceBackup::findOrFail((int) $backup->id)
|
|
);
|
|
}
|
|
$prepared = self::recordStableDeleteProbe(
|
|
(int) $backup->id, time()
|
|
);
|
|
if ($prepared['mode'] !== 'delete') {
|
|
return self::deletionResult($prepared['backup']);
|
|
}
|
|
$runtime = $prepared['runtime'];
|
|
$objectKey = $prepared['object_key'];
|
|
}
|
|
|
|
try {
|
|
$client->deleteObject(
|
|
$runtime['config'], $runtime['credentials'], $objectKey,
|
|
(string) $prepared['object_version_id']
|
|
);
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
if ($error->effectMayHaveApplied) {
|
|
self::markDeleteUncertain(
|
|
(int) $backup->id, $error->publicCode, time(), true
|
|
);
|
|
return self::deletionResult(
|
|
CloudInstanceBackup::findOrFail((int) $backup->id)
|
|
);
|
|
}
|
|
if (isset($prepared['previous_state'])) {
|
|
self::restoreAfterRejectedDelete(
|
|
(int) $backup->id,
|
|
(string) $prepared['previous_state'],
|
|
$error->publicCode
|
|
);
|
|
} else {
|
|
self::markDeleteAttention(
|
|
(int) $backup->id, $error->publicCode, time()
|
|
);
|
|
}
|
|
throw new CloudError($error->getMessage(), 502);
|
|
}
|
|
return self::finishDelete((int) $backup->id);
|
|
}
|
|
|
|
/** Sanitized administrator view of remote objects awaiting cleanup. */
|
|
public static function cleanupDebt(array $actor, array $input = []): array
|
|
{
|
|
if (($actor['is_admin'] ?? false) !== true) {
|
|
throw new CloudError('只有管理员可以查看对象清理异常', 403);
|
|
}
|
|
CloudInput::only($input, ['page', 'per_page']);
|
|
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
|
|
$perPage = self::libraryInteger($input['per_page'] ?? 20, 10, 50, 20);
|
|
$query = CloudInstanceBackup::whereIn('state', self::CLEANUP_DEBT_STATES);
|
|
$total = (int) (clone $query)->count('id');
|
|
$bytes = (int) (clone $query)->sum('object_size_bytes');
|
|
$attention = (int) (clone $query)
|
|
->where('state', 'delete_attention')->count('id');
|
|
$totalPages = max(1, (int) ceil($total / $perPage));
|
|
$page = min($page, $totalPages);
|
|
$rows = $query->orderByRaw(
|
|
"CASE WHEN state = 'delete_attention' THEN 0 ELSE 1 END"
|
|
)->orderBy('delete_next_check_at')->orderBy('id')
|
|
->offset(($page - 1) * $perPage)->limit($perPage)->get();
|
|
$items = [];
|
|
foreach ($rows as $row) {
|
|
$items[] = [
|
|
'id' => (string) $row->id,
|
|
'name' => (string) $row->name,
|
|
'source_instance_id' => (string) $row->source_instance_id,
|
|
'user_id' => (string) $row->user_id,
|
|
'state' => (string) $row->state,
|
|
'object_size_bytes' => max(0, (int) $row->object_size_bytes),
|
|
'delete_generation' => max(0, (int) $row->delete_generation),
|
|
'cycle_attempts' => max(0, (int) $row->delete_cycle_attempts),
|
|
'next_check_at' => max(0, (int) $row->delete_next_check_at),
|
|
'attention_at' => $row->delete_attention_at === null ? null
|
|
: max(0, (int) $row->delete_attention_at),
|
|
'error_code' => (string) $row->error_code,
|
|
'can_retry' => (string) $row->state === 'delete_attention'
|
|
&& ($actor['is_super_admin'] ?? false) === true
|
|
&& ($actor['can_manage'] ?? false) === true,
|
|
];
|
|
}
|
|
return [
|
|
'summary' => [
|
|
'count' => $total,
|
|
'bytes' => $bytes,
|
|
'attention_count' => $attention,
|
|
],
|
|
'items' => $items,
|
|
'pagination' => [
|
|
'page' => $page,
|
|
'per_page' => $perPage,
|
|
'total' => $total,
|
|
'total_pages' => $totalPages,
|
|
],
|
|
];
|
|
}
|
|
|
|
/** Begin one new bounded cycle after an administrator has inspected debt. */
|
|
public static function retryCleanup(CloudInstanceBackup $backup, array $actor,
|
|
?CloudObjectStorageS3Client $client = null): array
|
|
{
|
|
CloudAccess::manage($actor);
|
|
$client ??= new CloudObjectStorageS3Client();
|
|
$prepared = $backup->getConnection()->transaction(function () use ($backup): array {
|
|
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
->lockForUpdate()->firstOrFail();
|
|
if ((string) $locked->state !== 'delete_attention') {
|
|
throw new CloudError('只有待人工处理的对象可以重新进入清理流程', 409);
|
|
}
|
|
self::assertBackupDeletionAvailable($locked);
|
|
return [
|
|
'runtime' => self::storageRuntime($locked),
|
|
'object_key' => (string) $locked->object_key,
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
'backup' => $locked,
|
|
];
|
|
}, 3);
|
|
try {
|
|
$identity = $client->inspectObject(
|
|
$prepared['runtime']['config'],
|
|
$prepared['runtime']['credentials'],
|
|
$prepared['object_key'],
|
|
$prepared['object_version_id']
|
|
);
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
throw new CloudError($error->getMessage(), 502);
|
|
}
|
|
if ($identity === null) {
|
|
return self::finishDelete((int) $backup->id);
|
|
}
|
|
self::assertObjectIdentity($prepared['backup'], $identity, true);
|
|
$now = time();
|
|
$locked = $backup->getConnection()->transaction(function () use (
|
|
$backup, $now
|
|
): CloudInstanceBackup {
|
|
$row = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
->lockForUpdate()->firstOrFail();
|
|
if ((string) $row->state !== 'delete_attention') {
|
|
throw new CloudError('对象清理状态已发生变化', 409);
|
|
}
|
|
$row->state = 'delete_uncertain';
|
|
$row->delete_cycle_attempts = 0;
|
|
$row->delete_presence_confirmations = 1;
|
|
$row->delete_last_checked_at = $now;
|
|
$row->delete_next_check_at = $now + self::DELETE_STABLE_SECONDS;
|
|
$row->delete_attention_at = null;
|
|
$row->error_code = 'manual_retry_awaiting_stable_probe';
|
|
$row->updated_at = $now;
|
|
$row->save();
|
|
return $row;
|
|
}, 3);
|
|
return self::deletionResult($locked);
|
|
}
|
|
|
|
public static function create(CloudInstance $instance, array $actor,
|
|
array $input): array
|
|
{
|
|
return self::createInternal($instance, $actor, $input, null);
|
|
}
|
|
|
|
/** Queue one deterministic paid automatic backup for a scheduled slot. */
|
|
public static function createAutomatic(CloudInstance $instance,
|
|
CloudBackupSubscription $subscription, array $actor,
|
|
int $scheduledFor): array
|
|
{
|
|
if ($scheduledFor < 1) {
|
|
throw new CloudError('自动备份计划时间无效', 409);
|
|
}
|
|
$zone = new \DateTimeZone(CloudAutomaticBackupService::TIMEZONE);
|
|
$name = 'autobackup-'
|
|
. (new \DateTimeImmutable('@' . $scheduledFor))
|
|
->setTimezone($zone)->format('Ymd-His');
|
|
return self::createInternal($instance, $actor, [
|
|
'name' => $name,
|
|
'consistency_mode' => 'filesystem',
|
|
], [
|
|
'subscription_id' => (int) $subscription->id,
|
|
'scheduled_for' => $scheduledFor,
|
|
]);
|
|
}
|
|
|
|
private static function createInternal(CloudInstance $instance, array $actor,
|
|
array $input, ?array $automatic): array
|
|
{
|
|
if ($automatic === null) {
|
|
CloudAccess::manage($actor);
|
|
} elseif ((int) ($actor['user_id'] ?? 0)
|
|
!== (int) $instance->user_id) {
|
|
throw new CloudError('无权为该实例创建自动备份', 403);
|
|
}
|
|
CloudInput::only($input, ['name', 'consistency_mode']);
|
|
$name = CloudInput::snapshotName($input['name'] ?? null);
|
|
$mode = CloudInput::snapshotConsistency(
|
|
$input['consistency_mode'] ?? null
|
|
);
|
|
$requestTarget = '/v1/vms/'
|
|
. rawurlencode((string) $instance->uuid) . '/backups';
|
|
$canonicalRequest = [
|
|
'instance_id' => (int) $instance->id,
|
|
'name' => $name,
|
|
'consistency_mode' => $mode,
|
|
];
|
|
if ($automatic !== null) {
|
|
$canonicalRequest['source'] = 'automatic';
|
|
$canonicalRequest['subscription_id'] =
|
|
(int) $automatic['subscription_id'];
|
|
$canonicalRequest['scheduled_for'] =
|
|
(int) $automatic['scheduled_for'];
|
|
}
|
|
$fingerprint = CloudJobService::standaloneRequestFingerprint(
|
|
(int) $instance->host_id,
|
|
'backup_create',
|
|
'POST',
|
|
$requestTarget,
|
|
$canonicalRequest
|
|
);
|
|
$prior = self::priorJob(
|
|
$actor,
|
|
'backup_create',
|
|
(int) $instance->id,
|
|
null,
|
|
$fingerprint,
|
|
false
|
|
);
|
|
if ($prior !== null) {
|
|
$backup = CloudInstanceBackup::find((int) $prior->backup_id);
|
|
if ($backup === null) {
|
|
throw new CloudError('备份任务目录记录缺失', 409);
|
|
}
|
|
return ['backup' => self::displayBackup($backup, $prior),
|
|
'job' => $prior];
|
|
}
|
|
$runtime = CloudObjectStorageConfig::runtime(true);
|
|
$plan = null;
|
|
$objectKey = null;
|
|
try {
|
|
return $instance->getConnection()->transaction(function () use (
|
|
$instance, $actor, $name, $mode, $runtime, $requestTarget,
|
|
$canonicalRequest, $fingerprint, $automatic,
|
|
&$plan, &$objectKey
|
|
): array {
|
|
$locked = CloudInstance::where('id', (int) $instance->id)
|
|
->lockForUpdate()->first();
|
|
if ($locked === null || (string) $locked->status === 'deleted') {
|
|
throw new CloudError('虚拟机不存在', 404);
|
|
}
|
|
$prior = self::priorJob($actor, 'backup_create',
|
|
(int) $locked->id, null, $fingerprint);
|
|
if ($prior !== null) {
|
|
$backup = CloudInstanceBackup::find((int) $prior->backup_id);
|
|
if ($backup === null) {
|
|
throw new CloudError('备份任务目录记录缺失', 409);
|
|
}
|
|
return ['backup' => self::displayBackup($backup, $prior),
|
|
'job' => $prior];
|
|
}
|
|
$automaticSubscription = null;
|
|
if ($automatic !== null) {
|
|
$automaticSubscription = CloudBackupSubscription::where(
|
|
'id', (int) $automatic['subscription_id']
|
|
)->lockForUpdate()->first();
|
|
$scheduledFor = (int) $automatic['scheduled_for'];
|
|
$entitled = $automaticSubscription !== null
|
|
&& CloudBackupPeriodEntitlement::where(
|
|
'subscription_id',
|
|
(int) $automaticSubscription->id
|
|
)->where('instance_id', (int) $locked->id)
|
|
->where('user_id', (int) $locked->user_id)
|
|
->where('starts_at', '<=', $scheduledFor)
|
|
->where('ends_at', '>', $scheduledFor)
|
|
->lockForUpdate()->exists();
|
|
if ($automaticSubscription === null
|
|
|| !(bool) $automaticSubscription->enabled
|
|
|| (int) $automaticSubscription->instance_id
|
|
!== (int) $locked->id
|
|
|| (int) $automaticSubscription->user_id
|
|
!== (int) $locked->user_id
|
|
|| !$entitled) {
|
|
throw new CloudError('自动备份订阅与实例账本不一致', 409);
|
|
}
|
|
$existing = CloudInstanceBackup::where(
|
|
'subscription_id', (int) $automaticSubscription->id
|
|
)->where('scheduled_for',
|
|
$scheduledFor)->first();
|
|
if ($existing !== null) {
|
|
$job = CloudJob::where('kind', 'backup_create')
|
|
->where('backup_id', (int) $existing->id)
|
|
->orderByDesc('id')->first();
|
|
if ($job === null) {
|
|
throw new CloudError('自动备份计划账本不完整', 409);
|
|
}
|
|
CloudAutomaticBackupService::advanceAfterScheduled(
|
|
$automaticSubscription,
|
|
$scheduledFor,
|
|
(int) $existing->id
|
|
);
|
|
return [
|
|
'backup' => self::displayBackup($existing, $job),
|
|
'job' => $job,
|
|
];
|
|
}
|
|
}
|
|
if ((string) $locked->status !== 'active') {
|
|
throw new CloudError('只有活动实例可以创建备份', 409);
|
|
}
|
|
if ($automatic !== null
|
|
&& ((string) $locked->service_status !== 'active'
|
|
|| CloudAutomaticBackupService::
|
|
shouldPauseForInstanceExpiry($locked, time()))) {
|
|
throw new CloudError('实例服务当前不可创建自动备份', 409);
|
|
}
|
|
self::assertManagedSource($locked);
|
|
self::assertNoActiveJob($locked);
|
|
$usedQuery = CloudInstanceBackup::where(
|
|
'source_instance_id', (int) $locked->id
|
|
)->where('purpose', 'portable')
|
|
->whereIn('state', self::ACTIVE_BACKUP_STATES);
|
|
if ($automatic === null) {
|
|
$usedQuery->where('source', 'manual');
|
|
} else {
|
|
$usedQuery->where('source', 'automatic')
|
|
->where('subscription_id',
|
|
(int) $automaticSubscription->id)
|
|
->whereNotExists(static function ($query): void {
|
|
$query->selectRaw('1')
|
|
->from('cloud_deployment_snapshots')
|
|
->whereColumn(
|
|
'cloud_deployment_snapshots.backup_id',
|
|
'cloud_instance_backups.id'
|
|
)->where(
|
|
'cloud_deployment_snapshots.state',
|
|
'<>',
|
|
'deleted'
|
|
);
|
|
});
|
|
}
|
|
$used = (int) $usedQuery->count('id');
|
|
if ($automatic === null && $used >= self::MAX_PER_INSTANCE) {
|
|
throw new CloudError('每台实例最多保留十个有效备份', 409);
|
|
}
|
|
if ($automatic !== null
|
|
&& $used >= min(
|
|
CloudAutomaticBackupService::MAX_RETENTION,
|
|
max(1, (int) $automaticSubscription->retention_count)
|
|
) + 1) {
|
|
throw new CloudError('自动备份正在等待旧备份安全清理', 409);
|
|
}
|
|
$spec = self::freezeRestoreCompatibility(
|
|
self::decodeSpec((string) $locked->spec_json)
|
|
);
|
|
$resources = self::resourcesFromSpec($spec);
|
|
$maximumBytes = self::maximumObjectBytes(
|
|
$resources['disk_gib']
|
|
);
|
|
$key = bin2hex(random_bytes(16));
|
|
$prefix = trim((string) $runtime['config']['prefix'], '/');
|
|
$objectKey = ($prefix === '' ? '' : $prefix . '/')
|
|
. 'vm-backups/' . $key . '/system.qcow2';
|
|
$plan = (new CloudObjectStorageS3Client())->beginMultipartPlan(
|
|
$runtime['config'], $runtime['credentials'], $objectKey,
|
|
$maximumBytes, self::TRANSFER_URL_SECONDS
|
|
);
|
|
$agentBody = [
|
|
'backup_key' => $key,
|
|
'name' => $name,
|
|
'consistency_mode' => $mode,
|
|
'object_upload' => self::publicTransferPlan($plan),
|
|
];
|
|
$encodedAgentBody = json_encode(
|
|
$agentBody,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
if (strlen($encodedAgentBody) > self::MAX_AGENT_BODY_BYTES) {
|
|
throw new CloudError(
|
|
'对象存储返回的分段上传凭据过大,不能安全派发备份',
|
|
409
|
|
);
|
|
}
|
|
$now = time();
|
|
$backup = new CloudInstanceBackup();
|
|
$backup->backup_key = $key;
|
|
$backup->source_instance_id = (int) $locked->id;
|
|
$backup->source_host_id = (int) $locked->host_id;
|
|
$backup->user_id = (int) $locked->user_id;
|
|
$backup->source_uuid = strtolower((string) $locked->uuid);
|
|
$backup->name = $name;
|
|
$backup->purpose = 'portable';
|
|
$backup->source = $automatic === null
|
|
? 'manual' : 'automatic';
|
|
$backup->subscription_id = $automatic === null ? null
|
|
: (int) $automaticSubscription->id;
|
|
$backup->scheduled_for = $automatic === null ? null
|
|
: (int) $automatic['scheduled_for'];
|
|
$backup->state = 'creating';
|
|
$backup->consistency = $mode;
|
|
$backup->object_key = $objectKey;
|
|
$backup->object_size_bytes = 0;
|
|
$backup->virtual_size_bytes = 0;
|
|
$backup->sha256 = '';
|
|
$backup->spec_json = json_encode(
|
|
CloudInput::safeResult($spec),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
$backup->sealed_storage_runtime = CloudSecrets::encrypt([
|
|
'config' => $runtime['config'],
|
|
'credentials' => $runtime['credentials'],
|
|
'revision' => (int) $runtime['revision'],
|
|
], self::STORAGE_PURPOSE);
|
|
$backup->sealed_credential = $locked->sealed_credential;
|
|
$backup->storage_revision = (int) $runtime['revision'];
|
|
$backup->active_job_id = null;
|
|
$backup->last_operation_id = null;
|
|
$backup->error_code = '';
|
|
$backup->ready_at = null;
|
|
$backup->created_at = $backup->updated_at = $now;
|
|
$backup->save();
|
|
|
|
[$job, $created] = CloudJobService::
|
|
reserveStandaloneAgentMutation(
|
|
$actor,
|
|
(int) $locked->host_id,
|
|
'backup_create',
|
|
'POST',
|
|
$requestTarget,
|
|
$agentBody,
|
|
[
|
|
'owner_id' => (int) $locked->user_id,
|
|
'can_manage' => true,
|
|
'backup_key' => $key,
|
|
'maximum_size_bytes' => $maximumBytes,
|
|
],
|
|
$canonicalRequest
|
|
);
|
|
if (!$created) {
|
|
throw new CloudError('备份幂等任务状态冲突', 409);
|
|
}
|
|
$job->instance_id = (int) $locked->id;
|
|
$job->backup_id = (int) $backup->id;
|
|
$job->save();
|
|
$backup->active_job_id = (int) $job->id;
|
|
$backup->last_operation_id = (string) $job->operation_id;
|
|
$backup->updated_at = time();
|
|
$backup->save();
|
|
if ($automaticSubscription !== null) {
|
|
CloudAutomaticBackupService::advanceAfterScheduled(
|
|
$automaticSubscription,
|
|
(int) $automatic['scheduled_for'],
|
|
(int) $backup->id
|
|
);
|
|
}
|
|
return ['backup' => self::displayBackup($backup, $job),
|
|
'job' => $job];
|
|
}, self::CREATE_TRANSACTION_ATTEMPTS);
|
|
} catch (\Throwable $error) {
|
|
if (is_array($plan) && is_string($objectKey)) {
|
|
try {
|
|
(new CloudObjectStorageS3Client())->abortMultipart(
|
|
$runtime['config'], $runtime['credentials'], $objectKey,
|
|
(string) ($plan['upload_id'] ?? '')
|
|
);
|
|
} catch (\Throwable $cleanupError) {
|
|
error_log('cloud backup multipart cleanup unavailable: '
|
|
. get_class($cleanupError));
|
|
}
|
|
}
|
|
throw $error;
|
|
}
|
|
}
|
|
|
|
public static function restore(CloudInstanceBackup $backup, array $actor,
|
|
array $input): array
|
|
{
|
|
CloudAccess::manage($actor);
|
|
CloudInput::only($input,
|
|
['target_host_id', 'name', 'confirm_new_instance']);
|
|
$targetHostId = filter_var($input['target_host_id'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
if ($targetHostId === false) {
|
|
throw new CloudError('请选择目标宿主机');
|
|
}
|
|
$name = CloudInput::name($input['name'] ?? null);
|
|
if (($input['confirm_new_instance'] ?? false) !== true) {
|
|
throw new CloudError('请确认恢复会创建免计费的独立新实例');
|
|
}
|
|
$canonicalRequest = [
|
|
'backup_id' => (int) $backup->id,
|
|
'target_host_id' => (int) $targetHostId,
|
|
'name' => $name,
|
|
];
|
|
$idempotencyTarget = '/v1/backup-restores';
|
|
$fingerprint = CloudJobService::standaloneRequestFingerprint(
|
|
(int) $targetHostId,
|
|
'backup_restore',
|
|
'POST',
|
|
$idempotencyTarget,
|
|
$canonicalRequest
|
|
);
|
|
$prior = self::priorJob(
|
|
$actor,
|
|
'backup_restore',
|
|
null,
|
|
(int) $backup->id,
|
|
$fingerprint,
|
|
false
|
|
);
|
|
if ($prior !== null) {
|
|
$restore = CloudBackupRestore::find(
|
|
(int) $prior->backup_restore_id
|
|
);
|
|
if ($restore === null) {
|
|
throw new CloudError('恢复任务目录记录缺失', 409);
|
|
}
|
|
return ['restore' => self::displayRestore($restore),
|
|
'job' => $prior];
|
|
}
|
|
if ((string) $backup->purpose !== 'portable') {
|
|
throw new CloudError('只有普通可移植备份可以创建独立恢复实例', 409);
|
|
}
|
|
if ((string) $backup->state !== 'ready'
|
|
|| (int) $backup->object_size_bytes < 1
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $backup->sha256) !== 1) {
|
|
throw new CloudError('备份尚未处于可恢复状态', 409);
|
|
}
|
|
$runtime = self::storageRuntime($backup);
|
|
$object = (new CloudObjectStorageS3Client())->headObject(
|
|
$runtime['config'], $runtime['credentials'],
|
|
(string) $backup->object_key
|
|
);
|
|
if ((int) $object['size_bytes'] !== (int) $backup->object_size_bytes) {
|
|
throw new CloudError('备份对象大小与目录记录不一致,不能恢复', 409);
|
|
}
|
|
$sourceUrl = (new CloudObjectStorageS3Client())->presignedGet(
|
|
$runtime['config'], $runtime['credentials'],
|
|
(string) $backup->object_key, self::TRANSFER_URL_SECONDS
|
|
);
|
|
return $backup->getConnection()->transaction(function () use (
|
|
$backup, $actor, $targetHostId, $name, $sourceUrl,
|
|
$canonicalRequest, $idempotencyTarget, $fingerprint
|
|
): array {
|
|
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
->lockForUpdate()->first();
|
|
if ($locked === null || (string) $locked->state !== 'ready'
|
|
|| (int) $locked->object_size_bytes < 1
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $locked->sha256) !== 1) {
|
|
throw new CloudError('备份尚未处于可恢复状态', 409);
|
|
}
|
|
$prior = self::priorJob($actor, 'backup_restore', null,
|
|
(int) $locked->id, $fingerprint);
|
|
if ($prior !== null) {
|
|
$restore = CloudBackupRestore::find(
|
|
(int) $prior->backup_restore_id
|
|
);
|
|
if ($restore === null) {
|
|
throw new CloudError('恢复任务目录记录缺失', 409);
|
|
}
|
|
return ['restore' => self::displayRestore($restore),
|
|
'job' => $prior];
|
|
}
|
|
if (CloudInstance::where('host_id', (int) $targetHostId)
|
|
->where('name', $name)->where('status', '<>', 'deleted')
|
|
->exists()) {
|
|
throw new CloudError('目标宿主机已有同名实例', 409);
|
|
}
|
|
$spec = self::decodeSpec((string) $locked->spec_json);
|
|
$resources = self::resourcesFromSpec($spec);
|
|
$uuid = self::uuidV4();
|
|
$operationId = bin2hex(random_bytes(16));
|
|
$now = time();
|
|
$restore = new CloudBackupRestore();
|
|
$restore->restore_key = bin2hex(random_bytes(16));
|
|
$restore->backup_id = (int) $locked->id;
|
|
$restore->target_host_id = (int) $targetHostId;
|
|
$restore->target_instance_id = null;
|
|
$restore->reservation_id = null;
|
|
$restore->job_id = null;
|
|
$restore->operation_id = $operationId;
|
|
$restore->new_uuid = $uuid;
|
|
$restore->new_name = $name;
|
|
$restore->state = 'restoring';
|
|
$restore->error_code = '';
|
|
$restore->ready_at = null;
|
|
$restore->created_at = $restore->updated_at = $now;
|
|
$restore->save();
|
|
$placement = CloudPlacementService::reserveBackupRestore(
|
|
$restore, (int) $locked->user_id, $resources
|
|
);
|
|
$network = $placement['network'];
|
|
$instanceSpec = self::restoredSpec(
|
|
$spec, $locked, $network, $resources
|
|
);
|
|
$instance = new CloudInstance();
|
|
$instance->uuid = $uuid;
|
|
$instance->host_id = (int) $targetHostId;
|
|
$instance->user_id = (int) $locked->user_id;
|
|
$instance->order_id = null;
|
|
$instance->plan_id = null;
|
|
$instance->offer_id = null;
|
|
$instance->resource_pool_id = (int) $network['resource_pool_id'];
|
|
$instance->reservation_id = (int) $placement['reservation']->id;
|
|
$instance->current_period_id = null;
|
|
$instance->name = $name;
|
|
$instance->status = 'provisioning';
|
|
$instance->service_status = 'exempt';
|
|
$instance->network_status = 'pending';
|
|
$instance->expires_at = null;
|
|
$instance->renewal_price = null;
|
|
$instance->renewal_days = 0;
|
|
$instance->renewal_traffic_bytes = 0;
|
|
$instance->traffic_quota_bytes = 0;
|
|
$instance->traffic_download_bytes = 0;
|
|
$instance->traffic_upload_bytes = 0;
|
|
$instance->lifetime_download_bytes = 0;
|
|
$instance->lifetime_upload_bytes = 0;
|
|
$instance->traffic_synced_at = 0;
|
|
$instance->suspended_at = null;
|
|
$instance->network_suspended_at = null;
|
|
$instance->spec_json = json_encode($instanceSpec,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE);
|
|
$instance->sealed_credential = $locked->sealed_credential;
|
|
$instance->created_at = $instance->updated_at = $now;
|
|
$instance->save();
|
|
$reservation = $placement['reservation'];
|
|
$reservation->instance_id = (int) $instance->id;
|
|
$reservation->updated_at = time();
|
|
$reservation->save();
|
|
|
|
$policy = $spec['platform_network_policy_snapshot'] ?? [];
|
|
if (!is_array($policy) || array_is_list($policy)) {
|
|
$policy = [];
|
|
}
|
|
$body = [
|
|
'uuid' => $uuid,
|
|
'source' => '',
|
|
'name' => $name,
|
|
'description' => '',
|
|
'vcpu' => $resources['cpu_cores'],
|
|
'memory_mib' => $resources['memory_mib'],
|
|
'disk_gib' => $resources['disk_gib'],
|
|
'bandwidth_mbps' => $resources['bandwidth_mbps'],
|
|
'network' => 'bridge:br0',
|
|
'template_id' => '',
|
|
'template_revision' => 0,
|
|
'template_generation_token' => '',
|
|
'startup_script' => '',
|
|
'hostname' => CloudInput::hostname($name),
|
|
'guest_ssh_public_key' => '',
|
|
'root_password' => '',
|
|
'public_pool_key' => (string) $network['public_pool_key'],
|
|
'public_pool_cidr' => (string) $network['public_pool_cidr'],
|
|
'public_ip_cidr' => (string) $network['public_ip_cidr'],
|
|
'gateway' => (string) $network['gateway'],
|
|
'dns_servers' => array_values($network['dns_servers']),
|
|
'platform_network_policy_snapshot' => $policy,
|
|
'debian_12_network_fix' =>
|
|
($spec['debian_12_network_fix'] ?? null) === true,
|
|
'backup_key' => (string) $locked->backup_key,
|
|
'source_url' => $sourceUrl,
|
|
'source_size_bytes' => (int) $locked->object_size_bytes,
|
|
'source_sha256' => (string) $locked->sha256,
|
|
];
|
|
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
|
|
$actor, (int) $targetHostId, 'backup_restore', 'POST',
|
|
'/v1/vms/' . rawurlencode($uuid) . '/restore-backup',
|
|
$body,
|
|
[
|
|
'owner_id' => (int) $locked->user_id,
|
|
'can_manage' => true,
|
|
'backup_key' => (string) $locked->backup_key,
|
|
'restore_key' => (string) $restore->restore_key,
|
|
'reservation_id' => (int) $reservation->id,
|
|
],
|
|
$canonicalRequest,
|
|
$operationId,
|
|
$idempotencyTarget
|
|
);
|
|
if (!$created) {
|
|
throw new CloudError('恢复幂等任务状态冲突', 409);
|
|
}
|
|
$job->instance_id = (int) $instance->id;
|
|
$job->backup_id = (int) $locked->id;
|
|
$job->backup_restore_id = (int) $restore->id;
|
|
$job->save();
|
|
$restore->target_instance_id = (int) $instance->id;
|
|
$restore->reservation_id = (int) $reservation->id;
|
|
$restore->job_id = (int) $job->id;
|
|
$restore->updated_at = time();
|
|
$restore->save();
|
|
return ['restore' => self::displayRestore($restore), 'job' => $job];
|
|
}, 3);
|
|
}
|
|
|
|
/**
|
|
* Build the short-lived download capability for a paid create order.
|
|
* The caller must already hold and validate the owner/order row locks;
|
|
* signing is local and therefore does not extend that transaction across
|
|
* a remote object-storage request.
|
|
*
|
|
* @return array{source_url:string}
|
|
*/
|
|
public static function provisioningSource(
|
|
CloudInstanceBackup $backup
|
|
): array {
|
|
if ((string) $backup->purpose !== 'portable'
|
|
|| (string) $backup->state !== 'ready'
|
|
|| (int) $backup->object_size_bytes < 1
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $backup->sha256) !== 1) {
|
|
throw new CloudError('云备份尚未处于可创建实例状态', 409);
|
|
}
|
|
$runtime = self::storageRuntime($backup);
|
|
return [
|
|
'source_url' => (new CloudObjectStorageS3Client())->presignedGet(
|
|
$runtime['config'],
|
|
$runtime['credentials'],
|
|
(string) $backup->object_key,
|
|
self::TRANSFER_URL_SECONDS
|
|
),
|
|
];
|
|
}
|
|
|
|
/** Replace an owner's existing system disk from its own automatic backup. */
|
|
public static function recover(CloudInstance $instance,
|
|
CloudInstanceBackup $backup, array $actor, array $input): array
|
|
{
|
|
if ((int) ($actor['user_id'] ?? 0) !== (int) $instance->user_id) {
|
|
throw new CloudError('无权恢复该实例', 403);
|
|
}
|
|
CloudInput::only($input, ['confirm_restore']);
|
|
if (($input['confirm_restore'] ?? false) !== true) {
|
|
throw new CloudError('请确认恢复会覆盖当前系统盘内容');
|
|
}
|
|
$requestKey = $actor['request_idempotency_key'] ?? null;
|
|
if (!is_string($requestKey) || $requestKey === '') {
|
|
throw new CloudError('恢复云备份必须提供请求幂等标识', 400);
|
|
}
|
|
$target = '/v1/vms/' . rawurlencode((string) $instance->uuid)
|
|
. '/recover-backup';
|
|
$canonical = [
|
|
'instance_id' => (int) $instance->id,
|
|
'backup_id' => (int) $backup->id,
|
|
'confirm_restore' => true,
|
|
];
|
|
$fingerprint = CloudJobService::standaloneRequestFingerprint(
|
|
(int) $instance->host_id,
|
|
'backup_recover',
|
|
'POST',
|
|
$target,
|
|
$canonical
|
|
);
|
|
$prior = self::priorJob($actor, 'backup_recover',
|
|
(int) $instance->id, (int) $backup->id, $fingerprint, false);
|
|
if ($prior !== null) {
|
|
return self::priorRecoveryResult($prior);
|
|
}
|
|
CloudOriginalBackupRestorePolicy::assertEligible($instance, $backup);
|
|
|
|
$runtime = self::storageRuntime($backup);
|
|
$object = (new CloudObjectStorageS3Client())->headObject(
|
|
$runtime['config'], $runtime['credentials'],
|
|
(string) $backup->object_key
|
|
);
|
|
if ((int) $object['size_bytes'] !== (int) $backup->object_size_bytes) {
|
|
throw new CloudError('备份对象大小与目录记录不一致,不能恢复', 409);
|
|
}
|
|
$sourceUrl = (new CloudObjectStorageS3Client())->presignedGet(
|
|
$runtime['config'], $runtime['credentials'],
|
|
(string) $backup->object_key, self::TRANSFER_URL_SECONDS
|
|
);
|
|
|
|
return $instance->getConnection()->transaction(function () use (
|
|
$instance, $backup, $actor, $target, $canonical, $fingerprint,
|
|
$sourceUrl
|
|
): array {
|
|
$lockedInstance = CloudInstance::where(
|
|
'id', (int) $instance->id
|
|
)->lockForUpdate()->first();
|
|
$lockedBackup = CloudInstanceBackup::where(
|
|
'id', (int) $backup->id
|
|
)->lockForUpdate()->first();
|
|
if ($lockedInstance === null || $lockedBackup === null) {
|
|
throw new CloudError('实例或云备份不存在', 404);
|
|
}
|
|
$prior = self::priorJob($actor, 'backup_recover',
|
|
(int) $lockedInstance->id, (int) $lockedBackup->id,
|
|
$fingerprint);
|
|
if ($prior !== null) {
|
|
return self::priorRecoveryResult($prior);
|
|
}
|
|
CloudOriginalBackupRestorePolicy::assertEligible(
|
|
$lockedInstance, $lockedBackup
|
|
);
|
|
self::assertNoActiveJob($lockedInstance);
|
|
if (CloudBackupRecovery::where(
|
|
'instance_id', (int) $lockedInstance->id
|
|
)->whereIn('state', ['recovering', 'uncertain'])->exists()) {
|
|
throw new CloudError('实例已有恢复任务或未确认效果', 409);
|
|
}
|
|
|
|
$operationId = bin2hex(random_bytes(16));
|
|
$recoveryKey = bin2hex(random_bytes(16));
|
|
$now = time();
|
|
$recovery = new CloudBackupRecovery();
|
|
$recovery->recovery_key = $recoveryKey;
|
|
$recovery->backup_id = (int) $lockedBackup->id;
|
|
$recovery->instance_id = (int) $lockedInstance->id;
|
|
$recovery->job_id = null;
|
|
$recovery->operation_id = $operationId;
|
|
$recovery->state = 'recovering';
|
|
$recovery->error_code = '';
|
|
$recovery->completed_at = null;
|
|
$recovery->created_at = $recovery->updated_at = $now;
|
|
$recovery->save();
|
|
|
|
$body = [
|
|
'backup_key' => (string) $lockedBackup->backup_key,
|
|
'source_url' => $sourceUrl,
|
|
'source_size_bytes' => (int) $lockedBackup->object_size_bytes,
|
|
'source_sha256' => (string) $lockedBackup->sha256,
|
|
];
|
|
$jobActor = $actor;
|
|
$jobActor['can_manage'] = true;
|
|
[$job, $created] = CloudJobService::
|
|
reserveStandaloneAgentMutation(
|
|
$jobActor,
|
|
(int) $lockedInstance->host_id,
|
|
'backup_recover',
|
|
'POST',
|
|
$target,
|
|
$body,
|
|
[
|
|
'owner_id' => (int) $lockedInstance->user_id,
|
|
'can_manage' => true,
|
|
'backup_key' => (string) $lockedBackup->backup_key,
|
|
'recovery_key' => $recoveryKey,
|
|
],
|
|
$canonical,
|
|
$operationId
|
|
);
|
|
if (!$created) {
|
|
throw new CloudError('云备份恢复幂等任务状态冲突', 409);
|
|
}
|
|
$job->instance_id = (int) $lockedInstance->id;
|
|
$job->backup_id = (int) $lockedBackup->id;
|
|
$job->backup_recovery_id = (int) $recovery->id;
|
|
$job->save();
|
|
$recovery->job_id = (int) $job->id;
|
|
$recovery->updated_at = time();
|
|
$recovery->save();
|
|
return [
|
|
'recovery' => self::displayRecovery($recovery),
|
|
'job' => $job,
|
|
];
|
|
}, 3);
|
|
}
|
|
|
|
/** Apply a terminal/running Agent result inside CloudWorkerService's lock. */
|
|
public static function finishJobLocked(CloudJob $job, string $status,
|
|
$result, array $payload): void
|
|
{
|
|
if ($job->kind === 'backup_create') {
|
|
self::finishCreateLocked($job, $status, $result, $payload);
|
|
} elseif ($job->kind === 'backup_restore') {
|
|
self::finishRestoreLocked($job, $status, $result, $payload);
|
|
} elseif ($job->kind === 'backup_recover') {
|
|
self::finishRecoveryLocked($job, $status, $result, $payload);
|
|
} elseif ($job->kind === 'backup_workspace_cleanup') {
|
|
self::finishWorkspaceCleanupLocked(
|
|
$job, $status, $result, $payload
|
|
);
|
|
}
|
|
}
|
|
|
|
/** Queue one evidence-bound cleanup of a source-host backup workspace. */
|
|
public static function cleanupWorkspace(
|
|
CloudInstanceBackup $backup,
|
|
array $actor
|
|
): CloudJob {
|
|
CloudAccess::manage($actor);
|
|
if (($actor['is_super_admin'] ?? false) !== true) {
|
|
throw new CloudError('只有超级管理员可以清理备份工作区', 403);
|
|
}
|
|
$original = CloudJob::where(
|
|
'operation_id', (string) $backup->last_operation_id
|
|
)->first();
|
|
if ($original === null || (string) $original->kind !== 'backup_create'
|
|
|| (string) $original->phase !== 'terminal'
|
|
|| (string) $original->outcome !== 'succeeded'
|
|
|| (int) $original->backup_id !== (int) $backup->id
|
|
|| (int) $original->host_id !== (int) $backup->source_host_id) {
|
|
throw new CloudError('原备份任务身份不完整,不能清理工作区', 409);
|
|
}
|
|
$inspection = CloudAdminOperationRecoveryService::inspect(
|
|
$actor, (string) $original->id
|
|
);
|
|
$cleanup = is_array($inspection['cleanup'] ?? null)
|
|
? $inspection['cleanup'] : [];
|
|
$planId = ($cleanup['available'] ?? false) === true
|
|
&& is_string($cleanup['plan_id'] ?? null)
|
|
? strtolower((string) $cleanup['plan_id']) : '';
|
|
if ($planId !== ''
|
|
&& preg_match('/^[a-f0-9]{64}$/D', $planId) !== 1) {
|
|
throw new CloudError('Agent 返回的清理计划身份无效', 502);
|
|
}
|
|
|
|
return $backup->getConnection()->transaction(function () use (
|
|
$backup, $actor, $original, $planId
|
|
): CloudJob {
|
|
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
->lockForUpdate()->first();
|
|
if ($locked === null || (string) $locked->state !== 'ready'
|
|
|| (string) $locked->error_code
|
|
!== 'backup_local_cleanup_pending'
|
|
|| !hash_equals(
|
|
(string) $locked->last_operation_id,
|
|
(string) $original->operation_id
|
|
)) {
|
|
throw new CloudError('备份工作区状态已变化,请刷新后重试', 409);
|
|
}
|
|
$operationId = bin2hex(random_bytes(16));
|
|
$target = '/v1/operations/' . rawurlencode($operationId)
|
|
. '/cleanup-backup-workspace';
|
|
$body = [
|
|
'prior_operation_id' => (string) $original->operation_id,
|
|
'cleanup_plan_id' => $planId,
|
|
];
|
|
$canonical = [
|
|
'backup_id' => (int) $locked->id,
|
|
'backup_key' => (string) $locked->backup_key,
|
|
'prior_operation_id' => (string) $original->operation_id,
|
|
'cleanup_plan_id' => $planId,
|
|
];
|
|
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
|
|
$actor,
|
|
(int) $locked->source_host_id,
|
|
'backup_workspace_cleanup',
|
|
'POST',
|
|
$target,
|
|
$body,
|
|
[
|
|
'owner_id' => (int) $locked->user_id,
|
|
'can_manage' => true,
|
|
'backup_key' => (string) $locked->backup_key,
|
|
'prior_job_id' => (int) $original->id,
|
|
'prior_operation_id' => (string) $original->operation_id,
|
|
'cleanup_plan_id' => $planId,
|
|
],
|
|
$canonical,
|
|
$operationId,
|
|
'/v1/backups/' . (int) $locked->id . '/cleanup-workspace'
|
|
);
|
|
if ($created) {
|
|
$job->instance_id = (int) $locked->source_instance_id;
|
|
$job->backup_id = (int) $locked->id;
|
|
$job->save();
|
|
}
|
|
return $job;
|
|
}, 3);
|
|
}
|
|
|
|
private static function finishWorkspaceCleanupLocked(
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $payload
|
|
): void {
|
|
$backup = $job->backup_id === null ? null
|
|
: CloudInstanceBackup::where('id', (int) $job->backup_id)
|
|
->lockForUpdate()->first();
|
|
if ($backup === null
|
|
|| (int) $backup->source_host_id !== (int) $job->host_id
|
|
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|
|
|| !is_string($payload['prior_operation_id'] ?? null)
|
|
|| !hash_equals(
|
|
(string) $backup->last_operation_id,
|
|
(string) $payload['prior_operation_id']
|
|
)
|
|
|| !hash_equals(
|
|
(string) $backup->backup_key,
|
|
(string) ($payload['backup_key'] ?? '')
|
|
)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'备份工作区清理任务与目录账本不一致', 409
|
|
);
|
|
}
|
|
if ($status !== 'succeeded') {
|
|
return;
|
|
}
|
|
if (!is_array($result)
|
|
|| !hash_equals(
|
|
(string) $backup->backup_key,
|
|
(string) ($result['backup_key'] ?? '')
|
|
)
|
|
|| ($result['cleaned'] ?? null) !== true
|
|
|| ($result['workspace_absent'] ?? null) !== true
|
|
|| ($result['metadata_absent'] ?? null) !== true) {
|
|
throw new CloudError('Agent 清理成功结果缺少完整缺席证据', 502);
|
|
}
|
|
if ((string) $backup->error_code
|
|
=== 'backup_local_cleanup_pending') {
|
|
$backup->error_code = '';
|
|
$backup->updated_at = time();
|
|
$backup->save();
|
|
}
|
|
}
|
|
|
|
public static function markJobUncertain(CloudJob $job): void
|
|
{
|
|
if ($job->kind === 'backup_create' && $job->backup_id !== null) {
|
|
CloudInstanceBackup::where('id', (int) $job->backup_id)
|
|
->where('last_operation_id', (string) $job->operation_id)
|
|
->update(['state' => 'uncertain', 'error_code' =>
|
|
'finalization_incident', 'updated_at' => time()]);
|
|
} elseif ($job->kind === 'backup_restore'
|
|
&& $job->backup_restore_id !== null) {
|
|
CloudBackupRestore::where('id', (int) $job->backup_restore_id)
|
|
->where('operation_id', (string) $job->operation_id)
|
|
->update(['state' => 'uncertain', 'error_code' =>
|
|
'finalization_incident', 'updated_at' => time()]);
|
|
} elseif ($job->kind === 'backup_recover'
|
|
&& $job->backup_recovery_id !== null) {
|
|
CloudBackupRecovery::where(
|
|
'id', (int) $job->backup_recovery_id
|
|
)->where('operation_id', (string) $job->operation_id)
|
|
->update(['state' => 'uncertain', 'error_code' =>
|
|
'finalization_incident', 'updated_at' => time()]);
|
|
}
|
|
CloudInstanceMigrationService::markJobUncertain($job);
|
|
}
|
|
|
|
private static function finishCreateLocked(CloudJob $job, string $status,
|
|
$result, array $payload): void
|
|
{
|
|
$backup = $job->backup_id === null ? null
|
|
: CloudInstanceBackup::where('id', (int) $job->backup_id)
|
|
->lockForUpdate()->first();
|
|
if ($backup === null
|
|
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|
|
|| (int) $backup->source_host_id !== (int) $job->host_id
|
|
|| (int) $backup->user_id !== (int) ($payload['owner_id'] ?? 0)
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($payload['backup_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->last_operation_id,
|
|
(string) $job->operation_id)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'备份任务与面板目录账本不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
$now = time();
|
|
if ($status === 'uncertain') {
|
|
$backup->state = 'uncertain';
|
|
$backup->error_code = 'operation_effect_unknown';
|
|
} elseif ($status === 'failed') {
|
|
$backup->state = 'failed';
|
|
$backup->error_code = 'backup_failed';
|
|
$backup->active_job_id = null;
|
|
} elseif ($status === 'succeeded') {
|
|
$size = filter_var($result['size_bytes'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$virtual = filter_var($result['virtual_size_bytes'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$sha = $result['sha256'] ?? null;
|
|
$objectEtag = $result['object_etag'] ?? '';
|
|
$objectVersionId = $result['object_version_id'] ?? '';
|
|
$consistency = $result['consistency'] ?? null;
|
|
$cleanupPending = $result['cleanup_pending'] ?? false;
|
|
if (!is_array($result)
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($result['backup_key'] ?? ''))
|
|
|| $size === false || $virtual === false
|
|
|| !is_string($sha)
|
|
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|
|
|| !is_string($objectEtag) || strlen($objectEtag) > 256
|
|
|| preg_match('/[\x00-\x1f\x7f]/D', $objectEtag) === 1
|
|
|| !is_string($objectVersionId)
|
|
|| strlen($objectVersionId) > 1024
|
|
|| preg_match('/[\x00-\x1f\x7f]/D', $objectVersionId) === 1
|
|
|| ($objectVersionId !== '' && $objectEtag === '')
|
|
|| !in_array($consistency, ['filesystem', 'crash'], true)
|
|
|| !is_bool($cleanupPending)
|
|
|| (int) $size > (int) ($payload['maximum_size_bytes'] ?? 0)) {
|
|
throw new CloudError('Agent 成功结果缺少可信备份对象身份', 502);
|
|
}
|
|
$backup->state = 'ready';
|
|
$backup->consistency = $consistency;
|
|
$backup->object_size_bytes = (int) $size;
|
|
$backup->virtual_size_bytes = (int) $virtual;
|
|
$backup->sha256 = $sha;
|
|
$backup->object_etag = $objectEtag;
|
|
$backup->object_version_id = $objectVersionId;
|
|
$backup->ready_at = $now;
|
|
$backup->active_job_id = null;
|
|
// The remote object is complete and restorable. A host-side
|
|
// snapshot cleanup warning must not discard this sole immutable
|
|
// object identity or trigger another upload.
|
|
$backup->error_code = $cleanupPending
|
|
? 'backup_local_cleanup_pending' : '';
|
|
}
|
|
$backup->updated_at = $now;
|
|
$backup->save();
|
|
}
|
|
|
|
private static function finishRestoreLocked(CloudJob $job, string $status,
|
|
$result, array $payload): void
|
|
{
|
|
$restore = $job->backup_restore_id === null ? null
|
|
: CloudBackupRestore::where('id', (int) $job->backup_restore_id)
|
|
->lockForUpdate()->first();
|
|
$backup = $job->backup_id === null ? null
|
|
: CloudInstanceBackup::where('id', (int) $job->backup_id)
|
|
->lockForUpdate()->first();
|
|
$instance = $job->instance_id === null ? null
|
|
: CloudInstance::where('id', (int) $job->instance_id)
|
|
->lockForUpdate()->first();
|
|
if ($restore === null || $backup === null || $instance === null
|
|
|| (int) $restore->backup_id !== (int) $backup->id
|
|
|| (int) $restore->target_instance_id !== (int) $instance->id
|
|
|| (int) $restore->target_host_id !== (int) $job->host_id
|
|
|| !hash_equals((string) $restore->operation_id,
|
|
(string) $job->operation_id)
|
|
|| !hash_equals((string) $restore->restore_key,
|
|
(string) ($payload['restore_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($payload['backup_key'] ?? ''))
|
|
|| (int) $instance->user_id !== (int) ($payload['owner_id'] ?? 0)
|
|
|| !hash_equals(strtolower((string) $restore->new_uuid),
|
|
strtolower((string) $instance->uuid))) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'备份恢复任务与实例、资源或目录账本不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
$requestedIp = $payload['body']['public_ip_cidr'] ?? null;
|
|
if (!is_string($requestedIp)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'备份恢复任务缺少公网 IP 快照', 409
|
|
);
|
|
}
|
|
$now = time();
|
|
if ($status === 'uncertain') {
|
|
CloudPlacementService::finishBackupRestore(
|
|
$restore, (int) $instance->id, $requestedIp, 'uncertain'
|
|
);
|
|
$restore->state = 'uncertain';
|
|
$restore->error_code = 'operation_effect_unknown';
|
|
} elseif ($status === 'failed') {
|
|
CloudPlacementService::finishBackupRestore(
|
|
$restore, (int) $instance->id, $requestedIp, 'failed'
|
|
);
|
|
$restore->state = 'failed';
|
|
$restore->error_code = 'backup_restore_failed';
|
|
$instance->status = 'deleted';
|
|
$instance->network_status = 'inactive';
|
|
$instance->sealed_credential = null;
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
} elseif ($status === 'succeeded') {
|
|
$vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
|
|
if (!is_array($result) || !is_array($vm)
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($result['backup_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->sha256,
|
|
(string) ($result['sha256'] ?? ''))
|
|
|| !hash_equals(strtolower((string) $restore->new_uuid),
|
|
strtolower((string) ($vm['uuid'] ?? '')))
|
|
|| !hash_equals($requestedIp,
|
|
(string) ($vm['public_ip_cidr'] ?? ''))
|
|
|| !hash_equals((string) ($payload['body']['public_pool_key'] ?? ''),
|
|
(string) ($vm['public_pool_key'] ?? ''))) {
|
|
throw new CloudError(
|
|
'Agent 恢复成功结果与备份、实例或公网 IP 预留不一致',
|
|
502
|
|
);
|
|
}
|
|
CloudPlacementService::finishBackupRestore(
|
|
$restore, (int) $instance->id, $requestedIp, 'succeeded'
|
|
);
|
|
$spec = self::decodeSpec((string) $instance->spec_json);
|
|
$sourceSpec = self::decodeSpec((string) $backup->spec_json);
|
|
$spec = CloudInstanceMetadata::merge($spec, $vm);
|
|
foreach (['os_source', 'os_name', 'login_name'] as $field) {
|
|
if (isset($sourceSpec[$field])) {
|
|
$spec[$field] = $sourceSpec[$field];
|
|
}
|
|
}
|
|
$spec['managed'] = true;
|
|
$spec['restored_from_backup'] = (string) $backup->backup_key;
|
|
$instance->spec_json = json_encode($spec,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE);
|
|
$instance->status = 'active';
|
|
$instance->service_status = 'exempt';
|
|
$instance->network_status = 'active';
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
self::recordRuntime($instance, $vm);
|
|
$restore->state = 'ready';
|
|
$restore->error_code = '';
|
|
$restore->ready_at = $now;
|
|
}
|
|
$restore->updated_at = $now;
|
|
$restore->save();
|
|
}
|
|
|
|
private static function finishRecoveryLocked(CloudJob $job,
|
|
string $status, $result, array $payload): void
|
|
{
|
|
$recovery = $job->backup_recovery_id === null ? null
|
|
: CloudBackupRecovery::where(
|
|
'id', (int) $job->backup_recovery_id
|
|
)->lockForUpdate()->first();
|
|
$backup = $job->backup_id === null ? null
|
|
: CloudInstanceBackup::where('id', (int) $job->backup_id)
|
|
->lockForUpdate()->first();
|
|
$instance = $job->instance_id === null ? null
|
|
: CloudInstance::where('id', (int) $job->instance_id)
|
|
->lockForUpdate()->first();
|
|
$body = is_array($payload['body'] ?? null)
|
|
? $payload['body'] : [];
|
|
if ($recovery === null || $backup === null || $instance === null
|
|
|| (int) $recovery->job_id !== (int) $job->id
|
|
|| (int) $recovery->backup_id !== (int) $backup->id
|
|
|| (int) $recovery->instance_id !== (int) $instance->id
|
|
|| !hash_equals((string) $recovery->operation_id,
|
|
(string) $job->operation_id)
|
|
|| !hash_equals((string) $recovery->recovery_key,
|
|
(string) ($payload['recovery_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($payload['backup_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($body['backup_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->sha256,
|
|
(string) ($body['source_sha256'] ?? ''))
|
|
|| (int) $backup->object_size_bytes
|
|
!== (int) ($body['source_size_bytes'] ?? 0)
|
|
|| (int) $instance->user_id
|
|
!== (int) ($payload['owner_id'] ?? 0)
|
|
|| (int) $job->user_id !== (int) $instance->user_id
|
|
|| (int) $job->host_id !== (int) $instance->host_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'云备份恢复任务与实例、备份或恢复账本不一致',
|
|
409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
$now = time();
|
|
if ($status === 'uncertain') {
|
|
$recovery->state = 'uncertain';
|
|
$recovery->error_code = 'operation_effect_unknown';
|
|
$recovery->completed_at = null;
|
|
} elseif ($status === 'failed') {
|
|
$recovery->state = 'failed';
|
|
$recovery->error_code = 'backup_recovery_failed';
|
|
$recovery->completed_at = $now;
|
|
} elseif ($status === 'succeeded') {
|
|
if (!is_array($result)
|
|
|| ($result['restored'] ?? null) !== true
|
|
|| !hash_equals((string) $backup->backup_key,
|
|
(string) ($result['backup_key'] ?? ''))
|
|
|| !hash_equals((string) $backup->sha256,
|
|
(string) ($result['sha256'] ?? ''))
|
|
|| CloudInstanceRuntimeService::normalizeState(
|
|
$result['power_state'] ?? null
|
|
) === 'unknown') {
|
|
throw new CloudError(
|
|
'Agent 恢复成功结果缺少可信的备份或电源身份',
|
|
502
|
|
);
|
|
}
|
|
$backupSpec = self::decodeSpec((string) $backup->spec_json);
|
|
if (($backupSpec['managed'] ?? null) !== true
|
|
|| ($backupSpec['os_source'] ?? null) !== 'cloud_template'
|
|
|| !is_string($backupSpec['os_name'] ?? null)
|
|
|| trim((string) $backupSpec['os_name']) === ''
|
|
|| ($backupSpec['login_name'] ?? null) !== 'root'
|
|
|| !is_bool($backupSpec['debian_12_network_fix'] ?? null)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'云备份的客户机身份快照无效',
|
|
409
|
|
);
|
|
}
|
|
$instanceSpec = self::decodeSpec((string) $instance->spec_json);
|
|
foreach ([
|
|
'template_id', 'template_revision',
|
|
'template_generation_token', 'os_source', 'os_name',
|
|
'login_name', 'debian_12_network_fix',
|
|
] as $field) {
|
|
if (array_key_exists($field, $backupSpec)) {
|
|
$instanceSpec[$field] = $backupSpec[$field];
|
|
} else {
|
|
unset($instanceSpec[$field]);
|
|
}
|
|
}
|
|
unset($instanceSpec['install_iso']);
|
|
// Passwords and hostnames may have been changed inside the guest
|
|
// without passing through the panel. The restored system disk is
|
|
// authoritative, so the panel must not claim either value is
|
|
// known or cause the Agent's NoCloud seed to inject an old value.
|
|
unset($instanceSpec['guest_hostname'], $instanceSpec['hostname']);
|
|
$instanceSpec['managed'] = true;
|
|
$instanceSpec['restored_from_backup'] =
|
|
(string) $backup->backup_key;
|
|
$instance->spec_json = json_encode(
|
|
CloudInput::safeResult($instanceSpec),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
$instance->sealed_credential = null;
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
$recovery->state = 'succeeded';
|
|
$recovery->error_code = '';
|
|
$recovery->completed_at = $now;
|
|
self::recordRuntime($instance, $result);
|
|
}
|
|
$recovery->updated_at = $now;
|
|
$recovery->save();
|
|
}
|
|
|
|
private static function priorRecoveryResult(CloudJob $job): array
|
|
{
|
|
$recovery = $job->backup_recovery_id === null ? null
|
|
: CloudBackupRecovery::find((int) $job->backup_recovery_id);
|
|
if ($recovery === null) {
|
|
throw new CloudError('云备份恢复任务目录记录缺失', 409);
|
|
}
|
|
return [
|
|
'recovery' => self::displayRecovery($recovery),
|
|
'job' => $job,
|
|
];
|
|
}
|
|
|
|
private static function displayRecovery(CloudBackupRecovery $recovery): array
|
|
{
|
|
return [
|
|
'recovery_key' => (string) $recovery->recovery_key,
|
|
'backup_id' => (string) $recovery->backup_id,
|
|
'instance_id' => (string) $recovery->instance_id,
|
|
'state' => (string) $recovery->state,
|
|
'created_at' => max(0, (int) $recovery->created_at),
|
|
'completed_at' => $recovery->completed_at === null ? null
|
|
: max(0, (int) $recovery->completed_at),
|
|
];
|
|
}
|
|
|
|
private static function assertManagedSource(CloudInstance $instance): void
|
|
{
|
|
$spec = self::decodeSpec((string) $instance->spec_json);
|
|
if (($spec['managed'] ?? false) !== true
|
|
|| ($spec['os_source'] ?? null) !== 'cloud_template') {
|
|
throw new CloudError('仅面板创建的云镜像实例支持可移植备份', 409);
|
|
}
|
|
self::resourcesFromSpec($spec);
|
|
}
|
|
|
|
private static function assertNoActiveJob(CloudInstance $instance): void
|
|
{
|
|
CloudInstanceMigrationService::assertInstanceMutationAvailable(
|
|
$instance
|
|
);
|
|
if (CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->exists()) {
|
|
throw new CloudError('虚拟机已有任务,请等待完成', 409);
|
|
}
|
|
}
|
|
|
|
private static function assertBackupDeletionAvailable(
|
|
CloudInstanceBackup $backup
|
|
): void {
|
|
if ((string) $backup->purpose !== 'portable') {
|
|
throw new CloudError('该备份类型不能由此入口删除', 409);
|
|
}
|
|
if ((string) $backup->state === 'uncertain') {
|
|
throw new CloudError('备份效果尚未确认,不能删除', 409);
|
|
}
|
|
if ($backup->active_job_id !== null) {
|
|
throw new CloudError('备份仍有关联任务,不能删除', 409);
|
|
}
|
|
if (CloudBackupRestore::where('backup_id', (int) $backup->id)
|
|
->whereIn('state', ['restoring', 'uncertain'])->exists()) {
|
|
throw new CloudError('备份仍有恢复任务或未知效果,不能删除', 409);
|
|
}
|
|
if (CloudBackupRecovery::where('backup_id', (int) $backup->id)
|
|
->whereIn('state', ['recovering', 'uncertain'])->exists()) {
|
|
throw new CloudError('备份仍有原实例恢复任务或未知效果,不能删除', 409);
|
|
}
|
|
if (CloudJob::where('backup_id', (int) $backup->id)
|
|
->where(static function ($query): void {
|
|
$query->where('phase', '<>', 'terminal')
|
|
->orWhere('effect_resolution', 'unknown')
|
|
->orWhere('finalization_incident', 1);
|
|
})->exists()) {
|
|
throw new CloudError('备份仍有关联任务,不能删除', 409);
|
|
}
|
|
if (CloudDeploymentSnapshot::where('backup_id', (int) $backup->id)
|
|
->where('state', 'ready')->exists()) {
|
|
throw new CloudError('该对象已保存为云快照,请从云快照页面删除', 409);
|
|
}
|
|
}
|
|
|
|
/** @param array{size_bytes:int,etag:string,version_id:string} $identity */
|
|
private static function claimDeleteGeneration(int $backupId,
|
|
string $previousState, array $identity, int $now,
|
|
?callable $beforeDeleteLocked): array
|
|
{
|
|
$backup = CloudInstanceBackup::findOrFail($backupId);
|
|
return $backup->getConnection()->transaction(function () use (
|
|
$backupId, $previousState, $identity, $now, $beforeDeleteLocked
|
|
): array {
|
|
$locked = CloudInstanceBackup::where('id', $backupId)
|
|
->lockForUpdate()->firstOrFail();
|
|
if ((string) $locked->state === 'deleted') {
|
|
return ['mode' => 'deleted', 'backup' => $locked];
|
|
}
|
|
if (in_array((string) $locked->state,
|
|
self::CLEANUP_DEBT_STATES, true)) {
|
|
return ['mode' => 'waiting', 'backup' => $locked];
|
|
}
|
|
if ((string) $locked->state !== $previousState) {
|
|
throw new CloudError('备份删除状态已发生变化', 409);
|
|
}
|
|
self::assertBackupDeletionAvailable($locked);
|
|
if ($beforeDeleteLocked !== null) {
|
|
$beforeDeleteLocked($locked);
|
|
}
|
|
$locked->object_etag = (string) $identity['etag'];
|
|
$locked->object_version_id = (string) $identity['version_id'];
|
|
$locked->state = 'deleting';
|
|
$locked->delete_operation_id = bin2hex(random_bytes(16));
|
|
$locked->delete_generation = max(0,
|
|
(int) $locked->delete_generation) + 1;
|
|
$locked->delete_cycle_attempts = 1;
|
|
$locked->delete_presence_confirmations = 0;
|
|
$locked->delete_requested_at = $now;
|
|
$locked->delete_last_checked_at = null;
|
|
$locked->delete_next_check_at = 0;
|
|
$locked->delete_attention_at = null;
|
|
$locked->error_code = '';
|
|
$locked->updated_at = $now;
|
|
$locked->save();
|
|
return [
|
|
'mode' => 'delete',
|
|
'previous_state' => $previousState,
|
|
'backup' => $locked,
|
|
'runtime' => self::storageRuntime($locked),
|
|
'object_key' => (string) $locked->object_key,
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
];
|
|
}, 3);
|
|
}
|
|
|
|
private static function recordStableDeleteProbe(int $backupId,
|
|
int $now): array
|
|
{
|
|
$backup = CloudInstanceBackup::findOrFail($backupId);
|
|
return $backup->getConnection()->transaction(function () use (
|
|
$backupId, $now
|
|
): array {
|
|
$locked = CloudInstanceBackup::where('id', $backupId)
|
|
->lockForUpdate()->firstOrFail();
|
|
if ((string) $locked->state === 'deleted') {
|
|
return ['mode' => 'deleted', 'backup' => $locked];
|
|
}
|
|
if ((string) $locked->state === 'delete_attention') {
|
|
return ['mode' => 'attention', 'backup' => $locked];
|
|
}
|
|
if (!in_array((string) $locked->state,
|
|
['deleting', 'delete_uncertain'], true)) {
|
|
throw new CloudError('备份删除状态已发生变化', 409);
|
|
}
|
|
$last = max(0, (int) $locked->delete_last_checked_at);
|
|
if ($last > 0 && $now - $last < self::DELETE_STABLE_SECONDS) {
|
|
$locked->state = 'delete_uncertain';
|
|
$locked->delete_next_check_at = $last
|
|
+ self::DELETE_STABLE_SECONDS;
|
|
$locked->updated_at = $now;
|
|
$locked->save();
|
|
return ['mode' => 'waiting', 'backup' => $locked];
|
|
}
|
|
$confirmations = min(self::DELETE_STABLE_PROBES,
|
|
max(0, (int) $locked->delete_presence_confirmations) + 1);
|
|
$locked->state = 'delete_uncertain';
|
|
$locked->delete_presence_confirmations = $confirmations;
|
|
$locked->delete_last_checked_at = $now;
|
|
$locked->delete_next_check_at = $now
|
|
+ self::DELETE_STABLE_SECONDS;
|
|
$locked->error_code = 'object_delete_still_present';
|
|
$locked->updated_at = $now;
|
|
if ($confirmations < self::DELETE_STABLE_PROBES) {
|
|
$locked->save();
|
|
return ['mode' => 'waiting', 'backup' => $locked];
|
|
}
|
|
if ((int) $locked->delete_cycle_attempts
|
|
>= self::DELETE_MAX_AUTOMATIC_ATTEMPTS) {
|
|
$locked->state = 'delete_attention';
|
|
$locked->delete_attention_at = $now;
|
|
$locked->delete_next_check_at = 0;
|
|
$locked->error_code = 'object_delete_retry_exhausted';
|
|
$locked->save();
|
|
return ['mode' => 'attention', 'backup' => $locked];
|
|
}
|
|
$locked->state = 'deleting';
|
|
$locked->delete_operation_id = bin2hex(random_bytes(16));
|
|
$locked->delete_generation = max(0,
|
|
(int) $locked->delete_generation) + 1;
|
|
$locked->delete_cycle_attempts = max(0,
|
|
(int) $locked->delete_cycle_attempts) + 1;
|
|
$locked->delete_presence_confirmations = 0;
|
|
$locked->delete_next_check_at = 0;
|
|
$locked->error_code = '';
|
|
$locked->save();
|
|
return [
|
|
'mode' => 'delete',
|
|
'backup' => $locked,
|
|
'runtime' => self::storageRuntime($locked),
|
|
'object_key' => (string) $locked->object_key,
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
];
|
|
}, 3);
|
|
}
|
|
|
|
/** @param array{size_bytes:int,etag:string,version_id:string} $identity */
|
|
private static function assertObjectIdentity(CloudInstanceBackup $backup,
|
|
array $identity, bool $strict): void
|
|
{
|
|
if ((int) $identity['size_bytes'] !== (int) $backup->object_size_bytes) {
|
|
throw new CloudError('对象存储中的备份大小与目录账本不一致', 409);
|
|
}
|
|
if (!$strict) {
|
|
return;
|
|
}
|
|
$etag = (string) $backup->object_etag;
|
|
if ($etag === '' || !hash_equals($etag, (string) $identity['etag'])) {
|
|
throw new CloudError('对象存储中的备份 ETag 已发生变化', 409);
|
|
}
|
|
$versionId = (string) $backup->object_version_id;
|
|
if ($versionId !== ''
|
|
&& !hash_equals($versionId, (string) $identity['version_id'])) {
|
|
throw new CloudError('对象存储中的备份版本已发生变化', 409);
|
|
}
|
|
}
|
|
|
|
private static function markDeleteUncertain(int $backupId,
|
|
string $errorCode, int $now, bool $resetEvidence,
|
|
int $retryAfter = self::DELETE_STABLE_SECONDS): void
|
|
{
|
|
$values = [
|
|
'state' => 'delete_uncertain',
|
|
'delete_next_check_at' => $now + max(1, $retryAfter),
|
|
'error_code' => substr($errorCode, 0, 64),
|
|
'updated_at' => $now,
|
|
];
|
|
if ($resetEvidence) {
|
|
$values['delete_presence_confirmations'] = 0;
|
|
$values['delete_last_checked_at'] = null;
|
|
}
|
|
CloudInstanceBackup::where('id', $backupId)
|
|
->whereIn('state', ['deleting', 'delete_uncertain'])
|
|
->update($values);
|
|
}
|
|
|
|
private static function markDeleteAttention(int $backupId,
|
|
string $errorCode, int $now): void
|
|
{
|
|
CloudInstanceBackup::where('id', $backupId)
|
|
->whereIn('state', ['deleting', 'delete_uncertain'])
|
|
->update([
|
|
'state' => 'delete_attention',
|
|
'delete_next_check_at' => 0,
|
|
'delete_attention_at' => $now,
|
|
'error_code' => substr($errorCode, 0, 64),
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
|
|
private static function restoreAfterRejectedDelete(int $backupId,
|
|
string $previousState, string $errorCode): void
|
|
{
|
|
CloudInstanceBackup::where('id', $backupId)
|
|
->where('state', 'deleting')
|
|
->update([
|
|
'state' => $previousState,
|
|
'delete_operation_id' => null,
|
|
'delete_cycle_attempts' => 0,
|
|
'delete_presence_confirmations' => 0,
|
|
'delete_requested_at' => null,
|
|
'delete_last_checked_at' => null,
|
|
'delete_next_check_at' => 0,
|
|
'delete_attention_at' => null,
|
|
'error_code' => substr($errorCode, 0, 64),
|
|
'updated_at' => time(),
|
|
]);
|
|
}
|
|
|
|
private static function finishDelete(int $backupId): array
|
|
{
|
|
$backup = CloudInstanceBackup::findOrFail($backupId);
|
|
return $backup->getConnection()->transaction(function () use (
|
|
$backupId
|
|
): array {
|
|
$locked = CloudInstanceBackup::where('id', $backupId)
|
|
->lockForUpdate()->firstOrFail();
|
|
if ((string) $locked->state !== 'deleted') {
|
|
self::assertBackupDeletionAvailable($locked);
|
|
if (!in_array((string) $locked->state,
|
|
['ready', 'failed', 'deleting', 'delete_uncertain',
|
|
'delete_attention'], true)) {
|
|
throw new CloudError('备份删除状态已发生变化', 409);
|
|
}
|
|
$locked->state = 'deleted';
|
|
$locked->sealed_storage_runtime = '';
|
|
$locked->sealed_credential = null;
|
|
$locked->active_job_id = null;
|
|
$locked->delete_presence_confirmations = 0;
|
|
$locked->delete_next_check_at = 0;
|
|
$locked->delete_attention_at = null;
|
|
$locked->error_code = '';
|
|
$locked->updated_at = time();
|
|
$locked->save();
|
|
CloudDeploymentSnapshot::where('backup_id', $backupId)
|
|
->whereIn('state', ['deleting', 'delete_uncertain'])
|
|
->update([
|
|
'state' => 'deleted',
|
|
'deleted_at' => time(),
|
|
'error_code' => '',
|
|
'updated_at' => time(),
|
|
]);
|
|
}
|
|
return self::deletionResult($locked);
|
|
}, 3);
|
|
}
|
|
|
|
private static function deletionResult(CloudInstanceBackup $backup): array
|
|
{
|
|
$state = (string) $backup->state;
|
|
return [
|
|
'backup_id' => (string) $backup->id,
|
|
'state' => $state,
|
|
'deleted' => $state === 'deleted',
|
|
'requires_reconciliation' => in_array($state,
|
|
['deleting', 'delete_uncertain'], true),
|
|
'requires_attention' => $state === 'delete_attention',
|
|
'next_check_at' => max(0, (int) $backup->delete_next_check_at),
|
|
];
|
|
}
|
|
|
|
private static function priorJob(array $actor, string $kind,
|
|
?int $instanceId, ?int $backupId, string $fingerprint,
|
|
bool $lock = true): ?CloudJob
|
|
{
|
|
$key = $actor['request_idempotency_key'] ?? null;
|
|
if (!is_string($key) || $key === '') {
|
|
return null;
|
|
}
|
|
$query = CloudJob::where('user_id', (int) $actor['user_id'])
|
|
->where('request_idempotency_key', $key);
|
|
if ($lock) {
|
|
$query->lockForUpdate();
|
|
}
|
|
$job = $query->first();
|
|
if ($job === null) {
|
|
return null;
|
|
}
|
|
if ((string) $job->kind !== $kind
|
|
|| !hash_equals((string) $job->request_fingerprint,
|
|
$fingerprint)
|
|
|| ($instanceId !== null
|
|
&& (int) $job->instance_id !== $instanceId)
|
|
|| ($backupId !== null && (int) $job->backup_id !== $backupId)) {
|
|
throw new CloudError('同一请求标识不能用于不同的备份操作', 409);
|
|
}
|
|
return $job;
|
|
}
|
|
|
|
private static function resourcesFromSpec(array $spec): array
|
|
{
|
|
$cpu = filter_var($spec['vcpu'] ?? $spec['cpu_cores'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$memory = filter_var($spec['memory_mib'] ?? $spec['ram_mib'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$disk = filter_var($spec['disk_gib'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$bandwidth = filter_var($spec['bandwidth_mbps'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
if ($cpu === false || $memory === false || $disk === false
|
|
|| $bandwidth === false) {
|
|
throw new CloudError('实例缺少可信的资源规格快照', 409);
|
|
}
|
|
return ['cpu_cores' => (int) $cpu, 'memory_mib' => (int) $memory,
|
|
'disk_gib' => (int) $disk, 'bandwidth_mbps' => (int) $bandwidth];
|
|
}
|
|
|
|
/** Freeze template-specific boot compatibility into the backup manifest. */
|
|
private static function freezeRestoreCompatibility(array $spec): array
|
|
{
|
|
if (!is_bool($spec['debian_12_network_fix'] ?? null)) {
|
|
throw new CloudError(
|
|
'实例缺少创建时冻结的网络兼容快照,不能生成可移植备份',
|
|
409
|
|
);
|
|
}
|
|
return $spec;
|
|
}
|
|
|
|
private static function restoredSpec(array $source,
|
|
CloudInstanceBackup $backup, array $network, array $resources): array
|
|
{
|
|
$spec = CloudInput::safeResult($source);
|
|
foreach (['plan_id', 'offer_id', 'price_version', 'stock_mode',
|
|
'validity_days', 'traffic_quota_bytes', 'renewal_price',
|
|
'currency', 'public_ip', 'public_ip_cidr'] as $field) {
|
|
unset($spec[$field]);
|
|
}
|
|
return array_replace($spec, [
|
|
'managed' => false,
|
|
'vcpu' => $resources['cpu_cores'],
|
|
'cpu_cores' => $resources['cpu_cores'],
|
|
'memory_mib' => $resources['memory_mib'],
|
|
'disk_gib' => $resources['disk_gib'],
|
|
'bandwidth_mbps' => $resources['bandwidth_mbps'],
|
|
'network' => 'bridge:br0',
|
|
'public_pool_key' => (string) $network['public_pool_key'],
|
|
'public_pool_cidr' => (string) $network['public_pool_cidr'],
|
|
'public_ip_cidr' => (string) $network['public_ip_cidr'],
|
|
'gateway' => (string) $network['gateway'],
|
|
'dns_servers' => array_values($network['dns_servers']),
|
|
'resource_pool_id' => (int) $network['resource_pool_id'],
|
|
'resource_pool_name' => (string) $network['resource_pool_name'],
|
|
'line_id' => (int) $network['line_id'],
|
|
'line_name' => (string) $network['line_name'],
|
|
'region_id' => (int) $network['region_id'],
|
|
'region' => (string) $network['region_name'],
|
|
'country_code' => (string) $network['country_code'],
|
|
'restored_from_backup' => (string) $backup->backup_key,
|
|
]);
|
|
}
|
|
|
|
private static function decodeSpec(string $encoded,
|
|
bool $throw = true): ?array
|
|
{
|
|
$spec = json_decode($encoded, true);
|
|
if (!is_array($spec) || array_is_list($spec)) {
|
|
if ($throw) {
|
|
throw new CloudError('实例备份规格快照损坏', 409);
|
|
}
|
|
return null;
|
|
}
|
|
return $spec;
|
|
}
|
|
|
|
private static function storageRuntime(CloudInstanceBackup $backup): array
|
|
{
|
|
$runtime = CloudSecrets::decrypt(
|
|
(string) $backup->sealed_storage_runtime,
|
|
self::STORAGE_PURPOSE
|
|
);
|
|
if (!is_array($runtime['config'] ?? null)
|
|
|| !is_array($runtime['credentials'] ?? null)
|
|
|| (int) ($runtime['revision'] ?? -1)
|
|
!== (int) $backup->storage_revision) {
|
|
throw new CloudError('备份对象存储快照无效', 503);
|
|
}
|
|
return $runtime;
|
|
}
|
|
|
|
private static function publicTransferPlan(array $plan): array
|
|
{
|
|
return [
|
|
'part_size_bytes' => (int) $plan['part_size_bytes'],
|
|
'maximum_size_bytes' => (int) $plan['maximum_size_bytes'],
|
|
'part_urls' => array_values($plan['part_urls']),
|
|
'complete_url' => (string) $plan['complete_url'],
|
|
'abort_url' => (string) $plan['abort_url'],
|
|
'verify_url' => (string) $plan['verify_url'],
|
|
];
|
|
}
|
|
|
|
private static function maximumObjectBytes(int $diskGib): int
|
|
{
|
|
if ($diskGib < 1 || $diskGib > 2559) {
|
|
throw new CloudError('实例系统盘超出当前备份最大容量', 409);
|
|
}
|
|
return ($diskGib + 1) * self::GIB;
|
|
}
|
|
|
|
private static function displayBackup(CloudInstanceBackup $backup,
|
|
?CloudJob $job, bool $canManage = true): array
|
|
{
|
|
$state = in_array((string) $backup->state,
|
|
['creating', 'ready', 'failed', 'uncertain', 'deleting',
|
|
'delete_uncertain', 'delete_attention', 'deleted'], true)
|
|
? (string) $backup->state : 'uncertain';
|
|
$canDelete = $canManage && in_array($state,
|
|
['ready', 'failed', 'deleting', 'delete_uncertain'], true);
|
|
if ($canDelete) {
|
|
try {
|
|
self::assertBackupDeletionAvailable($backup);
|
|
} catch (CloudError $error) {
|
|
$canDelete = false;
|
|
}
|
|
}
|
|
return [
|
|
'id' => (string) $backup->id,
|
|
'backup_key' => (string) $backup->backup_key,
|
|
'name' => (string) $backup->name,
|
|
'state' => $state,
|
|
'consistency' => in_array((string) $backup->consistency,
|
|
['filesystem', 'crash'], true)
|
|
? (string) $backup->consistency : 'unknown',
|
|
'object_size_bytes' => max(0, (int) $backup->object_size_bytes),
|
|
'virtual_size_bytes' => max(0, (int) $backup->virtual_size_bytes),
|
|
'sha256' => preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $backup->sha256) === 1
|
|
? (string) $backup->sha256 : '',
|
|
'source_host_id' => (string) $backup->source_host_id,
|
|
'created_at' => max(0, (int) $backup->created_at),
|
|
'ready_at' => $backup->ready_at === null ? null
|
|
: max(0, (int) $backup->ready_at),
|
|
'updated_at' => max(0, (int) $backup->updated_at),
|
|
'task_id' => $job === null ? null : (string) $job->id,
|
|
'job_status' => $job === null ? null
|
|
: (CloudOperationErrorCatalog::userStatus(
|
|
(string) $job->status
|
|
) ?? 'unknown'),
|
|
'cleanup_pending' => (string) $backup->error_code
|
|
=== 'backup_local_cleanup_pending',
|
|
'can_cleanup_workspace' => $canManage
|
|
&& (string) $backup->error_code
|
|
=== 'backup_local_cleanup_pending',
|
|
'can_delete' => $canDelete,
|
|
'delete_action' => in_array($state,
|
|
['deleting', 'delete_uncertain'], true)
|
|
? 'reconcile' : 'delete',
|
|
'error' => $state === 'delete_attention'
|
|
? '对象自动清理已达到三次上限,需要管理员检查对象存储后手动恢复清理。'
|
|
: ($state === 'delete_uncertain'
|
|
? '删除结果尚未确认;系统会在两次稳定核对后创建新的删除代次。'
|
|
: null),
|
|
];
|
|
}
|
|
|
|
private static function displayRestore(CloudBackupRestore $restore): array
|
|
{
|
|
return [
|
|
'restore_key' => (string) $restore->restore_key,
|
|
'state' => (string) $restore->state,
|
|
'target_host_id' => (string) $restore->target_host_id,
|
|
'target_instance_id' => $restore->target_instance_id === null
|
|
? null : (string) $restore->target_instance_id,
|
|
'name' => (string) $restore->new_name,
|
|
'created_at' => max(0, (int) $restore->created_at),
|
|
'ready_at' => $restore->ready_at === null ? null
|
|
: max(0, (int) $restore->ready_at),
|
|
];
|
|
}
|
|
|
|
private static function uuidV4(): string
|
|
{
|
|
$bytes = random_bytes(16);
|
|
$bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40);
|
|
$bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80);
|
|
$hex = bin2hex($bytes);
|
|
return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-'
|
|
. substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-'
|
|
. substr($hex, 20, 12);
|
|
}
|
|
|
|
private static function libraryFilters(array $input): array
|
|
{
|
|
CloudInput::only($input, ['page', 'per_page', 'q', 'user_id',
|
|
'instance_id', 'host_id', 'state']);
|
|
foreach ($input as $value) {
|
|
if (!is_string($value) && !is_int($value)) {
|
|
throw new CloudError('备份库筛选参数无效', 400);
|
|
}
|
|
}
|
|
$filters = [];
|
|
foreach (['user_id', 'instance_id', 'host_id'] as $key) {
|
|
$value = (string) ($input[$key] ?? '');
|
|
if ($value !== '' && (preg_match('/^[1-9][0-9]{0,17}$/D', $value) !== 1)) {
|
|
throw new CloudError('备份库 ' . $key . ' 必须为有效的正整数', 400);
|
|
}
|
|
$filters[$key] = $value;
|
|
}
|
|
$state = (string) ($input['state'] ?? '');
|
|
if (!in_array($state, ['', 'creating', 'ready', 'restoring', 'failed',
|
|
'uncertain', 'deleting', 'delete_uncertain', 'delete_attention'], true)) {
|
|
throw new CloudError('备份库状态筛选无效', 400);
|
|
}
|
|
$q = trim((string) ($input['q'] ?? ''));
|
|
if (preg_match('//u', $q) !== 1 || mb_strlen($q, 'UTF-8') > 80
|
|
|| preg_match('/[\x00-\x1f\x7f]/u', $q) !== 0) {
|
|
throw new CloudError('备份名称关键词最多 80 个字符,且不能包含控制字符', 400);
|
|
}
|
|
return $filters + ['state' => $state, 'q' => $q];
|
|
}
|
|
|
|
private static function libraryInteger($value, int $minimum, int $maximum,
|
|
int $default): int
|
|
{
|
|
if ($value === null || $value === '') {
|
|
return $default;
|
|
}
|
|
if (is_int($value)) {
|
|
$parsed = $value;
|
|
} elseif (is_string($value)
|
|
&& preg_match('/^(?:0|[1-9][0-9]{0,9})$/D', $value) === 1) {
|
|
$parsed = (int) $value;
|
|
} else {
|
|
throw new CloudError('备份库分页参数无效', 400);
|
|
}
|
|
if ($parsed < $minimum || $parsed > $maximum) {
|
|
throw new CloudError('备份库分页参数超出允许范围', 400);
|
|
}
|
|
return $parsed;
|
|
}
|
|
|
|
private static function recordRuntime(CloudInstance $instance,
|
|
array $result): void
|
|
{
|
|
try {
|
|
CloudInstanceRuntimeService::seed(
|
|
(int) $instance->id, (int) $instance->host_id,
|
|
(string) $instance->uuid
|
|
);
|
|
$state = $result['status']
|
|
?? ($result['state'] ?? ($result['power_state'] ?? null));
|
|
if (CloudInstanceRuntimeService::normalizeState($state)
|
|
!== 'unknown') {
|
|
CloudInstanceRuntimeService::markObserved(
|
|
(int) $instance->id, (int) $instance->host_id,
|
|
(string) $instance->uuid, $state
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
error_log('cloud backup restore runtime cache unavailable: '
|
|
. get_class($error));
|
|
}
|
|
}
|
|
}
|