From fc38aa318f9b514a9b4fddd5e0d96eec444d8cd1 Mon Sep 17 00:00:00 2001 From: codex Date: Mon, 26 Oct 2026 03:43:20 +0800 Subject: [PATCH] fix: clean successful migration bundles --- docs/DEPLOYMENT.md | 2 +- docs/PORTABLE_VM_BACKUPS.md | 2 + docs/VALIDATION.md | 1 + web/app/Command/CloudWorker.php | 28 ++ web/app/Services/Cloud/CloudBackupService.php | 3 + .../Cloud/CloudMigrationBundleService.php | 436 +++++++++++++++++ .../Services/Cloud/CloudRuntimeHeartbeat.php | 2 + web/tests/CloudMigrationBundleCleanupTest.php | 463 ++++++++++++++++++ web/tests/CloudPortableBackupDeletionTest.php | 11 + web/tests/CloudRuntimeHeartbeatTest.php | 61 +++ 10 files changed, 1008 insertions(+), 1 deletion(-) create mode 100644 web/tests/CloudMigrationBundleCleanupTest.php diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index bd385017..365bf34c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -95,7 +95,7 @@ php xcat cloudRepairInstanceMetadata --apply 后台 `/admin/object-storage` 可配置一个全局 S3 兼容对象存储应用,适用于 AWS S3、MinIO 及实现 AWS Signature Version 4 的兼容服务。该功能复用业务库已有的 `configurations` 表,不需要导入新表。Endpoint 必须是带有效证书的 HTTPS origin;MinIO 通常选择路径寻址,AWS S3 通常选择虚拟主机寻址。 -Access Key ID 与 Secret Access Key 只在超级管理员提交时进入 PHP,使用 `cloud_data_key` 的独立用途密钥密封后保存,读取页面不会回显。修改任一配置或停用应用都会清除原就绪标记;必须重新执行“验证连接并启用”。连接验证只发送签名的 `HeadBucket` 请求,需要目标 Bucket 的 `s3:ListBucket` 权限,不上传、覆盖或删除对象。管理员虚拟机管理已接入完整系统盘备份和新实例恢复,目标可以是原宿主机或其他合格宿主机;运行账户还需要配置备份前缀范围内的 multipart 上传、`GetObject`、`HeadObject` 和 `DeleteObject` 权限。当前不包含增量链和自动保留策略,具体安全边界见 `docs/PORTABLE_VM_BACKUPS.md`。 +Access Key ID 与 Secret Access Key 只在超级管理员提交时进入 PHP,使用 `cloud_data_key` 的独立用途密钥密封后保存,读取页面不会回显。修改任一配置或停用应用都会清除原就绪标记;必须重新执行“验证连接并启用”。连接验证只发送签名的 `HeadBucket` 请求,需要目标 Bucket 的 `s3:ListBucket` 权限,不上传、覆盖或删除对象。管理员虚拟机管理已接入完整系统盘备份和新实例恢复,目标可以是原宿主机或其他合格宿主机;运行账户还需要配置 `vm-backups/` 与 `vm-migrations/` 前缀范围内的 multipart 上传、`GetObject`、`HeadObject` 和 `DeleteObject` 权限。成功维护迁移会由普通任务 Worker 自动回收隐藏迁移传输包;版本控制 Bucket 仍须为 `vm-migrations/` 配置非当前版本过期策略。当前不包含增量链和普通备份自动保留策略,具体安全边界见 `docs/PORTABLE_VM_BACKUPS.md`。 生产环境应为面板创建独立服务账户,把权限限制到配置的 Bucket 和备份前缀。数据库备份必须与原始 `cloud_data_key` 一起保管,重新生成主密钥无法恢复全局配置及各备份冻结的对象存储凭据。 diff --git a/docs/PORTABLE_VM_BACKUPS.md b/docs/PORTABLE_VM_BACKUPS.md index 6a5d77b3..561888ac 100644 --- a/docs/PORTABLE_VM_BACKUPS.md +++ b/docs/PORTABLE_VM_BACKUPS.md @@ -42,6 +42,8 @@ 任何删除响应超时、断连或回执丢失都进入待核对状态,系统不会重放源删除,也不会提前释放源资源。迁移对象使用独立的 `vm-migrations/{bundle_key}/` 前缀,根对象为 `system.qcow2`,快照对象以快照键命名;普通云备份继续使用 `vm-backups/` 且不携带快照。迁移目录和目标恢复属于隐藏的编排步骤,不会生成一个免计费的独立业务实例。 +目标迁移已经确认成功并进入 `completed` 或 `source_retained` 后,普通任务 Worker 会自动回收该次迁移的隐藏传输包:先逐个删除快照对象,全部确认不存在后才删除根对象并清除冻结的对象存储凭据。失败、回滚、源清理效果不确定或尚未完成切换的迁移不会触发该回收。对象 DELETE 响应未知时,后续只用 HEAD 核对,不会重放删除;因此不确定对象可能需要人工核对。若 Bucket 开启版本控制,DeleteObject 只会创建删除标记,仍须配置 `vm-migrations/` 前缀的非当前版本过期策略,才能真正释放历史版本占用。 + ## 失败与不确定效果 备份和恢复均为异步 HeavyIO 操作,单次宿主机 worker 最长可运行 24 小时。控制面只会派发一次同一 operation ID:超时、断连或回执丢失均视为效果未知,并通过 Agent Journal 查询原操作,不会重新执行备份或创建第二台 VM。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 91873092..d45195be 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -131,6 +131,7 @@ jq empty deploy/*.json.example - 使用受控的 S3/MinIO 测试 Bucket 分别验收路径寻址和虚拟主机寻址、错误 Region、错误凭据、缺少 `s3:ListBucket`、无效证书和连接超时;验证过程中 Bucket 对象数与内容不得变化。 - 可移植备份必须拒绝额外可写盘、backing chain、LUN 和活动 pivot/mirror;运行实例分别验证文件系统一致性与 Guest Agent 不可用时的崩溃一致性路径,关机实例验证稳定完整副本。 - 维护迁移 `MigrationBundle` 必须验证当前全部快照都生成独立对象、任一快照缺失或回执漂移即拒绝切换、目标恢复后快照目录整体改绑。目标在恢复和用户防火墙收敛期间必须保持关机;源机的运行/关机、autostart、断网状态与修改后主机名必须在独立激活步骤后保持一致。普通云备份仍只能产生单系统盘对象。 +- 迁移进入 `completed` 或 `source_retained` 后,普通任务 Worker 必须先删除全部快照传输对象、最后删除根对象并清除冻结凭据;失败、回滚和效果不确定的迁移必须保留传输包。故障注入 DELETE 超时或断连后,后续只能 HEAD 核对且不得发出第二次 DELETE;开启版本控制的 Bucket 还须验证非当前版本生命周期策略。 - 恢复必须选择不同宿主机,目标使用 `bridge:br0` 和全新预留公网 IP;校验下载对象字节数与 SHA-256,新实例 UUID、MAC、seed、NVRAM、网络过滤器和流量状态均不得复用源实例,且不得继承订单、账单、服务期、到期时间或流量周期。 - 对备份/恢复断连、超时和回执丢失执行故障注入:同一 operation ID 只能查询收敛,不能再次派发。恢复失败只有在全部托管产物明确不存在时才能释放容量与 IP;对象删除响应未知时只能 HEAD 核对不存在,不能重发 DELETE。 diff --git a/web/app/Command/CloudWorker.php b/web/app/Command/CloudWorker.php index 99fb1bd6..b8d1c737 100644 --- a/web/app/Command/CloudWorker.php +++ b/web/app/Command/CloudWorker.php @@ -3,11 +3,16 @@ namespace App\Command; use App\Services\Cloud\CloudInvoiceService; use App\Services\Cloud\CloudAutomaticOperationRecoveryService; +use App\Services\Cloud\CloudMigrationBundleService; +use App\Services\Cloud\CloudObjectStorageTransferError; use App\Services\Cloud\CloudRuntimeHeartbeat; use App\Services\Cloud\CloudWorkerService; final class CloudWorker { + private const CLEANUP_IDLE_SECONDS = 30; + private const CLEANUP_ERROR_SECONDS = 60; + public static function run(array $argv = [], ?CloudRuntimeHeartbeat $heartbeat = null): int { $once = in_array('--once', $argv, true); @@ -26,6 +31,7 @@ final class CloudWorker $worker = $recoveryOnly ? new CloudAutomaticOperationRecoveryService() : new CloudWorkerService(null, $policyOnly); + $nextMigrationBundleCleanupAt = 0; do { $errors = []; $heartbeat->busy( @@ -47,6 +53,28 @@ final class CloudWorker } $worked = false; } + $cleanupNow = time(); + if (!$policyOnly && !$recoveryOnly + && $cleanupNow >= $nextMigrationBundleCleanupAt) { + $heartbeat->busy('migration_bundle_cleanup'); + try { + $cleaned = CloudMigrationBundleService::cleanupOneEligible(); + $worked = $cleaned || $worked; + $nextMigrationBundleCleanupAt = $cleaned + ? 0 : $cleanupNow + self::CLEANUP_IDLE_SECONDS; + } catch (\Throwable $error) { + $nextMigrationBundleCleanupAt = $cleanupNow + + self::CLEANUP_ERROR_SECONDS; + $errors[] = 'migration_bundle_cleanup_failed'; + $heartbeat->error('migration_bundle_cleanup_failed'); + $detail = $error instanceof CloudObjectStorageTransferError + ? ':' . $error->publicCode : ''; + fwrite(STDERR, + 'migration bundle cleanup temporarily unavailable: ' + . get_class($error) . $detail . PHP_EOL + ); + } + } if ($errors !== []) { $heartbeat->error($errors[0]); } diff --git a/web/app/Services/Cloud/CloudBackupService.php b/web/app/Services/Cloud/CloudBackupService.php index 653d8e4b..434f306d 100644 --- a/web/app/Services/Cloud/CloudBackupService.php +++ b/web/app/Services/Cloud/CloudBackupService.php @@ -1147,6 +1147,9 @@ final class CloudBackupService private static function assertBackupDeletionAvailable( CloudInstanceBackup $backup ): void { + if ((string) $backup->purpose !== 'portable') { + throw new CloudError('维护迁移包由系统自动清理', 409); + } if ((string) $backup->state === 'uncertain') { throw new CloudError('备份效果尚未确认,不能删除', 409); } diff --git a/web/app/Services/Cloud/CloudMigrationBundleService.php b/web/app/Services/Cloud/CloudMigrationBundleService.php index e3dae9e4..fb25db63 100644 --- a/web/app/Services/Cloud/CloudMigrationBundleService.php +++ b/web/app/Services/Cloud/CloudMigrationBundleService.php @@ -3,6 +3,7 @@ namespace App\Services\Cloud; use App\Models\CloudInstance; +use App\Models\CloudInstanceBackup; use App\Models\CloudInstanceMigration; use App\Models\CloudInstanceSnapshot; use App\Models\CloudMigrationBundleArtifact; @@ -19,6 +20,10 @@ final class CloudMigrationBundleService { public const VERSION = 1; private const GIB = 1073741824; + private const CLEANUP_SUCCESS_STATES = ['completed', 'source_retained']; + private const CLEANUP_RETRY_SECONDS = 300; + private const CLEANUP_INFLIGHT_SECONDS = 60; + private const STORAGE_PURPOSE = 'object-storage-backup-runtime'; /** @param array $snapshots */ public static function additionalDiskGib(array $snapshots): int @@ -299,6 +304,437 @@ final class CloudMigrationBundleService } } + /** + * Delete at most one object from a migration whose target is proven ready. + * + * Snapshot transports are removed before the root image so the sealed S3 + * runtime remains available until every object has converged. A lost DELETE + * response changes the durable object state to delete_uncertain; every + * later attempt is a read-only HEAD and never replays the mutation. + */ + public static function cleanupOneEligible( + ?CloudObjectStorageS3Client $client = null, + ?int $now = null + ): bool { + $now ??= time(); + $client ??= new CloudObjectStorageS3Client(); + // Finish an already-drained bundle before starting another snapshot. + // eligibleRootBackup() itself proves that this bundle has no pending + // artifacts, so root-last remains strict without starving old roots. + $backup = self::eligibleRootBackup($now); + if ($backup !== null) { + $claim = self::claimRootCleanup((int) $backup->id, $now); + if ($claim === null) { + return false; + } + self::assertRunnableCleanupClaim($claim); + self::executeCleanupClaim($claim, $client, $now); + return true; + } + + $artifact = self::eligibleArtifact($now); + if ($artifact === null) { + return false; + } + $claim = self::claimArtifactCleanup((int) $artifact->id, $now); + if ($claim === null) { + return false; + } + self::assertRunnableCleanupClaim($claim); + self::executeCleanupClaim($claim, $client, $now); + return true; + } + + private static function eligibleArtifact( + int $now + ): ?CloudMigrationBundleArtifact { + return CloudMigrationBundleArtifact::query() + ->from('cloud_migration_bundle_artifacts as cleanup_artifacts') + ->select('cleanup_artifacts.*') + ->join('cloud_instance_migrations as cleanup_migrations', + 'cleanup_migrations.id', '=', + 'cleanup_artifacts.migration_id') + ->join('cloud_instance_backups as cleanup_backups', + 'cleanup_backups.id', '=', 'cleanup_migrations.backup_id') + ->whereIn('cleanup_migrations.state', self::CLEANUP_SUCCESS_STATES) + ->where('cleanup_backups.purpose', 'migration_bundle') + ->where('cleanup_backups.state', 'ready') + ->where(static function ($query) use ($now): void { + $query->where('cleanup_artifacts.state', 'restored') + ->orWhere(static function ($retry) use ($now): void { + $retry->where('cleanup_artifacts.state', + 'delete_rejected') + ->where('cleanup_artifacts.updated_at', '<=', + $now - self::CLEANUP_RETRY_SECONDS); + })->orWhere(static function ($uncertain) use ($now): void { + $uncertain->where('cleanup_artifacts.state', + 'delete_uncertain') + ->where('cleanup_artifacts.updated_at', '<=', + $now - self::CLEANUP_RETRY_SECONDS); + })->orWhere(static function ($stale) use ($now): void { + $stale->where('cleanup_artifacts.state', 'deleting') + ->where('cleanup_artifacts.updated_at', '<=', + $now - self::CLEANUP_INFLIGHT_SECONDS); + }); + })->orderBy('cleanup_artifacts.id')->first(); + } + + private static function eligibleRootBackup( + int $now + ): ?CloudInstanceBackup { + return CloudInstanceBackup::query() + ->from('cloud_instance_backups as cleanup_backups') + ->select('cleanup_backups.*') + ->join('cloud_instance_migrations as cleanup_migrations', + 'cleanup_migrations.backup_id', '=', 'cleanup_backups.id') + ->whereIn('cleanup_migrations.state', self::CLEANUP_SUCCESS_STATES) + ->where('cleanup_backups.purpose', 'migration_bundle') + ->whereNotExists(static function ($pending): void { + $pending->selectRaw('1') + ->from('cloud_migration_bundle_artifacts as pending_artifacts') + ->whereColumn('pending_artifacts.migration_id', + 'cleanup_migrations.id') + ->where('pending_artifacts.state', '<>', 'deleted'); + })->where(static function ($query) use ($now): void { + $query->where('cleanup_backups.state', 'ready') + ->orWhere(static function ($retry) use ($now): void { + $retry->where('cleanup_backups.state', 'delete_rejected') + ->where('cleanup_backups.updated_at', '<=', + $now - self::CLEANUP_RETRY_SECONDS); + })->orWhere(static function ($uncertain) use ($now): void { + $uncertain->where('cleanup_backups.state', + 'delete_uncertain') + ->where('cleanup_backups.updated_at', '<=', + $now - self::CLEANUP_RETRY_SECONDS); + })->orWhere(static function ($stale) use ($now): void { + $stale->where('cleanup_backups.state', 'deleting') + ->where('cleanup_backups.updated_at', '<=', + $now - self::CLEANUP_INFLIGHT_SECONDS); + }); + })->orderBy('cleanup_migrations.id')->first(); + } + + /** @return array|null */ + private static function claimArtifactCleanup(int $artifactId, + int $now): ?array + { + return (new CloudMigrationBundleArtifact())->getConnection() + ->transaction(function () use ($artifactId, $now): ?array { + $artifact = CloudMigrationBundleArtifact::where( + 'id', $artifactId + )->lockForUpdate()->first(); + if ($artifact === null) { + return null; + } + $migration = CloudInstanceMigration::where( + 'id', (int) $artifact->migration_id + )->lockForUpdate()->first(); + $backup = $migration === null ? null + : CloudInstanceBackup::where( + 'id', (int) $migration->backup_id + )->lockForUpdate()->first(); + if (!self::cleanupLedgerMatches($migration, $backup)) { + return null; + } + if ((string) $backup->state !== 'ready') { + return null; + } + $action = self::claimAction( + (string) $artifact->state, + (int) $artifact->updated_at, + $now, + 'restored' + ); + if ($action === null) { + return null; + } + try { + $runtime = self::cleanupRuntime($backup); + } catch (\Throwable $error) { + self::markInvalidCleanupRuntime( + $artifact, + $backup, + $now, + $action + ); + return ['action' => 'invalid_runtime']; + } + $artifact->state = $action === 'delete' + ? 'deleting' : 'delete_uncertain'; + $artifact->updated_at = $now; + $artifact->save(); + return [ + 'kind' => 'artifact', + 'id' => (int) $artifact->id, + 'migration_id' => (int) $migration->id, + 'backup_id' => (int) $backup->id, + 'object_key' => (string) $artifact->object_key, + 'action' => $action, + 'runtime' => $runtime, + ]; + }, 3); + } + + /** @return array|null */ + private static function claimRootCleanup(int $backupId, int $now): ?array + { + return (new CloudInstanceBackup())->getConnection()->transaction( + function () use ($backupId, $now): ?array { + $backup = CloudInstanceBackup::where('id', $backupId) + ->lockForUpdate()->first(); + $migration = $backup === null ? null + : CloudInstanceMigration::where( + 'backup_id', (int) $backup->id + )->lockForUpdate()->first(); + if (!self::cleanupLedgerMatches($migration, $backup) + || CloudMigrationBundleArtifact::where( + 'migration_id', (int) $migration->id + )->where('state', '<>', 'deleted')->exists()) { + return null; + } + $action = self::claimAction( + (string) $backup->state, + (int) $backup->updated_at, + $now, + 'ready' + ); + if ($action === null) { + return null; + } + try { + $runtime = self::cleanupRuntime($backup); + } catch (\Throwable $error) { + $backup->state = $action === 'probe' + ? 'delete_uncertain' : 'delete_rejected'; + $backup->error_code = 'migration_bundle_runtime_invalid'; + $backup->updated_at = $now; + $backup->save(); + return ['action' => 'invalid_runtime']; + } + $backup->state = $action === 'delete' + ? 'deleting' : 'delete_uncertain'; + $backup->error_code = ''; + $backup->updated_at = $now; + $backup->save(); + return [ + 'kind' => 'root', + 'id' => (int) $backup->id, + 'migration_id' => (int) $migration->id, + 'backup_id' => (int) $backup->id, + 'object_key' => (string) $backup->object_key, + 'action' => $action, + 'runtime' => $runtime, + ]; + }, 3 + ); + } + + private static function cleanupLedgerMatches( + ?CloudInstanceMigration $migration, + ?CloudInstanceBackup $backup + ): bool { + return $migration !== null && $backup !== null + && in_array((string) $migration->state, + self::CLEANUP_SUCCESS_STATES, true) + && (int) $migration->backup_id === (int) $backup->id + && (string) $backup->purpose === 'migration_bundle' + && (string) $backup->state !== 'deleted'; + } + + private static function markInvalidCleanupRuntime( + CloudMigrationBundleArtifact $artifact, + CloudInstanceBackup $backup, + int $now, + string $action + ): void { + $artifact->state = $action === 'probe' + ? 'delete_uncertain' : 'delete_rejected'; + $artifact->updated_at = $now; + $artifact->save(); + $backup->error_code = 'migration_bundle_runtime_invalid'; + $backup->updated_at = $now; + $backup->save(); + } + + /** @param array $claim */ + private static function assertRunnableCleanupClaim(array $claim): void + { + if (($claim['action'] ?? null) === 'invalid_runtime') { + throw new CloudObjectStorageTransferError( + '迁移包对象存储快照无效。', + 'migration_bundle_runtime_invalid' + ); + } + } + + private static function claimAction(string $state, int $updatedAt, + int $now, string $initialState): ?string + { + if ($state === $initialState) { + return 'delete'; + } + if ($state === 'delete_rejected' + && $updatedAt <= $now - self::CLEANUP_RETRY_SECONDS) { + return 'delete'; + } + if ($state === 'delete_uncertain' + && $updatedAt <= $now - self::CLEANUP_RETRY_SECONDS) { + return 'probe'; + } + if ($state === 'deleting' + && $updatedAt <= $now - self::CLEANUP_INFLIGHT_SECONDS) { + return 'probe'; + } + return null; + } + + /** @param array $claim */ + private static function executeCleanupClaim(array $claim, + CloudObjectStorageS3Client $client, int $now): void + { + $runtime = $claim['runtime']; + try { + if ($claim['action'] === 'delete') { + $client->deleteObject( + $runtime['config'], + $runtime['credentials'], + (string) $claim['object_key'] + ); + self::finishCleanupClaim($claim, $now); + return; + } + $presence = $client->objectPresence( + $runtime['config'], + $runtime['credentials'], + (string) $claim['object_key'] + ); + if ($presence === 'absent') { + self::finishCleanupClaim($claim, $now); + return; + } + self::markCleanupClaim($claim, 'delete_uncertain', + 'object_delete_still_present', $now); + throw new CloudObjectStorageTransferError( + '迁移包对象删除结果仍未收敛。', + 'object_delete_still_present', + true + ); + } catch (CloudObjectStorageTransferError $error) { + if ($error->publicCode === 'object_delete_still_present') { + throw $error; + } + $state = $claim['action'] === 'delete' + && !$error->effectMayHaveApplied + ? 'delete_rejected' : 'delete_uncertain'; + self::markCleanupClaim( + $claim, + $state, + $error->publicCode, + $now + ); + throw $error; + } + } + + /** @param array $claim */ + private static function finishCleanupClaim(array $claim, int $now): void + { + (new CloudInstanceBackup())->getConnection()->transaction( + function () use ($claim, $now): void { + if ($claim['kind'] === 'artifact') { + $artifact = CloudMigrationBundleArtifact::where( + 'id', (int) $claim['id'] + )->lockForUpdate()->first(); + if ($artifact === null || !in_array( + (string) $artifact->state, + ['deleting', 'delete_uncertain'], + true + )) { + return; + } + $artifact->state = 'deleted'; + $artifact->updated_at = $now; + $artifact->save(); + return; + } + $backup = CloudInstanceBackup::where( + 'id', (int) $claim['backup_id'] + )->lockForUpdate()->first(); + if ($backup === null || !in_array( + (string) $backup->state, + ['deleting', 'delete_uncertain'], + true + ) || CloudMigrationBundleArtifact::where( + 'migration_id', (int) $claim['migration_id'] + )->where('state', '<>', 'deleted')->exists()) { + return; + } + $backup->state = 'deleted'; + $backup->sealed_storage_runtime = ''; + $backup->sealed_credential = null; + $backup->active_job_id = null; + $backup->error_code = ''; + $backup->updated_at = $now; + $backup->save(); + }, 3 + ); + } + + /** @param array $claim */ + private static function markCleanupClaim(array $claim, string $state, + string $errorCode, int $now): void + { + (new CloudInstanceBackup())->getConnection()->transaction( + function () use ($claim, $state, $errorCode, $now): void { + if ($claim['kind'] === 'artifact') { + $updated = CloudMigrationBundleArtifact::where( + 'id', (int) $claim['id'] + )->whereIn('state', ['deleting', 'delete_uncertain']) + ->update(['state' => $state, 'updated_at' => $now]); + if ($updated !== 1) { + return; + } + } else { + CloudInstanceBackup::where( + 'id', (int) $claim['backup_id'] + )->whereIn('state', ['deleting', 'delete_uncertain']) + ->update([ + 'state' => $state, + 'error_code' => substr($errorCode, 0, 64), + 'updated_at' => $now, + ]); + return; + } + CloudInstanceBackup::where( + 'id', (int) $claim['backup_id'] + )->where('purpose', 'migration_bundle') + ->where('state', 'ready')->update([ + 'error_code' => substr($errorCode, 0, 64), + 'updated_at' => $now, + ]); + }, 3 + ); + } + + /** @return array{config:array,credentials:array} */ + private static function cleanupRuntime(CloudInstanceBackup $backup): array + { + $runtime = CloudSecrets::decrypt( + (string) $backup->sealed_storage_runtime, + self::STORAGE_PURPOSE + ); + if (!is_array($runtime['config'] ?? null) + || !is_array($runtime['credentials'] ?? null) + || (int) ($runtime['revision'] ?? -1) + !== (int) $backup->storage_revision) { + throw new CloudError('迁移包对象存储快照无效', 503); + } + return [ + 'config' => $runtime['config'], + 'credentials' => $runtime['credentials'], + ]; + } + /** @param array> $prepared */ public static function abortPreparedPlans(array $runtime, array $prepared): void diff --git a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php index c38e65f7..0850501a 100644 --- a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php +++ b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php @@ -10,6 +10,7 @@ final class CloudRuntimeHeartbeat 'lifecycle' => ['lifecycle', '生命周期调度']]; private const PHASES = ['startup' => '启动', 'worker_tick' => '任务消费或核对', 'recovery_tick' => '任务安全收敛', + 'migration_bundle_cleanup' => '迁移传输包清理', 'notification' => '邮件通知', 'lifecycle_tick' => '生命周期巡检', 'provision_expiry' => '开通账单过期处理', 'invoice_generation' => '续费账单生成', 'traffic_sync' => '流量同步', @@ -17,6 +18,7 @@ final class CloudRuntimeHeartbeat 'instance_reconcile' => '实例策略核对', 'idle' => '等待下一轮', 'stopped' => '已退出']; private const ERRORS = ['worker_tick_failed', 'recovery_tick_failed', + 'migration_bundle_cleanup_failed', 'notification_failed', 'lifecycle_tick_failed', 'provision_expiry_failed', 'invoice_generation_failed', 'traffic_sync_failed', 'runtime_status_sync_failed', diff --git a/web/tests/CloudMigrationBundleCleanupTest.php b/web/tests/CloudMigrationBundleCleanupTest.php new file mode 100644 index 00000000..90309760 --- /dev/null +++ b/web/tests/CloudMigrationBundleCleanupTest.php @@ -0,0 +1,463 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', + ], 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema('default'); + $schema->create('cloud_instance_backups', + static function (Blueprint $table): void { + $table->increments('id'); + $table->string('purpose'); + $table->string('state'); + $table->string('object_key'); + $table->text('sealed_storage_runtime'); + $table->text('sealed_credential')->nullable(); + $table->integer('storage_revision'); + $table->integer('active_job_id')->nullable(); + $table->string('error_code')->default(''); + $table->integer('updated_at'); + }); + $schema->create('cloud_instance_migrations', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('backup_id'); + $table->string('state'); + $table->string('error_code')->default(''); + $table->integer('updated_at'); + }); + $schema->create('cloud_migration_bundle_artifacts', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('migration_id'); + $table->string('object_key'); + $table->string('state'); + $table->integer('updated_at'); + }); + + Config::$dataKey = base64_encode(str_repeat('m', 32)); + $runtime = [ + 'config' => [ + 'endpoint' => 'https://objects.example.com', + 'region' => 'us-east-1', + 'bucket' => 'backup-bucket', + 'prefix' => 'virt', + 'path_style' => true, + ], + 'credentials' => [ + 'access_key_id' => 'MIGRATIONACCESS', + 'secret_access_key' => 'migration-secret-value', + ], + 'revision' => 9, + ]; + $sealedRuntime = CloudSecrets::encrypt( + $runtime, + 'object-storage-backup-runtime' + ); + $client = static function (array $responses, array &$history): + CloudObjectStorageS3Client { + $stack = HandlerStack::create(new MockHandler($responses)); + $stack->push(Middleware::history($history)); + return new CloudObjectStorageS3Client(new Client(['handler' => $stack])); + }; + $newMigration = static function (int $id, string $state, + int $artifactCount = 2) use ($sealedRuntime, $capsule): void { + $now = time() - 600; + $capsule->table('cloud_instance_backups')->insert([ + 'id' => 100 + $id, + 'purpose' => 'migration_bundle', + 'state' => 'ready', + 'object_key' => 'virt/vm-migrations/bundle-' . $id + . '/system.qcow2', + 'sealed_storage_runtime' => $sealedRuntime, + 'sealed_credential' => 'sealed-guest-credential', + 'storage_revision' => 9, + 'active_job_id' => null, + 'error_code' => '', + 'updated_at' => $now, + ]); + $capsule->table('cloud_instance_migrations')->insert([ + 'id' => $id, + 'backup_id' => 100 + $id, + 'state' => $state, + 'error_code' => '', + 'updated_at' => $now, + ]); + for ($index = 1; $index <= $artifactCount; $index++) { + $capsule->table('cloud_migration_bundle_artifacts')->insert([ + 'id' => $id * 10 + $index, + 'migration_id' => $id, + 'object_key' => 'virt/vm-migrations/bundle-' . $id + . '/snapshots/snapshot-' . $index . '.qcow2', + 'state' => 'restored', + 'updated_at' => $now, + ]); + } + }; + + // A completed migration must delete every snapshot transport before the + // root object, then remove its frozen object-store credentials. + $newMigration(1, 'completed'); + $history = []; + $storage = $client([ + new Response(204), + new Response(204), + new Response(204), + ], $history); + for ($step = 0; $step < 3; $step++) { + $assert(CloudMigrationBundleService::cleanupOneEligible( + $storage, + time() + $step + ), 'Successful migration bundle cleanup did not advance.'); + } + $backup = CloudInstanceBackup::findOrFail(101); + $assert(CloudMigrationBundleArtifact::where('migration_id', 1) + ->where('state', 'deleted')->count() === 2 + && (string) $backup->state === 'deleted' + && (string) $backup->sealed_storage_runtime === '' + && $backup->sealed_credential === null, + 'Successful migration retained object-store artifacts or credentials.'); + $assert(array_map( + static fn (array $entry): string => + $entry['request']->getMethod() . ' ' + . $entry['request']->getUri()->getPath(), + $history + ) === [ + 'DELETE /backup-bucket/virt/vm-migrations/bundle-1/snapshots/snapshot-1.qcow2', + 'DELETE /backup-bucket/virt/vm-migrations/bundle-1/snapshots/snapshot-2.qcow2', + 'DELETE /backup-bucket/virt/vm-migrations/bundle-1/system.qcow2', + ], 'Migration transport objects were not deleted snapshot-first/root-last.'); + + // A deliberately retained source is still a successful target migration; + // its object-store transport must not become a permanent hidden backup. + $newMigration(2, 'source_retained', 0); + $retainedHistory = []; + $assert(CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(204)], $retainedHistory), + time() + ), 'Source-retained migration did not clean its transport object.'); + $assert((string) CloudInstanceBackup::findOrFail(102)->state === 'deleted' + && (string) CloudInstanceMigration::findOrFail(2)->state + === 'source_retained', + 'Transport cleanup changed the retained-source migration contract.'); + + // Once DeleteObject has an unknown effect, every automatic follow-up must + // be read-only HEAD reconciliation; it must never replay DELETE. + $newMigration(3, 'completed', 1); + $unknownHistory = []; + $expectCleanupFailure(static function () use ( + $client, + &$unknownHistory + ): void { + CloudMigrationBundleService::cleanupOneEligible($client([ + new ConnectException('transport lost', new Request( + 'DELETE', 'https://objects.example.com' + )), + ], $unknownHistory), time()); + }, 'Unknown DeleteObject effect was not reported.'); + $artifact = CloudMigrationBundleArtifact::findOrFail(31); + $assert((string) $artifact->state === 'delete_uncertain' + && count($unknownHistory) === 1 + && $unknownHistory[0]['request']->getMethod() === 'DELETE', + 'Unknown object deletion did not retain its effect fence.'); + CloudInstanceBackup::where('id', 103)->update([ + 'sealed_storage_runtime' => 'invalid-ciphertext', + ]); + $invalidFenceHistory = []; + $expectCleanupFailure(static function () use ( + $client, + &$invalidFenceHistory + ): void { + CloudMigrationBundleService::cleanupOneEligible( + $client([], $invalidFenceHistory), + time() + 301 + ); + }, 'Invalid runtime did not surface during ambiguous reconciliation.'); + $assert($invalidFenceHistory === [] + && (string) CloudMigrationBundleArtifact::findOrFail(31)->state + === 'delete_uncertain', + 'Invalid runtime downgraded an unknown-effect deletion fence.'); + CloudInstanceBackup::where('id', 103)->update([ + 'sealed_storage_runtime' => $sealedRuntime, + ]); + $presentHistory = []; + $expectCleanupFailure(static function () use ( + $client, + &$presentHistory + ): void { + CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(200)], $presentHistory), + time() + 602 + ); + }, 'Still-present ambiguous object was not reported.'); + $assert((string) CloudMigrationBundleArtifact::findOrFail(31)->state + === 'delete_uncertain' + && count($presentHistory) === 1 + && $presentHistory[0]['request']->getMethod() === 'HEAD', + 'A present ambiguous object caused DeleteObject to be replayed.'); + $absentHistory = []; + $assert(CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(404)], $absentHistory), + time() + 903 + ), 'Absent ambiguous object did not converge to deleted.'); + $assert((string) CloudMigrationBundleArtifact::findOrFail(31)->state + === 'deleted' + && $absentHistory[0]['request']->getMethod() === 'HEAD', + 'HEAD 404 did not safely finish ambiguous object cleanup.'); + + // Finish migration 3's root before proving that a later failed migration + // cannot become eligible by accident. + $thirdRootHistory = []; + $assert(CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(204)], $thirdRootHistory), + time() + 904 + ) && (string) CloudInstanceBackup::findOrFail(103)->state === 'deleted', + 'A reconciled bundle did not finish by deleting its root object.'); + + // A failed migration is not a successful migration and must retain its + // bundle for rollback/recovery investigation. + $newMigration(4, 'rolled_back', 0); + $failedHistory = []; + $assert(!CloudMigrationBundleService::cleanupOneEligible( + $client([], $failedHistory), + time() + 1000 + ) && $failedHistory === [] + && (string) CloudInstanceBackup::findOrFail(104)->state === 'ready', + 'A non-successful migration lost its recovery bundle.'); + + // Successful migration alone is insufficient: the target must have + // restored every snapshot and the root backup must remain ready. + $newMigration(5, 'completed', 1); + CloudMigrationBundleArtifact::where('migration_id', 5)->update([ + 'state' => 'ready', + ]); + $newMigration(6, 'completed', 0); + CloudInstanceBackup::where('id', 106)->update(['state' => 'failed']); + $newMigration(7, 'cleanup_uncertain', 0); + $failClosedHistory = []; + $assert(!CloudMigrationBundleService::cleanupOneEligible( + $client([], $failClosedHistory), + time() + 1100 + ) && $failClosedHistory === [], + 'An incomplete or uncertain migration bundle became deletable.'); + + // A fresh deleting claim belongs to another worker. Once stale, it is an + // unknown effect and must be reconciled with HEAD instead of replaying it. + $newMigration(8, 'completed', 1); + $leaseNow = time() + 2000; + CloudMigrationBundleArtifact::where('migration_id', 8)->update([ + 'state' => 'deleting', + 'updated_at' => $leaseNow, + ]); + $freshHistory = []; + $assert(!CloudMigrationBundleService::cleanupOneEligible( + $client([], $freshHistory), + $leaseNow + 59 + ) && $freshHistory === [], + 'A fresh object-delete lease was stolen by another worker.'); + CloudInstanceBackup::where('id', 108)->update([ + 'sealed_storage_runtime' => 'invalid-ciphertext', + ]); + $staleInvalidHistory = []; + $expectCleanupFailure(static function () use ( + $client, + &$staleInvalidHistory, + $leaseNow + ): void { + CloudMigrationBundleService::cleanupOneEligible( + $client([], $staleInvalidHistory), + $leaseNow + 60 + ); + }, 'Invalid runtime did not surface for a stale delete claim.'); + $assert($staleInvalidHistory === [] + && (string) CloudMigrationBundleArtifact::where( + 'migration_id', 8 + )->value('state') === 'delete_uncertain', + 'A stale unknown-effect claim was downgraded to a replayable state.'); + CloudInstanceBackup::where('id', 108)->update([ + 'sealed_storage_runtime' => $sealedRuntime, + ]); + $staleHistory = []; + $assert(CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(404)], $staleHistory), + $leaseNow + 361 + ) && $staleHistory[0]['request']->getMethod() === 'HEAD', + 'A stale object-delete claim was replayed instead of reconciled.'); + $staleRootHistory = []; + $assert(CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(204)], $staleRootHistory), + $leaseNow + 362 + ) && (string) CloudInstanceBackup::findOrFail(108)->state === 'deleted', + 'The reconciled stale bundle did not delete its root last.'); + + // A definitive provider rejection may be retried, but only after the + // backoff window; this must not form a hot loop in the worker. + $newMigration(9, 'completed', 1); + $rejectedHistory = []; + $expectCleanupFailure(static function () use ( + $client, + &$rejectedHistory, + $leaseNow + ): void { + CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(403)], $rejectedHistory), + $leaseNow + 400 + ); + }, 'A definitive DeleteObject rejection was not reported.'); + $assert((string) CloudMigrationBundleArtifact::where( + 'migration_id', 9 + )->value('state') === 'delete_rejected', + 'A definitive DeleteObject rejection was not persisted.'); + $backoffHistory = []; + $assert(!CloudMigrationBundleService::cleanupOneEligible( + $client([], $backoffHistory), + $leaseNow + 699 + ) && $backoffHistory === [], + 'A rejected DeleteObject entered a retry hot loop.'); + $retryHistory = []; + $assert(CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(204)], $retryHistory), + $leaseNow + 700 + ) && $retryHistory[0]['request']->getMethod() === 'DELETE', + 'A definitively rejected delete was not retried after backoff.'); + $retryRootHistory = []; + $assert(CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(204)], $retryRootHistory), + $leaseNow + 701 + ), 'The retry test bundle root was not deleted.'); + + // Root deletion has the same unknown-effect fence: runtime damage after a + // lost DELETE response cannot turn it back into a replayable operation. + $newMigration(10, 'completed', 0); + $rootUnknownHistory = []; + $expectCleanupFailure(static function () use ( + $client, + &$rootUnknownHistory, + $leaseNow + ): void { + CloudMigrationBundleService::cleanupOneEligible($client([ + new ConnectException('transport lost', new Request( + 'DELETE', 'https://objects.example.com' + )), + ], $rootUnknownHistory), $leaseNow + 800); + }, 'Unknown root DeleteObject effect was not reported.'); + $assert((string) CloudInstanceBackup::findOrFail(110)->state + === 'delete_uncertain' + && $rootUnknownHistory[0]['request']->getMethod() === 'DELETE', + 'Unknown root deletion did not retain its effect fence.'); + CloudInstanceBackup::where('id', 110)->update([ + 'sealed_storage_runtime' => 'invalid-ciphertext', + ]); + $rootInvalidHistory = []; + $expectCleanupFailure(static function () use ( + $client, + &$rootInvalidHistory, + $leaseNow + ): void { + CloudMigrationBundleService::cleanupOneEligible( + $client([], $rootInvalidHistory), + $leaseNow + 1101 + ); + }, 'Invalid runtime did not surface for ambiguous root cleanup.'); + $assert($rootInvalidHistory === [] + && (string) CloudInstanceBackup::findOrFail(110)->state + === 'delete_uncertain', + 'Invalid runtime downgraded an ambiguous root deletion fence.'); + CloudInstanceBackup::where('id', 110)->update([ + 'sealed_storage_runtime' => $sealedRuntime, + ]); + $rootAbsentHistory = []; + $assert(CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(404)], $rootAbsentHistory), + $leaseNow + 1402 + ) && (string) CloudInstanceBackup::findOrFail(110)->state === 'deleted' + && $rootAbsentHistory[0]['request']->getMethod() === 'HEAD', + 'Ambiguous root deletion did not converge with HEAD only.'); + + // A corrupt frozen runtime is isolated with backoff and cannot starve a + // later valid bundle forever. + $newMigration(11, 'completed', 0); + CloudInstanceBackup::where('id', 111)->update([ + 'sealed_storage_runtime' => 'invalid-ciphertext', + ]); + $newMigration(12, 'completed', 0); + $invalidHistory = []; + $expectCleanupFailure(static function () use ( + $client, + &$invalidHistory, + $leaseNow + ): void { + CloudMigrationBundleService::cleanupOneEligible( + $client([], $invalidHistory), + $leaseNow + 1500 + ); + }, 'An invalid frozen runtime was not reported.'); + $assert($invalidHistory === [] + && (string) CloudInstanceBackup::findOrFail(111)->state + === 'delete_rejected', + 'An invalid frozen runtime was not isolated behind backoff.'); + $laterHistory = []; + $assert(CloudMigrationBundleService::cleanupOneEligible( + $client([new Response(204)], $laterHistory), + $leaseNow + 1501 + ) && (string) CloudInstanceBackup::findOrFail(112)->state === 'deleted', + 'One invalid runtime starved a later valid migration bundle.'); + + echo "Maintenance migration bundle cleanup and reconciliation passed.\n"; +} diff --git a/web/tests/CloudPortableBackupDeletionTest.php b/web/tests/CloudPortableBackupDeletionTest.php index 9c5290c6..a8e81fff 100644 --- a/web/tests/CloudPortableBackupDeletionTest.php +++ b/web/tests/CloudPortableBackupDeletionTest.php @@ -210,6 +210,17 @@ namespace { && $presentHistory[0]['request']->getMethod() === 'HEAD', 'An existing object after ambiguous deletion was replayed or lost its fence.'); + $migrationBundle = $newBackup(15, 'migration-bundle'); + $migrationBundle->purpose = 'migration_bundle'; + $migrationBundle->save(); + $expectConflict(static function () use ($migrationBundle, $actor): void { + CloudBackupService::delete($migrationBundle, $actor); + }); + $migrationBundle->refresh(); + $assert((string) $migrationBundle->state === 'ready' + && (string) $migrationBundle->sealed_storage_runtime !== '', + 'The ordinary backup endpoint modified a hidden migration bundle.'); + $blocked = $newBackup(13, 'four'); $restore = new CloudBackupRestore(); $restore->backup_id = (int) $blocked->id; diff --git a/web/tests/CloudRuntimeHeartbeatTest.php b/web/tests/CloudRuntimeHeartbeatTest.php index ed53b61b..22594561 100644 --- a/web/tests/CloudRuntimeHeartbeatTest.php +++ b/web/tests/CloudRuntimeHeartbeatTest.php @@ -330,6 +330,29 @@ try { $table->integer('updated_at')->default(0); $table->integer('sent_at')->nullable(); }); + $schema->create('cloud_instance_backups', static function ( + Blueprint $table + ): void { + $table->increments('id'); + $table->string('purpose'); + $table->string('state'); + $table->integer('updated_at'); + }); + $schema->create('cloud_instance_migrations', static function ( + Blueprint $table + ): void { + $table->increments('id'); + $table->integer('backup_id'); + $table->string('state'); + }); + $schema->create('cloud_migration_bundle_artifacts', static function ( + Blueprint $table + ): void { + $table->increments('id'); + $table->integer('migration_id'); + $table->string('state'); + $table->integer('updated_at'); + }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); foreach (['status', 'service_status', 'uuid', 'renewal_price'] as $column) { @@ -387,6 +410,10 @@ try { $xcat = file_get_contents(dirname(__DIR__) . '/app/Command/XCat.php'); $assert(str_contains($workerCommand, "'--recovery-only'") && str_contains($workerCommand, 'CloudAutomaticOperationRecoveryService') + && str_contains($workerCommand, + 'CloudMigrationBundleService::cleanupOneEligible()') + && str_contains($workerCommand, + "if (!\$policyOnly && !\$recoveryOnly)") && str_contains($workerCommand, '!$policyOnly && !$recoveryOnly') && str_contains($xcat, '--recovery-only') && str_contains($workerUnit, 'cloudWorker --recovery-only'), @@ -483,6 +510,40 @@ try { $assert($worker->tick() === false && $worker->observationErrors() === [], 'Worker observation errors were not reset for the next idle cycle.'); + // Migration transport cleanup is an independent best-effort worker stage: + // its own schema/runtime failure is observable, but cannot rewrite a + // migration already confirmed successful or stop policy/recovery workers. + Capsule::table('cloud_instance_backups')->insert([ + 'id' => 1, 'purpose' => 'migration_bundle', 'state' => 'ready', + 'updated_at' => time(), + ]); + Capsule::table('cloud_instance_migrations')->insert([ + 'id' => 1, 'backup_id' => 1, 'state' => 'completed', + ]); + $schema->drop('cloud_migration_bundle_artifacts'); + $cleanupFailureDirectory = $testRoot . '/cleanup-stage-failure'; + $cleanupFailure = CloudRuntimeHeartbeat::start( + 'normal', true, $cleanupFailureDirectory, $clock + ); + $assert(CloudWorker::run(['--once'], $cleanupFailure) === 0 + && $read($cleanupFailureDirectory)[0]['last_error'] + === 'migration_bundle_cleanup_failed' + && Capsule::table('cloud_instance_migrations')->value('state') + === 'completed', + 'Cleanup-stage failure changed migration truth or stopped the worker.'); + foreach (['policy', 'recovery'] as $isolatedKind) { + $isolatedDirectory = $testRoot . '/cleanup-stage-' . $isolatedKind; + $isolatedHeartbeat = CloudRuntimeHeartbeat::start( + $isolatedKind, true, $isolatedDirectory, $clock + ); + $arguments = $isolatedKind === 'policy' + ? ['--once', '--policy-only'] + : ['--once', '--recovery-only']; + $assert(CloudWorker::run($arguments, $isolatedHeartbeat) === 0 + && $read($isolatedDirectory)[0]['last_error'] === null, + 'A policy/recovery worker attempted migration-bundle cleanup.'); + } + echo "Cloud runtime private atomic files, bounded snapshots, long-call freshness, multi-worker identity, --once CLI and SQLite partial failures passed.\n"; } finally { ini_set('error_log', $oldLog === false ? '' : $oldLog);