From c13468f054beca4e7d740baee22e5d7e054a1452 Mon Sep 17 00:00:00 2001 From: codex Date: Fri, 30 Oct 2026 19:56:35 +0800 Subject: [PATCH] feat(template): make deletion two phase --- README.md | 2 +- docs/INTEGRATION_PLAN.md | 2 +- docs/TASK_EXECUTION.md | 2 + docs/VALIDATION.md | 1 + web/app/Command/CloudSetup.php | 6 +- web/app/Models/CloudTemplateArtifact.php | 6 + web/app/Services/Cloud/CloudJobService.php | 22 + .../Services/Cloud/CloudLifecycleService.php | 15 + .../Services/Cloud/CloudRuntimeHeartbeat.php | 2 + .../Cloud/CloudTemplateArtifactManager.php | 54 ++ .../Cloud/CloudTemplateCatalogService.php | 229 +++--- .../Cloud/CloudTemplateDeletionService.php | 742 ++++++++++++++++++ web/public/cloud/admin.js | 14 +- web/resources/views/venix/layout_assets.tpl | 2 +- web/sql/cloud.sql | 10 +- .../CloudAggregateRecoveryConvergenceTest.php | 130 ++- ...dAutomaticOperationRecoveryServiceTest.php | 66 +- web/tests/CloudRuntimeHeartbeatTest.php | 7 + .../CloudTemplateArtifactManagerTest.php | 7 + web/tests/CloudTemplateCatalogTest.php | 8 + .../CloudTemplateDeletionServiceTest.php | 354 +++++++++ web/tests/CloudTemplateErrorContractTest.php | 14 +- 22 files changed, 1529 insertions(+), 166 deletions(-) create mode 100644 web/app/Services/Cloud/CloudTemplateDeletionService.php create mode 100644 web/tests/CloudTemplateDeletionServiceTest.php diff --git a/README.md b/README.md index 1c9edba1..f2bc5e1b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Virt Cloud 是 PHP 8.5 面板与宿主机 Go Agent 组成的虚拟机销售平 - Agent 以 `panel_url + agent_id + agent_secret` 主动建立 WSS 控制长连接。PHP 先持久化任务并立即向浏览器返回 `202 Accepted`,再通过面板本机 Go Agent Gateway 派发一次带 SHA-256、时间戳、nonce、操作者上下文和 HMAC-SHA256 签名的请求。断线、超时或重启都不会重新派发 mutation。维护迁移的数据不经过面板:同资源池宿主机的 Agent 在 `peer_migration_port`(默认 `9443/TCP`)之间使用 TLS 1.3、一次性票据和源证书 SHA-256 锁定进行目标拉取;该端口只提供路径隔离的 PeerMigration V3 清单和工件 Range 读取,不暴露 libvirt 或通用管理 API。 - 同地区 VPC 使用宿主机既有私网作为 direct L3 underlay。PHP 负责地区 CIDR 池、VPC/IPAM、逻辑端口、宿主机完整路由投影和迁移编排;Agent 只按签名期望状态管理专属 bridge、精确 `/32` 路由、nftables 隔离规则以及一张可选 VM 私网网卡。VPC 不提供跨地区互联、NAT 或二层广播,也不改变公网 `br0`、公网防火墙和公网流量计费边界。 - 所有虚拟机使用宿主机能力支持的 Q35 UEFI,并明确关闭 Secure Boot。不要硬编码某个发行版的 OVMF 路径。 -- 每个云镜像可配置仅首次安装执行的启动脚本。创建任务会快照脚本版本;脚本成功、失败或中断后都不会自动重放,租户不能读取或覆盖脚本正文。 +- 每个云镜像版本拥有对象存储中的不可变主副本,并按地区/宿主机维护派生副本;可配置仅首次安装执行的启动脚本。创建任务会快照镜像代次和脚本版本;脚本成功、失败或中断后都不会自动重放,租户不能读取或覆盖脚本正文。删除先确认全部宿主机副本已清理,再删除主副本的精确对象版本;对象删除结果不明确时只做 HEAD 核对,不盲目重放 DELETE。 - 用户购买套餐开通一台 VM;每次购买或续费生成独立预付费账期及流量额度。新建实例账单的支付期限由后台按分钟配置,只影响之后生成的开通账单;续费账单的支付截止固定为其对应实例到期时间加三天,不会因账单延迟生成而顺延。额度用尽只断网、不关机;下一已付费账期恢复网络。最终到期只断网;未支付续费账单进入三天宽限期,之后永久删除 VM 定义和面板管理的磁盘。 - 用户页面及登录页保持原虚拟机面板样式,管理员页面保留 venix 主题。普通用户不获得宿主机管理、Agent 设置或退役节点管理入口。 - 任务只有执行中、成功和失败三种公开状态,终态只有 `succeeded` 或 `failed`;所有任务都有绝对截止时间。派发后超时或断线直接终态失败,并标记“效果可能已发生、不可重放原 operation ID”;任务不会停在“待核对”,也不会自动重试、退款或再次删除磁盘。实例设置可以用精确晚到回执,或 Agent 对同一 operation 的只读双标记证据,收敛 PHP 凭据/主机名账本与内部 effect fence;原任务仍保持失败,原宿主机 mutation 永不重放。不能可靠受 Go context 中止的 libvirt mutation 在独立子进程中执行,截止后由父 Agent 回收整个进程组。固定存储池配置是非破坏性的幂等例外:失败任务仍终止且不重放,但不保留跨任务资源栅栏;已定义的固定 pool 不做失败回滚,管理员可显式提交新任务核验或补全同路径固定池,绝不改绑、迁移或删除数据。 diff --git a/docs/INTEGRATION_PLAN.md b/docs/INTEGRATION_PLAN.md index 8416f2aa..cc91487a 100644 --- a/docs/INTEGRATION_PLAN.md +++ b/docs/INTEGRATION_PLAN.md @@ -84,7 +84,7 @@ Agent 在连接前注册并持续运行 libvirt event loop;每个连接 genera - 域 XML 使用结构化类型生成并验证输入,所有变更按 UUID 定位。创建请求的 `name` 保留为面板/Guest 显示输入并写入 `display-name` metadata;libvirt 内部名称只能由已验签 actor 的 `user_id` 与最终 UUID 派生为 `:vm-<前 12 位小写无连字符 UUID>`,不接受请求正文覆盖。部署每台宿主机时先查询 domain capabilities,把该机支持且不启用 Secure Boot 的 Q35/EFI loader 与 NVRAM template 写入本机 Agent 配置;创建请求不能传入任意固件路径。Agent 使用手工 loader/NVRAM XML 并保持 `secure="no"`、`SMM=off`,不混入只适用于固件自动选择的 feature 列表,也不在 PHP 中硬编码单一发行版 OVMF 路径。 - 宿主机网络接入由 PHP 面板统一保存:网页默认 Linux 网桥 `br0`,`default_network` 使用 `bridge:<名称>` 或 `network:<名称>`;Agent 配置不再接受 `network_selectors` 映射。Agent 只发现、验证和使用已存在且活动的 Linux 网桥或 libvirt network,不修改宿主机系统网络。`capabilities.network_selectors` 返回排序去重、最多 64 项的可用接入列表;面板所选值必须精确匹配,创建/换网时再次重验。 - Agent 配置使用 `revision`/`tested_revision` 就绪门闩。保存密钥、启用状态或宿主机 `default_network` 会推进 revision 并将 `tested_revision` 清零;只有同一 revision 经已认证出站会话完成 `/v1/health` 与 `/v1/capabilities` 且网络匹配,才记录为就绪。调度、Agent 客户端以及控制台路径均要求 `enabled=1`、`last_test_status=healthy` 且 `tested_revision=revision`。全新部署只接受规范的 `bridge:<名称>` 或 `network:<名称>`,不读取旧 alias、旧健康结果或已派发旧任务。 -- 云镜像由 PHP 保存目录元数据和启动脚本。创建任务快照镜像 generation、revision 和脚本正文后签名发送给目标 Agent;删除后重新导入的同名镜像不能继承旧脚本。 +- 云镜像由 PHP 保存目录元数据、启动脚本以及对象存储中不可变主副本的精确身份,宿主机只保存可重建的派生副本。创建任务快照镜像 generation、revision 和脚本正文后签名发送给目标 Agent;删除后重新导入的同名镜像不能继承旧脚本。删除采用两阶段收敛:先证明全部宿主机副本已删除,再删除主副本的精确对象版本;主副本清理完成前目录和原聚合任务保持占用。 - 启动脚本仅用于新云镜像实例的首次安装,不用于普通重启、网络设置、重装、救援或用户 ISO。脚本限制 32 KiB UTF-8,拒绝 NUL,CRLF 规范化为 LF,空值表示禁用。 - Agent 将脚本以 root 私有文件写入 cloud-init seed,由一次性 runner 原子记录 started 状态。成功、失败和中断都不自动重放;VM 创建成功不等待脚本结束。 - 镜像下载始终使用独立 HTTPS 请求,WSS 只传递一次性下载 URL/token/摘要等任务元数据,不传镜像字节。面板暂存上传还会把已捕获文件的精确大小写入签名任务;即使代理改用 chunked 响应,Agent 也按该大小有界读取,并拒绝长度不符、多余字节、短读或摘要不符。远程 URL 没有面板大小快照,仍要求来源返回有效 `Content-Length`。Agent 限制来源、解析后的公网地址、全部跳转、大小、超时和摘要;校验 QCOW2 格式,拒绝 backing file、外部 data file、加密或 raw 内容。 diff --git a/docs/TASK_EXECUTION.md b/docs/TASK_EXECUTION.md index b3d3844c..47a53de8 100644 --- a/docs/TASK_EXECUTION.md +++ b/docs/TASK_EXECUTION.md @@ -90,6 +90,8 @@ PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之 模板/用户 ISO 导入和删除的多宿主结果可以合法地部分成功。所有叶子均有确定效果、且副本身份与投影一致时,父任务保留原失败审计,并以 `effect_resolution=applied` 加 `effect_scope=partial` 表示“部分宿主机已应用,全部效果已知”,清除 incident 与目录 `active_job_id`。这不等于全量成功,也不把不完整镜像发布为可用;用户可显式创建新的删除任务,仅处理确认仍存在的副本,已不存在的副本不会重发删除。原父/子任务不会自动补偿或重放,仍有未知叶子或投影矛盾时继续保锁。 +云镜像全量删除额外包含对象存储主副本阶段。全部 Agent 叶子明确删除后,汇总父任务仍不能发布成功,也不能移除副本账本;PHP 先把精确 `object_key + version_id + size + ETag` 清理债务写入 `cloud_template_artifacts`。对象 DELETE 返回明确成功或同一精确版本 HEAD 明确不存在后,才在一个事务中删除主副本账本和宿主机副本账本、将目录置为 `retired` 并完成父任务。DELETE 超时、断线或 5xx 不得重放原请求;必须间隔至少五分钟得到两次同一对象身份仍存在的 HEAD 证据,才创建新的删除代次,每轮最多三代。身份变化、明确拒绝或超限进入 `delete_attention`,管理员入口只重启主副本核对,不重新派发已经完成的 Agent 删除。 + ## 非任务循环 服务守护循环、健康采集和只读指标刷新可以长期运行,但每一轮必须有独立超时、允许失败后返回主循环,且不能持有某个 mutation 的数据库租约、资源锁或浏览器忙状态。它们不属于任务重试,不得借此重新提交宿主机变更。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index c0aacfe1..8c356f79 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -91,6 +91,7 @@ jq empty deploy/*.json.example 六类恢复回归还必须覆盖: - 模板/ISO 导入与删除 A 宿主已应用、B 明确未应用:保留失败审计,显示部分结果,清除父任务占用;新的显式删除仅派发至仍有副本的宿主。未知叶子、错绑副本和伪造投影仍拒绝解锁。 +- 云镜像全部宿主机副本已删除但主副本仍存在:父任务不得提前发布成功,副本账本不得提前删除。对象 DELETE 明确成功或精确版本 HEAD 返回不存在后才原子退休目录;DELETE 结果不明确时原 operation 不重放,需两次间隔 HEAD 证明相同大小、ETag 和 version ID 后才生成新代次。连续三代仍不明确、对象身份变化或明确拒绝必须进入管理员核对;管理员重试只处理对象存储,不得再次调用 Agent。晚到宿主机成功回执也必须进入同一两阶段边界。 - 持续填充回执、叶子与聚合队列:三个通道均在有界 tick 内获得处理机会,空通道不能浪费整轮。 - Gateway 未监听及连接阶段超时且零字节发送:明确未执行并原子收尾;已发送后超时、连接复用、缺统计、多个传输中任一可能发送均不能当作未派发。开通任务只退款一次,不重复派发。 - 快照占用及各变更入口写入前校验失败:Journal 记录无副作用失败,同资源新 operation 可以进入;写调用之后及显式 uncertain 错误继续保留保护。 diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index 380708dd..226ec876 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -52,7 +52,10 @@ final class CloudSetup 'cloud_template_artifacts' => [ 'template_id', 'storage_revision', 'sealed_storage_runtime', 'object_key', 'object_version_id', 'object_etag', 'size_bytes', - 'sha256', 'state', 'error_code', 'verified_at', + 'sha256', 'state', 'error_code', 'delete_operation_id', + 'delete_generation', 'delete_cycle_attempts', + 'delete_presence_confirmations', 'delete_last_checked_at', + 'delete_next_check_at', 'delete_attention_at', 'verified_at', 'created_at', 'updated_at', ], 'cloud_template_replicas' => [ @@ -298,6 +301,7 @@ final class CloudSetup 'cloud_template_artifacts' => [ 'PRIMARY', 'cloud_template_artifact_object', 'cloud_template_artifact_state', + 'cloud_template_artifact_delete_due', ], 'cloud_template_replicas' => [ 'PRIMARY', 'cloud_template_replica_ready', diff --git a/web/app/Models/CloudTemplateArtifact.php b/web/app/Models/CloudTemplateArtifact.php index b30f1e36..028e10c0 100644 --- a/web/app/Models/CloudTemplateArtifact.php +++ b/web/app/Models/CloudTemplateArtifact.php @@ -14,6 +14,12 @@ final class CloudTemplateArtifact extends Model protected $casts = [ 'storage_revision' => 'int', 'size_bytes' => 'int', + 'delete_generation' => 'int', + 'delete_cycle_attempts' => 'int', + 'delete_presence_confirmations' => 'int', + 'delete_last_checked_at' => 'int', + 'delete_next_check_at' => 'int', + 'delete_attention_at' => 'int', 'verified_at' => 'int', 'created_at' => 'int', 'updated_at' => 'int', diff --git a/web/app/Services/Cloud/CloudJobService.php b/web/app/Services/Cloud/CloudJobService.php index 9f777da0..7f18bf12 100644 --- a/web/app/Services/Cloud/CloudJobService.php +++ b/web/app/Services/Cloud/CloudJobService.php @@ -1182,6 +1182,17 @@ final class CloudJobService $resolution ); } + if (!CloudTemplateCatalogService::aggregateEffectProjectionReadyLocked( + $parent, + $resolution + )) { + self::deferAggregateRecoveryLocked( + $parent, + 'recovery_child_convergence', + time() + ); + return; + } $publishedResolution = $metadataMixed || $partialMixed ? 'applied' : $resolution; $parent->effect_resolution = $publishedResolution; @@ -1436,6 +1447,17 @@ final class CloudJobService $resolution ); } + if (!CloudTemplateCatalogService::aggregateEffectProjectionReadyLocked( + $parent, + $resolution + )) { + self::deferAggregateRecoveryLocked( + $parent, + 'recovery_child_convergence', + $now + ); + return true; + } $publishedResolution = $metadataMixed || $partialMixed ? 'applied' : $resolution; $lastChild = $children->last(); diff --git a/web/app/Services/Cloud/CloudLifecycleService.php b/web/app/Services/Cloud/CloudLifecycleService.php index 26c872cc..2559e860 100644 --- a/web/app/Services/Cloud/CloudLifecycleService.php +++ b/web/app/Services/Cloud/CloudLifecycleService.php @@ -74,6 +74,21 @@ final class CloudLifecycleService error_log('cloud automatic backup retention unavailable: ' . get_class($error)); } + $this->heartbeat?->busy('template_artifact_cleanup'); + try { + $cleanup = CloudTemplateDeletionService::processDue( + min(3, $automaticLimit) + ); + $queued += (int) $cleanup['completed']; + if ((int) $cleanup['errors'] > 0 + || (int) $cleanup['attention'] > 0) { + $this->observeFailure('template_artifact_cleanup_failed'); + } + } catch (\Throwable $error) { + $this->observeFailure('template_artifact_cleanup_failed'); + error_log('cloud template artifact cleanup unavailable: ' + . get_class($error)); + } $now = time(); if ($now >= $this->templateReconcileAfter) { $this->templateReconcileAfter = $now + 300; diff --git a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php index 211f3c6a..0b79d735 100644 --- a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php +++ b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php @@ -15,6 +15,7 @@ final class CloudRuntimeHeartbeat 'provision_expiry' => '开通账单过期处理', 'invoice_generation' => '续费账单生成', 'automatic_backup_schedule' => '自动备份调度', 'automatic_backup_retention' => '自动备份保留数清理', + 'template_artifact_cleanup' => '云镜像主副本清理', 'template_reconcile' => '云镜像副本核对', 'traffic_sync' => '流量同步', 'runtime_status_sync' => '实例状态同步', @@ -25,6 +26,7 @@ final class CloudRuntimeHeartbeat 'lifecycle_tick_failed', 'provision_expiry_failed', 'invoice_generation_failed', 'automatic_backup_schedule_failed', 'automatic_backup_retention_failed', + 'template_artifact_cleanup_failed', 'template_reconcile_failed', 'traffic_sync_failed', 'runtime_status_sync_failed', 'runtime_status_sync_unconfigured', 'traffic_collection_unconfigured', diff --git a/web/app/Services/Cloud/CloudTemplateArtifactManager.php b/web/app/Services/Cloud/CloudTemplateArtifactManager.php index 0f7e7d96..b983588f 100644 --- a/web/app/Services/Cloud/CloudTemplateArtifactManager.php +++ b/web/app/Services/Cloud/CloudTemplateArtifactManager.php @@ -48,6 +48,13 @@ final class CloudTemplateArtifactManager $artifact->sha256 = (string) $template->sha256; $artifact->state = 'publishing'; $artifact->error_code = ''; + $artifact->delete_operation_id = ''; + $artifact->delete_generation = 0; + $artifact->delete_cycle_attempts = 0; + $artifact->delete_presence_confirmations = 0; + $artifact->delete_last_checked_at = 0; + $artifact->delete_next_check_at = 0; + $artifact->delete_attention_at = null; $artifact->verified_at = 0; $artifact->created_at = $artifact->updated_at = $now; $artifact->save(); @@ -167,6 +174,53 @@ final class CloudTemplateArtifactManager ]; } + /** Inspect the exact immutable version without exposing storage secrets. */ + public static function inspectDeletionTarget( + CloudTemplateArtifact $artifact, + ?CloudObjectStorageS3Client $client = null + ): ?array { + $runtime = self::runtime($artifact); + return ($client ?? new CloudObjectStorageS3Client())->inspectObject( + $runtime['config'], + $runtime['credentials'], + (string) $artifact->object_key, + (string) $artifact->object_version_id + ); + } + + /** Delete only the frozen provider version recorded by this artifact. */ + public static function deleteExact( + CloudTemplateArtifact $artifact, + ?CloudObjectStorageS3Client $client = null + ): void { + $runtime = self::runtime($artifact); + ($client ?? new CloudObjectStorageS3Client())->deleteObject( + $runtime['config'], + $runtime['credentials'], + (string) $artifact->object_key, + (string) $artifact->object_version_id + ); + } + + /** Refuse cleanup when the provider version no longer matches the ledger. */ + public static function deletionIdentityMatches( + CloudTemplateArtifact $artifact, + array $identity + ): bool { + return (int) ($identity['size_bytes'] ?? -1) + === (int) $artifact->size_bytes + && is_string($identity['etag'] ?? null) + && hash_equals( + (string) $artifact->object_etag, + (string) $identity['etag'] + ) + && is_string($identity['version_id'] ?? null) + && hash_equals( + (string) $artifact->object_version_id, + (string) $identity['version_id'] + ); + } + private static function recordPublishFailure(string $templateId, string $code, bool $unknown): void { diff --git a/web/app/Services/Cloud/CloudTemplateCatalogService.php b/web/app/Services/Cloud/CloudTemplateCatalogService.php index 2aee7a85..f7397d0f 100644 --- a/web/app/Services/Cloud/CloudTemplateCatalogService.php +++ b/web/app/Services/Cloud/CloudTemplateCatalogService.php @@ -6,6 +6,7 @@ use App\Models\CloudJob; use App\Models\CloudHost; use App\Models\CloudOrder; use App\Models\CloudTemplate; +use App\Models\CloudTemplateArtifact; use App\Models\CloudTemplateReplica; use App\Models\CloudUserIsoReplica; @@ -29,6 +30,11 @@ final class CloudTemplateCatalogService 'template_id', $templateIds )->orderBy('template_id')->orderBy('host_id')->get(); + $artifacts = $templateIds === [] ? collect() + : CloudTemplateArtifact::query()->whereIn( + 'template_id', + $templateIds + )->get()->keyBy('template_id'); $eligibleHostIds = array_fill_keys(array_map( 'intval', CloudAgentSettingsService::enabledHostIds() @@ -70,7 +76,8 @@ final class CloudTemplateCatalogService static fn (CloudTemplate $row): array => self::adminDto( $row, $replicasByTemplate[(string) $row->id] ?? [], - $hosts + $hosts, + $artifacts->get((string) $row->id) ) )->all(), ]; @@ -401,9 +408,9 @@ final class CloudTemplateCatalogService $id = self::templateId($id); self::validateVersion($expectedRevision, $generationToken); $connection = (new CloudTemplate())->getConnection(); - $parent = $connection->transaction(static function () use ( + $reserved = $connection->transaction(static function () use ( $id, $expectedRevision, $generationToken, $actor - ): CloudJob { + ): array { // Payment already owns its order before it snapshots a template. // Take the same order -> aggregate parent -> template sequence so // template deletion cannot deadlock with invoice settlement. @@ -419,14 +426,33 @@ final class CloudTemplateCatalogService && (string) $template->state === 'degraded') || ($partialCleanupKind === 'template_delete' && (string) $template->state === 'delete_failed'); - if ((string) $template->state !== 'active' && !$partialCleanup) { - throw new CloudError('云镜像存在未完成或失败任务,禁止删除', 409); - } if ((int) $template->revision !== $expectedRevision || !hash_equals((string) $template->generation_token, (string) $generationToken)) { throw new CloudError('云镜像版本已变化,请刷新后重试', 409); } + $artifact = CloudTemplateArtifact::query() + ->where('template_id', $id)->lockForUpdate()->first(); + if ((string) $template->state === 'delete_failed' + && $artifact !== null + && (string) $artifact->state === 'delete_attention') { + $activeParent = CloudJob::query() + ->where('id', (int) $template->active_job_id) + ->where('kind', 'template_delete') + ->where('execution_mode', 'aggregate') + ->lockForUpdate()->first(); + if ($activeParent === null) { + throw new CloudError('云镜像删除任务账本不完整', 409); + } + CloudTemplateDeletionService::retryAttentionLocked( + $template, + $activeParent + ); + return ['job' => $activeParent, 'restarted' => true]; + } + if ((string) $template->state !== 'active' && !$partialCleanup) { + throw new CloudError('云镜像存在未完成或失败任务,禁止删除', 409); + } self::assertNoJobReferencesLocked($id); $replicas = CloudTemplateReplica::query() ->where('template_id', $id)->orderBy('host_id') @@ -453,7 +479,7 @@ final class CloudTemplateCatalogService ] ); if (!$created) { - return $parent; + return ['job' => $parent, 'restarted' => false]; } if ($partialCleanupKind === 'template_import') { // A failed import replica is proven absent. Drop that catalog @@ -516,9 +542,11 @@ final class CloudTemplateCatalogService $replica->updated_at = time(); $replica->save(); } - return $parent; + return ['job' => $parent, 'restarted' => false]; }, 3); - return CloudJobService::acknowledgement($parent); + return CloudJobService::acknowledgement($reserved['job']) + [ + 'recovery_restarted' => (bool) $reserved['restarted'], + ]; } public static function finishJobLocked(CloudJob $job, string $status, @@ -570,7 +598,10 @@ final class CloudTemplateCatalogService ); } } else { - self::finishDeleteReplica($template, $replica, $status, + CloudTemplateDeletionService::finishReplicaLocked( + $template, + $replica, + $status, $operation, $recovery); } } @@ -641,11 +672,14 @@ final class CloudTemplateCatalogService $operation ); } else { - self::settleDeleteParent( - $template, - (int) $parent->id, - $operation - ); + if ($resolution === 'mixed') { + CloudTemplateDeletionService::settleMixedLocked( + $template, + $parent + ); + } else { + CloudTemplateDeletionService::armLocked($template, $parent); + } } if ($resolution === 'mixed' && (string) $parent->kind !== 'template_metadata_sync') { @@ -736,6 +770,15 @@ final class CloudTemplateCatalogService ); } + if ($kind === 'template_delete' && $resolution === 'applied') { + CloudTemplateDeletionService::assertSettledProjectionLocked( + $template, + $parent, + $resolution + ); + return; + } + if ($resolution === 'mixed' && $kind !== 'template_metadata_sync') { $expectedState = $kind === 'template_import' @@ -840,6 +883,27 @@ final class CloudTemplateCatalogService } } + /** Do not publish aggregate applied until local master cleanup is durable. */ + public static function aggregateEffectProjectionReadyLocked( + CloudJob $parent, + string $resolution + ): bool { + if ((string) $parent->kind !== 'template_delete' + || $resolution !== 'applied') { + return true; + } + $template = self::templateBoundToParentLocked($parent); + return (string) $template->state === 'retired' + && !CloudTemplateReplica::query()->where( + 'template_id', + (string) $template->id + )->lockForUpdate()->exists() + && !CloudTemplateArtifact::query()->where( + 'template_id', + (string) $template->id + )->lockForUpdate()->exists(); + } + /** * A recovered aggregate is stricter than ordinary incremental completion: * every child must own exactly one replica with the same host, operation @@ -850,6 +914,14 @@ final class CloudTemplateCatalogService CloudJob $parent, string $resolution ): void { + if ((string) $parent->kind === 'template_delete') { + CloudTemplateDeletionService::assertChildProjectionLocked( + $template, + $parent, + $resolution + ); + return; + } $children = CloudJob::where('parent_job_id', (int) $parent->id) ->orderBy('id')->lockForUpdate()->get(); $replicas = self::lockedReplicas((string) $template->id); @@ -966,29 +1038,9 @@ final class CloudTemplateCatalogService return; } if ((string) $parent->kind === 'template_delete') { - $replicas = CloudTemplateReplica::query()->where( - 'template_id', - (string) $template->id - )->orderBy('host_id')->lockForUpdate()->get(); - foreach ($replicas as $replica) { - CloudTemplateStateMachine::moveReplica( - $replica, - (int) $replica->revision === (int) $template->revision - ? 'active' : 'stale' - ); - $replica->error_code = ''; - $replica->updated_at = time(); - $replica->save(); - } - CloudTemplateStateMachine::moveTemplate($template, 'active'); - $template->updated_at = time(); - $template->save(); - CloudJobService::completeAggregateLocked( - $parent, - 'failed', - false, - true, - 'template_delete_not_applied' + CloudTemplateDeletionService::restoreNotAppliedLocked( + $template, + $parent ); return; } @@ -1213,92 +1265,6 @@ final class CloudTemplateCatalogService false, 'template_metadata_sync_failed'); } - private static function finishDeleteReplica(CloudTemplate $template, - CloudTemplateReplica $replica, string $status, - array $operation, bool $recovery): void - { - if ((string) $replica->state !== 'deleting' - && !($recovery - && (string) $replica->state === 'delete_failed')) { - throw new CloudError( - '云镜像副本状态与删除任务恢复证据不一致', - 409 - ); - } - if ($status === 'succeeded') { - if (!CloudAgentCatalogDeletion::confirms( - $operation, 'template', (string) $template->id - )) { - throw new CloudError('Agent 云镜像删除回执缺少清理证据', 502); - } - CloudTemplateStateMachine::moveReplica( - $replica, - 'delete_succeeded' - ); - $replica->error_code = ''; - } else { - CloudTemplateStateMachine::moveReplica($replica, 'delete_failed'); - $replica->error_code = self::errorCode( - $operation, 'template_delete_failed' - ); - } - CloudTemplateReplica::query() - ->where('host_id', (int) $replica->host_id) - ->where('template_id', (string) $replica->template_id) - ->where('job_id', (int) $replica->job_id)->update([ - 'state' => (string) $replica->state, - 'error_code' => (string) $replica->error_code, - 'updated_at' => time(), - ]); - } - - private static function settleDeleteParent(CloudTemplate $template, - int $parentId, array $operation, - ?int $completingChildId = null): void - { - $replicas = self::lockedReplicas((string) $template->id); - if ($replicas->isEmpty() || $replicas->contains(static fn ($row): bool => - (string) $row->state === 'deleting')) { - return; - } - $parent = self::parent($parentId, 'template_delete'); - self::assertTemplateParentBindingLocked($template, $parent); - if (CloudJobService::aggregateParentRecoveryPending( - $parent, - $completingChildId, - true - )) { - return; - } - $success = !$replicas->contains(static fn ($row): bool => - (string) $row->state !== 'delete_succeeded'); - if ($success) { - CloudTemplateReplica::query()->where( - 'template_id', (string) $template->id - )->delete(); - CloudTemplateStateMachine::moveTemplate($template, 'retired'); - $template->revision = (int) $template->revision + 1; - $template->updated_at = time(); - $template->save(); - CloudJobService::completeAggregateLocked($parent, 'succeeded', - true, false, '', [ - 'template_id' => (string) $template->id, - 'deleted' => true, - ]); - return; - } - $effect = $replicas->contains(static fn ($row): bool => - (string) $row->state === 'delete_succeeded') - || ($operation['effect_may_have_applied'] ?? false) === true - || CloudJob::where('parent_job_id', $parentId) - ->where('effect_may_have_applied', 1)->exists(); - CloudTemplateStateMachine::moveTemplate($template, 'delete_failed'); - $template->updated_at = time(); - $template->save(); - CloudJobService::completeAggregateLocked($parent, 'failed', $effect, - false, 'template_delete_failed'); - } - public static function expireAggregateLocked(CloudJob $parent): void { if (!in_array((string) $parent->kind, [ @@ -2412,7 +2378,8 @@ final class CloudTemplateCatalogService private static function adminDto( CloudTemplate $row, array $replicas, - array $hosts + array $hosts, + ?CloudTemplateArtifact $artifact ): array { $partialCleanupKind = self::settledMixedCleanupParentKind($row); @@ -2497,12 +2464,24 @@ final class CloudTemplateCatalogService 'updated_at' => (int) $row->updated_at, 'replica_summary' => $summary, 'replicas' => $matrix, + 'artifact' => $artifact === null ? null : [ + 'state' => (string) $artifact->state, + 'error_code' => (string) $artifact->error_code, + 'delete_generation' => + (int) $artifact->delete_generation, + 'next_check_at' => + (int) $artifact->delete_next_check_at, + ], ]; if ($partialCleanupKind !== null) { $result['cleanup_reason'] = $partialCleanupKind === 'template_import' ? 'partial_import' : 'partial_delete'; } + if ($artifact !== null + && (string) $artifact->state === 'delete_attention') { + $result['cleanup_reason'] = 'artifact_delete_attention'; + } return $result; } diff --git a/web/app/Services/Cloud/CloudTemplateDeletionService.php b/web/app/Services/Cloud/CloudTemplateDeletionService.php new file mode 100644 index 00000000..5c46bf32 --- /dev/null +++ b/web/app/Services/Cloud/CloudTemplateDeletionService.php @@ -0,0 +1,742 @@ +state !== 'deleting' + && !($recovery + && (string) $replica->state === 'delete_failed')) { + throw new CloudError( + '云镜像副本状态与删除任务恢复证据不一致', + 409 + ); + } + if ($status === 'succeeded') { + if (!CloudAgentCatalogDeletion::confirms( + $operation, + 'template', + (string) $template->id + )) { + throw new CloudError('Agent 云镜像删除回执缺少清理证据', 502); + } + CloudTemplateStateMachine::moveReplica( + $replica, + 'delete_succeeded' + ); + $replica->error_code = ''; + } else { + CloudTemplateStateMachine::moveReplica( + $replica, + 'delete_failed' + ); + $replica->error_code = self::errorCode( + $operation, + 'template_delete_failed' + ); + } + CloudTemplateReplica::query() + ->where('host_id', (int) $replica->host_id) + ->where('template_id', (string) $replica->template_id) + ->where('job_id', (int) $replica->job_id)->update([ + 'state' => (string) $replica->state, + 'error_code' => (string) $replica->error_code, + 'updated_at' => time(), + ]); + } + + /** Arm local master cleanup while the aggregate parent is held. */ + public static function armLocked( + CloudTemplate $template, + CloudJob $parent + ): void { + self::assertChildProjectionLocked($template, $parent, 'applied'); + $artifact = CloudTemplateArtifact::query() + ->where('template_id', (string) $template->id) + ->lockForUpdate()->first(); + if ($artifact === null) { + throw new CloudRecoveryInstanceStateConflict( + '云镜像主副本删除账本缺失', + 409 + ); + } + if ((string) $artifact->state === 'delete_attention') { + throw new CloudRecoveryInstanceStateConflict( + '云镜像主副本清理需要管理员重新核对', + 409 + ); + } + if (in_array((string) $artifact->state, self::DUE_STATES, true)) { + return; + } + if ((string) $artifact->state !== 'ready' + || !self::artifactMatchesTemplate($artifact, $template)) { + throw new CloudRecoveryInstanceStateConflict( + '云镜像主副本身份或状态不允许删除', + 409 + ); + } + $now = time(); + $artifact->state = 'delete_pending'; + $artifact->error_code = ''; + $artifact->delete_operation_id = ''; + $artifact->delete_generation = 0; + $artifact->delete_cycle_attempts = 0; + $artifact->delete_presence_confirmations = 0; + $artifact->delete_last_checked_at = 0; + $artifact->delete_next_check_at = $now; + $artifact->delete_attention_at = null; + $artifact->updated_at = $now; + $artifact->save(); + } + + /** Retry only local master cleanup; Agent delete operations stay immutable. */ + public static function retryAttentionLocked( + CloudTemplate $template, + CloudJob $parent + ): void { + self::assertChildProjectionLocked($template, $parent, 'applied'); + $artifact = CloudTemplateArtifact::query() + ->where('template_id', (string) $template->id) + ->lockForUpdate()->first(); + if ($artifact === null + || (string) $artifact->state !== 'delete_attention' + || !self::artifactMatchesTemplate($artifact, $template)) { + throw new CloudError('云镜像主副本不在可重试清理状态', 409); + } + CloudTemplateStateMachine::moveTemplate($template, 'deleting'); + $now = time(); + $template->updated_at = $now; + $template->save(); + $artifact->state = 'delete_pending'; + $artifact->error_code = ''; + $artifact->delete_operation_id = ''; + $artifact->delete_cycle_attempts = 0; + $artifact->delete_presence_confirmations = 0; + $artifact->delete_last_checked_at = 0; + $artifact->delete_next_check_at = $now; + $artifact->delete_attention_at = null; + $artifact->updated_at = $now; + $artifact->save(); + } + + /** Restore sellable projection only when every host proves no deletion. */ + public static function restoreNotAppliedLocked( + CloudTemplate $template, + CloudJob $parent + ): void { + self::assertChildProjectionLocked($template, $parent, 'not_applied'); + $replicas = self::lockedReplicas((string) $template->id); + foreach ($replicas as $replica) { + CloudTemplateStateMachine::moveReplica( + $replica, + (int) $replica->revision === (int) $template->revision + ? 'active' : 'stale' + ); + $replica->error_code = ''; + $replica->updated_at = time(); + $replica->save(); + } + CloudTemplateStateMachine::moveTemplate($template, 'active'); + $template->updated_at = time(); + $template->save(); + CloudJobService::completeAggregateLocked( + $parent, + 'failed', + false, + true, + 'template_delete_not_applied' + ); + } + + /** Publish a fully known partial host deletion without touching the master. */ + public static function settleMixedLocked( + CloudTemplate $template, + CloudJob $parent + ): void { + self::assertChildProjectionLocked($template, $parent, 'mixed'); + CloudTemplateStateMachine::moveTemplate($template, 'delete_failed'); + $template->updated_at = time(); + $template->save(); + CloudJobService::completeAggregateLocked( + $parent, + 'failed', + true, + false, + 'template_delete_failed' + ); + } + + /** + * Process a bounded number of local object cleanup ledgers. + * + * @return array{processed:int,completed:int,deferred:int,attention:int,errors:int} + */ + public static function processDue( + int $limit = 3, + ?CloudObjectStorageS3Client $client = null + ): array { + $limit = max(1, min(16, $limit)); + $now = time(); + $ids = CloudTemplateArtifact::query() + ->whereIn('state', self::DUE_STATES) + ->where('delete_next_check_at', '<=', $now) + ->orderBy('delete_next_check_at') + ->orderBy('template_id') + ->limit($limit) + ->pluck('template_id') + ->map(static fn ($id): string => (string) $id)->all(); + $report = [ + 'processed' => 0, + 'completed' => 0, + 'deferred' => 0, + 'attention' => 0, + 'errors' => 0, + ]; + $client ??= new CloudObjectStorageS3Client(); + foreach ($ids as $templateId) { + try { + $result = self::processOne($templateId, $client); + $report['processed']++; + $report[$result]++; + } catch (\Throwable $error) { + $report['errors']++; + error_log(sprintf( + 'cloud template artifact cleanup template=%s failed: %s', + $templateId, + get_class($error) + )); + } + } + return $report; + } + + /** Strict pre-settlement child/replica identity projection. */ + public static function assertChildProjectionLocked( + CloudTemplate $template, + CloudJob $parent, + string $resolution + ): void { + if ((string) $parent->kind !== 'template_delete' + || (string) $parent->execution_mode !== 'aggregate' + || (int) ($template->active_job_id ?? 0) !== (int) $parent->id + || !in_array($resolution, ['applied', 'not_applied', 'mixed'], true)) { + throw new CloudRecoveryAggregateStateConflict( + '云镜像删除汇总任务绑定无效', + 409 + ); + } + $children = CloudJob::where('parent_job_id', (int) $parent->id) + ->orderBy('id')->lockForUpdate()->get(); + $replicas = self::lockedReplicas((string) $template->id); + if ($children->isEmpty() || $children->count() !== $replicas->count()) { + throw new CloudRecoveryAggregateStateConflict( + '云镜像删除子任务与副本数量不一致', + 409 + ); + } + if (CloudUserIsoReplica::query()->whereIn( + 'job_id', + $children->map( + static fn (CloudJob $child): int => (int) $child->id + )->all() + )->exists()) { + throw new CloudRecoveryOperationIdentityConflict( + '云镜像删除子任务被 ISO 副本账本重复引用', + 409 + ); + } + foreach ($children as $child) { + $childResolution = $resolution === 'mixed' + ? (string) $child->effect_resolution : $resolution; + if (!in_array($childResolution, ['applied', 'not_applied'], true)) { + throw new CloudRecoveryAggregateStateConflict( + '云镜像删除子任务效果状态无效', + 409 + ); + } + $matches = $replicas->filter( + static fn (CloudTemplateReplica $replica): bool => + (int) ($replica->job_id ?? 0) === (int) $child->id + ); + $replica = $matches->count() === 1 ? $matches->first() : null; + $expectedState = $childResolution === 'applied' + ? 'delete_succeeded' : 'delete_failed'; + if ($replica === null + || (int) ($child->parent_job_id ?? 0) !== (int) $parent->id + || (string) $child->kind !== 'template_delete_replica' + || (string) $child->execution_mode !== 'agent' + || (int) $child->user_id !== (int) $parent->user_id + || (int) $child->host_id !== (int) $replica->host_id + || !hash_equals( + (string) $child->operation_id, + (string) $replica->operation_id + )) { + throw new CloudRecoveryOperationIdentityConflict( + '云镜像删除子任务与副本身份不一致', + 409 + ); + } + if (!hash_equals( + (string) $template->generation_token, + (string) $replica->generation_token + ) + || (string) $replica->state !== $expectedState + || (int) $replica->revision > (int) $template->revision + || (int) $replica->image_bytes !== (int) $template->image_bytes + || !hash_equals( + (string) $template->sha256, + (string) $replica->sha256 + )) { + throw new CloudRecoveryInstanceStateConflict( + '云镜像删除副本状态或内容不一致', + 409 + ); + } + } + } + + /** Strict post-settlement projection used by immutable parent rechecks. */ + public static function assertSettledProjectionLocked( + CloudTemplate $template, + CloudJob $parent, + string $resolution + ): void { + if ($resolution === 'applied') { + if ((string) $template->state !== 'retired' + || !self::lockedReplicas((string) $template->id)->isEmpty() + || CloudTemplateArtifact::query()->where( + 'template_id', + (string) $template->id + )->lockForUpdate()->exists()) { + throw new CloudRecoveryInstanceStateConflict( + '云镜像删除已结算投影仍保留受管工件', + 409 + ); + } + return; + } + self::assertChildProjectionLocked($template, $parent, $resolution); + if ($resolution === 'not_applied' + && (string) $template->state !== 'active') { + throw new CloudRecoveryInstanceStateConflict( + '云镜像删除未应用投影没有恢复为可用状态', + 409 + ); + } + } + + public static function artifactState(string $templateId): ?array + { + $artifact = CloudTemplateArtifact::find($templateId); + return $artifact === null ? null : [ + 'state' => (string) $artifact->state, + 'error_code' => (string) $artifact->error_code, + 'delete_generation' => (int) $artifact->delete_generation, + 'next_check_at' => (int) $artifact->delete_next_check_at, + 'attention_at' => $artifact->delete_attention_at === null + ? null : (int) $artifact->delete_attention_at, + ]; + } + + private static function processOne( + string $templateId, + CloudObjectStorageS3Client $client + ): string { + $prepared = self::prepare($templateId, time()); + if ($prepared === null) { + return 'deferred'; + } + /** @var CloudTemplateArtifact $artifact */ + $artifact = $prepared['artifact']; + try { + $identity = CloudTemplateArtifactManager::inspectDeletionTarget( + $artifact, + $client + ); + } catch (CloudObjectStorageTransferError $error) { + self::deferInspection( + $templateId, + $error->publicCode, + $prepared['mode'] === 'inspect' + ); + return 'deferred'; + } + if ($identity === null) { + self::finishSuccess($templateId); + return 'completed'; + } + if (!CloudTemplateArtifactManager::deletionIdentityMatches( + $artifact, + $identity + )) { + self::markAttention($templateId, 'template_object_identity_changed'); + return 'attention'; + } + if ($prepared['mode'] === 'inspect') { + $artifact = self::recordStablePresence($templateId, time()); + if ($artifact === null) { + $current = CloudTemplateArtifact::find($templateId); + if ($current !== null + && (int) $current->delete_cycle_attempts + >= self::MAX_DELETE_GENERATIONS + && (int) $current->delete_presence_confirmations >= 2) { + self::markAttention( + $templateId, + 'template_object_delete_limit_reached' + ); + return 'attention'; + } + return 'deferred'; + } + } + try { + CloudTemplateArtifactManager::deleteExact($artifact, $client); + } catch (CloudObjectStorageTransferError $error) { + if ($error->effectMayHaveApplied) { + self::markUncertain($templateId, $error->publicCode); + return 'deferred'; + } + self::markAttention($templateId, $error->publicCode); + return 'attention'; + } + self::finishSuccess($templateId); + return 'completed'; + } + + /** @return array{mode:string,artifact:CloudTemplateArtifact}|null */ + private static function prepare(string $templateId, int $now): ?array + { + $snapshot = CloudTemplate::find($templateId); + $parentId = (int) ($snapshot?->active_job_id ?? 0); + if ($parentId < 1) { + return null; + } + return (new CloudTemplate())->getConnection()->transaction( + static function () use ($templateId, $parentId, $now): ?array { + $parent = CloudJob::query()->where('id', $parentId) + ->lockForUpdate()->first(); + $template = CloudTemplate::query()->where('id', $templateId) + ->lockForUpdate()->first(); + if ($parent === null || $template === null + || (int) ($template->active_job_id ?? 0) !== $parentId + || !in_array((string) $template->state, [ + 'deleting', 'delete_failed', + ], true)) { + return null; + } + self::assertChildProjectionLocked( + $template, + $parent, + 'applied' + ); + $artifact = CloudTemplateArtifact::query() + ->where('template_id', $templateId) + ->lockForUpdate()->first(); + if ($artifact === null + || !in_array( + (string) $artifact->state, + self::DUE_STATES, + true + ) + || (int) $artifact->delete_next_check_at > $now + || !self::artifactMatchesTemplate($artifact, $template)) { + return null; + } + if ((string) $artifact->state === 'delete_pending') { + self::claimDeleteGeneration($artifact, $now); + return ['mode' => 'delete', 'artifact' => $artifact]; + } + return ['mode' => 'inspect', 'artifact' => $artifact]; + }, + 3 + ); + } + + private static function claimDeleteGeneration( + CloudTemplateArtifact $artifact, + int $now + ): void { + $artifact->state = 'deleting'; + $artifact->delete_operation_id = bin2hex(random_bytes(16)); + $artifact->delete_generation = max( + 0, + (int) $artifact->delete_generation + ) + 1; + $artifact->delete_cycle_attempts = max( + 0, + (int) $artifact->delete_cycle_attempts + ) + 1; + $artifact->delete_presence_confirmations = 0; + $artifact->delete_last_checked_at = 0; + $artifact->delete_next_check_at = $now + self::STABLE_SECONDS; + $artifact->error_code = ''; + $artifact->updated_at = $now; + $artifact->save(); + } + + private static function recordStablePresence( + string $templateId, + int $now + ): ?CloudTemplateArtifact { + return (new CloudTemplateArtifact())->getConnection()->transaction( + static function () use ($templateId, $now): ?CloudTemplateArtifact { + $artifact = CloudTemplateArtifact::query() + ->where('template_id', $templateId) + ->lockForUpdate()->first(); + if ($artifact === null || !in_array( + (string) $artifact->state, + ['deleting', 'delete_uncertain'], + true + )) { + return null; + } + $last = (int) $artifact->delete_last_checked_at; + if ($last > 0 && $now - $last < self::STABLE_SECONDS) { + $artifact->delete_next_check_at = $last + + self::STABLE_SECONDS; + $artifact->updated_at = $now; + $artifact->save(); + return null; + } + $confirmations = max( + 0, + (int) $artifact->delete_presence_confirmations + ) + 1; + $artifact->state = 'delete_uncertain'; + $artifact->delete_presence_confirmations = $confirmations; + $artifact->delete_last_checked_at = $now; + $artifact->delete_next_check_at = $now + + self::STABLE_SECONDS; + $artifact->error_code = 'template_object_still_present'; + $artifact->updated_at = $now; + if ($confirmations < 2) { + $artifact->save(); + return null; + } + if ((int) $artifact->delete_cycle_attempts + >= self::MAX_DELETE_GENERATIONS) { + $artifact->save(); + return null; + } + self::claimDeleteGeneration($artifact, $now); + return $artifact; + }, + 3 + ); + } + + private static function deferInspection( + string $templateId, + string $code, + bool $priorDeleteMayHaveApplied + ): void { + CloudTemplateArtifact::query()->where('template_id', $templateId) + ->whereIn('state', self::DUE_STATES)->update([ + 'state' => $priorDeleteMayHaveApplied + ? 'delete_uncertain' : 'delete_pending', + 'error_code' => substr($code, 0, 64), + 'delete_next_check_at' => time() + self::STABLE_SECONDS, + 'updated_at' => time(), + ]); + } + + private static function markUncertain( + string $templateId, + string $code + ): void { + CloudTemplateArtifact::query()->where('template_id', $templateId) + ->where('state', 'deleting')->update([ + 'state' => 'delete_uncertain', + 'error_code' => substr($code, 0, 64), + 'delete_presence_confirmations' => 0, + 'delete_last_checked_at' => 0, + 'delete_next_check_at' => time() + self::STABLE_SECONDS, + 'updated_at' => time(), + ]); + } + + private static function markAttention( + string $templateId, + string $code + ): void { + $snapshot = CloudTemplate::find($templateId); + $parentId = (int) ($snapshot?->active_job_id ?? 0); + if ($parentId < 1) { + return; + } + (new CloudTemplate())->getConnection()->transaction( + static function () use ($templateId, $parentId, $code): void { + $parent = CloudJob::query()->where('id', $parentId) + ->lockForUpdate()->first(); + $template = CloudTemplate::query()->where('id', $templateId) + ->lockForUpdate()->first(); + if ($parent === null || $template === null + || (int) ($template->active_job_id ?? 0) !== $parentId) { + return; + } + self::assertChildProjectionLocked( + $template, + $parent, + 'applied' + ); + $artifact = CloudTemplateArtifact::query() + ->where('template_id', $templateId) + ->lockForUpdate()->first(); + if ($artifact === null) { + return; + } + $now = time(); + $artifact->state = 'delete_attention'; + $artifact->error_code = substr($code, 0, 64); + $artifact->delete_next_check_at = 0; + $artifact->delete_attention_at = $now; + $artifact->updated_at = $now; + $artifact->save(); + if ((string) $template->state === 'deleting') { + CloudTemplateStateMachine::moveTemplate( + $template, + 'delete_failed' + ); + $template->updated_at = $now; + $template->save(); + } + CloudJobService::completeAggregateLocked( + $parent, + 'failed', + true, + false, + substr($code, 0, 64) + ); + }, + 3 + ); + } + + private static function finishSuccess(string $templateId): void + { + $snapshot = CloudTemplate::find($templateId); + $parentId = (int) ($snapshot?->active_job_id ?? 0); + if ($parentId < 1) { + return; + } + (new CloudTemplate())->getConnection()->transaction( + static function () use ($templateId, $parentId): void { + $parent = CloudJob::query()->where('id', $parentId) + ->lockForUpdate()->first(); + $template = CloudTemplate::query()->where('id', $templateId) + ->lockForUpdate()->first(); + if ($parent === null || $template === null + || (int) ($template->active_job_id ?? 0) !== $parentId) { + return; + } + self::assertChildProjectionLocked( + $template, + $parent, + 'applied' + ); + $artifact = CloudTemplateArtifact::query() + ->where('template_id', $templateId) + ->lockForUpdate()->first(); + if ($artifact === null || !in_array( + (string) $artifact->state, + self::DUE_STATES, + true + )) { + throw new CloudRecoveryInstanceStateConflict( + '云镜像主副本清理终结状态不一致', + 409 + ); + } + CloudTemplateArtifact::query() + ->where('template_id', $templateId)->delete(); + CloudTemplateReplica::query() + ->where('template_id', $templateId)->delete(); + CloudTemplateStateMachine::moveTemplate($template, 'retired'); + $template->revision = (int) $template->revision + 1; + $template->updated_at = time(); + $template->save(); + if ((string) $parent->phase !== 'terminal') { + CloudJobService::completeAggregateLocked( + $parent, + 'succeeded', + true, + false, + '', + ['template_id' => $templateId, 'deleted' => true] + ); + return; + } + if ((string) $parent->outcome === 'failed' + && (string) $parent->effect_resolution === 'unknown') { + $parent->effect_resolution = 'applied'; + $parent->effect_resolved_at = time(); + $parent->effect_resolved_by = null; + $parent->effect_resolution_reason = + 'template_master_cleanup_confirmed'; + $parent->finalization_incident = 0; + $parent->finalization_error_code = null; + $parent->available_at = 0; + $parent->updated_at = time(); + $parent->save(); + } + }, + 3 + ); + } + + private static function artifactMatchesTemplate( + CloudTemplateArtifact $artifact, + CloudTemplate $template + ): bool { + return (int) $artifact->size_bytes === (int) $template->image_bytes + && hash_equals( + (string) $artifact->sha256, + (string) $template->sha256 + ) + && (string) $artifact->object_key !== '' + && (string) $artifact->object_etag !== ''; + } + + private static function lockedReplicas(string $templateId) + { + return CloudTemplateReplica::query() + ->where('template_id', $templateId) + ->orderBy('host_id')->lockForUpdate()->get(); + } + + private static function errorCode(array $operation, string $fallback): string + { + $code = $operation['error_code'] ?? null; + return is_string($code) + && preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $code) === 1 + ? $code : $fallback; + } +} diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js index d0f0922c..3da5fe04 100644 --- a/web/public/cloud/admin.js +++ b/web/public/cloud/admin.js @@ -910,6 +910,8 @@ function bindCloudTemplateActions(entry, form, name, save, remove, startup) { ? `云镜像模版“${entry.name}”仅在部分宿主机导入成功。确定清理已存在的剩余副本?` : cleanupReason === 'partial_delete' ? `云镜像模版“${entry.name}”仍有部分宿主机副本未删除。确定继续清理剩余副本?` + : cleanupReason === 'artifact_delete_attention' + ? `云镜像模版“${entry.name}”的宿主机副本已删除,但对象存储主副本需要重新核对。确定只重试主副本清理?不会再次向宿主机发送删除请求。` : `删除全局云镜像模版“${entry.name}”?删除后不能再用于新建或重装;已创建实例不受影响。`; if (!confirm(message)) return; remove.disabled = true; state.busy = true; @@ -918,6 +920,10 @@ function bindCloudTemplateActions(entry, form, name, save, remove, startup) { method: 'DELETE', body: { expected_revision: Number(entry.revision), generation_token: String(entry.generation_token || '') }, }); if (result?.deleted === true) { await loadCatalog(); notice('镜像模版及其受管文件已确认删除。'); } + else if (result?.recovery_restarted === true) { + await loadCatalog(); + notice('主副本清理已重新进入安全核对;不会再次删除宿主机副本。'); + } else { importProgress(result, partialCleanup ? '清理镜像模版' : '删除镜像模版', async () => { await loadCatalog(); notice(partialCleanup ? '镜像模版剩余宿主机副本已清理。' : '镜像模版及其受管文件已确认删除。'); @@ -941,10 +947,12 @@ function cloudTemplateVersion(entry) { identity.append(element('small', '', `版本 ${entry.version_label || '未标记'} · 目录修订 r${Number(entry.revision || 0)}`)); const imageBytes = Number(entry.image_bytes || 0); const master = element('div', 'cloud-template-master-meta'); + const artifactState = String(entry.artifact?.state || 'missing'); master.append( element('span', '', imageBytes > 0 ? `${(imageBytes / 1024 ** 3).toFixed(2)} GiB` : '大小待确认'), element('span', 'admin-mono', `SHA ${compactIdentifier(entry.sha256, 14) || '—'}`), element('span', 'admin-mono', `GEN ${compactIdentifier(entry.generation_token, 10) || '—'}`), + element('span', 'admin-mono', `MASTER ${artifactState}`), ); header.append(identity, master); const form = element('form', 'cloud-template-version-form'); @@ -956,8 +964,10 @@ function cloudTemplateVersion(entry) { const actions = element('div', 'cloud-template-version-actions'); const save = element('button', 'btn btn-outline-primary', '保存名称'); save.type = 'submit'; save.disabled = !canManage || !active; const startup = element('button', 'btn btn-outline-secondary', canManage ? '启动脚本' : '查看脚本'); startup.type = 'button'; startup.disabled = !active; - const partialCleanup = ['partial_import', 'partial_delete'].includes(String(entry.cleanup_reason || '')); - const remove = element('button', 'btn btn-outline-danger', partialCleanup ? '清理残留' : '删除版本'); remove.type = 'button'; remove.disabled = !canManage; + const cleanupReason = String(entry.cleanup_reason || ''); + const partialCleanup = ['partial_import', 'partial_delete'].includes(cleanupReason); + const removeLabel = cleanupReason === 'artifact_delete_attention' ? '重新核对主副本' : (partialCleanup ? '清理残留' : '删除版本'); + const remove = element('button', 'btn btn-outline-danger', removeLabel); remove.type = 'button'; remove.disabled = !canManage; actions.append(save, startup, remove); form.append(nameWrap, actions); bindCloudTemplateActions(entry, form, name, save, remove, startup); card.append(header, form, replicaMatrix(entry)); diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 11e2d73e..0cd7f17b 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026103012' scope='global'} +{assign var=venix_asset_version value='2026103013' scope='global'} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index e64897e0..359523bb 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -346,11 +346,19 @@ CREATE TABLE cloud_template_artifacts ( sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, state VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'ready', error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + delete_operation_id VARCHAR(80) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + delete_generation INT UNSIGNED NOT NULL DEFAULT 0, + delete_cycle_attempts INT UNSIGNED NOT NULL DEFAULT 0, + delete_presence_confirmations INT UNSIGNED NOT NULL DEFAULT 0, + delete_last_checked_at BIGINT UNSIGNED NOT NULL DEFAULT 0, + delete_next_check_at BIGINT UNSIGNED NOT NULL DEFAULT 0, + delete_attention_at BIGINT UNSIGNED DEFAULT NULL, verified_at BIGINT UNSIGNED NOT NULL, created_at BIGINT UNSIGNED NOT NULL, updated_at BIGINT UNSIGNED NOT NULL, UNIQUE KEY cloud_template_artifact_object (object_key), - KEY cloud_template_artifact_state (state, verified_at, template_id) + KEY cloud_template_artifact_state (state, verified_at, template_id), + KEY cloud_template_artifact_delete_due (state, delete_next_check_at, template_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- A PHP catalog entry is usable on a host only after that Agent confirms the diff --git a/web/tests/CloudAggregateRecoveryConvergenceTest.php b/web/tests/CloudAggregateRecoveryConvergenceTest.php index 8a66643c..9e90d20d 100644 --- a/web/tests/CloudAggregateRecoveryConvergenceTest.php +++ b/web/tests/CloudAggregateRecoveryConvergenceTest.php @@ -212,6 +212,47 @@ namespace { $table->integer('created_at'); $table->integer('updated_at'); }); + $schema->create('cloud_template_artifacts', + static function (Blueprint $table): void { + $table->string('template_id')->primary(); + $table->integer('storage_revision')->default(1); + $table->text('sealed_storage_runtime'); + $table->string('object_key'); + $table->string('object_version_id')->default(''); + $table->string('object_etag')->default('"test-etag"'); + $table->integer('size_bytes'); + $table->string('sha256'); + $table->string('state')->default('ready'); + $table->string('error_code')->default(''); + $table->string('delete_operation_id')->default(''); + $table->integer('delete_generation')->default(0); + $table->integer('delete_cycle_attempts')->default(0); + $table->integer('delete_presence_confirmations')->default(0); + $table->integer('delete_last_checked_at')->default(0); + $table->integer('delete_next_check_at')->default(0); + $table->integer('delete_attention_at')->nullable(); + $table->integer('verified_at')->default(0); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + Capsule::connection('default')->statement(<<<'SQL' +CREATE TRIGGER cloud_template_artifact_fixture +AFTER INSERT ON cloud_templates +BEGIN + INSERT INTO cloud_template_artifacts ( + template_id, storage_revision, sealed_storage_runtime, object_key, + object_version_id, object_etag, size_bytes, sha256, state, error_code, + delete_operation_id, delete_generation, delete_cycle_attempts, + delete_presence_confirmations, delete_last_checked_at, + delete_next_check_at, delete_attention_at, verified_at, created_at, + updated_at + ) VALUES ( + NEW.id, 1, 'test-only', 'test/' || NEW.id, '', '"test-etag"', + NEW.image_bytes, NEW.sha256, 'ready', '', '', 0, 0, 0, 0, 0, NULL, + NEW.created_at, NEW.created_at, NEW.updated_at + ); +END +SQL); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); @@ -1330,8 +1371,28 @@ namespace { $resolved = CloudJob::whereIn('id', array_keys($legacyParents)) ->whereIn('effect_resolution', ['applied', 'not_applied']) ->count(); - $resolvedLegacyCount++; $current = CloudJob::find($parentId); + if ($expectation['kind'] === 'template_delete') { + $resourceId = $expectation['resource_id']; + $assert($resolved === $resolvedLegacyCount + && $current !== null + && (string) $current->effect_resolution === 'unknown' + && (bool) $current->finalization_incident + && (string) $current->finalization_error_code + === 'recovery_child_convergence' + && (string) Capsule::table('cloud_templates') + ->where('id', $resourceId)->value('state') + === 'delete_failed' + && (string) Capsule::table('cloud_template_artifacts') + ->where('template_id', $resourceId)->value('state') + === 'delete_pending' + && Capsule::table('cloud_template_replicas') + ->where('template_id', $resourceId)->exists(), + 'Template delete recovery published applied before the exact ' + . 'master object was confirmed absent.'); + continue; + } + $resolvedLegacyCount++; $assert($resolved === $resolvedLegacyCount && $current !== null && (string) $current->effect_resolution @@ -1357,11 +1418,6 @@ namespace { && !Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId) ->where('state', '<>', 'active')->exists(), - 'template_delete' => - (string) Capsule::table('cloud_templates') - ->where('id', $resourceId)->value('state') === 'retired' - && !Capsule::table('cloud_template_replicas') - ->where('template_id', $resourceId)->exists(), 'user_iso_import' => (string) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->value('state') === 'active', @@ -1867,16 +1923,22 @@ namespace { $deadlineDeleteParent->refresh(); $assert($immutableParentAudit($deadlineDeleteParent) === $deadlineDeleteAudit - && (string) $deadlineDeleteParent->effect_resolution === 'applied' + && (string) $deadlineDeleteParent->effect_resolution === 'unknown' + && (bool) $deadlineDeleteParent->finalization_incident + && (string) $deadlineDeleteParent->finalization_error_code + === 'recovery_child_convergence' && (string) Capsule::table('cloud_templates') ->where('id', $deadlineDeleteTemplateId)->value('state') - === 'retired' + === 'delete_failed' && (int) Capsule::table('cloud_templates') - ->where('id', $deadlineDeleteTemplateId)->value('revision') === 2 - && !Capsule::table('cloud_template_replicas') - ->where('template_id', $deadlineDeleteTemplateId)->exists(), - 'A recovered deadline delete did not retire its catalog projection ' - . 'without rewriting the failed task.'); + ->where('id', $deadlineDeleteTemplateId)->value('revision') === 1 + && Capsule::table('cloud_template_replicas') + ->where('template_id', $deadlineDeleteTemplateId)->exists() + && (string) Capsule::table('cloud_template_artifacts') + ->where('template_id', $deadlineDeleteTemplateId)->value('state') + === 'delete_pending', + 'A recovered deadline delete published applied before the master ' + . 'object cleanup was confirmed.'); $deadlineIsoParentId = $insertParent('user_iso_import'); $deadlineIsoParent = $markDeadlineAudit($deadlineIsoParentId); @@ -2681,15 +2743,25 @@ namespace { $resource = $parentKind === 'template_delete' ? \App\Models\CloudTemplate::findOrFail($resourceId) : \App\Models\CloudUserIso::findOrFail($resourceId); - $settleDelete = new ReflectionMethod($service, 'settleDeleteParent'); try { - $settleDelete->invoke( - null, - $resource, - $parentId, - ['effect_may_have_applied' => true], - $childId - ); + if ($parentKind === 'template_delete') { + CloudTemplateCatalogService::settleRecoveredParentLocked( + CloudJob::findOrFail($parentId), + 'applied' + ); + } else { + $settleDelete = new ReflectionMethod( + $service, + 'settleDeleteParent' + ); + $settleDelete->invoke( + null, + $resource, + $parentId, + ['effect_may_have_applied' => true], + $childId + ); + } throw new RuntimeException( 'Normal delete settle accepted duplicate main records.' ); @@ -3831,15 +3903,21 @@ namespace { && (string) $lateDeleteParent->outcome === 'failed' && (string) $lateDeleteParent->error_code === 'aggregate_deadline_exceeded' - && (string) $lateDeleteParent->effect_resolution === 'applied' - && !(bool) $lateDeleteParent->finalization_incident + && (string) $lateDeleteParent->effect_resolution === 'unknown' + && (bool) $lateDeleteParent->finalization_incident + && (string) $lateDeleteParent->finalization_error_code + === 'recovery_child_convergence' && (string) Capsule::table('cloud_templates') - ->where('id', $lateDeleteTemplateId)->value('state') === 'retired' - && !Capsule::table('cloud_template_replicas') + ->where('id', $lateDeleteTemplateId)->value('state') + === 'delete_failed' + && Capsule::table('cloud_template_replicas') ->where('template_id', $lateDeleteTemplateId)->exists() + && (string) Capsule::table('cloud_template_artifacts') + ->where('template_id', $lateDeleteTemplateId)->value('state') + === 'delete_pending' && CloudAgentSettingsService::$requests === $beforeLateDeleteRequests, 'A late template deletion receipt did not converge the original ' - . 'operation without replaying or rewriting its failure audit.'); + . 'host operation into the two-phase master cleanup fence.'); $lateIsoParentId = $insertParent('user_iso_delete'); $lateIsoChildId = $insertResolvedChild( diff --git a/web/tests/CloudAutomaticOperationRecoveryServiceTest.php b/web/tests/CloudAutomaticOperationRecoveryServiceTest.php index 8cef4ebb..fea03085 100644 --- a/web/tests/CloudAutomaticOperationRecoveryServiceTest.php +++ b/web/tests/CloudAutomaticOperationRecoveryServiceTest.php @@ -251,6 +251,29 @@ namespace { $table->integer('updated_at')->default(0); $table->primary(['host_id', 'template_id']); }); + $schema->create('cloud_template_artifacts', + static function (Blueprint $table): void { + $table->string('template_id')->primary(); + $table->integer('storage_revision'); + $table->text('sealed_storage_runtime'); + $table->string('object_key'); + $table->string('object_version_id')->default(''); + $table->string('object_etag'); + $table->integer('size_bytes'); + $table->string('sha256'); + $table->string('state'); + $table->string('error_code')->default(''); + $table->string('delete_operation_id')->default(''); + $table->integer('delete_generation')->default(0); + $table->integer('delete_cycle_attempts')->default(0); + $table->integer('delete_presence_confirmations')->default(0); + $table->integer('delete_last_checked_at')->default(0); + $table->integer('delete_next_check_at')->default(0); + $table->integer('delete_attention_at')->nullable(); + $table->integer('verified_at')->default(0); + $table->integer('created_at'); + $table->integer('updated_at'); + }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('host_id'); @@ -924,6 +947,32 @@ namespace { 'created_at' => $now - 120, 'updated_at' => $now, ]); + Capsule::table('cloud_template_artifacts')->insert([ + 'template_id' => $templateDeleteId, + 'storage_revision' => 1, + 'sealed_storage_runtime' => CloudSecrets::encrypt([ + 'config' => [], + 'credentials' => [], + 'revision' => 1, + ], 'template-artifact-storage-runtime'), + 'object_key' => 'templates/' . $templateDeleteId . '/system.qcow2', + 'object_version_id' => '', + 'object_etag' => str_repeat('f', 32), + 'size_bytes' => 4096, + 'sha256' => str_repeat('e', 64), + 'state' => 'ready', + 'error_code' => '', + 'delete_operation_id' => '', + 'delete_generation' => 0, + 'delete_cycle_attempts' => 0, + 'delete_presence_confirmations' => 0, + 'delete_last_checked_at' => 0, + 'delete_next_check_at' => 0, + 'delete_attention_at' => null, + 'verified_at' => $now, + 'created_at' => $now - 120, + 'updated_at' => $now, + ]); $client->responses[] = $inspection( $templateDeleteOperationId, 'succeeded', @@ -946,19 +995,24 @@ namespace { $assert((string) $templateDeleteChild->effect_resolution === 'applied' && !(bool) $templateDeleteChild->finalization_incident && $templateDeleteChild->finalization_error_code === null - && (string) $templateDeleteParent->effect_resolution === 'applied' - && !(bool) $templateDeleteParent->finalization_incident + && (string) $templateDeleteParent->effect_resolution === 'unknown' + && (bool) $templateDeleteParent->finalization_incident + && (string) $templateDeleteParent->finalization_error_code + === 'recovery_child_convergence' && (string) Capsule::table('cloud_templates') - ->where('id', $templateDeleteId)->value('state') === 'retired' - && !Capsule::table('cloud_template_replicas') + ->where('id', $templateDeleteId)->value('state') === 'deleting' + && Capsule::table('cloud_template_replicas') ->where('template_id', $templateDeleteId)->exists() + && (string) Capsule::table('cloud_template_artifacts') + ->where('template_id', $templateDeleteId)->value('state') + === 'delete_pending' && count($client->requests) === $beforeTemplateDeleteRequests + 1 && str_ends_with( $client->requests[$beforeTemplateDeleteRequests]['target'], '/inspect' ), - 'A successful template deletion inspection did not converge its ' - . 'existing operation and catalog ledger.'); + 'A successful template deletion inspection did not converge its host ' + . 'operation into the two-phase master cleanup fence.'); // A poisoned Agent completion outbox is never evidence for either effect // verdict. Even a syntactically complete not-applied projection remains diff --git a/web/tests/CloudRuntimeHeartbeatTest.php b/web/tests/CloudRuntimeHeartbeatTest.php index 45130485..8db0e7ff 100644 --- a/web/tests/CloudRuntimeHeartbeatTest.php +++ b/web/tests/CloudRuntimeHeartbeatTest.php @@ -307,6 +307,13 @@ try { $table->integer('job_id')->nullable(); $table->string('state')->default('pending'); }); + $schema->create('cloud_template_artifacts', static function ( + Blueprint $table + ): void { + $table->integer('template_id')->primary(); + $table->string('state'); + $table->integer('delete_next_check_at')->default(0); + }); $schema->create('cloud_user_isos', static function ( Blueprint $table ): void { diff --git a/web/tests/CloudTemplateArtifactManagerTest.php b/web/tests/CloudTemplateArtifactManagerTest.php index e8b2cabc..1bbeaa81 100644 --- a/web/tests/CloudTemplateArtifactManagerTest.php +++ b/web/tests/CloudTemplateArtifactManagerTest.php @@ -72,6 +72,13 @@ namespace { $table->string('sha256'); $table->string('state'); $table->string('error_code')->default(''); + $table->string('delete_operation_id')->default(''); + $table->integer('delete_generation')->default(0); + $table->integer('delete_cycle_attempts')->default(0); + $table->integer('delete_presence_confirmations')->default(0); + $table->integer('delete_last_checked_at')->default(0); + $table->integer('delete_next_check_at')->default(0); + $table->integer('delete_attention_at')->nullable(); $table->integer('verified_at'); $table->integer('created_at'); $table->integer('updated_at'); diff --git a/web/tests/CloudTemplateCatalogTest.php b/web/tests/CloudTemplateCatalogTest.php index f98dcc35..18d31c9e 100644 --- a/web/tests/CloudTemplateCatalogTest.php +++ b/web/tests/CloudTemplateCatalogTest.php @@ -66,6 +66,14 @@ $capsule->schema('default')->create('cloud_template_replicas', $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); }); +$capsule->schema('default')->create('cloud_template_artifacts', + static function (Blueprint $table): void { + $table->string('template_id')->primary(); + $table->string('state')->default('ready'); + $table->string('error_code')->default(''); + $table->integer('delete_generation')->default(0); + $table->integer('delete_next_check_at')->default(0); + }); $capsule->schema('default')->create('configurations', static function (Blueprint $table): void { $table->increments('id'); diff --git a/web/tests/CloudTemplateDeletionServiceTest.php b/web/tests/CloudTemplateDeletionServiceTest.php new file mode 100644 index 00000000..c0b1ad6f --- /dev/null +++ b/web/tests/CloudTemplateDeletionServiceTest.php @@ -0,0 +1,354 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', + ], 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema('default'); + $schema->create('cloud_jobs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('operation_id'); + $table->string('request_idempotency_key'); + $table->string('request_fingerprint'); + $table->integer('parent_job_id')->nullable(); + $table->integer('user_id'); + $table->integer('host_id')->nullable(); + $table->string('kind'); + $table->string('execution_mode'); + $table->string('phase'); + $table->string('outcome')->nullable(); + $table->string('status'); + $table->text('sealed_payload'); + $table->integer('attempts')->default(0); + $table->integer('dispatch_attempts')->default(0); + $table->integer('dispatch_deadline_at')->default(1); + $table->integer('dispatch_started_at')->nullable(); + $table->integer('agent_accepted_at')->nullable(); + $table->integer('dispatched_at')->nullable(); + $table->integer('receipt_deadline_at')->nullable(); + $table->integer('terminal_at')->nullable(); + $table->boolean('effect_may_have_applied')->nullable(); + $table->boolean('safe_to_resubmit')->nullable(); + $table->string('effect_resolution')->default('not_applicable'); + $table->integer('effect_resolved_at')->nullable(); + $table->integer('effect_resolved_by')->nullable(); + $table->string('effect_resolution_reason')->default(''); + $table->string('completion_event_id')->nullable(); + $table->integer('read_at')->nullable(); + $table->string('lease_token')->nullable(); + $table->integer('lease_until')->default(0); + $table->integer('available_at')->default(0); + $table->string('error_code')->nullable(); + $table->boolean('finalization_incident')->default(0); + $table->string('finalization_error_code')->nullable(); + $table->text('result_json')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_templates', static function (Blueprint $table): void { + $table->string('id')->primary(); + $table->string('catalog_key'); + $table->string('version_label'); + $table->boolean('is_default')->default(false); + $table->string('name'); + $table->string('os_family'); + $table->string('os_version'); + $table->string('source_type'); + $table->text('sealed_source_url'); + $table->integer('image_bytes'); + $table->string('sha256'); + $table->text('startup_script'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('active_job_id')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_template_replicas', static function (Blueprint $table): void { + $table->integer('host_id'); + $table->string('template_id'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('image_bytes'); + $table->string('sha256'); + $table->string('operation_id'); + $table->integer('job_id')->nullable(); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->primary(['host_id', 'template_id']); + }); + $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { + $table->integer('job_id')->nullable(); + }); + $schema->create('cloud_template_artifacts', static function (Blueprint $table): void { + $table->string('template_id')->primary(); + $table->integer('storage_revision'); + $table->text('sealed_storage_runtime'); + $table->string('object_key'); + $table->string('object_version_id')->default(''); + $table->string('object_etag'); + $table->integer('size_bytes'); + $table->string('sha256'); + $table->string('state'); + $table->string('error_code')->default(''); + $table->string('delete_operation_id')->default(''); + $table->integer('delete_generation')->default(0); + $table->integer('delete_cycle_attempts')->default(0); + $table->integer('delete_presence_confirmations')->default(0); + $table->integer('delete_last_checked_at')->default(0); + $table->integer('delete_next_check_at')->default(0); + $table->integer('delete_attention_at')->nullable(); + $table->integer('verified_at'); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + + $runtime = [ + 'config' => [ + 'endpoint' => 'https://objects.example.com', 'region' => 'ewr', + 'bucket' => 'templates', 'prefix' => 'virt', 'path_style' => true, + ], + 'credentials' => [ + 'access_key_id' => 'DELETEACCESSKEY', + 'secret_access_key' => 'delete-secret', + ], + 'revision' => 1, + ]; + $now = time(); + $fixture = static function (string $suffix) use ($runtime, $now): array { + $parentId = (int) Capsule::table('cloud_jobs')->insertGetId([ + 'operation_id' => md5('parent-' . $suffix), + 'request_idempotency_key' => md5('parent-' . $suffix), + 'request_fingerprint' => hash('sha256', 'parent-' . $suffix), + 'parent_job_id' => null, 'user_id' => 1, 'host_id' => null, + 'kind' => 'template_delete', 'execution_mode' => 'aggregate', + 'phase' => 'awaiting_receipt', 'outcome' => null, + 'status' => 'running', + 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), + 'attempts' => 0, 'dispatch_attempts' => 0, + 'dispatch_deadline_at' => $now + 300, + 'receipt_deadline_at' => $now + 3600, + 'effect_resolution' => 'not_applicable', + 'effect_resolution_reason' => '', 'lease_until' => 0, + 'available_at' => $now, 'finalization_incident' => 0, + 'created_at' => $now, 'updated_at' => $now, + ]); + $childOperation = md5('child-' . $suffix); + $childId = (int) Capsule::table('cloud_jobs')->insertGetId([ + 'operation_id' => $childOperation, + 'request_idempotency_key' => $childOperation, + 'request_fingerprint' => hash('sha256', 'child-' . $suffix), + 'parent_job_id' => $parentId, 'user_id' => 1, 'host_id' => 7, + 'kind' => 'template_delete_replica', 'execution_mode' => 'agent', + 'phase' => 'terminal', 'outcome' => 'succeeded', + 'status' => 'succeeded', + 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), + 'attempts' => 1, 'dispatch_attempts' => 1, + 'dispatch_deadline_at' => $now - 20, + 'dispatch_started_at' => $now - 19, + 'agent_accepted_at' => $now - 18, + 'dispatched_at' => $now - 17, + 'receipt_deadline_at' => $now + 900, + 'terminal_at' => $now - 1, + 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, + 'effect_resolution' => 'applied', + 'effect_resolution_reason' => '', 'lease_until' => 0, + 'available_at' => 0, 'finalization_incident' => 0, + 'created_at' => $now - 20, 'updated_at' => $now - 1, + ]); + $templateId = 'delete-template-' . $suffix; + $generation = md5('generation-' . $suffix); + $sha = hash('sha256', 'image-' . $suffix); + Capsule::table('cloud_templates')->insert([ + 'id' => $templateId, 'catalog_key' => 'debian-13', + 'version_label' => $suffix, 'is_default' => 0, + 'name' => 'Delete ' . $suffix, 'os_family' => 'Debian', + 'os_version' => '13', 'source_type' => 'object', + 'sealed_source_url' => '', 'image_bytes' => 4096, + 'sha256' => $sha, 'startup_script' => '', 'revision' => 1, + 'generation_token' => $generation, 'state' => 'deleting', + 'active_job_id' => $parentId, 'created_at' => $now, + 'updated_at' => $now, + ]); + Capsule::table('cloud_template_replicas')->insert([ + 'host_id' => 7, 'template_id' => $templateId, 'revision' => 1, + 'generation_token' => $generation, 'state' => 'delete_succeeded', + 'image_bytes' => 4096, 'sha256' => $sha, + 'operation_id' => $childOperation, 'job_id' => $childId, + 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, + ]); + Capsule::table('cloud_template_artifacts')->insert([ + 'template_id' => $templateId, 'storage_revision' => 1, + 'sealed_storage_runtime' => CloudSecrets::encrypt( + $runtime, 'template-artifact-runtime' + ), + 'object_key' => 'virt/cloud-templates/' . $templateId + . '/' . $generation . '/system.qcow2', + 'object_version_id' => 'version-' . $suffix, + 'object_etag' => '"etag-' . $suffix . '"', + 'size_bytes' => 4096, 'sha256' => $sha, 'state' => 'ready', + 'error_code' => '', 'delete_operation_id' => '', + 'delete_generation' => 0, 'delete_cycle_attempts' => 0, + 'delete_presence_confirmations' => 0, + 'delete_last_checked_at' => 0, 'delete_next_check_at' => 0, + 'delete_attention_at' => null, 'verified_at' => $now, + 'created_at' => $now, 'updated_at' => $now, + ]); + $template = CloudTemplate::findOrFail($templateId); + $parent = CloudJob::findOrFail($parentId); + $template->getConnection()->transaction( + static function () use ($templateId, $parentId): void { + $parent = CloudJob::where('id', $parentId) + ->lockForUpdate()->firstOrFail(); + $template = CloudTemplate::where('id', $templateId) + ->lockForUpdate()->firstOrFail(); + CloudTemplateDeletionService::armLocked($template, $parent); + }, + 3 + ); + return [$templateId, $parentId]; + }; + + [$normalId, $normalParentId] = $fixture('normal'); + $normalHistory = []; + $normalStack = HandlerStack::create(new MockHandler([ + new Response(200, [ + 'Content-Length' => '4096', 'ETag' => '"etag-normal"', + 'x-amz-version-id' => 'version-normal', + ]), + new Response(204), + ])); + $normalStack->push(Middleware::history($normalHistory)); + $normalClient = new CloudObjectStorageS3Client(new Client([ + 'handler' => $normalStack, + ])); + $normal = CloudTemplateDeletionService::processDue(1, $normalClient); + $assert($normal['completed'] === 1 + && CloudTemplateArtifact::find($normalId) === null + && (string) CloudTemplate::findOrFail($normalId)->state === 'retired' + && (int) CloudTemplate::findOrFail($normalId)->revision === 2 + && (string) CloudJob::findOrFail($normalParentId)->outcome + === 'succeeded' + && array_map( + static fn (array $item): string => $item['request']->getMethod(), + $normalHistory + ) === ['HEAD', 'DELETE'], + 'Confirmed master deletion did not retire the exact catalog version.'); + + [$unknownId, $unknownParentId] = $fixture('unknown'); + $unknownHistory = []; + $unknownStack = HandlerStack::create(new MockHandler([ + new Response(200, [ + 'Content-Length' => '4096', 'ETag' => '"etag-unknown"', + 'x-amz-version-id' => 'version-unknown', + ]), + new ConnectException('lost after delete', new Request( + 'DELETE', 'https://objects.example.com' + )), + new Response(200, [ + 'Content-Length' => '4096', 'ETag' => '"etag-unknown"', + 'x-amz-version-id' => 'version-unknown', + ]), + new Response(200, [ + 'Content-Length' => '4096', 'ETag' => '"etag-unknown"', + 'x-amz-version-id' => 'version-unknown', + ]), + new Response(204), + ])); + $unknownStack->push(Middleware::history($unknownHistory)); + $unknownClient = new CloudObjectStorageS3Client(new Client([ + 'handler' => $unknownStack, + ])); + CloudTemplateDeletionService::processDue(1, $unknownClient); + $unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId); + $assert((string) $unknownArtifact->state === 'delete_uncertain' + && (int) $unknownArtifact->delete_generation === 1, + 'Ambiguous DELETE was not fenced on its original generation.'); + Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId) + ->update(['delete_next_check_at' => 0]); + CloudTemplateDeletionService::processDue(1, $unknownClient); + $unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId); + $assert((int) $unknownArtifact->delete_presence_confirmations === 1 + && (int) $unknownArtifact->delete_generation === 1, + 'One presence probe incorrectly replayed an ambiguous DELETE.'); + Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId) + ->update([ + 'delete_last_checked_at' => time() - 301, + 'delete_next_check_at' => 0, + ]); + $converged = CloudTemplateDeletionService::processDue(1, $unknownClient); + $assert($converged['completed'] === 1 + && CloudTemplateArtifact::find($unknownId) === null + && (string) CloudTemplate::findOrFail($unknownId)->state === 'retired' + && (string) CloudJob::findOrFail($unknownParentId)->outcome + === 'succeeded' + && array_map( + static fn (array $item): string => $item['request']->getMethod(), + $unknownHistory + ) === ['HEAD', 'DELETE', 'HEAD', 'HEAD', 'DELETE'], + 'Stable two-probe evidence did not create exactly one fresh delete generation.'); + + [$attentionId, $attentionParentId] = $fixture('attention'); + $attentionClient = new CloudObjectStorageS3Client(new Client([ + 'handler' => HandlerStack::create(new MockHandler([ + new Response(200, [ + 'Content-Length' => '8192', 'ETag' => '"different"', + 'x-amz-version-id' => 'version-attention', + ]), + ])), + ])); + $attention = CloudTemplateDeletionService::processDue(1, $attentionClient); + $assert($attention['attention'] === 1 + && (string) CloudTemplateArtifact::findOrFail($attentionId)->state + === 'delete_attention' + && (string) CloudTemplate::findOrFail($attentionId)->state + === 'delete_failed' + && (string) CloudJob::findOrFail($attentionParentId)->outcome + === 'failed', + 'Changed object identity was not fenced for explicit administrator review.'); + + echo "Cloud-template two-phase deletion and recovery behavior passed.\n"; +} diff --git a/web/tests/CloudTemplateErrorContractTest.php b/web/tests/CloudTemplateErrorContractTest.php index e545e8b5..797b751f 100644 --- a/web/tests/CloudTemplateErrorContractTest.php +++ b/web/tests/CloudTemplateErrorContractTest.php @@ -13,6 +13,9 @@ $controller = file_get_contents( $catalog = file_get_contents( $root . '/app/Services/Cloud/CloudTemplateCatalogService.php' ); +$deletion = file_get_contents( + $root . '/app/Services/Cloud/CloudTemplateDeletionService.php' +); $routes = file_get_contents($root . '/config/routes_cloud.php'); $schema = file_get_contents($root . '/sql/cloud.sql'); $uploadStart = strpos($controller, 'public function cloudTemplateUpload('); @@ -40,8 +43,13 @@ $assert(is_string($catalog) && str_contains($catalog, 'public static function finishJobLocked(') && str_contains($catalog, 'public static function expireAggregateLocked(') && str_contains($catalog, 'CloudJobService::completeAggregateLocked(') - && str_contains($catalog, 'CloudAgentCatalogDeletion::confirms(') - && str_contains($catalog, 'Agent 云镜像删除回执缺少清理证据') + && is_string($deletion) + && str_contains($deletion, 'CloudAgentCatalogDeletion::confirms(') + && str_contains($deletion, 'Agent 云镜像删除回执缺少清理证据') + && str_contains($deletion, + 'CloudTemplateArtifactManager::inspectDeletionTarget(') + && str_contains($deletion, + 'CloudTemplateArtifactManager::deleteExact(') && !str_contains($catalog, 'CloudAgentClient') && !str_contains($catalog, '->request(') && str_contains($catalog, @@ -81,6 +89,8 @@ $assert(is_string($schema) && str_contains($schema, 'version_label VARCHAR(64)') && str_contains($schema, 'CREATE TABLE cloud_template_artifacts') && str_contains($schema, 'sealed_storage_runtime LONGTEXT NOT NULL') + && str_contains($schema, 'delete_generation INT UNSIGNED NOT NULL') + && str_contains($schema, 'cloud_template_artifact_delete_due') && str_contains($schema, 'sealed_source_url LONGTEXT NOT NULL') && !preg_match('/\bsource_url\s+(?:TEXT|LONGTEXT)/', $schema) && str_contains($schema, 'startup_script MEDIUMTEXT NOT NULL')