From 196a7d8ce5b34e4838037df74b36830a7c2baeb4 Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 28 Oct 2026 04:18:57 +0800 Subject: [PATCH] fix: reconcile object cleanup debt safely --- agent/hypervisor/libvirt_backup.go | 1 + .../libvirt_operation_admin_test.go | 4 + agent/hypervisor/types.go | 2 + agent/objecttransfer/client.go | 70 ++- agent/objecttransfer/client_test.go | 10 +- docs/AGENT_API.md | 3 +- docs/DEPLOYMENT.md | 4 +- docs/PORTABLE_VM_BACKUPS.md | 2 +- docs/VALIDATION.md | 2 +- web/app/Command/CloudSetup.php | 13 +- .../Admin/ObjectStorageController.php | 75 +++ .../CloudAdminInstanceController.php | 2 +- web/app/Models/CloudInstanceBackup.php | 12 +- .../Cloud/CloudAutomaticBackupService.php | 27 +- web/app/Services/Cloud/CloudBackupService.php | 456 ++++++++++++++++-- .../Cloud/CloudDeploymentSnapshotService.php | 11 +- .../Cloud/CloudObjectStorageS3Client.php | 112 +++-- web/config/routes.php | 2 + web/public/cloud/admin-instances.js | 19 +- web/public/cloud/backups.js | 2 +- web/public/cloud/cloud-snapshots.js | 2 +- web/public/cloud/detail.js | 2 + .../views/venix/admin/object_storage.tpl | 102 +++- web/resources/views/venix/layout_assets.tpl | 2 +- web/sql/cloud.sql | 13 +- ...CloudAutomaticBackupRetentionGraceTest.php | 52 +- .../CloudDeploymentSnapshotServiceTest.php | 50 +- web/tests/CloudFreshSchemaContractTest.php | 17 +- .../CloudObjectStorageAdminContractTest.php | 14 +- ...CloudPortableBackupAdminUiContractTest.php | 4 +- web/tests/CloudPortableBackupDeletionTest.php | 137 +++++- .../CloudPortableBackupFinalizerTest.php | 6 + web/tests/CloudS3SignerTest.php | 21 +- web/tests/LegalPagesContractTest.php | 2 +- web/tests/PublicPageLocaleContractTest.php | 2 +- web/tests/SiteBrandContractTest.php | 2 +- 36 files changed, 1084 insertions(+), 173 deletions(-) diff --git a/agent/hypervisor/libvirt_backup.go b/agent/hypervisor/libvirt_backup.go index 379c2192..55b97019 100644 --- a/agent/hypervisor/libvirt_backup.go +++ b/agent/hypervisor/libvirt_backup.go @@ -93,6 +93,7 @@ func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request result := BackupResult{ BackupKey: request.BackupKey, SizeBytes: upload.SizeBytes, VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256, + ObjectETag: upload.ETag, ObjectVersionID: upload.VersionID, Consistency: snapshot.Consistency, } cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx) diff --git a/agent/hypervisor/libvirt_operation_admin_test.go b/agent/hypervisor/libvirt_operation_admin_test.go index 1ef62bc1..37e140e1 100644 --- a/agent/hypervisor/libvirt_operation_admin_test.go +++ b/agent/hypervisor/libvirt_operation_admin_test.go @@ -412,6 +412,8 @@ func TestPortableBackupOperationProjectionPreservesFinalizerIdentity(t *testing. "size_bytes":1048576, "virtual_size_bytes":2147483648, "sha256":"`+digest+`", + "object_etag":"\"backup-etag\"", + "object_version_id":"version/7+opaque", "consistency":"filesystem", "source_url":"must-not-leak" }`)) @@ -421,6 +423,8 @@ func TestPortableBackupOperationProjectionPreservesFinalizerIdentity(t *testing. } if backupProjection.BackupKey != backupKey || backupProjection.SizeBytes != 1048576 || backupProjection.VirtualSizeBytes != 2147483648 || backupProjection.SHA256 != digest || + backupProjection.ObjectETag != `"backup-etag"` || + backupProjection.ObjectVersionID != "version/7+opaque" || backupProjection.Consistency != "filesystem" { t.Fatalf("backup projection omitted finalizer identity: %#v", backupProjection) } diff --git a/agent/hypervisor/types.go b/agent/hypervisor/types.go index 8f801ce1..0dfd05e3 100644 --- a/agent/hypervisor/types.go +++ b/agent/hypervisor/types.go @@ -228,6 +228,8 @@ type BackupResult struct { SizeBytes int64 `json:"size_bytes"` VirtualSizeBytes uint64 `json:"virtual_size_bytes"` SHA256 string `json:"sha256"` + ObjectETag string `json:"object_etag,omitempty"` + ObjectVersionID string `json:"object_version_id,omitempty"` Consistency string `json:"consistency"` CleanupPending bool `json:"cleanup_pending,omitempty"` } diff --git a/agent/objecttransfer/client.go b/agent/objecttransfer/client.go index 1341a189..5295368a 100644 --- a/agent/objecttransfer/client.go +++ b/agent/objecttransfer/client.go @@ -65,12 +65,15 @@ type UploadResult struct { SizeBytes int64 `json:"size_bytes"` SHA256 string `json:"sha256"` ETag string `json:"etag,omitempty"` + VersionID string `json:"version_id,omitempty"` Parts []CompletedPart `json:"parts"` } type DownloadResult struct { SizeBytes int64 `json:"size_bytes"` SHA256 string `json:"sha256"` + ETag string `json:"etag,omitempty"` + VersionID string `json:"version_id,omitempty"` } type Client struct { @@ -237,7 +240,7 @@ func (c *Client) UploadFile(ctx context.Context, path string, plan MultipartPlan return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL, "backup_source_changed", "backup source changed during upload") } - etag, err := c.complete(ctx, plan.CompleteURL, parts) + etag, versionID, err := c.complete(ctx, plan.CompleteURL, parts) if err != nil { if IsAmbiguous(err) { // Completion can commit remotely even if its response is lost. The @@ -246,14 +249,20 @@ func (c *Client) UploadFile(ctx context.Context, path string, plan MultipartPlan // ambiguous reply into success without aborting or replaying. verified, verifyErr := c.Download(ctx, plan.VerifyURL, size, digest, io.Discard) if verifyErr == nil && verified.SizeBytes == size && verified.SHA256 == digest { - return UploadResult{SizeBytes: size, SHA256: digest, Parts: parts}, nil + return UploadResult{ + SizeBytes: size, SHA256: digest, ETag: verified.ETag, + VersionID: verified.VersionID, Parts: parts, + }, nil } return UploadResult{}, err } return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL, "upload_complete_failed", "backup multipart completion was rejected") } - return UploadResult{SizeBytes: size, SHA256: digest, ETag: etag, Parts: parts}, nil + return UploadResult{ + SizeBytes: size, SHA256: digest, ETag: etag, + VersionID: versionID, Parts: parts, + }, nil } func (c *Client) putPart(ctx context.Context, capability string, body io.Reader, length int64) (string, error) { @@ -306,56 +315,73 @@ type completeMultipartResult struct { Code string `xml:"Code"` } -func (c *Client) complete(ctx context.Context, capability string, parts []CompletedPart) (string, error) { +func (c *Client) complete(ctx context.Context, capability string, + parts []CompletedPart) (string, string, error) { payload := completeMultipartUpload{Parts: make([]completeMultipartPart, len(parts))} for index, part := range parts { payload.Parts[index] = completeMultipartPart{PartNumber: part.PartNumber, ETag: part.ETag} } body, err := xml.Marshal(payload) if err != nil { - return "", transferError("upload_complete_invalid", "backup multipart completion could not be encoded", false) + return "", "", transferError("upload_complete_invalid", "backup multipart completion could not be encoded", false) } request, err := http.NewRequestWithContext(ctx, http.MethodPost, capability, strings.NewReader(string(body))) if err != nil { - return "", transferError("invalid_capability_url", "object transfer capability is invalid", false) + return "", "", transferError("invalid_capability_url", "object transfer capability is invalid", false) } request.ContentLength = int64(len(body)) request.Header.Set("Content-Type", "application/xml") response, err := c.http.Do(request) if err != nil { - return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) + return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } defer response.Body.Close() encoded, readErr := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1)) if readErr != nil || int64(len(encoded)) > maxResponseBytes { - return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) + return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } if response.StatusCode >= http.StatusInternalServerError { // A storage service or reverse proxy can lose the success response after // commit. A 5xx therefore cannot authorize AbortMultipartUpload. - return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) + return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } if response.StatusCode < 200 || response.StatusCode >= 300 { - return "", transferError("upload_complete_failed", "backup multipart completion was rejected", false) + return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false) } if len(strings.TrimSpace(string(encoded))) == 0 { - return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) + return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } var result completeMultipartResult if err := xml.Unmarshal(encoded, &result); err != nil { - return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) + return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } if result.XMLName.Local == "Error" || result.Code != "" { - return "", transferError("upload_complete_failed", "backup multipart completion was rejected", false) + return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false) } if result.XMLName.Local != "CompleteMultipartUploadResult" { - return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) + return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } etag := strings.TrimSpace(result.ETag) if !validETag(etag) { - return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) + return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } - return etag, nil + versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id")) + if !validVersionID(versionID) { + return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) + } + return etag, versionID, nil +} + +func validVersionID(value string) bool { + if len(value) > 1024 { + return false + } + for _, current := range []byte(value) { + if current < 0x20 || current == 0x7f { + return false + } + } + return true } func (c *Client) failAndAbort(ctx context.Context, capability, code, message string) error { @@ -445,7 +471,17 @@ func (c *Client) Download(ctx context.Context, capability string, expectedSize i if !strings.EqualFold(digest, expectedSHA256) { return DownloadResult{}, transferError("object_sha256_mismatch", "backup object SHA-256 does not match", false) } - return DownloadResult{SizeBytes: written, SHA256: digest}, nil + etag := strings.TrimSpace(response.Header.Get("ETag")) + if etag != "" && !validETag(etag) { + etag = "" + } + versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id")) + if !validVersionID(versionID) { + versionID = "" + } + return DownloadResult{ + SizeBytes: written, SHA256: digest, ETag: etag, VersionID: versionID, + }, nil } // String is useful in bounded diagnostics without revealing URL capabilities. diff --git a/agent/objecttransfer/client_test.go b/agent/objecttransfer/client_test.go index 523c0199..696a4408 100644 --- a/agent/objecttransfer/client_test.go +++ b/agent/objecttransfer/client_test.go @@ -76,6 +76,7 @@ func TestUploadFileCompletesMultipartWithOrderedParts(t *testing.T) { t.Errorf("unexpected completion parts: %#v", request.Parts) } w.Header().Set("Content-Type", "application/xml") + w.Header().Set("x-amz-version-id", "version/7+opaque") _, _ = io.WriteString(w, `"final"`) case http.MethodDelete: aborted = true @@ -94,7 +95,8 @@ func TestUploadFileCompletesMultipartWithOrderedParts(t *testing.T) { } wantHash := sha256.Sum256(contents) if result.SizeBytes != int64(len(contents)) || result.SHA256 != hex.EncodeToString(wantHash[:]) || - result.ETag != `"final"` || len(result.Parts) != 3 || aborted { + result.ETag != `"final"` || result.VersionID != "version/7+opaque" || + len(result.Parts) != 3 || aborted { t.Fatalf("unexpected result: %#v aborted=%v", result, aborted) } var reconstructed []byte @@ -241,6 +243,8 @@ func TestUploadFileVerifiesCommittedObjectAfterAmbiguousCompletion(t *testing.T) _ = connection.Close() case http.MethodGet: verified = true + w.Header().Set("ETag", `"verified-etag"`) + w.Header().Set("x-amz-version-id", "verified-version") _, _ = w.Write(contents) case http.MethodDelete: aborted = true @@ -256,7 +260,9 @@ func TestUploadFileVerifiesCommittedObjectAfterAmbiguousCompletion(t *testing.T) t.Fatalf("result=%#v err=%v verified=%v aborted=%v", result, err, verified, aborted) } wantHash := sha256.Sum256(contents) - if result.SHA256 != hex.EncodeToString(wantHash[:]) || len(result.Parts) != 1 { + if result.SHA256 != hex.EncodeToString(wantHash[:]) || + result.ETag != `"verified-etag"` || result.VersionID != "verified-version" || + len(result.Parts) != 1 { t.Fatalf("unexpected verified result: %#v", result) } } diff --git a/docs/AGENT_API.md b/docs/AGENT_API.md index ad14c816..b4987720 100644 --- a/docs/AGENT_API.md +++ b/docs/AGENT_API.md @@ -207,7 +207,8 @@ operation 临时卷/metadata 的 evidence-bound cleanup plan,但当前没有 cleanup 的 mutation endpoint。 成功结果只返回 `backup_key`、对象 `size_bytes`、qcow2 -`virtual_size_bytes`、`sha256`、实际 `consistency`,以及本地临时副本未能清理时的 +`virtual_size_bytes`、`sha256`、multipart 完成响应中的可选 `object_etag` 与 +`object_version_id`、实际 `consistency`,以及本地临时副本未能清理时的 可选 `cleanup_pending=true`,不返回预签名 URL 或本机路径。本地清理失败不会丢弃 已经验证成功的远端对象身份,也不会把可恢复的备份错误降级为远端效果不明。 后续只读 inspection 在残留仍精确归属于该 operation 且没有活跃 libvirt/block job 时, diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 4cdd5e00..c2b5f032 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -95,7 +95,9 @@ php xcat cloudRepairInstanceMetadata --apply 后台 `/admin/object-storage` 可配置一个全局 S3 兼容对象存储应用,适用于 AWS S3、MinIO 及实现 AWS Signature Version 4 的兼容服务。该功能复用业务库已有的 `configurations` 表,不需要导入新表。Endpoint 必须是带有效证书的 HTTPS origin;MinIO 通常选择路径寻址,AWS S3 通常选择虚拟主机寻址。 -Access Key ID 与 Secret Access Key 只在超级管理员提交时进入 PHP,使用 `cloud_data_key` 的独立用途密钥密封后保存,读取页面不会回显。修改任一配置或停用应用都会清除原就绪标记;必须重新执行“验证连接并启用”。连接验证只发送签名的 `HeadBucket` 请求,需要目标 Bucket 的 `s3:ListBucket` 权限,不上传、覆盖或删除对象。管理员虚拟机管理已接入完整系统盘备份和新实例恢复,目标可以是原宿主机或其他合格宿主机;用户自动备份使用同一对象存储,在实例设置中按每天或每周于 UTC 00:00 调度并保留最多两份。运行账户只需配置 `vm-backups/` 前缀范围内的 multipart 上传、`GetObject`、`HeadObject` 和 `DeleteObject` 权限。维护迁移使用宿主机间 PeerMigration V3 直传,不创建 S3 迁移对象,也不需要 `vm-migrations/` 权限。当前不包含增量链和管理员手工备份的自动保留策略,具体安全边界见 `docs/PORTABLE_VM_BACKUPS.md`。 +Access Key ID 与 Secret Access Key 只在超级管理员提交时进入 PHP,使用 `cloud_data_key` 的独立用途密钥密封后保存,读取页面不会回显。修改任一配置或停用应用都会清除原就绪标记;必须重新执行“验证连接并启用”。连接验证只发送签名的 `HeadBucket` 请求,需要目标 Bucket 的 `s3:ListBucket` 权限,不上传、覆盖或删除对象。管理员虚拟机管理已接入完整系统盘备份和新实例恢复,目标可以是原宿主机或其他合格宿主机;用户自动备份使用同一对象存储,在实例设置中按每天或每周于 UTC 00:00 调度并保留最多两份。运行账户只需配置 `vm-backups/` 前缀范围内的 multipart 上传、`GetObject`、`HeadObject` 和 `DeleteObject` 权限;开启 Bucket 版本控制时还必须授予 `s3:GetObjectVersion` 与 `s3:DeleteObjectVersion`,让 HEAD 与删除准确绑定 `versionId`。维护迁移使用宿主机间 PeerMigration V3 直传,不创建 S3 迁移对象,也不需要 `vm-migrations/` 权限。当前不包含增量链和管理员手工备份的自动保留策略,具体安全边界见 `docs/PORTABLE_VM_BACKUPS.md`。 + +生产 Bucket 必须把生命周期规则作为控制面之外的最后兜底:在短周期内终止未完成 multipart upload;开启版本控制时,按业务保留要求清理 `vm-backups/` 的非当前版本和过期删除标记。生命周期不得提前删除仍由面板目录引用的当前版本。面板会保存 HEAD 返回的 ETag 与 `version_id`,删除时精确定位该版本;删除结果不明时先进行两次间隔至少 60 秒的稳定 HEAD,只有对象身份完全一致才创建新的删除代次,每个自动周期最多三次。达到上限的对象进入后台“对象清理异常”,不再占用用户自动备份保留份数,但仍计入对象清理债务,必须由管理员核查后手动开启新的受控周期。 生产环境应为面板创建独立服务账户,把权限限制到配置的 Bucket 和备份前缀。数据库备份必须与原始 `cloud_data_key` 一起保管,重新生成主密钥无法恢复全局配置及各备份冻结的对象存储凭据。 diff --git a/docs/PORTABLE_VM_BACKUPS.md b/docs/PORTABLE_VM_BACKUPS.md index 8087a158..4d8762b0 100644 --- a/docs/PORTABLE_VM_BACKUPS.md +++ b/docs/PORTABLE_VM_BACKUPS.md @@ -72,4 +72,4 @@ PeerMigration 不创建 `vm-migrations/` 对象,也不持有 S3 凭据。目 恢复失败时,只有 Agent 明确证明新 Domain、所有 UUID 所属卷、seed、NVRAM、网络过滤器和流量状态均不存在,控制面才能释放容量与公网 IP。只要清理结果不确定,资源保持锁定等待人工核对。 -备份对象删除也遵守同一原则:明确删除成功才释放目录额度;DELETE 响应未知时只允许通过 HEAD 核对对象已经不存在,不会自动重放删除。若 Bucket 开启版本控制,S3 删除会隐藏当前对象但可能保留历史版本,生产环境还应配置仅覆盖备份前缀的历史版本过期策略。 +备份对象删除也遵守同一原则:首次删除前保存对象的大小、ETag 和可用的 `version_id`;版本控制 Bucket 的 HEAD 与 DELETE 都精确绑定该版本。DELETE 响应未知后不会重放原请求,而是等待两次间隔至少 60 秒、且大小/ETag/版本完全一致的 HEAD 证据,再以新的操作标识创建一个删除代次。每个自动清理周期最多三次,达到上限转为 `delete_attention`,由超级管理员在“对象存储 -> 对象清理异常”核查并手动恢复新的受控周期。删除意图一旦建立便不再占用用户自动备份保留份数;仍存在的远端数据单独计入对象清理债务。生产 Bucket 还必须配置终止未完成 multipart、清理非当前版本和过期删除标记的生命周期规则,但不得提前删除仍被目录引用的当前版本。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 4a5a13e5..d655f656 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -134,7 +134,7 @@ jq empty deploy/*.json.example - 目标分块暂存必须由 Agent 派生到第一个 `storage_path_roots` 下的 `.peer-migration-staging`,不能落在 Journal 系统盘。故障注入分块断连后应从已有偏移继续;成功导入删除暂存,过期中断会话只能由有界本地清理回收,不能删除不匹配目录。 - 目标在恢复和用户防火墙收敛期间必须保持关机;源机的运行/关机、autostart、断网状态与修改后主机名必须在独立激活步骤后保持一致。传输确定失败须同时提供续传和安全回滚,防火墙、激活、回滚、源清理的确定失败均有显式恢复出口。若源准备已存在精确成功回执但 PHP 最终化失败,只能将原准备效果收敛为 `applied` 后创建新的补偿回滚任务;Agent 明确确认源机恢复前不得释放目标预留或 IP。未知效果只能核对原 operation,不能重放。普通云备份仍只能产生单系统盘 S3 对象,PeerMigration 不得创建 S3 迁移对象。 - 独立恢复可选择原宿主机或其他合格宿主机;无论目标是否为原宿主机,都必须单独校验并预留容量与全新公网 IP。目标使用 `bridge:br0`;校验下载对象字节数与 SHA-256,新实例 UUID、MAC、seed、NVRAM、网络过滤器和流量状态均不得复用源实例,且不得继承订单、账单、服务期、到期时间或流量周期。 -- 对备份/恢复断连、超时和回执丢失执行故障注入:同一 operation ID 只能查询收敛,不能再次派发。恢复失败只有在全部托管产物明确不存在时才能释放容量与 IP;对象删除响应未知时只能 HEAD 核对不存在,不能重发 DELETE。 +- 对备份/恢复断连、超时和回执丢失执行故障注入:同一 Agent operation ID 只能查询收敛,不能再次派发。恢复失败只有在全部托管产物明确不存在时才能释放容量与 IP。对象删除响应未知时不得重放原 DELETE;必须用两次间隔 HEAD 证明同一大小、ETag 和 `version_id` 仍存在,才允许创建新的删除代次。验证每轮最多三次、超限转人工、手动重试重新建立稳定证据、精确版本删除、清理债务不占保留份数,并确认大量异常债务不会饿死新的保留清理。 ## 7. 非生产宿主机验收 diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index bc2b04f8..83f13ede 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -8,7 +8,7 @@ use App\Services\Config; /** Initializes only the clean-break PHP business and Agent log schemas. */ final class CloudSetup { - private const SCHEMA_VERSION = 'go-agent-wss-v11-peer-migration-v2'; + private const SCHEMA_VERSION = 'go-agent-wss-v12-object-cleanup-v1'; private const MAIN_TABLES = [ 'cloud_regions', 'cloud_network_lines', 'cloud_resource_pools', 'cloud_hosts', 'cloud_host_agents', @@ -85,10 +85,14 @@ final class CloudSetup 'user_id', 'source_uuid', 'name', 'purpose', 'source', 'subscription_id', 'scheduled_for', 'state', 'consistency', 'object_key', 'object_size_bytes', 'virtual_size_bytes', - 'sha256', 'spec_json', 'sealed_storage_runtime', + 'sha256', 'object_etag', 'object_version_id', 'spec_json', + 'sealed_storage_runtime', 'sealed_credential', 'storage_revision', 'active_job_id', - 'last_operation_id', 'error_code', 'ready_at', 'created_at', - 'updated_at', + 'last_operation_id', 'delete_operation_id', 'delete_generation', + 'delete_cycle_attempts', 'delete_presence_confirmations', + 'delete_requested_at', 'delete_last_checked_at', + 'delete_next_check_at', 'delete_attention_at', 'error_code', + 'ready_at', 'created_at', 'updated_at', ], 'cloud_deployment_snapshots' => [ 'id', 'snapshot_key', 'user_id', 'backup_id', @@ -226,6 +230,7 @@ final class CloudSetup 'cloud_instance_backup_object', 'cloud_instance_backup_schedule', 'cloud_instance_backup_source', 'cloud_instance_backup_owner', 'cloud_instance_backup_operation', + 'cloud_instance_backup_delete_due', ], 'cloud_deployment_snapshots' => [ 'PRIMARY', 'cloud_deployment_snapshot_key', diff --git a/web/app/Controllers/Admin/ObjectStorageController.php b/web/app/Controllers/Admin/ObjectStorageController.php index 31d536c2..c5b1f946 100644 --- a/web/app/Controllers/Admin/ObjectStorageController.php +++ b/web/app/Controllers/Admin/ObjectStorageController.php @@ -3,6 +3,9 @@ namespace App\Controllers\Admin; use App\Controllers\AdminController; +use App\Models\CloudInstanceBackup; +use App\Services\Cloud\CloudAccess; +use App\Services\Cloud\CloudBackupService; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudInput; use App\Services\Cloud\CloudObjectStorageConfig; @@ -13,6 +16,8 @@ final class ObjectStorageController extends AdminController { $state = null; $error = ''; + $cleanupDebt = null; + $cleanupDebtError = ''; $status = 200; try { $state = CloudObjectStorageConfig::view(); @@ -24,10 +29,26 @@ final class ObjectStorageController extends AdminController $error = '对象存储配置暂时无法读取,请检查数据库状态。'; $status = 503; } + try { + $query = $request->getQueryParams(); + $cleanupDebt = CloudBackupService::cleanupDebt( + CloudAccess::actor(false), [ + 'page' => $query['cleanup_page'] ?? 1, + 'per_page' => 20, + ] + ); + } catch (CloudError $exception) { + $cleanupDebtError = $exception->getMessage(); + } catch (\Throwable $exception) { + error_log('Object cleanup debt read failed: ' . get_class($exception)); + $cleanupDebtError = '对象清理债务暂时无法读取,请检查数据库结构。'; + } $smarty = $this->view()->assign([ 'object_storage' => $state, 'object_storage_error' => $error, + 'cleanup_debt' => $cleanupDebt, + 'cleanup_debt_error' => $cleanupDebtError, ]); $response->getBody()->write($smarty->fetch('admin/object_storage.tpl')); return $response->withStatus($status) @@ -81,6 +102,60 @@ final class ObjectStorageController extends AdminController } } + public function cleanupDebt($request, $response, $args) + { + try { + $payload = CloudBackupService::cleanupDebt( + CloudAccess::actor(false), $request->getQueryParams() + ); + return $response->withJson([ + 'ret' => 1, + 'msg' => '对象清理债务已刷新。', + 'cleanup_debt' => $payload, + ])->withHeader('Cache-Control', 'private, no-store'); + } catch (CloudError $exception) { + return $this->json($response, 0, $exception->getMessage(), + $exception->httpStatus); + } catch (\Throwable $exception) { + error_log('Object cleanup debt refresh failed: ' + . get_class($exception)); + return $this->json($response, 0, + '对象清理债务刷新失败,请稍后重试。', 500); + } + } + + public function retryCleanup($request, $response, $args) + { + try { + $backupId = filter_var($args['backup_id'] ?? null, + FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); + if ($backupId === false) { + throw new CloudError('对象清理记录无效', 400); + } + $backup = CloudInstanceBackup::find((int) $backupId); + if ($backup === null) { + throw new CloudError('对象清理记录不存在', 404); + } + $result = CloudBackupService::retryCleanup( + $backup, CloudAccess::actor(false) + ); + return $response->withJson([ + 'ret' => 1, + 'msg' => ($result['deleted'] ?? false) + ? '对象已确认不存在,清理债务已关闭。' + : '已恢复稳定性核对;第二次证据一致后才会创建新删除代次。', + 'cleanup' => $result, + ])->withHeader('Cache-Control', 'private, no-store'); + } catch (CloudError $exception) { + return $this->json($response, 0, $exception->getMessage(), + $exception->httpStatus); + } catch (\Throwable $exception) { + error_log('Object cleanup retry failed: ' . get_class($exception)); + return $this->json($response, 0, + '对象清理重试失败,请稍后重试。', 500); + } + } + /** @param array|null $state */ private function json($response, int $ret, string $message, int $status, ?array $state = null) { diff --git a/web/app/Controllers/CloudAdminInstanceController.php b/web/app/Controllers/CloudAdminInstanceController.php index edc7df13..67e47602 100644 --- a/web/app/Controllers/CloudAdminInstanceController.php +++ b/web/app/Controllers/CloudAdminInstanceController.php @@ -214,7 +214,7 @@ final class CloudAdminInstanceController ->withHeader('Cache-Control', 'no-store'); } - /** Delete one backup object, or HEAD-reconcile an ambiguous prior delete. */ + /** Delete one backup object, or advance its bounded HEAD reconciliation. */ public function deleteBackup($request, $response, $args) { $actor = $this->actor($request, true); diff --git a/web/app/Models/CloudInstanceBackup.php b/web/app/Models/CloudInstanceBackup.php index a0744e96..8af4fa45 100644 --- a/web/app/Models/CloudInstanceBackup.php +++ b/web/app/Models/CloudInstanceBackup.php @@ -7,7 +7,10 @@ final class CloudInstanceBackup extends Model { protected $connection = 'default'; protected $table = 'cloud_instance_backups'; - protected $hidden = ['sealed_storage_runtime', 'sealed_credential']; + protected $hidden = [ + 'sealed_storage_runtime', 'sealed_credential', 'object_etag', + 'object_version_id', 'delete_operation_id', + ]; protected $casts = [ 'id' => 'int', 'source_instance_id' => 'int', @@ -19,6 +22,13 @@ final class CloudInstanceBackup extends Model 'virtual_size_bytes' => 'int', 'storage_revision' => 'int', 'active_job_id' => 'int', + 'delete_generation' => 'int', + 'delete_cycle_attempts' => 'int', + 'delete_presence_confirmations' => 'int', + 'delete_requested_at' => 'int', + 'delete_last_checked_at' => 'int', + 'delete_next_check_at' => 'int', + 'delete_attention_at' => 'int', 'ready_at' => 'int', 'created_at' => 'int', 'updated_at' => 'int', diff --git a/web/app/Services/Cloud/CloudAutomaticBackupService.php b/web/app/Services/Cloud/CloudAutomaticBackupService.php index d49c23da..580e36d5 100644 --- a/web/app/Services/Cloud/CloudAutomaticBackupService.php +++ b/web/app/Services/Cloud/CloudAutomaticBackupService.php @@ -309,7 +309,10 @@ final class CloudAutomaticBackupService return $queued; } - /** Delete at most $limit oldest ready objects beyond each subscription's cap. */ + /** + * Prune at most $limit fresh retention candidates plus a separate bounded + * cleanup-debt batch, so stuck remote objects cannot starve new rotation. + */ public static function pruneRetention(int $limit = 10, ?int $now = null, ?CloudObjectStorageS3Client $client = null): int { @@ -324,15 +327,15 @@ final class CloudAutomaticBackupService ]; $processed = 0; $removed = 0; - // Ambiguous object deletion is reconciled by HEAD only. Keep it in a - // bounded system path; otherwise it drops out of the ready-count - // query and can never converge after the first uncertain DELETE. - $pending = CloudInstanceBackup::where('source', 'automatic') - ->where('purpose', 'portable') + // Cleanup debt has its own budget and due index. It never consumes the + // fresh retention budget and attention-required rows wait for an + // explicit administrator action instead of monopolising every batch. + $debtLimit = max(1, min(10, intdiv($limit + 1, 2))); + $pending = CloudInstanceBackup::where('purpose', 'portable') ->whereIn('state', ['deleting', 'delete_uncertain']) - ->orderBy('id'); - self::withoutDeploymentSnapshots($pending); - $pending = $pending->limit($limit)->get(); + ->where('delete_next_check_at', '<=', $now) + ->orderBy('delete_next_check_at')->orderBy('id'); + $pending = $pending->limit($debtLimit)->get(); foreach ($pending as $backup) { try { $result = CloudBackupService::delete( @@ -345,10 +348,6 @@ final class CloudAutomaticBackupService error_log('cloud automatic backup retention unavailable: ' . get_class($error)); } - $processed++; - } - if ($processed >= $limit) { - return $removed; } // A disabled plan keeps its already-paid recovery rights. Only after // the latest immutable entitlement has ended plus the three-day grace @@ -669,7 +668,7 @@ final class CloudAutomaticBackupService { $state = in_array((string) $backup->state, [ 'creating', 'ready', 'failed', 'uncertain', 'deleting', - 'delete_uncertain', + 'delete_uncertain', 'delete_attention', ], true) ? (string) $backup->state : 'uncertain'; return [ 'id' => (string) $backup->id, diff --git a/web/app/Services/Cloud/CloudBackupService.php b/web/app/Services/Cloud/CloudBackupService.php index 46e23d7d..02169c84 100644 --- a/web/app/Services/Cloud/CloudBackupService.php +++ b/web/app/Services/Cloud/CloudBackupService.php @@ -20,8 +20,15 @@ final class CloudBackupService private const STORAGE_PURPOSE = 'object-storage-backup-runtime'; private const ACTIVE_BACKUP_STATES = [ - 'creating', 'ready', 'uncertain', 'deleting', 'delete_uncertain', + 'creating', 'ready', 'uncertain', ]; + private const CLEANUP_DEBT_STATES = [ + 'deleting', 'delete_uncertain', 'delete_attention', + ]; + private const DELETE_STABLE_PROBES = 2; + private const DELETE_STABLE_SECONDS = 60; + private const DELETE_RETRY_SECONDS = 300; + private const DELETE_MAX_AUTOMATIC_ATTEMPTS = 3; private const TRANSFER_URL_SECONDS = 604800; private const MAX_AGENT_BODY_BYTES = 1048576; // The create transaction opens a remote S3 multipart upload. Database @@ -330,8 +337,9 @@ final class CloudBackupService /** * Delete or safely reconcile one frozen S3 backup object. * - * A second call after an ambiguous DELETE never issues another DELETE. It - * only performs HEAD and converges to deleted after an explicit 404. + * An ambiguous DELETE is never replayed. Two spaced HEAD observations of + * the same immutable identity may authorize a fresh deletion generation; + * at most three generations run automatically in one supervised cycle. */ public static function delete(CloudInstanceBackup $backup, array $actor, ?CloudObjectStorageS3Client $client = null, @@ -339,8 +347,9 @@ final class CloudBackupService { CloudAccess::manage($actor); $client ??= new CloudObjectStorageS3Client(); + $now = time(); $prepared = $backup->getConnection()->transaction(function () use ( - $backup, $beforeDeleteLocked + $backup, $beforeDeleteLocked, $now ): array { $locked = CloudInstanceBackup::where('id', (int) $backup->id) ->lockForUpdate()->first(); @@ -352,86 +361,252 @@ final class CloudBackupService return ['mode' => 'deleted', 'backup' => $locked]; } self::assertBackupDeletionAvailable($locked); + if ($state === 'delete_attention') { + return ['mode' => 'attention', 'backup' => $locked]; + } if (in_array($state, ['deleting', 'delete_uncertain'], true)) { + if ((int) $locked->delete_next_check_at > $now) { + return ['mode' => 'waiting', 'backup' => $locked]; + } return [ 'mode' => 'reconcile', 'backup' => $locked, 'runtime' => self::storageRuntime($locked), 'object_key' => (string) $locked->object_key, + 'object_etag' => (string) $locked->object_etag, + 'object_version_id' => (string) $locked->object_version_id, ]; } if (!in_array($state, ['ready', 'failed'], true)) { throw new CloudError('当前备份状态不允许删除', 409); } - if ($beforeDeleteLocked !== null) { - $beforeDeleteLocked($locked); - } - // Decrypt before crossing the durable deleting boundary. A crash - // after the state change is treated as unknown and only HEAD may - // follow; DELETE is never replayed. + // Decrypt before any remote request. The exact immutable identity + // is captured by HEAD before crossing the deleting boundary. $runtime = self::storageRuntime($locked); - $locked->state = 'deleting'; - $locked->error_code = ''; - $locked->updated_at = time(); - $locked->save(); return [ - 'mode' => 'delete', + 'mode' => 'identify', 'previous_state' => $state, 'backup' => $locked, 'runtime' => $runtime, 'object_key' => (string) $locked->object_key, + 'object_etag' => (string) $locked->object_etag, + 'object_version_id' => (string) $locked->object_version_id, ]; }, 3); - if ($prepared['mode'] === 'deleted') { + if (in_array($prepared['mode'], ['deleted', 'waiting', 'attention'], true)) { return self::deletionResult($prepared['backup']); } $runtime = $prepared['runtime']; $objectKey = $prepared['object_key']; + if ($prepared['mode'] === 'identify') { + try { + $identity = $client->inspectObject( + $runtime['config'], $runtime['credentials'], $objectKey, + (string) $prepared['object_version_id'] + ); + } catch (CloudObjectStorageTransferError $error) { + throw new CloudError($error->getMessage(), 502); + } + if ($identity === null) { + return self::finishDelete((int) $backup->id); + } + self::assertObjectIdentity( + $prepared['backup'], $identity, + (string) $prepared['object_etag'] !== '' + ); + $prepared = self::claimDeleteGeneration( + (int) $backup->id, + (string) $prepared['previous_state'], + $identity, + time(), + $beforeDeleteLocked + ); + if ($prepared['mode'] !== 'delete') { + return self::deletionResult($prepared['backup']); + } + } if ($prepared['mode'] === 'reconcile') { try { - $presence = $client->objectPresence( - $runtime['config'], $runtime['credentials'], $objectKey + $identity = $client->inspectObject( + $runtime['config'], $runtime['credentials'], $objectKey, + (string) $prepared['object_version_id'] ); } catch (CloudObjectStorageTransferError $error) { - self::markDeleteUncertain((int) $backup->id, - $error->publicCode); + self::markDeleteUncertain( + (int) $backup->id, $error->publicCode, time(), true, + self::DELETE_RETRY_SECONDS + ); return self::deletionResult( CloudInstanceBackup::findOrFail((int) $backup->id) ); } - if ($presence === 'absent') { + if ($identity === null) { return self::finishDelete((int) $backup->id); } - self::markDeleteUncertain((int) $backup->id, - 'object_delete_still_present'); - return self::deletionResult( - CloudInstanceBackup::findOrFail((int) $backup->id) + try { + self::assertObjectIdentity($prepared['backup'], $identity, true); + } catch (CloudError $error) { + self::markDeleteAttention( + (int) $backup->id, 'object_identity_changed', time() + ); + return self::deletionResult( + CloudInstanceBackup::findOrFail((int) $backup->id) + ); + } + $prepared = self::recordStableDeleteProbe( + (int) $backup->id, time() ); + if ($prepared['mode'] !== 'delete') { + return self::deletionResult($prepared['backup']); + } + $runtime = $prepared['runtime']; + $objectKey = $prepared['object_key']; } try { $client->deleteObject( - $runtime['config'], $runtime['credentials'], $objectKey + $runtime['config'], $runtime['credentials'], $objectKey, + (string) $prepared['object_version_id'] ); } catch (CloudObjectStorageTransferError $error) { if ($error->effectMayHaveApplied) { - self::markDeleteUncertain((int) $backup->id, - $error->publicCode); + self::markDeleteUncertain( + (int) $backup->id, $error->publicCode, time(), true + ); return self::deletionResult( CloudInstanceBackup::findOrFail((int) $backup->id) ); } - self::restoreAfterRejectedDelete( - (int) $backup->id, - (string) $prepared['previous_state'], - $error->publicCode - ); + if (isset($prepared['previous_state'])) { + self::restoreAfterRejectedDelete( + (int) $backup->id, + (string) $prepared['previous_state'], + $error->publicCode + ); + } else { + self::markDeleteAttention( + (int) $backup->id, $error->publicCode, time() + ); + } throw new CloudError($error->getMessage(), 502); } return self::finishDelete((int) $backup->id); } + /** Sanitized administrator view of remote objects awaiting cleanup. */ + public static function cleanupDebt(array $actor, array $input = []): array + { + if (($actor['is_admin'] ?? false) !== true) { + throw new CloudError('只有管理员可以查看对象清理异常', 403); + } + CloudInput::only($input, ['page', 'per_page']); + $page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1); + $perPage = self::libraryInteger($input['per_page'] ?? 20, 10, 50, 20); + $query = CloudInstanceBackup::whereIn('state', self::CLEANUP_DEBT_STATES); + $total = (int) (clone $query)->count('id'); + $bytes = (int) (clone $query)->sum('object_size_bytes'); + $attention = (int) (clone $query) + ->where('state', 'delete_attention')->count('id'); + $totalPages = max(1, (int) ceil($total / $perPage)); + $page = min($page, $totalPages); + $rows = $query->orderByRaw( + "CASE WHEN state = 'delete_attention' THEN 0 ELSE 1 END" + )->orderBy('delete_next_check_at')->orderBy('id') + ->offset(($page - 1) * $perPage)->limit($perPage)->get(); + $items = []; + foreach ($rows as $row) { + $items[] = [ + 'id' => (string) $row->id, + 'name' => (string) $row->name, + 'source_instance_id' => (string) $row->source_instance_id, + 'user_id' => (string) $row->user_id, + 'state' => (string) $row->state, + 'object_size_bytes' => max(0, (int) $row->object_size_bytes), + 'delete_generation' => max(0, (int) $row->delete_generation), + 'cycle_attempts' => max(0, (int) $row->delete_cycle_attempts), + 'next_check_at' => max(0, (int) $row->delete_next_check_at), + 'attention_at' => $row->delete_attention_at === null ? null + : max(0, (int) $row->delete_attention_at), + 'error_code' => (string) $row->error_code, + 'can_retry' => (string) $row->state === 'delete_attention' + && ($actor['is_super_admin'] ?? false) === true + && ($actor['can_manage'] ?? false) === true, + ]; + } + return [ + 'summary' => [ + 'count' => $total, + 'bytes' => $bytes, + 'attention_count' => $attention, + ], + 'items' => $items, + 'pagination' => [ + 'page' => $page, + 'per_page' => $perPage, + 'total' => $total, + 'total_pages' => $totalPages, + ], + ]; + } + + /** Begin one new bounded cycle after an administrator has inspected debt. */ + public static function retryCleanup(CloudInstanceBackup $backup, array $actor, + ?CloudObjectStorageS3Client $client = null): array + { + CloudAccess::manage($actor); + $client ??= new CloudObjectStorageS3Client(); + $prepared = $backup->getConnection()->transaction(function () use ($backup): array { + $locked = CloudInstanceBackup::where('id', (int) $backup->id) + ->lockForUpdate()->firstOrFail(); + if ((string) $locked->state !== 'delete_attention') { + throw new CloudError('只有待人工处理的对象可以重新进入清理流程', 409); + } + self::assertBackupDeletionAvailable($locked); + return [ + 'runtime' => self::storageRuntime($locked), + 'object_key' => (string) $locked->object_key, + 'object_version_id' => (string) $locked->object_version_id, + 'backup' => $locked, + ]; + }, 3); + try { + $identity = $client->inspectObject( + $prepared['runtime']['config'], + $prepared['runtime']['credentials'], + $prepared['object_key'], + $prepared['object_version_id'] + ); + } catch (CloudObjectStorageTransferError $error) { + throw new CloudError($error->getMessage(), 502); + } + if ($identity === null) { + return self::finishDelete((int) $backup->id); + } + self::assertObjectIdentity($prepared['backup'], $identity, true); + $now = time(); + $locked = $backup->getConnection()->transaction(function () use ( + $backup, $now + ): CloudInstanceBackup { + $row = CloudInstanceBackup::where('id', (int) $backup->id) + ->lockForUpdate()->firstOrFail(); + if ((string) $row->state !== 'delete_attention') { + throw new CloudError('对象清理状态已发生变化', 409); + } + $row->state = 'delete_uncertain'; + $row->delete_cycle_attempts = 0; + $row->delete_presence_confirmations = 1; + $row->delete_last_checked_at = $now; + $row->delete_next_check_at = $now + self::DELETE_STABLE_SECONDS; + $row->delete_attention_at = null; + $row->error_code = 'manual_retry_awaiting_stable_probe'; + $row->updated_at = $now; + $row->save(); + return $row; + }, 3); + return self::deletionResult($locked); + } + public static function create(CloudInstance $instance, array $actor, array $input): array { @@ -1213,6 +1388,8 @@ final class CloudBackupService $virtual = filter_var($result['virtual_size_bytes'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); $sha = $result['sha256'] ?? null; + $objectEtag = $result['object_etag'] ?? ''; + $objectVersionId = $result['object_version_id'] ?? ''; $consistency = $result['consistency'] ?? null; $cleanupPending = $result['cleanup_pending'] ?? false; if (!is_array($result) @@ -1221,6 +1398,12 @@ final class CloudBackupService || $size === false || $virtual === false || !is_string($sha) || preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1 + || !is_string($objectEtag) || strlen($objectEtag) > 256 + || preg_match('/[\x00-\x1f\x7f]/D', $objectEtag) === 1 + || !is_string($objectVersionId) + || strlen($objectVersionId) > 1024 + || preg_match('/[\x00-\x1f\x7f]/D', $objectVersionId) === 1 + || ($objectVersionId !== '' && $objectEtag === '') || !in_array($consistency, ['filesystem', 'crash'], true) || !is_bool($cleanupPending) || (int) $size > (int) ($payload['maximum_size_bytes'] ?? 0)) { @@ -1231,6 +1414,8 @@ final class CloudBackupService $backup->object_size_bytes = (int) $size; $backup->virtual_size_bytes = (int) $virtual; $backup->sha256 = $sha; + $backup->object_etag = $objectEtag; + $backup->object_version_id = $objectVersionId; $backup->ready_at = $now; $backup->active_job_id = null; // The remote object is complete and restorable. A host-side @@ -1577,15 +1762,178 @@ final class CloudBackupService } } + /** @param array{size_bytes:int,etag:string,version_id:string} $identity */ + private static function claimDeleteGeneration(int $backupId, + string $previousState, array $identity, int $now, + ?callable $beforeDeleteLocked): array + { + $backup = CloudInstanceBackup::findOrFail($backupId); + return $backup->getConnection()->transaction(function () use ( + $backupId, $previousState, $identity, $now, $beforeDeleteLocked + ): array { + $locked = CloudInstanceBackup::where('id', $backupId) + ->lockForUpdate()->firstOrFail(); + if ((string) $locked->state === 'deleted') { + return ['mode' => 'deleted', 'backup' => $locked]; + } + if (in_array((string) $locked->state, + self::CLEANUP_DEBT_STATES, true)) { + return ['mode' => 'waiting', 'backup' => $locked]; + } + if ((string) $locked->state !== $previousState) { + throw new CloudError('备份删除状态已发生变化', 409); + } + self::assertBackupDeletionAvailable($locked); + if ($beforeDeleteLocked !== null) { + $beforeDeleteLocked($locked); + } + $locked->object_etag = (string) $identity['etag']; + $locked->object_version_id = (string) $identity['version_id']; + $locked->state = 'deleting'; + $locked->delete_operation_id = bin2hex(random_bytes(16)); + $locked->delete_generation = max(0, + (int) $locked->delete_generation) + 1; + $locked->delete_cycle_attempts = 1; + $locked->delete_presence_confirmations = 0; + $locked->delete_requested_at = $now; + $locked->delete_last_checked_at = null; + $locked->delete_next_check_at = 0; + $locked->delete_attention_at = null; + $locked->error_code = ''; + $locked->updated_at = $now; + $locked->save(); + return [ + 'mode' => 'delete', + 'previous_state' => $previousState, + 'backup' => $locked, + 'runtime' => self::storageRuntime($locked), + 'object_key' => (string) $locked->object_key, + 'object_version_id' => (string) $locked->object_version_id, + ]; + }, 3); + } + + private static function recordStableDeleteProbe(int $backupId, + int $now): array + { + $backup = CloudInstanceBackup::findOrFail($backupId); + return $backup->getConnection()->transaction(function () use ( + $backupId, $now + ): array { + $locked = CloudInstanceBackup::where('id', $backupId) + ->lockForUpdate()->firstOrFail(); + if ((string) $locked->state === 'deleted') { + return ['mode' => 'deleted', 'backup' => $locked]; + } + if ((string) $locked->state === 'delete_attention') { + return ['mode' => 'attention', 'backup' => $locked]; + } + if (!in_array((string) $locked->state, + ['deleting', 'delete_uncertain'], true)) { + throw new CloudError('备份删除状态已发生变化', 409); + } + $last = max(0, (int) $locked->delete_last_checked_at); + if ($last > 0 && $now - $last < self::DELETE_STABLE_SECONDS) { + $locked->state = 'delete_uncertain'; + $locked->delete_next_check_at = $last + + self::DELETE_STABLE_SECONDS; + $locked->updated_at = $now; + $locked->save(); + return ['mode' => 'waiting', 'backup' => $locked]; + } + $confirmations = min(self::DELETE_STABLE_PROBES, + max(0, (int) $locked->delete_presence_confirmations) + 1); + $locked->state = 'delete_uncertain'; + $locked->delete_presence_confirmations = $confirmations; + $locked->delete_last_checked_at = $now; + $locked->delete_next_check_at = $now + + self::DELETE_STABLE_SECONDS; + $locked->error_code = 'object_delete_still_present'; + $locked->updated_at = $now; + if ($confirmations < self::DELETE_STABLE_PROBES) { + $locked->save(); + return ['mode' => 'waiting', 'backup' => $locked]; + } + if ((int) $locked->delete_cycle_attempts + >= self::DELETE_MAX_AUTOMATIC_ATTEMPTS) { + $locked->state = 'delete_attention'; + $locked->delete_attention_at = $now; + $locked->delete_next_check_at = 0; + $locked->error_code = 'object_delete_retry_exhausted'; + $locked->save(); + return ['mode' => 'attention', 'backup' => $locked]; + } + $locked->state = 'deleting'; + $locked->delete_operation_id = bin2hex(random_bytes(16)); + $locked->delete_generation = max(0, + (int) $locked->delete_generation) + 1; + $locked->delete_cycle_attempts = max(0, + (int) $locked->delete_cycle_attempts) + 1; + $locked->delete_presence_confirmations = 0; + $locked->delete_next_check_at = 0; + $locked->error_code = ''; + $locked->save(); + return [ + 'mode' => 'delete', + 'backup' => $locked, + 'runtime' => self::storageRuntime($locked), + 'object_key' => (string) $locked->object_key, + 'object_version_id' => (string) $locked->object_version_id, + ]; + }, 3); + } + + /** @param array{size_bytes:int,etag:string,version_id:string} $identity */ + private static function assertObjectIdentity(CloudInstanceBackup $backup, + array $identity, bool $strict): void + { + if ((int) $identity['size_bytes'] !== (int) $backup->object_size_bytes) { + throw new CloudError('对象存储中的备份大小与目录账本不一致', 409); + } + if (!$strict) { + return; + } + $etag = (string) $backup->object_etag; + if ($etag === '' || !hash_equals($etag, (string) $identity['etag'])) { + throw new CloudError('对象存储中的备份 ETag 已发生变化', 409); + } + $versionId = (string) $backup->object_version_id; + if ($versionId !== '' + && !hash_equals($versionId, (string) $identity['version_id'])) { + throw new CloudError('对象存储中的备份版本已发生变化', 409); + } + } + private static function markDeleteUncertain(int $backupId, - string $errorCode): void + string $errorCode, int $now, bool $resetEvidence, + int $retryAfter = self::DELETE_STABLE_SECONDS): void + { + $values = [ + 'state' => 'delete_uncertain', + 'delete_next_check_at' => $now + max(1, $retryAfter), + 'error_code' => substr($errorCode, 0, 64), + 'updated_at' => $now, + ]; + if ($resetEvidence) { + $values['delete_presence_confirmations'] = 0; + $values['delete_last_checked_at'] = null; + } + CloudInstanceBackup::where('id', $backupId) + ->whereIn('state', ['deleting', 'delete_uncertain']) + ->update($values); + } + + private static function markDeleteAttention(int $backupId, + string $errorCode, int $now): void { CloudInstanceBackup::where('id', $backupId) ->whereIn('state', ['deleting', 'delete_uncertain']) ->update([ - 'state' => 'delete_uncertain', + 'state' => 'delete_attention', + 'delete_next_check_at' => 0, + 'delete_attention_at' => $now, 'error_code' => substr($errorCode, 0, 64), - 'updated_at' => time(), + 'updated_at' => $now, ]); } @@ -1596,6 +1944,13 @@ final class CloudBackupService ->where('state', 'deleting') ->update([ 'state' => $previousState, + 'delete_operation_id' => null, + 'delete_cycle_attempts' => 0, + 'delete_presence_confirmations' => 0, + 'delete_requested_at' => null, + 'delete_last_checked_at' => null, + 'delete_next_check_at' => 0, + 'delete_attention_at' => null, 'error_code' => substr($errorCode, 0, 64), 'updated_at' => time(), ]); @@ -1612,16 +1967,28 @@ final class CloudBackupService if ((string) $locked->state !== 'deleted') { self::assertBackupDeletionAvailable($locked); if (!in_array((string) $locked->state, - ['deleting', 'delete_uncertain'], true)) { + ['ready', 'failed', 'deleting', 'delete_uncertain', + 'delete_attention'], true)) { throw new CloudError('备份删除状态已发生变化', 409); } $locked->state = 'deleted'; $locked->sealed_storage_runtime = ''; $locked->sealed_credential = null; $locked->active_job_id = null; + $locked->delete_presence_confirmations = 0; + $locked->delete_next_check_at = 0; + $locked->delete_attention_at = null; $locked->error_code = ''; $locked->updated_at = time(); $locked->save(); + CloudDeploymentSnapshot::where('backup_id', $backupId) + ->whereIn('state', ['deleting', 'delete_uncertain']) + ->update([ + 'state' => 'deleted', + 'deleted_at' => time(), + 'error_code' => '', + 'updated_at' => time(), + ]); } return self::deletionResult($locked); }, 3); @@ -1634,7 +2001,10 @@ final class CloudBackupService 'backup_id' => (string) $backup->id, 'state' => $state, 'deleted' => $state === 'deleted', - 'requires_reconciliation' => $state === 'delete_uncertain', + 'requires_reconciliation' => in_array($state, + ['deleting', 'delete_uncertain'], true), + 'requires_attention' => $state === 'delete_attention', + 'next_check_at' => max(0, (int) $backup->delete_next_check_at), ]; } @@ -1782,7 +2152,7 @@ final class CloudBackupService { $state = in_array((string) $backup->state, ['creating', 'ready', 'failed', 'uncertain', 'deleting', - 'delete_uncertain', 'deleted'], true) + 'delete_uncertain', 'delete_attention', 'deleted'], true) ? (string) $backup->state : 'uncertain'; $canDelete = $canManage && in_array($state, ['ready', 'failed', 'deleting', 'delete_uncertain'], true); @@ -1822,9 +2192,11 @@ final class CloudBackupService 'delete_action' => in_array($state, ['deleting', 'delete_uncertain'], true) ? 'reconcile' : 'delete', - 'error' => $state === 'delete_uncertain' - ? '删除结果尚未确认;再次核对只会读取对象状态,不会重发删除请求。' - : null, + 'error' => $state === 'delete_attention' + ? '对象自动清理已达到三次上限,需要管理员检查对象存储后手动恢复清理。' + : ($state === 'delete_uncertain' + ? '删除结果尚未确认;系统会在两次稳定核对后创建新的删除代次。' + : null), ]; } diff --git a/web/app/Services/Cloud/CloudDeploymentSnapshotService.php b/web/app/Services/Cloud/CloudDeploymentSnapshotService.php index 471659f6..e0e98ebb 100644 --- a/web/app/Services/Cloud/CloudDeploymentSnapshotService.php +++ b/web/app/Services/Cloud/CloudDeploymentSnapshotService.php @@ -269,7 +269,7 @@ final class CloudDeploymentSnapshotService 'id', (int) $prepared['backup']->id )->value('state') ?? ''); $effectUnknown = in_array($backingState, - ['deleting', 'delete_uncertain', 'deleted'], true); + ['deleting', 'delete_uncertain', 'delete_attention', 'deleted'], true); CloudDeploymentSnapshot::where( 'id', (int) $prepared['snapshot']->id )->where('state', 'deleting')->update([ @@ -291,9 +291,12 @@ final class CloudDeploymentSnapshotService $locked->state = 'deleted'; $locked->deleted_at = time(); $locked->error_code = ''; - } elseif (($result['requires_reconciliation'] ?? false) === true) { + } elseif (($result['requires_reconciliation'] ?? false) === true + || ($result['requires_attention'] ?? false) === true) { $locked->state = 'delete_uncertain'; - $locked->error_code = 'object_delete_uncertain'; + $locked->error_code = ($result['requires_attention'] ?? false) + ? 'object_delete_attention_required' + : 'object_delete_uncertain'; } else { $locked->state = 'deleting'; } @@ -351,7 +354,7 @@ final class CloudDeploymentSnapshotService 'active_invoice_count' => $activeInvoiceCount, 'has_blocking_use' => $blocked, 'error' => $state === 'delete_uncertain' - ? '删除结果尚未确认;再次核对只会读取对象状态,不会重发删除请求。' + ? '删除结果尚未确认;系统会按稳定对象证据继续受控清理,达到上限后转管理员处理。' : null, ]; } diff --git a/web/app/Services/Cloud/CloudObjectStorageS3Client.php b/web/app/Services/Cloud/CloudObjectStorageS3Client.php index b7dcd69a..6b781915 100644 --- a/web/app/Services/Cloud/CloudObjectStorageS3Client.php +++ b/web/app/Services/Cloud/CloudObjectStorageS3Client.php @@ -320,11 +320,38 @@ final class CloudObjectStorageS3Client public function headObject( array $config, array $credentials, - string $objectKey + string $objectKey, + string $versionId = '' ): array { + $identity = $this->inspectObject( + $config, $credentials, $objectKey, $versionId + ); + if ($identity === null) { + throw new CloudObjectStorageTransferError( + '备份对象不存在或当前凭据无权读取。', 'object_head_rejected' + ); + } + return $identity; + } + + /** + * Inspect one exact object version. A 404 is a proven absence; transport, + * permission and provider failures remain distinguishable from absence. + * + * @param array $config + * @param array{access_key_id:string,secret_access_key:string} $credentials + * @return array{size_bytes:int,etag:string,version_id:string}|null + */ + public function inspectObject( + array $config, + array $credentials, + string $objectKey, + string $versionId = '' + ): ?array { self::assertRuntime($config, $credentials); self::assertObjectKey($objectKey); - $url = self::objectUrl($config, $objectKey); + self::assertVersionId($versionId); + $url = self::versionedObjectUrl($config, $objectKey, $versionId); $headers = CloudS3Signer::sign( 'HEAD', $url, (string) $config['region'], $credentials['access_key_id'], $credentials['secret_access_key'], @@ -339,6 +366,9 @@ final class CloudObjectStorageS3Client '无法读取备份对象状态。', 'object_head_unavailable' ); } + if ($response->getStatusCode() === 404) { + return null; + } if ($response->getStatusCode() < 200 || $response->getStatusCode() >= 300) { throw new CloudObjectStorageTransferError( '备份对象不存在或当前凭据无权读取。', 'object_head_rejected' @@ -351,16 +381,26 @@ final class CloudObjectStorageS3Client '备份对象缺少可信的大小信息。', 'object_head_invalid' ); } + $etag = trim($response->getHeaderLine('ETag')); + if ($etag === '' || strlen($etag) > 256 + || preg_match('/[\x00-\x1f\x7f]/', $etag) === 1) { + throw new CloudObjectStorageTransferError( + '备份对象缺少可信的 ETag。', 'object_head_invalid' + ); + } + $reportedVersion = trim($response->getHeaderLine('x-amz-version-id')); + self::assertVersionId($reportedVersion); return [ 'size_bytes' => (int) $length, - 'etag' => trim($response->getHeaderLine('ETag')), - 'version_id' => trim($response->getHeaderLine('x-amz-version-id')), + 'etag' => $etag, + 'version_id' => $reportedVersion, ]; } /** - * Delete one exact backup object. A transport failure has unknown effect; - * callers must reconcile it with objectPresence() and never replay it. + * Delete one exact backup object version. A transport failure has unknown + * effect; callers must reconcile it and may only create a new generation + * after independently stable identity evidence. * * @param array $config * @param array{access_key_id:string,secret_access_key:string} $credentials @@ -368,11 +408,13 @@ final class CloudObjectStorageS3Client public function deleteObject( array $config, array $credentials, - string $objectKey + string $objectKey, + string $versionId = '' ): void { self::assertRuntime($config, $credentials); self::assertObjectKey($objectKey); - $url = self::objectUrl($config, $objectKey); + self::assertVersionId($versionId); + $url = self::versionedObjectUrl($config, $objectKey, $versionId); $headers = CloudS3Signer::sign( 'DELETE', $url, (string) $config['region'], $credentials['access_key_id'], $credentials['secret_access_key'], @@ -404,7 +446,7 @@ final class CloudObjectStorageS3Client } /** - * Read-only reconciliation for an ambiguous DeleteObject call. + * Read-only presence check for one exact object version. * * @param array $config * @param array{access_key_id:string,secret_access_key:string} $credentials @@ -413,37 +455,12 @@ final class CloudObjectStorageS3Client public function objectPresence( array $config, array $credentials, - string $objectKey + string $objectKey, + string $versionId = '' ): string { - self::assertRuntime($config, $credentials); - self::assertObjectKey($objectKey); - $url = self::objectUrl($config, $objectKey); - $headers = CloudS3Signer::sign( - 'HEAD', $url, (string) $config['region'], - $credentials['access_key_id'], $credentials['secret_access_key'], - time() - ); - try { - $response = $this->http()->request('HEAD', $url, [ - 'headers' => $headers, - ] + self::requestOptions(30.0)); - } catch (GuzzleException $exception) { - throw new CloudObjectStorageTransferError( - '无法核对备份对象是否仍然存在。', - 'object_delete_probe_unavailable' - ); - } - $status = $response->getStatusCode(); - if ($status >= 200 && $status < 300) { - return 'present'; - } - if ($status === 404) { - return 'absent'; - } - throw new CloudObjectStorageTransferError( - '对象存储拒绝核对备份对象状态。', - 'object_delete_probe_rejected' - ); + return $this->inspectObject( + $config, $credentials, $objectKey, $versionId + ) === null ? 'absent' : 'present'; } /** @@ -506,6 +523,23 @@ final class CloudObjectStorageS3Client return rtrim($base, '/') . '/' . $encoded; } + /** @param array $config */ + private static function versionedObjectUrl(array $config, + string $objectKey, string $versionId): string + { + $url = self::objectUrl($config, $objectKey); + return $versionId === '' ? $url + : $url . '?versionId=' . rawurlencode($versionId); + } + + private static function assertVersionId(string $versionId): void + { + if (strlen($versionId) > 1024 + || preg_match('/[\x00-\x1f\x7f]/D', $versionId) === 1) { + throw new \InvalidArgumentException('Invalid S3 object version identifier'); + } + } + /** * @param array $config * @param array{access_key_id:string,secret_access_key:string} $credentials diff --git a/web/config/routes.php b/web/config/routes.php index 476da928..bb616094 100644 --- a/web/config/routes.php +++ b/web/config/routes.php @@ -178,6 +178,8 @@ $app->group('/admin', function () { $this->post('/object-storage/config', 'App\Controllers\Admin\ObjectStorageController:save')->add(new SuperAdmin()); $this->post('/object-storage/test', 'App\Controllers\Admin\ObjectStorageController:test')->add(new SuperAdmin()); $this->post('/object-storage/disable', 'App\Controllers\Admin\ObjectStorageController:disable')->add(new SuperAdmin()); + $this->get('/object-storage/cleanup-debts', 'App\Controllers\Admin\ObjectStorageController:cleanupDebt'); + $this->post('/object-storage/cleanup-debts/{backup_id:[0-9]+}/retry', 'App\Controllers\Admin\ObjectStorageController:retryCleanup')->add(new SuperAdmin()); $this->get('/cloud/templates', 'App\Controllers\Admin\CloudPageController:templates'); $this->get('/cloud/rescue', 'App\Controllers\Admin\CloudPageController:rescue'); $this->get('/cloud/instances', 'App\Controllers\Admin\CloudPageController:instances'); diff --git a/web/public/cloud/admin-instances.js b/web/public/cloud/admin-instances.js index 5a7e3329..5f3cebc3 100644 --- a/web/public/cloud/admin-instances.js +++ b/web/public/cloud/admin-instances.js @@ -671,7 +671,8 @@ function backupStatus(value) { creating: ['正在备份', 'warning'], ready: ['可恢复', 'success'], restoring: ['正在恢复', 'warning'], failed: ['备份失败', 'danger'], uncertain: ['效果待核对', 'danger'], deleting: ['正在删除', 'warning'], - delete_uncertain: ['删除待核对', 'danger'], deleted: ['已删除', 'secondary'], + delete_uncertain: ['删除待核对', 'danger'], + delete_attention: ['清理需人工处理', 'danger'], deleted: ['已删除', 'secondary'], })[String(value || '').toLowerCase()] || ['状态未知', 'secondary']; } @@ -741,7 +742,8 @@ async function deleteBackupObject(vmId, backup, controller, library = false) { error.status = response.status; throw error; } - if (payload?.deleted !== true && payload?.requires_reconciliation !== true) { + if (payload?.deleted !== true && payload?.requires_reconciliation !== true + && payload?.requires_attention !== true) { const error = new Error('删除回执状态无效,请勿重复删除,先刷新核对。'); error.mutationMayHaveApplied = true; throw error; @@ -771,13 +773,15 @@ function backupDeleteButton(vmId, backup, library = false) { const controller = new AbortController(); state.backupMutationRequest = controller; button.disabled = true; backupSurfaceNotice(reconcile - ? '正在只读核对对象是否已删除;不会重发删除请求。' + ? '正在核对准确对象版本;只有两次间隔证据一致才会创建新的删除代次。' : '正在删除对象存储备份,请勿重复提交。', false, library); try { const result = await deleteBackupObject(vmId, backup, controller, library); backupSurfaceNotice(result.deleted === true ? '备份对象已确认删除。' - : '删除结果仍不确定,已保留对象边界和冻结凭据;稍后可再次只读核对。', + : (result.requires_attention === true + ? '自动清理已达到三次上限,请到对象存储的“对象清理异常”中核查。' + : '删除结果仍不确定;系统将按稳定证据继续受控核对。'), result.deleted !== true, library); refreshBackupSurface(vmId, library); } catch (error) { @@ -1238,11 +1242,14 @@ function renderBackups(payload, vmId) { '备份效果尚未确认。系统会保留任务和对象边界,请勿重复创建或恢复,先完成任务核对。')); } else if (backup.state === 'delete_uncertain' || backup.state === 'deleting') { body.append(el('p', 'alert alert-warning fs-13', - '删除结果尚未确认。核对操作只会读取对象状态,不会重发删除请求。')); + '删除结果尚未确认。系统会在两次间隔核对确认同一对象后创建新的删除代次。')); if (backup.can_delete === true) { const deleteActions = el('div', 'd-flex justify-content-end'); deleteActions.append(backupDeleteButton(vmId, backup)); body.append(deleteActions); } + } else if (backup.state === 'delete_attention') { + body.append(el('p', 'alert alert-danger fs-13 mb-0', + '对象清理已达到自动重试上限,请在“对象存储 -> 对象清理异常”中核查并手动恢复。')); } else if (backup.state === 'ready') { const targets = backupTargets(payload, backup); if (targets.length) body.append(restoreForm(vmId, backup, targets, @@ -1343,7 +1350,7 @@ function renderBackupLibrary(payload) { const actions = el('div', 'd-flex justify-content-end mt-3'); actions.append(backupDeleteButton(vmId, backup, true)); body.append(actions); - } else if (!canManage && ['ready', 'failed', 'deleting', 'delete_uncertain'].includes(backup.state)) { + } else if (!canManage && ['ready', 'failed', 'deleting', 'delete_uncertain', 'delete_attention'].includes(backup.state)) { body.append(el('p', 'text-muted fs-12 mb-0', '当前管理员仅可查看,超级管理员才可以恢复或删除备份。')); } diff --git a/web/public/cloud/backups.js b/web/public/cloud/backups.js index 7ba0a6f1..d9971b54 100644 --- a/web/public/cloud/backups.js +++ b/web/public/cloud/backups.js @@ -59,7 +59,7 @@ function backupState(value) { const values = { ready: ['可用', 'running'], creating: ['正在备份', 'warning'], queued: ['等待备份', 'warning'], failed: ['备份失败', 'stopped'], uncertain: ['等待核对', 'warning'], deleting: ['正在删除', 'warning'], - delete_uncertain: ['删除待核对', 'warning'], + delete_uncertain: ['删除待核对', 'warning'], delete_attention: ['清理需人工处理', 'stopped'], }; return values[normalized] || [normalized || '状态未知', 'other']; } diff --git a/web/public/cloud/cloud-snapshots.js b/web/public/cloud/cloud-snapshots.js index bc1342dc..9195ba3a 100644 --- a/web/public/cloud/cloud-snapshots.js +++ b/web/public/cloud/cloud-snapshots.js @@ -222,7 +222,7 @@ async function deleteSnapshot(snapshot) { if (resultState === 'deleted') { toast('云快照已删除。', 'success'); } else if (resultState === 'delete_uncertain') { - toast('删除结果待核对,请再次点击删除;系统只会读取对象状态,不会重复发送删除请求。', 'warning'); + toast('删除结果待核对;系统会在两次间隔证据一致后创建新的删除代次。', 'warning'); } else { toast('云快照正在删除,请稍后刷新状态。', 'warning'); } diff --git a/web/public/cloud/detail.js b/web/public/cloud/detail.js index ec0d267c..3e3a0245 100644 --- a/web/public/cloud/detail.js +++ b/web/public/cloud/detail.js @@ -1557,6 +1557,8 @@ function automaticBackupState(backup) { ready: ['可恢复', 'running'], creating: ['正在备份', 'warning'], queued: ['等待备份', 'warning'], failed: ['备份失败', 'stopped'], uncertain: ['等待核对', 'warning'], restoring: ['正在恢复', 'warning'], + deleting: ['正在删除', 'warning'], delete_uncertain: ['删除待核对', 'warning'], + delete_attention: ['清理需管理员处理', 'stopped'], }; return labels[value] || [value || '状态未知', 'other']; } diff --git a/web/resources/views/venix/admin/object_storage.tpl b/web/resources/views/venix/admin/object_storage.tpl index e881a238..e017348a 100644 --- a/web/resources/views/venix/admin/object_storage.tpl +++ b/web/resources/views/venix/admin/object_storage.tpl @@ -2,7 +2,7 @@ {include file='admin/static_header.tpl'} {literal}{/literal}
@@ -107,12 +107,61 @@ {/if} +
+
+
+

对象清理异常

+

删除意图已从用户保留份数中移除;这里单独跟踪仍需在对象存储收敛的清理债务。

+
+ +
+ {if ($cleanup_debt_error|default:'') != ''} +
{$cleanup_debt_error|escape}
+ {elseif $cleanup_debt|default:false} +
+
清理债务{$cleanup_debt.summary.count} 个对象
+
待人工处理{$cleanup_debt.summary.attention_count} 个对象
+
占用空间{$cleanup_debt.summary.bytes} B
+
+ {if $cleanup_debt.items|@count > 0} +
+ + + + {foreach $cleanup_debt.items as $item} + + + + + + + + + {/foreach} + +
备份用户 / 实例状态删除代次对象大小操作
#{$item.id} {$item.name|escape}
{$item.error_code|escape}
用户 #{$item.user_id}
实例 #{$item.source_instance_id}
{if $item.state == 'delete_attention'}需人工处理{else}自动核对中{/if}总计 {$item.delete_generation}
本轮 {$item.cycle_attempts} / 3
{$item.object_size_bytes} B{if $item.can_retry}{else}等待系统核对{/if}
+
+ {if $cleanup_debt.pagination.total_pages > 1} +
+ 上一页 + 第 {$cleanup_debt.pagination.page} / {$cleanup_debt.pagination.total_pages} 页 + 下一页 +
+ {/if} + {else} +
当前没有对象清理异常。
+ {/if} + {/if} +
+
+
-

当前功能边界

本阶段只建立安全、可验证的对象存储配置基础。

+

运行边界

云备份使用精确对象身份;Bucket 生命周期只作为控制面之外的最后兜底。

  • 保存 S3 Endpoint、Region、Bucket、前缀和寻址方式,并加密保存访问凭据。
  • 使用签名的 HeadBucket 请求验证 Bucket 可达性和 s3:ListBucket 权限;测试不会写入对象。
  • -
  • 云备份上传、增量链、保留策略、加密备份和异地恢复尚未接入,等待对象存储配置稳定后单独开发。
  • +
  • 云备份对象采用独立目录身份;版本控制 Bucket 会保存并按准确 version_id 删除,并需要 s3:GetObjectVersion / s3:DeleteObjectVersion 权限。
  • +
  • Bucket 仍应配置“终止未完成 multipart”和“清理非当前版本/过期删除标记”的生命周期规则作为最后兜底。
@@ -230,4 +279,51 @@ })(); {/literal} +{literal}{/literal} + {include file='admin/static_footer.tpl'} diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index cc9676fa..ba78139f 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026102801' scope='global'} +{assign var=venix_asset_version value='2026102802' scope='global'} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index c17dad4a..6f7ab813 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -599,12 +599,22 @@ CREATE TABLE cloud_instance_backups ( object_size_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0, virtual_size_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0, sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + object_etag VARCHAR(256) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + object_version_id VARCHAR(1024) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', spec_json LONGTEXT NOT NULL, sealed_storage_runtime LONGTEXT NOT NULL, sealed_credential LONGTEXT DEFAULT NULL, storage_revision BIGINT UNSIGNED NOT NULL, active_job_id BIGINT UNSIGNED DEFAULT NULL, last_operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, + delete_operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, + delete_generation INT UNSIGNED NOT NULL DEFAULT 0, + delete_cycle_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0, + delete_presence_confirmations TINYINT UNSIGNED NOT NULL DEFAULT 0, + delete_requested_at BIGINT UNSIGNED DEFAULT NULL, + delete_last_checked_at BIGINT UNSIGNED DEFAULT NULL, + delete_next_check_at BIGINT UNSIGNED NOT NULL DEFAULT 0, + delete_attention_at BIGINT UNSIGNED DEFAULT NULL, error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', ready_at BIGINT UNSIGNED DEFAULT NULL, created_at BIGINT UNSIGNED NOT NULL, @@ -614,7 +624,8 @@ CREATE TABLE cloud_instance_backups ( UNIQUE KEY cloud_instance_backup_schedule (subscription_id, scheduled_for), KEY cloud_instance_backup_source (source_instance_id, state, id), KEY cloud_instance_backup_owner (user_id, id), - KEY cloud_instance_backup_operation (last_operation_id) + KEY cloud_instance_backup_operation (last_operation_id), + KEY cloud_instance_backup_delete_due (state, delete_next_check_at, id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- A user-owned, durable deployment source promoted from one automatic cloud -- backup. The backing backup row remains the sole owner of the immutable S3 diff --git a/web/tests/CloudAutomaticBackupRetentionGraceTest.php b/web/tests/CloudAutomaticBackupRetentionGraceTest.php index f0749ef0..4d9df209 100644 --- a/web/tests/CloudAutomaticBackupRetentionGraceTest.php +++ b/web/tests/CloudAutomaticBackupRetentionGraceTest.php @@ -100,12 +100,22 @@ namespace { $table->integer('object_size_bytes'); $table->integer('virtual_size_bytes'); $table->string('sha256'); + $table->string('object_etag')->default(''); + $table->string('object_version_id')->default(''); $table->text('spec_json'); $table->text('sealed_storage_runtime'); $table->text('sealed_credential')->nullable(); $table->integer('storage_revision'); $table->integer('active_job_id')->nullable(); $table->string('last_operation_id')->nullable(); + $table->string('delete_operation_id')->nullable(); + $table->integer('delete_generation')->default(0); + $table->integer('delete_cycle_attempts')->default(0); + $table->integer('delete_presence_confirmations')->default(0); + $table->integer('delete_requested_at')->nullable(); + $table->integer('delete_last_checked_at')->nullable(); + $table->integer('delete_next_check_at')->default(0); + $table->integer('delete_attention_at')->nullable(); $table->string('error_code'); $table->integer('scheduled_for')->nullable(); $table->integer('ready_at')->nullable(); @@ -118,6 +128,9 @@ namespace { $table->increments('id'); $table->integer('backup_id')->unique(); $table->string('state'); + $table->string('error_code')->default(''); + $table->integer('deleted_at')->nullable(); + $table->integer('updated_at')->default(0); }); foreach (['cloud_backup_restores', 'cloud_backup_recoveries'] as $tableName) { $schema->create($tableName, static function (Blueprint $table): void { @@ -192,6 +205,7 @@ namespace { 'object_key' => 'virt/vm-backups/' . $suffix . '/system.qcow2', 'object_size_bytes' => 1024, 'virtual_size_bytes' => 2048, 'sha256' => str_repeat('a', 64), + 'object_etag' => '', 'object_version_id' => '', 'spec_json' => json_encode([ 'managed' => true, 'os_source' => 'cloud_template', 'os_name' => 'Debian 13', 'login_name' => 'root', @@ -199,6 +213,9 @@ namespace { ], JSON_THROW_ON_ERROR), 'sealed_storage_runtime' => $sealedRuntime, 'sealed_credential' => 'sealed-credential', 'storage_revision' => 1, + 'delete_generation' => 0, 'delete_cycle_attempts' => 0, + 'delete_presence_confirmations' => 0, + 'delete_next_check_at' => 0, 'error_code' => '', 'scheduled_for' => $now - 1000, 'ready_at' => $now - 900, 'created_at' => $now - 1000, 'updated_at' => $now - 1000, @@ -217,6 +234,17 @@ namespace { $newSubscription(15, [$now - 3 * 86400]), 'ordinary-expired' ); + $debt = $newBackup( + $newSubscription(16, [$now - 3 * 86400]), + 'cleanup-debt' + ); + $debt->state = 'delete_uncertain'; + $debt->object_etag = '"debt-etag"'; + $debt->object_version_id = 'debt-version'; + $debt->delete_generation = 1; + $debt->delete_cycle_attempts = 1; + $debt->delete_next_check_at = 0; + $debt->save(); $capsule->table('cloud_deployment_snapshots')->insert([ 'backup_id' => (int) $expired->id, 'state' => 'ready', @@ -228,24 +256,34 @@ namespace { && CloudAutomaticBackupService::withinRecoveryGrace($renewed, $now), 'Recovery eligibility did not follow the paid period plus three-day grace.'); $history = []; - $stack = HandlerStack::create(new MockHandler([new Response(204)])); + $stack = HandlerStack::create(new MockHandler([ + new Response(404), + new Response(200, [ + 'Content-Length' => '1024', + 'ETag' => '"fresh-etag"', + 'x-amz-version-id' => 'fresh-version', + ]), + new Response(204), + ])); $stack->push(Middleware::history($history)); $client = new CloudObjectStorageS3Client(new Client(['handler' => $stack])); - $removed = CloudAutomaticBackupService::pruneRetention(10, $now, $client); + $removed = CloudAutomaticBackupService::pruneRetention(1, $now, $client); - foreach ([$paid, $insideGrace, $expired, $renewed, $ordinaryExpired] + foreach ([$paid, $insideGrace, $expired, $renewed, $ordinaryExpired, $debt] as $backup) { $backup->refresh(); } - $assert($removed === 1 + $assert($removed === 2 && (string) $paid->state === 'ready' && (string) $insideGrace->state === 'ready' && (string) $expired->state === 'ready' && (string) $renewed->state === 'ready' && (string) $ordinaryExpired->state === 'deleted' - && count($history) === 1 - && $history[0]['request']->getMethod() === 'DELETE', - 'Automatic backups were not retained through grace or were pruned after promotion to a cloud snapshot.'); + && (string) $debt->state === 'deleted' + && count($history) === 3 + && array_map(static fn ($item): string => + $item['request']->getMethod(), $history) === ['HEAD', 'HEAD', 'DELETE'], + 'Cleanup debt starved fresh retention work or entitlement grace was not preserved.'); echo "Automatic backup entitlement grace retention tests passed.\n"; } diff --git a/web/tests/CloudDeploymentSnapshotServiceTest.php b/web/tests/CloudDeploymentSnapshotServiceTest.php index 4c19b050..0a24a31c 100644 --- a/web/tests/CloudDeploymentSnapshotServiceTest.php +++ b/web/tests/CloudDeploymentSnapshotServiceTest.php @@ -95,12 +95,22 @@ namespace { $table->integer('object_size_bytes'); $table->integer('virtual_size_bytes'); $table->string('sha256'); + $table->string('object_etag')->default(''); + $table->string('object_version_id')->default(''); $table->text('spec_json'); $table->text('sealed_storage_runtime'); $table->text('sealed_credential')->nullable(); $table->integer('storage_revision'); $table->integer('active_job_id')->nullable(); $table->string('last_operation_id')->nullable(); + $table->string('delete_operation_id')->nullable(); + $table->integer('delete_generation')->default(0); + $table->integer('delete_cycle_attempts')->default(0); + $table->integer('delete_presence_confirmations')->default(0); + $table->integer('delete_requested_at')->nullable(); + $table->integer('delete_last_checked_at')->nullable(); + $table->integer('delete_next_check_at')->default(0); + $table->integer('delete_attention_at')->nullable(); $table->string('error_code'); $table->integer('ready_at')->nullable(); $table->integer('created_at'); @@ -208,6 +218,8 @@ namespace { $backup->object_size_bytes = 1024; $backup->virtual_size_bytes = 2048; $backup->sha256 = str_repeat(dechex(($id % 15) + 1), 64); + $backup->object_etag = ''; + $backup->object_version_id = ''; $backup->spec_json = json_encode([ 'managed' => true, 'os_source' => 'cloud_template', @@ -220,6 +232,10 @@ namespace { $backup->storage_revision = 1; $backup->active_job_id = null; $backup->last_operation_id = null; + $backup->delete_generation = 0; + $backup->delete_cycle_attempts = 0; + $backup->delete_presence_confirmations = 0; + $backup->delete_next_check_at = 0; $backup->error_code = ''; $backup->ready_at = $now; $backup->created_at = $backup->updated_at = $now; @@ -398,13 +414,21 @@ namespace { $deleted = CloudDeploymentSnapshotService::delete( (string) $snapshot->snapshot_key, $actor, - $client([new Response(204)], $history) + $client([ + new Response(200, [ + 'Content-Length' => '1024', + 'ETag' => '"snapshot-etag"', + 'x-amz-version-id' => 'snapshot-version', + ]), + new Response(204), + ], $history) ); $good->refresh(); $assert(($deleted['state'] ?? '') === 'deleted' && (string) $good->state === 'deleted' - && count($history) === 1 - && $history[0]['request']->getMethod() === 'DELETE', + && count($history) === 2 + && $history[0]['request']->getMethod() === 'HEAD' + && $history[1]['request']->getMethod() === 'DELETE', 'Cloud snapshot deletion did not converge its backing object.'); $uncertainBackup = $newBackup(15, 7, 3); @@ -415,14 +439,24 @@ namespace { $uncertain = CloudDeploymentSnapshotService::delete( (string) $uncertainSnapshot->snapshot_key, $actor, - $client([new ConnectException( - 'timeout', new Request('DELETE', 'https://objects.example.com') - )], $uncertainHistory) + $client([ + new Response(200, [ + 'Content-Length' => '1024', + 'ETag' => '"uncertain-etag"', + 'x-amz-version-id' => 'uncertain-version', + ]), + new ConnectException( + 'timeout', new Request('DELETE', 'https://objects.example.com') + ), + ], $uncertainHistory) ); $assert(($uncertain['state'] ?? '') === 'delete_uncertain' - && count($uncertainHistory) === 1 - && $uncertainHistory[0]['request']->getMethod() === 'DELETE', + && count($uncertainHistory) === 2 + && $uncertainHistory[1]['request']->getMethod() === 'DELETE', 'Unknown object deletion was not reflected by the cloud snapshot.'); + $uncertainBackup->refresh(); + $uncertainBackup->delete_next_check_at = 0; + $uncertainBackup->save(); $reconcileHistory = []; $reconciled = CloudDeploymentSnapshotService::delete( (string) $uncertainSnapshot->snapshot_key, diff --git a/web/tests/CloudFreshSchemaContractTest.php b/web/tests/CloudFreshSchemaContractTest.php index d51ef32c..ea3e7e74 100644 --- a/web/tests/CloudFreshSchemaContractTest.php +++ b/web/tests/CloudFreshSchemaContractTest.php @@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false $assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema), 'cloud.sql retained an engine-prefixed identifier.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v11-peer-migration-v2'") + "SCHEMA_VERSION = 'go-agent-wss-v12-object-cleanup-v1'") && !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"), 'Outbound Agent schema must reject the incompatible earlier Go database.'); $assert(str_contains($schema, @@ -89,6 +89,21 @@ $assert(str_contains($schema, && str_contains($setup, "'cloud_job_backup_recovery'"), 'Fresh schema is missing portable-backup catalog or restore reservation constraints.'); +$assert(str_contains($schema, + 'object_etag VARCHAR(256) CHARACTER SET ascii COLLATE ascii_bin NOT NULL') + && str_contains($schema, + 'object_version_id VARCHAR(1024) CHARACTER SET ascii COLLATE ascii_bin NOT NULL') + && str_contains($schema, + 'delete_generation INT UNSIGNED NOT NULL DEFAULT 0') + && str_contains($schema, + 'delete_cycle_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0') + && str_contains($schema, + 'delete_presence_confirmations TINYINT UNSIGNED NOT NULL DEFAULT 0') + && str_contains($schema, + 'KEY cloud_instance_backup_delete_due (state, delete_next_check_at, id)') + && str_contains($setup, "'object_etag', 'object_version_id'") + && str_contains($setup, "'cloud_instance_backup_delete_due'"), + 'Fresh backup schema is missing exact-version cleanup debt fields or index.'); $assert(str_contains($schema, 'CREATE TABLE cloud_backup_period_entitlements (') && str_contains($schema, diff --git a/web/tests/CloudObjectStorageAdminContractTest.php b/web/tests/CloudObjectStorageAdminContractTest.php index 8fab835b..07c71362 100644 --- a/web/tests/CloudObjectStorageAdminContractTest.php +++ b/web/tests/CloudObjectStorageAdminContractTest.php @@ -31,6 +31,11 @@ foreach (['config' => 'save', 'test' => 'test', 'disable' => 'disable'] as $path $assert(str_contains($routes, $needle), 'Object-storage mutation lost SuperAdmin/CSRF middleware: ' . $action); } +$assert(str_contains($routes, + "get('/object-storage/cleanup-debts', 'App\\Controllers\\Admin\\ObjectStorageController:cleanupDebt')") + && str_contains($routes, + "post('/object-storage/cleanup-debts/{backup_id:[0-9]+}/retry', 'App\\Controllers\\Admin\\ObjectStorageController:retryCleanup')->add(new SuperAdmin())"), + 'Object cleanup debt list or protected manual retry route is missing.'); $assert(str_contains($sidebar, 'href="/admin/object-storage"') && str_contains($sidebar, '对象存储'), 'Object-storage navigation entry is missing.'); @@ -57,14 +62,21 @@ $assert(substr_count($template, 'name="access_key_id" type="password" value=""') && !str_contains($template, '$object_storage.access_key_id') && !str_contains($template, '$object_storage.secret_access_key') && str_contains($template, 'data-can-manage=') - && str_contains($template, 's3:ListBucket'), + && str_contains($template, 's3:ListBucket') + && str_contains($template, '对象清理异常') + && str_contains($template, 'os-cleanup-retry') + && !str_contains($template, '$item.object_version_id'), 'Admin UI can expose credentials or lost its permission/test boundary.'); $assert(str_contains($controller, "'Cache-Control', 'private, no-store'") + && str_contains($controller, 'CloudBackupService::cleanupDebt') + && str_contains($controller, 'CloudBackupService::retryCleanup') && preg_match('/error_log\\([^\\n]*getMessage/', $controller) !== 1, 'Object-storage responses lost no-store or may log raw exception messages.'); $assert(is_string($deployment) && str_contains($deployment, '### 对象存储应用') && str_contains($deployment, '可移植备份') && str_contains($deployment, 'multipart 上传、读取、HEAD 和删除权限') + && str_contains($deployment, '非当前版本和过期删除标记') + && str_contains($deployment, '终止未完成 multipart') && is_string($validation) && str_contains($validation, '对象存储配置页面不得回显'), 'Deployment or validation boundaries are not documented.'); diff --git a/web/tests/CloudPortableBackupAdminUiContractTest.php b/web/tests/CloudPortableBackupAdminUiContractTest.php index 98a1dd35..3e9be478 100644 --- a/web/tests/CloudPortableBackupAdminUiContractTest.php +++ b/web/tests/CloudPortableBackupAdminUiContractTest.php @@ -130,9 +130,9 @@ $assert(str_contains($runtime, $assert(str_contains($runtime, "method: 'DELETE'") && str_contains($runtime, '核对删除状态') - && str_contains($runtime, '不会重发删除请求') + && str_contains($runtime, '两次间隔证据一致') && str_contains($runtime, 'requires_reconciliation'), - 'Backup deletion UI must expose non-replaying read-only reconciliation after unknown effect.'); + 'Backup deletion UI must expose bounded stable-evidence reconciliation after unknown effect.'); $assert(str_contains($runtime, '统一接入 br0、分配全新公网 IP') && str_contains($runtime, '不会继承原 IP、账单、到期时间或流量周期') diff --git a/web/tests/CloudPortableBackupDeletionTest.php b/web/tests/CloudPortableBackupDeletionTest.php index db003848..53b55a4f 100644 --- a/web/tests/CloudPortableBackupDeletionTest.php +++ b/web/tests/CloudPortableBackupDeletionTest.php @@ -76,12 +76,22 @@ namespace { $table->integer('object_size_bytes'); $table->integer('virtual_size_bytes'); $table->string('sha256'); + $table->string('object_etag')->default(''); + $table->string('object_version_id')->default(''); $table->text('spec_json'); $table->text('sealed_storage_runtime'); $table->text('sealed_credential')->nullable(); $table->integer('storage_revision'); $table->integer('active_job_id')->nullable(); $table->string('last_operation_id')->nullable(); + $table->string('delete_operation_id')->nullable(); + $table->integer('delete_generation')->default(0); + $table->integer('delete_cycle_attempts')->default(0); + $table->integer('delete_presence_confirmations')->default(0); + $table->integer('delete_requested_at')->nullable(); + $table->integer('delete_last_checked_at')->nullable(); + $table->integer('delete_next_check_at')->default(0); + $table->integer('delete_attention_at')->nullable(); $table->string('error_code'); $table->integer('ready_at')->nullable(); $table->integer('created_at'); @@ -92,6 +102,9 @@ namespace { $table->increments('id'); $table->integer('backup_id')->unique(); $table->string('state'); + $table->string('error_code')->default(''); + $table->integer('deleted_at')->nullable(); + $table->integer('updated_at')->default(0); }); $schema->create('cloud_backup_restores', static function (Blueprint $table): void { @@ -153,12 +166,22 @@ namespace { $backup->object_size_bytes = 1024; $backup->virtual_size_bytes = 2048; $backup->sha256 = str_repeat('a', 64); + $backup->object_etag = ''; + $backup->object_version_id = ''; $backup->spec_json = '{}'; $backup->sealed_storage_runtime = $sealedRuntime; $backup->sealed_credential = 'sealed-guest-credential'; $backup->storage_revision = 7; $backup->active_job_id = null; $backup->last_operation_id = null; + $backup->delete_operation_id = null; + $backup->delete_generation = 0; + $backup->delete_cycle_attempts = 0; + $backup->delete_presence_confirmations = 0; + $backup->delete_requested_at = null; + $backup->delete_last_checked_at = null; + $backup->delete_next_check_at = 0; + $backup->delete_attention_at = null; $backup->error_code = ''; $backup->ready_at = time(); $backup->created_at = $backup->updated_at = time(); @@ -170,24 +193,33 @@ namespace { $stack->push(Middleware::history($history)); return new CloudObjectStorageS3Client(new Client(['handler' => $stack])); }; + $identity = static fn (): Response => new Response(200, [ + 'Content-Length' => '1024', + 'ETag' => '"catalog-etag"', + 'x-amz-version-id' => 'catalog-version', + ]); $history = []; $backup = $newBackup(10, 'one'); $deleted = CloudBackupService::delete( - $backup, $actor, $client([new Response(204)], $history) + $backup, $actor, $client([$identity(), new Response(204)], $history) ); $backup->refresh(); $assert($deleted['deleted'] === true && (string) $backup->state === 'deleted' && (string) $backup->sealed_storage_runtime === '' && $backup->sealed_credential === null - && count($history) === 1 - && $history[0]['request']->getMethod() === 'DELETE', + && count($history) === 2 + && $history[0]['request']->getMethod() === 'HEAD' + && $history[1]['request']->getMethod() === 'DELETE' + && str_contains((string) $history[1]['request']->getUri(), + 'versionId=catalog-version'), 'Explicit DeleteObject success did not tombstone the catalog and remove frozen credentials.'); $unknown = $newBackup(11, 'two'); $unknownHistory = []; $unknownResult = CloudBackupService::delete($unknown, $actor, $client([ + $identity(), new ConnectException('transport lost', new Request('DELETE', 'https://objects.example.com')), ], $unknownHistory)); @@ -196,6 +228,8 @@ namespace { && (string) $unknown->state === 'delete_uncertain' && (string) $unknown->sealed_storage_runtime !== '', 'Ambiguous DeleteObject did not retain the frozen runtime and unknown-effect fence.'); + $unknown->delete_next_check_at = 0; + $unknown->save(); $probeHistory = []; $reconciled = CloudBackupService::delete( $unknown, $actor, $client([new Response(404)], $probeHistory) @@ -208,12 +242,16 @@ namespace { $present = $newBackup(12, 'three'); $firstHistory = []; CloudBackupService::delete($present, $actor, $client([ + $identity(), new ConnectException('transport lost', new Request('DELETE', 'https://objects.example.com')), ], $firstHistory)); + $present->refresh(); + $present->delete_next_check_at = 0; + $present->save(); $presentHistory = []; $stillPresent = CloudBackupService::delete( - $present, $actor, $client([new Response(200)], $presentHistory) + $present, $actor, $client([$identity()], $presentHistory) ); $present->refresh(); $assert($stillPresent['requires_reconciliation'] === true @@ -256,7 +294,7 @@ namespace { $quotaHistory = []; CloudBackupService::delete( $quotaBackups[0], $actor, - $client([new Response(204)], $quotaHistory) + $client([$identity(), new Response(204)], $quotaHistory) ); $after = CloudInstanceBackup::where('source_instance_id', 99) ->whereIn('state', $activeStates)->count(); @@ -264,5 +302,94 @@ namespace { && $after === CloudBackupService::MAX_PER_INSTANCE - 1, 'Confirmed deletion did not release one retained-backup quota slot.'); + // One ambiguous generation may be retried only after two stable, spaced + // identity probes. The retry is a new deletion generation bound to the + // exact S3 version instead of a replay of the ambiguous request. + $retry = $newBackup(100, 'stable-retry', 'delete_uncertain'); + $retry->object_etag = '"stable-etag"'; + $retry->object_version_id = 'version/7+opaque'; + $retry->delete_operation_id = str_repeat('1', 32); + $retry->delete_generation = 1; + $retry->delete_cycle_attempts = 1; + $retry->delete_presence_confirmations = 1; + $retry->delete_requested_at = time() - 300; + $retry->delete_last_checked_at = time() - 120; + $retry->delete_next_check_at = 0; + $retry->save(); + $retryHistory = []; + $retryResult = CloudBackupService::delete($retry, $actor, $client([ + new Response(200, [ + 'Content-Length' => '1024', + 'ETag' => '"stable-etag"', + 'x-amz-version-id' => 'version/7+opaque', + ]), + new ConnectException('retry response lost', + new Request('DELETE', 'https://objects.example.com')), + ], $retryHistory)); + $retry->refresh(); + $retryUris = array_map(static fn ($entry): string => + (string) $entry['request']->getUri(), $retryHistory); + $assert($retryResult['requires_reconciliation'] === true + && (string) $retry->state === 'delete_uncertain' + && (int) $retry->delete_generation === 2 + && (int) $retry->delete_cycle_attempts === 2 + && count($retryHistory) === 2 + && $retryHistory[0]['request']->getMethod() === 'HEAD' + && $retryHistory[1]['request']->getMethod() === 'DELETE' + && str_contains($retryUris[0], 'versionId=version%2F7%2Bopaque') + && str_contains($retryUris[1], 'versionId=version%2F7%2Bopaque'), + 'Stable object identity did not create a fresh, exact-version deletion generation.'); + + $assert(!in_array('delete_uncertain', $activeStates, true), + 'Object cleanup debt still consumes the user retained-backup quota.'); + + $retry->delete_cycle_attempts = 3; + $retry->delete_generation = 3; + $retry->delete_presence_confirmations = 1; + $retry->delete_last_checked_at = time() - 120; + $retry->delete_next_check_at = 0; + $retry->save(); + $exhaustedHistory = []; + $exhausted = CloudBackupService::delete($retry, $actor, $client([ + new Response(200, [ + 'Content-Length' => '1024', + 'ETag' => '"stable-etag"', + 'x-amz-version-id' => 'version/7+opaque', + ]), + ], $exhaustedHistory)); + $retry->refresh(); + $assert(($exhausted['requires_attention'] ?? false) === true + && (string) $retry->state === 'delete_attention' + && count($exhaustedHistory) === 1 + && $exhaustedHistory[0]['request']->getMethod() === 'HEAD', + 'Three automatic deletion attempts did not stop at administrator attention.'); + + $debt = CloudBackupService::cleanupDebt($actor); + $debtItem = array_values(array_filter($debt['items'], + static fn (array $item): bool => $item['id'] === (string) $retry->id))[0] ?? null; + $assert(($debt['summary']['attention_count'] ?? 0) >= 1 + && is_array($debtItem) + && ($debtItem['can_retry'] ?? false) === true + && !array_key_exists('object_key', $debtItem) + && !array_key_exists('object_version_id', $debtItem), + 'Cleanup debt was not listed safely for administrators.'); + + $manualHistory = []; + $manual = CloudBackupService::retryCleanup($retry, $actor, $client([ + new Response(200, [ + 'Content-Length' => '1024', + 'ETag' => '"stable-etag"', + 'x-amz-version-id' => 'version/7+opaque', + ]), + ], $manualHistory)); + $retry->refresh(); + $assert($manual['requires_reconciliation'] === true + && (string) $retry->state === 'delete_uncertain' + && (int) $retry->delete_cycle_attempts === 0 + && (int) $retry->delete_presence_confirmations === 1 + && (int) $retry->delete_next_check_at > time() + && count($manualHistory) === 1, + 'Manual cleanup retry did not begin a new bounded stable-evidence cycle.'); + echo "Portable backup deletion, reconciliation, blocker and quota tests passed.\n"; } diff --git a/web/tests/CloudPortableBackupFinalizerTest.php b/web/tests/CloudPortableBackupFinalizerTest.php index c6c0a214..59048d73 100644 --- a/web/tests/CloudPortableBackupFinalizerTest.php +++ b/web/tests/CloudPortableBackupFinalizerTest.php @@ -40,6 +40,8 @@ $schema->create('cloud_instance_backups', $table->integer('object_size_bytes')->default(0); $table->integer('virtual_size_bytes')->default(0); $table->string('sha256')->default(''); + $table->string('object_etag')->default(''); + $table->string('object_version_id')->default(''); $table->text('spec_json'); $table->text('sealed_storage_runtime'); $table->text('sealed_credential')->nullable(); @@ -201,6 +203,8 @@ CloudBackupService::finishJobLocked($createJob, 'succeeded', [ 'size_bytes' => 1024, 'virtual_size_bytes' => 20 * 1073741824, 'sha256' => str_repeat('b', 64), + 'object_etag' => '"finalizer-etag"', + 'object_version_id' => 'finalizer-version', 'consistency' => 'filesystem', 'cleanup_pending' => true, ], [ @@ -214,6 +218,8 @@ $assert((string) $createdBackup->state === 'ready' === 'backup_local_cleanup_pending' && (int) $createdBackup->object_size_bytes === 1024 && (string) $createdBackup->sha256 === str_repeat('b', 64) + && (string) $createdBackup->object_etag === '"finalizer-etag"' + && (string) $createdBackup->object_version_id === 'finalizer-version' && $createdBackup->active_job_id === null && $createdBackup->ready_at !== null, 'Cleanup-pending backup lost its ready/restorable object identity.'); diff --git a/web/tests/CloudS3SignerTest.php b/web/tests/CloudS3SignerTest.php index e65cd02f..78313ec7 100644 --- a/web/tests/CloudS3SignerTest.php +++ b/web/tests/CloudS3SignerTest.php @@ -255,7 +255,11 @@ try { $deleteHistory = []; $deleteStack = HandlerStack::create(new MockHandler([ new Response(204), - new Response(200), + new Response(200, [ + 'Content-Length' => '4096', + 'ETag' => '"present-etag"', + 'x-amz-version-id' => 'present/version+1', + ]), new Response(404), ])); $deleteStack->push(Middleware::history($deleteHistory)); @@ -263,17 +267,24 @@ $deleteClient = new CloudObjectStorageS3Client(new Client([ 'handler' => $deleteStack, ])); $deleteClient->deleteObject( - $multipartConfig, $multipartCredentials, 'vm/delete.qcow2' + $multipartConfig, $multipartCredentials, 'vm/delete.qcow2', + 'delete/version+1' ); $assert($deleteClient->objectPresence( - $multipartConfig, $multipartCredentials, 'vm/present.qcow2' + $multipartConfig, $multipartCredentials, 'vm/present.qcow2', + 'present/version+1' ) === 'present' && $deleteClient->objectPresence( - $multipartConfig, $multipartCredentials, 'vm/absent.qcow2' + $multipartConfig, $multipartCredentials, 'vm/absent.qcow2', + 'absent/version+1' ) === 'absent' && array_map(static fn ($entry): string => $entry['request']->getMethod(), $deleteHistory) - === ['DELETE', 'HEAD', 'HEAD'], + === ['DELETE', 'HEAD', 'HEAD'] + && str_contains((string) $deleteHistory[0]['request']->getUri(), + 'versionId=delete%2Fversion%2B1') + && str_contains((string) $deleteHistory[1]['request']->getUri(), + 'versionId=present%2Fversion%2B1'), 'DeleteObject and read-only deletion reconciliation lost method or status semantics.'); $ambiguousDelete = new CloudObjectStorageS3Client(new Client([ diff --git a/web/tests/LegalPagesContractTest.php b/web/tests/LegalPagesContractTest.php index 5264362e..1a1fcf89 100644 --- a/web/tests/LegalPagesContractTest.php +++ b/web/tests/LegalPagesContractTest.php @@ -73,7 +73,7 @@ $assert(str_contains($auth, "Config::get('tos_reg') === 'true'") 'Legal consent is only client-side and can be bypassed.'); $assets = $read('web/resources/views/venix/layout_assets.tpl'); -$assert(str_contains($assets, "value='2026102801'") +$assert(str_contains($assets, "value='2026102802'") && str_contains($admin, '?v={$venix_asset_version}') && str_contains($read('web/resources/views/venix/home/layout_header.tpl'), '?v={$venix_asset_version}'), 'Shared browser asset version was not coupled to the new assets.'); diff --git a/web/tests/PublicPageLocaleContractTest.php b/web/tests/PublicPageLocaleContractTest.php index 3f7ff764..97e6f52c 100644 --- a/web/tests/PublicPageLocaleContractTest.php +++ b/web/tests/PublicPageLocaleContractTest.php @@ -80,7 +80,7 @@ $assert(preg_match('/\.landing-site-header\s*\{[^}]*position:\s*fixed;/s', $css) && str_contains($accountCss, 'padding-top: 88px;') && str_contains($accountCss, 'padding-top: 76px;'), 'Persistent desktop or mobile public-navigation styling is incomplete.'); -$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102801'"), +$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102802'"), 'The shared browser asset version was not advanced with the navigation styles.'); echo "Public page locale and shared-layout contract tests passed.\n"; diff --git a/web/tests/SiteBrandContractTest.php b/web/tests/SiteBrandContractTest.php index 0b7de9ec..369539fa 100644 --- a/web/tests/SiteBrandContractTest.php +++ b/web/tests/SiteBrandContractTest.php @@ -69,7 +69,7 @@ foreach ($templates as $template) { } $assets = $read('web/resources/views/venix/layout_assets.tpl'); -$assert(str_contains($assets, "value='2026102801'"), +$assert(str_contains($assets, "value='2026102802'"), 'The shared asset version was not updated for the canonical logo.'); echo "Shared site-brand contract passed across public, auth, user, cloud and admin layouts.\n";