fix: reconcile object cleanup debt safely

go
codex 4 weeks ago
parent 9f382920e1
commit 196a7d8ce5

@ -93,6 +93,7 @@ func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request
result := BackupResult{
BackupKey: request.BackupKey, SizeBytes: upload.SizeBytes,
VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256,
ObjectETag: upload.ETag, ObjectVersionID: upload.VersionID,
Consistency: snapshot.Consistency,
}
cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx)

@ -412,6 +412,8 @@ func TestPortableBackupOperationProjectionPreservesFinalizerIdentity(t *testing.
"size_bytes":1048576,
"virtual_size_bytes":2147483648,
"sha256":"`+digest+`",
"object_etag":"\"backup-etag\"",
"object_version_id":"version/7+opaque",
"consistency":"filesystem",
"source_url":"must-not-leak"
}`))
@ -421,6 +423,8 @@ func TestPortableBackupOperationProjectionPreservesFinalizerIdentity(t *testing.
}
if backupProjection.BackupKey != backupKey || backupProjection.SizeBytes != 1048576 ||
backupProjection.VirtualSizeBytes != 2147483648 || backupProjection.SHA256 != digest ||
backupProjection.ObjectETag != `"backup-etag"` ||
backupProjection.ObjectVersionID != "version/7+opaque" ||
backupProjection.Consistency != "filesystem" {
t.Fatalf("backup projection omitted finalizer identity: %#v", backupProjection)
}

@ -228,6 +228,8 @@ type BackupResult struct {
SizeBytes int64 `json:"size_bytes"`
VirtualSizeBytes uint64 `json:"virtual_size_bytes"`
SHA256 string `json:"sha256"`
ObjectETag string `json:"object_etag,omitempty"`
ObjectVersionID string `json:"object_version_id,omitempty"`
Consistency string `json:"consistency"`
CleanupPending bool `json:"cleanup_pending,omitempty"`
}

@ -65,12 +65,15 @@ type UploadResult struct {
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
ETag string `json:"etag,omitempty"`
VersionID string `json:"version_id,omitempty"`
Parts []CompletedPart `json:"parts"`
}
type DownloadResult struct {
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
ETag string `json:"etag,omitempty"`
VersionID string `json:"version_id,omitempty"`
}
type Client struct {
@ -237,7 +240,7 @@ func (c *Client) UploadFile(ctx context.Context, path string, plan MultipartPlan
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"backup_source_changed", "backup source changed during upload")
}
etag, err := c.complete(ctx, plan.CompleteURL, parts)
etag, versionID, err := c.complete(ctx, plan.CompleteURL, parts)
if err != nil {
if IsAmbiguous(err) {
// Completion can commit remotely even if its response is lost. The
@ -246,14 +249,20 @@ func (c *Client) UploadFile(ctx context.Context, path string, plan MultipartPlan
// ambiguous reply into success without aborting or replaying.
verified, verifyErr := c.Download(ctx, plan.VerifyURL, size, digest, io.Discard)
if verifyErr == nil && verified.SizeBytes == size && verified.SHA256 == digest {
return UploadResult{SizeBytes: size, SHA256: digest, Parts: parts}, nil
return UploadResult{
SizeBytes: size, SHA256: digest, ETag: verified.ETag,
VersionID: verified.VersionID, Parts: parts,
}, nil
}
return UploadResult{}, err
}
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"upload_complete_failed", "backup multipart completion was rejected")
}
return UploadResult{SizeBytes: size, SHA256: digest, ETag: etag, Parts: parts}, nil
return UploadResult{
SizeBytes: size, SHA256: digest, ETag: etag,
VersionID: versionID, Parts: parts,
}, nil
}
func (c *Client) putPart(ctx context.Context, capability string, body io.Reader, length int64) (string, error) {
@ -306,56 +315,73 @@ type completeMultipartResult struct {
Code string `xml:"Code"`
}
func (c *Client) complete(ctx context.Context, capability string, parts []CompletedPart) (string, error) {
func (c *Client) complete(ctx context.Context, capability string,
parts []CompletedPart) (string, string, error) {
payload := completeMultipartUpload{Parts: make([]completeMultipartPart, len(parts))}
for index, part := range parts {
payload.Parts[index] = completeMultipartPart{PartNumber: part.PartNumber, ETag: part.ETag}
}
body, err := xml.Marshal(payload)
if err != nil {
return "", transferError("upload_complete_invalid", "backup multipart completion could not be encoded", false)
return "", "", transferError("upload_complete_invalid", "backup multipart completion could not be encoded", false)
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, capability, strings.NewReader(string(body)))
if err != nil {
return "", transferError("invalid_capability_url", "object transfer capability is invalid", false)
return "", "", transferError("invalid_capability_url", "object transfer capability is invalid", false)
}
request.ContentLength = int64(len(body))
request.Header.Set("Content-Type", "application/xml")
response, err := c.http.Do(request)
if err != nil {
return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
defer response.Body.Close()
encoded, readErr := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1))
if readErr != nil || int64(len(encoded)) > maxResponseBytes {
return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
if response.StatusCode >= http.StatusInternalServerError {
// A storage service or reverse proxy can lose the success response after
// commit. A 5xx therefore cannot authorize AbortMultipartUpload.
return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
if response.StatusCode < 200 || response.StatusCode >= 300 {
return "", transferError("upload_complete_failed", "backup multipart completion was rejected", false)
return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false)
}
if len(strings.TrimSpace(string(encoded))) == 0 {
return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
var result completeMultipartResult
if err := xml.Unmarshal(encoded, &result); err != nil {
return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
if result.XMLName.Local == "Error" || result.Code != "" {
return "", transferError("upload_complete_failed", "backup multipart completion was rejected", false)
return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false)
}
if result.XMLName.Local != "CompleteMultipartUploadResult" {
return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
etag := strings.TrimSpace(result.ETag)
if !validETag(etag) {
return "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
return etag, nil
versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id"))
if !validVersionID(versionID) {
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
return etag, versionID, nil
}
func validVersionID(value string) bool {
if len(value) > 1024 {
return false
}
for _, current := range []byte(value) {
if current < 0x20 || current == 0x7f {
return false
}
}
return true
}
func (c *Client) failAndAbort(ctx context.Context, capability, code, message string) error {
@ -445,7 +471,17 @@ func (c *Client) Download(ctx context.Context, capability string, expectedSize i
if !strings.EqualFold(digest, expectedSHA256) {
return DownloadResult{}, transferError("object_sha256_mismatch", "backup object SHA-256 does not match", false)
}
return DownloadResult{SizeBytes: written, SHA256: digest}, nil
etag := strings.TrimSpace(response.Header.Get("ETag"))
if etag != "" && !validETag(etag) {
etag = ""
}
versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id"))
if !validVersionID(versionID) {
versionID = ""
}
return DownloadResult{
SizeBytes: written, SHA256: digest, ETag: etag, VersionID: versionID,
}, nil
}
// String is useful in bounded diagnostics without revealing URL capabilities.

@ -76,6 +76,7 @@ func TestUploadFileCompletesMultipartWithOrderedParts(t *testing.T) {
t.Errorf("unexpected completion parts: %#v", request.Parts)
}
w.Header().Set("Content-Type", "application/xml")
w.Header().Set("x-amz-version-id", "version/7+opaque")
_, _ = io.WriteString(w, `<CompleteMultipartUploadResult><ETag>"final"</ETag></CompleteMultipartUploadResult>`)
case http.MethodDelete:
aborted = true
@ -94,7 +95,8 @@ func TestUploadFileCompletesMultipartWithOrderedParts(t *testing.T) {
}
wantHash := sha256.Sum256(contents)
if result.SizeBytes != int64(len(contents)) || result.SHA256 != hex.EncodeToString(wantHash[:]) ||
result.ETag != `"final"` || len(result.Parts) != 3 || aborted {
result.ETag != `"final"` || result.VersionID != "version/7+opaque" ||
len(result.Parts) != 3 || aborted {
t.Fatalf("unexpected result: %#v aborted=%v", result, aborted)
}
var reconstructed []byte
@ -241,6 +243,8 @@ func TestUploadFileVerifiesCommittedObjectAfterAmbiguousCompletion(t *testing.T)
_ = connection.Close()
case http.MethodGet:
verified = true
w.Header().Set("ETag", `"verified-etag"`)
w.Header().Set("x-amz-version-id", "verified-version")
_, _ = w.Write(contents)
case http.MethodDelete:
aborted = true
@ -256,7 +260,9 @@ func TestUploadFileVerifiesCommittedObjectAfterAmbiguousCompletion(t *testing.T)
t.Fatalf("result=%#v err=%v verified=%v aborted=%v", result, err, verified, aborted)
}
wantHash := sha256.Sum256(contents)
if result.SHA256 != hex.EncodeToString(wantHash[:]) || len(result.Parts) != 1 {
if result.SHA256 != hex.EncodeToString(wantHash[:]) ||
result.ETag != `"verified-etag"` || result.VersionID != "verified-version" ||
len(result.Parts) != 1 {
t.Fatalf("unexpected verified result: %#v", result)
}
}

@ -207,7 +207,8 @@ operation 临时卷/metadata 的 evidence-bound cleanup plan,但当前没有
cleanup 的 mutation endpoint。
成功结果只返回 `backup_key`、对象 `size_bytes`、qcow2
`virtual_size_bytes`、`sha256`、实际 `consistency`,以及本地临时副本未能清理时的
`virtual_size_bytes`、`sha256`、multipart 完成响应中的可选 `object_etag` 与
`object_version_id`、实际 `consistency`,以及本地临时副本未能清理时的
可选 `cleanup_pending=true`,不返回预签名 URL 或本机路径。本地清理失败不会丢弃
已经验证成功的远端对象身份,也不会把可恢复的备份错误降级为远端效果不明。
后续只读 inspection 在残留仍精确归属于该 operation 且没有活跃 libvirt/block job 时,

@ -95,7 +95,9 @@ php xcat cloudRepairInstanceMetadata --apply
后台 `/admin/object-storage` 可配置一个全局 S3 兼容对象存储应用,适用于 AWS S3、MinIO 及实现 AWS Signature Version 4 的兼容服务。该功能复用业务库已有的 `configurations` 表,不需要导入新表。Endpoint 必须是带有效证书的 HTTPS origin;MinIO 通常选择路径寻址,AWS S3 通常选择虚拟主机寻址。
Access Key ID 与 Secret Access Key 只在超级管理员提交时进入 PHP,使用 `cloud_data_key` 的独立用途密钥密封后保存,读取页面不会回显。修改任一配置或停用应用都会清除原就绪标记;必须重新执行“验证连接并启用”。连接验证只发送签名的 `HeadBucket` 请求,需要目标 Bucket 的 `s3:ListBucket` 权限,不上传、覆盖或删除对象。管理员虚拟机管理已接入完整系统盘备份和新实例恢复,目标可以是原宿主机或其他合格宿主机;用户自动备份使用同一对象存储,在实例设置中按每天或每周于 UTC 00:00 调度并保留最多两份。运行账户只需配置 `vm-backups/` 前缀范围内的 multipart 上传、`GetObject`、`HeadObject` 和 `DeleteObject` 权限。维护迁移使用宿主机间 PeerMigration V3 直传,不创建 S3 迁移对象,也不需要 `vm-migrations/` 权限。当前不包含增量链和管理员手工备份的自动保留策略,具体安全边界见 `docs/PORTABLE_VM_BACKUPS.md`。
Access Key ID 与 Secret Access Key 只在超级管理员提交时进入 PHP,使用 `cloud_data_key` 的独立用途密钥密封后保存,读取页面不会回显。修改任一配置或停用应用都会清除原就绪标记;必须重新执行“验证连接并启用”。连接验证只发送签名的 `HeadBucket` 请求,需要目标 Bucket 的 `s3:ListBucket` 权限,不上传、覆盖或删除对象。管理员虚拟机管理已接入完整系统盘备份和新实例恢复,目标可以是原宿主机或其他合格宿主机;用户自动备份使用同一对象存储,在实例设置中按每天或每周于 UTC 00:00 调度并保留最多两份。运行账户只需配置 `vm-backups/` 前缀范围内的 multipart 上传、`GetObject`、`HeadObject` 和 `DeleteObject` 权限;开启 Bucket 版本控制时还必须授予 `s3:GetObjectVersion` 与 `s3:DeleteObjectVersion`,让 HEAD 与删除准确绑定 `versionId`。维护迁移使用宿主机间 PeerMigration V3 直传,不创建 S3 迁移对象,也不需要 `vm-migrations/` 权限。当前不包含增量链和管理员手工备份的自动保留策略,具体安全边界见 `docs/PORTABLE_VM_BACKUPS.md`。
生产 Bucket 必须把生命周期规则作为控制面之外的最后兜底:在短周期内终止未完成 multipart upload;开启版本控制时,按业务保留要求清理 `vm-backups/` 的非当前版本和过期删除标记。生命周期不得提前删除仍由面板目录引用的当前版本。面板会保存 HEAD 返回的 ETag 与 `version_id`,删除时精确定位该版本;删除结果不明时先进行两次间隔至少 60 秒的稳定 HEAD,只有对象身份完全一致才创建新的删除代次,每个自动周期最多三次。达到上限的对象进入后台“对象清理异常”,不再占用用户自动备份保留份数,但仍计入对象清理债务,必须由管理员核查后手动开启新的受控周期。
生产环境应为面板创建独立服务账户,把权限限制到配置的 Bucket 和备份前缀。数据库备份必须与原始 `cloud_data_key` 一起保管,重新生成主密钥无法恢复全局配置及各备份冻结的对象存储凭据。

@ -72,4 +72,4 @@ PeerMigration 不创建 `vm-migrations/` 对象,也不持有 S3 凭据。目
恢复失败时,只有 Agent 明确证明新 Domain、所有 UUID 所属卷、seed、NVRAM、网络过滤器和流量状态均不存在,控制面才能释放容量与公网 IP。只要清理结果不确定,资源保持锁定等待人工核对。
备份对象删除也遵守同一原则:明确删除成功才释放目录额度;DELETE 响应未知时只允许通过 HEAD 核对对象已经不存在,不会自动重放删除。若 Bucket 开启版本控制,S3 删除会隐藏当前对象但可能保留历史版本,生产环境还应配置仅覆盖备份前缀的历史版本过期策略。
备份对象删除也遵守同一原则:首次删除前保存对象的大小、ETag 和可用的 `version_id`;版本控制 Bucket 的 HEAD 与 DELETE 都精确绑定该版本。DELETE 响应未知后不会重放原请求,而是等待两次间隔至少 60 秒、且大小/ETag/版本完全一致的 HEAD 证据,再以新的操作标识创建一个删除代次。每个自动清理周期最多三次,达到上限转为 `delete_attention`,由超级管理员在“对象存储 -> 对象清理异常”核查并手动恢复新的受控周期。删除意图一旦建立便不再占用用户自动备份保留份数;仍存在的远端数据单独计入对象清理债务。生产 Bucket 还必须配置终止未完成 multipart、清理非当前版本和过期删除标记的生命周期规则,但不得提前删除仍被目录引用的当前版本。

@ -134,7 +134,7 @@ jq empty deploy/*.json.example
- 目标分块暂存必须由 Agent 派生到第一个 `storage_path_roots` 下的 `.peer-migration-staging`,不能落在 Journal 系统盘。故障注入分块断连后应从已有偏移继续;成功导入删除暂存,过期中断会话只能由有界本地清理回收,不能删除不匹配目录。
- 目标在恢复和用户防火墙收敛期间必须保持关机;源机的运行/关机、autostart、断网状态与修改后主机名必须在独立激活步骤后保持一致。传输确定失败须同时提供续传和安全回滚,防火墙、激活、回滚、源清理的确定失败均有显式恢复出口。若源准备已存在精确成功回执但 PHP 最终化失败,只能将原准备效果收敛为 `applied` 后创建新的补偿回滚任务;Agent 明确确认源机恢复前不得释放目标预留或 IP。未知效果只能核对原 operation,不能重放。普通云备份仍只能产生单系统盘 S3 对象,PeerMigration 不得创建 S3 迁移对象。
- 独立恢复可选择原宿主机或其他合格宿主机;无论目标是否为原宿主机,都必须单独校验并预留容量与全新公网 IP。目标使用 `bridge:br0`;校验下载对象字节数与 SHA-256,新实例 UUID、MAC、seed、NVRAM、网络过滤器和流量状态均不得复用源实例,且不得继承订单、账单、服务期、到期时间或流量周期。
- 对备份/恢复断连、超时和回执丢失执行故障注入:同一 operation ID 只能查询收敛,不能再次派发。恢复失败只有在全部托管产物明确不存在时才能释放容量与 IP;对象删除响应未知时只能 HEAD 核对不存在,不能重发 DELETE。
- 对备份/恢复断连、超时和回执丢失执行故障注入:同一 Agent operation ID 只能查询收敛,不能再次派发。恢复失败只有在全部托管产物明确不存在时才能释放容量与 IP。对象删除响应未知时不得重放原 DELETE;必须用两次间隔 HEAD 证明同一大小、ETag 和 `version_id` 仍存在,才允许创建新的删除代次。验证每轮最多三次、超限转人工、手动重试重新建立稳定证据、精确版本删除、清理债务不占保留份数,并确认大量异常债务不会饿死新的保留清理。
## 7. 非生产宿主机验收

@ -8,7 +8,7 @@ use App\Services\Config;
/** Initializes only the clean-break PHP business and Agent log schemas. */
final class CloudSetup
{
private const SCHEMA_VERSION = 'go-agent-wss-v11-peer-migration-v2';
private const SCHEMA_VERSION = 'go-agent-wss-v12-object-cleanup-v1';
private const MAIN_TABLES = [
'cloud_regions', 'cloud_network_lines', 'cloud_resource_pools',
'cloud_hosts', 'cloud_host_agents',
@ -85,10 +85,14 @@ final class CloudSetup
'user_id', 'source_uuid', 'name', 'purpose', 'source',
'subscription_id', 'scheduled_for', 'state', 'consistency',
'object_key', 'object_size_bytes', 'virtual_size_bytes',
'sha256', 'spec_json', 'sealed_storage_runtime',
'sha256', 'object_etag', 'object_version_id', 'spec_json',
'sealed_storage_runtime',
'sealed_credential', 'storage_revision', 'active_job_id',
'last_operation_id', 'error_code', 'ready_at', 'created_at',
'updated_at',
'last_operation_id', 'delete_operation_id', 'delete_generation',
'delete_cycle_attempts', 'delete_presence_confirmations',
'delete_requested_at', 'delete_last_checked_at',
'delete_next_check_at', 'delete_attention_at', 'error_code',
'ready_at', 'created_at', 'updated_at',
],
'cloud_deployment_snapshots' => [
'id', 'snapshot_key', 'user_id', 'backup_id',
@ -226,6 +230,7 @@ final class CloudSetup
'cloud_instance_backup_object', 'cloud_instance_backup_schedule',
'cloud_instance_backup_source',
'cloud_instance_backup_owner', 'cloud_instance_backup_operation',
'cloud_instance_backup_delete_due',
],
'cloud_deployment_snapshots' => [
'PRIMARY', 'cloud_deployment_snapshot_key',

@ -3,6 +3,9 @@
namespace App\Controllers\Admin;
use App\Controllers\AdminController;
use App\Models\CloudInstanceBackup;
use App\Services\Cloud\CloudAccess;
use App\Services\Cloud\CloudBackupService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudInput;
use App\Services\Cloud\CloudObjectStorageConfig;
@ -13,6 +16,8 @@ final class ObjectStorageController extends AdminController
{
$state = null;
$error = '';
$cleanupDebt = null;
$cleanupDebtError = '';
$status = 200;
try {
$state = CloudObjectStorageConfig::view();
@ -24,10 +29,26 @@ final class ObjectStorageController extends AdminController
$error = '对象存储配置暂时无法读取,请检查数据库状态。';
$status = 503;
}
try {
$query = $request->getQueryParams();
$cleanupDebt = CloudBackupService::cleanupDebt(
CloudAccess::actor(false), [
'page' => $query['cleanup_page'] ?? 1,
'per_page' => 20,
]
);
} catch (CloudError $exception) {
$cleanupDebtError = $exception->getMessage();
} catch (\Throwable $exception) {
error_log('Object cleanup debt read failed: ' . get_class($exception));
$cleanupDebtError = '对象清理债务暂时无法读取,请检查数据库结构。';
}
$smarty = $this->view()->assign([
'object_storage' => $state,
'object_storage_error' => $error,
'cleanup_debt' => $cleanupDebt,
'cleanup_debt_error' => $cleanupDebtError,
]);
$response->getBody()->write($smarty->fetch('admin/object_storage.tpl'));
return $response->withStatus($status)
@ -81,6 +102,60 @@ final class ObjectStorageController extends AdminController
}
}
public function cleanupDebt($request, $response, $args)
{
try {
$payload = CloudBackupService::cleanupDebt(
CloudAccess::actor(false), $request->getQueryParams()
);
return $response->withJson([
'ret' => 1,
'msg' => '对象清理债务已刷新。',
'cleanup_debt' => $payload,
])->withHeader('Cache-Control', 'private, no-store');
} catch (CloudError $exception) {
return $this->json($response, 0, $exception->getMessage(),
$exception->httpStatus);
} catch (\Throwable $exception) {
error_log('Object cleanup debt refresh failed: '
. get_class($exception));
return $this->json($response, 0,
'对象清理债务刷新失败,请稍后重试。', 500);
}
}
public function retryCleanup($request, $response, $args)
{
try {
$backupId = filter_var($args['backup_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
if ($backupId === false) {
throw new CloudError('对象清理记录无效', 400);
}
$backup = CloudInstanceBackup::find((int) $backupId);
if ($backup === null) {
throw new CloudError('对象清理记录不存在', 404);
}
$result = CloudBackupService::retryCleanup(
$backup, CloudAccess::actor(false)
);
return $response->withJson([
'ret' => 1,
'msg' => ($result['deleted'] ?? false)
? '对象已确认不存在,清理债务已关闭。'
: '已恢复稳定性核对;第二次证据一致后才会创建新删除代次。',
'cleanup' => $result,
])->withHeader('Cache-Control', 'private, no-store');
} catch (CloudError $exception) {
return $this->json($response, 0, $exception->getMessage(),
$exception->httpStatus);
} catch (\Throwable $exception) {
error_log('Object cleanup retry failed: ' . get_class($exception));
return $this->json($response, 0,
'对象清理重试失败,请稍后重试。', 500);
}
}
/** @param array<string, mixed>|null $state */
private function json($response, int $ret, string $message, int $status, ?array $state = null)
{

@ -214,7 +214,7 @@ final class CloudAdminInstanceController
->withHeader('Cache-Control', 'no-store');
}
/** Delete one backup object, or HEAD-reconcile an ambiguous prior delete. */
/** Delete one backup object, or advance its bounded HEAD reconciliation. */
public function deleteBackup($request, $response, $args)
{
$actor = $this->actor($request, true);

@ -7,7 +7,10 @@ final class CloudInstanceBackup extends Model
{
protected $connection = 'default';
protected $table = 'cloud_instance_backups';
protected $hidden = ['sealed_storage_runtime', 'sealed_credential'];
protected $hidden = [
'sealed_storage_runtime', 'sealed_credential', 'object_etag',
'object_version_id', 'delete_operation_id',
];
protected $casts = [
'id' => 'int',
'source_instance_id' => 'int',
@ -19,6 +22,13 @@ final class CloudInstanceBackup extends Model
'virtual_size_bytes' => 'int',
'storage_revision' => 'int',
'active_job_id' => 'int',
'delete_generation' => 'int',
'delete_cycle_attempts' => 'int',
'delete_presence_confirmations' => 'int',
'delete_requested_at' => 'int',
'delete_last_checked_at' => 'int',
'delete_next_check_at' => 'int',
'delete_attention_at' => 'int',
'ready_at' => 'int',
'created_at' => 'int',
'updated_at' => 'int',

@ -309,7 +309,10 @@ final class CloudAutomaticBackupService
return $queued;
}
/** Delete at most $limit oldest ready objects beyond each subscription's cap. */
/**
* Prune at most $limit fresh retention candidates plus a separate bounded
* cleanup-debt batch, so stuck remote objects cannot starve new rotation.
*/
public static function pruneRetention(int $limit = 10, ?int $now = null,
?CloudObjectStorageS3Client $client = null): int
{
@ -324,15 +327,15 @@ final class CloudAutomaticBackupService
];
$processed = 0;
$removed = 0;
// Ambiguous object deletion is reconciled by HEAD only. Keep it in a
// bounded system path; otherwise it drops out of the ready-count
// query and can never converge after the first uncertain DELETE.
$pending = CloudInstanceBackup::where('source', 'automatic')
->where('purpose', 'portable')
// Cleanup debt has its own budget and due index. It never consumes the
// fresh retention budget and attention-required rows wait for an
// explicit administrator action instead of monopolising every batch.
$debtLimit = max(1, min(10, intdiv($limit + 1, 2)));
$pending = CloudInstanceBackup::where('purpose', 'portable')
->whereIn('state', ['deleting', 'delete_uncertain'])
->orderBy('id');
self::withoutDeploymentSnapshots($pending);
$pending = $pending->limit($limit)->get();
->where('delete_next_check_at', '<=', $now)
->orderBy('delete_next_check_at')->orderBy('id');
$pending = $pending->limit($debtLimit)->get();
foreach ($pending as $backup) {
try {
$result = CloudBackupService::delete(
@ -345,10 +348,6 @@ final class CloudAutomaticBackupService
error_log('cloud automatic backup retention unavailable: '
. get_class($error));
}
$processed++;
}
if ($processed >= $limit) {
return $removed;
}
// A disabled plan keeps its already-paid recovery rights. Only after
// the latest immutable entitlement has ended plus the three-day grace
@ -669,7 +668,7 @@ final class CloudAutomaticBackupService
{
$state = in_array((string) $backup->state, [
'creating', 'ready', 'failed', 'uncertain', 'deleting',
'delete_uncertain',
'delete_uncertain', 'delete_attention',
], true) ? (string) $backup->state : 'uncertain';
return [
'id' => (string) $backup->id,

@ -20,8 +20,15 @@ final class CloudBackupService
private const STORAGE_PURPOSE = 'object-storage-backup-runtime';
private const ACTIVE_BACKUP_STATES = [
'creating', 'ready', 'uncertain', 'deleting', 'delete_uncertain',
'creating', 'ready', 'uncertain',
];
private const CLEANUP_DEBT_STATES = [
'deleting', 'delete_uncertain', 'delete_attention',
];
private const DELETE_STABLE_PROBES = 2;
private const DELETE_STABLE_SECONDS = 60;
private const DELETE_RETRY_SECONDS = 300;
private const DELETE_MAX_AUTOMATIC_ATTEMPTS = 3;
private const TRANSFER_URL_SECONDS = 604800;
private const MAX_AGENT_BODY_BYTES = 1048576;
// The create transaction opens a remote S3 multipart upload. Database
@ -330,8 +337,9 @@ final class CloudBackupService
/**
* Delete or safely reconcile one frozen S3 backup object.
*
* A second call after an ambiguous DELETE never issues another DELETE. It
* only performs HEAD and converges to deleted after an explicit 404.
* An ambiguous DELETE is never replayed. Two spaced HEAD observations of
* the same immutable identity may authorize a fresh deletion generation;
* at most three generations run automatically in one supervised cycle.
*/
public static function delete(CloudInstanceBackup $backup, array $actor,
?CloudObjectStorageS3Client $client = null,
@ -339,8 +347,9 @@ final class CloudBackupService
{
CloudAccess::manage($actor);
$client ??= new CloudObjectStorageS3Client();
$now = time();
$prepared = $backup->getConnection()->transaction(function () use (
$backup, $beforeDeleteLocked
$backup, $beforeDeleteLocked, $now
): array {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->first();
@ -352,86 +361,252 @@ final class CloudBackupService
return ['mode' => 'deleted', 'backup' => $locked];
}
self::assertBackupDeletionAvailable($locked);
if ($state === 'delete_attention') {
return ['mode' => 'attention', 'backup' => $locked];
}
if (in_array($state, ['deleting', 'delete_uncertain'], true)) {
if ((int) $locked->delete_next_check_at > $now) {
return ['mode' => 'waiting', 'backup' => $locked];
}
return [
'mode' => 'reconcile',
'backup' => $locked,
'runtime' => self::storageRuntime($locked),
'object_key' => (string) $locked->object_key,
'object_etag' => (string) $locked->object_etag,
'object_version_id' => (string) $locked->object_version_id,
];
}
if (!in_array($state, ['ready', 'failed'], true)) {
throw new CloudError('当前备份状态不允许删除', 409);
}
if ($beforeDeleteLocked !== null) {
$beforeDeleteLocked($locked);
}
// Decrypt before crossing the durable deleting boundary. A crash
// after the state change is treated as unknown and only HEAD may
// follow; DELETE is never replayed.
// Decrypt before any remote request. The exact immutable identity
// is captured by HEAD before crossing the deleting boundary.
$runtime = self::storageRuntime($locked);
$locked->state = 'deleting';
$locked->error_code = '';
$locked->updated_at = time();
$locked->save();
return [
'mode' => 'delete',
'mode' => 'identify',
'previous_state' => $state,
'backup' => $locked,
'runtime' => $runtime,
'object_key' => (string) $locked->object_key,
'object_etag' => (string) $locked->object_etag,
'object_version_id' => (string) $locked->object_version_id,
];
}, 3);
if ($prepared['mode'] === 'deleted') {
if (in_array($prepared['mode'], ['deleted', 'waiting', 'attention'], true)) {
return self::deletionResult($prepared['backup']);
}
$runtime = $prepared['runtime'];
$objectKey = $prepared['object_key'];
if ($prepared['mode'] === 'identify') {
try {
$identity = $client->inspectObject(
$runtime['config'], $runtime['credentials'], $objectKey,
(string) $prepared['object_version_id']
);
} catch (CloudObjectStorageTransferError $error) {
throw new CloudError($error->getMessage(), 502);
}
if ($identity === null) {
return self::finishDelete((int) $backup->id);
}
self::assertObjectIdentity(
$prepared['backup'], $identity,
(string) $prepared['object_etag'] !== ''
);
$prepared = self::claimDeleteGeneration(
(int) $backup->id,
(string) $prepared['previous_state'],
$identity,
time(),
$beforeDeleteLocked
);
if ($prepared['mode'] !== 'delete') {
return self::deletionResult($prepared['backup']);
}
}
if ($prepared['mode'] === 'reconcile') {
try {
$presence = $client->objectPresence(
$runtime['config'], $runtime['credentials'], $objectKey
$identity = $client->inspectObject(
$runtime['config'], $runtime['credentials'], $objectKey,
(string) $prepared['object_version_id']
);
} catch (CloudObjectStorageTransferError $error) {
self::markDeleteUncertain((int) $backup->id,
$error->publicCode);
self::markDeleteUncertain(
(int) $backup->id, $error->publicCode, time(), true,
self::DELETE_RETRY_SECONDS
);
return self::deletionResult(
CloudInstanceBackup::findOrFail((int) $backup->id)
);
}
if ($presence === 'absent') {
if ($identity === null) {
return self::finishDelete((int) $backup->id);
}
self::markDeleteUncertain((int) $backup->id,
'object_delete_still_present');
return self::deletionResult(
CloudInstanceBackup::findOrFail((int) $backup->id)
try {
self::assertObjectIdentity($prepared['backup'], $identity, true);
} catch (CloudError $error) {
self::markDeleteAttention(
(int) $backup->id, 'object_identity_changed', time()
);
return self::deletionResult(
CloudInstanceBackup::findOrFail((int) $backup->id)
);
}
$prepared = self::recordStableDeleteProbe(
(int) $backup->id, time()
);
if ($prepared['mode'] !== 'delete') {
return self::deletionResult($prepared['backup']);
}
$runtime = $prepared['runtime'];
$objectKey = $prepared['object_key'];
}
try {
$client->deleteObject(
$runtime['config'], $runtime['credentials'], $objectKey
$runtime['config'], $runtime['credentials'], $objectKey,
(string) $prepared['object_version_id']
);
} catch (CloudObjectStorageTransferError $error) {
if ($error->effectMayHaveApplied) {
self::markDeleteUncertain((int) $backup->id,
$error->publicCode);
self::markDeleteUncertain(
(int) $backup->id, $error->publicCode, time(), true
);
return self::deletionResult(
CloudInstanceBackup::findOrFail((int) $backup->id)
);
}
self::restoreAfterRejectedDelete(
(int) $backup->id,
(string) $prepared['previous_state'],
$error->publicCode
);
if (isset($prepared['previous_state'])) {
self::restoreAfterRejectedDelete(
(int) $backup->id,
(string) $prepared['previous_state'],
$error->publicCode
);
} else {
self::markDeleteAttention(
(int) $backup->id, $error->publicCode, time()
);
}
throw new CloudError($error->getMessage(), 502);
}
return self::finishDelete((int) $backup->id);
}
/** Sanitized administrator view of remote objects awaiting cleanup. */
public static function cleanupDebt(array $actor, array $input = []): array
{
if (($actor['is_admin'] ?? false) !== true) {
throw new CloudError('只有管理员可以查看对象清理异常', 403);
}
CloudInput::only($input, ['page', 'per_page']);
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
$perPage = self::libraryInteger($input['per_page'] ?? 20, 10, 50, 20);
$query = CloudInstanceBackup::whereIn('state', self::CLEANUP_DEBT_STATES);
$total = (int) (clone $query)->count('id');
$bytes = (int) (clone $query)->sum('object_size_bytes');
$attention = (int) (clone $query)
->where('state', 'delete_attention')->count('id');
$totalPages = max(1, (int) ceil($total / $perPage));
$page = min($page, $totalPages);
$rows = $query->orderByRaw(
"CASE WHEN state = 'delete_attention' THEN 0 ELSE 1 END"
)->orderBy('delete_next_check_at')->orderBy('id')
->offset(($page - 1) * $perPage)->limit($perPage)->get();
$items = [];
foreach ($rows as $row) {
$items[] = [
'id' => (string) $row->id,
'name' => (string) $row->name,
'source_instance_id' => (string) $row->source_instance_id,
'user_id' => (string) $row->user_id,
'state' => (string) $row->state,
'object_size_bytes' => max(0, (int) $row->object_size_bytes),
'delete_generation' => max(0, (int) $row->delete_generation),
'cycle_attempts' => max(0, (int) $row->delete_cycle_attempts),
'next_check_at' => max(0, (int) $row->delete_next_check_at),
'attention_at' => $row->delete_attention_at === null ? null
: max(0, (int) $row->delete_attention_at),
'error_code' => (string) $row->error_code,
'can_retry' => (string) $row->state === 'delete_attention'
&& ($actor['is_super_admin'] ?? false) === true
&& ($actor['can_manage'] ?? false) === true,
];
}
return [
'summary' => [
'count' => $total,
'bytes' => $bytes,
'attention_count' => $attention,
],
'items' => $items,
'pagination' => [
'page' => $page,
'per_page' => $perPage,
'total' => $total,
'total_pages' => $totalPages,
],
];
}
/** Begin one new bounded cycle after an administrator has inspected debt. */
public static function retryCleanup(CloudInstanceBackup $backup, array $actor,
?CloudObjectStorageS3Client $client = null): array
{
CloudAccess::manage($actor);
$client ??= new CloudObjectStorageS3Client();
$prepared = $backup->getConnection()->transaction(function () use ($backup): array {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->firstOrFail();
if ((string) $locked->state !== 'delete_attention') {
throw new CloudError('只有待人工处理的对象可以重新进入清理流程', 409);
}
self::assertBackupDeletionAvailable($locked);
return [
'runtime' => self::storageRuntime($locked),
'object_key' => (string) $locked->object_key,
'object_version_id' => (string) $locked->object_version_id,
'backup' => $locked,
];
}, 3);
try {
$identity = $client->inspectObject(
$prepared['runtime']['config'],
$prepared['runtime']['credentials'],
$prepared['object_key'],
$prepared['object_version_id']
);
} catch (CloudObjectStorageTransferError $error) {
throw new CloudError($error->getMessage(), 502);
}
if ($identity === null) {
return self::finishDelete((int) $backup->id);
}
self::assertObjectIdentity($prepared['backup'], $identity, true);
$now = time();
$locked = $backup->getConnection()->transaction(function () use (
$backup, $now
): CloudInstanceBackup {
$row = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->firstOrFail();
if ((string) $row->state !== 'delete_attention') {
throw new CloudError('对象清理状态已发生变化', 409);
}
$row->state = 'delete_uncertain';
$row->delete_cycle_attempts = 0;
$row->delete_presence_confirmations = 1;
$row->delete_last_checked_at = $now;
$row->delete_next_check_at = $now + self::DELETE_STABLE_SECONDS;
$row->delete_attention_at = null;
$row->error_code = 'manual_retry_awaiting_stable_probe';
$row->updated_at = $now;
$row->save();
return $row;
}, 3);
return self::deletionResult($locked);
}
public static function create(CloudInstance $instance, array $actor,
array $input): array
{
@ -1213,6 +1388,8 @@ final class CloudBackupService
$virtual = filter_var($result['virtual_size_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$sha = $result['sha256'] ?? null;
$objectEtag = $result['object_etag'] ?? '';
$objectVersionId = $result['object_version_id'] ?? '';
$consistency = $result['consistency'] ?? null;
$cleanupPending = $result['cleanup_pending'] ?? false;
if (!is_array($result)
@ -1221,6 +1398,12 @@ final class CloudBackupService
|| $size === false || $virtual === false
|| !is_string($sha)
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|| !is_string($objectEtag) || strlen($objectEtag) > 256
|| preg_match('/[\x00-\x1f\x7f]/D', $objectEtag) === 1
|| !is_string($objectVersionId)
|| strlen($objectVersionId) > 1024
|| preg_match('/[\x00-\x1f\x7f]/D', $objectVersionId) === 1
|| ($objectVersionId !== '' && $objectEtag === '')
|| !in_array($consistency, ['filesystem', 'crash'], true)
|| !is_bool($cleanupPending)
|| (int) $size > (int) ($payload['maximum_size_bytes'] ?? 0)) {
@ -1231,6 +1414,8 @@ final class CloudBackupService
$backup->object_size_bytes = (int) $size;
$backup->virtual_size_bytes = (int) $virtual;
$backup->sha256 = $sha;
$backup->object_etag = $objectEtag;
$backup->object_version_id = $objectVersionId;
$backup->ready_at = $now;
$backup->active_job_id = null;
// The remote object is complete and restorable. A host-side
@ -1577,15 +1762,178 @@ final class CloudBackupService
}
}
/** @param array{size_bytes:int,etag:string,version_id:string} $identity */
private static function claimDeleteGeneration(int $backupId,
string $previousState, array $identity, int $now,
?callable $beforeDeleteLocked): array
{
$backup = CloudInstanceBackup::findOrFail($backupId);
return $backup->getConnection()->transaction(function () use (
$backupId, $previousState, $identity, $now, $beforeDeleteLocked
): array {
$locked = CloudInstanceBackup::where('id', $backupId)
->lockForUpdate()->firstOrFail();
if ((string) $locked->state === 'deleted') {
return ['mode' => 'deleted', 'backup' => $locked];
}
if (in_array((string) $locked->state,
self::CLEANUP_DEBT_STATES, true)) {
return ['mode' => 'waiting', 'backup' => $locked];
}
if ((string) $locked->state !== $previousState) {
throw new CloudError('备份删除状态已发生变化', 409);
}
self::assertBackupDeletionAvailable($locked);
if ($beforeDeleteLocked !== null) {
$beforeDeleteLocked($locked);
}
$locked->object_etag = (string) $identity['etag'];
$locked->object_version_id = (string) $identity['version_id'];
$locked->state = 'deleting';
$locked->delete_operation_id = bin2hex(random_bytes(16));
$locked->delete_generation = max(0,
(int) $locked->delete_generation) + 1;
$locked->delete_cycle_attempts = 1;
$locked->delete_presence_confirmations = 0;
$locked->delete_requested_at = $now;
$locked->delete_last_checked_at = null;
$locked->delete_next_check_at = 0;
$locked->delete_attention_at = null;
$locked->error_code = '';
$locked->updated_at = $now;
$locked->save();
return [
'mode' => 'delete',
'previous_state' => $previousState,
'backup' => $locked,
'runtime' => self::storageRuntime($locked),
'object_key' => (string) $locked->object_key,
'object_version_id' => (string) $locked->object_version_id,
];
}, 3);
}
private static function recordStableDeleteProbe(int $backupId,
int $now): array
{
$backup = CloudInstanceBackup::findOrFail($backupId);
return $backup->getConnection()->transaction(function () use (
$backupId, $now
): array {
$locked = CloudInstanceBackup::where('id', $backupId)
->lockForUpdate()->firstOrFail();
if ((string) $locked->state === 'deleted') {
return ['mode' => 'deleted', 'backup' => $locked];
}
if ((string) $locked->state === 'delete_attention') {
return ['mode' => 'attention', 'backup' => $locked];
}
if (!in_array((string) $locked->state,
['deleting', 'delete_uncertain'], true)) {
throw new CloudError('备份删除状态已发生变化', 409);
}
$last = max(0, (int) $locked->delete_last_checked_at);
if ($last > 0 && $now - $last < self::DELETE_STABLE_SECONDS) {
$locked->state = 'delete_uncertain';
$locked->delete_next_check_at = $last
+ self::DELETE_STABLE_SECONDS;
$locked->updated_at = $now;
$locked->save();
return ['mode' => 'waiting', 'backup' => $locked];
}
$confirmations = min(self::DELETE_STABLE_PROBES,
max(0, (int) $locked->delete_presence_confirmations) + 1);
$locked->state = 'delete_uncertain';
$locked->delete_presence_confirmations = $confirmations;
$locked->delete_last_checked_at = $now;
$locked->delete_next_check_at = $now
+ self::DELETE_STABLE_SECONDS;
$locked->error_code = 'object_delete_still_present';
$locked->updated_at = $now;
if ($confirmations < self::DELETE_STABLE_PROBES) {
$locked->save();
return ['mode' => 'waiting', 'backup' => $locked];
}
if ((int) $locked->delete_cycle_attempts
>= self::DELETE_MAX_AUTOMATIC_ATTEMPTS) {
$locked->state = 'delete_attention';
$locked->delete_attention_at = $now;
$locked->delete_next_check_at = 0;
$locked->error_code = 'object_delete_retry_exhausted';
$locked->save();
return ['mode' => 'attention', 'backup' => $locked];
}
$locked->state = 'deleting';
$locked->delete_operation_id = bin2hex(random_bytes(16));
$locked->delete_generation = max(0,
(int) $locked->delete_generation) + 1;
$locked->delete_cycle_attempts = max(0,
(int) $locked->delete_cycle_attempts) + 1;
$locked->delete_presence_confirmations = 0;
$locked->delete_next_check_at = 0;
$locked->error_code = '';
$locked->save();
return [
'mode' => 'delete',
'backup' => $locked,
'runtime' => self::storageRuntime($locked),
'object_key' => (string) $locked->object_key,
'object_version_id' => (string) $locked->object_version_id,
];
}, 3);
}
/** @param array{size_bytes:int,etag:string,version_id:string} $identity */
private static function assertObjectIdentity(CloudInstanceBackup $backup,
array $identity, bool $strict): void
{
if ((int) $identity['size_bytes'] !== (int) $backup->object_size_bytes) {
throw new CloudError('对象存储中的备份大小与目录账本不一致', 409);
}
if (!$strict) {
return;
}
$etag = (string) $backup->object_etag;
if ($etag === '' || !hash_equals($etag, (string) $identity['etag'])) {
throw new CloudError('对象存储中的备份 ETag 已发生变化', 409);
}
$versionId = (string) $backup->object_version_id;
if ($versionId !== ''
&& !hash_equals($versionId, (string) $identity['version_id'])) {
throw new CloudError('对象存储中的备份版本已发生变化', 409);
}
}
private static function markDeleteUncertain(int $backupId,
string $errorCode): void
string $errorCode, int $now, bool $resetEvidence,
int $retryAfter = self::DELETE_STABLE_SECONDS): void
{
$values = [
'state' => 'delete_uncertain',
'delete_next_check_at' => $now + max(1, $retryAfter),
'error_code' => substr($errorCode, 0, 64),
'updated_at' => $now,
];
if ($resetEvidence) {
$values['delete_presence_confirmations'] = 0;
$values['delete_last_checked_at'] = null;
}
CloudInstanceBackup::where('id', $backupId)
->whereIn('state', ['deleting', 'delete_uncertain'])
->update($values);
}
private static function markDeleteAttention(int $backupId,
string $errorCode, int $now): void
{
CloudInstanceBackup::where('id', $backupId)
->whereIn('state', ['deleting', 'delete_uncertain'])
->update([
'state' => 'delete_uncertain',
'state' => 'delete_attention',
'delete_next_check_at' => 0,
'delete_attention_at' => $now,
'error_code' => substr($errorCode, 0, 64),
'updated_at' => time(),
'updated_at' => $now,
]);
}
@ -1596,6 +1944,13 @@ final class CloudBackupService
->where('state', 'deleting')
->update([
'state' => $previousState,
'delete_operation_id' => null,
'delete_cycle_attempts' => 0,
'delete_presence_confirmations' => 0,
'delete_requested_at' => null,
'delete_last_checked_at' => null,
'delete_next_check_at' => 0,
'delete_attention_at' => null,
'error_code' => substr($errorCode, 0, 64),
'updated_at' => time(),
]);
@ -1612,16 +1967,28 @@ final class CloudBackupService
if ((string) $locked->state !== 'deleted') {
self::assertBackupDeletionAvailable($locked);
if (!in_array((string) $locked->state,
['deleting', 'delete_uncertain'], true)) {
['ready', 'failed', 'deleting', 'delete_uncertain',
'delete_attention'], true)) {
throw new CloudError('备份删除状态已发生变化', 409);
}
$locked->state = 'deleted';
$locked->sealed_storage_runtime = '';
$locked->sealed_credential = null;
$locked->active_job_id = null;
$locked->delete_presence_confirmations = 0;
$locked->delete_next_check_at = 0;
$locked->delete_attention_at = null;
$locked->error_code = '';
$locked->updated_at = time();
$locked->save();
CloudDeploymentSnapshot::where('backup_id', $backupId)
->whereIn('state', ['deleting', 'delete_uncertain'])
->update([
'state' => 'deleted',
'deleted_at' => time(),
'error_code' => '',
'updated_at' => time(),
]);
}
return self::deletionResult($locked);
}, 3);
@ -1634,7 +2001,10 @@ final class CloudBackupService
'backup_id' => (string) $backup->id,
'state' => $state,
'deleted' => $state === 'deleted',
'requires_reconciliation' => $state === 'delete_uncertain',
'requires_reconciliation' => in_array($state,
['deleting', 'delete_uncertain'], true),
'requires_attention' => $state === 'delete_attention',
'next_check_at' => max(0, (int) $backup->delete_next_check_at),
];
}
@ -1782,7 +2152,7 @@ final class CloudBackupService
{
$state = in_array((string) $backup->state,
['creating', 'ready', 'failed', 'uncertain', 'deleting',
'delete_uncertain', 'deleted'], true)
'delete_uncertain', 'delete_attention', 'deleted'], true)
? (string) $backup->state : 'uncertain';
$canDelete = $canManage && in_array($state,
['ready', 'failed', 'deleting', 'delete_uncertain'], true);
@ -1822,9 +2192,11 @@ final class CloudBackupService
'delete_action' => in_array($state,
['deleting', 'delete_uncertain'], true)
? 'reconcile' : 'delete',
'error' => $state === 'delete_uncertain'
? '删除结果尚未确认;再次核对只会读取对象状态,不会重发删除请求。'
: null,
'error' => $state === 'delete_attention'
? '对象自动清理已达到三次上限,需要管理员检查对象存储后手动恢复清理。'
: ($state === 'delete_uncertain'
? '删除结果尚未确认;系统会在两次稳定核对后创建新的删除代次。'
: null),
];
}

@ -269,7 +269,7 @@ final class CloudDeploymentSnapshotService
'id', (int) $prepared['backup']->id
)->value('state') ?? '');
$effectUnknown = in_array($backingState,
['deleting', 'delete_uncertain', 'deleted'], true);
['deleting', 'delete_uncertain', 'delete_attention', 'deleted'], true);
CloudDeploymentSnapshot::where(
'id', (int) $prepared['snapshot']->id
)->where('state', 'deleting')->update([
@ -291,9 +291,12 @@ final class CloudDeploymentSnapshotService
$locked->state = 'deleted';
$locked->deleted_at = time();
$locked->error_code = '';
} elseif (($result['requires_reconciliation'] ?? false) === true) {
} elseif (($result['requires_reconciliation'] ?? false) === true
|| ($result['requires_attention'] ?? false) === true) {
$locked->state = 'delete_uncertain';
$locked->error_code = 'object_delete_uncertain';
$locked->error_code = ($result['requires_attention'] ?? false)
? 'object_delete_attention_required'
: 'object_delete_uncertain';
} else {
$locked->state = 'deleting';
}
@ -351,7 +354,7 @@ final class CloudDeploymentSnapshotService
'active_invoice_count' => $activeInvoiceCount,
'has_blocking_use' => $blocked,
'error' => $state === 'delete_uncertain'
? '删除结果尚未确认;再次核对只会读取对象状态,不会重发删除请求。'
? '删除结果尚未确认;系统会按稳定对象证据继续受控清理,达到上限后转管理员处理。'
: null,
];
}

@ -320,11 +320,38 @@ final class CloudObjectStorageS3Client
public function headObject(
array $config,
array $credentials,
string $objectKey
string $objectKey,
string $versionId = ''
): array {
$identity = $this->inspectObject(
$config, $credentials, $objectKey, $versionId
);
if ($identity === null) {
throw new CloudObjectStorageTransferError(
'备份对象不存在或当前凭据无权读取。', 'object_head_rejected'
);
}
return $identity;
}
/**
* Inspect one exact object version. A 404 is a proven absence; transport,
* permission and provider failures remain distinguishable from absence.
*
* @param array<string,mixed> $config
* @param array{access_key_id:string,secret_access_key:string} $credentials
* @return array{size_bytes:int,etag:string,version_id:string}|null
*/
public function inspectObject(
array $config,
array $credentials,
string $objectKey,
string $versionId = ''
): ?array {
self::assertRuntime($config, $credentials);
self::assertObjectKey($objectKey);
$url = self::objectUrl($config, $objectKey);
self::assertVersionId($versionId);
$url = self::versionedObjectUrl($config, $objectKey, $versionId);
$headers = CloudS3Signer::sign(
'HEAD', $url, (string) $config['region'],
$credentials['access_key_id'], $credentials['secret_access_key'],
@ -339,6 +366,9 @@ final class CloudObjectStorageS3Client
'无法读取备份对象状态。', 'object_head_unavailable'
);
}
if ($response->getStatusCode() === 404) {
return null;
}
if ($response->getStatusCode() < 200 || $response->getStatusCode() >= 300) {
throw new CloudObjectStorageTransferError(
'备份对象不存在或当前凭据无权读取。', 'object_head_rejected'
@ -351,16 +381,26 @@ final class CloudObjectStorageS3Client
'备份对象缺少可信的大小信息。', 'object_head_invalid'
);
}
$etag = trim($response->getHeaderLine('ETag'));
if ($etag === '' || strlen($etag) > 256
|| preg_match('/[\x00-\x1f\x7f]/', $etag) === 1) {
throw new CloudObjectStorageTransferError(
'备份对象缺少可信的 ETag。', 'object_head_invalid'
);
}
$reportedVersion = trim($response->getHeaderLine('x-amz-version-id'));
self::assertVersionId($reportedVersion);
return [
'size_bytes' => (int) $length,
'etag' => trim($response->getHeaderLine('ETag')),
'version_id' => trim($response->getHeaderLine('x-amz-version-id')),
'etag' => $etag,
'version_id' => $reportedVersion,
];
}
/**
* Delete one exact backup object. A transport failure has unknown effect;
* callers must reconcile it with objectPresence() and never replay it.
* Delete one exact backup object version. A transport failure has unknown
* effect; callers must reconcile it and may only create a new generation
* after independently stable identity evidence.
*
* @param array<string,mixed> $config
* @param array{access_key_id:string,secret_access_key:string} $credentials
@ -368,11 +408,13 @@ final class CloudObjectStorageS3Client
public function deleteObject(
array $config,
array $credentials,
string $objectKey
string $objectKey,
string $versionId = ''
): void {
self::assertRuntime($config, $credentials);
self::assertObjectKey($objectKey);
$url = self::objectUrl($config, $objectKey);
self::assertVersionId($versionId);
$url = self::versionedObjectUrl($config, $objectKey, $versionId);
$headers = CloudS3Signer::sign(
'DELETE', $url, (string) $config['region'],
$credentials['access_key_id'], $credentials['secret_access_key'],
@ -404,7 +446,7 @@ final class CloudObjectStorageS3Client
}
/**
* Read-only reconciliation for an ambiguous DeleteObject call.
* Read-only presence check for one exact object version.
*
* @param array<string,mixed> $config
* @param array{access_key_id:string,secret_access_key:string} $credentials
@ -413,37 +455,12 @@ final class CloudObjectStorageS3Client
public function objectPresence(
array $config,
array $credentials,
string $objectKey
string $objectKey,
string $versionId = ''
): string {
self::assertRuntime($config, $credentials);
self::assertObjectKey($objectKey);
$url = self::objectUrl($config, $objectKey);
$headers = CloudS3Signer::sign(
'HEAD', $url, (string) $config['region'],
$credentials['access_key_id'], $credentials['secret_access_key'],
time()
);
try {
$response = $this->http()->request('HEAD', $url, [
'headers' => $headers,
] + self::requestOptions(30.0));
} catch (GuzzleException $exception) {
throw new CloudObjectStorageTransferError(
'无法核对备份对象是否仍然存在。',
'object_delete_probe_unavailable'
);
}
$status = $response->getStatusCode();
if ($status >= 200 && $status < 300) {
return 'present';
}
if ($status === 404) {
return 'absent';
}
throw new CloudObjectStorageTransferError(
'对象存储拒绝核对备份对象状态。',
'object_delete_probe_rejected'
);
return $this->inspectObject(
$config, $credentials, $objectKey, $versionId
) === null ? 'absent' : 'present';
}
/**
@ -506,6 +523,23 @@ final class CloudObjectStorageS3Client
return rtrim($base, '/') . '/' . $encoded;
}
/** @param array<string,mixed> $config */
private static function versionedObjectUrl(array $config,
string $objectKey, string $versionId): string
{
$url = self::objectUrl($config, $objectKey);
return $versionId === '' ? $url
: $url . '?versionId=' . rawurlencode($versionId);
}
private static function assertVersionId(string $versionId): void
{
if (strlen($versionId) > 1024
|| preg_match('/[\x00-\x1f\x7f]/D', $versionId) === 1) {
throw new \InvalidArgumentException('Invalid S3 object version identifier');
}
}
/**
* @param array<string,mixed> $config
* @param array{access_key_id:string,secret_access_key:string} $credentials

@ -178,6 +178,8 @@ $app->group('/admin', function () {
$this->post('/object-storage/config', 'App\Controllers\Admin\ObjectStorageController:save')->add(new SuperAdmin());
$this->post('/object-storage/test', 'App\Controllers\Admin\ObjectStorageController:test')->add(new SuperAdmin());
$this->post('/object-storage/disable', 'App\Controllers\Admin\ObjectStorageController:disable')->add(new SuperAdmin());
$this->get('/object-storage/cleanup-debts', 'App\Controllers\Admin\ObjectStorageController:cleanupDebt');
$this->post('/object-storage/cleanup-debts/{backup_id:[0-9]+}/retry', 'App\Controllers\Admin\ObjectStorageController:retryCleanup')->add(new SuperAdmin());
$this->get('/cloud/templates', 'App\Controllers\Admin\CloudPageController:templates');
$this->get('/cloud/rescue', 'App\Controllers\Admin\CloudPageController:rescue');
$this->get('/cloud/instances', 'App\Controllers\Admin\CloudPageController:instances');

@ -671,7 +671,8 @@ function backupStatus(value) {
creating: ['正在备份', 'warning'], ready: ['可恢复', 'success'],
restoring: ['正在恢复', 'warning'], failed: ['备份失败', 'danger'],
uncertain: ['效果待核对', 'danger'], deleting: ['正在删除', 'warning'],
delete_uncertain: ['删除待核对', 'danger'], deleted: ['已删除', 'secondary'],
delete_uncertain: ['删除待核对', 'danger'],
delete_attention: ['清理需人工处理', 'danger'], deleted: ['已删除', 'secondary'],
})[String(value || '').toLowerCase()] || ['状态未知', 'secondary'];
}
@ -741,7 +742,8 @@ async function deleteBackupObject(vmId, backup, controller, library = false) {
error.status = response.status;
throw error;
}
if (payload?.deleted !== true && payload?.requires_reconciliation !== true) {
if (payload?.deleted !== true && payload?.requires_reconciliation !== true
&& payload?.requires_attention !== true) {
const error = new Error('删除回执状态无效,请勿重复删除,先刷新核对。');
error.mutationMayHaveApplied = true;
throw error;
@ -771,13 +773,15 @@ function backupDeleteButton(vmId, backup, library = false) {
const controller = new AbortController();
state.backupMutationRequest = controller; button.disabled = true;
backupSurfaceNotice(reconcile
? '正在只读核对对象是否已删除;不会重发删除请求。'
? '正在核对准确对象版本;只有两次间隔证据一致才会创建新的删除代次。'
: '正在删除对象存储备份,请勿重复提交。', false, library);
try {
const result = await deleteBackupObject(vmId, backup, controller, library);
backupSurfaceNotice(result.deleted === true
? '备份对象已确认删除。'
: '删除结果仍不确定,已保留对象边界和冻结凭据;稍后可再次只读核对。',
: (result.requires_attention === true
? '自动清理已达到三次上限,请到对象存储的“对象清理异常”中核查。'
: '删除结果仍不确定;系统将按稳定证据继续受控核对。'),
result.deleted !== true, library);
refreshBackupSurface(vmId, library);
} catch (error) {
@ -1238,11 +1242,14 @@ function renderBackups(payload, vmId) {
'备份效果尚未确认。系统会保留任务和对象边界,请勿重复创建或恢复,先完成任务核对。'));
} else if (backup.state === 'delete_uncertain' || backup.state === 'deleting') {
body.append(el('p', 'alert alert-warning fs-13',
'删除结果尚未确认。核对操作只会读取对象状态,不会重发删除请求。'));
'删除结果尚未确认。系统会在两次间隔核对确认同一对象后创建新的删除代次。'));
if (backup.can_delete === true) {
const deleteActions = el('div', 'd-flex justify-content-end');
deleteActions.append(backupDeleteButton(vmId, backup)); body.append(deleteActions);
}
} else if (backup.state === 'delete_attention') {
body.append(el('p', 'alert alert-danger fs-13 mb-0',
'对象清理已达到自动重试上限,请在“对象存储 -> 对象清理异常”中核查并手动恢复。'));
} else if (backup.state === 'ready') {
const targets = backupTargets(payload, backup);
if (targets.length) body.append(restoreForm(vmId, backup, targets,
@ -1343,7 +1350,7 @@ function renderBackupLibrary(payload) {
const actions = el('div', 'd-flex justify-content-end mt-3');
actions.append(backupDeleteButton(vmId, backup, true));
body.append(actions);
} else if (!canManage && ['ready', 'failed', 'deleting', 'delete_uncertain'].includes(backup.state)) {
} else if (!canManage && ['ready', 'failed', 'deleting', 'delete_uncertain', 'delete_attention'].includes(backup.state)) {
body.append(el('p', 'text-muted fs-12 mb-0',
'当前管理员仅可查看,超级管理员才可以恢复或删除备份。'));
}

@ -59,7 +59,7 @@ function backupState(value) {
const values = {
ready: ['可用', 'running'], creating: ['正在备份', 'warning'], queued: ['等待备份', 'warning'],
failed: ['备份失败', 'stopped'], uncertain: ['等待核对', 'warning'], deleting: ['正在删除', 'warning'],
delete_uncertain: ['删除待核对', 'warning'],
delete_uncertain: ['删除待核对', 'warning'], delete_attention: ['清理需人工处理', 'stopped'],
};
return values[normalized] || [normalized || '状态未知', 'other'];
}

@ -222,7 +222,7 @@ async function deleteSnapshot(snapshot) {
if (resultState === 'deleted') {
toast('云快照已删除。', 'success');
} else if (resultState === 'delete_uncertain') {
toast('删除结果待核对,请再次点击删除;系统只会读取对象状态,不会重复发送删除请求。', 'warning');
toast('删除结果待核对;系统会在两次间隔证据一致后创建新的删除代次。', 'warning');
} else {
toast('云快照正在删除,请稍后刷新状态。', 'warning');
}

@ -1557,6 +1557,8 @@ function automaticBackupState(backup) {
ready: ['可恢复', 'running'], creating: ['正在备份', 'warning'],
queued: ['等待备份', 'warning'], failed: ['备份失败', 'stopped'],
uncertain: ['等待核对', 'warning'], restoring: ['正在恢复', 'warning'],
deleting: ['正在删除', 'warning'], delete_uncertain: ['删除待核对', 'warning'],
delete_attention: ['清理需管理员处理', 'stopped'],
};
return labels[value] || [value || '状态未知', 'other'];
}

@ -2,7 +2,7 @@
{include file='admin/static_header.tpl'}
{literal}<style>
.os-page{display:grid;gap:20px;min-width:0;max-width:1080px;padding-bottom:30px}.os-head{display:flex;align-items:flex-start;justify-content:space-between;gap:18px}.os-head h1{margin:0 0 7px;font-size:28px;color:#0f172a}.os-head p{margin:0;color:#64748b;line-height:1.7}.os-card{min-width:0;border:1px solid #e2e8f0;border-radius:10px;background:#fff;box-shadow:0 10px 28px rgba(15,23,42,.05);padding:22px}.os-card-head{display:flex;align-items:flex-start;justify-content:space-between;gap:14px;margin-bottom:20px}.os-card-head h2{margin:0 0 5px;font-size:20px;color:#172033}.os-card-head p{margin:0;color:#64748b;line-height:1.65}.os-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:17px}.os-field{display:grid;gap:7px;min-width:0}.os-field.is-wide{grid-column:1/-1}.os-field label{font-weight:700;color:#334155}.os-help{font-size:12px;line-height:1.55;color:#64748b}.os-mono{font-family:SFMono-Regular,Consolas,Liberation Mono,monospace}.os-secret-state{display:inline-flex;align-items:center;gap:6px;color:#047857;font-weight:700}.os-actions{display:flex;flex-wrap:wrap;align-items:center;gap:10px;margin-top:22px}.os-notice{display:none;flex:1 1 300px;padding:11px 13px;border-radius:7px;font-size:13px;line-height:1.55}.os-notice.is-show{display:block}.os-notice.is-ok{background:#ecfdf5;color:#047857}.os-notice.is-error{background:#fef2f2;color:#b91c1c}.os-alert{padding:13px 15px;border-radius:8px;line-height:1.65}.os-alert.is-error{background:#fef2f2;color:#b91c1c}.os-alert.is-info{background:#eff6ff;color:#1d4ed8}.os-status{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px;margin-top:18px}.os-status-item{padding:14px;border:1px solid #e2e8f0;border-radius:8px;background:#f8fafc}.os-status-item span{display:block;margin-bottom:5px;color:#64748b;font-size:12px}.os-status-item strong{display:block;color:#1e293b;overflow-wrap:anywhere}.os-readonly{margin-bottom:16px;padding:11px 13px;border-radius:7px;background:#fff7ed;color:#9a3412}.os-scope{display:grid;gap:11px;margin:0;padding:0;list-style:none}.os-scope li{display:flex;align-items:flex-start;gap:10px;color:#475569;line-height:1.6}.os-scope i{margin-top:2px;color:#0ea5a4;font-size:18px}.os-scope li.is-muted i{color:#94a3b8}@media(max-width:760px){.os-page{gap:16px}.os-head{display:grid}.os-head h1{font-size:24px}.os-card{padding:17px}.os-grid,.os-status{grid-template-columns:1fr}.os-field.is-wide{grid-column:auto}.os-actions .btn{width:100%}.os-notice{flex-basis:100%}}
.os-page{display:grid;gap:20px;min-width:0;max-width:1080px;padding-bottom:30px}.os-head{display:flex;align-items:flex-start;justify-content:space-between;gap:18px}.os-head h1{margin:0 0 7px;font-size:28px;color:#0f172a}.os-head p{margin:0;color:#64748b;line-height:1.7}.os-card{min-width:0;border:1px solid #e2e8f0;border-radius:10px;background:#fff;box-shadow:0 10px 28px rgba(15,23,42,.05);padding:22px}.os-card-head{display:flex;align-items:flex-start;justify-content:space-between;gap:14px;margin-bottom:20px}.os-card-head h2{margin:0 0 5px;font-size:20px;color:#172033}.os-card-head p{margin:0;color:#64748b;line-height:1.65}.os-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:17px}.os-field{display:grid;gap:7px;min-width:0}.os-field.is-wide{grid-column:1/-1}.os-field label{font-weight:700;color:#334155}.os-help{font-size:12px;line-height:1.55;color:#64748b}.os-mono{font-family:SFMono-Regular,Consolas,Liberation Mono,monospace}.os-secret-state{display:inline-flex;align-items:center;gap:6px;color:#047857;font-weight:700}.os-actions{display:flex;flex-wrap:wrap;align-items:center;gap:10px;margin-top:22px}.os-notice{display:none;flex:1 1 300px;padding:11px 13px;border-radius:7px;font-size:13px;line-height:1.55}.os-notice.is-show{display:block}.os-notice.is-ok{background:#ecfdf5;color:#047857}.os-notice.is-error{background:#fef2f2;color:#b91c1c}.os-alert{padding:13px 15px;border-radius:8px;line-height:1.65}.os-alert.is-error{background:#fef2f2;color:#b91c1c}.os-alert.is-info{background:#eff6ff;color:#1d4ed8}.os-status{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px;margin-top:18px}.os-status-item{padding:14px;border:1px solid #e2e8f0;border-radius:8px;background:#f8fafc}.os-status-item span{display:block;margin-bottom:5px;color:#64748b;font-size:12px}.os-status-item strong{display:block;color:#1e293b;overflow-wrap:anywhere}.os-readonly{margin-bottom:16px;padding:11px 13px;border-radius:7px;background:#fff7ed;color:#9a3412}.os-scope{display:grid;gap:11px;margin:0;padding:0;list-style:none}.os-scope li{display:flex;align-items:flex-start;gap:10px;color:#475569;line-height:1.6}.os-scope i{margin-top:2px;color:#0ea5a4;font-size:18px}.os-scope li.is-muted i{color:#94a3b8}.os-debt-table{overflow-x:auto}.os-debt-table table{width:100%;min-width:780px;border-collapse:collapse}.os-debt-table th,.os-debt-table td{padding:12px 10px;border-bottom:1px solid #e2e8f0;text-align:left;vertical-align:middle}.os-debt-table th{color:#64748b;font-size:12px}.os-debt-table td{color:#334155}.os-debt-table tr:last-child td{border-bottom:0}.os-debt-state{display:inline-flex;padding:4px 8px;border-radius:999px;background:#fff7ed;color:#9a3412;font-size:12px;font-weight:700}.os-debt-state.is-attention{background:#fef2f2;color:#b91c1c}.os-empty{padding:20px;text-align:center;color:#64748b}@media(max-width:760px){.os-page{gap:16px}.os-head{display:grid}.os-head h1{font-size:24px}.os-card{padding:17px}.os-grid,.os-status{grid-template-columns:1fr}.os-field.is-wide{grid-column:auto}.os-actions .btn{width:100%}.os-notice{flex-basis:100%}}
</style>{/literal}
<div class="os-page">
@ -107,12 +107,61 @@
{/if}
</section>
<section class="os-card" id="objectCleanupDebt">
<div class="os-card-head">
<div>
<h2>对象清理异常</h2>
<p>删除意图已从用户保留份数中移除;这里单独跟踪仍需在对象存储收敛的清理债务。</p>
</div>
<button class="btn btn-outline-secondary btn-sm" id="osDebtRefresh" type="button"><i class="mdi mdi-refresh me-1"></i>刷新</button>
</div>
{if ($cleanup_debt_error|default:'') != ''}
<div class="os-alert is-error">{$cleanup_debt_error|escape}</div>
{elseif $cleanup_debt|default:false}
<div class="os-status" style="margin:0 0 18px">
<div class="os-status-item"><span>清理债务</span><strong>{$cleanup_debt.summary.count} 个对象</strong></div>
<div class="os-status-item"><span>待人工处理</span><strong>{$cleanup_debt.summary.attention_count} 个对象</strong></div>
<div class="os-status-item"><span>占用空间</span><strong class="os-byte-value" data-bytes="{$cleanup_debt.summary.bytes}">{$cleanup_debt.summary.bytes} B</strong></div>
</div>
{if $cleanup_debt.items|@count > 0}
<div class="os-debt-table">
<table>
<thead><tr><th>备份</th><th>用户 / 实例</th><th>状态</th><th>删除代次</th><th>对象大小</th><th>操作</th></tr></thead>
<tbody>
{foreach $cleanup_debt.items as $item}
<tr>
<td><strong>#{$item.id} {$item.name|escape}</strong><div class="os-help">{$item.error_code|escape}</div></td>
<td>用户 #{$item.user_id}<div class="os-help">实例 #{$item.source_instance_id}</div></td>
<td><span class="os-debt-state {if $item.state == 'delete_attention'}is-attention{/if}">{if $item.state == 'delete_attention'}需人工处理{else}自动核对中{/if}</span></td>
<td>总计 {$item.delete_generation}<div class="os-help">本轮 {$item.cycle_attempts} / 3</div></td>
<td><span class="os-byte-value" data-bytes="{$item.object_size_bytes}">{$item.object_size_bytes} B</span></td>
<td>{if $item.can_retry}<button class="btn btn-sm btn-outline-danger os-cleanup-retry" type="button" data-backup-id="{$item.id}">手动重试</button>{else}<span class="os-help">等待系统核对</span>{/if}</td>
</tr>
{/foreach}
</tbody>
</table>
</div>
{if $cleanup_debt.pagination.total_pages > 1}
<div class="os-actions" style="justify-content:flex-end;margin-top:16px">
<a class="btn btn-sm btn-outline-secondary {if $cleanup_debt.pagination.page <= 1}disabled{/if}" href="/admin/object-storage?cleanup_page={$cleanup_debt.pagination.page-1}">上一页</a>
<span class="os-help">第 {$cleanup_debt.pagination.page} / {$cleanup_debt.pagination.total_pages} 页</span>
<a class="btn btn-sm btn-outline-secondary {if $cleanup_debt.pagination.page >= $cleanup_debt.pagination.total_pages}disabled{/if}" href="/admin/object-storage?cleanup_page={$cleanup_debt.pagination.page+1}">下一页</a>
</div>
{/if}
{else}
<div class="os-empty">当前没有对象清理异常。</div>
{/if}
{/if}
<div class="os-notice" id="osDebtNotice" role="status" aria-live="polite"></div>
</section>
<section class="os-card">
<div class="os-card-head"><div><h2>当前功能边界</h2><p>本阶段只建立安全、可验证的对象存储配置基础。</p></div></div>
<div class="os-card-head"><div><h2>运行边界</h2><p>云备份使用精确对象身份;Bucket 生命周期只作为控制面之外的最后兜底。</p></div></div>
<ul class="os-scope">
<li><i class="mdi mdi-check-circle-outline"></i><span>保存 S3 Endpoint、Region、Bucket、前缀和寻址方式,并加密保存访问凭据。</span></li>
<li><i class="mdi mdi-check-circle-outline"></i><span>使用签名的 HeadBucket 请求验证 Bucket 可达性和 <span class="os-mono">s3:ListBucket</span> 权限;测试不会写入对象。</span></li>
<li class="is-muted"><i class="mdi mdi-clock-outline"></i><span>云备份上传、增量链、保留策略、加密备份和异地恢复尚未接入,等待对象存储配置稳定后单独开发。</span></li>
<li><i class="mdi mdi-check-circle-outline"></i><span>云备份对象采用独立目录身份;版本控制 Bucket 会保存并按准确 <span class="os-mono">version_id</span> 删除,并需要 <span class="os-mono">s3:GetObjectVersion</span> / <span class="os-mono">s3:DeleteObjectVersion</span> 权限。</span></li>
<li class="is-muted"><i class="mdi mdi-alert-outline"></i><span>Bucket 仍应配置“终止未完成 multipart”和“清理非当前版本/过期删除标记”的生命周期规则作为最后兜底。</span></li>
</ul>
</section>
</div>
@ -230,4 +279,51 @@
})();
</script>{/literal}
{literal}<script>
(function () {
'use strict';
function formatBytes(value) {
var bytes = Math.max(0, Number(value) || 0);
var units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
var index = 0;
while (bytes >= 1024 && index < units.length - 1) { bytes /= 1024; index++; }
return (index === 0 ? String(Math.round(bytes)) : bytes.toFixed(bytes >= 10 ? 1 : 2)) + ' ' + units[index];
}
document.querySelectorAll('.os-byte-value').forEach(function (node) {
node.textContent = formatBytes(node.getAttribute('data-bytes'));
});
var refresh = document.getElementById('osDebtRefresh');
if (refresh) refresh.addEventListener('click', function () { window.location.reload(); });
var notice = document.getElementById('osDebtNotice');
document.querySelectorAll('.os-cleanup-retry').forEach(function (button) {
button.addEventListener('click', function () {
var backupId = String(button.getAttribute('data-backup-id') || '');
if (!/^\d+$/.test(backupId) || button.disabled) return;
if (!window.confirm('确认重新开启该对象的受控清理周期?系统仍会先完成稳定 HEAD 核对。')) return;
button.disabled = true;
if (notice) {
notice.textContent = '正在核对准确对象版本...';
notice.className = 'os-notice is-show is-ok';
}
fetch('/admin/object-storage/cleanup-debts/' + encodeURIComponent(backupId) + '/retry', {
method: 'POST', credentials: 'same-origin',
headers: { 'Accept': 'application/json', 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' },
body: '{}'
}).then(function (response) {
return response.json().then(function (result) {
if (!response.ok || Number(result.ret) !== 1) throw new Error(result.msg || '请求失败');
return result;
});
}).then(function (result) {
if (notice) { notice.textContent = result.msg; notice.className = 'os-notice is-show is-ok'; }
window.setTimeout(function () { window.location.reload(); }, 700);
}).catch(function (error) {
if (notice) { notice.textContent = error.message || '请求失败'; notice.className = 'os-notice is-show is-error'; }
button.disabled = false;
});
});
});
})();
</script>{/literal}
{include file='admin/static_footer.tpl'}

@ -1,2 +1,2 @@
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
{assign var=venix_asset_version value='2026102801' scope='global'}
{assign var=venix_asset_version value='2026102802' scope='global'}

@ -599,12 +599,22 @@ CREATE TABLE cloud_instance_backups (
object_size_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
virtual_size_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
object_etag VARCHAR(256) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
object_version_id VARCHAR(1024) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
spec_json LONGTEXT NOT NULL,
sealed_storage_runtime LONGTEXT NOT NULL,
sealed_credential LONGTEXT DEFAULT NULL,
storage_revision BIGINT UNSIGNED NOT NULL,
active_job_id BIGINT UNSIGNED DEFAULT NULL,
last_operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
delete_operation_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
delete_generation INT UNSIGNED NOT NULL DEFAULT 0,
delete_cycle_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0,
delete_presence_confirmations TINYINT UNSIGNED NOT NULL DEFAULT 0,
delete_requested_at BIGINT UNSIGNED DEFAULT NULL,
delete_last_checked_at BIGINT UNSIGNED DEFAULT NULL,
delete_next_check_at BIGINT UNSIGNED NOT NULL DEFAULT 0,
delete_attention_at BIGINT UNSIGNED DEFAULT NULL,
error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
ready_at BIGINT UNSIGNED DEFAULT NULL,
created_at BIGINT UNSIGNED NOT NULL,
@ -614,7 +624,8 @@ CREATE TABLE cloud_instance_backups (
UNIQUE KEY cloud_instance_backup_schedule (subscription_id, scheduled_for),
KEY cloud_instance_backup_source (source_instance_id, state, id),
KEY cloud_instance_backup_owner (user_id, id),
KEY cloud_instance_backup_operation (last_operation_id)
KEY cloud_instance_backup_operation (last_operation_id),
KEY cloud_instance_backup_delete_due (state, delete_next_check_at, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- A user-owned, durable deployment source promoted from one automatic cloud
-- backup. The backing backup row remains the sole owner of the immutable S3

@ -100,12 +100,22 @@ namespace {
$table->integer('object_size_bytes');
$table->integer('virtual_size_bytes');
$table->string('sha256');
$table->string('object_etag')->default('');
$table->string('object_version_id')->default('');
$table->text('spec_json');
$table->text('sealed_storage_runtime');
$table->text('sealed_credential')->nullable();
$table->integer('storage_revision');
$table->integer('active_job_id')->nullable();
$table->string('last_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_requested_at')->nullable();
$table->integer('delete_last_checked_at')->nullable();
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->string('error_code');
$table->integer('scheduled_for')->nullable();
$table->integer('ready_at')->nullable();
@ -118,6 +128,9 @@ namespace {
$table->increments('id');
$table->integer('backup_id')->unique();
$table->string('state');
$table->string('error_code')->default('');
$table->integer('deleted_at')->nullable();
$table->integer('updated_at')->default(0);
});
foreach (['cloud_backup_restores', 'cloud_backup_recoveries'] as $tableName) {
$schema->create($tableName, static function (Blueprint $table): void {
@ -192,6 +205,7 @@ namespace {
'object_key' => 'virt/vm-backups/' . $suffix . '/system.qcow2',
'object_size_bytes' => 1024, 'virtual_size_bytes' => 2048,
'sha256' => str_repeat('a', 64),
'object_etag' => '', 'object_version_id' => '',
'spec_json' => json_encode([
'managed' => true, 'os_source' => 'cloud_template',
'os_name' => 'Debian 13', 'login_name' => 'root',
@ -199,6 +213,9 @@ namespace {
], JSON_THROW_ON_ERROR),
'sealed_storage_runtime' => $sealedRuntime,
'sealed_credential' => 'sealed-credential', 'storage_revision' => 1,
'delete_generation' => 0, 'delete_cycle_attempts' => 0,
'delete_presence_confirmations' => 0,
'delete_next_check_at' => 0,
'error_code' => '', 'scheduled_for' => $now - 1000,
'ready_at' => $now - 900, 'created_at' => $now - 1000,
'updated_at' => $now - 1000,
@ -217,6 +234,17 @@ namespace {
$newSubscription(15, [$now - 3 * 86400]),
'ordinary-expired'
);
$debt = $newBackup(
$newSubscription(16, [$now - 3 * 86400]),
'cleanup-debt'
);
$debt->state = 'delete_uncertain';
$debt->object_etag = '"debt-etag"';
$debt->object_version_id = 'debt-version';
$debt->delete_generation = 1;
$debt->delete_cycle_attempts = 1;
$debt->delete_next_check_at = 0;
$debt->save();
$capsule->table('cloud_deployment_snapshots')->insert([
'backup_id' => (int) $expired->id,
'state' => 'ready',
@ -228,24 +256,34 @@ namespace {
&& CloudAutomaticBackupService::withinRecoveryGrace($renewed, $now),
'Recovery eligibility did not follow the paid period plus three-day grace.');
$history = [];
$stack = HandlerStack::create(new MockHandler([new Response(204)]));
$stack = HandlerStack::create(new MockHandler([
new Response(404),
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"fresh-etag"',
'x-amz-version-id' => 'fresh-version',
]),
new Response(204),
]));
$stack->push(Middleware::history($history));
$client = new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
$removed = CloudAutomaticBackupService::pruneRetention(10, $now, $client);
$removed = CloudAutomaticBackupService::pruneRetention(1, $now, $client);
foreach ([$paid, $insideGrace, $expired, $renewed, $ordinaryExpired]
foreach ([$paid, $insideGrace, $expired, $renewed, $ordinaryExpired, $debt]
as $backup) {
$backup->refresh();
}
$assert($removed === 1
$assert($removed === 2
&& (string) $paid->state === 'ready'
&& (string) $insideGrace->state === 'ready'
&& (string) $expired->state === 'ready'
&& (string) $renewed->state === 'ready'
&& (string) $ordinaryExpired->state === 'deleted'
&& count($history) === 1
&& $history[0]['request']->getMethod() === 'DELETE',
'Automatic backups were not retained through grace or were pruned after promotion to a cloud snapshot.');
&& (string) $debt->state === 'deleted'
&& count($history) === 3
&& array_map(static fn ($item): string =>
$item['request']->getMethod(), $history) === ['HEAD', 'HEAD', 'DELETE'],
'Cleanup debt starved fresh retention work or entitlement grace was not preserved.');
echo "Automatic backup entitlement grace retention tests passed.\n";
}

@ -95,12 +95,22 @@ namespace {
$table->integer('object_size_bytes');
$table->integer('virtual_size_bytes');
$table->string('sha256');
$table->string('object_etag')->default('');
$table->string('object_version_id')->default('');
$table->text('spec_json');
$table->text('sealed_storage_runtime');
$table->text('sealed_credential')->nullable();
$table->integer('storage_revision');
$table->integer('active_job_id')->nullable();
$table->string('last_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_requested_at')->nullable();
$table->integer('delete_last_checked_at')->nullable();
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->string('error_code');
$table->integer('ready_at')->nullable();
$table->integer('created_at');
@ -208,6 +218,8 @@ namespace {
$backup->object_size_bytes = 1024;
$backup->virtual_size_bytes = 2048;
$backup->sha256 = str_repeat(dechex(($id % 15) + 1), 64);
$backup->object_etag = '';
$backup->object_version_id = '';
$backup->spec_json = json_encode([
'managed' => true,
'os_source' => 'cloud_template',
@ -220,6 +232,10 @@ namespace {
$backup->storage_revision = 1;
$backup->active_job_id = null;
$backup->last_operation_id = null;
$backup->delete_generation = 0;
$backup->delete_cycle_attempts = 0;
$backup->delete_presence_confirmations = 0;
$backup->delete_next_check_at = 0;
$backup->error_code = '';
$backup->ready_at = $now;
$backup->created_at = $backup->updated_at = $now;
@ -398,13 +414,21 @@ namespace {
$deleted = CloudDeploymentSnapshotService::delete(
(string) $snapshot->snapshot_key,
$actor,
$client([new Response(204)], $history)
$client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"snapshot-etag"',
'x-amz-version-id' => 'snapshot-version',
]),
new Response(204),
], $history)
);
$good->refresh();
$assert(($deleted['state'] ?? '') === 'deleted'
&& (string) $good->state === 'deleted'
&& count($history) === 1
&& $history[0]['request']->getMethod() === 'DELETE',
&& count($history) === 2
&& $history[0]['request']->getMethod() === 'HEAD'
&& $history[1]['request']->getMethod() === 'DELETE',
'Cloud snapshot deletion did not converge its backing object.');
$uncertainBackup = $newBackup(15, 7, 3);
@ -415,14 +439,24 @@ namespace {
$uncertain = CloudDeploymentSnapshotService::delete(
(string) $uncertainSnapshot->snapshot_key,
$actor,
$client([new ConnectException(
'timeout', new Request('DELETE', 'https://objects.example.com')
)], $uncertainHistory)
$client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"uncertain-etag"',
'x-amz-version-id' => 'uncertain-version',
]),
new ConnectException(
'timeout', new Request('DELETE', 'https://objects.example.com')
),
], $uncertainHistory)
);
$assert(($uncertain['state'] ?? '') === 'delete_uncertain'
&& count($uncertainHistory) === 1
&& $uncertainHistory[0]['request']->getMethod() === 'DELETE',
&& count($uncertainHistory) === 2
&& $uncertainHistory[1]['request']->getMethod() === 'DELETE',
'Unknown object deletion was not reflected by the cloud snapshot.');
$uncertainBackup->refresh();
$uncertainBackup->delete_next_check_at = 0;
$uncertainBackup->save();
$reconcileHistory = [];
$reconciled = CloudDeploymentSnapshotService::delete(
(string) $uncertainSnapshot->snapshot_key,

@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false
$assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema),
'cloud.sql retained an engine-prefixed identifier.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v11-peer-migration-v2'")
"SCHEMA_VERSION = 'go-agent-wss-v12-object-cleanup-v1'")
&& !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"),
'Outbound Agent schema must reject the incompatible earlier Go database.');
$assert(str_contains($schema,
@ -89,6 +89,21 @@ $assert(str_contains($schema,
&& str_contains($setup,
"'cloud_job_backup_recovery'"),
'Fresh schema is missing portable-backup catalog or restore reservation constraints.');
$assert(str_contains($schema,
'object_etag VARCHAR(256) CHARACTER SET ascii COLLATE ascii_bin NOT NULL')
&& str_contains($schema,
'object_version_id VARCHAR(1024) CHARACTER SET ascii COLLATE ascii_bin NOT NULL')
&& str_contains($schema,
'delete_generation INT UNSIGNED NOT NULL DEFAULT 0')
&& str_contains($schema,
'delete_cycle_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0')
&& str_contains($schema,
'delete_presence_confirmations TINYINT UNSIGNED NOT NULL DEFAULT 0')
&& str_contains($schema,
'KEY cloud_instance_backup_delete_due (state, delete_next_check_at, id)')
&& str_contains($setup, "'object_etag', 'object_version_id'")
&& str_contains($setup, "'cloud_instance_backup_delete_due'"),
'Fresh backup schema is missing exact-version cleanup debt fields or index.');
$assert(str_contains($schema,
'CREATE TABLE cloud_backup_period_entitlements (')
&& str_contains($schema,

@ -31,6 +31,11 @@ foreach (['config' => 'save', 'test' => 'test', 'disable' => 'disable'] as $path
$assert(str_contains($routes, $needle),
'Object-storage mutation lost SuperAdmin/CSRF middleware: ' . $action);
}
$assert(str_contains($routes,
"get('/object-storage/cleanup-debts', 'App\\Controllers\\Admin\\ObjectStorageController:cleanupDebt')")
&& str_contains($routes,
"post('/object-storage/cleanup-debts/{backup_id:[0-9]+}/retry', 'App\\Controllers\\Admin\\ObjectStorageController:retryCleanup')->add(new SuperAdmin())"),
'Object cleanup debt list or protected manual retry route is missing.');
$assert(str_contains($sidebar, 'href="/admin/object-storage"')
&& str_contains($sidebar, '<span>对象存储</span>'),
'Object-storage navigation entry is missing.');
@ -57,14 +62,21 @@ $assert(substr_count($template, 'name="access_key_id" type="password" value=""')
&& !str_contains($template, '$object_storage.access_key_id')
&& !str_contains($template, '$object_storage.secret_access_key')
&& str_contains($template, 'data-can-manage=')
&& str_contains($template, 's3:ListBucket'),
&& str_contains($template, 's3:ListBucket')
&& str_contains($template, '对象清理异常')
&& str_contains($template, 'os-cleanup-retry')
&& !str_contains($template, '$item.object_version_id'),
'Admin UI can expose credentials or lost its permission/test boundary.');
$assert(str_contains($controller, "'Cache-Control', 'private, no-store'")
&& str_contains($controller, 'CloudBackupService::cleanupDebt')
&& str_contains($controller, 'CloudBackupService::retryCleanup')
&& preg_match('/error_log\\([^\\n]*getMessage/', $controller) !== 1,
'Object-storage responses lost no-store or may log raw exception messages.');
$assert(is_string($deployment) && str_contains($deployment, '### 对象存储应用')
&& str_contains($deployment, '可移植备份')
&& str_contains($deployment, 'multipart 上传、读取、HEAD 和删除权限')
&& str_contains($deployment, '非当前版本和过期删除标记')
&& str_contains($deployment, '终止未完成 multipart')
&& is_string($validation) && str_contains($validation, '对象存储配置页面不得回显'),
'Deployment or validation boundaries are not documented.');

@ -130,9 +130,9 @@ $assert(str_contains($runtime,
$assert(str_contains($runtime, "method: 'DELETE'")
&& str_contains($runtime, '核对删除状态')
&& str_contains($runtime, '不会重发删除请求')
&& str_contains($runtime, '两次间隔证据一致')
&& str_contains($runtime, 'requires_reconciliation'),
'Backup deletion UI must expose non-replaying read-only reconciliation after unknown effect.');
'Backup deletion UI must expose bounded stable-evidence reconciliation after unknown effect.');
$assert(str_contains($runtime, '统一接入 br0、分配全新公网 IP')
&& str_contains($runtime, '不会继承原 IP、账单、到期时间或流量周期')

@ -76,12 +76,22 @@ namespace {
$table->integer('object_size_bytes');
$table->integer('virtual_size_bytes');
$table->string('sha256');
$table->string('object_etag')->default('');
$table->string('object_version_id')->default('');
$table->text('spec_json');
$table->text('sealed_storage_runtime');
$table->text('sealed_credential')->nullable();
$table->integer('storage_revision');
$table->integer('active_job_id')->nullable();
$table->string('last_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_requested_at')->nullable();
$table->integer('delete_last_checked_at')->nullable();
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->string('error_code');
$table->integer('ready_at')->nullable();
$table->integer('created_at');
@ -92,6 +102,9 @@ namespace {
$table->increments('id');
$table->integer('backup_id')->unique();
$table->string('state');
$table->string('error_code')->default('');
$table->integer('deleted_at')->nullable();
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_backup_restores',
static function (Blueprint $table): void {
@ -153,12 +166,22 @@ namespace {
$backup->object_size_bytes = 1024;
$backup->virtual_size_bytes = 2048;
$backup->sha256 = str_repeat('a', 64);
$backup->object_etag = '';
$backup->object_version_id = '';
$backup->spec_json = '{}';
$backup->sealed_storage_runtime = $sealedRuntime;
$backup->sealed_credential = 'sealed-guest-credential';
$backup->storage_revision = 7;
$backup->active_job_id = null;
$backup->last_operation_id = null;
$backup->delete_operation_id = null;
$backup->delete_generation = 0;
$backup->delete_cycle_attempts = 0;
$backup->delete_presence_confirmations = 0;
$backup->delete_requested_at = null;
$backup->delete_last_checked_at = null;
$backup->delete_next_check_at = 0;
$backup->delete_attention_at = null;
$backup->error_code = '';
$backup->ready_at = time();
$backup->created_at = $backup->updated_at = time();
@ -170,24 +193,33 @@ namespace {
$stack->push(Middleware::history($history));
return new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
};
$identity = static fn (): Response => new Response(200, [
'Content-Length' => '1024',
'ETag' => '"catalog-etag"',
'x-amz-version-id' => 'catalog-version',
]);
$history = [];
$backup = $newBackup(10, 'one');
$deleted = CloudBackupService::delete(
$backup, $actor, $client([new Response(204)], $history)
$backup, $actor, $client([$identity(), new Response(204)], $history)
);
$backup->refresh();
$assert($deleted['deleted'] === true
&& (string) $backup->state === 'deleted'
&& (string) $backup->sealed_storage_runtime === ''
&& $backup->sealed_credential === null
&& count($history) === 1
&& $history[0]['request']->getMethod() === 'DELETE',
&& count($history) === 2
&& $history[0]['request']->getMethod() === 'HEAD'
&& $history[1]['request']->getMethod() === 'DELETE'
&& str_contains((string) $history[1]['request']->getUri(),
'versionId=catalog-version'),
'Explicit DeleteObject success did not tombstone the catalog and remove frozen credentials.');
$unknown = $newBackup(11, 'two');
$unknownHistory = [];
$unknownResult = CloudBackupService::delete($unknown, $actor, $client([
$identity(),
new ConnectException('transport lost',
new Request('DELETE', 'https://objects.example.com')),
], $unknownHistory));
@ -196,6 +228,8 @@ namespace {
&& (string) $unknown->state === 'delete_uncertain'
&& (string) $unknown->sealed_storage_runtime !== '',
'Ambiguous DeleteObject did not retain the frozen runtime and unknown-effect fence.');
$unknown->delete_next_check_at = 0;
$unknown->save();
$probeHistory = [];
$reconciled = CloudBackupService::delete(
$unknown, $actor, $client([new Response(404)], $probeHistory)
@ -208,12 +242,16 @@ namespace {
$present = $newBackup(12, 'three');
$firstHistory = [];
CloudBackupService::delete($present, $actor, $client([
$identity(),
new ConnectException('transport lost',
new Request('DELETE', 'https://objects.example.com')),
], $firstHistory));
$present->refresh();
$present->delete_next_check_at = 0;
$present->save();
$presentHistory = [];
$stillPresent = CloudBackupService::delete(
$present, $actor, $client([new Response(200)], $presentHistory)
$present, $actor, $client([$identity()], $presentHistory)
);
$present->refresh();
$assert($stillPresent['requires_reconciliation'] === true
@ -256,7 +294,7 @@ namespace {
$quotaHistory = [];
CloudBackupService::delete(
$quotaBackups[0], $actor,
$client([new Response(204)], $quotaHistory)
$client([$identity(), new Response(204)], $quotaHistory)
);
$after = CloudInstanceBackup::where('source_instance_id', 99)
->whereIn('state', $activeStates)->count();
@ -264,5 +302,94 @@ namespace {
&& $after === CloudBackupService::MAX_PER_INSTANCE - 1,
'Confirmed deletion did not release one retained-backup quota slot.');
// One ambiguous generation may be retried only after two stable, spaced
// identity probes. The retry is a new deletion generation bound to the
// exact S3 version instead of a replay of the ambiguous request.
$retry = $newBackup(100, 'stable-retry', 'delete_uncertain');
$retry->object_etag = '"stable-etag"';
$retry->object_version_id = 'version/7+opaque';
$retry->delete_operation_id = str_repeat('1', 32);
$retry->delete_generation = 1;
$retry->delete_cycle_attempts = 1;
$retry->delete_presence_confirmations = 1;
$retry->delete_requested_at = time() - 300;
$retry->delete_last_checked_at = time() - 120;
$retry->delete_next_check_at = 0;
$retry->save();
$retryHistory = [];
$retryResult = CloudBackupService::delete($retry, $actor, $client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"stable-etag"',
'x-amz-version-id' => 'version/7+opaque',
]),
new ConnectException('retry response lost',
new Request('DELETE', 'https://objects.example.com')),
], $retryHistory));
$retry->refresh();
$retryUris = array_map(static fn ($entry): string =>
(string) $entry['request']->getUri(), $retryHistory);
$assert($retryResult['requires_reconciliation'] === true
&& (string) $retry->state === 'delete_uncertain'
&& (int) $retry->delete_generation === 2
&& (int) $retry->delete_cycle_attempts === 2
&& count($retryHistory) === 2
&& $retryHistory[0]['request']->getMethod() === 'HEAD'
&& $retryHistory[1]['request']->getMethod() === 'DELETE'
&& str_contains($retryUris[0], 'versionId=version%2F7%2Bopaque')
&& str_contains($retryUris[1], 'versionId=version%2F7%2Bopaque'),
'Stable object identity did not create a fresh, exact-version deletion generation.');
$assert(!in_array('delete_uncertain', $activeStates, true),
'Object cleanup debt still consumes the user retained-backup quota.');
$retry->delete_cycle_attempts = 3;
$retry->delete_generation = 3;
$retry->delete_presence_confirmations = 1;
$retry->delete_last_checked_at = time() - 120;
$retry->delete_next_check_at = 0;
$retry->save();
$exhaustedHistory = [];
$exhausted = CloudBackupService::delete($retry, $actor, $client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"stable-etag"',
'x-amz-version-id' => 'version/7+opaque',
]),
], $exhaustedHistory));
$retry->refresh();
$assert(($exhausted['requires_attention'] ?? false) === true
&& (string) $retry->state === 'delete_attention'
&& count($exhaustedHistory) === 1
&& $exhaustedHistory[0]['request']->getMethod() === 'HEAD',
'Three automatic deletion attempts did not stop at administrator attention.');
$debt = CloudBackupService::cleanupDebt($actor);
$debtItem = array_values(array_filter($debt['items'],
static fn (array $item): bool => $item['id'] === (string) $retry->id))[0] ?? null;
$assert(($debt['summary']['attention_count'] ?? 0) >= 1
&& is_array($debtItem)
&& ($debtItem['can_retry'] ?? false) === true
&& !array_key_exists('object_key', $debtItem)
&& !array_key_exists('object_version_id', $debtItem),
'Cleanup debt was not listed safely for administrators.');
$manualHistory = [];
$manual = CloudBackupService::retryCleanup($retry, $actor, $client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"stable-etag"',
'x-amz-version-id' => 'version/7+opaque',
]),
], $manualHistory));
$retry->refresh();
$assert($manual['requires_reconciliation'] === true
&& (string) $retry->state === 'delete_uncertain'
&& (int) $retry->delete_cycle_attempts === 0
&& (int) $retry->delete_presence_confirmations === 1
&& (int) $retry->delete_next_check_at > time()
&& count($manualHistory) === 1,
'Manual cleanup retry did not begin a new bounded stable-evidence cycle.');
echo "Portable backup deletion, reconciliation, blocker and quota tests passed.\n";
}

@ -40,6 +40,8 @@ $schema->create('cloud_instance_backups',
$table->integer('object_size_bytes')->default(0);
$table->integer('virtual_size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('object_etag')->default('');
$table->string('object_version_id')->default('');
$table->text('spec_json');
$table->text('sealed_storage_runtime');
$table->text('sealed_credential')->nullable();
@ -201,6 +203,8 @@ CloudBackupService::finishJobLocked($createJob, 'succeeded', [
'size_bytes' => 1024,
'virtual_size_bytes' => 20 * 1073741824,
'sha256' => str_repeat('b', 64),
'object_etag' => '"finalizer-etag"',
'object_version_id' => 'finalizer-version',
'consistency' => 'filesystem',
'cleanup_pending' => true,
], [
@ -214,6 +218,8 @@ $assert((string) $createdBackup->state === 'ready'
=== 'backup_local_cleanup_pending'
&& (int) $createdBackup->object_size_bytes === 1024
&& (string) $createdBackup->sha256 === str_repeat('b', 64)
&& (string) $createdBackup->object_etag === '"finalizer-etag"'
&& (string) $createdBackup->object_version_id === 'finalizer-version'
&& $createdBackup->active_job_id === null
&& $createdBackup->ready_at !== null,
'Cleanup-pending backup lost its ready/restorable object identity.');

@ -255,7 +255,11 @@ try {
$deleteHistory = [];
$deleteStack = HandlerStack::create(new MockHandler([
new Response(204),
new Response(200),
new Response(200, [
'Content-Length' => '4096',
'ETag' => '"present-etag"',
'x-amz-version-id' => 'present/version+1',
]),
new Response(404),
]));
$deleteStack->push(Middleware::history($deleteHistory));
@ -263,17 +267,24 @@ $deleteClient = new CloudObjectStorageS3Client(new Client([
'handler' => $deleteStack,
]));
$deleteClient->deleteObject(
$multipartConfig, $multipartCredentials, 'vm/delete.qcow2'
$multipartConfig, $multipartCredentials, 'vm/delete.qcow2',
'delete/version+1'
);
$assert($deleteClient->objectPresence(
$multipartConfig, $multipartCredentials, 'vm/present.qcow2'
$multipartConfig, $multipartCredentials, 'vm/present.qcow2',
'present/version+1'
) === 'present'
&& $deleteClient->objectPresence(
$multipartConfig, $multipartCredentials, 'vm/absent.qcow2'
$multipartConfig, $multipartCredentials, 'vm/absent.qcow2',
'absent/version+1'
) === 'absent'
&& array_map(static fn ($entry): string =>
$entry['request']->getMethod(), $deleteHistory)
=== ['DELETE', 'HEAD', 'HEAD'],
=== ['DELETE', 'HEAD', 'HEAD']
&& str_contains((string) $deleteHistory[0]['request']->getUri(),
'versionId=delete%2Fversion%2B1')
&& str_contains((string) $deleteHistory[1]['request']->getUri(),
'versionId=present%2Fversion%2B1'),
'DeleteObject and read-only deletion reconciliation lost method or status semantics.');
$ambiguousDelete = new CloudObjectStorageS3Client(new Client([

@ -73,7 +73,7 @@ $assert(str_contains($auth, "Config::get('tos_reg') === 'true'")
'Legal consent is only client-side and can be bypassed.');
$assets = $read('web/resources/views/venix/layout_assets.tpl');
$assert(str_contains($assets, "value='2026102801'")
$assert(str_contains($assets, "value='2026102802'")
&& str_contains($admin, '?v={$venix_asset_version}')
&& str_contains($read('web/resources/views/venix/home/layout_header.tpl'), '?v={$venix_asset_version}'),
'Shared browser asset version was not coupled to the new assets.');

@ -80,7 +80,7 @@ $assert(preg_match('/\.landing-site-header\s*\{[^}]*position:\s*fixed;/s', $css)
&& str_contains($accountCss, 'padding-top: 88px;')
&& str_contains($accountCss, 'padding-top: 76px;'),
'Persistent desktop or mobile public-navigation styling is incomplete.');
$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102801'"),
$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102802'"),
'The shared browser asset version was not advanced with the navigation styles.');
echo "Public page locale and shared-layout contract tests passed.\n";

@ -69,7 +69,7 @@ foreach ($templates as $template) {
}
$assets = $read('web/resources/views/venix/layout_assets.tpl');
$assert(str_contains($assets, "value='2026102801'"),
$assert(str_contains($assets, "value='2026102802'"),
'The shared asset version was not updated for the canonical logo.');
echo "Shared site-brand contract passed across public, auth, user, cloud and admin layouts.\n";

Loading…
Cancel
Save