feat: isolate portable backup workspaces

go
codex 3 weeks ago
parent 26c654d188
commit 52f4a61fe5

@ -1076,6 +1076,37 @@ func peerMigrationTargetCleanupInspectionMatches(prior operation.Record, planID
return true
}
func backupWorkspaceCleanupInspectionMatches(prior operation.Record, planID string,
inspection hypervisor.OperationInspection) bool {
if inspection.OperationID != prior.OperationID || inspection.Action != prior.Action ||
inspection.ResourceID != prior.ResourceID || inspection.RequestFingerprint != prior.Fingerprint ||
inspection.OperationStatus != prior.State || inspection.CheckedAt <= 0 ||
inspection.Outcome != hypervisor.InspectionOutcomeSucceeded ||
inspection.EffectResolution != hypervisor.InspectionEffectApplied ||
inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" ||
inspection.AbortJob.Available {
return false
}
if !inspection.Cleanup.Available {
return planID == "" &&
inspection.SummaryCode == hypervisor.InspectionSummarySucceeded
}
if inspection.SummaryCode != hypervisor.InspectionSummaryCleanupRequired ||
planID == "" || !strings.EqualFold(inspection.Cleanup.PlanID, planID) ||
len(inspection.Cleanup.Steps) == 0 {
return false
}
for _, step := range inspection.Cleanup.Steps {
switch step {
case hypervisor.CleanupStepDeleteSnapshotVolume,
hypervisor.CleanupStepRemoveSnapshotMetadata:
default:
return false
}
}
return true
}
func (s *Server) routeSnapshots(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) {
if len(segments) == 4 {
switch r.Method {
@ -1527,6 +1558,99 @@ func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segment
writeJSON(w, http.StatusOK, inspection)
return
}
if len(segments) == 4 && segments[3] == "cleanup-backup-workspace" &&
r.Method == http.MethodPost {
if !requireSuperAdminPrivilegedOperation(w, actor) {
return
}
if actor.OperationID == "" || actor.OperationID != operationID {
writePreDispatchError(w, http.StatusBadRequest, "operation_id_mismatch", "actor operation_id must match the cleanup operation")
return
}
request, ok := decodeRequired[hypervisor.CleanupBackupWorkspaceRequest](w, body)
if !ok {
return
}
if err := hypervisor.ValidateCleanupBackupWorkspaceRequest(request); err != nil {
writePreDispatchMappedError(w, err)
return
}
action := ""
existingReplay := false
trustedPrior, trustedPriorFound := trustedSupersededOperation(
r.Context(), request.PriorOperationID,
)
existing, existingErr := s.Journal.Get(operationID)
if existingErr == nil {
action = existing.Action
existingReplay = strings.HasPrefix(action, "cleanup-backup-workspace:") &&
existing.SupersedesOperationID == request.PriorOperationID
} else if !errors.Is(existingErr, operation.ErrNotFound) {
writeMappedError(w, existingErr, nil)
return
}
prior := trustedPrior
if !existingReplay {
var priorErr error
if !trustedPriorFound {
prior, priorErr = s.Journal.Get(request.PriorOperationID)
}
parts := strings.Split(prior.Action, ":")
if priorErr != nil || len(parts) != 3 || parts[0] != "create-backup" ||
!catalogIDPattern.MatchString(parts[1]) ||
prior.State != operation.StateSucceeded ||
prior.ResourceID == "" || prior.ExecutionFenced {
writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup requires the exact successful backup operation")
return
}
action = "cleanup-backup-workspace:" + parts[1]
if !trustedPriorFound {
_, incidentFound, incidentErr := s.Journal.CompletionDeliveryIncident(prior.OperationID)
if incidentErr != nil {
writeMappedError(w, incidentErr, nil)
return
}
if incidentFound {
writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup is blocked by a completion-delivery incident")
return
}
}
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, prior)
cancel()
if inspectErr != nil {
writeMappedError(w, inspectErr, nil)
return
}
if !backupWorkspaceCleanupInspectionMatches(
prior, request.CleanupPlanID, inspection,
) {
writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_plan_changed", "cleanup plan is stale or no longer safe")
return
}
request.PriorOperation = prior
}
parts := strings.Split(action, ":")
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) {
writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup action identity is invalid")
return
}
resourceID := prior.ResourceID
if existingReplay {
resourceID = existing.ResourceID
}
supersedes := []string{request.PriorOperationID}
if trustedPriorFound {
supersedes = nil
}
s.runMutationExpected(w, r, body, actor, operationID, resourceID,
action, nil, true, http.StatusOK,
hypervisor.CleanupBackupWorkspaceAdmission(resourceID, parts[1]),
func(ctx context.Context) (any, error) {
return s.Hypervisor.CleanupBackupWorkspace(ctx, resourceID, request)
}, supersedes...)
return
}
if len(segments) == 4 && segments[3] == "fence-resolution" && r.Method == http.MethodGet {
if !requireSuperAdminInspection(w, actor, operationID) {
return

@ -33,6 +33,8 @@ type testHypervisor struct {
configureCalls int
cleanupPeerTargetCalls int
cleanupPeerTargetLast hypervisor.CleanupPeerMigrationTargetRequest
cleanupBackupCalls int
cleanupBackupLast hypervisor.CleanupBackupWorkspaceRequest
inspectionResult *hypervisor.OperationInspection
inspectionErr error
}
@ -61,6 +63,22 @@ func (h *testHypervisor) CleanupPeerMigrationTarget(_ context.Context, _ string,
}, nil
}
func (h *testHypervisor) CleanupBackupWorkspace(_ context.Context, _ string,
request hypervisor.CleanupBackupWorkspaceRequest) (hypervisor.CleanupBackupWorkspaceResult, error) {
h.mu.Lock()
h.cleanupBackupCalls++
h.cleanupBackupLast = request
h.mu.Unlock()
parts := strings.Split(request.PriorOperation.Action, ":")
key := ""
if len(parts) == 3 {
key = parts[1]
}
return hypervisor.CleanupBackupWorkspaceResult{
BackupKey: key, Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true,
}, nil
}
func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) {
h.mu.Lock()
defer h.mu.Unlock()
@ -1141,7 +1159,7 @@ func TestCreateBackupUsesDedicatedAsyncActionAndHeavyAdmission(t *testing.T) {
for _, want := range []string{
"domain:" + uuid,
"volume:virt-cloud-managed/vm-" + uuid,
"volume:virt-cloud-managed/vsnap-123e4567e89b42d3a456426614174000-" + backupKey + ".qcow2",
"volume:virt-cloud-managed/vbackup-123e4567e89b42d3a456426614174000-" + backupKey + ".qcow2",
"catalog:backup/" + backupKey,
} {
if !containsTestString(record.LockKeys, want) {
@ -1162,6 +1180,95 @@ func TestCreateBackupUsesDedicatedAsyncActionAndHeavyAdmission(t *testing.T) {
}
}
func TestBackupWorkspaceCleanupRequiresSuccessfulPriorAndFreshExactPlan(t *testing.T) {
api := newTestAPI(t)
const (
uuid = "123e4567-e89b-42d3-a456-426614174000"
backupKey = "0123456789abcdef0123456789abcdef"
priorID = "operation-backup-cleanup-pending"
cleanupID = "operation-backup-workspace-cleanup"
)
priorSpec := operation.Spec{
OperationID: priorID, ResourceID: uuid,
Action: "create-backup:" + backupKey + ":filesystem",
Fingerprint: operation.Fingerprint(http.MethodPost,
"/v1/vms/"+uuid+"/backups", []byte(`{}`)),
LockKeys: hypervisor.CreateBackupAdmission(uuid, hypervisor.BackupRequest{
BackupKey: backupKey,
}).LockKeys,
HeavyIO: true,
}
if _, _, err := api.journal.Execute(context.Background(), priorSpec,
func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"backup_key":"` + backupKey + `","size_bytes":1024,"virtual_size_bytes":4096,"sha256":"` + strings.Repeat("a", 64) + `","consistency":"crash","cleanup_pending":true}`), nil
}); err != nil {
t.Fatal(err)
}
prior, err := api.journal.Get(priorID)
if err != nil {
t.Fatal(err)
}
planID := strings.Repeat("b", 64)
api.hv.inspectionResult = &hypervisor.OperationInspection{
OperationID: prior.OperationID, Action: prior.Action,
ResourceID: prior.ResourceID, RequestFingerprint: prior.Fingerprint,
OperationStatus: prior.State, CheckedAt: api.now.Unix(),
Outcome: hypervisor.InspectionOutcomeSucceeded,
SummaryCode: hypervisor.InspectionSummaryCleanupRequired,
EffectResolution: hypervisor.InspectionEffectApplied,
EffectProjection: append(json.RawMessage(nil), prior.Result...),
Cleanup: hypervisor.OperationCleanupPlan{
Available: true, PlanID: planID,
Steps: []string{hypervisor.CleanupStepDeleteSnapshotVolume},
},
}
executor := &fakeMutationExecutor{result: json.RawMessage(`{"backup_key":"` + backupKey + `","cleaned":true,"workspace_absent":true,"metadata_absent":true}`)}
if err := api.handler.SetMutationExecutor(executor); err != nil {
t.Fatal(err)
}
request := hypervisor.CleanupBackupWorkspaceRequest{
OperationID: cleanupID, PriorOperationID: priorID, CleanupPlanID: planID,
}
body, _ := json.Marshal(request)
actor := auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: cleanupID,
}
response := api.request(t, http.MethodPost,
"/v1/operations/"+cleanupID+"/cleanup-backup-workspace", body, actor)
if response.Code != http.StatusAccepted {
t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String())
}
terminal := waitForTerminal(t, api.journal, cleanupID)
if terminal.State != operation.StateSucceeded ||
terminal.SupersedesOperationID != priorID ||
terminal.Action != "cleanup-backup-workspace:"+backupKey ||
terminal.ResourceID != uuid {
t.Fatalf("cleanup terminal = %#v", terminal)
}
executor.mu.Lock()
if len(executor.requests) != 1 || executor.requests[0].SupersededOperation == nil ||
executor.requests[0].SupersededOperation.OperationID != priorID {
executor.mu.Unlock()
t.Fatalf("cleanup worker predecessor = %#v", executor.requests)
}
executor.mu.Unlock()
stale := request
stale.OperationID = "operation-backup-workspace-stale"
stale.CleanupPlanID = strings.Repeat("c", 64)
staleBody, _ := json.Marshal(stale)
staleActor := actor
staleActor.OperationID = stale.OperationID
denied := api.request(t, http.MethodPost,
"/v1/operations/"+stale.OperationID+"/cleanup-backup-workspace",
staleBody, staleActor)
if denied.Code != http.StatusConflict ||
!bytes.Contains(denied.Body.Bytes(), []byte(`"code":"backup_workspace_cleanup_plan_changed"`)) {
t.Fatalf("stale cleanup = %d %s", denied.Code, denied.Body.String())
}
}
func TestRecoverBackupUsesDestructiveDiskTransitionAdmission(t *testing.T) {
api := newTestAPI(t)
const (

@ -111,12 +111,24 @@ func CreateBackupAdmission(uuid string, request BackupRequest) MutationAdmission
uuid = strings.ToLower(uuid)
locks := []string{
domainLockKey(uuid), vmManagedVolumesLockKey(uuid),
volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, request.BackupKey)),
volumeLockKey(managedStoragePoolName, backupWorkspaceVolumeName(uuid, request.BackupKey)),
catalogLockKey("backup", request.BackupKey),
}
return MutationAdmission{HeavyIO: true, LockKeys: locks}
}
func CleanupBackupWorkspaceAdmission(uuid, backupKey string) MutationAdmission {
uuid = strings.ToLower(uuid)
return MutationAdmission{HeavyIO: true, LockKeys: []string{
domainLockKey(uuid), vmManagedVolumesLockKey(uuid),
volumeLockKey(managedStoragePoolName, backupWorkspaceVolumeName(uuid, backupKey)),
// Legacy backups created before the workspace split used the snapshot
// namespace. Keep that one exact lock until their residue is gone.
volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, backupKey)),
catalogLockKey("backup", backupKey),
}}
}
func RestoreBackupAdmission(request RestoreBackupRequest) MutationAdmission {
uuid := strings.ToLower(request.UUID)
locks := []string{
@ -328,6 +340,11 @@ func snapshotVolumeName(uuid, snapshotKey string) string {
return "vsnap-" + compact + "-" + snapshotKey + ".qcow2"
}
func backupWorkspaceVolumeName(uuid, backupKey string) string {
compact := strings.ReplaceAll(strings.ToLower(uuid), "-", "")
return "vbackup-" + compact + "-" + backupKey + ".qcow2"
}
func snapshotRestoreVolumeName(uuid, operationID string) string {
sum := sha256.Sum256([]byte(operationID))
compact := strings.ReplaceAll(strings.ToLower(uuid), "-", "")

@ -32,6 +32,20 @@ func ValidateBackupRequest(request BackupRequest) error {
return nil
}
func ValidateCleanupBackupWorkspaceRequest(request CleanupBackupWorkspaceRequest) error {
if !operationIdentifierPattern.MatchString(request.OperationID) ||
!operationIdentifierPattern.MatchString(request.PriorOperationID) ||
request.OperationID == request.PriorOperationID ||
(request.CleanupPlanID != "" &&
!sha256Pattern.MatchString(strings.ToLower(request.CleanupPlanID))) {
return NewError(
ErrInvalid, "invalid_backup_workspace_cleanup",
"backup workspace cleanup identity is invalid", nil,
)
}
return nil
}
func ValidateRestoreBackupRequest(request RestoreBackupRequest) error {
if !operationIdentifierPattern.MatchString(request.OperationID) ||
!catalogIDPattern.MatchString(request.BackupKey) ||

@ -97,6 +97,27 @@ func TestValidateBackupRequestChecksCapabilityBeforeDispatch(t *testing.T) {
}
}
func TestBackupWorkspaceIdentityAndCleanupRequestAreBound(t *testing.T) {
const uuid = "123e4567-e89b-42d3-a456-426614174000"
name := backupWorkspaceVolumeName(uuid, testBackupKey)
if name == snapshotVolumeName(uuid, testBackupKey) ||
!isVMOwnedManagedVolumeName(uuid, name) {
t.Fatalf("backup workspace identity was not isolated: %q", name)
}
request := CleanupBackupWorkspaceRequest{
OperationID: "operation-backup-cleanup-2",
PriorOperationID: "operation-backup-create-1",
CleanupPlanID: strings.Repeat("a", 64),
}
if err := ValidateCleanupBackupWorkspaceRequest(request); err != nil {
t.Fatalf("valid workspace cleanup rejected: %v", err)
}
request.OperationID = request.PriorOperationID
if err := ValidateCleanupBackupWorkspaceRequest(request); err == nil {
t.Fatal("cleanup accepted the original operation id as its successor")
}
}
func TestValidateRestoreBackupReusesCreateIntentWithoutReplacingCredentials(t *testing.T) {
request := validRestoreBackupRequest(t)
if err := ValidateRestoreBackupRequest(request); err != nil {
@ -139,13 +160,26 @@ func TestBackupAdmissionsFenceSourceAndNewVMArtifacts(t *testing.T) {
backup := CreateBackupAdmission(uuid, backupRequest)
for _, key := range []string{
domainLockKey(uuid), vmManagedVolumesLockKey(uuid),
volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, backupRequest.BackupKey)),
volumeLockKey(managedStoragePoolName, backupWorkspaceVolumeName(uuid, backupRequest.BackupKey)),
catalogLockKey("backup", backupRequest.BackupKey),
} {
if !containsString(backup.LockKeys, key) || !backup.HeavyIO {
t.Fatalf("backup admission missing %q: %#v", key, backup)
}
}
cleanup := CleanupBackupWorkspaceAdmission(uuid, backupRequest.BackupKey)
for _, key := range []string{
domainLockKey(uuid), vmManagedVolumesLockKey(uuid),
volumeLockKey(managedStoragePoolName,
backupWorkspaceVolumeName(uuid, backupRequest.BackupKey)),
volumeLockKey(managedStoragePoolName,
snapshotVolumeName(uuid, backupRequest.BackupKey)),
catalogLockKey("backup", backupRequest.BackupKey),
} {
if !containsString(cleanup.LockKeys, key) || !cleanup.HeavyIO {
t.Fatalf("cleanup admission missing %q: %#v", key, cleanup)
}
}
restore := RestoreBackupAdmission(validRestoreBackupRequest(t))
for _, key := range []string{
domainLockKey(uuid), vmManagedVolumesLockKey(uuid),

@ -64,12 +64,17 @@ func isVMOwnedManagedVolumeName(uuid, name string) bool {
return false
}
compactUUID := strings.ReplaceAll(strings.ToLower(uuid), "-", "")
prefix := "vsnap-" + compactUUID + "-"
if !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, ".qcow2") {
return false
for _, prefix := range []string{
"vsnap-" + compactUUID + "-",
"vbackup-" + compactUUID + "-",
} {
if !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, ".qcow2") {
continue
}
key := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".qcow2")
return catalogIDPattern.MatchString(key)
}
key := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".qcow2")
return catalogIDPattern.MatchString(key)
return false
}
func BuildDomainXML(definition DomainDefinition) (string, error) {

@ -40,16 +40,16 @@ func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request
}
var snapshot Snapshot
if active {
snapshot, err = l.CreateSnapshot(ctx, uuid, snapshotRequest)
snapshot, err = l.createBackupWorkspace(ctx, uuid, snapshotRequest)
} else {
snapshot, err = l.createStoppedBackupSnapshot(ctx, uuid, snapshotRequest)
snapshot, err = l.createStoppedBackupWorkspace(ctx, uuid, snapshotRequest)
}
if err != nil {
return BackupResult{}, err
}
volume, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, request.BackupKey),
l.storagePoolName(StorageRoleManaged), backupWorkspaceVolumeName(uuid, request.BackupKey),
)
if err != nil || volume == nil {
return BackupResult{}, operation.MarkUncertain(NewError(
@ -78,7 +78,7 @@ func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request
return BackupResult{}, translated
}
cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx)
_, cleanupErr := l.DeleteSnapshot(cleanupCtx, uuid, request.BackupKey, SnapshotMutationRequest{OperationID: request.OperationID})
cleanupErr := l.cleanupBackupWorkspace(cleanupCtx, uuid, request.BackupKey)
cleanupCancel()
if cleanupErr != nil {
return BackupResult{}, classifyBackupUploadFailure(translated, cleanupErr)
@ -97,7 +97,7 @@ func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request
Consistency: snapshot.Consistency,
}
cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx)
_, cleanupErr := l.DeleteSnapshot(cleanupCtx, uuid, request.BackupKey, SnapshotMutationRequest{OperationID: request.OperationID})
cleanupErr := l.cleanupBackupWorkspace(cleanupCtx, uuid, request.BackupKey)
cleanupCancel()
if cleanupErr != nil {
// The immutable remote object is already fully verified. Local temporary
@ -276,15 +276,22 @@ func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey stri
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
return false, err
}
existing, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, backupKey),
)
if err != nil {
return false, err
}
if existing != nil {
existing.Free()
return false, NewError(ErrConflict, "backup_exists", "local backup work already exists", nil)
for _, volumeName := range []string{
backupWorkspaceVolumeName(uuid, backupKey),
// Reject a legacy residue with the same key instead of silently
// allocating a second workspace for one logical backup.
snapshotVolumeName(uuid, backupKey),
} {
existing, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), volumeName,
)
if err != nil {
return false, err
}
if existing != nil {
existing.Free()
return false, NewError(ErrConflict, "backup_exists", "local backup work already exists", nil)
}
}
active, err := domain.IsActive()
if err != nil {
@ -400,13 +407,13 @@ func validateStandaloneQCOW2Volume(volume qcow2VolumeInspector, expectedCapacity
return info, nil
}
func (l *Libvirt) createStoppedBackupSnapshot(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) {
func (l *Libvirt) createStoppedBackupWorkspace(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return Snapshot{}, err
}
defer domain.Free()
document, definition, metadata, pool, source, err := l.managedRootContext(domain)
_, definition, metadata, pool, source, err := l.managedRootContext(domain)
if err != nil {
return Snapshot{}, err
}
@ -436,7 +443,7 @@ func (l *Libvirt) createStoppedBackupSnapshot(ctx context.Context, uuid string,
if sourceInfo.Capacity > poolInfo.Available {
return Snapshot{}, NewError(ErrConflict, "backup_storage_full", "managed storage pool does not have enough free space", nil)
}
volumeName := snapshotVolumeName(uuid, request.SnapshotKey)
volumeName := backupWorkspaceVolumeName(uuid, request.SnapshotKey)
volumeXML, err := storageVolumeXMLAllocated(volumeName, sourceInfo.Capacity, "qcow2")
if err != nil {
return Snapshot{}, err
@ -472,24 +479,6 @@ func (l *Libvirt) createStoppedBackupSnapshot(ctx context.Context, uuid string,
// previous shutdown was guest-coordinated, so it reports crash safety.
actualConsistency = "crash"
}
updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
stored.Snapshots = append(stored.Snapshots, domainSnapshotMetadataXML{
Key: request.SnapshotKey, Name: request.Name,
Consistency: actualConsistency, SizeBytes: info.Capacity, CreatedAt: created.Unix(),
})
return nil
})
if err != nil {
return Snapshot{}, err
}
defined, err := l.conn.DomainDefineXML(updated)
if err != nil {
// DomainDefineXML may have committed before the reply failed; retain the
// volume because metadata could now reference it.
targetOwned = false
return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("persist offline backup metadata", err))
}
defined.Free()
targetOwned = false
return Snapshot{
ID: request.SnapshotKey, SnapshotKey: request.SnapshotKey,

@ -0,0 +1,308 @@
//go:build libvirt
package hypervisor
import (
"context"
"errors"
"strings"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
)
// backupWorkspaceResidue resolves exactly one operation-owned portable-backup
// copy. New backups use vbackup-* without domain snapshot metadata. The
// legacy vsnap-* form is accepted only so an evidence-bound cleanup can retire
// residue produced before the workspace split.
func (l *Libvirt) backupWorkspaceResidue(uuid, key string) (
name string, volume *libvirt.StorageVol, legacy bool, err error,
) {
workspaceName := backupWorkspaceVolumeName(uuid, key)
workspace, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), workspaceName,
)
if err != nil {
return "", nil, false, err
}
legacyName := snapshotVolumeName(uuid, key)
legacyVolume, legacyErr := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), legacyName,
)
if legacyErr != nil {
if workspace != nil {
workspace.Free()
}
return "", nil, false, legacyErr
}
if workspace != nil && legacyVolume != nil {
workspace.Free()
legacyVolume.Free()
return "", nil, false, NewError(
ErrConflict, "backup_workspace_ambiguous",
"both current and legacy backup workspaces are present", nil,
)
}
if workspace != nil {
return workspaceName, workspace, false, nil
}
if legacyVolume != nil {
return legacyName, legacyVolume, true, nil
}
return workspaceName, nil, false, nil
}
// cleanupBackupWorkspace is the bounded fast-path used immediately after an
// upload. It never mutates a user snapshot: current workspaces have no
// snapshot metadata, while a legacy workspace must match its exact metadata
// before the existing snapshot deletion contract is reused.
func (l *Libvirt) cleanupBackupWorkspace(ctx context.Context, uuid, key string) error {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return err
}
defer domain.Free()
_, metadata, err := requireManagedDomain(domain)
if err != nil {
return err
}
metadataFound := false
for _, stored := range metadata.Snapshots {
if stored.Key == key {
metadataFound = true
break
}
}
name, volume, legacy, err := l.backupWorkspaceResidue(uuid, key)
if err != nil {
return err
}
if volume == nil {
if metadataFound {
return NewError(
ErrConflict, "backup_workspace_inconsistent",
"backup workspace metadata remains without its volume", nil,
)
}
return nil
}
volume.Free()
if legacy {
if !metadataFound {
return NewError(
ErrConflict, "backup_workspace_inconsistent",
"legacy backup workspace is missing its metadata", nil,
)
}
_, err := l.DeleteSnapshot(ctx, uuid, key, SnapshotMutationRequest{})
return err
}
if metadataFound {
return NewError(
ErrConflict, "backup_workspace_inconsistent",
"current backup workspace collides with user snapshot metadata", nil,
)
}
if name != backupWorkspaceVolumeName(uuid, key) {
return NewError(
ErrConflict, "backup_workspace_identity_invalid",
"backup workspace identity is invalid", nil,
)
}
return l.deleteUnreferencedBackupVolume(domain, uuid, key, name)
}
func (l *Libvirt) deleteUnreferencedBackupVolume(domain *libvirt.Domain,
uuid, key, volumeName string) error {
if !catalogIDPattern.MatchString(key) {
return operation.MarkNotApplied(NewError(
ErrConflict, "backup_workspace_identity_invalid",
"backup workspace identity is invalid", nil,
))
}
if volumeName != backupWorkspaceVolumeName(uuid, key) &&
volumeName != snapshotVolumeName(uuid, key) {
return operation.MarkNotApplied(NewError(
ErrConflict, "backup_workspace_identity_invalid",
"backup workspace identity is invalid", nil,
))
}
if domain != nil {
if err := ensureNoActiveDomainJobIfRunning(domain); err != nil {
return operation.MarkNotApplied(err)
}
}
volume, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), volumeName,
)
if err != nil {
return operation.MarkNotApplied(err)
}
if volume == nil {
return nil
}
defer volume.Free()
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
return operation.MarkNotApplied(err)
}
path, err := volume.GetPath()
if err != nil {
return operation.MarkNotApplied(
translateLibvirtError("resolve backup workspace path", err),
)
}
referenced, err := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), volumeName,
)
if err != nil {
return operation.MarkNotApplied(err)
}
if referenced {
return operation.MarkNotApplied(NewError(
ErrConflict, "backup_workspace_in_use",
"backup workspace is referenced by a domain", nil,
))
}
if err := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), volumeName,
)
if observed != nil {
observed.Free()
}
if inspectErr != nil || observed != nil {
return operation.MarkUncertain(errors.Join(
mutationLibvirtError("delete backup workspace", err), inspectErr,
))
}
}
return nil
}
// CleanupBackupWorkspace executes one fresh evidence-bound plan. It is a
// successor operation: the original successful backup receipt remains
// immutable and continues to prove the remote object identity.
func (l *Libvirt) CleanupBackupWorkspace(ctx context.Context, uuid string,
request CleanupBackupWorkspaceRequest) (CleanupBackupWorkspaceResult, error) {
if err := ValidateCleanupBackupWorkspaceRequest(request); err != nil {
return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(err)
}
prior := request.PriorOperation
parts := strings.Split(prior.Action, ":")
if prior.OperationID != request.PriorOperationID ||
!strings.EqualFold(prior.ResourceID, uuid) || len(parts) != 3 ||
parts[0] != "create-backup" || !catalogIDPattern.MatchString(parts[1]) ||
!successfulBackupCleanupPending(prior) {
return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "backup_workspace_cleanup_not_authorized",
"backup workspace cleanup requires the exact successful backup operation", nil,
))
}
key := parts[1]
domain, err := l.domainByUUID(ctx, uuid)
domainAbsent := errors.Is(err, ErrNotFound)
if err != nil && !domainAbsent {
return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(err)
}
if domain != nil {
defer domain.Free()
}
inspection := l.inspectSuccessfulBackupCleanupPending(ctx, prior)
if inspection.Cleanup.Available {
if request.CleanupPlanID == "" || !hashEqualsFold(
request.CleanupPlanID, inspection.Cleanup.PlanID,
) {
return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "backup_workspace_cleanup_plan_changed",
"backup workspace cleanup plan changed; inspect again", nil,
))
}
} else if request.CleanupPlanID != "" {
return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "backup_workspace_cleanup_plan_changed",
"backup workspace cleanup plan is no longer present", nil,
))
}
metadataFound := false
if !domainAbsent {
_, metadata, metadataErr := requireManagedDomain(domain)
if metadataErr != nil {
return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(metadataErr)
}
for _, stored := range metadata.Snapshots {
if stored.Key == key {
metadataFound = true
break
}
}
}
volumeName, volume, legacy, err := l.backupWorkspaceResidue(uuid, key)
if err != nil {
return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(err)
}
volumePresent := volume != nil
if volume != nil {
volume.Free()
}
if !volumePresent && !metadataFound {
return CleanupBackupWorkspaceResult{
BackupKey: key, Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true,
}, nil
}
if !inspection.Cleanup.Available {
return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "backup_workspace_cleanup_unavailable",
"backup workspace is not currently safe to clean", nil,
))
}
if metadataFound {
if volumePresent && !legacy {
return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "backup_workspace_inconsistent",
"current backup workspace collides with user snapshot metadata", nil,
))
}
if _, err := l.DeleteSnapshot(
ctx, uuid, key, SnapshotMutationRequest{OperationID: request.OperationID},
); err != nil {
return CleanupBackupWorkspaceResult{}, err
}
} else if volumePresent {
if err := l.deleteUnreferencedBackupVolume(
domain, uuid, key, volumeName,
); err != nil {
return CleanupBackupWorkspaceResult{}, err
}
}
_, remaining, _, err := l.backupWorkspaceResidue(uuid, key)
if err != nil {
return CleanupBackupWorkspaceResult{}, operation.MarkUncertain(err)
}
if remaining != nil {
remaining.Free()
return CleanupBackupWorkspaceResult{}, operation.MarkUncertain(NewError(
ErrConflict, "backup_workspace_cleanup_pending",
"backup workspace remains after cleanup", nil,
))
}
if !domainAbsent {
_, current, metadataErr := requireManagedDomain(domain)
if metadataErr != nil {
return CleanupBackupWorkspaceResult{}, operation.MarkUncertain(metadataErr)
}
for _, stored := range current.Snapshots {
if stored.Key == key {
return CleanupBackupWorkspaceResult{}, operation.MarkUncertain(NewError(
ErrConflict, "backup_workspace_cleanup_pending",
"legacy backup metadata remains after cleanup", nil,
))
}
}
}
return CleanupBackupWorkspaceResult{
BackupKey: key, Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true,
}, nil
}

@ -560,6 +560,12 @@ func (m *managedLibvirt) CreateBackup(ctx context.Context, uuid string, request
})
}
func (m *managedLibvirt) CleanupBackupWorkspace(ctx context.Context, uuid string, request CleanupBackupWorkspaceRequest) (CleanupBackupWorkspaceResult, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (CleanupBackupWorkspaceResult, error) {
return h.CleanupBackupWorkspace(ctx, uuid, request)
})
}
func (m *managedLibvirt) RestoreBackup(ctx context.Context, request RestoreBackupRequest) (RestoreBackupResult, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (RestoreBackupResult, error) {
return h.RestoreBackup(ctx, request)

@ -136,12 +136,31 @@ func storedOperationInspectionAvailable(record operation.Record) bool {
// inspectSuccessfulBackupCleanupPending preserves the immutable applied
// object projection while adding an optional, read-only cleanup plan for an
// exact operation-owned local snapshot residue. Host read failures never
// exact operation-owned local workspace residue. Host read failures never
// downgrade the already verified remote object identity.
func (l *Libvirt) inspectSuccessfulBackupCleanupPending(ctx context.Context,
record operation.Record) OperationInspection {
inspection := inspectionFromRecord(record)
domain, err := l.domainByUUID(ctx, record.ResourceID)
if errors.Is(err, ErrNotFound) {
parts := strings.Split(record.Action, ":")
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) {
return inspection
}
volumeName, volume, _, residueErr := l.backupWorkspaceResidue(
record.ResourceID, parts[1],
)
if residueErr != nil || volume == nil {
return inspection
}
defer volume.Free()
cleanup := l.absentDomainBackupWorkspaceCleanupInspection(
volume, volumeName,
)
return appliedBackupInspectionWithCleanup(
record, inspection, cleanup,
)
}
if err != nil {
return inspection
}
@ -193,6 +212,8 @@ func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record opera
}
cleanup = plan
resolution = inspectAbsentDeleteResolution(plan, record.Action == "force-delete-vm")
case strings.HasPrefix(record.Action, "cleanup-backup-workspace:"):
resolution = l.reconcileAbsentDomainBackupWorkspaceCleanup(record)
default:
if record.ExpectedState != nil {
resolution = expectedStateResourceAbsent(record)
@ -201,6 +222,39 @@ func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record opera
return buildOperationInspection(record, resolution, OperationAbortInspection{}, cleanup), nil
}
func (l *Libvirt) reconcileAbsentDomainBackupWorkspaceCleanup(
record operation.Record,
) operation.Resolution {
parts := strings.Split(record.Action, ":")
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) {
return operation.Resolution{}
}
_, volume, _, err := l.backupWorkspaceResidue(
record.ResourceID, parts[1],
)
if err != nil {
return operation.Resolution{}
}
if volume == nil {
result, _ := json.Marshal(CleanupBackupWorkspaceResult{
BackupKey: parts[1], Cleaned: true,
WorkspaceAbsent: true, MetadataAbsent: true,
})
return operation.Resolution{
Resolved: true, Succeeded: true, Result: result,
}
}
defer volume.Free()
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
return operation.Resolution{}
}
return operation.Resolution{
Resolved: true, Succeeded: false,
ErrorCode: "backup_workspace_cleanup_not_applied",
ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied",
}
}
func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePresent bool) (operation.Resolution, operationCleanupInspection) {
if !managedArtifactsAbsent {
return operation.Resolution{}, operationCleanupInspection{}
@ -265,6 +319,7 @@ func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record ope
}
if strings.HasPrefix(record.Action, "create-snapshot:") ||
strings.HasPrefix(record.Action, "create-backup:") ||
strings.HasPrefix(record.Action, "cleanup-backup-workspace:") ||
strings.HasPrefix(record.Action, "delete-snapshot:") ||
strings.HasPrefix(record.Action, "restore-snapshot:") ||
strings.HasPrefix(record.Action, "recover-backup:") {
@ -843,9 +898,18 @@ func (l *Libvirt) inspectSnapshotOperation(ctx context.Context, record operation
break
}
}
volume, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, key),
)
volumeName := snapshotVolumeName(record.ResourceID, key)
var volume *libvirt.StorageVol
legacyBackupVolume := false
if parts[0] == "create-backup" || parts[0] == "cleanup-backup-workspace" {
volumeName, volume, legacyBackupVolume, err = l.backupWorkspaceResidue(
record.ResourceID, key,
)
} else {
volume, err = l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), volumeName,
)
}
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
@ -856,12 +920,42 @@ func (l *Libvirt) inspectSnapshotOperation(ctx context.Context, record operation
switch parts[0] {
case "create-backup":
if stored != nil && volume != nil &&
volumeName != snapshotVolumeName(record.ResourceID, key) {
return operation.Resolution{}, operationCleanupInspection{}
}
if !backupResidueCleanupAllowed(record, stored) {
return operation.Resolution{}, operationCleanupInspection{}
}
return operation.Resolution{}, l.backupResidueCleanupInspection(
domain, volume, snapshotVolumeName(record.ResourceID, key), key, stored,
domain, volume, volumeName, key, stored,
)
case "cleanup-backup-workspace":
if len(parts) != 2 {
return operation.Resolution{}, operationCleanupInspection{}
}
if stored != nil && volume != nil &&
volumeName != snapshotVolumeName(record.ResourceID, key) {
return operation.Resolution{}, operationCleanupInspection{}
}
if stored == nil && !volumeExists {
result, _ := json.Marshal(CleanupBackupWorkspaceResult{
BackupKey: key, Cleaned: true,
WorkspaceAbsent: true, MetadataAbsent: true,
})
return operation.Resolution{
Resolved: true, Succeeded: true, Result: result,
}, operationCleanupInspection{}
}
if (stored == nil && volumeExists && !legacyBackupVolume) ||
(stored != nil && volumeExists && legacyBackupVolume) {
return operation.Resolution{
Resolved: true, Succeeded: false,
ErrorCode: "backup_workspace_cleanup_not_applied",
ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied",
}, operationCleanupInspection{}
}
return operation.Resolution{}, operationCleanupInspection{}
case "create-snapshot":
if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") {
return operation.Resolution{}, operationCleanupInspection{}
@ -1093,6 +1187,35 @@ func (l *Libvirt) backupResidueCleanupInspection(domain cleanupJobDomain,
)
}
func (l *Libvirt) absentDomainBackupWorkspaceCleanupInspection(
volume *libvirt.StorageVol, volumeName string,
) operationCleanupInspection {
if volume == nil {
return operationCleanupInspection{}
}
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
return operationCleanupInspection{}
}
digest, err := storageVolumeInspectionDigest(volume)
if err != nil {
return operationCleanupInspection{}
}
path, err := volume.GetPath()
if err != nil {
return operationCleanupInspection{}
}
referenced, err := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), volumeName,
)
if err != nil || referenced {
return operationCleanupInspection{}
}
return newOperationCleanupInspection(
[]string{CleanupStepDeleteSnapshotVolume},
"domain-state:absent", "backup-volume:"+volumeName+":"+digest,
)
}
func diskTransitionPowerStateMatches(metadata *domainMetadataXML, state libvirt.DomainState,
kind, operationID string) bool {
if !diskTransitionMarkerMatches(metadata, kind, operationID) {
@ -1299,9 +1422,7 @@ func (l *Libvirt) inspectMatchingBackupJob(record operation.Record, domain *libv
if err != nil {
return OperationAbortInspection{}
}
volume, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, parts[1]),
)
_, volume, _, err := l.backupWorkspaceResidue(record.ResourceID, parts[1])
if err != nil || volume == nil {
return OperationAbortInspection{}
}

@ -1797,6 +1797,7 @@ func TestCollectVMManagedVolumeNamesIncludesTransitionsSnapshotsAndOrphans(t *te
}
inventory := []string{
root, pending, snapshotVolumeName(uuid, registeredKey), snapshotVolumeName(uuid, orphanKey),
backupWorkspaceVolumeName(uuid, orphanKey),
orphanRestore, orphanReinstall, rootVolumeName(otherUUID), "template-debian-12.qcow2",
}
got, err := collectVMManagedVolumeNames(uuid, metadata, inventory)
@ -1805,6 +1806,7 @@ func TestCollectVMManagedVolumeNamesIncludesTransitionsSnapshotsAndOrphans(t *te
}
want := []string{
root, pending, snapshotVolumeName(uuid, registeredKey), snapshotVolumeName(uuid, orphanKey),
backupWorkspaceVolumeName(uuid, orphanKey),
orphanRestore, orphanReinstall,
}
sort.Strings(want)

@ -438,6 +438,21 @@ func abortBackupAfterThawFailure(domain snapshotBackupJob, thawErr error,
}
func (l *Libvirt) CreateSnapshot(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) {
return l.createFullDiskCopy(ctx, uuid, request, true)
}
// createBackupWorkspace creates the immutable full-system-disk copy used only
// by a portable backup operation. Unlike a user snapshot it is deliberately
// absent from domainMetadata.Snapshots, so a later local cleanup failure can
// consume storage and raise an alert without changing the VM snapshot catalog
// or blocking maintenance migration.
func (l *Libvirt) createBackupWorkspace(ctx context.Context, uuid string,
request SnapshotRequest) (_ Snapshot, resultErr error) {
return l.createFullDiskCopy(ctx, uuid, request, false)
}
func (l *Libvirt) createFullDiskCopy(ctx context.Context, uuid string,
request SnapshotRequest, registerSnapshot bool) (_ Snapshot, resultErr error) {
key := request.SnapshotKey
if key == "" {
key = request.SnapshotID
@ -486,6 +501,9 @@ func (l *Libvirt) CreateSnapshot(ctx context.Context, uuid string, request Snaps
return Snapshot{}, snapshotPreconditionFailure(NewError(ErrConflict, "snapshot_storage_full", "managed storage pool does not have enough free space", nil))
}
volumeName := snapshotVolumeName(uuid, key)
if !registerSnapshot {
volumeName = backupWorkspaceVolumeName(uuid, key)
}
if existing, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), volumeName); lookupErr != nil {
return Snapshot{}, snapshotPreconditionFailure(lookupErr)
} else if existing != nil {
@ -575,18 +593,20 @@ func (l *Libvirt) CreateSnapshot(ctx context.Context, uuid string, request Snaps
return Snapshot{}, operation.MarkUncertain(err)
}
created := time.Now().UTC()
updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
stored.Snapshots = append(stored.Snapshots, domainSnapshotMetadataXML{Key: key, Name: request.Name, Description: request.Description, Consistency: actualConsistency, SizeBytes: info.Capacity, CreatedAt: created.Unix()})
return nil
})
if err != nil {
return Snapshot{}, operation.MarkUncertain(err)
}
defined, err := l.conn.DomainDefineXML(updated)
if err != nil {
return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("persist snapshot metadata", err))
if registerSnapshot {
updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
stored.Snapshots = append(stored.Snapshots, domainSnapshotMetadataXML{Key: key, Name: request.Name, Description: request.Description, Consistency: actualConsistency, SizeBytes: info.Capacity, CreatedAt: created.Unix()})
return nil
})
if err != nil {
return Snapshot{}, operation.MarkUncertain(err)
}
defined, err := l.conn.DomainDefineXML(updated)
if err != nil {
return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("persist snapshot metadata", err))
}
defined.Free()
}
defined.Free()
return Snapshot{ID: key, SnapshotKey: key, Name: request.Name, Description: request.Description, CreatedAt: created, State: "ready", Consistency: actualConsistency, SizeBytes: info.Capacity, Created: true}, nil
}

@ -343,6 +343,23 @@ type CleanupPeerMigrationTargetRequest struct {
PriorOperation operation.Record `json:"-"`
}
// CleanupBackupWorkspaceRequest authorizes only the exact cleanup plan exposed
// by a prior successful create-backup operation. PriorOperation is injected by
// the Agent API from its immutable Journal and is never accepted from the wire.
type CleanupBackupWorkspaceRequest struct {
OperationID string `json:"operation_id,omitempty"`
PriorOperationID string `json:"prior_operation_id"`
CleanupPlanID string `json:"cleanup_plan_id,omitempty"`
PriorOperation operation.Record `json:"-"`
}
type CleanupBackupWorkspaceResult struct {
BackupKey string `json:"backup_key"`
Cleaned bool `json:"cleaned"`
WorkspaceAbsent bool `json:"workspace_absent"`
MetadataAbsent bool `json:"metadata_absent"`
}
// RecoverBackupRequest replaces the managed root disk of an existing VM with
// one immutable portable-backup object. Network and domain identity remain
// attached to the existing VM, while the Agent reduces its NoCloud seed to
@ -830,6 +847,7 @@ type Hypervisor interface {
GetVMMetadata(context.Context, string) (VMMetadata, error)
CreateVM(context.Context, CreateVMRequest) (VMMetadata, error)
CreateBackup(context.Context, string, BackupRequest) (BackupResult, error)
CleanupBackupWorkspace(context.Context, string, CleanupBackupWorkspaceRequest) (CleanupBackupWorkspaceResult, error)
RestoreBackup(context.Context, RestoreBackupRequest) (RestoreBackupResult, error)
RecoverBackup(context.Context, string, RecoverBackupRequest) (RecoverBackupResult, error)
PreparePeerMigration(context.Context, string, PreparePeerMigrationRequest) (PreparePeerMigrationResult, error)
@ -905,6 +923,9 @@ func (Unsupported) CreateVM(context.Context, CreateVMRequest) (VMMetadata, error
func (Unsupported) CreateBackup(context.Context, string, BackupRequest) (BackupResult, error) {
return BackupResult{}, unsupported("VM backup")
}
func (Unsupported) CleanupBackupWorkspace(context.Context, string, CleanupBackupWorkspaceRequest) (CleanupBackupWorkspaceResult, error) {
return CleanupBackupWorkspaceResult{}, unsupported("VM backup workspace cleanup")
}
func (Unsupported) RestoreBackup(context.Context, RestoreBackupRequest) (RestoreBackupResult, error) {
return RestoreBackupResult{}, unsupported("VM backup restore")
}

@ -2672,9 +2672,37 @@ func validPeerMigrationTargetCleanupSupersession(spec Spec, lockKeys []string, p
return lockKeysOverlap(lockKeys, predecessorKeys)
}
func validBackupWorkspaceCleanupSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID ||
!cleanupBackupActionPattern.MatchString(spec.Action) ||
predecessor.OperationID != spec.SupersedesOperationID ||
predecessor.ResourceID != spec.ResourceID ||
predecessor.State != StateSucceeded || predecessor.ExecutionFenced ||
!createBackupActionPattern.MatchString(predecessor.Action) {
return false
}
backupKey := strings.TrimPrefix(spec.Action, "cleanup-backup-workspace:")
priorParts := strings.Split(predecessor.Action, ":")
if len(priorParts) != 3 || priorParts[1] != backupKey {
return false
}
var result struct {
CleanupPending bool `json:"cleanup_pending"`
}
if json.Unmarshal(predecessor.Result, &result) != nil || !result.CleanupPending {
return false
}
predecessorKeys := effectiveLockKeys(
predecessor.OperationID, predecessor.ResourceID,
predecessor.Action, predecessor.LockKeys,
)
return lockKeysOverlap(lockKeys, predecessorKeys)
}
func validExplicitSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
return validForceDeleteSupersession(spec, lockKeys, predecessor) ||
validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor)
validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor) ||
validBackupWorkspaceCleanupSupersession(spec, lockKeys, predecessor)
}
// findResourceBlocker validates any explicit force-delete predecessor and
@ -2989,7 +3017,9 @@ func validateSpec(spec Spec) error {
if spec.SupersedesOperationID != "" {
if !operationIDPattern.MatchString(spec.SupersedesOperationID) ||
spec.SupersedesOperationID == spec.OperationID ||
(spec.Action != "force-delete-vm" && !cleanupPeerTargetActionPattern.MatchString(spec.Action)) {
(spec.Action != "force-delete-vm" &&
!cleanupPeerTargetActionPattern.MatchString(spec.Action) &&
!cleanupBackupActionPattern.MatchString(spec.Action)) {
return ErrInvalidSupersession
}
}

@ -18,6 +18,8 @@ var (
restoreBackupActionPattern = regexp.MustCompile(`^restore-backup:[a-f0-9]{32}$`)
receivePeerActionPattern = regexp.MustCompile(`^receive-peer-migration:[a-f0-9]{32}$`)
cleanupPeerTargetActionPattern = regexp.MustCompile(`^cleanup-peer-migration-target:[a-f0-9]{32}$`)
createBackupActionPattern = regexp.MustCompile(`^create-backup:[a-f0-9]{32}:(?:filesystem|crash)$`)
cleanupBackupActionPattern = regexp.MustCompile(`^cleanup-backup-workspace:[a-f0-9]{32}$`)
)
//go:embed recovery_contract.json

@ -255,6 +255,19 @@
"not_applied"
]
},
{
"kind": "backup_workspace_cleanup",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [],
"action_patterns": [
"^cleanup-backup-workspace:[a-f0-9]{32}$"
],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "backup_recover",
"recovery_mode": "generic",

@ -187,7 +187,7 @@ Go context 无法可靠中止已进入 CGo/libvirt 的调用。父 Agent 到达
校验。远程 URL 导入不携带该字段,仍必须返回有效的 `Content-Length`。
任何导入源都不得使用压缩 `Content-Encoding`。
不存在 mutation `reconcile`、`confirm-inspection`、`abort-job` 或 `cleanup` endpoint。`inspect` 不取得 mutation admission,不写 Journal,不继续原目标,也不能调用任何 libvirt 变更。不可变 Journal 成功或明确无副作用失败可以为策略表中的任何类型提供精确结论;其他已失败且影响未知的类型只有在该 action 的专用投影证据完整匹配时才能输出 `applied/not_applied`,否则保持 `partial/inconclusive`。例如实例设置只有在 Domain metadata 与 NoCloud seed 都匹配同一 transition ID 时才是 `applied`,两者都确认未提交时才是 `not_applied`。电源、网络链路或服务的“当前状态刚好符合目标”不是原 operation 的强证据,必须保持 `inconclusive`。
不存在通用 mutation `reconcile`、`confirm-inspection`、`abort-job` 或任意资源 `cleanup` endpoint。`inspect` 不取得 mutation admission,不写 Journal,不继续原目标,也不能调用任何 libvirt 变更。唯一的备份工作区清理接口在下节说明:它只能消费原成功备份返回的最新 evidence-bound plan,不能接受路径、卷名或任意清理步骤。不可变 Journal 成功或明确无副作用失败可以为策略表中的任何类型提供精确结论;其他已失败且影响未知的类型只有在该 action 的专用投影证据完整匹配时才能输出 `applied/not_applied`,否则保持 `partial/inconclusive`。例如实例设置只有在 Domain metadata 与 NoCloud seed 都匹配同一 transition ID 时才是 `applied`,两者都确认未提交时才是 `not_applied`。电源、网络链路或服务的“当前状态刚好符合目标”不是原 operation 的强证据,必须保持 `inconclusive`。
### 可移植备份与异地恢复
@ -195,16 +195,16 @@ Go context 无法可靠中止已进入 CGo/libvirt 的调用。父 Agent 到达
`consistency_mode=filesystem|crash` 及 `object_upload`。`object_upload` 只有
短期预签名的 `part_urls`、`complete_url`、`abort_url`、只读 `verify_url`、part 大小及
总大小上限;Agent 永远不接收或保存 S3 access key。运行中 VM 使用 libvirt 完整 push backup,
关机 VM 使用稳定的独立克隆;文件系统冻结不可用时 `filesystem` 会保守降级并在
关机 VM 使用稳定的独立克隆。两者都写入独立 `vbackup-*` 工作区卷,不注册到
Domain 的正式 `Snapshots` 目录;文件系统冻结不可用时 `filesystem` 会保守降级并在
结果中报告 `crash`。首版只接受一个可写 `vda`,拒绝额外可写盘、救援挂载、
原生/外部快照或带 backing file 的系统盘。multipart 完成得到确定成功后删除本地
临时快照。若完成响应丢失,Agent 只通过 `verify_url` 重新读取并核对对象大小及
临时工作区。若完成响应丢失,Agent 只通过 `verify_url` 重新读取并核对对象大小及
SHA-256;完全匹配才继续成功清理,不匹配或暂不可读时保留本地快照和 effect fence,
绝不自动重放、Abort 或误判为未执行。普通上传失败或取消在远端 Abort 已确认后,
使用独立的两分钟本地清理上下文删除临时卷;只有进程硬崩溃、远端完成不明确或
清理失败才可能留下本地副本。read-only operation inspection 可输出只覆盖该
operation 临时卷/metadata 的 evidence-bound cleanup plan,但当前没有自动执行
cleanup 的 mutation endpoint。
operation 工作区卷(以及旧版本遗留的同 key 快照 metadata)的 evidence-bound cleanup plan。
成功结果只返回 `backup_key`、对象 `size_bytes`、qcow2
`virtual_size_bytes`、`sha256`、multipart 完成响应中的可选 `object_etag` 与
@ -212,8 +212,13 @@ cleanup 的 mutation endpoint。
可选 `cleanup_pending=true`,不返回预签名 URL 或本机路径。本地清理失败不会丢弃
已经验证成功的远端对象身份,也不会把可恢复的备份错误降级为远端效果不明。
后续只读 inspection 在残留仍精确归属于该 operation 且没有活跃 libvirt/block job 时,
会同时保留 `applied` 对象投影并附带 cleanup plan;该 plan 只用于管理员核对,首版仍不
提供执行它的 mutation endpoint。
会同时保留 `applied` 对象投影并附带 cleanup plan。超级管理员可为一个新的 operation ID
调用 `POST /v1/operations/{cleanup_operation_id}/cleanup-backup-workspace`,正文只含
`prior_operation_id` 与刚读取的 `cleanup_plan_id`。Agent 在 mutation admission 内重新读取
原 Journal、活动 block job、精确卷摘要、引用关系和 metadata;计划变化即拒绝。成功结果必须
同时返回 `cleaned=true`、`workspace_absent=true`、`metadata_absent=true`。原备份成功回执不会
被改写;清理 operation 通过 `supersedes_operation_id` 保留审计关系。若第一次响应丢失,管理员
必须重新 inspect 后创建新的清理 operation,不能盲目重放旧删除请求。
action 为 `create-backup:{backup_key}:{consistency}`。
维护迁移不复用备份 endpoint。`POST /v1/vms/{uuid}/peer-migration/prepare`

@ -68,7 +68,7 @@ PeerMigration 不创建 `vm-migrations/` 对象,也不持有 S3 凭据。目
备份和恢复均为异步 HeavyIO 操作,单次宿主机 worker 最长可运行 24 小时。控制面只会派发一次同一 operation ID:超时、断连或回执丢失均视为效果未知,并通过 Agent Journal 查询原操作,不会重新执行备份或创建第二台 VM。
若远端对象已经完整提交并校验,但源宿主机的临时快照清理失败,Agent 会保留对象大小、SHA-256 和一致性结果,把备份标记为可恢复并单独提示本地清理待核对;该提示不会触发重新上传或删除远端对象。
普通云备份使用独立 `vbackup-*` 工作区卷,不把临时副本加入用户快照目录。因此即使远端对象已经完整提交后本地删除暂时失败,维护迁移仍只读取正式快照,不会被该残留阻断。Agent 会保留对象大小、SHA-256 和一致性结果,把备份标记为可恢复并单独提示本地工作区待清理;该提示不会触发重新上传或删除远端对象。超级管理员的“核对并清理临时工作区”会先读取原成功 operation 的最新计划,再以新的 operation ID 执行精确清理;计划摘要、卷身份、引用关系或活动作业任一变化都会拒绝执行。实例已经迁移或源 Domain 已删除时,清理仍绑定备份冻结的源宿主机和 UUID,只能在确认卷未被任何 Domain 引用后删除;若卷已经随源实例清理消失,则只确认缺席并解除提示。旧版本留下的 `vsnap-*` 临时卷和对应 metadata 也只能通过同一证据绑定入口清理。
恢复失败时,只有 Agent 明确证明新 Domain、所有 UUID 所属卷、seed、NVRAM、网络过滤器和流量状态均不存在,控制面才能释放容量与公网 IP。只要清理结果不确定,资源保持锁定等待人工核对。

@ -183,6 +183,21 @@ final class CloudAdminInstanceController
)->withHeader('Cache-Control', 'no-store');
}
public function cleanupLibraryBackupWorkspace($request, $response, $args)
{
$actor = $this->actor($request, true);
$backupId = filter_var($args['backup_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$backup = $backupId === false ? null
: CloudInstanceBackup::find((int) $backupId);
if ($backup === null || (string) $backup->state === 'deleted') {
throw new CloudError('备份不存在', 404);
}
$job = CloudBackupService::cleanupWorkspace($backup, $actor);
return $response->withJson(CloudJobService::acknowledgement($job), 202)
->withHeader('Cache-Control', 'no-store');
}
/** Queue a new independent VM restored from a ready portable backup. */
public function restoreBackup($request, $response, $args)
{

@ -146,6 +146,8 @@ final class CloudAdminOperationRecoveryService
'snapshot_not_created' => '快照没有创建',
'snapshot_not_deleted' => '快照仍然存在',
'snapshot_not_restored' => '系统盘没有切换到快照',
'backup_workspace_cleanup_not_applied' =>
'备份临时工作区仍完整存在,清理没有生效',
'storage_configuration_mismatch' => '固定存储池与面板配置不一致',
'storage_configuration_incomplete' => '存储池配置尚未完成',
'import_not_completed' => '目录资源导入未完成',
@ -1029,6 +1031,14 @@ final class CloudAdminOperationRecoveryService
if ($kind === 'backup_create') {
return self::BACKUP_PROJECTION;
}
if ($kind === 'backup_workspace_cleanup') {
return [
'backup_key' => 'string',
'cleaned' => 'boolean',
'workspace_absent' => 'boolean',
'metadata_absent' => 'boolean',
];
}
if ($kind === 'backup_recover') {
return [
'backup_key' => 'string',
@ -1118,6 +1128,10 @@ final class CloudAdminOperationRecoveryService
'backup_key', 'size_bytes', 'virtual_size_bytes', 'sha256',
'consistency',
],
'backup_workspace_cleanup' => [
'backup_key', 'cleaned', 'workspace_absent',
'metadata_absent',
],
'backup_recover' => [
'backup_key', 'sha256', 'restored', 'power_state',
],

@ -1322,6 +1322,143 @@ final class CloudBackupService
self::finishRestoreLocked($job, $status, $result, $payload);
} elseif ($job->kind === 'backup_recover') {
self::finishRecoveryLocked($job, $status, $result, $payload);
} elseif ($job->kind === 'backup_workspace_cleanup') {
self::finishWorkspaceCleanupLocked(
$job, $status, $result, $payload
);
}
}
/** Queue one evidence-bound cleanup of a source-host backup workspace. */
public static function cleanupWorkspace(
CloudInstanceBackup $backup,
array $actor
): CloudJob {
CloudAccess::manage($actor);
if (($actor['is_super_admin'] ?? false) !== true) {
throw new CloudError('只有超级管理员可以清理备份工作区', 403);
}
$original = CloudJob::where(
'operation_id', (string) $backup->last_operation_id
)->first();
if ($original === null || (string) $original->kind !== 'backup_create'
|| (string) $original->phase !== 'terminal'
|| (string) $original->outcome !== 'succeeded'
|| (int) $original->backup_id !== (int) $backup->id
|| (int) $original->host_id !== (int) $backup->source_host_id) {
throw new CloudError('原备份任务身份不完整,不能清理工作区', 409);
}
$inspection = CloudAdminOperationRecoveryService::inspect(
$actor, (string) $original->id
);
$cleanup = is_array($inspection['cleanup'] ?? null)
? $inspection['cleanup'] : [];
$planId = ($cleanup['available'] ?? false) === true
&& is_string($cleanup['plan_id'] ?? null)
? strtolower((string) $cleanup['plan_id']) : '';
if ($planId !== ''
&& preg_match('/^[a-f0-9]{64}$/D', $planId) !== 1) {
throw new CloudError('Agent 返回的清理计划身份无效', 502);
}
return $backup->getConnection()->transaction(function () use (
$backup, $actor, $original, $planId
): CloudJob {
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
->lockForUpdate()->first();
if ($locked === null || (string) $locked->state !== 'ready'
|| (string) $locked->error_code
!== 'backup_local_cleanup_pending'
|| !hash_equals(
(string) $locked->last_operation_id,
(string) $original->operation_id
)) {
throw new CloudError('备份工作区状态已变化,请刷新后重试', 409);
}
$operationId = bin2hex(random_bytes(16));
$target = '/v1/operations/' . rawurlencode($operationId)
. '/cleanup-backup-workspace';
$body = [
'prior_operation_id' => (string) $original->operation_id,
'cleanup_plan_id' => $planId,
];
$canonical = [
'backup_id' => (int) $locked->id,
'backup_key' => (string) $locked->backup_key,
'prior_operation_id' => (string) $original->operation_id,
'cleanup_plan_id' => $planId,
];
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
$actor,
(int) $locked->source_host_id,
'backup_workspace_cleanup',
'POST',
$target,
$body,
[
'owner_id' => (int) $locked->user_id,
'can_manage' => true,
'backup_key' => (string) $locked->backup_key,
'prior_job_id' => (int) $original->id,
'prior_operation_id' => (string) $original->operation_id,
'cleanup_plan_id' => $planId,
],
$canonical,
$operationId,
'/v1/backups/' . (int) $locked->id . '/cleanup-workspace'
);
if ($created) {
$job->instance_id = (int) $locked->source_instance_id;
$job->backup_id = (int) $locked->id;
$job->save();
}
return $job;
}, 3);
}
private static function finishWorkspaceCleanupLocked(
CloudJob $job,
string $status,
$result,
array $payload
): void {
$backup = $job->backup_id === null ? null
: CloudInstanceBackup::where('id', (int) $job->backup_id)
->lockForUpdate()->first();
if ($backup === null
|| (int) $backup->source_host_id !== (int) $job->host_id
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|| !is_string($payload['prior_operation_id'] ?? null)
|| !hash_equals(
(string) $backup->last_operation_id,
(string) $payload['prior_operation_id']
)
|| !hash_equals(
(string) $backup->backup_key,
(string) ($payload['backup_key'] ?? '')
)) {
throw new CloudRecoveryInstanceStateConflict(
'备份工作区清理任务与目录账本不一致', 409
);
}
if ($status !== 'succeeded') {
return;
}
if (!is_array($result)
|| !hash_equals(
(string) $backup->backup_key,
(string) ($result['backup_key'] ?? '')
)
|| ($result['cleaned'] ?? null) !== true
|| ($result['workspace_absent'] ?? null) !== true
|| ($result['metadata_absent'] ?? null) !== true) {
throw new CloudError('Agent 清理成功结果缺少完整缺席证据', 502);
}
if ((string) $backup->error_code
=== 'backup_local_cleanup_pending') {
$backup->error_code = '';
$backup->updated_at = time();
$backup->save();
}
}
@ -2153,6 +2290,9 @@ final class CloudBackupService
) ?? 'unknown'),
'cleanup_pending' => (string) $backup->error_code
=== 'backup_local_cleanup_pending',
'can_cleanup_workspace' => $canManage
&& (string) $backup->error_code
=== 'backup_local_cleanup_pending',
'can_delete' => $canDelete,
'delete_action' => in_array($state,
['deleting', 'delete_uncertain'], true)

@ -25,6 +25,7 @@ final class CloudOperationCatalog
'snapshot_restore' => '恢复快照',
'snapshot_delete' => '删除快照',
'backup_create' => '创建可移植备份',
'backup_workspace_cleanup' => '清理备份工作区',
'backup_restore' => '从备份创建实例',
'backup_recover' => '从云备份恢复实例',
'instance_migration_prepare' => '准备维护迁移源实例',
@ -74,6 +75,7 @@ final class CloudOperationCatalog
'snapshot_restore',
'snapshot_delete',
'backup_create',
'backup_workspace_cleanup',
'backup_restore',
'backup_recover',
'instance_migration_prepare',

@ -505,6 +505,7 @@ final class CloudOperationErrorCatalog
'snapshot_restore' => '恢复快照失败;任务已结束且不会自动重试',
'snapshot_delete' => '删除快照失败;任务已结束且不会自动重试',
'backup_create' => '创建云备份失败;任务已结束且不会自动重试',
'backup_workspace_cleanup' => '清理备份工作区失败;远端备份不受影响,可重新核对后创建新的清理任务',
'backup_restore' => '从备份创建实例失败;任务已结束且不会自动重试',
'backup_recover' => '从云备份恢复实例失败;任务已结束且不会自动重试',
'instance_migration_prepare' => '维护迁移源准备失败;请按迁移恢复出口处理',
@ -631,6 +632,7 @@ final class CloudOperationErrorCatalog
'service_was_running', 'service_was_running_hint',
'autostart', 'managed', 'created', 'restored', 'deleted',
'reinstall_committed', 'cleanup_pending', 'forced',
'cleaned', 'workspace_absent', 'metadata_absent',
'source_stopped', 'source_was_running',
'source_autostart_disabled',
'managed_artifacts_absent', 'active', 'staged',
@ -705,10 +707,11 @@ final class CloudOperationErrorCatalog
'source_autostart_disabled',
'autostart', 'managed', 'created', 'restored', 'deleted',
'forced', 'managed_artifacts_absent', 'active', 'staged',
'cleaned', 'workspace_absent', 'metadata_absent',
'pending_manual_restart'], true)) {
return null;
}
if ($key === 'snapshot_key'
if (in_array($key, ['snapshot_key', 'backup_key'], true)
&& preg_match('/^[a-f0-9]{32}$/D', $value) !== 1) {
return null;
}

@ -140,6 +140,13 @@ final class CloudOperationRecoveryContract
'action_patterns' => ['^create-backup:[a-f0-9]{32}:(?:filesystem|crash)$'],
'system_resolutions' => ['applied', 'not_applied'],
],
'backup_workspace_cleanup' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => [],
'action_patterns' => ['^cleanup-backup-workspace:[a-f0-9]{32}$'],
'system_resolutions' => ['applied', 'not_applied'],
],
'backup_recover' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',

@ -261,6 +261,11 @@ final class CloudOperationRecoveryIdentity
bool $forAppliedEffect
): array {
$kind = (string) $job->kind;
if ($kind === 'backup_workspace_cleanup') {
return self::backupWorkspaceCleanupAction(
$job, $payload, $method, $target, $body
);
}
if (in_array($kind,
['instance_migration_prepare', 'instance_migration_transfer',
'instance_migration_target_clean',
@ -1591,6 +1596,69 @@ final class CloudOperationRecoveryIdentity
return 'restore-backup:' . $key;
}
private static function backupWorkspaceCleanupAction(
CloudJob $job,
array $payload,
string $method,
string $target,
array $body
): array {
self::expectRequest(
$method,
$target,
'POST',
'/v1/operations/' . rawurlencode((string) $job->operation_id)
. '/cleanup-backup-workspace'
);
self::assertAllowedBody($body, [
'operation_id', 'prior_operation_id', 'cleanup_plan_id',
]);
$backupId = filter_var($job->backup_id, FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]);
$backup = $backupId === false ? null
: CloudInstanceBackup::find((int) $backupId);
$priorJobId = filter_var($payload['prior_job_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$prior = $priorJobId === false ? null
: CloudJob::find((int) $priorJobId);
$key = $payload['backup_key'] ?? null;
$planId = $body['cleanup_plan_id'] ?? null;
if ($backup === null || $prior === null
|| (int) $backup->source_instance_id !== (int) $job->instance_id
|| (int) $backup->source_host_id !== (int) $job->host_id
|| (int) ($payload['owner_id'] ?? 0) !== (int) $backup->user_id
|| ((int) $job->user_id !== (int) $backup->user_id
&& ($payload['can_manage'] ?? false) !== true)
|| (int) $prior->backup_id !== (int) $backup->id
|| (int) $prior->instance_id
!== (int) $backup->source_instance_id
|| (int) $prior->host_id !== (int) $job->host_id
|| (string) $prior->kind !== 'backup_create'
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'succeeded'
|| !is_string($key)
|| preg_match(self::CATALOG_ID_PATTERN, $key) !== 1
|| !hash_equals((string) $backup->backup_key, $key)
|| !hash_equals((string) $backup->last_operation_id,
(string) $prior->operation_id)
|| !hash_equals((string) $prior->operation_id,
(string) ($payload['prior_operation_id'] ?? ''))
|| !hash_equals((string) $prior->operation_id,
(string) ($body['prior_operation_id'] ?? ''))
|| !is_string($planId)
|| ($planId !== ''
&& preg_match('/^[a-f0-9]{64}$/D', $planId) !== 1)
|| !hash_equals((string) ($payload['cleanup_plan_id'] ?? ''),
$planId)) {
self::invalidPayload('备份工作区清理请求无效');
}
$resource = strtolower((string) $backup->source_uuid);
if (preg_match(self::UUID_PATTERN, $resource) !== 1) {
self::invalidPayload('备份工作区清理资源身份无效');
}
return ['cleanup-backup-workspace:' . $key, $resource];
}
private static function backupRecoverAction(
CloudJob $job,
array $payload,

@ -97,6 +97,8 @@ $app->group('/api/v1', function () {
'App\Controllers\CloudAdminInstanceController:restoreLibraryBackup');
$this->delete('/admin/backups/{backup_id:[0-9]+}',
'App\Controllers\CloudAdminInstanceController:deleteLibraryBackup');
$this->post('/admin/backups/{backup_id:[0-9]+}/cleanup-workspace',
'App\Controllers\CloudAdminInstanceController:cleanupLibraryBackupWorkspace');
$this->map(['GET', 'POST'], '/admin/instances/{id:[0-9]+}/backups',
'App\Controllers\CloudAdminInstanceController:backups');
$this->post('/admin/instances/{id:[0-9]+}/maintenance-migrations',

@ -796,6 +796,35 @@ function backupDeleteButton(vmId, backup, library = false) {
return button;
}
function backupWorkspaceCleanupButton(vmId, backup, library = false) {
const button = el('button', 'btn btn-outline-warning btn-sm', '核对并清理临时工作区');
button.type = 'button';
button.addEventListener('click', async () => {
if (state.backupMutationRequest || backup.can_cleanup_workspace !== true) return;
const controller = new AbortController();
state.backupMutationRequest = controller; button.disabled = true;
backupSurfaceNotice('正在读取原备份操作的最新宿主机证据并创建精确清理任务。', false, library);
try {
const receipt = await mutate(
`/api/v1/admin/backups/${encodeURIComponent(backup.id)}/cleanup-workspace`,
{}, controller, 'POST',
);
backupSurfaceNotice(`工作区清理任务 #${receipt.task_id || '—'} 已接收。`, false, library);
if (!library) watchBackupTask(vmId, receipt, '清理备份工作区');
else announceTaskAccepted(receipt, { title: '清理备份工作区' });
refreshBackupSurface(vmId, library);
} catch (error) {
if (controller.signal.aborted && controller.signal.reason !== 'timeout') return;
backupSurfaceNotice(error.message, true, library);
refreshBackupSurface(vmId, library);
} finally {
if (state.backupMutationRequest === controller) state.backupMutationRequest = null;
button.disabled = false;
}
});
return button;
}
function watchBackupTask(vmId, receipt, title) {
const serial = state.backupSerial;
const taskId = receipt?.task_id || '—';
@ -1235,7 +1264,12 @@ function renderBackups(payload, vmId) {
if (backup.error) body.append(el('p', 'text-danger fs-13', backup.error));
if (backup.cleanup_pending === true) {
body.append(el('p', 'text-warning fs-13',
'远端备份已完整保存并可恢复,但源宿主机的临时快照清理仍需管理员核对;系统不会重复上传。'));
'远端备份已完整保存并可恢复,但源宿主机的临时工作区仍待清理;清理不会重复上传或修改备份对象。'));
if (backup.can_cleanup_workspace === true) {
const cleanupActions = el('div', 'd-flex justify-content-end mb-3');
cleanupActions.append(backupWorkspaceCleanupButton(vmId, backup));
body.append(cleanupActions);
}
}
if (backup.state === 'uncertain') {
body.append(el('p', 'alert alert-warning fs-13 mb-0',
@ -1334,7 +1368,12 @@ function renderBackupLibrary(payload) {
}
if (backup.cleanup_pending === true) {
body.append(el('p', 'text-warning fs-13',
'远端备份可恢复,但源宿主机的临时快照清理仍需核对。'));
'远端备份可恢复,但源宿主机的临时工作区仍需核对。'));
if (backup.can_cleanup_workspace === true) {
const cleanupActions = el('div', 'd-flex justify-content-end mb-3');
cleanupActions.append(backupWorkspaceCleanupButton(vmId, backup, true));
body.append(cleanupActions);
}
}
if (backup.error) body.append(el('p', 'text-danger fs-13', backup.error));
const targets = backupTargets(payload, backup);

@ -1,2 +1,2 @@
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
{assign var=venix_asset_version value='2026102802' scope='global'}
{assign var=venix_asset_version value='2026102803' scope='global'}

@ -1364,6 +1364,136 @@ namespace {
);
}
$cleanupBackupId = 205;
$cleanupBackupKey = hash('md5',
'recovery-matrix-backup:' . $cleanupBackupId);
$cleanupPlanId = str_repeat('8', 64);
$cleanupPrior = $insertJob(
'backup_create',
static function (int $jobId, string $operationId) use (
$instanceUuid,
$ownerId,
$cleanupBackupKey
): array {
return [
'method' => 'POST',
'request_target' => '/v1/vms/'
. rawurlencode($instanceUuid) . '/backups',
'body' => [
'operation_id' => $operationId,
'backup_key' => $cleanupBackupKey,
'name' => 'matrix-cleanup-source',
'consistency_mode' => 'filesystem',
'object_upload' => [
'part_size_bytes' => 8388608,
'maximum_size_bytes' => 42949672960,
'part_urls' => [
'https://objects.invalid/part-1',
],
'complete_url' =>
'https://objects.invalid/complete',
'abort_url' => 'https://objects.invalid/abort',
'verify_url' => 'https://objects.invalid/verify',
],
],
'owner_id' => $ownerId,
'can_manage' => true,
'backup_key' => $cleanupBackupKey,
'maximum_size_bytes' => 42949672960,
];
},
$instanceId,
null,
null,
null,
'agent',
$ownerId,
$cleanupBackupId
);
Capsule::table('cloud_jobs')->where('id', $cleanupPrior['id'])->update([
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'result_json' => json_encode([
'backup_key' => $cleanupBackupKey,
'size_bytes' => 1073741824,
'virtual_size_bytes' => 42949672960,
'sha256' => str_repeat('6', 64),
'consistency' => 'filesystem',
'cleanup_pending' => true,
], JSON_THROW_ON_ERROR),
]);
Capsule::table('cloud_instance_backups')->insert([
'id' => $cleanupBackupId,
'backup_key' => $cleanupBackupKey,
'source_instance_id' => $instanceId,
'source_host_id' => $hostId,
'user_id' => $ownerId,
'source_uuid' => $instanceUuid,
'name' => 'matrix-cleanup-source',
'state' => 'ready',
'consistency' => 'filesystem',
'object_key' => 'vm-backups/' . $cleanupBackupKey . '/system.qcow2',
'object_size_bytes' => 1073741824,
'virtual_size_bytes' => 42949672960,
'sha256' => str_repeat('6', 64),
'spec_json' => '{}',
'sealed_storage_runtime' => 'sealed-runtime',
'last_operation_id' => $cleanupPrior['operation_id'],
'error_code' => 'backup_local_cleanup_pending',
'created_at' => $now - 180,
'updated_at' => $now - 20,
]);
$cleanupRecord = $insertJob(
'backup_workspace_cleanup',
static function (int $jobId, string $operationId) use (
$ownerId,
$cleanupBackupKey,
$cleanupPlanId,
$cleanupPrior
): array {
return [
'method' => 'POST',
'request_target' => '/v1/operations/'
. rawurlencode($operationId)
. '/cleanup-backup-workspace',
'body' => [
'operation_id' => $operationId,
'prior_operation_id' => $cleanupPrior['operation_id'],
'cleanup_plan_id' => $cleanupPlanId,
],
'owner_id' => $ownerId,
'can_manage' => true,
'backup_key' => $cleanupBackupKey,
'prior_job_id' => $cleanupPrior['id'],
'prior_operation_id' => $cleanupPrior['operation_id'],
'cleanup_plan_id' => $cleanupPlanId,
];
},
$instanceId,
null,
null,
null,
'agent',
$ownerId,
$cleanupBackupId
);
$addFixture(
'backup_workspace_cleanup',
'cleanup',
$cleanupRecord,
'cleanup-backup-workspace:' . $cleanupBackupKey,
$instanceUuid,
[
'backup_key' => $cleanupBackupKey,
'cleaned' => true,
'workspace_absent' => true,
'metadata_absent' => true,
]
);
$recoverBackupId = 204;
$recoverBackupKey = hash('md5',
'recovery-matrix-backup:' . $recoverBackupId);
@ -2296,6 +2426,23 @@ namespace {
'An upgrade with the wrong final power state crossed the projection boundary.'
);
$workspaceCleanupFixture = $fixtures['backup_workspace_cleanup'][0];
Capsule::table('cloud_instances')->where('id', $instanceId)->update([
'host_id' => $hostId + 1,
'status' => 'deleted',
]);
$migratedCleanupIdentity = CloudOperationRecoveryIdentity::
expectedFromPayload(
$workspaceCleanupFixture['job'],
$workspaceCleanupFixture['payload']
);
$assert($migratedCleanupIdentity['resource_id'] === $instanceUuid,
'Backup workspace cleanup lost its frozen source-host identity after migration.');
Capsule::table('cloud_instances')->where('id', $instanceId)->update([
'host_id' => $hostId,
'status' => 'active',
]);
foreach ($fixtures as $kind => $kindFixtures) {
$entry = $manifestOperations[$kind];
$assert(CloudOperationRecoveryPolicy::supports($kind)

@ -67,6 +67,7 @@ namespace {
'snapshot_restore',
'snapshot_delete',
'backup_create',
'backup_workspace_cleanup',
'backup_recover',
'firewall_apply',
'service_suspend',
@ -82,8 +83,8 @@ namespace {
];
$assert(CloudOperationRecoveryPolicy::kinds() === $expectedKinds,
'The Agent leaf recovery policy changed without updating its contract.');
$assert(count(CloudOperationRecoveryPolicy::kinds()) === 31,
'The recovery policy contract must contain exactly 31 Agent leaf kinds.');
$assert(count(CloudOperationRecoveryPolicy::kinds()) === 32,
'The recovery policy contract must contain exactly 32 Agent leaf kinds.');
foreach ($expectedKinds as $kind) {
$assert(CloudOperationRecoveryPolicy::supports($kind),
'The recovery policy omitted ' . $kind . '.');

@ -28,6 +28,8 @@ $assert(str_contains($routes,
&& str_contains($routes, "'/admin/backups'")
&& str_contains($routes, "'/admin/backups/{backup_id:[0-9]+}/restore'")
&& str_contains($routes, "'/admin/backups/{backup_id:[0-9]+}'")
&& str_contains($routes,
"'/admin/backups/{backup_id:[0-9]+}/cleanup-workspace'")
&& str_contains($routes,
"'/admin/instances/{id:[0-9]+}/backups/{backup_id:[0-9]+}/restore'")
&& str_contains($routes,
@ -42,6 +44,7 @@ $assert(str_contains($routes,
&& str_contains($controller, 'CloudBackupService::create(')
&& str_contains($controller, 'CloudBackupService::restore(')
&& str_contains($controller, 'CloudBackupService::delete(')
&& str_contains($controller, 'CloudBackupService::cleanupWorkspace(')
&& str_contains($controller, 'CloudBackupService::library(')
&& substr_count($controller, 'CloudAccess::manage($actor);') >= 2
&& str_contains($controller, 'CloudJobService::acknowledgement($job)'),
@ -103,6 +106,8 @@ $assert(str_contains($template, 'id="instances-backup-library"')
&& str_contains($template, 'id="backup-library"')
&& str_contains($runtime, 'async function loadBackupLibrary(')
&& str_contains($runtime, '/api/v1/admin/backups?page=')
&& str_contains($runtime,
'/api/v1/admin/backups/${encodeURIComponent(backup.id)}/cleanup-workspace')
&& str_contains($runtime, '源实例已删除')
&& str_contains($runtime, 'backupMutationUrl('),
'The bounded global backup library must remain usable after source VM deletion.');
@ -136,8 +141,8 @@ $assert(str_contains($runtime, "method: 'DELETE'")
$assert(str_contains($runtime, '统一接入 br0、分配全新公网 IP')
&& str_contains($runtime, '不会继承原 IP、账单、到期时间或流量周期')
&& str_contains($runtime, '临时快照清理仍需管理员核对')
&& str_contains($runtime, '系统不会重复上传')
&& str_contains($runtime, '临时工作区仍待清理')
&& str_contains($runtime, '清理不会重复上传或修改备份对象')
&& str_contains($runtime, 'target_host_id')
&& str_contains($runtime, 'available_ip_count'),
'The restore UI must describe and collect the independent br0/new-IP target intent.');

@ -224,6 +224,33 @@ $assert((string) $createdBackup->state === 'ready'
&& $createdBackup->ready_at !== null,
'Cleanup-pending backup lost its ready/restorable object identity.');
$cleanupOperation = str_repeat('9', 32);
$capsule->table('cloud_jobs')->insert([
'id' => 2,
'kind' => 'backup_workspace_cleanup',
'host_id' => 3,
'instance_id' => 101,
'backup_id' => 1,
'backup_restore_id' => null,
'operation_id' => $cleanupOperation,
]);
$cleanupJob = CloudJob::findOrFail(2);
CloudBackupService::finishJobLocked($cleanupJob, 'succeeded', [
'backup_key' => str_repeat('a', 32),
'cleaned' => true,
'workspace_absent' => true,
'metadata_absent' => true,
], [
'owner_id' => 9,
'backup_key' => str_repeat('a', 32),
'prior_operation_id' => $createOperation,
]);
$cleanedBackup = CloudInstanceBackup::findOrFail(1);
$assert((string) $cleanedBackup->state === 'ready'
&& (string) $cleanedBackup->error_code === ''
&& (string) $cleanedBackup->sha256 === str_repeat('b', 64),
'Evidence-bound workspace cleanup did not preserve the remote backup identity.');
$fixture = static function (int $id, string $outcome) use (
$capsule, $sourceSpec, $now
): array {

@ -633,7 +633,8 @@ namespace {
'action', 'reinstall', 'instance_settings', 'network_change',
'instance_upgrade', 'firewall_apply', 'snapshot_create',
'snapshot_restore', 'snapshot_delete', 'backup_create',
'backup_restore', 'backup_recover', 'instance_migration_prepare',
'backup_workspace_cleanup', 'backup_restore', 'backup_recover',
'instance_migration_prepare',
'instance_migration_transfer', 'instance_migration_target_clean',
'instance_migration_cleanup',
'instance_migration_rollback', 'instance_migration_activate', 'rescue',

@ -73,7 +73,7 @@ $assert(str_contains($auth, "Config::get('tos_reg') === 'true'")
'Legal consent is only client-side and can be bypassed.');
$assets = $read('web/resources/views/venix/layout_assets.tpl');
$assert(str_contains($assets, "value='2026102802'")
$assert(str_contains($assets, "value='2026102803'")
&& str_contains($admin, '?v={$venix_asset_version}')
&& str_contains($read('web/resources/views/venix/home/layout_header.tpl'), '?v={$venix_asset_version}'),
'Shared browser asset version was not coupled to the new assets.');

@ -80,7 +80,7 @@ $assert(preg_match('/\.landing-site-header\s*\{[^}]*position:\s*fixed;/s', $css)
&& str_contains($accountCss, 'padding-top: 88px;')
&& str_contains($accountCss, 'padding-top: 76px;'),
'Persistent desktop or mobile public-navigation styling is incomplete.');
$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102802'"),
$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102803'"),
'The shared browser asset version was not advanced with the navigation styles.');
echo "Public page locale and shared-layout contract tests passed.\n";

@ -69,7 +69,7 @@ foreach ($templates as $template) {
}
$assets = $read('web/resources/views/venix/layout_assets.tpl');
$assert(str_contains($assets, "value='2026102802'"),
$assert(str_contains($assets, "value='2026102803'"),
'The shared asset version was not updated for the canonical logo.');
echo "Shared site-brand contract passed across public, auth, user, cloud and admin layouts.\n";

Loading…
Cancel
Save