From 52f4a61fe547855be2a8a4c2a2f45b2ae6731a09 Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 28 Oct 2026 05:44:31 +0800 Subject: [PATCH] feat: isolate portable backup workspaces --- agent/api/server.go | 124 +++++++ agent/api/server_test.go | 109 ++++++- agent/hypervisor/admission.go | 19 +- agent/hypervisor/backup.go | 14 + agent/hypervisor/backup_test.go | 36 +- agent/hypervisor/domain_xml.go | 15 +- agent/hypervisor/libvirt_backup.go | 59 ++-- agent/hypervisor/libvirt_backup_workspace.go | 308 ++++++++++++++++++ agent/hypervisor/libvirt_connection.go | 6 + agent/hypervisor/libvirt_operation_admin.go | 137 +++++++- agent/hypervisor/libvirt_safety_test.go | 2 + agent/hypervisor/libvirt_snapshot_features.go | 42 ++- agent/hypervisor/types.go | 21 ++ agent/operation/journal.go | 34 +- agent/operation/recovery_contract.go | 2 + agent/operation/recovery_contract.json | 13 + docs/AGENT_API.md | 19 +- docs/PORTABLE_VM_BACKUPS.md | 2 +- .../CloudAdminInstanceController.php | 15 + .../CloudAdminOperationRecoveryService.php | 14 + web/app/Services/Cloud/CloudBackupService.php | 140 ++++++++ .../Services/Cloud/CloudOperationCatalog.php | 2 + .../Cloud/CloudOperationErrorCatalog.php | 5 +- .../Cloud/CloudOperationRecoveryContract.php | 7 + .../Cloud/CloudOperationRecoveryIdentity.php | 68 ++++ web/config/routes_cloud.php | 2 + web/public/cloud/admin-instances.js | 43 ++- web/resources/views/venix/layout_assets.tpl | 2 +- .../CloudOperationRecoveryMatrixTest.php | 147 +++++++++ .../CloudOperationRecoveryPolicyTest.php | 5 +- ...CloudPortableBackupAdminUiContractTest.php | 9 +- .../CloudPortableBackupFinalizerTest.php | 27 ++ web/tests/CloudSnapshotWorkflowTest.php | 3 +- web/tests/LegalPagesContractTest.php | 2 +- web/tests/PublicPageLocaleContractTest.php | 2 +- web/tests/SiteBrandContractTest.php | 2 +- 36 files changed, 1373 insertions(+), 84 deletions(-) create mode 100644 agent/hypervisor/libvirt_backup_workspace.go diff --git a/agent/api/server.go b/agent/api/server.go index f9c79430..52f3526a 100644 --- a/agent/api/server.go +++ b/agent/api/server.go @@ -1076,6 +1076,37 @@ func peerMigrationTargetCleanupInspectionMatches(prior operation.Record, planID return true } +func backupWorkspaceCleanupInspectionMatches(prior operation.Record, planID string, + inspection hypervisor.OperationInspection) bool { + if inspection.OperationID != prior.OperationID || inspection.Action != prior.Action || + inspection.ResourceID != prior.ResourceID || inspection.RequestFingerprint != prior.Fingerprint || + inspection.OperationStatus != prior.State || inspection.CheckedAt <= 0 || + inspection.Outcome != hypervisor.InspectionOutcomeSucceeded || + inspection.EffectResolution != hypervisor.InspectionEffectApplied || + inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" || + inspection.AbortJob.Available { + return false + } + if !inspection.Cleanup.Available { + return planID == "" && + inspection.SummaryCode == hypervisor.InspectionSummarySucceeded + } + if inspection.SummaryCode != hypervisor.InspectionSummaryCleanupRequired || + planID == "" || !strings.EqualFold(inspection.Cleanup.PlanID, planID) || + len(inspection.Cleanup.Steps) == 0 { + return false + } + for _, step := range inspection.Cleanup.Steps { + switch step { + case hypervisor.CleanupStepDeleteSnapshotVolume, + hypervisor.CleanupStepRemoveSnapshotMetadata: + default: + return false + } + } + return true +} + func (s *Server) routeSnapshots(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) { if len(segments) == 4 { switch r.Method { @@ -1527,6 +1558,99 @@ func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segment writeJSON(w, http.StatusOK, inspection) return } + if len(segments) == 4 && segments[3] == "cleanup-backup-workspace" && + r.Method == http.MethodPost { + if !requireSuperAdminPrivilegedOperation(w, actor) { + return + } + if actor.OperationID == "" || actor.OperationID != operationID { + writePreDispatchError(w, http.StatusBadRequest, "operation_id_mismatch", "actor operation_id must match the cleanup operation") + return + } + request, ok := decodeRequired[hypervisor.CleanupBackupWorkspaceRequest](w, body) + if !ok { + return + } + if err := hypervisor.ValidateCleanupBackupWorkspaceRequest(request); err != nil { + writePreDispatchMappedError(w, err) + return + } + action := "" + existingReplay := false + trustedPrior, trustedPriorFound := trustedSupersededOperation( + r.Context(), request.PriorOperationID, + ) + existing, existingErr := s.Journal.Get(operationID) + if existingErr == nil { + action = existing.Action + existingReplay = strings.HasPrefix(action, "cleanup-backup-workspace:") && + existing.SupersedesOperationID == request.PriorOperationID + } else if !errors.Is(existingErr, operation.ErrNotFound) { + writeMappedError(w, existingErr, nil) + return + } + prior := trustedPrior + if !existingReplay { + var priorErr error + if !trustedPriorFound { + prior, priorErr = s.Journal.Get(request.PriorOperationID) + } + parts := strings.Split(prior.Action, ":") + if priorErr != nil || len(parts) != 3 || parts[0] != "create-backup" || + !catalogIDPattern.MatchString(parts[1]) || + prior.State != operation.StateSucceeded || + prior.ResourceID == "" || prior.ExecutionFenced { + writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup requires the exact successful backup operation") + return + } + action = "cleanup-backup-workspace:" + parts[1] + if !trustedPriorFound { + _, incidentFound, incidentErr := s.Journal.CompletionDeliveryIncident(prior.OperationID) + if incidentErr != nil { + writeMappedError(w, incidentErr, nil) + return + } + if incidentFound { + writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup is blocked by a completion-delivery incident") + return + } + } + ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) + inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, prior) + cancel() + if inspectErr != nil { + writeMappedError(w, inspectErr, nil) + return + } + if !backupWorkspaceCleanupInspectionMatches( + prior, request.CleanupPlanID, inspection, + ) { + writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_plan_changed", "cleanup plan is stale or no longer safe") + return + } + request.PriorOperation = prior + } + parts := strings.Split(action, ":") + if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) { + writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup action identity is invalid") + return + } + resourceID := prior.ResourceID + if existingReplay { + resourceID = existing.ResourceID + } + supersedes := []string{request.PriorOperationID} + if trustedPriorFound { + supersedes = nil + } + s.runMutationExpected(w, r, body, actor, operationID, resourceID, + action, nil, true, http.StatusOK, + hypervisor.CleanupBackupWorkspaceAdmission(resourceID, parts[1]), + func(ctx context.Context) (any, error) { + return s.Hypervisor.CleanupBackupWorkspace(ctx, resourceID, request) + }, supersedes...) + return + } if len(segments) == 4 && segments[3] == "fence-resolution" && r.Method == http.MethodGet { if !requireSuperAdminInspection(w, actor, operationID) { return diff --git a/agent/api/server_test.go b/agent/api/server_test.go index b313ae00..a187dbe0 100644 --- a/agent/api/server_test.go +++ b/agent/api/server_test.go @@ -33,6 +33,8 @@ type testHypervisor struct { configureCalls int cleanupPeerTargetCalls int cleanupPeerTargetLast hypervisor.CleanupPeerMigrationTargetRequest + cleanupBackupCalls int + cleanupBackupLast hypervisor.CleanupBackupWorkspaceRequest inspectionResult *hypervisor.OperationInspection inspectionErr error } @@ -61,6 +63,22 @@ func (h *testHypervisor) CleanupPeerMigrationTarget(_ context.Context, _ string, }, nil } +func (h *testHypervisor) CleanupBackupWorkspace(_ context.Context, _ string, + request hypervisor.CleanupBackupWorkspaceRequest) (hypervisor.CleanupBackupWorkspaceResult, error) { + h.mu.Lock() + h.cleanupBackupCalls++ + h.cleanupBackupLast = request + h.mu.Unlock() + parts := strings.Split(request.PriorOperation.Action, ":") + key := "" + if len(parts) == 3 { + key = parts[1] + } + return hypervisor.CleanupBackupWorkspaceResult{ + BackupKey: key, Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true, + }, nil +} + func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) { h.mu.Lock() defer h.mu.Unlock() @@ -1141,7 +1159,7 @@ func TestCreateBackupUsesDedicatedAsyncActionAndHeavyAdmission(t *testing.T) { for _, want := range []string{ "domain:" + uuid, "volume:virt-cloud-managed/vm-" + uuid, - "volume:virt-cloud-managed/vsnap-123e4567e89b42d3a456426614174000-" + backupKey + ".qcow2", + "volume:virt-cloud-managed/vbackup-123e4567e89b42d3a456426614174000-" + backupKey + ".qcow2", "catalog:backup/" + backupKey, } { if !containsTestString(record.LockKeys, want) { @@ -1162,6 +1180,95 @@ func TestCreateBackupUsesDedicatedAsyncActionAndHeavyAdmission(t *testing.T) { } } +func TestBackupWorkspaceCleanupRequiresSuccessfulPriorAndFreshExactPlan(t *testing.T) { + api := newTestAPI(t) + const ( + uuid = "123e4567-e89b-42d3-a456-426614174000" + backupKey = "0123456789abcdef0123456789abcdef" + priorID = "operation-backup-cleanup-pending" + cleanupID = "operation-backup-workspace-cleanup" + ) + priorSpec := operation.Spec{ + OperationID: priorID, ResourceID: uuid, + Action: "create-backup:" + backupKey + ":filesystem", + Fingerprint: operation.Fingerprint(http.MethodPost, + "/v1/vms/"+uuid+"/backups", []byte(`{}`)), + LockKeys: hypervisor.CreateBackupAdmission(uuid, hypervisor.BackupRequest{ + BackupKey: backupKey, + }).LockKeys, + HeavyIO: true, + } + if _, _, err := api.journal.Execute(context.Background(), priorSpec, + func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"backup_key":"` + backupKey + `","size_bytes":1024,"virtual_size_bytes":4096,"sha256":"` + strings.Repeat("a", 64) + `","consistency":"crash","cleanup_pending":true}`), nil + }); err != nil { + t.Fatal(err) + } + prior, err := api.journal.Get(priorID) + if err != nil { + t.Fatal(err) + } + planID := strings.Repeat("b", 64) + api.hv.inspectionResult = &hypervisor.OperationInspection{ + OperationID: prior.OperationID, Action: prior.Action, + ResourceID: prior.ResourceID, RequestFingerprint: prior.Fingerprint, + OperationStatus: prior.State, CheckedAt: api.now.Unix(), + Outcome: hypervisor.InspectionOutcomeSucceeded, + SummaryCode: hypervisor.InspectionSummaryCleanupRequired, + EffectResolution: hypervisor.InspectionEffectApplied, + EffectProjection: append(json.RawMessage(nil), prior.Result...), + Cleanup: hypervisor.OperationCleanupPlan{ + Available: true, PlanID: planID, + Steps: []string{hypervisor.CleanupStepDeleteSnapshotVolume}, + }, + } + executor := &fakeMutationExecutor{result: json.RawMessage(`{"backup_key":"` + backupKey + `","cleaned":true,"workspace_absent":true,"metadata_absent":true}`)} + if err := api.handler.SetMutationExecutor(executor); err != nil { + t.Fatal(err) + } + request := hypervisor.CleanupBackupWorkspaceRequest{ + OperationID: cleanupID, PriorOperationID: priorID, CleanupPlanID: planID, + } + body, _ := json.Marshal(request) + actor := auth.Actor{ + UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, + OperationID: cleanupID, + } + response := api.request(t, http.MethodPost, + "/v1/operations/"+cleanupID+"/cleanup-backup-workspace", body, actor) + if response.Code != http.StatusAccepted { + t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String()) + } + terminal := waitForTerminal(t, api.journal, cleanupID) + if terminal.State != operation.StateSucceeded || + terminal.SupersedesOperationID != priorID || + terminal.Action != "cleanup-backup-workspace:"+backupKey || + terminal.ResourceID != uuid { + t.Fatalf("cleanup terminal = %#v", terminal) + } + executor.mu.Lock() + if len(executor.requests) != 1 || executor.requests[0].SupersededOperation == nil || + executor.requests[0].SupersededOperation.OperationID != priorID { + executor.mu.Unlock() + t.Fatalf("cleanup worker predecessor = %#v", executor.requests) + } + executor.mu.Unlock() + + stale := request + stale.OperationID = "operation-backup-workspace-stale" + stale.CleanupPlanID = strings.Repeat("c", 64) + staleBody, _ := json.Marshal(stale) + staleActor := actor + staleActor.OperationID = stale.OperationID + denied := api.request(t, http.MethodPost, + "/v1/operations/"+stale.OperationID+"/cleanup-backup-workspace", + staleBody, staleActor) + if denied.Code != http.StatusConflict || + !bytes.Contains(denied.Body.Bytes(), []byte(`"code":"backup_workspace_cleanup_plan_changed"`)) { + t.Fatalf("stale cleanup = %d %s", denied.Code, denied.Body.String()) + } +} + func TestRecoverBackupUsesDestructiveDiskTransitionAdmission(t *testing.T) { api := newTestAPI(t) const ( diff --git a/agent/hypervisor/admission.go b/agent/hypervisor/admission.go index 88800b73..ed9f01cc 100644 --- a/agent/hypervisor/admission.go +++ b/agent/hypervisor/admission.go @@ -111,12 +111,24 @@ func CreateBackupAdmission(uuid string, request BackupRequest) MutationAdmission uuid = strings.ToLower(uuid) locks := []string{ domainLockKey(uuid), vmManagedVolumesLockKey(uuid), - volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, request.BackupKey)), + volumeLockKey(managedStoragePoolName, backupWorkspaceVolumeName(uuid, request.BackupKey)), catalogLockKey("backup", request.BackupKey), } return MutationAdmission{HeavyIO: true, LockKeys: locks} } +func CleanupBackupWorkspaceAdmission(uuid, backupKey string) MutationAdmission { + uuid = strings.ToLower(uuid) + return MutationAdmission{HeavyIO: true, LockKeys: []string{ + domainLockKey(uuid), vmManagedVolumesLockKey(uuid), + volumeLockKey(managedStoragePoolName, backupWorkspaceVolumeName(uuid, backupKey)), + // Legacy backups created before the workspace split used the snapshot + // namespace. Keep that one exact lock until their residue is gone. + volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, backupKey)), + catalogLockKey("backup", backupKey), + }} +} + func RestoreBackupAdmission(request RestoreBackupRequest) MutationAdmission { uuid := strings.ToLower(request.UUID) locks := []string{ @@ -328,6 +340,11 @@ func snapshotVolumeName(uuid, snapshotKey string) string { return "vsnap-" + compact + "-" + snapshotKey + ".qcow2" } +func backupWorkspaceVolumeName(uuid, backupKey string) string { + compact := strings.ReplaceAll(strings.ToLower(uuid), "-", "") + return "vbackup-" + compact + "-" + backupKey + ".qcow2" +} + func snapshotRestoreVolumeName(uuid, operationID string) string { sum := sha256.Sum256([]byte(operationID)) compact := strings.ReplaceAll(strings.ToLower(uuid), "-", "") diff --git a/agent/hypervisor/backup.go b/agent/hypervisor/backup.go index e66afb60..41b30551 100644 --- a/agent/hypervisor/backup.go +++ b/agent/hypervisor/backup.go @@ -32,6 +32,20 @@ func ValidateBackupRequest(request BackupRequest) error { return nil } +func ValidateCleanupBackupWorkspaceRequest(request CleanupBackupWorkspaceRequest) error { + if !operationIdentifierPattern.MatchString(request.OperationID) || + !operationIdentifierPattern.MatchString(request.PriorOperationID) || + request.OperationID == request.PriorOperationID || + (request.CleanupPlanID != "" && + !sha256Pattern.MatchString(strings.ToLower(request.CleanupPlanID))) { + return NewError( + ErrInvalid, "invalid_backup_workspace_cleanup", + "backup workspace cleanup identity is invalid", nil, + ) + } + return nil +} + func ValidateRestoreBackupRequest(request RestoreBackupRequest) error { if !operationIdentifierPattern.MatchString(request.OperationID) || !catalogIDPattern.MatchString(request.BackupKey) || diff --git a/agent/hypervisor/backup_test.go b/agent/hypervisor/backup_test.go index 281ceca7..2a2f66fb 100644 --- a/agent/hypervisor/backup_test.go +++ b/agent/hypervisor/backup_test.go @@ -97,6 +97,27 @@ func TestValidateBackupRequestChecksCapabilityBeforeDispatch(t *testing.T) { } } +func TestBackupWorkspaceIdentityAndCleanupRequestAreBound(t *testing.T) { + const uuid = "123e4567-e89b-42d3-a456-426614174000" + name := backupWorkspaceVolumeName(uuid, testBackupKey) + if name == snapshotVolumeName(uuid, testBackupKey) || + !isVMOwnedManagedVolumeName(uuid, name) { + t.Fatalf("backup workspace identity was not isolated: %q", name) + } + request := CleanupBackupWorkspaceRequest{ + OperationID: "operation-backup-cleanup-2", + PriorOperationID: "operation-backup-create-1", + CleanupPlanID: strings.Repeat("a", 64), + } + if err := ValidateCleanupBackupWorkspaceRequest(request); err != nil { + t.Fatalf("valid workspace cleanup rejected: %v", err) + } + request.OperationID = request.PriorOperationID + if err := ValidateCleanupBackupWorkspaceRequest(request); err == nil { + t.Fatal("cleanup accepted the original operation id as its successor") + } +} + func TestValidateRestoreBackupReusesCreateIntentWithoutReplacingCredentials(t *testing.T) { request := validRestoreBackupRequest(t) if err := ValidateRestoreBackupRequest(request); err != nil { @@ -139,13 +160,26 @@ func TestBackupAdmissionsFenceSourceAndNewVMArtifacts(t *testing.T) { backup := CreateBackupAdmission(uuid, backupRequest) for _, key := range []string{ domainLockKey(uuid), vmManagedVolumesLockKey(uuid), - volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, backupRequest.BackupKey)), + volumeLockKey(managedStoragePoolName, backupWorkspaceVolumeName(uuid, backupRequest.BackupKey)), catalogLockKey("backup", backupRequest.BackupKey), } { if !containsString(backup.LockKeys, key) || !backup.HeavyIO { t.Fatalf("backup admission missing %q: %#v", key, backup) } } + cleanup := CleanupBackupWorkspaceAdmission(uuid, backupRequest.BackupKey) + for _, key := range []string{ + domainLockKey(uuid), vmManagedVolumesLockKey(uuid), + volumeLockKey(managedStoragePoolName, + backupWorkspaceVolumeName(uuid, backupRequest.BackupKey)), + volumeLockKey(managedStoragePoolName, + snapshotVolumeName(uuid, backupRequest.BackupKey)), + catalogLockKey("backup", backupRequest.BackupKey), + } { + if !containsString(cleanup.LockKeys, key) || !cleanup.HeavyIO { + t.Fatalf("cleanup admission missing %q: %#v", key, cleanup) + } + } restore := RestoreBackupAdmission(validRestoreBackupRequest(t)) for _, key := range []string{ domainLockKey(uuid), vmManagedVolumesLockKey(uuid), diff --git a/agent/hypervisor/domain_xml.go b/agent/hypervisor/domain_xml.go index ee72f48a..560100d7 100644 --- a/agent/hypervisor/domain_xml.go +++ b/agent/hypervisor/domain_xml.go @@ -64,12 +64,17 @@ func isVMOwnedManagedVolumeName(uuid, name string) bool { return false } compactUUID := strings.ReplaceAll(strings.ToLower(uuid), "-", "") - prefix := "vsnap-" + compactUUID + "-" - if !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, ".qcow2") { - return false + for _, prefix := range []string{ + "vsnap-" + compactUUID + "-", + "vbackup-" + compactUUID + "-", + } { + if !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, ".qcow2") { + continue + } + key := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".qcow2") + return catalogIDPattern.MatchString(key) } - key := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".qcow2") - return catalogIDPattern.MatchString(key) + return false } func BuildDomainXML(definition DomainDefinition) (string, error) { diff --git a/agent/hypervisor/libvirt_backup.go b/agent/hypervisor/libvirt_backup.go index 55b97019..8ef4e65e 100644 --- a/agent/hypervisor/libvirt_backup.go +++ b/agent/hypervisor/libvirt_backup.go @@ -40,16 +40,16 @@ func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request } var snapshot Snapshot if active { - snapshot, err = l.CreateSnapshot(ctx, uuid, snapshotRequest) + snapshot, err = l.createBackupWorkspace(ctx, uuid, snapshotRequest) } else { - snapshot, err = l.createStoppedBackupSnapshot(ctx, uuid, snapshotRequest) + snapshot, err = l.createStoppedBackupWorkspace(ctx, uuid, snapshotRequest) } if err != nil { return BackupResult{}, err } volume, err := l.lookupOptionalManagedVolume( - l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, request.BackupKey), + l.storagePoolName(StorageRoleManaged), backupWorkspaceVolumeName(uuid, request.BackupKey), ) if err != nil || volume == nil { return BackupResult{}, operation.MarkUncertain(NewError( @@ -78,7 +78,7 @@ func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request return BackupResult{}, translated } cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx) - _, cleanupErr := l.DeleteSnapshot(cleanupCtx, uuid, request.BackupKey, SnapshotMutationRequest{OperationID: request.OperationID}) + cleanupErr := l.cleanupBackupWorkspace(cleanupCtx, uuid, request.BackupKey) cleanupCancel() if cleanupErr != nil { return BackupResult{}, classifyBackupUploadFailure(translated, cleanupErr) @@ -97,7 +97,7 @@ func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request Consistency: snapshot.Consistency, } cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx) - _, cleanupErr := l.DeleteSnapshot(cleanupCtx, uuid, request.BackupKey, SnapshotMutationRequest{OperationID: request.OperationID}) + cleanupErr := l.cleanupBackupWorkspace(cleanupCtx, uuid, request.BackupKey) cleanupCancel() if cleanupErr != nil { // The immutable remote object is already fully verified. Local temporary @@ -276,15 +276,22 @@ func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey stri if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil { return false, err } - existing, err := l.lookupOptionalManagedVolume( - l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, backupKey), - ) - if err != nil { - return false, err - } - if existing != nil { - existing.Free() - return false, NewError(ErrConflict, "backup_exists", "local backup work already exists", nil) + for _, volumeName := range []string{ + backupWorkspaceVolumeName(uuid, backupKey), + // Reject a legacy residue with the same key instead of silently + // allocating a second workspace for one logical backup. + snapshotVolumeName(uuid, backupKey), + } { + existing, err := l.lookupOptionalManagedVolume( + l.storagePoolName(StorageRoleManaged), volumeName, + ) + if err != nil { + return false, err + } + if existing != nil { + existing.Free() + return false, NewError(ErrConflict, "backup_exists", "local backup work already exists", nil) + } } active, err := domain.IsActive() if err != nil { @@ -400,13 +407,13 @@ func validateStandaloneQCOW2Volume(volume qcow2VolumeInspector, expectedCapacity return info, nil } -func (l *Libvirt) createStoppedBackupSnapshot(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) { +func (l *Libvirt) createStoppedBackupWorkspace(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) { domain, err := l.domainByUUID(ctx, uuid) if err != nil { return Snapshot{}, err } defer domain.Free() - document, definition, metadata, pool, source, err := l.managedRootContext(domain) + _, definition, metadata, pool, source, err := l.managedRootContext(domain) if err != nil { return Snapshot{}, err } @@ -436,7 +443,7 @@ func (l *Libvirt) createStoppedBackupSnapshot(ctx context.Context, uuid string, if sourceInfo.Capacity > poolInfo.Available { return Snapshot{}, NewError(ErrConflict, "backup_storage_full", "managed storage pool does not have enough free space", nil) } - volumeName := snapshotVolumeName(uuid, request.SnapshotKey) + volumeName := backupWorkspaceVolumeName(uuid, request.SnapshotKey) volumeXML, err := storageVolumeXMLAllocated(volumeName, sourceInfo.Capacity, "qcow2") if err != nil { return Snapshot{}, err @@ -472,24 +479,6 @@ func (l *Libvirt) createStoppedBackupSnapshot(ctx context.Context, uuid string, // previous shutdown was guest-coordinated, so it reports crash safety. actualConsistency = "crash" } - updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error { - stored.Snapshots = append(stored.Snapshots, domainSnapshotMetadataXML{ - Key: request.SnapshotKey, Name: request.Name, - Consistency: actualConsistency, SizeBytes: info.Capacity, CreatedAt: created.Unix(), - }) - return nil - }) - if err != nil { - return Snapshot{}, err - } - defined, err := l.conn.DomainDefineXML(updated) - if err != nil { - // DomainDefineXML may have committed before the reply failed; retain the - // volume because metadata could now reference it. - targetOwned = false - return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("persist offline backup metadata", err)) - } - defined.Free() targetOwned = false return Snapshot{ ID: request.SnapshotKey, SnapshotKey: request.SnapshotKey, diff --git a/agent/hypervisor/libvirt_backup_workspace.go b/agent/hypervisor/libvirt_backup_workspace.go new file mode 100644 index 00000000..acea4c43 --- /dev/null +++ b/agent/hypervisor/libvirt_backup_workspace.go @@ -0,0 +1,308 @@ +//go:build libvirt + +package hypervisor + +import ( + "context" + "errors" + "strings" + + "git.luoli.live/root/virt/agent/operation" + "libvirt.org/go/libvirt" +) + +// backupWorkspaceResidue resolves exactly one operation-owned portable-backup +// copy. New backups use vbackup-* without domain snapshot metadata. The +// legacy vsnap-* form is accepted only so an evidence-bound cleanup can retire +// residue produced before the workspace split. +func (l *Libvirt) backupWorkspaceResidue(uuid, key string) ( + name string, volume *libvirt.StorageVol, legacy bool, err error, +) { + workspaceName := backupWorkspaceVolumeName(uuid, key) + workspace, err := l.lookupOptionalManagedVolume( + l.storagePoolName(StorageRoleManaged), workspaceName, + ) + if err != nil { + return "", nil, false, err + } + legacyName := snapshotVolumeName(uuid, key) + legacyVolume, legacyErr := l.lookupOptionalManagedVolume( + l.storagePoolName(StorageRoleManaged), legacyName, + ) + if legacyErr != nil { + if workspace != nil { + workspace.Free() + } + return "", nil, false, legacyErr + } + if workspace != nil && legacyVolume != nil { + workspace.Free() + legacyVolume.Free() + return "", nil, false, NewError( + ErrConflict, "backup_workspace_ambiguous", + "both current and legacy backup workspaces are present", nil, + ) + } + if workspace != nil { + return workspaceName, workspace, false, nil + } + if legacyVolume != nil { + return legacyName, legacyVolume, true, nil + } + return workspaceName, nil, false, nil +} + +// cleanupBackupWorkspace is the bounded fast-path used immediately after an +// upload. It never mutates a user snapshot: current workspaces have no +// snapshot metadata, while a legacy workspace must match its exact metadata +// before the existing snapshot deletion contract is reused. +func (l *Libvirt) cleanupBackupWorkspace(ctx context.Context, uuid, key string) error { + domain, err := l.domainByUUID(ctx, uuid) + if err != nil { + return err + } + defer domain.Free() + _, metadata, err := requireManagedDomain(domain) + if err != nil { + return err + } + metadataFound := false + for _, stored := range metadata.Snapshots { + if stored.Key == key { + metadataFound = true + break + } + } + name, volume, legacy, err := l.backupWorkspaceResidue(uuid, key) + if err != nil { + return err + } + if volume == nil { + if metadataFound { + return NewError( + ErrConflict, "backup_workspace_inconsistent", + "backup workspace metadata remains without its volume", nil, + ) + } + return nil + } + volume.Free() + if legacy { + if !metadataFound { + return NewError( + ErrConflict, "backup_workspace_inconsistent", + "legacy backup workspace is missing its metadata", nil, + ) + } + _, err := l.DeleteSnapshot(ctx, uuid, key, SnapshotMutationRequest{}) + return err + } + if metadataFound { + return NewError( + ErrConflict, "backup_workspace_inconsistent", + "current backup workspace collides with user snapshot metadata", nil, + ) + } + if name != backupWorkspaceVolumeName(uuid, key) { + return NewError( + ErrConflict, "backup_workspace_identity_invalid", + "backup workspace identity is invalid", nil, + ) + } + return l.deleteUnreferencedBackupVolume(domain, uuid, key, name) +} + +func (l *Libvirt) deleteUnreferencedBackupVolume(domain *libvirt.Domain, + uuid, key, volumeName string) error { + if !catalogIDPattern.MatchString(key) { + return operation.MarkNotApplied(NewError( + ErrConflict, "backup_workspace_identity_invalid", + "backup workspace identity is invalid", nil, + )) + } + if volumeName != backupWorkspaceVolumeName(uuid, key) && + volumeName != snapshotVolumeName(uuid, key) { + return operation.MarkNotApplied(NewError( + ErrConflict, "backup_workspace_identity_invalid", + "backup workspace identity is invalid", nil, + )) + } + if domain != nil { + if err := ensureNoActiveDomainJobIfRunning(domain); err != nil { + return operation.MarkNotApplied(err) + } + } + volume, err := l.lookupOptionalManagedVolume( + l.storagePoolName(StorageRoleManaged), volumeName, + ) + if err != nil { + return operation.MarkNotApplied(err) + } + if volume == nil { + return nil + } + defer volume.Free() + if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil { + return operation.MarkNotApplied(err) + } + path, err := volume.GetPath() + if err != nil { + return operation.MarkNotApplied( + translateLibvirtError("resolve backup workspace path", err), + ) + } + referenced, err := l.anyDomainUsesVolume( + path, l.storagePoolName(StorageRoleManaged), volumeName, + ) + if err != nil { + return operation.MarkNotApplied(err) + } + if referenced { + return operation.MarkNotApplied(NewError( + ErrConflict, "backup_workspace_in_use", + "backup workspace is referenced by a domain", nil, + )) + } + if err := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil { + observed, inspectErr := l.lookupOptionalManagedVolume( + l.storagePoolName(StorageRoleManaged), volumeName, + ) + if observed != nil { + observed.Free() + } + if inspectErr != nil || observed != nil { + return operation.MarkUncertain(errors.Join( + mutationLibvirtError("delete backup workspace", err), inspectErr, + )) + } + } + return nil +} + +// CleanupBackupWorkspace executes one fresh evidence-bound plan. It is a +// successor operation: the original successful backup receipt remains +// immutable and continues to prove the remote object identity. +func (l *Libvirt) CleanupBackupWorkspace(ctx context.Context, uuid string, + request CleanupBackupWorkspaceRequest) (CleanupBackupWorkspaceResult, error) { + if err := ValidateCleanupBackupWorkspaceRequest(request); err != nil { + return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(err) + } + prior := request.PriorOperation + parts := strings.Split(prior.Action, ":") + if prior.OperationID != request.PriorOperationID || + !strings.EqualFold(prior.ResourceID, uuid) || len(parts) != 3 || + parts[0] != "create-backup" || !catalogIDPattern.MatchString(parts[1]) || + !successfulBackupCleanupPending(prior) { + return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError( + ErrConflict, "backup_workspace_cleanup_not_authorized", + "backup workspace cleanup requires the exact successful backup operation", nil, + )) + } + key := parts[1] + domain, err := l.domainByUUID(ctx, uuid) + domainAbsent := errors.Is(err, ErrNotFound) + if err != nil && !domainAbsent { + return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(err) + } + if domain != nil { + defer domain.Free() + } + + inspection := l.inspectSuccessfulBackupCleanupPending(ctx, prior) + if inspection.Cleanup.Available { + if request.CleanupPlanID == "" || !hashEqualsFold( + request.CleanupPlanID, inspection.Cleanup.PlanID, + ) { + return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError( + ErrConflict, "backup_workspace_cleanup_plan_changed", + "backup workspace cleanup plan changed; inspect again", nil, + )) + } + } else if request.CleanupPlanID != "" { + return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError( + ErrConflict, "backup_workspace_cleanup_plan_changed", + "backup workspace cleanup plan is no longer present", nil, + )) + } + + metadataFound := false + if !domainAbsent { + _, metadata, metadataErr := requireManagedDomain(domain) + if metadataErr != nil { + return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(metadataErr) + } + for _, stored := range metadata.Snapshots { + if stored.Key == key { + metadataFound = true + break + } + } + } + volumeName, volume, legacy, err := l.backupWorkspaceResidue(uuid, key) + if err != nil { + return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(err) + } + volumePresent := volume != nil + if volume != nil { + volume.Free() + } + if !volumePresent && !metadataFound { + return CleanupBackupWorkspaceResult{ + BackupKey: key, Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true, + }, nil + } + if !inspection.Cleanup.Available { + return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError( + ErrConflict, "backup_workspace_cleanup_unavailable", + "backup workspace is not currently safe to clean", nil, + )) + } + if metadataFound { + if volumePresent && !legacy { + return CleanupBackupWorkspaceResult{}, operation.MarkNotApplied(NewError( + ErrConflict, "backup_workspace_inconsistent", + "current backup workspace collides with user snapshot metadata", nil, + )) + } + if _, err := l.DeleteSnapshot( + ctx, uuid, key, SnapshotMutationRequest{OperationID: request.OperationID}, + ); err != nil { + return CleanupBackupWorkspaceResult{}, err + } + } else if volumePresent { + if err := l.deleteUnreferencedBackupVolume( + domain, uuid, key, volumeName, + ); err != nil { + return CleanupBackupWorkspaceResult{}, err + } + } + + _, remaining, _, err := l.backupWorkspaceResidue(uuid, key) + if err != nil { + return CleanupBackupWorkspaceResult{}, operation.MarkUncertain(err) + } + if remaining != nil { + remaining.Free() + return CleanupBackupWorkspaceResult{}, operation.MarkUncertain(NewError( + ErrConflict, "backup_workspace_cleanup_pending", + "backup workspace remains after cleanup", nil, + )) + } + if !domainAbsent { + _, current, metadataErr := requireManagedDomain(domain) + if metadataErr != nil { + return CleanupBackupWorkspaceResult{}, operation.MarkUncertain(metadataErr) + } + for _, stored := range current.Snapshots { + if stored.Key == key { + return CleanupBackupWorkspaceResult{}, operation.MarkUncertain(NewError( + ErrConflict, "backup_workspace_cleanup_pending", + "legacy backup metadata remains after cleanup", nil, + )) + } + } + } + return CleanupBackupWorkspaceResult{ + BackupKey: key, Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true, + }, nil +} diff --git a/agent/hypervisor/libvirt_connection.go b/agent/hypervisor/libvirt_connection.go index a7b47aba..196c994f 100644 --- a/agent/hypervisor/libvirt_connection.go +++ b/agent/hypervisor/libvirt_connection.go @@ -560,6 +560,12 @@ func (m *managedLibvirt) CreateBackup(ctx context.Context, uuid string, request }) } +func (m *managedLibvirt) CleanupBackupWorkspace(ctx context.Context, uuid string, request CleanupBackupWorkspaceRequest) (CleanupBackupWorkspaceResult, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (CleanupBackupWorkspaceResult, error) { + return h.CleanupBackupWorkspace(ctx, uuid, request) + }) +} + func (m *managedLibvirt) RestoreBackup(ctx context.Context, request RestoreBackupRequest) (RestoreBackupResult, error) { return managedCall(m, ctx, true, func(h Hypervisor) (RestoreBackupResult, error) { return h.RestoreBackup(ctx, request) diff --git a/agent/hypervisor/libvirt_operation_admin.go b/agent/hypervisor/libvirt_operation_admin.go index 43cd9fa1..fc9bb25b 100644 --- a/agent/hypervisor/libvirt_operation_admin.go +++ b/agent/hypervisor/libvirt_operation_admin.go @@ -136,12 +136,31 @@ func storedOperationInspectionAvailable(record operation.Record) bool { // inspectSuccessfulBackupCleanupPending preserves the immutable applied // object projection while adding an optional, read-only cleanup plan for an -// exact operation-owned local snapshot residue. Host read failures never +// exact operation-owned local workspace residue. Host read failures never // downgrade the already verified remote object identity. func (l *Libvirt) inspectSuccessfulBackupCleanupPending(ctx context.Context, record operation.Record) OperationInspection { inspection := inspectionFromRecord(record) domain, err := l.domainByUUID(ctx, record.ResourceID) + if errors.Is(err, ErrNotFound) { + parts := strings.Split(record.Action, ":") + if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) { + return inspection + } + volumeName, volume, _, residueErr := l.backupWorkspaceResidue( + record.ResourceID, parts[1], + ) + if residueErr != nil || volume == nil { + return inspection + } + defer volume.Free() + cleanup := l.absentDomainBackupWorkspaceCleanupInspection( + volume, volumeName, + ) + return appliedBackupInspectionWithCleanup( + record, inspection, cleanup, + ) + } if err != nil { return inspection } @@ -193,6 +212,8 @@ func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record opera } cleanup = plan resolution = inspectAbsentDeleteResolution(plan, record.Action == "force-delete-vm") + case strings.HasPrefix(record.Action, "cleanup-backup-workspace:"): + resolution = l.reconcileAbsentDomainBackupWorkspaceCleanup(record) default: if record.ExpectedState != nil { resolution = expectedStateResourceAbsent(record) @@ -201,6 +222,39 @@ func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record opera return buildOperationInspection(record, resolution, OperationAbortInspection{}, cleanup), nil } +func (l *Libvirt) reconcileAbsentDomainBackupWorkspaceCleanup( + record operation.Record, +) operation.Resolution { + parts := strings.Split(record.Action, ":") + if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) { + return operation.Resolution{} + } + _, volume, _, err := l.backupWorkspaceResidue( + record.ResourceID, parts[1], + ) + if err != nil { + return operation.Resolution{} + } + if volume == nil { + result, _ := json.Marshal(CleanupBackupWorkspaceResult{ + BackupKey: parts[1], Cleaned: true, + WorkspaceAbsent: true, MetadataAbsent: true, + }) + return operation.Resolution{ + Resolved: true, Succeeded: true, Result: result, + } + } + defer volume.Free() + if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil { + return operation.Resolution{} + } + return operation.Resolution{ + Resolved: true, Succeeded: false, + ErrorCode: "backup_workspace_cleanup_not_applied", + ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied", + } +} + func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePresent bool) (operation.Resolution, operationCleanupInspection) { if !managedArtifactsAbsent { return operation.Resolution{}, operationCleanupInspection{} @@ -265,6 +319,7 @@ func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record ope } if strings.HasPrefix(record.Action, "create-snapshot:") || strings.HasPrefix(record.Action, "create-backup:") || + strings.HasPrefix(record.Action, "cleanup-backup-workspace:") || strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:") || strings.HasPrefix(record.Action, "recover-backup:") { @@ -843,9 +898,18 @@ func (l *Libvirt) inspectSnapshotOperation(ctx context.Context, record operation break } } - volume, err := l.lookupOptionalManagedVolume( - l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, key), - ) + volumeName := snapshotVolumeName(record.ResourceID, key) + var volume *libvirt.StorageVol + legacyBackupVolume := false + if parts[0] == "create-backup" || parts[0] == "cleanup-backup-workspace" { + volumeName, volume, legacyBackupVolume, err = l.backupWorkspaceResidue( + record.ResourceID, key, + ) + } else { + volume, err = l.lookupOptionalManagedVolume( + l.storagePoolName(StorageRoleManaged), volumeName, + ) + } if err != nil { return operation.Resolution{}, operationCleanupInspection{} } @@ -856,12 +920,42 @@ func (l *Libvirt) inspectSnapshotOperation(ctx context.Context, record operation switch parts[0] { case "create-backup": + if stored != nil && volume != nil && + volumeName != snapshotVolumeName(record.ResourceID, key) { + return operation.Resolution{}, operationCleanupInspection{} + } if !backupResidueCleanupAllowed(record, stored) { return operation.Resolution{}, operationCleanupInspection{} } return operation.Resolution{}, l.backupResidueCleanupInspection( - domain, volume, snapshotVolumeName(record.ResourceID, key), key, stored, + domain, volume, volumeName, key, stored, ) + case "cleanup-backup-workspace": + if len(parts) != 2 { + return operation.Resolution{}, operationCleanupInspection{} + } + if stored != nil && volume != nil && + volumeName != snapshotVolumeName(record.ResourceID, key) { + return operation.Resolution{}, operationCleanupInspection{} + } + if stored == nil && !volumeExists { + result, _ := json.Marshal(CleanupBackupWorkspaceResult{ + BackupKey: key, Cleaned: true, + WorkspaceAbsent: true, MetadataAbsent: true, + }) + return operation.Resolution{ + Resolved: true, Succeeded: true, Result: result, + }, operationCleanupInspection{} + } + if (stored == nil && volumeExists && !legacyBackupVolume) || + (stored != nil && volumeExists && legacyBackupVolume) { + return operation.Resolution{ + Resolved: true, Succeeded: false, + ErrorCode: "backup_workspace_cleanup_not_applied", + ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied", + }, operationCleanupInspection{} + } + return operation.Resolution{}, operationCleanupInspection{} case "create-snapshot": if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") { return operation.Resolution{}, operationCleanupInspection{} @@ -1093,6 +1187,35 @@ func (l *Libvirt) backupResidueCleanupInspection(domain cleanupJobDomain, ) } +func (l *Libvirt) absentDomainBackupWorkspaceCleanupInspection( + volume *libvirt.StorageVol, volumeName string, +) operationCleanupInspection { + if volume == nil { + return operationCleanupInspection{} + } + if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil { + return operationCleanupInspection{} + } + digest, err := storageVolumeInspectionDigest(volume) + if err != nil { + return operationCleanupInspection{} + } + path, err := volume.GetPath() + if err != nil { + return operationCleanupInspection{} + } + referenced, err := l.anyDomainUsesVolume( + path, l.storagePoolName(StorageRoleManaged), volumeName, + ) + if err != nil || referenced { + return operationCleanupInspection{} + } + return newOperationCleanupInspection( + []string{CleanupStepDeleteSnapshotVolume}, + "domain-state:absent", "backup-volume:"+volumeName+":"+digest, + ) +} + func diskTransitionPowerStateMatches(metadata *domainMetadataXML, state libvirt.DomainState, kind, operationID string) bool { if !diskTransitionMarkerMatches(metadata, kind, operationID) { @@ -1299,9 +1422,7 @@ func (l *Libvirt) inspectMatchingBackupJob(record operation.Record, domain *libv if err != nil { return OperationAbortInspection{} } - volume, err := l.lookupOptionalManagedVolume( - l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, parts[1]), - ) + _, volume, _, err := l.backupWorkspaceResidue(record.ResourceID, parts[1]) if err != nil || volume == nil { return OperationAbortInspection{} } diff --git a/agent/hypervisor/libvirt_safety_test.go b/agent/hypervisor/libvirt_safety_test.go index 91f2952a..53a8dd04 100644 --- a/agent/hypervisor/libvirt_safety_test.go +++ b/agent/hypervisor/libvirt_safety_test.go @@ -1797,6 +1797,7 @@ func TestCollectVMManagedVolumeNamesIncludesTransitionsSnapshotsAndOrphans(t *te } inventory := []string{ root, pending, snapshotVolumeName(uuid, registeredKey), snapshotVolumeName(uuid, orphanKey), + backupWorkspaceVolumeName(uuid, orphanKey), orphanRestore, orphanReinstall, rootVolumeName(otherUUID), "template-debian-12.qcow2", } got, err := collectVMManagedVolumeNames(uuid, metadata, inventory) @@ -1805,6 +1806,7 @@ func TestCollectVMManagedVolumeNamesIncludesTransitionsSnapshotsAndOrphans(t *te } want := []string{ root, pending, snapshotVolumeName(uuid, registeredKey), snapshotVolumeName(uuid, orphanKey), + backupWorkspaceVolumeName(uuid, orphanKey), orphanRestore, orphanReinstall, } sort.Strings(want) diff --git a/agent/hypervisor/libvirt_snapshot_features.go b/agent/hypervisor/libvirt_snapshot_features.go index 16f65806..2cae1e67 100644 --- a/agent/hypervisor/libvirt_snapshot_features.go +++ b/agent/hypervisor/libvirt_snapshot_features.go @@ -438,6 +438,21 @@ func abortBackupAfterThawFailure(domain snapshotBackupJob, thawErr error, } func (l *Libvirt) CreateSnapshot(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) { + return l.createFullDiskCopy(ctx, uuid, request, true) +} + +// createBackupWorkspace creates the immutable full-system-disk copy used only +// by a portable backup operation. Unlike a user snapshot it is deliberately +// absent from domainMetadata.Snapshots, so a later local cleanup failure can +// consume storage and raise an alert without changing the VM snapshot catalog +// or blocking maintenance migration. +func (l *Libvirt) createBackupWorkspace(ctx context.Context, uuid string, + request SnapshotRequest) (_ Snapshot, resultErr error) { + return l.createFullDiskCopy(ctx, uuid, request, false) +} + +func (l *Libvirt) createFullDiskCopy(ctx context.Context, uuid string, + request SnapshotRequest, registerSnapshot bool) (_ Snapshot, resultErr error) { key := request.SnapshotKey if key == "" { key = request.SnapshotID @@ -486,6 +501,9 @@ func (l *Libvirt) CreateSnapshot(ctx context.Context, uuid string, request Snaps return Snapshot{}, snapshotPreconditionFailure(NewError(ErrConflict, "snapshot_storage_full", "managed storage pool does not have enough free space", nil)) } volumeName := snapshotVolumeName(uuid, key) + if !registerSnapshot { + volumeName = backupWorkspaceVolumeName(uuid, key) + } if existing, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), volumeName); lookupErr != nil { return Snapshot{}, snapshotPreconditionFailure(lookupErr) } else if existing != nil { @@ -575,18 +593,20 @@ func (l *Libvirt) CreateSnapshot(ctx context.Context, uuid string, request Snaps return Snapshot{}, operation.MarkUncertain(err) } created := time.Now().UTC() - updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error { - stored.Snapshots = append(stored.Snapshots, domainSnapshotMetadataXML{Key: key, Name: request.Name, Description: request.Description, Consistency: actualConsistency, SizeBytes: info.Capacity, CreatedAt: created.Unix()}) - return nil - }) - if err != nil { - return Snapshot{}, operation.MarkUncertain(err) - } - defined, err := l.conn.DomainDefineXML(updated) - if err != nil { - return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("persist snapshot metadata", err)) + if registerSnapshot { + updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error { + stored.Snapshots = append(stored.Snapshots, domainSnapshotMetadataXML{Key: key, Name: request.Name, Description: request.Description, Consistency: actualConsistency, SizeBytes: info.Capacity, CreatedAt: created.Unix()}) + return nil + }) + if err != nil { + return Snapshot{}, operation.MarkUncertain(err) + } + defined, err := l.conn.DomainDefineXML(updated) + if err != nil { + return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("persist snapshot metadata", err)) + } + defined.Free() } - defined.Free() return Snapshot{ID: key, SnapshotKey: key, Name: request.Name, Description: request.Description, CreatedAt: created, State: "ready", Consistency: actualConsistency, SizeBytes: info.Capacity, Created: true}, nil } diff --git a/agent/hypervisor/types.go b/agent/hypervisor/types.go index 0dfd05e3..ec830253 100644 --- a/agent/hypervisor/types.go +++ b/agent/hypervisor/types.go @@ -343,6 +343,23 @@ type CleanupPeerMigrationTargetRequest struct { PriorOperation operation.Record `json:"-"` } +// CleanupBackupWorkspaceRequest authorizes only the exact cleanup plan exposed +// by a prior successful create-backup operation. PriorOperation is injected by +// the Agent API from its immutable Journal and is never accepted from the wire. +type CleanupBackupWorkspaceRequest struct { + OperationID string `json:"operation_id,omitempty"` + PriorOperationID string `json:"prior_operation_id"` + CleanupPlanID string `json:"cleanup_plan_id,omitempty"` + PriorOperation operation.Record `json:"-"` +} + +type CleanupBackupWorkspaceResult struct { + BackupKey string `json:"backup_key"` + Cleaned bool `json:"cleaned"` + WorkspaceAbsent bool `json:"workspace_absent"` + MetadataAbsent bool `json:"metadata_absent"` +} + // RecoverBackupRequest replaces the managed root disk of an existing VM with // one immutable portable-backup object. Network and domain identity remain // attached to the existing VM, while the Agent reduces its NoCloud seed to @@ -830,6 +847,7 @@ type Hypervisor interface { GetVMMetadata(context.Context, string) (VMMetadata, error) CreateVM(context.Context, CreateVMRequest) (VMMetadata, error) CreateBackup(context.Context, string, BackupRequest) (BackupResult, error) + CleanupBackupWorkspace(context.Context, string, CleanupBackupWorkspaceRequest) (CleanupBackupWorkspaceResult, error) RestoreBackup(context.Context, RestoreBackupRequest) (RestoreBackupResult, error) RecoverBackup(context.Context, string, RecoverBackupRequest) (RecoverBackupResult, error) PreparePeerMigration(context.Context, string, PreparePeerMigrationRequest) (PreparePeerMigrationResult, error) @@ -905,6 +923,9 @@ func (Unsupported) CreateVM(context.Context, CreateVMRequest) (VMMetadata, error func (Unsupported) CreateBackup(context.Context, string, BackupRequest) (BackupResult, error) { return BackupResult{}, unsupported("VM backup") } +func (Unsupported) CleanupBackupWorkspace(context.Context, string, CleanupBackupWorkspaceRequest) (CleanupBackupWorkspaceResult, error) { + return CleanupBackupWorkspaceResult{}, unsupported("VM backup workspace cleanup") +} func (Unsupported) RestoreBackup(context.Context, RestoreBackupRequest) (RestoreBackupResult, error) { return RestoreBackupResult{}, unsupported("VM backup restore") } diff --git a/agent/operation/journal.go b/agent/operation/journal.go index 6e94ebe1..63fb7c1d 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -2672,9 +2672,37 @@ func validPeerMigrationTargetCleanupSupersession(spec Spec, lockKeys []string, p return lockKeysOverlap(lockKeys, predecessorKeys) } +func validBackupWorkspaceCleanupSupersession(spec Spec, lockKeys []string, predecessor Record) bool { + if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID || + !cleanupBackupActionPattern.MatchString(spec.Action) || + predecessor.OperationID != spec.SupersedesOperationID || + predecessor.ResourceID != spec.ResourceID || + predecessor.State != StateSucceeded || predecessor.ExecutionFenced || + !createBackupActionPattern.MatchString(predecessor.Action) { + return false + } + backupKey := strings.TrimPrefix(spec.Action, "cleanup-backup-workspace:") + priorParts := strings.Split(predecessor.Action, ":") + if len(priorParts) != 3 || priorParts[1] != backupKey { + return false + } + var result struct { + CleanupPending bool `json:"cleanup_pending"` + } + if json.Unmarshal(predecessor.Result, &result) != nil || !result.CleanupPending { + return false + } + predecessorKeys := effectiveLockKeys( + predecessor.OperationID, predecessor.ResourceID, + predecessor.Action, predecessor.LockKeys, + ) + return lockKeysOverlap(lockKeys, predecessorKeys) +} + func validExplicitSupersession(spec Spec, lockKeys []string, predecessor Record) bool { return validForceDeleteSupersession(spec, lockKeys, predecessor) || - validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor) + validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor) || + validBackupWorkspaceCleanupSupersession(spec, lockKeys, predecessor) } // findResourceBlocker validates any explicit force-delete predecessor and @@ -2989,7 +3017,9 @@ func validateSpec(spec Spec) error { if spec.SupersedesOperationID != "" { if !operationIDPattern.MatchString(spec.SupersedesOperationID) || spec.SupersedesOperationID == spec.OperationID || - (spec.Action != "force-delete-vm" && !cleanupPeerTargetActionPattern.MatchString(spec.Action)) { + (spec.Action != "force-delete-vm" && + !cleanupPeerTargetActionPattern.MatchString(spec.Action) && + !cleanupBackupActionPattern.MatchString(spec.Action)) { return ErrInvalidSupersession } } diff --git a/agent/operation/recovery_contract.go b/agent/operation/recovery_contract.go index 2bd95b5b..fe75c02c 100644 --- a/agent/operation/recovery_contract.go +++ b/agent/operation/recovery_contract.go @@ -18,6 +18,8 @@ var ( restoreBackupActionPattern = regexp.MustCompile(`^restore-backup:[a-f0-9]{32}$`) receivePeerActionPattern = regexp.MustCompile(`^receive-peer-migration:[a-f0-9]{32}$`) cleanupPeerTargetActionPattern = regexp.MustCompile(`^cleanup-peer-migration-target:[a-f0-9]{32}$`) + createBackupActionPattern = regexp.MustCompile(`^create-backup:[a-f0-9]{32}:(?:filesystem|crash)$`) + cleanupBackupActionPattern = regexp.MustCompile(`^cleanup-backup-workspace:[a-f0-9]{32}$`) ) //go:embed recovery_contract.json diff --git a/agent/operation/recovery_contract.json b/agent/operation/recovery_contract.json index a7f03f0b..1e18ef60 100644 --- a/agent/operation/recovery_contract.json +++ b/agent/operation/recovery_contract.json @@ -255,6 +255,19 @@ "not_applied" ] }, + { + "kind": "backup_workspace_cleanup", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [], + "action_patterns": [ + "^cleanup-backup-workspace:[a-f0-9]{32}$" + ], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, { "kind": "backup_recover", "recovery_mode": "generic", diff --git a/docs/AGENT_API.md b/docs/AGENT_API.md index b4987720..f3436617 100644 --- a/docs/AGENT_API.md +++ b/docs/AGENT_API.md @@ -187,7 +187,7 @@ Go context 无法可靠中止已进入 CGo/libvirt 的调用。父 Agent 到达 校验。远程 URL 导入不携带该字段,仍必须返回有效的 `Content-Length`。 任何导入源都不得使用压缩 `Content-Encoding`。 -不存在 mutation `reconcile`、`confirm-inspection`、`abort-job` 或 `cleanup` endpoint。`inspect` 不取得 mutation admission,不写 Journal,不继续原目标,也不能调用任何 libvirt 变更。不可变 Journal 成功或明确无副作用失败可以为策略表中的任何类型提供精确结论;其他已失败且影响未知的类型只有在该 action 的专用投影证据完整匹配时才能输出 `applied/not_applied`,否则保持 `partial/inconclusive`。例如实例设置只有在 Domain metadata 与 NoCloud seed 都匹配同一 transition ID 时才是 `applied`,两者都确认未提交时才是 `not_applied`。电源、网络链路或服务的“当前状态刚好符合目标”不是原 operation 的强证据,必须保持 `inconclusive`。 +不存在通用 mutation `reconcile`、`confirm-inspection`、`abort-job` 或任意资源 `cleanup` endpoint。`inspect` 不取得 mutation admission,不写 Journal,不继续原目标,也不能调用任何 libvirt 变更。唯一的备份工作区清理接口在下节说明:它只能消费原成功备份返回的最新 evidence-bound plan,不能接受路径、卷名或任意清理步骤。不可变 Journal 成功或明确无副作用失败可以为策略表中的任何类型提供精确结论;其他已失败且影响未知的类型只有在该 action 的专用投影证据完整匹配时才能输出 `applied/not_applied`,否则保持 `partial/inconclusive`。例如实例设置只有在 Domain metadata 与 NoCloud seed 都匹配同一 transition ID 时才是 `applied`,两者都确认未提交时才是 `not_applied`。电源、网络链路或服务的“当前状态刚好符合目标”不是原 operation 的强证据,必须保持 `inconclusive`。 ### 可移植备份与异地恢复 @@ -195,16 +195,16 @@ Go context 无法可靠中止已进入 CGo/libvirt 的调用。父 Agent 到达 `consistency_mode=filesystem|crash` 及 `object_upload`。`object_upload` 只有 短期预签名的 `part_urls`、`complete_url`、`abort_url`、只读 `verify_url`、part 大小及 总大小上限;Agent 永远不接收或保存 S3 access key。运行中 VM 使用 libvirt 完整 push backup, -关机 VM 使用稳定的独立克隆;文件系统冻结不可用时 `filesystem` 会保守降级并在 +关机 VM 使用稳定的独立克隆。两者都写入独立 `vbackup-*` 工作区卷,不注册到 +Domain 的正式 `Snapshots` 目录;文件系统冻结不可用时 `filesystem` 会保守降级并在 结果中报告 `crash`。首版只接受一个可写 `vda`,拒绝额外可写盘、救援挂载、 原生/外部快照或带 backing file 的系统盘。multipart 完成得到确定成功后删除本地 -临时快照。若完成响应丢失,Agent 只通过 `verify_url` 重新读取并核对对象大小及 +临时工作区。若完成响应丢失,Agent 只通过 `verify_url` 重新读取并核对对象大小及 SHA-256;完全匹配才继续成功清理,不匹配或暂不可读时保留本地快照和 effect fence, 绝不自动重放、Abort 或误判为未执行。普通上传失败或取消在远端 Abort 已确认后, 使用独立的两分钟本地清理上下文删除临时卷;只有进程硬崩溃、远端完成不明确或 清理失败才可能留下本地副本。read-only operation inspection 可输出只覆盖该 -operation 临时卷/metadata 的 evidence-bound cleanup plan,但当前没有自动执行 -cleanup 的 mutation endpoint。 +operation 工作区卷(以及旧版本遗留的同 key 快照 metadata)的 evidence-bound cleanup plan。 成功结果只返回 `backup_key`、对象 `size_bytes`、qcow2 `virtual_size_bytes`、`sha256`、multipart 完成响应中的可选 `object_etag` 与 @@ -212,8 +212,13 @@ cleanup 的 mutation endpoint。 可选 `cleanup_pending=true`,不返回预签名 URL 或本机路径。本地清理失败不会丢弃 已经验证成功的远端对象身份,也不会把可恢复的备份错误降级为远端效果不明。 后续只读 inspection 在残留仍精确归属于该 operation 且没有活跃 libvirt/block job 时, -会同时保留 `applied` 对象投影并附带 cleanup plan;该 plan 只用于管理员核对,首版仍不 -提供执行它的 mutation endpoint。 +会同时保留 `applied` 对象投影并附带 cleanup plan。超级管理员可为一个新的 operation ID +调用 `POST /v1/operations/{cleanup_operation_id}/cleanup-backup-workspace`,正文只含 +`prior_operation_id` 与刚读取的 `cleanup_plan_id`。Agent 在 mutation admission 内重新读取 +原 Journal、活动 block job、精确卷摘要、引用关系和 metadata;计划变化即拒绝。成功结果必须 +同时返回 `cleaned=true`、`workspace_absent=true`、`metadata_absent=true`。原备份成功回执不会 +被改写;清理 operation 通过 `supersedes_operation_id` 保留审计关系。若第一次响应丢失,管理员 +必须重新 inspect 后创建新的清理 operation,不能盲目重放旧删除请求。 action 为 `create-backup:{backup_key}:{consistency}`。 维护迁移不复用备份 endpoint。`POST /v1/vms/{uuid}/peer-migration/prepare` diff --git a/docs/PORTABLE_VM_BACKUPS.md b/docs/PORTABLE_VM_BACKUPS.md index 4d8762b0..8fa7ac32 100644 --- a/docs/PORTABLE_VM_BACKUPS.md +++ b/docs/PORTABLE_VM_BACKUPS.md @@ -68,7 +68,7 @@ PeerMigration 不创建 `vm-migrations/` 对象,也不持有 S3 凭据。目 备份和恢复均为异步 HeavyIO 操作,单次宿主机 worker 最长可运行 24 小时。控制面只会派发一次同一 operation ID:超时、断连或回执丢失均视为效果未知,并通过 Agent Journal 查询原操作,不会重新执行备份或创建第二台 VM。 -若远端对象已经完整提交并校验,但源宿主机的临时快照清理失败,Agent 会保留对象大小、SHA-256 和一致性结果,把备份标记为可恢复并单独提示本地清理待核对;该提示不会触发重新上传或删除远端对象。 +普通云备份使用独立 `vbackup-*` 工作区卷,不把临时副本加入用户快照目录。因此即使远端对象已经完整提交后本地删除暂时失败,维护迁移仍只读取正式快照,不会被该残留阻断。Agent 会保留对象大小、SHA-256 和一致性结果,把备份标记为可恢复并单独提示本地工作区待清理;该提示不会触发重新上传或删除远端对象。超级管理员的“核对并清理临时工作区”会先读取原成功 operation 的最新计划,再以新的 operation ID 执行精确清理;计划摘要、卷身份、引用关系或活动作业任一变化都会拒绝执行。实例已经迁移或源 Domain 已删除时,清理仍绑定备份冻结的源宿主机和 UUID,只能在确认卷未被任何 Domain 引用后删除;若卷已经随源实例清理消失,则只确认缺席并解除提示。旧版本留下的 `vsnap-*` 临时卷和对应 metadata 也只能通过同一证据绑定入口清理。 恢复失败时,只有 Agent 明确证明新 Domain、所有 UUID 所属卷、seed、NVRAM、网络过滤器和流量状态均不存在,控制面才能释放容量与公网 IP。只要清理结果不确定,资源保持锁定等待人工核对。 diff --git a/web/app/Controllers/CloudAdminInstanceController.php b/web/app/Controllers/CloudAdminInstanceController.php index 67e47602..f9d5ab57 100644 --- a/web/app/Controllers/CloudAdminInstanceController.php +++ b/web/app/Controllers/CloudAdminInstanceController.php @@ -183,6 +183,21 @@ final class CloudAdminInstanceController )->withHeader('Cache-Control', 'no-store'); } + public function cleanupLibraryBackupWorkspace($request, $response, $args) + { + $actor = $this->actor($request, true); + $backupId = filter_var($args['backup_id'] ?? null, + FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); + $backup = $backupId === false ? null + : CloudInstanceBackup::find((int) $backupId); + if ($backup === null || (string) $backup->state === 'deleted') { + throw new CloudError('备份不存在', 404); + } + $job = CloudBackupService::cleanupWorkspace($backup, $actor); + return $response->withJson(CloudJobService::acknowledgement($job), 202) + ->withHeader('Cache-Control', 'no-store'); + } + /** Queue a new independent VM restored from a ready portable backup. */ public function restoreBackup($request, $response, $args) { diff --git a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php index 3f93bbc5..f5efd008 100644 --- a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php @@ -146,6 +146,8 @@ final class CloudAdminOperationRecoveryService 'snapshot_not_created' => '快照没有创建', 'snapshot_not_deleted' => '快照仍然存在', 'snapshot_not_restored' => '系统盘没有切换到快照', + 'backup_workspace_cleanup_not_applied' => + '备份临时工作区仍完整存在,清理没有生效', 'storage_configuration_mismatch' => '固定存储池与面板配置不一致', 'storage_configuration_incomplete' => '存储池配置尚未完成', 'import_not_completed' => '目录资源导入未完成', @@ -1029,6 +1031,14 @@ final class CloudAdminOperationRecoveryService if ($kind === 'backup_create') { return self::BACKUP_PROJECTION; } + if ($kind === 'backup_workspace_cleanup') { + return [ + 'backup_key' => 'string', + 'cleaned' => 'boolean', + 'workspace_absent' => 'boolean', + 'metadata_absent' => 'boolean', + ]; + } if ($kind === 'backup_recover') { return [ 'backup_key' => 'string', @@ -1118,6 +1128,10 @@ final class CloudAdminOperationRecoveryService 'backup_key', 'size_bytes', 'virtual_size_bytes', 'sha256', 'consistency', ], + 'backup_workspace_cleanup' => [ + 'backup_key', 'cleaned', 'workspace_absent', + 'metadata_absent', + ], 'backup_recover' => [ 'backup_key', 'sha256', 'restored', 'power_state', ], diff --git a/web/app/Services/Cloud/CloudBackupService.php b/web/app/Services/Cloud/CloudBackupService.php index c628c5b9..d7faf3d7 100644 --- a/web/app/Services/Cloud/CloudBackupService.php +++ b/web/app/Services/Cloud/CloudBackupService.php @@ -1322,6 +1322,143 @@ final class CloudBackupService self::finishRestoreLocked($job, $status, $result, $payload); } elseif ($job->kind === 'backup_recover') { self::finishRecoveryLocked($job, $status, $result, $payload); + } elseif ($job->kind === 'backup_workspace_cleanup') { + self::finishWorkspaceCleanupLocked( + $job, $status, $result, $payload + ); + } + } + + /** Queue one evidence-bound cleanup of a source-host backup workspace. */ + public static function cleanupWorkspace( + CloudInstanceBackup $backup, + array $actor + ): CloudJob { + CloudAccess::manage($actor); + if (($actor['is_super_admin'] ?? false) !== true) { + throw new CloudError('只有超级管理员可以清理备份工作区', 403); + } + $original = CloudJob::where( + 'operation_id', (string) $backup->last_operation_id + )->first(); + if ($original === null || (string) $original->kind !== 'backup_create' + || (string) $original->phase !== 'terminal' + || (string) $original->outcome !== 'succeeded' + || (int) $original->backup_id !== (int) $backup->id + || (int) $original->host_id !== (int) $backup->source_host_id) { + throw new CloudError('原备份任务身份不完整,不能清理工作区', 409); + } + $inspection = CloudAdminOperationRecoveryService::inspect( + $actor, (string) $original->id + ); + $cleanup = is_array($inspection['cleanup'] ?? null) + ? $inspection['cleanup'] : []; + $planId = ($cleanup['available'] ?? false) === true + && is_string($cleanup['plan_id'] ?? null) + ? strtolower((string) $cleanup['plan_id']) : ''; + if ($planId !== '' + && preg_match('/^[a-f0-9]{64}$/D', $planId) !== 1) { + throw new CloudError('Agent 返回的清理计划身份无效', 502); + } + + return $backup->getConnection()->transaction(function () use ( + $backup, $actor, $original, $planId + ): CloudJob { + $locked = CloudInstanceBackup::where('id', (int) $backup->id) + ->lockForUpdate()->first(); + if ($locked === null || (string) $locked->state !== 'ready' + || (string) $locked->error_code + !== 'backup_local_cleanup_pending' + || !hash_equals( + (string) $locked->last_operation_id, + (string) $original->operation_id + )) { + throw new CloudError('备份工作区状态已变化,请刷新后重试', 409); + } + $operationId = bin2hex(random_bytes(16)); + $target = '/v1/operations/' . rawurlencode($operationId) + . '/cleanup-backup-workspace'; + $body = [ + 'prior_operation_id' => (string) $original->operation_id, + 'cleanup_plan_id' => $planId, + ]; + $canonical = [ + 'backup_id' => (int) $locked->id, + 'backup_key' => (string) $locked->backup_key, + 'prior_operation_id' => (string) $original->operation_id, + 'cleanup_plan_id' => $planId, + ]; + [$job, $created] = CloudJobService::reserveStandaloneAgentMutation( + $actor, + (int) $locked->source_host_id, + 'backup_workspace_cleanup', + 'POST', + $target, + $body, + [ + 'owner_id' => (int) $locked->user_id, + 'can_manage' => true, + 'backup_key' => (string) $locked->backup_key, + 'prior_job_id' => (int) $original->id, + 'prior_operation_id' => (string) $original->operation_id, + 'cleanup_plan_id' => $planId, + ], + $canonical, + $operationId, + '/v1/backups/' . (int) $locked->id . '/cleanup-workspace' + ); + if ($created) { + $job->instance_id = (int) $locked->source_instance_id; + $job->backup_id = (int) $locked->id; + $job->save(); + } + return $job; + }, 3); + } + + private static function finishWorkspaceCleanupLocked( + CloudJob $job, + string $status, + $result, + array $payload + ): void { + $backup = $job->backup_id === null ? null + : CloudInstanceBackup::where('id', (int) $job->backup_id) + ->lockForUpdate()->first(); + if ($backup === null + || (int) $backup->source_host_id !== (int) $job->host_id + || (int) $backup->source_instance_id !== (int) $job->instance_id + || !is_string($payload['prior_operation_id'] ?? null) + || !hash_equals( + (string) $backup->last_operation_id, + (string) $payload['prior_operation_id'] + ) + || !hash_equals( + (string) $backup->backup_key, + (string) ($payload['backup_key'] ?? '') + )) { + throw new CloudRecoveryInstanceStateConflict( + '备份工作区清理任务与目录账本不一致', 409 + ); + } + if ($status !== 'succeeded') { + return; + } + if (!is_array($result) + || !hash_equals( + (string) $backup->backup_key, + (string) ($result['backup_key'] ?? '') + ) + || ($result['cleaned'] ?? null) !== true + || ($result['workspace_absent'] ?? null) !== true + || ($result['metadata_absent'] ?? null) !== true) { + throw new CloudError('Agent 清理成功结果缺少完整缺席证据', 502); + } + if ((string) $backup->error_code + === 'backup_local_cleanup_pending') { + $backup->error_code = ''; + $backup->updated_at = time(); + $backup->save(); } } @@ -2153,6 +2290,9 @@ final class CloudBackupService ) ?? 'unknown'), 'cleanup_pending' => (string) $backup->error_code === 'backup_local_cleanup_pending', + 'can_cleanup_workspace' => $canManage + && (string) $backup->error_code + === 'backup_local_cleanup_pending', 'can_delete' => $canDelete, 'delete_action' => in_array($state, ['deleting', 'delete_uncertain'], true) diff --git a/web/app/Services/Cloud/CloudOperationCatalog.php b/web/app/Services/Cloud/CloudOperationCatalog.php index fd490e47..19865e94 100644 --- a/web/app/Services/Cloud/CloudOperationCatalog.php +++ b/web/app/Services/Cloud/CloudOperationCatalog.php @@ -25,6 +25,7 @@ final class CloudOperationCatalog 'snapshot_restore' => '恢复快照', 'snapshot_delete' => '删除快照', 'backup_create' => '创建可移植备份', + 'backup_workspace_cleanup' => '清理备份工作区', 'backup_restore' => '从备份创建实例', 'backup_recover' => '从云备份恢复实例', 'instance_migration_prepare' => '准备维护迁移源实例', @@ -74,6 +75,7 @@ final class CloudOperationCatalog 'snapshot_restore', 'snapshot_delete', 'backup_create', + 'backup_workspace_cleanup', 'backup_restore', 'backup_recover', 'instance_migration_prepare', diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php index e407e0b7..9a7b1e59 100644 --- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php +++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php @@ -505,6 +505,7 @@ final class CloudOperationErrorCatalog 'snapshot_restore' => '恢复快照失败;任务已结束且不会自动重试', 'snapshot_delete' => '删除快照失败;任务已结束且不会自动重试', 'backup_create' => '创建云备份失败;任务已结束且不会自动重试', + 'backup_workspace_cleanup' => '清理备份工作区失败;远端备份不受影响,可重新核对后创建新的清理任务', 'backup_restore' => '从备份创建实例失败;任务已结束且不会自动重试', 'backup_recover' => '从云备份恢复实例失败;任务已结束且不会自动重试', 'instance_migration_prepare' => '维护迁移源准备失败;请按迁移恢复出口处理', @@ -631,6 +632,7 @@ final class CloudOperationErrorCatalog 'service_was_running', 'service_was_running_hint', 'autostart', 'managed', 'created', 'restored', 'deleted', 'reinstall_committed', 'cleanup_pending', 'forced', + 'cleaned', 'workspace_absent', 'metadata_absent', 'source_stopped', 'source_was_running', 'source_autostart_disabled', 'managed_artifacts_absent', 'active', 'staged', @@ -705,10 +707,11 @@ final class CloudOperationErrorCatalog 'source_autostart_disabled', 'autostart', 'managed', 'created', 'restored', 'deleted', 'forced', 'managed_artifacts_absent', 'active', 'staged', + 'cleaned', 'workspace_absent', 'metadata_absent', 'pending_manual_restart'], true)) { return null; } - if ($key === 'snapshot_key' + if (in_array($key, ['snapshot_key', 'backup_key'], true) && preg_match('/^[a-f0-9]{32}$/D', $value) !== 1) { return null; } diff --git a/web/app/Services/Cloud/CloudOperationRecoveryContract.php b/web/app/Services/Cloud/CloudOperationRecoveryContract.php index 3d2a308f..16fbba0e 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryContract.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryContract.php @@ -140,6 +140,13 @@ final class CloudOperationRecoveryContract 'action_patterns' => ['^create-backup:[a-f0-9]{32}:(?:filesystem|crash)$'], 'system_resolutions' => ['applied', 'not_applied'], ], + 'backup_workspace_cleanup' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => [], + 'action_patterns' => ['^cleanup-backup-workspace:[a-f0-9]{32}$'], + 'system_resolutions' => ['applied', 'not_applied'], + ], 'backup_recover' => [ 'recovery_mode' => 'generic', 'identity_scope' => 'instance', diff --git a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php index 4288eefb..b9af3ecb 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php @@ -261,6 +261,11 @@ final class CloudOperationRecoveryIdentity bool $forAppliedEffect ): array { $kind = (string) $job->kind; + if ($kind === 'backup_workspace_cleanup') { + return self::backupWorkspaceCleanupAction( + $job, $payload, $method, $target, $body + ); + } if (in_array($kind, ['instance_migration_prepare', 'instance_migration_transfer', 'instance_migration_target_clean', @@ -1591,6 +1596,69 @@ final class CloudOperationRecoveryIdentity return 'restore-backup:' . $key; } + private static function backupWorkspaceCleanupAction( + CloudJob $job, + array $payload, + string $method, + string $target, + array $body + ): array { + self::expectRequest( + $method, + $target, + 'POST', + '/v1/operations/' . rawurlencode((string) $job->operation_id) + . '/cleanup-backup-workspace' + ); + self::assertAllowedBody($body, [ + 'operation_id', 'prior_operation_id', 'cleanup_plan_id', + ]); + $backupId = filter_var($job->backup_id, FILTER_VALIDATE_INT, + ['options' => ['min_range' => 1]]); + $backup = $backupId === false ? null + : CloudInstanceBackup::find((int) $backupId); + $priorJobId = filter_var($payload['prior_job_id'] ?? null, + FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); + $prior = $priorJobId === false ? null + : CloudJob::find((int) $priorJobId); + $key = $payload['backup_key'] ?? null; + $planId = $body['cleanup_plan_id'] ?? null; + if ($backup === null || $prior === null + || (int) $backup->source_instance_id !== (int) $job->instance_id + || (int) $backup->source_host_id !== (int) $job->host_id + || (int) ($payload['owner_id'] ?? 0) !== (int) $backup->user_id + || ((int) $job->user_id !== (int) $backup->user_id + && ($payload['can_manage'] ?? false) !== true) + || (int) $prior->backup_id !== (int) $backup->id + || (int) $prior->instance_id + !== (int) $backup->source_instance_id + || (int) $prior->host_id !== (int) $job->host_id + || (string) $prior->kind !== 'backup_create' + || (string) $prior->phase !== 'terminal' + || (string) $prior->outcome !== 'succeeded' + || !is_string($key) + || preg_match(self::CATALOG_ID_PATTERN, $key) !== 1 + || !hash_equals((string) $backup->backup_key, $key) + || !hash_equals((string) $backup->last_operation_id, + (string) $prior->operation_id) + || !hash_equals((string) $prior->operation_id, + (string) ($payload['prior_operation_id'] ?? '')) + || !hash_equals((string) $prior->operation_id, + (string) ($body['prior_operation_id'] ?? '')) + || !is_string($planId) + || ($planId !== '' + && preg_match('/^[a-f0-9]{64}$/D', $planId) !== 1) + || !hash_equals((string) ($payload['cleanup_plan_id'] ?? ''), + $planId)) { + self::invalidPayload('备份工作区清理请求无效'); + } + $resource = strtolower((string) $backup->source_uuid); + if (preg_match(self::UUID_PATTERN, $resource) !== 1) { + self::invalidPayload('备份工作区清理资源身份无效'); + } + return ['cleanup-backup-workspace:' . $key, $resource]; + } + private static function backupRecoverAction( CloudJob $job, array $payload, diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index 452eef9b..422b4cf9 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -97,6 +97,8 @@ $app->group('/api/v1', function () { 'App\Controllers\CloudAdminInstanceController:restoreLibraryBackup'); $this->delete('/admin/backups/{backup_id:[0-9]+}', 'App\Controllers\CloudAdminInstanceController:deleteLibraryBackup'); + $this->post('/admin/backups/{backup_id:[0-9]+}/cleanup-workspace', + 'App\Controllers\CloudAdminInstanceController:cleanupLibraryBackupWorkspace'); $this->map(['GET', 'POST'], '/admin/instances/{id:[0-9]+}/backups', 'App\Controllers\CloudAdminInstanceController:backups'); $this->post('/admin/instances/{id:[0-9]+}/maintenance-migrations', diff --git a/web/public/cloud/admin-instances.js b/web/public/cloud/admin-instances.js index 5f3cebc3..e9996cf6 100644 --- a/web/public/cloud/admin-instances.js +++ b/web/public/cloud/admin-instances.js @@ -796,6 +796,35 @@ function backupDeleteButton(vmId, backup, library = false) { return button; } +function backupWorkspaceCleanupButton(vmId, backup, library = false) { + const button = el('button', 'btn btn-outline-warning btn-sm', '核对并清理临时工作区'); + button.type = 'button'; + button.addEventListener('click', async () => { + if (state.backupMutationRequest || backup.can_cleanup_workspace !== true) return; + const controller = new AbortController(); + state.backupMutationRequest = controller; button.disabled = true; + backupSurfaceNotice('正在读取原备份操作的最新宿主机证据并创建精确清理任务。', false, library); + try { + const receipt = await mutate( + `/api/v1/admin/backups/${encodeURIComponent(backup.id)}/cleanup-workspace`, + {}, controller, 'POST', + ); + backupSurfaceNotice(`工作区清理任务 #${receipt.task_id || '—'} 已接收。`, false, library); + if (!library) watchBackupTask(vmId, receipt, '清理备份工作区'); + else announceTaskAccepted(receipt, { title: '清理备份工作区' }); + refreshBackupSurface(vmId, library); + } catch (error) { + if (controller.signal.aborted && controller.signal.reason !== 'timeout') return; + backupSurfaceNotice(error.message, true, library); + refreshBackupSurface(vmId, library); + } finally { + if (state.backupMutationRequest === controller) state.backupMutationRequest = null; + button.disabled = false; + } + }); + return button; +} + function watchBackupTask(vmId, receipt, title) { const serial = state.backupSerial; const taskId = receipt?.task_id || '—'; @@ -1235,7 +1264,12 @@ function renderBackups(payload, vmId) { if (backup.error) body.append(el('p', 'text-danger fs-13', backup.error)); if (backup.cleanup_pending === true) { body.append(el('p', 'text-warning fs-13', - '远端备份已完整保存并可恢复,但源宿主机的临时快照清理仍需管理员核对;系统不会重复上传。')); + '远端备份已完整保存并可恢复,但源宿主机的临时工作区仍待清理;清理不会重复上传或修改备份对象。')); + if (backup.can_cleanup_workspace === true) { + const cleanupActions = el('div', 'd-flex justify-content-end mb-3'); + cleanupActions.append(backupWorkspaceCleanupButton(vmId, backup)); + body.append(cleanupActions); + } } if (backup.state === 'uncertain') { body.append(el('p', 'alert alert-warning fs-13 mb-0', @@ -1334,7 +1368,12 @@ function renderBackupLibrary(payload) { } if (backup.cleanup_pending === true) { body.append(el('p', 'text-warning fs-13', - '远端备份可恢复,但源宿主机的临时快照清理仍需核对。')); + '远端备份可恢复,但源宿主机的临时工作区仍需核对。')); + if (backup.can_cleanup_workspace === true) { + const cleanupActions = el('div', 'd-flex justify-content-end mb-3'); + cleanupActions.append(backupWorkspaceCleanupButton(vmId, backup, true)); + body.append(cleanupActions); + } } if (backup.error) body.append(el('p', 'text-danger fs-13', backup.error)); const targets = backupTargets(payload, backup); diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index ba78139f..e871cf4b 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026102802' scope='global'} +{assign var=venix_asset_version value='2026102803' scope='global'} diff --git a/web/tests/CloudOperationRecoveryMatrixTest.php b/web/tests/CloudOperationRecoveryMatrixTest.php index f426e762..2edf88e4 100644 --- a/web/tests/CloudOperationRecoveryMatrixTest.php +++ b/web/tests/CloudOperationRecoveryMatrixTest.php @@ -1364,6 +1364,136 @@ namespace { ); } + $cleanupBackupId = 205; + $cleanupBackupKey = hash('md5', + 'recovery-matrix-backup:' . $cleanupBackupId); + $cleanupPlanId = str_repeat('8', 64); + $cleanupPrior = $insertJob( + 'backup_create', + static function (int $jobId, string $operationId) use ( + $instanceUuid, + $ownerId, + $cleanupBackupKey + ): array { + return [ + 'method' => 'POST', + 'request_target' => '/v1/vms/' + . rawurlencode($instanceUuid) . '/backups', + 'body' => [ + 'operation_id' => $operationId, + 'backup_key' => $cleanupBackupKey, + 'name' => 'matrix-cleanup-source', + 'consistency_mode' => 'filesystem', + 'object_upload' => [ + 'part_size_bytes' => 8388608, + 'maximum_size_bytes' => 42949672960, + 'part_urls' => [ + 'https://objects.invalid/part-1', + ], + 'complete_url' => + 'https://objects.invalid/complete', + 'abort_url' => 'https://objects.invalid/abort', + 'verify_url' => 'https://objects.invalid/verify', + ], + ], + 'owner_id' => $ownerId, + 'can_manage' => true, + 'backup_key' => $cleanupBackupKey, + 'maximum_size_bytes' => 42949672960, + ]; + }, + $instanceId, + null, + null, + null, + 'agent', + $ownerId, + $cleanupBackupId + ); + Capsule::table('cloud_jobs')->where('id', $cleanupPrior['id'])->update([ + 'outcome' => 'succeeded', + 'status' => 'succeeded', + 'effect_may_have_applied' => 0, + 'safe_to_resubmit' => 0, + 'effect_resolution' => 'applied', + 'result_json' => json_encode([ + 'backup_key' => $cleanupBackupKey, + 'size_bytes' => 1073741824, + 'virtual_size_bytes' => 42949672960, + 'sha256' => str_repeat('6', 64), + 'consistency' => 'filesystem', + 'cleanup_pending' => true, + ], JSON_THROW_ON_ERROR), + ]); + Capsule::table('cloud_instance_backups')->insert([ + 'id' => $cleanupBackupId, + 'backup_key' => $cleanupBackupKey, + 'source_instance_id' => $instanceId, + 'source_host_id' => $hostId, + 'user_id' => $ownerId, + 'source_uuid' => $instanceUuid, + 'name' => 'matrix-cleanup-source', + 'state' => 'ready', + 'consistency' => 'filesystem', + 'object_key' => 'vm-backups/' . $cleanupBackupKey . '/system.qcow2', + 'object_size_bytes' => 1073741824, + 'virtual_size_bytes' => 42949672960, + 'sha256' => str_repeat('6', 64), + 'spec_json' => '{}', + 'sealed_storage_runtime' => 'sealed-runtime', + 'last_operation_id' => $cleanupPrior['operation_id'], + 'error_code' => 'backup_local_cleanup_pending', + 'created_at' => $now - 180, + 'updated_at' => $now - 20, + ]); + $cleanupRecord = $insertJob( + 'backup_workspace_cleanup', + static function (int $jobId, string $operationId) use ( + $ownerId, + $cleanupBackupKey, + $cleanupPlanId, + $cleanupPrior + ): array { + return [ + 'method' => 'POST', + 'request_target' => '/v1/operations/' + . rawurlencode($operationId) + . '/cleanup-backup-workspace', + 'body' => [ + 'operation_id' => $operationId, + 'prior_operation_id' => $cleanupPrior['operation_id'], + 'cleanup_plan_id' => $cleanupPlanId, + ], + 'owner_id' => $ownerId, + 'can_manage' => true, + 'backup_key' => $cleanupBackupKey, + 'prior_job_id' => $cleanupPrior['id'], + 'prior_operation_id' => $cleanupPrior['operation_id'], + 'cleanup_plan_id' => $cleanupPlanId, + ]; + }, + $instanceId, + null, + null, + null, + 'agent', + $ownerId, + $cleanupBackupId + ); + $addFixture( + 'backup_workspace_cleanup', + 'cleanup', + $cleanupRecord, + 'cleanup-backup-workspace:' . $cleanupBackupKey, + $instanceUuid, + [ + 'backup_key' => $cleanupBackupKey, + 'cleaned' => true, + 'workspace_absent' => true, + 'metadata_absent' => true, + ] + ); + $recoverBackupId = 204; $recoverBackupKey = hash('md5', 'recovery-matrix-backup:' . $recoverBackupId); @@ -2296,6 +2426,23 @@ namespace { 'An upgrade with the wrong final power state crossed the projection boundary.' ); + $workspaceCleanupFixture = $fixtures['backup_workspace_cleanup'][0]; + Capsule::table('cloud_instances')->where('id', $instanceId)->update([ + 'host_id' => $hostId + 1, + 'status' => 'deleted', + ]); + $migratedCleanupIdentity = CloudOperationRecoveryIdentity:: + expectedFromPayload( + $workspaceCleanupFixture['job'], + $workspaceCleanupFixture['payload'] + ); + $assert($migratedCleanupIdentity['resource_id'] === $instanceUuid, + 'Backup workspace cleanup lost its frozen source-host identity after migration.'); + Capsule::table('cloud_instances')->where('id', $instanceId)->update([ + 'host_id' => $hostId, + 'status' => 'active', + ]); + foreach ($fixtures as $kind => $kindFixtures) { $entry = $manifestOperations[$kind]; $assert(CloudOperationRecoveryPolicy::supports($kind) diff --git a/web/tests/CloudOperationRecoveryPolicyTest.php b/web/tests/CloudOperationRecoveryPolicyTest.php index 8f698ba0..bbbfbb98 100644 --- a/web/tests/CloudOperationRecoveryPolicyTest.php +++ b/web/tests/CloudOperationRecoveryPolicyTest.php @@ -67,6 +67,7 @@ namespace { 'snapshot_restore', 'snapshot_delete', 'backup_create', + 'backup_workspace_cleanup', 'backup_recover', 'firewall_apply', 'service_suspend', @@ -82,8 +83,8 @@ namespace { ]; $assert(CloudOperationRecoveryPolicy::kinds() === $expectedKinds, 'The Agent leaf recovery policy changed without updating its contract.'); - $assert(count(CloudOperationRecoveryPolicy::kinds()) === 31, - 'The recovery policy contract must contain exactly 31 Agent leaf kinds.'); + $assert(count(CloudOperationRecoveryPolicy::kinds()) === 32, + 'The recovery policy contract must contain exactly 32 Agent leaf kinds.'); foreach ($expectedKinds as $kind) { $assert(CloudOperationRecoveryPolicy::supports($kind), 'The recovery policy omitted ' . $kind . '.'); diff --git a/web/tests/CloudPortableBackupAdminUiContractTest.php b/web/tests/CloudPortableBackupAdminUiContractTest.php index 3e9be478..27c76250 100644 --- a/web/tests/CloudPortableBackupAdminUiContractTest.php +++ b/web/tests/CloudPortableBackupAdminUiContractTest.php @@ -28,6 +28,8 @@ $assert(str_contains($routes, && str_contains($routes, "'/admin/backups'") && str_contains($routes, "'/admin/backups/{backup_id:[0-9]+}/restore'") && str_contains($routes, "'/admin/backups/{backup_id:[0-9]+}'") + && str_contains($routes, + "'/admin/backups/{backup_id:[0-9]+}/cleanup-workspace'") && str_contains($routes, "'/admin/instances/{id:[0-9]+}/backups/{backup_id:[0-9]+}/restore'") && str_contains($routes, @@ -42,6 +44,7 @@ $assert(str_contains($routes, && str_contains($controller, 'CloudBackupService::create(') && str_contains($controller, 'CloudBackupService::restore(') && str_contains($controller, 'CloudBackupService::delete(') + && str_contains($controller, 'CloudBackupService::cleanupWorkspace(') && str_contains($controller, 'CloudBackupService::library(') && substr_count($controller, 'CloudAccess::manage($actor);') >= 2 && str_contains($controller, 'CloudJobService::acknowledgement($job)'), @@ -103,6 +106,8 @@ $assert(str_contains($template, 'id="instances-backup-library"') && str_contains($template, 'id="backup-library"') && str_contains($runtime, 'async function loadBackupLibrary(') && str_contains($runtime, '/api/v1/admin/backups?page=') + && str_contains($runtime, + '/api/v1/admin/backups/${encodeURIComponent(backup.id)}/cleanup-workspace') && str_contains($runtime, '源实例已删除') && str_contains($runtime, 'backupMutationUrl('), 'The bounded global backup library must remain usable after source VM deletion.'); @@ -136,8 +141,8 @@ $assert(str_contains($runtime, "method: 'DELETE'") $assert(str_contains($runtime, '统一接入 br0、分配全新公网 IP') && str_contains($runtime, '不会继承原 IP、账单、到期时间或流量周期') - && str_contains($runtime, '临时快照清理仍需管理员核对') - && str_contains($runtime, '系统不会重复上传') + && str_contains($runtime, '临时工作区仍待清理') + && str_contains($runtime, '清理不会重复上传或修改备份对象') && str_contains($runtime, 'target_host_id') && str_contains($runtime, 'available_ip_count'), 'The restore UI must describe and collect the independent br0/new-IP target intent.'); diff --git a/web/tests/CloudPortableBackupFinalizerTest.php b/web/tests/CloudPortableBackupFinalizerTest.php index 59048d73..1953cd5b 100644 --- a/web/tests/CloudPortableBackupFinalizerTest.php +++ b/web/tests/CloudPortableBackupFinalizerTest.php @@ -224,6 +224,33 @@ $assert((string) $createdBackup->state === 'ready' && $createdBackup->ready_at !== null, 'Cleanup-pending backup lost its ready/restorable object identity.'); +$cleanupOperation = str_repeat('9', 32); +$capsule->table('cloud_jobs')->insert([ + 'id' => 2, + 'kind' => 'backup_workspace_cleanup', + 'host_id' => 3, + 'instance_id' => 101, + 'backup_id' => 1, + 'backup_restore_id' => null, + 'operation_id' => $cleanupOperation, +]); +$cleanupJob = CloudJob::findOrFail(2); +CloudBackupService::finishJobLocked($cleanupJob, 'succeeded', [ + 'backup_key' => str_repeat('a', 32), + 'cleaned' => true, + 'workspace_absent' => true, + 'metadata_absent' => true, +], [ + 'owner_id' => 9, + 'backup_key' => str_repeat('a', 32), + 'prior_operation_id' => $createOperation, +]); +$cleanedBackup = CloudInstanceBackup::findOrFail(1); +$assert((string) $cleanedBackup->state === 'ready' + && (string) $cleanedBackup->error_code === '' + && (string) $cleanedBackup->sha256 === str_repeat('b', 64), + 'Evidence-bound workspace cleanup did not preserve the remote backup identity.'); + $fixture = static function (int $id, string $outcome) use ( $capsule, $sourceSpec, $now ): array { diff --git a/web/tests/CloudSnapshotWorkflowTest.php b/web/tests/CloudSnapshotWorkflowTest.php index c1212cc5..c64ab5f0 100644 --- a/web/tests/CloudSnapshotWorkflowTest.php +++ b/web/tests/CloudSnapshotWorkflowTest.php @@ -633,7 +633,8 @@ namespace { 'action', 'reinstall', 'instance_settings', 'network_change', 'instance_upgrade', 'firewall_apply', 'snapshot_create', 'snapshot_restore', 'snapshot_delete', 'backup_create', - 'backup_restore', 'backup_recover', 'instance_migration_prepare', + 'backup_workspace_cleanup', 'backup_restore', 'backup_recover', + 'instance_migration_prepare', 'instance_migration_transfer', 'instance_migration_target_clean', 'instance_migration_cleanup', 'instance_migration_rollback', 'instance_migration_activate', 'rescue', diff --git a/web/tests/LegalPagesContractTest.php b/web/tests/LegalPagesContractTest.php index 1a1fcf89..6b313949 100644 --- a/web/tests/LegalPagesContractTest.php +++ b/web/tests/LegalPagesContractTest.php @@ -73,7 +73,7 @@ $assert(str_contains($auth, "Config::get('tos_reg') === 'true'") 'Legal consent is only client-side and can be bypassed.'); $assets = $read('web/resources/views/venix/layout_assets.tpl'); -$assert(str_contains($assets, "value='2026102802'") +$assert(str_contains($assets, "value='2026102803'") && str_contains($admin, '?v={$venix_asset_version}') && str_contains($read('web/resources/views/venix/home/layout_header.tpl'), '?v={$venix_asset_version}'), 'Shared browser asset version was not coupled to the new assets.'); diff --git a/web/tests/PublicPageLocaleContractTest.php b/web/tests/PublicPageLocaleContractTest.php index 97e6f52c..1f9448cc 100644 --- a/web/tests/PublicPageLocaleContractTest.php +++ b/web/tests/PublicPageLocaleContractTest.php @@ -80,7 +80,7 @@ $assert(preg_match('/\.landing-site-header\s*\{[^}]*position:\s*fixed;/s', $css) && str_contains($accountCss, 'padding-top: 88px;') && str_contains($accountCss, 'padding-top: 76px;'), 'Persistent desktop or mobile public-navigation styling is incomplete.'); -$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102802'"), +$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102803'"), 'The shared browser asset version was not advanced with the navigation styles.'); echo "Public page locale and shared-layout contract tests passed.\n"; diff --git a/web/tests/SiteBrandContractTest.php b/web/tests/SiteBrandContractTest.php index 369539fa..7da3a274 100644 --- a/web/tests/SiteBrandContractTest.php +++ b/web/tests/SiteBrandContractTest.php @@ -69,7 +69,7 @@ foreach ($templates as $template) { } $assets = $read('web/resources/views/venix/layout_assets.tpl'); -$assert(str_contains($assets, "value='2026102802'"), +$assert(str_contains($assets, "value='2026102803'"), 'The shared asset version was not updated for the canonical logo.'); echo "Shared site-brand contract passed across public, auth, user, cloud and admin layouts.\n";