You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_snapshot_features.go

752 lines
29 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"path/filepath"
"strings"
"time"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
func snapshotVolumeName(uuid, snapshotKey string) string {
compact := strings.ReplaceAll(strings.ToLower(uuid), "-", "")
return "vsnap-" + compact + "-" + snapshotKey + ".qcow2"
}
func snapshotRestoreVolumeName(uuid, operationID string) string {
sum := sha256.Sum256([]byte(operationID))
compact := strings.ReplaceAll(strings.ToLower(uuid), "-", "")
return "vrestore-" + compact + "-" + hex.EncodeToString(sum[:8]) + ".qcow2"
}
func deterministicArtifactCreateFailure(action string, mutationErr, inspectErr error, artifactExists bool) error {
err := mutationLibvirtError(action, mutationErr)
if inspectErr != nil || artifactExists {
return operation.MarkUncertain(errors.Join(err, inspectErr))
}
return err
}
func deterministicVolumeCreateFailure(action string, mutationErr, inspectErr error, artifactExists bool) error {
return deterministicArtifactCreateFailure(action, mutationErr, inspectErr, artifactExists)
}
func validateSnapshotDeletePresence(metadataExists, volumeExists bool) error {
if !metadataExists && !volumeExists {
return NewError(ErrNotFound, "snapshot_not_found", "snapshot was not found", nil)
}
if !metadataExists && volumeExists {
return NewError(ErrConflict, "snapshot_artifacts_inconsistent", "snapshot volume exists without managed metadata", nil)
}
return nil
}
func (l *Libvirt) managedRootContext(domain *libvirt.Domain) (string, *libvirtxml.Domain, *domainMetadataXML, *libvirt.StoragePool, *libvirt.StorageVol, error) {
if err := requirePersistentWithoutSnapshots(domain); err != nil {
return "", nil, nil, nil, nil, err
}
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return "", nil, nil, nil, nil, err
}
definition, _, err := parseManagedDomain(document)
if err != nil {
return "", nil, nil, nil, nil, err
}
rootDisk, err := managedRootDisk(definition)
if err != nil {
return "", nil, nil, nil, nil, err
}
pool, err := l.conn.LookupStoragePoolByName(l.local.ManagedStoragePool)
if err != nil {
return "", nil, nil, nil, nil, translateLibvirtError("lookup managed storage pool", err)
}
volume, err := pool.LookupStorageVolByName(metadata.RootVolumeName)
if err != nil {
pool.Free()
return "", nil, nil, nil, nil, translateLibvirtError("lookup managed root volume", err)
}
path, err := volume.GetPath()
if err != nil || rootDisk.Source == nil || rootDisk.Source.File == nil || filepath.Clean(rootDisk.Source.File.File) != filepath.Clean(path) {
volume.Free()
pool.Free()
return "", nil, nil, nil, nil, NewError(ErrConflict, "managed_disk_invalid", "domain root source does not match the managed storage volume", err)
}
if _, err := validateQCOW2Volume(volume, 0); err != nil {
volume.Free()
pool.Free()
return "", nil, nil, nil, nil, err
}
return document, definition, metadata, pool, volume, nil
}
func managedRootDisk(definition *libvirtxml.Domain) (*libvirtxml.DomainDisk, error) {
if definition == nil || definition.Devices == nil {
return nil, NewError(ErrConflict, "managed_disk_invalid", "managed domain devices are missing", nil)
}
var root *libvirtxml.DomainDisk
for index := range definition.Devices.Disks {
disk := &definition.Devices.Disks[index]
if disk.Device == "disk" && disk.Target != nil && disk.Target.Dev == "vda" {
if root != nil {
return nil, NewError(ErrConflict, "managed_disk_invalid", "managed domain has multiple vda disks", nil)
}
root = disk
}
}
if root == nil || root.Driver == nil || root.Driver.Type != "qcow2" {
return nil, NewError(ErrConflict, "managed_disk_invalid", "managed domain must have one qcow2 vda disk", nil)
}
return root, nil
}
func validateQCOW2Volume(volume *libvirt.StorageVol, expectedCapacity uint64) (*libvirt.StorageVolInfo, error) {
document, err := volume.GetXMLDesc(0)
if err != nil {
return nil, translateLibvirtError("read qcow2 volume XML", err)
}
var definition libvirtxml.StorageVolume
if err := definition.Unmarshal(document); err != nil || definition.Target == nil || definition.Target.Format == nil || definition.Target.Format.Type != "qcow2" {
return nil, NewError(ErrConflict, "invalid_qcow2_volume", "managed volume is not qcow2", err)
}
info, err := volume.GetInfo()
if err != nil {
return nil, translateLibvirtError("read qcow2 volume size", err)
}
if info.Capacity == 0 || (expectedCapacity != 0 && info.Capacity != expectedCapacity) {
return nil, NewError(ErrConflict, "invalid_qcow2_volume", "managed qcow2 volume capacity is invalid", nil)
}
return info, nil
}
func storageVolumeXMLAllocated(name string, capacity uint64, format string) (string, error) {
definition := libvirtxml.StorageVolume{
Name: name,
Capacity: &libvirtxml.StorageVolumeSize{Value: capacity, Unit: "bytes"},
Allocation: &libvirtxml.StorageVolumeSize{Value: capacity, Unit: "bytes"},
Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: format}},
}
return definition.Marshal()
}
func (l *Libvirt) directManagedVolumePath(pool *libvirt.StoragePool, volume *libvirt.StorageVol) (string, error) {
poolXML, err := pool.GetXMLDesc(0)
if err != nil {
return "", translateLibvirtError("read managed storage pool XML", err)
}
var definition libvirtxml.StoragePool
if err := definition.Unmarshal(poolXML); err != nil || definition.Target == nil || definition.Target.Path == "" || (definition.Type != "dir" && definition.Type != "fs" && definition.Type != "netfs") {
return "", NewError(ErrConflict, "snapshot_pool_unsupported", "snapshots require a file-backed storage pool", err)
}
path, err := volume.GetPath()
if err != nil {
return "", translateLibvirtError("read managed volume path", err)
}
path = filepath.Clean(path)
if !filepath.IsAbs(path) || filepath.Dir(path) != filepath.Clean(definition.Target.Path) {
return "", NewError(ErrConflict, "snapshot_pool_unsupported", "snapshot target must be a direct file in the managed pool", nil)
}
return path, nil
}
func ensureNoActiveDomainJob(domain *libvirt.Domain) error {
job, err := domain.GetJobStats(0)
if err != nil {
return NewError(ErrUnsupported, "live_snapshot_unsupported", "libvirt cannot report domain job state", err)
}
if job.Type != libvirt.DOMAIN_JOB_NONE {
return NewError(ErrConflict, "domain_job_active", "domain already has an active libvirt job", nil)
}
return nil
}
func ensureNoActiveDomainJobIfRunning(domain *libvirt.Domain) error {
active, err := domain.IsActive()
if err != nil {
return translateLibvirtError("check domain activity", err)
}
if !active {
return nil
}
return ensureNoActiveDomainJob(domain)
}
func consumePreviousBackupEvidence(domain *libvirt.Domain) error {
job, err := domain.GetJobStats(libvirt.DOMAIN_JOB_STATS_COMPLETED | libvirt.DOMAIN_JOB_STATS_KEEP_COMPLETED)
if err != nil {
return NewError(ErrUnsupported, "live_snapshot_unsupported", "libvirt cannot retain completed backup evidence", err)
}
if job.Type == libvirt.DOMAIN_JOB_NONE {
return nil
}
if !job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
return operation.MarkUncertain(NewError(ErrConflict, "foreign_job_evidence", "retained libvirt job evidence is not a VM backup", nil))
}
consumed, err := domain.GetJobStats(libvirt.DOMAIN_JOB_STATS_COMPLETED)
if err != nil || consumed.Type != job.Type || !consumed.OperationSet || consumed.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
return operation.MarkUncertain(NewError(ErrConflict, "backup_evidence_not_consumed", "previous backup evidence could not be consumed safely", err))
}
return nil
}
func waitForBackupTerminal(ctx context.Context, domain *libvirt.Domain) (libvirt.DomainJobType, error) {
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
job, err := domain.GetJobStats(0)
if err != nil {
return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(translateLibvirtError("inspect live snapshot backup", err))
}
if job.Type == libvirt.DOMAIN_JOB_FAILED || job.Type == libvirt.DOMAIN_JOB_CANCELLED {
return job.Type, nil
}
if job.Type == libvirt.DOMAIN_JOB_NONE {
terminal, err := domain.GetJobStats(libvirt.DOMAIN_JOB_STATS_COMPLETED | libvirt.DOMAIN_JOB_STATS_KEEP_COMPLETED)
if err != nil || !terminal.OperationSet || terminal.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(NewError(ErrConflict, "snapshot_terminal_unknown", "live snapshot terminal evidence is unavailable", err))
}
return terminal.Type, nil
}
if job.OperationSet && job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(NewError(ErrConflict, "foreign_domain_job", "active libvirt job is not the expected backup", nil))
}
select {
case <-ctx.Done():
if err := domain.AbortJob(); err != nil {
return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(errors.Join(ctx.Err(), mutationLibvirtError("abort timed out snapshot backup", err)))
}
return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(ctx.Err())
case <-ticker.C:
}
}
}
func (l *Libvirt) CreateSnapshot(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) {
key := request.SnapshotKey
if key == "" {
key = request.SnapshotID
}
if !catalogIDPattern.MatchString(key) || !displayNamePattern.MatchString(request.Name) || len(request.Description) > 1024 || (request.ConsistencyMode != "filesystem" && request.ConsistencyMode != "crash") {
return Snapshot{}, NewError(ErrInvalid, "invalid_snapshot", "snapshot key, name, description, or consistency is invalid", nil)
}
domainMutationMu.Lock()
defer domainMutationMu.Unlock()
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return Snapshot{}, err
}
defer domain.Free()
document, _, metadata, pool, source, err := l.managedRootContext(domain)
if err != nil {
return Snapshot{}, err
}
defer pool.Free()
defer source.Free()
active, err := domain.IsActive()
if err != nil {
return Snapshot{}, translateLibvirtError("check domain activity for snapshot", err)
}
if !active {
return Snapshot{}, NewError(ErrConflict, "snapshot_requires_running_vm", "online snapshot requires a running VM", nil)
}
for _, stored := range metadata.Snapshots {
if stored.Key == key {
return Snapshot{}, NewError(ErrConflict, "snapshot_exists", "snapshot already exists", nil)
}
}
if err := ensureNoActiveDomainJob(domain); err != nil {
return Snapshot{}, err
}
if err := consumePreviousBackupEvidence(domain); err != nil {
return Snapshot{}, err
}
sourceInfo, err := source.GetInfo()
if err != nil || sourceInfo.Capacity == 0 {
return Snapshot{}, translateLibvirtError("inspect snapshot source volume", err)
}
poolInfo, err := pool.GetInfo()
if err != nil {
return Snapshot{}, translateLibvirtError("inspect snapshot storage capacity", err)
}
if sourceInfo.Capacity > poolInfo.Available {
return Snapshot{}, NewError(ErrConflict, "snapshot_storage_full", "managed storage pool does not have enough free space", nil)
}
volumeName := snapshotVolumeName(uuid, key)
if existing, lookupErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, volumeName); lookupErr != nil {
return Snapshot{}, lookupErr
} else if existing != nil {
existing.Free()
return Snapshot{}, NewError(ErrConflict, "snapshot_exists", "snapshot target volume already exists", nil)
}
volumeXML, err := storageVolumeXMLAllocated(volumeName, sourceInfo.Capacity, "qcow2")
if err != nil {
return Snapshot{}, err
}
target, err := pool.StorageVolCreateXML(volumeXML, 0)
if err != nil {
return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("reserve snapshot volume", err))
}
targetOwned := true
defer func() {
if targetOwned {
if cleanupErr := target.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr, mutationLibvirtError("roll back snapshot volume", cleanupErr)))
}
}
_ = target.Free()
}()
targetPath, err := l.directManagedVolumePath(pool, target)
if err != nil {
return Snapshot{}, err
}
frozen := false
if request.ConsistencyMode == "filesystem" {
if err := domain.FSFreeze(nil, 0); err != nil {
thawErr := domain.FSThaw(nil, 0)
freezeErr := NewError(ErrConflict, "snapshot_guest_agent_unavailable", "QEMU guest agent could not freeze guest filesystems", err)
if thawErr != nil {
return Snapshot{}, operation.MarkUncertain(errors.Join(freezeErr, mutationLibvirtError("thaw guest filesystems after ambiguous freeze", thawErr)))
}
return Snapshot{}, freezeErr
}
frozen = true
}
backup := libvirtxml.DomainBackup{Push: &libvirtxml.DomainBackupPush{Disks: &libvirtxml.DomainBackupPushDisks{Disks: []libvirtxml.DomainBackupPushDisk{{
Name: "vda", Backup: "yes", Driver: &libvirtxml.DomainBackupDiskDriver{Type: "qcow2"},
Target: &libvirtxml.DomainDiskSource{File: &libvirtxml.DomainDiskSourceFile{File: targetPath}},
}}}}}
backupXML, err := backup.Marshal()
if err != nil {
if frozen {
if thawErr := domain.FSThaw(nil, 0); thawErr != nil {
return Snapshot{}, operation.MarkUncertain(errors.Join(err, mutationLibvirtError("thaw guest filesystems after snapshot preparation", thawErr)))
}
}
return Snapshot{}, err
}
beginErr := domain.BackupBegin(backupXML, "", libvirt.DOMAIN_BACKUP_BEGIN_REUSE_EXTERNAL)
if frozen {
thawErr := domain.FSThaw(nil, 0)
frozen = false
if thawErr != nil {
targetOwned = false
_ = domain.AbortJob()
return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("thaw guest filesystems after snapshot start", thawErr))
}
}
if beginErr != nil {
job, inspectErr := domain.GetJobStats(0)
if inspectErr != nil || (job.Type != libvirt.DOMAIN_JOB_NONE && (!job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP)) {
targetOwned = false
return Snapshot{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("start live snapshot backup", beginErr), translateLibvirtError("inspect ambiguous snapshot start", inspectErr)))
}
if job.Type == libvirt.DOMAIN_JOB_NONE {
return Snapshot{}, mutationLibvirtError("start live snapshot backup", beginErr)
}
}
targetOwned = false
terminal, err := waitForBackupTerminal(ctx, domain)
if err != nil {
return Snapshot{}, err
}
if terminal != libvirt.DOMAIN_JOB_COMPLETED {
targetOwned = true
return Snapshot{}, NewError(ErrUnavailable, "snapshot_backup_failed", "live snapshot backup did not complete successfully", nil)
}
info, err := validateQCOW2Volume(target, sourceInfo.Capacity)
if err != nil {
return Snapshot{}, operation.MarkUncertain(err)
}
created := time.Now().UTC()
updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
stored.Snapshots = append(stored.Snapshots, domainSnapshotMetadataXML{Key: key, Name: request.Name, Description: request.Description, Consistency: request.ConsistencyMode, SizeBytes: info.Capacity, CreatedAt: created.Unix()})
return nil
})
if err != nil {
return Snapshot{}, operation.MarkUncertain(err)
}
defined, err := l.conn.DomainDefineXML(updated)
if err != nil {
return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("persist snapshot metadata", err))
}
defined.Free()
return Snapshot{ID: key, SnapshotKey: key, Name: request.Name, Description: request.Description, CreatedAt: created, State: "ready", Consistency: request.ConsistencyMode, SizeBytes: info.Capacity, Created: true}, nil
}
func (l *Libvirt) DeleteSnapshot(ctx context.Context, uuid, key string, _ SnapshotMutationRequest) (SnapshotMutationResult, error) {
if !catalogIDPattern.MatchString(key) {
return SnapshotMutationResult{}, NewError(ErrInvalid, "invalid_snapshot_id", "snapshot key is invalid", nil)
}
domainMutationMu.Lock()
defer domainMutationMu.Unlock()
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return SnapshotMutationResult{}, err
}
defer domain.Free()
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return SnapshotMutationResult{}, err
}
found := false
for _, stored := range metadata.Snapshots {
found = found || stored.Key == key
}
volumeName := snapshotVolumeName(uuid, key)
volume, err := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, volumeName)
if err != nil {
return SnapshotMutationResult{}, err
}
if err := validateSnapshotDeletePresence(found, volume != nil); err != nil {
if volume != nil {
volume.Free()
}
return SnapshotMutationResult{}, err
}
if err := ensureNoActiveDomainJobIfRunning(domain); err != nil {
if volume != nil {
volume.Free()
}
return SnapshotMutationResult{}, err
}
if volume != nil {
defer volume.Free()
if _, err := validateQCOW2Volume(volume, 0); err != nil {
return SnapshotMutationResult{}, err
}
path, err := volume.GetPath()
if err != nil {
return SnapshotMutationResult{}, translateLibvirtError("resolve snapshot volume path", err)
}
referenced, err := l.anyDomainUsesPath(path)
if err != nil {
return SnapshotMutationResult{}, err
}
if referenced {
return SnapshotMutationResult{}, NewError(ErrConflict, "snapshot_in_use", "snapshot volume is referenced by a domain", nil)
}
if err := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, volumeName)
if inspectErr != nil || observed != nil {
if observed != nil {
observed.Free()
}
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("delete snapshot volume", err), inspectErr))
}
}
}
updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
kept := stored.Snapshots[:0]
for _, snapshot := range stored.Snapshots {
if snapshot.Key != key {
kept = append(kept, snapshot)
}
}
stored.Snapshots = kept
return nil
})
if err != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(err)
}
defined, err := l.conn.DomainDefineXML(updated)
if err != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(mutationLibvirtError("remove snapshot metadata", err))
}
defined.Free()
return SnapshotMutationResult{SnapshotKey: key, Deleted: true}, nil
}
func (l *Libvirt) RestoreSnapshot(ctx context.Context, uuid, key string, request SnapshotMutationRequest) (SnapshotMutationResult, error) {
if !catalogIDPattern.MatchString(key) || !operationIdentifierPattern.MatchString(request.OperationID) {
return SnapshotMutationResult{}, NewError(ErrInvalid, "invalid_snapshot_restore", "snapshot key or operation id is invalid", nil)
}
domainMutationMu.Lock()
defer domainMutationMu.Unlock()
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return SnapshotMutationResult{}, err
}
defer domain.Free()
originalXML, _, metadata, pool, oldRoot, err := l.managedRootContext(domain)
if err != nil {
return SnapshotMutationResult{}, err
}
defer pool.Free()
defer oldRoot.Free()
autostart, err := domain.GetAutostart()
if err != nil {
return SnapshotMutationResult{}, translateLibvirtError("read snapshot restore domain autostart", err)
}
found := false
for _, stored := range metadata.Snapshots {
found = found || stored.Key == key
}
if !found {
return SnapshotMutationResult{}, NewError(ErrNotFound, "snapshot_not_found", "snapshot was not found", nil)
}
snapshot, err := pool.LookupStorageVolByName(snapshotVolumeName(uuid, key))
if err != nil {
return SnapshotMutationResult{}, translateLibvirtError("lookup snapshot volume", err)
}
defer snapshot.Free()
snapshotInfo, err := validateQCOW2Volume(snapshot, 0)
if err != nil {
return SnapshotMutationResult{}, err
}
snapshotPath, err := snapshot.GetPath()
if err != nil {
return SnapshotMutationResult{}, translateLibvirtError("resolve snapshot path", err)
}
referenced, err := l.anyDomainUsesPath(snapshotPath)
if err != nil {
return SnapshotMutationResult{}, err
}
if referenced {
return SnapshotMutationResult{}, NewError(ErrConflict, "snapshot_in_use", "immutable snapshot is referenced by a domain", nil)
}
replacementName := snapshotRestoreVolumeName(uuid, request.OperationID)
if existing, lookupErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, replacementName); lookupErr != nil {
return SnapshotMutationResult{}, lookupErr
} else if existing != nil {
existing.Free()
return SnapshotMutationResult{}, operation.MarkUncertain(NewError(ErrConflict, "restore_volume_exists", "deterministic restore volume already exists", nil))
}
poolInfo, err := pool.GetInfo()
if err != nil {
return SnapshotMutationResult{}, translateLibvirtError("inspect restore storage capacity", err)
}
if snapshotInfo.Capacity > poolInfo.Available {
return SnapshotMutationResult{}, NewError(ErrConflict, "restore_storage_full", "managed storage pool does not have enough free space", nil)
}
replacementXML, err := storageVolumeXMLAllocated(replacementName, snapshotInfo.Capacity, "qcow2")
if err != nil {
return SnapshotMutationResult{}, err
}
replacement, err := pool.StorageVolCreateXMLFrom(replacementXML, snapshot, 0)
if err != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, replacementName)
artifactExists := observed != nil
if observed != nil {
observed.Free()
}
return SnapshotMutationResult{}, deterministicVolumeCreateFailure(
"clone snapshot restore volume", err, inspectErr, artifactExists,
)
}
defer replacement.Free()
if _, err := validateQCOW2Volume(replacement, snapshotInfo.Capacity); err != nil {
if cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(err, mutationLibvirtError("roll back restore volume", cleanupErr)))
}
return SnapshotMutationResult{}, err
}
replacementPath, err := replacement.GetPath()
if err != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(translateLibvirtError("resolve restore volume path", err))
}
wasRunning, err := stopForDefinitionChange(ctx, domain)
if err != nil {
if cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(err, mutationLibvirtError("roll back restore volume", cleanupErr)))
}
return SnapshotMutationResult{}, err
}
updatedXML, err := mutateDomainDocument(originalXML, func(definition *libvirtxml.Domain, stored *domainMetadataXML) error {
root, rootErr := managedRootDisk(definition)
if rootErr != nil {
return rootErr
}
root.Source = &libvirtxml.DomainDiskSource{File: &libvirtxml.DomainDiskSourceFile{File: replacementPath}}
stored.PendingCleanupVolume = stored.RootVolumeName
stored.RootVolumeName = replacementName
stored.DiskTransitionKind = "snapshot_restore"
stored.DiskTransitionID = request.OperationID
stored.DiskTransitionRunning = wasRunning
return nil
})
if err != nil {
restartErr := restartOriginalDomain(domain, wasRunning)
cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
if restartErr != nil || cleanupErr != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(
err, restartErr,
optionalMutationLibvirtError("roll back restore volume", cleanupErr),
))
}
return SnapshotMutationResult{}, err
}
defined, err := l.conn.DomainDefineXML(updatedXML)
if err != nil {
currentXML, currentMetadata, inspectErr := requireManagedDomain(domain)
_ = currentXML
if inspectErr != nil || currentMetadata.RootVolumeName != metadata.RootVolumeName {
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("switch restored system disk", err), inspectErr))
}
restartErr := restartOriginalDomain(domain, wasRunning)
cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
if restartErr != nil || cleanupErr != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(
mutationLibvirtError("switch restored system disk", err), restartErr,
optionalMutationLibvirtError("roll back restore volume", cleanupErr),
))
}
return SnapshotMutationResult{}, mutationLibvirtError("switch restored system disk", err)
}
defer defined.Free()
if err := defined.SetAutostart(autostart); err != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(mutationLibvirtError("preserve restored domain autostart", err))
}
if wasRunning {
if err := defined.Create(); err != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(mutationLibvirtError("start restored domain", err))
}
}
observedXML, observed, err := requireManagedDomain(defined)
if err != nil || observed.RootVolumeName != replacementName || observed.DiskTransitionID != request.OperationID {
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(NewError(ErrConflict, "snapshot_restore_observation_failed", "restored domain did not expose the committed disk transition", nil), err))
}
oldPath, err := oldRoot.GetPath()
if err != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(translateLibvirtError("resolve old system disk", err))
}
used, err := l.anyDomainUsesPath(oldPath)
if err != nil || used {
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(err, NewError(ErrConflict, "snapshot_restore_cleanup_pending", "snapshot restore committed but the old root volume is still referenced", nil)))
}
if err := oldRoot.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(mutationLibvirtError("delete previous root after snapshot restore", err))
}
cleanXML, err := mutateManagedMetadata(observedXML, func(stored *domainMetadataXML) error {
stored.PendingCleanupVolume = ""
// Keep the completed operation marker so a crash between this
// definition write and journal success remains reconcilable.
return nil
})
if err != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(err)
}
cleaned, err := l.conn.DomainDefineXML(cleanXML)
if err != nil {
return SnapshotMutationResult{}, operation.MarkUncertain(mutationLibvirtError("clear snapshot restore transition metadata", err))
}
cleaned.Free()
state := VMStateShutoff
if wasRunning {
state = VMStateRunning
}
return SnapshotMutationResult{SnapshotKey: key, Restored: true, PowerState: state, ActiveVolumeName: replacementName}, nil
}
func restartOriginalDomain(domain *libvirt.Domain, shouldRun bool) error {
if !shouldRun {
return nil
}
active, err := domain.IsActive()
if err != nil {
return translateLibvirtError("inspect original domain during rollback", err)
}
if active {
return nil
}
if err := domain.Create(); err != nil {
return mutationLibvirtError("restart original domain during rollback", err)
}
return nil
}
func (l *Libvirt) anyDomainUsesPath(path string) (bool, error) {
return l.otherDomainUsesPath(path, "")
}
func (l *Libvirt) otherDomainUsesPath(path, excludedUUID string) (bool, error) {
path = filepath.Clean(path)
domains, err := l.conn.ListAllDomains(0)
if err != nil {
return false, translateLibvirtError("list domains for volume reference check", err)
}
for index := range domains {
domain := &domains[index]
if excludedUUID != "" {
candidateUUID, uuidErr := domain.GetUUIDString()
if uuidErr != nil {
domain.Free()
return false, translateLibvirtError("read domain UUID for volume reference", uuidErr)
}
if strings.EqualFold(candidateUUID, excludedUUID) {
domain.Free()
continue
}
}
flags := []libvirt.DomainXMLFlags{0}
persistent, persistentErr := domain.IsPersistent()
if persistentErr != nil {
domain.Free()
return false, translateLibvirtError("inspect domain persistence for volume reference", persistentErr)
}
if persistent {
flags = append(flags, libvirt.DOMAIN_XML_INACTIVE)
}
for _, flag := range flags {
document, xmlErr := domain.GetXMLDesc(flag)
if xmlErr != nil {
domain.Free()
return false, translateLibvirtError("read domain XML for volume reference", xmlErr)
}
var definition libvirtxml.Domain
if err := definition.Unmarshal(document); err != nil {
domain.Free()
return false, fmt.Errorf("parse domain XML for volume reference: %w", err)
}
if domainReferencesPath(&definition, path) {
domain.Free()
return true, nil
}
}
domain.Free()
}
return false, nil
}
func domainReferencesPath(definition *libvirtxml.Domain, path string) bool {
if definition == nil {
return false
}
path = filepath.Clean(path)
if definition.OS != nil {
if definition.OS.NVRam != nil {
if definition.OS.NVRam.NVRam != "" &&
filepath.Clean(definition.OS.NVRam.NVRam) == path {
return true
}
if definition.OS.NVRam.Source != nil &&
definition.OS.NVRam.Source.File != nil &&
filepath.Clean(definition.OS.NVRam.Source.File.File) == path {
return true
}
}
if definition.OS.VarStore != nil && definition.OS.VarStore.Path != "" &&
filepath.Clean(definition.OS.VarStore.Path) == path {
return true
}
}
if definition.Devices == nil {
return false
}
for _, disk := range definition.Devices.Disks {
if disk.Source != nil && disk.Source.File != nil &&
filepath.Clean(disk.Source.File.File) == path {
return true
}
}
return false
}