You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1333 lines
50 KiB
Go
1333 lines
50 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
const snapshotGuestAgentPing = `{"execute":"guest-ping"}`
|
|
|
|
const (
|
|
snapshotBackupPollInterval = 500 * time.Millisecond
|
|
snapshotBackupAbortConfirmationTimeout = 30 * time.Second
|
|
guestAgentResponseTimeoutMinVersion = 5_010_000
|
|
)
|
|
|
|
type snapshotGuestAgent interface {
|
|
QemuAgentCommand(string, libvirt.DomainQemuAgentCommandTimeout, uint32) (string, error)
|
|
AgentSetResponseTimeout(int, uint32) error
|
|
FSFreeze([]string, uint32) error
|
|
FSThaw([]string, uint32) error
|
|
}
|
|
|
|
type snapshotBackupJob interface {
|
|
GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error)
|
|
AbortJob() error
|
|
}
|
|
|
|
type snapshotBackupWaitTiming struct {
|
|
pollInterval time.Duration
|
|
abortConfirmationTimeout time.Duration
|
|
}
|
|
|
|
var productionSnapshotBackupWaitTiming = snapshotBackupWaitTiming{
|
|
pollInterval: snapshotBackupPollInterval,
|
|
abortConfirmationTimeout: snapshotBackupAbortConfirmationTimeout,
|
|
}
|
|
|
|
func snapshotConsistencyMatchesRequest(requested, actual string) bool {
|
|
return requested == actual || (requested == "filesystem" && actual == "crash")
|
|
}
|
|
|
|
func snapshotGuestAgentProbeUnavailable(err error) bool {
|
|
return errors.Is(err, libvirt.ERR_AGENT_UNRESPONSIVE) ||
|
|
errors.Is(err, libvirt.ERR_ARGUMENT_UNSUPPORTED) ||
|
|
errors.Is(err, libvirt.ERR_AGENT_COMMAND_TIMEOUT)
|
|
}
|
|
|
|
func snapshotGuestAgentTimeoutUnsupported(err error) bool {
|
|
return errors.Is(err, libvirt.ERR_NO_SUPPORT) ||
|
|
errors.Is(err, libvirt.ERR_OPERATION_UNSUPPORTED) ||
|
|
errors.Is(err, libvirt.ERR_ARGUMENT_UNSUPPORTED)
|
|
}
|
|
|
|
func supportsGuestAgentResponseTimeout(clientVersion, daemonVersion uint32) bool {
|
|
return clientVersion >= guestAgentResponseTimeoutMinVersion &&
|
|
daemonVersion >= guestAgentResponseTimeoutMinVersion
|
|
}
|
|
|
|
func prepareSnapshotConsistency(domain snapshotGuestAgent, requested string,
|
|
responseTimeoutSeconds int, responseTimeoutSupported bool) (string, bool, error) {
|
|
if requested != "filesystem" {
|
|
return requested, false, nil
|
|
}
|
|
if !responseTimeoutSupported {
|
|
return "crash", false, nil
|
|
}
|
|
if _, err := domain.QemuAgentCommand(snapshotGuestAgentPing, 5, 0); err != nil {
|
|
if snapshotGuestAgentProbeUnavailable(err) {
|
|
return "crash", false, nil
|
|
}
|
|
return "", false, translateLibvirtError("probe QEMU guest agent for filesystem snapshot", err)
|
|
}
|
|
if err := domain.AgentSetResponseTimeout(responseTimeoutSeconds, 0); err != nil {
|
|
if snapshotGuestAgentTimeoutUnsupported(err) {
|
|
return "crash", false, nil
|
|
}
|
|
return "", false, translateLibvirtError("set QEMU guest agent response timeout for filesystem snapshot", err)
|
|
}
|
|
if err := domain.FSFreeze(nil, 0); err != nil {
|
|
thawErr := domain.FSThaw(nil, 0)
|
|
freezeErr := NewError(ErrConflict, "snapshot_filesystem_freeze_failed", "QEMU guest agent could not freeze guest filesystems", err)
|
|
if thawErr != nil {
|
|
return "", false, operation.MarkUncertain(errors.Join(freezeErr, mutationLibvirtError("thaw guest filesystems after ambiguous freeze", thawErr)))
|
|
}
|
|
return "", false, freezeErr
|
|
}
|
|
return "filesystem", true, nil
|
|
}
|
|
|
|
func deterministicArtifactCreateFailure(action string, mutationErr, inspectErr error, artifactExists bool) error {
|
|
err := mutationLibvirtError(action, mutationErr)
|
|
if inspectErr != nil || artifactExists {
|
|
return operation.MarkUncertain(errors.Join(err, inspectErr))
|
|
}
|
|
return err
|
|
}
|
|
|
|
func deterministicVolumeCreateFailure(action string, mutationErr, inspectErr error, artifactExists bool) error {
|
|
return deterministicArtifactCreateFailure(action, mutationErr, inspectErr, artifactExists)
|
|
}
|
|
|
|
// snapshotPreconditionFailure is only used before the first storage, domain,
|
|
// or guest mutation belonging to the requested snapshot operation. It gives
|
|
// the durable Journal explicit no-effect semantics without changing the
|
|
// classification of any failure after that boundary.
|
|
func snapshotPreconditionFailure(err error) error {
|
|
return operation.MarkNotApplied(err)
|
|
}
|
|
|
|
func snapshotInUsePrecondition(message string) error {
|
|
return snapshotPreconditionFailure(NewError(ErrConflict, "snapshot_in_use", message, nil))
|
|
}
|
|
|
|
func validateSnapshotDeletePresence(metadataExists, volumeExists bool) error {
|
|
if !metadataExists && !volumeExists {
|
|
return NewError(ErrNotFound, "snapshot_not_found", "snapshot was not found", nil)
|
|
}
|
|
if !metadataExists && volumeExists {
|
|
return NewError(ErrConflict, "snapshot_artifacts_inconsistent", "snapshot volume exists without managed metadata", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) managedRootContext(domain *libvirt.Domain) (string, *libvirtxml.Domain, *domainMetadataXML, *libvirt.StoragePool, *libvirt.StorageVol, error) {
|
|
if err := requirePersistentWithoutSnapshots(domain); err != nil {
|
|
return "", nil, nil, nil, nil, err
|
|
}
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return "", nil, nil, nil, nil, err
|
|
}
|
|
definition, _, err := parseManagedDomain(document)
|
|
if err != nil {
|
|
return "", nil, nil, nil, nil, err
|
|
}
|
|
rootDisk, err := managedRootDisk(definition)
|
|
if err != nil {
|
|
return "", nil, nil, nil, nil, err
|
|
}
|
|
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return "", nil, nil, nil, nil, translateLibvirtError("lookup managed storage pool", err)
|
|
}
|
|
volume, err := pool.LookupStorageVolByName(metadata.RootVolumeName)
|
|
if err != nil {
|
|
pool.Free()
|
|
return "", nil, nil, nil, nil, translateLibvirtError("lookup managed root volume", err)
|
|
}
|
|
path, err := volume.GetPath()
|
|
if err != nil || rootDisk.Source == nil || rootDisk.Source.File == nil || filepath.Clean(rootDisk.Source.File.File) != filepath.Clean(path) {
|
|
volume.Free()
|
|
pool.Free()
|
|
return "", nil, nil, nil, nil, NewError(ErrConflict, "managed_disk_invalid", "domain root source does not match the managed storage volume", err)
|
|
}
|
|
if _, err := validateQCOW2Volume(volume, 0); err != nil {
|
|
volume.Free()
|
|
pool.Free()
|
|
return "", nil, nil, nil, nil, err
|
|
}
|
|
return document, definition, metadata, pool, volume, nil
|
|
}
|
|
|
|
func managedRootDisk(definition *libvirtxml.Domain) (*libvirtxml.DomainDisk, error) {
|
|
if definition == nil || definition.Devices == nil {
|
|
return nil, NewError(ErrConflict, "managed_disk_invalid", "managed domain devices are missing", nil)
|
|
}
|
|
var root *libvirtxml.DomainDisk
|
|
for index := range definition.Devices.Disks {
|
|
disk := &definition.Devices.Disks[index]
|
|
if disk.Device == "disk" && disk.Target != nil && disk.Target.Dev == "vda" {
|
|
if root != nil {
|
|
return nil, NewError(ErrConflict, "managed_disk_invalid", "managed domain has multiple vda disks", nil)
|
|
}
|
|
root = disk
|
|
}
|
|
}
|
|
if root == nil || root.Driver == nil || root.Driver.Type != "qcow2" {
|
|
return nil, NewError(ErrConflict, "managed_disk_invalid", "managed domain must have one qcow2 vda disk", nil)
|
|
}
|
|
return root, nil
|
|
}
|
|
|
|
type qcow2VolumeInspector interface {
|
|
GetXMLDesc(uint32) (string, error)
|
|
GetInfo() (*libvirt.StorageVolInfo, error)
|
|
}
|
|
|
|
func validateQCOW2Volume(volume qcow2VolumeInspector, expectedCapacity uint64) (*libvirt.StorageVolInfo, error) {
|
|
document, err := volume.GetXMLDesc(0)
|
|
if err != nil {
|
|
return nil, translateLibvirtError("read qcow2 volume XML", err)
|
|
}
|
|
var definition libvirtxml.StorageVolume
|
|
if err := definition.Unmarshal(document); err != nil || definition.Target == nil || definition.Target.Format == nil || definition.Target.Format.Type != "qcow2" {
|
|
return nil, NewError(ErrConflict, "invalid_qcow2_volume", "managed volume is not qcow2", err)
|
|
}
|
|
info, err := volume.GetInfo()
|
|
if err != nil {
|
|
return nil, translateLibvirtError("read qcow2 volume size", err)
|
|
}
|
|
if info.Capacity == 0 || (expectedCapacity != 0 && info.Capacity != expectedCapacity) {
|
|
return nil, NewError(ErrConflict, "invalid_qcow2_volume", "managed qcow2 volume capacity is invalid", nil)
|
|
}
|
|
return info, nil
|
|
}
|
|
|
|
func storageVolumeXMLAllocated(name string, capacity uint64, format string) (string, error) {
|
|
definition := libvirtxml.StorageVolume{
|
|
Name: name,
|
|
Capacity: &libvirtxml.StorageVolumeSize{Value: capacity, Unit: "bytes"},
|
|
Allocation: &libvirtxml.StorageVolumeSize{Value: capacity, Unit: "bytes"},
|
|
Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: format}},
|
|
}
|
|
return definition.Marshal()
|
|
}
|
|
|
|
func (l *Libvirt) directManagedVolumePath(pool *libvirt.StoragePool, volume *libvirt.StorageVol) (string, error) {
|
|
poolXML, err := pool.GetXMLDesc(0)
|
|
if err != nil {
|
|
return "", translateLibvirtError("read managed storage pool XML", err)
|
|
}
|
|
var definition libvirtxml.StoragePool
|
|
if err := definition.Unmarshal(poolXML); err != nil || definition.Target == nil || definition.Target.Path == "" || (definition.Type != "dir" && definition.Type != "fs" && definition.Type != "netfs") {
|
|
return "", NewError(ErrConflict, "snapshot_pool_unsupported", "snapshots require a file-backed storage pool", err)
|
|
}
|
|
path, err := volume.GetPath()
|
|
if err != nil {
|
|
return "", translateLibvirtError("read managed volume path", err)
|
|
}
|
|
path = filepath.Clean(path)
|
|
if !filepath.IsAbs(path) || filepath.Dir(path) != filepath.Clean(definition.Target.Path) {
|
|
return "", NewError(ErrConflict, "snapshot_pool_unsupported", "snapshot target must be a direct file in the managed pool", nil)
|
|
}
|
|
return path, nil
|
|
}
|
|
|
|
func ensureNoActiveDomainJob(domain *libvirt.Domain) error {
|
|
job, err := domain.GetJobStats(0)
|
|
if err != nil {
|
|
return NewError(ErrUnsupported, "live_snapshot_unsupported", "libvirt cannot report domain job state", err)
|
|
}
|
|
if job.Type != libvirt.DOMAIN_JOB_NONE {
|
|
return NewError(ErrConflict, "domain_job_active", "domain already has an active libvirt job", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ensureNoActiveDomainJobIfRunning(domain *libvirt.Domain) error {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return translateLibvirtError("check domain activity", err)
|
|
}
|
|
if !active {
|
|
return nil
|
|
}
|
|
return ensureNoActiveDomainJob(domain)
|
|
}
|
|
|
|
func consumePreviousBackupEvidence(domain *libvirt.Domain) error {
|
|
job, err := domain.GetJobStats(libvirt.DOMAIN_JOB_STATS_COMPLETED | libvirt.DOMAIN_JOB_STATS_KEEP_COMPLETED)
|
|
if err != nil {
|
|
return NewError(ErrUnsupported, "live_snapshot_unsupported", "libvirt cannot retain completed backup evidence", err)
|
|
}
|
|
if job.Type == libvirt.DOMAIN_JOB_NONE {
|
|
return nil
|
|
}
|
|
if !job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
|
|
return operation.MarkUncertain(NewError(ErrConflict, "foreign_job_evidence", "retained libvirt job evidence is not a VM backup", nil))
|
|
}
|
|
consumed, err := domain.GetJobStats(libvirt.DOMAIN_JOB_STATS_COMPLETED)
|
|
if err != nil || consumed.Type != job.Type || !consumed.OperationSet || consumed.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
|
|
return operation.MarkUncertain(NewError(ErrConflict, "backup_evidence_not_consumed", "previous backup evidence could not be consumed safely", err))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func inspectCompletedBackupTerminal(domain snapshotBackupJob) (libvirt.DomainJobType, bool, error) {
|
|
job, err := domain.GetJobStats(libvirt.DOMAIN_JOB_STATS_COMPLETED | libvirt.DOMAIN_JOB_STATS_KEEP_COMPLETED)
|
|
if err != nil {
|
|
return libvirt.DOMAIN_JOB_NONE, false, translateLibvirtError("inspect completed live snapshot backup", err)
|
|
}
|
|
if job == nil || job.Type == libvirt.DOMAIN_JOB_NONE {
|
|
return libvirt.DOMAIN_JOB_NONE, false, NewError(ErrConflict, "snapshot_terminal_unknown", "live snapshot terminal evidence is unavailable", nil)
|
|
}
|
|
if !job.OperationSet {
|
|
return libvirt.DOMAIN_JOB_NONE, false, NewError(ErrConflict, "snapshot_terminal_unknown", "live snapshot terminal evidence does not identify its operation", nil)
|
|
}
|
|
if job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
|
|
return libvirt.DOMAIN_JOB_NONE, false, NewError(ErrConflict, "foreign_domain_job", "active libvirt job is not the expected backup", nil)
|
|
}
|
|
switch job.Type {
|
|
case libvirt.DOMAIN_JOB_COMPLETED, libvirt.DOMAIN_JOB_FAILED, libvirt.DOMAIN_JOB_CANCELLED:
|
|
return job.Type, true, nil
|
|
default:
|
|
return libvirt.DOMAIN_JOB_NONE, false, NewError(ErrConflict, "snapshot_terminal_unknown", "live snapshot terminal evidence is invalid", nil)
|
|
}
|
|
}
|
|
|
|
func inspectBackupTerminal(domain snapshotBackupJob) (libvirt.DomainJobType, bool, error) {
|
|
job, err := domain.GetJobStats(0)
|
|
if err != nil {
|
|
terminal, done, completedErr := inspectCompletedBackupTerminal(domain)
|
|
if done {
|
|
return terminal, true, nil
|
|
}
|
|
return libvirt.DOMAIN_JOB_NONE, false, NewError(
|
|
ErrUnavailable,
|
|
"snapshot_terminal_unknown",
|
|
"live snapshot job state is unavailable",
|
|
errors.Join(translateLibvirtError("inspect live snapshot backup", err), completedErr),
|
|
)
|
|
}
|
|
if job == nil {
|
|
return libvirt.DOMAIN_JOB_NONE, false, NewError(ErrUnavailable, "snapshot_terminal_unknown", "live snapshot job state is unavailable", nil)
|
|
}
|
|
if job.Type == libvirt.DOMAIN_JOB_NONE {
|
|
return inspectCompletedBackupTerminal(domain)
|
|
}
|
|
if !job.OperationSet {
|
|
return libvirt.DOMAIN_JOB_NONE, false, NewError(ErrConflict, "snapshot_terminal_unknown", "active libvirt job does not identify its operation", nil)
|
|
}
|
|
if job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
|
|
return libvirt.DOMAIN_JOB_NONE, false, NewError(ErrConflict, "foreign_domain_job", "active libvirt job is not the expected backup", nil)
|
|
}
|
|
switch job.Type {
|
|
case libvirt.DOMAIN_JOB_COMPLETED, libvirt.DOMAIN_JOB_FAILED, libvirt.DOMAIN_JOB_CANCELLED:
|
|
return job.Type, true, nil
|
|
case libvirt.DOMAIN_JOB_BOUNDED, libvirt.DOMAIN_JOB_UNBOUNDED:
|
|
return libvirt.DOMAIN_JOB_NONE, false, nil
|
|
default:
|
|
return libvirt.DOMAIN_JOB_NONE, false, NewError(ErrConflict, "snapshot_terminal_unknown", "live snapshot job state is invalid", nil)
|
|
}
|
|
}
|
|
|
|
func abortBackupAndWaitForTerminal(domain snapshotBackupJob, cause error,
|
|
timing snapshotBackupWaitTiming) (libvirt.DomainJobType, error) {
|
|
terminal, done, inspectErr := inspectBackupTerminal(domain)
|
|
if done {
|
|
return terminal, nil
|
|
}
|
|
if inspectErr != nil {
|
|
return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(NewError(
|
|
ErrUnavailable,
|
|
"snapshot_abort_unconfirmed",
|
|
"live snapshot backup could not be safely identified for cancellation",
|
|
errors.Join(cause, inspectErr),
|
|
))
|
|
}
|
|
abortErr := domain.AbortJob()
|
|
ticker := time.NewTicker(timing.pollInterval)
|
|
defer ticker.Stop()
|
|
timer := time.NewTimer(timing.abortConfirmationTimeout)
|
|
defer timer.Stop()
|
|
var lastInspectErr error
|
|
for {
|
|
_, jobWake := libvirtJobEvents.snapshot()
|
|
terminal, done, inspectErr = inspectBackupTerminal(domain)
|
|
if done {
|
|
// A retained terminal result is stronger evidence than an error from
|
|
// AbortJob, which may merely have raced with natural completion.
|
|
return terminal, nil
|
|
}
|
|
if inspectErr != nil {
|
|
lastInspectErr = inspectErr
|
|
}
|
|
select {
|
|
case <-jobWake:
|
|
case <-ticker.C:
|
|
case <-timer.C:
|
|
// Close the boundary race between the last polling tick and the
|
|
// confirmation deadline. Only report uncertainty after one final
|
|
// attempt to read retained terminal evidence.
|
|
terminal, done, inspectErr := inspectBackupTerminal(domain)
|
|
if done {
|
|
return terminal, nil
|
|
}
|
|
if inspectErr != nil {
|
|
lastInspectErr = inspectErr
|
|
}
|
|
return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(NewError(
|
|
ErrUnavailable,
|
|
"snapshot_abort_unconfirmed",
|
|
"live snapshot backup did not reach a terminal state after cancellation",
|
|
errors.Join(cause, optionalMutationLibvirtError("abort live snapshot backup", abortErr), lastInspectErr),
|
|
))
|
|
}
|
|
}
|
|
}
|
|
|
|
func waitForBackupTerminalWithTiming(ctx context.Context, domain snapshotBackupJob,
|
|
timing snapshotBackupWaitTiming) (libvirt.DomainJobType, error) {
|
|
ticker := time.NewTicker(timing.pollInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
_, jobWake := libvirtJobEvents.snapshot()
|
|
terminal, done, err := inspectBackupTerminal(domain)
|
|
if err != nil {
|
|
return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(err)
|
|
}
|
|
if done {
|
|
return terminal, nil
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return abortBackupAndWaitForTerminal(domain, err, timing)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return abortBackupAndWaitForTerminal(domain, ctx.Err(), timing)
|
|
case <-jobWake:
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func waitForBackupTerminal(ctx context.Context, domain snapshotBackupJob) (libvirt.DomainJobType, error) {
|
|
return waitForBackupTerminalWithTiming(ctx, domain, productionSnapshotBackupWaitTiming)
|
|
}
|
|
|
|
func abortBackupAfterThawFailure(domain snapshotBackupJob, thawErr error,
|
|
timing snapshotBackupWaitTiming) (libvirt.DomainJobType, error) {
|
|
terminal, confirmErr := abortBackupAndWaitForTerminal(domain, nil, timing)
|
|
return terminal, operation.MarkUncertain(errors.Join(
|
|
mutationLibvirtError("thaw guest filesystems after snapshot start", thawErr),
|
|
confirmErr,
|
|
))
|
|
}
|
|
|
|
func (l *Libvirt) CreateSnapshot(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) {
|
|
return l.createFullDiskCopy(ctx, uuid, request, true)
|
|
}
|
|
|
|
// createBackupWorkspace creates the immutable full-system-disk copy used only
|
|
// by a portable backup operation. Unlike a user snapshot it is deliberately
|
|
// absent from domainMetadata.Snapshots, so a later local cleanup failure can
|
|
// consume storage and raise an alert without changing the VM snapshot catalog
|
|
// or blocking maintenance migration.
|
|
func (l *Libvirt) createBackupWorkspace(ctx context.Context, uuid string,
|
|
request SnapshotRequest) (_ Snapshot, resultErr error) {
|
|
return l.createFullDiskCopy(ctx, uuid, request, false)
|
|
}
|
|
|
|
func (l *Libvirt) createFullDiskCopy(ctx context.Context, uuid string,
|
|
request SnapshotRequest, registerSnapshot bool) (_ Snapshot, resultErr error) {
|
|
key := request.SnapshotKey
|
|
if key == "" {
|
|
key = request.SnapshotID
|
|
}
|
|
if !catalogIDPattern.MatchString(key) || !displayNamePattern.MatchString(request.Name) || len(request.Description) > 1024 || (request.ConsistencyMode != "filesystem" && request.ConsistencyMode != "crash") {
|
|
return Snapshot{}, snapshotPreconditionFailure(NewError(ErrInvalid, "invalid_snapshot", "snapshot key, name, description, or consistency is invalid", nil))
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return Snapshot{}, snapshotPreconditionFailure(err)
|
|
}
|
|
defer domain.Free()
|
|
document, _, metadata, pool, source, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
return Snapshot{}, snapshotPreconditionFailure(err)
|
|
}
|
|
defer pool.Free()
|
|
defer source.Free()
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return Snapshot{}, snapshotPreconditionFailure(translateLibvirtError("check domain activity for snapshot", err))
|
|
}
|
|
if !active {
|
|
return Snapshot{}, snapshotPreconditionFailure(NewError(ErrConflict, "snapshot_requires_running_vm", "online snapshot requires a running VM", nil))
|
|
}
|
|
for _, stored := range metadata.Snapshots {
|
|
if stored.Key == key {
|
|
return Snapshot{}, snapshotPreconditionFailure(NewError(ErrConflict, "snapshot_exists", "snapshot already exists", nil))
|
|
}
|
|
}
|
|
if err := ensureNoActiveDomainJob(domain); err != nil {
|
|
return Snapshot{}, snapshotPreconditionFailure(err)
|
|
}
|
|
if err := consumePreviousBackupEvidence(domain); err != nil {
|
|
return Snapshot{}, snapshotPreconditionFailure(err)
|
|
}
|
|
sourceInfo, err := source.GetInfo()
|
|
if err != nil || sourceInfo.Capacity == 0 {
|
|
return Snapshot{}, snapshotPreconditionFailure(translateLibvirtError("inspect snapshot source volume", err))
|
|
}
|
|
poolInfo, err := pool.GetInfo()
|
|
if err != nil {
|
|
return Snapshot{}, snapshotPreconditionFailure(translateLibvirtError("inspect snapshot storage capacity", err))
|
|
}
|
|
if sourceInfo.Capacity > poolInfo.Available {
|
|
return Snapshot{}, snapshotPreconditionFailure(NewError(ErrConflict, "snapshot_storage_full", "managed storage pool does not have enough free space", nil))
|
|
}
|
|
volumeName := snapshotVolumeName(uuid, key)
|
|
if !registerSnapshot {
|
|
volumeName = backupWorkspaceVolumeName(uuid, key)
|
|
}
|
|
if existing, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), volumeName); lookupErr != nil {
|
|
return Snapshot{}, snapshotPreconditionFailure(lookupErr)
|
|
} else if existing != nil {
|
|
existing.Free()
|
|
return Snapshot{}, snapshotPreconditionFailure(NewError(ErrConflict, "snapshot_exists", "snapshot target volume already exists", nil))
|
|
}
|
|
volumeXML, err := storageVolumeXMLAllocated(volumeName, sourceInfo.Capacity, "qcow2")
|
|
if err != nil {
|
|
return Snapshot{}, snapshotPreconditionFailure(err)
|
|
}
|
|
target, err := pool.StorageVolCreateXML(volumeXML, 0)
|
|
if err != nil {
|
|
return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("reserve snapshot volume", err))
|
|
}
|
|
targetOwned := true
|
|
defer func() {
|
|
if targetOwned {
|
|
if cleanupErr := target.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
|
|
resultErr = operation.MarkUncertain(errors.Join(resultErr, mutationLibvirtError("roll back snapshot volume", cleanupErr)))
|
|
}
|
|
}
|
|
_ = target.Free()
|
|
}()
|
|
targetPath, err := l.directManagedVolumePath(pool, target)
|
|
if err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
actualConsistency, frozen, err := prepareSnapshotConsistency(
|
|
domain, request.ConsistencyMode, l.local.GuestAgentTimeoutSeconds,
|
|
l.guestAgentResponseTimeoutSupported,
|
|
)
|
|
if err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
backup := libvirtxml.DomainBackup{Push: &libvirtxml.DomainBackupPush{Disks: &libvirtxml.DomainBackupPushDisks{Disks: []libvirtxml.DomainBackupPushDisk{{
|
|
Name: "vda", Backup: "yes", Driver: &libvirtxml.DomainBackupDiskDriver{Type: "qcow2"},
|
|
Target: &libvirtxml.DomainDiskSource{File: &libvirtxml.DomainDiskSourceFile{File: targetPath}},
|
|
}}}}}
|
|
backupXML, err := backup.Marshal()
|
|
if err != nil {
|
|
if frozen {
|
|
if thawErr := domain.FSThaw(nil, 0); thawErr != nil {
|
|
return Snapshot{}, operation.MarkUncertain(errors.Join(err, mutationLibvirtError("thaw guest filesystems after snapshot preparation", thawErr)))
|
|
}
|
|
}
|
|
return Snapshot{}, err
|
|
}
|
|
beginErr := domain.BackupBegin(backupXML, "", libvirt.DOMAIN_BACKUP_BEGIN_REUSE_EXTERNAL)
|
|
if frozen {
|
|
thawErr := domain.FSThaw(nil, 0)
|
|
frozen = false
|
|
if thawErr != nil {
|
|
targetOwned = false
|
|
terminal, waitErr := abortBackupAfterThawFailure(
|
|
domain, thawErr, productionSnapshotBackupWaitTiming,
|
|
)
|
|
if terminal == libvirt.DOMAIN_JOB_FAILED || terminal == libvirt.DOMAIN_JOB_CANCELLED {
|
|
// The push target is invalid after a confirmed failure/cancel.
|
|
// Once the job is terminal it is safe for the deferred cleanup to
|
|
// delete the reserved volume.
|
|
targetOwned = true
|
|
}
|
|
return Snapshot{}, waitErr
|
|
}
|
|
}
|
|
if beginErr != nil {
|
|
job, inspectErr := domain.GetJobStats(0)
|
|
if inspectErr != nil || (job.Type != libvirt.DOMAIN_JOB_NONE && (!job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP)) {
|
|
targetOwned = false
|
|
return Snapshot{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("start live snapshot backup", beginErr), translateLibvirtError("inspect ambiguous snapshot start", inspectErr)))
|
|
}
|
|
if job.Type == libvirt.DOMAIN_JOB_NONE {
|
|
return Snapshot{}, mutationLibvirtError("start live snapshot backup", beginErr)
|
|
}
|
|
}
|
|
targetOwned = false
|
|
terminal, err := waitForBackupTerminal(ctx, domain)
|
|
if err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
if terminal != libvirt.DOMAIN_JOB_COMPLETED {
|
|
targetOwned = true
|
|
return Snapshot{}, NewError(ErrUnavailable, "snapshot_backup_failed", "live snapshot backup did not complete successfully", nil)
|
|
}
|
|
info, err := validateQCOW2Volume(target, sourceInfo.Capacity)
|
|
if err != nil {
|
|
return Snapshot{}, operation.MarkUncertain(err)
|
|
}
|
|
created := time.Now().UTC()
|
|
if registerSnapshot {
|
|
updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
|
|
stored.Snapshots = append(stored.Snapshots, domainSnapshotMetadataXML{Key: key, Name: request.Name, Description: request.Description, Consistency: actualConsistency, SizeBytes: info.Capacity, CreatedAt: created.Unix()})
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return Snapshot{}, operation.MarkUncertain(err)
|
|
}
|
|
defined, err := l.conn.DomainDefineXML(updated)
|
|
if err != nil {
|
|
return Snapshot{}, operation.MarkUncertain(mutationLibvirtError("persist snapshot metadata", err))
|
|
}
|
|
defined.Free()
|
|
}
|
|
return Snapshot{ID: key, SnapshotKey: key, Name: request.Name, Description: request.Description, CreatedAt: created, State: "ready", Consistency: actualConsistency, SizeBytes: info.Capacity, Created: true}, nil
|
|
}
|
|
|
|
func (l *Libvirt) DeleteSnapshot(ctx context.Context, uuid, key string, _ SnapshotMutationRequest) (SnapshotMutationResult, error) {
|
|
if !catalogIDPattern.MatchString(key) {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(NewError(ErrInvalid, "invalid_snapshot_id", "snapshot key is invalid", nil))
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
defer domain.Free()
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
found := false
|
|
for _, stored := range metadata.Snapshots {
|
|
found = found || stored.Key == key
|
|
}
|
|
volumeName := snapshotVolumeName(uuid, key)
|
|
volume, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), volumeName)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
if err := validateSnapshotDeletePresence(found, volume != nil); err != nil {
|
|
if volume != nil {
|
|
volume.Free()
|
|
}
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
if err := ensureNoActiveDomainJobIfRunning(domain); err != nil {
|
|
if volume != nil {
|
|
volume.Free()
|
|
}
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
if volume != nil {
|
|
defer volume.Free()
|
|
if _, err := validateQCOW2Volume(volume, 0); err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
path, err := volume.GetPath()
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(translateLibvirtError("resolve snapshot volume path", err))
|
|
}
|
|
referenced, err := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), volumeName,
|
|
)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
if referenced {
|
|
return SnapshotMutationResult{}, snapshotInUsePrecondition("snapshot volume is referenced by a domain")
|
|
}
|
|
if err := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil {
|
|
observed, inspectErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), volumeName)
|
|
if inspectErr != nil || observed != nil {
|
|
if observed != nil {
|
|
observed.Free()
|
|
}
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("delete snapshot volume", err), inspectErr))
|
|
}
|
|
}
|
|
}
|
|
updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
|
|
kept := stored.Snapshots[:0]
|
|
for _, snapshot := range stored.Snapshots {
|
|
if snapshot.Key != key {
|
|
kept = append(kept, snapshot)
|
|
}
|
|
}
|
|
stored.Snapshots = kept
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(err)
|
|
}
|
|
defined, err := l.conn.DomainDefineXML(updated)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(mutationLibvirtError("remove snapshot metadata", err))
|
|
}
|
|
defined.Free()
|
|
return SnapshotMutationResult{SnapshotKey: key, Deleted: true}, nil
|
|
}
|
|
|
|
func (l *Libvirt) RestoreSnapshot(ctx context.Context, uuid, key string, request SnapshotMutationRequest) (SnapshotMutationResult, error) {
|
|
if !catalogIDPattern.MatchString(key) || !operationIdentifierPattern.MatchString(request.OperationID) {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(NewError(ErrInvalid, "invalid_snapshot_restore", "snapshot key or operation id is invalid", nil))
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
defer domain.Free()
|
|
originalXML, _, metadata, pool, oldRoot, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
defer pool.Free()
|
|
defer oldRoot.Free()
|
|
autostart, err := domain.GetAutostart()
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(translateLibvirtError("read snapshot restore domain autostart", err))
|
|
}
|
|
found := false
|
|
for _, stored := range metadata.Snapshots {
|
|
found = found || stored.Key == key
|
|
}
|
|
if !found {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(NewError(ErrNotFound, "snapshot_not_found", "snapshot was not found", nil))
|
|
}
|
|
snapshot, err := pool.LookupStorageVolByName(snapshotVolumeName(uuid, key))
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(translateLibvirtError("lookup snapshot volume", err))
|
|
}
|
|
defer snapshot.Free()
|
|
snapshotInfo, err := validateQCOW2Volume(snapshot, 0)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
snapshotPath, err := snapshot.GetPath()
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(translateLibvirtError("resolve snapshot path", err))
|
|
}
|
|
referenced, err := l.anyDomainUsesVolume(
|
|
snapshotPath, l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, key),
|
|
)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
if referenced {
|
|
return SnapshotMutationResult{}, snapshotInUsePrecondition("immutable snapshot is referenced by a domain")
|
|
}
|
|
replacementName := snapshotRestoreVolumeName(uuid, request.OperationID)
|
|
if existing, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName); lookupErr != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(lookupErr)
|
|
} else if existing != nil {
|
|
existing.Free()
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(NewError(ErrConflict, "restore_volume_exists", "deterministic restore volume already exists", nil))
|
|
}
|
|
poolInfo, err := pool.GetInfo()
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(translateLibvirtError("inspect restore storage capacity", err))
|
|
}
|
|
if snapshotInfo.Capacity > poolInfo.Available {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(NewError(ErrConflict, "restore_storage_full", "managed storage pool does not have enough free space", nil))
|
|
}
|
|
replacementXML, err := storageVolumeXMLAllocated(replacementName, snapshotInfo.Capacity, "qcow2")
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, snapshotPreconditionFailure(err)
|
|
}
|
|
replacement, err := pool.StorageVolCreateXMLFrom(replacementXML, snapshot, 0)
|
|
if err != nil {
|
|
observed, inspectErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
|
|
artifactExists := observed != nil
|
|
if observed != nil {
|
|
observed.Free()
|
|
}
|
|
return SnapshotMutationResult{}, deterministicVolumeCreateFailure(
|
|
"clone snapshot restore volume", err, inspectErr, artifactExists,
|
|
)
|
|
}
|
|
defer replacement.Free()
|
|
if _, err := validateQCOW2Volume(replacement, snapshotInfo.Capacity); err != nil {
|
|
if cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(err, mutationLibvirtError("roll back restore volume", cleanupErr)))
|
|
}
|
|
return SnapshotMutationResult{}, err
|
|
}
|
|
replacementPath, err := replacement.GetPath()
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(translateLibvirtError("resolve restore volume path", err))
|
|
}
|
|
wasRunning, err := stopForDefinitionChange(ctx, domain)
|
|
if err != nil {
|
|
if cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(err, mutationLibvirtError("roll back restore volume", cleanupErr)))
|
|
}
|
|
return SnapshotMutationResult{}, err
|
|
}
|
|
updatedXML, err := mutateDomainDocument(originalXML, func(definition *libvirtxml.Domain, stored *domainMetadataXML) error {
|
|
root, rootErr := managedRootDisk(definition)
|
|
if rootErr != nil {
|
|
return rootErr
|
|
}
|
|
root.Source = &libvirtxml.DomainDiskSource{File: &libvirtxml.DomainDiskSourceFile{File: replacementPath}}
|
|
stored.PendingCleanupVolume = stored.RootVolumeName
|
|
stored.RootVolumeName = replacementName
|
|
stored.DiskTransitionKind = "snapshot_restore"
|
|
stored.DiskTransitionID = request.OperationID
|
|
stored.DiskTransitionRunning = wasRunning
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
restartErr := l.restartOriginalDomain(ctx, domain, wasRunning)
|
|
cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
if restartErr != nil || cleanupErr != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(
|
|
err, restartErr,
|
|
optionalMutationLibvirtError("roll back restore volume", cleanupErr),
|
|
))
|
|
}
|
|
return SnapshotMutationResult{}, err
|
|
}
|
|
defined, err := l.conn.DomainDefineXML(updatedXML)
|
|
if err != nil {
|
|
currentXML, currentMetadata, inspectErr := requireManagedDomain(domain)
|
|
_ = currentXML
|
|
if inspectErr != nil || currentMetadata.RootVolumeName != metadata.RootVolumeName {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("switch restored system disk", err), inspectErr))
|
|
}
|
|
restartErr := l.restartOriginalDomain(ctx, domain, wasRunning)
|
|
cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
if restartErr != nil || cleanupErr != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(
|
|
mutationLibvirtError("switch restored system disk", err), restartErr,
|
|
optionalMutationLibvirtError("roll back restore volume", cleanupErr),
|
|
))
|
|
}
|
|
return SnapshotMutationResult{}, mutationLibvirtError("switch restored system disk", err)
|
|
}
|
|
defer defined.Free()
|
|
if err := defined.SetAutostart(autostart); err != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(mutationLibvirtError("preserve restored domain autostart", err))
|
|
}
|
|
if wasRunning {
|
|
if err := l.startManagedDomain(ctx, defined, "start restored domain"); err != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(err)
|
|
}
|
|
}
|
|
observedXML, observed, err := requireManagedDomain(defined)
|
|
if err != nil || observed.RootVolumeName != replacementName || observed.DiskTransitionID != request.OperationID {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(NewError(ErrConflict, "snapshot_restore_observation_failed", "restored domain did not expose the committed disk transition", nil), err))
|
|
}
|
|
oldPath, err := oldRoot.GetPath()
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(translateLibvirtError("resolve old system disk", err))
|
|
}
|
|
used, err := l.anyDomainUsesVolume(
|
|
oldPath, l.storagePoolName(StorageRoleManaged), metadata.RootVolumeName,
|
|
)
|
|
if err != nil || used {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(errors.Join(err, NewError(ErrConflict, "snapshot_restore_cleanup_pending", "snapshot restore committed but the old root volume is still referenced", nil)))
|
|
}
|
|
if err := oldRoot.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(mutationLibvirtError("delete previous root after snapshot restore", err))
|
|
}
|
|
cleanXML, err := mutateManagedMetadata(observedXML, func(stored *domainMetadataXML) error {
|
|
stored.PendingCleanupVolume = ""
|
|
// Keep the completed operation marker so a crash between this
|
|
// definition write and journal success remains reconcilable.
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(err)
|
|
}
|
|
cleaned, err := l.conn.DomainDefineXML(cleanXML)
|
|
if err != nil {
|
|
return SnapshotMutationResult{}, operation.MarkUncertain(mutationLibvirtError("clear snapshot restore transition metadata", err))
|
|
}
|
|
cleaned.Free()
|
|
state := VMStateShutoff
|
|
if wasRunning {
|
|
state = VMStateRunning
|
|
}
|
|
return SnapshotMutationResult{SnapshotKey: key, Restored: true, PowerState: state, ActiveVolumeName: replacementName}, nil
|
|
}
|
|
|
|
func (l *Libvirt) restartOriginalDomain(ctx context.Context, domain *libvirt.Domain, shouldRun bool) error {
|
|
if !shouldRun {
|
|
return nil
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return translateLibvirtError("inspect original domain during rollback", err)
|
|
}
|
|
if active {
|
|
return nil
|
|
}
|
|
if err := l.startManagedDomain(ctx, domain, "restart original domain during rollback"); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) anyDomainUsesPath(path string) (bool, error) {
|
|
return l.otherDomainUsesStorage(domainStorageReference{Path: path}, "")
|
|
}
|
|
|
|
func (l *Libvirt) otherDomainUsesPath(path, excludedUUID string) (bool, error) {
|
|
return l.otherDomainUsesStorage(domainStorageReference{Path: path}, excludedUUID)
|
|
}
|
|
|
|
type domainStorageReference struct {
|
|
Path string
|
|
Pool string
|
|
Volume string
|
|
}
|
|
|
|
func (l *Libvirt) anyDomainUsesVolume(path, pool, volume string) (bool, error) {
|
|
return l.otherDomainUsesStorage(domainStorageReference{
|
|
Path: path, Pool: pool, Volume: volume,
|
|
}, "")
|
|
}
|
|
|
|
func (l *Libvirt) otherDomainUsesVolume(path, pool, volume, excludedUUID string) (bool, error) {
|
|
return l.otherDomainUsesStorage(domainStorageReference{
|
|
Path: path, Pool: pool, Volume: volume,
|
|
}, excludedUUID)
|
|
}
|
|
|
|
func (l *Libvirt) otherDomainUsesStorage(target domainStorageReference, excludedUUID string) (bool, error) {
|
|
if target.Path != "" {
|
|
target.Path = filepath.Clean(target.Path)
|
|
}
|
|
domains, err := l.conn.ListAllDomains(0)
|
|
if err != nil {
|
|
return false, translateLibvirtError("list domains for volume reference check", err)
|
|
}
|
|
for index := range domains {
|
|
domain := &domains[index]
|
|
if excludedUUID != "" {
|
|
candidateUUID, uuidErr := domain.GetUUIDString()
|
|
if uuidErr != nil {
|
|
domain.Free()
|
|
return false, translateLibvirtError("read domain UUID for volume reference", uuidErr)
|
|
}
|
|
if strings.EqualFold(candidateUUID, excludedUUID) {
|
|
domain.Free()
|
|
continue
|
|
}
|
|
}
|
|
flags := []libvirt.DomainXMLFlags{0}
|
|
persistent, persistentErr := domain.IsPersistent()
|
|
if persistentErr != nil {
|
|
domain.Free()
|
|
return false, translateLibvirtError("inspect domain persistence for volume reference", persistentErr)
|
|
}
|
|
if persistent {
|
|
flags = append(flags, libvirt.DOMAIN_XML_INACTIVE)
|
|
}
|
|
for _, flag := range flags {
|
|
document, xmlErr := domain.GetXMLDesc(flag)
|
|
if xmlErr != nil {
|
|
domain.Free()
|
|
return false, translateLibvirtError("read domain XML for volume reference", xmlErr)
|
|
}
|
|
var definition libvirtxml.Domain
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
domain.Free()
|
|
return false, fmt.Errorf("parse domain XML for volume reference: %w", err)
|
|
}
|
|
if domainReferencesStorage(&definition, target) {
|
|
domain.Free()
|
|
return true, nil
|
|
}
|
|
}
|
|
referenced, historyErr := domainHistoryReferencesStorage(domain, target)
|
|
if historyErr != nil {
|
|
domain.Free()
|
|
return false, historyErr
|
|
}
|
|
if referenced {
|
|
domain.Free()
|
|
return true, nil
|
|
}
|
|
domain.Free()
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func domainHistoryReferencesStorage(domain *libvirt.Domain, target domainStorageReference) (bool, error) {
|
|
managedSave, present, err := managedSaveDomainDefinition(domain)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if present && domainReferencesStorage(managedSave, target) {
|
|
return true, nil
|
|
}
|
|
|
|
snapshots, err := domain.ListAllSnapshots(0)
|
|
if err != nil {
|
|
return false, translateLibvirtError("list domain snapshots for volume reference check", err)
|
|
}
|
|
defer func() {
|
|
for index := range snapshots {
|
|
_ = snapshots[index].Free()
|
|
}
|
|
}()
|
|
for index := range snapshots {
|
|
document, xmlErr := snapshots[index].GetXMLDesc(0)
|
|
if xmlErr != nil {
|
|
return false, translateLibvirtError("read domain snapshot XML for volume reference", xmlErr)
|
|
}
|
|
var definition libvirtxml.DomainSnapshot
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
return false, NewError(ErrConflict, "domain_snapshot_xml_invalid",
|
|
"domain snapshot XML could not be inspected for storage references", err)
|
|
}
|
|
if domainSnapshotReferencesStorage(&definition, target) {
|
|
return true, nil
|
|
}
|
|
privateState, parseErr := parseNativeSnapshotPrivateXML(document)
|
|
if parseErr != nil {
|
|
return false, NewError(ErrConflict, "domain_snapshot_xml_invalid",
|
|
"private domain snapshot XML could not be inspected for storage references", parseErr)
|
|
}
|
|
if nativeSnapshotPrivateReferencesStorage(privateState, target) {
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
checkpoints, err := domain.ListAllCheckpoints(0)
|
|
if err != nil {
|
|
return false, translateLibvirtError("list domain checkpoints for volume reference check", err)
|
|
}
|
|
defer func() {
|
|
for index := range checkpoints {
|
|
_ = checkpoints[index].Free()
|
|
}
|
|
}()
|
|
for index := range checkpoints {
|
|
document, xmlErr := checkpoints[index].GetXMLDesc(0)
|
|
if xmlErr != nil {
|
|
return false, translateLibvirtError("read domain checkpoint XML for volume reference", xmlErr)
|
|
}
|
|
var definition libvirtxml.DomainCheckpoint
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
return false, NewError(ErrConflict, "domain_checkpoint_xml_invalid",
|
|
"domain checkpoint XML could not be inspected for storage references", err)
|
|
}
|
|
if definition.Domain != nil && domainReferencesStorage(definition.Domain, target) {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func managedSaveDomainDefinition(domain *libvirt.Domain) (*libvirtxml.Domain, bool, error) {
|
|
present, err := domain.HasManagedSaveImage(0)
|
|
if err != nil {
|
|
return nil, false, translateLibvirtError("inspect domain managed-save image", err)
|
|
}
|
|
if !present {
|
|
return nil, false, nil
|
|
}
|
|
document, err := domain.ManagedSaveGetXMLDesc(0)
|
|
if err != nil {
|
|
return nil, false, translateLibvirtError("read domain managed-save XML", err)
|
|
}
|
|
definition, err := parseHistoricalDomainXML(document, "managed_save")
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
return definition, true, nil
|
|
}
|
|
|
|
func parseHistoricalDomainXML(document, kind string) (*libvirtxml.Domain, error) {
|
|
var definition libvirtxml.Domain
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
return nil, NewError(ErrConflict, "domain_"+kind+"_xml_invalid",
|
|
"historical domain XML could not be inspected for resource references", err)
|
|
}
|
|
return &definition, nil
|
|
}
|
|
|
|
func domainReferencesPath(definition *libvirtxml.Domain, path string) bool {
|
|
return domainReferencesStorage(definition, domainStorageReference{Path: path})
|
|
}
|
|
|
|
func domainReferencesStorage(definition *libvirtxml.Domain, target domainStorageReference) bool {
|
|
if definition == nil {
|
|
return false
|
|
}
|
|
if target.Path != "" {
|
|
target.Path = filepath.Clean(target.Path)
|
|
}
|
|
if definition.OS != nil {
|
|
if definition.OS.NVRam != nil {
|
|
if storagePathMatches(definition.OS.NVRam.NVRam, target.Path) {
|
|
return true
|
|
}
|
|
if domainDiskSourceReferences(definition.OS.NVRam.Source, target) {
|
|
return true
|
|
}
|
|
}
|
|
if definition.OS.VarStore != nil &&
|
|
storagePathMatches(definition.OS.VarStore.Path, target.Path) {
|
|
return true
|
|
}
|
|
}
|
|
if definition.Devices == nil {
|
|
return false
|
|
}
|
|
for _, disk := range definition.Devices.Disks {
|
|
if domainDiskSourceReferences(disk.Source, target) ||
|
|
domainBackingStoreReferences(disk.BackingStore, target) ||
|
|
domainMirrorReferences(disk.Mirror, target) {
|
|
return true
|
|
}
|
|
}
|
|
for _, filesystem := range definition.Devices.Filesystems {
|
|
if filesystem.Source == nil {
|
|
continue
|
|
}
|
|
if (filesystem.Source.File != nil &&
|
|
storagePathMatches(filesystem.Source.File.File, target.Path)) ||
|
|
(filesystem.Source.Block != nil &&
|
|
storagePathMatches(filesystem.Source.Block.Dev, target.Path)) ||
|
|
(filesystem.Source.Volume != nil &&
|
|
storageVolumeMatches(filesystem.Source.Volume.Pool,
|
|
filesystem.Source.Volume.Volume, target)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func domainSnapshotReferencesStorage(snapshot *libvirtxml.DomainSnapshot, target domainStorageReference) bool {
|
|
if snapshot == nil {
|
|
return false
|
|
}
|
|
if snapshot.Domain != nil && domainReferencesStorage(snapshot.Domain, target) {
|
|
return true
|
|
}
|
|
if snapshot.InactiveDomain != nil &&
|
|
domainReferencesStorage(&snapshot.InactiveDomain.Domain, target) {
|
|
return true
|
|
}
|
|
if snapshot.Memory != nil && storagePathMatches(snapshot.Memory.File, target.Path) {
|
|
return true
|
|
}
|
|
if snapshot.Disks != nil {
|
|
for _, disk := range snapshot.Disks.Disks {
|
|
if domainDiskSourceReferences(disk.Source, target) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
type nativeSnapshotPrivateDisk struct {
|
|
Source *nativeSnapshotPrivateSource `xml:"source"`
|
|
SnapshotDeleteInProgress *struct{} `xml:"snapshotDeleteInProgress"`
|
|
}
|
|
|
|
type nativeSnapshotPrivateSource struct {
|
|
File string `xml:"file,attr"`
|
|
Dev string `xml:"dev,attr"`
|
|
Dir string `xml:"dir,attr"`
|
|
Pool string `xml:"pool,attr"`
|
|
Volume string `xml:"volume,attr"`
|
|
DataStore *nativeSnapshotPrivateDataStore `xml:"dataStore"`
|
|
}
|
|
|
|
type nativeSnapshotPrivateDataStore struct {
|
|
Source *nativeSnapshotPrivateSource `xml:"source"`
|
|
}
|
|
|
|
type nativeSnapshotPrivateDisks struct {
|
|
Disks []nativeSnapshotPrivateDisk `xml:"disk"`
|
|
}
|
|
|
|
type nativeSnapshotPrivateXML struct {
|
|
XMLName xml.Name `xml:"domainsnapshot"`
|
|
Disks nativeSnapshotPrivateDisks `xml:"disks"`
|
|
RevertDisks nativeSnapshotPrivateDisks `xml:"revertDisks"`
|
|
}
|
|
|
|
func parseNativeSnapshotPrivateXML(document string) (*nativeSnapshotPrivateXML, error) {
|
|
var state nativeSnapshotPrivateXML
|
|
if err := xml.Unmarshal([]byte(document), &state); err != nil {
|
|
return nil, err
|
|
}
|
|
return &state, nil
|
|
}
|
|
|
|
func nativeSnapshotPrivateReferencesStorage(state *nativeSnapshotPrivateXML,
|
|
target domainStorageReference) bool {
|
|
if state == nil {
|
|
return false
|
|
}
|
|
for _, group := range []nativeSnapshotPrivateDisks{state.Disks, state.RevertDisks} {
|
|
for _, disk := range group.Disks {
|
|
if nativeSnapshotPrivateSourceReferences(disk.Source, target) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func nativeSnapshotPrivateSourceReferences(source *nativeSnapshotPrivateSource,
|
|
target domainStorageReference) bool {
|
|
if source == nil {
|
|
return false
|
|
}
|
|
if storagePathMatches(source.File, target.Path) ||
|
|
storagePathMatches(source.Dev, target.Path) ||
|
|
storagePathMatches(source.Dir, target.Path) ||
|
|
storageVolumeMatches(source.Pool, source.Volume, target) {
|
|
return true
|
|
}
|
|
return source.DataStore != nil &&
|
|
nativeSnapshotPrivateSourceReferences(source.DataStore.Source, target)
|
|
}
|
|
|
|
func validateNativeSnapshotDeleteSafety(document string) error {
|
|
var snapshot libvirtxml.DomainSnapshot
|
|
if err := snapshot.Unmarshal(document); err != nil {
|
|
return NewError(ErrConflict, "domain_snapshot_xml_invalid",
|
|
"native snapshot XML could not be inspected before delete", err)
|
|
}
|
|
privateState, err := parseNativeSnapshotPrivateXML(document)
|
|
if err != nil {
|
|
return NewError(ErrConflict, "domain_snapshot_xml_invalid",
|
|
"native snapshot delete state could not be inspected", err)
|
|
}
|
|
for _, group := range []nativeSnapshotPrivateDisks{privateState.Disks, privateState.RevertDisks} {
|
|
for _, disk := range group.Disks {
|
|
if disk.SnapshotDeleteInProgress != nil {
|
|
return NewError(ErrConflict, "native_snapshot_delete_in_progress",
|
|
"native snapshot deletion is still in progress", nil)
|
|
}
|
|
}
|
|
}
|
|
if len(privateState.RevertDisks.Disks) != 0 {
|
|
return NewError(ErrConflict, "external_snapshot_cleanup_required",
|
|
"native snapshot rollback metadata must be completed before deleting the VM", nil)
|
|
}
|
|
if snapshot.Memory != nil &&
|
|
(!nativeSnapshotContentModeSafe(snapshot.Memory.Snapshot) || snapshot.Memory.File != "") {
|
|
return NewError(ErrConflict, "external_snapshot_cleanup_required",
|
|
"external native snapshot memory must be removed before deleting the VM", nil)
|
|
}
|
|
if snapshot.Disks != nil {
|
|
for _, disk := range snapshot.Disks.Disks {
|
|
if !nativeSnapshotContentModeSafe(disk.Snapshot) ||
|
|
domainDiskSourceHasLocator(disk.Source) {
|
|
return NewError(ErrConflict, "external_snapshot_cleanup_required",
|
|
"external native snapshot disks must be removed before deleting the VM", nil)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func nativeSnapshotContentModeSafe(mode string) bool {
|
|
switch strings.ToLower(strings.TrimSpace(mode)) {
|
|
case "", "no", "internal":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func domainDiskSourceHasLocator(source *libvirtxml.DomainDiskSource) bool {
|
|
if source == nil {
|
|
return false
|
|
}
|
|
if (source.File != nil && source.File.File != "") ||
|
|
(source.Block != nil && source.Block.Dev != "") ||
|
|
(source.Dir != nil && source.Dir.Dir != "") ||
|
|
(source.Volume != nil && (source.Volume.Pool != "" || source.Volume.Volume != "")) ||
|
|
(source.Network != nil && (source.Network.Name != "" || len(source.Network.Hosts) != 0)) {
|
|
return true
|
|
}
|
|
return source.DataStore != nil && domainDiskSourceHasLocator(source.DataStore.Source)
|
|
}
|
|
|
|
func storagePathMatches(candidate, target string) bool {
|
|
if candidate == "" || target == "" {
|
|
return false
|
|
}
|
|
if filepath.Clean(candidate) == target {
|
|
return true
|
|
}
|
|
candidateInfo, candidateStatErr := os.Stat(candidate)
|
|
targetInfo, targetStatErr := os.Stat(target)
|
|
if candidateStatErr == nil && targetStatErr == nil &&
|
|
os.SameFile(candidateInfo, targetInfo) {
|
|
return true
|
|
}
|
|
resolvedCandidate, candidateErr := filepath.EvalSymlinks(candidate)
|
|
resolvedTarget, targetErr := filepath.EvalSymlinks(target)
|
|
return candidateErr == nil && targetErr == nil &&
|
|
filepath.Clean(resolvedCandidate) == filepath.Clean(resolvedTarget)
|
|
}
|
|
|
|
func storageVolumeMatches(pool, volume string, target domainStorageReference) bool {
|
|
return pool != "" && volume != "" && target.Pool != "" && target.Volume != "" &&
|
|
pool == target.Pool && volume == target.Volume
|
|
}
|
|
|
|
func domainDiskSourceReferences(source *libvirtxml.DomainDiskSource, target domainStorageReference) bool {
|
|
if source == nil {
|
|
return false
|
|
}
|
|
if (source.File != nil && storagePathMatches(source.File.File, target.Path)) ||
|
|
(source.Block != nil && storagePathMatches(source.Block.Dev, target.Path)) ||
|
|
(source.Dir != nil && storagePathMatches(source.Dir.Dir, target.Path)) ||
|
|
(source.Volume != nil && storageVolumeMatches(source.Volume.Pool, source.Volume.Volume, target)) {
|
|
return true
|
|
}
|
|
return source.DataStore != nil && domainDiskSourceReferences(source.DataStore.Source, target)
|
|
}
|
|
|
|
func domainBackingStoreReferences(store *libvirtxml.DomainDiskBackingStore, target domainStorageReference) bool {
|
|
for store != nil {
|
|
if domainDiskSourceReferences(store.Source, target) {
|
|
return true
|
|
}
|
|
store = store.BackingStore
|
|
}
|
|
return false
|
|
}
|
|
|
|
func domainMirrorReferences(mirror *libvirtxml.DomainDiskMirror, target domainStorageReference) bool {
|
|
return mirror != nil && (domainDiskSourceReferences(mirror.Source, target) ||
|
|
domainBackingStoreReferences(mirror.BackingStore, target))
|
|
}
|