You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
805 lines
30 KiB
Go
805 lines
30 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/cloudinit"
|
|
"git.luoli.live/root/virt/agent/objecttransfer"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
func (l *Libvirt) CreateBackup(ctx context.Context, uuid string, request BackupRequest) (_ BackupResult, resultErr error) {
|
|
if err := ValidateBackupRequest(request); err != nil {
|
|
return BackupResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
return l.createPortableBackup(ctx, uuid, request)
|
|
}
|
|
|
|
// GetBackupReadiness reports only transient blockers which automatic backup
|
|
// may safely defer. It deliberately does not reserve resources or consume
|
|
// retained job evidence, and CreateBackup repeats these checks before mutation.
|
|
func (l *Libvirt) GetBackupReadiness(ctx context.Context, uuid string) (BackupReadiness, error) {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return BackupReadiness{}, err
|
|
}
|
|
defer domain.Free()
|
|
_, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return BackupReadiness{}, err
|
|
}
|
|
if reason := backupTransientBlocker(metadata); reason != "" {
|
|
return BackupReadiness{Eligible: false, Retryable: true, Reason: reason}, nil
|
|
}
|
|
if err := ensureNoActiveDomainJobIfRunning(domain); err != nil {
|
|
var public interface{ PublicCode() string }
|
|
if errors.As(err, &public) && public.PublicCode() == "domain_job_active" {
|
|
return BackupReadiness{
|
|
Eligible: false, Retryable: true, Reason: "domain_job_active",
|
|
}, nil
|
|
}
|
|
return BackupReadiness{}, err
|
|
}
|
|
return BackupReadiness{Eligible: true, Retryable: false}, nil
|
|
}
|
|
|
|
func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request BackupRequest) (_ BackupResult, resultErr error) {
|
|
client := objecttransfer.New()
|
|
defer client.CloseIdleConnections()
|
|
active, sourceContract, err := l.validateBackupSource(ctx, uuid, request.BackupKey)
|
|
if err != nil {
|
|
return BackupResult{}, abortRejectedBackupPlan(
|
|
ctx, client.AbortMultipart, request.ObjectUpload, err,
|
|
)
|
|
}
|
|
snapshotRequest := SnapshotRequest{
|
|
OperationID: request.OperationID, SnapshotKey: request.BackupKey,
|
|
Name: request.Name, ConsistencyMode: request.ConsistencyMode,
|
|
}
|
|
var snapshot Snapshot
|
|
if active {
|
|
snapshot, err = l.createBackupWorkspace(ctx, uuid, snapshotRequest)
|
|
} else {
|
|
snapshot, err = l.createStoppedBackupWorkspace(ctx, uuid, snapshotRequest)
|
|
}
|
|
if err != nil {
|
|
return BackupResult{}, err
|
|
}
|
|
|
|
volume, err := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), backupWorkspaceVolumeName(uuid, request.BackupKey),
|
|
)
|
|
if err != nil || volume == nil {
|
|
return BackupResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "backup_snapshot_unavailable", "local backup snapshot is unavailable", err,
|
|
))
|
|
}
|
|
defer volume.Free()
|
|
info, err := validateStandaloneQCOW2Volume(volume, snapshot.SizeBytes)
|
|
if err != nil {
|
|
return BackupResult{}, operation.MarkUncertain(err)
|
|
}
|
|
path, err := volume.GetPath()
|
|
if err != nil || !filepath.IsAbs(path) {
|
|
return BackupResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "backup_snapshot_unavailable", "local backup snapshot path is unavailable",
|
|
translateOptionalLibvirtError("resolve backup snapshot path", err),
|
|
))
|
|
}
|
|
|
|
upload, err := client.UploadFile(ctx, path, request.ObjectUpload)
|
|
if err != nil {
|
|
translated := translateObjectTransferError(err)
|
|
if objecttransfer.IsAmbiguous(err) {
|
|
// Multipart completion may already have committed. Retain the local
|
|
// immutable snapshot and the operation fence for explicit recovery.
|
|
return BackupResult{}, translated
|
|
}
|
|
cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx)
|
|
cleanupErr := l.cleanupBackupWorkspace(cleanupCtx, uuid, request.BackupKey)
|
|
cleanupCancel()
|
|
if cleanupErr != nil {
|
|
return BackupResult{}, classifyBackupUploadFailure(translated, cleanupErr)
|
|
}
|
|
return BackupResult{}, classifyBackupUploadFailure(translated, nil)
|
|
}
|
|
if upload.SizeBytes <= 0 || upload.SHA256 == "" {
|
|
return BackupResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "backup_upload_invalid", "backup upload result is invalid", nil,
|
|
))
|
|
}
|
|
result := BackupResult{
|
|
BackupKey: request.BackupKey, SizeBytes: upload.SizeBytes,
|
|
VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256,
|
|
ObjectETag: upload.ETag, ObjectVersionID: upload.VersionID,
|
|
Consistency: snapshot.Consistency,
|
|
SourceContract: &sourceContract,
|
|
}
|
|
cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx)
|
|
cleanupErr := l.cleanupBackupWorkspace(cleanupCtx, uuid, request.BackupKey)
|
|
cleanupCancel()
|
|
if cleanupErr != nil {
|
|
// The immutable remote object is already fully verified. Local temporary
|
|
// cleanup cannot revoke that success or discard its only SHA/size receipt;
|
|
// publish a bounded cleanup marker for follow-up instead of fencing the
|
|
// usable backup as an unknown remote effect.
|
|
result.CleanupPending = true
|
|
return result, nil
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
type maintenanceMigrationSourceState struct {
|
|
wasRunning bool
|
|
autostart bool
|
|
}
|
|
|
|
func (l *Libvirt) inspectMaintenanceMigrationSource(ctx context.Context,
|
|
uuid string) (maintenanceMigrationSourceState, error) {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return maintenanceMigrationSourceState{}, err
|
|
}
|
|
defer domain.Free()
|
|
autostart, err := domain.GetAutostart()
|
|
if err != nil {
|
|
return maintenanceMigrationSourceState{}, operation.MarkNotApplied(
|
|
translateLibvirtError("read maintenance migration autostart", err),
|
|
)
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return maintenanceMigrationSourceState{}, operation.MarkNotApplied(
|
|
translateLibvirtError("read maintenance migration power state", err),
|
|
)
|
|
}
|
|
return maintenanceMigrationSourceState{wasRunning: active, autostart: autostart}, nil
|
|
}
|
|
|
|
func (l *Libvirt) stopMaintenanceMigrationSource(ctx context.Context, uuid string,
|
|
state maintenanceMigrationSourceState) error {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return operation.MarkNotApplied(err)
|
|
}
|
|
defer domain.Free()
|
|
autostart, autostartErr := domain.GetAutostart()
|
|
active, activeErr := domain.IsActive()
|
|
if autostartErr != nil || activeErr != nil {
|
|
return operation.MarkNotApplied(errors.Join(
|
|
translateOptionalLibvirtError("recheck maintenance migration autostart", autostartErr),
|
|
translateOptionalLibvirtError("recheck maintenance migration power state", activeErr),
|
|
))
|
|
}
|
|
if autostart != state.autostart || active != state.wasRunning {
|
|
return operation.MarkNotApplied(NewError(
|
|
ErrConflict, "maintenance_source_state_changed",
|
|
"maintenance migration source state changed during preflight", nil,
|
|
))
|
|
}
|
|
if err := setServiceAutostart(domain, false); err != nil {
|
|
if rollbackErr := l.restoreMaintenanceMigrationDomain(domain, state, func() error {
|
|
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
|
|
}); rollbackErr != nil {
|
|
return operation.MarkUncertain(errors.Join(err, rollbackErr))
|
|
}
|
|
return err
|
|
}
|
|
if err := requestGracefulDomainShutoff(ctx, domain, 60*time.Second); err != nil {
|
|
if rollbackErr := l.restoreMaintenanceMigrationDomain(domain, state, func() error {
|
|
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
|
|
}); rollbackErr != nil {
|
|
return operation.MarkUncertain(errors.Join(err, rollbackErr))
|
|
}
|
|
return err
|
|
}
|
|
observedAutostart, err := domain.GetAutostart()
|
|
if err != nil {
|
|
confirmationErr := operation.MarkUncertain(translateLibvirtError(
|
|
"confirm maintenance migration autostart", err,
|
|
))
|
|
if rollbackErr := l.restoreMaintenanceMigrationDomain(domain, state, func() error {
|
|
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
|
|
}); rollbackErr != nil {
|
|
return operation.MarkUncertain(errors.Join(confirmationErr, rollbackErr))
|
|
}
|
|
return confirmationErr
|
|
}
|
|
observedActive, err := domain.IsActive()
|
|
if err != nil || observedAutostart || observedActive {
|
|
confirmationErr := operation.MarkUncertain(errors.Join(
|
|
translateOptionalLibvirtError("confirm maintenance migration power state", err),
|
|
NewError(ErrConflict, "maintenance_source_not_stopped",
|
|
"maintenance migration source did not reach the stopped state", nil),
|
|
))
|
|
if rollbackErr := l.restoreMaintenanceMigrationDomain(domain, state, func() error {
|
|
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
|
|
}); rollbackErr != nil {
|
|
return operation.MarkUncertain(errors.Join(confirmationErr, rollbackErr))
|
|
}
|
|
return confirmationErr
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) restoreMaintenanceMigrationSource(ctx context.Context,
|
|
uuid string, state maintenanceMigrationSourceState) error {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return operation.MarkUncertain(err)
|
|
}
|
|
defer domain.Free()
|
|
return l.restoreMaintenanceMigrationDomain(domain, state, func() error {
|
|
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
|
|
})
|
|
}
|
|
|
|
type maintenanceMigrationDomainControl interface {
|
|
serviceStateDomainControl
|
|
Create() error
|
|
}
|
|
|
|
func (l *Libvirt) restoreMaintenanceMigrationDomain(
|
|
domain maintenanceMigrationDomainControl,
|
|
state maintenanceMigrationSourceState,
|
|
start func() error,
|
|
) error {
|
|
if err := domain.SetAutostart(state.autostart); err != nil {
|
|
return operation.MarkUncertain(mutationLibvirtError(
|
|
"restore maintenance migration autostart", err,
|
|
))
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.MarkUncertain(translateLibvirtError(
|
|
"inspect maintenance migration rollback power state", err,
|
|
))
|
|
}
|
|
if state.wasRunning && !active {
|
|
if start == nil {
|
|
return operation.MarkUncertain(NewError(ErrUnavailable,
|
|
"public_guard_unavailable", "maintenance migration restart guard is unavailable", nil))
|
|
}
|
|
if err := start(); err != nil {
|
|
return operation.MarkUncertain(mutationLibvirtError(
|
|
"restart maintenance migration source", err,
|
|
))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func abortRejectedBackupPlan(ctx context.Context,
|
|
abort func(context.Context, objecttransfer.MultipartPlan) error,
|
|
plan objecttransfer.MultipartPlan, rejection error) error {
|
|
if cleanupErr := abort(ctx, plan); cleanupErr != nil {
|
|
// The VM and backup object were never mutated, but PHP must not treat
|
|
// the original transient preflight cause as proof that multipart
|
|
// cleanup completed. Keep retry-safe semantics while publishing a
|
|
// distinct bounded cause that requires operator reconciliation.
|
|
return operation.MarkNotApplied(NewError(
|
|
ErrUnavailable,
|
|
"backup_upload_abort_failed",
|
|
"backup upload cleanup could not be confirmed",
|
|
errors.Join(rejection, cleanupErr),
|
|
))
|
|
}
|
|
return operation.MarkNotApplied(rejection)
|
|
}
|
|
|
|
func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey string) (bool, PortableRestoreContract, error) {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return false, PortableRestoreContract{}, err
|
|
}
|
|
defer domain.Free()
|
|
_, definition, metadata, pool, volume, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
return false, PortableRestoreContract{}, err
|
|
}
|
|
defer pool.Free()
|
|
defer volume.Free()
|
|
if err := validateBackupTransientState(metadata); err != nil {
|
|
return false, PortableRestoreContract{}, err
|
|
}
|
|
for _, stored := range metadata.Snapshots {
|
|
if stored.Key == backupKey {
|
|
return false, PortableRestoreContract{}, NewError(ErrConflict, "backup_exists", "local backup work already exists", nil)
|
|
}
|
|
}
|
|
if err := validateSingleWritableRoot(definition); err != nil {
|
|
return false, PortableRestoreContract{}, err
|
|
}
|
|
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
|
|
return false, PortableRestoreContract{}, err
|
|
}
|
|
sourceContract, err := l.validatePortableRestoreDomain(definition, metadata)
|
|
if err != nil {
|
|
return false, PortableRestoreContract{}, err
|
|
}
|
|
for _, volumeName := range []string{
|
|
backupWorkspaceVolumeName(uuid, backupKey),
|
|
// Reject a legacy residue with the same key instead of silently
|
|
// allocating a second workspace for one logical backup.
|
|
snapshotVolumeName(uuid, backupKey),
|
|
} {
|
|
existing, err := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), volumeName,
|
|
)
|
|
if err != nil {
|
|
return false, PortableRestoreContract{}, err
|
|
}
|
|
if existing != nil {
|
|
existing.Free()
|
|
return false, PortableRestoreContract{}, NewError(ErrConflict, "backup_exists", "local backup work already exists", nil)
|
|
}
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return false, PortableRestoreContract{}, translateLibvirtError("check VM state for backup", err)
|
|
}
|
|
if active {
|
|
liveDocument, liveErr := domain.GetXMLDesc(0)
|
|
if liveErr != nil {
|
|
return false, PortableRestoreContract{}, translateLibvirtError("read live VM disk layout for backup", liveErr)
|
|
}
|
|
liveDefinition, liveMetadata, parseErr := parseManagedDomain(liveDocument)
|
|
if parseErr != nil {
|
|
return false, PortableRestoreContract{}, NewError(ErrConflict, "backup_platform_unsupported", "live VM platform is invalid for portable backup", nil)
|
|
}
|
|
liveContract, contractErr := l.validatePortableRestoreDomain(liveDefinition, liveMetadata)
|
|
if contractErr != nil || liveContract != sourceContract {
|
|
return false, PortableRestoreContract{}, NewError(ErrConflict, "backup_platform_unsupported", "live VM platform is unavailable for portable backup", nil)
|
|
}
|
|
managedPath, pathErr := volume.GetPath()
|
|
if pathErr != nil {
|
|
return false, PortableRestoreContract{}, translateLibvirtError("resolve live managed root for backup", pathErr)
|
|
}
|
|
if err := validateLiveBackupDefinition(liveDefinition, managedPath); err != nil {
|
|
return false, PortableRestoreContract{}, err
|
|
}
|
|
}
|
|
return active, sourceContract, nil
|
|
}
|
|
|
|
func (l *Libvirt) validatePortableRestoreDomain(definition *libvirtxml.Domain,
|
|
metadata *domainMetadataXML) (PortableRestoreContract, error) {
|
|
reject := func() (PortableRestoreContract, error) {
|
|
return PortableRestoreContract{}, NewError(
|
|
ErrConflict, "backup_platform_unsupported",
|
|
"VM platform is unavailable for portable backup", nil,
|
|
)
|
|
}
|
|
if definition == nil || metadata == nil || definition.Type != "kvm" ||
|
|
definition.OS == nil || definition.OS.Type == nil ||
|
|
definition.OS.Type.Type != "hvm" || definition.OS.Type.Arch != "x86_64" ||
|
|
!portableQ35Machine(definition.OS.Type.Machine) ||
|
|
definition.OS.Loader == nil ||
|
|
definition.OS.Loader.Path != l.local.FirmwareLoader ||
|
|
definition.OS.Loader.Type != "pflash" ||
|
|
definition.OS.Loader.Readonly != "yes" ||
|
|
definition.OS.Loader.Secure != "no" ||
|
|
definition.OS.NVRam == nil ||
|
|
definition.Features == nil || definition.Features.SMM == nil ||
|
|
definition.Features.SMM.State != "off" {
|
|
return reject()
|
|
}
|
|
if _, err := validateManagedDomainNVRAMPaths(
|
|
definition, l.local.NVRAMDirectory, definition.UUID,
|
|
); err != nil {
|
|
return reject()
|
|
}
|
|
if err := validateSingleWritableRoot(definition); err != nil {
|
|
return reject()
|
|
}
|
|
root, err := managedRootDisk(definition)
|
|
if err != nil || root.Driver == nil || root.Driver.Name != "qemu" ||
|
|
root.Driver.Type != "qcow2" || root.Target == nil ||
|
|
root.Target.Dev != "vda" || root.Target.Bus != "virtio" ||
|
|
root.Source == nil || root.Source.File == nil ||
|
|
domainBackingChainPresent(root.BackingStore) || root.Mirror != nil {
|
|
return reject()
|
|
}
|
|
public, err := metadata.PublicNetwork.public()
|
|
if err != nil || public.Contract != fixedPortableRestoreContract().PublicNetworkContract ||
|
|
!managedNetworkLayoutMatches(definition, metadata) {
|
|
return reject()
|
|
}
|
|
source, err := ParseNetworkSelector(public.NetworkSelector)
|
|
if err != nil {
|
|
return reject()
|
|
}
|
|
publicIndex, found := managedPublicInterfaceIndex(
|
|
definition, networkFilterName(definition.UUID), public,
|
|
)
|
|
if !found || !domainNetworkSourceMatches(
|
|
definition.Devices.Interfaces[publicIndex], source.Type, source.Source,
|
|
) {
|
|
return reject()
|
|
}
|
|
return fixedPortableRestoreContract(), nil
|
|
}
|
|
|
|
func portableQ35Machine(machine string) bool {
|
|
machine = strings.ToLower(strings.TrimSpace(machine))
|
|
return machine == "q35" || strings.HasPrefix(machine, "pc-q35-")
|
|
}
|
|
|
|
func backupTransientBlocker(metadata *domainMetadataXML) string {
|
|
if metadata != nil && metadata.RescueTemplateID != "" {
|
|
return "backup_rescue_active"
|
|
}
|
|
// Reinstall and snapshot restore deliberately retain their transition kind
|
|
// and operation ID after the old root has been deleted. Those markers prove
|
|
// a completed root switch during reconciliation; only a pending old root
|
|
// means that the disk transition is still active.
|
|
if metadata != nil && metadata.PendingCleanupVolume != "" {
|
|
return "backup_disk_transition_active"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func validateBackupTransientState(metadata *domainMetadataXML) error {
|
|
switch backupTransientBlocker(metadata) {
|
|
case "backup_rescue_active":
|
|
return NewError(ErrConflict, "backup_rescue_active", "VM backup is unavailable while rescue media is attached", nil)
|
|
case "backup_disk_transition_active":
|
|
return NewError(ErrConflict, "backup_disk_transition_active", "VM backup is unavailable during a disk transition", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateBackupTransitionState(metadata *domainMetadataXML) error {
|
|
return validateBackupTransientState(metadata)
|
|
}
|
|
|
|
func validateSingleWritableRoot(definition *libvirtxml.Domain) error {
|
|
if definition == nil || definition.Devices == nil {
|
|
return NewError(ErrConflict, "backup_disk_layout_unsupported", "VM disk layout is unavailable for portable backup", nil)
|
|
}
|
|
writable := 0
|
|
for index := range definition.Devices.Disks {
|
|
disk := &definition.Devices.Disks[index]
|
|
if disk.ReadOnly != nil {
|
|
continue
|
|
}
|
|
if disk.Device != "disk" {
|
|
return NewError(ErrConflict, "backup_extra_writable_disk", "portable backup supports only the managed vda system disk", nil)
|
|
}
|
|
writable++
|
|
if disk.Target == nil || disk.Target.Dev != "vda" {
|
|
return NewError(ErrConflict, "backup_extra_writable_disk", "portable backup supports only the managed vda system disk", nil)
|
|
}
|
|
}
|
|
if writable != 1 {
|
|
return NewError(ErrConflict, "backup_disk_layout_unsupported", "portable backup requires exactly one writable system disk", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateLiveBackupDefinition(definition *libvirtxml.Domain, managedRootPath string) error {
|
|
if err := validateSingleWritableRoot(definition); err != nil {
|
|
return err
|
|
}
|
|
root, err := managedRootDisk(definition)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if root.Source == nil || root.Source.File == nil || !filepath.IsAbs(managedRootPath) ||
|
|
filepath.Clean(root.Source.File.File) != filepath.Clean(managedRootPath) ||
|
|
domainBackingChainPresent(root.BackingStore) || root.Mirror != nil {
|
|
return NewError(ErrConflict, "backup_live_root_unsupported", "live VM root disk does not match the standalone managed system disk", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func domainBackingChainPresent(store *libvirtxml.DomainDiskBackingStore) bool {
|
|
for store != nil {
|
|
// libvirt commonly serializes an empty <backingStore/> sentinel for a
|
|
// standalone qcow2 disk. libvirtxml materializes that sentinel with a
|
|
// non-nil, zero-valued Source, so pointer presence alone is not evidence
|
|
// of a backing layer. Any actual source content, format or index remains
|
|
// fail-closed because it makes the portable snapshot unsafe.
|
|
if store.Index != 0 || store.Format != nil ||
|
|
domainDiskSourcePresent(store.Source) {
|
|
return true
|
|
}
|
|
store = store.BackingStore
|
|
}
|
|
return false
|
|
}
|
|
|
|
func domainDiskSourcePresent(source *libvirtxml.DomainDiskSource) bool {
|
|
return source != nil && !reflect.ValueOf(*source).IsZero()
|
|
}
|
|
|
|
func validateStandaloneQCOW2Volume(volume qcow2VolumeInspector, expectedCapacity uint64) (*libvirt.StorageVolInfo, error) {
|
|
info, err := validateQCOW2Volume(volume, expectedCapacity)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
document, err := volume.GetXMLDesc(0)
|
|
if err != nil {
|
|
return nil, translateLibvirtError("read standalone qcow2 volume XML", err)
|
|
}
|
|
var definition libvirtxml.StorageVolume
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
return nil, NewError(ErrConflict, "invalid_qcow2_volume", "managed qcow2 volume metadata is invalid", err)
|
|
}
|
|
if definition.BackingStore != nil && definition.BackingStore.Path != "" {
|
|
return nil, NewError(ErrConflict, "backup_external_backing_unsupported", "portable backup requires a standalone system disk", nil)
|
|
}
|
|
return info, nil
|
|
}
|
|
|
|
func (l *Libvirt) createStoppedBackupWorkspace(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
defer domain.Free()
|
|
_, definition, metadata, pool, source, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
defer pool.Free()
|
|
defer source.Free()
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return Snapshot{}, translateLibvirtError("check VM state for offline backup", err)
|
|
}
|
|
if active {
|
|
return Snapshot{}, NewError(ErrConflict, "backup_state_changed", "VM state changed while preparing backup", nil)
|
|
}
|
|
if err := validateSingleWritableRoot(definition); err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
if metadata.RescueTemplateID != "" {
|
|
return Snapshot{}, NewError(ErrConflict, "backup_rescue_active", "VM backup is unavailable while rescue media is attached", nil)
|
|
}
|
|
sourceInfo, err := validateStandaloneQCOW2Volume(source, 0)
|
|
if err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
poolInfo, err := pool.GetInfo()
|
|
if err != nil {
|
|
return Snapshot{}, translateLibvirtError("inspect backup storage capacity", err)
|
|
}
|
|
if sourceInfo.Capacity > poolInfo.Available {
|
|
return Snapshot{}, NewError(ErrConflict, "backup_storage_full", "managed storage pool does not have enough free space", nil)
|
|
}
|
|
volumeName := backupWorkspaceVolumeName(uuid, request.SnapshotKey)
|
|
volumeXML, err := storageVolumeXMLAllocated(volumeName, sourceInfo.Capacity, "qcow2")
|
|
if err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
target, err := pool.StorageVolCreateXMLFrom(volumeXML, source, 0)
|
|
if err != nil {
|
|
absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName)
|
|
return Snapshot{}, deterministicVolumeCreateFailure(
|
|
"create offline backup snapshot", err, inspectErr, !absent,
|
|
)
|
|
}
|
|
targetOwned := true
|
|
defer func() {
|
|
if targetOwned {
|
|
if cleanupErr := target.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
|
|
resultErr = operation.MarkUncertain(errors.Join(resultErr,
|
|
mutationLibvirtError("roll back offline backup snapshot", cleanupErr)))
|
|
}
|
|
}
|
|
_ = target.Free()
|
|
}()
|
|
if err := establishNewRootVolumeCapacity(target, sourceInfo.Capacity); err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
info, err := validateStandaloneQCOW2Volume(target, sourceInfo.Capacity)
|
|
if err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
created := time.Now().UTC()
|
|
actualConsistency := request.ConsistencyMode
|
|
if actualConsistency == "filesystem" {
|
|
// A powered-off disk is stable, but the Agent cannot prove that the
|
|
// previous shutdown was guest-coordinated, so it reports crash safety.
|
|
actualConsistency = "crash"
|
|
}
|
|
targetOwned = false
|
|
return Snapshot{
|
|
ID: request.SnapshotKey, SnapshotKey: request.SnapshotKey,
|
|
Name: request.Name, CreatedAt: created, State: "ready",
|
|
Consistency: actualConsistency, SizeBytes: info.Capacity, Created: true,
|
|
}, nil
|
|
}
|
|
|
|
func (l *Libvirt) RestoreBackup(ctx context.Context, request RestoreBackupRequest) (_ RestoreBackupResult, resultErr error) {
|
|
if err := ValidateRestoreBackupRequest(request); err != nil {
|
|
return RestoreBackupResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
if err := l.ensureRestoreTargetAbsent(ctx, request.UUID); err != nil {
|
|
return RestoreBackupResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
create := request.CreateVMRequest
|
|
create.Source = "backup"
|
|
create.TemplateID = backupTemplateID(request.BackupKey)
|
|
|
|
attachment, err := l.resolvePublicNetwork(ctx, create)
|
|
if err != nil {
|
|
return RestoreBackupResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
mac := vmMACAddress(create.UUID)
|
|
conflict, err := l.networkIdentityConflict(ctx, create.UUID, mac, create.PublicNetwork)
|
|
if err != nil {
|
|
return RestoreBackupResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
if conflict {
|
|
return RestoreBackupResult{}, operation.MarkNotApplied(NewError(ErrConflict, "network_identity_conflict", "VM network identity is already in use", nil))
|
|
}
|
|
attachment.MAC = mac
|
|
seed, err := cloudinit.BuildISO(cloudinit.Request{
|
|
UUID: create.UUID, Name: create.Name, Hostname: create.Hostname,
|
|
PreserveGuestCredentials: true,
|
|
Debian12NetworkFix: *request.Debian12NetworkFix,
|
|
Network: cloudinit.Network{
|
|
MAC: mac, Public: create.PublicNetwork, InterfaceKey: "public0",
|
|
},
|
|
})
|
|
if err != nil {
|
|
return RestoreBackupResult{}, operation.MarkNotApplied(NewError(ErrInvalid, "invalid_cloud_init", "backup restore cloud-init request is invalid", err))
|
|
}
|
|
root, err := l.createRootVolumeFromBackup(ctx, request)
|
|
if err != nil {
|
|
return RestoreBackupResult{}, err
|
|
}
|
|
metadata, err := l.createVMWithPreparedRoot(ctx, create, attachment, seed, root)
|
|
if err != nil {
|
|
return RestoreBackupResult{}, err
|
|
}
|
|
return RestoreBackupResult{
|
|
VMMetadata: metadata, BackupKey: request.BackupKey,
|
|
SHA256: request.SourceSHA256,
|
|
}, nil
|
|
}
|
|
|
|
func (l *Libvirt) ensureRestoreTargetAbsent(ctx context.Context, uuid string) error {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
switch {
|
|
case err == nil:
|
|
_ = domain.Free()
|
|
return NewError(ErrConflict, "restore_target_exists", "backup restore target VM already exists", nil)
|
|
case !errors.Is(err, ErrNotFound):
|
|
return err
|
|
}
|
|
absent, err := l.deleteArtifactsAbsent(uuid)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !absent || l.traffic.contains(uuid) {
|
|
return NewError(ErrConflict, "restore_target_artifacts_exist", "managed artifacts already exist for the backup restore target", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) createRootVolumeFromBackup(ctx context.Context, request RestoreBackupRequest) (*libvirt.StorageVol, error) {
|
|
capacity := request.DiskGiB << 30
|
|
initialCapacity := capacity
|
|
if objectBytes := uint64(request.SourceSizeBytes); objectBytes > initialCapacity {
|
|
// StorageVolUpload bounds the byte stream by the volume capacity. A
|
|
// fully allocated qcow2 file can be slightly larger than its virtual
|
|
// capacity because of qcow2 metadata, so reserve enough room for the
|
|
// exact object before inspecting and resizing its virtual size.
|
|
initialCapacity = objectBytes
|
|
}
|
|
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(translateLibvirtError("lookup managed storage pool", err))
|
|
}
|
|
defer pool.Free()
|
|
definition := libvirtxml.StorageVolume{
|
|
Name: rootVolumeName(request.UUID),
|
|
Capacity: &libvirtxml.StorageVolumeSize{Value: initialCapacity, Unit: "bytes"},
|
|
Target: &libvirtxml.StorageVolumeTarget{
|
|
Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"},
|
|
},
|
|
}
|
|
document, err := definition.Marshal()
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(fmt.Errorf("marshal restored root volume XML: %w", err))
|
|
}
|
|
volume, err := pool.StorageVolCreateXML(document, 0)
|
|
if err != nil {
|
|
absent, inspectErr := l.dirPoolVolumeAbsent(pool, rootVolumeName(request.UUID))
|
|
return nil, createVMStorageCloneStage.wrap(deterministicArtifactCreateFailure(
|
|
"create restored system disk", err, inspectErr, !absent,
|
|
))
|
|
}
|
|
stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK)
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, rootVolumeName(request.UUID), err, "roll back restored system disk",
|
|
))
|
|
}
|
|
transfer := &storageStreamTransfer{stream: stream}
|
|
if err := volume.Upload(stream, 0, uint64(request.SourceSizeBytes), 0); err != nil {
|
|
cause := transfer.abort(err)
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, rootVolumeName(request.UUID), cause, "roll back restored system disk",
|
|
))
|
|
}
|
|
client := objecttransfer.New()
|
|
defer client.CloseIdleConnections()
|
|
download, downloadErr := client.Download(
|
|
ctx, request.SourceURL, request.SourceSizeBytes, request.SourceSHA256,
|
|
&storageStreamWriter{ctx: ctx, transfer: transfer},
|
|
)
|
|
if downloadErr != nil {
|
|
cause := transfer.abort(translateObjectTransferError(downloadErr))
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, rootVolumeName(request.UUID), cause, "roll back restored system disk",
|
|
))
|
|
}
|
|
if err := transfer.finish(ctx); err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, rootVolumeName(request.UUID), err, "roll back restored system disk",
|
|
))
|
|
}
|
|
if download.SizeBytes != request.SourceSizeBytes || !strings.EqualFold(download.SHA256, request.SourceSHA256) {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, rootVolumeName(request.UUID),
|
|
NewError(ErrConflict, "backup_object_mismatch", "downloaded backup object identity does not match", nil),
|
|
"roll back restored system disk",
|
|
))
|
|
}
|
|
if err := pool.Refresh(0); err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, rootVolumeName(request.UUID),
|
|
translateLibvirtError("refresh restored system disk", err), "roll back restored system disk",
|
|
))
|
|
}
|
|
info, err := validateStandaloneQCOW2Volume(volume, 0)
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, rootVolumeName(request.UUID), err, "roll back restored system disk",
|
|
))
|
|
}
|
|
if info.Capacity > capacity {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, rootVolumeName(request.UUID),
|
|
NewError(ErrInvalid, "disk_smaller_than_backup", "disk_gib is smaller than the backup system disk", nil),
|
|
"roll back restored system disk",
|
|
))
|
|
}
|
|
if err := establishNewRootVolumeCapacity(volume, capacity); err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, rootVolumeName(request.UUID), err, "roll back restored system disk",
|
|
))
|
|
}
|
|
return volume, nil
|
|
}
|
|
|
|
func restoreStreamFailureAllowsVolumeRollback(cause error) bool {
|
|
return !operation.IsUncertain(cause)
|
|
}
|
|
|
|
func (l *Libvirt) rollbackRestoreStreamFailure(pool *libvirt.StoragePool,
|
|
volume *libvirt.StorageVol, volumeName string, cause error, action string) error {
|
|
if !restoreStreamFailureAllowsVolumeRollback(cause) {
|
|
// A retained stream may still be owned by libvirt. Releasing this Go
|
|
// handle is safe, but deleting the backing volume would race an active
|
|
// writer. Preserve the artifact and the effect fence for reconciliation.
|
|
_ = volume.Free()
|
|
return cause
|
|
}
|
|
return l.rollbackUnpublishedVolume(pool, volume, volumeName, cause, action)
|
|
}
|