You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_backup.go

757 lines
29 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"errors"
"fmt"
"path/filepath"
"reflect"
"strings"
"time"
"git.luoli.live/root/virt/agent/cloudinit"
"git.luoli.live/root/virt/agent/objecttransfer"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
func (l *Libvirt) CreateBackup(ctx context.Context, uuid string, request BackupRequest) (_ BackupResult, resultErr error) {
if err := ValidateBackupRequest(request); err != nil {
return BackupResult{}, operation.MarkNotApplied(err)
}
return l.createPortableBackup(ctx, uuid, request)
}
func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request BackupRequest) (_ BackupResult, resultErr error) {
client := objecttransfer.New()
defer client.CloseIdleConnections()
active, sourceContract, err := l.validateBackupSource(ctx, uuid, request.BackupKey)
if err != nil {
return BackupResult{}, abortRejectedBackupPlan(
ctx, client.AbortMultipart, request.ObjectUpload, err,
)
}
snapshotRequest := SnapshotRequest{
OperationID: request.OperationID, SnapshotKey: request.BackupKey,
Name: request.Name, ConsistencyMode: request.ConsistencyMode,
}
var snapshot Snapshot
if active {
snapshot, err = l.createBackupWorkspace(ctx, uuid, snapshotRequest)
} else {
snapshot, err = l.createStoppedBackupWorkspace(ctx, uuid, snapshotRequest)
}
if err != nil {
return BackupResult{}, err
}
volume, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), backupWorkspaceVolumeName(uuid, request.BackupKey),
)
if err != nil || volume == nil {
return BackupResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "backup_snapshot_unavailable", "local backup snapshot is unavailable", err,
))
}
defer volume.Free()
info, err := validateStandaloneQCOW2Volume(volume, snapshot.SizeBytes)
if err != nil {
return BackupResult{}, operation.MarkUncertain(err)
}
path, err := volume.GetPath()
if err != nil || !filepath.IsAbs(path) {
return BackupResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "backup_snapshot_unavailable", "local backup snapshot path is unavailable",
translateOptionalLibvirtError("resolve backup snapshot path", err),
))
}
upload, err := client.UploadFile(ctx, path, request.ObjectUpload)
if err != nil {
translated := translateObjectTransferError(err)
if objecttransfer.IsAmbiguous(err) {
// Multipart completion may already have committed. Retain the local
// immutable snapshot and the operation fence for explicit recovery.
return BackupResult{}, translated
}
cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx)
cleanupErr := l.cleanupBackupWorkspace(cleanupCtx, uuid, request.BackupKey)
cleanupCancel()
if cleanupErr != nil {
return BackupResult{}, classifyBackupUploadFailure(translated, cleanupErr)
}
return BackupResult{}, classifyBackupUploadFailure(translated, nil)
}
if upload.SizeBytes <= 0 || upload.SHA256 == "" {
return BackupResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "backup_upload_invalid", "backup upload result is invalid", nil,
))
}
result := BackupResult{
BackupKey: request.BackupKey, SizeBytes: upload.SizeBytes,
VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256,
ObjectETag: upload.ETag, ObjectVersionID: upload.VersionID,
Consistency: snapshot.Consistency,
SourceContract: &sourceContract,
}
cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx)
cleanupErr := l.cleanupBackupWorkspace(cleanupCtx, uuid, request.BackupKey)
cleanupCancel()
if cleanupErr != nil {
// The immutable remote object is already fully verified. Local temporary
// cleanup cannot revoke that success or discard its only SHA/size receipt;
// publish a bounded cleanup marker for follow-up instead of fencing the
// usable backup as an unknown remote effect.
result.CleanupPending = true
return result, nil
}
return result, nil
}
type maintenanceMigrationSourceState struct {
wasRunning bool
autostart bool
}
func (l *Libvirt) inspectMaintenanceMigrationSource(ctx context.Context,
uuid string) (maintenanceMigrationSourceState, error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return maintenanceMigrationSourceState{}, err
}
defer domain.Free()
autostart, err := domain.GetAutostart()
if err != nil {
return maintenanceMigrationSourceState{}, operation.MarkNotApplied(
translateLibvirtError("read maintenance migration autostart", err),
)
}
active, err := domain.IsActive()
if err != nil {
return maintenanceMigrationSourceState{}, operation.MarkNotApplied(
translateLibvirtError("read maintenance migration power state", err),
)
}
return maintenanceMigrationSourceState{wasRunning: active, autostart: autostart}, nil
}
func (l *Libvirt) stopMaintenanceMigrationSource(ctx context.Context, uuid string,
state maintenanceMigrationSourceState) error {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return operation.MarkNotApplied(err)
}
defer domain.Free()
autostart, autostartErr := domain.GetAutostart()
active, activeErr := domain.IsActive()
if autostartErr != nil || activeErr != nil {
return operation.MarkNotApplied(errors.Join(
translateOptionalLibvirtError("recheck maintenance migration autostart", autostartErr),
translateOptionalLibvirtError("recheck maintenance migration power state", activeErr),
))
}
if autostart != state.autostart || active != state.wasRunning {
return operation.MarkNotApplied(NewError(
ErrConflict, "maintenance_source_state_changed",
"maintenance migration source state changed during preflight", nil,
))
}
if err := setServiceAutostart(domain, false); err != nil {
if rollbackErr := l.restoreMaintenanceMigrationDomain(domain, state, func() error {
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
}); rollbackErr != nil {
return operation.MarkUncertain(errors.Join(err, rollbackErr))
}
return err
}
if err := requestGracefulDomainShutoff(ctx, domain, 60*time.Second); err != nil {
if rollbackErr := l.restoreMaintenanceMigrationDomain(domain, state, func() error {
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
}); rollbackErr != nil {
return operation.MarkUncertain(errors.Join(err, rollbackErr))
}
return err
}
observedAutostart, err := domain.GetAutostart()
if err != nil {
confirmationErr := operation.MarkUncertain(translateLibvirtError(
"confirm maintenance migration autostart", err,
))
if rollbackErr := l.restoreMaintenanceMigrationDomain(domain, state, func() error {
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
}); rollbackErr != nil {
return operation.MarkUncertain(errors.Join(confirmationErr, rollbackErr))
}
return confirmationErr
}
observedActive, err := domain.IsActive()
if err != nil || observedAutostart || observedActive {
confirmationErr := operation.MarkUncertain(errors.Join(
translateOptionalLibvirtError("confirm maintenance migration power state", err),
NewError(ErrConflict, "maintenance_source_not_stopped",
"maintenance migration source did not reach the stopped state", nil),
))
if rollbackErr := l.restoreMaintenanceMigrationDomain(domain, state, func() error {
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
}); rollbackErr != nil {
return operation.MarkUncertain(errors.Join(confirmationErr, rollbackErr))
}
return confirmationErr
}
return nil
}
func (l *Libvirt) restoreMaintenanceMigrationSource(ctx context.Context,
uuid string, state maintenanceMigrationSourceState) error {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return operation.MarkUncertain(err)
}
defer domain.Free()
return l.restoreMaintenanceMigrationDomain(domain, state, func() error {
return l.startManagedDomain(ctx, domain, "restart maintenance migration source")
})
}
type maintenanceMigrationDomainControl interface {
serviceStateDomainControl
Create() error
}
func (l *Libvirt) restoreMaintenanceMigrationDomain(
domain maintenanceMigrationDomainControl,
state maintenanceMigrationSourceState,
start func() error,
) error {
if err := domain.SetAutostart(state.autostart); err != nil {
return operation.MarkUncertain(mutationLibvirtError(
"restore maintenance migration autostart", err,
))
}
active, err := domain.IsActive()
if err != nil {
return operation.MarkUncertain(translateLibvirtError(
"inspect maintenance migration rollback power state", err,
))
}
if state.wasRunning && !active {
if start == nil {
return operation.MarkUncertain(NewError(ErrUnavailable,
"public_guard_unavailable", "maintenance migration restart guard is unavailable", nil))
}
if err := start(); err != nil {
return operation.MarkUncertain(mutationLibvirtError(
"restart maintenance migration source", err,
))
}
}
return nil
}
func abortRejectedBackupPlan(ctx context.Context,
abort func(context.Context, objecttransfer.MultipartPlan) error,
plan objecttransfer.MultipartPlan, rejection error) error {
if cleanupErr := abort(ctx, plan); cleanupErr != nil {
// The VM and backup object were never mutated. Retain retry-safe
// semantics and the original preflight cause while keeping the bounded
// multipart cleanup failure in the private Agent error chain.
return operation.MarkNotApplied(errors.Join(rejection, cleanupErr))
}
return operation.MarkNotApplied(rejection)
}
func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey string) (bool, PortableRestoreContract, error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return false, PortableRestoreContract{}, err
}
defer domain.Free()
_, definition, metadata, pool, volume, err := l.managedRootContext(domain)
if err != nil {
return false, PortableRestoreContract{}, err
}
defer pool.Free()
defer volume.Free()
if metadata.RescueTemplateID != "" {
return false, PortableRestoreContract{}, NewError(ErrConflict, "backup_rescue_active", "VM backup is unavailable while rescue media is attached", nil)
}
if err := validateBackupTransitionState(metadata); err != nil {
return false, PortableRestoreContract{}, err
}
for _, stored := range metadata.Snapshots {
if stored.Key == backupKey {
return false, PortableRestoreContract{}, NewError(ErrConflict, "backup_exists", "local backup work already exists", nil)
}
}
if err := validateSingleWritableRoot(definition); err != nil {
return false, PortableRestoreContract{}, err
}
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
return false, PortableRestoreContract{}, err
}
sourceContract, err := l.validatePortableRestoreDomain(definition, metadata)
if err != nil {
return false, PortableRestoreContract{}, err
}
for _, volumeName := range []string{
backupWorkspaceVolumeName(uuid, backupKey),
// Reject a legacy residue with the same key instead of silently
// allocating a second workspace for one logical backup.
snapshotVolumeName(uuid, backupKey),
} {
existing, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), volumeName,
)
if err != nil {
return false, PortableRestoreContract{}, err
}
if existing != nil {
existing.Free()
return false, PortableRestoreContract{}, NewError(ErrConflict, "backup_exists", "local backup work already exists", nil)
}
}
active, err := domain.IsActive()
if err != nil {
return false, PortableRestoreContract{}, translateLibvirtError("check VM state for backup", err)
}
if active {
liveDocument, liveErr := domain.GetXMLDesc(0)
if liveErr != nil {
return false, PortableRestoreContract{}, translateLibvirtError("read live VM disk layout for backup", liveErr)
}
liveDefinition, liveMetadata, parseErr := parseManagedDomain(liveDocument)
if parseErr != nil {
return false, PortableRestoreContract{}, NewError(ErrConflict, "backup_platform_unsupported", "live VM platform is invalid for portable backup", nil)
}
liveContract, contractErr := l.validatePortableRestoreDomain(liveDefinition, liveMetadata)
if contractErr != nil || liveContract != sourceContract {
return false, PortableRestoreContract{}, NewError(ErrConflict, "backup_platform_unsupported", "live VM platform is unavailable for portable backup", nil)
}
managedPath, pathErr := volume.GetPath()
if pathErr != nil {
return false, PortableRestoreContract{}, translateLibvirtError("resolve live managed root for backup", pathErr)
}
if err := validateLiveBackupDefinition(liveDefinition, managedPath); err != nil {
return false, PortableRestoreContract{}, err
}
}
return active, sourceContract, nil
}
func (l *Libvirt) validatePortableRestoreDomain(definition *libvirtxml.Domain,
metadata *domainMetadataXML) (PortableRestoreContract, error) {
reject := func() (PortableRestoreContract, error) {
return PortableRestoreContract{}, NewError(
ErrConflict, "backup_platform_unsupported",
"VM platform is unavailable for portable backup", nil,
)
}
if definition == nil || metadata == nil || definition.Type != "kvm" ||
definition.OS == nil || definition.OS.Type == nil ||
definition.OS.Type.Type != "hvm" || definition.OS.Type.Arch != "x86_64" ||
!portableQ35Machine(definition.OS.Type.Machine) ||
definition.OS.Loader == nil ||
definition.OS.Loader.Path != l.local.FirmwareLoader ||
definition.OS.Loader.Type != "pflash" ||
definition.OS.Loader.Readonly != "yes" ||
definition.OS.Loader.Secure != "no" ||
definition.OS.NVRam == nil ||
definition.Features == nil || definition.Features.SMM == nil ||
definition.Features.SMM.State != "off" {
return reject()
}
if _, err := validateManagedDomainNVRAMPaths(
definition, l.local.NVRAMDirectory, definition.UUID,
); err != nil {
return reject()
}
if err := validateSingleWritableRoot(definition); err != nil {
return reject()
}
root, err := managedRootDisk(definition)
if err != nil || root.Driver == nil || root.Driver.Name != "qemu" ||
root.Driver.Type != "qcow2" || root.Target == nil ||
root.Target.Dev != "vda" || root.Target.Bus != "virtio" ||
root.Source == nil || root.Source.File == nil ||
domainBackingChainPresent(root.BackingStore) || root.Mirror != nil {
return reject()
}
public, err := metadata.PublicNetwork.public()
if err != nil || public.Contract != fixedPortableRestoreContract().PublicNetworkContract ||
!managedNetworkLayoutMatches(definition, metadata) {
return reject()
}
source, err := ParseNetworkSelector(public.NetworkSelector)
if err != nil {
return reject()
}
publicIndex, found := managedPublicInterfaceIndex(
definition, networkFilterName(definition.UUID), public,
)
if !found || !domainNetworkSourceMatches(
definition.Devices.Interfaces[publicIndex], source.Type, source.Source,
) {
return reject()
}
return fixedPortableRestoreContract(), nil
}
func portableQ35Machine(machine string) bool {
machine = strings.ToLower(strings.TrimSpace(machine))
return machine == "q35" || strings.HasPrefix(machine, "pc-q35-")
}
func validateBackupTransitionState(metadata *domainMetadataXML) error {
// Reinstall and snapshot restore deliberately retain their transition kind
// and operation ID after the old root has been deleted. Those markers prove
// a completed root switch during reconciliation; only a pending old root
// means that the disk transition is still active.
if metadata != nil && metadata.PendingCleanupVolume != "" {
return NewError(ErrConflict, "backup_disk_transition_active", "VM backup is unavailable during a disk transition", nil)
}
return nil
}
func validateSingleWritableRoot(definition *libvirtxml.Domain) error {
if definition == nil || definition.Devices == nil {
return NewError(ErrConflict, "backup_disk_layout_unsupported", "VM disk layout is unavailable for portable backup", nil)
}
writable := 0
for index := range definition.Devices.Disks {
disk := &definition.Devices.Disks[index]
if disk.ReadOnly != nil {
continue
}
if disk.Device != "disk" {
return NewError(ErrConflict, "backup_extra_writable_disk", "portable backup supports only the managed vda system disk", nil)
}
writable++
if disk.Target == nil || disk.Target.Dev != "vda" {
return NewError(ErrConflict, "backup_extra_writable_disk", "portable backup supports only the managed vda system disk", nil)
}
}
if writable != 1 {
return NewError(ErrConflict, "backup_disk_layout_unsupported", "portable backup requires exactly one writable system disk", nil)
}
return nil
}
func validateLiveBackupDefinition(definition *libvirtxml.Domain, managedRootPath string) error {
if err := validateSingleWritableRoot(definition); err != nil {
return err
}
root, err := managedRootDisk(definition)
if err != nil {
return err
}
if root.Source == nil || root.Source.File == nil || !filepath.IsAbs(managedRootPath) ||
filepath.Clean(root.Source.File.File) != filepath.Clean(managedRootPath) ||
domainBackingChainPresent(root.BackingStore) || root.Mirror != nil {
return NewError(ErrConflict, "backup_live_root_unsupported", "live VM root disk does not match the standalone managed system disk", nil)
}
return nil
}
func domainBackingChainPresent(store *libvirtxml.DomainDiskBackingStore) bool {
for store != nil {
// libvirt commonly serializes an empty <backingStore/> sentinel for a
// standalone qcow2 disk. libvirtxml materializes that sentinel with a
// non-nil, zero-valued Source, so pointer presence alone is not evidence
// of a backing layer. Any actual source content, format or index remains
// fail-closed because it makes the portable snapshot unsafe.
if store.Index != 0 || store.Format != nil ||
domainDiskSourcePresent(store.Source) {
return true
}
store = store.BackingStore
}
return false
}
func domainDiskSourcePresent(source *libvirtxml.DomainDiskSource) bool {
return source != nil && !reflect.ValueOf(*source).IsZero()
}
func validateStandaloneQCOW2Volume(volume qcow2VolumeInspector, expectedCapacity uint64) (*libvirt.StorageVolInfo, error) {
info, err := validateQCOW2Volume(volume, expectedCapacity)
if err != nil {
return nil, err
}
document, err := volume.GetXMLDesc(0)
if err != nil {
return nil, translateLibvirtError("read standalone qcow2 volume XML", err)
}
var definition libvirtxml.StorageVolume
if err := definition.Unmarshal(document); err != nil {
return nil, NewError(ErrConflict, "invalid_qcow2_volume", "managed qcow2 volume metadata is invalid", err)
}
if definition.BackingStore != nil && definition.BackingStore.Path != "" {
return nil, NewError(ErrConflict, "backup_external_backing_unsupported", "portable backup requires a standalone system disk", nil)
}
return info, nil
}
func (l *Libvirt) createStoppedBackupWorkspace(ctx context.Context, uuid string, request SnapshotRequest) (_ Snapshot, resultErr error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return Snapshot{}, err
}
defer domain.Free()
_, definition, metadata, pool, source, err := l.managedRootContext(domain)
if err != nil {
return Snapshot{}, err
}
defer pool.Free()
defer source.Free()
active, err := domain.IsActive()
if err != nil {
return Snapshot{}, translateLibvirtError("check VM state for offline backup", err)
}
if active {
return Snapshot{}, NewError(ErrConflict, "backup_state_changed", "VM state changed while preparing backup", nil)
}
if err := validateSingleWritableRoot(definition); err != nil {
return Snapshot{}, err
}
if metadata.RescueTemplateID != "" {
return Snapshot{}, NewError(ErrConflict, "backup_rescue_active", "VM backup is unavailable while rescue media is attached", nil)
}
sourceInfo, err := validateStandaloneQCOW2Volume(source, 0)
if err != nil {
return Snapshot{}, err
}
poolInfo, err := pool.GetInfo()
if err != nil {
return Snapshot{}, translateLibvirtError("inspect backup storage capacity", err)
}
if sourceInfo.Capacity > poolInfo.Available {
return Snapshot{}, NewError(ErrConflict, "backup_storage_full", "managed storage pool does not have enough free space", nil)
}
volumeName := backupWorkspaceVolumeName(uuid, request.SnapshotKey)
volumeXML, err := storageVolumeXMLAllocated(volumeName, sourceInfo.Capacity, "qcow2")
if err != nil {
return Snapshot{}, err
}
target, err := pool.StorageVolCreateXMLFrom(volumeXML, source, 0)
if err != nil {
absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName)
return Snapshot{}, deterministicVolumeCreateFailure(
"create offline backup snapshot", err, inspectErr, !absent,
)
}
targetOwned := true
defer func() {
if targetOwned {
if cleanupErr := target.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr,
mutationLibvirtError("roll back offline backup snapshot", cleanupErr)))
}
}
_ = target.Free()
}()
if err := establishNewRootVolumeCapacity(target, sourceInfo.Capacity); err != nil {
return Snapshot{}, err
}
info, err := validateStandaloneQCOW2Volume(target, sourceInfo.Capacity)
if err != nil {
return Snapshot{}, err
}
created := time.Now().UTC()
actualConsistency := request.ConsistencyMode
if actualConsistency == "filesystem" {
// A powered-off disk is stable, but the Agent cannot prove that the
// previous shutdown was guest-coordinated, so it reports crash safety.
actualConsistency = "crash"
}
targetOwned = false
return Snapshot{
ID: request.SnapshotKey, SnapshotKey: request.SnapshotKey,
Name: request.Name, CreatedAt: created, State: "ready",
Consistency: actualConsistency, SizeBytes: info.Capacity, Created: true,
}, nil
}
func (l *Libvirt) RestoreBackup(ctx context.Context, request RestoreBackupRequest) (_ RestoreBackupResult, resultErr error) {
if err := ValidateRestoreBackupRequest(request); err != nil {
return RestoreBackupResult{}, operation.MarkNotApplied(err)
}
if err := l.ensureRestoreTargetAbsent(ctx, request.UUID); err != nil {
return RestoreBackupResult{}, operation.MarkNotApplied(err)
}
create := request.CreateVMRequest
create.Source = "backup"
create.TemplateID = backupTemplateID(request.BackupKey)
attachment, err := l.resolvePublicNetwork(ctx, create)
if err != nil {
return RestoreBackupResult{}, operation.MarkNotApplied(err)
}
mac := vmMACAddress(create.UUID)
conflict, err := l.networkIdentityConflict(ctx, create.UUID, mac, create.PublicNetwork)
if err != nil {
return RestoreBackupResult{}, operation.MarkNotApplied(err)
}
if conflict {
return RestoreBackupResult{}, operation.MarkNotApplied(NewError(ErrConflict, "network_identity_conflict", "VM network identity is already in use", nil))
}
attachment.MAC = mac
seed, err := cloudinit.BuildISO(cloudinit.Request{
UUID: create.UUID, Name: create.Name, Hostname: create.Hostname,
PreserveGuestCredentials: true,
Debian12NetworkFix: *request.Debian12NetworkFix,
Network: cloudinit.Network{
MAC: mac, Public: create.PublicNetwork, InterfaceKey: "public0",
},
})
if err != nil {
return RestoreBackupResult{}, operation.MarkNotApplied(NewError(ErrInvalid, "invalid_cloud_init", "backup restore cloud-init request is invalid", err))
}
root, err := l.createRootVolumeFromBackup(ctx, request)
if err != nil {
return RestoreBackupResult{}, err
}
metadata, err := l.createVMWithPreparedRoot(ctx, create, attachment, seed, root)
if err != nil {
return RestoreBackupResult{}, err
}
return RestoreBackupResult{
VMMetadata: metadata, BackupKey: request.BackupKey,
SHA256: request.SourceSHA256,
}, nil
}
func (l *Libvirt) ensureRestoreTargetAbsent(ctx context.Context, uuid string) error {
domain, err := l.domainByUUID(ctx, uuid)
switch {
case err == nil:
_ = domain.Free()
return NewError(ErrConflict, "restore_target_exists", "backup restore target VM already exists", nil)
case !errors.Is(err, ErrNotFound):
return err
}
absent, err := l.deleteArtifactsAbsent(uuid)
if err != nil {
return err
}
if !absent || l.traffic.contains(uuid) {
return NewError(ErrConflict, "restore_target_artifacts_exist", "managed artifacts already exist for the backup restore target", nil)
}
return nil
}
func (l *Libvirt) createRootVolumeFromBackup(ctx context.Context, request RestoreBackupRequest) (*libvirt.StorageVol, error) {
capacity := request.DiskGiB << 30
initialCapacity := capacity
if objectBytes := uint64(request.SourceSizeBytes); objectBytes > initialCapacity {
// StorageVolUpload bounds the byte stream by the volume capacity. A
// fully allocated qcow2 file can be slightly larger than its virtual
// capacity because of qcow2 metadata, so reserve enough room for the
// exact object before inspecting and resizing its virtual size.
initialCapacity = objectBytes
}
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
if err != nil {
return nil, createVMStorageCloneStage.wrap(translateLibvirtError("lookup managed storage pool", err))
}
defer pool.Free()
definition := libvirtxml.StorageVolume{
Name: rootVolumeName(request.UUID),
Capacity: &libvirtxml.StorageVolumeSize{Value: initialCapacity, Unit: "bytes"},
Target: &libvirtxml.StorageVolumeTarget{
Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"},
},
}
document, err := definition.Marshal()
if err != nil {
return nil, createVMStorageCloneStage.wrap(fmt.Errorf("marshal restored root volume XML: %w", err))
}
volume, err := pool.StorageVolCreateXML(document, 0)
if err != nil {
absent, inspectErr := l.dirPoolVolumeAbsent(pool, rootVolumeName(request.UUID))
return nil, createVMStorageCloneStage.wrap(deterministicArtifactCreateFailure(
"create restored system disk", err, inspectErr, !absent,
))
}
stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK)
if err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, rootVolumeName(request.UUID), err, "roll back restored system disk",
))
}
transfer := &storageStreamTransfer{stream: stream}
if err := volume.Upload(stream, 0, uint64(request.SourceSizeBytes), 0); err != nil {
cause := transfer.abort(err)
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, rootVolumeName(request.UUID), cause, "roll back restored system disk",
))
}
client := objecttransfer.New()
defer client.CloseIdleConnections()
download, downloadErr := client.Download(
ctx, request.SourceURL, request.SourceSizeBytes, request.SourceSHA256,
&storageStreamWriter{ctx: ctx, transfer: transfer},
)
if downloadErr != nil {
cause := transfer.abort(translateObjectTransferError(downloadErr))
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, rootVolumeName(request.UUID), cause, "roll back restored system disk",
))
}
if err := transfer.finish(ctx); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, rootVolumeName(request.UUID), err, "roll back restored system disk",
))
}
if download.SizeBytes != request.SourceSizeBytes || !strings.EqualFold(download.SHA256, request.SourceSHA256) {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, rootVolumeName(request.UUID),
NewError(ErrConflict, "backup_object_mismatch", "downloaded backup object identity does not match", nil),
"roll back restored system disk",
))
}
if err := pool.Refresh(0); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, rootVolumeName(request.UUID),
translateLibvirtError("refresh restored system disk", err), "roll back restored system disk",
))
}
info, err := validateStandaloneQCOW2Volume(volume, 0)
if err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, rootVolumeName(request.UUID), err, "roll back restored system disk",
))
}
if info.Capacity > capacity {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, rootVolumeName(request.UUID),
NewError(ErrInvalid, "disk_smaller_than_backup", "disk_gib is smaller than the backup system disk", nil),
"roll back restored system disk",
))
}
if err := establishNewRootVolumeCapacity(volume, capacity); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, rootVolumeName(request.UUID), err, "roll back restored system disk",
))
}
return volume, nil
}
func restoreStreamFailureAllowsVolumeRollback(cause error) bool {
return !operation.IsUncertain(cause)
}
func (l *Libvirt) rollbackRestoreStreamFailure(pool *libvirt.StoragePool,
volume *libvirt.StorageVol, volumeName string, cause error, action string) error {
if !restoreStreamFailureAllowsVolumeRollback(cause) {
// A retained stream may still be owned by libvirt. Releasing this Go
// handle is safe, but deleting the backing volume would race an active
// writer. Preserve the artifact and the effect fence for reconciliation.
_ = volume.Free()
return cause
}
return l.rollbackUnpublishedVolume(pool, volume, volumeName, cause, action)
}