You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_adapter.go

2447 lines
86 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"git.luoli.live/root/virt/agent/cloudinit"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/publicguard"
"git.luoli.live/root/virt/agent/publicnetwork"
"git.luoli.live/root/virt/agent/vpcrouting"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
type Libvirt struct {
conn *libvirt.Connect
local LocalConfig
traffic *trafficStore
vpcRouter *vpcrouting.Manager
publicGuard *publicguard.Manager
guestAgentResponseTimeoutSupported bool
}
const deleteGracefulShutdownTimeout = 30 * time.Second
type createVMFailureStage struct {
code string
message string
}
var (
createVMTemplateLookupStage = createVMFailureStage{
code: "template_lookup_failed", message: "template metadata or volume could not be located",
}
createVMTemplateCapacityStage = createVMFailureStage{
code: "template_capacity_failed", message: "template capacity could not satisfy the requested disk",
}
createVMStorageCloneStage = createVMFailureStage{
code: "storage_clone_failed", message: "system disk could not be cloned from the template",
}
createVMSeedCreateStage = createVMFailureStage{
code: "seed_create_failed", message: "cloud-init seed volume could not be created",
}
createVMSeedUploadStage = createVMFailureStage{
code: "seed_upload_failed", message: "cloud-init seed data could not be uploaded",
}
createVMNWFilterDefineStage = createVMFailureStage{
code: "nwfilter_define_failed", message: "virtual machine network filter could not be defined",
}
createVMPublicGuardStage = createVMFailureStage{
code: "public_guard_failed", message: "virtual machine public network guard could not be applied",
}
createVMDomainDefineStage = createVMFailureStage{
code: "domain_define_failed", message: "virtual machine definition could not be created",
}
createVMDomainAutostartStage = createVMFailureStage{
code: "domain_autostart_failed", message: "virtual machine autostart could not be enabled",
}
createVMDomainStartStage = createVMFailureStage{
code: "domain_start_failed", message: "virtual machine could not be started",
}
)
type createVMStageError struct {
stage createVMFailureStage
cause error
}
func (e *createVMStageError) Error() string { return e.stage.message }
func (e *createVMStageError) Unwrap() error { return e.cause }
func (e *createVMStageError) PublicCode() string { return e.stage.code }
func (e *createVMStageError) PublicMessage() string { return e.stage.message }
// wrap exposes only a stable stage code and bounded public message. The full
// cause remains in the error chain for local diagnostics and, importantly,
// preserves operation uncertainty and the original hypervisor error kind.
func (stage createVMFailureStage) wrap(cause error) error {
if cause == nil {
return nil
}
return &createVMStageError{stage: stage, cause: cause}
}
func OpenLibvirt(uri string, local LocalConfig) (Hypervisor, error) {
if uri != "qemu:///system" {
return nil, NewError(ErrInvalid, "invalid_libvirt_uri", "only qemu:///system is allowed", nil)
}
if err := startLibvirtEventLoop(); err != nil {
return nil, translateLibvirtError("start libvirt event loop", err)
}
clientVersion, err := libvirt.GetVersion()
if err != nil {
return nil, translateLibvirtError("get local libvirt version", err)
}
traffic, err := openTrafficStore(local.TrafficStatePath)
if err != nil {
return nil, NewError(ErrUnavailable, "traffic_state_unavailable", "durable traffic state could not be opened", err)
}
return openManagedLibvirt(uri, local, clientVersion, traffic), nil
}
func (l *Libvirt) Close() error {
_, err := l.conn.Close()
return err
}
func (l *Libvirt) Ping(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
alive, err := l.conn.IsAlive()
if err != nil {
return translateLibvirtError("ping", err)
}
if !alive {
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt connection is not alive", nil)
}
return nil
}
func (l *Libvirt) Capabilities(ctx context.Context) (Capabilities, error) {
if err := l.Ping(ctx); err != nil {
return Capabilities{}, err
}
libVersion, err := l.conn.GetLibVersion()
if err != nil {
return Capabilities{}, translateLibvirtError("get libvirt version", err)
}
qemuVersion, err := l.conn.GetVersion()
if err != nil {
return Capabilities{}, translateLibvirtError("get QEMU version", err)
}
if err := ctx.Err(); err != nil {
return Capabilities{}, err
}
features, checks, networkSelectors := l.probeRuntimeCapabilities(ctx)
vpcCapability, vpcChecks := probeVPCL3Capability(l.vpcRouter)
for name, check := range vpcChecks {
checks[name] = check
}
vpcFilterCheck := l.probeVPCCleanTrafficFilter()
checks["vpc_clean_traffic_filter"] = vpcFilterCheck
features.VPCL3 = applyVPCCleanTrafficGate(vpcCapability, vpcFilterCheck)
peerCapability, peerCheck := l.peerMigrationCapability(features.Backups)
checks["peer_migration"] = peerCheck
features.PeerMigration = peerCapability != nil
powerActions := []PowerAction(nil)
if features.Power {
powerActions = []PowerAction{PowerStart, PowerShutdown, PowerReboot, PowerReset, PowerDestroy}
}
firmware := []string(nil)
if features.UEFI {
firmware = []string{"uefi"}
}
return Capabilities{
Backend: "libvirt", LibvirtVersion: formatLibvirtVersion(libVersion),
QEMUVersion: formatLibvirtVersion(qemuVersion),
Features: features, PowerActions: powerActions, Firmware: firmware,
NetworkSelectors: networkSelectors, Checks: checks,
PeerMigration: peerCapability,
VPCL3: vpcCapability,
PublicNetwork: publicNetworkCapability(checks),
}, nil
}
func (l *Libvirt) ListVMs(ctx context.Context) ([]VMMetadata, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
domains, err := l.conn.ListAllDomains(0)
if err != nil {
return nil, translateLibvirtError("list domains", err)
}
result := make([]VMMetadata, 0, len(domains))
for index := range domains {
domain := &domains[index]
document, xmlErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
_ = domain.Free()
if xmlErr != nil {
return nil, translateLibvirtError("read domain XML", xmlErr)
}
_, _, managed, inspectErr := inspectManagedDomain(document)
if inspectErr != nil {
return nil, inspectErr
}
if !managed {
continue
}
metadata, parseErr := metadataFromXML(document)
if parseErr != nil {
return nil, parseErr
}
result = append(result, metadata)
}
return result, nil
}
func (l *Libvirt) GetVMState(ctx context.Context, uuid string) (VMState, error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return VMState{}, err
}
defer domain.Free()
state, reason, err := domain.GetState()
if err != nil {
return VMState{}, translateLibvirtError("get domain state", err)
}
name, _ := domain.GetName()
return VMState{UUID: uuid, Name: name, State: mapDomainState(state), Reason: reason}, nil
}
func (l *Libvirt) GetVMMetadata(ctx context.Context, uuid string) (VMMetadata, error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return VMMetadata{}, err
}
defer domain.Free()
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return VMMetadata{}, translateLibvirtError("get domain XML", err)
}
return metadataFromXML(document)
}
func (l *Libvirt) CreateVM(ctx context.Context, request CreateVMRequest) (_ VMMetadata, resultErr error) {
if err := ctx.Err(); err != nil {
return VMMetadata{}, err
}
if err := ValidateCreateVM(request); err != nil {
return VMMetadata{}, err
}
attachment, err := l.resolvePublicNetwork(ctx, request)
if err != nil {
return VMMetadata{}, err
}
mac := vmMACAddress(request.UUID)
conflict, err := l.networkIdentityConflict(ctx, request.UUID, mac, request.PublicNetwork)
if err != nil {
return VMMetadata{}, err
}
if conflict {
return VMMetadata{}, NewError(ErrConflict, "network_identity_conflict", "VM network identity is already in use", nil)
}
attachment.MAC = mac
templateRecord, err := l.loadImageTemplate(ctx, request.TemplateID)
if err != nil {
return VMMetadata{}, createVMTemplateLookupStage.wrap(err)
}
if templateRecord.GenerationToken != request.TemplateGenerationToken {
return VMMetadata{}, NewError(ErrConflict, "template_generation_changed", "template generation token no longer matches", nil)
}
seed, err := cloudinit.BuildISO(cloudinit.Request{
UUID: request.UUID, Name: request.Name, Hostname: request.Hostname,
GuestSSHPublicKey: request.GuestSSHPublicKey, RootPassword: request.RootPassword,
StartupScript: request.StartupScript,
Debian12NetworkFix: isDebian12Template(templateRecord),
Network: cloudinit.Network{
MAC: mac, Public: request.PublicNetwork, InterfaceKey: "public0",
},
})
if err != nil {
return VMMetadata{}, NewError(ErrInvalid, "invalid_cloud_init", "cloud-init request is invalid", err)
}
root, err := l.createRootVolumeFromTemplate(ctx, request)
if err != nil {
return VMMetadata{}, err
}
return l.createVMWithPreparedRoot(ctx, request, attachment, seed, root)
}
// createVMWithPreparedRoot owns root after entry. It is the single create tail
// for template provisioning and backup restore, ensuring both paths generate a
// fresh seed, MAC, public-IP filter, NVRAM, traffic baseline, and domain.
func (l *Libvirt) createVMWithPreparedRoot(ctx context.Context, request CreateVMRequest,
attachment NetworkAttachment, seed []byte, root *libvirt.StorageVol) (_ VMMetadata, resultErr error) {
return l.createVMWithPreparedRootOptions(ctx, request, attachment, seed, root,
preparedVMOptions{Autostart: true, Start: true})
}
type preparedVMOptions struct {
Autostart bool
Start bool
NetworkSuspended bool
VPCPort *VPCPortMetadata
Snapshots []domainSnapshotMetadataXML
SnapshotVolumes []*libvirt.StorageVol
}
type storageVolumeCleanup interface {
Delete(libvirt.StorageVolDeleteFlags) error
Free() error
}
type preparedVolumeOwnership struct {
root storageVolumeCleanup
snapshots []storageVolumeCleanup
rootOwned bool
snapshotsOwned bool
}
func newPreparedVolumeOwnership(root *libvirt.StorageVol, snapshots []*libvirt.StorageVol) *preparedVolumeOwnership {
owned := &preparedVolumeOwnership{root: root, rootOwned: root != nil, snapshotsOwned: len(snapshots) > 0}
owned.snapshots = make([]storageVolumeCleanup, 0, len(snapshots))
for _, snapshot := range snapshots {
if snapshot != nil {
owned.snapshots = append(owned.snapshots, snapshot)
}
}
return owned
}
func (owned *preparedVolumeOwnership) preserve() {
if owned == nil {
return
}
owned.rootOwned = false
owned.snapshotsOwned = false
}
func (owned *preparedVolumeOwnership) cleanup(resultErr *error) {
if owned == nil {
return
}
for _, volume := range owned.snapshots {
if owned.snapshotsOwned {
if err := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil && resultErr != nil {
*resultErr = operation.MarkUncertain(errors.Join(*resultErr,
mutationLibvirtError("roll back restored migration snapshot", err)))
}
}
_ = volume.Free()
}
owned.snapshots = nil
owned.snapshotsOwned = false
if owned.root != nil {
if owned.rootOwned {
if err := owned.root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil && resultErr != nil {
*resultErr = operation.MarkUncertain(errors.Join(*resultErr,
mutationLibvirtError("roll back cloned root volume", err)))
}
}
_ = owned.root.Free()
}
owned.root = nil
owned.rootOwned = false
}
func (l *Libvirt) createVMWithPreparedRootOptions(ctx context.Context,
request CreateVMRequest, attachment NetworkAttachment, seed []byte,
root *libvirt.StorageVol, options preparedVMOptions) (_ VMMetadata, resultErr error) {
volumeOwnership := newPreparedVolumeOwnership(root, options.SnapshotVolumes)
defer volumeOwnership.cleanup(&resultErr)
if root == nil {
return VMMetadata{}, NewError(ErrInvalid, "missing_root_volume", "prepared root volume is required", nil)
}
var domainVPCPort *VPCPortMetadata
var seedVPCMetadata *domainVPCPortMetadataXML
if options.VPCPort != nil {
value := *options.VPCPort
if err := validateVPCPortMetadata(value); err != nil {
return VMMetadata{}, err
}
domainVPCPort = &value
metadata := domainVPCPortFromPublic(value)
seedVPCMetadata = &metadata
}
if _, err := preservedVPCCloudInitNetwork(seed, seedVPCMetadata); err != nil {
return VMMetadata{}, err
}
rootPath, err := root.GetPath()
if err != nil {
return VMMetadata{}, createVMStorageCloneStage.wrap(
mutationLibvirtError("resolve cloned root volume path", err))
}
seedVolume, err := l.createSeedVolume(ctx, request.UUID, seed)
if err != nil {
return VMMetadata{}, err
}
seedPath, err := seedVolume.GetPath()
if err != nil {
cleanupErr := seedVolume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = seedVolume.Free()
if cleanupErr != nil {
return VMMetadata{}, operation.MarkUncertain(createVMSeedCreateStage.wrap(errors.Join(
mutationLibvirtError("resolve NoCloud volume path", err),
mutationLibvirtError("roll back NoCloud seed volume", cleanupErr),
)))
}
return VMMetadata{}, createVMSeedCreateStage.wrap(
mutationLibvirtError("resolve NoCloud volume path", err))
}
seedOwned := true
defer func() {
if seedOwned {
if cleanupErr := seedVolume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr,
mutationLibvirtError("roll back NoCloud seed volume", cleanupErr)))
}
}
_ = seedVolume.Free()
}()
filterName := networkFilterName(request.UUID)
filterDocument, err := BuildPlatformFilterXML(filterName, request.PublicNetwork, request.PlatformNetworkPolicy)
if err != nil {
return VMMetadata{}, createVMNWFilterDefineStage.wrap(err)
}
filter, err := l.conn.NWFilterDefineXML(filterDocument)
if err != nil {
observed, inspectErr := l.lookupOptionalNetworkFilter(filterName)
artifactExists := observed != nil
if observed != nil {
_ = observed.Free()
}
return VMMetadata{}, createVMNWFilterDefineStage.wrap(
deterministicArtifactCreateFailure(
"define VM network filter", err, inspectErr, artifactExists,
))
}
filterOwned := true
defer func() {
if filterOwned {
if cleanupErr := filter.Undefine(); cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr,
mutationLibvirtError("roll back VM network filter", cleanupErr)))
}
}
_ = filter.Free()
}()
attachment.LinkDown = options.NetworkSuspended
networks := []NetworkAttachment{attachment}
if domainVPCPort != nil {
networks = append(networks, NetworkAttachment{
Bridge: domainVPCPort.Bridge,
MAC: domainVPCPort.MAC,
LinkDown: options.NetworkSuspended,
})
}
document, err := BuildDomainXML(DomainDefinition{
UUID: request.UUID, Name: request.DomainName, DisplayName: request.Name, Description: request.Description,
VCPUs: request.VCPUs, MemoryMiB: request.MemoryMiB, DiskGiB: request.DiskGiB,
BandwidthMbps: request.BandwidthMbps, Hostname: request.Hostname,
PublicNetwork: request.PublicNetwork, TemplateID: request.TemplateID,
RootDiskPath: rootPath, RootVolumeName: rootVolumeName(request.UUID), RootDiskFormat: "qcow2", SeedDiskPath: seedPath,
Firmware: Firmware{
Mode: "uefi", Loader: l.local.FirmwareLoader, NVRAMTemplate: l.local.FirmwareNVRAMTemplate,
NVRAMPath: filepath.Join(l.local.NVRAMDirectory, request.UUID+"_VARS.fd"), SecureBoot: false,
},
Network: networks, VPCPort: domainVPCPort, FilterName: filterName,
PlatformNetworkPolicy: request.PlatformNetworkPolicy,
Snapshots: append([]domainSnapshotMetadataXML(nil), options.Snapshots...),
})
if err != nil {
return VMMetadata{}, createVMDomainDefineStage.wrap(err)
}
// Serialize the traffic baseline only after the storage, seed, and filter
// mutations have returned. A slow metrics read can no longer make a request
// wait before its first libvirt mutation after Journal dispatch.
unlockTrafficSample := l.traffic.lockSample(request.UUID)
defer unlockTrafficSample()
trafficCreated, err := l.traffic.initialize(request.UUID, time.Now().UTC())
if err != nil {
return VMMetadata{}, NewError(ErrUnavailable, "traffic_state_unavailable", "new VM traffic baseline could not be initialized", err)
}
trafficOwned := trafficCreated
defer func() {
if trafficOwned {
if cleanupErr := l.traffic.remove(request.UUID); cleanupErr != nil {
resultErr = markTrafficCleanupUncertain(resultErr, cleanupErr)
}
}
}()
if domainVPCPort != nil {
if l.vpcRouter == nil {
return VMMetadata{}, NewError(ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil)
}
_, routeLease, routeErr := l.vpcRouter.AcquireLocalPortLease(
domainVPCPort.VPCKey, domainVPCPort.HostRevision, domainVPCPort.HostDigest,
domainVPCPort.CIDR, domainVPCPort.Gateway, domainVPCPort.PrivateIP, domainVPCPort.MAC,
)
if routeErr != nil {
return VMMetadata{}, NewError(ErrConflict, "vpc_host_projection_not_ready", "VPC host routing must authorize the exact migration port before the domain is defined", routeErr)
}
defer routeLease.Release()
}
guardBinding, err := publicGuardBinding(request.UUID, attachment.MAC, request.PublicNetwork)
if err != nil {
return VMMetadata{}, createVMPublicGuardStage.wrap(err)
}
guardLease, err := l.acquirePublicGuardTransition(ctx)
if err != nil {
return VMMetadata{}, createVMPublicGuardStage.wrap(err)
}
defer guardLease.Release()
if err := l.reconcilePublicGuardLease(ctx, guardLease, &guardBinding); err != nil {
return VMMetadata{}, createVMPublicGuardStage.wrap(err)
}
defer func() {
cleanupCtx, cancelCleanup := storageRollbackContext(ctx)
cleanupErr := l.reconcilePublicGuardLease(cleanupCtx, guardLease, nil)
cancelCleanup()
if cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr,
createVMPublicGuardStage.wrap(cleanupErr)))
}
}()
preserveDefinedDomainArtifacts := func() {
volumeOwnership.preserve()
seedOwned = false
filterOwned = false
trafficOwned = false
}
domain, err := l.conn.DomainDefineXML(document)
if err != nil {
observed, inspectErr := l.conn.LookupDomainByUUIDString(request.UUID)
if observed != nil {
_ = observed.Free()
}
if domainMayExistAfterDefine(inspectErr) {
// DomainDefineXML may have committed before its response failed. Do
// not let deferred cleanup remove disks, firewall, or accounting
// state that the possibly-defined domain owns.
preserveDefinedDomainArtifacts()
return VMMetadata{}, operation.MarkUncertain(createVMDomainDefineStage.wrap(errors.Join(
mutationLibvirtError("define domain", err),
translateOptionalLibvirtError("inspect domain after define failure", inspectErr),
)))
}
return VMMetadata{}, createVMDomainDefineStage.wrap(mutationLibvirtError("define domain", err))
}
defer domain.Free()
if err := domain.SetAutostart(options.Autostart); err != nil {
rollbackErr := domain.UndefineFlags(libvirt.DOMAIN_UNDEFINE_NVRAM)
absent, inspectErr := l.deleteDomainAbsent(ctx, request.UUID)
if inspectErr != nil || !absent {
preserveDefinedDomainArtifacts()
return VMMetadata{}, operation.MarkUncertain(createVMDomainAutostartStage.wrap(errors.Join(
mutationLibvirtError("enable domain autostart", err),
optionalMutationLibvirtError("roll back domain definition", rollbackErr),
inspectErr,
)))
}
if rollbackErr != nil {
return VMMetadata{}, operation.MarkUncertain(createVMDomainAutostartStage.wrap(errors.Join(
mutationLibvirtError("enable domain autostart", err),
mutationLibvirtError("roll back domain definition", rollbackErr),
)))
}
return VMMetadata{}, createVMDomainAutostartStage.wrap(
mutationLibvirtError("enable domain autostart", err))
}
preserveDefinedDomainArtifacts()
if !options.Start {
if err := l.verifyStagedMigrationDomain(
domain, request, filterName, attachment.MAC,
options.NetworkSuspended, options.Snapshots, options.VPCPort,
); err != nil {
return VMMetadata{}, operation.MarkUncertain(err)
}
metadata, err := metadataFromXML(document)
if err != nil {
return VMMetadata{}, operation.MarkUncertain(err)
}
metadata.Status = VMStateShutoff
metadata.DiskGiB = request.DiskGiB
metadata.BandwidthMbps = request.BandwidthMbps
metadata.Hostname = request.Hostname
metadata.PublicNetwork = request.PublicNetwork
metadata.TemplateID = request.TemplateID
return metadata, nil
}
// From this point onward Domain.Create may have taken effect even if its
// response is lost. Retain the zero baseline so reconciliation and the
// first successful sample account for all traffic since initial boot.
if err := l.startManagedDomainWithLease(ctx, guardLease, domain, "start newly defined domain"); err != nil {
return VMMetadata{}, operation.MarkUncertain(createVMDomainStartStage.wrap(
err))
}
if err := l.verifyCreatedDomain(domain, request, filterName, attachment.MAC); err != nil {
return VMMetadata{}, operation.MarkUncertain(err)
}
metadata, err := metadataFromXML(document)
if err != nil {
return VMMetadata{}, operation.MarkUncertain(err)
}
metadata.Status = VMStateRunning
metadata.DiskGiB = request.DiskGiB
metadata.BandwidthMbps = request.BandwidthMbps
metadata.Hostname = request.Hostname
metadata.PublicNetwork = request.PublicNetwork
metadata.TemplateID = request.TemplateID
return metadata, nil
}
func markTrafficCleanupUncertain(prior, cleanupErr error) error {
return operation.MarkUncertain(errors.Join(prior, NewError(
ErrUnavailable, "traffic_state_unavailable",
"unused VM traffic baseline could not be removed", cleanupErr,
)))
}
func isDebian12Template(template ImageTemplate) bool {
return strings.EqualFold(strings.TrimSpace(template.OSFamily), "debian") && strings.TrimSpace(template.Version) == "12"
}
func (l *Libvirt) resolvePublicNetwork(ctx context.Context,
request CreateVMRequest) (NetworkAttachment, error) {
if err := ValidatePublicNetworkIntent(request.PublicNetwork); err != nil {
return NetworkAttachment{}, err
}
source, err := l.ResolveNetworkSelector(ctx, request.PublicNetwork.NetworkSelector)
if err != nil {
return NetworkAttachment{}, err
}
attachment := NetworkAttachment{}
switch source.Type {
case "bridge":
attachment.Bridge = source.Source
case "network":
attachment.Network = source.Source
default:
return NetworkAttachment{}, NewError(ErrUnavailable, "invalid_host_network_config", "host network source type is invalid", nil)
}
return attachment, nil
}
func (l *Libvirt) createRootVolumeFromTemplate(ctx context.Context, request CreateVMRequest) (*libvirt.StorageVol, error) {
templateRecord, err := l.loadImageTemplate(ctx, request.TemplateID)
if err != nil {
return nil, createVMTemplateLookupStage.wrap(err)
}
if templateRecord.GenerationToken != request.TemplateGenerationToken {
return nil, NewError(ErrConflict, "template_generation_changed", "template generation token no longer matches", nil)
}
templatePool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleTemplate))
if err != nil {
return nil, createVMTemplateLookupStage.wrap(
translateLibvirtError("lookup template storage pool", err))
}
defer templatePool.Free()
template, err := templatePool.LookupStorageVolByName(templateVolumeName(request.TemplateID))
if err != nil {
return nil, createVMTemplateLookupStage.wrap(
translateLibvirtError("lookup template volume", err))
}
defer template.Free()
templateInfo, err := template.GetInfo()
if err != nil {
return nil, createVMTemplateCapacityStage.wrap(
translateLibvirtError("read template volume size", err))
}
capacity := request.DiskGiB << 30
if capacity < templateInfo.Capacity {
return nil, createVMTemplateCapacityStage.wrap(NewError(
ErrInvalid, "disk_smaller_than_template", "disk_gib is smaller than the selected template", nil))
}
managedPool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
if err != nil {
return nil, createVMStorageCloneStage.wrap(
translateLibvirtError("lookup managed storage pool", err))
}
defer managedPool.Free()
definition := libvirtxml.StorageVolume{
Name: rootVolumeName(request.UUID),
Capacity: &libvirtxml.StorageVolumeSize{Value: capacity, Unit: "bytes"},
Target: &libvirtxml.StorageVolumeTarget{
Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"},
},
}
document, err := definition.Marshal()
if err != nil {
return nil, createVMStorageCloneStage.wrap(fmt.Errorf("marshal root volume XML: %w", err))
}
volume, err := managedPool.StorageVolCreateXMLFrom(document, template, 0)
if err != nil {
artifactAbsent, inspectErr := l.dirPoolVolumeAbsent(
managedPool, rootVolumeName(request.UUID),
)
return nil, createVMStorageCloneStage.wrap(deterministicArtifactCreateFailure(
"create system disk from template", err, inspectErr, !artifactAbsent,
))
}
// A qcow2 clone in a dir pool is implemented with qemu-img convert. The
// clone inherits the source image's virtual size even when the destination
// XML asks for a larger capacity. Make the requested capacity a mandatory
// part of this unpublished construction step: no domain can reference the
// volume and the caller cannot observe it until resize and verification have
// both succeeded.
if err := establishNewRootVolumeCapacity(volume, capacity); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
managedPool, volume, rootVolumeName(request.UUID), err,
"roll back incomplete system disk",
))
}
return volume, nil
}
type newRootVolumeCapacityTarget interface {
qcow2VolumeInspector
Resize(uint64, libvirt.StorageVolResizeFlags) error
}
func establishNewRootVolumeCapacity(volume newRootVolumeCapacityTarget, capacity uint64) error {
if capacity == 0 {
return NewError(ErrInvalid, "invalid_disk_capacity", "system disk capacity must be positive", nil)
}
// Do not first inspect and conditionally repair a clone. Resize is an
// unconditional part of constructing every new root volume and flags=0
// keeps the additional qcow2 address space sparse.
if err := volume.Resize(capacity, 0); err != nil {
return translateLibvirtError("set new system disk capacity", err)
}
_, err := validateQCOW2Volume(volume, capacity)
return err
}
func (l *Libvirt) rollbackUnpublishedVolume(pool *libvirt.StoragePool,
volume *libvirt.StorageVol, volumeName string, cause error, action string) error {
cleanupErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = volume.Free()
absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName)
return classifyUnpublishedVolumeRollback(cause, cleanupErr, inspectErr, absent, action)
}
func classifyUnpublishedVolumeRollback(cause, cleanupErr, inspectErr error,
absent bool, action string) error {
if inspectErr == nil && absent {
// Independent libvirt inventory and filesystem evidence proves the
// temporary volume is gone, even if Delete itself returned an error.
return errors.Join(cause, translateOptionalLibvirtError(action, cleanupErr))
}
if inspectErr == nil {
inspectErr = NewError(ErrConflict, "storage_cleanup_incomplete",
"incomplete storage volume remains after rollback", nil)
}
return operation.MarkUncertain(errors.Join(
cause,
optionalMutationLibvirtError(action, cleanupErr),
inspectErr,
))
}
func (l *Libvirt) createSeedVolume(ctx context.Context, uuid string, contents []byte) (*libvirt.StorageVol, error) {
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleSeed))
if err != nil {
return nil, createVMSeedCreateStage.wrap(
translateLibvirtError("lookup seed storage pool", err))
}
defer pool.Free()
definition := libvirtxml.StorageVolume{
Name: seedVolumeName(uuid),
Capacity: &libvirtxml.StorageVolumeSize{Value: uint64(len(contents)), Unit: "bytes"},
Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: "raw"}},
}
document, err := definition.Marshal()
if err != nil {
return nil, createVMSeedCreateStage.wrap(fmt.Errorf("marshal seed volume XML: %w", err))
}
volume, err := pool.StorageVolCreateXML(document, 0)
if err != nil {
artifactAbsent, inspectErr := l.dirPoolVolumeAbsent(pool, seedVolumeName(uuid))
return nil, createVMSeedCreateStage.wrap(deterministicArtifactCreateFailure(
"create NoCloud volume", err, inspectErr, !artifactAbsent,
))
}
if err := l.uploadVolume(ctx, volume, contents); err != nil {
uploadErr := createVMSeedUploadStage.wrap(err)
if operation.IsUncertain(err) {
_ = volume.Free()
return nil, operation.MarkUncertain(uploadErr)
}
cleanupErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = volume.Free()
if cleanupErr != nil {
return nil, operation.MarkUncertain(errors.Join(uploadErr,
mutationLibvirtError("roll back NoCloud seed volume", cleanupErr)))
}
return nil, uploadErr
}
return volume, nil
}
func (l *Libvirt) uploadVolume(ctx context.Context, volume *libvirt.StorageVol, contents []byte) error {
if err := ctx.Err(); err != nil {
return err
}
stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK)
if err != nil {
return translateLibvirtError("create storage upload stream", err)
}
transfer := &storageStreamTransfer{stream: stream}
if err := volume.Upload(stream, 0, uint64(len(contents)), 0); err != nil {
return mutationLibvirtError("open storage upload", transfer.abort(err))
}
if _, err := transfer.send(ctx, contents); err != nil {
return mutationLibvirtError("upload storage volume", transfer.abort(err))
}
if err := transfer.finish(ctx); err != nil {
return mutationLibvirtError("finish storage upload", err)
}
return nil
}
func (l *Libvirt) networkIdentityConflict(ctx context.Context, requestedUUID, mac string, public publicnetwork.Intent) (bool, error) {
if err := ValidatePublicNetworkIntent(public); err != nil {
return false, err
}
domains, err := l.conn.ListAllDomains(0)
if err != nil {
return false, translateLibvirtError("list domains for network identity validation", err)
}
for index := range domains {
if err := ctx.Err(); err != nil {
return false, err
}
domain := &domains[index]
document, xmlErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
_ = domain.Free()
if xmlErr != nil {
return false, translateLibvirtError("read domain network identity", xmlErr)
}
definition, metadata, managed, parseErr := inspectManagedDomain(document)
if parseErr != nil {
return false, parseErr
}
if !managed || strings.EqualFold(definition.UUID, requestedUUID) {
continue
}
if publicNetworkAddressConflict(metadata.PublicNetwork.publicMust(), public) {
return true, nil
}
if definition.Devices != nil {
for _, iface := range definition.Devices.Interfaces {
if iface.MAC != nil && strings.EqualFold(iface.MAC.Address, mac) {
return true, nil
}
}
}
}
return false, nil
}
func (l *Libvirt) verifyCreatedDomain(domain *libvirt.Domain, request CreateVMRequest, filterName, mac string) error {
autostart, err := domain.GetAutostart()
if err != nil || !autostart {
return mutationLibvirtError("verify domain autostart", err)
}
active, err := domain.IsActive()
if err != nil || !active {
return mutationLibvirtError("verify newly created domain state", err)
}
document, err := domain.GetXMLDesc(0)
if err != nil {
return translateLibvirtError("verify live domain XML", err)
}
definition, metadata, err := parseManagedDomain(document)
if err != nil {
return err
}
if !strings.EqualFold(definition.UUID, request.UUID) || definition.Name != request.DomainName || metadata.DisplayName != request.Name || !publicNetworkEqual(metadata.PublicNetwork.publicMust(), request.PublicNetwork) || definition.Devices == nil || len(definition.Devices.Interfaces) != 1 {
return NewError(ErrConflict, "create_observation_mismatch", "new domain identity or network metadata did not match the requested intent", nil)
}
var storedPolicy PlatformNetworkPolicySnapshot
if err := json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &storedPolicy); err != nil || storedPolicy.Digest != request.PlatformNetworkPolicy.Digest {
return NewError(ErrConflict, "firewall_observation_mismatch", "new domain platform policy metadata did not match the requested snapshot", err)
}
iface := definition.Devices.Interfaces[0]
if iface.MAC == nil || !strings.EqualFold(iface.MAC.Address, mac) ||
!managedInterfaceFilterIPMatches(&iface, filterName, request.PublicNetwork) {
return NewError(ErrConflict, "firewall_observation_mismatch", "new domain did not attach the expected network filter", nil)
}
return l.verifyManagedFilter(filterName, storedPolicy)
}
func (l *Libvirt) verifyStagedMigrationDomain(domain *libvirt.Domain,
request CreateVMRequest, filterName, mac string, networkSuspended bool,
snapshots []domainSnapshotMetadataXML, vpcPort *VPCPortMetadata) error {
autostart, err := domain.GetAutostart()
if err != nil || autostart {
return mutationLibvirtError("verify staged domain autostart", err)
}
active, err := domain.IsActive()
if err != nil || active {
return mutationLibvirtError("verify staged domain power state", err)
}
document, err := domain.GetXMLDesc(0)
if err != nil {
return translateLibvirtError("verify staged domain XML", err)
}
if err := validateStagedMigrationDomainDocument(
document, request, filterName, mac, networkSuspended, snapshots, vpcPort,
); err != nil {
return err
}
return l.verifyManagedFilter(filterName, request.PlatformNetworkPolicy)
}
func validateStagedMigrationDomainDocument(document string, request CreateVMRequest,
filterName, mac string, networkSuspended bool, snapshots []domainSnapshotMetadataXML,
vpcPort *VPCPortMetadata) error {
definition, metadata, err := parseManagedDomain(document)
if err != nil {
return err
}
expectedInterfaces := 1
if vpcPort != nil {
expectedInterfaces = 2
}
if !strings.EqualFold(definition.UUID, request.UUID) ||
definition.Name != request.DomainName ||
metadata.DisplayName != request.Name ||
!publicNetworkEqual(metadata.PublicNetwork.publicMust(), request.PublicNetwork) ||
definition.Devices == nil || len(definition.Devices.Interfaces) != expectedInterfaces ||
!managedNetworkLayoutMatches(definition, metadata) ||
len(metadata.Snapshots) != len(snapshots) {
return NewError(ErrConflict, "migration_stage_observation_mismatch",
"staged migration domain identity or snapshot catalog did not match", nil)
}
for index := range snapshots {
if metadata.Snapshots[index] != snapshots[index] {
return NewError(ErrConflict, "migration_stage_observation_mismatch",
"staged migration snapshot catalog did not match", nil)
}
}
if vpcPort == nil && metadata.VPCPort != nil || vpcPort != nil &&
(metadata.VPCPort == nil || *metadata.VPCPort != domainVPCPortFromPublic(*vpcPort)) {
return NewError(ErrConflict, "migration_stage_network_mismatch",
"staged migration VPC identity did not match", nil)
}
publicIndex, found := managedPublicInterfaceIndex(definition, filterName, request.PublicNetwork)
if !found {
return NewError(ErrConflict, "migration_stage_network_mismatch",
"staged migration public interface did not match", nil)
}
iface := definition.Devices.Interfaces[publicIndex]
if iface.MAC == nil ||
!strings.EqualFold(iface.MAC.Address, mac) ||
!managedInterfaceFilterIPMatches(&iface, filterName, request.PublicNetwork) {
return NewError(ErrConflict, "migration_stage_network_mismatch",
"staged migration network state did not match", nil)
}
for index := range definition.Devices.Interfaces {
linkDown := definition.Devices.Interfaces[index].Link != nil &&
definition.Devices.Interfaces[index].Link.State == "down"
if linkDown != networkSuspended {
return NewError(ErrConflict, "migration_stage_network_mismatch",
"staged migration interface link state did not match", nil)
}
}
return nil
}
func (l *Libvirt) verifyManagedFilter(filterName string, policy PlatformNetworkPolicySnapshot) error {
filter, err := l.conn.LookupNWFilterByName(filterName)
if err != nil {
return translateLibvirtError("verify VM network filter", err)
}
defer filter.Free()
filterXML, err := filter.GetXMLDesc(0)
if err != nil {
return translateLibvirtError("verify VM network policy", err)
}
if !managedFilterSemanticsValid(filterXML, filterName, policy) {
return NewError(ErrConflict, "firewall_observation_mismatch", "new VM filter does not match the project-owned anti-spoof and platform policy contract", nil)
}
return nil
}
func (l *Libvirt) DeleteVM(ctx context.Context, uuid string, request DeleteVMRequest) (_ DeleteVMResult, resultErr error) {
mutationStarted := false
defer func() {
if !mutationStarted {
resultErr = operation.MarkNotApplied(resultErr)
}
}()
if !request.RemoveManagedDisks {
return DeleteVMResult{}, NewError(ErrInvalid, "managed_disk_cleanup_required", "remove_managed_disks must be true", nil)
}
domain, err := l.domainByUUID(ctx, uuid)
if result, absent, cleanupErr := completeDeleteForMissingDomain(err, func() (DeleteVMResult, error) {
mutationStarted = true
return l.completeDeletedVM(ctx, uuid, false)
}); absent {
return result, cleanupErr
}
if err != nil {
return DeleteVMResult{}, err
}
defer domain.Free()
_, managed, err := requireManagedDomain(domain)
if err != nil {
return DeleteVMResult{}, err
}
if err := l.validateDeleteVMArtifacts(ctx, uuid, managed); err != nil {
return DeleteVMResult{}, err
}
if _, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain); err != nil {
return DeleteVMResult{}, err
}
if err := ensureNoCleanupBlockingJobs(domain); err != nil {
return DeleteVMResult{}, err
}
completionCtx := context.WithoutCancel(ctx)
mutationStarted = true
if err := driveGracefulDeleteDomainToAbsent(completionCtx, domain, deleteGracefulShutdownTimeout, func(completionCtx context.Context) error {
if _, err := l.inspectDeleteDomainEvidence(completionCtx, uuid, domain); err != nil {
return err
}
return ensureNoCleanupBlockingJobs(domain)
}, func(completionCtx context.Context) (bool, error) {
return l.deleteDomainAbsent(completionCtx, uuid)
}); err != nil {
return DeleteVMResult{}, err
}
return l.completeDeletedVM(completionCtx, uuid, false)
}
func (l *Libvirt) ForceDeleteVM(ctx context.Context, uuid string, request ForceDeleteVMRequest) (_ DeleteVMResult, resultErr error) {
mutationStarted := false
defer func() {
if !mutationStarted {
resultErr = operation.MarkNotApplied(resultErr)
}
}()
if !request.RemoveManagedDisks {
return DeleteVMResult{}, NewError(ErrInvalid, "managed_disk_cleanup_required", "remove_managed_disks must be true", nil)
}
if !request.AckDataLoss {
return DeleteVMResult{}, NewError(ErrInvalid, "data_loss_ack_required", "ack_data_loss must be true", nil)
}
if request.PriorOperationID == "" {
return DeleteVMResult{}, NewError(ErrInvalid, "invalid_prior_operation_id", "prior_operation_id is required", nil)
}
domain, err := l.domainByUUID(ctx, uuid)
if result, absent, cleanupErr := completeDeleteForMissingDomain(err, func() (DeleteVMResult, error) {
mutationStarted = true
return l.completeDeletedVM(ctx, uuid, true)
}); absent {
return result, cleanupErr
}
if err != nil {
return DeleteVMResult{}, err
}
defer domain.Free()
_, managed, err := requireManagedDomain(domain)
if err != nil {
return DeleteVMResult{}, err
}
if err := l.validateDeleteVMArtifacts(ctx, uuid, managed); err != nil {
return DeleteVMResult{}, err
}
if _, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain); err != nil {
return DeleteVMResult{}, err
}
if err := ensureNoCleanupBlockingJobs(domain); err != nil {
return DeleteVMResult{}, err
}
mutationStarted = true
if err := driveForceDeleteDomainToAbsent(domain, func() error {
if _, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain); err != nil {
return err
}
return ensureNoCleanupBlockingJobs(domain)
}, func() (bool, error) {
return l.deleteDomainAbsent(ctx, uuid)
}); err != nil {
return DeleteVMResult{}, err
}
return l.completeDeletedVM(ctx, uuid, true)
}
// completeDeleteForMissingDomain makes an already absent definition enter the
// same UUID-owned cleanup and final absence check as an in-request Undefine.
// This covers external removal and late graceful completion without inventing
// another domain mutation. The callback keeps the lookup decision directly
// testable without a live libvirt daemon.
func completeDeleteForMissingDomain(lookupErr error,
complete func() (DeleteVMResult, error)) (DeleteVMResult, bool, error) {
if !errors.Is(lookupErr, ErrNotFound) {
return DeleteVMResult{}, false, nil
}
result, err := complete()
return result, true, err
}
// validateDeleteVMArtifacts proves that every disk selected by a VM deletion
// is Agent-owned and is not referenced by another live or persistent domain.
// It deliberately runs before the domain is stopped so a corrupt ownership
// marker cannot turn an otherwise rejected request into a partial deletion.
func (l *Libvirt) validateDeleteVMArtifacts(ctx context.Context, uuid string,
managed *domainMetadataXML) error {
managedPool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
if err != nil {
return translateLibvirtError("lookup managed storage pool", err)
}
defer managedPool.Free()
inventory, err := managedPool.ListStorageVolumes()
if err != nil {
return translateLibvirtError("list managed volumes for VM deletion", err)
}
volumeNames, err := collectVMManagedVolumeNames(uuid, managed, inventory)
if err != nil {
return err
}
for _, name := range volumeNames {
if err := ctx.Err(); err != nil {
return err
}
volume, lookupErr := managedPool.LookupStorageVolByName(name)
if isNoStorageVolume(lookupErr) {
continue
}
if lookupErr != nil {
return translateLibvirtError("lookup managed VM volume", lookupErr)
}
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
_ = volume.Free()
return validateErr
}
path, pathErr := volume.GetPath()
if pathErr != nil {
_ = volume.Free()
return translateLibvirtError("resolve managed VM volume", pathErr)
}
referenced, referenceErr := l.otherDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), name, uuid,
)
if referenceErr != nil {
_ = volume.Free()
return referenceErr
}
if referenced {
_ = volume.Free()
return NewError(ErrConflict, "managed_disk_shared", "managed VM volume is referenced by another domain", nil)
}
_ = volume.Free()
}
seedVolume, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
if err != nil {
return err
}
if seedVolume != nil {
seedPath, pathErr := seedVolume.GetPath()
if pathErr != nil {
_ = seedVolume.Free()
return translateLibvirtError("resolve managed NoCloud seed volume", pathErr)
}
referenced, referenceErr := l.otherDomainUsesVolume(
seedPath, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid,
)
if referenceErr != nil {
_ = seedVolume.Free()
return referenceErr
}
if referenced {
_ = seedVolume.Free()
return NewError(ErrConflict, "managed_disk_shared", "managed NoCloud seed volume is referenced by another domain", nil)
}
_ = seedVolume.Free()
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if err != nil {
return err
}
if filter != nil {
document, xmlErr := filter.GetXMLDesc(0)
_ = filter.Free()
if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) {
return NewError(ErrConflict, "managed_filter_invalid",
"network filter cannot be confirmed as panel managed", xmlErr)
}
referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), uuid)
if referenceErr != nil {
return referenceErr
}
if referenced {
return NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil)
}
}
return nil
}
type gracefulShutdownDomainControl interface {
IsActive() (bool, error)
ShutdownFlags(libvirt.DomainShutdownFlags) error
}
type deleteDomainControl interface {
gracefulShutdownDomainControl
DestroyFlags(libvirt.DomainDestroyFlags) error
UndefineFlags(libvirt.DomainUndefineFlagsValues) error
}
func driveGracefulDeleteDomainToAbsent(ctx context.Context, domain deleteDomainControl,
timeout time.Duration, beforeUndefine func(context.Context) error,
domainAbsent func(context.Context) (bool, error)) error {
ctx = context.WithoutCancel(ctx)
if err := requestGracefulDomainShutoff(ctx, domain, timeout); err != nil {
return err
}
if beforeUndefine != nil {
if err := beforeUndefine(ctx); err != nil {
return err
}
}
err := driveInactiveDeleteDomainToAbsent(domain, func() (bool, error) {
return domainAbsent(ctx)
})
if !isForceRequired(err) {
return err
}
// The guest was initially inactive (or stopped gracefully) but became active
// before Undefine. That race is not evidence that a graceful wait expired.
// Issue a fresh ACPI request and perform its full bounded wait before a
// force_required result can ever authorize a separate force operation.
if err := requestGracefulDomainShutoff(ctx, domain, timeout); err != nil {
return err
}
if beforeUndefine != nil {
if err := beforeUndefine(ctx); err != nil {
return err
}
}
err = driveInactiveDeleteDomainToAbsent(domain, func() (bool, error) {
return domainAbsent(ctx)
})
if isForceRequired(err) {
return operation.MarkUncertain(NewError(ErrConflict,
"graceful_delete_domain_reactivated",
"domain became active again after a confirmed graceful shutdown", err))
}
return err
}
func driveForceDeleteDomainToAbsent(domain deleteDomainControl,
beforeUndefine func() error, domainAbsent func() (bool, error)) error {
active, err := domain.IsActive()
if err != nil {
return translateLibvirtError("check domain activity", err)
}
destroyDispatched := false
if active {
destroyDispatched = true
destroyErr := domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT)
stillActive, inspectErr := domain.IsActive()
if inspectErr != nil || stillActive {
var mutationErr, observedErr error
if destroyErr != nil {
mutationErr = mutationLibvirtError("force domain off before delete", destroyErr)
} else {
mutationErr = NewError(ErrConflict, "force_delete_domain_still_active",
"domain remained active after force delete", nil)
}
if inspectErr != nil {
observedErr = translateLibvirtError("recheck domain activity after force delete", inspectErr)
}
return operation.MarkUncertain(errors.Join(mutationErr, observedErr))
}
}
if beforeUndefine != nil {
if err := beforeUndefine(); err != nil {
if destroyDispatched {
return operation.MarkUncertain(err)
}
return err
}
}
err = driveInactiveDeleteDomainToAbsent(domain, domainAbsent)
if isForceRequired(err) {
// The explicit force authorization remains attached to this operation.
// If an external actor restarted the guest between Destroy and Undefine,
// keep the operation uncertain so its own reconciliation may safely
// continue; never terminate it as a new graceful force_required state.
return operation.MarkUncertain(NewError(ErrConflict, "force_delete_domain_reactivated",
"domain became active again during the explicit force delete", err))
}
if destroyDispatched && err != nil && !operation.IsUncertain(err) {
return operation.MarkUncertain(err)
}
return err
}
// continueForceDeleteReconciliation never calls DestroyFlags. An uncertain
// force operation may have already crossed that destructive boundary before
// its response was lost; an explicitly authorized cleanup pass may only
// observe an active guest or continue from a confirmed inactive state.
func continueForceDeleteReconciliation(domain deleteDomainControl,
beforeUndefine func() error, domainAbsent func() (bool, error)) (bool, error) {
active, err := domain.IsActive()
if err != nil {
return false, translateLibvirtError("inspect force-delete domain activity", err)
}
if active {
return false, nil
}
if beforeUndefine != nil {
if err := beforeUndefine(); err != nil {
return false, err
}
}
err = driveInactiveDeleteDomainToAbsent(domain, domainAbsent)
if isForceRequired(err) {
return false, operation.MarkUncertain(NewError(ErrConflict, "force_delete_domain_reactivated",
"domain became active again while reconciling the explicit force delete", err))
}
return err == nil, err
}
func isForceRequired(err error) bool {
var backend *Error
return errors.As(err, &backend) && backend.Code == "force_required"
}
func driveInactiveDeleteDomainToAbsent(domain deleteDomainControl,
domainAbsent func() (bool, error)) error {
active, err := domain.IsActive()
if err != nil {
return translateLibvirtError("check domain activity before undefine", err)
}
if active {
return NewError(ErrConflict, "force_required",
"guest did not stop after the graceful shutdown request; explicit force delete is required", nil)
}
flags := libvirt.DOMAIN_UNDEFINE_NVRAM |
libvirt.DOMAIN_UNDEFINE_MANAGED_SAVE |
libvirt.DOMAIN_UNDEFINE_SNAPSHOTS_METADATA |
libvirt.DOMAIN_UNDEFINE_CHECKPOINTS_METADATA
undefineErr := domain.UndefineFlags(flags)
absent, inspectErr := domainAbsent()
if undefineErr != nil || inspectErr != nil || !absent {
var mutationErr error
if undefineErr != nil {
mutationErr = mutationLibvirtError("undefine domain", undefineErr)
} else {
mutationErr = NewError(ErrConflict, "delete_domain_still_defined",
"domain remained present after undefine", nil)
}
return operation.MarkUncertain(errors.Join(mutationErr, inspectErr))
}
return nil
}
// requestGracefulDomainShutoff is shared by VM deletion, service expiry, and
// paid resource upgrades. It has no force method in its interface, so callers
// cannot silently escalate a graceful lifecycle request to DestroyFlags.
func requestGracefulDomainShutoff(ctx context.Context, domain gracefulShutdownDomainControl,
timeout time.Duration) error {
active, err := domain.IsActive()
if err != nil {
return translateLibvirtError("check domain activity before graceful shutdown", err)
}
if !active {
return nil
}
shutdownErr := domain.ShutdownFlags(libvirt.DOMAIN_SHUTDOWN_ACPI_POWER_BTN)
if shutdownErr != nil {
stillActive, inspectErr := domain.IsActive()
if inspectErr != nil {
return ambiguousGracefulShutdown(errors.Join(
mutationLibvirtError("request graceful domain shutdown", shutdownErr),
translateLibvirtError("confirm graceful domain shutdown request", inspectErr),
))
}
if !stillActive {
return nil
}
// A failed RPC response does not prove whether libvirt delivered the
// request. Observe the guest for the full graceful window, but never
// turn an unconfirmed request into force authorization. If the guest
// stays active, this operation fails and a later request must start a
// fresh graceful attempt with a new operation ID.
waitErr := waitForGracefulDomainShutoff(context.WithoutCancel(ctx), domain, timeout)
if waitErr == nil {
return nil
}
if isForceRequired(waitErr) {
return NewError(ErrConflict, "graceful_shutdown_request_failed",
"graceful shutdown request was not confirmed; retry graceful shutdown before force delete",
shutdownErr)
}
return ambiguousGracefulShutdown(errors.Join(
mutationLibvirtError("request graceful domain shutdown", shutdownErr), waitErr,
))
}
// Once ShutdownFlags returned success, finish the bounded observation even
// if the HTTP/request context disappears. Journal.Execute can return an
// uncertain response to the caller while this late result is persisted;
// abandoning the wait here would otherwise strand a safely recoverable
// graceful request without a legal force boundary.
return waitForGracefulDomainShutoff(context.WithoutCancel(ctx), domain, timeout)
}
func ambiguousGracefulShutdown(cause error) error {
return operation.MarkUncertain(NewError(
ErrUnavailable, "graceful_shutdown_ambiguous",
"graceful shutdown result could not be confirmed", cause,
))
}
func waitForGracefulDomainShutoff(ctx context.Context, domain interface{ IsActive() (bool, error) }, timeout time.Duration) error {
if timeout <= 0 {
timeout = deleteGracefulShutdownTimeout
}
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
timer := time.NewTimer(timeout)
defer timer.Stop()
for {
active, err := domain.IsActive()
if err != nil {
return operation.MarkUncertain(translateLibvirtError("wait for graceful domain shutdown", err))
}
if !active {
return nil
}
select {
case <-ctx.Done():
return operation.MarkUncertain(ctx.Err())
case <-timer.C:
active, inspectErr := domain.IsActive()
if inspectErr != nil {
return operation.MarkUncertain(translateLibvirtError("confirm graceful domain shutdown timeout", inspectErr))
}
if !active {
return nil
}
return NewError(ErrConflict, "force_required",
"guest did not stop after the graceful shutdown request; explicit force delete is required", nil)
case <-ticker.C:
}
}
}
// completeDeletedVM is shared by the initial request and reconciliation. A
// successful operation therefore always means a fresh UUID lookup, every
// panel-managed volume, the seed, network filter, NVRAM file, and local
// traffic state were all observed absent.
func (l *Libvirt) completeDeletedVM(ctx context.Context, uuid string, forced bool) (DeleteVMResult, error) {
if err := l.cleanupDeletedVMArtifacts(ctx, uuid); err != nil {
return DeleteVMResult{}, operation.MarkUncertain(err)
}
domainAbsent, err := l.deleteDomainAbsent(ctx, uuid)
if err != nil {
return DeleteVMResult{}, operation.MarkUncertain(err)
}
if !domainAbsent {
return DeleteVMResult{}, operation.MarkUncertain(NewError(
ErrConflict, "delete_cleanup_incomplete", "deleted VM domain could not be confirmed absent", nil,
))
}
if err := l.reconcilePublicGuard(ctx, nil); err != nil {
return DeleteVMResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "public_guard_cleanup_unconfirmed", "deleted VM public network guard could not be reconciled", err,
))
}
artifactsAbsent, err := l.deleteArtifactsAbsent(uuid)
if err != nil {
return DeleteVMResult{}, operation.MarkUncertain(err)
}
return confirmDeletedVMAbsence(domainAbsent, artifactsAbsent, l.traffic.contains(uuid), forced)
}
func confirmDeletedVMAbsence(domainAbsent, artifactsAbsent, trafficPresent, forced bool) (DeleteVMResult, error) {
if !domainAbsent || !artifactsAbsent || trafficPresent {
return DeleteVMResult{}, operation.MarkUncertain(NewError(
ErrConflict, "delete_cleanup_incomplete",
"deleted VM resources could not all be confirmed absent", nil,
))
}
return DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true, Forced: forced}, nil
}
// inspectDeleteDomainEvidence verifies all domain-bound state that
// UndefineFlags is about to remove. The evidence is private and is used only
// to bind an administrator cleanup plan; no XML or host path crosses the API.
func (l *Libvirt) inspectDeleteDomainEvidence(ctx context.Context, uuid string,
domain *libvirt.Domain) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return nil, translateLibvirtError("read persistent domain delete evidence", err)
}
var definition libvirtxml.Domain
if err := definition.Unmarshal(document); err != nil {
return nil, NewError(ErrConflict, "managed_domain_xml_invalid", "managed domain XML is invalid", err)
}
evidence := make([]string, 0, 12)
documentDigest := sha256.Sum256([]byte(document))
evidence = append(evidence, "delete-domain-xml:"+hex.EncodeToString(documentDigest[:]))
nvramPaths, err := validateManagedDomainNVRAMPaths(&definition, l.local.NVRAMDirectory, uuid)
if err != nil {
return nil, err
}
for _, path := range nvramPaths {
referenced, referenceErr := l.otherDomainUsesPath(path, uuid)
if referenceErr != nil {
return nil, referenceErr
}
if referenced {
return nil, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil)
}
info, statErr := os.Lstat(path)
switch {
case errors.Is(statErr, os.ErrNotExist):
evidence = append(evidence, "domain-nvram:"+path+":absent")
case statErr != nil:
return nil, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr)
case !info.Mode().IsRegular():
return nil, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil)
default:
evidence = append(evidence, "domain-nvram:"+path+":"+
strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10))
}
}
managedSave, err := domain.HasManagedSaveImage(0)
if err != nil {
return nil, translateLibvirtError("inspect domain managed save image", err)
}
evidence = append(evidence, "managed-save:"+strconv.FormatBool(managedSave))
snapshots, err := domain.ListAllSnapshots(0)
if err != nil {
return nil, translateLibvirtError("list domain snapshot metadata", err)
}
for index := range snapshots {
if err := ctx.Err(); err != nil {
for remaining := index; remaining < len(snapshots); remaining++ {
_ = snapshots[remaining].Free()
}
return nil, err
}
name, nameErr := snapshots[index].GetName()
snapshotXML, xmlErr := snapshots[index].GetXMLDesc(0)
_ = snapshots[index].Free()
if nameErr != nil || xmlErr != nil {
for remaining := index + 1; remaining < len(snapshots); remaining++ {
_ = snapshots[remaining].Free()
}
return nil, translateLibvirtError("read domain snapshot metadata", errors.Join(nameErr, xmlErr))
}
if safetyErr := validateNativeSnapshotDeleteSafety(snapshotXML); safetyErr != nil {
for remaining := index + 1; remaining < len(snapshots); remaining++ {
_ = snapshots[remaining].Free()
}
return nil, operation.MarkUncertain(safetyErr)
}
digest := sha256.Sum256([]byte(snapshotXML))
evidence = append(evidence, "domain-snapshot:"+name+":"+hex.EncodeToString(digest[:]))
}
evidence = append(evidence, "domain-snapshot-count:"+strconv.Itoa(len(snapshots)))
checkpoints, err := domain.ListAllCheckpoints(0)
if err != nil {
return nil, translateLibvirtError("list domain checkpoint metadata", err)
}
for index := range checkpoints {
if err := ctx.Err(); err != nil {
for remaining := index; remaining < len(checkpoints); remaining++ {
_ = checkpoints[remaining].Free()
}
return nil, err
}
name, nameErr := checkpoints[index].GetName()
checkpointXML, xmlErr := checkpoints[index].GetXMLDesc(0)
_ = checkpoints[index].Free()
if nameErr != nil || xmlErr != nil {
for remaining := index + 1; remaining < len(checkpoints); remaining++ {
_ = checkpoints[remaining].Free()
}
return nil, translateLibvirtError("read domain checkpoint metadata", errors.Join(nameErr, xmlErr))
}
var checkpoint libvirtxml.DomainCheckpoint
if err := checkpoint.Unmarshal(checkpointXML); err != nil {
for remaining := index + 1; remaining < len(checkpoints); remaining++ {
_ = checkpoints[remaining].Free()
}
return nil, NewError(ErrConflict, "domain_checkpoint_xml_invalid",
"native checkpoint XML could not be inspected before delete", err)
}
digest := sha256.Sum256([]byte(checkpointXML))
evidence = append(evidence, "domain-checkpoint:"+name+":"+hex.EncodeToString(digest[:]))
}
evidence = append(evidence, "domain-checkpoint-count:"+strconv.Itoa(len(checkpoints)))
filterName := networkFilterName(uuid)
referenced, err := l.otherDomainUsesFilter(filterName, uuid)
if err != nil {
return nil, err
}
if referenced {
return nil, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil)
}
evidence = append(evidence, "network-filter-exclusive:"+filterName)
return evidence, nil
}
func validateManagedDomainNVRAMPaths(definition *libvirtxml.Domain, directory, uuid string) ([]string, error) {
paths := make(map[string]struct{}, 2)
if definition != nil && definition.OS != nil {
if nvram := definition.OS.NVRam; nvram != nil {
if nvram.NVRam != "" {
paths[filepath.Clean(nvram.NVRam)] = struct{}{}
}
if nvram.Source != nil && nvram.Source.File != nil && nvram.Source.File.File != "" {
paths[filepath.Clean(nvram.Source.File.File)] = struct{}{}
}
}
if definition.OS.VarStore != nil && definition.OS.VarStore.Path != "" {
paths[filepath.Clean(definition.OS.VarStore.Path)] = struct{}{}
}
}
if len(paths) == 0 {
return nil, NewError(ErrConflict, "managed_nvram_invalid", "managed domain NVRAM path is missing", nil)
}
expected := make(map[string]struct{}, len(managedNVRAMArtifactNames(uuid)))
for _, name := range managedNVRAMArtifactNames(uuid) {
expected[filepath.Clean(filepath.Join(directory, name))] = struct{}{}
}
result := make([]string, 0, len(paths))
for path := range paths {
if !filepath.IsAbs(path) {
return nil, NewError(ErrConflict, "managed_nvram_invalid", "managed domain NVRAM path is invalid", nil)
}
if _, ok := expected[path]; !ok {
return nil, NewError(ErrConflict, "managed_nvram_path_mismatch", "managed domain NVRAM path is outside the UUID-owned location", nil)
}
result = append(result, path)
}
sort.Strings(result)
return result, nil
}
func (l *Libvirt) otherDomainUsesFilter(filterName, excludedUUID string) (bool, error) {
filterReferenced, err := l.otherFilterReferencesFilter(filterName)
if err != nil || filterReferenced {
return filterReferenced, err
}
domains, err := l.conn.ListAllDomains(0)
if err != nil {
return false, translateLibvirtError("list domains for network filter reference check", err)
}
for index := range domains {
domain := &domains[index]
candidateUUID, uuidErr := domain.GetUUIDString()
if uuidErr != nil {
_ = domain.Free()
return false, translateLibvirtError("read domain UUID for network filter reference", uuidErr)
}
if excludedUUID != "" && strings.EqualFold(candidateUUID, excludedUUID) {
_ = domain.Free()
continue
}
flags := []libvirt.DomainXMLFlags{0}
persistent, persistentErr := domain.IsPersistent()
if persistentErr != nil {
_ = domain.Free()
return false, translateLibvirtError("inspect domain persistence for network filter reference", persistentErr)
}
if persistent {
flags = append(flags, libvirt.DOMAIN_XML_INACTIVE)
}
for _, flag := range flags {
document, xmlErr := domain.GetXMLDesc(flag)
if xmlErr != nil {
_ = domain.Free()
return false, translateLibvirtError("read domain XML for network filter reference", xmlErr)
}
var definition libvirtxml.Domain
if err := definition.Unmarshal(document); err != nil {
_ = domain.Free()
return false, NewError(ErrConflict, "domain_xml_invalid", "domain XML could not be inspected for network filter references", err)
}
if domainReferencesFilter(&definition, filterName) {
_ = domain.Free()
return true, nil
}
}
referenced, historyErr := domainHistoryReferencesFilter(domain, filterName)
if historyErr != nil {
_ = domain.Free()
return false, historyErr
}
if referenced {
_ = domain.Free()
return true, nil
}
_ = domain.Free()
}
return false, nil
}
func domainHistoryReferencesFilter(domain *libvirt.Domain, filterName string) (bool, error) {
managedSave, present, err := managedSaveDomainDefinition(domain)
if err != nil {
return false, err
}
if present && domainReferencesFilter(managedSave, filterName) {
return true, nil
}
snapshots, err := domain.ListAllSnapshots(0)
if err != nil {
return false, translateLibvirtError("list domain snapshots for network filter reference check", err)
}
defer func() {
for index := range snapshots {
_ = snapshots[index].Free()
}
}()
for index := range snapshots {
document, xmlErr := snapshots[index].GetXMLDesc(0)
if xmlErr != nil {
return false, translateLibvirtError("read domain snapshot XML for network filter reference", xmlErr)
}
var definition libvirtxml.DomainSnapshot
if err := definition.Unmarshal(document); err != nil {
return false, NewError(ErrConflict, "domain_snapshot_xml_invalid",
"domain snapshot XML could not be inspected for network filter references", err)
}
if domainSnapshotReferencesFilter(&definition, filterName) {
return true, nil
}
}
checkpoints, err := domain.ListAllCheckpoints(0)
if err != nil {
return false, translateLibvirtError("list domain checkpoints for network filter reference check", err)
}
defer func() {
for index := range checkpoints {
_ = checkpoints[index].Free()
}
}()
for index := range checkpoints {
document, xmlErr := checkpoints[index].GetXMLDesc(0)
if xmlErr != nil {
return false, translateLibvirtError("read domain checkpoint XML for network filter reference", xmlErr)
}
var definition libvirtxml.DomainCheckpoint
if err := definition.Unmarshal(document); err != nil {
return false, NewError(ErrConflict, "domain_checkpoint_xml_invalid",
"domain checkpoint XML could not be inspected for network filter references", err)
}
if definition.Domain != nil && domainReferencesFilter(definition.Domain, filterName) {
return true, nil
}
}
return false, nil
}
func (l *Libvirt) otherFilterReferencesFilter(filterName string) (bool, error) {
filters, err := l.conn.ListAllNWFilters(0)
if err != nil {
return false, translateLibvirtError("list network filters for dependency check", err)
}
defer func() {
for index := range filters {
_ = filters[index].Free()
}
}()
for index := range filters {
filter := &filters[index]
name, nameErr := filter.GetName()
if nameErr != nil {
return false, translateLibvirtError("read network filter name for dependency check", nameErr)
}
if name == filterName {
continue
}
document, xmlErr := filter.GetXMLDesc(0)
if xmlErr != nil {
return false, translateLibvirtError("read network filter dependency XML", xmlErr)
}
var definition libvirtxml.NWFilter
if err := definition.Unmarshal(document); err != nil {
return false, NewError(ErrConflict, "network_filter_xml_invalid",
"network filter dependencies could not be inspected", err)
}
if networkFilterReferencesFilter(&definition, filterName) {
return true, nil
}
}
return false, nil
}
func networkFilterReferencesFilter(definition *libvirtxml.NWFilter, filterName string) bool {
if definition == nil || filterName == "" {
return false
}
for _, entry := range definition.Entries {
if entry.Ref != nil && entry.Ref.Filter == filterName {
return true
}
}
return false
}
func domainReferencesFilter(definition *libvirtxml.Domain, filterName string) bool {
if definition == nil || definition.Devices == nil {
return false
}
for _, iface := range definition.Devices.Interfaces {
if iface.FilterRef != nil && iface.FilterRef.Filter == filterName {
return true
}
}
return false
}
func domainSnapshotReferencesFilter(snapshot *libvirtxml.DomainSnapshot, filterName string) bool {
if snapshot == nil {
return false
}
if snapshot.Domain != nil && domainReferencesFilter(snapshot.Domain, filterName) {
return true
}
return snapshot.InactiveDomain != nil &&
domainReferencesFilter(&snapshot.InactiveDomain.Domain, filterName)
}
func (l *Libvirt) deleteDomainAbsent(ctx context.Context, uuid string) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
if !uuidPattern.MatchString(uuid) {
return false, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
}
domain, err := l.conn.LookupDomainByUUIDString(uuid)
if err == nil {
_ = domain.Free()
return false, nil
}
if isNoDomain(err) {
return true, nil
}
return false, translateLibvirtError("confirm deleted domain absence", err)
}
func isNoDomain(err error) bool {
var libvirtErr libvirt.Error
return errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_DOMAIN
}
func domainMayExistAfterDefine(inspectErr error) bool {
return inspectErr == nil || !isNoDomain(inspectErr)
}
func collectVMManagedVolumeNames(uuid string, metadata *domainMetadataXML, inventory []string) ([]string, error) {
if !uuidPattern.MatchString(uuid) {
return nil, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
}
names := make(map[string]struct{})
for _, name := range inventory {
if isVMOwnedManagedVolumeName(uuid, name) {
names[name] = struct{}{}
}
}
if metadata != nil {
if !isVMOwnedRootVolumeName(uuid, metadata.RootVolumeName) {
return nil, NewError(ErrConflict, "managed_metadata_invalid", "managed root volume is not owned by the VM UUID", nil)
}
names[metadata.RootVolumeName] = struct{}{}
if metadata.PendingCleanupVolume != "" {
if !isVMOwnedRootVolumeName(uuid, metadata.PendingCleanupVolume) {
return nil, NewError(ErrConflict, "managed_metadata_invalid", "pending cleanup volume is not owned by the VM UUID", nil)
}
names[metadata.PendingCleanupVolume] = struct{}{}
}
for _, snapshot := range metadata.Snapshots {
name := snapshotVolumeName(uuid, snapshot.Key)
if !isVMOwnedManagedVolumeName(uuid, name) {
return nil, NewError(ErrConflict, "managed_metadata_invalid", "snapshot volume is not owned by the VM UUID", nil)
}
names[name] = struct{}{}
}
}
result := make([]string, 0, len(names))
for name := range names {
result = append(result, name)
}
sort.Strings(result)
return result, nil
}
func (l *Libvirt) lookupOptionalManagedVolume(poolName, volumeName string) (*libvirt.StorageVol, error) {
pool, err := l.conn.LookupStoragePoolByName(poolName)
if err != nil {
return nil, translateLibvirtError("lookup managed storage pool", err)
}
defer pool.Free()
volume, err := pool.LookupStorageVolByName(volumeName)
if err == nil {
return volume, nil
}
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_STORAGE_VOL {
return nil, nil
}
return nil, translateLibvirtError("lookup managed storage volume", err)
}
func (l *Libvirt) lookupOptionalNetworkFilter(name string) (*libvirt.NWFilter, error) {
filter, err := l.conn.LookupNWFilterByName(name)
if err == nil {
return filter, nil
}
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_NWFILTER {
return nil, nil
}
return nil, translateLibvirtError("lookup VM network filter", err)
}
func (l *Libvirt) Power(ctx context.Context, uuid string, request PowerRequest) (_ VMState, resultErr error) {
mutationStarted := false
defer func() {
if !mutationStarted {
resultErr = operation.MarkNotApplied(resultErr)
}
}()
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return VMState{}, err
}
defer domain.Free()
if _, _, err := requireManagedDomain(domain); err != nil {
return VMState{}, err
}
if powerActionRequiresPublicGuard(request.Action) {
if err := l.reconcilePublicGuard(ctx, nil); err != nil {
return VMState{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "public_guard_unavailable",
"public network guard could not be confirmed before the power action", err,
))
}
}
mutationStarted = true
if err := dispatchPowerMutation(domain, request.Action, func() error {
return l.startManagedDomain(ctx, domain, "perform power start")
}); err != nil {
return VMState{}, err
}
name, _ := domain.GetName()
if request.Action == PowerShutdown || request.Action == PowerReboot || request.Action == PowerReset {
// libvirt returning success is the durable effect for command-style
// lifecycle requests. The guest may remain running while ACPI shutdown
// or reboot is still in progress, so an immediate state sample is
// informational and can never turn an accepted command into failure.
result := VMState{UUID: uuid, Name: name, State: VMStateUnknown, CommandAccepted: true}
if state, reason, stateErr := domain.GetState(); stateErr == nil {
result.State = mapDomainState(state)
result.Reason = reason
}
return result, nil
}
state, reason, err := domain.GetState()
if err != nil {
return VMState{}, operation.MarkUncertain(translateLibvirtError("confirm power action", err))
}
mapped := mapDomainState(state)
if err := confirmPowerTargetState(request.Action, mapped); err != nil {
return VMState{}, err
}
return VMState{UUID: uuid, Name: name, State: mapped, Reason: reason, CommandAccepted: true, EffectConfirmed: true}, nil
}
func powerActionRequiresPublicGuard(action PowerAction) bool {
return action == PowerStart || action == PowerReboot || action == PowerReset
}
type powerDomainControl interface {
Create() error
ShutdownFlags(libvirt.DomainShutdownFlags) error
Reboot(libvirt.DomainRebootFlagValues) error
Reset(uint32) error
DestroyFlags(libvirt.DomainDestroyFlags) error
}
func dispatchPowerMutation(domain powerDomainControl, action PowerAction, start func() error) error {
var err error
switch action {
case PowerStart:
if start == nil {
return operation.MarkNotApplied(NewError(ErrUnavailable, "public_guard_unavailable", "power start guard is unavailable", nil))
}
err = start()
case PowerShutdown:
err = domain.ShutdownFlags(libvirt.DOMAIN_SHUTDOWN_ACPI_POWER_BTN)
case PowerReboot:
err = domain.Reboot(libvirt.DOMAIN_REBOOT_ACPI_POWER_BTN)
case PowerReset:
err = domain.Reset(0)
case PowerDestroy:
err = domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT)
default:
return operation.MarkNotApplied(NewError(ErrInvalid, "invalid_power_action", "unknown power action", nil))
}
if err != nil {
if operation.IsNotApplied(err) {
return err
}
// A libvirt lifecycle call can take effect before its transport reports
// an error. Keep the exact operation reconcilable instead of recording a
// terminal failure that could invite a second destructive request.
return operation.MarkUncertain(mutationLibvirtError("perform power action", err))
}
return nil
}
func confirmPowerTargetState(action PowerAction, state VMStateName) error {
reached := true
switch action {
case PowerStart:
reached = state == VMStateRunning || state == VMStateBlocked
case PowerDestroy:
reached = state == VMStateShutoff
}
if !reached {
return operation.MarkUncertain(NewError(
ErrConflict, "power_state_unconfirmed",
"power action did not reach the requested target state", nil,
))
}
return nil
}
func (l *Libvirt) GetConsole(ctx context.Context, uuid string) (ConsoleSession, error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return ConsoleSession{}, err
}
defer domain.Free()
active, err := domain.IsActive()
if err != nil {
return ConsoleSession{}, translateLibvirtError("check console domain activity", err)
}
if !active {
return ConsoleSession{}, NewError(ErrNotFound, "console_not_available", "VM has no active local VNC console", nil)
}
document, err := domain.GetXMLDesc(0)
if err != nil {
return ConsoleSession{}, translateLibvirtError("get live console XML", err)
}
metadata := struct {
Devices struct {
Graphics []struct {
Type string `xml:"type,attr"`
Port int `xml:"port,attr"`
} `xml:"graphics"`
} `xml:"devices"`
}{}
if err := xml.Unmarshal([]byte(document), &metadata); err != nil {
return ConsoleSession{}, fmt.Errorf("parse console XML: %w", err)
}
for _, graphic := range metadata.Devices.Graphics {
if graphic.Type == "vnc" && graphic.Port > 0 && graphic.Port <= 65535 {
return ConsoleSession{Protocol: "vnc", Address: net.JoinHostPort("127.0.0.1", strconv.Itoa(graphic.Port))}, nil
}
}
return ConsoleSession{}, NewError(ErrNotFound, "console_not_available", "VM has no active local VNC console", nil)
}
func resolvedDeleteSuccess(result DeleteVMResult) operation.Resolution {
encoded, _ := json.Marshal(result)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
}
// cleanupDeletedVMArtifacts resumes the idempotent tail of a delete after the
// domain definition is already gone. Every target is derived from the VM UUID,
// validated as Agent-owned, and checked for cross-domain references first.
func (l *Libvirt) cleanupDeletedVMArtifacts(ctx context.Context, uuid string) error {
if err := ctx.Err(); err != nil {
return err
}
unlockTrafficSample := l.traffic.lockSample(uuid)
defer unlockTrafficSample()
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
if err != nil {
return translateLibvirtError("lookup managed storage pool", err)
}
defer pool.Free()
inventory, err := pool.ListStorageVolumes()
if err != nil {
return translateLibvirtError("list managed volumes for VM cleanup", err)
}
names, err := collectVMManagedVolumeNames(uuid, nil, inventory)
if err != nil {
return err
}
for _, name := range names {
if err := ctx.Err(); err != nil {
return err
}
volume, lookupErr := pool.LookupStorageVolByName(name)
if isNoStorageVolume(lookupErr) {
continue
}
if lookupErr != nil {
return translateLibvirtError("lookup managed VM cleanup volume", lookupErr)
}
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
_ = volume.Free()
return validateErr
}
path, pathErr := volume.GetPath()
if pathErr != nil {
_ = volume.Free()
return translateLibvirtError("resolve managed VM cleanup volume", pathErr)
}
referenced, referenceErr := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), name,
)
if referenceErr != nil {
_ = volume.Free()
return referenceErr
}
if referenced {
_ = volume.Free()
return NewError(ErrConflict, "managed_disk_shared", "managed VM cleanup volume is referenced by a domain", nil)
}
deleteErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = volume.Free()
if deleteErr != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), name)
if observed != nil {
_ = observed.Free()
}
if inspectErr != nil || observed != nil {
return operation.MarkUncertain(errors.Join(
mutationLibvirtError("delete managed VM cleanup volume "+name, deleteErr), inspectErr,
))
}
}
}
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
if err != nil {
return err
}
if seed != nil {
path, pathErr := seed.GetPath()
if pathErr != nil {
_ = seed.Free()
return translateLibvirtError("resolve managed NoCloud cleanup volume", pathErr)
}
referenced, referenceErr := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
)
if referenceErr != nil {
_ = seed.Free()
return referenceErr
}
if referenced {
_ = seed.Free()
return NewError(ErrConflict, "managed_disk_shared", "managed NoCloud cleanup volume is referenced by a domain", nil)
}
deleteErr := seed.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = seed.Free()
if deleteErr != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
if observed != nil {
_ = observed.Free()
}
if inspectErr != nil || observed != nil {
return operation.MarkUncertain(errors.Join(
mutationLibvirtError("delete NoCloud cleanup volume", deleteErr), inspectErr,
))
}
}
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if err != nil {
return err
}
if filter != nil {
document, xmlErr := filter.GetXMLDesc(0)
if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) {
_ = filter.Free()
return NewError(ErrConflict, "managed_filter_invalid",
"network filter cannot be confirmed as panel managed", xmlErr)
}
referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), "")
if referenceErr != nil {
_ = filter.Free()
return referenceErr
}
if referenced {
_ = filter.Free()
return NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by a domain", nil)
}
undefineErr := filter.Undefine()
_ = filter.Free()
if undefineErr != nil {
observed, inspectErr := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if observed != nil {
_ = observed.Free()
}
if inspectErr != nil || observed != nil {
return operation.MarkUncertain(errors.Join(
mutationLibvirtError("undefine VM cleanup network filter", undefineErr), inspectErr,
))
}
}
}
for _, name := range managedNVRAMArtifactNames(uuid) {
path := filepath.Join(l.local.NVRAMDirectory, name)
referenced, referenceErr := l.anyDomainUsesPath(path)
if referenceErr != nil {
return referenceErr
}
if referenced {
return NewError(ErrConflict, "managed_nvram_shared",
"managed VM firmware state is referenced by another domain", nil)
}
}
if err := removeManagedNVRAMArtifacts(l.local.NVRAMDirectory, uuid); err != nil {
return NewError(ErrUnavailable, "nvram_cleanup_pending", "managed VM firmware cleanup is incomplete", err)
}
if err := l.traffic.remove(uuid); err != nil {
return NewError(ErrUnavailable, "traffic_state_unavailable", "deleted VM traffic state could not be removed", err)
}
return nil
}
func (l *Libvirt) deleteArtifactsAbsent(uuid string) (bool, error) {
if !uuidPattern.MatchString(uuid) {
return false, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
}
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
if err != nil {
return false, translateLibvirtError("lookup managed storage pool", err)
}
defer pool.Free()
rootAbsent, err := l.dirPoolVolumeAbsent(pool, rootVolumeName(uuid))
if err != nil {
return false, err
}
if !rootAbsent {
return false, nil
}
inventory, err := pool.ListStorageVolumes()
if err != nil {
return false, translateLibvirtError("list managed volumes for reconciliation", err)
}
owned, err := collectVMManagedVolumeNames(uuid, nil, inventory)
if err != nil {
return false, err
}
if len(owned) != 0 {
return false, nil
}
seedPool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleSeed))
if err != nil {
return false, translateLibvirtError("lookup seed storage pool", err)
}
defer seedPool.Free()
seedAbsent, err := l.dirPoolVolumeAbsent(seedPool, seedVolumeName(uuid))
if err != nil {
return false, err
}
if !seedAbsent {
return false, nil
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if err != nil {
return false, err
}
if filter != nil {
_ = filter.Free()
return false, nil
}
nvramAbsent, err := managedNVRAMArtifactAbsent(l.local.NVRAMDirectory, uuid)
if err != nil {
return false, NewError(ErrUnavailable, "nvram_state_unavailable",
"managed VM firmware state could not be inspected", err)
}
if !nvramAbsent {
return false, nil
}
return true, nil
}
func managedNVRAMArtifactAbsent(directory, uuid string) (bool, error) {
candidates := managedNVRAMArtifactNames(uuid)
for _, name := range candidates {
if _, err := os.Lstat(filepath.Join(directory, name)); err == nil {
return false, nil
} else if !errors.Is(err, os.ErrNotExist) {
return false, err
}
}
return true, nil
}
func managedNVRAMArtifactNames(uuid string) []string {
normalized := strings.ToLower(uuid)
candidates := []string{normalized + "_VARS.fd"}
if uuid != normalized {
// CreateVM historically accepted an uppercase UUID and used its exact
// spelling for the NVRAM filename. Check both spellings so absence is
// never claimed while either managed artifact remains.
candidates = append(candidates, uuid+"_VARS.fd")
}
return candidates
}
func removeManagedNVRAMArtifacts(directory, uuid string) error {
for _, name := range managedNVRAMArtifactNames(uuid) {
path := filepath.Join(directory, name)
info, err := os.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return fmt.Errorf("managed NVRAM artifact is not a regular file")
}
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
return nil
}
func (l *Libvirt) domainByUUID(ctx context.Context, uuid string) (*libvirt.Domain, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if !uuidPattern.MatchString(uuid) {
return nil, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
}
domain, err := l.conn.LookupDomainByUUIDString(uuid)
if err != nil {
return nil, translateLibvirtError("lookup domain by UUID", err)
}
if _, _, err := requireManagedDomain(domain); err != nil {
domain.Free()
return nil, err
}
return domain, nil
}
func waitForShutoff(ctx context.Context, domain *libvirt.Domain) error {
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
state, _, err := domain.GetState()
if err != nil {
return translateLibvirtError("wait for shutdown", err)
}
if state == libvirt.DOMAIN_SHUTOFF {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
}
func mapDomainState(state libvirt.DomainState) VMStateName {
switch state {
case libvirt.DOMAIN_RUNNING:
return VMStateRunning
case libvirt.DOMAIN_BLOCKED:
return VMStateBlocked
case libvirt.DOMAIN_PAUSED:
return VMStatePaused
case libvirt.DOMAIN_SHUTDOWN:
return VMStateShutdown
case libvirt.DOMAIN_SHUTOFF:
return VMStateShutoff
case libvirt.DOMAIN_CRASHED:
return VMStateCrashed
case libvirt.DOMAIN_PMSUSPENDED:
return VMStateSuspended
default:
return VMStateUnknown
}
}
func translateLibvirtError(action string, err error) error {
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) {
switch libvirtErr.Code {
case libvirt.ERR_NO_DOMAIN:
return NewError(ErrNotFound, "vm_not_found", "virtual machine was not found", err)
case libvirt.ERR_NO_DOMAIN_SNAPSHOT:
return NewError(ErrNotFound, "snapshot_not_found", "virtual machine snapshot was not found", err)
case libvirt.ERR_NO_STORAGE_VOL:
return NewError(ErrNotFound, "storage_volume_not_found", "storage volume was not found", err)
case libvirt.ERR_NO_STORAGE_POOL:
return NewError(ErrUnavailable, "storage_pool_unavailable", "configured storage pool was not found", err)
case libvirt.ERR_NO_NWFILTER:
return NewError(ErrNotFound, "firewall_filter_not_found", "network filter was not found", err)
case libvirt.ERR_DOM_EXIST, libvirt.ERR_STORAGE_VOL_EXIST, libvirt.ERR_OPERATION_INVALID, libvirt.ERR_OPERATION_DENIED:
return NewError(ErrConflict, "libvirt_conflict", action+" conflicts with current hypervisor state", err)
case libvirt.ERR_INVALID_ARG:
return NewError(ErrInvalid, "invalid_libvirt_request", action+" was rejected as invalid", err)
case libvirt.ERR_OPERATION_UNSUPPORTED:
return NewError(ErrUnsupported, "unsupported", action+" is not supported", err)
case libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_AUTH_FAILED:
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", err)
}
}
return NewError(ErrUnavailable, "libvirt_error", fmt.Sprintf("%s failed", action), err)
}
func mutationLibvirtError(action string, err error) error {
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) {
switch libvirtErr.Code {
case libvirt.ERR_SYSTEM_ERROR, libvirt.ERR_RPC, libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_OPERATION_TIMEOUT:
return operation.MarkUncertain(translateLibvirtError(action, err))
}
}
return translateLibvirtError(action, err)
}
func formatLibvirtVersion(version uint32) string {
return fmt.Sprintf("%d.%d.%d", version/1_000_000, (version/1_000)%1_000, version%1_000)
}