You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_adapter.go

1560 lines
53 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"crypto/rand"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"net"
"net/netip"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"git.luoli.live/root/virt/agent/cloudinit"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
type Libvirt struct {
conn *libvirt.Connect
local LocalConfig
traffic *trafficStore
}
type createVMFailureStage struct {
code string
message string
}
var (
createVMTemplateLookupStage = createVMFailureStage{
code: "template_lookup_failed", message: "template metadata or volume could not be located",
}
createVMTemplateCapacityStage = createVMFailureStage{
code: "template_capacity_failed", message: "template capacity could not satisfy the requested disk",
}
createVMStorageCloneStage = createVMFailureStage{
code: "storage_clone_failed", message: "system disk could not be cloned from the template",
}
createVMSeedCreateStage = createVMFailureStage{
code: "seed_create_failed", message: "cloud-init seed volume could not be created",
}
createVMSeedUploadStage = createVMFailureStage{
code: "seed_upload_failed", message: "cloud-init seed data could not be uploaded",
}
createVMNWFilterDefineStage = createVMFailureStage{
code: "nwfilter_define_failed", message: "virtual machine network filter could not be defined",
}
createVMDomainDefineStage = createVMFailureStage{
code: "domain_define_failed", message: "virtual machine definition could not be created",
}
createVMDomainAutostartStage = createVMFailureStage{
code: "domain_autostart_failed", message: "virtual machine autostart could not be enabled",
}
createVMDomainStartStage = createVMFailureStage{
code: "domain_start_failed", message: "virtual machine could not be started",
}
)
type createVMStageError struct {
stage createVMFailureStage
cause error
}
func (e *createVMStageError) Error() string { return e.stage.message }
func (e *createVMStageError) Unwrap() error { return e.cause }
func (e *createVMStageError) PublicCode() string { return e.stage.code }
func (e *createVMStageError) PublicMessage() string { return e.stage.message }
// wrap exposes only a stable stage code and bounded public message. The full
// cause remains in the error chain for local diagnostics and, importantly,
// preserves operation uncertainty and the original hypervisor error kind.
func (stage createVMFailureStage) wrap(cause error) error {
if cause == nil {
return nil
}
return &createVMStageError{stage: stage, cause: cause}
}
func OpenLibvirt(uri string, local LocalConfig) (Hypervisor, error) {
if uri != "qemu:///system" {
return nil, NewError(ErrInvalid, "invalid_libvirt_uri", "only qemu:///system is allowed", nil)
}
conn, err := libvirt.NewConnect(uri)
if err != nil {
return nil, translateLibvirtError("connect", err)
}
traffic, err := openTrafficStore(local.TrafficStatePath)
if err != nil {
_, _ = conn.Close()
return nil, NewError(ErrUnavailable, "traffic_state_unavailable", "durable traffic state could not be opened", err)
}
return &Libvirt{conn: conn, local: local, traffic: traffic}, nil
}
func (l *Libvirt) Close() error {
_, err := l.conn.Close()
return err
}
func (l *Libvirt) Ping(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
alive, err := l.conn.IsAlive()
if err != nil {
return translateLibvirtError("ping", err)
}
if !alive {
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt connection is not alive", nil)
}
return nil
}
func (l *Libvirt) Capabilities(ctx context.Context) (Capabilities, error) {
if err := l.Ping(ctx); err != nil {
return Capabilities{}, err
}
libVersion, err := l.conn.GetLibVersion()
if err != nil {
return Capabilities{}, translateLibvirtError("get libvirt version", err)
}
qemuVersion, err := l.conn.GetVersion()
if err != nil {
return Capabilities{}, translateLibvirtError("get QEMU version", err)
}
if err := ctx.Err(); err != nil {
return Capabilities{}, err
}
features, checks, networkSelectors := l.probeRuntimeCapabilities()
powerActions := []PowerAction(nil)
if features.Power {
powerActions = []PowerAction{PowerStart, PowerShutdown, PowerReboot, PowerReset, PowerDestroy}
}
firmware := []string(nil)
if features.UEFI {
firmware = []string{"uefi"}
}
return Capabilities{
Backend: "libvirt", LibvirtVersion: formatLibvirtVersion(libVersion),
QEMUVersion: formatLibvirtVersion(qemuVersion),
Features: features, PowerActions: powerActions, Firmware: firmware,
NetworkSelectors: networkSelectors, Checks: checks,
}, nil
}
func (l *Libvirt) ListVMs(ctx context.Context) ([]VMMetadata, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
domains, err := l.conn.ListAllDomains(0)
if err != nil {
return nil, translateLibvirtError("list domains", err)
}
result := make([]VMMetadata, 0, len(domains))
for index := range domains {
domain := &domains[index]
document, xmlErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
_ = domain.Free()
if xmlErr != nil {
return nil, translateLibvirtError("read domain XML", xmlErr)
}
_, _, managed, inspectErr := inspectManagedDomain(document)
if inspectErr != nil {
return nil, inspectErr
}
if !managed {
continue
}
metadata, parseErr := metadataFromXML(document)
if parseErr != nil {
return nil, parseErr
}
result = append(result, metadata)
}
return result, nil
}
func (l *Libvirt) GetVMState(ctx context.Context, uuid string) (VMState, error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return VMState{}, err
}
defer domain.Free()
state, reason, err := domain.GetState()
if err != nil {
return VMState{}, translateLibvirtError("get domain state", err)
}
name, _ := domain.GetName()
return VMState{UUID: uuid, Name: name, State: mapDomainState(state), Reason: reason}, nil
}
func (l *Libvirt) GetVMMetadata(ctx context.Context, uuid string) (VMMetadata, error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return VMMetadata{}, err
}
defer domain.Free()
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return VMMetadata{}, translateLibvirtError("get domain XML", err)
}
return metadataFromXML(document)
}
func (l *Libvirt) CreateVM(ctx context.Context, request CreateVMRequest) (_ VMMetadata, resultErr error) {
if err := ctx.Err(); err != nil {
return VMMetadata{}, err
}
if err := ValidateCreateVM(request); err != nil {
return VMMetadata{}, err
}
domainMutationMu.Lock()
defer domainMutationMu.Unlock()
unlockTrafficSample := l.traffic.lockSample(request.UUID)
defer unlockTrafficSample()
attachment, err := l.resolvePublicNetwork(ctx, request)
if err != nil {
return VMMetadata{}, err
}
var mac string
for attempts := 0; attempts < 32; attempts++ {
mac, err = randomMAC()
if err != nil {
return VMMetadata{}, NewError(ErrUnavailable, "entropy_unavailable", "could not generate VM network identity", err)
}
conflict, inspectErr := l.networkIdentityConflict(ctx, request.UUID, mac, request.PublicIPCIDR)
if inspectErr != nil {
return VMMetadata{}, inspectErr
}
if !conflict {
break
}
mac = ""
}
if mac == "" {
return VMMetadata{}, NewError(ErrConflict, "network_identity_exhausted", "could not allocate a unique VM network identity", nil)
}
attachment.MAC = mac
templateRecord, err := l.loadImageTemplate(request.TemplateID)
if err != nil {
return VMMetadata{}, createVMTemplateLookupStage.wrap(err)
}
if templateRecord.GenerationToken != request.TemplateGenerationToken {
return VMMetadata{}, NewError(ErrConflict, "template_generation_changed", "template generation token no longer matches", nil)
}
seed, err := cloudinit.BuildISO(cloudinit.Request{
UUID: request.UUID, Name: request.Name, Hostname: request.Hostname,
GuestSSHPublicKey: request.GuestSSHPublicKey, RootPassword: request.RootPassword,
StartupScript: request.StartupScript,
Debian12NetworkFix: isDebian12Template(templateRecord),
Network: cloudinit.Network{
MAC: mac, PublicCIDR: request.PublicIPCIDR, Gateway: request.Gateway,
DNSServers: append([]string(nil), request.DNSServers...), InterfaceKey: "public0",
},
})
if err != nil {
return VMMetadata{}, NewError(ErrInvalid, "invalid_cloud_init", "cloud-init request is invalid", err)
}
root, err := l.cloneRootVolume(request)
if err != nil {
return VMMetadata{}, err
}
rootPath, err := root.GetPath()
if err != nil {
cleanupErr := root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = root.Free()
if cleanupErr != nil {
return VMMetadata{}, operation.MarkUncertain(createVMStorageCloneStage.wrap(errors.Join(
mutationLibvirtError("resolve cloned root volume path", err),
mutationLibvirtError("roll back cloned root volume", cleanupErr),
)))
}
return VMMetadata{}, createVMStorageCloneStage.wrap(
mutationLibvirtError("resolve cloned root volume path", err))
}
rootOwned := true
defer func() {
if rootOwned {
if cleanupErr := root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr,
mutationLibvirtError("roll back cloned root volume", cleanupErr)))
}
}
_ = root.Free()
}()
seedVolume, err := l.createSeedVolume(request.UUID, seed)
if err != nil {
return VMMetadata{}, err
}
seedPath, err := seedVolume.GetPath()
if err != nil {
cleanupErr := seedVolume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = seedVolume.Free()
if cleanupErr != nil {
return VMMetadata{}, operation.MarkUncertain(createVMSeedCreateStage.wrap(errors.Join(
mutationLibvirtError("resolve NoCloud volume path", err),
mutationLibvirtError("roll back NoCloud seed volume", cleanupErr),
)))
}
return VMMetadata{}, createVMSeedCreateStage.wrap(
mutationLibvirtError("resolve NoCloud volume path", err))
}
seedOwned := true
defer func() {
if seedOwned {
if cleanupErr := seedVolume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr,
mutationLibvirtError("roll back NoCloud seed volume", cleanupErr)))
}
}
_ = seedVolume.Free()
}()
filterName := networkFilterName(request.UUID)
filterDocument, err := BuildPlatformFilterXML(filterName, request.PlatformNetworkPolicy)
if err != nil {
return VMMetadata{}, createVMNWFilterDefineStage.wrap(err)
}
filter, err := l.conn.NWFilterDefineXML(filterDocument)
if err != nil {
observed, inspectErr := l.lookupOptionalNetworkFilter(filterName)
artifactExists := observed != nil
if observed != nil {
_ = observed.Free()
}
return VMMetadata{}, createVMNWFilterDefineStage.wrap(
deterministicArtifactCreateFailure(
"define VM network filter", err, inspectErr, artifactExists,
))
}
filterOwned := true
defer func() {
if filterOwned {
if cleanupErr := filter.Undefine(); cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr,
mutationLibvirtError("roll back VM network filter", cleanupErr)))
}
}
_ = filter.Free()
}()
document, err := BuildDomainXML(DomainDefinition{
UUID: request.UUID, Name: request.Name, Description: request.Description,
VCPUs: request.VCPUs, MemoryMiB: request.MemoryMiB, DiskGiB: request.DiskGiB,
BandwidthMbps: request.BandwidthMbps, Hostname: request.Hostname,
PublicIPCIDR: request.PublicIPCIDR, PublicPoolKey: request.PublicPoolKey,
PublicPoolCIDR: request.PublicPoolCIDR, Gateway: request.Gateway,
DNSServers: append([]string(nil), request.DNSServers...), NetworkSelector: request.NetworkSelector,
TemplateID: request.TemplateID,
RootDiskPath: rootPath, RootVolumeName: rootVolumeName(request.UUID), RootDiskFormat: "qcow2", SeedDiskPath: seedPath,
Firmware: Firmware{
Mode: "uefi", Loader: l.local.FirmwareLoader, NVRAMTemplate: l.local.FirmwareNVRAMTemplate,
NVRAMPath: filepath.Join(l.local.NVRAMDirectory, request.UUID+"_VARS.fd"), SecureBoot: false,
},
Network: []NetworkAttachment{attachment}, FilterName: filterName,
PlatformNetworkPolicy: request.PlatformNetworkPolicy,
})
if err != nil {
return VMMetadata{}, createVMDomainDefineStage.wrap(err)
}
trafficCreated, err := l.traffic.initialize(request.UUID, time.Now().UTC())
if err != nil {
return VMMetadata{}, NewError(ErrUnavailable, "traffic_state_unavailable", "new VM traffic baseline could not be initialized", err)
}
trafficOwned := trafficCreated
defer func() {
if trafficOwned {
if cleanupErr := l.traffic.remove(request.UUID); cleanupErr != nil {
resultErr = markTrafficCleanupUncertain(resultErr, cleanupErr)
}
}
}()
preserveDefinedDomainArtifacts := func() {
rootOwned = false
seedOwned = false
filterOwned = false
trafficOwned = false
}
domain, err := l.conn.DomainDefineXML(document)
if err != nil {
observed, inspectErr := l.conn.LookupDomainByUUIDString(request.UUID)
if observed != nil {
_ = observed.Free()
}
if domainMayExistAfterDefine(inspectErr) {
// DomainDefineXML may have committed before its response failed. Do
// not let deferred cleanup remove disks, firewall, or accounting
// state that the possibly-defined domain owns.
preserveDefinedDomainArtifacts()
return VMMetadata{}, operation.MarkUncertain(createVMDomainDefineStage.wrap(errors.Join(
mutationLibvirtError("define domain", err),
translateOptionalLibvirtError("inspect domain after define failure", inspectErr),
)))
}
return VMMetadata{}, createVMDomainDefineStage.wrap(mutationLibvirtError("define domain", err))
}
defer domain.Free()
if err := domain.SetAutostart(true); err != nil {
rollbackErr := domain.UndefineFlags(libvirt.DOMAIN_UNDEFINE_NVRAM)
absent, inspectErr := l.deleteDomainAbsent(ctx, request.UUID)
if inspectErr != nil || !absent {
preserveDefinedDomainArtifacts()
return VMMetadata{}, operation.MarkUncertain(createVMDomainAutostartStage.wrap(errors.Join(
mutationLibvirtError("enable domain autostart", err),
optionalMutationLibvirtError("roll back domain definition", rollbackErr),
inspectErr,
)))
}
if rollbackErr != nil {
return VMMetadata{}, operation.MarkUncertain(createVMDomainAutostartStage.wrap(errors.Join(
mutationLibvirtError("enable domain autostart", err),
mutationLibvirtError("roll back domain definition", rollbackErr),
)))
}
return VMMetadata{}, createVMDomainAutostartStage.wrap(
mutationLibvirtError("enable domain autostart", err))
}
preserveDefinedDomainArtifacts()
// From this point onward Domain.Create may have taken effect even if its
// response is lost. Retain the zero baseline so reconciliation and the
// first successful sample account for all traffic since initial boot.
if err := domain.Create(); err != nil {
return VMMetadata{}, operation.MarkUncertain(createVMDomainStartStage.wrap(
mutationLibvirtError("start newly defined domain", err)))
}
if err := l.verifyCreatedDomain(domain, request, filterName, mac); err != nil {
return VMMetadata{}, operation.MarkUncertain(err)
}
metadata, err := metadataFromXML(document)
if err != nil {
return VMMetadata{}, operation.MarkUncertain(err)
}
metadata.Status = VMStateRunning
metadata.DiskGiB = request.DiskGiB
metadata.BandwidthMbps = request.BandwidthMbps
metadata.Hostname = request.Hostname
metadata.PublicIPCIDR = request.PublicIPCIDR
metadata.TemplateID = request.TemplateID
return metadata, nil
}
func markTrafficCleanupUncertain(prior, cleanupErr error) error {
return operation.MarkUncertain(errors.Join(prior, NewError(
ErrUnavailable, "traffic_state_unavailable",
"unused VM traffic baseline could not be removed", cleanupErr,
)))
}
func isDebian12Template(template ImageTemplate) bool {
return strings.EqualFold(strings.TrimSpace(template.OSFamily), "debian") && strings.TrimSpace(template.Version) == "12"
}
func (l *Libvirt) resolvePublicNetwork(ctx context.Context,
request CreateVMRequest) (NetworkAttachment, error) {
if err := ValidatePublicNetworkIntent(
request.NetworkSelector, request.PublicPoolKey, request.PublicPoolCIDR,
request.PublicIPCIDR, request.Gateway, request.DNSServers,
); err != nil {
return NetworkAttachment{}, err
}
source, err := l.ResolveNetworkSelector(ctx, request.NetworkSelector)
if err != nil {
return NetworkAttachment{}, err
}
attachment := NetworkAttachment{}
switch source.Type {
case "bridge":
attachment.Bridge = source.Source
case "network":
attachment.Network = source.Source
default:
return NetworkAttachment{}, NewError(ErrUnavailable, "invalid_host_network_config", "host network source type is invalid", nil)
}
return attachment, nil
}
func (l *Libvirt) cloneRootVolume(request CreateVMRequest) (*libvirt.StorageVol, error) {
templateRecord, err := l.loadImageTemplate(request.TemplateID)
if err != nil {
return nil, createVMTemplateLookupStage.wrap(err)
}
if templateRecord.GenerationToken != request.TemplateGenerationToken {
return nil, NewError(ErrConflict, "template_generation_changed", "template generation token no longer matches", nil)
}
templatePool, err := l.conn.LookupStoragePoolByName(l.local.TemplateStoragePool)
if err != nil {
return nil, createVMTemplateLookupStage.wrap(
translateLibvirtError("lookup template storage pool", err))
}
defer templatePool.Free()
template, err := templatePool.LookupStorageVolByName(templateVolumeName(request.TemplateID))
if err != nil {
return nil, createVMTemplateLookupStage.wrap(
translateLibvirtError("lookup template volume", err))
}
defer template.Free()
templateInfo, err := template.GetInfo()
if err != nil {
return nil, createVMTemplateCapacityStage.wrap(
translateLibvirtError("read template volume size", err))
}
capacity := request.DiskGiB << 30
if capacity < templateInfo.Capacity {
return nil, createVMTemplateCapacityStage.wrap(NewError(
ErrInvalid, "disk_smaller_than_template", "disk_gib is smaller than the selected template", nil))
}
managedPool, err := l.conn.LookupStoragePoolByName(l.local.ManagedStoragePool)
if err != nil {
return nil, createVMStorageCloneStage.wrap(
translateLibvirtError("lookup managed storage pool", err))
}
defer managedPool.Free()
definition := libvirtxml.StorageVolume{
Name: rootVolumeName(request.UUID),
Capacity: &libvirtxml.StorageVolumeSize{Value: capacity, Unit: "bytes"},
Target: &libvirtxml.StorageVolumeTarget{
Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"},
},
}
document, err := definition.Marshal()
if err != nil {
return nil, createVMStorageCloneStage.wrap(fmt.Errorf("marshal root volume XML: %w", err))
}
volume, err := managedPool.StorageVolCreateXMLFrom(document, template, 0)
if err != nil {
artifactAbsent, inspectErr := l.dirPoolVolumeAbsent(
managedPool, rootVolumeName(request.UUID),
)
return nil, createVMStorageCloneStage.wrap(deterministicArtifactCreateFailure(
"clone template volume", err, inspectErr, !artifactAbsent,
))
}
return volume, nil
}
func (l *Libvirt) createSeedVolume(uuid string, contents []byte) (*libvirt.StorageVol, error) {
pool, err := l.conn.LookupStoragePoolByName(l.local.SeedStoragePool)
if err != nil {
return nil, createVMSeedCreateStage.wrap(
translateLibvirtError("lookup seed storage pool", err))
}
defer pool.Free()
definition := libvirtxml.StorageVolume{
Name: seedVolumeName(uuid),
Capacity: &libvirtxml.StorageVolumeSize{Value: uint64(len(contents)), Unit: "bytes"},
Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: "raw"}},
}
document, err := definition.Marshal()
if err != nil {
return nil, createVMSeedCreateStage.wrap(fmt.Errorf("marshal seed volume XML: %w", err))
}
volume, err := pool.StorageVolCreateXML(document, 0)
if err != nil {
artifactAbsent, inspectErr := l.dirPoolVolumeAbsent(pool, seedVolumeName(uuid))
return nil, createVMSeedCreateStage.wrap(deterministicArtifactCreateFailure(
"create NoCloud volume", err, inspectErr, !artifactAbsent,
))
}
if err := l.uploadVolume(volume, contents); err != nil {
uploadErr := createVMSeedUploadStage.wrap(err)
cleanupErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = volume.Free()
if cleanupErr != nil {
return nil, operation.MarkUncertain(errors.Join(uploadErr,
mutationLibvirtError("roll back NoCloud seed volume", cleanupErr)))
}
return nil, uploadErr
}
return volume, nil
}
func (l *Libvirt) uploadVolume(volume *libvirt.StorageVol, contents []byte) error {
stream, err := l.conn.NewStream(0)
if err != nil {
return translateLibvirtError("create storage upload stream", err)
}
defer stream.Free()
if err := volume.Upload(stream, 0, uint64(len(contents)), 0); err != nil {
_ = stream.Abort()
return mutationLibvirtError("open storage upload", err)
}
for len(contents) > 0 {
count, sendErr := stream.Send(contents)
if sendErr != nil || count <= 0 {
_ = stream.Abort()
if sendErr == nil {
sendErr = errors.New("storage stream made no progress")
}
return mutationLibvirtError("upload storage volume", sendErr)
}
contents = contents[count:]
}
if err := stream.Finish(); err != nil {
return mutationLibvirtError("finish storage upload", err)
}
return nil
}
func randomMAC() (string, error) {
random := make([]byte, 3)
if _, err := rand.Read(random); err != nil {
return "", err
}
return fmt.Sprintf("52:54:00:%02x:%02x:%02x", random[0], random[1], random[2]), nil
}
func (l *Libvirt) networkIdentityConflict(ctx context.Context, requestedUUID, mac, publicCIDR string) (bool, error) {
requestedPrefix, err := netip.ParsePrefix(publicCIDR)
if err != nil {
return false, NewError(ErrInvalid, "invalid_public_ip", "public_ip_cidr is invalid", err)
}
domains, err := l.conn.ListAllDomains(0)
if err != nil {
return false, translateLibvirtError("list domains for network identity validation", err)
}
for index := range domains {
if err := ctx.Err(); err != nil {
return false, err
}
domain := &domains[index]
document, xmlErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
_ = domain.Free()
if xmlErr != nil {
return false, translateLibvirtError("read domain network identity", xmlErr)
}
definition, metadata, managed, parseErr := inspectManagedDomain(document)
if parseErr != nil {
return false, parseErr
}
if !managed || strings.EqualFold(definition.UUID, requestedUUID) {
continue
}
if existing, parseErr := netip.ParsePrefix(metadata.PublicIPCIDR); parseErr == nil && existing.Addr() == requestedPrefix.Addr() {
return true, nil
}
if definition.Devices != nil {
for _, iface := range definition.Devices.Interfaces {
if iface.MAC != nil && strings.EqualFold(iface.MAC.Address, mac) {
return true, nil
}
}
}
}
return false, nil
}
func (l *Libvirt) verifyCreatedDomain(domain *libvirt.Domain, request CreateVMRequest, filterName, mac string) error {
autostart, err := domain.GetAutostart()
if err != nil || !autostart {
return mutationLibvirtError("verify domain autostart", err)
}
active, err := domain.IsActive()
if err != nil || !active {
return mutationLibvirtError("verify newly created domain state", err)
}
document, err := domain.GetXMLDesc(0)
if err != nil {
return translateLibvirtError("verify live domain XML", err)
}
definition, metadata, err := parseManagedDomain(document)
if err != nil {
return err
}
if !strings.EqualFold(definition.UUID, request.UUID) || metadata.PublicPoolKey != request.PublicPoolKey || metadata.PublicIPCIDR != request.PublicIPCIDR || definition.Devices == nil || len(definition.Devices.Interfaces) != 1 {
return NewError(ErrConflict, "create_observation_mismatch", "new domain identity or network metadata did not match the requested intent", nil)
}
var storedPolicy PlatformNetworkPolicySnapshot
if err := json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &storedPolicy); err != nil || storedPolicy.Digest != request.PlatformNetworkPolicy.Digest {
return NewError(ErrConflict, "firewall_observation_mismatch", "new domain platform policy metadata did not match the requested snapshot", err)
}
iface := definition.Devices.Interfaces[0]
if iface.MAC == nil || !strings.EqualFold(iface.MAC.Address, mac) || iface.FilterRef == nil || iface.FilterRef.Filter != filterName {
return NewError(ErrConflict, "firewall_observation_mismatch", "new domain did not attach the expected network filter", nil)
}
return l.verifyManagedFilter(filterName, storedPolicy)
}
func (l *Libvirt) verifyManagedFilter(filterName string, policy PlatformNetworkPolicySnapshot) error {
filter, err := l.conn.LookupNWFilterByName(filterName)
if err != nil {
return translateLibvirtError("verify VM network filter", err)
}
defer filter.Free()
filterXML, err := filter.GetXMLDesc(0)
if err != nil {
return translateLibvirtError("verify VM network policy", err)
}
if !strings.Contains(filterXML, `filter='clean-traffic'`) && !strings.Contains(filterXML, `filter="clean-traffic"`) {
return NewError(ErrConflict, "firewall_observation_mismatch", "new VM filter is missing the platform clean-traffic baseline", nil)
}
for _, rule := range policy.Rules {
if !strings.Contains(filterXML, "comment='"+rule.ID+"'") && !strings.Contains(filterXML, "comment=\""+rule.ID+"\"") {
return NewError(ErrConflict, "firewall_observation_mismatch", "new VM filter is missing a platform policy rule", nil)
}
}
return nil
}
func templateVolumeName(id string) string { return "template-" + id + ".qcow2" }
func rootVolumeName(uuid string) string { return strings.ToLower(uuid) + "-root.qcow2" }
func seedVolumeName(uuid string) string { return strings.ToLower(uuid) + "-seed.iso" }
func networkFilterName(uuid string) string { return "virt-" + strings.ToLower(uuid) }
func (l *Libvirt) DeleteVM(ctx context.Context, uuid string, request DeleteVMRequest) error {
if !request.RemoveManagedDisks {
return NewError(ErrInvalid, "managed_disk_cleanup_required", "remove_managed_disks must be true", nil)
}
domainMutationMu.Lock()
defer domainMutationMu.Unlock()
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return err
}
defer domain.Free()
_, managed, err := requireManagedDomain(domain)
if err != nil {
return err
}
if err := l.validateDeleteVMArtifacts(ctx, uuid, managed); err != nil {
return err
}
if err := driveDeleteDomainToAbsent(domain, func() (bool, error) {
return l.deleteDomainAbsent(ctx, uuid)
}); err != nil {
return err
}
if err := l.cleanupDeletedVMArtifacts(ctx, uuid); err != nil {
// Domain absence is already confirmed, so any later failure describes a
// partially completed cleanup tail and must never become a terminal
// failed delete in the journal.
return operation.MarkUncertain(err)
}
return nil
}
// validateDeleteVMArtifacts proves that every disk selected by a VM deletion
// is Agent-owned and is not referenced by another live or persistent domain.
// It deliberately runs before the domain is stopped so a corrupt ownership
// marker cannot turn an otherwise rejected request into a partial deletion.
func (l *Libvirt) validateDeleteVMArtifacts(ctx context.Context, uuid string,
managed *domainMetadataXML) error {
managedPool, err := l.conn.LookupStoragePoolByName(l.local.ManagedStoragePool)
if err != nil {
return translateLibvirtError("lookup managed storage pool", err)
}
defer managedPool.Free()
inventory, err := managedPool.ListStorageVolumes()
if err != nil {
return translateLibvirtError("list managed volumes for VM deletion", err)
}
volumeNames, err := collectVMManagedVolumeNames(uuid, managed, inventory)
if err != nil {
return err
}
for _, name := range volumeNames {
if err := ctx.Err(); err != nil {
return err
}
volume, lookupErr := managedPool.LookupStorageVolByName(name)
if isNoStorageVolume(lookupErr) {
continue
}
if lookupErr != nil {
return translateLibvirtError("lookup managed VM volume", lookupErr)
}
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
_ = volume.Free()
return validateErr
}
path, pathErr := volume.GetPath()
if pathErr != nil {
_ = volume.Free()
return translateLibvirtError("resolve managed VM volume", pathErr)
}
referenced, referenceErr := l.otherDomainUsesPath(path, uuid)
if referenceErr != nil {
_ = volume.Free()
return referenceErr
}
if referenced {
_ = volume.Free()
return NewError(ErrConflict, "managed_disk_shared", "managed VM volume is referenced by another domain", nil)
}
_ = volume.Free()
}
seedVolume, err := l.lookupOptionalManagedVolume(l.local.SeedStoragePool, seedVolumeName(uuid))
if err != nil {
return err
}
if seedVolume != nil {
seedPath, pathErr := seedVolume.GetPath()
if pathErr != nil {
_ = seedVolume.Free()
return translateLibvirtError("resolve managed NoCloud seed volume", pathErr)
}
referenced, referenceErr := l.otherDomainUsesPath(seedPath, uuid)
if referenceErr != nil {
_ = seedVolume.Free()
return referenceErr
}
if referenced {
_ = seedVolume.Free()
return NewError(ErrConflict, "managed_disk_shared", "managed NoCloud seed volume is referenced by another domain", nil)
}
_ = seedVolume.Free()
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if err != nil {
return err
}
if filter != nil {
_ = filter.Free()
}
return nil
}
type deleteDomainControl interface {
IsActive() (bool, error)
DestroyFlags(libvirt.DomainDestroyFlags) error
UndefineFlags(libvirt.DomainUndefineFlagsValues) error
}
// driveDeleteDomainToAbsent advances only the remaining stages of the exact
// delete operation being reconciled. Mutation-call errors are never accepted
// on their own: the observed domain state must prove that the requested stage
// took effect, otherwise the operation remains uncertain and can be reconciled
// again under the same operation ID.
func driveDeleteDomainToAbsent(domain deleteDomainControl,
domainAbsent func() (bool, error)) error {
active, err := domain.IsActive()
if err != nil {
return translateLibvirtError("check domain activity", err)
}
if active {
destroyErr := domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT)
stillActive, inspectErr := domain.IsActive()
if inspectErr != nil || stillActive {
var mutationErr, observedErr error
if destroyErr != nil {
mutationErr = mutationLibvirtError("destroy domain before undefine", destroyErr)
} else {
mutationErr = NewError(ErrConflict, "delete_domain_still_active",
"domain remained active after destroy", nil)
}
if inspectErr != nil {
observedErr = translateLibvirtError("recheck domain activity after destroy", inspectErr)
}
return operation.MarkUncertain(errors.Join(mutationErr, observedErr))
}
}
flags := libvirt.DOMAIN_UNDEFINE_NVRAM |
libvirt.DOMAIN_UNDEFINE_SNAPSHOTS_METADATA |
libvirt.DOMAIN_UNDEFINE_CHECKPOINTS_METADATA
undefineErr := domain.UndefineFlags(flags)
absent, inspectErr := domainAbsent()
if inspectErr != nil || !absent {
var mutationErr error
if undefineErr != nil {
mutationErr = mutationLibvirtError("undefine domain", undefineErr)
} else {
mutationErr = NewError(ErrConflict, "delete_domain_still_defined",
"domain remained present after undefine", nil)
}
return operation.MarkUncertain(errors.Join(mutationErr, inspectErr))
}
return nil
}
func (l *Libvirt) deleteDomainAbsent(ctx context.Context, uuid string) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
if !uuidPattern.MatchString(uuid) {
return false, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
}
domain, err := l.conn.LookupDomainByUUIDString(uuid)
if err == nil {
_ = domain.Free()
return false, nil
}
if isNoDomain(err) {
return true, nil
}
return false, translateLibvirtError("confirm deleted domain absence", err)
}
func isNoDomain(err error) bool {
var libvirtErr libvirt.Error
return errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_DOMAIN
}
func domainMayExistAfterDefine(inspectErr error) bool {
return inspectErr == nil || !isNoDomain(inspectErr)
}
func collectVMManagedVolumeNames(uuid string, metadata *domainMetadataXML, inventory []string) ([]string, error) {
if !uuidPattern.MatchString(uuid) {
return nil, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
}
names := make(map[string]struct{})
for _, name := range inventory {
if isVMOwnedManagedVolumeName(uuid, name) {
names[name] = struct{}{}
}
}
if metadata != nil {
if !isVMOwnedRootVolumeName(uuid, metadata.RootVolumeName) {
return nil, NewError(ErrConflict, "managed_metadata_invalid", "managed root volume is not owned by the VM UUID", nil)
}
names[metadata.RootVolumeName] = struct{}{}
if metadata.PendingCleanupVolume != "" {
if !isVMOwnedRootVolumeName(uuid, metadata.PendingCleanupVolume) {
return nil, NewError(ErrConflict, "managed_metadata_invalid", "pending cleanup volume is not owned by the VM UUID", nil)
}
names[metadata.PendingCleanupVolume] = struct{}{}
}
for _, snapshot := range metadata.Snapshots {
name := snapshotVolumeName(uuid, snapshot.Key)
if !isVMOwnedManagedVolumeName(uuid, name) {
return nil, NewError(ErrConflict, "managed_metadata_invalid", "snapshot volume is not owned by the VM UUID", nil)
}
names[name] = struct{}{}
}
}
result := make([]string, 0, len(names))
for name := range names {
result = append(result, name)
}
sort.Strings(result)
return result, nil
}
func (l *Libvirt) lookupOptionalManagedVolume(poolName, volumeName string) (*libvirt.StorageVol, error) {
pool, err := l.conn.LookupStoragePoolByName(poolName)
if err != nil {
return nil, translateLibvirtError("lookup managed storage pool", err)
}
defer pool.Free()
volume, err := pool.LookupStorageVolByName(volumeName)
if err == nil {
return volume, nil
}
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_STORAGE_VOL {
return nil, nil
}
return nil, translateLibvirtError("lookup managed storage volume", err)
}
func (l *Libvirt) lookupOptionalNetworkFilter(name string) (*libvirt.NWFilter, error) {
filter, err := l.conn.LookupNWFilterByName(name)
if err == nil {
return filter, nil
}
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_NWFILTER {
return nil, nil
}
return nil, translateLibvirtError("lookup VM network filter", err)
}
func (l *Libvirt) Power(ctx context.Context, uuid string, request PowerRequest) (VMState, error) {
domainMutationMu.Lock()
defer domainMutationMu.Unlock()
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return VMState{}, err
}
defer domain.Free()
if _, _, err := requireManagedDomain(domain); err != nil {
return VMState{}, err
}
if err := dispatchPowerMutation(domain, request.Action); err != nil {
return VMState{}, err
}
if request.Action == PowerShutdown {
if err := waitForShutoff(ctx, domain); err != nil {
return VMState{}, operation.MarkUncertain(err)
}
}
state, reason, err := domain.GetState()
if err != nil {
return VMState{}, operation.MarkUncertain(translateLibvirtError("confirm power action", err))
}
mapped := mapDomainState(state)
if err := confirmPowerTargetState(request.Action, mapped); err != nil {
return VMState{}, err
}
name, _ := domain.GetName()
return VMState{UUID: uuid, Name: name, State: mapped, Reason: reason}, nil
}
type powerDomainControl interface {
Create() error
ShutdownFlags(libvirt.DomainShutdownFlags) error
Reboot(libvirt.DomainRebootFlagValues) error
Reset(uint32) error
DestroyFlags(libvirt.DomainDestroyFlags) error
}
func dispatchPowerMutation(domain powerDomainControl, action PowerAction) error {
var err error
switch action {
case PowerStart:
err = domain.Create()
case PowerShutdown:
err = domain.ShutdownFlags(libvirt.DOMAIN_SHUTDOWN_ACPI_POWER_BTN)
case PowerReboot:
err = domain.Reboot(libvirt.DOMAIN_REBOOT_ACPI_POWER_BTN)
case PowerReset:
err = domain.Reset(0)
case PowerDestroy:
err = domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT)
default:
return NewError(ErrInvalid, "invalid_power_action", "unknown power action", nil)
}
if err != nil {
// A libvirt lifecycle call can take effect before its transport reports
// an error. Keep the exact operation reconcilable instead of recording a
// terminal failure that could invite a second destructive request.
return operation.MarkUncertain(mutationLibvirtError("perform power action", err))
}
return nil
}
func confirmPowerTargetState(action PowerAction, state VMStateName) error {
reached := true
switch action {
case PowerStart:
reached = state == VMStateRunning || state == VMStateBlocked
case PowerShutdown, PowerDestroy:
reached = state == VMStateShutoff
}
if !reached {
return operation.MarkUncertain(NewError(
ErrConflict, "power_state_unconfirmed",
"power action did not reach the requested target state", nil,
))
}
return nil
}
func (l *Libvirt) GetConsole(ctx context.Context, uuid string) (ConsoleSession, error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return ConsoleSession{}, err
}
defer domain.Free()
active, err := domain.IsActive()
if err != nil {
return ConsoleSession{}, translateLibvirtError("check console domain activity", err)
}
if !active {
return ConsoleSession{}, NewError(ErrNotFound, "console_not_available", "VM has no active local VNC console", nil)
}
document, err := domain.GetXMLDesc(0)
if err != nil {
return ConsoleSession{}, translateLibvirtError("get live console XML", err)
}
metadata := struct {
Devices struct {
Graphics []struct {
Type string `xml:"type,attr"`
Port int `xml:"port,attr"`
} `xml:"graphics"`
} `xml:"devices"`
}{}
if err := xml.Unmarshal([]byte(document), &metadata); err != nil {
return ConsoleSession{}, fmt.Errorf("parse console XML: %w", err)
}
for _, graphic := range metadata.Devices.Graphics {
if graphic.Type == "vnc" && graphic.Port > 0 && graphic.Port <= 65535 {
return ConsoleSession{Protocol: "vnc", Address: net.JoinHostPort("127.0.0.1", strconv.Itoa(graphic.Port))}, nil
}
}
return ConsoleSession{}, NewError(ErrNotFound, "console_not_available", "VM has no active local VNC console", nil)
}
func (l *Libvirt) Reconcile(ctx context.Context, record operation.Record) (operation.Resolution, error) {
if err := ctx.Err(); err != nil {
return operation.Resolution{}, err
}
if resolution, handled, err := l.reconcileCatalogOperation(record); handled {
return resolution, err
}
// These reconciliation paths can continue only target-state or cleanup
// steps uniquely owned by the original operation. Serialize them with
// normal domain mutations; the journal already holds the per-VM lock.
if reconciliationNeedsDomainMutationLock(record.Action) {
domainMutationMu.Lock()
defer domainMutationMu.Unlock()
}
domain, err := l.domainByUUID(ctx, record.ResourceID)
if errors.Is(err, ErrNotFound) {
if record.Action == "create-vm" {
absent, inspectErr := l.deleteArtifactsAbsent(record.ResourceID)
if inspectErr != nil {
return operation.Resolution{}, inspectErr
}
if absent {
if cleanupErr := l.removeAbsentVMTrafficState(record.ResourceID); cleanupErr != nil {
return operation.Resolution{}, cleanupErr
}
return operation.Resolution{
Resolved: true, Succeeded: false,
Result: []byte(`{"creation_absent":true,"managed_artifacts_absent":true}`),
ErrorCode: "creation_absent",
ErrorMessage: "host evidence confirms the VM and managed artifacts are absent",
}, nil
}
}
if record.Action == "delete-vm" {
// The original delete already crossed its dispatch boundary and the
// domain is now absent. Continue only deterministic, UUID-owned
// cleanup; never invoke DeleteVM again or touch panel/IP state.
if cleanupErr := l.cleanupDeletedVMArtifacts(ctx, record.ResourceID); cleanupErr != nil {
return operation.Resolution{}, cleanupErr
}
absent, inspectErr := l.deleteArtifactsAbsent(record.ResourceID)
if inspectErr != nil {
return operation.Resolution{}, inspectErr
}
if absent {
return operation.Resolution{Resolved: true, Succeeded: true, Result: []byte(`{"absent":true,"managed_disks_absent":true}`)}, nil
}
}
if record.ExpectedState != nil {
return expectedStateResourceAbsent(record), nil
}
return operation.Resolution{}, nil
}
if err != nil {
return operation.Resolution{}, err
}
defer domain.Free()
if record.Action == "delete-vm" {
return l.reconcileExistingDelete(ctx, record.ResourceID, domain)
}
if resolution, handled, err := l.reconcileDomainFeatureOperation(record, domain); handled {
return resolution, err
}
state, _, err := domain.GetState()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile domain state", err)
}
mapped := mapDomainState(state)
switch record.Action {
case "create-vm":
return l.reconcileCreatedVM(domain)
case "power-start":
if mapped == VMStateRunning || mapped == VMStateBlocked {
return operation.Resolution{Resolved: true, Succeeded: true, Result: []byte(`{"state":"running"}`)}, nil
}
case "power-shutdown", "power-destroy":
if mapped == VMStateShutoff {
return operation.Resolution{Resolved: true, Succeeded: true, Result: []byte(`{"state":"shutoff"}`)}, nil
}
}
// Reboot/reset and mismatched states are not proof that dispatch did not
// happen; keep them uncertain instead of replaying.
return operation.Resolution{}, nil
}
func reconciliationNeedsDomainMutationLock(action string) bool {
return action == "create-vm" || action == "delete-vm" || action == "reinstall-vm" ||
strings.HasPrefix(action, "create-snapshot:") ||
strings.HasPrefix(action, "delete-snapshot:") ||
strings.HasPrefix(action, "restore-snapshot:")
}
func (l *Libvirt) reconcileExistingDelete(ctx context.Context, uuid string,
domain *libvirt.Domain) (operation.Resolution, error) {
_, managed, err := requireManagedDomain(domain)
if err != nil {
return operation.Resolution{}, err
}
if err := l.validateDeleteVMArtifacts(ctx, uuid, managed); err != nil {
return operation.Resolution{}, err
}
if err := driveDeleteDomainToAbsent(domain, func() (bool, error) {
return l.deleteDomainAbsent(ctx, uuid)
}); err != nil {
return operation.Resolution{}, err
}
if err := l.cleanupDeletedVMArtifacts(ctx, uuid); err != nil {
return operation.Resolution{}, err
}
absent, err := l.deleteArtifactsAbsent(uuid)
if err != nil {
return operation.Resolution{}, err
}
if !absent {
return operation.Resolution{}, nil
}
return operation.Resolution{
Resolved: true, Succeeded: true,
Result: []byte(`{"absent":true,"managed_disks_absent":true}`),
}, nil
}
func (l *Libvirt) removeAbsentVMTrafficState(uuid string) error {
unlockTrafficSample := l.traffic.lockSample(uuid)
defer unlockTrafficSample()
if err := l.traffic.remove(uuid); err != nil {
return NewError(ErrUnavailable, "traffic_state_unavailable",
"absent VM traffic state could not be removed", err)
}
return nil
}
// cleanupDeletedVMArtifacts resumes the idempotent tail of a delete after the
// domain definition is already gone. Every target is derived from the VM UUID,
// validated as Agent-owned, and checked for cross-domain references first.
func (l *Libvirt) cleanupDeletedVMArtifacts(ctx context.Context, uuid string) error {
if err := ctx.Err(); err != nil {
return err
}
unlockTrafficSample := l.traffic.lockSample(uuid)
defer unlockTrafficSample()
pool, err := l.conn.LookupStoragePoolByName(l.local.ManagedStoragePool)
if err != nil {
return translateLibvirtError("lookup managed storage pool", err)
}
defer pool.Free()
inventory, err := pool.ListStorageVolumes()
if err != nil {
return translateLibvirtError("list managed volumes for VM cleanup", err)
}
names, err := collectVMManagedVolumeNames(uuid, nil, inventory)
if err != nil {
return err
}
for _, name := range names {
if err := ctx.Err(); err != nil {
return err
}
volume, lookupErr := pool.LookupStorageVolByName(name)
if isNoStorageVolume(lookupErr) {
continue
}
if lookupErr != nil {
return translateLibvirtError("lookup managed VM cleanup volume", lookupErr)
}
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
_ = volume.Free()
return validateErr
}
path, pathErr := volume.GetPath()
if pathErr != nil {
_ = volume.Free()
return translateLibvirtError("resolve managed VM cleanup volume", pathErr)
}
referenced, referenceErr := l.anyDomainUsesPath(path)
if referenceErr != nil {
_ = volume.Free()
return referenceErr
}
if referenced {
_ = volume.Free()
return NewError(ErrConflict, "managed_disk_shared", "managed VM cleanup volume is referenced by a domain", nil)
}
deleteErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = volume.Free()
if deleteErr != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, name)
if observed != nil {
_ = observed.Free()
}
if inspectErr != nil || observed != nil {
return operation.MarkUncertain(errors.Join(
mutationLibvirtError("delete managed VM cleanup volume "+name, deleteErr), inspectErr,
))
}
}
}
seed, err := l.lookupOptionalManagedVolume(l.local.SeedStoragePool, seedVolumeName(uuid))
if err != nil {
return err
}
if seed != nil {
path, pathErr := seed.GetPath()
if pathErr != nil {
_ = seed.Free()
return translateLibvirtError("resolve managed NoCloud cleanup volume", pathErr)
}
referenced, referenceErr := l.anyDomainUsesPath(path)
if referenceErr != nil {
_ = seed.Free()
return referenceErr
}
if referenced {
_ = seed.Free()
return NewError(ErrConflict, "managed_disk_shared", "managed NoCloud cleanup volume is referenced by a domain", nil)
}
deleteErr := seed.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = seed.Free()
if deleteErr != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(l.local.SeedStoragePool, seedVolumeName(uuid))
if observed != nil {
_ = observed.Free()
}
if inspectErr != nil || observed != nil {
return operation.MarkUncertain(errors.Join(
mutationLibvirtError("delete NoCloud cleanup volume", deleteErr), inspectErr,
))
}
}
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if err != nil {
return err
}
if filter != nil {
undefineErr := filter.Undefine()
_ = filter.Free()
if undefineErr != nil {
observed, inspectErr := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if observed != nil {
_ = observed.Free()
}
if inspectErr != nil || observed != nil {
return operation.MarkUncertain(errors.Join(
mutationLibvirtError("undefine VM cleanup network filter", undefineErr), inspectErr,
))
}
}
}
for _, name := range managedNVRAMArtifactNames(uuid) {
path := filepath.Join(l.local.NVRAMDirectory, name)
referenced, referenceErr := l.anyDomainUsesPath(path)
if referenceErr != nil {
return referenceErr
}
if referenced {
return NewError(ErrConflict, "managed_nvram_shared",
"managed VM firmware state is referenced by another domain", nil)
}
}
if err := removeManagedNVRAMArtifacts(l.local.NVRAMDirectory, uuid); err != nil {
return NewError(ErrUnavailable, "nvram_cleanup_pending", "managed VM firmware cleanup is incomplete", err)
}
if err := l.traffic.remove(uuid); err != nil {
return NewError(ErrUnavailable, "traffic_state_unavailable", "deleted VM traffic state could not be removed", err)
}
return nil
}
func (l *Libvirt) deleteArtifactsAbsent(uuid string) (bool, error) {
if !uuidPattern.MatchString(uuid) {
return false, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
}
pool, err := l.conn.LookupStoragePoolByName(l.local.ManagedStoragePool)
if err != nil {
return false, translateLibvirtError("lookup managed storage pool", err)
}
defer pool.Free()
rootAbsent, err := l.dirPoolVolumeAbsent(pool, rootVolumeName(uuid))
if err != nil {
return false, err
}
if !rootAbsent {
return false, nil
}
inventory, err := pool.ListStorageVolumes()
if err != nil {
return false, translateLibvirtError("list managed volumes for reconciliation", err)
}
owned, err := collectVMManagedVolumeNames(uuid, nil, inventory)
if err != nil {
return false, err
}
if len(owned) != 0 {
return false, nil
}
seedPool, err := l.conn.LookupStoragePoolByName(l.local.SeedStoragePool)
if err != nil {
return false, translateLibvirtError("lookup seed storage pool", err)
}
defer seedPool.Free()
seedAbsent, err := l.dirPoolVolumeAbsent(seedPool, seedVolumeName(uuid))
if err != nil {
return false, err
}
if !seedAbsent {
return false, nil
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if err != nil {
return false, err
}
if filter != nil {
_ = filter.Free()
return false, nil
}
nvramAbsent, err := managedNVRAMArtifactAbsent(l.local.NVRAMDirectory, uuid)
if err != nil {
return false, NewError(ErrUnavailable, "nvram_state_unavailable",
"managed VM firmware state could not be inspected", err)
}
if !nvramAbsent {
return false, nil
}
return true, nil
}
func managedNVRAMArtifactAbsent(directory, uuid string) (bool, error) {
candidates := managedNVRAMArtifactNames(uuid)
for _, name := range candidates {
if _, err := os.Lstat(filepath.Join(directory, name)); err == nil {
return false, nil
} else if !errors.Is(err, os.ErrNotExist) {
return false, err
}
}
return true, nil
}
func managedNVRAMArtifactNames(uuid string) []string {
normalized := strings.ToLower(uuid)
candidates := []string{normalized + "_VARS.fd"}
if uuid != normalized {
// CreateVM historically accepted an uppercase UUID and used its exact
// spelling for the NVRAM filename. Check both spellings so absence is
// never claimed while either managed artifact remains.
candidates = append(candidates, uuid+"_VARS.fd")
}
return candidates
}
func removeManagedNVRAMArtifacts(directory, uuid string) error {
for _, name := range managedNVRAMArtifactNames(uuid) {
path := filepath.Join(directory, name)
info, err := os.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return fmt.Errorf("managed NVRAM artifact is not a regular file")
}
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
return nil
}
func (l *Libvirt) domainByUUID(ctx context.Context, uuid string) (*libvirt.Domain, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if !uuidPattern.MatchString(uuid) {
return nil, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
}
domain, err := l.conn.LookupDomainByUUIDString(uuid)
if err != nil {
return nil, translateLibvirtError("lookup domain by UUID", err)
}
if _, _, err := requireManagedDomain(domain); err != nil {
domain.Free()
return nil, err
}
return domain, nil
}
func waitForShutoff(ctx context.Context, domain *libvirt.Domain) error {
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
state, _, err := domain.GetState()
if err != nil {
return translateLibvirtError("wait for shutdown", err)
}
if state == libvirt.DOMAIN_SHUTOFF {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
}
func mapDomainState(state libvirt.DomainState) VMStateName {
switch state {
case libvirt.DOMAIN_RUNNING:
return VMStateRunning
case libvirt.DOMAIN_BLOCKED:
return VMStateBlocked
case libvirt.DOMAIN_PAUSED:
return VMStatePaused
case libvirt.DOMAIN_SHUTDOWN:
return VMStateShutdown
case libvirt.DOMAIN_SHUTOFF:
return VMStateShutoff
case libvirt.DOMAIN_CRASHED:
return VMStateCrashed
case libvirt.DOMAIN_PMSUSPENDED:
return VMStateSuspended
default:
return VMStateUnknown
}
}
func translateLibvirtError(action string, err error) error {
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) {
switch libvirtErr.Code {
case libvirt.ERR_NO_DOMAIN:
return NewError(ErrNotFound, "vm_not_found", "virtual machine was not found", err)
case libvirt.ERR_NO_DOMAIN_SNAPSHOT:
return NewError(ErrNotFound, "snapshot_not_found", "virtual machine snapshot was not found", err)
case libvirt.ERR_NO_STORAGE_VOL:
return NewError(ErrNotFound, "storage_volume_not_found", "storage volume was not found", err)
case libvirt.ERR_NO_STORAGE_POOL:
return NewError(ErrUnavailable, "storage_pool_unavailable", "configured storage pool was not found", err)
case libvirt.ERR_NO_NWFILTER:
return NewError(ErrNotFound, "firewall_filter_not_found", "network filter was not found", err)
case libvirt.ERR_DOM_EXIST, libvirt.ERR_STORAGE_VOL_EXIST, libvirt.ERR_OPERATION_INVALID, libvirt.ERR_OPERATION_DENIED:
return NewError(ErrConflict, "libvirt_conflict", action+" conflicts with current hypervisor state", err)
case libvirt.ERR_INVALID_ARG:
return NewError(ErrInvalid, "invalid_libvirt_request", action+" was rejected as invalid", err)
case libvirt.ERR_OPERATION_UNSUPPORTED:
return NewError(ErrUnsupported, "unsupported", action+" is not supported", err)
case libvirt.ERR_NO_CONNECT, libvirt.ERR_AUTH_FAILED:
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", err)
}
}
return NewError(ErrUnavailable, "libvirt_error", fmt.Sprintf("%s failed", action), err)
}
func mutationLibvirtError(action string, err error) error {
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) {
switch libvirtErr.Code {
case libvirt.ERR_SYSTEM_ERROR, libvirt.ERR_RPC, libvirt.ERR_NO_CONNECT, libvirt.ERR_OPERATION_TIMEOUT:
return operation.MarkUncertain(translateLibvirtError(action, err))
}
}
return translateLibvirtError(action, err)
}
func formatLibvirtVersion(version uint32) string {
return fmt.Sprintf("%d.%d.%d", version/1_000_000, (version/1_000)%1_000, version%1_000)
}