You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2104 lines
74 KiB
Go
2104 lines
74 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/netip"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/cloudinit"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
type Libvirt struct {
|
|
conn *libvirt.Connect
|
|
local LocalConfig
|
|
traffic *trafficStore
|
|
guestAgentResponseTimeoutSupported bool
|
|
}
|
|
|
|
const deleteGracefulShutdownTimeout = 30 * time.Second
|
|
|
|
type createVMFailureStage struct {
|
|
code string
|
|
message string
|
|
}
|
|
|
|
var (
|
|
createVMTemplateLookupStage = createVMFailureStage{
|
|
code: "template_lookup_failed", message: "template metadata or volume could not be located",
|
|
}
|
|
createVMTemplateCapacityStage = createVMFailureStage{
|
|
code: "template_capacity_failed", message: "template capacity could not satisfy the requested disk",
|
|
}
|
|
createVMStorageCloneStage = createVMFailureStage{
|
|
code: "storage_clone_failed", message: "system disk could not be cloned from the template",
|
|
}
|
|
createVMSeedCreateStage = createVMFailureStage{
|
|
code: "seed_create_failed", message: "cloud-init seed volume could not be created",
|
|
}
|
|
createVMSeedUploadStage = createVMFailureStage{
|
|
code: "seed_upload_failed", message: "cloud-init seed data could not be uploaded",
|
|
}
|
|
createVMNWFilterDefineStage = createVMFailureStage{
|
|
code: "nwfilter_define_failed", message: "virtual machine network filter could not be defined",
|
|
}
|
|
createVMDomainDefineStage = createVMFailureStage{
|
|
code: "domain_define_failed", message: "virtual machine definition could not be created",
|
|
}
|
|
createVMDomainAutostartStage = createVMFailureStage{
|
|
code: "domain_autostart_failed", message: "virtual machine autostart could not be enabled",
|
|
}
|
|
createVMDomainStartStage = createVMFailureStage{
|
|
code: "domain_start_failed", message: "virtual machine could not be started",
|
|
}
|
|
)
|
|
|
|
type createVMStageError struct {
|
|
stage createVMFailureStage
|
|
cause error
|
|
}
|
|
|
|
func (e *createVMStageError) Error() string { return e.stage.message }
|
|
func (e *createVMStageError) Unwrap() error { return e.cause }
|
|
func (e *createVMStageError) PublicCode() string { return e.stage.code }
|
|
func (e *createVMStageError) PublicMessage() string { return e.stage.message }
|
|
|
|
// wrap exposes only a stable stage code and bounded public message. The full
|
|
// cause remains in the error chain for local diagnostics and, importantly,
|
|
// preserves operation uncertainty and the original hypervisor error kind.
|
|
func (stage createVMFailureStage) wrap(cause error) error {
|
|
if cause == nil {
|
|
return nil
|
|
}
|
|
return &createVMStageError{stage: stage, cause: cause}
|
|
}
|
|
|
|
func OpenLibvirt(uri string, local LocalConfig) (Hypervisor, error) {
|
|
if uri != "qemu:///system" {
|
|
return nil, NewError(ErrInvalid, "invalid_libvirt_uri", "only qemu:///system is allowed", nil)
|
|
}
|
|
if err := startLibvirtEventLoop(); err != nil {
|
|
return nil, translateLibvirtError("start libvirt event loop", err)
|
|
}
|
|
clientVersion, err := libvirt.GetVersion()
|
|
if err != nil {
|
|
return nil, translateLibvirtError("get local libvirt version", err)
|
|
}
|
|
traffic, err := openTrafficStore(local.TrafficStatePath)
|
|
if err != nil {
|
|
return nil, NewError(ErrUnavailable, "traffic_state_unavailable", "durable traffic state could not be opened", err)
|
|
}
|
|
return openManagedLibvirt(uri, local, clientVersion, traffic), nil
|
|
}
|
|
|
|
func (l *Libvirt) Close() error {
|
|
_, err := l.conn.Close()
|
|
return err
|
|
}
|
|
|
|
func (l *Libvirt) Ping(ctx context.Context) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
alive, err := l.conn.IsAlive()
|
|
if err != nil {
|
|
return translateLibvirtError("ping", err)
|
|
}
|
|
if !alive {
|
|
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt connection is not alive", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) Capabilities(ctx context.Context) (Capabilities, error) {
|
|
if err := l.Ping(ctx); err != nil {
|
|
return Capabilities{}, err
|
|
}
|
|
libVersion, err := l.conn.GetLibVersion()
|
|
if err != nil {
|
|
return Capabilities{}, translateLibvirtError("get libvirt version", err)
|
|
}
|
|
qemuVersion, err := l.conn.GetVersion()
|
|
if err != nil {
|
|
return Capabilities{}, translateLibvirtError("get QEMU version", err)
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return Capabilities{}, err
|
|
}
|
|
features, checks, networkSelectors := l.probeRuntimeCapabilities()
|
|
powerActions := []PowerAction(nil)
|
|
if features.Power {
|
|
powerActions = []PowerAction{PowerStart, PowerShutdown, PowerReboot, PowerReset, PowerDestroy}
|
|
}
|
|
firmware := []string(nil)
|
|
if features.UEFI {
|
|
firmware = []string{"uefi"}
|
|
}
|
|
return Capabilities{
|
|
Backend: "libvirt", LibvirtVersion: formatLibvirtVersion(libVersion),
|
|
QEMUVersion: formatLibvirtVersion(qemuVersion),
|
|
Features: features, PowerActions: powerActions, Firmware: firmware,
|
|
NetworkSelectors: networkSelectors, Checks: checks,
|
|
}, nil
|
|
}
|
|
|
|
func (l *Libvirt) ListVMs(ctx context.Context) ([]VMMetadata, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
domains, err := l.conn.ListAllDomains(0)
|
|
if err != nil {
|
|
return nil, translateLibvirtError("list domains", err)
|
|
}
|
|
result := make([]VMMetadata, 0, len(domains))
|
|
for index := range domains {
|
|
domain := &domains[index]
|
|
document, xmlErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
_ = domain.Free()
|
|
if xmlErr != nil {
|
|
return nil, translateLibvirtError("read domain XML", xmlErr)
|
|
}
|
|
_, _, managed, inspectErr := inspectManagedDomain(document)
|
|
if inspectErr != nil {
|
|
return nil, inspectErr
|
|
}
|
|
if !managed {
|
|
continue
|
|
}
|
|
metadata, parseErr := metadataFromXML(document)
|
|
if parseErr != nil {
|
|
return nil, parseErr
|
|
}
|
|
result = append(result, metadata)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (l *Libvirt) GetVMState(ctx context.Context, uuid string) (VMState, error) {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return VMState{}, err
|
|
}
|
|
defer domain.Free()
|
|
state, reason, err := domain.GetState()
|
|
if err != nil {
|
|
return VMState{}, translateLibvirtError("get domain state", err)
|
|
}
|
|
name, _ := domain.GetName()
|
|
return VMState{UUID: uuid, Name: name, State: mapDomainState(state), Reason: reason}, nil
|
|
}
|
|
|
|
func (l *Libvirt) GetVMMetadata(ctx context.Context, uuid string) (VMMetadata, error) {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return VMMetadata{}, err
|
|
}
|
|
defer domain.Free()
|
|
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if err != nil {
|
|
return VMMetadata{}, translateLibvirtError("get domain XML", err)
|
|
}
|
|
return metadataFromXML(document)
|
|
}
|
|
|
|
func (l *Libvirt) CreateVM(ctx context.Context, request CreateVMRequest) (_ VMMetadata, resultErr error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return VMMetadata{}, err
|
|
}
|
|
if err := ValidateCreateVM(request); err != nil {
|
|
return VMMetadata{}, err
|
|
}
|
|
attachment, err := l.resolvePublicNetwork(ctx, request)
|
|
if err != nil {
|
|
return VMMetadata{}, err
|
|
}
|
|
mac := vmMACAddress(request.UUID)
|
|
conflict, err := l.networkIdentityConflict(ctx, request.UUID, mac, request.PublicIPCIDR)
|
|
if err != nil {
|
|
return VMMetadata{}, err
|
|
}
|
|
if conflict {
|
|
return VMMetadata{}, NewError(ErrConflict, "network_identity_conflict", "VM network identity is already in use", nil)
|
|
}
|
|
attachment.MAC = mac
|
|
templateRecord, err := l.loadImageTemplate(ctx, request.TemplateID)
|
|
if err != nil {
|
|
return VMMetadata{}, createVMTemplateLookupStage.wrap(err)
|
|
}
|
|
if templateRecord.GenerationToken != request.TemplateGenerationToken {
|
|
return VMMetadata{}, NewError(ErrConflict, "template_generation_changed", "template generation token no longer matches", nil)
|
|
}
|
|
seed, err := cloudinit.BuildISO(cloudinit.Request{
|
|
UUID: request.UUID, Name: request.Name, Hostname: request.Hostname,
|
|
GuestSSHPublicKey: request.GuestSSHPublicKey, RootPassword: request.RootPassword,
|
|
StartupScript: request.StartupScript,
|
|
Debian12NetworkFix: isDebian12Template(templateRecord),
|
|
Network: cloudinit.Network{
|
|
MAC: mac, PublicCIDR: request.PublicIPCIDR, Gateway: request.Gateway,
|
|
DNSServers: append([]string(nil), request.DNSServers...), InterfaceKey: "public0",
|
|
},
|
|
})
|
|
if err != nil {
|
|
return VMMetadata{}, NewError(ErrInvalid, "invalid_cloud_init", "cloud-init request is invalid", err)
|
|
}
|
|
|
|
root, err := l.cloneRootVolume(ctx, request)
|
|
if err != nil {
|
|
return VMMetadata{}, err
|
|
}
|
|
rootPath, err := root.GetPath()
|
|
if err != nil {
|
|
cleanupErr := root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
_ = root.Free()
|
|
if cleanupErr != nil {
|
|
return VMMetadata{}, operation.MarkUncertain(createVMStorageCloneStage.wrap(errors.Join(
|
|
mutationLibvirtError("resolve cloned root volume path", err),
|
|
mutationLibvirtError("roll back cloned root volume", cleanupErr),
|
|
)))
|
|
}
|
|
return VMMetadata{}, createVMStorageCloneStage.wrap(
|
|
mutationLibvirtError("resolve cloned root volume path", err))
|
|
}
|
|
rootOwned := true
|
|
defer func() {
|
|
if rootOwned {
|
|
if cleanupErr := root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
|
|
resultErr = operation.MarkUncertain(errors.Join(resultErr,
|
|
mutationLibvirtError("roll back cloned root volume", cleanupErr)))
|
|
}
|
|
}
|
|
_ = root.Free()
|
|
}()
|
|
|
|
seedVolume, err := l.createSeedVolume(ctx, request.UUID, seed)
|
|
if err != nil {
|
|
return VMMetadata{}, err
|
|
}
|
|
seedPath, err := seedVolume.GetPath()
|
|
if err != nil {
|
|
cleanupErr := seedVolume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
_ = seedVolume.Free()
|
|
if cleanupErr != nil {
|
|
return VMMetadata{}, operation.MarkUncertain(createVMSeedCreateStage.wrap(errors.Join(
|
|
mutationLibvirtError("resolve NoCloud volume path", err),
|
|
mutationLibvirtError("roll back NoCloud seed volume", cleanupErr),
|
|
)))
|
|
}
|
|
return VMMetadata{}, createVMSeedCreateStage.wrap(
|
|
mutationLibvirtError("resolve NoCloud volume path", err))
|
|
}
|
|
seedOwned := true
|
|
defer func() {
|
|
if seedOwned {
|
|
if cleanupErr := seedVolume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
|
|
resultErr = operation.MarkUncertain(errors.Join(resultErr,
|
|
mutationLibvirtError("roll back NoCloud seed volume", cleanupErr)))
|
|
}
|
|
}
|
|
_ = seedVolume.Free()
|
|
}()
|
|
|
|
filterName := networkFilterName(request.UUID)
|
|
filterDocument, err := BuildPlatformFilterXML(filterName, request.PlatformNetworkPolicy)
|
|
if err != nil {
|
|
return VMMetadata{}, createVMNWFilterDefineStage.wrap(err)
|
|
}
|
|
filter, err := l.conn.NWFilterDefineXML(filterDocument)
|
|
if err != nil {
|
|
observed, inspectErr := l.lookupOptionalNetworkFilter(filterName)
|
|
artifactExists := observed != nil
|
|
if observed != nil {
|
|
_ = observed.Free()
|
|
}
|
|
return VMMetadata{}, createVMNWFilterDefineStage.wrap(
|
|
deterministicArtifactCreateFailure(
|
|
"define VM network filter", err, inspectErr, artifactExists,
|
|
))
|
|
}
|
|
filterOwned := true
|
|
defer func() {
|
|
if filterOwned {
|
|
if cleanupErr := filter.Undefine(); cleanupErr != nil {
|
|
resultErr = operation.MarkUncertain(errors.Join(resultErr,
|
|
mutationLibvirtError("roll back VM network filter", cleanupErr)))
|
|
}
|
|
}
|
|
_ = filter.Free()
|
|
}()
|
|
|
|
document, err := BuildDomainXML(DomainDefinition{
|
|
UUID: request.UUID, Name: request.DomainName, DisplayName: request.Name, Description: request.Description,
|
|
VCPUs: request.VCPUs, MemoryMiB: request.MemoryMiB, DiskGiB: request.DiskGiB,
|
|
BandwidthMbps: request.BandwidthMbps, Hostname: request.Hostname,
|
|
PublicIPCIDR: request.PublicIPCIDR, PublicPoolKey: request.PublicPoolKey,
|
|
PublicPoolCIDR: request.PublicPoolCIDR, Gateway: request.Gateway,
|
|
DNSServers: append([]string(nil), request.DNSServers...), NetworkSelector: request.NetworkSelector,
|
|
TemplateID: request.TemplateID,
|
|
RootDiskPath: rootPath, RootVolumeName: rootVolumeName(request.UUID), RootDiskFormat: "qcow2", SeedDiskPath: seedPath,
|
|
Firmware: Firmware{
|
|
Mode: "uefi", Loader: l.local.FirmwareLoader, NVRAMTemplate: l.local.FirmwareNVRAMTemplate,
|
|
NVRAMPath: filepath.Join(l.local.NVRAMDirectory, request.UUID+"_VARS.fd"), SecureBoot: false,
|
|
},
|
|
Network: []NetworkAttachment{attachment}, FilterName: filterName,
|
|
PlatformNetworkPolicy: request.PlatformNetworkPolicy,
|
|
})
|
|
if err != nil {
|
|
return VMMetadata{}, createVMDomainDefineStage.wrap(err)
|
|
}
|
|
// Serialize the traffic baseline only after the storage, seed, and filter
|
|
// mutations have returned. A slow metrics read can no longer make a request
|
|
// wait before its first libvirt mutation after Journal dispatch.
|
|
unlockTrafficSample := l.traffic.lockSample(request.UUID)
|
|
defer unlockTrafficSample()
|
|
trafficCreated, err := l.traffic.initialize(request.UUID, time.Now().UTC())
|
|
if err != nil {
|
|
return VMMetadata{}, NewError(ErrUnavailable, "traffic_state_unavailable", "new VM traffic baseline could not be initialized", err)
|
|
}
|
|
trafficOwned := trafficCreated
|
|
defer func() {
|
|
if trafficOwned {
|
|
if cleanupErr := l.traffic.remove(request.UUID); cleanupErr != nil {
|
|
resultErr = markTrafficCleanupUncertain(resultErr, cleanupErr)
|
|
}
|
|
}
|
|
}()
|
|
preserveDefinedDomainArtifacts := func() {
|
|
rootOwned = false
|
|
seedOwned = false
|
|
filterOwned = false
|
|
trafficOwned = false
|
|
}
|
|
domain, err := l.conn.DomainDefineXML(document)
|
|
if err != nil {
|
|
observed, inspectErr := l.conn.LookupDomainByUUIDString(request.UUID)
|
|
if observed != nil {
|
|
_ = observed.Free()
|
|
}
|
|
if domainMayExistAfterDefine(inspectErr) {
|
|
// DomainDefineXML may have committed before its response failed. Do
|
|
// not let deferred cleanup remove disks, firewall, or accounting
|
|
// state that the possibly-defined domain owns.
|
|
preserveDefinedDomainArtifacts()
|
|
return VMMetadata{}, operation.MarkUncertain(createVMDomainDefineStage.wrap(errors.Join(
|
|
mutationLibvirtError("define domain", err),
|
|
translateOptionalLibvirtError("inspect domain after define failure", inspectErr),
|
|
)))
|
|
}
|
|
return VMMetadata{}, createVMDomainDefineStage.wrap(mutationLibvirtError("define domain", err))
|
|
}
|
|
defer domain.Free()
|
|
if err := domain.SetAutostart(true); err != nil {
|
|
rollbackErr := domain.UndefineFlags(libvirt.DOMAIN_UNDEFINE_NVRAM)
|
|
absent, inspectErr := l.deleteDomainAbsent(ctx, request.UUID)
|
|
if inspectErr != nil || !absent {
|
|
preserveDefinedDomainArtifacts()
|
|
return VMMetadata{}, operation.MarkUncertain(createVMDomainAutostartStage.wrap(errors.Join(
|
|
mutationLibvirtError("enable domain autostart", err),
|
|
optionalMutationLibvirtError("roll back domain definition", rollbackErr),
|
|
inspectErr,
|
|
)))
|
|
}
|
|
if rollbackErr != nil {
|
|
return VMMetadata{}, operation.MarkUncertain(createVMDomainAutostartStage.wrap(errors.Join(
|
|
mutationLibvirtError("enable domain autostart", err),
|
|
mutationLibvirtError("roll back domain definition", rollbackErr),
|
|
)))
|
|
}
|
|
return VMMetadata{}, createVMDomainAutostartStage.wrap(
|
|
mutationLibvirtError("enable domain autostart", err))
|
|
}
|
|
preserveDefinedDomainArtifacts()
|
|
// From this point onward Domain.Create may have taken effect even if its
|
|
// response is lost. Retain the zero baseline so reconciliation and the
|
|
// first successful sample account for all traffic since initial boot.
|
|
if err := domain.Create(); err != nil {
|
|
return VMMetadata{}, operation.MarkUncertain(createVMDomainStartStage.wrap(
|
|
mutationLibvirtError("start newly defined domain", err)))
|
|
}
|
|
if err := l.verifyCreatedDomain(domain, request, filterName, mac); err != nil {
|
|
return VMMetadata{}, operation.MarkUncertain(err)
|
|
}
|
|
metadata, err := metadataFromXML(document)
|
|
if err != nil {
|
|
return VMMetadata{}, operation.MarkUncertain(err)
|
|
}
|
|
metadata.Status = VMStateRunning
|
|
metadata.DiskGiB = request.DiskGiB
|
|
metadata.BandwidthMbps = request.BandwidthMbps
|
|
metadata.Hostname = request.Hostname
|
|
metadata.PublicIPCIDR = request.PublicIPCIDR
|
|
metadata.TemplateID = request.TemplateID
|
|
return metadata, nil
|
|
}
|
|
|
|
func markTrafficCleanupUncertain(prior, cleanupErr error) error {
|
|
return operation.MarkUncertain(errors.Join(prior, NewError(
|
|
ErrUnavailable, "traffic_state_unavailable",
|
|
"unused VM traffic baseline could not be removed", cleanupErr,
|
|
)))
|
|
}
|
|
|
|
func isDebian12Template(template ImageTemplate) bool {
|
|
return strings.EqualFold(strings.TrimSpace(template.OSFamily), "debian") && strings.TrimSpace(template.Version) == "12"
|
|
}
|
|
|
|
func (l *Libvirt) resolvePublicNetwork(ctx context.Context,
|
|
request CreateVMRequest) (NetworkAttachment, error) {
|
|
if err := ValidatePublicNetworkIntent(
|
|
request.NetworkSelector, request.PublicPoolKey, request.PublicPoolCIDR,
|
|
request.PublicIPCIDR, request.Gateway, request.DNSServers,
|
|
); err != nil {
|
|
return NetworkAttachment{}, err
|
|
}
|
|
source, err := l.ResolveNetworkSelector(ctx, request.NetworkSelector)
|
|
if err != nil {
|
|
return NetworkAttachment{}, err
|
|
}
|
|
attachment := NetworkAttachment{}
|
|
switch source.Type {
|
|
case "bridge":
|
|
attachment.Bridge = source.Source
|
|
case "network":
|
|
attachment.Network = source.Source
|
|
default:
|
|
return NetworkAttachment{}, NewError(ErrUnavailable, "invalid_host_network_config", "host network source type is invalid", nil)
|
|
}
|
|
return attachment, nil
|
|
}
|
|
|
|
func (l *Libvirt) cloneRootVolume(ctx context.Context, request CreateVMRequest) (*libvirt.StorageVol, error) {
|
|
templateRecord, err := l.loadImageTemplate(ctx, request.TemplateID)
|
|
if err != nil {
|
|
return nil, createVMTemplateLookupStage.wrap(err)
|
|
}
|
|
if templateRecord.GenerationToken != request.TemplateGenerationToken {
|
|
return nil, NewError(ErrConflict, "template_generation_changed", "template generation token no longer matches", nil)
|
|
}
|
|
templatePool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleTemplate))
|
|
if err != nil {
|
|
return nil, createVMTemplateLookupStage.wrap(
|
|
translateLibvirtError("lookup template storage pool", err))
|
|
}
|
|
defer templatePool.Free()
|
|
template, err := templatePool.LookupStorageVolByName(templateVolumeName(request.TemplateID))
|
|
if err != nil {
|
|
return nil, createVMTemplateLookupStage.wrap(
|
|
translateLibvirtError("lookup template volume", err))
|
|
}
|
|
defer template.Free()
|
|
templateInfo, err := template.GetInfo()
|
|
if err != nil {
|
|
return nil, createVMTemplateCapacityStage.wrap(
|
|
translateLibvirtError("read template volume size", err))
|
|
}
|
|
capacity := request.DiskGiB << 30
|
|
if capacity < templateInfo.Capacity {
|
|
return nil, createVMTemplateCapacityStage.wrap(NewError(
|
|
ErrInvalid, "disk_smaller_than_template", "disk_gib is smaller than the selected template", nil))
|
|
}
|
|
managedPool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(
|
|
translateLibvirtError("lookup managed storage pool", err))
|
|
}
|
|
defer managedPool.Free()
|
|
definition := libvirtxml.StorageVolume{
|
|
Name: rootVolumeName(request.UUID),
|
|
Capacity: &libvirtxml.StorageVolumeSize{Value: capacity, Unit: "bytes"},
|
|
Target: &libvirtxml.StorageVolumeTarget{
|
|
Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"},
|
|
},
|
|
}
|
|
document, err := definition.Marshal()
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(fmt.Errorf("marshal root volume XML: %w", err))
|
|
}
|
|
volume, err := managedPool.StorageVolCreateXMLFrom(document, template, 0)
|
|
if err != nil {
|
|
artifactAbsent, inspectErr := l.dirPoolVolumeAbsent(
|
|
managedPool, rootVolumeName(request.UUID),
|
|
)
|
|
return nil, createVMStorageCloneStage.wrap(deterministicArtifactCreateFailure(
|
|
"clone template volume", err, inspectErr, !artifactAbsent,
|
|
))
|
|
}
|
|
return volume, nil
|
|
}
|
|
|
|
func (l *Libvirt) createSeedVolume(ctx context.Context, uuid string, contents []byte) (*libvirt.StorageVol, error) {
|
|
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleSeed))
|
|
if err != nil {
|
|
return nil, createVMSeedCreateStage.wrap(
|
|
translateLibvirtError("lookup seed storage pool", err))
|
|
}
|
|
defer pool.Free()
|
|
definition := libvirtxml.StorageVolume{
|
|
Name: seedVolumeName(uuid),
|
|
Capacity: &libvirtxml.StorageVolumeSize{Value: uint64(len(contents)), Unit: "bytes"},
|
|
Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: "raw"}},
|
|
}
|
|
document, err := definition.Marshal()
|
|
if err != nil {
|
|
return nil, createVMSeedCreateStage.wrap(fmt.Errorf("marshal seed volume XML: %w", err))
|
|
}
|
|
volume, err := pool.StorageVolCreateXML(document, 0)
|
|
if err != nil {
|
|
artifactAbsent, inspectErr := l.dirPoolVolumeAbsent(pool, seedVolumeName(uuid))
|
|
return nil, createVMSeedCreateStage.wrap(deterministicArtifactCreateFailure(
|
|
"create NoCloud volume", err, inspectErr, !artifactAbsent,
|
|
))
|
|
}
|
|
if err := l.uploadVolume(ctx, volume, contents); err != nil {
|
|
uploadErr := createVMSeedUploadStage.wrap(err)
|
|
if operation.IsUncertain(err) {
|
|
_ = volume.Free()
|
|
return nil, operation.MarkUncertain(uploadErr)
|
|
}
|
|
cleanupErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
_ = volume.Free()
|
|
if cleanupErr != nil {
|
|
return nil, operation.MarkUncertain(errors.Join(uploadErr,
|
|
mutationLibvirtError("roll back NoCloud seed volume", cleanupErr)))
|
|
}
|
|
return nil, uploadErr
|
|
}
|
|
return volume, nil
|
|
}
|
|
|
|
func (l *Libvirt) uploadVolume(ctx context.Context, volume *libvirt.StorageVol, contents []byte) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK)
|
|
if err != nil {
|
|
return translateLibvirtError("create storage upload stream", err)
|
|
}
|
|
transfer := &storageStreamTransfer{stream: stream}
|
|
if err := volume.Upload(stream, 0, uint64(len(contents)), 0); err != nil {
|
|
return mutationLibvirtError("open storage upload", transfer.abort(err))
|
|
}
|
|
if _, err := transfer.send(ctx, contents); err != nil {
|
|
return mutationLibvirtError("upload storage volume", transfer.abort(err))
|
|
}
|
|
if err := transfer.finish(ctx); err != nil {
|
|
return mutationLibvirtError("finish storage upload", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) networkIdentityConflict(ctx context.Context, requestedUUID, mac, publicCIDR string) (bool, error) {
|
|
requestedPrefix, err := netip.ParsePrefix(publicCIDR)
|
|
if err != nil {
|
|
return false, NewError(ErrInvalid, "invalid_public_ip", "public_ip_cidr is invalid", err)
|
|
}
|
|
domains, err := l.conn.ListAllDomains(0)
|
|
if err != nil {
|
|
return false, translateLibvirtError("list domains for network identity validation", err)
|
|
}
|
|
for index := range domains {
|
|
if err := ctx.Err(); err != nil {
|
|
return false, err
|
|
}
|
|
domain := &domains[index]
|
|
document, xmlErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
_ = domain.Free()
|
|
if xmlErr != nil {
|
|
return false, translateLibvirtError("read domain network identity", xmlErr)
|
|
}
|
|
definition, metadata, managed, parseErr := inspectManagedDomain(document)
|
|
if parseErr != nil {
|
|
return false, parseErr
|
|
}
|
|
if !managed || strings.EqualFold(definition.UUID, requestedUUID) {
|
|
continue
|
|
}
|
|
if existing, parseErr := netip.ParsePrefix(metadata.PublicIPCIDR); parseErr == nil && existing.Addr() == requestedPrefix.Addr() {
|
|
return true, nil
|
|
}
|
|
if definition.Devices != nil {
|
|
for _, iface := range definition.Devices.Interfaces {
|
|
if iface.MAC != nil && strings.EqualFold(iface.MAC.Address, mac) {
|
|
return true, nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func (l *Libvirt) verifyCreatedDomain(domain *libvirt.Domain, request CreateVMRequest, filterName, mac string) error {
|
|
autostart, err := domain.GetAutostart()
|
|
if err != nil || !autostart {
|
|
return mutationLibvirtError("verify domain autostart", err)
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil || !active {
|
|
return mutationLibvirtError("verify newly created domain state", err)
|
|
}
|
|
document, err := domain.GetXMLDesc(0)
|
|
if err != nil {
|
|
return translateLibvirtError("verify live domain XML", err)
|
|
}
|
|
definition, metadata, err := parseManagedDomain(document)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !strings.EqualFold(definition.UUID, request.UUID) || definition.Name != request.DomainName || metadata.DisplayName != request.Name || metadata.PublicPoolKey != request.PublicPoolKey || metadata.PublicIPCIDR != request.PublicIPCIDR || definition.Devices == nil || len(definition.Devices.Interfaces) != 1 {
|
|
return NewError(ErrConflict, "create_observation_mismatch", "new domain identity or network metadata did not match the requested intent", nil)
|
|
}
|
|
var storedPolicy PlatformNetworkPolicySnapshot
|
|
if err := json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &storedPolicy); err != nil || storedPolicy.Digest != request.PlatformNetworkPolicy.Digest {
|
|
return NewError(ErrConflict, "firewall_observation_mismatch", "new domain platform policy metadata did not match the requested snapshot", err)
|
|
}
|
|
iface := definition.Devices.Interfaces[0]
|
|
if iface.MAC == nil || !strings.EqualFold(iface.MAC.Address, mac) || iface.FilterRef == nil || iface.FilterRef.Filter != filterName {
|
|
return NewError(ErrConflict, "firewall_observation_mismatch", "new domain did not attach the expected network filter", nil)
|
|
}
|
|
return l.verifyManagedFilter(filterName, storedPolicy)
|
|
}
|
|
|
|
func (l *Libvirt) verifyManagedFilter(filterName string, policy PlatformNetworkPolicySnapshot) error {
|
|
filter, err := l.conn.LookupNWFilterByName(filterName)
|
|
if err != nil {
|
|
return translateLibvirtError("verify VM network filter", err)
|
|
}
|
|
defer filter.Free()
|
|
filterXML, err := filter.GetXMLDesc(0)
|
|
if err != nil {
|
|
return translateLibvirtError("verify VM network policy", err)
|
|
}
|
|
if !strings.Contains(filterXML, `filter='clean-traffic'`) && !strings.Contains(filterXML, `filter="clean-traffic"`) {
|
|
return NewError(ErrConflict, "firewall_observation_mismatch", "new VM filter is missing the platform clean-traffic baseline", nil)
|
|
}
|
|
for _, rule := range policy.Rules {
|
|
if !strings.Contains(filterXML, "comment='"+rule.ID+"'") && !strings.Contains(filterXML, "comment=\""+rule.ID+"\"") {
|
|
return NewError(ErrConflict, "firewall_observation_mismatch", "new VM filter is missing a platform policy rule", nil)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) DeleteVM(ctx context.Context, uuid string, request DeleteVMRequest) (DeleteVMResult, error) {
|
|
if !request.RemoveManagedDisks {
|
|
return DeleteVMResult{}, NewError(ErrInvalid, "managed_disk_cleanup_required", "remove_managed_disks must be true", nil)
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if result, absent, cleanupErr := completeDeleteForMissingDomain(err, func() (DeleteVMResult, error) {
|
|
return l.completeDeletedVM(ctx, uuid, false)
|
|
}); absent {
|
|
return result, cleanupErr
|
|
}
|
|
if err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
defer domain.Free()
|
|
_, managed, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
if err := l.validateDeleteVMArtifacts(ctx, uuid, managed); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
if _, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
if err := ensureNoCleanupBlockingJobs(domain); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
completionCtx := context.WithoutCancel(ctx)
|
|
if err := driveGracefulDeleteDomainToAbsent(completionCtx, domain, deleteGracefulShutdownTimeout, func(completionCtx context.Context) error {
|
|
if _, err := l.inspectDeleteDomainEvidence(completionCtx, uuid, domain); err != nil {
|
|
return err
|
|
}
|
|
return ensureNoCleanupBlockingJobs(domain)
|
|
}, func(completionCtx context.Context) (bool, error) {
|
|
return l.deleteDomainAbsent(completionCtx, uuid)
|
|
}); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
return l.completeDeletedVM(completionCtx, uuid, false)
|
|
}
|
|
|
|
func (l *Libvirt) ForceDeleteVM(ctx context.Context, uuid string, request ForceDeleteVMRequest) (DeleteVMResult, error) {
|
|
if !request.RemoveManagedDisks {
|
|
return DeleteVMResult{}, NewError(ErrInvalid, "managed_disk_cleanup_required", "remove_managed_disks must be true", nil)
|
|
}
|
|
if !request.AckDataLoss {
|
|
return DeleteVMResult{}, NewError(ErrInvalid, "data_loss_ack_required", "ack_data_loss must be true", nil)
|
|
}
|
|
if request.PriorOperationID == "" {
|
|
return DeleteVMResult{}, NewError(ErrInvalid, "invalid_prior_operation_id", "prior_operation_id is required", nil)
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if result, absent, cleanupErr := completeDeleteForMissingDomain(err, func() (DeleteVMResult, error) {
|
|
return l.completeDeletedVM(ctx, uuid, true)
|
|
}); absent {
|
|
return result, cleanupErr
|
|
}
|
|
if err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
defer domain.Free()
|
|
_, managed, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
if err := l.validateDeleteVMArtifacts(ctx, uuid, managed); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
if _, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
if err := ensureNoCleanupBlockingJobs(domain); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
if err := driveForceDeleteDomainToAbsent(domain, func() error {
|
|
if _, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain); err != nil {
|
|
return err
|
|
}
|
|
return ensureNoCleanupBlockingJobs(domain)
|
|
}, func() (bool, error) {
|
|
return l.deleteDomainAbsent(ctx, uuid)
|
|
}); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
return l.completeDeletedVM(ctx, uuid, true)
|
|
}
|
|
|
|
// completeDeleteForMissingDomain makes an already absent definition enter the
|
|
// same UUID-owned cleanup and final absence check as an in-request Undefine.
|
|
// This covers external removal and late graceful completion without inventing
|
|
// another domain mutation. The callback keeps the lookup decision directly
|
|
// testable without a live libvirt daemon.
|
|
func completeDeleteForMissingDomain(lookupErr error,
|
|
complete func() (DeleteVMResult, error)) (DeleteVMResult, bool, error) {
|
|
if !errors.Is(lookupErr, ErrNotFound) {
|
|
return DeleteVMResult{}, false, nil
|
|
}
|
|
result, err := complete()
|
|
return result, true, err
|
|
}
|
|
|
|
// validateDeleteVMArtifacts proves that every disk selected by a VM deletion
|
|
// is Agent-owned and is not referenced by another live or persistent domain.
|
|
// It deliberately runs before the domain is stopped so a corrupt ownership
|
|
// marker cannot turn an otherwise rejected request into a partial deletion.
|
|
func (l *Libvirt) validateDeleteVMArtifacts(ctx context.Context, uuid string,
|
|
managed *domainMetadataXML) error {
|
|
managedPool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return translateLibvirtError("lookup managed storage pool", err)
|
|
}
|
|
defer managedPool.Free()
|
|
inventory, err := managedPool.ListStorageVolumes()
|
|
if err != nil {
|
|
return translateLibvirtError("list managed volumes for VM deletion", err)
|
|
}
|
|
volumeNames, err := collectVMManagedVolumeNames(uuid, managed, inventory)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, name := range volumeNames {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
volume, lookupErr := managedPool.LookupStorageVolByName(name)
|
|
if isNoStorageVolume(lookupErr) {
|
|
continue
|
|
}
|
|
if lookupErr != nil {
|
|
return translateLibvirtError("lookup managed VM volume", lookupErr)
|
|
}
|
|
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
|
|
_ = volume.Free()
|
|
return validateErr
|
|
}
|
|
path, pathErr := volume.GetPath()
|
|
if pathErr != nil {
|
|
_ = volume.Free()
|
|
return translateLibvirtError("resolve managed VM volume", pathErr)
|
|
}
|
|
referenced, referenceErr := l.otherDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), name, uuid,
|
|
)
|
|
if referenceErr != nil {
|
|
_ = volume.Free()
|
|
return referenceErr
|
|
}
|
|
if referenced {
|
|
_ = volume.Free()
|
|
return NewError(ErrConflict, "managed_disk_shared", "managed VM volume is referenced by another domain", nil)
|
|
}
|
|
_ = volume.Free()
|
|
}
|
|
seedVolume, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if seedVolume != nil {
|
|
seedPath, pathErr := seedVolume.GetPath()
|
|
if pathErr != nil {
|
|
_ = seedVolume.Free()
|
|
return translateLibvirtError("resolve managed NoCloud seed volume", pathErr)
|
|
}
|
|
referenced, referenceErr := l.otherDomainUsesVolume(
|
|
seedPath, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid,
|
|
)
|
|
if referenceErr != nil {
|
|
_ = seedVolume.Free()
|
|
return referenceErr
|
|
}
|
|
if referenced {
|
|
_ = seedVolume.Free()
|
|
return NewError(ErrConflict, "managed_disk_shared", "managed NoCloud seed volume is referenced by another domain", nil)
|
|
}
|
|
_ = seedVolume.Free()
|
|
}
|
|
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if filter != nil {
|
|
document, xmlErr := filter.GetXMLDesc(0)
|
|
_ = filter.Free()
|
|
if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) {
|
|
return NewError(ErrConflict, "managed_filter_invalid",
|
|
"network filter cannot be confirmed as panel managed", xmlErr)
|
|
}
|
|
referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), uuid)
|
|
if referenceErr != nil {
|
|
return referenceErr
|
|
}
|
|
if referenced {
|
|
return NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type gracefulShutdownDomainControl interface {
|
|
IsActive() (bool, error)
|
|
ShutdownFlags(libvirt.DomainShutdownFlags) error
|
|
}
|
|
|
|
type deleteDomainControl interface {
|
|
gracefulShutdownDomainControl
|
|
DestroyFlags(libvirt.DomainDestroyFlags) error
|
|
UndefineFlags(libvirt.DomainUndefineFlagsValues) error
|
|
}
|
|
|
|
func driveGracefulDeleteDomainToAbsent(ctx context.Context, domain deleteDomainControl,
|
|
timeout time.Duration, beforeUndefine func(context.Context) error,
|
|
domainAbsent func(context.Context) (bool, error)) error {
|
|
ctx = context.WithoutCancel(ctx)
|
|
if err := requestGracefulDomainShutoff(ctx, domain, timeout); err != nil {
|
|
return err
|
|
}
|
|
if beforeUndefine != nil {
|
|
if err := beforeUndefine(ctx); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
err := driveInactiveDeleteDomainToAbsent(domain, func() (bool, error) {
|
|
return domainAbsent(ctx)
|
|
})
|
|
if !isForceRequired(err) {
|
|
return err
|
|
}
|
|
// The guest was initially inactive (or stopped gracefully) but became active
|
|
// before Undefine. That race is not evidence that a graceful wait expired.
|
|
// Issue a fresh ACPI request and perform its full bounded wait before a
|
|
// force_required result can ever authorize a separate force operation.
|
|
if err := requestGracefulDomainShutoff(ctx, domain, timeout); err != nil {
|
|
return err
|
|
}
|
|
if beforeUndefine != nil {
|
|
if err := beforeUndefine(ctx); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
err = driveInactiveDeleteDomainToAbsent(domain, func() (bool, error) {
|
|
return domainAbsent(ctx)
|
|
})
|
|
if isForceRequired(err) {
|
|
return operation.MarkUncertain(NewError(ErrConflict,
|
|
"graceful_delete_domain_reactivated",
|
|
"domain became active again after a confirmed graceful shutdown", err))
|
|
}
|
|
return err
|
|
}
|
|
|
|
func driveForceDeleteDomainToAbsent(domain deleteDomainControl,
|
|
beforeUndefine func() error, domainAbsent func() (bool, error)) error {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return translateLibvirtError("check domain activity", err)
|
|
}
|
|
destroyDispatched := false
|
|
if active {
|
|
destroyDispatched = true
|
|
destroyErr := domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT)
|
|
stillActive, inspectErr := domain.IsActive()
|
|
if inspectErr != nil || stillActive {
|
|
var mutationErr, observedErr error
|
|
if destroyErr != nil {
|
|
mutationErr = mutationLibvirtError("force domain off before delete", destroyErr)
|
|
} else {
|
|
mutationErr = NewError(ErrConflict, "force_delete_domain_still_active",
|
|
"domain remained active after force delete", nil)
|
|
}
|
|
if inspectErr != nil {
|
|
observedErr = translateLibvirtError("recheck domain activity after force delete", inspectErr)
|
|
}
|
|
return operation.MarkUncertain(errors.Join(mutationErr, observedErr))
|
|
}
|
|
}
|
|
if beforeUndefine != nil {
|
|
if err := beforeUndefine(); err != nil {
|
|
if destroyDispatched {
|
|
return operation.MarkUncertain(err)
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
err = driveInactiveDeleteDomainToAbsent(domain, domainAbsent)
|
|
if isForceRequired(err) {
|
|
// The explicit force authorization remains attached to this operation.
|
|
// If an external actor restarted the guest between Destroy and Undefine,
|
|
// keep the operation uncertain so its own reconciliation may safely
|
|
// continue; never terminate it as a new graceful force_required state.
|
|
return operation.MarkUncertain(NewError(ErrConflict, "force_delete_domain_reactivated",
|
|
"domain became active again during the explicit force delete", err))
|
|
}
|
|
if destroyDispatched && err != nil && !operation.IsUncertain(err) {
|
|
return operation.MarkUncertain(err)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// continueForceDeleteReconciliation never calls DestroyFlags. An uncertain
|
|
// force operation may have already crossed that destructive boundary before
|
|
// its response was lost; an explicitly authorized cleanup pass may only
|
|
// observe an active guest or continue from a confirmed inactive state.
|
|
func continueForceDeleteReconciliation(domain deleteDomainControl,
|
|
beforeUndefine func() error, domainAbsent func() (bool, error)) (bool, error) {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return false, translateLibvirtError("inspect force-delete domain activity", err)
|
|
}
|
|
if active {
|
|
return false, nil
|
|
}
|
|
if beforeUndefine != nil {
|
|
if err := beforeUndefine(); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
err = driveInactiveDeleteDomainToAbsent(domain, domainAbsent)
|
|
if isForceRequired(err) {
|
|
return false, operation.MarkUncertain(NewError(ErrConflict, "force_delete_domain_reactivated",
|
|
"domain became active again while reconciling the explicit force delete", err))
|
|
}
|
|
return err == nil, err
|
|
}
|
|
|
|
func isForceRequired(err error) bool {
|
|
var backend *Error
|
|
return errors.As(err, &backend) && backend.Code == "force_required"
|
|
}
|
|
|
|
func driveInactiveDeleteDomainToAbsent(domain deleteDomainControl,
|
|
domainAbsent func() (bool, error)) error {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return translateLibvirtError("check domain activity before undefine", err)
|
|
}
|
|
if active {
|
|
return NewError(ErrConflict, "force_required",
|
|
"guest did not stop after the graceful shutdown request; explicit force delete is required", nil)
|
|
}
|
|
flags := libvirt.DOMAIN_UNDEFINE_NVRAM |
|
|
libvirt.DOMAIN_UNDEFINE_MANAGED_SAVE |
|
|
libvirt.DOMAIN_UNDEFINE_SNAPSHOTS_METADATA |
|
|
libvirt.DOMAIN_UNDEFINE_CHECKPOINTS_METADATA
|
|
undefineErr := domain.UndefineFlags(flags)
|
|
absent, inspectErr := domainAbsent()
|
|
if undefineErr != nil || inspectErr != nil || !absent {
|
|
var mutationErr error
|
|
if undefineErr != nil {
|
|
mutationErr = mutationLibvirtError("undefine domain", undefineErr)
|
|
} else {
|
|
mutationErr = NewError(ErrConflict, "delete_domain_still_defined",
|
|
"domain remained present after undefine", nil)
|
|
}
|
|
return operation.MarkUncertain(errors.Join(mutationErr, inspectErr))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// requestGracefulDomainShutoff is shared by VM deletion and service expiry.
|
|
// It has no force method in its interface, so neither caller can silently
|
|
// escalate a graceful lifecycle request to DestroyFlags.
|
|
func requestGracefulDomainShutoff(ctx context.Context, domain gracefulShutdownDomainControl,
|
|
timeout time.Duration) error {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return translateLibvirtError("check domain activity before graceful shutdown", err)
|
|
}
|
|
if !active {
|
|
return nil
|
|
}
|
|
shutdownErr := domain.ShutdownFlags(libvirt.DOMAIN_SHUTDOWN_ACPI_POWER_BTN)
|
|
if shutdownErr != nil {
|
|
stillActive, inspectErr := domain.IsActive()
|
|
if inspectErr != nil {
|
|
return ambiguousGracefulShutdown(errors.Join(
|
|
mutationLibvirtError("request graceful domain shutdown", shutdownErr),
|
|
translateLibvirtError("confirm graceful domain shutdown request", inspectErr),
|
|
))
|
|
}
|
|
if !stillActive {
|
|
return nil
|
|
}
|
|
// A failed RPC response does not prove whether libvirt delivered the
|
|
// request. Observe the guest for the full graceful window, but never
|
|
// turn an unconfirmed request into force authorization. If the guest
|
|
// stays active, this operation fails and a later request must start a
|
|
// fresh graceful attempt with a new operation ID.
|
|
waitErr := waitForGracefulDomainShutoff(context.WithoutCancel(ctx), domain, timeout)
|
|
if waitErr == nil {
|
|
return nil
|
|
}
|
|
if isForceRequired(waitErr) {
|
|
return NewError(ErrConflict, "graceful_shutdown_request_failed",
|
|
"graceful shutdown request was not confirmed; retry graceful shutdown before force delete",
|
|
shutdownErr)
|
|
}
|
|
return ambiguousGracefulShutdown(errors.Join(
|
|
mutationLibvirtError("request graceful domain shutdown", shutdownErr), waitErr,
|
|
))
|
|
}
|
|
// Once ShutdownFlags returned success, finish the bounded observation even
|
|
// if the HTTP/request context disappears. Journal.Execute can return an
|
|
// uncertain response to the caller while this late result is persisted;
|
|
// abandoning the wait here would otherwise strand a safely recoverable
|
|
// graceful request without a legal force boundary.
|
|
return waitForGracefulDomainShutoff(context.WithoutCancel(ctx), domain, timeout)
|
|
}
|
|
|
|
func ambiguousGracefulShutdown(cause error) error {
|
|
return operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "graceful_shutdown_ambiguous",
|
|
"graceful shutdown result could not be confirmed", cause,
|
|
))
|
|
}
|
|
|
|
func waitForGracefulDomainShutoff(ctx context.Context, domain interface{ IsActive() (bool, error) }, timeout time.Duration) error {
|
|
if timeout <= 0 {
|
|
timeout = deleteGracefulShutdownTimeout
|
|
}
|
|
ticker := time.NewTicker(500 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
timer := time.NewTimer(timeout)
|
|
defer timer.Stop()
|
|
for {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.MarkUncertain(translateLibvirtError("wait for graceful domain shutdown", err))
|
|
}
|
|
if !active {
|
|
return nil
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return operation.MarkUncertain(ctx.Err())
|
|
case <-timer.C:
|
|
active, inspectErr := domain.IsActive()
|
|
if inspectErr != nil {
|
|
return operation.MarkUncertain(translateLibvirtError("confirm graceful domain shutdown timeout", inspectErr))
|
|
}
|
|
if !active {
|
|
return nil
|
|
}
|
|
return NewError(ErrConflict, "force_required",
|
|
"guest did not stop after the graceful shutdown request; explicit force delete is required", nil)
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// completeDeletedVM is shared by the initial request and reconciliation. A
|
|
// successful operation therefore always means a fresh UUID lookup, every
|
|
// panel-managed volume, the seed, network filter, NVRAM file, and local
|
|
// traffic state were all observed absent.
|
|
func (l *Libvirt) completeDeletedVM(ctx context.Context, uuid string, forced bool) (DeleteVMResult, error) {
|
|
if err := l.cleanupDeletedVMArtifacts(ctx, uuid); err != nil {
|
|
return DeleteVMResult{}, operation.MarkUncertain(err)
|
|
}
|
|
domainAbsent, err := l.deleteDomainAbsent(ctx, uuid)
|
|
if err != nil {
|
|
return DeleteVMResult{}, operation.MarkUncertain(err)
|
|
}
|
|
artifactsAbsent, err := l.deleteArtifactsAbsent(uuid)
|
|
if err != nil {
|
|
return DeleteVMResult{}, operation.MarkUncertain(err)
|
|
}
|
|
return confirmDeletedVMAbsence(domainAbsent, artifactsAbsent, l.traffic.contains(uuid), forced)
|
|
}
|
|
|
|
func confirmDeletedVMAbsence(domainAbsent, artifactsAbsent, trafficPresent, forced bool) (DeleteVMResult, error) {
|
|
if !domainAbsent || !artifactsAbsent || trafficPresent {
|
|
return DeleteVMResult{}, operation.MarkUncertain(NewError(
|
|
ErrConflict, "delete_cleanup_incomplete",
|
|
"deleted VM resources could not all be confirmed absent", nil,
|
|
))
|
|
}
|
|
return DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true, Forced: forced}, nil
|
|
}
|
|
|
|
// inspectDeleteDomainEvidence verifies all domain-bound state that
|
|
// UndefineFlags is about to remove. The evidence is private and is used only
|
|
// to bind an administrator cleanup plan; no XML or host path crosses the API.
|
|
func (l *Libvirt) inspectDeleteDomainEvidence(ctx context.Context, uuid string,
|
|
domain *libvirt.Domain) ([]string, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if err != nil {
|
|
return nil, translateLibvirtError("read persistent domain delete evidence", err)
|
|
}
|
|
var definition libvirtxml.Domain
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
return nil, NewError(ErrConflict, "managed_domain_xml_invalid", "managed domain XML is invalid", err)
|
|
}
|
|
evidence := make([]string, 0, 12)
|
|
documentDigest := sha256.Sum256([]byte(document))
|
|
evidence = append(evidence, "delete-domain-xml:"+hex.EncodeToString(documentDigest[:]))
|
|
|
|
nvramPaths, err := validateManagedDomainNVRAMPaths(&definition, l.local.NVRAMDirectory, uuid)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, path := range nvramPaths {
|
|
referenced, referenceErr := l.otherDomainUsesPath(path, uuid)
|
|
if referenceErr != nil {
|
|
return nil, referenceErr
|
|
}
|
|
if referenced {
|
|
return nil, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil)
|
|
}
|
|
info, statErr := os.Lstat(path)
|
|
switch {
|
|
case errors.Is(statErr, os.ErrNotExist):
|
|
evidence = append(evidence, "domain-nvram:"+path+":absent")
|
|
case statErr != nil:
|
|
return nil, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr)
|
|
case !info.Mode().IsRegular():
|
|
return nil, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil)
|
|
default:
|
|
evidence = append(evidence, "domain-nvram:"+path+":"+
|
|
strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10))
|
|
}
|
|
}
|
|
|
|
managedSave, err := domain.HasManagedSaveImage(0)
|
|
if err != nil {
|
|
return nil, translateLibvirtError("inspect domain managed save image", err)
|
|
}
|
|
evidence = append(evidence, "managed-save:"+strconv.FormatBool(managedSave))
|
|
|
|
snapshots, err := domain.ListAllSnapshots(0)
|
|
if err != nil {
|
|
return nil, translateLibvirtError("list domain snapshot metadata", err)
|
|
}
|
|
for index := range snapshots {
|
|
if err := ctx.Err(); err != nil {
|
|
for remaining := index; remaining < len(snapshots); remaining++ {
|
|
_ = snapshots[remaining].Free()
|
|
}
|
|
return nil, err
|
|
}
|
|
name, nameErr := snapshots[index].GetName()
|
|
snapshotXML, xmlErr := snapshots[index].GetXMLDesc(0)
|
|
_ = snapshots[index].Free()
|
|
if nameErr != nil || xmlErr != nil {
|
|
for remaining := index + 1; remaining < len(snapshots); remaining++ {
|
|
_ = snapshots[remaining].Free()
|
|
}
|
|
return nil, translateLibvirtError("read domain snapshot metadata", errors.Join(nameErr, xmlErr))
|
|
}
|
|
if safetyErr := validateNativeSnapshotDeleteSafety(snapshotXML); safetyErr != nil {
|
|
for remaining := index + 1; remaining < len(snapshots); remaining++ {
|
|
_ = snapshots[remaining].Free()
|
|
}
|
|
return nil, operation.MarkUncertain(safetyErr)
|
|
}
|
|
digest := sha256.Sum256([]byte(snapshotXML))
|
|
evidence = append(evidence, "domain-snapshot:"+name+":"+hex.EncodeToString(digest[:]))
|
|
}
|
|
evidence = append(evidence, "domain-snapshot-count:"+strconv.Itoa(len(snapshots)))
|
|
|
|
checkpoints, err := domain.ListAllCheckpoints(0)
|
|
if err != nil {
|
|
return nil, translateLibvirtError("list domain checkpoint metadata", err)
|
|
}
|
|
for index := range checkpoints {
|
|
if err := ctx.Err(); err != nil {
|
|
for remaining := index; remaining < len(checkpoints); remaining++ {
|
|
_ = checkpoints[remaining].Free()
|
|
}
|
|
return nil, err
|
|
}
|
|
name, nameErr := checkpoints[index].GetName()
|
|
checkpointXML, xmlErr := checkpoints[index].GetXMLDesc(0)
|
|
_ = checkpoints[index].Free()
|
|
if nameErr != nil || xmlErr != nil {
|
|
for remaining := index + 1; remaining < len(checkpoints); remaining++ {
|
|
_ = checkpoints[remaining].Free()
|
|
}
|
|
return nil, translateLibvirtError("read domain checkpoint metadata", errors.Join(nameErr, xmlErr))
|
|
}
|
|
var checkpoint libvirtxml.DomainCheckpoint
|
|
if err := checkpoint.Unmarshal(checkpointXML); err != nil {
|
|
for remaining := index + 1; remaining < len(checkpoints); remaining++ {
|
|
_ = checkpoints[remaining].Free()
|
|
}
|
|
return nil, NewError(ErrConflict, "domain_checkpoint_xml_invalid",
|
|
"native checkpoint XML could not be inspected before delete", err)
|
|
}
|
|
digest := sha256.Sum256([]byte(checkpointXML))
|
|
evidence = append(evidence, "domain-checkpoint:"+name+":"+hex.EncodeToString(digest[:]))
|
|
}
|
|
evidence = append(evidence, "domain-checkpoint-count:"+strconv.Itoa(len(checkpoints)))
|
|
|
|
filterName := networkFilterName(uuid)
|
|
referenced, err := l.otherDomainUsesFilter(filterName, uuid)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if referenced {
|
|
return nil, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil)
|
|
}
|
|
evidence = append(evidence, "network-filter-exclusive:"+filterName)
|
|
return evidence, nil
|
|
}
|
|
|
|
func validateManagedDomainNVRAMPaths(definition *libvirtxml.Domain, directory, uuid string) ([]string, error) {
|
|
paths := make(map[string]struct{}, 2)
|
|
if definition != nil && definition.OS != nil {
|
|
if nvram := definition.OS.NVRam; nvram != nil {
|
|
if nvram.NVRam != "" {
|
|
paths[filepath.Clean(nvram.NVRam)] = struct{}{}
|
|
}
|
|
if nvram.Source != nil && nvram.Source.File != nil && nvram.Source.File.File != "" {
|
|
paths[filepath.Clean(nvram.Source.File.File)] = struct{}{}
|
|
}
|
|
}
|
|
if definition.OS.VarStore != nil && definition.OS.VarStore.Path != "" {
|
|
paths[filepath.Clean(definition.OS.VarStore.Path)] = struct{}{}
|
|
}
|
|
}
|
|
if len(paths) == 0 {
|
|
return nil, NewError(ErrConflict, "managed_nvram_invalid", "managed domain NVRAM path is missing", nil)
|
|
}
|
|
expected := make(map[string]struct{}, len(managedNVRAMArtifactNames(uuid)))
|
|
for _, name := range managedNVRAMArtifactNames(uuid) {
|
|
expected[filepath.Clean(filepath.Join(directory, name))] = struct{}{}
|
|
}
|
|
result := make([]string, 0, len(paths))
|
|
for path := range paths {
|
|
if !filepath.IsAbs(path) {
|
|
return nil, NewError(ErrConflict, "managed_nvram_invalid", "managed domain NVRAM path is invalid", nil)
|
|
}
|
|
if _, ok := expected[path]; !ok {
|
|
return nil, NewError(ErrConflict, "managed_nvram_path_mismatch", "managed domain NVRAM path is outside the UUID-owned location", nil)
|
|
}
|
|
result = append(result, path)
|
|
}
|
|
sort.Strings(result)
|
|
return result, nil
|
|
}
|
|
|
|
func (l *Libvirt) otherDomainUsesFilter(filterName, excludedUUID string) (bool, error) {
|
|
filterReferenced, err := l.otherFilterReferencesFilter(filterName)
|
|
if err != nil || filterReferenced {
|
|
return filterReferenced, err
|
|
}
|
|
domains, err := l.conn.ListAllDomains(0)
|
|
if err != nil {
|
|
return false, translateLibvirtError("list domains for network filter reference check", err)
|
|
}
|
|
for index := range domains {
|
|
domain := &domains[index]
|
|
candidateUUID, uuidErr := domain.GetUUIDString()
|
|
if uuidErr != nil {
|
|
_ = domain.Free()
|
|
return false, translateLibvirtError("read domain UUID for network filter reference", uuidErr)
|
|
}
|
|
if excludedUUID != "" && strings.EqualFold(candidateUUID, excludedUUID) {
|
|
_ = domain.Free()
|
|
continue
|
|
}
|
|
flags := []libvirt.DomainXMLFlags{0}
|
|
persistent, persistentErr := domain.IsPersistent()
|
|
if persistentErr != nil {
|
|
_ = domain.Free()
|
|
return false, translateLibvirtError("inspect domain persistence for network filter reference", persistentErr)
|
|
}
|
|
if persistent {
|
|
flags = append(flags, libvirt.DOMAIN_XML_INACTIVE)
|
|
}
|
|
for _, flag := range flags {
|
|
document, xmlErr := domain.GetXMLDesc(flag)
|
|
if xmlErr != nil {
|
|
_ = domain.Free()
|
|
return false, translateLibvirtError("read domain XML for network filter reference", xmlErr)
|
|
}
|
|
var definition libvirtxml.Domain
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
_ = domain.Free()
|
|
return false, NewError(ErrConflict, "domain_xml_invalid", "domain XML could not be inspected for network filter references", err)
|
|
}
|
|
if domainReferencesFilter(&definition, filterName) {
|
|
_ = domain.Free()
|
|
return true, nil
|
|
}
|
|
}
|
|
referenced, historyErr := domainHistoryReferencesFilter(domain, filterName)
|
|
if historyErr != nil {
|
|
_ = domain.Free()
|
|
return false, historyErr
|
|
}
|
|
if referenced {
|
|
_ = domain.Free()
|
|
return true, nil
|
|
}
|
|
_ = domain.Free()
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func domainHistoryReferencesFilter(domain *libvirt.Domain, filterName string) (bool, error) {
|
|
managedSave, present, err := managedSaveDomainDefinition(domain)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if present && domainReferencesFilter(managedSave, filterName) {
|
|
return true, nil
|
|
}
|
|
|
|
snapshots, err := domain.ListAllSnapshots(0)
|
|
if err != nil {
|
|
return false, translateLibvirtError("list domain snapshots for network filter reference check", err)
|
|
}
|
|
defer func() {
|
|
for index := range snapshots {
|
|
_ = snapshots[index].Free()
|
|
}
|
|
}()
|
|
for index := range snapshots {
|
|
document, xmlErr := snapshots[index].GetXMLDesc(0)
|
|
if xmlErr != nil {
|
|
return false, translateLibvirtError("read domain snapshot XML for network filter reference", xmlErr)
|
|
}
|
|
var definition libvirtxml.DomainSnapshot
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
return false, NewError(ErrConflict, "domain_snapshot_xml_invalid",
|
|
"domain snapshot XML could not be inspected for network filter references", err)
|
|
}
|
|
if domainSnapshotReferencesFilter(&definition, filterName) {
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
checkpoints, err := domain.ListAllCheckpoints(0)
|
|
if err != nil {
|
|
return false, translateLibvirtError("list domain checkpoints for network filter reference check", err)
|
|
}
|
|
defer func() {
|
|
for index := range checkpoints {
|
|
_ = checkpoints[index].Free()
|
|
}
|
|
}()
|
|
for index := range checkpoints {
|
|
document, xmlErr := checkpoints[index].GetXMLDesc(0)
|
|
if xmlErr != nil {
|
|
return false, translateLibvirtError("read domain checkpoint XML for network filter reference", xmlErr)
|
|
}
|
|
var definition libvirtxml.DomainCheckpoint
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
return false, NewError(ErrConflict, "domain_checkpoint_xml_invalid",
|
|
"domain checkpoint XML could not be inspected for network filter references", err)
|
|
}
|
|
if definition.Domain != nil && domainReferencesFilter(definition.Domain, filterName) {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func (l *Libvirt) otherFilterReferencesFilter(filterName string) (bool, error) {
|
|
filters, err := l.conn.ListAllNWFilters(0)
|
|
if err != nil {
|
|
return false, translateLibvirtError("list network filters for dependency check", err)
|
|
}
|
|
defer func() {
|
|
for index := range filters {
|
|
_ = filters[index].Free()
|
|
}
|
|
}()
|
|
for index := range filters {
|
|
filter := &filters[index]
|
|
name, nameErr := filter.GetName()
|
|
if nameErr != nil {
|
|
return false, translateLibvirtError("read network filter name for dependency check", nameErr)
|
|
}
|
|
if name == filterName {
|
|
continue
|
|
}
|
|
document, xmlErr := filter.GetXMLDesc(0)
|
|
if xmlErr != nil {
|
|
return false, translateLibvirtError("read network filter dependency XML", xmlErr)
|
|
}
|
|
var definition libvirtxml.NWFilter
|
|
if err := definition.Unmarshal(document); err != nil {
|
|
return false, NewError(ErrConflict, "network_filter_xml_invalid",
|
|
"network filter dependencies could not be inspected", err)
|
|
}
|
|
if networkFilterReferencesFilter(&definition, filterName) {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func networkFilterReferencesFilter(definition *libvirtxml.NWFilter, filterName string) bool {
|
|
if definition == nil || filterName == "" {
|
|
return false
|
|
}
|
|
for _, entry := range definition.Entries {
|
|
if entry.Ref != nil && entry.Ref.Filter == filterName {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func domainReferencesFilter(definition *libvirtxml.Domain, filterName string) bool {
|
|
if definition == nil || definition.Devices == nil {
|
|
return false
|
|
}
|
|
for _, iface := range definition.Devices.Interfaces {
|
|
if iface.FilterRef != nil && iface.FilterRef.Filter == filterName {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func domainSnapshotReferencesFilter(snapshot *libvirtxml.DomainSnapshot, filterName string) bool {
|
|
if snapshot == nil {
|
|
return false
|
|
}
|
|
if snapshot.Domain != nil && domainReferencesFilter(snapshot.Domain, filterName) {
|
|
return true
|
|
}
|
|
return snapshot.InactiveDomain != nil &&
|
|
domainReferencesFilter(&snapshot.InactiveDomain.Domain, filterName)
|
|
}
|
|
|
|
func (l *Libvirt) deleteDomainAbsent(ctx context.Context, uuid string) (bool, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return false, err
|
|
}
|
|
if !uuidPattern.MatchString(uuid) {
|
|
return false, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
|
|
}
|
|
domain, err := l.conn.LookupDomainByUUIDString(uuid)
|
|
if err == nil {
|
|
_ = domain.Free()
|
|
return false, nil
|
|
}
|
|
if isNoDomain(err) {
|
|
return true, nil
|
|
}
|
|
return false, translateLibvirtError("confirm deleted domain absence", err)
|
|
}
|
|
|
|
func isNoDomain(err error) bool {
|
|
var libvirtErr libvirt.Error
|
|
return errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_DOMAIN
|
|
}
|
|
|
|
func domainMayExistAfterDefine(inspectErr error) bool {
|
|
return inspectErr == nil || !isNoDomain(inspectErr)
|
|
}
|
|
|
|
func collectVMManagedVolumeNames(uuid string, metadata *domainMetadataXML, inventory []string) ([]string, error) {
|
|
if !uuidPattern.MatchString(uuid) {
|
|
return nil, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
|
|
}
|
|
names := make(map[string]struct{})
|
|
for _, name := range inventory {
|
|
if isVMOwnedManagedVolumeName(uuid, name) {
|
|
names[name] = struct{}{}
|
|
}
|
|
}
|
|
if metadata != nil {
|
|
if !isVMOwnedRootVolumeName(uuid, metadata.RootVolumeName) {
|
|
return nil, NewError(ErrConflict, "managed_metadata_invalid", "managed root volume is not owned by the VM UUID", nil)
|
|
}
|
|
names[metadata.RootVolumeName] = struct{}{}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
if !isVMOwnedRootVolumeName(uuid, metadata.PendingCleanupVolume) {
|
|
return nil, NewError(ErrConflict, "managed_metadata_invalid", "pending cleanup volume is not owned by the VM UUID", nil)
|
|
}
|
|
names[metadata.PendingCleanupVolume] = struct{}{}
|
|
}
|
|
for _, snapshot := range metadata.Snapshots {
|
|
name := snapshotVolumeName(uuid, snapshot.Key)
|
|
if !isVMOwnedManagedVolumeName(uuid, name) {
|
|
return nil, NewError(ErrConflict, "managed_metadata_invalid", "snapshot volume is not owned by the VM UUID", nil)
|
|
}
|
|
names[name] = struct{}{}
|
|
}
|
|
}
|
|
result := make([]string, 0, len(names))
|
|
for name := range names {
|
|
result = append(result, name)
|
|
}
|
|
sort.Strings(result)
|
|
return result, nil
|
|
}
|
|
|
|
func (l *Libvirt) lookupOptionalManagedVolume(poolName, volumeName string) (*libvirt.StorageVol, error) {
|
|
pool, err := l.conn.LookupStoragePoolByName(poolName)
|
|
if err != nil {
|
|
return nil, translateLibvirtError("lookup managed storage pool", err)
|
|
}
|
|
defer pool.Free()
|
|
volume, err := pool.LookupStorageVolByName(volumeName)
|
|
if err == nil {
|
|
return volume, nil
|
|
}
|
|
var libvirtErr libvirt.Error
|
|
if errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_STORAGE_VOL {
|
|
return nil, nil
|
|
}
|
|
return nil, translateLibvirtError("lookup managed storage volume", err)
|
|
}
|
|
|
|
func (l *Libvirt) lookupOptionalNetworkFilter(name string) (*libvirt.NWFilter, error) {
|
|
filter, err := l.conn.LookupNWFilterByName(name)
|
|
if err == nil {
|
|
return filter, nil
|
|
}
|
|
var libvirtErr libvirt.Error
|
|
if errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_NWFILTER {
|
|
return nil, nil
|
|
}
|
|
return nil, translateLibvirtError("lookup VM network filter", err)
|
|
}
|
|
|
|
func (l *Libvirt) Power(ctx context.Context, uuid string, request PowerRequest) (VMState, error) {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return VMState{}, err
|
|
}
|
|
defer domain.Free()
|
|
if _, _, err := requireManagedDomain(domain); err != nil {
|
|
return VMState{}, err
|
|
}
|
|
if err := dispatchPowerMutation(domain, request.Action); err != nil {
|
|
return VMState{}, err
|
|
}
|
|
name, _ := domain.GetName()
|
|
if request.Action == PowerShutdown || request.Action == PowerReboot || request.Action == PowerReset {
|
|
// libvirt returning success is the durable effect for command-style
|
|
// lifecycle requests. The guest may remain running while ACPI shutdown
|
|
// or reboot is still in progress, so an immediate state sample is
|
|
// informational and can never turn an accepted command into failure.
|
|
result := VMState{UUID: uuid, Name: name, State: VMStateUnknown, CommandAccepted: true}
|
|
if state, reason, stateErr := domain.GetState(); stateErr == nil {
|
|
result.State = mapDomainState(state)
|
|
result.Reason = reason
|
|
}
|
|
return result, nil
|
|
}
|
|
state, reason, err := domain.GetState()
|
|
if err != nil {
|
|
return VMState{}, operation.MarkUncertain(translateLibvirtError("confirm power action", err))
|
|
}
|
|
mapped := mapDomainState(state)
|
|
if err := confirmPowerTargetState(request.Action, mapped); err != nil {
|
|
return VMState{}, err
|
|
}
|
|
return VMState{UUID: uuid, Name: name, State: mapped, Reason: reason, CommandAccepted: true, EffectConfirmed: true}, nil
|
|
}
|
|
|
|
type powerDomainControl interface {
|
|
Create() error
|
|
ShutdownFlags(libvirt.DomainShutdownFlags) error
|
|
Reboot(libvirt.DomainRebootFlagValues) error
|
|
Reset(uint32) error
|
|
DestroyFlags(libvirt.DomainDestroyFlags) error
|
|
}
|
|
|
|
func dispatchPowerMutation(domain powerDomainControl, action PowerAction) error {
|
|
var err error
|
|
switch action {
|
|
case PowerStart:
|
|
err = domain.Create()
|
|
case PowerShutdown:
|
|
err = domain.ShutdownFlags(libvirt.DOMAIN_SHUTDOWN_ACPI_POWER_BTN)
|
|
case PowerReboot:
|
|
err = domain.Reboot(libvirt.DOMAIN_REBOOT_ACPI_POWER_BTN)
|
|
case PowerReset:
|
|
err = domain.Reset(0)
|
|
case PowerDestroy:
|
|
err = domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT)
|
|
default:
|
|
return NewError(ErrInvalid, "invalid_power_action", "unknown power action", nil)
|
|
}
|
|
if err != nil {
|
|
// A libvirt lifecycle call can take effect before its transport reports
|
|
// an error. Keep the exact operation reconcilable instead of recording a
|
|
// terminal failure that could invite a second destructive request.
|
|
return operation.MarkUncertain(mutationLibvirtError("perform power action", err))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func confirmPowerTargetState(action PowerAction, state VMStateName) error {
|
|
reached := true
|
|
switch action {
|
|
case PowerStart:
|
|
reached = state == VMStateRunning || state == VMStateBlocked
|
|
case PowerDestroy:
|
|
reached = state == VMStateShutoff
|
|
}
|
|
if !reached {
|
|
return operation.MarkUncertain(NewError(
|
|
ErrConflict, "power_state_unconfirmed",
|
|
"power action did not reach the requested target state", nil,
|
|
))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) GetConsole(ctx context.Context, uuid string) (ConsoleSession, error) {
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return ConsoleSession{}, err
|
|
}
|
|
defer domain.Free()
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return ConsoleSession{}, translateLibvirtError("check console domain activity", err)
|
|
}
|
|
if !active {
|
|
return ConsoleSession{}, NewError(ErrNotFound, "console_not_available", "VM has no active local VNC console", nil)
|
|
}
|
|
document, err := domain.GetXMLDesc(0)
|
|
if err != nil {
|
|
return ConsoleSession{}, translateLibvirtError("get live console XML", err)
|
|
}
|
|
metadata := struct {
|
|
Devices struct {
|
|
Graphics []struct {
|
|
Type string `xml:"type,attr"`
|
|
Port int `xml:"port,attr"`
|
|
} `xml:"graphics"`
|
|
} `xml:"devices"`
|
|
}{}
|
|
if err := xml.Unmarshal([]byte(document), &metadata); err != nil {
|
|
return ConsoleSession{}, fmt.Errorf("parse console XML: %w", err)
|
|
}
|
|
for _, graphic := range metadata.Devices.Graphics {
|
|
if graphic.Type == "vnc" && graphic.Port > 0 && graphic.Port <= 65535 {
|
|
return ConsoleSession{Protocol: "vnc", Address: net.JoinHostPort("127.0.0.1", strconv.Itoa(graphic.Port))}, nil
|
|
}
|
|
}
|
|
return ConsoleSession{}, NewError(ErrNotFound, "console_not_available", "VM has no active local VNC console", nil)
|
|
}
|
|
|
|
func resolvedDeleteSuccess(result DeleteVMResult) operation.Resolution {
|
|
encoded, _ := json.Marshal(result)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
|
|
}
|
|
|
|
// cleanupDeletedVMArtifacts resumes the idempotent tail of a delete after the
|
|
// domain definition is already gone. Every target is derived from the VM UUID,
|
|
// validated as Agent-owned, and checked for cross-domain references first.
|
|
func (l *Libvirt) cleanupDeletedVMArtifacts(ctx context.Context, uuid string) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
unlockTrafficSample := l.traffic.lockSample(uuid)
|
|
defer unlockTrafficSample()
|
|
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return translateLibvirtError("lookup managed storage pool", err)
|
|
}
|
|
defer pool.Free()
|
|
inventory, err := pool.ListStorageVolumes()
|
|
if err != nil {
|
|
return translateLibvirtError("list managed volumes for VM cleanup", err)
|
|
}
|
|
names, err := collectVMManagedVolumeNames(uuid, nil, inventory)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, name := range names {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
volume, lookupErr := pool.LookupStorageVolByName(name)
|
|
if isNoStorageVolume(lookupErr) {
|
|
continue
|
|
}
|
|
if lookupErr != nil {
|
|
return translateLibvirtError("lookup managed VM cleanup volume", lookupErr)
|
|
}
|
|
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
|
|
_ = volume.Free()
|
|
return validateErr
|
|
}
|
|
path, pathErr := volume.GetPath()
|
|
if pathErr != nil {
|
|
_ = volume.Free()
|
|
return translateLibvirtError("resolve managed VM cleanup volume", pathErr)
|
|
}
|
|
referenced, referenceErr := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), name,
|
|
)
|
|
if referenceErr != nil {
|
|
_ = volume.Free()
|
|
return referenceErr
|
|
}
|
|
if referenced {
|
|
_ = volume.Free()
|
|
return NewError(ErrConflict, "managed_disk_shared", "managed VM cleanup volume is referenced by a domain", nil)
|
|
}
|
|
deleteErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
_ = volume.Free()
|
|
if deleteErr != nil {
|
|
observed, inspectErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), name)
|
|
if observed != nil {
|
|
_ = observed.Free()
|
|
}
|
|
if inspectErr != nil || observed != nil {
|
|
return operation.MarkUncertain(errors.Join(
|
|
mutationLibvirtError("delete managed VM cleanup volume "+name, deleteErr), inspectErr,
|
|
))
|
|
}
|
|
}
|
|
}
|
|
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if seed != nil {
|
|
path, pathErr := seed.GetPath()
|
|
if pathErr != nil {
|
|
_ = seed.Free()
|
|
return translateLibvirtError("resolve managed NoCloud cleanup volume", pathErr)
|
|
}
|
|
referenced, referenceErr := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
|
|
)
|
|
if referenceErr != nil {
|
|
_ = seed.Free()
|
|
return referenceErr
|
|
}
|
|
if referenced {
|
|
_ = seed.Free()
|
|
return NewError(ErrConflict, "managed_disk_shared", "managed NoCloud cleanup volume is referenced by a domain", nil)
|
|
}
|
|
deleteErr := seed.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
_ = seed.Free()
|
|
if deleteErr != nil {
|
|
observed, inspectErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
|
|
if observed != nil {
|
|
_ = observed.Free()
|
|
}
|
|
if inspectErr != nil || observed != nil {
|
|
return operation.MarkUncertain(errors.Join(
|
|
mutationLibvirtError("delete NoCloud cleanup volume", deleteErr), inspectErr,
|
|
))
|
|
}
|
|
}
|
|
}
|
|
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if filter != nil {
|
|
document, xmlErr := filter.GetXMLDesc(0)
|
|
if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) {
|
|
_ = filter.Free()
|
|
return NewError(ErrConflict, "managed_filter_invalid",
|
|
"network filter cannot be confirmed as panel managed", xmlErr)
|
|
}
|
|
referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), "")
|
|
if referenceErr != nil {
|
|
_ = filter.Free()
|
|
return referenceErr
|
|
}
|
|
if referenced {
|
|
_ = filter.Free()
|
|
return NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by a domain", nil)
|
|
}
|
|
undefineErr := filter.Undefine()
|
|
_ = filter.Free()
|
|
if undefineErr != nil {
|
|
observed, inspectErr := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
|
|
if observed != nil {
|
|
_ = observed.Free()
|
|
}
|
|
if inspectErr != nil || observed != nil {
|
|
return operation.MarkUncertain(errors.Join(
|
|
mutationLibvirtError("undefine VM cleanup network filter", undefineErr), inspectErr,
|
|
))
|
|
}
|
|
}
|
|
}
|
|
for _, name := range managedNVRAMArtifactNames(uuid) {
|
|
path := filepath.Join(l.local.NVRAMDirectory, name)
|
|
referenced, referenceErr := l.anyDomainUsesPath(path)
|
|
if referenceErr != nil {
|
|
return referenceErr
|
|
}
|
|
if referenced {
|
|
return NewError(ErrConflict, "managed_nvram_shared",
|
|
"managed VM firmware state is referenced by another domain", nil)
|
|
}
|
|
}
|
|
if err := removeManagedNVRAMArtifacts(l.local.NVRAMDirectory, uuid); err != nil {
|
|
return NewError(ErrUnavailable, "nvram_cleanup_pending", "managed VM firmware cleanup is incomplete", err)
|
|
}
|
|
if err := l.traffic.remove(uuid); err != nil {
|
|
return NewError(ErrUnavailable, "traffic_state_unavailable", "deleted VM traffic state could not be removed", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) deleteArtifactsAbsent(uuid string) (bool, error) {
|
|
if !uuidPattern.MatchString(uuid) {
|
|
return false, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
|
|
}
|
|
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return false, translateLibvirtError("lookup managed storage pool", err)
|
|
}
|
|
defer pool.Free()
|
|
rootAbsent, err := l.dirPoolVolumeAbsent(pool, rootVolumeName(uuid))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !rootAbsent {
|
|
return false, nil
|
|
}
|
|
inventory, err := pool.ListStorageVolumes()
|
|
if err != nil {
|
|
return false, translateLibvirtError("list managed volumes for reconciliation", err)
|
|
}
|
|
owned, err := collectVMManagedVolumeNames(uuid, nil, inventory)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if len(owned) != 0 {
|
|
return false, nil
|
|
}
|
|
seedPool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleSeed))
|
|
if err != nil {
|
|
return false, translateLibvirtError("lookup seed storage pool", err)
|
|
}
|
|
defer seedPool.Free()
|
|
seedAbsent, err := l.dirPoolVolumeAbsent(seedPool, seedVolumeName(uuid))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !seedAbsent {
|
|
return false, nil
|
|
}
|
|
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if filter != nil {
|
|
_ = filter.Free()
|
|
return false, nil
|
|
}
|
|
nvramAbsent, err := managedNVRAMArtifactAbsent(l.local.NVRAMDirectory, uuid)
|
|
if err != nil {
|
|
return false, NewError(ErrUnavailable, "nvram_state_unavailable",
|
|
"managed VM firmware state could not be inspected", err)
|
|
}
|
|
if !nvramAbsent {
|
|
return false, nil
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func managedNVRAMArtifactAbsent(directory, uuid string) (bool, error) {
|
|
candidates := managedNVRAMArtifactNames(uuid)
|
|
for _, name := range candidates {
|
|
if _, err := os.Lstat(filepath.Join(directory, name)); err == nil {
|
|
return false, nil
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
return false, err
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func managedNVRAMArtifactNames(uuid string) []string {
|
|
normalized := strings.ToLower(uuid)
|
|
candidates := []string{normalized + "_VARS.fd"}
|
|
if uuid != normalized {
|
|
// CreateVM historically accepted an uppercase UUID and used its exact
|
|
// spelling for the NVRAM filename. Check both spellings so absence is
|
|
// never claimed while either managed artifact remains.
|
|
candidates = append(candidates, uuid+"_VARS.fd")
|
|
}
|
|
return candidates
|
|
}
|
|
|
|
func removeManagedNVRAMArtifacts(directory, uuid string) error {
|
|
for _, name := range managedNVRAMArtifactNames(uuid) {
|
|
path := filepath.Join(directory, name)
|
|
info, err := os.Lstat(path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return fmt.Errorf("managed NVRAM artifact is not a regular file")
|
|
}
|
|
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) domainByUUID(ctx context.Context, uuid string) (*libvirt.Domain, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if !uuidPattern.MatchString(uuid) {
|
|
return nil, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil)
|
|
}
|
|
domain, err := l.conn.LookupDomainByUUIDString(uuid)
|
|
if err != nil {
|
|
return nil, translateLibvirtError("lookup domain by UUID", err)
|
|
}
|
|
if _, _, err := requireManagedDomain(domain); err != nil {
|
|
domain.Free()
|
|
return nil, err
|
|
}
|
|
return domain, nil
|
|
}
|
|
|
|
func waitForShutoff(ctx context.Context, domain *libvirt.Domain) error {
|
|
ticker := time.NewTicker(500 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
for {
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return translateLibvirtError("wait for shutdown", err)
|
|
}
|
|
if state == libvirt.DOMAIN_SHUTOFF {
|
|
return nil
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func mapDomainState(state libvirt.DomainState) VMStateName {
|
|
switch state {
|
|
case libvirt.DOMAIN_RUNNING:
|
|
return VMStateRunning
|
|
case libvirt.DOMAIN_BLOCKED:
|
|
return VMStateBlocked
|
|
case libvirt.DOMAIN_PAUSED:
|
|
return VMStatePaused
|
|
case libvirt.DOMAIN_SHUTDOWN:
|
|
return VMStateShutdown
|
|
case libvirt.DOMAIN_SHUTOFF:
|
|
return VMStateShutoff
|
|
case libvirt.DOMAIN_CRASHED:
|
|
return VMStateCrashed
|
|
case libvirt.DOMAIN_PMSUSPENDED:
|
|
return VMStateSuspended
|
|
default:
|
|
return VMStateUnknown
|
|
}
|
|
}
|
|
|
|
func translateLibvirtError(action string, err error) error {
|
|
var libvirtErr libvirt.Error
|
|
if errors.As(err, &libvirtErr) {
|
|
switch libvirtErr.Code {
|
|
case libvirt.ERR_NO_DOMAIN:
|
|
return NewError(ErrNotFound, "vm_not_found", "virtual machine was not found", err)
|
|
case libvirt.ERR_NO_DOMAIN_SNAPSHOT:
|
|
return NewError(ErrNotFound, "snapshot_not_found", "virtual machine snapshot was not found", err)
|
|
case libvirt.ERR_NO_STORAGE_VOL:
|
|
return NewError(ErrNotFound, "storage_volume_not_found", "storage volume was not found", err)
|
|
case libvirt.ERR_NO_STORAGE_POOL:
|
|
return NewError(ErrUnavailable, "storage_pool_unavailable", "configured storage pool was not found", err)
|
|
case libvirt.ERR_NO_NWFILTER:
|
|
return NewError(ErrNotFound, "firewall_filter_not_found", "network filter was not found", err)
|
|
case libvirt.ERR_DOM_EXIST, libvirt.ERR_STORAGE_VOL_EXIST, libvirt.ERR_OPERATION_INVALID, libvirt.ERR_OPERATION_DENIED:
|
|
return NewError(ErrConflict, "libvirt_conflict", action+" conflicts with current hypervisor state", err)
|
|
case libvirt.ERR_INVALID_ARG:
|
|
return NewError(ErrInvalid, "invalid_libvirt_request", action+" was rejected as invalid", err)
|
|
case libvirt.ERR_OPERATION_UNSUPPORTED:
|
|
return NewError(ErrUnsupported, "unsupported", action+" is not supported", err)
|
|
case libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_AUTH_FAILED:
|
|
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", err)
|
|
}
|
|
}
|
|
return NewError(ErrUnavailable, "libvirt_error", fmt.Sprintf("%s failed", action), err)
|
|
}
|
|
|
|
func mutationLibvirtError(action string, err error) error {
|
|
var libvirtErr libvirt.Error
|
|
if errors.As(err, &libvirtErr) {
|
|
switch libvirtErr.Code {
|
|
case libvirt.ERR_SYSTEM_ERROR, libvirt.ERR_RPC, libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_OPERATION_TIMEOUT:
|
|
return operation.MarkUncertain(translateLibvirtError(action, err))
|
|
}
|
|
}
|
|
return translateLibvirtError(action, err)
|
|
}
|
|
|
|
func formatLibvirtVersion(version uint32) string {
|
|
return fmt.Sprintf("%d.%d.%d", version/1_000_000, (version/1_000)%1_000, version%1_000)
|
|
}
|