//go:build libvirt
package hypervisor
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"git.luoli.live/root/virt/agent/cloudinit"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/publicguard"
"git.luoli.live/root/virt/agent/publicnetwork"
"git.luoli.live/root/virt/agent/vpcrouting"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
type Libvirt struct {
conn * libvirt . Connect
local LocalConfig
traffic * trafficStore
vpcRouter * vpcrouting . Manager
publicGuard * publicguard . Manager
guestAgentResponseTimeoutSupported bool
}
const deleteGracefulShutdownTimeout = 30 * time . Second
type createVMFailureStage struct {
code string
message string
}
var (
createVMTemplateLookupStage = createVMFailureStage {
code : "template_lookup_failed" , message : "template metadata or volume could not be located" ,
}
createVMTemplateCapacityStage = createVMFailureStage {
code : "template_capacity_failed" , message : "template capacity could not satisfy the requested disk" ,
}
createVMStorageCloneStage = createVMFailureStage {
code : "storage_clone_failed" , message : "system disk could not be cloned from the template" ,
}
createVMSeedCreateStage = createVMFailureStage {
code : "seed_create_failed" , message : "cloud-init seed volume could not be created" ,
}
createVMSeedUploadStage = createVMFailureStage {
code : "seed_upload_failed" , message : "cloud-init seed data could not be uploaded" ,
}
createVMNWFilterDefineStage = createVMFailureStage {
code : "nwfilter_define_failed" , message : "virtual machine network filter could not be defined" ,
}
createVMPublicGuardStage = createVMFailureStage {
code : "public_guard_failed" , message : "virtual machine public network guard could not be applied" ,
}
createVMDomainDefineStage = createVMFailureStage {
code : "domain_define_failed" , message : "virtual machine definition could not be created" ,
}
createVMDomainAutostartStage = createVMFailureStage {
code : "domain_autostart_failed" , message : "virtual machine autostart could not be enabled" ,
}
createVMDomainStartStage = createVMFailureStage {
code : "domain_start_failed" , message : "virtual machine could not be started" ,
}
)
type createVMStageError struct {
stage createVMFailureStage
cause error
}
func ( e * createVMStageError ) Error ( ) string { return e . stage . message }
func ( e * createVMStageError ) Unwrap ( ) error { return e . cause }
func ( e * createVMStageError ) PublicCode ( ) string { return e . stage . code }
func ( e * createVMStageError ) PublicMessage ( ) string { return e . stage . message }
// wrap exposes only a stable stage code and bounded public message. The full
// cause remains in the error chain for local diagnostics and, importantly,
// preserves operation uncertainty and the original hypervisor error kind.
func ( stage createVMFailureStage ) wrap ( cause error ) error {
if cause == nil {
return nil
}
return & createVMStageError { stage : stage , cause : cause }
}
func OpenLibvirt ( uri string , local LocalConfig ) ( Hypervisor , error ) {
if uri != "qemu:///system" {
return nil , NewError ( ErrInvalid , "invalid_libvirt_uri" , "only qemu:///system is allowed" , nil )
}
if err := startLibvirtEventLoop ( ) ; err != nil {
return nil , translateLibvirtError ( "start libvirt event loop" , err )
}
clientVersion , err := libvirt . GetVersion ( )
if err != nil {
return nil , translateLibvirtError ( "get local libvirt version" , err )
}
traffic , err := openTrafficStore ( local . TrafficStatePath )
if err != nil {
return nil , NewError ( ErrUnavailable , "traffic_state_unavailable" , "durable traffic state could not be opened" , err )
}
return openManagedLibvirt ( uri , local , clientVersion , traffic ) , nil
}
func ( l * Libvirt ) Close ( ) error {
_ , err := l . conn . Close ( )
return err
}
func ( l * Libvirt ) Ping ( ctx context . Context ) error {
if err := ctx . Err ( ) ; err != nil {
return err
}
alive , err := l . conn . IsAlive ( )
if err != nil {
return translateLibvirtError ( "ping" , err )
}
if ! alive {
return NewError ( ErrUnavailable , "libvirt_unavailable" , "libvirt connection is not alive" , nil )
}
return nil
}
func ( l * Libvirt ) Capabilities ( ctx context . Context ) ( Capabilities , error ) {
if err := l . Ping ( ctx ) ; err != nil {
return Capabilities { } , err
}
libVersion , err := l . conn . GetLibVersion ( )
if err != nil {
return Capabilities { } , translateLibvirtError ( "get libvirt version" , err )
}
qemuVersion , err := l . conn . GetVersion ( )
if err != nil {
return Capabilities { } , translateLibvirtError ( "get QEMU version" , err )
}
if err := ctx . Err ( ) ; err != nil {
return Capabilities { } , err
}
features , checks , networkSelectors := l . probeRuntimeCapabilities ( ctx )
vpcCapability , vpcChecks := probeVPCL3Capability ( l . vpcRouter )
for name , check := range vpcChecks {
checks [ name ] = check
}
vpcFilterCheck := l . probeVPCCleanTrafficFilter ( )
checks [ "vpc_clean_traffic_filter" ] = vpcFilterCheck
features . VPCL3 = applyVPCCleanTrafficGate ( vpcCapability , vpcFilterCheck )
peerCapability , peerCheck := l . peerMigrationCapability ( features . Backups )
checks [ "peer_migration" ] = peerCheck
features . PeerMigration = peerCapability != nil
powerActions := [ ] PowerAction ( nil )
if features . Power {
powerActions = [ ] PowerAction { PowerStart , PowerShutdown , PowerReboot , PowerReset , PowerDestroy }
}
firmware := [ ] string ( nil )
if features . UEFI {
firmware = [ ] string { "uefi" }
}
publicNetwork := publicNetworkCapability ( checks )
return Capabilities {
Backend : "libvirt" , LibvirtVersion : formatLibvirtVersion ( libVersion ) ,
QEMUVersion : formatLibvirtVersion ( qemuVersion ) ,
Features : features , PowerActions : powerActions , Firmware : firmware ,
NetworkSelectors : networkSelectors , Checks : checks ,
PeerMigration : peerCapability ,
VPCL3 : vpcCapability ,
PublicNetwork : publicNetwork ,
PortableRestore : portableRestoreCapability ( checks , publicNetwork ) ,
} , nil
}
func ( l * Libvirt ) ListVMs ( ctx context . Context ) ( [ ] VMMetadata , error ) {
if err := ctx . Err ( ) ; err != nil {
return nil , err
}
domains , err := l . conn . ListAllDomains ( 0 )
if err != nil {
return nil , translateLibvirtError ( "list domains" , err )
}
result := make ( [ ] VMMetadata , 0 , len ( domains ) )
for index := range domains {
domain := & domains [ index ]
document , xmlErr := domain . GetXMLDesc ( libvirt . DOMAIN_XML_INACTIVE )
_ = domain . Free ( )
if xmlErr != nil {
return nil , translateLibvirtError ( "read domain XML" , xmlErr )
}
_ , _ , managed , inspectErr := inspectManagedDomain ( document )
if inspectErr != nil {
return nil , inspectErr
}
if ! managed {
continue
}
metadata , parseErr := metadataFromXML ( document )
if parseErr != nil {
return nil , parseErr
}
result = append ( result , metadata )
}
return result , nil
}
func ( l * Libvirt ) GetVMState ( ctx context . Context , uuid string ) ( VMState , error ) {
domain , err := l . domainByUUID ( ctx , uuid )
if err != nil {
return VMState { } , err
}
defer domain . Free ( )
state , reason , err := domain . GetState ( )
if err != nil {
return VMState { } , translateLibvirtError ( "get domain state" , err )
}
name , _ := domain . GetName ( )
return VMState { UUID : uuid , Name : name , State : mapDomainState ( state ) , Reason : reason } , nil
}
func ( l * Libvirt ) GetVMMetadata ( ctx context . Context , uuid string ) ( VMMetadata , error ) {
domain , err := l . domainByUUID ( ctx , uuid )
if err != nil {
return VMMetadata { } , err
}
defer domain . Free ( )
document , err := domain . GetXMLDesc ( libvirt . DOMAIN_XML_INACTIVE )
if err != nil {
return VMMetadata { } , translateLibvirtError ( "get domain XML" , err )
}
return metadataFromXML ( document )
}
func ( l * Libvirt ) CreateVM ( ctx context . Context , request CreateVMRequest ) ( _ VMMetadata , resultErr error ) {
if err := ctx . Err ( ) ; err != nil {
return VMMetadata { } , err
}
if err := ValidateCreateVM ( request ) ; err != nil {
return VMMetadata { } , err
}
attachment , err := l . resolvePublicNetwork ( ctx , request )
if err != nil {
return VMMetadata { } , err
}
mac := vmMACAddress ( request . UUID )
conflict , err := l . networkIdentityConflict ( ctx , request . UUID , mac , request . PublicNetwork )
if err != nil {
return VMMetadata { } , err
}
if conflict {
return VMMetadata { } , NewError ( ErrConflict , "network_identity_conflict" , "VM network identity is already in use" , nil )
}
attachment . MAC = mac
templateRecord , err := l . loadImageTemplate ( ctx , request . TemplateID )
if err != nil {
return VMMetadata { } , createVMTemplateLookupStage . wrap ( err )
}
if templateRecord . GenerationToken != request . TemplateGenerationToken {
return VMMetadata { } , NewError ( ErrConflict , "template_generation_changed" , "template generation token no longer matches" , nil )
}
seed , err := cloudinit . BuildISO ( cloudinit . Request {
UUID : request . UUID , Name : request . Name , Hostname : request . Hostname ,
GuestSSHPublicKey : request . GuestSSHPublicKey , RootPassword : request . RootPassword ,
StartupScript : request . StartupScript ,
Debian12NetworkFix : isDebian12Template ( templateRecord ) ,
Network : cloudinit . Network {
MAC : mac , Public : request . PublicNetwork , InterfaceKey : "public0" ,
} ,
} )
if err != nil {
return VMMetadata { } , NewError ( ErrInvalid , "invalid_cloud_init" , "cloud-init request is invalid" , err )
}
root , err := l . createRootVolumeFromTemplate ( ctx , request )
if err != nil {
return VMMetadata { } , err
}
return l . createVMWithPreparedRoot ( ctx , request , attachment , seed , root )
}
// createVMWithPreparedRoot owns root after entry. It is the single create tail
// for template provisioning and backup restore, ensuring both paths generate a
// fresh seed, MAC, public-IP filter, NVRAM, traffic baseline, and domain.
func ( l * Libvirt ) createVMWithPreparedRoot ( ctx context . Context , request CreateVMRequest ,
attachment NetworkAttachment , seed [ ] byte , root * libvirt . StorageVol ) ( _ VMMetadata , resultErr error ) {
return l . createVMWithPreparedRootOptions ( ctx , request , attachment , seed , root ,
preparedVMOptions { Autostart : true , Start : true } )
}
type preparedVMOptions struct {
Autostart bool
Start bool
NetworkSuspended bool
VPCPort * VPCPortMetadata
Snapshots [ ] domainSnapshotMetadataXML
SnapshotVolumes [ ] * libvirt . StorageVol
}
type storageVolumeCleanup interface {
Delete ( libvirt . StorageVolDeleteFlags ) error
Free ( ) error
}
type preparedVolumeOwnership struct {
root storageVolumeCleanup
snapshots [ ] storageVolumeCleanup
rootOwned bool
snapshotsOwned bool
}
func newPreparedVolumeOwnership ( root * libvirt . StorageVol , snapshots [ ] * libvirt . StorageVol ) * preparedVolumeOwnership {
owned := & preparedVolumeOwnership { root : root , rootOwned : root != nil , snapshotsOwned : len ( snapshots ) > 0 }
owned . snapshots = make ( [ ] storageVolumeCleanup , 0 , len ( snapshots ) )
for _ , snapshot := range snapshots {
if snapshot != nil {
owned . snapshots = append ( owned . snapshots , snapshot )
}
}
return owned
}
func ( owned * preparedVolumeOwnership ) preserve ( ) {
if owned == nil {
return
}
owned . rootOwned = false
owned . snapshotsOwned = false
}
func ( owned * preparedVolumeOwnership ) cleanup ( resultErr * error ) {
if owned == nil {
return
}
for _ , volume := range owned . snapshots {
if owned . snapshotsOwned {
if err := volume . Delete ( libvirt . STORAGE_VOL_DELETE_NORMAL ) ; err != nil && resultErr != nil {
* resultErr = operation . MarkUncertain ( errors . Join ( * resultErr ,
mutationLibvirtError ( "roll back restored migration snapshot" , err ) ) )
}
}
_ = volume . Free ( )
}
owned . snapshots = nil
owned . snapshotsOwned = false
if owned . root != nil {
if owned . rootOwned {
if err := owned . root . Delete ( libvirt . STORAGE_VOL_DELETE_NORMAL ) ; err != nil && resultErr != nil {
* resultErr = operation . MarkUncertain ( errors . Join ( * resultErr ,
mutationLibvirtError ( "roll back cloned root volume" , err ) ) )
}
}
_ = owned . root . Free ( )
}
owned . root = nil
owned . rootOwned = false
}
func ( l * Libvirt ) createVMWithPreparedRootOptions ( ctx context . Context ,
request CreateVMRequest , attachment NetworkAttachment , seed [ ] byte ,
root * libvirt . StorageVol , options preparedVMOptions ) ( _ VMMetadata , resultErr error ) {
volumeOwnership := newPreparedVolumeOwnership ( root , options . SnapshotVolumes )
defer volumeOwnership . cleanup ( & resultErr )
if root == nil {
return VMMetadata { } , NewError ( ErrInvalid , "missing_root_volume" , "prepared root volume is required" , nil )
}
var domainVPCPort * VPCPortMetadata
var seedVPCMetadata * domainVPCPortMetadataXML
if options . VPCPort != nil {
value := * options . VPCPort
if err := validateVPCPortMetadata ( value ) ; err != nil {
return VMMetadata { } , err
}
domainVPCPort = & value
metadata := domainVPCPortFromPublic ( value )
seedVPCMetadata = & metadata
}
if _ , err := preservedVPCCloudInitNetwork ( seed , seedVPCMetadata ) ; err != nil {
return VMMetadata { } , err
}
rootPath , err := root . GetPath ( )
if err != nil {
return VMMetadata { } , createVMStorageCloneStage . wrap (
mutationLibvirtError ( "resolve cloned root volume path" , err ) )
}
seedVolume , err := l . createSeedVolume ( ctx , request . UUID , seed )
if err != nil {
return VMMetadata { } , err
}
seedPath , err := seedVolume . GetPath ( )
if err != nil {
cleanupErr := seedVolume . Delete ( libvirt . STORAGE_VOL_DELETE_NORMAL )
_ = seedVolume . Free ( )
if cleanupErr != nil {
return VMMetadata { } , operation . MarkUncertain ( createVMSeedCreateStage . wrap ( errors . Join (
mutationLibvirtError ( "resolve NoCloud volume path" , err ) ,
mutationLibvirtError ( "roll back NoCloud seed volume" , cleanupErr ) ,
) ) )
}
return VMMetadata { } , createVMSeedCreateStage . wrap (
mutationLibvirtError ( "resolve NoCloud volume path" , err ) )
}
seedOwned := true
defer func ( ) {
if seedOwned {
if cleanupErr := seedVolume . Delete ( libvirt . STORAGE_VOL_DELETE_NORMAL ) ; cleanupErr != nil {
resultErr = operation . MarkUncertain ( errors . Join ( resultErr ,
mutationLibvirtError ( "roll back NoCloud seed volume" , cleanupErr ) ) )
}
}
_ = seedVolume . Free ( )
} ( )
filterName := networkFilterName ( request . UUID )
filterDocument , err := BuildPlatformFilterXML ( filterName , request . PublicNetwork , request . PlatformNetworkPolicy )
if err != nil {
return VMMetadata { } , createVMNWFilterDefineStage . wrap ( err )
}
filter , err := l . conn . NWFilterDefineXML ( filterDocument )
if err != nil {
observed , inspectErr := l . lookupOptionalNetworkFilter ( filterName )
artifactExists := observed != nil
if observed != nil {
_ = observed . Free ( )
}
return VMMetadata { } , createVMNWFilterDefineStage . wrap (
deterministicArtifactCreateFailure (
"define VM network filter" , err , inspectErr , artifactExists ,
) )
}
filterOwned := true
defer func ( ) {
if filterOwned {
if cleanupErr := filter . Undefine ( ) ; cleanupErr != nil {
resultErr = operation . MarkUncertain ( errors . Join ( resultErr ,
mutationLibvirtError ( "roll back VM network filter" , cleanupErr ) ) )
}
}
_ = filter . Free ( )
} ( )
attachment . LinkDown = options . NetworkSuspended
networks := [ ] NetworkAttachment { attachment }
if domainVPCPort != nil {
networks = append ( networks , NetworkAttachment {
Bridge : domainVPCPort . Bridge ,
MAC : domainVPCPort . MAC ,
LinkDown : options . NetworkSuspended ,
} )
}
document , err := BuildDomainXML ( DomainDefinition {
UUID : request . UUID , Name : request . DomainName , DisplayName : request . Name , Description : request . Description ,
VCPUs : request . VCPUs , MemoryMiB : request . MemoryMiB , DiskGiB : request . DiskGiB ,
BandwidthMbps : request . BandwidthMbps , Hostname : request . Hostname ,
PublicNetwork : request . PublicNetwork , TemplateID : request . TemplateID ,
RootDiskPath : rootPath , RootVolumeName : rootVolumeName ( request . UUID ) , RootDiskFormat : "qcow2" , SeedDiskPath : seedPath ,
Firmware : Firmware {
Mode : "uefi" , Loader : l . local . FirmwareLoader , NVRAMTemplate : l . local . FirmwareNVRAMTemplate ,
NVRAMPath : filepath . Join ( l . local . NVRAMDirectory , request . UUID + "_VARS.fd" ) , SecureBoot : false ,
} ,
Network : networks , VPCPort : domainVPCPort , FilterName : filterName ,
PlatformNetworkPolicy : request . PlatformNetworkPolicy ,
Snapshots : append ( [ ] domainSnapshotMetadataXML ( nil ) , options . Snapshots ... ) ,
} )
if err != nil {
return VMMetadata { } , createVMDomainDefineStage . wrap ( err )
}
// Serialize the traffic baseline only after the storage, seed, and filter
// mutations have returned. A slow metrics read can no longer make a request
// wait before its first libvirt mutation after Journal dispatch.
unlockTrafficSample := l . traffic . lockSample ( request . UUID )
defer unlockTrafficSample ( )
trafficCreated , err := l . traffic . initialize ( request . UUID , time . Now ( ) . UTC ( ) )
if err != nil {
return VMMetadata { } , NewError ( ErrUnavailable , "traffic_state_unavailable" , "new VM traffic baseline could not be initialized" , err )
}
trafficOwned := trafficCreated
defer func ( ) {
if trafficOwned {
if cleanupErr := l . traffic . remove ( request . UUID ) ; cleanupErr != nil {
resultErr = markTrafficCleanupUncertain ( resultErr , cleanupErr )
}
}
} ( )
if domainVPCPort != nil {
if l . vpcRouter == nil {
return VMMetadata { } , NewError ( ErrUnavailable , "vpc_router_unavailable" , "VPC host router is unavailable" , nil )
}
_ , routeLease , routeErr := l . vpcRouter . AcquireLocalPortLease (
domainVPCPort . VPCKey , domainVPCPort . HostRevision , domainVPCPort . HostDigest ,
domainVPCPort . CIDR , domainVPCPort . Gateway , domainVPCPort . PrivateIP , domainVPCPort . MAC ,
)
if routeErr != nil {
return VMMetadata { } , NewError ( ErrConflict , "vpc_host_projection_not_ready" , "VPC host routing must authorize the exact migration port before the domain is defined" , routeErr )
}
defer routeLease . Release ( )
}
guardBinding , err := publicGuardBinding ( request . UUID , attachment . MAC , request . PublicNetwork )
if err != nil {
return VMMetadata { } , createVMPublicGuardStage . wrap ( err )
}
guardLease , err := l . acquirePublicGuardTransition ( ctx )
if err != nil {
return VMMetadata { } , createVMPublicGuardStage . wrap ( err )
}
defer guardLease . Release ( )
if err := l . reconcilePublicGuardLease ( ctx , guardLease , & guardBinding ) ; err != nil {
return VMMetadata { } , createVMPublicGuardStage . wrap ( err )
}
defer func ( ) {
cleanupCtx , cancelCleanup := storageRollbackContext ( ctx )
cleanupErr := l . reconcilePublicGuardLease ( cleanupCtx , guardLease , nil )
cancelCleanup ( )
if cleanupErr != nil {
resultErr = operation . MarkUncertain ( errors . Join ( resultErr ,
createVMPublicGuardStage . wrap ( cleanupErr ) ) )
}
} ( )
preserveDefinedDomainArtifacts := func ( ) {
volumeOwnership . preserve ( )
seedOwned = false
filterOwned = false
trafficOwned = false
}
domain , err := l . conn . DomainDefineXML ( document )
if err != nil {
observed , inspectErr := l . conn . LookupDomainByUUIDString ( request . UUID )
if observed != nil {
_ = observed . Free ( )
}
if domainMayExistAfterDefine ( inspectErr ) {
// DomainDefineXML may have committed before its response failed. Do
// not let deferred cleanup remove disks, firewall, or accounting
// state that the possibly-defined domain owns.
preserveDefinedDomainArtifacts ( )
return VMMetadata { } , operation . MarkUncertain ( createVMDomainDefineStage . wrap ( errors . Join (
mutationLibvirtError ( "define domain" , err ) ,
translateOptionalLibvirtError ( "inspect domain after define failure" , inspectErr ) ,
) ) )
}
return VMMetadata { } , createVMDomainDefineStage . wrap ( mutationLibvirtError ( "define domain" , err ) )
}
defer domain . Free ( )
if err := domain . SetAutostart ( options . Autostart ) ; err != nil {
rollbackErr := domain . UndefineFlags ( libvirt . DOMAIN_UNDEFINE_NVRAM )
absent , inspectErr := l . deleteDomainAbsent ( ctx , request . UUID )
if inspectErr != nil || ! absent {
preserveDefinedDomainArtifacts ( )
return VMMetadata { } , operation . MarkUncertain ( createVMDomainAutostartStage . wrap ( errors . Join (
mutationLibvirtError ( "enable domain autostart" , err ) ,
optionalMutationLibvirtError ( "roll back domain definition" , rollbackErr ) ,
inspectErr ,
) ) )
}
if rollbackErr != nil {
return VMMetadata { } , operation . MarkUncertain ( createVMDomainAutostartStage . wrap ( errors . Join (
mutationLibvirtError ( "enable domain autostart" , err ) ,
mutationLibvirtError ( "roll back domain definition" , rollbackErr ) ,
) ) )
}
return VMMetadata { } , createVMDomainAutostartStage . wrap (
mutationLibvirtError ( "enable domain autostart" , err ) )
}
preserveDefinedDomainArtifacts ( )
if ! options . Start {
if err := l . verifyStagedMigrationDomain (
domain , request , filterName , attachment . MAC ,
options . NetworkSuspended , options . Snapshots , options . VPCPort ,
) ; err != nil {
return VMMetadata { } , operation . MarkUncertain ( err )
}
metadata , err := metadataFromXML ( document )
if err != nil {
return VMMetadata { } , operation . MarkUncertain ( err )
}
metadata . Status = VMStateShutoff
metadata . DiskGiB = request . DiskGiB
metadata . BandwidthMbps = request . BandwidthMbps
metadata . Hostname = request . Hostname
metadata . PublicNetwork = request . PublicNetwork
metadata . TemplateID = request . TemplateID
return metadata , nil
}
// From this point onward Domain.Create may have taken effect even if its
// response is lost. Retain the zero baseline so reconciliation and the
// first successful sample account for all traffic since initial boot.
if err := l . startManagedDomainWithLease ( ctx , guardLease , domain , "start newly defined domain" ) ; err != nil {
return VMMetadata { } , operation . MarkUncertain ( createVMDomainStartStage . wrap (
err ) )
}
if err := l . verifyCreatedDomain ( domain , request , filterName , attachment . MAC ) ; err != nil {
return VMMetadata { } , operation . MarkUncertain ( err )
}
metadata , err := metadataFromXML ( document )
if err != nil {
return VMMetadata { } , operation . MarkUncertain ( err )
}
metadata . Status = VMStateRunning
metadata . DiskGiB = request . DiskGiB
metadata . BandwidthMbps = request . BandwidthMbps
metadata . Hostname = request . Hostname
metadata . PublicNetwork = request . PublicNetwork
metadata . TemplateID = request . TemplateID
return metadata , nil
}
func markTrafficCleanupUncertain ( prior , cleanupErr error ) error {
return operation . MarkUncertain ( errors . Join ( prior , NewError (
ErrUnavailable , "traffic_state_unavailable" ,
"unused VM traffic baseline could not be removed" , cleanupErr ,
) ) )
}
func isDebian12Template ( template ImageTemplate ) bool {
return strings . EqualFold ( strings . TrimSpace ( template . OSFamily ) , "debian" ) && strings . TrimSpace ( template . Version ) == "12"
}
func ( l * Libvirt ) resolvePublicNetwork ( ctx context . Context ,
request CreateVMRequest ) ( NetworkAttachment , error ) {
if err := ValidatePublicNetworkIntent ( request . PublicNetwork ) ; err != nil {
return NetworkAttachment { } , err
}
source , err := l . ResolveNetworkSelector ( ctx , request . PublicNetwork . NetworkSelector )
if err != nil {
return NetworkAttachment { } , err
}
attachment := NetworkAttachment { }
switch source . Type {
case "bridge" :
attachment . Bridge = source . Source
case "network" :
attachment . Network = source . Source
default :
return NetworkAttachment { } , NewError ( ErrUnavailable , "invalid_host_network_config" , "host network source type is invalid" , nil )
}
return attachment , nil
}
func ( l * Libvirt ) createRootVolumeFromTemplate ( ctx context . Context , request CreateVMRequest ) ( * libvirt . StorageVol , error ) {
templateRecord , err := l . loadImageTemplate ( ctx , request . TemplateID )
if err != nil {
return nil , createVMTemplateLookupStage . wrap ( err )
}
if templateRecord . GenerationToken != request . TemplateGenerationToken {
return nil , NewError ( ErrConflict , "template_generation_changed" , "template generation token no longer matches" , nil )
}
templatePool , err := l . conn . LookupStoragePoolByName ( l . storagePoolName ( StorageRoleTemplate ) )
if err != nil {
return nil , createVMTemplateLookupStage . wrap (
translateLibvirtError ( "lookup template storage pool" , err ) )
}
defer templatePool . Free ( )
template , err := templatePool . LookupStorageVolByName ( templateVolumeName ( request . TemplateID ) )
if err != nil {
return nil , createVMTemplateLookupStage . wrap (
translateLibvirtError ( "lookup template volume" , err ) )
}
defer template . Free ( )
templateInfo , err := template . GetInfo ( )
if err != nil {
return nil , createVMTemplateCapacityStage . wrap (
translateLibvirtError ( "read template volume size" , err ) )
}
capacity := request . DiskGiB << 30
if capacity < templateInfo . Capacity {
return nil , createVMTemplateCapacityStage . wrap ( NewError (
ErrInvalid , "disk_smaller_than_template" , "disk_gib is smaller than the selected template" , nil ) )
}
managedPool , err := l . conn . LookupStoragePoolByName ( l . storagePoolName ( StorageRoleManaged ) )
if err != nil {
return nil , createVMStorageCloneStage . wrap (
translateLibvirtError ( "lookup managed storage pool" , err ) )
}
defer managedPool . Free ( )
definition := libvirtxml . StorageVolume {
Name : rootVolumeName ( request . UUID ) ,
Capacity : & libvirtxml . StorageVolumeSize { Value : capacity , Unit : "bytes" } ,
Target : & libvirtxml . StorageVolumeTarget {
Format : & libvirtxml . StorageVolumeTargetFormat { Type : "qcow2" } ,
} ,
}
document , err := definition . Marshal ( )
if err != nil {
return nil , createVMStorageCloneStage . wrap ( fmt . Errorf ( "marshal root volume XML: %w" , err ) )
}
volume , err := managedPool . StorageVolCreateXMLFrom ( document , template , 0 )
if err != nil {
artifactAbsent , inspectErr := l . dirPoolVolumeAbsent (
managedPool , rootVolumeName ( request . UUID ) ,
)
return nil , createVMStorageCloneStage . wrap ( deterministicArtifactCreateFailure (
"create system disk from template" , err , inspectErr , ! artifactAbsent ,
) )
}
// A qcow2 clone in a dir pool is implemented with qemu-img convert. The
// clone inherits the source image's virtual size even when the destination
// XML asks for a larger capacity. Make the requested capacity a mandatory
// part of this unpublished construction step: no domain can reference the
// volume and the caller cannot observe it until resize and verification have
// both succeeded.
if err := establishNewRootVolumeCapacity ( volume , capacity ) ; err != nil {
return nil , createVMStorageCloneStage . wrap ( l . rollbackUnpublishedVolume (
managedPool , volume , rootVolumeName ( request . UUID ) , err ,
"roll back incomplete system disk" ,
) )
}
return volume , nil
}
type newRootVolumeCapacityTarget interface {
qcow2VolumeInspector
Resize ( uint64 , libvirt . StorageVolResizeFlags ) error
}
func establishNewRootVolumeCapacity ( volume newRootVolumeCapacityTarget , capacity uint64 ) error {
if capacity == 0 {
return NewError ( ErrInvalid , "invalid_disk_capacity" , "system disk capacity must be positive" , nil )
}
// Do not first inspect and conditionally repair a clone. Resize is an
// unconditional part of constructing every new root volume and flags=0
// keeps the additional qcow2 address space sparse.
if err := volume . Resize ( capacity , 0 ) ; err != nil {
return translateLibvirtError ( "set new system disk capacity" , err )
}
_ , err := validateQCOW2Volume ( volume , capacity )
return err
}
func ( l * Libvirt ) rollbackUnpublishedVolume ( pool * libvirt . StoragePool ,
volume * libvirt . StorageVol , volumeName string , cause error , action string ) error {
cleanupErr := volume . Delete ( libvirt . STORAGE_VOL_DELETE_NORMAL )
_ = volume . Free ( )
absent , inspectErr := l . dirPoolVolumeAbsent ( pool , volumeName )
return classifyUnpublishedVolumeRollback ( cause , cleanupErr , inspectErr , absent , action )
}
func classifyUnpublishedVolumeRollback ( cause , cleanupErr , inspectErr error ,
absent bool , action string ) error {
if inspectErr == nil && absent {
// Independent libvirt inventory and filesystem evidence proves the
// temporary volume is gone, even if Delete itself returned an error.
return errors . Join ( cause , translateOptionalLibvirtError ( action , cleanupErr ) )
}
if inspectErr == nil {
inspectErr = NewError ( ErrConflict , "storage_cleanup_incomplete" ,
"incomplete storage volume remains after rollback" , nil )
}
return operation . MarkUncertain ( errors . Join (
cause ,
optionalMutationLibvirtError ( action , cleanupErr ) ,
inspectErr ,
) )
}
func ( l * Libvirt ) createSeedVolume ( ctx context . Context , uuid string , contents [ ] byte ) ( * libvirt . StorageVol , error ) {
pool , err := l . conn . LookupStoragePoolByName ( l . storagePoolName ( StorageRoleSeed ) )
if err != nil {
return nil , createVMSeedCreateStage . wrap (
translateLibvirtError ( "lookup seed storage pool" , err ) )
}
defer pool . Free ( )
definition := libvirtxml . StorageVolume {
Name : seedVolumeName ( uuid ) ,
Capacity : & libvirtxml . StorageVolumeSize { Value : uint64 ( len ( contents ) ) , Unit : "bytes" } ,
Target : & libvirtxml . StorageVolumeTarget { Format : & libvirtxml . StorageVolumeTargetFormat { Type : "raw" } } ,
}
document , err := definition . Marshal ( )
if err != nil {
return nil , createVMSeedCreateStage . wrap ( fmt . Errorf ( "marshal seed volume XML: %w" , err ) )
}
volume , err := pool . StorageVolCreateXML ( document , 0 )
if err != nil {
artifactAbsent , inspectErr := l . dirPoolVolumeAbsent ( pool , seedVolumeName ( uuid ) )
return nil , createVMSeedCreateStage . wrap ( deterministicArtifactCreateFailure (
"create NoCloud volume" , err , inspectErr , ! artifactAbsent ,
) )
}
if err := l . uploadVolume ( ctx , volume , contents ) ; err != nil {
uploadErr := createVMSeedUploadStage . wrap ( err )
if operation . IsUncertain ( err ) {
_ = volume . Free ( )
return nil , operation . MarkUncertain ( uploadErr )
}
cleanupErr := volume . Delete ( libvirt . STORAGE_VOL_DELETE_NORMAL )
_ = volume . Free ( )
if cleanupErr != nil {
return nil , operation . MarkUncertain ( errors . Join ( uploadErr ,
mutationLibvirtError ( "roll back NoCloud seed volume" , cleanupErr ) ) )
}
return nil , uploadErr
}
return volume , nil
}
func ( l * Libvirt ) uploadVolume ( ctx context . Context , volume * libvirt . StorageVol , contents [ ] byte ) error {
if err := ctx . Err ( ) ; err != nil {
return err
}
stream , err := l . conn . NewStream ( libvirt . STREAM_NONBLOCK )
if err != nil {
return translateLibvirtError ( "create storage upload stream" , err )
}
transfer := & storageStreamTransfer { stream : stream }
if err := volume . Upload ( stream , 0 , uint64 ( len ( contents ) ) , 0 ) ; err != nil {
return mutationLibvirtError ( "open storage upload" , transfer . abort ( err ) )
}
if _ , err := transfer . send ( ctx , contents ) ; err != nil {
return mutationLibvirtError ( "upload storage volume" , transfer . abort ( err ) )
}
if err := transfer . finish ( ctx ) ; err != nil {
return mutationLibvirtError ( "finish storage upload" , err )
}
return nil
}
func ( l * Libvirt ) networkIdentityConflict ( ctx context . Context , requestedUUID , mac string , public publicnetwork . Intent ) ( bool , error ) {
if err := ValidatePublicNetworkIntent ( public ) ; err != nil {
return false , err
}
domains , err := l . conn . ListAllDomains ( 0 )
if err != nil {
return false , translateLibvirtError ( "list domains for network identity validation" , err )
}
for index := range domains {
if err := ctx . Err ( ) ; err != nil {
return false , err
}
domain := & domains [ index ]
document , xmlErr := domain . GetXMLDesc ( libvirt . DOMAIN_XML_INACTIVE )
_ = domain . Free ( )
if xmlErr != nil {
return false , translateLibvirtError ( "read domain network identity" , xmlErr )
}
definition , metadata , managed , parseErr := inspectManagedDomain ( document )
if parseErr != nil {
return false , parseErr
}
if ! managed || strings . EqualFold ( definition . UUID , requestedUUID ) {
continue
}
if publicNetworkAddressConflict ( metadata . PublicNetwork . publicMust ( ) , public ) {
return true , nil
}
if definition . Devices != nil {
for _ , iface := range definition . Devices . Interfaces {
if iface . MAC != nil && strings . EqualFold ( iface . MAC . Address , mac ) {
return true , nil
}
}
}
}
return false , nil
}
func ( l * Libvirt ) verifyCreatedDomain ( domain * libvirt . Domain , request CreateVMRequest , filterName , mac string ) error {
autostart , err := domain . GetAutostart ( )
if err != nil || ! autostart {
return mutationLibvirtError ( "verify domain autostart" , err )
}
active , err := domain . IsActive ( )
if err != nil || ! active {
return mutationLibvirtError ( "verify newly created domain state" , err )
}
document , err := domain . GetXMLDesc ( 0 )
if err != nil {
return translateLibvirtError ( "verify live domain XML" , err )
}
definition , metadata , err := parseManagedDomain ( document )
if err != nil {
return err
}
if ! strings . EqualFold ( definition . UUID , request . UUID ) || definition . Name != request . DomainName || metadata . DisplayName != request . Name || ! publicNetworkEqual ( metadata . PublicNetwork . publicMust ( ) , request . PublicNetwork ) || definition . Devices == nil || len ( definition . Devices . Interfaces ) != 1 {
return NewError ( ErrConflict , "create_observation_mismatch" , "new domain identity or network metadata did not match the requested intent" , nil )
}
var storedPolicy PlatformNetworkPolicySnapshot
if err := json . Unmarshal ( [ ] byte ( metadata . PlatformPolicyJSON ) , & storedPolicy ) ; err != nil || storedPolicy . Digest != request . PlatformNetworkPolicy . Digest {
return NewError ( ErrConflict , "firewall_observation_mismatch" , "new domain platform policy metadata did not match the requested snapshot" , err )
}
iface := definition . Devices . Interfaces [ 0 ]
if iface . MAC == nil || ! strings . EqualFold ( iface . MAC . Address , mac ) ||
! managedInterfaceFilterIPMatches ( & iface , filterName , request . PublicNetwork ) {
return NewError ( ErrConflict , "firewall_observation_mismatch" , "new domain did not attach the expected network filter" , nil )
}
return l . verifyManagedFilter ( filterName , storedPolicy )
}
func ( l * Libvirt ) verifyStagedMigrationDomain ( domain * libvirt . Domain ,
request CreateVMRequest , filterName , mac string , networkSuspended bool ,
snapshots [ ] domainSnapshotMetadataXML , vpcPort * VPCPortMetadata ) error {
autostart , err := domain . GetAutostart ( )
if err != nil || autostart {
return mutationLibvirtError ( "verify staged domain autostart" , err )
}
active , err := domain . IsActive ( )
if err != nil || active {
return mutationLibvirtError ( "verify staged domain power state" , err )
}
document , err := domain . GetXMLDesc ( 0 )
if err != nil {
return translateLibvirtError ( "verify staged domain XML" , err )
}
if err := validateStagedMigrationDomainDocument (
document , request , filterName , mac , networkSuspended , snapshots , vpcPort ,
) ; err != nil {
return err
}
return l . verifyManagedFilter ( filterName , request . PlatformNetworkPolicy )
}
func validateStagedMigrationDomainDocument ( document string , request CreateVMRequest ,
filterName , mac string , networkSuspended bool , snapshots [ ] domainSnapshotMetadataXML ,
vpcPort * VPCPortMetadata ) error {
definition , metadata , err := parseManagedDomain ( document )
if err != nil {
return err
}
expectedInterfaces := 1
if vpcPort != nil {
expectedInterfaces = 2
}
if ! strings . EqualFold ( definition . UUID , request . UUID ) ||
definition . Name != request . DomainName ||
metadata . DisplayName != request . Name ||
! publicNetworkEqual ( metadata . PublicNetwork . publicMust ( ) , request . PublicNetwork ) ||
definition . Devices == nil || len ( definition . Devices . Interfaces ) != expectedInterfaces ||
! managedNetworkLayoutMatches ( definition , metadata ) ||
len ( metadata . Snapshots ) != len ( snapshots ) {
return NewError ( ErrConflict , "migration_stage_observation_mismatch" ,
"staged migration domain identity or snapshot catalog did not match" , nil )
}
for index := range snapshots {
if metadata . Snapshots [ index ] != snapshots [ index ] {
return NewError ( ErrConflict , "migration_stage_observation_mismatch" ,
"staged migration snapshot catalog did not match" , nil )
}
}
if vpcPort == nil && metadata . VPCPort != nil || vpcPort != nil &&
( metadata . VPCPort == nil || * metadata . VPCPort != domainVPCPortFromPublic ( * vpcPort ) ) {
return NewError ( ErrConflict , "migration_stage_network_mismatch" ,
"staged migration VPC identity did not match" , nil )
}
publicIndex , found := managedPublicInterfaceIndex ( definition , filterName , request . PublicNetwork )
if ! found {
return NewError ( ErrConflict , "migration_stage_network_mismatch" ,
"staged migration public interface did not match" , nil )
}
iface := definition . Devices . Interfaces [ publicIndex ]
if iface . MAC == nil ||
! strings . EqualFold ( iface . MAC . Address , mac ) ||
! managedInterfaceFilterIPMatches ( & iface , filterName , request . PublicNetwork ) {
return NewError ( ErrConflict , "migration_stage_network_mismatch" ,
"staged migration network state did not match" , nil )
}
for index := range definition . Devices . Interfaces {
linkDown := definition . Devices . Interfaces [ index ] . Link != nil &&
definition . Devices . Interfaces [ index ] . Link . State == "down"
if linkDown != networkSuspended {
return NewError ( ErrConflict , "migration_stage_network_mismatch" ,
"staged migration interface link state did not match" , nil )
}
}
return nil
}
func ( l * Libvirt ) verifyManagedFilter ( filterName string , policy PlatformNetworkPolicySnapshot ) error {
filter , err := l . conn . LookupNWFilterByName ( filterName )
if err != nil {
return translateLibvirtError ( "verify VM network filter" , err )
}
defer filter . Free ( )
filterXML , err := filter . GetXMLDesc ( 0 )
if err != nil {
return translateLibvirtError ( "verify VM network policy" , err )
}
if ! managedFilterSemanticsValid ( filterXML , filterName , policy ) {
return NewError ( ErrConflict , "firewall_observation_mismatch" , "new VM filter does not match the project-owned anti-spoof and platform policy contract" , nil )
}
return nil
}
func ( l * Libvirt ) DeleteVM ( ctx context . Context , uuid string , request DeleteVMRequest ) ( _ DeleteVMResult , resultErr error ) {
mutationStarted := false
defer func ( ) {
if ! mutationStarted {
resultErr = operation . MarkNotApplied ( resultErr )
}
} ( )
if ! request . RemoveManagedDisks {
return DeleteVMResult { } , NewError ( ErrInvalid , "managed_disk_cleanup_required" , "remove_managed_disks must be true" , nil )
}
domain , err := l . domainByUUID ( ctx , uuid )
if result , absent , cleanupErr := completeDeleteForMissingDomain ( err , func ( ) ( DeleteVMResult , error ) {
mutationStarted = true
return l . completeDeletedVM ( ctx , uuid , false )
} ) ; absent {
return result , cleanupErr
}
if err != nil {
return DeleteVMResult { } , err
}
defer domain . Free ( )
_ , managed , err := requireManagedDomain ( domain )
if err != nil {
return DeleteVMResult { } , err
}
if err := l . validateDeleteVMArtifacts ( ctx , uuid , managed ) ; err != nil {
return DeleteVMResult { } , err
}
if _ , err := l . inspectDeleteDomainEvidence ( ctx , uuid , domain ) ; err != nil {
return DeleteVMResult { } , err
}
if err := ensureNoCleanupBlockingJobs ( domain ) ; err != nil {
return DeleteVMResult { } , err
}
completionCtx := context . WithoutCancel ( ctx )
mutationStarted = true
if err := driveGracefulDeleteDomainToAbsent ( completionCtx , domain , deleteGracefulShutdownTimeout , func ( completionCtx context . Context ) error {
if _ , err := l . inspectDeleteDomainEvidence ( completionCtx , uuid , domain ) ; err != nil {
return err
}
return ensureNoCleanupBlockingJobs ( domain )
} , func ( completionCtx context . Context ) ( bool , error ) {
return l . deleteDomainAbsent ( completionCtx , uuid )
} ) ; err != nil {
return DeleteVMResult { } , err
}
return l . completeDeletedVM ( completionCtx , uuid , false )
}
func ( l * Libvirt ) ForceDeleteVM ( ctx context . Context , uuid string , request ForceDeleteVMRequest ) ( _ DeleteVMResult , resultErr error ) {
mutationStarted := false
defer func ( ) {
if ! mutationStarted {
resultErr = operation . MarkNotApplied ( resultErr )
}
} ( )
if ! request . RemoveManagedDisks {
return DeleteVMResult { } , NewError ( ErrInvalid , "managed_disk_cleanup_required" , "remove_managed_disks must be true" , nil )
}
if ! request . AckDataLoss {
return DeleteVMResult { } , NewError ( ErrInvalid , "data_loss_ack_required" , "ack_data_loss must be true" , nil )
}
if request . PriorOperationID == "" {
return DeleteVMResult { } , NewError ( ErrInvalid , "invalid_prior_operation_id" , "prior_operation_id is required" , nil )
}
domain , err := l . domainByUUID ( ctx , uuid )
if result , absent , cleanupErr := completeDeleteForMissingDomain ( err , func ( ) ( DeleteVMResult , error ) {
mutationStarted = true
return l . completeDeletedVM ( ctx , uuid , true )
} ) ; absent {
return result , cleanupErr
}
if err != nil {
return DeleteVMResult { } , err
}
defer domain . Free ( )
_ , managed , err := requireManagedDomain ( domain )
if err != nil {
return DeleteVMResult { } , err
}
if err := l . validateDeleteVMArtifacts ( ctx , uuid , managed ) ; err != nil {
return DeleteVMResult { } , err
}
if _ , err := l . inspectDeleteDomainEvidence ( ctx , uuid , domain ) ; err != nil {
return DeleteVMResult { } , err
}
if err := ensureNoCleanupBlockingJobs ( domain ) ; err != nil {
return DeleteVMResult { } , err
}
mutationStarted = true
if err := driveForceDeleteDomainToAbsent ( domain , func ( ) error {
if _ , err := l . inspectDeleteDomainEvidence ( ctx , uuid , domain ) ; err != nil {
return err
}
return ensureNoCleanupBlockingJobs ( domain )
} , func ( ) ( bool , error ) {
return l . deleteDomainAbsent ( ctx , uuid )
} ) ; err != nil {
return DeleteVMResult { } , err
}
return l . completeDeletedVM ( ctx , uuid , true )
}
// completeDeleteForMissingDomain makes an already absent definition enter the
// same UUID-owned cleanup and final absence check as an in-request Undefine.
// This covers external removal and late graceful completion without inventing
// another domain mutation. The callback keeps the lookup decision directly
// testable without a live libvirt daemon.
func completeDeleteForMissingDomain ( lookupErr error ,
complete func ( ) ( DeleteVMResult , error ) ) ( DeleteVMResult , bool , error ) {
if ! errors . Is ( lookupErr , ErrNotFound ) {
return DeleteVMResult { } , false , nil
}
result , err := complete ( )
return result , true , err
}
// validateDeleteVMArtifacts proves that every disk selected by a VM deletion
// is Agent-owned and is not referenced by another live or persistent domain.
// It deliberately runs before the domain is stopped so a corrupt ownership
// marker cannot turn an otherwise rejected request into a partial deletion.
func ( l * Libvirt ) validateDeleteVMArtifacts ( ctx context . Context , uuid string ,
managed * domainMetadataXML ) error {
managedPool , err := l . conn . LookupStoragePoolByName ( l . storagePoolName ( StorageRoleManaged ) )
if err != nil {
return translateLibvirtError ( "lookup managed storage pool" , err )
}
defer managedPool . Free ( )
inventory , err := managedPool . ListStorageVolumes ( )
if err != nil {
return translateLibvirtError ( "list managed volumes for VM deletion" , err )
}
volumeNames , err := collectVMManagedVolumeNames ( uuid , managed , inventory )
if err != nil {
return err
}
for _ , name := range volumeNames {
if err := ctx . Err ( ) ; err != nil {
return err
}
volume , lookupErr := managedPool . LookupStorageVolByName ( name )
if isNoStorageVolume ( lookupErr ) {
continue
}
if lookupErr != nil {
return translateLibvirtError ( "lookup managed VM volume" , lookupErr )
}
if _ , validateErr := validateQCOW2Volume ( volume , 0 ) ; validateErr != nil {
_ = volume . Free ( )
return validateErr
}
path , pathErr := volume . GetPath ( )
if pathErr != nil {
_ = volume . Free ( )
return translateLibvirtError ( "resolve managed VM volume" , pathErr )
}
referenced , referenceErr := l . otherDomainUsesVolume (
path , l . storagePoolName ( StorageRoleManaged ) , name , uuid ,
)
if referenceErr != nil {
_ = volume . Free ( )
return referenceErr
}
if referenced {
_ = volume . Free ( )
return NewError ( ErrConflict , "managed_disk_shared" , "managed VM volume is referenced by another domain" , nil )
}
_ = volume . Free ( )
}
seedVolume , err := l . lookupOptionalManagedVolume ( l . storagePoolName ( StorageRoleSeed ) , seedVolumeName ( uuid ) )
if err != nil {
return err
}
if seedVolume != nil {
seedPath , pathErr := seedVolume . GetPath ( )
if pathErr != nil {
_ = seedVolume . Free ( )
return translateLibvirtError ( "resolve managed NoCloud seed volume" , pathErr )
}
referenced , referenceErr := l . otherDomainUsesVolume (
seedPath , l . storagePoolName ( StorageRoleSeed ) , seedVolumeName ( uuid ) , uuid ,
)
if referenceErr != nil {
_ = seedVolume . Free ( )
return referenceErr
}
if referenced {
_ = seedVolume . Free ( )
return NewError ( ErrConflict , "managed_disk_shared" , "managed NoCloud seed volume is referenced by another domain" , nil )
}
_ = seedVolume . Free ( )
}
filter , err := l . lookupOptionalNetworkFilter ( networkFilterName ( uuid ) )
if err != nil {
return err
}
if filter != nil {
document , xmlErr := filter . GetXMLDesc ( 0 )
_ = filter . Free ( )
if xmlErr != nil || ! managedFilterDocument ( document , networkFilterName ( uuid ) ) {
return NewError ( ErrConflict , "managed_filter_invalid" ,
"network filter cannot be confirmed as panel managed" , xmlErr )
}
referenced , referenceErr := l . otherDomainUsesFilter ( networkFilterName ( uuid ) , uuid )
if referenceErr != nil {
return referenceErr
}
if referenced {
return NewError ( ErrConflict , "managed_filter_shared" , "managed VM network filter is referenced by another domain" , nil )
}
}
return nil
}
type gracefulShutdownDomainControl interface {
IsActive ( ) ( bool , error )
ShutdownFlags ( libvirt . DomainShutdownFlags ) error
}
type deleteDomainControl interface {
gracefulShutdownDomainControl
DestroyFlags ( libvirt . DomainDestroyFlags ) error
UndefineFlags ( libvirt . DomainUndefineFlagsValues ) error
}
func driveGracefulDeleteDomainToAbsent ( ctx context . Context , domain deleteDomainControl ,
timeout time . Duration , beforeUndefine func ( context . Context ) error ,
domainAbsent func ( context . Context ) ( bool , error ) ) error {
ctx = context . WithoutCancel ( ctx )
if err := requestGracefulDomainShutoff ( ctx , domain , timeout ) ; err != nil {
return err
}
if beforeUndefine != nil {
if err := beforeUndefine ( ctx ) ; err != nil {
return err
}
}
err := driveInactiveDeleteDomainToAbsent ( domain , func ( ) ( bool , error ) {
return domainAbsent ( ctx )
} )
if ! isForceRequired ( err ) {
return err
}
// The guest was initially inactive (or stopped gracefully) but became active
// before Undefine. That race is not evidence that a graceful wait expired.
// Issue a fresh ACPI request and perform its full bounded wait before a
// force_required result can ever authorize a separate force operation.
if err := requestGracefulDomainShutoff ( ctx , domain , timeout ) ; err != nil {
return err
}
if beforeUndefine != nil {
if err := beforeUndefine ( ctx ) ; err != nil {
return err
}
}
err = driveInactiveDeleteDomainToAbsent ( domain , func ( ) ( bool , error ) {
return domainAbsent ( ctx )
} )
if isForceRequired ( err ) {
return operation . MarkUncertain ( NewError ( ErrConflict ,
"graceful_delete_domain_reactivated" ,
"domain became active again after a confirmed graceful shutdown" , err ) )
}
return err
}
func driveForceDeleteDomainToAbsent ( domain deleteDomainControl ,
beforeUndefine func ( ) error , domainAbsent func ( ) ( bool , error ) ) error {
active , err := domain . IsActive ( )
if err != nil {
return translateLibvirtError ( "check domain activity" , err )
}
destroyDispatched := false
if active {
destroyDispatched = true
destroyErr := domain . DestroyFlags ( libvirt . DOMAIN_DESTROY_DEFAULT )
stillActive , inspectErr := domain . IsActive ( )
if inspectErr != nil || stillActive {
var mutationErr , observedErr error
if destroyErr != nil {
mutationErr = mutationLibvirtError ( "force domain off before delete" , destroyErr )
} else {
mutationErr = NewError ( ErrConflict , "force_delete_domain_still_active" ,
"domain remained active after force delete" , nil )
}
if inspectErr != nil {
observedErr = translateLibvirtError ( "recheck domain activity after force delete" , inspectErr )
}
return operation . MarkUncertain ( errors . Join ( mutationErr , observedErr ) )
}
}
if beforeUndefine != nil {
if err := beforeUndefine ( ) ; err != nil {
if destroyDispatched {
return operation . MarkUncertain ( err )
}
return err
}
}
err = driveInactiveDeleteDomainToAbsent ( domain , domainAbsent )
if isForceRequired ( err ) {
// The explicit force authorization remains attached to this operation.
// If an external actor restarted the guest between Destroy and Undefine,
// keep the operation uncertain so its own reconciliation may safely
// continue; never terminate it as a new graceful force_required state.
return operation . MarkUncertain ( NewError ( ErrConflict , "force_delete_domain_reactivated" ,
"domain became active again during the explicit force delete" , err ) )
}
if destroyDispatched && err != nil && ! operation . IsUncertain ( err ) {
return operation . MarkUncertain ( err )
}
return err
}
// continueForceDeleteReconciliation never calls DestroyFlags. An uncertain
// force operation may have already crossed that destructive boundary before
// its response was lost; an explicitly authorized cleanup pass may only
// observe an active guest or continue from a confirmed inactive state.
func continueForceDeleteReconciliation ( domain deleteDomainControl ,
beforeUndefine func ( ) error , domainAbsent func ( ) ( bool , error ) ) ( bool , error ) {
active , err := domain . IsActive ( )
if err != nil {
return false , translateLibvirtError ( "inspect force-delete domain activity" , err )
}
if active {
return false , nil
}
if beforeUndefine != nil {
if err := beforeUndefine ( ) ; err != nil {
return false , err
}
}
err = driveInactiveDeleteDomainToAbsent ( domain , domainAbsent )
if isForceRequired ( err ) {
return false , operation . MarkUncertain ( NewError ( ErrConflict , "force_delete_domain_reactivated" ,
"domain became active again while reconciling the explicit force delete" , err ) )
}
return err == nil , err
}
func isForceRequired ( err error ) bool {
var backend * Error
return errors . As ( err , & backend ) && backend . Code == "force_required"
}
func driveInactiveDeleteDomainToAbsent ( domain deleteDomainControl ,
domainAbsent func ( ) ( bool , error ) ) error {
active , err := domain . IsActive ( )
if err != nil {
return translateLibvirtError ( "check domain activity before undefine" , err )
}
if active {
return NewError ( ErrConflict , "force_required" ,
"guest did not stop after the graceful shutdown request; explicit force delete is required" , nil )
}
flags := libvirt . DOMAIN_UNDEFINE_NVRAM |
libvirt . DOMAIN_UNDEFINE_MANAGED_SAVE |
libvirt . DOMAIN_UNDEFINE_SNAPSHOTS_METADATA |
libvirt . DOMAIN_UNDEFINE_CHECKPOINTS_METADATA
undefineErr := domain . UndefineFlags ( flags )
absent , inspectErr := domainAbsent ( )
if undefineErr != nil || inspectErr != nil || ! absent {
var mutationErr error
if undefineErr != nil {
mutationErr = mutationLibvirtError ( "undefine domain" , undefineErr )
} else {
mutationErr = NewError ( ErrConflict , "delete_domain_still_defined" ,
"domain remained present after undefine" , nil )
}
return operation . MarkUncertain ( errors . Join ( mutationErr , inspectErr ) )
}
return nil
}
// requestGracefulDomainShutoff is shared by VM deletion, service expiry, and
// paid resource upgrades. It has no force method in its interface, so callers
// cannot silently escalate a graceful lifecycle request to DestroyFlags.
func requestGracefulDomainShutoff ( ctx context . Context , domain gracefulShutdownDomainControl ,
timeout time . Duration ) error {
active , err := domain . IsActive ( )
if err != nil {
return translateLibvirtError ( "check domain activity before graceful shutdown" , err )
}
if ! active {
return nil
}
shutdownErr := domain . ShutdownFlags ( libvirt . DOMAIN_SHUTDOWN_ACPI_POWER_BTN )
if shutdownErr != nil {
stillActive , inspectErr := domain . IsActive ( )
if inspectErr != nil {
return ambiguousGracefulShutdown ( errors . Join (
mutationLibvirtError ( "request graceful domain shutdown" , shutdownErr ) ,
translateLibvirtError ( "confirm graceful domain shutdown request" , inspectErr ) ,
) )
}
if ! stillActive {
return nil
}
// A failed RPC response does not prove whether libvirt delivered the
// request. Observe the guest for the full graceful window, but never
// turn an unconfirmed request into force authorization. If the guest
// stays active, this operation fails and a later request must start a
// fresh graceful attempt with a new operation ID.
waitErr := waitForGracefulDomainShutoff ( context . WithoutCancel ( ctx ) , domain , timeout )
if waitErr == nil {
return nil
}
if isForceRequired ( waitErr ) {
return NewError ( ErrConflict , "graceful_shutdown_request_failed" ,
"graceful shutdown request was not confirmed; retry graceful shutdown before force delete" ,
shutdownErr )
}
return ambiguousGracefulShutdown ( errors . Join (
mutationLibvirtError ( "request graceful domain shutdown" , shutdownErr ) , waitErr ,
) )
}
// Once ShutdownFlags returned success, finish the bounded observation even
// if the HTTP/request context disappears. Journal.Execute can return an
// uncertain response to the caller while this late result is persisted;
// abandoning the wait here would otherwise strand a safely recoverable
// graceful request without a legal force boundary.
return waitForGracefulDomainShutoff ( context . WithoutCancel ( ctx ) , domain , timeout )
}
func ambiguousGracefulShutdown ( cause error ) error {
return operation . MarkUncertain ( NewError (
ErrUnavailable , "graceful_shutdown_ambiguous" ,
"graceful shutdown result could not be confirmed" , cause ,
) )
}
func waitForGracefulDomainShutoff ( ctx context . Context , domain interface { IsActive ( ) ( bool , error ) } , timeout time . Duration ) error {
if timeout <= 0 {
timeout = deleteGracefulShutdownTimeout
}
ticker := time . NewTicker ( 500 * time . Millisecond )
defer ticker . Stop ( )
timer := time . NewTimer ( timeout )
defer timer . Stop ( )
for {
active , err := domain . IsActive ( )
if err != nil {
return operation . MarkUncertain ( translateLibvirtError ( "wait for graceful domain shutdown" , err ) )
}
if ! active {
return nil
}
select {
case <- ctx . Done ( ) :
return operation . MarkUncertain ( ctx . Err ( ) )
case <- timer . C :
active , inspectErr := domain . IsActive ( )
if inspectErr != nil {
return operation . MarkUncertain ( translateLibvirtError ( "confirm graceful domain shutdown timeout" , inspectErr ) )
}
if ! active {
return nil
}
return NewError ( ErrConflict , "force_required" ,
"guest did not stop after the graceful shutdown request; explicit force delete is required" , nil )
case <- ticker . C :
}
}
}
// completeDeletedVM is shared by the initial request and reconciliation. A
// successful operation therefore always means a fresh UUID lookup, every
// panel-managed volume, the seed, network filter, NVRAM file, and local
// traffic state were all observed absent.
func ( l * Libvirt ) completeDeletedVM ( ctx context . Context , uuid string , forced bool ) ( DeleteVMResult , error ) {
if err := l . cleanupDeletedVMArtifacts ( ctx , uuid ) ; err != nil {
return DeleteVMResult { } , operation . MarkUncertain ( err )
}
domainAbsent , err := l . deleteDomainAbsent ( ctx , uuid )
if err != nil {
return DeleteVMResult { } , operation . MarkUncertain ( err )
}
if ! domainAbsent {
return DeleteVMResult { } , operation . MarkUncertain ( NewError (
ErrConflict , "delete_cleanup_incomplete" , "deleted VM domain could not be confirmed absent" , nil ,
) )
}
if err := l . reconcilePublicGuard ( ctx , nil ) ; err != nil {
return DeleteVMResult { } , operation . MarkUncertain ( NewError (
ErrUnavailable , "public_guard_cleanup_unconfirmed" , "deleted VM public network guard could not be reconciled" , err ,
) )
}
artifactsAbsent , err := l . deleteArtifactsAbsent ( uuid )
if err != nil {
return DeleteVMResult { } , operation . MarkUncertain ( err )
}
return confirmDeletedVMAbsence ( domainAbsent , artifactsAbsent , l . traffic . contains ( uuid ) , forced )
}
func confirmDeletedVMAbsence ( domainAbsent , artifactsAbsent , trafficPresent , forced bool ) ( DeleteVMResult , error ) {
if ! domainAbsent || ! artifactsAbsent || trafficPresent {
return DeleteVMResult { } , operation . MarkUncertain ( NewError (
ErrConflict , "delete_cleanup_incomplete" ,
"deleted VM resources could not all be confirmed absent" , nil ,
) )
}
return DeleteVMResult { Deleted : true , ManagedArtifactsAbsent : true , Forced : forced } , nil
}
// inspectDeleteDomainEvidence verifies all domain-bound state that
// UndefineFlags is about to remove. The evidence is private and is used only
// to bind an administrator cleanup plan; no XML or host path crosses the API.
func ( l * Libvirt ) inspectDeleteDomainEvidence ( ctx context . Context , uuid string ,
domain * libvirt . Domain ) ( [ ] string , error ) {
if err := ctx . Err ( ) ; err != nil {
return nil , err
}
document , err := domain . GetXMLDesc ( libvirt . DOMAIN_XML_INACTIVE )
if err != nil {
return nil , translateLibvirtError ( "read persistent domain delete evidence" , err )
}
var definition libvirtxml . Domain
if err := definition . Unmarshal ( document ) ; err != nil {
return nil , NewError ( ErrConflict , "managed_domain_xml_invalid" , "managed domain XML is invalid" , err )
}
evidence := make ( [ ] string , 0 , 12 )
documentDigest := sha256 . Sum256 ( [ ] byte ( document ) )
evidence = append ( evidence , "delete-domain-xml:" + hex . EncodeToString ( documentDigest [ : ] ) )
nvramPaths , err := validateManagedDomainNVRAMPaths ( & definition , l . local . NVRAMDirectory , uuid )
if err != nil {
return nil , err
}
for _ , path := range nvramPaths {
referenced , referenceErr := l . otherDomainUsesPath ( path , uuid )
if referenceErr != nil {
return nil , referenceErr
}
if referenced {
return nil , NewError ( ErrConflict , "managed_nvram_shared" , "managed firmware state is referenced by another domain" , nil )
}
info , statErr := os . Lstat ( path )
switch {
case errors . Is ( statErr , os . ErrNotExist ) :
evidence = append ( evidence , "domain-nvram:" + path + ":absent" )
case statErr != nil :
return nil , NewError ( ErrUnavailable , "nvram_state_unavailable" , "managed firmware state could not be inspected" , statErr )
case ! info . Mode ( ) . IsRegular ( ) :
return nil , NewError ( ErrConflict , "managed_nvram_invalid" , "managed firmware cleanup target is invalid" , nil )
default :
evidence = append ( evidence , "domain-nvram:" + path + ":" +
strconv . FormatInt ( info . Size ( ) , 10 ) + ":" + strconv . FormatInt ( info . ModTime ( ) . UnixNano ( ) , 10 ) )
}
}
managedSave , err := domain . HasManagedSaveImage ( 0 )
if err != nil {
return nil , translateLibvirtError ( "inspect domain managed save image" , err )
}
evidence = append ( evidence , "managed-save:" + strconv . FormatBool ( managedSave ) )
snapshots , err := domain . ListAllSnapshots ( 0 )
if err != nil {
return nil , translateLibvirtError ( "list domain snapshot metadata" , err )
}
for index := range snapshots {
if err := ctx . Err ( ) ; err != nil {
for remaining := index ; remaining < len ( snapshots ) ; remaining ++ {
_ = snapshots [ remaining ] . Free ( )
}
return nil , err
}
name , nameErr := snapshots [ index ] . GetName ( )
snapshotXML , xmlErr := snapshots [ index ] . GetXMLDesc ( 0 )
_ = snapshots [ index ] . Free ( )
if nameErr != nil || xmlErr != nil {
for remaining := index + 1 ; remaining < len ( snapshots ) ; remaining ++ {
_ = snapshots [ remaining ] . Free ( )
}
return nil , translateLibvirtError ( "read domain snapshot metadata" , errors . Join ( nameErr , xmlErr ) )
}
if safetyErr := validateNativeSnapshotDeleteSafety ( snapshotXML ) ; safetyErr != nil {
for remaining := index + 1 ; remaining < len ( snapshots ) ; remaining ++ {
_ = snapshots [ remaining ] . Free ( )
}
return nil , operation . MarkUncertain ( safetyErr )
}
digest := sha256 . Sum256 ( [ ] byte ( snapshotXML ) )
evidence = append ( evidence , "domain-snapshot:" + name + ":" + hex . EncodeToString ( digest [ : ] ) )
}
evidence = append ( evidence , "domain-snapshot-count:" + strconv . Itoa ( len ( snapshots ) ) )
checkpoints , err := domain . ListAllCheckpoints ( 0 )
if err != nil {
return nil , translateLibvirtError ( "list domain checkpoint metadata" , err )
}
for index := range checkpoints {
if err := ctx . Err ( ) ; err != nil {
for remaining := index ; remaining < len ( checkpoints ) ; remaining ++ {
_ = checkpoints [ remaining ] . Free ( )
}
return nil , err
}
name , nameErr := checkpoints [ index ] . GetName ( )
checkpointXML , xmlErr := checkpoints [ index ] . GetXMLDesc ( 0 )
_ = checkpoints [ index ] . Free ( )
if nameErr != nil || xmlErr != nil {
for remaining := index + 1 ; remaining < len ( checkpoints ) ; remaining ++ {
_ = checkpoints [ remaining ] . Free ( )
}
return nil , translateLibvirtError ( "read domain checkpoint metadata" , errors . Join ( nameErr , xmlErr ) )
}
var checkpoint libvirtxml . DomainCheckpoint
if err := checkpoint . Unmarshal ( checkpointXML ) ; err != nil {
for remaining := index + 1 ; remaining < len ( checkpoints ) ; remaining ++ {
_ = checkpoints [ remaining ] . Free ( )
}
return nil , NewError ( ErrConflict , "domain_checkpoint_xml_invalid" ,
"native checkpoint XML could not be inspected before delete" , err )
}
digest := sha256 . Sum256 ( [ ] byte ( checkpointXML ) )
evidence = append ( evidence , "domain-checkpoint:" + name + ":" + hex . EncodeToString ( digest [ : ] ) )
}
evidence = append ( evidence , "domain-checkpoint-count:" + strconv . Itoa ( len ( checkpoints ) ) )
filterName := networkFilterName ( uuid )
referenced , err := l . otherDomainUsesFilter ( filterName , uuid )
if err != nil {
return nil , err
}
if referenced {
return nil , NewError ( ErrConflict , "managed_filter_shared" , "managed VM network filter is referenced by another domain" , nil )
}
evidence = append ( evidence , "network-filter-exclusive:" + filterName )
return evidence , nil
}
func validateManagedDomainNVRAMPaths ( definition * libvirtxml . Domain , directory , uuid string ) ( [ ] string , error ) {
paths := make ( map [ string ] struct { } , 2 )
if definition != nil && definition . OS != nil {
if nvram := definition . OS . NVRam ; nvram != nil {
if nvram . NVRam != "" {
paths [ filepath . Clean ( nvram . NVRam ) ] = struct { } { }
}
if nvram . Source != nil && nvram . Source . File != nil && nvram . Source . File . File != "" {
paths [ filepath . Clean ( nvram . Source . File . File ) ] = struct { } { }
}
}
if definition . OS . VarStore != nil && definition . OS . VarStore . Path != "" {
paths [ filepath . Clean ( definition . OS . VarStore . Path ) ] = struct { } { }
}
}
if len ( paths ) == 0 {
return nil , NewError ( ErrConflict , "managed_nvram_invalid" , "managed domain NVRAM path is missing" , nil )
}
expected := make ( map [ string ] struct { } , len ( managedNVRAMArtifactNames ( uuid ) ) )
for _ , name := range managedNVRAMArtifactNames ( uuid ) {
expected [ filepath . Clean ( filepath . Join ( directory , name ) ) ] = struct { } { }
}
result := make ( [ ] string , 0 , len ( paths ) )
for path := range paths {
if ! filepath . IsAbs ( path ) {
return nil , NewError ( ErrConflict , "managed_nvram_invalid" , "managed domain NVRAM path is invalid" , nil )
}
if _ , ok := expected [ path ] ; ! ok {
return nil , NewError ( ErrConflict , "managed_nvram_path_mismatch" , "managed domain NVRAM path is outside the UUID-owned location" , nil )
}
result = append ( result , path )
}
sort . Strings ( result )
return result , nil
}
func ( l * Libvirt ) otherDomainUsesFilter ( filterName , excludedUUID string ) ( bool , error ) {
filterReferenced , err := l . otherFilterReferencesFilter ( filterName )
if err != nil || filterReferenced {
return filterReferenced , err
}
domains , err := l . conn . ListAllDomains ( 0 )
if err != nil {
return false , translateLibvirtError ( "list domains for network filter reference check" , err )
}
for index := range domains {
domain := & domains [ index ]
candidateUUID , uuidErr := domain . GetUUIDString ( )
if uuidErr != nil {
_ = domain . Free ( )
return false , translateLibvirtError ( "read domain UUID for network filter reference" , uuidErr )
}
if excludedUUID != "" && strings . EqualFold ( candidateUUID , excludedUUID ) {
_ = domain . Free ( )
continue
}
flags := [ ] libvirt . DomainXMLFlags { 0 }
persistent , persistentErr := domain . IsPersistent ( )
if persistentErr != nil {
_ = domain . Free ( )
return false , translateLibvirtError ( "inspect domain persistence for network filter reference" , persistentErr )
}
if persistent {
flags = append ( flags , libvirt . DOMAIN_XML_INACTIVE )
}
for _ , flag := range flags {
document , xmlErr := domain . GetXMLDesc ( flag )
if xmlErr != nil {
_ = domain . Free ( )
return false , translateLibvirtError ( "read domain XML for network filter reference" , xmlErr )
}
var definition libvirtxml . Domain
if err := definition . Unmarshal ( document ) ; err != nil {
_ = domain . Free ( )
return false , NewError ( ErrConflict , "domain_xml_invalid" , "domain XML could not be inspected for network filter references" , err )
}
if domainReferencesFilter ( & definition , filterName ) {
_ = domain . Free ( )
return true , nil
}
}
referenced , historyErr := domainHistoryReferencesFilter ( domain , filterName )
if historyErr != nil {
_ = domain . Free ( )
return false , historyErr
}
if referenced {
_ = domain . Free ( )
return true , nil
}
_ = domain . Free ( )
}
return false , nil
}
func domainHistoryReferencesFilter ( domain * libvirt . Domain , filterName string ) ( bool , error ) {
managedSave , present , err := managedSaveDomainDefinition ( domain )
if err != nil {
return false , err
}
if present && domainReferencesFilter ( managedSave , filterName ) {
return true , nil
}
snapshots , err := domain . ListAllSnapshots ( 0 )
if err != nil {
return false , translateLibvirtError ( "list domain snapshots for network filter reference check" , err )
}
defer func ( ) {
for index := range snapshots {
_ = snapshots [ index ] . Free ( )
}
} ( )
for index := range snapshots {
document , xmlErr := snapshots [ index ] . GetXMLDesc ( 0 )
if xmlErr != nil {
return false , translateLibvirtError ( "read domain snapshot XML for network filter reference" , xmlErr )
}
var definition libvirtxml . DomainSnapshot
if err := definition . Unmarshal ( document ) ; err != nil {
return false , NewError ( ErrConflict , "domain_snapshot_xml_invalid" ,
"domain snapshot XML could not be inspected for network filter references" , err )
}
if domainSnapshotReferencesFilter ( & definition , filterName ) {
return true , nil
}
}
checkpoints , err := domain . ListAllCheckpoints ( 0 )
if err != nil {
return false , translateLibvirtError ( "list domain checkpoints for network filter reference check" , err )
}
defer func ( ) {
for index := range checkpoints {
_ = checkpoints [ index ] . Free ( )
}
} ( )
for index := range checkpoints {
document , xmlErr := checkpoints [ index ] . GetXMLDesc ( 0 )
if xmlErr != nil {
return false , translateLibvirtError ( "read domain checkpoint XML for network filter reference" , xmlErr )
}
var definition libvirtxml . DomainCheckpoint
if err := definition . Unmarshal ( document ) ; err != nil {
return false , NewError ( ErrConflict , "domain_checkpoint_xml_invalid" ,
"domain checkpoint XML could not be inspected for network filter references" , err )
}
if definition . Domain != nil && domainReferencesFilter ( definition . Domain , filterName ) {
return true , nil
}
}
return false , nil
}
func ( l * Libvirt ) otherFilterReferencesFilter ( filterName string ) ( bool , error ) {
filters , err := l . conn . ListAllNWFilters ( 0 )
if err != nil {
return false , translateLibvirtError ( "list network filters for dependency check" , err )
}
defer func ( ) {
for index := range filters {
_ = filters [ index ] . Free ( )
}
} ( )
for index := range filters {
filter := & filters [ index ]
name , nameErr := filter . GetName ( )
if nameErr != nil {
return false , translateLibvirtError ( "read network filter name for dependency check" , nameErr )
}
if name == filterName {
continue
}
document , xmlErr := filter . GetXMLDesc ( 0 )
if xmlErr != nil {
return false , translateLibvirtError ( "read network filter dependency XML" , xmlErr )
}
var definition libvirtxml . NWFilter
if err := definition . Unmarshal ( document ) ; err != nil {
return false , NewError ( ErrConflict , "network_filter_xml_invalid" ,
"network filter dependencies could not be inspected" , err )
}
if networkFilterReferencesFilter ( & definition , filterName ) {
return true , nil
}
}
return false , nil
}
func networkFilterReferencesFilter ( definition * libvirtxml . NWFilter , filterName string ) bool {
if definition == nil || filterName == "" {
return false
}
for _ , entry := range definition . Entries {
if entry . Ref != nil && entry . Ref . Filter == filterName {
return true
}
}
return false
}
func domainReferencesFilter ( definition * libvirtxml . Domain , filterName string ) bool {
if definition == nil || definition . Devices == nil {
return false
}
for _ , iface := range definition . Devices . Interfaces {
if iface . FilterRef != nil && iface . FilterRef . Filter == filterName {
return true
}
}
return false
}
func domainSnapshotReferencesFilter ( snapshot * libvirtxml . DomainSnapshot , filterName string ) bool {
if snapshot == nil {
return false
}
if snapshot . Domain != nil && domainReferencesFilter ( snapshot . Domain , filterName ) {
return true
}
return snapshot . InactiveDomain != nil &&
domainReferencesFilter ( & snapshot . InactiveDomain . Domain , filterName )
}
func ( l * Libvirt ) deleteDomainAbsent ( ctx context . Context , uuid string ) ( bool , error ) {
if err := ctx . Err ( ) ; err != nil {
return false , err
}
if ! uuidPattern . MatchString ( uuid ) {
return false , NewError ( ErrInvalid , "invalid_uuid" , "uuid must be canonical" , nil )
}
domain , err := l . conn . LookupDomainByUUIDString ( uuid )
if err == nil {
_ = domain . Free ( )
return false , nil
}
if isNoDomain ( err ) {
return true , nil
}
return false , translateLibvirtError ( "confirm deleted domain absence" , err )
}
func isNoDomain ( err error ) bool {
var libvirtErr libvirt . Error
return errors . As ( err , & libvirtErr ) && libvirtErr . Code == libvirt . ERR_NO_DOMAIN
}
func domainMayExistAfterDefine ( inspectErr error ) bool {
return inspectErr == nil || ! isNoDomain ( inspectErr )
}
func collectVMManagedVolumeNames ( uuid string , metadata * domainMetadataXML , inventory [ ] string ) ( [ ] string , error ) {
if ! uuidPattern . MatchString ( uuid ) {
return nil , NewError ( ErrInvalid , "invalid_uuid" , "uuid must be canonical" , nil )
}
names := make ( map [ string ] struct { } )
for _ , name := range inventory {
if isVMOwnedManagedVolumeName ( uuid , name ) {
names [ name ] = struct { } { }
}
}
if metadata != nil {
if ! isVMOwnedRootVolumeName ( uuid , metadata . RootVolumeName ) {
return nil , NewError ( ErrConflict , "managed_metadata_invalid" , "managed root volume is not owned by the VM UUID" , nil )
}
names [ metadata . RootVolumeName ] = struct { } { }
if metadata . PendingCleanupVolume != "" {
if ! isVMOwnedRootVolumeName ( uuid , metadata . PendingCleanupVolume ) {
return nil , NewError ( ErrConflict , "managed_metadata_invalid" , "pending cleanup volume is not owned by the VM UUID" , nil )
}
names [ metadata . PendingCleanupVolume ] = struct { } { }
}
for _ , snapshot := range metadata . Snapshots {
name := snapshotVolumeName ( uuid , snapshot . Key )
if ! isVMOwnedManagedVolumeName ( uuid , name ) {
return nil , NewError ( ErrConflict , "managed_metadata_invalid" , "snapshot volume is not owned by the VM UUID" , nil )
}
names [ name ] = struct { } { }
}
}
result := make ( [ ] string , 0 , len ( names ) )
for name := range names {
result = append ( result , name )
}
sort . Strings ( result )
return result , nil
}
func ( l * Libvirt ) lookupOptionalManagedVolume ( poolName , volumeName string ) ( * libvirt . StorageVol , error ) {
pool , err := l . conn . LookupStoragePoolByName ( poolName )
if err != nil {
return nil , translateLibvirtError ( "lookup managed storage pool" , err )
}
defer pool . Free ( )
volume , err := pool . LookupStorageVolByName ( volumeName )
if err == nil {
return volume , nil
}
var libvirtErr libvirt . Error
if errors . As ( err , & libvirtErr ) && libvirtErr . Code == libvirt . ERR_NO_STORAGE_VOL {
return nil , nil
}
return nil , translateLibvirtError ( "lookup managed storage volume" , err )
}
func ( l * Libvirt ) lookupOptionalNetworkFilter ( name string ) ( * libvirt . NWFilter , error ) {
filter , err := l . conn . LookupNWFilterByName ( name )
if err == nil {
return filter , nil
}
var libvirtErr libvirt . Error
if errors . As ( err , & libvirtErr ) && libvirtErr . Code == libvirt . ERR_NO_NWFILTER {
return nil , nil
}
return nil , translateLibvirtError ( "lookup VM network filter" , err )
}
func ( l * Libvirt ) Power ( ctx context . Context , uuid string , request PowerRequest ) ( _ VMState , resultErr error ) {
mutationStarted := false
defer func ( ) {
if ! mutationStarted {
resultErr = operation . MarkNotApplied ( resultErr )
}
} ( )
domain , err := l . domainByUUID ( ctx , uuid )
if err != nil {
return VMState { } , err
}
defer domain . Free ( )
if _ , _ , err := requireManagedDomain ( domain ) ; err != nil {
return VMState { } , err
}
if powerActionRequiresPublicGuard ( request . Action ) {
if err := l . reconcilePublicGuard ( ctx , nil ) ; err != nil {
return VMState { } , operation . MarkNotApplied ( NewError (
ErrUnavailable , "public_guard_unavailable" ,
"public network guard could not be confirmed before the power action" , err ,
) )
}
}
mutationStarted = true
if err := dispatchPowerMutation ( domain , request . Action , func ( ) error {
return l . startManagedDomain ( ctx , domain , "perform power start" )
} ) ; err != nil {
return VMState { } , err
}
name , _ := domain . GetName ( )
if request . Action == PowerShutdown || request . Action == PowerReboot || request . Action == PowerReset {
// libvirt returning success is the durable effect for command-style
// lifecycle requests. The guest may remain running while ACPI shutdown
// or reboot is still in progress, so an immediate state sample is
// informational and can never turn an accepted command into failure.
result := VMState { UUID : uuid , Name : name , State : VMStateUnknown , CommandAccepted : true }
if state , reason , stateErr := domain . GetState ( ) ; stateErr == nil {
result . State = mapDomainState ( state )
result . Reason = reason
}
return result , nil
}
state , reason , err := domain . GetState ( )
if err != nil {
return VMState { } , operation . MarkUncertain ( translateLibvirtError ( "confirm power action" , err ) )
}
mapped := mapDomainState ( state )
if err := confirmPowerTargetState ( request . Action , mapped ) ; err != nil {
return VMState { } , err
}
return VMState { UUID : uuid , Name : name , State : mapped , Reason : reason , CommandAccepted : true , EffectConfirmed : true } , nil
}
func powerActionRequiresPublicGuard ( action PowerAction ) bool {
return action == PowerStart || action == PowerReboot || action == PowerReset
}
type powerDomainControl interface {
Create ( ) error
ShutdownFlags ( libvirt . DomainShutdownFlags ) error
Reboot ( libvirt . DomainRebootFlagValues ) error
Reset ( uint32 ) error
DestroyFlags ( libvirt . DomainDestroyFlags ) error
}
func dispatchPowerMutation ( domain powerDomainControl , action PowerAction , start func ( ) error ) error {
var err error
switch action {
case PowerStart :
if start == nil {
return operation . MarkNotApplied ( NewError ( ErrUnavailable , "public_guard_unavailable" , "power start guard is unavailable" , nil ) )
}
err = start ( )
case PowerShutdown :
err = domain . ShutdownFlags ( libvirt . DOMAIN_SHUTDOWN_ACPI_POWER_BTN )
case PowerReboot :
err = domain . Reboot ( libvirt . DOMAIN_REBOOT_ACPI_POWER_BTN )
case PowerReset :
err = domain . Reset ( 0 )
case PowerDestroy :
err = domain . DestroyFlags ( libvirt . DOMAIN_DESTROY_DEFAULT )
default :
return operation . MarkNotApplied ( NewError ( ErrInvalid , "invalid_power_action" , "unknown power action" , nil ) )
}
if err != nil {
if operation . IsNotApplied ( err ) {
return err
}
// A libvirt lifecycle call can take effect before its transport reports
// an error. Keep the exact operation reconcilable instead of recording a
// terminal failure that could invite a second destructive request.
return operation . MarkUncertain ( mutationLibvirtError ( "perform power action" , err ) )
}
return nil
}
func confirmPowerTargetState ( action PowerAction , state VMStateName ) error {
reached := true
switch action {
case PowerStart :
reached = state == VMStateRunning || state == VMStateBlocked
case PowerDestroy :
reached = state == VMStateShutoff
}
if ! reached {
return operation . MarkUncertain ( NewError (
ErrConflict , "power_state_unconfirmed" ,
"power action did not reach the requested target state" , nil ,
) )
}
return nil
}
func ( l * Libvirt ) GetConsole ( ctx context . Context , uuid string ) ( ConsoleSession , error ) {
domain , err := l . domainByUUID ( ctx , uuid )
if err != nil {
return ConsoleSession { } , err
}
defer domain . Free ( )
active , err := domain . IsActive ( )
if err != nil {
return ConsoleSession { } , translateLibvirtError ( "check console domain activity" , err )
}
if ! active {
return ConsoleSession { } , NewError ( ErrNotFound , "console_not_available" , "VM has no active local VNC console" , nil )
}
document , err := domain . GetXMLDesc ( 0 )
if err != nil {
return ConsoleSession { } , translateLibvirtError ( "get live console XML" , err )
}
metadata := struct {
Devices struct {
Graphics [ ] struct {
Type string ` xml:"type,attr" `
Port int ` xml:"port,attr" `
} ` xml:"graphics" `
} ` xml:"devices" `
} { }
if err := xml . Unmarshal ( [ ] byte ( document ) , & metadata ) ; err != nil {
return ConsoleSession { } , fmt . Errorf ( "parse console XML: %w" , err )
}
for _ , graphic := range metadata . Devices . Graphics {
if graphic . Type == "vnc" && graphic . Port > 0 && graphic . Port <= 65535 {
return ConsoleSession { Protocol : "vnc" , Address : net . JoinHostPort ( "127.0.0.1" , strconv . Itoa ( graphic . Port ) ) } , nil
}
}
return ConsoleSession { } , NewError ( ErrNotFound , "console_not_available" , "VM has no active local VNC console" , nil )
}
func resolvedDeleteSuccess ( result DeleteVMResult ) operation . Resolution {
encoded , _ := json . Marshal ( result )
return operation . Resolution { Resolved : true , Succeeded : true , Result : encoded }
}
// cleanupDeletedVMArtifacts resumes the idempotent tail of a delete after the
// domain definition is already gone. Every target is derived from the VM UUID,
// validated as Agent-owned, and checked for cross-domain references first.
func ( l * Libvirt ) cleanupDeletedVMArtifacts ( ctx context . Context , uuid string ) error {
if err := ctx . Err ( ) ; err != nil {
return err
}
unlockTrafficSample := l . traffic . lockSample ( uuid )
defer unlockTrafficSample ( )
pool , err := l . conn . LookupStoragePoolByName ( l . storagePoolName ( StorageRoleManaged ) )
if err != nil {
return translateLibvirtError ( "lookup managed storage pool" , err )
}
defer pool . Free ( )
inventory , err := pool . ListStorageVolumes ( )
if err != nil {
return translateLibvirtError ( "list managed volumes for VM cleanup" , err )
}
names , err := collectVMManagedVolumeNames ( uuid , nil , inventory )
if err != nil {
return err
}
for _ , name := range names {
if err := ctx . Err ( ) ; err != nil {
return err
}
volume , lookupErr := pool . LookupStorageVolByName ( name )
if isNoStorageVolume ( lookupErr ) {
continue
}
if lookupErr != nil {
return translateLibvirtError ( "lookup managed VM cleanup volume" , lookupErr )
}
if _ , validateErr := validateQCOW2Volume ( volume , 0 ) ; validateErr != nil {
_ = volume . Free ( )
return validateErr
}
path , pathErr := volume . GetPath ( )
if pathErr != nil {
_ = volume . Free ( )
return translateLibvirtError ( "resolve managed VM cleanup volume" , pathErr )
}
referenced , referenceErr := l . anyDomainUsesVolume (
path , l . storagePoolName ( StorageRoleManaged ) , name ,
)
if referenceErr != nil {
_ = volume . Free ( )
return referenceErr
}
if referenced {
_ = volume . Free ( )
return NewError ( ErrConflict , "managed_disk_shared" , "managed VM cleanup volume is referenced by a domain" , nil )
}
deleteErr := volume . Delete ( libvirt . STORAGE_VOL_DELETE_NORMAL )
_ = volume . Free ( )
if deleteErr != nil {
observed , inspectErr := l . lookupOptionalManagedVolume ( l . storagePoolName ( StorageRoleManaged ) , name )
if observed != nil {
_ = observed . Free ( )
}
if inspectErr != nil || observed != nil {
return operation . MarkUncertain ( errors . Join (
mutationLibvirtError ( "delete managed VM cleanup volume " + name , deleteErr ) , inspectErr ,
) )
}
}
}
seed , err := l . lookupOptionalManagedVolume ( l . storagePoolName ( StorageRoleSeed ) , seedVolumeName ( uuid ) )
if err != nil {
return err
}
if seed != nil {
path , pathErr := seed . GetPath ( )
if pathErr != nil {
_ = seed . Free ( )
return translateLibvirtError ( "resolve managed NoCloud cleanup volume" , pathErr )
}
referenced , referenceErr := l . anyDomainUsesVolume (
path , l . storagePoolName ( StorageRoleSeed ) , seedVolumeName ( uuid ) ,
)
if referenceErr != nil {
_ = seed . Free ( )
return referenceErr
}
if referenced {
_ = seed . Free ( )
return NewError ( ErrConflict , "managed_disk_shared" , "managed NoCloud cleanup volume is referenced by a domain" , nil )
}
deleteErr := seed . Delete ( libvirt . STORAGE_VOL_DELETE_NORMAL )
_ = seed . Free ( )
if deleteErr != nil {
observed , inspectErr := l . lookupOptionalManagedVolume ( l . storagePoolName ( StorageRoleSeed ) , seedVolumeName ( uuid ) )
if observed != nil {
_ = observed . Free ( )
}
if inspectErr != nil || observed != nil {
return operation . MarkUncertain ( errors . Join (
mutationLibvirtError ( "delete NoCloud cleanup volume" , deleteErr ) , inspectErr ,
) )
}
}
}
filter , err := l . lookupOptionalNetworkFilter ( networkFilterName ( uuid ) )
if err != nil {
return err
}
if filter != nil {
document , xmlErr := filter . GetXMLDesc ( 0 )
if xmlErr != nil || ! managedFilterDocument ( document , networkFilterName ( uuid ) ) {
_ = filter . Free ( )
return NewError ( ErrConflict , "managed_filter_invalid" ,
"network filter cannot be confirmed as panel managed" , xmlErr )
}
referenced , referenceErr := l . otherDomainUsesFilter ( networkFilterName ( uuid ) , "" )
if referenceErr != nil {
_ = filter . Free ( )
return referenceErr
}
if referenced {
_ = filter . Free ( )
return NewError ( ErrConflict , "managed_filter_shared" , "managed VM network filter is referenced by a domain" , nil )
}
undefineErr := filter . Undefine ( )
_ = filter . Free ( )
if undefineErr != nil {
observed , inspectErr := l . lookupOptionalNetworkFilter ( networkFilterName ( uuid ) )
if observed != nil {
_ = observed . Free ( )
}
if inspectErr != nil || observed != nil {
return operation . MarkUncertain ( errors . Join (
mutationLibvirtError ( "undefine VM cleanup network filter" , undefineErr ) , inspectErr ,
) )
}
}
}
for _ , name := range managedNVRAMArtifactNames ( uuid ) {
path := filepath . Join ( l . local . NVRAMDirectory , name )
referenced , referenceErr := l . anyDomainUsesPath ( path )
if referenceErr != nil {
return referenceErr
}
if referenced {
return NewError ( ErrConflict , "managed_nvram_shared" ,
"managed VM firmware state is referenced by another domain" , nil )
}
}
if err := removeManagedNVRAMArtifacts ( l . local . NVRAMDirectory , uuid ) ; err != nil {
return NewError ( ErrUnavailable , "nvram_cleanup_pending" , "managed VM firmware cleanup is incomplete" , err )
}
if err := l . traffic . remove ( uuid ) ; err != nil {
return NewError ( ErrUnavailable , "traffic_state_unavailable" , "deleted VM traffic state could not be removed" , err )
}
return nil
}
func ( l * Libvirt ) deleteArtifactsAbsent ( uuid string ) ( bool , error ) {
if ! uuidPattern . MatchString ( uuid ) {
return false , NewError ( ErrInvalid , "invalid_uuid" , "uuid must be canonical" , nil )
}
pool , err := l . conn . LookupStoragePoolByName ( l . storagePoolName ( StorageRoleManaged ) )
if err != nil {
return false , translateLibvirtError ( "lookup managed storage pool" , err )
}
defer pool . Free ( )
rootAbsent , err := l . dirPoolVolumeAbsent ( pool , rootVolumeName ( uuid ) )
if err != nil {
return false , err
}
if ! rootAbsent {
return false , nil
}
inventory , err := pool . ListStorageVolumes ( )
if err != nil {
return false , translateLibvirtError ( "list managed volumes for reconciliation" , err )
}
owned , err := collectVMManagedVolumeNames ( uuid , nil , inventory )
if err != nil {
return false , err
}
if len ( owned ) != 0 {
return false , nil
}
seedPool , err := l . conn . LookupStoragePoolByName ( l . storagePoolName ( StorageRoleSeed ) )
if err != nil {
return false , translateLibvirtError ( "lookup seed storage pool" , err )
}
defer seedPool . Free ( )
seedAbsent , err := l . dirPoolVolumeAbsent ( seedPool , seedVolumeName ( uuid ) )
if err != nil {
return false , err
}
if ! seedAbsent {
return false , nil
}
filter , err := l . lookupOptionalNetworkFilter ( networkFilterName ( uuid ) )
if err != nil {
return false , err
}
if filter != nil {
_ = filter . Free ( )
return false , nil
}
nvramAbsent , err := managedNVRAMArtifactAbsent ( l . local . NVRAMDirectory , uuid )
if err != nil {
return false , NewError ( ErrUnavailable , "nvram_state_unavailable" ,
"managed VM firmware state could not be inspected" , err )
}
if ! nvramAbsent {
return false , nil
}
return true , nil
}
func managedNVRAMArtifactAbsent ( directory , uuid string ) ( bool , error ) {
candidates := managedNVRAMArtifactNames ( uuid )
for _ , name := range candidates {
if _ , err := os . Lstat ( filepath . Join ( directory , name ) ) ; err == nil {
return false , nil
} else if ! errors . Is ( err , os . ErrNotExist ) {
return false , err
}
}
return true , nil
}
func managedNVRAMArtifactNames ( uuid string ) [ ] string {
normalized := strings . ToLower ( uuid )
candidates := [ ] string { normalized + "_VARS.fd" }
if uuid != normalized {
// CreateVM historically accepted an uppercase UUID and used its exact
// spelling for the NVRAM filename. Check both spellings so absence is
// never claimed while either managed artifact remains.
candidates = append ( candidates , uuid + "_VARS.fd" )
}
return candidates
}
func removeManagedNVRAMArtifacts ( directory , uuid string ) error {
for _ , name := range managedNVRAMArtifactNames ( uuid ) {
path := filepath . Join ( directory , name )
info , err := os . Lstat ( path )
if errors . Is ( err , os . ErrNotExist ) {
continue
}
if err != nil {
return err
}
if ! info . Mode ( ) . IsRegular ( ) {
return fmt . Errorf ( "managed NVRAM artifact is not a regular file" )
}
if err := os . Remove ( path ) ; err != nil && ! errors . Is ( err , os . ErrNotExist ) {
return err
}
}
return nil
}
func ( l * Libvirt ) domainByUUID ( ctx context . Context , uuid string ) ( * libvirt . Domain , error ) {
if err := ctx . Err ( ) ; err != nil {
return nil , err
}
if ! uuidPattern . MatchString ( uuid ) {
return nil , NewError ( ErrInvalid , "invalid_uuid" , "uuid must be canonical" , nil )
}
domain , err := l . conn . LookupDomainByUUIDString ( uuid )
if err != nil {
return nil , translateLibvirtError ( "lookup domain by UUID" , err )
}
if _ , _ , err := requireManagedDomain ( domain ) ; err != nil {
domain . Free ( )
return nil , err
}
return domain , nil
}
func waitForShutoff ( ctx context . Context , domain * libvirt . Domain ) error {
ticker := time . NewTicker ( 500 * time . Millisecond )
defer ticker . Stop ( )
for {
state , _ , err := domain . GetState ( )
if err != nil {
return translateLibvirtError ( "wait for shutdown" , err )
}
if state == libvirt . DOMAIN_SHUTOFF {
return nil
}
select {
case <- ctx . Done ( ) :
return ctx . Err ( )
case <- ticker . C :
}
}
}
func mapDomainState ( state libvirt . DomainState ) VMStateName {
switch state {
case libvirt . DOMAIN_RUNNING :
return VMStateRunning
case libvirt . DOMAIN_BLOCKED :
return VMStateBlocked
case libvirt . DOMAIN_PAUSED :
return VMStatePaused
case libvirt . DOMAIN_SHUTDOWN :
return VMStateShutdown
case libvirt . DOMAIN_SHUTOFF :
return VMStateShutoff
case libvirt . DOMAIN_CRASHED :
return VMStateCrashed
case libvirt . DOMAIN_PMSUSPENDED :
return VMStateSuspended
default :
return VMStateUnknown
}
}
func translateLibvirtError ( action string , err error ) error {
var libvirtErr libvirt . Error
if errors . As ( err , & libvirtErr ) {
switch libvirtErr . Code {
case libvirt . ERR_NO_DOMAIN :
return NewError ( ErrNotFound , "vm_not_found" , "virtual machine was not found" , err )
case libvirt . ERR_NO_DOMAIN_SNAPSHOT :
return NewError ( ErrNotFound , "snapshot_not_found" , "virtual machine snapshot was not found" , err )
case libvirt . ERR_NO_STORAGE_VOL :
return NewError ( ErrNotFound , "storage_volume_not_found" , "storage volume was not found" , err )
case libvirt . ERR_NO_STORAGE_POOL :
return NewError ( ErrUnavailable , "storage_pool_unavailable" , "configured storage pool was not found" , err )
case libvirt . ERR_NO_NWFILTER :
return NewError ( ErrNotFound , "firewall_filter_not_found" , "network filter was not found" , err )
case libvirt . ERR_DOM_EXIST , libvirt . ERR_STORAGE_VOL_EXIST , libvirt . ERR_OPERATION_INVALID , libvirt . ERR_OPERATION_DENIED :
return NewError ( ErrConflict , "libvirt_conflict" , action + " conflicts with current hypervisor state" , err )
case libvirt . ERR_INVALID_ARG :
return NewError ( ErrInvalid , "invalid_libvirt_request" , action + " was rejected as invalid" , err )
case libvirt . ERR_OPERATION_UNSUPPORTED :
return NewError ( ErrUnsupported , "unsupported" , action + " is not supported" , err )
case libvirt . ERR_INVALID_CONN , libvirt . ERR_NO_CONNECT , libvirt . ERR_AUTH_FAILED :
return NewError ( ErrUnavailable , "libvirt_unavailable" , "libvirt is unavailable" , err )
}
}
return NewError ( ErrUnavailable , "libvirt_error" , fmt . Sprintf ( "%s failed" , action ) , err )
}
func mutationLibvirtError ( action string , err error ) error {
var libvirtErr libvirt . Error
if errors . As ( err , & libvirtErr ) {
switch libvirtErr . Code {
case libvirt . ERR_SYSTEM_ERROR , libvirt . ERR_RPC , libvirt . ERR_INVALID_CONN , libvirt . ERR_NO_CONNECT , libvirt . ERR_OPERATION_TIMEOUT :
return operation . MarkUncertain ( translateLibvirtError ( action , err ) )
}
}
return translateLibvirtError ( action , err )
}
func formatLibvirtVersion ( version uint32 ) string {
return fmt . Sprintf ( "%d.%d.%d" , version / 1_000_000 , ( version / 1_000 ) % 1_000 , version % 1_000 )
}