//go:build libvirt package hypervisor import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "encoding/xml" "errors" "fmt" "net" "os" "path/filepath" "sort" "strconv" "strings" "time" "git.luoli.live/root/virt/agent/cloudinit" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/publicguard" "git.luoli.live/root/virt/agent/publicnetwork" "git.luoli.live/root/virt/agent/vpcrouting" "libvirt.org/go/libvirt" "libvirt.org/go/libvirtxml" ) type Libvirt struct { conn *libvirt.Connect local LocalConfig traffic *trafficStore vpcRouter *vpcrouting.Manager publicGuard *publicguard.Manager guestAgentResponseTimeoutSupported bool } const deleteGracefulShutdownTimeout = 30 * time.Second type createVMFailureStage struct { code string message string } var ( createVMTemplateLookupStage = createVMFailureStage{ code: "template_lookup_failed", message: "template metadata or volume could not be located", } createVMTemplateCapacityStage = createVMFailureStage{ code: "template_capacity_failed", message: "template capacity could not satisfy the requested disk", } createVMStorageCloneStage = createVMFailureStage{ code: "storage_clone_failed", message: "system disk could not be cloned from the template", } createVMSeedCreateStage = createVMFailureStage{ code: "seed_create_failed", message: "cloud-init seed volume could not be created", } createVMSeedUploadStage = createVMFailureStage{ code: "seed_upload_failed", message: "cloud-init seed data could not be uploaded", } createVMNWFilterDefineStage = createVMFailureStage{ code: "nwfilter_define_failed", message: "virtual machine network filter could not be defined", } createVMPublicGuardStage = createVMFailureStage{ code: "public_guard_failed", message: "virtual machine public network guard could not be applied", } createVMDomainDefineStage = createVMFailureStage{ code: "domain_define_failed", message: "virtual machine definition could not be created", } createVMDomainAutostartStage = createVMFailureStage{ code: "domain_autostart_failed", message: "virtual machine autostart could not be enabled", } createVMDomainStartStage = createVMFailureStage{ code: "domain_start_failed", message: "virtual machine could not be started", } ) type createVMStageError struct { stage createVMFailureStage cause error } func (e *createVMStageError) Error() string { return e.stage.message } func (e *createVMStageError) Unwrap() error { return e.cause } func (e *createVMStageError) PublicCode() string { return e.stage.code } func (e *createVMStageError) PublicMessage() string { return e.stage.message } // wrap exposes only a stable stage code and bounded public message. The full // cause remains in the error chain for local diagnostics and, importantly, // preserves operation uncertainty and the original hypervisor error kind. func (stage createVMFailureStage) wrap(cause error) error { if cause == nil { return nil } return &createVMStageError{stage: stage, cause: cause} } func OpenLibvirt(uri string, local LocalConfig) (Hypervisor, error) { if uri != "qemu:///system" { return nil, NewError(ErrInvalid, "invalid_libvirt_uri", "only qemu:///system is allowed", nil) } if err := startLibvirtEventLoop(); err != nil { return nil, translateLibvirtError("start libvirt event loop", err) } clientVersion, err := libvirt.GetVersion() if err != nil { return nil, translateLibvirtError("get local libvirt version", err) } traffic, err := openTrafficStore(local.TrafficStatePath) if err != nil { return nil, NewError(ErrUnavailable, "traffic_state_unavailable", "durable traffic state could not be opened", err) } return openManagedLibvirt(uri, local, clientVersion, traffic), nil } func (l *Libvirt) Close() error { _, err := l.conn.Close() return err } func (l *Libvirt) Ping(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } alive, err := l.conn.IsAlive() if err != nil { return translateLibvirtError("ping", err) } if !alive { return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt connection is not alive", nil) } return nil } func (l *Libvirt) Capabilities(ctx context.Context) (Capabilities, error) { if err := l.Ping(ctx); err != nil { return Capabilities{}, err } libVersion, err := l.conn.GetLibVersion() if err != nil { return Capabilities{}, translateLibvirtError("get libvirt version", err) } qemuVersion, err := l.conn.GetVersion() if err != nil { return Capabilities{}, translateLibvirtError("get QEMU version", err) } if err := ctx.Err(); err != nil { return Capabilities{}, err } features, checks, networkSelectors := l.probeRuntimeCapabilities(ctx) vpcCapability, vpcChecks := probeVPCL3Capability(l.vpcRouter) for name, check := range vpcChecks { checks[name] = check } vpcFilterCheck := l.probeVPCCleanTrafficFilter() checks["vpc_clean_traffic_filter"] = vpcFilterCheck features.VPCL3 = applyVPCCleanTrafficGate(vpcCapability, vpcFilterCheck) peerCapability, peerCheck := l.peerMigrationCapability(features.Backups) checks["peer_migration"] = peerCheck features.PeerMigration = peerCapability != nil powerActions := []PowerAction(nil) if features.Power { powerActions = []PowerAction{PowerStart, PowerShutdown, PowerReboot, PowerReset, PowerDestroy} } firmware := []string(nil) if features.UEFI { firmware = []string{"uefi"} } publicNetwork := publicNetworkCapability(checks) return Capabilities{ Backend: "libvirt", LibvirtVersion: formatLibvirtVersion(libVersion), QEMUVersion: formatLibvirtVersion(qemuVersion), Features: features, PowerActions: powerActions, Firmware: firmware, NetworkSelectors: networkSelectors, Checks: checks, PeerMigration: peerCapability, VPCL3: vpcCapability, PublicNetwork: publicNetwork, PortableRestore: portableRestoreCapability(checks, publicNetwork), }, nil } func (l *Libvirt) ListVMs(ctx context.Context) ([]VMMetadata, error) { if err := ctx.Err(); err != nil { return nil, err } domains, err := l.conn.ListAllDomains(0) if err != nil { return nil, translateLibvirtError("list domains", err) } result := make([]VMMetadata, 0, len(domains)) for index := range domains { domain := &domains[index] document, xmlErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) _ = domain.Free() if xmlErr != nil { return nil, translateLibvirtError("read domain XML", xmlErr) } _, _, managed, inspectErr := inspectManagedDomain(document) if inspectErr != nil { return nil, inspectErr } if !managed { continue } metadata, parseErr := metadataFromXML(document) if parseErr != nil { return nil, parseErr } result = append(result, metadata) } return result, nil } func (l *Libvirt) GetVMState(ctx context.Context, uuid string) (VMState, error) { domain, err := l.domainByUUID(ctx, uuid) if err != nil { return VMState{}, err } defer domain.Free() state, reason, err := domain.GetState() if err != nil { return VMState{}, translateLibvirtError("get domain state", err) } name, _ := domain.GetName() return VMState{UUID: uuid, Name: name, State: mapDomainState(state), Reason: reason}, nil } func (l *Libvirt) GetVMMetadata(ctx context.Context, uuid string) (VMMetadata, error) { domain, err := l.domainByUUID(ctx, uuid) if err != nil { return VMMetadata{}, err } defer domain.Free() document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if err != nil { return VMMetadata{}, translateLibvirtError("get domain XML", err) } return metadataFromXML(document) } func (l *Libvirt) CreateVM(ctx context.Context, request CreateVMRequest) (_ VMMetadata, resultErr error) { if err := ctx.Err(); err != nil { return VMMetadata{}, err } if err := ValidateCreateVM(request); err != nil { return VMMetadata{}, err } attachment, err := l.resolvePublicNetwork(ctx, request) if err != nil { return VMMetadata{}, err } mac := vmMACAddress(request.UUID) conflict, err := l.networkIdentityConflict(ctx, request.UUID, mac, request.PublicNetwork) if err != nil { return VMMetadata{}, err } if conflict { return VMMetadata{}, NewError(ErrConflict, "network_identity_conflict", "VM network identity is already in use", nil) } attachment.MAC = mac templateRecord, err := l.loadImageTemplate(ctx, request.TemplateID) if err != nil { return VMMetadata{}, createVMTemplateLookupStage.wrap(err) } if templateRecord.GenerationToken != request.TemplateGenerationToken { return VMMetadata{}, NewError(ErrConflict, "template_generation_changed", "template generation token no longer matches", nil) } seed, err := cloudinit.BuildISO(cloudinit.Request{ UUID: request.UUID, Name: request.Name, Hostname: request.Hostname, GuestSSHPublicKey: request.GuestSSHPublicKey, RootPassword: request.RootPassword, StartupScript: request.StartupScript, Debian12NetworkFix: isDebian12Template(templateRecord), Network: cloudinit.Network{ MAC: mac, Public: request.PublicNetwork, InterfaceKey: "public0", }, }) if err != nil { return VMMetadata{}, NewError(ErrInvalid, "invalid_cloud_init", "cloud-init request is invalid", err) } root, err := l.createRootVolumeFromTemplate(ctx, request) if err != nil { return VMMetadata{}, err } return l.createVMWithPreparedRoot(ctx, request, attachment, seed, root) } // createVMWithPreparedRoot owns root after entry. It is the single create tail // for template provisioning and backup restore, ensuring both paths generate a // fresh seed, MAC, public-IP filter, NVRAM, traffic baseline, and domain. func (l *Libvirt) createVMWithPreparedRoot(ctx context.Context, request CreateVMRequest, attachment NetworkAttachment, seed []byte, root *libvirt.StorageVol) (_ VMMetadata, resultErr error) { return l.createVMWithPreparedRootOptions(ctx, request, attachment, seed, root, preparedVMOptions{Autostart: true, Start: true}) } type preparedVMOptions struct { Autostart bool Start bool NetworkSuspended bool VPCPort *VPCPortMetadata Snapshots []domainSnapshotMetadataXML SnapshotVolumes []*libvirt.StorageVol } type storageVolumeCleanup interface { Delete(libvirt.StorageVolDeleteFlags) error Free() error } type preparedVolumeOwnership struct { root storageVolumeCleanup snapshots []storageVolumeCleanup rootOwned bool snapshotsOwned bool } func newPreparedVolumeOwnership(root *libvirt.StorageVol, snapshots []*libvirt.StorageVol) *preparedVolumeOwnership { owned := &preparedVolumeOwnership{root: root, rootOwned: root != nil, snapshotsOwned: len(snapshots) > 0} owned.snapshots = make([]storageVolumeCleanup, 0, len(snapshots)) for _, snapshot := range snapshots { if snapshot != nil { owned.snapshots = append(owned.snapshots, snapshot) } } return owned } func (owned *preparedVolumeOwnership) preserve() { if owned == nil { return } owned.rootOwned = false owned.snapshotsOwned = false } func (owned *preparedVolumeOwnership) cleanup(resultErr *error) { if owned == nil { return } for _, volume := range owned.snapshots { if owned.snapshotsOwned { if err := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil && resultErr != nil { *resultErr = operation.MarkUncertain(errors.Join(*resultErr, mutationLibvirtError("roll back restored migration snapshot", err))) } } _ = volume.Free() } owned.snapshots = nil owned.snapshotsOwned = false if owned.root != nil { if owned.rootOwned { if err := owned.root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil && resultErr != nil { *resultErr = operation.MarkUncertain(errors.Join(*resultErr, mutationLibvirtError("roll back cloned root volume", err))) } } _ = owned.root.Free() } owned.root = nil owned.rootOwned = false } func (l *Libvirt) createVMWithPreparedRootOptions(ctx context.Context, request CreateVMRequest, attachment NetworkAttachment, seed []byte, root *libvirt.StorageVol, options preparedVMOptions) (_ VMMetadata, resultErr error) { volumeOwnership := newPreparedVolumeOwnership(root, options.SnapshotVolumes) defer volumeOwnership.cleanup(&resultErr) if root == nil { return VMMetadata{}, NewError(ErrInvalid, "missing_root_volume", "prepared root volume is required", nil) } var domainVPCPort *VPCPortMetadata var seedVPCMetadata *domainVPCPortMetadataXML if options.VPCPort != nil { value := *options.VPCPort if err := validateVPCPortMetadata(value); err != nil { return VMMetadata{}, err } domainVPCPort = &value metadata := domainVPCPortFromPublic(value) seedVPCMetadata = &metadata } if _, err := preservedVPCCloudInitNetwork(seed, seedVPCMetadata); err != nil { return VMMetadata{}, err } rootPath, err := root.GetPath() if err != nil { return VMMetadata{}, createVMStorageCloneStage.wrap( mutationLibvirtError("resolve cloned root volume path", err)) } seedVolume, err := l.createSeedVolume(ctx, request.UUID, seed) if err != nil { return VMMetadata{}, err } seedPath, err := seedVolume.GetPath() if err != nil { cleanupErr := seedVolume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) _ = seedVolume.Free() if cleanupErr != nil { return VMMetadata{}, operation.MarkUncertain(createVMSeedCreateStage.wrap(errors.Join( mutationLibvirtError("resolve NoCloud volume path", err), mutationLibvirtError("roll back NoCloud seed volume", cleanupErr), ))) } return VMMetadata{}, createVMSeedCreateStage.wrap( mutationLibvirtError("resolve NoCloud volume path", err)) } seedOwned := true defer func() { if seedOwned { if cleanupErr := seedVolume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil { resultErr = operation.MarkUncertain(errors.Join(resultErr, mutationLibvirtError("roll back NoCloud seed volume", cleanupErr))) } } _ = seedVolume.Free() }() filterName := networkFilterName(request.UUID) filterDocument, err := BuildPlatformFilterXML(filterName, request.PublicNetwork, request.PlatformNetworkPolicy) if err != nil { return VMMetadata{}, createVMNWFilterDefineStage.wrap(err) } filter, err := l.conn.NWFilterDefineXML(filterDocument) if err != nil { observed, inspectErr := l.lookupOptionalNetworkFilter(filterName) artifactExists := observed != nil if observed != nil { _ = observed.Free() } return VMMetadata{}, createVMNWFilterDefineStage.wrap( deterministicArtifactCreateFailure( "define VM network filter", err, inspectErr, artifactExists, )) } filterOwned := true defer func() { if filterOwned { if cleanupErr := filter.Undefine(); cleanupErr != nil { resultErr = operation.MarkUncertain(errors.Join(resultErr, mutationLibvirtError("roll back VM network filter", cleanupErr))) } } _ = filter.Free() }() attachment.LinkDown = options.NetworkSuspended networks := []NetworkAttachment{attachment} if domainVPCPort != nil { networks = append(networks, NetworkAttachment{ Bridge: domainVPCPort.Bridge, MAC: domainVPCPort.MAC, LinkDown: options.NetworkSuspended, }) } document, err := BuildDomainXML(DomainDefinition{ UUID: request.UUID, Name: request.DomainName, DisplayName: request.Name, Description: request.Description, VCPUs: request.VCPUs, MemoryMiB: request.MemoryMiB, DiskGiB: request.DiskGiB, BandwidthMbps: request.BandwidthMbps, Hostname: request.Hostname, PublicNetwork: request.PublicNetwork, TemplateID: request.TemplateID, RootDiskPath: rootPath, RootVolumeName: rootVolumeName(request.UUID), RootDiskFormat: "qcow2", SeedDiskPath: seedPath, Firmware: Firmware{ Mode: "uefi", Loader: l.local.FirmwareLoader, NVRAMTemplate: l.local.FirmwareNVRAMTemplate, NVRAMPath: filepath.Join(l.local.NVRAMDirectory, request.UUID+"_VARS.fd"), SecureBoot: false, }, Network: networks, VPCPort: domainVPCPort, FilterName: filterName, PlatformNetworkPolicy: request.PlatformNetworkPolicy, Snapshots: append([]domainSnapshotMetadataXML(nil), options.Snapshots...), }) if err != nil { return VMMetadata{}, createVMDomainDefineStage.wrap(err) } // Serialize the traffic baseline only after the storage, seed, and filter // mutations have returned. A slow metrics read can no longer make a request // wait before its first libvirt mutation after Journal dispatch. unlockTrafficSample := l.traffic.lockSample(request.UUID) defer unlockTrafficSample() trafficCreated, err := l.traffic.initialize(request.UUID, time.Now().UTC()) if err != nil { return VMMetadata{}, NewError(ErrUnavailable, "traffic_state_unavailable", "new VM traffic baseline could not be initialized", err) } trafficOwned := trafficCreated defer func() { if trafficOwned { if cleanupErr := l.traffic.remove(request.UUID); cleanupErr != nil { resultErr = markTrafficCleanupUncertain(resultErr, cleanupErr) } } }() if domainVPCPort != nil { if l.vpcRouter == nil { return VMMetadata{}, NewError(ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil) } _, routeLease, routeErr := l.vpcRouter.AcquireLocalPortLease( domainVPCPort.VPCKey, domainVPCPort.HostRevision, domainVPCPort.HostDigest, domainVPCPort.CIDR, domainVPCPort.Gateway, domainVPCPort.PrivateIP, domainVPCPort.MAC, ) if routeErr != nil { return VMMetadata{}, NewError(ErrConflict, "vpc_host_projection_not_ready", "VPC host routing must authorize the exact migration port before the domain is defined", routeErr) } defer routeLease.Release() } guardBinding, err := publicGuardBinding(request.UUID, attachment.MAC, request.PublicNetwork) if err != nil { return VMMetadata{}, createVMPublicGuardStage.wrap(err) } guardLease, err := l.acquirePublicGuardTransition(ctx) if err != nil { return VMMetadata{}, createVMPublicGuardStage.wrap(err) } defer guardLease.Release() if err := l.reconcilePublicGuardLease(ctx, guardLease, &guardBinding); err != nil { return VMMetadata{}, createVMPublicGuardStage.wrap(err) } defer func() { cleanupCtx, cancelCleanup := storageRollbackContext(ctx) cleanupErr := l.reconcilePublicGuardLease(cleanupCtx, guardLease, nil) cancelCleanup() if cleanupErr != nil { resultErr = operation.MarkUncertain(errors.Join(resultErr, createVMPublicGuardStage.wrap(cleanupErr))) } }() preserveDefinedDomainArtifacts := func() { volumeOwnership.preserve() seedOwned = false filterOwned = false trafficOwned = false } domain, err := l.conn.DomainDefineXML(document) if err != nil { observed, inspectErr := l.conn.LookupDomainByUUIDString(request.UUID) if observed != nil { _ = observed.Free() } if domainMayExistAfterDefine(inspectErr) { // DomainDefineXML may have committed before its response failed. Do // not let deferred cleanup remove disks, firewall, or accounting // state that the possibly-defined domain owns. preserveDefinedDomainArtifacts() return VMMetadata{}, operation.MarkUncertain(createVMDomainDefineStage.wrap(errors.Join( mutationLibvirtError("define domain", err), translateOptionalLibvirtError("inspect domain after define failure", inspectErr), ))) } return VMMetadata{}, createVMDomainDefineStage.wrap(mutationLibvirtError("define domain", err)) } defer domain.Free() if err := domain.SetAutostart(options.Autostart); err != nil { rollbackErr := domain.UndefineFlags(libvirt.DOMAIN_UNDEFINE_NVRAM) absent, inspectErr := l.deleteDomainAbsent(ctx, request.UUID) if inspectErr != nil || !absent { preserveDefinedDomainArtifacts() return VMMetadata{}, operation.MarkUncertain(createVMDomainAutostartStage.wrap(errors.Join( mutationLibvirtError("enable domain autostart", err), optionalMutationLibvirtError("roll back domain definition", rollbackErr), inspectErr, ))) } if rollbackErr != nil { return VMMetadata{}, operation.MarkUncertain(createVMDomainAutostartStage.wrap(errors.Join( mutationLibvirtError("enable domain autostart", err), mutationLibvirtError("roll back domain definition", rollbackErr), ))) } return VMMetadata{}, createVMDomainAutostartStage.wrap( mutationLibvirtError("enable domain autostart", err)) } preserveDefinedDomainArtifacts() if !options.Start { if err := l.verifyStagedMigrationDomain( domain, request, filterName, attachment.MAC, options.NetworkSuspended, options.Snapshots, options.VPCPort, ); err != nil { return VMMetadata{}, operation.MarkUncertain(err) } metadata, err := metadataFromXML(document) if err != nil { return VMMetadata{}, operation.MarkUncertain(err) } metadata.Status = VMStateShutoff metadata.DiskGiB = request.DiskGiB metadata.BandwidthMbps = request.BandwidthMbps metadata.Hostname = request.Hostname metadata.PublicNetwork = request.PublicNetwork metadata.TemplateID = request.TemplateID return metadata, nil } // From this point onward Domain.Create may have taken effect even if its // response is lost. Retain the zero baseline so reconciliation and the // first successful sample account for all traffic since initial boot. if err := l.startManagedDomainWithLease(ctx, guardLease, domain, "start newly defined domain"); err != nil { return VMMetadata{}, operation.MarkUncertain(createVMDomainStartStage.wrap( err)) } if err := l.verifyCreatedDomain(domain, request, filterName, attachment.MAC); err != nil { return VMMetadata{}, operation.MarkUncertain(err) } metadata, err := metadataFromXML(document) if err != nil { return VMMetadata{}, operation.MarkUncertain(err) } metadata.Status = VMStateRunning metadata.DiskGiB = request.DiskGiB metadata.BandwidthMbps = request.BandwidthMbps metadata.Hostname = request.Hostname metadata.PublicNetwork = request.PublicNetwork metadata.TemplateID = request.TemplateID return metadata, nil } func markTrafficCleanupUncertain(prior, cleanupErr error) error { return operation.MarkUncertain(errors.Join(prior, NewError( ErrUnavailable, "traffic_state_unavailable", "unused VM traffic baseline could not be removed", cleanupErr, ))) } func isDebian12Template(template ImageTemplate) bool { return strings.EqualFold(strings.TrimSpace(template.OSFamily), "debian") && strings.TrimSpace(template.Version) == "12" } func (l *Libvirt) resolvePublicNetwork(ctx context.Context, request CreateVMRequest) (NetworkAttachment, error) { if err := ValidatePublicNetworkIntent(request.PublicNetwork); err != nil { return NetworkAttachment{}, err } source, err := l.ResolveNetworkSelector(ctx, request.PublicNetwork.NetworkSelector) if err != nil { return NetworkAttachment{}, err } attachment := NetworkAttachment{} switch source.Type { case "bridge": attachment.Bridge = source.Source case "network": attachment.Network = source.Source default: return NetworkAttachment{}, NewError(ErrUnavailable, "invalid_host_network_config", "host network source type is invalid", nil) } return attachment, nil } func (l *Libvirt) createRootVolumeFromTemplate(ctx context.Context, request CreateVMRequest) (*libvirt.StorageVol, error) { templateRecord, err := l.loadImageTemplate(ctx, request.TemplateID) if err != nil { return nil, createVMTemplateLookupStage.wrap(err) } if templateRecord.GenerationToken != request.TemplateGenerationToken { return nil, NewError(ErrConflict, "template_generation_changed", "template generation token no longer matches", nil) } templatePool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleTemplate)) if err != nil { return nil, createVMTemplateLookupStage.wrap( translateLibvirtError("lookup template storage pool", err)) } defer templatePool.Free() template, err := templatePool.LookupStorageVolByName(templateVolumeName(request.TemplateID)) if err != nil { return nil, createVMTemplateLookupStage.wrap( translateLibvirtError("lookup template volume", err)) } defer template.Free() templateInfo, err := template.GetInfo() if err != nil { return nil, createVMTemplateCapacityStage.wrap( translateLibvirtError("read template volume size", err)) } capacity := request.DiskGiB << 30 if capacity < templateInfo.Capacity { return nil, createVMTemplateCapacityStage.wrap(NewError( ErrInvalid, "disk_smaller_than_template", "disk_gib is smaller than the selected template", nil)) } managedPool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged)) if err != nil { return nil, createVMStorageCloneStage.wrap( translateLibvirtError("lookup managed storage pool", err)) } defer managedPool.Free() definition := libvirtxml.StorageVolume{ Name: rootVolumeName(request.UUID), Capacity: &libvirtxml.StorageVolumeSize{Value: capacity, Unit: "bytes"}, Target: &libvirtxml.StorageVolumeTarget{ Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"}, }, } document, err := definition.Marshal() if err != nil { return nil, createVMStorageCloneStage.wrap(fmt.Errorf("marshal root volume XML: %w", err)) } volume, err := managedPool.StorageVolCreateXMLFrom(document, template, 0) if err != nil { artifactAbsent, inspectErr := l.dirPoolVolumeAbsent( managedPool, rootVolumeName(request.UUID), ) return nil, createVMStorageCloneStage.wrap(deterministicArtifactCreateFailure( "create system disk from template", err, inspectErr, !artifactAbsent, )) } // A qcow2 clone in a dir pool is implemented with qemu-img convert. The // clone inherits the source image's virtual size even when the destination // XML asks for a larger capacity. Make the requested capacity a mandatory // part of this unpublished construction step: no domain can reference the // volume and the caller cannot observe it until resize and verification have // both succeeded. if err := establishNewRootVolumeCapacity(volume, capacity); err != nil { return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume( managedPool, volume, rootVolumeName(request.UUID), err, "roll back incomplete system disk", )) } return volume, nil } type newRootVolumeCapacityTarget interface { qcow2VolumeInspector Resize(uint64, libvirt.StorageVolResizeFlags) error } func establishNewRootVolumeCapacity(volume newRootVolumeCapacityTarget, capacity uint64) error { if capacity == 0 { return NewError(ErrInvalid, "invalid_disk_capacity", "system disk capacity must be positive", nil) } // Do not first inspect and conditionally repair a clone. Resize is an // unconditional part of constructing every new root volume and flags=0 // keeps the additional qcow2 address space sparse. if err := volume.Resize(capacity, 0); err != nil { return translateLibvirtError("set new system disk capacity", err) } _, err := validateQCOW2Volume(volume, capacity) return err } func (l *Libvirt) rollbackUnpublishedVolume(pool *libvirt.StoragePool, volume *libvirt.StorageVol, volumeName string, cause error, action string) error { cleanupErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) _ = volume.Free() absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName) return classifyUnpublishedVolumeRollback(cause, cleanupErr, inspectErr, absent, action) } func classifyUnpublishedVolumeRollback(cause, cleanupErr, inspectErr error, absent bool, action string) error { if inspectErr == nil && absent { // Independent libvirt inventory and filesystem evidence proves the // temporary volume is gone, even if Delete itself returned an error. return errors.Join(cause, translateOptionalLibvirtError(action, cleanupErr)) } if inspectErr == nil { inspectErr = NewError(ErrConflict, "storage_cleanup_incomplete", "incomplete storage volume remains after rollback", nil) } return operation.MarkUncertain(errors.Join( cause, optionalMutationLibvirtError(action, cleanupErr), inspectErr, )) } func (l *Libvirt) createSeedVolume(ctx context.Context, uuid string, contents []byte) (*libvirt.StorageVol, error) { pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleSeed)) if err != nil { return nil, createVMSeedCreateStage.wrap( translateLibvirtError("lookup seed storage pool", err)) } defer pool.Free() definition := libvirtxml.StorageVolume{ Name: seedVolumeName(uuid), Capacity: &libvirtxml.StorageVolumeSize{Value: uint64(len(contents)), Unit: "bytes"}, Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: "raw"}}, } document, err := definition.Marshal() if err != nil { return nil, createVMSeedCreateStage.wrap(fmt.Errorf("marshal seed volume XML: %w", err)) } volume, err := pool.StorageVolCreateXML(document, 0) if err != nil { artifactAbsent, inspectErr := l.dirPoolVolumeAbsent(pool, seedVolumeName(uuid)) return nil, createVMSeedCreateStage.wrap(deterministicArtifactCreateFailure( "create NoCloud volume", err, inspectErr, !artifactAbsent, )) } if err := l.uploadVolume(ctx, volume, contents); err != nil { uploadErr := createVMSeedUploadStage.wrap(err) if operation.IsUncertain(err) { _ = volume.Free() return nil, operation.MarkUncertain(uploadErr) } cleanupErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) _ = volume.Free() if cleanupErr != nil { return nil, operation.MarkUncertain(errors.Join(uploadErr, mutationLibvirtError("roll back NoCloud seed volume", cleanupErr))) } return nil, uploadErr } return volume, nil } func (l *Libvirt) uploadVolume(ctx context.Context, volume *libvirt.StorageVol, contents []byte) error { if err := ctx.Err(); err != nil { return err } stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK) if err != nil { return translateLibvirtError("create storage upload stream", err) } transfer := &storageStreamTransfer{stream: stream} if err := volume.Upload(stream, 0, uint64(len(contents)), 0); err != nil { return mutationLibvirtError("open storage upload", transfer.abort(err)) } if _, err := transfer.send(ctx, contents); err != nil { return mutationLibvirtError("upload storage volume", transfer.abort(err)) } if err := transfer.finish(ctx); err != nil { return mutationLibvirtError("finish storage upload", err) } return nil } func (l *Libvirt) networkIdentityConflict(ctx context.Context, requestedUUID, mac string, public publicnetwork.Intent) (bool, error) { if err := ValidatePublicNetworkIntent(public); err != nil { return false, err } domains, err := l.conn.ListAllDomains(0) if err != nil { return false, translateLibvirtError("list domains for network identity validation", err) } for index := range domains { if err := ctx.Err(); err != nil { return false, err } domain := &domains[index] document, xmlErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) _ = domain.Free() if xmlErr != nil { return false, translateLibvirtError("read domain network identity", xmlErr) } definition, metadata, managed, parseErr := inspectManagedDomain(document) if parseErr != nil { return false, parseErr } if !managed || strings.EqualFold(definition.UUID, requestedUUID) { continue } if publicNetworkAddressConflict(metadata.PublicNetwork.publicMust(), public) { return true, nil } if definition.Devices != nil { for _, iface := range definition.Devices.Interfaces { if iface.MAC != nil && strings.EqualFold(iface.MAC.Address, mac) { return true, nil } } } } return false, nil } func (l *Libvirt) verifyCreatedDomain(domain *libvirt.Domain, request CreateVMRequest, filterName, mac string) error { autostart, err := domain.GetAutostart() if err != nil || !autostart { return mutationLibvirtError("verify domain autostart", err) } active, err := domain.IsActive() if err != nil || !active { return mutationLibvirtError("verify newly created domain state", err) } document, err := domain.GetXMLDesc(0) if err != nil { return translateLibvirtError("verify live domain XML", err) } definition, metadata, err := parseManagedDomain(document) if err != nil { return err } if !strings.EqualFold(definition.UUID, request.UUID) || definition.Name != request.DomainName || metadata.DisplayName != request.Name || !publicNetworkEqual(metadata.PublicNetwork.publicMust(), request.PublicNetwork) || definition.Devices == nil || len(definition.Devices.Interfaces) != 1 { return NewError(ErrConflict, "create_observation_mismatch", "new domain identity or network metadata did not match the requested intent", nil) } var storedPolicy PlatformNetworkPolicySnapshot if err := json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &storedPolicy); err != nil || storedPolicy.Digest != request.PlatformNetworkPolicy.Digest { return NewError(ErrConflict, "firewall_observation_mismatch", "new domain platform policy metadata did not match the requested snapshot", err) } iface := definition.Devices.Interfaces[0] if iface.MAC == nil || !strings.EqualFold(iface.MAC.Address, mac) || !managedInterfaceFilterIPMatches(&iface, filterName, request.PublicNetwork) { return NewError(ErrConflict, "firewall_observation_mismatch", "new domain did not attach the expected network filter", nil) } return l.verifyManagedFilter(filterName, storedPolicy) } func (l *Libvirt) verifyStagedMigrationDomain(domain *libvirt.Domain, request CreateVMRequest, filterName, mac string, networkSuspended bool, snapshots []domainSnapshotMetadataXML, vpcPort *VPCPortMetadata) error { autostart, err := domain.GetAutostart() if err != nil || autostart { return mutationLibvirtError("verify staged domain autostart", err) } active, err := domain.IsActive() if err != nil || active { return mutationLibvirtError("verify staged domain power state", err) } document, err := domain.GetXMLDesc(0) if err != nil { return translateLibvirtError("verify staged domain XML", err) } if err := validateStagedMigrationDomainDocument( document, request, filterName, mac, networkSuspended, snapshots, vpcPort, ); err != nil { return err } return l.verifyManagedFilter(filterName, request.PlatformNetworkPolicy) } func validateStagedMigrationDomainDocument(document string, request CreateVMRequest, filterName, mac string, networkSuspended bool, snapshots []domainSnapshotMetadataXML, vpcPort *VPCPortMetadata) error { definition, metadata, err := parseManagedDomain(document) if err != nil { return err } expectedInterfaces := 1 if vpcPort != nil { expectedInterfaces = 2 } if !strings.EqualFold(definition.UUID, request.UUID) || definition.Name != request.DomainName || metadata.DisplayName != request.Name || !publicNetworkEqual(metadata.PublicNetwork.publicMust(), request.PublicNetwork) || definition.Devices == nil || len(definition.Devices.Interfaces) != expectedInterfaces || !managedNetworkLayoutMatches(definition, metadata) || len(metadata.Snapshots) != len(snapshots) { return NewError(ErrConflict, "migration_stage_observation_mismatch", "staged migration domain identity or snapshot catalog did not match", nil) } for index := range snapshots { if metadata.Snapshots[index] != snapshots[index] { return NewError(ErrConflict, "migration_stage_observation_mismatch", "staged migration snapshot catalog did not match", nil) } } if vpcPort == nil && metadata.VPCPort != nil || vpcPort != nil && (metadata.VPCPort == nil || *metadata.VPCPort != domainVPCPortFromPublic(*vpcPort)) { return NewError(ErrConflict, "migration_stage_network_mismatch", "staged migration VPC identity did not match", nil) } publicIndex, found := managedPublicInterfaceIndex(definition, filterName, request.PublicNetwork) if !found { return NewError(ErrConflict, "migration_stage_network_mismatch", "staged migration public interface did not match", nil) } iface := definition.Devices.Interfaces[publicIndex] if iface.MAC == nil || !strings.EqualFold(iface.MAC.Address, mac) || !managedInterfaceFilterIPMatches(&iface, filterName, request.PublicNetwork) { return NewError(ErrConflict, "migration_stage_network_mismatch", "staged migration network state did not match", nil) } for index := range definition.Devices.Interfaces { linkDown := definition.Devices.Interfaces[index].Link != nil && definition.Devices.Interfaces[index].Link.State == "down" if linkDown != networkSuspended { return NewError(ErrConflict, "migration_stage_network_mismatch", "staged migration interface link state did not match", nil) } } return nil } func (l *Libvirt) verifyManagedFilter(filterName string, policy PlatformNetworkPolicySnapshot) error { filter, err := l.conn.LookupNWFilterByName(filterName) if err != nil { return translateLibvirtError("verify VM network filter", err) } defer filter.Free() filterXML, err := filter.GetXMLDesc(0) if err != nil { return translateLibvirtError("verify VM network policy", err) } if !managedFilterSemanticsValid(filterXML, filterName, policy) { return NewError(ErrConflict, "firewall_observation_mismatch", "new VM filter does not match the project-owned anti-spoof and platform policy contract", nil) } return nil } func (l *Libvirt) DeleteVM(ctx context.Context, uuid string, request DeleteVMRequest) (_ DeleteVMResult, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() if !request.RemoveManagedDisks { return DeleteVMResult{}, NewError(ErrInvalid, "managed_disk_cleanup_required", "remove_managed_disks must be true", nil) } domain, err := l.domainByUUID(ctx, uuid) if result, absent, cleanupErr := completeDeleteForMissingDomain(err, func() (DeleteVMResult, error) { mutationStarted = true return l.completeDeletedVM(ctx, uuid, false) }); absent { return result, cleanupErr } if err != nil { return DeleteVMResult{}, err } defer domain.Free() _, managed, err := requireManagedDomain(domain) if err != nil { return DeleteVMResult{}, err } if err := l.validateDeleteVMArtifacts(ctx, uuid, managed); err != nil { return DeleteVMResult{}, err } if _, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain); err != nil { return DeleteVMResult{}, err } if err := ensureNoCleanupBlockingJobs(domain); err != nil { return DeleteVMResult{}, err } completionCtx := context.WithoutCancel(ctx) mutationStarted = true if err := driveGracefulDeleteDomainToAbsent(completionCtx, domain, deleteGracefulShutdownTimeout, func(completionCtx context.Context) error { if _, err := l.inspectDeleteDomainEvidence(completionCtx, uuid, domain); err != nil { return err } return ensureNoCleanupBlockingJobs(domain) }, func(completionCtx context.Context) (bool, error) { return l.deleteDomainAbsent(completionCtx, uuid) }); err != nil { return DeleteVMResult{}, err } return l.completeDeletedVM(completionCtx, uuid, false) } func (l *Libvirt) ForceDeleteVM(ctx context.Context, uuid string, request ForceDeleteVMRequest) (_ DeleteVMResult, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() if !request.RemoveManagedDisks { return DeleteVMResult{}, NewError(ErrInvalid, "managed_disk_cleanup_required", "remove_managed_disks must be true", nil) } if !request.AckDataLoss { return DeleteVMResult{}, NewError(ErrInvalid, "data_loss_ack_required", "ack_data_loss must be true", nil) } if request.PriorOperationID == "" { return DeleteVMResult{}, NewError(ErrInvalid, "invalid_prior_operation_id", "prior_operation_id is required", nil) } domain, err := l.domainByUUID(ctx, uuid) if result, absent, cleanupErr := completeDeleteForMissingDomain(err, func() (DeleteVMResult, error) { mutationStarted = true return l.completeDeletedVM(ctx, uuid, true) }); absent { return result, cleanupErr } if err != nil { return DeleteVMResult{}, err } defer domain.Free() _, managed, err := requireManagedDomain(domain) if err != nil { return DeleteVMResult{}, err } if err := l.validateDeleteVMArtifacts(ctx, uuid, managed); err != nil { return DeleteVMResult{}, err } if _, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain); err != nil { return DeleteVMResult{}, err } if err := ensureNoCleanupBlockingJobs(domain); err != nil { return DeleteVMResult{}, err } mutationStarted = true if err := driveForceDeleteDomainToAbsent(domain, func() error { if _, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain); err != nil { return err } return ensureNoCleanupBlockingJobs(domain) }, func() (bool, error) { return l.deleteDomainAbsent(ctx, uuid) }); err != nil { return DeleteVMResult{}, err } return l.completeDeletedVM(ctx, uuid, true) } // completeDeleteForMissingDomain makes an already absent definition enter the // same UUID-owned cleanup and final absence check as an in-request Undefine. // This covers external removal and late graceful completion without inventing // another domain mutation. The callback keeps the lookup decision directly // testable without a live libvirt daemon. func completeDeleteForMissingDomain(lookupErr error, complete func() (DeleteVMResult, error)) (DeleteVMResult, bool, error) { if !errors.Is(lookupErr, ErrNotFound) { return DeleteVMResult{}, false, nil } result, err := complete() return result, true, err } // validateDeleteVMArtifacts proves that every disk selected by a VM deletion // is Agent-owned and is not referenced by another live or persistent domain. // It deliberately runs before the domain is stopped so a corrupt ownership // marker cannot turn an otherwise rejected request into a partial deletion. func (l *Libvirt) validateDeleteVMArtifacts(ctx context.Context, uuid string, managed *domainMetadataXML) error { managedPool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged)) if err != nil { return translateLibvirtError("lookup managed storage pool", err) } defer managedPool.Free() inventory, err := managedPool.ListStorageVolumes() if err != nil { return translateLibvirtError("list managed volumes for VM deletion", err) } volumeNames, err := collectVMManagedVolumeNames(uuid, managed, inventory) if err != nil { return err } for _, name := range volumeNames { if err := ctx.Err(); err != nil { return err } volume, lookupErr := managedPool.LookupStorageVolByName(name) if isNoStorageVolume(lookupErr) { continue } if lookupErr != nil { return translateLibvirtError("lookup managed VM volume", lookupErr) } if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil { _ = volume.Free() return validateErr } path, pathErr := volume.GetPath() if pathErr != nil { _ = volume.Free() return translateLibvirtError("resolve managed VM volume", pathErr) } referenced, referenceErr := l.otherDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), name, uuid, ) if referenceErr != nil { _ = volume.Free() return referenceErr } if referenced { _ = volume.Free() return NewError(ErrConflict, "managed_disk_shared", "managed VM volume is referenced by another domain", nil) } _ = volume.Free() } seedVolume, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid)) if err != nil { return err } if seedVolume != nil { seedPath, pathErr := seedVolume.GetPath() if pathErr != nil { _ = seedVolume.Free() return translateLibvirtError("resolve managed NoCloud seed volume", pathErr) } referenced, referenceErr := l.otherDomainUsesVolume( seedPath, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid, ) if referenceErr != nil { _ = seedVolume.Free() return referenceErr } if referenced { _ = seedVolume.Free() return NewError(ErrConflict, "managed_disk_shared", "managed NoCloud seed volume is referenced by another domain", nil) } _ = seedVolume.Free() } filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid)) if err != nil { return err } if filter != nil { document, xmlErr := filter.GetXMLDesc(0) _ = filter.Free() if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) { return NewError(ErrConflict, "managed_filter_invalid", "network filter cannot be confirmed as panel managed", xmlErr) } referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), uuid) if referenceErr != nil { return referenceErr } if referenced { return NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil) } } return nil } type gracefulShutdownDomainControl interface { IsActive() (bool, error) ShutdownFlags(libvirt.DomainShutdownFlags) error } type deleteDomainControl interface { gracefulShutdownDomainControl DestroyFlags(libvirt.DomainDestroyFlags) error UndefineFlags(libvirt.DomainUndefineFlagsValues) error } func driveGracefulDeleteDomainToAbsent(ctx context.Context, domain deleteDomainControl, timeout time.Duration, beforeUndefine func(context.Context) error, domainAbsent func(context.Context) (bool, error)) error { ctx = context.WithoutCancel(ctx) if err := requestGracefulDomainShutoff(ctx, domain, timeout); err != nil { return err } if beforeUndefine != nil { if err := beforeUndefine(ctx); err != nil { return err } } err := driveInactiveDeleteDomainToAbsent(domain, func() (bool, error) { return domainAbsent(ctx) }) if !isForceRequired(err) { return err } // The guest was initially inactive (or stopped gracefully) but became active // before Undefine. That race is not evidence that a graceful wait expired. // Issue a fresh ACPI request and perform its full bounded wait before a // force_required result can ever authorize a separate force operation. if err := requestGracefulDomainShutoff(ctx, domain, timeout); err != nil { return err } if beforeUndefine != nil { if err := beforeUndefine(ctx); err != nil { return err } } err = driveInactiveDeleteDomainToAbsent(domain, func() (bool, error) { return domainAbsent(ctx) }) if isForceRequired(err) { return operation.MarkUncertain(NewError(ErrConflict, "graceful_delete_domain_reactivated", "domain became active again after a confirmed graceful shutdown", err)) } return err } func driveForceDeleteDomainToAbsent(domain deleteDomainControl, beforeUndefine func() error, domainAbsent func() (bool, error)) error { active, err := domain.IsActive() if err != nil { return translateLibvirtError("check domain activity", err) } destroyDispatched := false if active { destroyDispatched = true destroyErr := domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT) stillActive, inspectErr := domain.IsActive() if inspectErr != nil || stillActive { var mutationErr, observedErr error if destroyErr != nil { mutationErr = mutationLibvirtError("force domain off before delete", destroyErr) } else { mutationErr = NewError(ErrConflict, "force_delete_domain_still_active", "domain remained active after force delete", nil) } if inspectErr != nil { observedErr = translateLibvirtError("recheck domain activity after force delete", inspectErr) } return operation.MarkUncertain(errors.Join(mutationErr, observedErr)) } } if beforeUndefine != nil { if err := beforeUndefine(); err != nil { if destroyDispatched { return operation.MarkUncertain(err) } return err } } err = driveInactiveDeleteDomainToAbsent(domain, domainAbsent) if isForceRequired(err) { // The explicit force authorization remains attached to this operation. // If an external actor restarted the guest between Destroy and Undefine, // keep the operation uncertain so its own reconciliation may safely // continue; never terminate it as a new graceful force_required state. return operation.MarkUncertain(NewError(ErrConflict, "force_delete_domain_reactivated", "domain became active again during the explicit force delete", err)) } if destroyDispatched && err != nil && !operation.IsUncertain(err) { return operation.MarkUncertain(err) } return err } // continueForceDeleteReconciliation never calls DestroyFlags. An uncertain // force operation may have already crossed that destructive boundary before // its response was lost; an explicitly authorized cleanup pass may only // observe an active guest or continue from a confirmed inactive state. func continueForceDeleteReconciliation(domain deleteDomainControl, beforeUndefine func() error, domainAbsent func() (bool, error)) (bool, error) { active, err := domain.IsActive() if err != nil { return false, translateLibvirtError("inspect force-delete domain activity", err) } if active { return false, nil } if beforeUndefine != nil { if err := beforeUndefine(); err != nil { return false, err } } err = driveInactiveDeleteDomainToAbsent(domain, domainAbsent) if isForceRequired(err) { return false, operation.MarkUncertain(NewError(ErrConflict, "force_delete_domain_reactivated", "domain became active again while reconciling the explicit force delete", err)) } return err == nil, err } func isForceRequired(err error) bool { var backend *Error return errors.As(err, &backend) && backend.Code == "force_required" } func driveInactiveDeleteDomainToAbsent(domain deleteDomainControl, domainAbsent func() (bool, error)) error { active, err := domain.IsActive() if err != nil { return translateLibvirtError("check domain activity before undefine", err) } if active { return NewError(ErrConflict, "force_required", "guest did not stop after the graceful shutdown request; explicit force delete is required", nil) } flags := libvirt.DOMAIN_UNDEFINE_NVRAM | libvirt.DOMAIN_UNDEFINE_MANAGED_SAVE | libvirt.DOMAIN_UNDEFINE_SNAPSHOTS_METADATA | libvirt.DOMAIN_UNDEFINE_CHECKPOINTS_METADATA undefineErr := domain.UndefineFlags(flags) absent, inspectErr := domainAbsent() if undefineErr != nil || inspectErr != nil || !absent { var mutationErr error if undefineErr != nil { mutationErr = mutationLibvirtError("undefine domain", undefineErr) } else { mutationErr = NewError(ErrConflict, "delete_domain_still_defined", "domain remained present after undefine", nil) } return operation.MarkUncertain(errors.Join(mutationErr, inspectErr)) } return nil } // requestGracefulDomainShutoff is shared by VM deletion, service expiry, and // paid resource upgrades. It has no force method in its interface, so callers // cannot silently escalate a graceful lifecycle request to DestroyFlags. func requestGracefulDomainShutoff(ctx context.Context, domain gracefulShutdownDomainControl, timeout time.Duration) error { active, err := domain.IsActive() if err != nil { return translateLibvirtError("check domain activity before graceful shutdown", err) } if !active { return nil } shutdownErr := domain.ShutdownFlags(libvirt.DOMAIN_SHUTDOWN_ACPI_POWER_BTN) if shutdownErr != nil { stillActive, inspectErr := domain.IsActive() if inspectErr != nil { return ambiguousGracefulShutdown(errors.Join( mutationLibvirtError("request graceful domain shutdown", shutdownErr), translateLibvirtError("confirm graceful domain shutdown request", inspectErr), )) } if !stillActive { return nil } // A failed RPC response does not prove whether libvirt delivered the // request. Observe the guest for the full graceful window, but never // turn an unconfirmed request into force authorization. If the guest // stays active, this operation fails and a later request must start a // fresh graceful attempt with a new operation ID. waitErr := waitForGracefulDomainShutoff(context.WithoutCancel(ctx), domain, timeout) if waitErr == nil { return nil } if isForceRequired(waitErr) { return NewError(ErrConflict, "graceful_shutdown_request_failed", "graceful shutdown request was not confirmed; retry graceful shutdown before force delete", shutdownErr) } return ambiguousGracefulShutdown(errors.Join( mutationLibvirtError("request graceful domain shutdown", shutdownErr), waitErr, )) } // Once ShutdownFlags returned success, finish the bounded observation even // if the HTTP/request context disappears. Journal.Execute can return an // uncertain response to the caller while this late result is persisted; // abandoning the wait here would otherwise strand a safely recoverable // graceful request without a legal force boundary. return waitForGracefulDomainShutoff(context.WithoutCancel(ctx), domain, timeout) } func ambiguousGracefulShutdown(cause error) error { return operation.MarkUncertain(NewError( ErrUnavailable, "graceful_shutdown_ambiguous", "graceful shutdown result could not be confirmed", cause, )) } func waitForGracefulDomainShutoff(ctx context.Context, domain interface{ IsActive() (bool, error) }, timeout time.Duration) error { if timeout <= 0 { timeout = deleteGracefulShutdownTimeout } ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() timer := time.NewTimer(timeout) defer timer.Stop() for { active, err := domain.IsActive() if err != nil { return operation.MarkUncertain(translateLibvirtError("wait for graceful domain shutdown", err)) } if !active { return nil } select { case <-ctx.Done(): return operation.MarkUncertain(ctx.Err()) case <-timer.C: active, inspectErr := domain.IsActive() if inspectErr != nil { return operation.MarkUncertain(translateLibvirtError("confirm graceful domain shutdown timeout", inspectErr)) } if !active { return nil } return NewError(ErrConflict, "force_required", "guest did not stop after the graceful shutdown request; explicit force delete is required", nil) case <-ticker.C: } } } // completeDeletedVM is shared by the initial request and reconciliation. A // successful operation therefore always means a fresh UUID lookup, every // panel-managed volume, the seed, network filter, NVRAM file, and local // traffic state were all observed absent. func (l *Libvirt) completeDeletedVM(ctx context.Context, uuid string, forced bool) (DeleteVMResult, error) { if err := l.cleanupDeletedVMArtifacts(ctx, uuid); err != nil { return DeleteVMResult{}, operation.MarkUncertain(err) } domainAbsent, err := l.deleteDomainAbsent(ctx, uuid) if err != nil { return DeleteVMResult{}, operation.MarkUncertain(err) } if !domainAbsent { return DeleteVMResult{}, operation.MarkUncertain(NewError( ErrConflict, "delete_cleanup_incomplete", "deleted VM domain could not be confirmed absent", nil, )) } if err := l.reconcilePublicGuard(ctx, nil); err != nil { return DeleteVMResult{}, operation.MarkUncertain(NewError( ErrUnavailable, "public_guard_cleanup_unconfirmed", "deleted VM public network guard could not be reconciled", err, )) } artifactsAbsent, err := l.deleteArtifactsAbsent(uuid) if err != nil { return DeleteVMResult{}, operation.MarkUncertain(err) } return confirmDeletedVMAbsence(domainAbsent, artifactsAbsent, l.traffic.contains(uuid), forced) } func confirmDeletedVMAbsence(domainAbsent, artifactsAbsent, trafficPresent, forced bool) (DeleteVMResult, error) { if !domainAbsent || !artifactsAbsent || trafficPresent { return DeleteVMResult{}, operation.MarkUncertain(NewError( ErrConflict, "delete_cleanup_incomplete", "deleted VM resources could not all be confirmed absent", nil, )) } return DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true, Forced: forced}, nil } // inspectDeleteDomainEvidence verifies all domain-bound state that // UndefineFlags is about to remove. The evidence is private and is used only // to bind an administrator cleanup plan; no XML or host path crosses the API. func (l *Libvirt) inspectDeleteDomainEvidence(ctx context.Context, uuid string, domain *libvirt.Domain) ([]string, error) { if err := ctx.Err(); err != nil { return nil, err } document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if err != nil { return nil, translateLibvirtError("read persistent domain delete evidence", err) } var definition libvirtxml.Domain if err := definition.Unmarshal(document); err != nil { return nil, NewError(ErrConflict, "managed_domain_xml_invalid", "managed domain XML is invalid", err) } evidence := make([]string, 0, 12) documentDigest := sha256.Sum256([]byte(document)) evidence = append(evidence, "delete-domain-xml:"+hex.EncodeToString(documentDigest[:])) nvramPaths, err := validateManagedDomainNVRAMPaths(&definition, l.local.NVRAMDirectory, uuid) if err != nil { return nil, err } for _, path := range nvramPaths { referenced, referenceErr := l.otherDomainUsesPath(path, uuid) if referenceErr != nil { return nil, referenceErr } if referenced { return nil, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil) } info, statErr := os.Lstat(path) switch { case errors.Is(statErr, os.ErrNotExist): evidence = append(evidence, "domain-nvram:"+path+":absent") case statErr != nil: return nil, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr) case !info.Mode().IsRegular(): return nil, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil) default: evidence = append(evidence, "domain-nvram:"+path+":"+ strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10)) } } managedSave, err := domain.HasManagedSaveImage(0) if err != nil { return nil, translateLibvirtError("inspect domain managed save image", err) } evidence = append(evidence, "managed-save:"+strconv.FormatBool(managedSave)) snapshots, err := domain.ListAllSnapshots(0) if err != nil { return nil, translateLibvirtError("list domain snapshot metadata", err) } for index := range snapshots { if err := ctx.Err(); err != nil { for remaining := index; remaining < len(snapshots); remaining++ { _ = snapshots[remaining].Free() } return nil, err } name, nameErr := snapshots[index].GetName() snapshotXML, xmlErr := snapshots[index].GetXMLDesc(0) _ = snapshots[index].Free() if nameErr != nil || xmlErr != nil { for remaining := index + 1; remaining < len(snapshots); remaining++ { _ = snapshots[remaining].Free() } return nil, translateLibvirtError("read domain snapshot metadata", errors.Join(nameErr, xmlErr)) } if safetyErr := validateNativeSnapshotDeleteSafety(snapshotXML); safetyErr != nil { for remaining := index + 1; remaining < len(snapshots); remaining++ { _ = snapshots[remaining].Free() } return nil, operation.MarkUncertain(safetyErr) } digest := sha256.Sum256([]byte(snapshotXML)) evidence = append(evidence, "domain-snapshot:"+name+":"+hex.EncodeToString(digest[:])) } evidence = append(evidence, "domain-snapshot-count:"+strconv.Itoa(len(snapshots))) checkpoints, err := domain.ListAllCheckpoints(0) if err != nil { return nil, translateLibvirtError("list domain checkpoint metadata", err) } for index := range checkpoints { if err := ctx.Err(); err != nil { for remaining := index; remaining < len(checkpoints); remaining++ { _ = checkpoints[remaining].Free() } return nil, err } name, nameErr := checkpoints[index].GetName() checkpointXML, xmlErr := checkpoints[index].GetXMLDesc(0) _ = checkpoints[index].Free() if nameErr != nil || xmlErr != nil { for remaining := index + 1; remaining < len(checkpoints); remaining++ { _ = checkpoints[remaining].Free() } return nil, translateLibvirtError("read domain checkpoint metadata", errors.Join(nameErr, xmlErr)) } var checkpoint libvirtxml.DomainCheckpoint if err := checkpoint.Unmarshal(checkpointXML); err != nil { for remaining := index + 1; remaining < len(checkpoints); remaining++ { _ = checkpoints[remaining].Free() } return nil, NewError(ErrConflict, "domain_checkpoint_xml_invalid", "native checkpoint XML could not be inspected before delete", err) } digest := sha256.Sum256([]byte(checkpointXML)) evidence = append(evidence, "domain-checkpoint:"+name+":"+hex.EncodeToString(digest[:])) } evidence = append(evidence, "domain-checkpoint-count:"+strconv.Itoa(len(checkpoints))) filterName := networkFilterName(uuid) referenced, err := l.otherDomainUsesFilter(filterName, uuid) if err != nil { return nil, err } if referenced { return nil, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil) } evidence = append(evidence, "network-filter-exclusive:"+filterName) return evidence, nil } func validateManagedDomainNVRAMPaths(definition *libvirtxml.Domain, directory, uuid string) ([]string, error) { paths := make(map[string]struct{}, 2) if definition != nil && definition.OS != nil { if nvram := definition.OS.NVRam; nvram != nil { if nvram.NVRam != "" { paths[filepath.Clean(nvram.NVRam)] = struct{}{} } if nvram.Source != nil && nvram.Source.File != nil && nvram.Source.File.File != "" { paths[filepath.Clean(nvram.Source.File.File)] = struct{}{} } } if definition.OS.VarStore != nil && definition.OS.VarStore.Path != "" { paths[filepath.Clean(definition.OS.VarStore.Path)] = struct{}{} } } if len(paths) == 0 { return nil, NewError(ErrConflict, "managed_nvram_invalid", "managed domain NVRAM path is missing", nil) } expected := make(map[string]struct{}, len(managedNVRAMArtifactNames(uuid))) for _, name := range managedNVRAMArtifactNames(uuid) { expected[filepath.Clean(filepath.Join(directory, name))] = struct{}{} } result := make([]string, 0, len(paths)) for path := range paths { if !filepath.IsAbs(path) { return nil, NewError(ErrConflict, "managed_nvram_invalid", "managed domain NVRAM path is invalid", nil) } if _, ok := expected[path]; !ok { return nil, NewError(ErrConflict, "managed_nvram_path_mismatch", "managed domain NVRAM path is outside the UUID-owned location", nil) } result = append(result, path) } sort.Strings(result) return result, nil } func (l *Libvirt) otherDomainUsesFilter(filterName, excludedUUID string) (bool, error) { filterReferenced, err := l.otherFilterReferencesFilter(filterName) if err != nil || filterReferenced { return filterReferenced, err } domains, err := l.conn.ListAllDomains(0) if err != nil { return false, translateLibvirtError("list domains for network filter reference check", err) } for index := range domains { domain := &domains[index] candidateUUID, uuidErr := domain.GetUUIDString() if uuidErr != nil { _ = domain.Free() return false, translateLibvirtError("read domain UUID for network filter reference", uuidErr) } if excludedUUID != "" && strings.EqualFold(candidateUUID, excludedUUID) { _ = domain.Free() continue } flags := []libvirt.DomainXMLFlags{0} persistent, persistentErr := domain.IsPersistent() if persistentErr != nil { _ = domain.Free() return false, translateLibvirtError("inspect domain persistence for network filter reference", persistentErr) } if persistent { flags = append(flags, libvirt.DOMAIN_XML_INACTIVE) } for _, flag := range flags { document, xmlErr := domain.GetXMLDesc(flag) if xmlErr != nil { _ = domain.Free() return false, translateLibvirtError("read domain XML for network filter reference", xmlErr) } var definition libvirtxml.Domain if err := definition.Unmarshal(document); err != nil { _ = domain.Free() return false, NewError(ErrConflict, "domain_xml_invalid", "domain XML could not be inspected for network filter references", err) } if domainReferencesFilter(&definition, filterName) { _ = domain.Free() return true, nil } } referenced, historyErr := domainHistoryReferencesFilter(domain, filterName) if historyErr != nil { _ = domain.Free() return false, historyErr } if referenced { _ = domain.Free() return true, nil } _ = domain.Free() } return false, nil } func domainHistoryReferencesFilter(domain *libvirt.Domain, filterName string) (bool, error) { managedSave, present, err := managedSaveDomainDefinition(domain) if err != nil { return false, err } if present && domainReferencesFilter(managedSave, filterName) { return true, nil } snapshots, err := domain.ListAllSnapshots(0) if err != nil { return false, translateLibvirtError("list domain snapshots for network filter reference check", err) } defer func() { for index := range snapshots { _ = snapshots[index].Free() } }() for index := range snapshots { document, xmlErr := snapshots[index].GetXMLDesc(0) if xmlErr != nil { return false, translateLibvirtError("read domain snapshot XML for network filter reference", xmlErr) } var definition libvirtxml.DomainSnapshot if err := definition.Unmarshal(document); err != nil { return false, NewError(ErrConflict, "domain_snapshot_xml_invalid", "domain snapshot XML could not be inspected for network filter references", err) } if domainSnapshotReferencesFilter(&definition, filterName) { return true, nil } } checkpoints, err := domain.ListAllCheckpoints(0) if err != nil { return false, translateLibvirtError("list domain checkpoints for network filter reference check", err) } defer func() { for index := range checkpoints { _ = checkpoints[index].Free() } }() for index := range checkpoints { document, xmlErr := checkpoints[index].GetXMLDesc(0) if xmlErr != nil { return false, translateLibvirtError("read domain checkpoint XML for network filter reference", xmlErr) } var definition libvirtxml.DomainCheckpoint if err := definition.Unmarshal(document); err != nil { return false, NewError(ErrConflict, "domain_checkpoint_xml_invalid", "domain checkpoint XML could not be inspected for network filter references", err) } if definition.Domain != nil && domainReferencesFilter(definition.Domain, filterName) { return true, nil } } return false, nil } func (l *Libvirt) otherFilterReferencesFilter(filterName string) (bool, error) { filters, err := l.conn.ListAllNWFilters(0) if err != nil { return false, translateLibvirtError("list network filters for dependency check", err) } defer func() { for index := range filters { _ = filters[index].Free() } }() for index := range filters { filter := &filters[index] name, nameErr := filter.GetName() if nameErr != nil { return false, translateLibvirtError("read network filter name for dependency check", nameErr) } if name == filterName { continue } document, xmlErr := filter.GetXMLDesc(0) if xmlErr != nil { return false, translateLibvirtError("read network filter dependency XML", xmlErr) } var definition libvirtxml.NWFilter if err := definition.Unmarshal(document); err != nil { return false, NewError(ErrConflict, "network_filter_xml_invalid", "network filter dependencies could not be inspected", err) } if networkFilterReferencesFilter(&definition, filterName) { return true, nil } } return false, nil } func networkFilterReferencesFilter(definition *libvirtxml.NWFilter, filterName string) bool { if definition == nil || filterName == "" { return false } for _, entry := range definition.Entries { if entry.Ref != nil && entry.Ref.Filter == filterName { return true } } return false } func domainReferencesFilter(definition *libvirtxml.Domain, filterName string) bool { if definition == nil || definition.Devices == nil { return false } for _, iface := range definition.Devices.Interfaces { if iface.FilterRef != nil && iface.FilterRef.Filter == filterName { return true } } return false } func domainSnapshotReferencesFilter(snapshot *libvirtxml.DomainSnapshot, filterName string) bool { if snapshot == nil { return false } if snapshot.Domain != nil && domainReferencesFilter(snapshot.Domain, filterName) { return true } return snapshot.InactiveDomain != nil && domainReferencesFilter(&snapshot.InactiveDomain.Domain, filterName) } func (l *Libvirt) deleteDomainAbsent(ctx context.Context, uuid string) (bool, error) { if err := ctx.Err(); err != nil { return false, err } if !uuidPattern.MatchString(uuid) { return false, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil) } domain, err := l.conn.LookupDomainByUUIDString(uuid) if err == nil { _ = domain.Free() return false, nil } if isNoDomain(err) { return true, nil } return false, translateLibvirtError("confirm deleted domain absence", err) } func isNoDomain(err error) bool { var libvirtErr libvirt.Error return errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_DOMAIN } func domainMayExistAfterDefine(inspectErr error) bool { return inspectErr == nil || !isNoDomain(inspectErr) } func collectVMManagedVolumeNames(uuid string, metadata *domainMetadataXML, inventory []string) ([]string, error) { if !uuidPattern.MatchString(uuid) { return nil, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil) } names := make(map[string]struct{}) for _, name := range inventory { if isVMOwnedManagedVolumeName(uuid, name) { names[name] = struct{}{} } } if metadata != nil { if !isVMOwnedRootVolumeName(uuid, metadata.RootVolumeName) { return nil, NewError(ErrConflict, "managed_metadata_invalid", "managed root volume is not owned by the VM UUID", nil) } names[metadata.RootVolumeName] = struct{}{} if metadata.PendingCleanupVolume != "" { if !isVMOwnedRootVolumeName(uuid, metadata.PendingCleanupVolume) { return nil, NewError(ErrConflict, "managed_metadata_invalid", "pending cleanup volume is not owned by the VM UUID", nil) } names[metadata.PendingCleanupVolume] = struct{}{} } for _, snapshot := range metadata.Snapshots { name := snapshotVolumeName(uuid, snapshot.Key) if !isVMOwnedManagedVolumeName(uuid, name) { return nil, NewError(ErrConflict, "managed_metadata_invalid", "snapshot volume is not owned by the VM UUID", nil) } names[name] = struct{}{} } } result := make([]string, 0, len(names)) for name := range names { result = append(result, name) } sort.Strings(result) return result, nil } func (l *Libvirt) lookupOptionalManagedVolume(poolName, volumeName string) (*libvirt.StorageVol, error) { pool, err := l.conn.LookupStoragePoolByName(poolName) if err != nil { return nil, translateLibvirtError("lookup managed storage pool", err) } defer pool.Free() volume, err := pool.LookupStorageVolByName(volumeName) if err == nil { return volume, nil } var libvirtErr libvirt.Error if errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_STORAGE_VOL { return nil, nil } return nil, translateLibvirtError("lookup managed storage volume", err) } func (l *Libvirt) lookupOptionalNetworkFilter(name string) (*libvirt.NWFilter, error) { filter, err := l.conn.LookupNWFilterByName(name) if err == nil { return filter, nil } var libvirtErr libvirt.Error if errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_NWFILTER { return nil, nil } return nil, translateLibvirtError("lookup VM network filter", err) } func (l *Libvirt) Power(ctx context.Context, uuid string, request PowerRequest) (_ VMState, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() domain, err := l.domainByUUID(ctx, uuid) if err != nil { return VMState{}, err } defer domain.Free() if _, _, err := requireManagedDomain(domain); err != nil { return VMState{}, err } if powerActionRequiresPublicGuard(request.Action) { if err := l.reconcilePublicGuard(ctx, nil); err != nil { return VMState{}, operation.MarkNotApplied(NewError( ErrUnavailable, "public_guard_unavailable", "public network guard could not be confirmed before the power action", err, )) } } mutationStarted = true if err := dispatchPowerMutation(domain, request.Action, func() error { return l.startManagedDomain(ctx, domain, "perform power start") }); err != nil { return VMState{}, err } name, _ := domain.GetName() if request.Action == PowerShutdown || request.Action == PowerReboot || request.Action == PowerReset { // libvirt returning success is the durable effect for command-style // lifecycle requests. The guest may remain running while ACPI shutdown // or reboot is still in progress, so an immediate state sample is // informational and can never turn an accepted command into failure. result := VMState{UUID: uuid, Name: name, State: VMStateUnknown, CommandAccepted: true} if state, reason, stateErr := domain.GetState(); stateErr == nil { result.State = mapDomainState(state) result.Reason = reason } return result, nil } state, reason, err := domain.GetState() if err != nil { return VMState{}, operation.MarkUncertain(translateLibvirtError("confirm power action", err)) } mapped := mapDomainState(state) if err := confirmPowerTargetState(request.Action, mapped); err != nil { return VMState{}, err } return VMState{UUID: uuid, Name: name, State: mapped, Reason: reason, CommandAccepted: true, EffectConfirmed: true}, nil } func powerActionRequiresPublicGuard(action PowerAction) bool { return action == PowerStart || action == PowerReboot || action == PowerReset } type powerDomainControl interface { Create() error ShutdownFlags(libvirt.DomainShutdownFlags) error Reboot(libvirt.DomainRebootFlagValues) error Reset(uint32) error DestroyFlags(libvirt.DomainDestroyFlags) error } func dispatchPowerMutation(domain powerDomainControl, action PowerAction, start func() error) error { var err error switch action { case PowerStart: if start == nil { return operation.MarkNotApplied(NewError(ErrUnavailable, "public_guard_unavailable", "power start guard is unavailable", nil)) } err = start() case PowerShutdown: err = domain.ShutdownFlags(libvirt.DOMAIN_SHUTDOWN_ACPI_POWER_BTN) case PowerReboot: err = domain.Reboot(libvirt.DOMAIN_REBOOT_ACPI_POWER_BTN) case PowerReset: err = domain.Reset(0) case PowerDestroy: err = domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT) default: return operation.MarkNotApplied(NewError(ErrInvalid, "invalid_power_action", "unknown power action", nil)) } if err != nil { if operation.IsNotApplied(err) { return err } // A libvirt lifecycle call can take effect before its transport reports // an error. Keep the exact operation reconcilable instead of recording a // terminal failure that could invite a second destructive request. return operation.MarkUncertain(mutationLibvirtError("perform power action", err)) } return nil } func confirmPowerTargetState(action PowerAction, state VMStateName) error { reached := true switch action { case PowerStart: reached = state == VMStateRunning || state == VMStateBlocked case PowerDestroy: reached = state == VMStateShutoff } if !reached { return operation.MarkUncertain(NewError( ErrConflict, "power_state_unconfirmed", "power action did not reach the requested target state", nil, )) } return nil } func (l *Libvirt) GetConsole(ctx context.Context, uuid string) (ConsoleSession, error) { domain, err := l.domainByUUID(ctx, uuid) if err != nil { return ConsoleSession{}, err } defer domain.Free() active, err := domain.IsActive() if err != nil { return ConsoleSession{}, translateLibvirtError("check console domain activity", err) } if !active { return ConsoleSession{}, NewError(ErrNotFound, "console_not_available", "VM has no active local VNC console", nil) } document, err := domain.GetXMLDesc(0) if err != nil { return ConsoleSession{}, translateLibvirtError("get live console XML", err) } metadata := struct { Devices struct { Graphics []struct { Type string `xml:"type,attr"` Port int `xml:"port,attr"` } `xml:"graphics"` } `xml:"devices"` }{} if err := xml.Unmarshal([]byte(document), &metadata); err != nil { return ConsoleSession{}, fmt.Errorf("parse console XML: %w", err) } for _, graphic := range metadata.Devices.Graphics { if graphic.Type == "vnc" && graphic.Port > 0 && graphic.Port <= 65535 { return ConsoleSession{Protocol: "vnc", Address: net.JoinHostPort("127.0.0.1", strconv.Itoa(graphic.Port))}, nil } } return ConsoleSession{}, NewError(ErrNotFound, "console_not_available", "VM has no active local VNC console", nil) } func resolvedDeleteSuccess(result DeleteVMResult) operation.Resolution { encoded, _ := json.Marshal(result) return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded} } // cleanupDeletedVMArtifacts resumes the idempotent tail of a delete after the // domain definition is already gone. Every target is derived from the VM UUID, // validated as Agent-owned, and checked for cross-domain references first. func (l *Libvirt) cleanupDeletedVMArtifacts(ctx context.Context, uuid string) error { if err := ctx.Err(); err != nil { return err } unlockTrafficSample := l.traffic.lockSample(uuid) defer unlockTrafficSample() pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged)) if err != nil { return translateLibvirtError("lookup managed storage pool", err) } defer pool.Free() inventory, err := pool.ListStorageVolumes() if err != nil { return translateLibvirtError("list managed volumes for VM cleanup", err) } names, err := collectVMManagedVolumeNames(uuid, nil, inventory) if err != nil { return err } for _, name := range names { if err := ctx.Err(); err != nil { return err } volume, lookupErr := pool.LookupStorageVolByName(name) if isNoStorageVolume(lookupErr) { continue } if lookupErr != nil { return translateLibvirtError("lookup managed VM cleanup volume", lookupErr) } if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil { _ = volume.Free() return validateErr } path, pathErr := volume.GetPath() if pathErr != nil { _ = volume.Free() return translateLibvirtError("resolve managed VM cleanup volume", pathErr) } referenced, referenceErr := l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), name, ) if referenceErr != nil { _ = volume.Free() return referenceErr } if referenced { _ = volume.Free() return NewError(ErrConflict, "managed_disk_shared", "managed VM cleanup volume is referenced by a domain", nil) } deleteErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) _ = volume.Free() if deleteErr != nil { observed, inspectErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), name) if observed != nil { _ = observed.Free() } if inspectErr != nil || observed != nil { return operation.MarkUncertain(errors.Join( mutationLibvirtError("delete managed VM cleanup volume "+name, deleteErr), inspectErr, )) } } } seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid)) if err != nil { return err } if seed != nil { path, pathErr := seed.GetPath() if pathErr != nil { _ = seed.Free() return translateLibvirtError("resolve managed NoCloud cleanup volume", pathErr) } referenced, referenceErr := l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), ) if referenceErr != nil { _ = seed.Free() return referenceErr } if referenced { _ = seed.Free() return NewError(ErrConflict, "managed_disk_shared", "managed NoCloud cleanup volume is referenced by a domain", nil) } deleteErr := seed.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) _ = seed.Free() if deleteErr != nil { observed, inspectErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid)) if observed != nil { _ = observed.Free() } if inspectErr != nil || observed != nil { return operation.MarkUncertain(errors.Join( mutationLibvirtError("delete NoCloud cleanup volume", deleteErr), inspectErr, )) } } } filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid)) if err != nil { return err } if filter != nil { document, xmlErr := filter.GetXMLDesc(0) if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) { _ = filter.Free() return NewError(ErrConflict, "managed_filter_invalid", "network filter cannot be confirmed as panel managed", xmlErr) } referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), "") if referenceErr != nil { _ = filter.Free() return referenceErr } if referenced { _ = filter.Free() return NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by a domain", nil) } undefineErr := filter.Undefine() _ = filter.Free() if undefineErr != nil { observed, inspectErr := l.lookupOptionalNetworkFilter(networkFilterName(uuid)) if observed != nil { _ = observed.Free() } if inspectErr != nil || observed != nil { return operation.MarkUncertain(errors.Join( mutationLibvirtError("undefine VM cleanup network filter", undefineErr), inspectErr, )) } } } for _, name := range managedNVRAMArtifactNames(uuid) { path := filepath.Join(l.local.NVRAMDirectory, name) referenced, referenceErr := l.anyDomainUsesPath(path) if referenceErr != nil { return referenceErr } if referenced { return NewError(ErrConflict, "managed_nvram_shared", "managed VM firmware state is referenced by another domain", nil) } } if err := removeManagedNVRAMArtifacts(l.local.NVRAMDirectory, uuid); err != nil { return NewError(ErrUnavailable, "nvram_cleanup_pending", "managed VM firmware cleanup is incomplete", err) } if err := l.traffic.remove(uuid); err != nil { return NewError(ErrUnavailable, "traffic_state_unavailable", "deleted VM traffic state could not be removed", err) } return nil } func (l *Libvirt) deleteArtifactsAbsent(uuid string) (bool, error) { if !uuidPattern.MatchString(uuid) { return false, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil) } pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged)) if err != nil { return false, translateLibvirtError("lookup managed storage pool", err) } defer pool.Free() rootAbsent, err := l.dirPoolVolumeAbsent(pool, rootVolumeName(uuid)) if err != nil { return false, err } if !rootAbsent { return false, nil } inventory, err := pool.ListStorageVolumes() if err != nil { return false, translateLibvirtError("list managed volumes for reconciliation", err) } owned, err := collectVMManagedVolumeNames(uuid, nil, inventory) if err != nil { return false, err } if len(owned) != 0 { return false, nil } seedPool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleSeed)) if err != nil { return false, translateLibvirtError("lookup seed storage pool", err) } defer seedPool.Free() seedAbsent, err := l.dirPoolVolumeAbsent(seedPool, seedVolumeName(uuid)) if err != nil { return false, err } if !seedAbsent { return false, nil } filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid)) if err != nil { return false, err } if filter != nil { _ = filter.Free() return false, nil } nvramAbsent, err := managedNVRAMArtifactAbsent(l.local.NVRAMDirectory, uuid) if err != nil { return false, NewError(ErrUnavailable, "nvram_state_unavailable", "managed VM firmware state could not be inspected", err) } if !nvramAbsent { return false, nil } return true, nil } func managedNVRAMArtifactAbsent(directory, uuid string) (bool, error) { candidates := managedNVRAMArtifactNames(uuid) for _, name := range candidates { if _, err := os.Lstat(filepath.Join(directory, name)); err == nil { return false, nil } else if !errors.Is(err, os.ErrNotExist) { return false, err } } return true, nil } func managedNVRAMArtifactNames(uuid string) []string { normalized := strings.ToLower(uuid) candidates := []string{normalized + "_VARS.fd"} if uuid != normalized { // CreateVM historically accepted an uppercase UUID and used its exact // spelling for the NVRAM filename. Check both spellings so absence is // never claimed while either managed artifact remains. candidates = append(candidates, uuid+"_VARS.fd") } return candidates } func removeManagedNVRAMArtifacts(directory, uuid string) error { for _, name := range managedNVRAMArtifactNames(uuid) { path := filepath.Join(directory, name) info, err := os.Lstat(path) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { return err } if !info.Mode().IsRegular() { return fmt.Errorf("managed NVRAM artifact is not a regular file") } if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } return nil } func (l *Libvirt) domainByUUID(ctx context.Context, uuid string) (*libvirt.Domain, error) { if err := ctx.Err(); err != nil { return nil, err } if !uuidPattern.MatchString(uuid) { return nil, NewError(ErrInvalid, "invalid_uuid", "uuid must be canonical", nil) } domain, err := l.conn.LookupDomainByUUIDString(uuid) if err != nil { return nil, translateLibvirtError("lookup domain by UUID", err) } if _, _, err := requireManagedDomain(domain); err != nil { domain.Free() return nil, err } return domain, nil } func waitForShutoff(ctx context.Context, domain *libvirt.Domain) error { ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() for { state, _, err := domain.GetState() if err != nil { return translateLibvirtError("wait for shutdown", err) } if state == libvirt.DOMAIN_SHUTOFF { return nil } select { case <-ctx.Done(): return ctx.Err() case <-ticker.C: } } } func mapDomainState(state libvirt.DomainState) VMStateName { switch state { case libvirt.DOMAIN_RUNNING: return VMStateRunning case libvirt.DOMAIN_BLOCKED: return VMStateBlocked case libvirt.DOMAIN_PAUSED: return VMStatePaused case libvirt.DOMAIN_SHUTDOWN: return VMStateShutdown case libvirt.DOMAIN_SHUTOFF: return VMStateShutoff case libvirt.DOMAIN_CRASHED: return VMStateCrashed case libvirt.DOMAIN_PMSUSPENDED: return VMStateSuspended default: return VMStateUnknown } } func translateLibvirtError(action string, err error) error { var libvirtErr libvirt.Error if errors.As(err, &libvirtErr) { switch libvirtErr.Code { case libvirt.ERR_NO_DOMAIN: return NewError(ErrNotFound, "vm_not_found", "virtual machine was not found", err) case libvirt.ERR_NO_DOMAIN_SNAPSHOT: return NewError(ErrNotFound, "snapshot_not_found", "virtual machine snapshot was not found", err) case libvirt.ERR_NO_STORAGE_VOL: return NewError(ErrNotFound, "storage_volume_not_found", "storage volume was not found", err) case libvirt.ERR_NO_STORAGE_POOL: return NewError(ErrUnavailable, "storage_pool_unavailable", "configured storage pool was not found", err) case libvirt.ERR_NO_NWFILTER: return NewError(ErrNotFound, "firewall_filter_not_found", "network filter was not found", err) case libvirt.ERR_DOM_EXIST, libvirt.ERR_STORAGE_VOL_EXIST, libvirt.ERR_OPERATION_INVALID, libvirt.ERR_OPERATION_DENIED: return NewError(ErrConflict, "libvirt_conflict", action+" conflicts with current hypervisor state", err) case libvirt.ERR_INVALID_ARG: return NewError(ErrInvalid, "invalid_libvirt_request", action+" was rejected as invalid", err) case libvirt.ERR_OPERATION_UNSUPPORTED: return NewError(ErrUnsupported, "unsupported", action+" is not supported", err) case libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_AUTH_FAILED: return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", err) } } return NewError(ErrUnavailable, "libvirt_error", fmt.Sprintf("%s failed", action), err) } func mutationLibvirtError(action string, err error) error { var libvirtErr libvirt.Error if errors.As(err, &libvirtErr) { switch libvirtErr.Code { case libvirt.ERR_SYSTEM_ERROR, libvirt.ERR_RPC, libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_OPERATION_TIMEOUT: return operation.MarkUncertain(translateLibvirtError(action, err)) } } return translateLibvirtError(action, err) } func formatLibvirtVersion(version uint32) string { return fmt.Sprintf("%d.%d.%d", version/1_000_000, (version/1_000)%1_000, version%1_000) }