//go:build libvirt package hypervisor import ( "context" "encoding/json" "encoding/xml" "errors" "fmt" "path/filepath" "strings" "time" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/publicguard" "git.luoli.live/root/virt/agent/publicnetwork" "libvirt.org/go/libvirt" "libvirt.org/go/libvirtxml" ) func expectedMutationState(ctx context.Context, kind string) (operation.ExpectedState, error) { expected, ok := operation.ExpectedStateFromContext(ctx) if !ok || expected.Kind != kind || !operationIdentifierPattern.MatchString(expected.TransitionID) { return operation.ExpectedState{}, NewError(ErrConflict, "reconciliation_descriptor_missing", "durable mutation reconciliation descriptor is missing", nil) } return expected, nil } func (l *Libvirt) UpdateSettings(ctx context.Context, uuid string, request SettingsRequest) (_ VMMetadata, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() if !request.ConfirmRestart { return VMMetadata{}, NewError(ErrInvalid, "restart_confirmation_required", "confirm_restart must be true", nil) } if request.Hostname == nil && request.RootPassword == nil && request.SSHPublicKey == nil { return VMMetadata{}, NewError(ErrInvalid, "empty_settings", "at least one setting is required", nil) } expected, err := expectedMutationState(ctx, operation.ExpectedUpdateSettings) if err != nil { return VMMetadata{}, err } expectedHostname := "" if request.Hostname != nil { expectedHostname = *request.Hostname } if expected.HostnameChanged != (request.Hostname != nil) || expected.Hostname != expectedHostname || expected.RootPasswordChanged != (request.RootPassword != nil) || expected.SSHKeyChanged != (request.SSHPublicKey != nil) { return VMMetadata{}, NewError(ErrConflict, "reconciliation_descriptor_mismatch", "settings request does not match its durable reconciliation descriptor", nil) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return VMMetadata{}, err } defer domain.Free() if err := requirePersistentWithoutSnapshots(domain); err != nil { return VMMetadata{}, err } seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid)) if err != nil { return VMMetadata{}, err } if seed == nil { return VMMetadata{}, NewError(ErrConflict, "seed_not_found", "managed NoCloud seed volume was not found", nil) } defer seed.Free() originalSeed, err := l.downloadSmallVolume(ctx, seed, maxSeedImageBytes) if err != nil { return VMMetadata{}, err } replacement, hostname, err := mutateSeedSettings(originalSeed, request, expected.TransitionID) if err != nil { return VMMetadata{}, err } mutationStarted = true wasRunning, err := stopForDefinitionChange(ctx, domain) if err != nil { return VMMetadata{}, err } originalXML, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if err != nil { return VMMetadata{}, operation.MarkUncertain(translateLibvirtError("read stopped domain XML", err)) } updatedXML, err := mutateManagedMetadata(originalXML, func(metadata *domainMetadataXML) error { if request.Hostname != nil { metadata.Hostname = hostname } metadata.SettingsTransitionID = expected.TransitionID return nil }) if err != nil { return VMMetadata{}, err } if err := l.replaceSeedAndDefinition(ctx, seed, originalSeed, replacement, originalXML, updatedXML, wasRunning, nil); err != nil { return VMMetadata{}, err } metadata, err := metadataFromXML(updatedXML) if err != nil { return VMMetadata{}, operation.MarkUncertain(err) } metadata.Status = VMStateShutoff if wasRunning { metadata.Status = VMStateRunning } metadata.SettingsChanged = true return metadata, nil } func (l *Libvirt) ChangeNetwork(ctx context.Context, uuid string, request ChangeNetworkRequest) (_ VMMetadata, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() if !request.ConfirmRestart { return VMMetadata{}, NewError(ErrInvalid, "restart_confirmation_required", "confirm_restart must be true", nil) } if err := ValidatePublicNetworkIntent(request.PublicNetwork); err != nil { return VMMetadata{}, err } expected, err := expectedMutationState(ctx, operation.ExpectedChangeNetwork) if err != nil { return VMMetadata{}, err } if !publicNetworkEqual(expected.PublicNetwork, request.PublicNetwork) { return VMMetadata{}, NewError(ErrConflict, "reconciliation_descriptor_mismatch", "network request does not match its durable reconciliation descriptor", nil) } source, err := l.ResolveNetworkSelector(ctx, request.PublicNetwork.NetworkSelector) if err != nil { return VMMetadata{}, err } if expected.NetworkSourceType != source.Type || expected.NetworkSourceName != source.Source { return VMMetadata{}, NewError(ErrConflict, "reconciliation_descriptor_mismatch", "network source does not match its durable reconciliation descriptor", nil) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return VMMetadata{}, err } defer domain.Free() if err := requirePersistentWithoutSnapshots(domain); err != nil { return VMMetadata{}, err } preflightXML, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if err != nil { return VMMetadata{}, translateLibvirtError("read domain network filter binding", err) } preflightDefinition, preflightMetadata, err := parseManagedDomain(preflightXML) if err != nil { return VMMetadata{}, err } if !managedNetworkLayoutMatches(preflightDefinition, preflightMetadata) { return VMMetadata{}, NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its metadata", nil) } publicIndex, _ := managedPublicInterfaceIndex( preflightDefinition, networkFilterName(uuid), preflightMetadata.PublicNetwork.publicMust(), ) if !managedInterfaceFilterIPMatches( &preflightDefinition.Devices.Interfaces[publicIndex], networkFilterName(uuid), preflightMetadata.PublicNetwork.publicMust(), ) { return VMMetadata{}, NewError(ErrConflict, "network_filter_binding_changed", "managed VM network filter IP binding does not match its recorded public address", nil) } publicInterface := preflightDefinition.Devices.Interfaces[publicIndex] if publicInterface.MAC == nil { return VMMetadata{}, NewError(ErrConflict, "network_identity_changed", "managed VM public interface has no MAC address", nil) } if err := ensureTargetNetworkIdentityAvailable( ctx, uuid, strings.ToLower(publicInterface.MAC.Address), request.PublicNetwork, l.networkIdentityConflict, ); err != nil { return VMMetadata{}, err } seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid)) if err != nil { return VMMetadata{}, err } if seed == nil { return VMMetadata{}, NewError(ErrConflict, "seed_not_found", "managed NoCloud seed volume was not found", nil) } defer seed.Free() originalSeed, err := l.downloadSmallVolume(ctx, seed, maxSeedImageBytes) if err != nil { return VMMetadata{}, err } replacement, mac, err := mutateSeedNetwork(originalSeed, request.PublicNetwork, expected.TransitionID) if err != nil { return VMMetadata{}, err } targetGuard, err := publicGuardBinding(uuid, mac, request.PublicNetwork) if err != nil { return VMMetadata{}, err } guardLease, err := l.acquirePublicGuardTransition(ctx) if err != nil { return VMMetadata{}, err } defer guardLease.Release() defer func() { reconcileCtx, cancelReconcile := storageRollbackContext(ctx) guardErr := l.reconcilePublicGuardLease(reconcileCtx, guardLease, nil) cancelReconcile() if guardErr != nil { resultErr = operation.MarkUncertain(errors.Join(resultErr, NewError( ErrUnavailable, "public_guard_reconcile_failed", "public network guard could not be reconciled after network change", guardErr, ))) } }() mutationStarted = true wasRunning, err := stopForDefinitionChange(ctx, domain) if err != nil { return VMMetadata{}, err } originalXML, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if err != nil { return VMMetadata{}, operation.MarkUncertain(translateLibvirtError("read stopped domain XML", err)) } updatedXML, err := mutateDomainDocument(originalXML, func(document *libvirtxml.Domain, metadata *domainMetadataXML) error { if !managedNetworkLayoutMatches(document, metadata) { return NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its metadata", nil) } publicIndex, _ := managedPublicInterfaceIndex( document, networkFilterName(uuid), metadata.PublicNetwork.publicMust(), ) iface := &document.Devices.Interfaces[publicIndex] if iface.MAC == nil || !strings.EqualFold(iface.MAC.Address, mac) { return NewError(ErrConflict, "network_identity_changed", "domain and NoCloud interface identities do not match", nil) } if err := updateManagedInterfaceFilterIP( iface, networkFilterName(uuid), metadata.PublicNetwork.publicMust(), request.PublicNetwork, ); err != nil { return err } iface.Source = &libvirtxml.DomainInterfaceSource{} switch source.Type { case "bridge": iface.Source.Bridge = &libvirtxml.DomainInterfaceSourceBridge{Bridge: source.Source} case "network": iface.Source.Network = &libvirtxml.DomainInterfaceSourceNetwork{Network: source.Source} default: return NewError(ErrUnavailable, "invalid_host_network_config", "host network selector has an invalid source type", nil) } metadata.PublicNetwork = domainPublicNetworkFromPublic(request.PublicNetwork) metadata.NetworkTransitionID = expected.TransitionID return nil }) if err != nil { return VMMetadata{}, err } if err := l.reconcilePublicGuardLease(ctx, guardLease, &targetGuard); err != nil { rollbackCtx, cancelRollback := storageRollbackContext(ctx) defer cancelRollback() reconcileErr := l.reconcilePublicGuardLease(rollbackCtx, guardLease, nil) if reconcileErr != nil { // The target guard may have committed before its response was lost. // Keep the domain stopped until the projection from the still-current // definition is restored; restarting here would risk an unguarded or // incorrectly guarded public interface. return VMMetadata{}, operation.MarkUncertain(errors.Join( NewError(ErrUnavailable, "public_guard_apply_failed", "target public network guard could not be applied", err), NewError(ErrUnavailable, "public_guard_reconcile_failed", "original public network guard could not be restored", reconcileErr), )) } if wasRunning { if restartErr := l.startManagedDomainWithLease(rollbackCtx, guardLease, domain, "restart original domain after public guard failure"); restartErr != nil { return VMMetadata{}, operation.MarkUncertain(errors.Join( NewError(ErrUnavailable, "public_guard_apply_failed", "target public network guard could not be applied", err), mutationLibvirtError("restart original domain after public guard failure", restartErr), )) } } return VMMetadata{}, NewError(ErrUnavailable, "public_guard_apply_failed", "target public network guard could not be applied", err) } replaceErr := l.replaceSeedAndDefinition(ctx, seed, originalSeed, replacement, originalXML, updatedXML, wasRunning, guardLease) if replaceErr != nil { return VMMetadata{}, replaceErr } metadata, err := metadataFromXML(updatedXML) if err != nil { return VMMetadata{}, operation.MarkUncertain(err) } metadata.Status = VMStateShutoff if wasRunning { metadata.Status = VMStateRunning } metadata.NetworkChanged = true return metadata, nil } func ensureTargetNetworkIdentityAvailable(ctx context.Context, uuid, mac string, public publicnetwork.Intent, check func(context.Context, string, string, publicnetwork.Intent) (bool, error)) error { if check == nil { return NewError(ErrUnavailable, "network_identity_check_unavailable", "VM network identity inventory is unavailable", nil) } conflict, err := check(ctx, uuid, mac, public) if err != nil { return err } if conflict { return NewError(ErrConflict, "network_identity_conflict", "VM network identity is already in use", nil) } return nil } func (l *Libvirt) ResolveNetworkSelector(ctx context.Context, selector string) (LocalNetworkSource, error) { if err := ctx.Err(); err != nil { return LocalNetworkSource{}, err } return resolveNetworkSelector(selector, l.probeNetworkSource) } func (l *Libvirt) replaceSeedAndDefinition(ctx context.Context, seed *libvirt.StorageVol, originalSeed, replacement []byte, originalXML, updatedXML string, restart bool, guardLease *publicguard.Lease) error { info, err := seed.GetInfo() if err != nil { return translateLibvirtError("read NoCloud volume capacity", err) } if uint64(len(replacement)) > info.Capacity { if err := seed.Resize(uint64(len(replacement)), 0); err != nil { return operation.MarkUncertain(mutationLibvirtError("grow NoCloud volume", err)) } } seedChanged := false definitionChanged := false if err := l.uploadVolume(ctx, seed, replacement); err != nil { return operation.MarkUncertain(err) } seedChanged = true defined, err := l.conn.DomainDefineXML(updatedXML) if err == nil { definitionChanged = true defer defined.Free() if restart { if guardLease != nil { err = l.startManagedDomainWithLease(ctx, guardLease, defined, "start updated domain") } else { err = l.startManagedDomain(ctx, defined, "start updated domain") } } } if err == nil { return nil } if definitionChanged && restart { active, inspectErr := defined.IsActive() if inspectErr != nil || active { return operation.MarkUncertain(errors.Join(mutationLibvirtError("start updated domain", err), translateOptionalLibvirtError("inspect updated domain after start failure", inspectErr))) } } rollbackCtx, cancelRollback := storageRollbackContext(ctx) defer cancelRollback() original, rollbackErr := l.conn.DomainDefineXML(originalXML) if rollbackErr == nil { defer original.Free() if seedChanged { rollbackErr = l.uploadVolume(rollbackCtx, seed, originalSeed) } if rollbackErr == nil && restart { if guardLease != nil { rollbackErr = l.startManagedDomainWithLease(rollbackCtx, guardLease, original, "restart original domain after rollback") } else { rollbackErr = l.startManagedDomain(rollbackCtx, original, "restart original domain after rollback") } } } if rollbackErr != nil || definitionChanged { return operation.MarkUncertain(errors.Join(mutationLibvirtError("apply VM definition", err), optionalMutationLibvirtError("roll back VM definition", rollbackErr))) } return mutationLibvirtError("apply VM definition", err) } func (l *Libvirt) SetNetworkState(ctx context.Context, uuid string, enabled bool, _ NetworkStateRequest) (resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() expected, err := expectedMutationState(ctx, operation.ExpectedNetworkState) if err != nil { return err } if expected.Enabled != enabled { return NewError(ErrConflict, "reconciliation_descriptor_mismatch", "network state does not match its durable reconciliation descriptor", nil) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return err } defer domain.Free() if _, _, err := requireManagedDomain(domain); err != nil { return err } persistent, err := domain.IsPersistent() if err != nil { return translateLibvirtError("check domain persistence", err) } if !persistent { return NewError(ErrConflict, "persistent_vm_required", "network state can only be changed for a persistent VM", nil) } if err := ensureNoActiveDomainJobIfRunning(domain); err != nil { return err } inactive, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if err != nil { return translateLibvirtError("read inactive domain XML", err) } inactiveDefinition, inactiveMetadata, err := parseManagedDomain(inactive) if err != nil { return err } if !managedNetworkLayoutMatches(inactiveDefinition, inactiveMetadata) { return NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its metadata", nil) } documents, err := networkInterfaceLinkDocuments(inactive, enabled) if err != nil { return err } active, err := domain.IsActive() if err != nil { return translateLibvirtError("check domain activity", err) } var guardLease *publicguard.Lease if enabled { guardLease, err = l.acquirePublicGuardTransition(ctx) if err != nil { return err } defer guardLease.Release() if err := l.reconcilePublicGuardLease(ctx, guardLease, nil); err != nil { return NewError(ErrUnavailable, "public_guard_apply_failed", "public network guard is not ready for network restore", err) } } mutationStarted = true for _, document := range documents { if err := domain.UpdateDeviceFlags(document, libvirt.DOMAIN_DEVICE_MODIFY_CONFIG); err != nil { return operation.MarkUncertain(mutationLibvirtError("persist interface link state", err)) } if active { if err := domain.UpdateDeviceFlags(document, libvirt.DOMAIN_DEVICE_MODIFY_LIVE); err != nil { return operation.MarkUncertain(mutationLibvirtError("apply live interface link state", err)) } } } return nil } func networkInterfaceLinkDocuments(document string, enabled bool) ([]string, error) { var definition libvirtxml.Domain if err := definition.Unmarshal(document); err != nil || definition.Devices == nil || len(definition.Devices.Interfaces) == 0 { return nil, NewError(ErrConflict, "network_layout_changed", "domain has no managed network interfaces", err) } state := "down" if enabled { state = "up" } documents := make([]string, 0, len(definition.Devices.Interfaces)) for index := range definition.Devices.Interfaces { iface := definition.Devices.Interfaces[index] iface.Link = &libvirtxml.DomainInterfaceLink{State: state} encoded, err := iface.Marshal() if err != nil { return nil, err } documents = append(documents, encoded) } return documents, nil } func (l *Libvirt) SetServiceState(ctx context.Context, uuid string, enabled bool, request ServiceStateRequest) (_ VMState, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() expected, err := expectedMutationState(ctx, operation.ExpectedServiceState) if err != nil { return VMState{}, err } if expected.Enabled != enabled || expected.StartIfSuspended != request.StartIfSuspended { return VMState{}, NewError(ErrConflict, "reconciliation_descriptor_mismatch", "service state does not match its durable reconciliation descriptor", nil) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return VMState{}, err } defer domain.Free() if _, _, err := requireManagedDomain(domain); err != nil { return VMState{}, err } persistent, err := domain.IsPersistent() if err != nil { return VMState{}, translateLibvirtError("check domain persistence", err) } if !persistent { return VMState{}, NewError(ErrConflict, "persistent_vm_required", "service state can only be changed for a persistent VM", nil) } mutationStarted = true if enabled { if err := setServiceAutostart(domain, true); err != nil { return VMState{}, err } active, err := domain.IsActive() if err != nil { return VMState{}, operation.MarkUncertain(translateLibvirtError("check domain activity after enabling service", err)) } if request.StartIfSuspended && !active { if err := l.startManagedDomain(ctx, domain, "start service-resumed domain"); err != nil { if confirmed, confirmErr := domain.IsActive(); confirmErr != nil || !confirmed { return VMState{}, operation.MarkUncertain(errors.Join( mutationLibvirtError("start service-resumed domain", err), translateOptionalLibvirtError("confirm service-resumed domain", confirmErr), )) } } } } else { if err := setServiceAutostart(domain, false); err != nil { return VMState{}, err } // Service suspension is also a graceful lifecycle request. It must // never silently escalate to DestroyFlags(DEFAULT); the panel can // authorize a separate force-delete operation when appropriate. if err := requestGracefulDomainShutoff(ctx, domain, 30*time.Second); err != nil { return VMState{}, err } } if _, err := confirmServiceTargetState(domain, enabled, request.StartIfSuspended); err != nil { return VMState{}, err } state, reason, err := domain.GetState() if err != nil { return VMState{}, operation.MarkUncertain(translateLibvirtError("confirm service state", err)) } name, _ := domain.GetName() return VMState{UUID: uuid, Name: name, State: mapDomainState(state), Reason: reason}, nil } type serviceStateDomainControl interface { SetAutostart(bool) error GetAutostart() (bool, error) IsActive() (bool, error) } func setServiceAutostart(domain serviceStateDomainControl, enabled bool) error { if err := domain.SetAutostart(enabled); err != nil { label := "enable domain autostart" if !enabled { label = "disable domain autostart" } return operation.MarkUncertain(mutationLibvirtError(label, err)) } return nil } func confirmServiceTargetState(domain serviceStateDomainControl, enabled, startIfSuspended bool) (bool, error) { autostart, err := domain.GetAutostart() if err != nil { return false, operation.MarkUncertain(translateLibvirtError( "confirm service autostart", err, )) } active, err := domain.IsActive() if err != nil { return false, operation.MarkUncertain(translateLibvirtError( "confirm service activity", err, )) } if autostart != enabled || (!enabled && active) || (enabled && startIfSuspended && !active) { return active, operation.MarkUncertain(NewError( ErrConflict, "service_state_unconfirmed", "service state did not reach the requested target", nil, )) } return active, nil } func translateOptionalLibvirtError(label string, err error) error { if err == nil { return nil } return translateLibvirtError(label, err) } func optionalMutationLibvirtError(label string, err error) error { if err == nil { return nil } return mutationLibvirtError(label, err) } func (l *Libvirt) GetMetrics(ctx context.Context, uuid string) (Metrics, error) { unlockSample := l.traffic.lockSample(uuid) defer unlockSample() domain, err := l.domainByUUID(ctx, uuid) if err != nil { return Metrics{}, err } defer domain.Free() info, err := domain.GetInfo() if err != nil { return Metrics{}, translateLibvirtError("read domain metrics", err) } result := Metrics{UUID: uuid, CollectedAt: time.Now().UTC(), CPUTimeNS: info.CpuTime} active, err := domain.IsActive() if err != nil { return Metrics{}, translateLibvirtError("check domain activity for metrics", err) } if !active { totalRX, totalTX, observeErr := l.traffic.observe( uuid, result.CollectedAt, availableTrafficObservation(0, 0), ) if observeErr != nil { return Metrics{}, NewError(ErrUnavailable, "traffic_state_unavailable", "durable traffic counters could not be updated", observeErr) } result.RXBytes, result.TXBytes = totalRX, totalTX return result, nil } document, err := domain.GetXMLDesc(0) if err != nil { return Metrics{}, translateLibvirtError("read live domain XML for metrics", err) } definition, metadata, err := parseManagedDomain(document) if err != nil { return Metrics{}, err } if !managedNetworkLayoutMatches(definition, metadata) { return Metrics{}, NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its metadata", nil) } publicIndex, _ := managedPublicInterfaceIndex( definition, networkFilterName(uuid), metadata.PublicNetwork.publicMust(), ) memoryStats, memoryErr := domain.MemoryStats(uint32(libvirt.DOMAIN_MEMORY_STAT_NR), 0) if memoryErr == nil { usage := memoryUsageStats{} for _, stat := range memoryStats { value := stat.Val switch libvirt.DomainMemoryStatTags(stat.Tag) { case libvirt.DOMAIN_MEMORY_STAT_AVAILABLE: usage.AvailableKiB = &value case libvirt.DOMAIN_MEMORY_STAT_UNUSED: usage.UnusedKiB = &value } } result.MemoryUsedKiB = estimateMemoryUsedKiB(usage) } interfaceSamples := make([]interfaceTrafficSample, 0, 1) if definition.Devices != nil { for _, disk := range definition.Devices.Disks { if disk.Target == nil || disk.Device != "disk" { continue } stats, statErr := domain.BlockStats(disk.Target.Dev) if statErr != nil { return Metrics{}, translateLibvirtError("read domain block metrics", statErr) } if stats.RdBytes > 0 { result.DiskReadBytes += uint64(stats.RdBytes) } if stats.WrBytes > 0 { result.DiskWriteBytes += uint64(stats.WrBytes) } } publicInterface := definition.Devices.Interfaces[publicIndex] if publicInterface.Target == nil || publicInterface.Target.Dev == "" { interfaceSamples = append(interfaceSamples, interfaceTrafficSample{}) } else { stats, statErr := domain.InterfaceStats(publicInterface.Target.Dev) if statErr != nil { return Metrics{}, translateLibvirtError("read public interface network metrics", statErr) } interfaceSamples = append(interfaceSamples, interfaceTrafficSample{ TargetAvailable: true, RXSet: stats.RxBytesSet, RXBytes: stats.RxBytes, TXSet: stats.TxBytesSet, TXBytes: stats.TxBytes, }) } } observation := aggregateInterfaceTraffic(interfaceSamples) totalRX, totalTX, err := l.traffic.observe(uuid, result.CollectedAt, observation) if err != nil { return Metrics{}, NewError(ErrUnavailable, "traffic_state_unavailable", "durable traffic counters could not be updated", err) } result.RXBytes, result.TXBytes = totalRX, totalTX return result, nil } func (l *Libvirt) ListSnapshots(ctx context.Context, uuid string) ([]Snapshot, error) { domain, err := l.domainByUUID(ctx, uuid) if err != nil { return nil, err } defer domain.Free() _, metadata, err := requireManagedDomain(domain) if err != nil { return nil, err } result := make([]Snapshot, 0, len(metadata.Snapshots)) for _, stored := range metadata.Snapshots { state := "ready" volume, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, stored.Key)) if lookupErr != nil { return nil, lookupErr } if volume == nil { state = "missing" } else { info, infoErr := volume.GetInfo() _ = volume.Free() if infoErr != nil { return nil, translateLibvirtError("inspect snapshot volume", infoErr) } if info.Capacity != stored.SizeBytes { state = "invalid" } } result = append(result, Snapshot{ ID: stored.Key, SnapshotKey: stored.Key, Name: stored.Name, Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(), State: state, Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, }) } return result, nil } func (l *Libvirt) GetFirewall(ctx context.Context, uuid string) (FirewallPolicy, error) { domain, err := l.domainByUUID(ctx, uuid) if err != nil { return FirewallPolicy{}, err } defer domain.Free() document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if err != nil { return FirewallPolicy{}, translateLibvirtError("read domain firewall metadata", err) } _, stored, err := parseManagedDomain(document) if err != nil { return FirewallPolicy{}, err } if stored.Firewall == nil { return FirewallPolicy{DefaultIn: "accept", DefaultOut: "accept", Rules: []FirewallRule{}}, nil } return firewallFromMetadata(stored.Firewall), nil } func (l *Libvirt) ApplyFirewall(ctx context.Context, uuid string, policy FirewallPolicy) (_ FirewallPolicy, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() validated, err := validateFirewallPolicy(policy) if err != nil { return FirewallPolicy{}, err } expected, err := expectedMutationState(ctx, operation.ExpectedFirewall) if err != nil { return FirewallPolicy{}, err } if expected.FirewallRevision != validated.Revision || expected.FirewallDigest != validated.PolicyDigest { return FirewallPolicy{}, NewError(ErrConflict, "reconciliation_descriptor_mismatch", "firewall policy does not match its durable reconciliation descriptor", nil) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return FirewallPolicy{}, err } defer domain.Free() if _, _, err := requireManagedDomain(domain); err != nil { return FirewallPolicy{}, err } filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid)) if err != nil { return FirewallPolicy{}, err } if filter == nil { return FirewallPolicy{}, NewError(ErrConflict, "firewall_filter_missing", "managed VM network filter was not found", nil) } baseXML, err := filter.GetXMLDesc(0) filter.Free() if err != nil { return FirewallPolicy{}, translateLibvirtError("read VM network filter", err) } updatedFilter, err := buildUserFirewallXML(baseXML, validated) if err != nil { return FirewallPolicy{}, err } mutationStarted = true definedFilter, err := l.conn.NWFilterDefineXML(updatedFilter) if err != nil { return FirewallPolicy{}, operation.MarkUncertain(mutationLibvirtError("define VM firewall filter", err)) } definedFilter.Free() document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if err != nil { return FirewallPolicy{}, operation.MarkUncertain(translateLibvirtError("read domain firewall metadata", err)) } updatedDomain, err := mutateManagedMetadata(document, func(metadata *domainMetadataXML) error { metadata.Firewall = firewallToMetadata(validated) metadata.FirewallTransitionID = expected.TransitionID return nil }) if err != nil { return FirewallPolicy{}, operation.MarkUncertain(err) } defined, err := l.conn.DomainDefineXML(updatedDomain) if err != nil { return FirewallPolicy{}, operation.MarkUncertain(mutationLibvirtError("persist domain firewall metadata", err)) } defined.Free() return validated, nil } func (l *Libvirt) GetRescue(ctx context.Context, uuid string) (RescueState, error) { domain, err := l.domainByUUID(ctx, uuid) if err != nil { return RescueState{}, err } defer domain.Free() return l.rescueState(ctx, domain) } func (l *Libvirt) EnterRescue(ctx context.Context, uuid string, request RescueRequest) (_ RescueState, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() if err := ValidateRescueTemplateSnapshot( request.TemplateID, request.Revision, request.GenerationToken, ); err != nil { return RescueState{}, err } template, err := l.loadRescueTemplate(ctx, request.TemplateID) if err != nil { return RescueState{}, err } if template.Revision != request.Revision || template.GenerationToken != request.GenerationToken { return RescueState{}, NewError( ErrConflict, "rescue_generation_mismatch", "rescue template generation does not match the requested snapshot", nil, ) } volume, err := l.rescueVolume(ctx, request.TemplateID) if err != nil { return RescueState{}, err } defer volume.Free() path, err := volume.GetPath() if err != nil { return RescueState{}, translateLibvirtError("read rescue image path", err) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return RescueState{}, err } defer domain.Free() if err := requirePersistentWithoutSnapshots(domain); err != nil { return RescueState{}, err } document, metadata, err := requireManagedDomain(domain) if err != nil { return RescueState{}, err } if metadata.RescueTemplateID != "" { return RescueState{}, NewError(ErrConflict, "rescue_already_active", "VM already has a staged or active rescue boot", nil) } autostart, err := domain.GetAutostart() if err != nil { return RescueState{}, translateLibvirtError("read rescue domain autostart", err) } wasRunning, err := domain.IsActive() if err != nil { return RescueState{}, translateLibvirtError("check rescue domain activity", err) } updated, err := mutateDomainDocument(document, func(definition *libvirtxml.Domain, metadata *domainMetadataXML) error { if definition.OS == nil || definition.Devices == nil { return NewError(ErrConflict, "invalid_domain", "managed domain XML is incomplete", nil) } if err := stageRescueBoot(definition, path); err != nil { return err } metadata.RescueTemplateID = request.TemplateID metadata.RescueTemplateRevision = request.Revision metadata.RescueTemplateGeneration = request.GenerationToken metadata.RescueWasRunning = wasRunning metadata.RescueAutostart = autostart return nil }) if err != nil { return RescueState{}, err } mutationStarted = true defined, err := l.conn.DomainDefineXML(updated) if err != nil { observed, inspectErr := l.rescueState(ctx, domain) if inspectErr != nil || observed.TemplateID == request.TemplateID { return RescueState{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("stage rescue domain", err), inspectErr)) } return RescueState{}, mutationLibvirtError("stage rescue domain", err) } defer defined.Free() if err := defined.SetAutostart(autostart); err != nil { return RescueState{}, operation.MarkUncertain(mutationLibvirtError("preserve rescue domain autostart", err)) } if err := validateRescueDefinition( updated, request.TemplateID, request.Revision, request.GenerationToken, path, ); err != nil { return RescueState{}, operation.MarkUncertain(err) } if wasRunning { shutdownErr := domain.ShutdownFlags(libvirt.DOMAIN_SHUTDOWN_ACPI_POWER_BTN) if shutdownErr == nil { shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second) shutdownErr = waitForShutoff(shutdownCtx, domain) cancel() } if shutdownErr != nil { stillActive, inspectErr := domain.IsActive() if inspectErr != nil { return RescueState{}, operation.MarkUncertain(errors.Join(shutdownErr, translateLibvirtError("inspect staged rescue domain", inspectErr))) } if stillActive { state, stateErr := l.rescueState(ctx, domain) if stateErr != nil || !state.Staged { return RescueState{}, operation.MarkUncertain(errors.Join(shutdownErr, stateErr)) } state.Warning = "guest shutdown did not complete; rescue boot is staged and requires explicit force-start" state.PendingManualRestart = true return state, nil } } } if err := l.startManagedDomain(ctx, defined, "start rescue domain"); err != nil { state, inspectErr := l.rescueState(ctx, defined) if inspectErr == nil && state.Active { state.Warning = "rescue guest is running although libvirt reported a start error" return state, nil } return RescueState{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("start rescue domain", err), inspectErr)) } state, err := l.rescueState(ctx, defined) if err != nil || !state.Active || state.TemplateID != request.TemplateID { return RescueState{}, operation.MarkUncertain(errors.Join(NewError(ErrConflict, "rescue_start_unconfirmed", "rescue guest start could not be confirmed", nil), err)) } return state, nil } func (l *Libvirt) ForceStartRescue(ctx context.Context, uuid string, request ForceRescueStartRequest) (_ RescueState, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() if !request.AckDataLoss { return RescueState{}, NewError(ErrInvalid, "data_loss_ack_required", "ack_data_loss must be true", nil) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return RescueState{}, err } defer domain.Free() if err := requirePersistentWithoutSnapshots(domain); err != nil { return RescueState{}, err } state, err := l.rescueState(ctx, domain) if err != nil { return RescueState{}, err } if state.Active || !state.Staged || state.TemplateID == "" { return RescueState{}, NewError(ErrConflict, "rescue_not_staged", "a verified staged rescue boot is required", nil) } _, metadata, err := requireManagedDomain(domain) if err != nil { return RescueState{}, err } active, err := domain.IsActive() if err != nil { return RescueState{}, translateLibvirtError("check staged rescue domain activity", err) } mutationStarted = true if active { if err := domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT); err != nil { stillActive, inspectErr := domain.IsActive() if inspectErr != nil || stillActive { return RescueState{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("force off staged rescue domain", err), inspectErr)) } } } if err := l.startManagedDomain(ctx, domain, "cold start staged rescue domain"); err != nil { observed, inspectErr := l.rescueState(ctx, domain) if inspectErr == nil && observed.Active { observed.Warning = "rescue guest is running although libvirt reported a start error" return observed, nil } return RescueState{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("cold start staged rescue domain", err), inspectErr)) } observed, err := l.rescueState(ctx, domain) if err != nil || !observed.Active || observed.TemplateID != state.TemplateID || observed.Revision != state.Revision || observed.GenerationToken != state.GenerationToken { return RescueState{}, operation.MarkUncertain(errors.Join(NewError(ErrConflict, "rescue_force_start_unconfirmed", "forced rescue start could not be confirmed", nil), err)) } autostart, err := domain.GetAutostart() if err != nil || autostart != metadata.RescueAutostart { return RescueState{}, operation.MarkUncertain(errors.Join(NewError(ErrConflict, "rescue_autostart_changed", "rescue force-start did not preserve autostart", nil), err)) } return observed, nil } func (l *Libvirt) ExitRescue(ctx context.Context, uuid string, request ExitRescueRequest) (_ RescueState, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() if request.NetworkSuspended == nil { return RescueState{}, NewError(ErrInvalid, "network_state_required", "network_suspended must be provided", nil) } if !request.AckDataLoss { return RescueState{}, NewError(ErrInvalid, "data_loss_ack_required", "ack_data_loss must be true", nil) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return RescueState{}, err } defer domain.Free() if err := requirePersistentWithoutSnapshots(domain); err != nil { return RescueState{}, err } state, err := l.rescueState(ctx, domain) if err != nil { return RescueState{}, err } if state.TemplateID == "" || (!state.Active && !state.Staged) { return RescueState{}, NewError(ErrConflict, "rescue_not_active", "VM has no staged or active rescue boot", nil) } document, metadata, err := requireManagedDomain(domain) if err != nil { return RescueState{}, err } active, err := domain.IsActive() if err != nil { return RescueState{}, translateLibvirtError("check rescue domain activity", err) } updated, err := mutateDomainDocument(document, func(definition *libvirtxml.Domain, metadata *domainMetadataXML) error { if metadata.RescueTemplateID == "" { return NewError(ErrConflict, "rescue_not_active", "VM is not in rescue mode", nil) } if definition.OS == nil || !managedNetworkLayoutMatches(definition, metadata) { return NewError(ErrConflict, "invalid_domain", "managed domain XML is incomplete", nil) } restoreNormalBoot(definition) linkState := "up" if *request.NetworkSuspended { linkState = "down" } for index := range definition.Devices.Interfaces { definition.Devices.Interfaces[index].Link = &libvirtxml.DomainInterfaceLink{State: linkState} } metadata.RescueTemplateID = "" metadata.RescueTemplateRevision = 0 metadata.RescueTemplateGeneration = "" metadata.RescueWasRunning = false metadata.RescueAutostart = false return nil }) if err != nil { return RescueState{}, err } liveLinkDocuments, err := networkInterfaceLinkDocuments(updated, !*request.NetworkSuspended) if err != nil { return RescueState{}, err } stagedOriginalStillRunning := state.Staged && active mutationStarted = true if active && state.Active { if err := forceStopRescueForExit(domain); err != nil { return RescueState{}, err } } defined, err := l.conn.DomainDefineXML(updated) if err != nil { return RescueState{}, operation.MarkUncertain(mutationLibvirtError("restore normal domain definition", err)) } defer defined.Free() if err := defined.SetAutostart(metadata.RescueAutostart); err != nil { return RescueState{}, operation.MarkUncertain(mutationLibvirtError("preserve normal domain autostart", err)) } startWarning := "" if stagedOriginalStillRunning { var guardLease *publicguard.Lease if !*request.NetworkSuspended { guardLease, err = l.acquirePublicGuardTransition(ctx) if err != nil { return RescueState{}, err } defer guardLease.Release() if err := l.reconcilePublicGuardLease(ctx, guardLease, nil); err != nil { return RescueState{}, NewError(ErrUnavailable, "public_guard_apply_failed", "public network guard is not ready for rescue network restore", err) } } for _, interfaceDocument := range liveLinkDocuments { if err := defined.UpdateDeviceFlags(interfaceDocument, libvirt.DOMAIN_DEVICE_MODIFY_LIVE); err != nil { return RescueState{}, operation.MarkUncertain(mutationLibvirtError("restore live normal-domain interface link state", err)) } } } else { if err := l.startManagedDomain(ctx, defined, "start domain after rescue"); err != nil { observed, inspectErr := l.rescueState(ctx, defined) if inspectErr == nil && !observed.Active && !observed.Staged { isActive, activeErr := defined.IsActive() if activeErr == nil && isActive { startWarning = "normal guest is running although libvirt reported a start error" } else { return RescueState{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("start domain after rescue", err), inspectErr, translateOptionalLibvirtError("inspect normal domain after start failure", activeErr))) } } else { return RescueState{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("start domain after rescue", err), inspectErr)) } } } observed, err := l.rescueState(ctx, defined) if err != nil || observed.Active || observed.Staged { return RescueState{}, operation.MarkUncertain(errors.Join(NewError(ErrConflict, "rescue_exit_unconfirmed", "normal boot restoration could not be confirmed", nil), err)) } running, err := defined.IsActive() if err != nil || !running { return RescueState{}, operation.MarkUncertain(errors.Join(NewError(ErrConflict, "rescue_exit_unconfirmed", "normal guest is not running after rescue exit", nil), err)) } live, err := defined.GetXMLDesc(0) if err != nil || !domainNetworkLinksMatch(live, !*request.NetworkSuspended) { return RescueState{}, operation.MarkUncertain(errors.Join(NewError(ErrConflict, "rescue_exit_network_unconfirmed", "normal guest live network state could not be confirmed after rescue exit", nil), translateOptionalLibvirtError("read live normal-domain network state", err))) } observed.NetworkSuspended = *request.NetworkSuspended observed.Warning = startWarning return observed, nil } type rescueExitDomainControl interface { IsActive() (bool, error) DestroyFlags(libvirt.DomainDestroyFlags) error } func forceStopRescueForExit(domain rescueExitDomainControl) error { destroyErr := domain.DestroyFlags(libvirt.DOMAIN_DESTROY_DEFAULT) stillActive, inspectErr := domain.IsActive() if inspectErr == nil && !stillActive { return nil } var mutationErr, observedErr error if destroyErr != nil { mutationErr = mutationLibvirtError("force off rescue domain before exit", destroyErr) } else { mutationErr = NewError(ErrConflict, "rescue_exit_domain_still_active", "rescue domain remained active after force stop", nil) } if inspectErr != nil { observedErr = translateLibvirtError("recheck rescue domain activity after force stop", inspectErr) } return operation.MarkUncertain(errors.Join(mutationErr, observedErr)) } func (l *Libvirt) rescueState(ctx context.Context, domain *libvirt.Domain) (RescueState, error) { document, metadata, err := requireManagedDomain(domain) if err != nil { return RescueState{}, err } definition, _, err := parseManagedDomain(document) if err != nil { return RescueState{}, err } result := RescueState{ TemplateID: metadata.RescueTemplateID, Revision: metadata.RescueTemplateRevision, GenerationToken: metadata.RescueTemplateGeneration, NetworkSuspended: definitionNetworkSuspended(definition), } if metadata.RescueTemplateID == "" { return result, nil } template, err := l.loadRescueTemplate(ctx, metadata.RescueTemplateID) if err != nil { return RescueState{}, err } if template.Revision != metadata.RescueTemplateRevision || template.GenerationToken != metadata.RescueTemplateGeneration { return RescueState{}, NewError( ErrConflict, "rescue_generation_mismatch", "staged rescue metadata does not match the host catalog generation", nil, ) } volume, err := l.rescueVolume(ctx, metadata.RescueTemplateID) if err != nil { return RescueState{}, err } defer volume.Free() path, err := volume.GetPath() if err != nil { return RescueState{}, translateLibvirtError("read staged rescue image path", err) } if err := validateRescueDefinition( document, metadata.RescueTemplateID, metadata.RescueTemplateRevision, metadata.RescueTemplateGeneration, path, ); err != nil { return RescueState{}, err } active, err := domain.IsActive() if err != nil { return RescueState{}, translateLibvirtError("check rescue activity", err) } if !active { result.Staged = true result.PendingManualRestart = true return result, nil } liveDocument, err := domain.GetXMLDesc(0) if err != nil { return RescueState{}, translateLibvirtError("read live rescue definition", err) } _, liveMetadata, err := parseManagedDomain(liveDocument) if err != nil { return RescueState{}, err } switch liveMetadata.RescueTemplateID { case metadata.RescueTemplateID: if liveMetadata.RescueTemplateRevision != metadata.RescueTemplateRevision || liveMetadata.RescueTemplateGeneration != metadata.RescueTemplateGeneration { return RescueState{}, NewError(ErrConflict, "rescue_state_mismatch", "live and persistent rescue generations do not match", nil) } if err := validateRescueDefinition( liveDocument, metadata.RescueTemplateID, metadata.RescueTemplateRevision, metadata.RescueTemplateGeneration, path, ); err != nil { return RescueState{}, err } result.Active = true case "": result.Staged = true result.PendingManualRestart = true default: return RescueState{}, NewError(ErrConflict, "rescue_state_mismatch", "live and persistent rescue markers do not match", nil) } return result, nil } func validateRescueDefinition( document string, templateID string, revision uint64, generationToken string, expectedPath string, ) error { definition, metadata, err := parseManagedDomain(document) if err != nil { return err } if metadata.RescueTemplateID != templateID || metadata.RescueTemplateRevision != revision || metadata.RescueTemplateGeneration != generationToken || definition.OS == nil || definition.Devices == nil { return NewError(ErrConflict, "rescue_definition_invalid", "rescue boot metadata or order is invalid", nil) } legacyBootOrder := len(definition.OS.BootDevices) == 2 && definition.OS.BootDevices[0].Dev == "cdrom" && definition.OS.BootDevices[1].Dev == "hd" deviceBootOrder := len(definition.OS.BootDevices) == 0 if !legacyBootOrder && !deviceBootOrder { return NewError(ErrConflict, "rescue_definition_invalid", "rescue boot metadata or order is invalid", nil) } rescueMatches := 0 rootMatches := 0 bootDevices := 0 for _, disk := range definition.Devices.Disks { if disk.Boot != nil { bootDevices++ } if disk.Device == "disk" && disk.Target != nil && disk.Target.Dev == "vda" { if deviceBootOrder && (disk.Boot == nil || disk.Boot.Order != 2) { return NewError(ErrConflict, "rescue_definition_invalid", "managed root disk is not the rescue fallback boot device", nil) } if legacyBootOrder && disk.Boot != nil { return NewError(ErrConflict, "rescue_definition_invalid", "legacy rescue boot mixes global and per-device order", nil) } rootMatches++ continue } if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sdb" { if deviceBootOrder && (disk.Boot == nil || disk.Boot.Order != 1) { return NewError(ErrConflict, "rescue_definition_invalid", "managed rescue disk is not the first boot device", nil) } if legacyBootOrder && disk.Boot != nil { return NewError(ErrConflict, "rescue_definition_invalid", "legacy rescue boot mixes global and per-device order", nil) } if disk.Driver == nil || disk.Driver.Type != "raw" || disk.Source == nil || disk.Source.File == nil || filepath.Clean(disk.Source.File.File) != filepath.Clean(expectedPath) || disk.ReadOnly == nil { return NewError(ErrConflict, "rescue_definition_invalid", "rescue boot disk does not match the managed template", nil) } rescueMatches++ continue } if disk.Boot != nil { return NewError(ErrConflict, "rescue_definition_invalid", "an unmanaged disk participates in rescue boot order", nil) } } expectedBootDevices := 2 if legacyBootOrder { expectedBootDevices = 0 } if rescueMatches != 1 || rootMatches != 1 || bootDevices != expectedBootDevices { return NewError(ErrConflict, "rescue_definition_invalid", "rescue boot must contain exactly one rescue disk and one managed root fallback", nil) } return nil } func stageRescueBoot(definition *libvirtxml.Domain, path string) error { definition.OS.BootDevices = nil rootIndex := -1 for index := range definition.Devices.Disks { disk := &definition.Devices.Disks[index] disk.Boot = nil if disk.Device == "disk" && disk.Target != nil && disk.Target.Dev == "vda" { if rootIndex >= 0 { return NewError(ErrConflict, "invalid_domain", "managed domain contains multiple root disks", nil) } rootIndex = index } if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sdb" { return NewError(ErrConflict, "invalid_domain", "managed rescue disk target is already in use", nil) } } if rootIndex < 0 { return NewError(ErrConflict, "invalid_domain", "managed root disk is missing", nil) } definition.Devices.Disks[rootIndex].Boot = &libvirtxml.DomainDeviceBoot{Order: 2} definition.Devices.Disks = append(definition.Devices.Disks, libvirtxml.DomainDisk{ Device: "cdrom", Driver: &libvirtxml.DomainDiskDriver{Name: "qemu", Type: "raw"}, Source: &libvirtxml.DomainDiskSource{File: &libvirtxml.DomainDiskSourceFile{File: path}}, Target: &libvirtxml.DomainDiskTarget{Dev: "sdb", Bus: "sata"}, ReadOnly: &libvirtxml.DomainDiskReadOnly{}, Boot: &libvirtxml.DomainDeviceBoot{Order: 1}, }) return nil } func restoreNormalBoot(definition *libvirtxml.Domain) { definition.OS.BootDevices = []libvirtxml.DomainBootDevice{{Dev: "hd"}} disks := definition.Devices.Disks[:0] for _, disk := range definition.Devices.Disks { if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sdb" { continue } disk.Boot = nil disks = append(disks, disk) } definition.Devices.Disks = disks } func definitionNetworkSuspended(definition *libvirtxml.Domain) bool { if definition == nil || definition.Devices == nil || len(definition.Devices.Interfaces) == 0 { return false } for _, iface := range definition.Devices.Interfaces { if iface.Link == nil || iface.Link.State != "down" { return false } } return true } func requirePersistentWithoutSnapshots(domain *libvirt.Domain) error { if _, _, err := requireManagedDomain(domain); err != nil { return err } persistent, err := domain.IsPersistent() if err != nil { return translateLibvirtError("check domain persistence", err) } if !persistent { return NewError(ErrConflict, "persistent_vm_required", "operation requires a persistent VM", nil) } count, err := domain.SnapshotNum(0) if err != nil { return translateLibvirtError("check domain snapshots", err) } if count != 0 { return NewError(ErrConflict, "snapshots_present", "operation is blocked while snapshots exist", nil) } managedSave, err := domain.HasManagedSaveImage(0) if err != nil { return translateLibvirtError("check domain managed save image", err) } if managedSave { return NewError(ErrConflict, "managed_save_present", "operation is blocked while a managed save image exists", nil) } return nil } func stopForDefinitionChange(ctx context.Context, domain *libvirt.Domain) (bool, error) { active, err := domain.IsActive() if err != nil { return false, translateLibvirtError("check domain activity", err) } if !active { return false, nil } if err := domain.ShutdownFlags(libvirt.DOMAIN_SHUTDOWN_ACPI_POWER_BTN); err != nil { confirmed, inspectErr := domain.IsActive() if inspectErr != nil || confirmed { return true, operation.MarkUncertain(mutationLibvirtError("request guest shutdown", errors.Join(err, inspectErr))) } } if err := waitForShutoff(ctx, domain); err != nil { return true, operation.MarkUncertain(err) } return true, nil } func (l *Libvirt) downloadSmallVolume(ctx context.Context, volume *libvirt.StorageVol, maximum int) ([]byte, error) { info, err := volume.GetInfo() if err != nil { return nil, translateLibvirtError("read storage volume size", err) } if info.Capacity == 0 || info.Capacity > uint64(maximum) { return nil, NewError(ErrConflict, "invalid_volume_size", "managed volume has an invalid size", nil) } result, err := l.downloadStorageVolume(ctx, volume, 0, info.Capacity) if err != nil { return nil, translateLibvirtError("download storage volume", err) } return result, nil } func parseManagedDomain(document string) (*libvirtxml.Domain, *domainMetadataXML, error) { definition, metadata, managed, err := inspectManagedDomain(document) if err != nil { return nil, nil, err } if !managed { return nil, nil, NewError(ErrConflict, "managed_metadata_missing", "domain is not managed by Virt Cloud", nil) } return definition, metadata, nil } func inspectManagedDomain(document string) (*libvirtxml.Domain, *domainMetadataXML, bool, error) { var definition libvirtxml.Domain if err := definition.Unmarshal(document); err != nil { return nil, nil, false, fmt.Errorf("parse libvirt domain XML: %w", err) } if definition.Metadata == nil || definition.Metadata.XML == "" { return &definition, nil, false, nil } var marker struct { XMLName xml.Name } if err := xml.Unmarshal([]byte(definition.Metadata.XML), &marker); err != nil { return nil, nil, false, NewError(ErrConflict, "managed_metadata_invalid", "domain metadata XML is invalid", err) } if marker.XMLName.Local != "virt-cloud" || marker.XMLName.Space != domainMetadataNamespace { return &definition, nil, false, nil } var metadata domainMetadataXML if err := xml.Unmarshal([]byte(definition.Metadata.XML), &metadata); err != nil { return nil, nil, false, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud domain metadata is invalid", err) } if metadata.XMLName.Local != "virt-cloud" || metadata.XMLName.Space != domainMetadataNamespace { return &definition, nil, false, nil } if !uuidPattern.MatchString(definition.UUID) || !templateIDPattern.MatchString(metadata.TemplateID) || !isVMOwnedRootVolumeName(definition.UUID, metadata.RootVolumeName) { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud domain ownership metadata is incomplete", nil) } if metadata.DisplayName != "" && !namePattern.MatchString(metadata.DisplayName) { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud display name metadata is invalid", nil) } if _, err := metadata.PublicNetwork.public(); err != nil { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud public network metadata is invalid", err) } var platformPolicy PlatformNetworkPolicySnapshot if err := json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &platformPolicy); err != nil || ValidatePlatformNetworkPolicy(platformPolicy) != nil { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud platform policy metadata is invalid", err) } seenSnapshots := make(map[string]struct{}, len(metadata.Snapshots)) for _, snapshot := range metadata.Snapshots { if !catalogIDPattern.MatchString(snapshot.Key) || !displayNamePattern.MatchString(snapshot.Name) || len(snapshot.Description) > 1024 || (snapshot.Consistency != "filesystem" && snapshot.Consistency != "crash") || snapshot.SizeBytes == 0 || snapshot.CreatedAt <= 0 { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud snapshot metadata is invalid", nil) } if _, exists := seenSnapshots[snapshot.Key]; exists { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud snapshot metadata contains duplicates", nil) } seenSnapshots[snapshot.Key] = struct{}{} } if metadata.PendingCleanupVolume != "" && !isVMOwnedRootVolumeName(definition.UUID, metadata.PendingCleanupVolume) { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud pending cleanup volume metadata is invalid", nil) } rescueIdentityPresent := metadata.RescueTemplateID != "" && catalogIDPattern.MatchString(metadata.RescueTemplateID) && metadata.RescueTemplateRevision > 0 && catalogIDPattern.MatchString(metadata.RescueTemplateGeneration) rescueIdentityAbsent := metadata.RescueTemplateID == "" && metadata.RescueTemplateRevision == 0 && metadata.RescueTemplateGeneration == "" && !metadata.RescueWasRunning && !metadata.RescueAutostart if !rescueIdentityPresent && !rescueIdentityAbsent { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud rescue metadata is invalid", nil) } if (metadata.DiskTransitionKind == "") != (metadata.DiskTransitionID == "") || (metadata.DiskTransitionKind != "" && metadata.DiskTransitionKind != "reinstall" && metadata.DiskTransitionKind != "snapshot_restore" && metadata.DiskTransitionKind != "backup_recover") || (metadata.DiskTransitionID != "" && !operationIdentifierPattern.MatchString(metadata.DiskTransitionID)) { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud disk transition metadata is invalid", nil) } for _, transitionID := range []string{metadata.SettingsTransitionID, metadata.ResourcesTransitionID, metadata.NetworkTransitionID, metadata.VPCTransitionID, metadata.FirewallTransitionID} { if transitionID != "" && !operationIdentifierPattern.MatchString(transitionID) { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud operation transition metadata is invalid", nil) } } if metadata.VPCPort != nil { value := metadata.VPCPort.public() if err := validateVPCPortMetadata(value); err != nil { return nil, nil, true, NewError(ErrConflict, "managed_metadata_invalid", "Virt Cloud VPC port metadata is invalid", err) } } if definition.Devices == nil { return nil, nil, true, NewError(ErrConflict, "managed_disk_invalid", "managed domain has no device definition", nil) } rootMatches := 0 for _, disk := range definition.Devices.Disks { if disk.Device != "disk" || disk.Target == nil || disk.Target.Dev != "vda" { continue } if disk.Driver == nil || disk.Driver.Type != "qcow2" || disk.Source == nil || disk.Source.File == nil || filepath.Base(disk.Source.File.File) != metadata.RootVolumeName { return nil, nil, true, NewError(ErrConflict, "managed_disk_invalid", "managed root disk does not match its ownership metadata", nil) } rootMatches++ } if rootMatches != 1 { return nil, nil, true, NewError(ErrConflict, "managed_disk_invalid", "managed domain must have exactly one qcow2 vda root disk", nil) } return &definition, &metadata, true, nil } type managedDomainXMLReader interface { GetXMLDesc(libvirt.DomainXMLFlags) (string, error) } func requireManagedDomain(domain managedDomainXMLReader) (string, *domainMetadataXML, error) { document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if err != nil { return "", nil, translateLibvirtError("read managed domain XML", err) } _, metadata, err := parseManagedDomain(document) if err != nil { return "", nil, err } return document, metadata, nil } func mutateManagedMetadata(document string, mutate func(*domainMetadataXML) error) (string, error) { return mutateDomainDocument(document, func(_ *libvirtxml.Domain, metadata *domainMetadataXML) error { return mutate(metadata) }) } func mutateDomainDocument(document string, mutate func(*libvirtxml.Domain, *domainMetadataXML) error) (string, error) { definition, metadata, err := parseManagedDomain(document) if err != nil { return "", err } if err := mutate(definition, metadata); err != nil { return "", err } metadata.XMLNS = domainMetadataNamespace encoded, err := xml.Marshal(metadata) if err != nil { return "", fmt.Errorf("marshal Virt Cloud domain metadata: %w", err) } definition.Metadata.XML = string(encoded) return definition.Marshal() } func firewallToMetadata(policy FirewallPolicy) *domainFirewallMetadataXML { result := &domainFirewallMetadataXML{Revision: policy.Revision, Digest: policy.PolicyDigest, DefaultIn: policy.DefaultIn, DefaultOut: policy.DefaultOut, Rules: make([]domainFirewallRuleMetadataXML, 0, len(policy.Rules))} for _, rule := range policy.Rules { result.Rules = append(result.Rules, domainFirewallRuleMetadataXML{Direction: rule.Direction, Action: rule.Action, Protocol: rule.Protocol, CIDR: rule.CIDR, PortFrom: rule.PortFrom, PortTo: rule.PortTo}) } return result } func firewallFromMetadata(stored *domainFirewallMetadataXML) FirewallPolicy { result := FirewallPolicy{Revision: stored.Revision, PolicyDigest: stored.Digest, DefaultIn: stored.DefaultIn, DefaultOut: stored.DefaultOut, Rules: make([]FirewallRule, 0, len(stored.Rules))} for _, rule := range stored.Rules { result.Rules = append(result.Rules, FirewallRule{Direction: rule.Direction, Action: rule.Action, Protocol: rule.Protocol, CIDR: rule.CIDR, PortFrom: rule.PortFrom, PortTo: rule.PortTo}) } return result } func storageVolumeXML(name string, capacity uint64, format string) (string, error) { definition := libvirtxml.StorageVolume{Name: name, Capacity: &libvirtxml.StorageVolumeSize{Value: capacity, Unit: "bytes"}, Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: format}}} return definition.Marshal() } func (l *Libvirt) rescueVolume(ctx context.Context, id string) (*libvirt.StorageVol, error) { if _, err := l.loadRescueTemplate(ctx, id); err != nil { return nil, err } volume, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleRescue), rescueImageVolumeName(id)) if err != nil { return nil, err } if volume != nil { return volume, nil } return nil, NewError(ErrNotFound, "rescue_template_not_found", "rescue template is not present on this host", nil) }