You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_reconcile_features.go

818 lines
31 KiB
Go

//go:build libvirt
package hypervisor
import (
"encoding/json"
"errors"
"slices"
"strings"
"time"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
func (l *Libvirt) reconcileCreatedVM(domain *libvirt.Domain) (operation.Resolution, error) {
document, definition, metadata, pool, root, err := l.managedRootContext(domain)
if err != nil {
return operation.Resolution{}, nil
}
defer pool.Free()
defer root.Free()
autostart, err := domain.GetAutostart()
if err != nil || !autostart {
return operation.Resolution{}, nil
}
active, err := domain.IsActive()
if err != nil || !active {
return operation.Resolution{}, nil
}
state, _, err := domain.GetState()
if err != nil || (state != libvirt.DOMAIN_RUNNING && state != libvirt.DOMAIN_BLOCKED) {
return operation.Resolution{}, nil
}
seed, err := l.lookupOptionalManagedVolume(l.local.SeedStoragePool, seedVolumeName(definition.UUID))
if err != nil || seed == nil {
return operation.Resolution{}, err
}
seedPath, err := seed.GetPath()
seed.Free()
if err != nil || definition.Devices == nil {
return operation.Resolution{}, nil
}
seedAttached := false
filterName := networkFilterName(definition.UUID)
filterAttached := len(definition.Devices.Interfaces) == 1 && definition.Devices.Interfaces[0].FilterRef != nil && definition.Devices.Interfaces[0].FilterRef.Filter == filterName
for _, disk := range definition.Devices.Disks {
if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sda" && disk.Source != nil && disk.Source.File != nil && disk.Source.File.File == seedPath && disk.ReadOnly != nil {
seedAttached = true
}
}
if !seedAttached || !filterAttached {
return operation.Resolution{}, nil
}
var policy PlatformNetworkPolicySnapshot
if json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &policy) != nil || ValidatePlatformNetworkPolicy(policy) != nil || l.verifyManagedFilter(filterName, policy) != nil {
return operation.Resolution{}, nil
}
result, err := metadataFromXML(document)
if err != nil {
return operation.Resolution{}, nil
}
result.Status = VMStateRunning
encoded, err := json.Marshal(result)
if err != nil {
return operation.Resolution{}, err
}
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, nil
}
func (l *Libvirt) reconcileDomainFeatureOperation(record operation.Record, domain *libvirt.Domain) (operation.Resolution, bool, error) {
if record.ExpectedState != nil {
switch record.ExpectedState.Kind {
case operation.ExpectedUpdateSettings:
resolution, err := l.reconcileSettings(record, domain)
return resolution, true, err
case operation.ExpectedChangeNetwork:
resolution, err := l.reconcileNetworkChange(record, domain)
return resolution, true, err
case operation.ExpectedNetworkState:
resolution, err := l.reconcileNetworkState(record, domain)
return resolution, true, err
case operation.ExpectedServiceState:
resolution, err := l.reconcileServiceState(record, domain)
return resolution, true, err
case operation.ExpectedFirewall:
resolution, err := l.reconcileFirewall(record, domain)
return resolution, true, err
}
}
if record.Action == "reinstall-vm" {
return l.reconcileReinstall(record, domain), true, nil
}
if strings.HasPrefix(record.Action, "create-snapshot:") || strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:") {
return l.reconcileSnapshot(record, domain), true, nil
}
if record.Action == "enter-rescue" || record.Action == "force-start-rescue" || strings.HasPrefix(record.Action, "exit-rescue:") {
return l.reconcileRescue(record, domain), true, nil
}
return operation.Resolution{}, false, nil
}
func (l *Libvirt) reconcileSettings(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return expectedEvidenceError("settings", err)
}
seed, err := l.managedSeedContents(record.ResourceID)
if err != nil {
return expectedEvidenceError("settings", err)
}
seedMarker, markerErr := seedMetadataMarker(seed, "virt-cloud-settings-operation")
if markerErr != nil {
// A rewritten or damaged seed is evidence that the first phase may
// have crossed its commit boundary. It is not proof of failure.
return operation.Resolution{}, nil
}
domainMatches := metadata.SettingsTransitionID == expected.TransitionID
seedMatches := seedMarker == expected.TransitionID
if !domainMatches || !seedMatches {
return classifyTransitionMarkers("settings", domainMatches, seedMatches), nil
}
if expected.HostnameChanged && metadata.Hostname != expected.Hostname {
// Both durable markers belong to this operation, so it crossed the
// commit boundary. A mismatched payload can be corruption or a later
// out-of-band edit; it is not evidence that PHP may roll state back.
return operation.Resolution{}, nil
}
result, err := metadataFromXML(document)
if err != nil {
return operation.Resolution{}, err
}
state, _, err := domain.GetState()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile settings domain state", err)
}
result.Status = mapDomainState(state)
result.SettingsChanged = true
return resolvedExpectedSuccess(result)
}
func (l *Libvirt) reconcileNetworkChange(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return operation.Resolution{}, translateLibvirtError("read network definition for reconciliation", err)
}
definition, metadata, err := parseManagedDomain(document)
if err != nil {
return expectedEvidenceError("network", err)
}
seed, err := l.managedSeedContents(record.ResourceID)
if err != nil {
return expectedEvidenceError("network", err)
}
seedMarker, markerErr := seedMetadataMarker(seed, "virt-cloud-network-operation")
if markerErr != nil {
return operation.Resolution{}, nil
}
domainMatches := metadata.NetworkTransitionID == expected.TransitionID
seedMatches := seedMarker == expected.TransitionID
if !domainMatches || !seedMatches {
return classifyTransitionMarkers("network", domainMatches, seedMatches), nil
}
mac, publicCIDR, gateway, dns, networkErr := seedNetwork(seed)
if networkErr != nil || expected.NetworkSelector != metadata.NetworkSelector || expected.PublicPoolKey != metadata.PublicPoolKey ||
expected.PublicPoolCIDR != metadata.PublicPoolCIDR || expected.PublicIPCIDR != metadata.PublicIPCIDR ||
expected.Gateway != metadata.Gateway || !slices.Equal(expected.DNSServers, metadata.DNSServers) ||
expected.PublicIPCIDR != publicCIDR || expected.Gateway != gateway || !slices.Equal(expected.DNSServers, dns) ||
definition.Devices == nil || len(definition.Devices.Interfaces) != 1 || definition.Devices.Interfaces[0].MAC == nil ||
!strings.EqualFold(definition.Devices.Interfaces[0].MAC.Address, mac) ||
!domainNetworkSourceMatches(definition.Devices.Interfaces[0], expected.NetworkSourceType, expected.NetworkSourceName) {
// Matching transition markers prove that both phases wrote state for
// this operation. A payload mismatch is therefore ambiguous and must
// not authorize PHP to release or restore an address assignment.
return operation.Resolution{}, nil
}
result, err := metadataFromXML(document)
if err != nil {
return operation.Resolution{}, err
}
state, _, err := domain.GetState()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile network domain state", err)
}
result.Status = mapDomainState(state)
result.NetworkChanged = true
return resolvedExpectedSuccess(result)
}
func (l *Libvirt) reconcileNetworkState(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
inactive, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile persistent network state", err)
}
if _, _, err := parseManagedDomain(inactive); err != nil {
return expectedEvidenceError("network", err)
}
if !domainNetworkLinksMatch(inactive, expected.Enabled) {
return resolvedExpectedFailure("network_state_not_applied", "persistent interface link state does not match the requested state"), nil
}
active, err := domain.IsActive()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile live network state", err)
}
if active {
live, err := domain.GetXMLDesc(0)
if err != nil {
return operation.Resolution{}, translateLibvirtError("read live network state", err)
}
if !domainNetworkLinksMatch(live, expected.Enabled) {
return resolvedExpectedFailure("network_state_not_applied", "live interface link state does not match the requested state"), nil
}
}
return resolvedExpectedSuccess(map[string]bool{"network_enabled": expected.Enabled})
}
func (l *Libvirt) reconcileServiceState(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
if _, _, err := requireManagedDomain(domain); err != nil {
return expectedEvidenceError("service", err)
}
autostart, err := domain.GetAutostart()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile service autostart", err)
}
active, err := domain.IsActive()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile service activity", err)
}
if autostart != expected.Enabled || (!expected.Enabled && active) || (expected.Enabled && expected.StartIfSuspended && !active) {
return resolvedExpectedFailure("service_state_not_applied", "host service state does not match the requested state"), nil
}
state, reason, err := domain.GetState()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile service domain state", err)
}
name, _ := domain.GetName()
return resolvedExpectedSuccess(VMState{UUID: record.ResourceID, Name: name, State: mapDomainState(state), Reason: reason})
}
func (l *Libvirt) reconcileFirewall(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
_, metadata, err := requireManagedDomain(domain)
if err != nil {
return expectedEvidenceError("firewall", err)
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(record.ResourceID))
if err != nil {
return operation.Resolution{}, err
}
if filter == nil {
return operation.Resolution{}, nil
}
filterDocument, err := filter.GetXMLDesc(0)
filter.Free()
if err != nil {
return operation.Resolution{}, translateLibvirtError("read firewall filter for reconciliation", err)
}
domainMarker := metadata.FirewallTransitionID == expected.TransitionID
domainMatches := domainMarker && metadata.Firewall != nil &&
metadata.Firewall.Revision == expected.FirewallRevision &&
metadata.Firewall.Digest == expected.FirewallDigest
if domainMatches {
policy, validateErr := validateFirewallPolicy(firewallFromMetadata(metadata.Firewall))
if validateErr != nil || policy.PolicyDigest != expected.FirewallDigest ||
!firewallDocumentMatchesPolicy(filterDocument, policy) {
// Domain metadata committed this operation, but the filter cannot be
// proven to match it. Never report failure and roll PHP state back.
return operation.Resolution{}, nil
}
return resolvedExpectedSuccess(policy)
}
if domainMarker || !firewallDocumentMatchesStoredState(filterDocument, metadata.Firewall) {
// Either this operation wrote only the filter, or the domain marker is
// present with malformed/stale policy metadata. Both are half-commit
// evidence, so keep the exact operation uncertain.
return operation.Resolution{}, nil
}
return resolvedExpectedFailure("firewall_not_committed", "host evidence confirms the operation did not commit its transition markers"), nil
}
func (l *Libvirt) managedSeedContents(uuid string) ([]byte, error) {
seed, err := l.lookupOptionalManagedVolume(l.local.SeedStoragePool, seedVolumeName(uuid))
if err != nil {
return nil, err
}
if seed == nil {
return nil, NewError(ErrNotFound, "seed_not_found", "managed NoCloud seed volume was not found", nil)
}
defer seed.Free()
return l.downloadSmallVolume(seed, maxSeedImageBytes)
}
func seedMetadataMarker(seed []byte, key string) (string, error) {
files, err := readSeedISO(seed)
if err != nil {
return "", err
}
var metadata map[string]any
if err := json.Unmarshal(files.MetaData, &metadata); err != nil {
return "", err
}
value, _ := metadata[key].(string)
if value != "" && !operationIdentifierPattern.MatchString(value) {
return "", errors.New("invalid NoCloud transition marker")
}
return value, nil
}
func domainNetworkLinksMatch(document string, enabled bool) bool {
var definition libvirtxml.Domain
if definition.Unmarshal(document) != nil || definition.Devices == nil || len(definition.Devices.Interfaces) == 0 {
return false
}
for _, iface := range definition.Devices.Interfaces {
state := "up"
if iface.Link != nil && iface.Link.State != "" {
state = iface.Link.State
}
if (state == "up") != enabled {
return false
}
}
return true
}
func domainNetworkSourceMatches(iface libvirtxml.DomainInterface, sourceType, sourceName string) bool {
if iface.Source == nil {
return false
}
switch sourceType {
case "bridge":
return iface.Source.Bridge != nil && iface.Source.Bridge.Bridge == sourceName
case "network":
return iface.Source.Network != nil && iface.Source.Network.Network == sourceName
default:
return false
}
}
func firewallDocumentMatchesPolicy(document string, policy FirewallPolicy) bool {
var current libvirtxml.NWFilter
if current.Unmarshal(document) != nil {
return false
}
normalized, err := current.Marshal()
if err != nil {
return false
}
expected, err := buildUserFirewallXML(document, policy)
return err == nil && normalized == expected
}
// firewallDocumentMatchesStoredState proves that the filter still represents
// the policy durably recorded before an operation. A nil policy means the
// untouched platform filter, which must contain no user/default policy rules.
func firewallDocumentMatchesStoredState(document string, stored *domainFirewallMetadataXML) bool {
if stored != nil {
policy, err := validateFirewallPolicy(firewallFromMetadata(stored))
return err == nil && firewallDocumentMatchesPolicy(document, policy)
}
var current libvirtxml.NWFilter
if current.Unmarshal(document) != nil {
return false
}
for _, entry := range current.Entries {
comment := nwfilterRuleComment(entry.Rule)
if strings.HasPrefix(comment, "virt-user:") || strings.HasPrefix(comment, "virt-default:") {
return false
}
}
return true
}
func classifyTransitionMarkers(kind string, primary, secondary bool) operation.Resolution {
if !primary && !secondary {
return resolvedExpectedFailure(kind+"_not_committed", "host evidence confirms the operation did not commit its transition markers")
}
// One matching marker proves that the mutation crossed its first commit
// boundary. It does not prove which side contains the authoritative state:
// for example, a network seed may already contain the new public address
// while the persistent domain still has the old bridge. Finalizing this as
// a failure would let PHP release or roll back resources that the guest may
// already be using. Keep the exact operation uncertain for administrator
// reconciliation; never replay the original two-phase mutation.
return operation.Resolution{}
}
func expectedStateResourceAbsent(_ operation.Record) operation.Resolution {
return resolvedExpectedFailure("expected_state_resource_absent", "host evidence confirms the target VM is absent and the expected state cannot be satisfied")
}
func expectedEvidenceError(_ string, err error) (operation.Resolution, error) {
if errors.Is(err, ErrUnavailable) {
return operation.Resolution{}, err
}
// Missing, malformed, or ownership-inconsistent evidence cannot prove that
// an expected-state mutation did not commit. In particular, a network
// change may already have written the requested address into either the
// domain definition or NoCloud seed. Finalizing it as failed would let the
// panel release that address while the guest may still be using it. Keep the
// exact operation uncertain until an administrator restores readable
// evidence; never infer a safe rollback from damaged host state.
return operation.Resolution{}, nil
}
func resolvedExpectedFailure(code, message string) operation.Resolution {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: code, ErrorMessage: message}
}
func resolvedExpectedSuccess(value any) (operation.Resolution, error) {
encoded, err := json.Marshal(value)
if err != nil {
return operation.Resolution{}, err
}
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, nil
}
func (l *Libvirt) reconcileRescue(record operation.Record, domain *libvirt.Domain) operation.Resolution {
state, err := l.rescueState(domain)
if err != nil {
return operation.Resolution{}
}
switch record.Action {
case "enter-rescue":
if state.Active || state.Staged {
encoded, _ := json.Marshal(state)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
}
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_not_committed", ErrorMessage: "host evidence confirms rescue boot was not staged"}
case "force-start-rescue":
if state.Active {
encoded, _ := json.Marshal(state)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
}
if state.Staged {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_force_start_incomplete", ErrorMessage: "rescue boot remains staged but is not running"}
}
case "exit-rescue:connected", "exit-rescue:suspended":
if state.Active || state.Staged {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_exit_incomplete", ErrorMessage: "host evidence confirms rescue boot remains staged or active"}
}
expectedSuspended := record.Action == "exit-rescue:suspended"
if state.NetworkSuspended != expectedSuspended {
return operation.Resolution{}
}
active, activeErr := domain.IsActive()
if activeErr != nil || !active {
return operation.Resolution{}
}
live, liveErr := domain.GetXMLDesc(0)
if liveErr != nil || !domainNetworkLinksMatch(live, !expectedSuspended) {
return operation.Resolution{}
}
state.NetworkSuspended = expectedSuspended
encoded, _ := json.Marshal(state)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
}
return operation.Resolution{}
}
func (l *Libvirt) reconcileReinstall(record operation.Record, domain *libvirt.Domain) operation.Resolution {
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return operation.Resolution{}
}
replacementName := reinstallVolumeName(record.ResourceID, record.OperationID)
replacement, lookupErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, replacementName)
if lookupErr != nil {
return operation.Resolution{}
}
replacementExists := replacement != nil
if replacement != nil {
replacement.Free()
}
if diskTransitionCommitted(metadata, replacementName, "reinstall", record.OperationID) && replacementExists {
result, parseErr := metadataFromXML(document)
if parseErr != nil {
return operation.Resolution{}
}
state, _, stateErr := domain.GetState()
if stateErr != nil {
return operation.Resolution{}
}
if diskTransitionMarkerMatches(metadata, "reinstall", record.OperationID) {
if metadata.DiskTransitionRunning && state != libvirt.DOMAIN_RUNNING && state != libvirt.DOMAIN_BLOCKED {
return operation.Resolution{}
}
if !metadata.DiskTransitionRunning && state != libvirt.DOMAIN_SHUTOFF && state != libvirt.DOMAIN_SHUTDOWN {
return operation.Resolution{}
}
}
if metadata.PendingCleanupVolume != "" {
cleaned, cleanErr := l.completeDiskTransitionCleanup(
domain, document, metadata, "reinstall", record.OperationID,
)
if cleanErr != nil {
return operation.Resolution{}
}
document = cleaned
result, parseErr = metadataFromXML(document)
if parseErr != nil {
return operation.Resolution{}
}
}
result.Status = mapDomainState(state)
result.ReinstallCommitted = true
result.CleanupPending = false
encoded, _ := json.Marshal(result)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
}
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "reinstall_not_committed", ErrorMessage: "host evidence confirms the reinstall did not switch the root volume"}
}
return operation.Resolution{}
}
func (l *Libvirt) reconcileSnapshot(record operation.Record, domain *libvirt.Domain) operation.Resolution {
parts := strings.Split(record.Action, ":")
if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) {
return operation.Resolution{}
}
key := parts[1]
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return operation.Resolution{}
}
var stored *domainSnapshotMetadataXML
for index := range metadata.Snapshots {
if metadata.Snapshots[index].Key == key {
stored = &metadata.Snapshots[index]
break
}
}
snapshot, lookupErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, snapshotVolumeName(record.ResourceID, key))
if lookupErr != nil {
return operation.Resolution{}
}
snapshotExists := snapshot != nil
if snapshot != nil {
defer snapshot.Free()
}
switch parts[0] {
case "create-snapshot":
if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") {
return operation.Resolution{}
}
if stored != nil && snapshotExists && stored.Consistency == parts[2] {
info, validateErr := validateQCOW2Volume(snapshot, stored.SizeBytes)
if validateErr != nil || info.Capacity != stored.SizeBytes {
return operation.Resolution{}
}
result := Snapshot{ID: key, SnapshotKey: key, Name: stored.Name, Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(), State: "ready", Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, Created: true}
encoded, _ := json.Marshal(result)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
}
if stored == nil && !snapshotExists {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_created", ErrorMessage: "host evidence confirms the snapshot volume is absent"}
}
if stored == nil && snapshotExists {
if err := l.deleteSnapshotVolumeDuringReconcile(domain,
record.ResourceID, key, snapshot); err != nil {
return operation.Resolution{}
}
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "snapshot_not_created",
ErrorMessage: "snapshot data completed without durable metadata and was rolled back"}
}
if stored != nil && !snapshotExists {
if _, err := l.removeSnapshotMetadataDuringReconcile(domain,
document, key); err != nil {
return operation.Resolution{}
}
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "snapshot_not_created",
ErrorMessage: "snapshot metadata had no managed volume and was removed"}
}
case "delete-snapshot":
if len(parts) != 2 {
return operation.Resolution{}
}
if stored == nil && !snapshotExists {
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
}
if stored != nil && snapshotExists {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_deleted", ErrorMessage: "host evidence confirms the snapshot remains present"}
}
if stored != nil && !snapshotExists {
if _, err := l.removeSnapshotMetadataDuringReconcile(domain,
document, key); err != nil {
return operation.Resolution{}
}
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
}
if stored == nil && snapshotExists {
if err := l.deleteSnapshotVolumeDuringReconcile(domain,
record.ResourceID, key, snapshot); err != nil {
return operation.Resolution{}
}
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
}
case "restore-snapshot":
if len(parts) != 2 || stored == nil || !snapshotExists {
return operation.Resolution{}
}
replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID)
replacement, replacementErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, replacementName)
if replacementErr != nil {
return operation.Resolution{}
}
replacementExists := replacement != nil
if replacement != nil {
replacement.Free()
}
if diskTransitionCommitted(metadata, replacementName, "snapshot_restore", record.OperationID) && replacementExists {
state, _, stateErr := domain.GetState()
if stateErr != nil {
return operation.Resolution{}
}
if diskTransitionMarkerMatches(metadata, "snapshot_restore", record.OperationID) {
if metadata.DiskTransitionRunning && state != libvirt.DOMAIN_RUNNING && state != libvirt.DOMAIN_BLOCKED {
return operation.Resolution{}
}
if !metadata.DiskTransitionRunning && state != libvirt.DOMAIN_SHUTOFF && state != libvirt.DOMAIN_SHUTDOWN {
return operation.Resolution{}
}
}
if metadata.PendingCleanupVolume != "" {
if _, cleanErr := l.completeDiskTransitionCleanup(
domain, document, metadata, "snapshot_restore", record.OperationID,
); cleanErr != nil {
return operation.Resolution{}
}
}
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Restored: true, PowerState: mapDomainState(state), ActiveVolumeName: replacementName, CleanupPending: false})
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
}
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_restored", ErrorMessage: "host evidence confirms the root volume was not switched"}
}
}
return operation.Resolution{}
}
func (l *Libvirt) deleteSnapshotVolumeDuringReconcile(domain *libvirt.Domain,
uuid, key string, volume *libvirt.StorageVol) error {
if volume == nil || snapshotVolumeName(uuid, key) == "" {
return NewError(ErrConflict, "snapshot_cleanup_invalid",
"snapshot cleanup target is invalid", nil)
}
if err := ensureNoActiveDomainJobIfRunning(domain); err != nil {
return err
}
if _, err := validateQCOW2Volume(volume, 0); err != nil {
return err
}
path, err := volume.GetPath()
if err != nil {
return translateLibvirtError("resolve snapshot cleanup volume", err)
}
referenced, err := l.anyDomainUsesPath(path)
if err != nil {
return err
}
if referenced {
return NewError(ErrConflict, "snapshot_in_use",
"snapshot cleanup volume is referenced by a domain", nil)
}
if err := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(
l.local.ManagedStoragePool, snapshotVolumeName(uuid, key),
)
if observed != nil {
_ = observed.Free()
}
if inspectErr != nil || observed != nil {
return operation.MarkUncertain(errors.Join(
mutationLibvirtError("delete snapshot cleanup volume", err), inspectErr,
))
}
}
return nil
}
func (l *Libvirt) removeSnapshotMetadataDuringReconcile(_ *libvirt.Domain,
document, key string) (string, error) {
updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
kept := stored.Snapshots[:0]
for _, snapshot := range stored.Snapshots {
if snapshot.Key != key {
kept = append(kept, snapshot)
}
}
stored.Snapshots = kept
return nil
})
if err != nil {
return "", err
}
defined, err := l.conn.DomainDefineXML(updated)
if err != nil {
return "", operation.MarkUncertain(mutationLibvirtError(
"remove orphan snapshot metadata", err,
))
}
_ = defined.Free()
return updated, nil
}
// diskTransitionCommitted accepts both the retained completion marker used by
// current Agents and the marker-free state written by older Agents. The root
// volume name is derived from the exact operation ID, so a matching active
// root plus an existing replacement volume is durable commit evidence.
func diskTransitionCommitted(metadata *domainMetadataXML, replacementName, kind,
operationID string) bool {
if metadata == nil || metadata.RootVolumeName != replacementName {
return false
}
if diskTransitionMarkerMatches(metadata, kind, operationID) {
return true
}
return metadata.PendingCleanupVolume == "" &&
metadata.DiskTransitionKind == "" && metadata.DiskTransitionID == ""
}
func diskTransitionMarkerMatches(metadata *domainMetadataXML, kind,
operationID string) bool {
return metadata != nil && metadata.DiskTransitionKind == kind &&
metadata.DiskTransitionID == operationID
}
// completeDiskTransitionCleanup resumes only the deterministic cleanup tail
// after read-only evidence has proved the new root is committed. It never
// repeats the root switch, power action, seed write, or original restore.
func (l *Libvirt) completeDiskTransitionCleanup(domain *libvirt.Domain,
document string, metadata *domainMetadataXML, kind, operationID string) (string, error) {
pending := metadata.PendingCleanupVolume
if pending == "" {
return document, nil
}
if !isVMOwnedRootVolumeName(metadataUUID(document), pending) ||
pending == metadata.RootVolumeName {
return "", NewError(ErrConflict, "disk_transition_cleanup_invalid",
"pending disk cleanup does not identify an old VM-owned root", nil)
}
old, err := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, pending)
if err != nil {
return "", err
}
if old != nil {
if _, err := validateQCOW2Volume(old, 0); err != nil {
_ = old.Free()
return "", err
}
path, err := old.GetPath()
if err != nil {
_ = old.Free()
return "", translateLibvirtError("resolve previous root during reconciliation", err)
}
used, err := l.anyDomainUsesPath(path)
if err != nil {
_ = old.Free()
return "", err
}
if used {
_ = old.Free()
return "", NewError(ErrConflict, kind+"_cleanup_pending",
"committed root transition still references the previous root", nil)
}
deleteErr := old.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = old.Free()
if deleteErr != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, pending)
if observed != nil {
_ = observed.Free()
}
if inspectErr != nil || observed != nil {
return "", operation.MarkUncertain(errors.Join(
mutationLibvirtError("delete previous root during reconciliation", deleteErr), inspectErr,
))
}
}
}
cleaned, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
stored.PendingCleanupVolume = ""
stored.DiskTransitionKind = kind
stored.DiskTransitionID = operationID
return nil
})
if err != nil {
return "", err
}
defined, err := l.conn.DomainDefineXML(cleaned)
if err != nil {
return "", operation.MarkUncertain(mutationLibvirtError(
"persist completed disk cleanup", err,
))
}
_ = defined.Free()
return cleaned, nil
}
func metadataUUID(document string) string {
var definition libvirtxml.Domain
if definition.Unmarshal(document) != nil {
return ""
}
return definition.UUID
}