You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
818 lines
31 KiB
Go
818 lines
31 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
func (l *Libvirt) reconcileCreatedVM(domain *libvirt.Domain) (operation.Resolution, error) {
|
|
document, definition, metadata, pool, root, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
defer pool.Free()
|
|
defer root.Free()
|
|
autostart, err := domain.GetAutostart()
|
|
if err != nil || !autostart {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil || !active {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil || (state != libvirt.DOMAIN_RUNNING && state != libvirt.DOMAIN_BLOCKED) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
seed, err := l.lookupOptionalManagedVolume(l.local.SeedStoragePool, seedVolumeName(definition.UUID))
|
|
if err != nil || seed == nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
seedPath, err := seed.GetPath()
|
|
seed.Free()
|
|
if err != nil || definition.Devices == nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
seedAttached := false
|
|
filterName := networkFilterName(definition.UUID)
|
|
filterAttached := len(definition.Devices.Interfaces) == 1 && definition.Devices.Interfaces[0].FilterRef != nil && definition.Devices.Interfaces[0].FilterRef.Filter == filterName
|
|
for _, disk := range definition.Devices.Disks {
|
|
if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sda" && disk.Source != nil && disk.Source.File != nil && disk.Source.File.File == seedPath && disk.ReadOnly != nil {
|
|
seedAttached = true
|
|
}
|
|
}
|
|
if !seedAttached || !filterAttached {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
var policy PlatformNetworkPolicySnapshot
|
|
if json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &policy) != nil || ValidatePlatformNetworkPolicy(policy) != nil || l.verifyManagedFilter(filterName, policy) != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
result, err := metadataFromXML(document)
|
|
if err != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
result.Status = VMStateRunning
|
|
encoded, err := json.Marshal(result)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, nil
|
|
}
|
|
|
|
func (l *Libvirt) reconcileDomainFeatureOperation(record operation.Record, domain *libvirt.Domain) (operation.Resolution, bool, error) {
|
|
if record.ExpectedState != nil {
|
|
switch record.ExpectedState.Kind {
|
|
case operation.ExpectedUpdateSettings:
|
|
resolution, err := l.reconcileSettings(record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedChangeNetwork:
|
|
resolution, err := l.reconcileNetworkChange(record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedNetworkState:
|
|
resolution, err := l.reconcileNetworkState(record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedServiceState:
|
|
resolution, err := l.reconcileServiceState(record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedFirewall:
|
|
resolution, err := l.reconcileFirewall(record, domain)
|
|
return resolution, true, err
|
|
}
|
|
}
|
|
if record.Action == "reinstall-vm" {
|
|
return l.reconcileReinstall(record, domain), true, nil
|
|
}
|
|
if strings.HasPrefix(record.Action, "create-snapshot:") || strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:") {
|
|
return l.reconcileSnapshot(record, domain), true, nil
|
|
}
|
|
if record.Action == "enter-rescue" || record.Action == "force-start-rescue" || strings.HasPrefix(record.Action, "exit-rescue:") {
|
|
return l.reconcileRescue(record, domain), true, nil
|
|
}
|
|
return operation.Resolution{}, false, nil
|
|
}
|
|
|
|
func (l *Libvirt) reconcileSettings(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return expectedEvidenceError("settings", err)
|
|
}
|
|
seed, err := l.managedSeedContents(record.ResourceID)
|
|
if err != nil {
|
|
return expectedEvidenceError("settings", err)
|
|
}
|
|
seedMarker, markerErr := seedMetadataMarker(seed, "virt-cloud-settings-operation")
|
|
if markerErr != nil {
|
|
// A rewritten or damaged seed is evidence that the first phase may
|
|
// have crossed its commit boundary. It is not proof of failure.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
domainMatches := metadata.SettingsTransitionID == expected.TransitionID
|
|
seedMatches := seedMarker == expected.TransitionID
|
|
if !domainMatches || !seedMatches {
|
|
return classifyTransitionMarkers("settings", domainMatches, seedMatches), nil
|
|
}
|
|
if expected.HostnameChanged && metadata.Hostname != expected.Hostname {
|
|
// Both durable markers belong to this operation, so it crossed the
|
|
// commit boundary. A mismatched payload can be corruption or a later
|
|
// out-of-band edit; it is not evidence that PHP may roll state back.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
result, err := metadataFromXML(document)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile settings domain state", err)
|
|
}
|
|
result.Status = mapDomainState(state)
|
|
result.SettingsChanged = true
|
|
return resolvedExpectedSuccess(result)
|
|
}
|
|
|
|
func (l *Libvirt) reconcileNetworkChange(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("read network definition for reconciliation", err)
|
|
}
|
|
definition, metadata, err := parseManagedDomain(document)
|
|
if err != nil {
|
|
return expectedEvidenceError("network", err)
|
|
}
|
|
seed, err := l.managedSeedContents(record.ResourceID)
|
|
if err != nil {
|
|
return expectedEvidenceError("network", err)
|
|
}
|
|
seedMarker, markerErr := seedMetadataMarker(seed, "virt-cloud-network-operation")
|
|
if markerErr != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
domainMatches := metadata.NetworkTransitionID == expected.TransitionID
|
|
seedMatches := seedMarker == expected.TransitionID
|
|
if !domainMatches || !seedMatches {
|
|
return classifyTransitionMarkers("network", domainMatches, seedMatches), nil
|
|
}
|
|
mac, publicCIDR, gateway, dns, networkErr := seedNetwork(seed)
|
|
if networkErr != nil || expected.NetworkSelector != metadata.NetworkSelector || expected.PublicPoolKey != metadata.PublicPoolKey ||
|
|
expected.PublicPoolCIDR != metadata.PublicPoolCIDR || expected.PublicIPCIDR != metadata.PublicIPCIDR ||
|
|
expected.Gateway != metadata.Gateway || !slices.Equal(expected.DNSServers, metadata.DNSServers) ||
|
|
expected.PublicIPCIDR != publicCIDR || expected.Gateway != gateway || !slices.Equal(expected.DNSServers, dns) ||
|
|
definition.Devices == nil || len(definition.Devices.Interfaces) != 1 || definition.Devices.Interfaces[0].MAC == nil ||
|
|
!strings.EqualFold(definition.Devices.Interfaces[0].MAC.Address, mac) ||
|
|
!domainNetworkSourceMatches(definition.Devices.Interfaces[0], expected.NetworkSourceType, expected.NetworkSourceName) {
|
|
// Matching transition markers prove that both phases wrote state for
|
|
// this operation. A payload mismatch is therefore ambiguous and must
|
|
// not authorize PHP to release or restore an address assignment.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
result, err := metadataFromXML(document)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile network domain state", err)
|
|
}
|
|
result.Status = mapDomainState(state)
|
|
result.NetworkChanged = true
|
|
return resolvedExpectedSuccess(result)
|
|
}
|
|
|
|
func (l *Libvirt) reconcileNetworkState(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
inactive, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile persistent network state", err)
|
|
}
|
|
if _, _, err := parseManagedDomain(inactive); err != nil {
|
|
return expectedEvidenceError("network", err)
|
|
}
|
|
if !domainNetworkLinksMatch(inactive, expected.Enabled) {
|
|
return resolvedExpectedFailure("network_state_not_applied", "persistent interface link state does not match the requested state"), nil
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile live network state", err)
|
|
}
|
|
if active {
|
|
live, err := domain.GetXMLDesc(0)
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("read live network state", err)
|
|
}
|
|
if !domainNetworkLinksMatch(live, expected.Enabled) {
|
|
return resolvedExpectedFailure("network_state_not_applied", "live interface link state does not match the requested state"), nil
|
|
}
|
|
}
|
|
return resolvedExpectedSuccess(map[string]bool{"network_enabled": expected.Enabled})
|
|
}
|
|
|
|
func (l *Libvirt) reconcileServiceState(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
if _, _, err := requireManagedDomain(domain); err != nil {
|
|
return expectedEvidenceError("service", err)
|
|
}
|
|
autostart, err := domain.GetAutostart()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile service autostart", err)
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile service activity", err)
|
|
}
|
|
if autostart != expected.Enabled || (!expected.Enabled && active) || (expected.Enabled && expected.StartIfSuspended && !active) {
|
|
return resolvedExpectedFailure("service_state_not_applied", "host service state does not match the requested state"), nil
|
|
}
|
|
state, reason, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile service domain state", err)
|
|
}
|
|
name, _ := domain.GetName()
|
|
return resolvedExpectedSuccess(VMState{UUID: record.ResourceID, Name: name, State: mapDomainState(state), Reason: reason})
|
|
}
|
|
|
|
func (l *Libvirt) reconcileFirewall(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
_, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return expectedEvidenceError("firewall", err)
|
|
}
|
|
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(record.ResourceID))
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
if filter == nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
filterDocument, err := filter.GetXMLDesc(0)
|
|
filter.Free()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("read firewall filter for reconciliation", err)
|
|
}
|
|
|
|
domainMarker := metadata.FirewallTransitionID == expected.TransitionID
|
|
domainMatches := domainMarker && metadata.Firewall != nil &&
|
|
metadata.Firewall.Revision == expected.FirewallRevision &&
|
|
metadata.Firewall.Digest == expected.FirewallDigest
|
|
if domainMatches {
|
|
policy, validateErr := validateFirewallPolicy(firewallFromMetadata(metadata.Firewall))
|
|
if validateErr != nil || policy.PolicyDigest != expected.FirewallDigest ||
|
|
!firewallDocumentMatchesPolicy(filterDocument, policy) {
|
|
// Domain metadata committed this operation, but the filter cannot be
|
|
// proven to match it. Never report failure and roll PHP state back.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
return resolvedExpectedSuccess(policy)
|
|
}
|
|
if domainMarker || !firewallDocumentMatchesStoredState(filterDocument, metadata.Firewall) {
|
|
// Either this operation wrote only the filter, or the domain marker is
|
|
// present with malformed/stale policy metadata. Both are half-commit
|
|
// evidence, so keep the exact operation uncertain.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
return resolvedExpectedFailure("firewall_not_committed", "host evidence confirms the operation did not commit its transition markers"), nil
|
|
}
|
|
|
|
func (l *Libvirt) managedSeedContents(uuid string) ([]byte, error) {
|
|
seed, err := l.lookupOptionalManagedVolume(l.local.SeedStoragePool, seedVolumeName(uuid))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if seed == nil {
|
|
return nil, NewError(ErrNotFound, "seed_not_found", "managed NoCloud seed volume was not found", nil)
|
|
}
|
|
defer seed.Free()
|
|
return l.downloadSmallVolume(seed, maxSeedImageBytes)
|
|
}
|
|
|
|
func seedMetadataMarker(seed []byte, key string) (string, error) {
|
|
files, err := readSeedISO(seed)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var metadata map[string]any
|
|
if err := json.Unmarshal(files.MetaData, &metadata); err != nil {
|
|
return "", err
|
|
}
|
|
value, _ := metadata[key].(string)
|
|
if value != "" && !operationIdentifierPattern.MatchString(value) {
|
|
return "", errors.New("invalid NoCloud transition marker")
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func domainNetworkLinksMatch(document string, enabled bool) bool {
|
|
var definition libvirtxml.Domain
|
|
if definition.Unmarshal(document) != nil || definition.Devices == nil || len(definition.Devices.Interfaces) == 0 {
|
|
return false
|
|
}
|
|
for _, iface := range definition.Devices.Interfaces {
|
|
state := "up"
|
|
if iface.Link != nil && iface.Link.State != "" {
|
|
state = iface.Link.State
|
|
}
|
|
if (state == "up") != enabled {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func domainNetworkSourceMatches(iface libvirtxml.DomainInterface, sourceType, sourceName string) bool {
|
|
if iface.Source == nil {
|
|
return false
|
|
}
|
|
switch sourceType {
|
|
case "bridge":
|
|
return iface.Source.Bridge != nil && iface.Source.Bridge.Bridge == sourceName
|
|
case "network":
|
|
return iface.Source.Network != nil && iface.Source.Network.Network == sourceName
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func firewallDocumentMatchesPolicy(document string, policy FirewallPolicy) bool {
|
|
var current libvirtxml.NWFilter
|
|
if current.Unmarshal(document) != nil {
|
|
return false
|
|
}
|
|
normalized, err := current.Marshal()
|
|
if err != nil {
|
|
return false
|
|
}
|
|
expected, err := buildUserFirewallXML(document, policy)
|
|
return err == nil && normalized == expected
|
|
}
|
|
|
|
// firewallDocumentMatchesStoredState proves that the filter still represents
|
|
// the policy durably recorded before an operation. A nil policy means the
|
|
// untouched platform filter, which must contain no user/default policy rules.
|
|
func firewallDocumentMatchesStoredState(document string, stored *domainFirewallMetadataXML) bool {
|
|
if stored != nil {
|
|
policy, err := validateFirewallPolicy(firewallFromMetadata(stored))
|
|
return err == nil && firewallDocumentMatchesPolicy(document, policy)
|
|
}
|
|
var current libvirtxml.NWFilter
|
|
if current.Unmarshal(document) != nil {
|
|
return false
|
|
}
|
|
for _, entry := range current.Entries {
|
|
comment := nwfilterRuleComment(entry.Rule)
|
|
if strings.HasPrefix(comment, "virt-user:") || strings.HasPrefix(comment, "virt-default:") {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func classifyTransitionMarkers(kind string, primary, secondary bool) operation.Resolution {
|
|
if !primary && !secondary {
|
|
return resolvedExpectedFailure(kind+"_not_committed", "host evidence confirms the operation did not commit its transition markers")
|
|
}
|
|
// One matching marker proves that the mutation crossed its first commit
|
|
// boundary. It does not prove which side contains the authoritative state:
|
|
// for example, a network seed may already contain the new public address
|
|
// while the persistent domain still has the old bridge. Finalizing this as
|
|
// a failure would let PHP release or roll back resources that the guest may
|
|
// already be using. Keep the exact operation uncertain for administrator
|
|
// reconciliation; never replay the original two-phase mutation.
|
|
return operation.Resolution{}
|
|
}
|
|
|
|
func expectedStateResourceAbsent(_ operation.Record) operation.Resolution {
|
|
return resolvedExpectedFailure("expected_state_resource_absent", "host evidence confirms the target VM is absent and the expected state cannot be satisfied")
|
|
}
|
|
|
|
func expectedEvidenceError(_ string, err error) (operation.Resolution, error) {
|
|
if errors.Is(err, ErrUnavailable) {
|
|
return operation.Resolution{}, err
|
|
}
|
|
// Missing, malformed, or ownership-inconsistent evidence cannot prove that
|
|
// an expected-state mutation did not commit. In particular, a network
|
|
// change may already have written the requested address into either the
|
|
// domain definition or NoCloud seed. Finalizing it as failed would let the
|
|
// panel release that address while the guest may still be using it. Keep the
|
|
// exact operation uncertain until an administrator restores readable
|
|
// evidence; never infer a safe rollback from damaged host state.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
|
|
func resolvedExpectedFailure(code, message string) operation.Resolution {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: code, ErrorMessage: message}
|
|
}
|
|
|
|
func resolvedExpectedSuccess(value any) (operation.Resolution, error) {
|
|
encoded, err := json.Marshal(value)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, nil
|
|
}
|
|
|
|
func (l *Libvirt) reconcileRescue(record operation.Record, domain *libvirt.Domain) operation.Resolution {
|
|
state, err := l.rescueState(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
switch record.Action {
|
|
case "enter-rescue":
|
|
if state.Active || state.Staged {
|
|
encoded, _ := json.Marshal(state)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_not_committed", ErrorMessage: "host evidence confirms rescue boot was not staged"}
|
|
case "force-start-rescue":
|
|
if state.Active {
|
|
encoded, _ := json.Marshal(state)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
|
|
}
|
|
if state.Staged {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_force_start_incomplete", ErrorMessage: "rescue boot remains staged but is not running"}
|
|
}
|
|
case "exit-rescue:connected", "exit-rescue:suspended":
|
|
if state.Active || state.Staged {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_exit_incomplete", ErrorMessage: "host evidence confirms rescue boot remains staged or active"}
|
|
}
|
|
expectedSuspended := record.Action == "exit-rescue:suspended"
|
|
if state.NetworkSuspended != expectedSuspended {
|
|
return operation.Resolution{}
|
|
}
|
|
active, activeErr := domain.IsActive()
|
|
if activeErr != nil || !active {
|
|
return operation.Resolution{}
|
|
}
|
|
live, liveErr := domain.GetXMLDesc(0)
|
|
if liveErr != nil || !domainNetworkLinksMatch(live, !expectedSuspended) {
|
|
return operation.Resolution{}
|
|
}
|
|
state.NetworkSuspended = expectedSuspended
|
|
encoded, _ := json.Marshal(state)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
|
|
}
|
|
return operation.Resolution{}
|
|
}
|
|
|
|
func (l *Libvirt) reconcileReinstall(record operation.Record, domain *libvirt.Domain) operation.Resolution {
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
replacementName := reinstallVolumeName(record.ResourceID, record.OperationID)
|
|
replacement, lookupErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, replacementName)
|
|
if lookupErr != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
replacementExists := replacement != nil
|
|
if replacement != nil {
|
|
replacement.Free()
|
|
}
|
|
if diskTransitionCommitted(metadata, replacementName, "reinstall", record.OperationID) && replacementExists {
|
|
result, parseErr := metadataFromXML(document)
|
|
if parseErr != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
if diskTransitionMarkerMatches(metadata, "reinstall", record.OperationID) {
|
|
if metadata.DiskTransitionRunning && state != libvirt.DOMAIN_RUNNING && state != libvirt.DOMAIN_BLOCKED {
|
|
return operation.Resolution{}
|
|
}
|
|
if !metadata.DiskTransitionRunning && state != libvirt.DOMAIN_SHUTOFF && state != libvirt.DOMAIN_SHUTDOWN {
|
|
return operation.Resolution{}
|
|
}
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
cleaned, cleanErr := l.completeDiskTransitionCleanup(
|
|
domain, document, metadata, "reinstall", record.OperationID,
|
|
)
|
|
if cleanErr != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
document = cleaned
|
|
result, parseErr = metadataFromXML(document)
|
|
if parseErr != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
}
|
|
result.Status = mapDomainState(state)
|
|
result.ReinstallCommitted = true
|
|
result.CleanupPending = false
|
|
encoded, _ := json.Marshal(result)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
|
|
}
|
|
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "reinstall_not_committed", ErrorMessage: "host evidence confirms the reinstall did not switch the root volume"}
|
|
}
|
|
return operation.Resolution{}
|
|
}
|
|
|
|
func (l *Libvirt) reconcileSnapshot(record operation.Record, domain *libvirt.Domain) operation.Resolution {
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) {
|
|
return operation.Resolution{}
|
|
}
|
|
key := parts[1]
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
var stored *domainSnapshotMetadataXML
|
|
for index := range metadata.Snapshots {
|
|
if metadata.Snapshots[index].Key == key {
|
|
stored = &metadata.Snapshots[index]
|
|
break
|
|
}
|
|
}
|
|
snapshot, lookupErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, snapshotVolumeName(record.ResourceID, key))
|
|
if lookupErr != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
snapshotExists := snapshot != nil
|
|
if snapshot != nil {
|
|
defer snapshot.Free()
|
|
}
|
|
switch parts[0] {
|
|
case "create-snapshot":
|
|
if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") {
|
|
return operation.Resolution{}
|
|
}
|
|
if stored != nil && snapshotExists && stored.Consistency == parts[2] {
|
|
info, validateErr := validateQCOW2Volume(snapshot, stored.SizeBytes)
|
|
if validateErr != nil || info.Capacity != stored.SizeBytes {
|
|
return operation.Resolution{}
|
|
}
|
|
result := Snapshot{ID: key, SnapshotKey: key, Name: stored.Name, Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(), State: "ready", Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, Created: true}
|
|
encoded, _ := json.Marshal(result)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
|
|
}
|
|
if stored == nil && !snapshotExists {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_created", ErrorMessage: "host evidence confirms the snapshot volume is absent"}
|
|
}
|
|
if stored == nil && snapshotExists {
|
|
if err := l.deleteSnapshotVolumeDuringReconcile(domain,
|
|
record.ResourceID, key, snapshot); err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_created",
|
|
ErrorMessage: "snapshot data completed without durable metadata and was rolled back"}
|
|
}
|
|
if stored != nil && !snapshotExists {
|
|
if _, err := l.removeSnapshotMetadataDuringReconcile(domain,
|
|
document, key); err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_created",
|
|
ErrorMessage: "snapshot metadata had no managed volume and was removed"}
|
|
}
|
|
case "delete-snapshot":
|
|
if len(parts) != 2 {
|
|
return operation.Resolution{}
|
|
}
|
|
if stored == nil && !snapshotExists {
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
|
|
}
|
|
if stored != nil && snapshotExists {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_deleted", ErrorMessage: "host evidence confirms the snapshot remains present"}
|
|
}
|
|
if stored != nil && !snapshotExists {
|
|
if _, err := l.removeSnapshotMetadataDuringReconcile(domain,
|
|
document, key); err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
|
|
}
|
|
if stored == nil && snapshotExists {
|
|
if err := l.deleteSnapshotVolumeDuringReconcile(domain,
|
|
record.ResourceID, key, snapshot); err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
|
|
}
|
|
case "restore-snapshot":
|
|
if len(parts) != 2 || stored == nil || !snapshotExists {
|
|
return operation.Resolution{}
|
|
}
|
|
replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID)
|
|
replacement, replacementErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, replacementName)
|
|
if replacementErr != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
replacementExists := replacement != nil
|
|
if replacement != nil {
|
|
replacement.Free()
|
|
}
|
|
if diskTransitionCommitted(metadata, replacementName, "snapshot_restore", record.OperationID) && replacementExists {
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
if diskTransitionMarkerMatches(metadata, "snapshot_restore", record.OperationID) {
|
|
if metadata.DiskTransitionRunning && state != libvirt.DOMAIN_RUNNING && state != libvirt.DOMAIN_BLOCKED {
|
|
return operation.Resolution{}
|
|
}
|
|
if !metadata.DiskTransitionRunning && state != libvirt.DOMAIN_SHUTOFF && state != libvirt.DOMAIN_SHUTDOWN {
|
|
return operation.Resolution{}
|
|
}
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
if _, cleanErr := l.completeDiskTransitionCleanup(
|
|
domain, document, metadata, "snapshot_restore", record.OperationID,
|
|
); cleanErr != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
}
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Restored: true, PowerState: mapDomainState(state), ActiveVolumeName: replacementName, CleanupPending: false})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
|
|
}
|
|
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_restored", ErrorMessage: "host evidence confirms the root volume was not switched"}
|
|
}
|
|
}
|
|
return operation.Resolution{}
|
|
}
|
|
|
|
func (l *Libvirt) deleteSnapshotVolumeDuringReconcile(domain *libvirt.Domain,
|
|
uuid, key string, volume *libvirt.StorageVol) error {
|
|
if volume == nil || snapshotVolumeName(uuid, key) == "" {
|
|
return NewError(ErrConflict, "snapshot_cleanup_invalid",
|
|
"snapshot cleanup target is invalid", nil)
|
|
}
|
|
if err := ensureNoActiveDomainJobIfRunning(domain); err != nil {
|
|
return err
|
|
}
|
|
if _, err := validateQCOW2Volume(volume, 0); err != nil {
|
|
return err
|
|
}
|
|
path, err := volume.GetPath()
|
|
if err != nil {
|
|
return translateLibvirtError("resolve snapshot cleanup volume", err)
|
|
}
|
|
referenced, err := l.anyDomainUsesPath(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if referenced {
|
|
return NewError(ErrConflict, "snapshot_in_use",
|
|
"snapshot cleanup volume is referenced by a domain", nil)
|
|
}
|
|
if err := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil {
|
|
observed, inspectErr := l.lookupOptionalManagedVolume(
|
|
l.local.ManagedStoragePool, snapshotVolumeName(uuid, key),
|
|
)
|
|
if observed != nil {
|
|
_ = observed.Free()
|
|
}
|
|
if inspectErr != nil || observed != nil {
|
|
return operation.MarkUncertain(errors.Join(
|
|
mutationLibvirtError("delete snapshot cleanup volume", err), inspectErr,
|
|
))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) removeSnapshotMetadataDuringReconcile(_ *libvirt.Domain,
|
|
document, key string) (string, error) {
|
|
updated, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
|
|
kept := stored.Snapshots[:0]
|
|
for _, snapshot := range stored.Snapshots {
|
|
if snapshot.Key != key {
|
|
kept = append(kept, snapshot)
|
|
}
|
|
}
|
|
stored.Snapshots = kept
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defined, err := l.conn.DomainDefineXML(updated)
|
|
if err != nil {
|
|
return "", operation.MarkUncertain(mutationLibvirtError(
|
|
"remove orphan snapshot metadata", err,
|
|
))
|
|
}
|
|
_ = defined.Free()
|
|
return updated, nil
|
|
}
|
|
|
|
// diskTransitionCommitted accepts both the retained completion marker used by
|
|
// current Agents and the marker-free state written by older Agents. The root
|
|
// volume name is derived from the exact operation ID, so a matching active
|
|
// root plus an existing replacement volume is durable commit evidence.
|
|
func diskTransitionCommitted(metadata *domainMetadataXML, replacementName, kind,
|
|
operationID string) bool {
|
|
if metadata == nil || metadata.RootVolumeName != replacementName {
|
|
return false
|
|
}
|
|
if diskTransitionMarkerMatches(metadata, kind, operationID) {
|
|
return true
|
|
}
|
|
return metadata.PendingCleanupVolume == "" &&
|
|
metadata.DiskTransitionKind == "" && metadata.DiskTransitionID == ""
|
|
}
|
|
|
|
func diskTransitionMarkerMatches(metadata *domainMetadataXML, kind,
|
|
operationID string) bool {
|
|
return metadata != nil && metadata.DiskTransitionKind == kind &&
|
|
metadata.DiskTransitionID == operationID
|
|
}
|
|
|
|
// completeDiskTransitionCleanup resumes only the deterministic cleanup tail
|
|
// after read-only evidence has proved the new root is committed. It never
|
|
// repeats the root switch, power action, seed write, or original restore.
|
|
func (l *Libvirt) completeDiskTransitionCleanup(domain *libvirt.Domain,
|
|
document string, metadata *domainMetadataXML, kind, operationID string) (string, error) {
|
|
pending := metadata.PendingCleanupVolume
|
|
if pending == "" {
|
|
return document, nil
|
|
}
|
|
if !isVMOwnedRootVolumeName(metadataUUID(document), pending) ||
|
|
pending == metadata.RootVolumeName {
|
|
return "", NewError(ErrConflict, "disk_transition_cleanup_invalid",
|
|
"pending disk cleanup does not identify an old VM-owned root", nil)
|
|
}
|
|
old, err := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, pending)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if old != nil {
|
|
if _, err := validateQCOW2Volume(old, 0); err != nil {
|
|
_ = old.Free()
|
|
return "", err
|
|
}
|
|
path, err := old.GetPath()
|
|
if err != nil {
|
|
_ = old.Free()
|
|
return "", translateLibvirtError("resolve previous root during reconciliation", err)
|
|
}
|
|
used, err := l.anyDomainUsesPath(path)
|
|
if err != nil {
|
|
_ = old.Free()
|
|
return "", err
|
|
}
|
|
if used {
|
|
_ = old.Free()
|
|
return "", NewError(ErrConflict, kind+"_cleanup_pending",
|
|
"committed root transition still references the previous root", nil)
|
|
}
|
|
deleteErr := old.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
_ = old.Free()
|
|
if deleteErr != nil {
|
|
observed, inspectErr := l.lookupOptionalManagedVolume(l.local.ManagedStoragePool, pending)
|
|
if observed != nil {
|
|
_ = observed.Free()
|
|
}
|
|
if inspectErr != nil || observed != nil {
|
|
return "", operation.MarkUncertain(errors.Join(
|
|
mutationLibvirtError("delete previous root during reconciliation", deleteErr), inspectErr,
|
|
))
|
|
}
|
|
}
|
|
}
|
|
cleaned, err := mutateManagedMetadata(document, func(stored *domainMetadataXML) error {
|
|
stored.PendingCleanupVolume = ""
|
|
stored.DiskTransitionKind = kind
|
|
stored.DiskTransitionID = operationID
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defined, err := l.conn.DomainDefineXML(cleaned)
|
|
if err != nil {
|
|
return "", operation.MarkUncertain(mutationLibvirtError(
|
|
"persist completed disk cleanup", err,
|
|
))
|
|
}
|
|
_ = defined.Free()
|
|
return cleaned, nil
|
|
}
|
|
|
|
func metadataUUID(document string) string {
|
|
var definition libvirtxml.Domain
|
|
if definition.Unmarshal(document) != nil {
|
|
return ""
|
|
}
|
|
return definition.UUID
|
|
}
|