You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_reconcile_features.go

723 lines
28 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"encoding/json"
"errors"
"strings"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
func (l *Libvirt) reconcileCreatedVM(ctx context.Context, record operation.Record,
domain *libvirt.Domain) (operation.Resolution, error) {
document, definition, metadata, pool, root, err := l.managedRootContext(domain)
if err != nil {
return operation.Resolution{}, nil
}
defer pool.Free()
defer root.Free()
autostart, err := domain.GetAutostart()
if err != nil || !autostart {
return operation.Resolution{}, nil
}
active, err := domain.IsActive()
if err != nil || !active {
return operation.Resolution{}, nil
}
state, _, err := domain.GetState()
if err != nil || (state != libvirt.DOMAIN_RUNNING && state != libvirt.DOMAIN_BLOCKED) {
return operation.Resolution{}, nil
}
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(definition.UUID))
if err != nil || seed == nil {
return operation.Resolution{}, err
}
seedPath, err := seed.GetPath()
seed.Free()
if err != nil || definition.Devices == nil {
return operation.Resolution{}, nil
}
seedAttached := false
filterName := networkFilterName(definition.UUID)
filterAttached := managedNetworkLayoutMatches(definition, metadata)
for _, disk := range definition.Devices.Disks {
if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sda" && disk.Source != nil && disk.Source.File != nil && disk.Source.File.File == seedPath && disk.ReadOnly != nil {
seedAttached = true
}
}
if !seedAttached || !filterAttached {
return operation.Resolution{}, nil
}
var policy PlatformNetworkPolicySnapshot
if json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &policy) != nil || ValidatePlatformNetworkPolicy(policy) != nil || l.verifyManagedFilter(filterName, policy) != nil {
return operation.Resolution{}, nil
}
if record.ExpectedState != nil && record.ExpectedState.Kind == operation.ExpectedCreateLikeNetwork {
seedContents, seedErr := l.managedSeedContents(ctx, record.ResourceID)
if seedErr != nil {
return expectedEvidenceError("create-like network", seedErr)
}
if !createLikeNetworkEvidenceMatches(
record, definition, metadata, seedContents,
) {
return operation.Resolution{}, nil
}
}
result, err := metadataFromXML(document)
if err != nil {
return operation.Resolution{}, nil
}
result.Status = VMStateRunning
encoded, err := json.Marshal(result)
if err != nil {
return operation.Resolution{}, err
}
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, nil
}
// inspectExpectedStateOperation is deliberately limited to observation. It is
// the only expected-state helper reachable from InspectOperation: every call
// below reads durable domain, seed, network-filter, or runtime state and none
// defines, starts, stops, deletes, or otherwise changes a host resource.
func (l *Libvirt) inspectExpectedStateOperation(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, bool, error) {
if record.ExpectedState != nil {
switch record.ExpectedState.Kind {
case operation.ExpectedUpdateSettings:
resolution, err := l.reconcileSettings(ctx, record, domain)
return resolution, true, err
case operation.ExpectedResizeResources:
resolution, err := l.reconcileResizeResources(record, domain)
return resolution, true, err
case operation.ExpectedChangeNetwork:
resolution, err := l.reconcileNetworkChange(ctx, record, domain)
return resolution, true, err
case operation.ExpectedNetworkState:
resolution, err := l.reconcileNetworkState(record, domain)
return resolution, true, err
case operation.ExpectedServiceState:
resolution, err := l.reconcileServiceState(record, domain)
return resolution, true, err
case operation.ExpectedPeerMigrationState:
resolution, err := l.reconcilePeerMigrationState(record, domain)
return resolution, true, err
case operation.ExpectedFirewall:
resolution, err := l.reconcileFirewall(record, domain)
return resolution, true, err
case operation.ExpectedVPCPort:
resolution, err := l.reconcileVPCPort(ctx, record, domain)
return resolution, true, err
case operation.ExpectedCreateLikeNetwork:
switch {
case strings.HasPrefix(record.Action, "restore-backup:"):
resolution, err := l.reconcileCreatedVM(ctx, record, domain)
return resolution, true, err
case strings.HasPrefix(record.Action, "receive-peer-migration:"):
resolution, err := l.reconcileReceivedPeerMigration(ctx, record, domain)
return resolution, true, err
default:
// Journal validation does not permit this state/action pairing.
// Keep a damaged record fenced instead of falling through to a
// weaker action-only reconciler.
return operation.Resolution{}, true, nil
}
}
}
return operation.Resolution{}, false, nil
}
// createLikeNetworkEvidenceMatches binds an ambiguous restore or peer receive
// to the complete, immutable network identity persisted before dispatch. It
// deliberately performs no mutation. A mismatch remains unresolved so the
// panel cannot release an address or VPC identity which the guest may use.
func createLikeNetworkEvidenceMatches(record operation.Record,
definition *libvirtxml.Domain, metadata *domainMetadataXML, seed []byte) bool {
expected := record.ExpectedState
if expected == nil || expected.Kind != operation.ExpectedCreateLikeNetwork ||
expected.TransitionID != record.OperationID || definition == nil ||
metadata == nil || !strings.EqualFold(definition.UUID, record.ResourceID) ||
definition.Devices == nil {
return false
}
public, err := metadata.PublicNetwork.public()
if err != nil || !publicNetworkEqual(public, expected.PublicNetwork) ||
!managedNetworkLayoutMatches(definition, metadata) {
return false
}
seedMAC, seedPublic, err := seedNetwork(seed)
if err != nil || !strings.EqualFold(seedMAC, vmMACAddress(record.ResourceID)) ||
!seedPublic.matches(expected.PublicNetwork, seedMAC) {
return false
}
publicIndex, found := managedPublicInterfaceIndex(
definition, networkFilterName(record.ResourceID), expected.PublicNetwork,
)
if !found {
return false
}
publicInterface := definition.Devices.Interfaces[publicIndex]
if publicInterface.MAC == nil ||
!strings.EqualFold(publicInterface.MAC.Address, seedMAC) ||
!domainNetworkSourceMatches(
publicInterface, expected.NetworkSourceType,
expected.NetworkSourceName,
) {
return false
}
seedPort, err := seedVPCPort(seed)
if err != nil {
return false
}
if !expected.VPCPresent {
return metadata.VPCPort == nil && seedPort == nil
}
if metadata.VPCPort == nil || seedPort == nil {
return false
}
want := expectedVPCPortMetadata(*expected)
return metadata.VPCPort.public() == want && *seedPort == (seedVPCNetwork{
MAC: want.MAC, PrivateIP: want.PrivateIP,
CIDR: want.CIDR, Gateway: want.Gateway,
})
}
func (l *Libvirt) reconcileVPCPort(ctx context.Context, record operation.Record,
domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return expectedEvidenceError("VPC port", err)
}
seed, err := l.managedSeedContents(ctx, record.ResourceID)
if err != nil {
return expectedEvidenceError("VPC port", err)
}
seedMarker, markerErr := seedMetadataMarker(seed, vpcPortOperationMarker)
if markerErr != nil {
return operation.Resolution{}, nil
}
domainMatches := metadata.VPCTransitionID == expected.TransitionID
seedMatches := seedMarker == expected.TransitionID
if !domainMatches || !seedMatches {
return classifyTransitionMarkers("VPC port", domainMatches, seedMatches), nil
}
definition, parsedMetadata, err := parseManagedDomain(document)
if err != nil || !managedNetworkLayoutMatches(definition, parsedMetadata) {
return operation.Resolution{}, nil
}
seedPort, err := seedVPCPort(seed)
if err != nil {
return operation.Resolution{}, nil
}
want := expectedVPCPortMetadata(*expected)
if expected.VPCPresent {
if metadata.VPCPort == nil || metadata.VPCPort.public() != want ||
seedPort == nil || *seedPort != (seedVPCNetwork{
MAC: want.MAC, PrivateIP: want.PrivateIP,
CIDR: want.CIDR, Gateway: want.Gateway,
}) {
return operation.Resolution{}, nil
}
} else if metadata.VPCPort != nil || seedPort != nil {
return operation.Resolution{}, nil
}
state, _, err := domain.GetState()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile VPC port domain state", err)
}
result := VPCPortResult{
UUID: record.ResourceID, PortKey: expected.VPCPortKey,
VPCKey: expected.VPCKey, Status: mapDomainState(state),
Attached: expected.VPCPresent, Removed: !expected.VPCPresent,
}
if expected.VPCPresent {
result.VPCPort = &want
}
return resolvedExpectedSuccess(result)
}
func expectedVPCPortMetadata(expected operation.ExpectedState) VPCPortMetadata {
return VPCPortMetadata{
PortKey: expected.VPCPortKey, VPCKey: expected.VPCKey,
PrivateIP: expected.VPCPrivateIP, CIDR: expected.VPCCIDR,
Gateway: expected.VPCGateway, MAC: expected.VPCMAC,
Bridge: expected.VPCBridge, HostRevision: expected.VPCRevision,
HostDigest: expected.VPCDigest,
}
}
func (l *Libvirt) reconcilePeerMigrationState(record operation.Record,
domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
if _, _, err := requireManagedDomain(domain); err != nil {
return expectedEvidenceError("peer migration state", err)
}
autostart, err := domain.GetAutostart()
if err != nil {
return operation.Resolution{}, translateLibvirtError(
"reconcile peer migration autostart", err,
)
}
active, err := domain.IsActive()
if err != nil {
return operation.Resolution{}, translateLibvirtError(
"reconcile peer migration activity", err,
)
}
if autostart != expected.Enabled || active != expected.StartIfSuspended {
return resolvedExpectedFailure("peer_migration_state_not_applied",
"host state does not match the migration source state"), nil
}
state, reason, err := domain.GetState()
if err != nil {
return operation.Resolution{}, translateLibvirtError(
"reconcile peer migration domain state", err,
)
}
name, _ := domain.GetName()
return resolvedExpectedSuccess(PeerMigrationStateResult{
VMState: VMState{
UUID: record.ResourceID, Name: name,
State: mapDomainState(state), Reason: reason,
},
Running: active, AutostartEnabled: autostart,
})
}
func (l *Libvirt) reconcileSettings(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return expectedEvidenceError("settings", err)
}
seed, err := l.managedSeedContents(ctx, record.ResourceID)
if err != nil {
return expectedEvidenceError("settings", err)
}
seedMarker, markerErr := seedMetadataMarker(seed, "virt-cloud-settings-operation")
if markerErr != nil {
// A rewritten or damaged seed is evidence that the first phase may
// have crossed its commit boundary. It is not proof of failure.
return operation.Resolution{}, nil
}
domainMatches := metadata.SettingsTransitionID == expected.TransitionID
seedMatches := seedMarker == expected.TransitionID
if !domainMatches || !seedMatches {
return classifyTransitionMarkers("settings", domainMatches, seedMatches), nil
}
if expected.HostnameChanged && metadata.Hostname != expected.Hostname {
// Both durable markers belong to this operation, so it crossed the
// commit boundary. A mismatched payload can be corruption or a later
// out-of-band edit; it is not evidence that PHP may roll state back.
return operation.Resolution{}, nil
}
result, err := metadataFromXML(document)
if err != nil {
return operation.Resolution{}, err
}
state, _, err := domain.GetState()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile settings domain state", err)
}
result.Status = mapDomainState(state)
result.SettingsChanged = true
return resolvedExpectedSuccess(result)
}
func (l *Libvirt) reconcileNetworkChange(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return operation.Resolution{}, translateLibvirtError("read network definition for reconciliation", err)
}
definition, metadata, err := parseManagedDomain(document)
if err != nil {
return expectedEvidenceError("network", err)
}
seed, err := l.managedSeedContents(ctx, record.ResourceID)
if err != nil {
return expectedEvidenceError("network", err)
}
seedMarker, markerErr := seedMetadataMarker(seed, "virt-cloud-network-operation")
if markerErr != nil {
return operation.Resolution{}, nil
}
domainMatches := metadata.NetworkTransitionID == expected.TransitionID
seedMatches := seedMarker == expected.TransitionID
if !domainMatches || !seedMatches {
return classifyTransitionMarkers("network", domainMatches, seedMatches), nil
}
mac, seedPublic, networkErr := seedNetwork(seed)
if networkErr != nil || !publicNetworkEqual(expected.PublicNetwork, metadata.PublicNetwork.publicMust()) ||
!seedPublic.matches(expected.PublicNetwork, mac) ||
definition.Devices == nil || !managedNetworkLayoutMatches(definition, metadata) {
// Matching transition markers prove that both phases wrote state for
// this operation. A payload mismatch is therefore ambiguous and must
// not authorize PHP to release or restore an address assignment.
return operation.Resolution{}, nil
}
publicIndex, _ := managedPublicInterfaceIndex(
definition, networkFilterName(record.ResourceID), expected.PublicNetwork,
)
publicInterface := definition.Devices.Interfaces[publicIndex]
if publicInterface.MAC == nil || !strings.EqualFold(publicInterface.MAC.Address, mac) ||
!domainNetworkSourceMatches(publicInterface, expected.NetworkSourceType, expected.NetworkSourceName) {
return operation.Resolution{}, nil
}
result, err := metadataFromXML(document)
if err != nil {
return operation.Resolution{}, err
}
state, _, err := domain.GetState()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile network domain state", err)
}
result.Status = mapDomainState(state)
result.NetworkChanged = true
return resolvedExpectedSuccess(result)
}
func (l *Libvirt) reconcileNetworkState(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
inactive, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile persistent network state", err)
}
definition, metadata, err := parseManagedDomain(inactive)
if err != nil {
return expectedEvidenceError("network", err)
}
if !managedNetworkLayoutMatches(definition, metadata) {
return operation.Resolution{}, nil
}
if !domainNetworkLinksMatch(inactive, expected.Enabled) {
return resolvedExpectedFailure("network_state_not_applied", "persistent interface link state does not match the requested state"), nil
}
active, err := domain.IsActive()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile live network state", err)
}
if active {
live, err := domain.GetXMLDesc(0)
if err != nil {
return operation.Resolution{}, translateLibvirtError("read live network state", err)
}
liveDefinition, liveMetadata, parseErr := parseManagedDomain(live)
if parseErr != nil || !managedNetworkLayoutMatches(liveDefinition, liveMetadata) {
return operation.Resolution{}, nil
}
if !domainNetworkLinksMatch(live, expected.Enabled) {
return resolvedExpectedFailure("network_state_not_applied", "live interface link state does not match the requested state"), nil
}
}
return resolvedExpectedSuccess(map[string]bool{"network_enabled": expected.Enabled})
}
func (l *Libvirt) reconcileServiceState(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
if _, _, err := requireManagedDomain(domain); err != nil {
return expectedEvidenceError("service", err)
}
autostart, err := domain.GetAutostart()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile service autostart", err)
}
active, err := domain.IsActive()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile service activity", err)
}
if resolution, handled := inspectActiveServiceSuspension(record, active); handled {
return resolution, nil
}
if autostart != expected.Enabled || (expected.Enabled && expected.StartIfSuspended && !active) {
return resolvedExpectedFailure("service_state_not_applied", "host service state does not match the requested state"), nil
}
state, reason, err := domain.GetState()
if err != nil {
return operation.Resolution{}, translateLibvirtError("reconcile service domain state", err)
}
name, _ := domain.GetName()
return resolvedExpectedSuccess(VMState{UUID: record.ResourceID, Name: name, State: mapDomainState(state), Reason: reason})
}
func inspectActiveServiceSuspension(record operation.Record, active bool) (operation.Resolution, bool) {
if record.ExpectedState == nil || record.ExpectedState.Enabled || !active {
return operation.Resolution{}, false
}
if operation.RequiresExplicitForceDelete(record) {
// Only the original SetServiceState call can establish that ShutdownFlags
// was issued and its graceful wait expired. Read-only inspection preserves
// that terminal result without authorizing a follow-up mutation.
return resolvedExpectedFailure("force_required", "guest remains active after the graceful service suspension request"), true
}
if record.CauseCode == "graceful_shutdown_ambiguous" ||
recoveredGracefulAttemptRequiresRetry(record) {
// A fresh read proves only that the guest is active. It resolves the
// old ambiguous or restart-interrupted graceful attempt as failed without
// creating a force boundary or dispatching another mutation.
return resolvedExpectedFailure("graceful_shutdown_unconfirmed",
"graceful service suspension result was not confirmed; the resource remains blocked for manual remediation"), true
}
return operation.Resolution{}, true
}
func (l *Libvirt) reconcileFirewall(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState
_, metadata, err := requireManagedDomain(domain)
if err != nil {
return expectedEvidenceError("firewall", err)
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(record.ResourceID))
if err != nil {
return operation.Resolution{}, err
}
if filter == nil {
return operation.Resolution{}, nil
}
filterDocument, err := filter.GetXMLDesc(0)
filter.Free()
if err != nil {
return operation.Resolution{}, translateLibvirtError("read firewall filter for reconciliation", err)
}
domainMarker := metadata.FirewallTransitionID == expected.TransitionID
domainMatches := domainMarker && metadata.Firewall != nil &&
metadata.Firewall.Revision == expected.FirewallRevision &&
metadata.Firewall.Digest == expected.FirewallDigest
if domainMatches {
policy, validateErr := validateFirewallPolicy(firewallFromMetadata(metadata.Firewall))
if validateErr != nil || policy.PolicyDigest != expected.FirewallDigest ||
!firewallDocumentMatchesPolicy(filterDocument, policy) {
// Domain metadata committed this operation, but the filter cannot be
// proven to match it. Never report failure and roll PHP state back.
return operation.Resolution{}, nil
}
return resolvedExpectedSuccess(policy)
}
if domainMarker || !firewallDocumentMatchesStoredState(filterDocument, metadata.Firewall) {
// Either this operation wrote only the filter, or the domain marker is
// present with malformed/stale policy metadata. Both are half-commit
// evidence, so keep the exact operation uncertain.
return operation.Resolution{}, nil
}
return resolvedExpectedFailure("firewall_not_committed", "host evidence confirms the operation did not commit its transition markers"), nil
}
func (l *Libvirt) managedSeedContents(ctx context.Context, uuid string) ([]byte, error) {
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
if err != nil {
return nil, err
}
if seed == nil {
return nil, NewError(ErrNotFound, "seed_not_found", "managed NoCloud seed volume was not found", nil)
}
defer seed.Free()
return l.downloadSmallVolume(ctx, seed, maxSeedImageBytes)
}
func seedMetadataMarker(seed []byte, key string) (string, error) {
files, err := readSeedISO(seed)
if err != nil {
return "", err
}
var metadata map[string]any
if err := json.Unmarshal(files.MetaData, &metadata); err != nil {
return "", err
}
value, _ := metadata[key].(string)
if value != "" && !operationIdentifierPattern.MatchString(value) {
return "", errors.New("invalid NoCloud transition marker")
}
return value, nil
}
func domainNetworkLinksMatch(document string, enabled bool) bool {
var definition libvirtxml.Domain
if definition.Unmarshal(document) != nil || definition.Devices == nil || len(definition.Devices.Interfaces) == 0 {
return false
}
for _, iface := range definition.Devices.Interfaces {
state := "up"
if iface.Link != nil && iface.Link.State != "" {
state = iface.Link.State
}
if (state == "up") != enabled {
return false
}
}
return true
}
func domainNetworkSourceMatches(iface libvirtxml.DomainInterface, sourceType, sourceName string) bool {
if iface.Source == nil {
return false
}
switch sourceType {
case "bridge":
return iface.Source.Bridge != nil && iface.Source.Bridge.Bridge == sourceName
case "network":
return iface.Source.Network != nil && iface.Source.Network.Network == sourceName
default:
return false
}
}
func firewallDocumentMatchesPolicy(document string, policy FirewallPolicy) bool {
var current libvirtxml.NWFilter
if current.Unmarshal(document) != nil {
return false
}
normalized, err := current.Marshal()
if err != nil {
return false
}
expected, err := buildUserFirewallXML(document, policy)
return err == nil && normalized == expected
}
// firewallDocumentMatchesStoredState proves that the filter still represents
// the policy durably recorded before an operation. A nil policy means the
// untouched platform filter, which must contain no user/default policy rules.
func firewallDocumentMatchesStoredState(document string, stored *domainFirewallMetadataXML) bool {
if stored != nil {
policy, err := validateFirewallPolicy(firewallFromMetadata(stored))
return err == nil && firewallDocumentMatchesPolicy(document, policy)
}
var current libvirtxml.NWFilter
if current.Unmarshal(document) != nil {
return false
}
for _, entry := range current.Entries {
comment := nwfilterRuleComment(entry.Rule)
if strings.HasPrefix(comment, "virt-user:") || strings.HasPrefix(comment, "virt-default:") {
return false
}
}
return true
}
func classifyTransitionMarkers(kind string, primary, secondary bool) operation.Resolution {
if !primary && !secondary {
return resolvedExpectedFailure(kind+"_not_committed", "host evidence confirms the operation did not commit its transition markers")
}
// One matching marker proves that the mutation crossed its first commit
// boundary. It does not prove which side contains the authoritative state:
// for example, a network seed may already contain the new public address
// while the persistent domain still has the old bridge. Finalizing this as
// a failure would let PHP release or roll back resources that the guest may
// already be using. Keep the exact operation uncertain for administrator
// reconciliation; never replay the original two-phase mutation.
return operation.Resolution{}
}
func expectedStateResourceAbsent(record operation.Record) operation.Resolution {
if record.Action == "service-suspend" && record.ExpectedState != nil &&
record.ExpectedState.Kind == operation.ExpectedServiceState && !record.ExpectedState.Enabled {
// An absent persistent Domain is definitive evidence that the requested
// disabled service cannot be active or autostart. Treat the graceful
// target as reached so expiry cleanup can continue under its separate
// delete operation; this does not perform or authorize force deletion.
result, _ := json.Marshal(VMState{
UUID: record.ResourceID, State: VMStateShutoff,
})
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
}
return resolvedExpectedFailure("expected_state_resource_absent", "host evidence confirms the target VM is absent and the expected state cannot be satisfied")
}
func expectedEvidenceError(_ string, err error) (operation.Resolution, error) {
if errors.Is(err, ErrUnavailable) {
return operation.Resolution{}, err
}
// Missing, malformed, or ownership-inconsistent evidence cannot prove that
// an expected-state mutation did not commit. In particular, a network
// change may already have written the requested address into either the
// domain definition or NoCloud seed. Finalizing it as failed would let the
// panel release that address while the guest may still be using it. Keep the
// exact operation uncertain until an administrator restores readable
// evidence; never infer a safe rollback from damaged host state.
return operation.Resolution{}, nil
}
func resolvedExpectedFailure(code, message string) operation.Resolution {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: code, ErrorMessage: message}
}
func resolvedExpectedSuccess(value any) (operation.Resolution, error) {
encoded, err := json.Marshal(value)
if err != nil {
return operation.Resolution{}, err
}
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, nil
}
func (l *Libvirt) reconcileRescue(ctx context.Context, record operation.Record, domain *libvirt.Domain) operation.Resolution {
state, err := l.rescueState(ctx, domain)
if err != nil {
return operation.Resolution{}
}
switch record.Action {
case "enter-rescue":
if state.Active || state.Staged {
encoded, _ := json.Marshal(state)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
}
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_not_committed", ErrorMessage: "host evidence confirms rescue boot was not staged"}
case "force-start-rescue":
if state.Active {
encoded, _ := json.Marshal(state)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
}
if state.Staged {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_force_start_incomplete", ErrorMessage: "rescue boot remains staged but is not running"}
}
case "exit-rescue:connected", "exit-rescue:suspended":
if state.Active || state.Staged {
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_exit_incomplete", ErrorMessage: "host evidence confirms rescue boot remains staged or active"}
}
expectedSuspended := record.Action == "exit-rescue:suspended"
if state.NetworkSuspended != expectedSuspended {
return operation.Resolution{}
}
active, activeErr := domain.IsActive()
if activeErr != nil || !active {
return operation.Resolution{}
}
live, liveErr := domain.GetXMLDesc(0)
if liveErr != nil || !domainNetworkLinksMatch(live, !expectedSuspended) {
return operation.Resolution{}
}
state.NetworkSuspended = expectedSuspended
encoded, _ := json.Marshal(state)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
}
return operation.Resolution{}
}
// diskTransitionCommitted accepts only the current operation's retained
// transition marker together with its deterministic active root. Marker-free
// metadata is not commit evidence in the clean-break Agent protocol.
func diskTransitionCommitted(metadata *domainMetadataXML, replacementName, kind,
operationID string) bool {
return metadata != nil && metadata.RootVolumeName == replacementName &&
diskTransitionMarkerMatches(metadata, kind, operationID)
}
func diskTransitionMarkerMatches(metadata *domainMetadataXML, kind,
operationID string) bool {
return metadata != nil && metadata.DiskTransitionKind == kind &&
metadata.DiskTransitionID == operationID
}
func metadataUUID(document string) string {
var definition libvirtxml.Domain
if definition.Unmarshal(document) != nil {
return ""
}
return definition.UUID
}