You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
723 lines
28 KiB
Go
723 lines
28 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
func (l *Libvirt) reconcileCreatedVM(ctx context.Context, record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, error) {
|
|
document, definition, metadata, pool, root, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
defer pool.Free()
|
|
defer root.Free()
|
|
autostart, err := domain.GetAutostart()
|
|
if err != nil || !autostart {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil || !active {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil || (state != libvirt.DOMAIN_RUNNING && state != libvirt.DOMAIN_BLOCKED) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(definition.UUID))
|
|
if err != nil || seed == nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
seedPath, err := seed.GetPath()
|
|
seed.Free()
|
|
if err != nil || definition.Devices == nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
seedAttached := false
|
|
filterName := networkFilterName(definition.UUID)
|
|
filterAttached := managedNetworkLayoutMatches(definition, metadata)
|
|
for _, disk := range definition.Devices.Disks {
|
|
if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sda" && disk.Source != nil && disk.Source.File != nil && disk.Source.File.File == seedPath && disk.ReadOnly != nil {
|
|
seedAttached = true
|
|
}
|
|
}
|
|
if !seedAttached || !filterAttached {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
var policy PlatformNetworkPolicySnapshot
|
|
if json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &policy) != nil || ValidatePlatformNetworkPolicy(policy) != nil || l.verifyManagedFilter(filterName, policy) != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
if record.ExpectedState != nil && record.ExpectedState.Kind == operation.ExpectedCreateLikeNetwork {
|
|
seedContents, seedErr := l.managedSeedContents(ctx, record.ResourceID)
|
|
if seedErr != nil {
|
|
return expectedEvidenceError("create-like network", seedErr)
|
|
}
|
|
if !createLikeNetworkEvidenceMatches(
|
|
record, definition, metadata, seedContents,
|
|
) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
}
|
|
result, err := metadataFromXML(document)
|
|
if err != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
result.Status = VMStateRunning
|
|
encoded, err := json.Marshal(result)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, nil
|
|
}
|
|
|
|
// inspectExpectedStateOperation is deliberately limited to observation. It is
|
|
// the only expected-state helper reachable from InspectOperation: every call
|
|
// below reads durable domain, seed, network-filter, or runtime state and none
|
|
// defines, starts, stops, deletes, or otherwise changes a host resource.
|
|
func (l *Libvirt) inspectExpectedStateOperation(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, bool, error) {
|
|
if record.ExpectedState != nil {
|
|
switch record.ExpectedState.Kind {
|
|
case operation.ExpectedUpdateSettings:
|
|
resolution, err := l.reconcileSettings(ctx, record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedResizeResources:
|
|
resolution, err := l.reconcileResizeResources(record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedChangeNetwork:
|
|
resolution, err := l.reconcileNetworkChange(ctx, record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedNetworkState:
|
|
resolution, err := l.reconcileNetworkState(record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedServiceState:
|
|
resolution, err := l.reconcileServiceState(record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedPeerMigrationState:
|
|
resolution, err := l.reconcilePeerMigrationState(record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedFirewall:
|
|
resolution, err := l.reconcileFirewall(record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedVPCPort:
|
|
resolution, err := l.reconcileVPCPort(ctx, record, domain)
|
|
return resolution, true, err
|
|
case operation.ExpectedCreateLikeNetwork:
|
|
switch {
|
|
case strings.HasPrefix(record.Action, "restore-backup:"):
|
|
resolution, err := l.reconcileCreatedVM(ctx, record, domain)
|
|
return resolution, true, err
|
|
case strings.HasPrefix(record.Action, "receive-peer-migration:"):
|
|
resolution, err := l.reconcileReceivedPeerMigration(ctx, record, domain)
|
|
return resolution, true, err
|
|
default:
|
|
// Journal validation does not permit this state/action pairing.
|
|
// Keep a damaged record fenced instead of falling through to a
|
|
// weaker action-only reconciler.
|
|
return operation.Resolution{}, true, nil
|
|
}
|
|
}
|
|
}
|
|
return operation.Resolution{}, false, nil
|
|
}
|
|
|
|
// createLikeNetworkEvidenceMatches binds an ambiguous restore or peer receive
|
|
// to the complete, immutable network identity persisted before dispatch. It
|
|
// deliberately performs no mutation. A mismatch remains unresolved so the
|
|
// panel cannot release an address or VPC identity which the guest may use.
|
|
func createLikeNetworkEvidenceMatches(record operation.Record,
|
|
definition *libvirtxml.Domain, metadata *domainMetadataXML, seed []byte) bool {
|
|
expected := record.ExpectedState
|
|
if expected == nil || expected.Kind != operation.ExpectedCreateLikeNetwork ||
|
|
expected.TransitionID != record.OperationID || definition == nil ||
|
|
metadata == nil || !strings.EqualFold(definition.UUID, record.ResourceID) ||
|
|
definition.Devices == nil {
|
|
return false
|
|
}
|
|
public, err := metadata.PublicNetwork.public()
|
|
if err != nil || !publicNetworkEqual(public, expected.PublicNetwork) ||
|
|
!managedNetworkLayoutMatches(definition, metadata) {
|
|
return false
|
|
}
|
|
seedMAC, seedPublic, err := seedNetwork(seed)
|
|
if err != nil || !strings.EqualFold(seedMAC, vmMACAddress(record.ResourceID)) ||
|
|
!seedPublic.matches(expected.PublicNetwork, seedMAC) {
|
|
return false
|
|
}
|
|
publicIndex, found := managedPublicInterfaceIndex(
|
|
definition, networkFilterName(record.ResourceID), expected.PublicNetwork,
|
|
)
|
|
if !found {
|
|
return false
|
|
}
|
|
publicInterface := definition.Devices.Interfaces[publicIndex]
|
|
if publicInterface.MAC == nil ||
|
|
!strings.EqualFold(publicInterface.MAC.Address, seedMAC) ||
|
|
!domainNetworkSourceMatches(
|
|
publicInterface, expected.NetworkSourceType,
|
|
expected.NetworkSourceName,
|
|
) {
|
|
return false
|
|
}
|
|
seedPort, err := seedVPCPort(seed)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
if !expected.VPCPresent {
|
|
return metadata.VPCPort == nil && seedPort == nil
|
|
}
|
|
if metadata.VPCPort == nil || seedPort == nil {
|
|
return false
|
|
}
|
|
want := expectedVPCPortMetadata(*expected)
|
|
return metadata.VPCPort.public() == want && *seedPort == (seedVPCNetwork{
|
|
MAC: want.MAC, PrivateIP: want.PrivateIP,
|
|
CIDR: want.CIDR, Gateway: want.Gateway,
|
|
})
|
|
}
|
|
|
|
func (l *Libvirt) reconcileVPCPort(ctx context.Context, record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return expectedEvidenceError("VPC port", err)
|
|
}
|
|
seed, err := l.managedSeedContents(ctx, record.ResourceID)
|
|
if err != nil {
|
|
return expectedEvidenceError("VPC port", err)
|
|
}
|
|
seedMarker, markerErr := seedMetadataMarker(seed, vpcPortOperationMarker)
|
|
if markerErr != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
domainMatches := metadata.VPCTransitionID == expected.TransitionID
|
|
seedMatches := seedMarker == expected.TransitionID
|
|
if !domainMatches || !seedMatches {
|
|
return classifyTransitionMarkers("VPC port", domainMatches, seedMatches), nil
|
|
}
|
|
definition, parsedMetadata, err := parseManagedDomain(document)
|
|
if err != nil || !managedNetworkLayoutMatches(definition, parsedMetadata) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
seedPort, err := seedVPCPort(seed)
|
|
if err != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
want := expectedVPCPortMetadata(*expected)
|
|
if expected.VPCPresent {
|
|
if metadata.VPCPort == nil || metadata.VPCPort.public() != want ||
|
|
seedPort == nil || *seedPort != (seedVPCNetwork{
|
|
MAC: want.MAC, PrivateIP: want.PrivateIP,
|
|
CIDR: want.CIDR, Gateway: want.Gateway,
|
|
}) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
} else if metadata.VPCPort != nil || seedPort != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile VPC port domain state", err)
|
|
}
|
|
result := VPCPortResult{
|
|
UUID: record.ResourceID, PortKey: expected.VPCPortKey,
|
|
VPCKey: expected.VPCKey, Status: mapDomainState(state),
|
|
Attached: expected.VPCPresent, Removed: !expected.VPCPresent,
|
|
}
|
|
if expected.VPCPresent {
|
|
result.VPCPort = &want
|
|
}
|
|
return resolvedExpectedSuccess(result)
|
|
}
|
|
|
|
func expectedVPCPortMetadata(expected operation.ExpectedState) VPCPortMetadata {
|
|
return VPCPortMetadata{
|
|
PortKey: expected.VPCPortKey, VPCKey: expected.VPCKey,
|
|
PrivateIP: expected.VPCPrivateIP, CIDR: expected.VPCCIDR,
|
|
Gateway: expected.VPCGateway, MAC: expected.VPCMAC,
|
|
Bridge: expected.VPCBridge, HostRevision: expected.VPCRevision,
|
|
HostDigest: expected.VPCDigest,
|
|
}
|
|
}
|
|
|
|
func (l *Libvirt) reconcilePeerMigrationState(record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
if _, _, err := requireManagedDomain(domain); err != nil {
|
|
return expectedEvidenceError("peer migration state", err)
|
|
}
|
|
autostart, err := domain.GetAutostart()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError(
|
|
"reconcile peer migration autostart", err,
|
|
)
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError(
|
|
"reconcile peer migration activity", err,
|
|
)
|
|
}
|
|
if autostart != expected.Enabled || active != expected.StartIfSuspended {
|
|
return resolvedExpectedFailure("peer_migration_state_not_applied",
|
|
"host state does not match the migration source state"), nil
|
|
}
|
|
state, reason, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError(
|
|
"reconcile peer migration domain state", err,
|
|
)
|
|
}
|
|
name, _ := domain.GetName()
|
|
return resolvedExpectedSuccess(PeerMigrationStateResult{
|
|
VMState: VMState{
|
|
UUID: record.ResourceID, Name: name,
|
|
State: mapDomainState(state), Reason: reason,
|
|
},
|
|
Running: active, AutostartEnabled: autostart,
|
|
})
|
|
}
|
|
|
|
func (l *Libvirt) reconcileSettings(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return expectedEvidenceError("settings", err)
|
|
}
|
|
seed, err := l.managedSeedContents(ctx, record.ResourceID)
|
|
if err != nil {
|
|
return expectedEvidenceError("settings", err)
|
|
}
|
|
seedMarker, markerErr := seedMetadataMarker(seed, "virt-cloud-settings-operation")
|
|
if markerErr != nil {
|
|
// A rewritten or damaged seed is evidence that the first phase may
|
|
// have crossed its commit boundary. It is not proof of failure.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
domainMatches := metadata.SettingsTransitionID == expected.TransitionID
|
|
seedMatches := seedMarker == expected.TransitionID
|
|
if !domainMatches || !seedMatches {
|
|
return classifyTransitionMarkers("settings", domainMatches, seedMatches), nil
|
|
}
|
|
if expected.HostnameChanged && metadata.Hostname != expected.Hostname {
|
|
// Both durable markers belong to this operation, so it crossed the
|
|
// commit boundary. A mismatched payload can be corruption or a later
|
|
// out-of-band edit; it is not evidence that PHP may roll state back.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
result, err := metadataFromXML(document)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile settings domain state", err)
|
|
}
|
|
result.Status = mapDomainState(state)
|
|
result.SettingsChanged = true
|
|
return resolvedExpectedSuccess(result)
|
|
}
|
|
|
|
func (l *Libvirt) reconcileNetworkChange(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("read network definition for reconciliation", err)
|
|
}
|
|
definition, metadata, err := parseManagedDomain(document)
|
|
if err != nil {
|
|
return expectedEvidenceError("network", err)
|
|
}
|
|
seed, err := l.managedSeedContents(ctx, record.ResourceID)
|
|
if err != nil {
|
|
return expectedEvidenceError("network", err)
|
|
}
|
|
seedMarker, markerErr := seedMetadataMarker(seed, "virt-cloud-network-operation")
|
|
if markerErr != nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
domainMatches := metadata.NetworkTransitionID == expected.TransitionID
|
|
seedMatches := seedMarker == expected.TransitionID
|
|
if !domainMatches || !seedMatches {
|
|
return classifyTransitionMarkers("network", domainMatches, seedMatches), nil
|
|
}
|
|
mac, seedPublic, networkErr := seedNetwork(seed)
|
|
if networkErr != nil || !publicNetworkEqual(expected.PublicNetwork, metadata.PublicNetwork.publicMust()) ||
|
|
!seedPublic.matches(expected.PublicNetwork, mac) ||
|
|
definition.Devices == nil || !managedNetworkLayoutMatches(definition, metadata) {
|
|
// Matching transition markers prove that both phases wrote state for
|
|
// this operation. A payload mismatch is therefore ambiguous and must
|
|
// not authorize PHP to release or restore an address assignment.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
publicIndex, _ := managedPublicInterfaceIndex(
|
|
definition, networkFilterName(record.ResourceID), expected.PublicNetwork,
|
|
)
|
|
publicInterface := definition.Devices.Interfaces[publicIndex]
|
|
if publicInterface.MAC == nil || !strings.EqualFold(publicInterface.MAC.Address, mac) ||
|
|
!domainNetworkSourceMatches(publicInterface, expected.NetworkSourceType, expected.NetworkSourceName) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
result, err := metadataFromXML(document)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile network domain state", err)
|
|
}
|
|
result.Status = mapDomainState(state)
|
|
result.NetworkChanged = true
|
|
return resolvedExpectedSuccess(result)
|
|
}
|
|
|
|
func (l *Libvirt) reconcileNetworkState(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
inactive, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile persistent network state", err)
|
|
}
|
|
definition, metadata, err := parseManagedDomain(inactive)
|
|
if err != nil {
|
|
return expectedEvidenceError("network", err)
|
|
}
|
|
if !managedNetworkLayoutMatches(definition, metadata) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
if !domainNetworkLinksMatch(inactive, expected.Enabled) {
|
|
return resolvedExpectedFailure("network_state_not_applied", "persistent interface link state does not match the requested state"), nil
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile live network state", err)
|
|
}
|
|
if active {
|
|
live, err := domain.GetXMLDesc(0)
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("read live network state", err)
|
|
}
|
|
liveDefinition, liveMetadata, parseErr := parseManagedDomain(live)
|
|
if parseErr != nil || !managedNetworkLayoutMatches(liveDefinition, liveMetadata) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
if !domainNetworkLinksMatch(live, expected.Enabled) {
|
|
return resolvedExpectedFailure("network_state_not_applied", "live interface link state does not match the requested state"), nil
|
|
}
|
|
}
|
|
return resolvedExpectedSuccess(map[string]bool{"network_enabled": expected.Enabled})
|
|
}
|
|
|
|
func (l *Libvirt) reconcileServiceState(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
if _, _, err := requireManagedDomain(domain); err != nil {
|
|
return expectedEvidenceError("service", err)
|
|
}
|
|
autostart, err := domain.GetAutostart()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile service autostart", err)
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile service activity", err)
|
|
}
|
|
if resolution, handled := inspectActiveServiceSuspension(record, active); handled {
|
|
return resolution, nil
|
|
}
|
|
if autostart != expected.Enabled || (expected.Enabled && expected.StartIfSuspended && !active) {
|
|
return resolvedExpectedFailure("service_state_not_applied", "host service state does not match the requested state"), nil
|
|
}
|
|
state, reason, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile service domain state", err)
|
|
}
|
|
name, _ := domain.GetName()
|
|
return resolvedExpectedSuccess(VMState{UUID: record.ResourceID, Name: name, State: mapDomainState(state), Reason: reason})
|
|
}
|
|
|
|
func inspectActiveServiceSuspension(record operation.Record, active bool) (operation.Resolution, bool) {
|
|
if record.ExpectedState == nil || record.ExpectedState.Enabled || !active {
|
|
return operation.Resolution{}, false
|
|
}
|
|
if operation.RequiresExplicitForceDelete(record) {
|
|
// Only the original SetServiceState call can establish that ShutdownFlags
|
|
// was issued and its graceful wait expired. Read-only inspection preserves
|
|
// that terminal result without authorizing a follow-up mutation.
|
|
return resolvedExpectedFailure("force_required", "guest remains active after the graceful service suspension request"), true
|
|
}
|
|
if record.CauseCode == "graceful_shutdown_ambiguous" ||
|
|
recoveredGracefulAttemptRequiresRetry(record) {
|
|
// A fresh read proves only that the guest is active. It resolves the
|
|
// old ambiguous or restart-interrupted graceful attempt as failed without
|
|
// creating a force boundary or dispatching another mutation.
|
|
return resolvedExpectedFailure("graceful_shutdown_unconfirmed",
|
|
"graceful service suspension result was not confirmed; the resource remains blocked for manual remediation"), true
|
|
}
|
|
return operation.Resolution{}, true
|
|
}
|
|
|
|
func (l *Libvirt) reconcileFirewall(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
expected := record.ExpectedState
|
|
_, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return expectedEvidenceError("firewall", err)
|
|
}
|
|
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(record.ResourceID))
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
if filter == nil {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
filterDocument, err := filter.GetXMLDesc(0)
|
|
filter.Free()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("read firewall filter for reconciliation", err)
|
|
}
|
|
|
|
domainMarker := metadata.FirewallTransitionID == expected.TransitionID
|
|
domainMatches := domainMarker && metadata.Firewall != nil &&
|
|
metadata.Firewall.Revision == expected.FirewallRevision &&
|
|
metadata.Firewall.Digest == expected.FirewallDigest
|
|
if domainMatches {
|
|
policy, validateErr := validateFirewallPolicy(firewallFromMetadata(metadata.Firewall))
|
|
if validateErr != nil || policy.PolicyDigest != expected.FirewallDigest ||
|
|
!firewallDocumentMatchesPolicy(filterDocument, policy) {
|
|
// Domain metadata committed this operation, but the filter cannot be
|
|
// proven to match it. Never report failure and roll PHP state back.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
return resolvedExpectedSuccess(policy)
|
|
}
|
|
if domainMarker || !firewallDocumentMatchesStoredState(filterDocument, metadata.Firewall) {
|
|
// Either this operation wrote only the filter, or the domain marker is
|
|
// present with malformed/stale policy metadata. Both are half-commit
|
|
// evidence, so keep the exact operation uncertain.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
return resolvedExpectedFailure("firewall_not_committed", "host evidence confirms the operation did not commit its transition markers"), nil
|
|
}
|
|
|
|
func (l *Libvirt) managedSeedContents(ctx context.Context, uuid string) ([]byte, error) {
|
|
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if seed == nil {
|
|
return nil, NewError(ErrNotFound, "seed_not_found", "managed NoCloud seed volume was not found", nil)
|
|
}
|
|
defer seed.Free()
|
|
return l.downloadSmallVolume(ctx, seed, maxSeedImageBytes)
|
|
}
|
|
|
|
func seedMetadataMarker(seed []byte, key string) (string, error) {
|
|
files, err := readSeedISO(seed)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var metadata map[string]any
|
|
if err := json.Unmarshal(files.MetaData, &metadata); err != nil {
|
|
return "", err
|
|
}
|
|
value, _ := metadata[key].(string)
|
|
if value != "" && !operationIdentifierPattern.MatchString(value) {
|
|
return "", errors.New("invalid NoCloud transition marker")
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func domainNetworkLinksMatch(document string, enabled bool) bool {
|
|
var definition libvirtxml.Domain
|
|
if definition.Unmarshal(document) != nil || definition.Devices == nil || len(definition.Devices.Interfaces) == 0 {
|
|
return false
|
|
}
|
|
for _, iface := range definition.Devices.Interfaces {
|
|
state := "up"
|
|
if iface.Link != nil && iface.Link.State != "" {
|
|
state = iface.Link.State
|
|
}
|
|
if (state == "up") != enabled {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func domainNetworkSourceMatches(iface libvirtxml.DomainInterface, sourceType, sourceName string) bool {
|
|
if iface.Source == nil {
|
|
return false
|
|
}
|
|
switch sourceType {
|
|
case "bridge":
|
|
return iface.Source.Bridge != nil && iface.Source.Bridge.Bridge == sourceName
|
|
case "network":
|
|
return iface.Source.Network != nil && iface.Source.Network.Network == sourceName
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func firewallDocumentMatchesPolicy(document string, policy FirewallPolicy) bool {
|
|
var current libvirtxml.NWFilter
|
|
if current.Unmarshal(document) != nil {
|
|
return false
|
|
}
|
|
normalized, err := current.Marshal()
|
|
if err != nil {
|
|
return false
|
|
}
|
|
expected, err := buildUserFirewallXML(document, policy)
|
|
return err == nil && normalized == expected
|
|
}
|
|
|
|
// firewallDocumentMatchesStoredState proves that the filter still represents
|
|
// the policy durably recorded before an operation. A nil policy means the
|
|
// untouched platform filter, which must contain no user/default policy rules.
|
|
func firewallDocumentMatchesStoredState(document string, stored *domainFirewallMetadataXML) bool {
|
|
if stored != nil {
|
|
policy, err := validateFirewallPolicy(firewallFromMetadata(stored))
|
|
return err == nil && firewallDocumentMatchesPolicy(document, policy)
|
|
}
|
|
var current libvirtxml.NWFilter
|
|
if current.Unmarshal(document) != nil {
|
|
return false
|
|
}
|
|
for _, entry := range current.Entries {
|
|
comment := nwfilterRuleComment(entry.Rule)
|
|
if strings.HasPrefix(comment, "virt-user:") || strings.HasPrefix(comment, "virt-default:") {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func classifyTransitionMarkers(kind string, primary, secondary bool) operation.Resolution {
|
|
if !primary && !secondary {
|
|
return resolvedExpectedFailure(kind+"_not_committed", "host evidence confirms the operation did not commit its transition markers")
|
|
}
|
|
// One matching marker proves that the mutation crossed its first commit
|
|
// boundary. It does not prove which side contains the authoritative state:
|
|
// for example, a network seed may already contain the new public address
|
|
// while the persistent domain still has the old bridge. Finalizing this as
|
|
// a failure would let PHP release or roll back resources that the guest may
|
|
// already be using. Keep the exact operation uncertain for administrator
|
|
// reconciliation; never replay the original two-phase mutation.
|
|
return operation.Resolution{}
|
|
}
|
|
|
|
func expectedStateResourceAbsent(record operation.Record) operation.Resolution {
|
|
if record.Action == "service-suspend" && record.ExpectedState != nil &&
|
|
record.ExpectedState.Kind == operation.ExpectedServiceState && !record.ExpectedState.Enabled {
|
|
// An absent persistent Domain is definitive evidence that the requested
|
|
// disabled service cannot be active or autostart. Treat the graceful
|
|
// target as reached so expiry cleanup can continue under its separate
|
|
// delete operation; this does not perform or authorize force deletion.
|
|
result, _ := json.Marshal(VMState{
|
|
UUID: record.ResourceID, State: VMStateShutoff,
|
|
})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}
|
|
}
|
|
return resolvedExpectedFailure("expected_state_resource_absent", "host evidence confirms the target VM is absent and the expected state cannot be satisfied")
|
|
}
|
|
|
|
func expectedEvidenceError(_ string, err error) (operation.Resolution, error) {
|
|
if errors.Is(err, ErrUnavailable) {
|
|
return operation.Resolution{}, err
|
|
}
|
|
// Missing, malformed, or ownership-inconsistent evidence cannot prove that
|
|
// an expected-state mutation did not commit. In particular, a network
|
|
// change may already have written the requested address into either the
|
|
// domain definition or NoCloud seed. Finalizing it as failed would let the
|
|
// panel release that address while the guest may still be using it. Keep the
|
|
// exact operation uncertain until an administrator restores readable
|
|
// evidence; never infer a safe rollback from damaged host state.
|
|
return operation.Resolution{}, nil
|
|
}
|
|
|
|
func resolvedExpectedFailure(code, message string) operation.Resolution {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: code, ErrorMessage: message}
|
|
}
|
|
|
|
func resolvedExpectedSuccess(value any) (operation.Resolution, error) {
|
|
encoded, err := json.Marshal(value)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, nil
|
|
}
|
|
|
|
func (l *Libvirt) reconcileRescue(ctx context.Context, record operation.Record, domain *libvirt.Domain) operation.Resolution {
|
|
state, err := l.rescueState(ctx, domain)
|
|
if err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
switch record.Action {
|
|
case "enter-rescue":
|
|
if state.Active || state.Staged {
|
|
encoded, _ := json.Marshal(state)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_not_committed", ErrorMessage: "host evidence confirms rescue boot was not staged"}
|
|
case "force-start-rescue":
|
|
if state.Active {
|
|
encoded, _ := json.Marshal(state)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
|
|
}
|
|
if state.Staged {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_force_start_incomplete", ErrorMessage: "rescue boot remains staged but is not running"}
|
|
}
|
|
case "exit-rescue:connected", "exit-rescue:suspended":
|
|
if state.Active || state.Staged {
|
|
return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "rescue_exit_incomplete", ErrorMessage: "host evidence confirms rescue boot remains staged or active"}
|
|
}
|
|
expectedSuspended := record.Action == "exit-rescue:suspended"
|
|
if state.NetworkSuspended != expectedSuspended {
|
|
return operation.Resolution{}
|
|
}
|
|
active, activeErr := domain.IsActive()
|
|
if activeErr != nil || !active {
|
|
return operation.Resolution{}
|
|
}
|
|
live, liveErr := domain.GetXMLDesc(0)
|
|
if liveErr != nil || !domainNetworkLinksMatch(live, !expectedSuspended) {
|
|
return operation.Resolution{}
|
|
}
|
|
state.NetworkSuspended = expectedSuspended
|
|
encoded, _ := json.Marshal(state)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}
|
|
}
|
|
return operation.Resolution{}
|
|
}
|
|
|
|
// diskTransitionCommitted accepts only the current operation's retained
|
|
// transition marker together with its deterministic active root. Marker-free
|
|
// metadata is not commit evidence in the clean-break Agent protocol.
|
|
func diskTransitionCommitted(metadata *domainMetadataXML, replacementName, kind,
|
|
operationID string) bool {
|
|
return metadata != nil && metadata.RootVolumeName == replacementName &&
|
|
diskTransitionMarkerMatches(metadata, kind, operationID)
|
|
}
|
|
|
|
func diskTransitionMarkerMatches(metadata *domainMetadataXML, kind,
|
|
operationID string) bool {
|
|
return metadata != nil && metadata.DiskTransitionKind == kind &&
|
|
metadata.DiskTransitionID == operationID
|
|
}
|
|
|
|
func metadataUUID(document string) string {
|
|
var definition libvirtxml.Domain
|
|
if definition.Unmarshal(document) != nil {
|
|
return ""
|
|
}
|
|
return definition.UUID
|
|
}
|