fix: reconcile exact create-like network state

go
codex 2 weeks ago
parent e51ce26195
commit e6bb6723a3

@ -0,0 +1,380 @@
//go:build libvirt
package hypervisor
import (
"bytes"
"encoding/json"
"testing"
"git.luoli.live/root/virt/agent/cloudinit"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/publicnetwork"
"libvirt.org/go/libvirtxml"
)
type createLikeNetworkEvidenceFixture struct {
record operation.Record
definition *libvirtxml.Domain
metadata *domainMetadataXML
seed []byte
}
func newCreateLikeNetworkEvidenceFixture(t *testing.T,
withVPC bool) createLikeNetworkEvidenceFixture {
t.Helper()
definition := validDomainDefinition()
definition.Network[0].MAC = vmMACAddress(definition.UUID)
var port *VPCPortMetadata
if withVPC {
value := validVPCPortMetadata()
port = &value
definition.VPCPort = port
definition.Network = append(definition.Network, NetworkAttachment{
Bridge: port.Bridge, MAC: port.MAC,
})
}
document, err := BuildDomainXML(definition)
if err != nil {
t.Fatal(err)
}
parsed, metadata, err := parseManagedDomain(document)
if err != nil {
t.Fatal(err)
}
seed := buildCreateLikeNetworkSeed(
t, definition.UUID, definition.PublicNetwork,
definition.Network[0].MAC, port,
)
source, err := ParseNetworkSelector(definition.PublicNetwork.NetworkSelector)
if err != nil {
t.Fatal(err)
}
expected := &operation.ExpectedState{
Kind: operation.ExpectedCreateLikeNetwork,
TransitionID: "operation-create-like-network-evidence",
NetworkSourceType: source.Type, NetworkSourceName: source.Source,
PublicNetwork: definition.PublicNetwork,
VPCPresent: port != nil,
}
if port != nil {
expected.VPCKey = port.VPCKey
expected.VPCRevision = port.HostRevision
expected.VPCDigest = port.HostDigest
expected.VPCPortKey = port.PortKey
expected.VPCCIDR = port.CIDR
expected.VPCGateway = port.Gateway
expected.VPCPrivateIP = port.PrivateIP
expected.VPCMAC = port.MAC
expected.VPCBridge = port.Bridge
}
return createLikeNetworkEvidenceFixture{
record: operation.Record{
OperationID: expected.TransitionID,
ResourceID: definition.UUID,
Action: "receive-peer-migration:0123456789abcdef0123456789abcdef",
ExpectedState: expected,
},
definition: parsed, metadata: metadata, seed: seed,
}
}
func buildCreateLikeNetworkSeed(t *testing.T, uuid string,
public publicnetwork.Intent, mac string, port *VPCPortMetadata) []byte {
t.Helper()
var private *cloudinit.PrivateNetwork
if port != nil {
private = &cloudinit.PrivateNetwork{
MAC: port.MAC, PrivateIP: port.PrivateIP,
CIDR: port.CIDR, Gateway: port.Gateway,
InterfaceKey: seedVPCInterfaceKey,
}
}
image, err := cloudinit.BuildISO(cloudinit.Request{
UUID: uuid, Name: "vm-create-like", PreserveGuestCredentials: true,
Network: cloudinit.Network{
MAC: mac, Public: public, InterfaceKey: seedPublicInterfaceKey,
Private: private,
},
})
if err != nil {
t.Fatal(err)
}
return image
}
func TestCreateLikeNetworkEvidenceRequiresExactDualStackIdentity(t *testing.T) {
for _, withVPC := range []bool{false, true} {
fixture := newCreateLikeNetworkEvidenceFixture(t, withVPC)
if !createLikeNetworkEvidenceMatches(
fixture.record, fixture.definition, fixture.metadata, fixture.seed,
) {
t.Fatalf("exact create-like network evidence rejected (VPC=%v)", withVPC)
}
}
}
func TestCreateLikeNetworkEvidenceDriftRemainsUnresolved(t *testing.T) {
tests := map[string]func(*createLikeNetworkEvidenceFixture){
"transition": func(value *createLikeNetworkEvidenceFixture) {
value.record.ExpectedState.TransitionID = "operation-other"
},
"resource UUID": func(value *createLikeNetworkEvidenceFixture) {
value.record.ResourceID = "123e4567-e89b-42d3-a456-426614174999"
},
"metadata IPv4": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.PublicNetwork.IPv4.AddressCIDR = "8.8.8.11/24"
},
"metadata IPv6": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.PublicNetwork.IPv6.AddressCIDR = "2404:c140:156:3a::1001/64"
},
"metadata DNS": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.PublicNetwork.DNSServers = []string{"9.9.9.9", "2620:fe::fe"}
},
"metadata selector": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.PublicNetwork.NetworkSelector = "bridge:br1"
},
"metadata IPv4 gateway": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.PublicNetwork.IPv4.Gateway = "8.8.8.2"
},
"metadata IPv6 gateway": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.PublicNetwork.IPv6.Gateway = "2404:c140:156::2"
},
"network source": func(value *createLikeNetworkEvidenceFixture) {
value.definition.Devices.Interfaces[0].Source.Bridge.Bridge = "br1"
},
"public MAC": func(value *createLikeNetworkEvidenceFixture) {
value.definition.Devices.Interfaces[0].MAC.Address = "02:00:00:00:00:44"
},
"nwfilter IPv4": func(value *createLikeNetworkEvidenceFixture) {
value.definition.Devices.Interfaces[0].FilterRef.Parameters[0].Value = "8.8.8.11"
},
"nwfilter IPv6": func(value *createLikeNetworkEvidenceFixture) {
value.definition.Devices.Interfaces[0].FilterRef.Parameters[1].Value = "2404:c140:156:3a::1001"
},
"nwfilter link-local": func(value *createLikeNetworkEvidenceFixture) {
value.definition.Devices.Interfaces[0].FilterRef.Parameters[2].Value = "fe80::1"
},
"nwfilter name": func(value *createLikeNetworkEvidenceFixture) {
value.definition.Devices.Interfaces[0].FilterRef.Filter = "virt-forged"
},
"seed IPv4": func(value *createLikeNetworkEvidenceFixture) {
public := value.record.ExpectedState.PublicNetwork
public.IPv4.AddressCIDR = "8.8.8.11/24"
value.seed = buildCreateLikeNetworkSeed(
t, value.record.ResourceID, public,
vmMACAddress(value.record.ResourceID), nil,
)
},
"seed IPv6": func(value *createLikeNetworkEvidenceFixture) {
public := value.record.ExpectedState.PublicNetwork
public.IPv6.AddressCIDR = "2404:c140:156:3a::1001/64"
value.seed = buildCreateLikeNetworkSeed(
t, value.record.ResourceID, public,
vmMACAddress(value.record.ResourceID), nil,
)
},
"seed IPv4 gateway": func(value *createLikeNetworkEvidenceFixture) {
public := value.record.ExpectedState.PublicNetwork
public.IPv4.Gateway = "8.8.8.2"
value.seed = buildCreateLikeNetworkSeed(
t, value.record.ResourceID, public,
vmMACAddress(value.record.ResourceID), nil,
)
},
"seed IPv6 gateway": func(value *createLikeNetworkEvidenceFixture) {
public := value.record.ExpectedState.PublicNetwork
public.IPv6.Gateway = "2404:c140:156::2"
value.seed = buildCreateLikeNetworkSeed(
t, value.record.ResourceID, public,
vmMACAddress(value.record.ResourceID), nil,
)
},
"seed DNS": func(value *createLikeNetworkEvidenceFixture) {
public := value.record.ExpectedState.PublicNetwork
public.DNSServers = []string{"9.9.9.9", "2620:fe::fe"}
value.seed = buildCreateLikeNetworkSeed(
t, value.record.ResourceID, public,
vmMACAddress(value.record.ResourceID), nil,
)
},
"seed link-local": func(value *createLikeNetworkEvidenceFixture) {
value.seed = rewriteCreateLikeSeedLinkLocal(t, value.seed, "fe80::1/64")
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
fixture := newCreateLikeNetworkEvidenceFixture(t, false)
mutate(&fixture)
if createLikeNetworkEvidenceMatches(
fixture.record, fixture.definition, fixture.metadata, fixture.seed,
) {
t.Fatal("drifted create-like network evidence was accepted")
}
})
}
}
func TestCreateLikeNetworkEvidenceRequiresExactOptionalVPCIdentity(t *testing.T) {
tests := map[string]func(*createLikeNetworkEvidenceFixture){
"unexpected VPC": func(value *createLikeNetworkEvidenceFixture) {
value.record.ExpectedState.VPCPresent = false
clearExpectedCreateLikeVPC(value.record.ExpectedState)
},
"missing metadata": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort = nil
},
"port key": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort.PortKey = "3123456789abcdef0123456789abcdef"
},
"VPC key": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort.VPCKey = "4123456789abcdef0123456789abcdef"
},
"host revision": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort.HostRevision++
},
"host digest": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort.HostDigest = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
},
"private IP": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort.PrivateIP = "10.60.0.10"
},
"CIDR": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort.CIDR = "10.61.0.0/24"
},
"gateway": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort.Gateway = "10.60.0.2"
},
"MAC": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort.MAC = "02:00:00:00:00:10"
},
"bridge": func(value *createLikeNetworkEvidenceFixture) {
value.metadata.VPCPort.Bridge = "vcffffffffffff"
},
"seed private IP": func(value *createLikeNetworkEvidenceFixture) {
value.seed = rewriteCreateLikeSeedVPC(t, value.seed, func(entry map[string]any) {
entry["addresses"] = []any{"10.60.0.10/32"}
})
},
"seed VPC MAC": func(value *createLikeNetworkEvidenceFixture) {
value.seed = rewriteCreateLikeSeedVPC(t, value.seed, func(entry map[string]any) {
entry["match"] = map[string]any{"macaddress": "02:00:00:00:00:10"}
})
},
"seed VPC CIDR": func(value *createLikeNetworkEvidenceFixture) {
value.seed = rewriteCreateLikeSeedVPC(t, value.seed, func(entry map[string]any) {
routes := entry["routes"].([]any)
routes[0].(map[string]any)["to"] = "10.61.0.0/24"
})
},
"seed VPC gateway": func(value *createLikeNetworkEvidenceFixture) {
value.seed = rewriteCreateLikeSeedVPC(t, value.seed, func(entry map[string]any) {
routes := entry["routes"].([]any)
routes[0].(map[string]any)["via"] = "10.60.0.2"
})
},
"VPC interface source": func(value *createLikeNetworkEvidenceFixture) {
value.definition.Devices.Interfaces[1].Source.Bridge.Bridge = "vcffffffffffff"
},
"VPC anti-spoof parameter": func(value *createLikeNetworkEvidenceFixture) {
value.definition.Devices.Interfaces[1].FilterRef.Parameters[0].Value = "10.60.0.10"
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
fixture := newCreateLikeNetworkEvidenceFixture(t, true)
mutate(&fixture)
if createLikeNetworkEvidenceMatches(
fixture.record, fixture.definition, fixture.metadata, fixture.seed,
) {
t.Fatal("drifted VPC evidence was accepted")
}
})
}
fixture := newCreateLikeNetworkEvidenceFixture(t, false)
fixture.record.ExpectedState.VPCPresent = true
port := validVPCPortMetadata()
fixture.record.ExpectedState.VPCKey = port.VPCKey
fixture.record.ExpectedState.VPCRevision = port.HostRevision
fixture.record.ExpectedState.VPCDigest = port.HostDigest
fixture.record.ExpectedState.VPCPortKey = port.PortKey
fixture.record.ExpectedState.VPCCIDR = port.CIDR
fixture.record.ExpectedState.VPCGateway = port.Gateway
fixture.record.ExpectedState.VPCPrivateIP = port.PrivateIP
fixture.record.ExpectedState.VPCMAC = port.MAC
fixture.record.ExpectedState.VPCBridge = port.Bridge
if createLikeNetworkEvidenceMatches(
fixture.record, fixture.definition, fixture.metadata, fixture.seed,
) {
t.Fatal("missing VPC domain and seed evidence was accepted")
}
}
func clearExpectedCreateLikeVPC(expected *operation.ExpectedState) {
expected.VPCKey = ""
expected.VPCRevision = 0
expected.VPCDigest = ""
expected.VPCPortKey = ""
expected.VPCCIDR = ""
expected.VPCGateway = ""
expected.VPCPrivateIP = ""
expected.VPCMAC = ""
expected.VPCBridge = ""
}
func rewriteCreateLikeSeedLinkLocal(t *testing.T, seed []byte,
replacement string) []byte {
t.Helper()
return rewriteCreateLikeSeedNetwork(t, seed, func(ethernets map[string]any) {
entry, ok := ethernets[seedPublicInterfaceKey].(map[string]any)
if !ok {
t.Fatal("public seed interface is missing")
}
addresses := entry["addresses"].([]any)
for index, raw := range addresses {
if value, ok := raw.(string); ok && bytes.HasPrefix([]byte(value), []byte("fe80:")) {
addresses[index] = replacement
}
}
})
}
func rewriteCreateLikeSeedVPC(t *testing.T, seed []byte,
mutate func(map[string]any)) []byte {
t.Helper()
return rewriteCreateLikeSeedNetwork(t, seed, func(ethernets map[string]any) {
entry, ok := ethernets[seedVPCInterfaceKey].(map[string]any)
if !ok {
t.Fatal("VPC seed interface is missing")
}
mutate(entry)
})
}
func rewriteCreateLikeSeedNetwork(t *testing.T, seed []byte,
mutate func(map[string]any)) []byte {
t.Helper()
files, err := readSeedISO(seed)
if err != nil {
t.Fatal(err)
}
var network map[string]any
if err := json.Unmarshal(bytes.TrimSpace(files.NetworkConfig), &network); err != nil {
t.Fatal(err)
}
ethernets, err := managedSeedEthernets(network)
if err != nil {
t.Fatal(err)
}
mutate(ethernets)
files.NetworkConfig, err = marshalJSONLine(network)
if err != nil {
t.Fatal(err)
}
result, err := buildSeedISO(files)
if err != nil {
t.Fatal(err)
}
return result
}

@ -342,7 +342,7 @@ func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record ope
mapped := mapDomainState(state)
switch {
case operation.IsCreateLikeAction(record.Action):
resolution, err := l.reconcileCreatedVM(domain)
resolution, err := l.reconcileCreatedVM(ctx, record, domain)
return resolution, operationCleanupInspection{}, err
case record.Action == "power-start":
if mapped == VMStateRunning || mapped == VMStateBlocked {
@ -429,7 +429,7 @@ func (l *Libvirt) reconcilePreparedPeerMigration(record operation.Record,
return operation.Resolution{}, nil
}
func (l *Libvirt) reconcileReceivedPeerMigration(_ context.Context,
func (l *Libvirt) reconcileReceivedPeerMigration(ctx context.Context,
record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
migrationID, ok := peerMigrationActionID(record.Action, "receive-peer-migration")
if !ok {
@ -451,7 +451,7 @@ func (l *Libvirt) reconcileReceivedPeerMigration(_ context.Context,
if autoErr != nil || activeErr != nil || autostart || active {
return operation.Resolution{}, errors.Join(autoErr, activeErr)
}
vm, complete, err := l.reconcileStagedPeerMigrationVM(domain, receipt)
vm, complete, err := l.reconcileStagedPeerMigrationVM(ctx, record, domain, receipt)
if err != nil || !complete {
return operation.Resolution{}, err
}
@ -466,7 +466,8 @@ func (l *Libvirt) reconcileReceivedPeerMigration(_ context.Context,
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, nil
}
func (l *Libvirt) reconcileStagedPeerMigrationVM(domain *libvirt.Domain,
func (l *Libvirt) reconcileStagedPeerMigrationVM(ctx context.Context,
record operation.Record, domain *libvirt.Domain,
receipt peermigration.TargetReceipt) (VMMetadata, bool, error) {
document, definition, metadata, pool, root, err := l.managedRootContext(domain)
if err != nil {
@ -551,6 +552,18 @@ func (l *Libvirt) reconcileStagedPeerMigrationVM(domain *libvirt.Domain,
l.verifyManagedFilter(filterName, policy) != nil {
return VMMetadata{}, false, nil
}
if record.ExpectedState != nil && record.ExpectedState.Kind == operation.ExpectedCreateLikeNetwork {
seedContents, seedContentsErr := l.managedSeedContents(ctx, record.ResourceID)
if seedContentsErr != nil {
_, evidenceErr := expectedEvidenceError("create-like network", seedContentsErr)
return VMMetadata{}, false, evidenceErr
}
if !createLikeNetworkEvidenceMatches(
record, definition, metadata, seedContents,
) {
return VMMetadata{}, false, nil
}
}
vm, err := metadataFromXML(document)
if err != nil {
return VMMetadata{}, false, nil

@ -13,7 +13,8 @@ import (
"libvirt.org/go/libvirtxml"
)
func (l *Libvirt) reconcileCreatedVM(domain *libvirt.Domain) (operation.Resolution, error) {
func (l *Libvirt) reconcileCreatedVM(ctx context.Context, record operation.Record,
domain *libvirt.Domain) (operation.Resolution, error) {
document, definition, metadata, pool, root, err := l.managedRootContext(domain)
if err != nil {
return operation.Resolution{}, nil
@ -56,6 +57,17 @@ func (l *Libvirt) reconcileCreatedVM(domain *libvirt.Domain) (operation.Resoluti
if json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &policy) != nil || ValidatePlatformNetworkPolicy(policy) != nil || l.verifyManagedFilter(filterName, policy) != nil {
return operation.Resolution{}, nil
}
if record.ExpectedState != nil && record.ExpectedState.Kind == operation.ExpectedCreateLikeNetwork {
seedContents, seedErr := l.managedSeedContents(ctx, record.ResourceID)
if seedErr != nil {
return expectedEvidenceError("create-like network", seedErr)
}
if !createLikeNetworkEvidenceMatches(
record, definition, metadata, seedContents,
) {
return operation.Resolution{}, nil
}
}
result, err := metadataFromXML(document)
if err != nil {
return operation.Resolution{}, nil
@ -99,11 +111,80 @@ func (l *Libvirt) inspectExpectedStateOperation(ctx context.Context, record oper
case operation.ExpectedVPCPort:
resolution, err := l.reconcileVPCPort(ctx, record, domain)
return resolution, true, err
case operation.ExpectedCreateLikeNetwork:
switch {
case strings.HasPrefix(record.Action, "restore-backup:"):
resolution, err := l.reconcileCreatedVM(ctx, record, domain)
return resolution, true, err
case strings.HasPrefix(record.Action, "receive-peer-migration:"):
resolution, err := l.reconcileReceivedPeerMigration(ctx, record, domain)
return resolution, true, err
default:
// Journal validation does not permit this state/action pairing.
// Keep a damaged record fenced instead of falling through to a
// weaker action-only reconciler.
return operation.Resolution{}, true, nil
}
}
}
return operation.Resolution{}, false, nil
}
// createLikeNetworkEvidenceMatches binds an ambiguous restore or peer receive
// to the complete, immutable network identity persisted before dispatch. It
// deliberately performs no mutation. A mismatch remains unresolved so the
// panel cannot release an address or VPC identity which the guest may use.
func createLikeNetworkEvidenceMatches(record operation.Record,
definition *libvirtxml.Domain, metadata *domainMetadataXML, seed []byte) bool {
expected := record.ExpectedState
if expected == nil || expected.Kind != operation.ExpectedCreateLikeNetwork ||
expected.TransitionID != record.OperationID || definition == nil ||
metadata == nil || !strings.EqualFold(definition.UUID, record.ResourceID) ||
definition.Devices == nil {
return false
}
public, err := metadata.PublicNetwork.public()
if err != nil || !publicNetworkEqual(public, expected.PublicNetwork) ||
!managedNetworkLayoutMatches(definition, metadata) {
return false
}
seedMAC, seedPublic, err := seedNetwork(seed)
if err != nil || !strings.EqualFold(seedMAC, vmMACAddress(record.ResourceID)) ||
!seedPublic.matches(expected.PublicNetwork, seedMAC) {
return false
}
publicIndex, found := managedPublicInterfaceIndex(
definition, networkFilterName(record.ResourceID), expected.PublicNetwork,
)
if !found {
return false
}
publicInterface := definition.Devices.Interfaces[publicIndex]
if publicInterface.MAC == nil ||
!strings.EqualFold(publicInterface.MAC.Address, seedMAC) ||
!domainNetworkSourceMatches(
publicInterface, expected.NetworkSourceType,
expected.NetworkSourceName,
) {
return false
}
seedPort, err := seedVPCPort(seed)
if err != nil {
return false
}
if !expected.VPCPresent {
return metadata.VPCPort == nil && seedPort == nil
}
if metadata.VPCPort == nil || seedPort == nil {
return false
}
want := expectedVPCPortMetadata(*expected)
return metadata.VPCPort.public() == want && *seedPort == (seedVPCNetwork{
MAC: want.MAC, PrivateIP: want.PrivateIP,
CIDR: want.CIDR, Gateway: want.Gateway,
})
}
func (l *Libvirt) reconcileVPCPort(ctx context.Context, record operation.Record,
domain *libvirt.Domain) (operation.Resolution, error) {
expected := record.ExpectedState

Loading…
Cancel
Save