diff --git a/agent/hypervisor/libvirt_create_like_reconcile_test.go b/agent/hypervisor/libvirt_create_like_reconcile_test.go new file mode 100644 index 00000000..5a1b48a6 --- /dev/null +++ b/agent/hypervisor/libvirt_create_like_reconcile_test.go @@ -0,0 +1,380 @@ +//go:build libvirt + +package hypervisor + +import ( + "bytes" + "encoding/json" + "testing" + + "git.luoli.live/root/virt/agent/cloudinit" + "git.luoli.live/root/virt/agent/operation" + "git.luoli.live/root/virt/agent/publicnetwork" + "libvirt.org/go/libvirtxml" +) + +type createLikeNetworkEvidenceFixture struct { + record operation.Record + definition *libvirtxml.Domain + metadata *domainMetadataXML + seed []byte +} + +func newCreateLikeNetworkEvidenceFixture(t *testing.T, + withVPC bool) createLikeNetworkEvidenceFixture { + t.Helper() + definition := validDomainDefinition() + definition.Network[0].MAC = vmMACAddress(definition.UUID) + var port *VPCPortMetadata + if withVPC { + value := validVPCPortMetadata() + port = &value + definition.VPCPort = port + definition.Network = append(definition.Network, NetworkAttachment{ + Bridge: port.Bridge, MAC: port.MAC, + }) + } + document, err := BuildDomainXML(definition) + if err != nil { + t.Fatal(err) + } + parsed, metadata, err := parseManagedDomain(document) + if err != nil { + t.Fatal(err) + } + seed := buildCreateLikeNetworkSeed( + t, definition.UUID, definition.PublicNetwork, + definition.Network[0].MAC, port, + ) + source, err := ParseNetworkSelector(definition.PublicNetwork.NetworkSelector) + if err != nil { + t.Fatal(err) + } + expected := &operation.ExpectedState{ + Kind: operation.ExpectedCreateLikeNetwork, + TransitionID: "operation-create-like-network-evidence", + NetworkSourceType: source.Type, NetworkSourceName: source.Source, + PublicNetwork: definition.PublicNetwork, + VPCPresent: port != nil, + } + if port != nil { + expected.VPCKey = port.VPCKey + expected.VPCRevision = port.HostRevision + expected.VPCDigest = port.HostDigest + expected.VPCPortKey = port.PortKey + expected.VPCCIDR = port.CIDR + expected.VPCGateway = port.Gateway + expected.VPCPrivateIP = port.PrivateIP + expected.VPCMAC = port.MAC + expected.VPCBridge = port.Bridge + } + return createLikeNetworkEvidenceFixture{ + record: operation.Record{ + OperationID: expected.TransitionID, + ResourceID: definition.UUID, + Action: "receive-peer-migration:0123456789abcdef0123456789abcdef", + ExpectedState: expected, + }, + definition: parsed, metadata: metadata, seed: seed, + } +} + +func buildCreateLikeNetworkSeed(t *testing.T, uuid string, + public publicnetwork.Intent, mac string, port *VPCPortMetadata) []byte { + t.Helper() + var private *cloudinit.PrivateNetwork + if port != nil { + private = &cloudinit.PrivateNetwork{ + MAC: port.MAC, PrivateIP: port.PrivateIP, + CIDR: port.CIDR, Gateway: port.Gateway, + InterfaceKey: seedVPCInterfaceKey, + } + } + image, err := cloudinit.BuildISO(cloudinit.Request{ + UUID: uuid, Name: "vm-create-like", PreserveGuestCredentials: true, + Network: cloudinit.Network{ + MAC: mac, Public: public, InterfaceKey: seedPublicInterfaceKey, + Private: private, + }, + }) + if err != nil { + t.Fatal(err) + } + return image +} + +func TestCreateLikeNetworkEvidenceRequiresExactDualStackIdentity(t *testing.T) { + for _, withVPC := range []bool{false, true} { + fixture := newCreateLikeNetworkEvidenceFixture(t, withVPC) + if !createLikeNetworkEvidenceMatches( + fixture.record, fixture.definition, fixture.metadata, fixture.seed, + ) { + t.Fatalf("exact create-like network evidence rejected (VPC=%v)", withVPC) + } + } +} + +func TestCreateLikeNetworkEvidenceDriftRemainsUnresolved(t *testing.T) { + tests := map[string]func(*createLikeNetworkEvidenceFixture){ + "transition": func(value *createLikeNetworkEvidenceFixture) { + value.record.ExpectedState.TransitionID = "operation-other" + }, + "resource UUID": func(value *createLikeNetworkEvidenceFixture) { + value.record.ResourceID = "123e4567-e89b-42d3-a456-426614174999" + }, + "metadata IPv4": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.PublicNetwork.IPv4.AddressCIDR = "8.8.8.11/24" + }, + "metadata IPv6": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.PublicNetwork.IPv6.AddressCIDR = "2404:c140:156:3a::1001/64" + }, + "metadata DNS": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.PublicNetwork.DNSServers = []string{"9.9.9.9", "2620:fe::fe"} + }, + "metadata selector": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.PublicNetwork.NetworkSelector = "bridge:br1" + }, + "metadata IPv4 gateway": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.PublicNetwork.IPv4.Gateway = "8.8.8.2" + }, + "metadata IPv6 gateway": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.PublicNetwork.IPv6.Gateway = "2404:c140:156::2" + }, + "network source": func(value *createLikeNetworkEvidenceFixture) { + value.definition.Devices.Interfaces[0].Source.Bridge.Bridge = "br1" + }, + "public MAC": func(value *createLikeNetworkEvidenceFixture) { + value.definition.Devices.Interfaces[0].MAC.Address = "02:00:00:00:00:44" + }, + "nwfilter IPv4": func(value *createLikeNetworkEvidenceFixture) { + value.definition.Devices.Interfaces[0].FilterRef.Parameters[0].Value = "8.8.8.11" + }, + "nwfilter IPv6": func(value *createLikeNetworkEvidenceFixture) { + value.definition.Devices.Interfaces[0].FilterRef.Parameters[1].Value = "2404:c140:156:3a::1001" + }, + "nwfilter link-local": func(value *createLikeNetworkEvidenceFixture) { + value.definition.Devices.Interfaces[0].FilterRef.Parameters[2].Value = "fe80::1" + }, + "nwfilter name": func(value *createLikeNetworkEvidenceFixture) { + value.definition.Devices.Interfaces[0].FilterRef.Filter = "virt-forged" + }, + "seed IPv4": func(value *createLikeNetworkEvidenceFixture) { + public := value.record.ExpectedState.PublicNetwork + public.IPv4.AddressCIDR = "8.8.8.11/24" + value.seed = buildCreateLikeNetworkSeed( + t, value.record.ResourceID, public, + vmMACAddress(value.record.ResourceID), nil, + ) + }, + "seed IPv6": func(value *createLikeNetworkEvidenceFixture) { + public := value.record.ExpectedState.PublicNetwork + public.IPv6.AddressCIDR = "2404:c140:156:3a::1001/64" + value.seed = buildCreateLikeNetworkSeed( + t, value.record.ResourceID, public, + vmMACAddress(value.record.ResourceID), nil, + ) + }, + "seed IPv4 gateway": func(value *createLikeNetworkEvidenceFixture) { + public := value.record.ExpectedState.PublicNetwork + public.IPv4.Gateway = "8.8.8.2" + value.seed = buildCreateLikeNetworkSeed( + t, value.record.ResourceID, public, + vmMACAddress(value.record.ResourceID), nil, + ) + }, + "seed IPv6 gateway": func(value *createLikeNetworkEvidenceFixture) { + public := value.record.ExpectedState.PublicNetwork + public.IPv6.Gateway = "2404:c140:156::2" + value.seed = buildCreateLikeNetworkSeed( + t, value.record.ResourceID, public, + vmMACAddress(value.record.ResourceID), nil, + ) + }, + "seed DNS": func(value *createLikeNetworkEvidenceFixture) { + public := value.record.ExpectedState.PublicNetwork + public.DNSServers = []string{"9.9.9.9", "2620:fe::fe"} + value.seed = buildCreateLikeNetworkSeed( + t, value.record.ResourceID, public, + vmMACAddress(value.record.ResourceID), nil, + ) + }, + "seed link-local": func(value *createLikeNetworkEvidenceFixture) { + value.seed = rewriteCreateLikeSeedLinkLocal(t, value.seed, "fe80::1/64") + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + fixture := newCreateLikeNetworkEvidenceFixture(t, false) + mutate(&fixture) + if createLikeNetworkEvidenceMatches( + fixture.record, fixture.definition, fixture.metadata, fixture.seed, + ) { + t.Fatal("drifted create-like network evidence was accepted") + } + }) + } +} + +func TestCreateLikeNetworkEvidenceRequiresExactOptionalVPCIdentity(t *testing.T) { + tests := map[string]func(*createLikeNetworkEvidenceFixture){ + "unexpected VPC": func(value *createLikeNetworkEvidenceFixture) { + value.record.ExpectedState.VPCPresent = false + clearExpectedCreateLikeVPC(value.record.ExpectedState) + }, + "missing metadata": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort = nil + }, + "port key": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort.PortKey = "3123456789abcdef0123456789abcdef" + }, + "VPC key": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort.VPCKey = "4123456789abcdef0123456789abcdef" + }, + "host revision": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort.HostRevision++ + }, + "host digest": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort.HostDigest = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + }, + "private IP": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort.PrivateIP = "10.60.0.10" + }, + "CIDR": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort.CIDR = "10.61.0.0/24" + }, + "gateway": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort.Gateway = "10.60.0.2" + }, + "MAC": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort.MAC = "02:00:00:00:00:10" + }, + "bridge": func(value *createLikeNetworkEvidenceFixture) { + value.metadata.VPCPort.Bridge = "vcffffffffffff" + }, + "seed private IP": func(value *createLikeNetworkEvidenceFixture) { + value.seed = rewriteCreateLikeSeedVPC(t, value.seed, func(entry map[string]any) { + entry["addresses"] = []any{"10.60.0.10/32"} + }) + }, + "seed VPC MAC": func(value *createLikeNetworkEvidenceFixture) { + value.seed = rewriteCreateLikeSeedVPC(t, value.seed, func(entry map[string]any) { + entry["match"] = map[string]any{"macaddress": "02:00:00:00:00:10"} + }) + }, + "seed VPC CIDR": func(value *createLikeNetworkEvidenceFixture) { + value.seed = rewriteCreateLikeSeedVPC(t, value.seed, func(entry map[string]any) { + routes := entry["routes"].([]any) + routes[0].(map[string]any)["to"] = "10.61.0.0/24" + }) + }, + "seed VPC gateway": func(value *createLikeNetworkEvidenceFixture) { + value.seed = rewriteCreateLikeSeedVPC(t, value.seed, func(entry map[string]any) { + routes := entry["routes"].([]any) + routes[0].(map[string]any)["via"] = "10.60.0.2" + }) + }, + "VPC interface source": func(value *createLikeNetworkEvidenceFixture) { + value.definition.Devices.Interfaces[1].Source.Bridge.Bridge = "vcffffffffffff" + }, + "VPC anti-spoof parameter": func(value *createLikeNetworkEvidenceFixture) { + value.definition.Devices.Interfaces[1].FilterRef.Parameters[0].Value = "10.60.0.10" + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + fixture := newCreateLikeNetworkEvidenceFixture(t, true) + mutate(&fixture) + if createLikeNetworkEvidenceMatches( + fixture.record, fixture.definition, fixture.metadata, fixture.seed, + ) { + t.Fatal("drifted VPC evidence was accepted") + } + }) + } + + fixture := newCreateLikeNetworkEvidenceFixture(t, false) + fixture.record.ExpectedState.VPCPresent = true + port := validVPCPortMetadata() + fixture.record.ExpectedState.VPCKey = port.VPCKey + fixture.record.ExpectedState.VPCRevision = port.HostRevision + fixture.record.ExpectedState.VPCDigest = port.HostDigest + fixture.record.ExpectedState.VPCPortKey = port.PortKey + fixture.record.ExpectedState.VPCCIDR = port.CIDR + fixture.record.ExpectedState.VPCGateway = port.Gateway + fixture.record.ExpectedState.VPCPrivateIP = port.PrivateIP + fixture.record.ExpectedState.VPCMAC = port.MAC + fixture.record.ExpectedState.VPCBridge = port.Bridge + if createLikeNetworkEvidenceMatches( + fixture.record, fixture.definition, fixture.metadata, fixture.seed, + ) { + t.Fatal("missing VPC domain and seed evidence was accepted") + } +} + +func clearExpectedCreateLikeVPC(expected *operation.ExpectedState) { + expected.VPCKey = "" + expected.VPCRevision = 0 + expected.VPCDigest = "" + expected.VPCPortKey = "" + expected.VPCCIDR = "" + expected.VPCGateway = "" + expected.VPCPrivateIP = "" + expected.VPCMAC = "" + expected.VPCBridge = "" +} + +func rewriteCreateLikeSeedLinkLocal(t *testing.T, seed []byte, + replacement string) []byte { + t.Helper() + return rewriteCreateLikeSeedNetwork(t, seed, func(ethernets map[string]any) { + entry, ok := ethernets[seedPublicInterfaceKey].(map[string]any) + if !ok { + t.Fatal("public seed interface is missing") + } + addresses := entry["addresses"].([]any) + for index, raw := range addresses { + if value, ok := raw.(string); ok && bytes.HasPrefix([]byte(value), []byte("fe80:")) { + addresses[index] = replacement + } + } + }) +} + +func rewriteCreateLikeSeedVPC(t *testing.T, seed []byte, + mutate func(map[string]any)) []byte { + t.Helper() + return rewriteCreateLikeSeedNetwork(t, seed, func(ethernets map[string]any) { + entry, ok := ethernets[seedVPCInterfaceKey].(map[string]any) + if !ok { + t.Fatal("VPC seed interface is missing") + } + mutate(entry) + }) +} + +func rewriteCreateLikeSeedNetwork(t *testing.T, seed []byte, + mutate func(map[string]any)) []byte { + t.Helper() + files, err := readSeedISO(seed) + if err != nil { + t.Fatal(err) + } + var network map[string]any + if err := json.Unmarshal(bytes.TrimSpace(files.NetworkConfig), &network); err != nil { + t.Fatal(err) + } + ethernets, err := managedSeedEthernets(network) + if err != nil { + t.Fatal(err) + } + mutate(ethernets) + files.NetworkConfig, err = marshalJSONLine(network) + if err != nil { + t.Fatal(err) + } + result, err := buildSeedISO(files) + if err != nil { + t.Fatal(err) + } + return result +} diff --git a/agent/hypervisor/libvirt_operation_admin.go b/agent/hypervisor/libvirt_operation_admin.go index 53e521da..46965926 100644 --- a/agent/hypervisor/libvirt_operation_admin.go +++ b/agent/hypervisor/libvirt_operation_admin.go @@ -342,7 +342,7 @@ func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record ope mapped := mapDomainState(state) switch { case operation.IsCreateLikeAction(record.Action): - resolution, err := l.reconcileCreatedVM(domain) + resolution, err := l.reconcileCreatedVM(ctx, record, domain) return resolution, operationCleanupInspection{}, err case record.Action == "power-start": if mapped == VMStateRunning || mapped == VMStateBlocked { @@ -429,7 +429,7 @@ func (l *Libvirt) reconcilePreparedPeerMigration(record operation.Record, return operation.Resolution{}, nil } -func (l *Libvirt) reconcileReceivedPeerMigration(_ context.Context, +func (l *Libvirt) reconcileReceivedPeerMigration(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) { migrationID, ok := peerMigrationActionID(record.Action, "receive-peer-migration") if !ok { @@ -451,7 +451,7 @@ func (l *Libvirt) reconcileReceivedPeerMigration(_ context.Context, if autoErr != nil || activeErr != nil || autostart || active { return operation.Resolution{}, errors.Join(autoErr, activeErr) } - vm, complete, err := l.reconcileStagedPeerMigrationVM(domain, receipt) + vm, complete, err := l.reconcileStagedPeerMigrationVM(ctx, record, domain, receipt) if err != nil || !complete { return operation.Resolution{}, err } @@ -466,7 +466,8 @@ func (l *Libvirt) reconcileReceivedPeerMigration(_ context.Context, return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, nil } -func (l *Libvirt) reconcileStagedPeerMigrationVM(domain *libvirt.Domain, +func (l *Libvirt) reconcileStagedPeerMigrationVM(ctx context.Context, + record operation.Record, domain *libvirt.Domain, receipt peermigration.TargetReceipt) (VMMetadata, bool, error) { document, definition, metadata, pool, root, err := l.managedRootContext(domain) if err != nil { @@ -551,6 +552,18 @@ func (l *Libvirt) reconcileStagedPeerMigrationVM(domain *libvirt.Domain, l.verifyManagedFilter(filterName, policy) != nil { return VMMetadata{}, false, nil } + if record.ExpectedState != nil && record.ExpectedState.Kind == operation.ExpectedCreateLikeNetwork { + seedContents, seedContentsErr := l.managedSeedContents(ctx, record.ResourceID) + if seedContentsErr != nil { + _, evidenceErr := expectedEvidenceError("create-like network", seedContentsErr) + return VMMetadata{}, false, evidenceErr + } + if !createLikeNetworkEvidenceMatches( + record, definition, metadata, seedContents, + ) { + return VMMetadata{}, false, nil + } + } vm, err := metadataFromXML(document) if err != nil { return VMMetadata{}, false, nil diff --git a/agent/hypervisor/libvirt_reconcile_features.go b/agent/hypervisor/libvirt_reconcile_features.go index 873b59e8..655212b2 100644 --- a/agent/hypervisor/libvirt_reconcile_features.go +++ b/agent/hypervisor/libvirt_reconcile_features.go @@ -13,7 +13,8 @@ import ( "libvirt.org/go/libvirtxml" ) -func (l *Libvirt) reconcileCreatedVM(domain *libvirt.Domain) (operation.Resolution, error) { +func (l *Libvirt) reconcileCreatedVM(ctx context.Context, record operation.Record, + domain *libvirt.Domain) (operation.Resolution, error) { document, definition, metadata, pool, root, err := l.managedRootContext(domain) if err != nil { return operation.Resolution{}, nil @@ -56,6 +57,17 @@ func (l *Libvirt) reconcileCreatedVM(domain *libvirt.Domain) (operation.Resoluti if json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &policy) != nil || ValidatePlatformNetworkPolicy(policy) != nil || l.verifyManagedFilter(filterName, policy) != nil { return operation.Resolution{}, nil } + if record.ExpectedState != nil && record.ExpectedState.Kind == operation.ExpectedCreateLikeNetwork { + seedContents, seedErr := l.managedSeedContents(ctx, record.ResourceID) + if seedErr != nil { + return expectedEvidenceError("create-like network", seedErr) + } + if !createLikeNetworkEvidenceMatches( + record, definition, metadata, seedContents, + ) { + return operation.Resolution{}, nil + } + } result, err := metadataFromXML(document) if err != nil { return operation.Resolution{}, nil @@ -99,11 +111,80 @@ func (l *Libvirt) inspectExpectedStateOperation(ctx context.Context, record oper case operation.ExpectedVPCPort: resolution, err := l.reconcileVPCPort(ctx, record, domain) return resolution, true, err + case operation.ExpectedCreateLikeNetwork: + switch { + case strings.HasPrefix(record.Action, "restore-backup:"): + resolution, err := l.reconcileCreatedVM(ctx, record, domain) + return resolution, true, err + case strings.HasPrefix(record.Action, "receive-peer-migration:"): + resolution, err := l.reconcileReceivedPeerMigration(ctx, record, domain) + return resolution, true, err + default: + // Journal validation does not permit this state/action pairing. + // Keep a damaged record fenced instead of falling through to a + // weaker action-only reconciler. + return operation.Resolution{}, true, nil + } } } return operation.Resolution{}, false, nil } +// createLikeNetworkEvidenceMatches binds an ambiguous restore or peer receive +// to the complete, immutable network identity persisted before dispatch. It +// deliberately performs no mutation. A mismatch remains unresolved so the +// panel cannot release an address or VPC identity which the guest may use. +func createLikeNetworkEvidenceMatches(record operation.Record, + definition *libvirtxml.Domain, metadata *domainMetadataXML, seed []byte) bool { + expected := record.ExpectedState + if expected == nil || expected.Kind != operation.ExpectedCreateLikeNetwork || + expected.TransitionID != record.OperationID || definition == nil || + metadata == nil || !strings.EqualFold(definition.UUID, record.ResourceID) || + definition.Devices == nil { + return false + } + public, err := metadata.PublicNetwork.public() + if err != nil || !publicNetworkEqual(public, expected.PublicNetwork) || + !managedNetworkLayoutMatches(definition, metadata) { + return false + } + seedMAC, seedPublic, err := seedNetwork(seed) + if err != nil || !strings.EqualFold(seedMAC, vmMACAddress(record.ResourceID)) || + !seedPublic.matches(expected.PublicNetwork, seedMAC) { + return false + } + publicIndex, found := managedPublicInterfaceIndex( + definition, networkFilterName(record.ResourceID), expected.PublicNetwork, + ) + if !found { + return false + } + publicInterface := definition.Devices.Interfaces[publicIndex] + if publicInterface.MAC == nil || + !strings.EqualFold(publicInterface.MAC.Address, seedMAC) || + !domainNetworkSourceMatches( + publicInterface, expected.NetworkSourceType, + expected.NetworkSourceName, + ) { + return false + } + seedPort, err := seedVPCPort(seed) + if err != nil { + return false + } + if !expected.VPCPresent { + return metadata.VPCPort == nil && seedPort == nil + } + if metadata.VPCPort == nil || seedPort == nil { + return false + } + want := expectedVPCPortMetadata(*expected) + return metadata.VPCPort.public() == want && *seedPort == (seedVPCNetwork{ + MAC: want.MAC, PrivateIP: want.PrivateIP, + CIDR: want.CIDR, Gateway: want.Gateway, + }) +} + func (l *Libvirt) reconcileVPCPort(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) { expected := record.ExpectedState