You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_vpc_port.go

369 lines
13 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"strings"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
func (l *Libvirt) ApplyVPCPort(ctx context.Context, uuid string,
request VPCPortRequest) (result VPCPortResult, resultErr error) {
return l.changeVPCPort(ctx, uuid, request, true)
}
func (l *Libvirt) RemoveVPCPort(ctx context.Context, uuid string,
request VPCPortRequest) (result VPCPortResult, resultErr error) {
return l.changeVPCPort(ctx, uuid, request, false)
}
func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string,
request VPCPortRequest, present bool) (_ VPCPortResult, resultErr error) {
mutationStarted := false
defer func() {
if !mutationStarted {
resultErr = operation.MarkNotApplied(resultErr)
}
}()
request, err := NormalizeVPCPortRequest(request)
if err != nil {
return VPCPortResult{}, err
}
if strings.EqualFold(request.MAC, vmMACAddress(uuid)) {
return VPCPortResult{}, NewError(ErrConflict, "vpc_mac_conflicts_with_public", "VPC port MAC conflicts with the managed public interface", nil)
}
expected, err := expectedMutationState(ctx, operation.ExpectedVPCPort)
if err != nil {
return VPCPortResult{}, err
}
if !vpcPortRequestMatchesExpected(request, expected, present) {
return VPCPortResult{}, NewError(ErrConflict, "reconciliation_descriptor_mismatch", "VPC port request does not match its durable reconciliation descriptor", nil)
}
if l.vpcRouter == nil {
return VPCPortResult{}, NewError(ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil)
}
_, routeLease, err := l.vpcRouter.AcquireLocalPortLease(
request.VPCKey, request.HostRevision, request.HostDigest,
request.CIDR, request.Gateway, request.PrivateIP, request.MAC,
)
if err != nil {
return VPCPortResult{}, NewError(ErrConflict, "vpc_host_projection_not_ready", "VPC host routing must contain this exact local port before the VM interface changes", err)
}
defer routeLease.Release()
port := vpcPortMetadata(request)
if present {
conflict, err := l.vpcPortIdentityConflict(ctx, uuid, port)
if err != nil {
return VPCPortResult{}, err
}
if conflict {
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_identity_conflict", "VPC port identity is already attached to another managed VM", nil)
}
}
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return VPCPortResult{}, err
}
defer domain.Free()
if err := requirePersistentWithoutSnapshots(domain); err != nil {
return VPCPortResult{}, err
}
seed, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
)
if err != nil {
return VPCPortResult{}, err
}
if seed == nil {
return VPCPortResult{}, NewError(ErrConflict, "seed_not_found", "managed NoCloud seed volume was not found", nil)
}
defer seed.Free()
originalSeed, err := l.downloadSmallVolume(ctx, seed, maxSeedImageBytes)
if err != nil {
return VPCPortResult{}, err
}
preflightXML, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return VPCPortResult{}, translateLibvirtError("read VPC port domain definition", err)
}
preflightDefinition, preflightMetadata, err := parseManagedDomain(preflightXML)
if err != nil {
return VPCPortResult{}, err
}
if request.VPCRepair {
if err := validateRepairableVPCPortDomain(
preflightDefinition, preflightMetadata, port,
); err != nil {
return VPCPortResult{}, err
}
} else {
if !managedNetworkLayoutMatches(preflightDefinition, preflightMetadata) {
return VPCPortResult{}, NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its durable metadata", nil)
}
if present {
if preflightMetadata.VPCPort != nil {
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil)
}
} else if preflightMetadata.VPCPort == nil ||
!sameVPCPortIdentity(preflightMetadata.VPCPort.public(), port) {
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_identity_changed", "VM VPC port no longer matches the requested detach identity", nil)
}
}
var replacementSeed []byte
if request.VPCRepair {
replacementSeed, err = repairSeedVPCPort(
originalSeed, port, expected.TransitionID, present,
)
} else {
replacementSeed, err = mutateSeedVPCPort(
originalSeed, port, expected.TransitionID, present,
)
}
if err != nil {
return VPCPortResult{}, err
}
mutationStarted = true
wasRunning, err := stopForDefinitionChange(ctx, domain)
if err != nil {
return VPCPortResult{}, err
}
originalXML, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return VPCPortResult{}, operation.MarkUncertain(
translateLibvirtError("read stopped VPC port domain definition", err),
)
}
updatedXML, err := mutateDomainDocument(originalXML,
func(definition *libvirtxml.Domain, metadata *domainMetadataXML) error {
if request.VPCRepair {
if err := repairVPCPortDomain(
definition, metadata, port, present,
); err != nil {
return err
}
metadata.VPCTransitionID = expected.TransitionID
return nil
}
if !managedNetworkLayoutMatches(definition, metadata) {
return NewError(ErrConflict, "network_layout_changed", "managed VM network layout changed while applying the VPC port", nil)
}
if present {
if metadata.VPCPort != nil {
return NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil)
}
definition.Devices.Interfaces = append(
definition.Devices.Interfaces,
canonicalVPCInterface(definition, metadata, port),
)
stored := domainVPCPortFromPublic(port)
metadata.VPCPort = &stored
} else {
if metadata.VPCPort == nil || !sameVPCPortIdentity(metadata.VPCPort.public(), port) {
return NewError(ErrConflict, "vpc_port_identity_changed", "VM VPC port changed while detaching", nil)
}
index, found := managedVPCInterfaceIndex(definition, metadata.VPCPort)
if !found {
return NewError(ErrConflict, "vpc_port_interface_changed", "managed VPC interface no longer matches its metadata", nil)
}
definition.Devices.Interfaces = append(
definition.Devices.Interfaces[:index],
definition.Devices.Interfaces[index+1:]...,
)
metadata.VPCPort = nil
}
metadata.VPCTransitionID = expected.TransitionID
return nil
})
if err != nil {
return VPCPortResult{}, err
}
if err := l.replaceSeedAndDefinition(
ctx, seed, originalSeed, replacementSeed, originalXML, updatedXML, wasRunning, nil,
); err != nil {
return VPCPortResult{}, err
}
status := VMStateShutoff
if wasRunning {
status = VMStateRunning
}
result := VPCPortResult{
UUID: uuid, PortKey: request.PortKey, VPCKey: request.VPCKey,
Status: status, Attached: present, Removed: !present,
}
if present {
result.VPCPort = &port
}
return result, nil
}
// validateRepairableVPCPortDomain proves that every non-public interface is
// either the exact panel-owned VPC NIC or is already bound to that identity by
// durable metadata. It never adopts or deletes an unrelated guest interface.
func validateRepairableVPCPortDomain(definition *libvirtxml.Domain,
metadata *domainMetadataXML, port VPCPortMetadata) error {
if definition == nil || definition.Devices == nil || metadata == nil {
return NewError(ErrConflict, "network_layout_changed", "managed VM network layout is incomplete", nil)
}
publicIndex, found := managedPublicInterfaceIndex(
definition, networkFilterName(definition.UUID), metadata.PublicNetwork.publicMust(),
)
if !found || len(definition.Devices.Interfaces) > 2 {
return NewError(ErrConflict, "network_layout_changed", "managed VM has an unrepairable network layout", nil)
}
metadataOwnsPort := metadata.VPCPort != nil
if metadataOwnsPort &&
!sameVPCPortIdentity(metadata.VPCPort.public(), port) {
return NewError(ErrConflict, "vpc_repair_intent_changed", "domain metadata contains a different VPC identity", nil)
}
if len(definition.Devices.Interfaces) == 2 {
privateIndex := 1 - publicIndex
iface := definition.Devices.Interfaces[privateIndex]
exactInterface := iface.MAC != nil &&
strings.EqualFold(iface.MAC.Address, port.MAC) &&
iface.Source != nil && iface.Source.Bridge != nil &&
iface.Source.Bridge.Bridge == port.Bridge &&
iface.Source.Network == nil &&
(iface.FilterRef == nil ||
managedVPCInterfaceFilterIPMatches(&iface, port.PrivateIP))
if !metadataOwnsPort && !exactInterface {
return NewError(ErrConflict, "vpc_repair_unowned_interface", "unrelated private interface cannot be changed by VPC repair", nil)
}
}
return nil
}
func repairVPCPortDomain(definition *libvirtxml.Domain,
metadata *domainMetadataXML, port VPCPortMetadata, present bool) error {
if err := validateRepairableVPCPortDomain(definition, metadata, port); err != nil {
return err
}
publicIndex, _ := managedPublicInterfaceIndex(
definition, networkFilterName(definition.UUID), metadata.PublicNetwork.publicMust(),
)
if len(definition.Devices.Interfaces) == 2 {
privateIndex := 1 - publicIndex
definition.Devices.Interfaces = append(
definition.Devices.Interfaces[:privateIndex],
definition.Devices.Interfaces[privateIndex+1:]...,
)
}
if present {
definition.Devices.Interfaces = append(
definition.Devices.Interfaces,
canonicalVPCInterface(definition, metadata, port),
)
stored := domainVPCPortFromPublic(port)
metadata.VPCPort = &stored
} else {
metadata.VPCPort = nil
}
return nil
}
func canonicalVPCInterface(definition *libvirtxml.Domain,
metadata *domainMetadataXML, port VPCPortMetadata) libvirtxml.DomainInterface {
queues := uint(1)
if definition.VCPU != nil {
queues = min(definition.VCPU.Value, 16)
}
iface := libvirtxml.DomainInterface{
MAC: &libvirtxml.DomainInterfaceMAC{Address: port.MAC},
Model: &libvirtxml.DomainInterfaceModel{Type: "virtio"},
Driver: &libvirtxml.DomainInterfaceDriver{Queues: queues},
Source: &libvirtxml.DomainInterfaceSource{Bridge: &libvirtxml.DomainInterfaceSourceBridge{Bridge: port.Bridge}},
FilterRef: newManagedVPCInterfaceFilterRef(port.PrivateIP),
}
if metadata.BandwidthMbps > 0 {
rate := bandwidthRateKiBPerSecond(metadata.BandwidthMbps)
iface.Bandwidth = &libvirtxml.DomainInterfaceBandwidth{
Inbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
Outbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
}
}
if definitionNetworkSuspended(definition) {
iface.Link = &libvirtxml.DomainInterfaceLink{State: "down"}
}
return iface
}
func (l *Libvirt) vpcPortIdentityConflict(ctx context.Context,
requestedUUID string, port VPCPortMetadata) (bool, error) {
domains, err := l.conn.ListAllDomains(0)
if err != nil {
return false, translateLibvirtError("list domains for VPC port identity validation", err)
}
for index := range domains {
domain := &domains[index]
conflict, inspectErr := func() (bool, error) {
defer domain.Free()
if err := ctx.Err(); err != nil {
return false, err
}
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
return false, translateLibvirtError("read domain VPC identity", err)
}
definition, metadata, managed, err := inspectManagedDomain(document)
if err != nil {
return false, err
}
if !managed {
return false, nil
}
return vpcPortConflictsWithManagedDomain(
requestedUUID, port, definition, metadata,
), nil
}()
if inspectErr != nil || conflict {
for remaining := index + 1; remaining < len(domains); remaining++ {
_ = domains[remaining].Free()
}
return conflict, inspectErr
}
}
return false, nil
}
func vpcPortConflictsWithManagedDomain(requestedUUID string,
port VPCPortMetadata, definition *libvirtxml.Domain,
metadata *domainMetadataXML) bool {
if definition == nil || metadata == nil ||
strings.EqualFold(definition.UUID, requestedUUID) {
return false
}
if existing := metadata.VPCPort; existing != nil &&
(existing.PortKey == port.PortKey ||
(existing.VPCKey == port.VPCKey && existing.PrivateIP == port.PrivateIP)) {
return true
}
if definition.Devices != nil {
for _, iface := range definition.Devices.Interfaces {
if iface.MAC != nil && strings.EqualFold(iface.MAC.Address, port.MAC) {
return true
}
}
}
return false
}
func vpcPortRequestMatchesExpected(request VPCPortRequest,
expected operation.ExpectedState, present bool) bool {
port := vpcPortMetadata(request)
return expected.VPCPresent == present && expected.VPCPortKey == port.PortKey &&
expected.VPCKey == port.VPCKey && expected.VPCPrivateIP == port.PrivateIP &&
expected.VPCCIDR == port.CIDR && expected.VPCGateway == port.Gateway &&
expected.VPCMAC == port.MAC && expected.VPCBridge == port.Bridge &&
expected.VPCRevision == port.HostRevision && expected.VPCDigest == port.HostDigest
}
func sameVPCPortIdentity(left, right VPCPortMetadata) bool {
return left.PortKey == right.PortKey && left.VPCKey == right.VPCKey &&
left.PrivateIP == right.PrivateIP && left.CIDR == right.CIDR &&
left.Gateway == right.Gateway && left.MAC == right.MAC &&
left.Bridge == right.Bridge
}