You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
369 lines
13 KiB
Go
369 lines
13 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
func (l *Libvirt) ApplyVPCPort(ctx context.Context, uuid string,
|
|
request VPCPortRequest) (result VPCPortResult, resultErr error) {
|
|
return l.changeVPCPort(ctx, uuid, request, true)
|
|
}
|
|
|
|
func (l *Libvirt) RemoveVPCPort(ctx context.Context, uuid string,
|
|
request VPCPortRequest) (result VPCPortResult, resultErr error) {
|
|
return l.changeVPCPort(ctx, uuid, request, false)
|
|
}
|
|
|
|
func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string,
|
|
request VPCPortRequest, present bool) (_ VPCPortResult, resultErr error) {
|
|
mutationStarted := false
|
|
defer func() {
|
|
if !mutationStarted {
|
|
resultErr = operation.MarkNotApplied(resultErr)
|
|
}
|
|
}()
|
|
request, err := NormalizeVPCPortRequest(request)
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
if strings.EqualFold(request.MAC, vmMACAddress(uuid)) {
|
|
return VPCPortResult{}, NewError(ErrConflict, "vpc_mac_conflicts_with_public", "VPC port MAC conflicts with the managed public interface", nil)
|
|
}
|
|
expected, err := expectedMutationState(ctx, operation.ExpectedVPCPort)
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
if !vpcPortRequestMatchesExpected(request, expected, present) {
|
|
return VPCPortResult{}, NewError(ErrConflict, "reconciliation_descriptor_mismatch", "VPC port request does not match its durable reconciliation descriptor", nil)
|
|
}
|
|
if l.vpcRouter == nil {
|
|
return VPCPortResult{}, NewError(ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil)
|
|
}
|
|
_, routeLease, err := l.vpcRouter.AcquireLocalPortLease(
|
|
request.VPCKey, request.HostRevision, request.HostDigest,
|
|
request.CIDR, request.Gateway, request.PrivateIP, request.MAC,
|
|
)
|
|
if err != nil {
|
|
return VPCPortResult{}, NewError(ErrConflict, "vpc_host_projection_not_ready", "VPC host routing must contain this exact local port before the VM interface changes", err)
|
|
}
|
|
defer routeLease.Release()
|
|
port := vpcPortMetadata(request)
|
|
if present {
|
|
conflict, err := l.vpcPortIdentityConflict(ctx, uuid, port)
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
if conflict {
|
|
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_identity_conflict", "VPC port identity is already attached to another managed VM", nil)
|
|
}
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
defer domain.Free()
|
|
if err := requirePersistentWithoutSnapshots(domain); err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
seed, err := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
|
|
)
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
if seed == nil {
|
|
return VPCPortResult{}, NewError(ErrConflict, "seed_not_found", "managed NoCloud seed volume was not found", nil)
|
|
}
|
|
defer seed.Free()
|
|
originalSeed, err := l.downloadSmallVolume(ctx, seed, maxSeedImageBytes)
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
preflightXML, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if err != nil {
|
|
return VPCPortResult{}, translateLibvirtError("read VPC port domain definition", err)
|
|
}
|
|
preflightDefinition, preflightMetadata, err := parseManagedDomain(preflightXML)
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
if request.VPCRepair {
|
|
if err := validateRepairableVPCPortDomain(
|
|
preflightDefinition, preflightMetadata, port,
|
|
); err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
} else {
|
|
if !managedNetworkLayoutMatches(preflightDefinition, preflightMetadata) {
|
|
return VPCPortResult{}, NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its durable metadata", nil)
|
|
}
|
|
if present {
|
|
if preflightMetadata.VPCPort != nil {
|
|
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil)
|
|
}
|
|
} else if preflightMetadata.VPCPort == nil ||
|
|
!sameVPCPortIdentity(preflightMetadata.VPCPort.public(), port) {
|
|
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_identity_changed", "VM VPC port no longer matches the requested detach identity", nil)
|
|
}
|
|
}
|
|
var replacementSeed []byte
|
|
if request.VPCRepair {
|
|
replacementSeed, err = repairSeedVPCPort(
|
|
originalSeed, port, expected.TransitionID, present,
|
|
)
|
|
} else {
|
|
replacementSeed, err = mutateSeedVPCPort(
|
|
originalSeed, port, expected.TransitionID, present,
|
|
)
|
|
}
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
mutationStarted = true
|
|
wasRunning, err := stopForDefinitionChange(ctx, domain)
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
originalXML, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if err != nil {
|
|
return VPCPortResult{}, operation.MarkUncertain(
|
|
translateLibvirtError("read stopped VPC port domain definition", err),
|
|
)
|
|
}
|
|
updatedXML, err := mutateDomainDocument(originalXML,
|
|
func(definition *libvirtxml.Domain, metadata *domainMetadataXML) error {
|
|
if request.VPCRepair {
|
|
if err := repairVPCPortDomain(
|
|
definition, metadata, port, present,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
metadata.VPCTransitionID = expected.TransitionID
|
|
return nil
|
|
}
|
|
if !managedNetworkLayoutMatches(definition, metadata) {
|
|
return NewError(ErrConflict, "network_layout_changed", "managed VM network layout changed while applying the VPC port", nil)
|
|
}
|
|
if present {
|
|
if metadata.VPCPort != nil {
|
|
return NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil)
|
|
}
|
|
definition.Devices.Interfaces = append(
|
|
definition.Devices.Interfaces,
|
|
canonicalVPCInterface(definition, metadata, port),
|
|
)
|
|
stored := domainVPCPortFromPublic(port)
|
|
metadata.VPCPort = &stored
|
|
} else {
|
|
if metadata.VPCPort == nil || !sameVPCPortIdentity(metadata.VPCPort.public(), port) {
|
|
return NewError(ErrConflict, "vpc_port_identity_changed", "VM VPC port changed while detaching", nil)
|
|
}
|
|
index, found := managedVPCInterfaceIndex(definition, metadata.VPCPort)
|
|
if !found {
|
|
return NewError(ErrConflict, "vpc_port_interface_changed", "managed VPC interface no longer matches its metadata", nil)
|
|
}
|
|
definition.Devices.Interfaces = append(
|
|
definition.Devices.Interfaces[:index],
|
|
definition.Devices.Interfaces[index+1:]...,
|
|
)
|
|
metadata.VPCPort = nil
|
|
}
|
|
metadata.VPCTransitionID = expected.TransitionID
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
if err := l.replaceSeedAndDefinition(
|
|
ctx, seed, originalSeed, replacementSeed, originalXML, updatedXML, wasRunning, nil,
|
|
); err != nil {
|
|
return VPCPortResult{}, err
|
|
}
|
|
status := VMStateShutoff
|
|
if wasRunning {
|
|
status = VMStateRunning
|
|
}
|
|
result := VPCPortResult{
|
|
UUID: uuid, PortKey: request.PortKey, VPCKey: request.VPCKey,
|
|
Status: status, Attached: present, Removed: !present,
|
|
}
|
|
if present {
|
|
result.VPCPort = &port
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// validateRepairableVPCPortDomain proves that every non-public interface is
|
|
// either the exact panel-owned VPC NIC or is already bound to that identity by
|
|
// durable metadata. It never adopts or deletes an unrelated guest interface.
|
|
func validateRepairableVPCPortDomain(definition *libvirtxml.Domain,
|
|
metadata *domainMetadataXML, port VPCPortMetadata) error {
|
|
if definition == nil || definition.Devices == nil || metadata == nil {
|
|
return NewError(ErrConflict, "network_layout_changed", "managed VM network layout is incomplete", nil)
|
|
}
|
|
publicIndex, found := managedPublicInterfaceIndex(
|
|
definition, networkFilterName(definition.UUID), metadata.PublicNetwork.publicMust(),
|
|
)
|
|
if !found || len(definition.Devices.Interfaces) > 2 {
|
|
return NewError(ErrConflict, "network_layout_changed", "managed VM has an unrepairable network layout", nil)
|
|
}
|
|
metadataOwnsPort := metadata.VPCPort != nil
|
|
if metadataOwnsPort &&
|
|
!sameVPCPortIdentity(metadata.VPCPort.public(), port) {
|
|
return NewError(ErrConflict, "vpc_repair_intent_changed", "domain metadata contains a different VPC identity", nil)
|
|
}
|
|
if len(definition.Devices.Interfaces) == 2 {
|
|
privateIndex := 1 - publicIndex
|
|
iface := definition.Devices.Interfaces[privateIndex]
|
|
exactInterface := iface.MAC != nil &&
|
|
strings.EqualFold(iface.MAC.Address, port.MAC) &&
|
|
iface.Source != nil && iface.Source.Bridge != nil &&
|
|
iface.Source.Bridge.Bridge == port.Bridge &&
|
|
iface.Source.Network == nil &&
|
|
(iface.FilterRef == nil ||
|
|
managedVPCInterfaceFilterIPMatches(&iface, port.PrivateIP))
|
|
if !metadataOwnsPort && !exactInterface {
|
|
return NewError(ErrConflict, "vpc_repair_unowned_interface", "unrelated private interface cannot be changed by VPC repair", nil)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func repairVPCPortDomain(definition *libvirtxml.Domain,
|
|
metadata *domainMetadataXML, port VPCPortMetadata, present bool) error {
|
|
if err := validateRepairableVPCPortDomain(definition, metadata, port); err != nil {
|
|
return err
|
|
}
|
|
publicIndex, _ := managedPublicInterfaceIndex(
|
|
definition, networkFilterName(definition.UUID), metadata.PublicNetwork.publicMust(),
|
|
)
|
|
if len(definition.Devices.Interfaces) == 2 {
|
|
privateIndex := 1 - publicIndex
|
|
definition.Devices.Interfaces = append(
|
|
definition.Devices.Interfaces[:privateIndex],
|
|
definition.Devices.Interfaces[privateIndex+1:]...,
|
|
)
|
|
}
|
|
if present {
|
|
definition.Devices.Interfaces = append(
|
|
definition.Devices.Interfaces,
|
|
canonicalVPCInterface(definition, metadata, port),
|
|
)
|
|
stored := domainVPCPortFromPublic(port)
|
|
metadata.VPCPort = &stored
|
|
} else {
|
|
metadata.VPCPort = nil
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func canonicalVPCInterface(definition *libvirtxml.Domain,
|
|
metadata *domainMetadataXML, port VPCPortMetadata) libvirtxml.DomainInterface {
|
|
queues := uint(1)
|
|
if definition.VCPU != nil {
|
|
queues = min(definition.VCPU.Value, 16)
|
|
}
|
|
iface := libvirtxml.DomainInterface{
|
|
MAC: &libvirtxml.DomainInterfaceMAC{Address: port.MAC},
|
|
Model: &libvirtxml.DomainInterfaceModel{Type: "virtio"},
|
|
Driver: &libvirtxml.DomainInterfaceDriver{Queues: queues},
|
|
Source: &libvirtxml.DomainInterfaceSource{Bridge: &libvirtxml.DomainInterfaceSourceBridge{Bridge: port.Bridge}},
|
|
FilterRef: newManagedVPCInterfaceFilterRef(port.PrivateIP),
|
|
}
|
|
if metadata.BandwidthMbps > 0 {
|
|
rate := bandwidthRateKiBPerSecond(metadata.BandwidthMbps)
|
|
iface.Bandwidth = &libvirtxml.DomainInterfaceBandwidth{
|
|
Inbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
|
|
Outbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
|
|
}
|
|
}
|
|
if definitionNetworkSuspended(definition) {
|
|
iface.Link = &libvirtxml.DomainInterfaceLink{State: "down"}
|
|
}
|
|
return iface
|
|
}
|
|
|
|
func (l *Libvirt) vpcPortIdentityConflict(ctx context.Context,
|
|
requestedUUID string, port VPCPortMetadata) (bool, error) {
|
|
domains, err := l.conn.ListAllDomains(0)
|
|
if err != nil {
|
|
return false, translateLibvirtError("list domains for VPC port identity validation", err)
|
|
}
|
|
for index := range domains {
|
|
domain := &domains[index]
|
|
conflict, inspectErr := func() (bool, error) {
|
|
defer domain.Free()
|
|
if err := ctx.Err(); err != nil {
|
|
return false, err
|
|
}
|
|
document, err := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if err != nil {
|
|
return false, translateLibvirtError("read domain VPC identity", err)
|
|
}
|
|
definition, metadata, managed, err := inspectManagedDomain(document)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !managed {
|
|
return false, nil
|
|
}
|
|
return vpcPortConflictsWithManagedDomain(
|
|
requestedUUID, port, definition, metadata,
|
|
), nil
|
|
}()
|
|
if inspectErr != nil || conflict {
|
|
for remaining := index + 1; remaining < len(domains); remaining++ {
|
|
_ = domains[remaining].Free()
|
|
}
|
|
return conflict, inspectErr
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func vpcPortConflictsWithManagedDomain(requestedUUID string,
|
|
port VPCPortMetadata, definition *libvirtxml.Domain,
|
|
metadata *domainMetadataXML) bool {
|
|
if definition == nil || metadata == nil ||
|
|
strings.EqualFold(definition.UUID, requestedUUID) {
|
|
return false
|
|
}
|
|
if existing := metadata.VPCPort; existing != nil &&
|
|
(existing.PortKey == port.PortKey ||
|
|
(existing.VPCKey == port.VPCKey && existing.PrivateIP == port.PrivateIP)) {
|
|
return true
|
|
}
|
|
if definition.Devices != nil {
|
|
for _, iface := range definition.Devices.Interfaces {
|
|
if iface.MAC != nil && strings.EqualFold(iface.MAC.Address, port.MAC) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func vpcPortRequestMatchesExpected(request VPCPortRequest,
|
|
expected operation.ExpectedState, present bool) bool {
|
|
port := vpcPortMetadata(request)
|
|
return expected.VPCPresent == present && expected.VPCPortKey == port.PortKey &&
|
|
expected.VPCKey == port.VPCKey && expected.VPCPrivateIP == port.PrivateIP &&
|
|
expected.VPCCIDR == port.CIDR && expected.VPCGateway == port.Gateway &&
|
|
expected.VPCMAC == port.MAC && expected.VPCBridge == port.Bridge &&
|
|
expected.VPCRevision == port.HostRevision && expected.VPCDigest == port.HostDigest
|
|
}
|
|
|
|
func sameVPCPortIdentity(left, right VPCPortMetadata) bool {
|
|
return left.PortKey == right.PortKey && left.VPCKey == right.VPCKey &&
|
|
left.PrivateIP == right.PrivateIP && left.CIDR == right.CIDR &&
|
|
left.Gateway == right.Gateway && left.MAC == right.MAC &&
|
|
left.Bridge == right.Bridge
|
|
}
|