feat(agent): preserve VPC ports during migration

go
codex 4 weeks ago
parent a0861247e0
commit a1ae5e6b62

@ -193,7 +193,7 @@ func PreparePeerMigrationAdmission(uuid string, request PreparePeerMigrationRequ
func ReceivePeerMigrationAdmission(request ReceivePeerMigrationRequest) MutationAdmission {
uuid := strings.ToLower(request.UUID)
return MutationAdmission{HeavyIO: true, LockKeys: []string{
locks := []string{
domainLockKey(uuid), vmManagedVolumesLockKey(uuid),
volumeLockKey(managedStoragePoolName, rootVolumeName(uuid)),
volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)),
@ -201,7 +201,17 @@ func ReceivePeerMigrationAdmission(request ReceivePeerMigrationRequest) Mutation
networkIdentityLockKey(request.PublicIPCIDR),
networkIdentityLockKey(vmMACAddress(uuid)),
catalogLockKey("peer-migration", request.MigrationID),
}}
}
if request.VPCPort != nil {
locks = append(locks,
"catalog:vpc-host/state",
catalogLockKey("vpc", request.VPCPort.VPCKey),
catalogLockKey("vpc-port", request.VPCPort.PortKey),
networkIdentityLockKey(request.VPCPort.PrivateIP),
networkIdentityLockKey(request.VPCPort.MAC),
)
}
return MutationAdmission{HeavyIO: true, LockKeys: locks}
}
func CleanupPeerMigrationTargetAdmission(uuid, migrationID string) MutationAdmission {

@ -59,6 +59,29 @@ func TestVPCPortAdmissionSerializesDomainSeedAndHostProjection(t *testing.T) {
}
}
func TestPeerMigrationReceiveAdmissionIncludesVPCIdentity(t *testing.T) {
const uuid = "123e4567-e89b-42d3-a456-426614174000"
port := validVPCPortMetadata()
admission := ReceivePeerMigrationAdmission(ReceivePeerMigrationRequest{
CreateVMRequest: CreateVMRequest{
UUID: uuid, PublicIPCIDR: "192.0.2.10/24",
},
MigrationID: strings.Repeat("1", 32),
VPCPort: &port,
})
for _, required := range []string{
"catalog:vpc-host/state",
catalogLockKey("vpc", port.VPCKey),
catalogLockKey("vpc-port", port.PortKey),
networkIdentityLockKey(port.PrivateIP),
networkIdentityLockKey(port.MAC),
} {
if !slices.Contains(admission.LockKeys, required) {
t.Fatalf("peer migration receive admission missing %q: %#v", required, admission)
}
}
}
func TestSnapshotAndReinstallAdmissionsUseDeterministicVolumes(t *testing.T) {
const uuid = "123e4567-e89b-42d3-a456-426614174000"
reinstall := ReinstallVMAdmission(uuid, ReinstallRequest{

@ -290,6 +290,7 @@ type preparedVMOptions struct {
Autostart bool
Start bool
NetworkSuspended bool
VPCPort *VPCPortMetadata
Snapshots []domainSnapshotMetadataXML
SnapshotVolumes []*libvirt.StorageVol
}
@ -297,6 +298,20 @@ type preparedVMOptions struct {
func (l *Libvirt) createVMWithPreparedRootOptions(ctx context.Context,
request CreateVMRequest, attachment NetworkAttachment, seed []byte,
root *libvirt.StorageVol, options preparedVMOptions) (_ VMMetadata, resultErr error) {
var domainVPCPort *VPCPortMetadata
var seedVPCMetadata *domainVPCPortMetadataXML
if options.VPCPort != nil {
value := *options.VPCPort
if err := validateVPCPortMetadata(value); err != nil {
return VMMetadata{}, err
}
domainVPCPort = &value
metadata := domainVPCPortFromPublic(value)
seedVPCMetadata = &metadata
}
if _, err := preservedVPCCloudInitNetwork(seed, seedVPCMetadata); err != nil {
return VMMetadata{}, err
}
snapshotVolumesOwned := len(options.SnapshotVolumes) > 0
defer func() {
for _, volume := range options.SnapshotVolumes {
@ -392,6 +407,14 @@ func (l *Libvirt) createVMWithPreparedRootOptions(ctx context.Context,
_ = filter.Free()
}()
attachment.LinkDown = options.NetworkSuspended
networks := []NetworkAttachment{attachment}
if domainVPCPort != nil {
networks = append(networks, NetworkAttachment{
Bridge: domainVPCPort.Bridge,
MAC: domainVPCPort.MAC,
LinkDown: options.NetworkSuspended,
})
}
document, err := BuildDomainXML(DomainDefinition{
UUID: request.UUID, Name: request.DomainName, DisplayName: request.Name, Description: request.Description,
@ -406,7 +429,7 @@ func (l *Libvirt) createVMWithPreparedRootOptions(ctx context.Context,
Mode: "uefi", Loader: l.local.FirmwareLoader, NVRAMTemplate: l.local.FirmwareNVRAMTemplate,
NVRAMPath: filepath.Join(l.local.NVRAMDirectory, request.UUID+"_VARS.fd"), SecureBoot: false,
},
Network: []NetworkAttachment{attachment}, FilterName: filterName,
Network: networks, VPCPort: domainVPCPort, FilterName: filterName,
PlatformNetworkPolicy: request.PlatformNetworkPolicy,
Snapshots: append([]domainSnapshotMetadataXML(nil), options.Snapshots...),
})

@ -219,6 +219,38 @@ func (l *Libvirt) ReceivePeerMigration(ctx context.Context,
if err := l.ensureRestoreTargetAbsent(ctx, request.UUID); err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
}
var targetVPCPort *VPCPortMetadata
var privateNetwork *cloudinit.PrivateNetwork
if request.VPCPort != nil {
if l.vpcRouter == nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil,
))
}
port := *request.VPCPort
if _, err := l.vpcRouter.AuthorizeLocalPort(
port.VPCKey, port.HostRevision, port.HostDigest,
port.CIDR, port.Gateway, port.PrivateIP, port.MAC,
); err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "vpc_host_projection_not_ready", "peer migration target routing does not authorize the VPC port", err,
))
}
conflict, err := l.vpcPortIdentityConflict(ctx, request.UUID, port)
if err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
}
if conflict {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "vpc_port_identity_conflict", "peer migration VPC port is already attached to another managed VM", nil,
))
}
targetVPCPort = &port
privateNetwork = &cloudinit.PrivateNetwork{
MAC: port.MAC, PrivateIP: port.PrivateIP, CIDR: port.CIDR,
Gateway: port.Gateway, InterfaceKey: seedVPCInterfaceKey,
}
}
store, err := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
@ -289,6 +321,7 @@ func (l *Libvirt) ReceivePeerMigration(ctx context.Context,
Network: cloudinit.Network{
MAC: mac, PublicCIDR: create.PublicIPCIDR, Gateway: create.Gateway,
DNSServers: append([]string(nil), create.DNSServers...), InterfaceKey: "public0",
Private: privateNetwork,
},
})
if err != nil {
@ -345,6 +378,7 @@ func (l *Libvirt) ReceivePeerMigration(ctx context.Context,
vm, err := l.createVMWithPreparedRootOptions(ctx, create, attachment, seed, root,
preparedVMOptions{
Autostart: false, Start: false, NetworkSuspended: request.NetworkSuspended,
VPCPort: targetVPCPort,
Snapshots: snapshotMetadata, SnapshotVolumes: volumes,
})
if err != nil {

@ -78,6 +78,14 @@ func ValidateReceivePeerMigrationRequest(request ReceivePeerMigrationRequest) er
if err := ValidateCreateVM(candidate); err != nil {
return err
}
if request.VPCPort != nil {
if err := validateVPCPortMetadata(*request.VPCPort); err != nil {
return NewError(ErrInvalid, "invalid_peer_migration_vpc_port", "peer migration VPC port is invalid", err)
}
if strings.EqualFold(request.VPCPort.MAC, vmMACAddress(request.UUID)) {
return NewError(ErrConflict, "vpc_mac_conflicts_with_public", "peer migration VPC MAC conflicts with the managed public interface", nil)
}
}
return nil
}

@ -0,0 +1,53 @@
package hypervisor
import (
"strings"
"testing"
)
func validReceivePeerMigrationRequest(t *testing.T) ReceivePeerMigrationRequest {
t.Helper()
create := validIntentCreateRequest(t)
create.Source = ""
create.TemplateID = ""
create.TemplateRevision = 0
create.TemplateGenerationToken = ""
create.StartupScript = ""
create.GuestSSHPublicKey = ""
create.RootPassword = ""
create.OperationID = "operation-peer-migration-receive"
port := validVPCPortMetadata()
return ReceivePeerMigrationRequest{
CreateVMRequest: create,
MigrationID: strings.Repeat("1", 32),
Ticket: strings.Repeat("2", 64),
SourceEndpoints: []string{"https://10.20.0.11:9443"},
SourceIdentitySHA256: strings.Repeat("3", 64),
SourceUUID: "123e4567-e89b-42d3-a456-426614174001",
ManifestDigest: strings.Repeat("4", 64),
VPCPort: &port,
}
}
func TestValidateReceivePeerMigrationBindsOptionalVPCPort(t *testing.T) {
request := validReceivePeerMigrationRequest(t)
if err := ValidateReceivePeerMigrationRequest(request); err != nil {
t.Fatalf("valid VPC migration receive rejected: %v", err)
}
invalid := request
port := *request.VPCPort
port.Bridge = "wrong-bridge"
invalid.VPCPort = &port
if err := ValidateReceivePeerMigrationRequest(invalid); err == nil {
t.Fatal("migration receive accepted a VPC port with a forged bridge")
}
conflicting := request
port = *request.VPCPort
port.MAC = vmMACAddress(request.UUID)
conflicting.VPCPort = &port
if err := ValidateReceivePeerMigrationRequest(conflicting); err == nil {
t.Fatal("migration receive accepted one MAC for public and VPC interfaces")
}
}

@ -355,6 +355,10 @@ type ReceivePeerMigrationRequest struct {
ManifestDigest string `json:"manifest_digest"`
Debian12NetworkFix bool `json:"debian_12_network_fix"`
NetworkSuspended bool `json:"network_suspended"`
// VPCPort is the immutable logical port being moved. The target Agent
// accepts it only after its local host projection authorizes this exact
// identity; an omitted value preserves the ordinary public-only flow.
VPCPort *VPCPortMetadata `json:"vpc_port,omitempty"`
}
type ReceivePeerMigrationResult struct {

@ -11,7 +11,7 @@ import (
)
const (
vpcL3CapabilityVersion = uint64(2)
vpcL3CapabilityVersion = uint64(3)
maxVPCTransportCandidates = 64
vpcNoTransportCandidates = "no_private_transport_addresses"
vpcIPv4ForwardingDisabled = "ipv4_forwarding_disabled"

@ -11,7 +11,7 @@ namespace App\Services\Cloud;
*/
final class CloudVpcTransportCapability
{
public const VERSION = 2;
public const VERSION = 3;
private const MAX_CANDIDATES = 64;
public static function inspect(array $payload): array

@ -15,7 +15,7 @@ $payload = [
'backend' => 'libvirt',
'features' => ['vpc_l3' => true],
'vpc_l3' => [
'version' => 2,
'version' => 3,
'apply_supported' => true,
'vm_port_supported' => true,
'ipv4_forwarding' => true,

Loading…
Cancel
Save