You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_peer_migration.go

819 lines
33 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"os"
"path/filepath"
"strings"
"time"
"git.luoli.live/root/virt/agent/cloudinit"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/peermigration"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
func (l *Libvirt) PreparePeerMigration(ctx context.Context, uuid string,
request PreparePeerMigrationRequest) (_ PreparePeerMigrationResult, resultErr error) {
if err := ValidatePreparePeerMigrationRequest(request); err != nil {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(err)
}
store, err := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
)
if err != nil {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "peer_migration_state_unavailable", "peer migration state is unavailable", err,
))
}
if _, err := store.LoadSource(request.MigrationID); err == nil {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "peer_migration_exists", "peer migration source session already exists", nil,
))
} else if !errors.Is(err, os.ErrNotExist) {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "peer_migration_state_unavailable", "peer migration source state cannot be inspected", err,
))
}
if _, err := store.LoadSourceIntent(request.MigrationID); err == nil {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "peer_migration_exists", "peer migration source intent already exists", nil,
))
} else if !errors.Is(err, os.ErrNotExist) {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "peer_migration_state_unavailable", "peer migration source intent cannot be inspected", err,
))
}
if _, err := l.validateBackupSource(ctx, uuid, request.MigrationID); err != nil {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(err)
}
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(err)
}
_, _, metadata, rootPool, rootVolume, err := l.managedRootContext(domain)
if err != nil {
domain.Free()
return PreparePeerMigrationResult{}, operation.MarkNotApplied(err)
}
rootPath, err := rootVolume.GetPath()
rootInfo, infoErr := validateStandaloneQCOW2Volume(rootVolume, 0)
rootVolume.Free()
rootPool.Free()
domain.Free()
if err != nil || infoErr != nil || !filepath.IsAbs(rootPath) {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "peer_migration_root_unavailable", "peer migration root disk is unavailable", errors.Join(err, infoErr),
))
}
snapshotMetadata := make(map[string]domainSnapshotMetadataXML, len(metadata.Snapshots))
for _, snapshot := range metadata.Snapshots {
snapshotMetadata[snapshot.Key] = snapshot
}
type pendingArtifact struct {
request PeerMigrationSourceArtifact
path string
meta domainSnapshotMetadataXML
virtual uint64
}
pending := make([]pendingArtifact, 0, len(request.Artifacts))
for _, requested := range request.Artifacts {
if requested.Role == "root" {
pending = append(pending, pendingArtifact{request: requested, path: rootPath, virtual: rootInfo.Capacity})
continue
}
stored, ok := snapshotMetadata[requested.SnapshotKey]
if !ok {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "migration_snapshot_catalog_changed", "peer migration snapshot catalog changed", nil,
))
}
volume, lookupErr := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, stored.Key),
)
if lookupErr != nil || volume == nil {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "migration_snapshot_unavailable", "peer migration snapshot is unavailable", lookupErr,
))
}
info, inspectErr := validateStandaloneQCOW2Volume(volume, stored.SizeBytes)
path, pathErr := volume.GetPath()
volume.Free()
if inspectErr != nil || pathErr != nil || !filepath.IsAbs(path) {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "migration_snapshot_unavailable", "peer migration snapshot is unavailable", errors.Join(inspectErr, pathErr),
))
}
pending = append(pending, pendingArtifact{request: requested, path: path, meta: stored, virtual: info.Capacity})
}
if len(snapshotMetadata) != len(pending)-1 {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "migration_snapshot_catalog_changed", "peer migration must include every current snapshot", nil,
))
}
sourceState, err := l.inspectMaintenanceMigrationSource(ctx, uuid)
if err != nil {
return PreparePeerMigrationResult{}, err
}
ticketDigest, _ := peermigration.TicketDigest(request.Ticket)
intentArtifacts := make([]peermigration.SourceIntentArtifact, 0, len(pending))
for _, candidate := range pending {
intentArtifacts = append(intentArtifacts, peermigration.SourceIntentArtifact{
ID: candidate.request.ID, Role: candidate.request.Role,
SnapshotKey: candidate.request.SnapshotKey, Path: candidate.path,
})
}
intent := peermigration.SourceIntent{
Version: peermigration.ProtocolVersion, MigrationID: request.MigrationID,
SourceUUID: strings.ToLower(uuid), ExpiresAt: request.ExpiresAt,
TicketSHA256: ticketDigest, SourceWasRunning: sourceState.wasRunning,
SourceAutostartEnabled: sourceState.autostart, Artifacts: intentArtifacts,
}
if err := store.SaveSourceIntent(intent); err != nil {
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "peer_migration_intent_unavailable", "peer migration recovery intent could not be persisted", err,
))
}
if err := l.stopMaintenanceMigrationSource(ctx, uuid, sourceState); err != nil {
if !operation.IsUncertain(err) {
_ = store.RemoveSource(request.MigrationID)
}
return PreparePeerMigrationResult{}, err
}
defer func() {
if resultErr == nil {
return
}
if rollbackErr := l.restoreMaintenanceMigrationSource(context.WithoutCancel(ctx), uuid, sourceState); rollbackErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr, rollbackErr))
return
}
_ = store.RemoveSource(request.MigrationID)
}()
artifacts := make([]peermigration.Artifact, 0, len(pending))
for _, candidate := range pending {
size, digest, hashErr := hashRegularFile(ctx, candidate.path)
if hashErr != nil {
return PreparePeerMigrationResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "peer_migration_artifact_unavailable", "peer migration artifact could not be frozen", hashErr,
))
}
artifact := peermigration.Artifact{
ID: candidate.request.ID, Role: candidate.request.Role,
Consistency: "crash", SizeBytes: size, VirtualSizeBytes: candidate.virtual,
SHA256: digest, CreatedAt: time.Now().Unix(), Path: candidate.path,
}
if candidate.request.Role == "snapshot" {
artifact.SnapshotKey = candidate.meta.Key
artifact.Name = candidate.meta.Name
artifact.Consistency = candidate.meta.Consistency
artifact.CreatedAt = candidate.meta.CreatedAt
}
artifacts = append(artifacts, artifact)
}
stored := peermigration.StoredManifest{
Manifest: peermigration.Manifest{
Version: peermigration.ProtocolVersion, MigrationID: request.MigrationID,
SourceUUID: strings.ToLower(uuid), ExpiresAt: request.ExpiresAt,
Artifacts: artifacts,
},
TicketSHA256: ticketDigest, SourceWasRunning: sourceState.wasRunning,
SourceAutostartEnabled: sourceState.autostart,
}
if err := store.SaveSource(stored); err != nil {
return PreparePeerMigrationResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "peer_migration_manifest_unavailable", "peer migration manifest could not be published", err,
))
}
digest, err := stored.Public().Digest()
if err != nil {
return PreparePeerMigrationResult{}, operation.MarkUncertain(err)
}
publicArtifacts := stored.Public().Artifacts
return PreparePeerMigrationResult{
MigrationID: request.MigrationID, ManifestDigest: digest,
Artifacts: publicArtifacts, SourceStopped: true,
SourceWasRunning: sourceState.wasRunning,
SourceAutostartEnabled: sourceState.autostart,
SourceAutostartDisabled: true,
}, nil
}
func (l *Libvirt) ReceivePeerMigration(ctx context.Context,
request ReceivePeerMigrationRequest) (_ ReceivePeerMigrationResult, resultErr error) {
if err := ValidateReceivePeerMigrationRequest(request); err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
}
if err := l.ensureRestoreTargetAbsent(ctx, request.UUID); err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
}
var targetVPCPort *VPCPortMetadata
var privateNetwork *cloudinit.PrivateNetwork
if request.VPCPort != nil {
if l.vpcRouter == nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil,
))
}
port := *request.VPCPort
if _, err := l.vpcRouter.AuthorizeLocalPort(
port.VPCKey, port.HostRevision, port.HostDigest,
port.CIDR, port.Gateway, port.PrivateIP, port.MAC,
); err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "vpc_host_projection_not_ready", "peer migration target routing does not authorize the VPC port", err,
))
}
conflict, err := l.vpcPortIdentityConflict(ctx, request.UUID, port)
if err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
}
if conflict {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "vpc_port_identity_conflict", "peer migration VPC port is already attached to another managed VM", nil,
))
}
targetVPCPort = &port
privateNetwork = &cloudinit.PrivateNetwork{
MAC: port.MAC, PrivateIP: port.PrivateIP, CIDR: port.CIDR,
Gateway: port.Gateway, InterfaceKey: seedVPCInterfaceKey,
}
}
store, err := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
)
if err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
}
client, err := peermigration.NewClient(store, strings.ToLower(request.SourceIdentitySHA256))
if err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "peer_migration_tls_unavailable", "peer migration source identity is unavailable", err,
))
}
defer client.CloseIdleConnections()
manifest, selectedEndpoint, err := client.FetchManifest(ctx, request.SourceEndpoints, request.MigrationID,
request.Ticket, strings.ToLower(request.ManifestDigest))
if err != nil || !strings.EqualFold(manifest.SourceUUID, request.SourceUUID) {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "peer_migration_manifest_mismatch", "peer migration source manifest does not match", err,
))
}
publicArtifacts := make([]peermigration.Artifact, len(manifest.Artifacts))
copy(publicArtifacts, manifest.Artifacts)
for index := range publicArtifacts {
publicArtifacts[index].Path = ""
}
receipt := peermigration.TargetReceipt{
Version: peermigration.ProtocolVersion, MigrationID: request.MigrationID,
ManifestDigest: strings.ToLower(request.ManifestDigest),
SourceUUID: strings.ToLower(request.SourceUUID), TargetUUID: strings.ToLower(request.UUID),
ExpiresAt: manifest.ExpiresAt, Artifacts: publicArtifacts,
StagedPoweredOff: true, CreatedAt: time.Now().Unix(),
}
if err := store.SaveTargetReceipt(receipt); err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "peer_migration_receipt_unavailable", "peer migration target receipt could not be persisted", err,
))
}
paths, err := client.Download(ctx, selectedEndpoint, request.Ticket, manifest)
if err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "peer_migration_transfer_failed", "peer migration transfer did not complete", err,
))
}
create := request.CreateVMRequest
create.Source = "backup"
create.TemplateID = "peer-" + request.MigrationID
attachment, err := l.resolvePublicNetwork(ctx, create)
if err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
}
mac := vmMACAddress(create.UUID)
conflict, err := l.networkIdentityConflict(ctx, create.UUID, mac, create.PublicNetwork)
if err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
}
if conflict {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "network_identity_conflict", "VM network identity is already in use", nil,
))
}
attachment.MAC = mac
seed, err := cloudinit.BuildISO(cloudinit.Request{
UUID: create.UUID, Name: create.Name, Hostname: create.Hostname,
PreserveGuestCredentials: true,
Debian12NetworkFix: request.Debian12NetworkFix,
Network: cloudinit.Network{
MAC: mac, Public: create.PublicNetwork, InterfaceKey: "public0",
Private: privateNetwork,
},
})
if err != nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
ErrInvalid, "invalid_cloud_init", "peer migration cloud-init request is invalid", err,
))
}
var rootArtifact *peermigration.Artifact
snapshotArtifacts := make([]peermigration.Artifact, 0, len(manifest.Artifacts)-1)
for index := range manifest.Artifacts {
artifact := &manifest.Artifacts[index]
if artifact.Role == "root" {
rootArtifact = artifact
} else {
snapshotArtifacts = append(snapshotArtifacts, *artifact)
}
}
if rootArtifact == nil {
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(errors.New("peer migration root artifact is missing"))
}
root, err := l.createPeerMigrationVolume(ctx, create.UUID, rootVolumeName(create.UUID),
paths[rootArtifact.ID], *rootArtifact, create.DiskGiB<<30)
if err != nil {
return ReceivePeerMigrationResult{}, err
}
volumes := make([]*libvirt.StorageVol, 0, len(snapshotArtifacts))
snapshotMetadata := make([]domainSnapshotMetadataXML, 0, len(snapshotArtifacts))
handedOff := false
defer func() {
if handedOff {
return
}
for _, volume := range volumes {
_ = volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = volume.Free()
}
_ = root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
_ = root.Free()
}()
for _, artifact := range snapshotArtifacts {
volume, volumeErr := l.createPeerMigrationVolume(ctx, create.UUID,
snapshotVolumeName(create.UUID, artifact.SnapshotKey), paths[artifact.ID],
artifact, artifact.VirtualSizeBytes)
if volumeErr != nil {
return ReceivePeerMigrationResult{}, volumeErr
}
volumes = append(volumes, volume)
snapshotMetadata = append(snapshotMetadata, domainSnapshotMetadataXML{
Key: artifact.SnapshotKey, Name: artifact.Name,
Consistency: artifact.Consistency, SizeBytes: artifact.VirtualSizeBytes,
CreatedAt: artifact.CreatedAt,
})
}
// createVMWithPreparedRootOptions owns every volume immediately on entry.
// Relinquish the receive-stage owner before crossing that call boundary so
// no failure path can Delete or Free the same libvirt handle twice.
handedOff = true
vm, err := l.createVMWithPreparedRootOptions(ctx, create, attachment, seed, root,
preparedVMOptions{
Autostart: false, Start: false, NetworkSuspended: request.NetworkSuspended,
VPCPort: targetVPCPort,
Snapshots: snapshotMetadata, SnapshotVolumes: volumes,
})
if err != nil {
return ReceivePeerMigrationResult{}, err
}
// The immutable qcow2 inputs are no longer needed once libvirt owns every
// published volume. Keep the separately persisted target receipt until
// activation so an interrupted Journal commit can still be reconciled.
_ = store.RemoveTarget(request.MigrationID)
return ReceivePeerMigrationResult{
VMMetadata: vm, MigrationID: request.MigrationID,
ManifestDigest: strings.ToLower(request.ManifestDigest),
Artifacts: publicArtifacts, StagedPoweredOff: true,
}, nil
}
func (l *Libvirt) RollbackPeerMigration(ctx context.Context, uuid string,
request RollbackPeerMigrationRequest) (PeerMigrationStateResult, error) {
if err := ValidateRollbackPeerMigrationRequest(request); err != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
}
store, err := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
)
if err != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
}
intent, err := store.LoadSourceIntent(request.MigrationID)
if err != nil || !strings.EqualFold(intent.SourceUUID, uuid) ||
intent.SourceWasRunning != request.SourceWasRunning ||
intent.SourceAutostartEnabled != request.SourceAutostartEnabled {
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "peer_migration_source_mismatch", "peer migration source state does not match", err,
))
}
if err := l.restoreMaintenanceMigrationSource(ctx, uuid, maintenanceMigrationSourceState{
wasRunning: intent.SourceWasRunning, autostart: intent.SourceAutostartEnabled,
}); err != nil {
return PeerMigrationStateResult{}, err
}
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return PeerMigrationStateResult{}, operation.MarkUncertain(err)
}
defer domain.Free()
observedAutostart, autostartErr := domain.GetAutostart()
observedActive, activeErr := domain.IsActive()
state, reason, stateErr := domain.GetState()
if autostartErr != nil || activeErr != nil || stateErr != nil ||
observedAutostart != request.SourceAutostartEnabled ||
observedActive != request.SourceWasRunning {
return PeerMigrationStateResult{}, operation.MarkUncertain(errors.Join(
translateOptionalLibvirtError("confirm peer migration source autostart", autostartErr),
translateOptionalLibvirtError("confirm peer migration source activity", activeErr),
translateOptionalLibvirtError("confirm peer migration source state", stateErr),
NewError(ErrConflict, "peer_migration_source_state_unconfirmed",
"peer migration source did not return to its original state", nil),
))
}
if err := store.RemoveSource(request.MigrationID); err != nil {
return PeerMigrationStateResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "peer_migration_state_cleanup_failed", "peer migration rollback state could not be removed", err,
))
}
name, _ := domain.GetName()
return PeerMigrationStateResult{VMState: VMState{
UUID: strings.ToLower(uuid), Name: name,
State: mapDomainState(state), Reason: reason,
}, Running: observedActive, AutostartEnabled: observedAutostart}, nil
}
func (l *Libvirt) ActivatePeerMigration(ctx context.Context, uuid string,
request ActivatePeerMigrationRequest) (_ PeerMigrationStateResult, resultErr error) {
if err := ValidateActivatePeerMigrationRequest(request); err != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
}
expected, err := expectedMutationState(ctx, operation.ExpectedPeerMigrationState)
if err != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
}
if expected.Enabled != request.TargetAutostartEnabled ||
expected.StartIfSuspended != request.TargetWasRunning {
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "reconciliation_descriptor_mismatch",
"peer migration target state does not match its durable reconciliation descriptor", nil,
))
}
store, err := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
)
if err != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
}
receipt, err := store.LoadTargetReceipt(request.MigrationID)
if err != nil || !strings.EqualFold(receipt.TargetUUID, uuid) {
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "peer_migration_target_mismatch",
"peer migration target receipt does not match", err,
))
}
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
}
defer domain.Free()
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
}
if metadata.VPCPort != nil {
definition, parsedMetadata, parseErr := parseManagedDomain(document)
if parseErr != nil || !managedNetworkLayoutMatches(definition, parsedMetadata) {
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "network_layout_changed", "peer migration target network layout no longer matches its metadata", parseErr,
))
}
if l.vpcRouter == nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil,
))
}
port := metadata.VPCPort.public()
if err := validateVPCPortMetadata(port); err != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
}
_, routeLease, routeErr := l.vpcRouter.AcquireLocalPortLease(
port.VPCKey, port.HostRevision, port.HostDigest,
port.CIDR, port.Gateway, port.PrivateIP, port.MAC,
)
if routeErr != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "vpc_host_projection_not_ready", "peer migration target routing no longer authorizes the exact VPC port", routeErr,
))
}
defer routeLease.Release()
}
persistent, err := domain.IsPersistent()
if err != nil {
return PeerMigrationStateResult{}, operation.MarkNotApplied(translateLibvirtError(
"check peer migration target persistence", err,
))
}
if !persistent {
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
ErrConflict, "persistent_vm_required",
"peer migration target must be persistent", nil,
))
}
mutationStarted := false
defer func() {
if !mutationStarted {
resultErr = operation.MarkNotApplied(resultErr)
}
}()
mutationStarted = true
if err := setServiceAutostart(domain, request.TargetAutostartEnabled); err != nil {
return PeerMigrationStateResult{}, err
}
active, err := domain.IsActive()
if err != nil {
return PeerMigrationStateResult{}, operation.MarkUncertain(translateLibvirtError(
"inspect peer migration target activity", err,
))
}
if request.TargetWasRunning && !active {
if err := l.startManagedDomain(ctx, domain, "start peer migration target"); err != nil {
return PeerMigrationStateResult{}, operation.MarkUncertain(mutationLibvirtError(
"start peer migration target", err,
))
}
} else if !request.TargetWasRunning && active {
if err := requestGracefulDomainShutoff(ctx, domain, 30*time.Second); err != nil {
return PeerMigrationStateResult{}, err
}
}
observedAutostart, autostartErr := domain.GetAutostart()
observedActive, activeErr := domain.IsActive()
if autostartErr != nil || activeErr != nil ||
observedAutostart != request.TargetAutostartEnabled ||
observedActive != request.TargetWasRunning {
return PeerMigrationStateResult{}, operation.MarkUncertain(errors.Join(
translateOptionalLibvirtError("confirm peer migration target autostart", autostartErr),
translateOptionalLibvirtError("confirm peer migration target activity", activeErr),
NewError(ErrConflict, "peer_migration_target_state_unconfirmed",
"peer migration target did not reach the source state", nil),
))
}
state, reason, err := domain.GetState()
if err != nil {
return PeerMigrationStateResult{}, operation.MarkUncertain(translateLibvirtError(
"confirm peer migration target state", err,
))
}
// Activation has its own durable expected-state descriptor, so the receive
// receipt is no longer needed after the target power state is confirmed.
_ = store.RemoveTargetReceipt(request.MigrationID)
name, _ := domain.GetName()
return PeerMigrationStateResult{VMState: VMState{
UUID: strings.ToLower(uuid), Name: name,
State: mapDomainState(state), Reason: reason,
}, Running: observedActive, AutostartEnabled: observedAutostart}, nil
}
func (l *Libvirt) CleanupPeerMigration(ctx context.Context, uuid string,
request CleanupPeerMigrationRequest) (DeleteVMResult, error) {
if err := ValidateCleanupPeerMigrationRequest(request); err != nil {
return DeleteVMResult{}, operation.MarkNotApplied(err)
}
result, err := l.DeleteVM(ctx, uuid, DeleteVMRequest{
OperationID: request.OperationID, RemoveManagedDisks: true,
})
if err != nil {
return DeleteVMResult{}, err
}
store, storeErr := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
)
if storeErr != nil {
return result, nil
}
_ = store.RemoveSource(request.MigrationID)
_ = store.RemoveTarget(request.MigrationID)
_ = store.RemoveTargetReceipt(request.MigrationID)
return result, nil
}
// CleanupPeerMigrationTarget removes only UUID-owned artifacts left by one
// failed target receive. It never destroys or undefines a domain. The exact
// inventory is inspected twice: once by the API before Journal admission and
// again here under the successor's resource leases, closing the TOCTOU gap.
func (l *Libvirt) CleanupPeerMigrationTarget(ctx context.Context, uuid string,
request CleanupPeerMigrationTargetRequest) (_ DeleteVMResult, resultErr error) {
mutationStarted := false
defer func() {
if !mutationStarted {
resultErr = operation.MarkNotApplied(resultErr)
}
}()
if err := ValidateCleanupPeerMigrationTargetRequest(request); err != nil {
return DeleteVMResult{}, err
}
prior := request.PriorOperation
expectedAction := "receive-peer-migration:" + request.MigrationID
if prior.OperationID != request.PriorOperationID || prior.ResourceID != uuid ||
prior.Action != expectedAction || prior.State != operation.StateFailed ||
!prior.EffectMayHaveApplied || prior.SafeToResubmit || prior.ExecutionFenced {
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_cleanup_not_authorized", "peer migration target cleanup is not authorized by the prior receive", nil)
}
domain, err := l.domainByUUID(ctx, uuid)
if err == nil {
_ = domain.Free()
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_domain_present", "peer migration target cleanup refuses to remove an existing domain", nil)
}
if !errors.Is(err, ErrNotFound) {
return DeleteVMResult{}, err
}
plan, err := l.inspectDeleteCleanupSteps(ctx, uuid, nil, nil)
if err != nil {
return DeleteVMResult{}, err
}
if len(plan.steps) == 0 || len(plan.evidence) == 0 ||
!hashEqualsFold(request.CleanupPlanID,
operationInspectionToken(prior, "cleanup", plan.tokenEvidence())) {
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_changed", "peer migration target cleanup plan changed; inspect again", nil)
}
for _, step := range plan.steps {
switch step {
case CleanupStepDeleteManagedVolumes, CleanupStepDeleteSeedVolume,
CleanupStepUndefineNetworkFilter, CleanupStepDeleteNVRAM,
CleanupStepRemoveTrafficState:
default:
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_invalid", "peer migration target cleanup plan contains an unsafe step", nil)
}
}
mutationStarted = true
completionCtx := context.WithoutCancel(ctx)
if err := l.cleanupDeletedVMArtifacts(completionCtx, uuid); err != nil {
return DeleteVMResult{}, err
}
store, err := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
)
if err != nil {
return DeleteVMResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target state cleanup is incomplete", err,
))
}
if err := store.RemoveTarget(request.MigrationID); err != nil {
return DeleteVMResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target staging cleanup is incomplete", err,
))
}
if err := store.RemoveTargetReceipt(request.MigrationID); err != nil {
return DeleteVMResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target receipt cleanup is incomplete", err,
))
}
absent, err := l.deleteArtifactsAbsent(uuid)
if err != nil || !absent {
return DeleteVMResult{}, operation.MarkUncertain(errors.Join(err,
NewError(ErrConflict, "peer_migration_target_cleanup_unconfirmed", "peer migration target artifacts are still present", nil),
))
}
return DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true}, nil
}
func hashEqualsFold(left, right string) bool {
return len(left) == len(right) && strings.EqualFold(left, right)
}
func (l *Libvirt) createPeerMigrationVolume(ctx context.Context, _ string, volumeName,
path string, artifact peermigration.Artifact, finalCapacity uint64) (*libvirt.StorageVol, error) {
if finalCapacity < artifact.VirtualSizeBytes {
return nil, operation.MarkNotApplied(NewError(
ErrInvalid, "disk_smaller_than_migration", "target disk is smaller than the peer migration artifact", nil,
))
}
initialCapacity := finalCapacity
if uint64(artifact.SizeBytes) > initialCapacity {
initialCapacity = uint64(artifact.SizeBytes)
}
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
if err != nil {
return nil, createVMStorageCloneStage.wrap(translateLibvirtError("lookup managed storage pool", err))
}
defer pool.Free()
definition := libvirtxml.StorageVolume{
Name: volumeName,
Capacity: &libvirtxml.StorageVolumeSize{Value: initialCapacity, Unit: "bytes"},
Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"}},
}
document, err := definition.Marshal()
if err != nil {
return nil, createVMStorageCloneStage.wrap(err)
}
volume, err := pool.StorageVolCreateXML(document, 0)
if err != nil {
absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName)
return nil, createVMStorageCloneStage.wrap(deterministicArtifactCreateFailure(
"create peer migration volume", err, inspectErr, !absent,
))
}
stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK)
if err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, volumeName, err, "roll back peer migration volume",
))
}
transfer := &storageStreamTransfer{stream: stream}
if err := volume.Upload(stream, 0, uint64(artifact.SizeBytes), 0); err != nil {
cause := transfer.abort(err)
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, volumeName, cause, "roll back peer migration volume",
))
}
file, err := os.Open(path)
if err != nil {
cause := transfer.abort(err)
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, volumeName, cause, "roll back peer migration volume",
))
}
_, copyErr := io.Copy(&storageStreamWriter{ctx: ctx, transfer: transfer}, file)
closeErr := file.Close()
if copyErr != nil || closeErr != nil {
cause := transfer.abort(errors.Join(copyErr, closeErr))
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, volumeName, cause, "roll back peer migration volume",
))
}
if err := transfer.finish(ctx); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, volumeName, err, "roll back peer migration volume",
))
}
if err := pool.Refresh(0); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, volumeName, err, "roll back peer migration volume",
))
}
info, err := validateStandaloneQCOW2Volume(volume, 0)
if err != nil || info.Capacity != artifact.VirtualSizeBytes {
if err == nil {
err = NewError(ErrConflict, "peer_migration_capacity_mismatch", "peer migration qcow2 capacity does not match its manifest", nil)
}
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, volumeName, err, "roll back peer migration volume",
))
}
if finalCapacity != artifact.VirtualSizeBytes {
if err := establishNewRootVolumeCapacity(volume, finalCapacity); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, volumeName, err, "roll back peer migration volume",
))
}
}
return volume, nil
}
func hashRegularFile(ctx context.Context, path string) (int64, string, error) {
file, err := os.Open(path)
if err != nil {
return 0, "", err
}
defer file.Close()
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() || info.Size() < 1 {
return 0, "", errors.Join(err, errors.New("artifact is not a non-empty regular file"))
}
hash := sha256.New()
buffer := make([]byte, 4<<20)
for {
if err := ctx.Err(); err != nil {
return 0, "", err
}
read, readErr := file.Read(buffer)
if read > 0 {
if _, err := hash.Write(buffer[:read]); err != nil {
return 0, "", err
}
}
if errors.Is(readErr, io.EOF) {
break
}
if readErr != nil {
return 0, "", readErr
}
}
return info.Size(), hex.EncodeToString(hash.Sum(nil)), nil
}