You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
819 lines
33 KiB
Go
819 lines
33 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/cloudinit"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"git.luoli.live/root/virt/agent/peermigration"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
func (l *Libvirt) PreparePeerMigration(ctx context.Context, uuid string,
|
|
request PreparePeerMigrationRequest) (_ PreparePeerMigrationResult, resultErr error) {
|
|
if err := ValidatePreparePeerMigrationRequest(request); err != nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
store, err := peermigration.NewStore(
|
|
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
|
|
l.local.PeerMigrationStagingPath,
|
|
)
|
|
if err != nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "peer_migration_state_unavailable", "peer migration state is unavailable", err,
|
|
))
|
|
}
|
|
if _, err := store.LoadSource(request.MigrationID); err == nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "peer_migration_exists", "peer migration source session already exists", nil,
|
|
))
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "peer_migration_state_unavailable", "peer migration source state cannot be inspected", err,
|
|
))
|
|
}
|
|
if _, err := store.LoadSourceIntent(request.MigrationID); err == nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "peer_migration_exists", "peer migration source intent already exists", nil,
|
|
))
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "peer_migration_state_unavailable", "peer migration source intent cannot be inspected", err,
|
|
))
|
|
}
|
|
if _, _, err := l.validateBackupSource(ctx, uuid, request.MigrationID); err != nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
_, _, metadata, rootPool, rootVolume, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
domain.Free()
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
rootPath, err := rootVolume.GetPath()
|
|
rootInfo, infoErr := validateStandaloneQCOW2Volume(rootVolume, 0)
|
|
rootVolume.Free()
|
|
rootPool.Free()
|
|
domain.Free()
|
|
if err != nil || infoErr != nil || !filepath.IsAbs(rootPath) {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "peer_migration_root_unavailable", "peer migration root disk is unavailable", errors.Join(err, infoErr),
|
|
))
|
|
}
|
|
|
|
snapshotMetadata := make(map[string]domainSnapshotMetadataXML, len(metadata.Snapshots))
|
|
for _, snapshot := range metadata.Snapshots {
|
|
snapshotMetadata[snapshot.Key] = snapshot
|
|
}
|
|
type pendingArtifact struct {
|
|
request PeerMigrationSourceArtifact
|
|
path string
|
|
meta domainSnapshotMetadataXML
|
|
virtual uint64
|
|
}
|
|
pending := make([]pendingArtifact, 0, len(request.Artifacts))
|
|
for _, requested := range request.Artifacts {
|
|
if requested.Role == "root" {
|
|
pending = append(pending, pendingArtifact{request: requested, path: rootPath, virtual: rootInfo.Capacity})
|
|
continue
|
|
}
|
|
stored, ok := snapshotMetadata[requested.SnapshotKey]
|
|
if !ok {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "migration_snapshot_catalog_changed", "peer migration snapshot catalog changed", nil,
|
|
))
|
|
}
|
|
volume, lookupErr := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, stored.Key),
|
|
)
|
|
if lookupErr != nil || volume == nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "migration_snapshot_unavailable", "peer migration snapshot is unavailable", lookupErr,
|
|
))
|
|
}
|
|
info, inspectErr := validateStandaloneQCOW2Volume(volume, stored.SizeBytes)
|
|
path, pathErr := volume.GetPath()
|
|
volume.Free()
|
|
if inspectErr != nil || pathErr != nil || !filepath.IsAbs(path) {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "migration_snapshot_unavailable", "peer migration snapshot is unavailable", errors.Join(inspectErr, pathErr),
|
|
))
|
|
}
|
|
pending = append(pending, pendingArtifact{request: requested, path: path, meta: stored, virtual: info.Capacity})
|
|
}
|
|
if len(snapshotMetadata) != len(pending)-1 {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "migration_snapshot_catalog_changed", "peer migration must include every current snapshot", nil,
|
|
))
|
|
}
|
|
|
|
sourceState, err := l.inspectMaintenanceMigrationSource(ctx, uuid)
|
|
if err != nil {
|
|
return PreparePeerMigrationResult{}, err
|
|
}
|
|
ticketDigest, _ := peermigration.TicketDigest(request.Ticket)
|
|
intentArtifacts := make([]peermigration.SourceIntentArtifact, 0, len(pending))
|
|
for _, candidate := range pending {
|
|
intentArtifacts = append(intentArtifacts, peermigration.SourceIntentArtifact{
|
|
ID: candidate.request.ID, Role: candidate.request.Role,
|
|
SnapshotKey: candidate.request.SnapshotKey, Path: candidate.path,
|
|
})
|
|
}
|
|
intent := peermigration.SourceIntent{
|
|
Version: peermigration.ProtocolVersion, MigrationID: request.MigrationID,
|
|
SourceUUID: strings.ToLower(uuid), ExpiresAt: request.ExpiresAt,
|
|
TicketSHA256: ticketDigest, SourceWasRunning: sourceState.wasRunning,
|
|
SourceAutostartEnabled: sourceState.autostart, Artifacts: intentArtifacts,
|
|
}
|
|
if err := store.SaveSourceIntent(intent); err != nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "peer_migration_intent_unavailable", "peer migration recovery intent could not be persisted", err,
|
|
))
|
|
}
|
|
if err := l.stopMaintenanceMigrationSource(ctx, uuid, sourceState); err != nil {
|
|
if !operation.IsUncertain(err) {
|
|
_ = store.RemoveSource(request.MigrationID)
|
|
}
|
|
return PreparePeerMigrationResult{}, err
|
|
}
|
|
defer func() {
|
|
if resultErr == nil {
|
|
return
|
|
}
|
|
if rollbackErr := l.restoreMaintenanceMigrationSource(context.WithoutCancel(ctx), uuid, sourceState); rollbackErr != nil {
|
|
resultErr = operation.MarkUncertain(errors.Join(resultErr, rollbackErr))
|
|
return
|
|
}
|
|
_ = store.RemoveSource(request.MigrationID)
|
|
}()
|
|
|
|
artifacts := make([]peermigration.Artifact, 0, len(pending))
|
|
for _, candidate := range pending {
|
|
size, digest, hashErr := hashRegularFile(ctx, candidate.path)
|
|
if hashErr != nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "peer_migration_artifact_unavailable", "peer migration artifact could not be frozen", hashErr,
|
|
))
|
|
}
|
|
artifact := peermigration.Artifact{
|
|
ID: candidate.request.ID, Role: candidate.request.Role,
|
|
Consistency: "crash", SizeBytes: size, VirtualSizeBytes: candidate.virtual,
|
|
SHA256: digest, CreatedAt: time.Now().Unix(), Path: candidate.path,
|
|
}
|
|
if candidate.request.Role == "snapshot" {
|
|
artifact.SnapshotKey = candidate.meta.Key
|
|
artifact.Name = candidate.meta.Name
|
|
artifact.Consistency = candidate.meta.Consistency
|
|
artifact.CreatedAt = candidate.meta.CreatedAt
|
|
}
|
|
artifacts = append(artifacts, artifact)
|
|
}
|
|
stored := peermigration.StoredManifest{
|
|
Manifest: peermigration.Manifest{
|
|
Version: peermigration.ProtocolVersion, MigrationID: request.MigrationID,
|
|
SourceUUID: strings.ToLower(uuid), ExpiresAt: request.ExpiresAt,
|
|
Artifacts: artifacts,
|
|
},
|
|
TicketSHA256: ticketDigest, SourceWasRunning: sourceState.wasRunning,
|
|
SourceAutostartEnabled: sourceState.autostart,
|
|
}
|
|
if err := store.SaveSource(stored); err != nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "peer_migration_manifest_unavailable", "peer migration manifest could not be published", err,
|
|
))
|
|
}
|
|
digest, err := stored.Public().Digest()
|
|
if err != nil {
|
|
return PreparePeerMigrationResult{}, operation.MarkUncertain(err)
|
|
}
|
|
publicArtifacts := stored.Public().Artifacts
|
|
return PreparePeerMigrationResult{
|
|
MigrationID: request.MigrationID, ManifestDigest: digest,
|
|
Artifacts: publicArtifacts, SourceStopped: true,
|
|
SourceWasRunning: sourceState.wasRunning,
|
|
SourceAutostartEnabled: sourceState.autostart,
|
|
SourceAutostartDisabled: true,
|
|
}, nil
|
|
}
|
|
|
|
func (l *Libvirt) ReceivePeerMigration(ctx context.Context,
|
|
request ReceivePeerMigrationRequest) (_ ReceivePeerMigrationResult, resultErr error) {
|
|
if err := ValidateReceivePeerMigrationRequest(request); err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
if err := l.ensureRestoreTargetAbsent(ctx, request.UUID); err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
var targetVPCPort *VPCPortMetadata
|
|
var privateNetwork *cloudinit.PrivateNetwork
|
|
if request.VPCPort != nil {
|
|
if l.vpcRouter == nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil,
|
|
))
|
|
}
|
|
port := *request.VPCPort
|
|
if _, err := l.vpcRouter.AuthorizeLocalPort(
|
|
port.VPCKey, port.HostRevision, port.HostDigest,
|
|
port.CIDR, port.Gateway, port.PrivateIP, port.MAC,
|
|
); err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "vpc_host_projection_not_ready", "peer migration target routing does not authorize the VPC port", err,
|
|
))
|
|
}
|
|
conflict, err := l.vpcPortIdentityConflict(ctx, request.UUID, port)
|
|
if err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
if conflict {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "vpc_port_identity_conflict", "peer migration VPC port is already attached to another managed VM", nil,
|
|
))
|
|
}
|
|
targetVPCPort = &port
|
|
privateNetwork = &cloudinit.PrivateNetwork{
|
|
MAC: port.MAC, PrivateIP: port.PrivateIP, CIDR: port.CIDR,
|
|
Gateway: port.Gateway, InterfaceKey: seedVPCInterfaceKey,
|
|
}
|
|
}
|
|
store, err := peermigration.NewStore(
|
|
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
|
|
l.local.PeerMigrationStagingPath,
|
|
)
|
|
if err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
client, err := peermigration.NewClient(store, strings.ToLower(request.SourceIdentitySHA256))
|
|
if err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "peer_migration_tls_unavailable", "peer migration source identity is unavailable", err,
|
|
))
|
|
}
|
|
defer client.CloseIdleConnections()
|
|
manifest, selectedEndpoint, err := client.FetchManifest(ctx, request.SourceEndpoints, request.MigrationID,
|
|
request.Ticket, strings.ToLower(request.ManifestDigest))
|
|
if err != nil || !strings.EqualFold(manifest.SourceUUID, request.SourceUUID) {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "peer_migration_manifest_mismatch", "peer migration source manifest does not match", err,
|
|
))
|
|
}
|
|
publicArtifacts := make([]peermigration.Artifact, len(manifest.Artifacts))
|
|
copy(publicArtifacts, manifest.Artifacts)
|
|
for index := range publicArtifacts {
|
|
publicArtifacts[index].Path = ""
|
|
}
|
|
receipt := peermigration.TargetReceipt{
|
|
Version: peermigration.ProtocolVersion, MigrationID: request.MigrationID,
|
|
ManifestDigest: strings.ToLower(request.ManifestDigest),
|
|
SourceUUID: strings.ToLower(request.SourceUUID), TargetUUID: strings.ToLower(request.UUID),
|
|
ExpiresAt: manifest.ExpiresAt, Artifacts: publicArtifacts,
|
|
StagedPoweredOff: true, CreatedAt: time.Now().Unix(),
|
|
}
|
|
if err := store.SaveTargetReceipt(receipt); err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "peer_migration_receipt_unavailable", "peer migration target receipt could not be persisted", err,
|
|
))
|
|
}
|
|
paths, err := client.Download(ctx, selectedEndpoint, request.Ticket, manifest)
|
|
if err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "peer_migration_transfer_failed", "peer migration transfer did not complete", err,
|
|
))
|
|
}
|
|
|
|
create := request.CreateVMRequest
|
|
create.Source = "backup"
|
|
create.TemplateID = "peer-" + request.MigrationID
|
|
attachment, err := l.resolvePublicNetwork(ctx, create)
|
|
if err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
mac := vmMACAddress(create.UUID)
|
|
conflict, err := l.networkIdentityConflict(ctx, create.UUID, mac, create.PublicNetwork)
|
|
if err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
if conflict {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "network_identity_conflict", "VM network identity is already in use", nil,
|
|
))
|
|
}
|
|
attachment.MAC = mac
|
|
seed, err := cloudinit.BuildISO(cloudinit.Request{
|
|
UUID: create.UUID, Name: create.Name, Hostname: create.Hostname,
|
|
PreserveGuestCredentials: true,
|
|
Debian12NetworkFix: request.Debian12NetworkFix,
|
|
Network: cloudinit.Network{
|
|
MAC: mac, Public: create.PublicNetwork, InterfaceKey: "public0",
|
|
Private: privateNetwork,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError(
|
|
ErrInvalid, "invalid_cloud_init", "peer migration cloud-init request is invalid", err,
|
|
))
|
|
}
|
|
var rootArtifact *peermigration.Artifact
|
|
snapshotArtifacts := make([]peermigration.Artifact, 0, len(manifest.Artifacts)-1)
|
|
for index := range manifest.Artifacts {
|
|
artifact := &manifest.Artifacts[index]
|
|
if artifact.Role == "root" {
|
|
rootArtifact = artifact
|
|
} else {
|
|
snapshotArtifacts = append(snapshotArtifacts, *artifact)
|
|
}
|
|
}
|
|
if rootArtifact == nil {
|
|
return ReceivePeerMigrationResult{}, operation.MarkNotApplied(errors.New("peer migration root artifact is missing"))
|
|
}
|
|
root, err := l.createPeerMigrationVolume(ctx, create.UUID, rootVolumeName(create.UUID),
|
|
paths[rootArtifact.ID], *rootArtifact, create.DiskGiB<<30)
|
|
if err != nil {
|
|
return ReceivePeerMigrationResult{}, err
|
|
}
|
|
volumes := make([]*libvirt.StorageVol, 0, len(snapshotArtifacts))
|
|
snapshotMetadata := make([]domainSnapshotMetadataXML, 0, len(snapshotArtifacts))
|
|
handedOff := false
|
|
defer func() {
|
|
if handedOff {
|
|
return
|
|
}
|
|
for _, volume := range volumes {
|
|
_ = volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
_ = volume.Free()
|
|
}
|
|
_ = root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
|
|
_ = root.Free()
|
|
}()
|
|
for _, artifact := range snapshotArtifacts {
|
|
volume, volumeErr := l.createPeerMigrationVolume(ctx, create.UUID,
|
|
snapshotVolumeName(create.UUID, artifact.SnapshotKey), paths[artifact.ID],
|
|
artifact, artifact.VirtualSizeBytes)
|
|
if volumeErr != nil {
|
|
return ReceivePeerMigrationResult{}, volumeErr
|
|
}
|
|
volumes = append(volumes, volume)
|
|
snapshotMetadata = append(snapshotMetadata, domainSnapshotMetadataXML{
|
|
Key: artifact.SnapshotKey, Name: artifact.Name,
|
|
Consistency: artifact.Consistency, SizeBytes: artifact.VirtualSizeBytes,
|
|
CreatedAt: artifact.CreatedAt,
|
|
})
|
|
}
|
|
// createVMWithPreparedRootOptions owns every volume immediately on entry.
|
|
// Relinquish the receive-stage owner before crossing that call boundary so
|
|
// no failure path can Delete or Free the same libvirt handle twice.
|
|
handedOff = true
|
|
vm, err := l.createVMWithPreparedRootOptions(ctx, create, attachment, seed, root,
|
|
preparedVMOptions{
|
|
Autostart: false, Start: false, NetworkSuspended: request.NetworkSuspended,
|
|
VPCPort: targetVPCPort,
|
|
Snapshots: snapshotMetadata, SnapshotVolumes: volumes,
|
|
})
|
|
if err != nil {
|
|
return ReceivePeerMigrationResult{}, err
|
|
}
|
|
// The immutable qcow2 inputs are no longer needed once libvirt owns every
|
|
// published volume. Keep the separately persisted target receipt until
|
|
// activation so an interrupted Journal commit can still be reconciled.
|
|
_ = store.RemoveTarget(request.MigrationID)
|
|
return ReceivePeerMigrationResult{
|
|
VMMetadata: vm, MigrationID: request.MigrationID,
|
|
ManifestDigest: strings.ToLower(request.ManifestDigest),
|
|
Artifacts: publicArtifacts, StagedPoweredOff: true,
|
|
}, nil
|
|
}
|
|
|
|
func (l *Libvirt) RollbackPeerMigration(ctx context.Context, uuid string,
|
|
request RollbackPeerMigrationRequest) (PeerMigrationStateResult, error) {
|
|
if err := ValidateRollbackPeerMigrationRequest(request); err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
store, err := peermigration.NewStore(
|
|
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
|
|
l.local.PeerMigrationStagingPath,
|
|
)
|
|
if err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
intent, err := store.LoadSourceIntent(request.MigrationID)
|
|
if err != nil || !strings.EqualFold(intent.SourceUUID, uuid) ||
|
|
intent.SourceWasRunning != request.SourceWasRunning ||
|
|
intent.SourceAutostartEnabled != request.SourceAutostartEnabled {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "peer_migration_source_mismatch", "peer migration source state does not match", err,
|
|
))
|
|
}
|
|
if err := l.restoreMaintenanceMigrationSource(ctx, uuid, maintenanceMigrationSourceState{
|
|
wasRunning: intent.SourceWasRunning, autostart: intent.SourceAutostartEnabled,
|
|
}); err != nil {
|
|
return PeerMigrationStateResult{}, err
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkUncertain(err)
|
|
}
|
|
defer domain.Free()
|
|
observedAutostart, autostartErr := domain.GetAutostart()
|
|
observedActive, activeErr := domain.IsActive()
|
|
state, reason, stateErr := domain.GetState()
|
|
if autostartErr != nil || activeErr != nil || stateErr != nil ||
|
|
observedAutostart != request.SourceAutostartEnabled ||
|
|
observedActive != request.SourceWasRunning {
|
|
return PeerMigrationStateResult{}, operation.MarkUncertain(errors.Join(
|
|
translateOptionalLibvirtError("confirm peer migration source autostart", autostartErr),
|
|
translateOptionalLibvirtError("confirm peer migration source activity", activeErr),
|
|
translateOptionalLibvirtError("confirm peer migration source state", stateErr),
|
|
NewError(ErrConflict, "peer_migration_source_state_unconfirmed",
|
|
"peer migration source did not return to its original state", nil),
|
|
))
|
|
}
|
|
if err := store.RemoveSource(request.MigrationID); err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "peer_migration_state_cleanup_failed", "peer migration rollback state could not be removed", err,
|
|
))
|
|
}
|
|
name, _ := domain.GetName()
|
|
return PeerMigrationStateResult{VMState: VMState{
|
|
UUID: strings.ToLower(uuid), Name: name,
|
|
State: mapDomainState(state), Reason: reason,
|
|
}, Running: observedActive, AutostartEnabled: observedAutostart}, nil
|
|
}
|
|
|
|
func (l *Libvirt) ActivatePeerMigration(ctx context.Context, uuid string,
|
|
request ActivatePeerMigrationRequest) (_ PeerMigrationStateResult, resultErr error) {
|
|
if err := ValidateActivatePeerMigrationRequest(request); err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
expected, err := expectedMutationState(ctx, operation.ExpectedPeerMigrationState)
|
|
if err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
if expected.Enabled != request.TargetAutostartEnabled ||
|
|
expected.StartIfSuspended != request.TargetWasRunning {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "reconciliation_descriptor_mismatch",
|
|
"peer migration target state does not match its durable reconciliation descriptor", nil,
|
|
))
|
|
}
|
|
store, err := peermigration.NewStore(
|
|
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
|
|
l.local.PeerMigrationStagingPath,
|
|
)
|
|
if err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
receipt, err := store.LoadTargetReceipt(request.MigrationID)
|
|
if err != nil || !strings.EqualFold(receipt.TargetUUID, uuid) {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "peer_migration_target_mismatch",
|
|
"peer migration target receipt does not match", err,
|
|
))
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
defer domain.Free()
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
if metadata.VPCPort != nil {
|
|
definition, parsedMetadata, parseErr := parseManagedDomain(document)
|
|
if parseErr != nil || !managedNetworkLayoutMatches(definition, parsedMetadata) {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "network_layout_changed", "peer migration target network layout no longer matches its metadata", parseErr,
|
|
))
|
|
}
|
|
if l.vpcRouter == nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
|
|
ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil,
|
|
))
|
|
}
|
|
port := metadata.VPCPort.public()
|
|
if err := validateVPCPortMetadata(port); err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
_, routeLease, routeErr := l.vpcRouter.AcquireLocalPortLease(
|
|
port.VPCKey, port.HostRevision, port.HostDigest,
|
|
port.CIDR, port.Gateway, port.PrivateIP, port.MAC,
|
|
)
|
|
if routeErr != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "vpc_host_projection_not_ready", "peer migration target routing no longer authorizes the exact VPC port", routeErr,
|
|
))
|
|
}
|
|
defer routeLease.Release()
|
|
}
|
|
persistent, err := domain.IsPersistent()
|
|
if err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(translateLibvirtError(
|
|
"check peer migration target persistence", err,
|
|
))
|
|
}
|
|
if !persistent {
|
|
return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError(
|
|
ErrConflict, "persistent_vm_required",
|
|
"peer migration target must be persistent", nil,
|
|
))
|
|
}
|
|
mutationStarted := false
|
|
defer func() {
|
|
if !mutationStarted {
|
|
resultErr = operation.MarkNotApplied(resultErr)
|
|
}
|
|
}()
|
|
mutationStarted = true
|
|
if err := setServiceAutostart(domain, request.TargetAutostartEnabled); err != nil {
|
|
return PeerMigrationStateResult{}, err
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkUncertain(translateLibvirtError(
|
|
"inspect peer migration target activity", err,
|
|
))
|
|
}
|
|
if request.TargetWasRunning && !active {
|
|
if err := l.startManagedDomain(ctx, domain, "start peer migration target"); err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkUncertain(mutationLibvirtError(
|
|
"start peer migration target", err,
|
|
))
|
|
}
|
|
} else if !request.TargetWasRunning && active {
|
|
if err := requestGracefulDomainShutoff(ctx, domain, 30*time.Second); err != nil {
|
|
return PeerMigrationStateResult{}, err
|
|
}
|
|
}
|
|
observedAutostart, autostartErr := domain.GetAutostart()
|
|
observedActive, activeErr := domain.IsActive()
|
|
if autostartErr != nil || activeErr != nil ||
|
|
observedAutostart != request.TargetAutostartEnabled ||
|
|
observedActive != request.TargetWasRunning {
|
|
return PeerMigrationStateResult{}, operation.MarkUncertain(errors.Join(
|
|
translateOptionalLibvirtError("confirm peer migration target autostart", autostartErr),
|
|
translateOptionalLibvirtError("confirm peer migration target activity", activeErr),
|
|
NewError(ErrConflict, "peer_migration_target_state_unconfirmed",
|
|
"peer migration target did not reach the source state", nil),
|
|
))
|
|
}
|
|
state, reason, err := domain.GetState()
|
|
if err != nil {
|
|
return PeerMigrationStateResult{}, operation.MarkUncertain(translateLibvirtError(
|
|
"confirm peer migration target state", err,
|
|
))
|
|
}
|
|
// Activation has its own durable expected-state descriptor, so the receive
|
|
// receipt is no longer needed after the target power state is confirmed.
|
|
_ = store.RemoveTargetReceipt(request.MigrationID)
|
|
name, _ := domain.GetName()
|
|
return PeerMigrationStateResult{VMState: VMState{
|
|
UUID: strings.ToLower(uuid), Name: name,
|
|
State: mapDomainState(state), Reason: reason,
|
|
}, Running: observedActive, AutostartEnabled: observedAutostart}, nil
|
|
}
|
|
|
|
func (l *Libvirt) CleanupPeerMigration(ctx context.Context, uuid string,
|
|
request CleanupPeerMigrationRequest) (DeleteVMResult, error) {
|
|
if err := ValidateCleanupPeerMigrationRequest(request); err != nil {
|
|
return DeleteVMResult{}, operation.MarkNotApplied(err)
|
|
}
|
|
result, err := l.DeleteVM(ctx, uuid, DeleteVMRequest{
|
|
OperationID: request.OperationID, RemoveManagedDisks: true,
|
|
})
|
|
if err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
store, storeErr := peermigration.NewStore(
|
|
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
|
|
l.local.PeerMigrationStagingPath,
|
|
)
|
|
if storeErr != nil {
|
|
return result, nil
|
|
}
|
|
_ = store.RemoveSource(request.MigrationID)
|
|
_ = store.RemoveTarget(request.MigrationID)
|
|
_ = store.RemoveTargetReceipt(request.MigrationID)
|
|
return result, nil
|
|
}
|
|
|
|
// CleanupPeerMigrationTarget removes only UUID-owned artifacts left by one
|
|
// failed target receive. It never destroys or undefines a domain. The exact
|
|
// inventory is inspected twice: once by the API before Journal admission and
|
|
// again here under the successor's resource leases, closing the TOCTOU gap.
|
|
func (l *Libvirt) CleanupPeerMigrationTarget(ctx context.Context, uuid string,
|
|
request CleanupPeerMigrationTargetRequest) (_ DeleteVMResult, resultErr error) {
|
|
mutationStarted := false
|
|
defer func() {
|
|
if !mutationStarted {
|
|
resultErr = operation.MarkNotApplied(resultErr)
|
|
}
|
|
}()
|
|
if err := ValidateCleanupPeerMigrationTargetRequest(request); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
prior := request.PriorOperation
|
|
expectedAction := "receive-peer-migration:" + request.MigrationID
|
|
if prior.OperationID != request.PriorOperationID || prior.ResourceID != uuid ||
|
|
prior.Action != expectedAction || prior.State != operation.StateFailed ||
|
|
!prior.EffectMayHaveApplied || prior.SafeToResubmit || prior.ExecutionFenced {
|
|
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_cleanup_not_authorized", "peer migration target cleanup is not authorized by the prior receive", nil)
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err == nil {
|
|
_ = domain.Free()
|
|
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_domain_present", "peer migration target cleanup refuses to remove an existing domain", nil)
|
|
}
|
|
if !errors.Is(err, ErrNotFound) {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
plan, err := l.inspectDeleteCleanupSteps(ctx, uuid, nil, nil)
|
|
if err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
if len(plan.steps) == 0 || len(plan.evidence) == 0 ||
|
|
!hashEqualsFold(request.CleanupPlanID,
|
|
operationInspectionToken(prior, "cleanup", plan.tokenEvidence())) {
|
|
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_changed", "peer migration target cleanup plan changed; inspect again", nil)
|
|
}
|
|
for _, step := range plan.steps {
|
|
switch step {
|
|
case CleanupStepDeleteManagedVolumes, CleanupStepDeleteSeedVolume,
|
|
CleanupStepUndefineNetworkFilter, CleanupStepDeleteNVRAM,
|
|
CleanupStepRemoveTrafficState:
|
|
default:
|
|
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_invalid", "peer migration target cleanup plan contains an unsafe step", nil)
|
|
}
|
|
}
|
|
mutationStarted = true
|
|
completionCtx := context.WithoutCancel(ctx)
|
|
if err := l.cleanupDeletedVMArtifacts(completionCtx, uuid); err != nil {
|
|
return DeleteVMResult{}, err
|
|
}
|
|
store, err := peermigration.NewStore(
|
|
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
|
|
l.local.PeerMigrationStagingPath,
|
|
)
|
|
if err != nil {
|
|
return DeleteVMResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target state cleanup is incomplete", err,
|
|
))
|
|
}
|
|
if err := store.RemoveTarget(request.MigrationID); err != nil {
|
|
return DeleteVMResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target staging cleanup is incomplete", err,
|
|
))
|
|
}
|
|
if err := store.RemoveTargetReceipt(request.MigrationID); err != nil {
|
|
return DeleteVMResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target receipt cleanup is incomplete", err,
|
|
))
|
|
}
|
|
absent, err := l.deleteArtifactsAbsent(uuid)
|
|
if err != nil || !absent {
|
|
return DeleteVMResult{}, operation.MarkUncertain(errors.Join(err,
|
|
NewError(ErrConflict, "peer_migration_target_cleanup_unconfirmed", "peer migration target artifacts are still present", nil),
|
|
))
|
|
}
|
|
return DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true}, nil
|
|
}
|
|
|
|
func hashEqualsFold(left, right string) bool {
|
|
return len(left) == len(right) && strings.EqualFold(left, right)
|
|
}
|
|
|
|
func (l *Libvirt) createPeerMigrationVolume(ctx context.Context, _ string, volumeName,
|
|
path string, artifact peermigration.Artifact, finalCapacity uint64) (*libvirt.StorageVol, error) {
|
|
if finalCapacity < artifact.VirtualSizeBytes {
|
|
return nil, operation.MarkNotApplied(NewError(
|
|
ErrInvalid, "disk_smaller_than_migration", "target disk is smaller than the peer migration artifact", nil,
|
|
))
|
|
}
|
|
initialCapacity := finalCapacity
|
|
if uint64(artifact.SizeBytes) > initialCapacity {
|
|
initialCapacity = uint64(artifact.SizeBytes)
|
|
}
|
|
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(translateLibvirtError("lookup managed storage pool", err))
|
|
}
|
|
defer pool.Free()
|
|
definition := libvirtxml.StorageVolume{
|
|
Name: volumeName,
|
|
Capacity: &libvirtxml.StorageVolumeSize{Value: initialCapacity, Unit: "bytes"},
|
|
Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"}},
|
|
}
|
|
document, err := definition.Marshal()
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(err)
|
|
}
|
|
volume, err := pool.StorageVolCreateXML(document, 0)
|
|
if err != nil {
|
|
absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName)
|
|
return nil, createVMStorageCloneStage.wrap(deterministicArtifactCreateFailure(
|
|
"create peer migration volume", err, inspectErr, !absent,
|
|
))
|
|
}
|
|
stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK)
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, volumeName, err, "roll back peer migration volume",
|
|
))
|
|
}
|
|
transfer := &storageStreamTransfer{stream: stream}
|
|
if err := volume.Upload(stream, 0, uint64(artifact.SizeBytes), 0); err != nil {
|
|
cause := transfer.abort(err)
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, volumeName, cause, "roll back peer migration volume",
|
|
))
|
|
}
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
cause := transfer.abort(err)
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, volumeName, cause, "roll back peer migration volume",
|
|
))
|
|
}
|
|
_, copyErr := io.Copy(&storageStreamWriter{ctx: ctx, transfer: transfer}, file)
|
|
closeErr := file.Close()
|
|
if copyErr != nil || closeErr != nil {
|
|
cause := transfer.abort(errors.Join(copyErr, closeErr))
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, volumeName, cause, "roll back peer migration volume",
|
|
))
|
|
}
|
|
if err := transfer.finish(ctx); err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, volumeName, err, "roll back peer migration volume",
|
|
))
|
|
}
|
|
if err := pool.Refresh(0); err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, volumeName, err, "roll back peer migration volume",
|
|
))
|
|
}
|
|
info, err := validateStandaloneQCOW2Volume(volume, 0)
|
|
if err != nil || info.Capacity != artifact.VirtualSizeBytes {
|
|
if err == nil {
|
|
err = NewError(ErrConflict, "peer_migration_capacity_mismatch", "peer migration qcow2 capacity does not match its manifest", nil)
|
|
}
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, volumeName, err, "roll back peer migration volume",
|
|
))
|
|
}
|
|
if finalCapacity != artifact.VirtualSizeBytes {
|
|
if err := establishNewRootVolumeCapacity(volume, finalCapacity); err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, volumeName, err, "roll back peer migration volume",
|
|
))
|
|
}
|
|
}
|
|
return volume, nil
|
|
}
|
|
|
|
func hashRegularFile(ctx context.Context, path string) (int64, string, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return 0, "", err
|
|
}
|
|
defer file.Close()
|
|
info, err := file.Stat()
|
|
if err != nil || !info.Mode().IsRegular() || info.Size() < 1 {
|
|
return 0, "", errors.Join(err, errors.New("artifact is not a non-empty regular file"))
|
|
}
|
|
hash := sha256.New()
|
|
buffer := make([]byte, 4<<20)
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
return 0, "", err
|
|
}
|
|
read, readErr := file.Read(buffer)
|
|
if read > 0 {
|
|
if _, err := hash.Write(buffer[:read]); err != nil {
|
|
return 0, "", err
|
|
}
|
|
}
|
|
if errors.Is(readErr, io.EOF) {
|
|
break
|
|
}
|
|
if readErr != nil {
|
|
return 0, "", readErr
|
|
}
|
|
}
|
|
return info.Size(), hex.EncodeToString(hash.Sum(nil)), nil
|
|
}
|