You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_migration_bundle.go

403 lines
14 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"git.luoli.live/root/virt/agent/objecttransfer"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
// createMigrationBundle exports the immutable local snapshot set first and
// stops the source only for the final system-disk capture. A partially
// completed multi-object upload is deliberately fenced as uncertain: the
// Agent has no object-store delete credential and must never pretend the
// already committed artifacts do not exist.
func (l *Libvirt) createMigrationBundle(ctx context.Context, uuid string,
request BackupRequest) (_ BackupResult, resultErr error) {
client := objecttransfer.New()
defer client.CloseIdleConnections()
stored, err := l.validateMigrationBundleSource(ctx, uuid, request)
if err != nil {
return BackupResult{}, l.abortRejectedMigrationBundlePlans(
ctx, client, request, err,
)
}
artifacts := make([]MigrationBundleArtifactResult, 0, len(stored))
for index, metadata := range stored {
volume, lookupErr := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged),
snapshotVolumeName(uuid, metadata.Key),
)
if lookupErr != nil || volume == nil {
return BackupResult{}, l.migrationBundleExportFailure(
ctx, client, request, index, len(artifacts) > 0,
NewError(ErrUnavailable, "migration_snapshot_unavailable",
"migration snapshot volume is unavailable", lookupErr),
)
}
info, inspectErr := validateStandaloneQCOW2Volume(volume, metadata.SizeBytes)
path, pathErr := volume.GetPath()
_ = volume.Free()
if inspectErr != nil || pathErr != nil || !filepath.IsAbs(path) {
cause := inspectErr
if cause == nil {
cause = translateOptionalLibvirtError(
"resolve migration snapshot path", pathErr,
)
}
return BackupResult{}, l.migrationBundleExportFailure(
ctx, client, request, index, len(artifacts) > 0, cause,
)
}
upload, uploadErr := client.UploadFile(
ctx, path, request.MigrationBundle.Snapshots[index].ObjectUpload,
)
if uploadErr != nil {
return BackupResult{}, l.migrationBundleExportFailure(
ctx, client, request, index+1, len(artifacts) > 0,
translateObjectTransferError(uploadErr),
)
}
if upload.SizeBytes <= 0 || upload.SHA256 == "" {
return BackupResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "migration_snapshot_upload_invalid",
"migration snapshot upload result is invalid", nil,
))
}
artifacts = append(artifacts, MigrationBundleArtifactResult{
SnapshotKey: metadata.Key, Name: metadata.Name,
Consistency: metadata.Consistency, SizeBytes: upload.SizeBytes,
VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256,
CreatedAt: metadata.CreatedAt,
})
}
maintenanceState, err := l.prepareMaintenanceMigrationSource(ctx, uuid)
if err != nil {
return BackupResult{}, operation.MarkUncertain(err)
}
defer func() {
if resultErr == nil {
return
}
if rollbackErr := l.restoreMaintenanceMigrationSource(
context.WithoutCancel(ctx), uuid, maintenanceState,
); rollbackErr != nil {
resultErr = operation.MarkUncertain(errors.Join(resultErr, rollbackErr))
}
}()
snapshot, err := l.createStoppedBackupSnapshot(ctx, uuid, SnapshotRequest{
OperationID: request.OperationID, SnapshotKey: request.BackupKey,
Name: request.Name, ConsistencyMode: request.ConsistencyMode,
})
if err != nil {
return BackupResult{}, operation.MarkUncertain(err)
}
volume, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, request.BackupKey),
)
if err != nil || volume == nil {
return BackupResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "backup_snapshot_unavailable",
"local migration root snapshot is unavailable", err,
))
}
info, inspectErr := validateStandaloneQCOW2Volume(volume, snapshot.SizeBytes)
path, pathErr := volume.GetPath()
_ = volume.Free()
if inspectErr != nil || pathErr != nil || !filepath.IsAbs(path) {
cause := inspectErr
if cause == nil {
cause = translateOptionalLibvirtError(
"resolve migration root snapshot path", pathErr,
)
}
return BackupResult{}, operation.MarkUncertain(cause)
}
upload, err := client.UploadFile(ctx, path, request.ObjectUpload)
if err != nil {
return BackupResult{}, operation.MarkUncertain(
translateObjectTransferError(err),
)
}
if upload.SizeBytes <= 0 || upload.SHA256 == "" {
return BackupResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "backup_upload_invalid",
"migration root upload result is invalid", nil,
))
}
result := BackupResult{
BackupKey: request.BackupKey, SizeBytes: upload.SizeBytes,
VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256,
Consistency: snapshot.Consistency, SourceStopped: true,
SourceWasRunning: maintenanceState.wasRunning,
SourceAutostartEnabled: maintenanceState.autostart,
SourceAutostartDisabled: true, MigrationBundleVersion: 1,
SnapshotArtifacts: artifacts,
}
cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx)
_, cleanupErr := l.DeleteSnapshot(cleanupCtx, uuid, request.BackupKey,
SnapshotMutationRequest{OperationID: request.OperationID})
cleanupCancel()
if cleanupErr != nil {
result.CleanupPending = true
}
return result, nil
}
func (l *Libvirt) validateMigrationBundleSource(ctx context.Context, uuid string,
request BackupRequest) ([]domainSnapshotMetadataXML, error) {
if _, err := l.validateBackupSource(ctx, uuid, request.BackupKey); err != nil {
return nil, err
}
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return nil, err
}
defer domain.Free()
return validateMigrationBundleSnapshotCatalog(
domain, request.MigrationBundle.Snapshots,
)
}
func validateMigrationBundleSnapshotCatalog(domain managedDomainXMLReader,
requested []MigrationBundleSnapshotUpload) ([]domainSnapshotMetadataXML, error) {
_, metadata, err := requireManagedDomain(domain)
if err != nil {
return nil, err
}
if len(metadata.Snapshots) != len(requested) {
return nil, NewError(ErrConflict, "migration_snapshot_catalog_changed",
"migration bundle must include every current snapshot", nil)
}
byKey := make(map[string]domainSnapshotMetadataXML, len(metadata.Snapshots))
for _, snapshot := range metadata.Snapshots {
byKey[snapshot.Key] = snapshot
}
ordered := make([]domainSnapshotMetadataXML, 0, len(metadata.Snapshots))
for _, expected := range requested {
snapshot, ok := byKey[expected.SnapshotKey]
if !ok {
return nil, NewError(ErrConflict, "migration_snapshot_catalog_changed",
"migration snapshot catalog changed before export", nil)
}
ordered = append(ordered, snapshot)
}
return ordered, nil
}
func (l *Libvirt) abortRejectedMigrationBundlePlans(ctx context.Context,
client *objecttransfer.Client, request BackupRequest, rejection error) error {
errs := []error{rejection}
if err := client.AbortMultipart(ctx, request.ObjectUpload); err != nil {
errs = append(errs, err)
}
for _, snapshot := range request.MigrationBundle.Snapshots {
if err := client.AbortMultipart(ctx, snapshot.ObjectUpload); err != nil {
errs = append(errs, err)
}
}
return operation.MarkNotApplied(errors.Join(errs...))
}
// migrationBundleExportFailure aborts plans that have not started. Once an
// artifact completed, the overall multi-object effect can no longer be called
// not-applied even when a later upload fails deterministically.
func (l *Libvirt) migrationBundleExportFailure(ctx context.Context,
client *objecttransfer.Client, request BackupRequest, nextSnapshot int,
completed bool, cause error) error {
errs := []error{cause}
if err := client.AbortMultipart(ctx, request.ObjectUpload); err != nil {
errs = append(errs, err)
}
for index := nextSnapshot; index < len(request.MigrationBundle.Snapshots); index++ {
if err := client.AbortMultipart(
ctx, request.MigrationBundle.Snapshots[index].ObjectUpload,
); err != nil {
errs = append(errs, err)
}
}
joined := errors.Join(errs...)
if completed || len(errs) > 1 || operation.IsUncertain(cause) {
return operation.MarkUncertain(joined)
}
return operation.MarkNotApplied(joined)
}
func (l *Libvirt) restoreMigrationBundle(ctx context.Context,
request RestoreBackupRequest, create CreateVMRequest,
attachment NetworkAttachment, seed []byte,
root *libvirt.StorageVol) (_ RestoreBackupResult, resultErr error) {
volumes := make([]*libvirt.StorageVol, 0,
len(request.MigrationBundle.Snapshots))
metadata := make([]domainSnapshotMetadataXML, 0,
len(request.MigrationBundle.Snapshots))
artifacts := make([]MigrationBundleArtifactResult, 0,
len(request.MigrationBundle.Snapshots))
for _, source := range request.MigrationBundle.Snapshots {
volume, err := l.createMigrationSnapshotVolume(ctx, request.UUID, source)
if err != nil {
for _, prior := range volumes {
if cleanupErr := prior.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(
err, mutationLibvirtError(
"roll back restored migration snapshot", cleanupErr,
),
))
}
_ = prior.Free()
}
if cleanupErr := root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
resultErr = operation.MarkUncertain(errors.Join(
resultErr, err, mutationLibvirtError(
"roll back restored migration root", cleanupErr,
),
))
}
_ = root.Free()
if resultErr != nil {
return RestoreBackupResult{}, resultErr
}
return RestoreBackupResult{}, err
}
volumes = append(volumes, volume)
metadata = append(metadata, domainSnapshotMetadataXML{
Key: source.SnapshotKey, Name: source.Name,
Consistency: source.Consistency, SizeBytes: source.VirtualSizeBytes,
CreatedAt: source.CreatedAt,
})
artifacts = append(artifacts, MigrationBundleArtifactResult{
SnapshotKey: source.SnapshotKey, Name: source.Name,
Consistency: source.Consistency, SizeBytes: source.SourceSizeBytes,
VirtualSizeBytes: source.VirtualSizeBytes,
SHA256: source.SourceSHA256, CreatedAt: source.CreatedAt,
})
}
vm, err := l.createVMWithPreparedRootOptions(
ctx, create, attachment, seed, root, preparedVMOptions{
Autostart: false, Start: false,
NetworkSuspended: request.MigrationBundle.NetworkSuspended,
Snapshots: metadata, SnapshotVolumes: volumes,
},
)
if err != nil {
return RestoreBackupResult{}, err
}
return RestoreBackupResult{
VMMetadata: vm, BackupKey: request.BackupKey,
SHA256: request.SourceSHA256, MigrationBundleVersion: 1,
SnapshotArtifacts: artifacts, StagedPoweredOff: true,
}, nil
}
func (l *Libvirt) createMigrationSnapshotVolume(ctx context.Context,
uuid string, source MigrationBundleSnapshotSource) (*libvirt.StorageVol, error) {
volumeName := snapshotVolumeName(uuid, source.SnapshotKey)
initialCapacity := source.VirtualSizeBytes
if objectBytes := uint64(source.SourceSizeBytes); objectBytes > initialCapacity {
initialCapacity = objectBytes
}
pool, err := l.conn.LookupStoragePoolByName(
l.storagePoolName(StorageRoleManaged),
)
if err != nil {
return nil, createVMStorageCloneStage.wrap(
translateLibvirtError("lookup managed storage pool", err),
)
}
defer pool.Free()
definition := libvirtxml.StorageVolume{
Name: volumeName,
Capacity: &libvirtxml.StorageVolumeSize{
Value: initialCapacity, Unit: "bytes",
},
Target: &libvirtxml.StorageVolumeTarget{
Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"},
},
}
document, err := definition.Marshal()
if err != nil {
return nil, createVMStorageCloneStage.wrap(
fmt.Errorf("marshal restored migration snapshot XML: %w", err),
)
}
volume, err := pool.StorageVolCreateXML(document, 0)
if err != nil {
absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName)
return nil, createVMStorageCloneStage.wrap(
deterministicArtifactCreateFailure(
"create restored migration snapshot", err, inspectErr, !absent,
),
)
}
stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK)
if err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, volumeName, err,
"roll back restored migration snapshot",
))
}
transfer := &storageStreamTransfer{stream: stream}
if err := volume.Upload(stream, 0, uint64(source.SourceSizeBytes), 0); err != nil {
cause := transfer.abort(err)
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, volumeName, cause,
"roll back restored migration snapshot",
))
}
client := objecttransfer.New()
defer client.CloseIdleConnections()
download, downloadErr := client.Download(
ctx, source.SourceURL, source.SourceSizeBytes, source.SourceSHA256,
&storageStreamWriter{ctx: ctx, transfer: transfer},
)
if downloadErr != nil {
cause := transfer.abort(translateObjectTransferError(downloadErr))
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, volumeName, cause,
"roll back restored migration snapshot",
))
}
if err := transfer.finish(ctx); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
pool, volume, volumeName, err,
"roll back restored migration snapshot",
))
}
if download.SizeBytes != source.SourceSizeBytes ||
!strings.EqualFold(download.SHA256, source.SourceSHA256) {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, volumeName,
NewError(ErrConflict, "backup_object_mismatch",
"downloaded migration snapshot identity does not match", nil),
"roll back restored migration snapshot",
))
}
if err := pool.Refresh(0); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, volumeName,
translateLibvirtError("refresh restored migration snapshot", err),
"roll back restored migration snapshot",
))
}
if _, err := validateStandaloneQCOW2Volume(
volume, source.VirtualSizeBytes,
); err != nil {
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
pool, volume, volumeName, err,
"roll back restored migration snapshot",
))
}
return volume, nil
}