You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
403 lines
14 KiB
Go
403 lines
14 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"git.luoli.live/root/virt/agent/objecttransfer"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
// createMigrationBundle exports the immutable local snapshot set first and
|
|
// stops the source only for the final system-disk capture. A partially
|
|
// completed multi-object upload is deliberately fenced as uncertain: the
|
|
// Agent has no object-store delete credential and must never pretend the
|
|
// already committed artifacts do not exist.
|
|
func (l *Libvirt) createMigrationBundle(ctx context.Context, uuid string,
|
|
request BackupRequest) (_ BackupResult, resultErr error) {
|
|
client := objecttransfer.New()
|
|
defer client.CloseIdleConnections()
|
|
|
|
stored, err := l.validateMigrationBundleSource(ctx, uuid, request)
|
|
if err != nil {
|
|
return BackupResult{}, l.abortRejectedMigrationBundlePlans(
|
|
ctx, client, request, err,
|
|
)
|
|
}
|
|
artifacts := make([]MigrationBundleArtifactResult, 0, len(stored))
|
|
for index, metadata := range stored {
|
|
volume, lookupErr := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged),
|
|
snapshotVolumeName(uuid, metadata.Key),
|
|
)
|
|
if lookupErr != nil || volume == nil {
|
|
return BackupResult{}, l.migrationBundleExportFailure(
|
|
ctx, client, request, index, len(artifacts) > 0,
|
|
NewError(ErrUnavailable, "migration_snapshot_unavailable",
|
|
"migration snapshot volume is unavailable", lookupErr),
|
|
)
|
|
}
|
|
info, inspectErr := validateStandaloneQCOW2Volume(volume, metadata.SizeBytes)
|
|
path, pathErr := volume.GetPath()
|
|
_ = volume.Free()
|
|
if inspectErr != nil || pathErr != nil || !filepath.IsAbs(path) {
|
|
cause := inspectErr
|
|
if cause == nil {
|
|
cause = translateOptionalLibvirtError(
|
|
"resolve migration snapshot path", pathErr,
|
|
)
|
|
}
|
|
return BackupResult{}, l.migrationBundleExportFailure(
|
|
ctx, client, request, index, len(artifacts) > 0, cause,
|
|
)
|
|
}
|
|
upload, uploadErr := client.UploadFile(
|
|
ctx, path, request.MigrationBundle.Snapshots[index].ObjectUpload,
|
|
)
|
|
if uploadErr != nil {
|
|
return BackupResult{}, l.migrationBundleExportFailure(
|
|
ctx, client, request, index+1, len(artifacts) > 0,
|
|
translateObjectTransferError(uploadErr),
|
|
)
|
|
}
|
|
if upload.SizeBytes <= 0 || upload.SHA256 == "" {
|
|
return BackupResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "migration_snapshot_upload_invalid",
|
|
"migration snapshot upload result is invalid", nil,
|
|
))
|
|
}
|
|
artifacts = append(artifacts, MigrationBundleArtifactResult{
|
|
SnapshotKey: metadata.Key, Name: metadata.Name,
|
|
Consistency: metadata.Consistency, SizeBytes: upload.SizeBytes,
|
|
VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256,
|
|
CreatedAt: metadata.CreatedAt,
|
|
})
|
|
}
|
|
|
|
maintenanceState, err := l.prepareMaintenanceMigrationSource(ctx, uuid)
|
|
if err != nil {
|
|
return BackupResult{}, operation.MarkUncertain(err)
|
|
}
|
|
defer func() {
|
|
if resultErr == nil {
|
|
return
|
|
}
|
|
if rollbackErr := l.restoreMaintenanceMigrationSource(
|
|
context.WithoutCancel(ctx), uuid, maintenanceState,
|
|
); rollbackErr != nil {
|
|
resultErr = operation.MarkUncertain(errors.Join(resultErr, rollbackErr))
|
|
}
|
|
}()
|
|
|
|
snapshot, err := l.createStoppedBackupSnapshot(ctx, uuid, SnapshotRequest{
|
|
OperationID: request.OperationID, SnapshotKey: request.BackupKey,
|
|
Name: request.Name, ConsistencyMode: request.ConsistencyMode,
|
|
})
|
|
if err != nil {
|
|
return BackupResult{}, operation.MarkUncertain(err)
|
|
}
|
|
volume, err := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, request.BackupKey),
|
|
)
|
|
if err != nil || volume == nil {
|
|
return BackupResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "backup_snapshot_unavailable",
|
|
"local migration root snapshot is unavailable", err,
|
|
))
|
|
}
|
|
info, inspectErr := validateStandaloneQCOW2Volume(volume, snapshot.SizeBytes)
|
|
path, pathErr := volume.GetPath()
|
|
_ = volume.Free()
|
|
if inspectErr != nil || pathErr != nil || !filepath.IsAbs(path) {
|
|
cause := inspectErr
|
|
if cause == nil {
|
|
cause = translateOptionalLibvirtError(
|
|
"resolve migration root snapshot path", pathErr,
|
|
)
|
|
}
|
|
return BackupResult{}, operation.MarkUncertain(cause)
|
|
}
|
|
upload, err := client.UploadFile(ctx, path, request.ObjectUpload)
|
|
if err != nil {
|
|
return BackupResult{}, operation.MarkUncertain(
|
|
translateObjectTransferError(err),
|
|
)
|
|
}
|
|
if upload.SizeBytes <= 0 || upload.SHA256 == "" {
|
|
return BackupResult{}, operation.MarkUncertain(NewError(
|
|
ErrUnavailable, "backup_upload_invalid",
|
|
"migration root upload result is invalid", nil,
|
|
))
|
|
}
|
|
result := BackupResult{
|
|
BackupKey: request.BackupKey, SizeBytes: upload.SizeBytes,
|
|
VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256,
|
|
Consistency: snapshot.Consistency, SourceStopped: true,
|
|
SourceWasRunning: maintenanceState.wasRunning,
|
|
SourceAutostartEnabled: maintenanceState.autostart,
|
|
SourceAutostartDisabled: true, MigrationBundleVersion: 1,
|
|
SnapshotArtifacts: artifacts,
|
|
}
|
|
cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx)
|
|
_, cleanupErr := l.DeleteSnapshot(cleanupCtx, uuid, request.BackupKey,
|
|
SnapshotMutationRequest{OperationID: request.OperationID})
|
|
cleanupCancel()
|
|
if cleanupErr != nil {
|
|
result.CleanupPending = true
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (l *Libvirt) validateMigrationBundleSource(ctx context.Context, uuid string,
|
|
request BackupRequest) ([]domainSnapshotMetadataXML, error) {
|
|
if _, err := l.validateBackupSource(ctx, uuid, request.BackupKey); err != nil {
|
|
return nil, err
|
|
}
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer domain.Free()
|
|
return validateMigrationBundleSnapshotCatalog(
|
|
domain, request.MigrationBundle.Snapshots,
|
|
)
|
|
}
|
|
|
|
func validateMigrationBundleSnapshotCatalog(domain managedDomainXMLReader,
|
|
requested []MigrationBundleSnapshotUpload) ([]domainSnapshotMetadataXML, error) {
|
|
_, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(metadata.Snapshots) != len(requested) {
|
|
return nil, NewError(ErrConflict, "migration_snapshot_catalog_changed",
|
|
"migration bundle must include every current snapshot", nil)
|
|
}
|
|
byKey := make(map[string]domainSnapshotMetadataXML, len(metadata.Snapshots))
|
|
for _, snapshot := range metadata.Snapshots {
|
|
byKey[snapshot.Key] = snapshot
|
|
}
|
|
ordered := make([]domainSnapshotMetadataXML, 0, len(metadata.Snapshots))
|
|
for _, expected := range requested {
|
|
snapshot, ok := byKey[expected.SnapshotKey]
|
|
if !ok {
|
|
return nil, NewError(ErrConflict, "migration_snapshot_catalog_changed",
|
|
"migration snapshot catalog changed before export", nil)
|
|
}
|
|
ordered = append(ordered, snapshot)
|
|
}
|
|
return ordered, nil
|
|
}
|
|
|
|
func (l *Libvirt) abortRejectedMigrationBundlePlans(ctx context.Context,
|
|
client *objecttransfer.Client, request BackupRequest, rejection error) error {
|
|
errs := []error{rejection}
|
|
if err := client.AbortMultipart(ctx, request.ObjectUpload); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
for _, snapshot := range request.MigrationBundle.Snapshots {
|
|
if err := client.AbortMultipart(ctx, snapshot.ObjectUpload); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
return operation.MarkNotApplied(errors.Join(errs...))
|
|
}
|
|
|
|
// migrationBundleExportFailure aborts plans that have not started. Once an
|
|
// artifact completed, the overall multi-object effect can no longer be called
|
|
// not-applied even when a later upload fails deterministically.
|
|
func (l *Libvirt) migrationBundleExportFailure(ctx context.Context,
|
|
client *objecttransfer.Client, request BackupRequest, nextSnapshot int,
|
|
completed bool, cause error) error {
|
|
errs := []error{cause}
|
|
if err := client.AbortMultipart(ctx, request.ObjectUpload); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
for index := nextSnapshot; index < len(request.MigrationBundle.Snapshots); index++ {
|
|
if err := client.AbortMultipart(
|
|
ctx, request.MigrationBundle.Snapshots[index].ObjectUpload,
|
|
); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
joined := errors.Join(errs...)
|
|
if completed || len(errs) > 1 || operation.IsUncertain(cause) {
|
|
return operation.MarkUncertain(joined)
|
|
}
|
|
return operation.MarkNotApplied(joined)
|
|
}
|
|
|
|
func (l *Libvirt) restoreMigrationBundle(ctx context.Context,
|
|
request RestoreBackupRequest, create CreateVMRequest,
|
|
attachment NetworkAttachment, seed []byte,
|
|
root *libvirt.StorageVol) (_ RestoreBackupResult, resultErr error) {
|
|
volumes := make([]*libvirt.StorageVol, 0,
|
|
len(request.MigrationBundle.Snapshots))
|
|
metadata := make([]domainSnapshotMetadataXML, 0,
|
|
len(request.MigrationBundle.Snapshots))
|
|
artifacts := make([]MigrationBundleArtifactResult, 0,
|
|
len(request.MigrationBundle.Snapshots))
|
|
for _, source := range request.MigrationBundle.Snapshots {
|
|
volume, err := l.createMigrationSnapshotVolume(ctx, request.UUID, source)
|
|
if err != nil {
|
|
for _, prior := range volumes {
|
|
if cleanupErr := prior.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
|
|
resultErr = operation.MarkUncertain(errors.Join(
|
|
err, mutationLibvirtError(
|
|
"roll back restored migration snapshot", cleanupErr,
|
|
),
|
|
))
|
|
}
|
|
_ = prior.Free()
|
|
}
|
|
if cleanupErr := root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil {
|
|
resultErr = operation.MarkUncertain(errors.Join(
|
|
resultErr, err, mutationLibvirtError(
|
|
"roll back restored migration root", cleanupErr,
|
|
),
|
|
))
|
|
}
|
|
_ = root.Free()
|
|
if resultErr != nil {
|
|
return RestoreBackupResult{}, resultErr
|
|
}
|
|
return RestoreBackupResult{}, err
|
|
}
|
|
volumes = append(volumes, volume)
|
|
metadata = append(metadata, domainSnapshotMetadataXML{
|
|
Key: source.SnapshotKey, Name: source.Name,
|
|
Consistency: source.Consistency, SizeBytes: source.VirtualSizeBytes,
|
|
CreatedAt: source.CreatedAt,
|
|
})
|
|
artifacts = append(artifacts, MigrationBundleArtifactResult{
|
|
SnapshotKey: source.SnapshotKey, Name: source.Name,
|
|
Consistency: source.Consistency, SizeBytes: source.SourceSizeBytes,
|
|
VirtualSizeBytes: source.VirtualSizeBytes,
|
|
SHA256: source.SourceSHA256, CreatedAt: source.CreatedAt,
|
|
})
|
|
}
|
|
vm, err := l.createVMWithPreparedRootOptions(
|
|
ctx, create, attachment, seed, root, preparedVMOptions{
|
|
Autostart: false, Start: false,
|
|
NetworkSuspended: request.MigrationBundle.NetworkSuspended,
|
|
Snapshots: metadata, SnapshotVolumes: volumes,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return RestoreBackupResult{}, err
|
|
}
|
|
return RestoreBackupResult{
|
|
VMMetadata: vm, BackupKey: request.BackupKey,
|
|
SHA256: request.SourceSHA256, MigrationBundleVersion: 1,
|
|
SnapshotArtifacts: artifacts, StagedPoweredOff: true,
|
|
}, nil
|
|
}
|
|
|
|
func (l *Libvirt) createMigrationSnapshotVolume(ctx context.Context,
|
|
uuid string, source MigrationBundleSnapshotSource) (*libvirt.StorageVol, error) {
|
|
volumeName := snapshotVolumeName(uuid, source.SnapshotKey)
|
|
initialCapacity := source.VirtualSizeBytes
|
|
if objectBytes := uint64(source.SourceSizeBytes); objectBytes > initialCapacity {
|
|
initialCapacity = objectBytes
|
|
}
|
|
pool, err := l.conn.LookupStoragePoolByName(
|
|
l.storagePoolName(StorageRoleManaged),
|
|
)
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(
|
|
translateLibvirtError("lookup managed storage pool", err),
|
|
)
|
|
}
|
|
defer pool.Free()
|
|
definition := libvirtxml.StorageVolume{
|
|
Name: volumeName,
|
|
Capacity: &libvirtxml.StorageVolumeSize{
|
|
Value: initialCapacity, Unit: "bytes",
|
|
},
|
|
Target: &libvirtxml.StorageVolumeTarget{
|
|
Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"},
|
|
},
|
|
}
|
|
document, err := definition.Marshal()
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(
|
|
fmt.Errorf("marshal restored migration snapshot XML: %w", err),
|
|
)
|
|
}
|
|
volume, err := pool.StorageVolCreateXML(document, 0)
|
|
if err != nil {
|
|
absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName)
|
|
return nil, createVMStorageCloneStage.wrap(
|
|
deterministicArtifactCreateFailure(
|
|
"create restored migration snapshot", err, inspectErr, !absent,
|
|
),
|
|
)
|
|
}
|
|
stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK)
|
|
if err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, volumeName, err,
|
|
"roll back restored migration snapshot",
|
|
))
|
|
}
|
|
transfer := &storageStreamTransfer{stream: stream}
|
|
if err := volume.Upload(stream, 0, uint64(source.SourceSizeBytes), 0); err != nil {
|
|
cause := transfer.abort(err)
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, volumeName, cause,
|
|
"roll back restored migration snapshot",
|
|
))
|
|
}
|
|
client := objecttransfer.New()
|
|
defer client.CloseIdleConnections()
|
|
download, downloadErr := client.Download(
|
|
ctx, source.SourceURL, source.SourceSizeBytes, source.SourceSHA256,
|
|
&storageStreamWriter{ctx: ctx, transfer: transfer},
|
|
)
|
|
if downloadErr != nil {
|
|
cause := transfer.abort(translateObjectTransferError(downloadErr))
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, volumeName, cause,
|
|
"roll back restored migration snapshot",
|
|
))
|
|
}
|
|
if err := transfer.finish(ctx); err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure(
|
|
pool, volume, volumeName, err,
|
|
"roll back restored migration snapshot",
|
|
))
|
|
}
|
|
if download.SizeBytes != source.SourceSizeBytes ||
|
|
!strings.EqualFold(download.SHA256, source.SourceSHA256) {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, volumeName,
|
|
NewError(ErrConflict, "backup_object_mismatch",
|
|
"downloaded migration snapshot identity does not match", nil),
|
|
"roll back restored migration snapshot",
|
|
))
|
|
}
|
|
if err := pool.Refresh(0); err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, volumeName,
|
|
translateLibvirtError("refresh restored migration snapshot", err),
|
|
"roll back restored migration snapshot",
|
|
))
|
|
}
|
|
if _, err := validateStandaloneQCOW2Volume(
|
|
volume, source.VirtualSizeBytes,
|
|
); err != nil {
|
|
return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume(
|
|
pool, volume, volumeName, err,
|
|
"roll back restored migration snapshot",
|
|
))
|
|
}
|
|
return volume, nil
|
|
}
|