//go:build libvirt package hypervisor import ( "context" "crypto/sha256" "encoding/hex" "errors" "io" "os" "path/filepath" "strings" "time" "git.luoli.live/root/virt/agent/cloudinit" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/peermigration" "libvirt.org/go/libvirt" "libvirt.org/go/libvirtxml" ) func (l *Libvirt) PreparePeerMigration(ctx context.Context, uuid string, request PreparePeerMigrationRequest) (_ PreparePeerMigrationResult, resultErr error) { if err := ValidatePreparePeerMigrationRequest(request); err != nil { return PreparePeerMigrationResult{}, operation.MarkNotApplied(err) } store, err := peermigration.NewStore( l.local.PeerMigrationStatePath, l.local.StoragePathRoots, l.local.PeerMigrationStagingPath, ) if err != nil { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "peer_migration_state_unavailable", "peer migration state is unavailable", err, )) } if _, err := store.LoadSource(request.MigrationID); err == nil { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrConflict, "peer_migration_exists", "peer migration source session already exists", nil, )) } else if !errors.Is(err, os.ErrNotExist) { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "peer_migration_state_unavailable", "peer migration source state cannot be inspected", err, )) } if _, err := store.LoadSourceIntent(request.MigrationID); err == nil { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrConflict, "peer_migration_exists", "peer migration source intent already exists", nil, )) } else if !errors.Is(err, os.ErrNotExist) { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "peer_migration_state_unavailable", "peer migration source intent cannot be inspected", err, )) } if _, _, err := l.validateBackupSource(ctx, uuid, request.MigrationID); err != nil { return PreparePeerMigrationResult{}, operation.MarkNotApplied(err) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return PreparePeerMigrationResult{}, operation.MarkNotApplied(err) } _, _, metadata, rootPool, rootVolume, err := l.managedRootContext(domain) if err != nil { domain.Free() return PreparePeerMigrationResult{}, operation.MarkNotApplied(err) } rootPath, err := rootVolume.GetPath() rootInfo, infoErr := validateStandaloneQCOW2Volume(rootVolume, 0) rootVolume.Free() rootPool.Free() domain.Free() if err != nil || infoErr != nil || !filepath.IsAbs(rootPath) { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "peer_migration_root_unavailable", "peer migration root disk is unavailable", errors.Join(err, infoErr), )) } snapshotMetadata := make(map[string]domainSnapshotMetadataXML, len(metadata.Snapshots)) for _, snapshot := range metadata.Snapshots { snapshotMetadata[snapshot.Key] = snapshot } type pendingArtifact struct { request PeerMigrationSourceArtifact path string meta domainSnapshotMetadataXML virtual uint64 } pending := make([]pendingArtifact, 0, len(request.Artifacts)) for _, requested := range request.Artifacts { if requested.Role == "root" { pending = append(pending, pendingArtifact{request: requested, path: rootPath, virtual: rootInfo.Capacity}) continue } stored, ok := snapshotMetadata[requested.SnapshotKey] if !ok { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrConflict, "migration_snapshot_catalog_changed", "peer migration snapshot catalog changed", nil, )) } volume, lookupErr := l.lookupOptionalManagedVolume( l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, stored.Key), ) if lookupErr != nil || volume == nil { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "migration_snapshot_unavailable", "peer migration snapshot is unavailable", lookupErr, )) } info, inspectErr := validateStandaloneQCOW2Volume(volume, stored.SizeBytes) path, pathErr := volume.GetPath() volume.Free() if inspectErr != nil || pathErr != nil || !filepath.IsAbs(path) { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "migration_snapshot_unavailable", "peer migration snapshot is unavailable", errors.Join(inspectErr, pathErr), )) } pending = append(pending, pendingArtifact{request: requested, path: path, meta: stored, virtual: info.Capacity}) } if len(snapshotMetadata) != len(pending)-1 { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrConflict, "migration_snapshot_catalog_changed", "peer migration must include every current snapshot", nil, )) } sourceState, err := l.inspectMaintenanceMigrationSource(ctx, uuid) if err != nil { return PreparePeerMigrationResult{}, err } ticketDigest, _ := peermigration.TicketDigest(request.Ticket) intentArtifacts := make([]peermigration.SourceIntentArtifact, 0, len(pending)) for _, candidate := range pending { intentArtifacts = append(intentArtifacts, peermigration.SourceIntentArtifact{ ID: candidate.request.ID, Role: candidate.request.Role, SnapshotKey: candidate.request.SnapshotKey, Path: candidate.path, }) } intent := peermigration.SourceIntent{ Version: peermigration.ProtocolVersion, MigrationID: request.MigrationID, SourceUUID: strings.ToLower(uuid), ExpiresAt: request.ExpiresAt, TicketSHA256: ticketDigest, SourceWasRunning: sourceState.wasRunning, SourceAutostartEnabled: sourceState.autostart, Artifacts: intentArtifacts, } if err := store.SaveSourceIntent(intent); err != nil { return PreparePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "peer_migration_intent_unavailable", "peer migration recovery intent could not be persisted", err, )) } if err := l.stopMaintenanceMigrationSource(ctx, uuid, sourceState); err != nil { if !operation.IsUncertain(err) { _ = store.RemoveSource(request.MigrationID) } return PreparePeerMigrationResult{}, err } defer func() { if resultErr == nil { return } if rollbackErr := l.restoreMaintenanceMigrationSource(context.WithoutCancel(ctx), uuid, sourceState); rollbackErr != nil { resultErr = operation.MarkUncertain(errors.Join(resultErr, rollbackErr)) return } _ = store.RemoveSource(request.MigrationID) }() artifacts := make([]peermigration.Artifact, 0, len(pending)) for _, candidate := range pending { size, digest, hashErr := hashRegularFile(ctx, candidate.path) if hashErr != nil { return PreparePeerMigrationResult{}, operation.MarkUncertain(NewError( ErrUnavailable, "peer_migration_artifact_unavailable", "peer migration artifact could not be frozen", hashErr, )) } artifact := peermigration.Artifact{ ID: candidate.request.ID, Role: candidate.request.Role, Consistency: "crash", SizeBytes: size, VirtualSizeBytes: candidate.virtual, SHA256: digest, CreatedAt: time.Now().Unix(), Path: candidate.path, } if candidate.request.Role == "snapshot" { artifact.SnapshotKey = candidate.meta.Key artifact.Name = candidate.meta.Name artifact.Consistency = candidate.meta.Consistency artifact.CreatedAt = candidate.meta.CreatedAt } artifacts = append(artifacts, artifact) } stored := peermigration.StoredManifest{ Manifest: peermigration.Manifest{ Version: peermigration.ProtocolVersion, MigrationID: request.MigrationID, SourceUUID: strings.ToLower(uuid), ExpiresAt: request.ExpiresAt, Artifacts: artifacts, }, TicketSHA256: ticketDigest, SourceWasRunning: sourceState.wasRunning, SourceAutostartEnabled: sourceState.autostart, } if err := store.SaveSource(stored); err != nil { return PreparePeerMigrationResult{}, operation.MarkUncertain(NewError( ErrUnavailable, "peer_migration_manifest_unavailable", "peer migration manifest could not be published", err, )) } digest, err := stored.Public().Digest() if err != nil { return PreparePeerMigrationResult{}, operation.MarkUncertain(err) } publicArtifacts := stored.Public().Artifacts return PreparePeerMigrationResult{ MigrationID: request.MigrationID, ManifestDigest: digest, Artifacts: publicArtifacts, SourceStopped: true, SourceWasRunning: sourceState.wasRunning, SourceAutostartEnabled: sourceState.autostart, SourceAutostartDisabled: true, }, nil } func (l *Libvirt) ReceivePeerMigration(ctx context.Context, request ReceivePeerMigrationRequest) (_ ReceivePeerMigrationResult, resultErr error) { if err := ValidateReceivePeerMigrationRequest(request); err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err) } if err := l.ensureRestoreTargetAbsent(ctx, request.UUID); err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err) } var targetVPCPort *VPCPortMetadata var privateNetwork *cloudinit.PrivateNetwork if request.VPCPort != nil { if l.vpcRouter == nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil, )) } port := *request.VPCPort if _, err := l.vpcRouter.AuthorizeLocalPort( port.VPCKey, port.HostRevision, port.HostDigest, port.CIDR, port.Gateway, port.PrivateIP, port.MAC, ); err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrConflict, "vpc_host_projection_not_ready", "peer migration target routing does not authorize the VPC port", err, )) } conflict, err := l.vpcPortIdentityConflict(ctx, request.UUID, port) if err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err) } if conflict { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrConflict, "vpc_port_identity_conflict", "peer migration VPC port is already attached to another managed VM", nil, )) } targetVPCPort = &port privateNetwork = &cloudinit.PrivateNetwork{ MAC: port.MAC, PrivateIP: port.PrivateIP, CIDR: port.CIDR, Gateway: port.Gateway, InterfaceKey: seedVPCInterfaceKey, } } store, err := peermigration.NewStore( l.local.PeerMigrationStatePath, l.local.StoragePathRoots, l.local.PeerMigrationStagingPath, ) if err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err) } client, err := peermigration.NewClient(store, strings.ToLower(request.SourceIdentitySHA256)) if err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "peer_migration_tls_unavailable", "peer migration source identity is unavailable", err, )) } defer client.CloseIdleConnections() manifest, selectedEndpoint, err := client.FetchManifest(ctx, request.SourceEndpoints, request.MigrationID, request.Ticket, strings.ToLower(request.ManifestDigest)) if err != nil || !strings.EqualFold(manifest.SourceUUID, request.SourceUUID) { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrConflict, "peer_migration_manifest_mismatch", "peer migration source manifest does not match", err, )) } publicArtifacts := make([]peermigration.Artifact, len(manifest.Artifacts)) copy(publicArtifacts, manifest.Artifacts) for index := range publicArtifacts { publicArtifacts[index].Path = "" } receipt := peermigration.TargetReceipt{ Version: peermigration.ProtocolVersion, MigrationID: request.MigrationID, ManifestDigest: strings.ToLower(request.ManifestDigest), SourceUUID: strings.ToLower(request.SourceUUID), TargetUUID: strings.ToLower(request.UUID), ExpiresAt: manifest.ExpiresAt, Artifacts: publicArtifacts, StagedPoweredOff: true, CreatedAt: time.Now().Unix(), } if err := store.SaveTargetReceipt(receipt); err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "peer_migration_receipt_unavailable", "peer migration target receipt could not be persisted", err, )) } paths, err := client.Download(ctx, selectedEndpoint, request.Ticket, manifest) if err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "peer_migration_transfer_failed", "peer migration transfer did not complete", err, )) } create := request.CreateVMRequest create.Source = "backup" create.TemplateID = "peer-" + request.MigrationID attachment, err := l.resolvePublicNetwork(ctx, create) if err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err) } mac := vmMACAddress(create.UUID) conflict, err := l.networkIdentityConflict(ctx, create.UUID, mac, create.PublicNetwork) if err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(err) } if conflict { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrConflict, "network_identity_conflict", "VM network identity is already in use", nil, )) } attachment.MAC = mac seed, err := cloudinit.BuildISO(cloudinit.Request{ UUID: create.UUID, Name: create.Name, Hostname: create.Hostname, PreserveGuestCredentials: true, Debian12NetworkFix: request.Debian12NetworkFix, Network: cloudinit.Network{ MAC: mac, Public: create.PublicNetwork, InterfaceKey: "public0", Private: privateNetwork, }, }) if err != nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(NewError( ErrInvalid, "invalid_cloud_init", "peer migration cloud-init request is invalid", err, )) } var rootArtifact *peermigration.Artifact snapshotArtifacts := make([]peermigration.Artifact, 0, len(manifest.Artifacts)-1) for index := range manifest.Artifacts { artifact := &manifest.Artifacts[index] if artifact.Role == "root" { rootArtifact = artifact } else { snapshotArtifacts = append(snapshotArtifacts, *artifact) } } if rootArtifact == nil { return ReceivePeerMigrationResult{}, operation.MarkNotApplied(errors.New("peer migration root artifact is missing")) } root, err := l.createPeerMigrationVolume(ctx, create.UUID, rootVolumeName(create.UUID), paths[rootArtifact.ID], *rootArtifact, create.DiskGiB<<30) if err != nil { return ReceivePeerMigrationResult{}, err } volumes := make([]*libvirt.StorageVol, 0, len(snapshotArtifacts)) snapshotMetadata := make([]domainSnapshotMetadataXML, 0, len(snapshotArtifacts)) handedOff := false defer func() { if handedOff { return } for _, volume := range volumes { _ = volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) _ = volume.Free() } _ = root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) _ = root.Free() }() for _, artifact := range snapshotArtifacts { volume, volumeErr := l.createPeerMigrationVolume(ctx, create.UUID, snapshotVolumeName(create.UUID, artifact.SnapshotKey), paths[artifact.ID], artifact, artifact.VirtualSizeBytes) if volumeErr != nil { return ReceivePeerMigrationResult{}, volumeErr } volumes = append(volumes, volume) snapshotMetadata = append(snapshotMetadata, domainSnapshotMetadataXML{ Key: artifact.SnapshotKey, Name: artifact.Name, Consistency: artifact.Consistency, SizeBytes: artifact.VirtualSizeBytes, CreatedAt: artifact.CreatedAt, }) } // createVMWithPreparedRootOptions owns every volume immediately on entry. // Relinquish the receive-stage owner before crossing that call boundary so // no failure path can Delete or Free the same libvirt handle twice. handedOff = true vm, err := l.createVMWithPreparedRootOptions(ctx, create, attachment, seed, root, preparedVMOptions{ Autostart: false, Start: false, NetworkSuspended: request.NetworkSuspended, VPCPort: targetVPCPort, Snapshots: snapshotMetadata, SnapshotVolumes: volumes, }) if err != nil { return ReceivePeerMigrationResult{}, err } // The immutable qcow2 inputs are no longer needed once libvirt owns every // published volume. Keep the separately persisted target receipt until // activation so an interrupted Journal commit can still be reconciled. _ = store.RemoveTarget(request.MigrationID) return ReceivePeerMigrationResult{ VMMetadata: vm, MigrationID: request.MigrationID, ManifestDigest: strings.ToLower(request.ManifestDigest), Artifacts: publicArtifacts, StagedPoweredOff: true, }, nil } func (l *Libvirt) RollbackPeerMigration(ctx context.Context, uuid string, request RollbackPeerMigrationRequest) (PeerMigrationStateResult, error) { if err := ValidateRollbackPeerMigrationRequest(request); err != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(err) } store, err := peermigration.NewStore( l.local.PeerMigrationStatePath, l.local.StoragePathRoots, l.local.PeerMigrationStagingPath, ) if err != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(err) } intent, err := store.LoadSourceIntent(request.MigrationID) if err != nil || !strings.EqualFold(intent.SourceUUID, uuid) || intent.SourceWasRunning != request.SourceWasRunning || intent.SourceAutostartEnabled != request.SourceAutostartEnabled { return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError( ErrConflict, "peer_migration_source_mismatch", "peer migration source state does not match", err, )) } if err := l.restoreMaintenanceMigrationSource(ctx, uuid, maintenanceMigrationSourceState{ wasRunning: intent.SourceWasRunning, autostart: intent.SourceAutostartEnabled, }); err != nil { return PeerMigrationStateResult{}, err } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return PeerMigrationStateResult{}, operation.MarkUncertain(err) } defer domain.Free() observedAutostart, autostartErr := domain.GetAutostart() observedActive, activeErr := domain.IsActive() state, reason, stateErr := domain.GetState() if autostartErr != nil || activeErr != nil || stateErr != nil || observedAutostart != request.SourceAutostartEnabled || observedActive != request.SourceWasRunning { return PeerMigrationStateResult{}, operation.MarkUncertain(errors.Join( translateOptionalLibvirtError("confirm peer migration source autostart", autostartErr), translateOptionalLibvirtError("confirm peer migration source activity", activeErr), translateOptionalLibvirtError("confirm peer migration source state", stateErr), NewError(ErrConflict, "peer_migration_source_state_unconfirmed", "peer migration source did not return to its original state", nil), )) } if err := store.RemoveSource(request.MigrationID); err != nil { return PeerMigrationStateResult{}, operation.MarkUncertain(NewError( ErrUnavailable, "peer_migration_state_cleanup_failed", "peer migration rollback state could not be removed", err, )) } name, _ := domain.GetName() return PeerMigrationStateResult{VMState: VMState{ UUID: strings.ToLower(uuid), Name: name, State: mapDomainState(state), Reason: reason, }, Running: observedActive, AutostartEnabled: observedAutostart}, nil } func (l *Libvirt) ActivatePeerMigration(ctx context.Context, uuid string, request ActivatePeerMigrationRequest) (_ PeerMigrationStateResult, resultErr error) { if err := ValidateActivatePeerMigrationRequest(request); err != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(err) } expected, err := expectedMutationState(ctx, operation.ExpectedPeerMigrationState) if err != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(err) } if expected.Enabled != request.TargetAutostartEnabled || expected.StartIfSuspended != request.TargetWasRunning { return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError( ErrConflict, "reconciliation_descriptor_mismatch", "peer migration target state does not match its durable reconciliation descriptor", nil, )) } store, err := peermigration.NewStore( l.local.PeerMigrationStatePath, l.local.StoragePathRoots, l.local.PeerMigrationStagingPath, ) if err != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(err) } receipt, err := store.LoadTargetReceipt(request.MigrationID) if err != nil || !strings.EqualFold(receipt.TargetUUID, uuid) { return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError( ErrConflict, "peer_migration_target_mismatch", "peer migration target receipt does not match", err, )) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(err) } defer domain.Free() document, metadata, err := requireManagedDomain(domain) if err != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(err) } if metadata.VPCPort != nil { definition, parsedMetadata, parseErr := parseManagedDomain(document) if parseErr != nil || !managedNetworkLayoutMatches(definition, parsedMetadata) { return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError( ErrConflict, "network_layout_changed", "peer migration target network layout no longer matches its metadata", parseErr, )) } if l.vpcRouter == nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError( ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil, )) } port := metadata.VPCPort.public() if err := validateVPCPortMetadata(port); err != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(err) } _, routeLease, routeErr := l.vpcRouter.AcquireLocalPortLease( port.VPCKey, port.HostRevision, port.HostDigest, port.CIDR, port.Gateway, port.PrivateIP, port.MAC, ) if routeErr != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError( ErrConflict, "vpc_host_projection_not_ready", "peer migration target routing no longer authorizes the exact VPC port", routeErr, )) } defer routeLease.Release() } persistent, err := domain.IsPersistent() if err != nil { return PeerMigrationStateResult{}, operation.MarkNotApplied(translateLibvirtError( "check peer migration target persistence", err, )) } if !persistent { return PeerMigrationStateResult{}, operation.MarkNotApplied(NewError( ErrConflict, "persistent_vm_required", "peer migration target must be persistent", nil, )) } mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() mutationStarted = true if err := setServiceAutostart(domain, request.TargetAutostartEnabled); err != nil { return PeerMigrationStateResult{}, err } active, err := domain.IsActive() if err != nil { return PeerMigrationStateResult{}, operation.MarkUncertain(translateLibvirtError( "inspect peer migration target activity", err, )) } if request.TargetWasRunning && !active { if err := l.startManagedDomain(ctx, domain, "start peer migration target"); err != nil { return PeerMigrationStateResult{}, operation.MarkUncertain(mutationLibvirtError( "start peer migration target", err, )) } } else if !request.TargetWasRunning && active { if err := requestGracefulDomainShutoff(ctx, domain, 30*time.Second); err != nil { return PeerMigrationStateResult{}, err } } observedAutostart, autostartErr := domain.GetAutostart() observedActive, activeErr := domain.IsActive() if autostartErr != nil || activeErr != nil || observedAutostart != request.TargetAutostartEnabled || observedActive != request.TargetWasRunning { return PeerMigrationStateResult{}, operation.MarkUncertain(errors.Join( translateOptionalLibvirtError("confirm peer migration target autostart", autostartErr), translateOptionalLibvirtError("confirm peer migration target activity", activeErr), NewError(ErrConflict, "peer_migration_target_state_unconfirmed", "peer migration target did not reach the source state", nil), )) } state, reason, err := domain.GetState() if err != nil { return PeerMigrationStateResult{}, operation.MarkUncertain(translateLibvirtError( "confirm peer migration target state", err, )) } // Activation has its own durable expected-state descriptor, so the receive // receipt is no longer needed after the target power state is confirmed. _ = store.RemoveTargetReceipt(request.MigrationID) name, _ := domain.GetName() return PeerMigrationStateResult{VMState: VMState{ UUID: strings.ToLower(uuid), Name: name, State: mapDomainState(state), Reason: reason, }, Running: observedActive, AutostartEnabled: observedAutostart}, nil } func (l *Libvirt) CleanupPeerMigration(ctx context.Context, uuid string, request CleanupPeerMigrationRequest) (DeleteVMResult, error) { if err := ValidateCleanupPeerMigrationRequest(request); err != nil { return DeleteVMResult{}, operation.MarkNotApplied(err) } result, err := l.DeleteVM(ctx, uuid, DeleteVMRequest{ OperationID: request.OperationID, RemoveManagedDisks: true, }) if err != nil { return DeleteVMResult{}, err } store, storeErr := peermigration.NewStore( l.local.PeerMigrationStatePath, l.local.StoragePathRoots, l.local.PeerMigrationStagingPath, ) if storeErr != nil { return result, nil } _ = store.RemoveSource(request.MigrationID) _ = store.RemoveTarget(request.MigrationID) _ = store.RemoveTargetReceipt(request.MigrationID) return result, nil } // CleanupPeerMigrationTarget removes only UUID-owned artifacts left by one // failed target receive. It never destroys or undefines a domain. The exact // inventory is inspected twice: once by the API before Journal admission and // again here under the successor's resource leases, closing the TOCTOU gap. func (l *Libvirt) CleanupPeerMigrationTarget(ctx context.Context, uuid string, request CleanupPeerMigrationTargetRequest) (_ DeleteVMResult, resultErr error) { mutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() if err := ValidateCleanupPeerMigrationTargetRequest(request); err != nil { return DeleteVMResult{}, err } prior := request.PriorOperation expectedAction := "receive-peer-migration:" + request.MigrationID if prior.OperationID != request.PriorOperationID || prior.ResourceID != uuid || prior.Action != expectedAction || prior.State != operation.StateFailed || !prior.EffectMayHaveApplied || prior.SafeToResubmit || prior.ExecutionFenced { return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_cleanup_not_authorized", "peer migration target cleanup is not authorized by the prior receive", nil) } domain, err := l.domainByUUID(ctx, uuid) if err == nil { _ = domain.Free() return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_domain_present", "peer migration target cleanup refuses to remove an existing domain", nil) } if !errors.Is(err, ErrNotFound) { return DeleteVMResult{}, err } plan, err := l.inspectDeleteCleanupSteps(ctx, uuid, nil, nil) if err != nil { return DeleteVMResult{}, err } if len(plan.steps) == 0 || len(plan.evidence) == 0 || !hashEqualsFold(request.CleanupPlanID, operationInspectionToken(prior, "cleanup", plan.tokenEvidence())) { return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_changed", "peer migration target cleanup plan changed; inspect again", nil) } for _, step := range plan.steps { switch step { case CleanupStepDeleteManagedVolumes, CleanupStepDeleteSeedVolume, CleanupStepUndefineNetworkFilter, CleanupStepDeleteNVRAM, CleanupStepRemoveTrafficState: default: return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_invalid", "peer migration target cleanup plan contains an unsafe step", nil) } } mutationStarted = true completionCtx := context.WithoutCancel(ctx) if err := l.cleanupDeletedVMArtifacts(completionCtx, uuid); err != nil { return DeleteVMResult{}, err } store, err := peermigration.NewStore( l.local.PeerMigrationStatePath, l.local.StoragePathRoots, l.local.PeerMigrationStagingPath, ) if err != nil { return DeleteVMResult{}, operation.MarkUncertain(NewError( ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target state cleanup is incomplete", err, )) } if err := store.RemoveTarget(request.MigrationID); err != nil { return DeleteVMResult{}, operation.MarkUncertain(NewError( ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target staging cleanup is incomplete", err, )) } if err := store.RemoveTargetReceipt(request.MigrationID); err != nil { return DeleteVMResult{}, operation.MarkUncertain(NewError( ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target receipt cleanup is incomplete", err, )) } absent, err := l.deleteArtifactsAbsent(uuid) if err != nil || !absent { return DeleteVMResult{}, operation.MarkUncertain(errors.Join(err, NewError(ErrConflict, "peer_migration_target_cleanup_unconfirmed", "peer migration target artifacts are still present", nil), )) } return DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true}, nil } func hashEqualsFold(left, right string) bool { return len(left) == len(right) && strings.EqualFold(left, right) } func (l *Libvirt) createPeerMigrationVolume(ctx context.Context, _ string, volumeName, path string, artifact peermigration.Artifact, finalCapacity uint64) (*libvirt.StorageVol, error) { if finalCapacity < artifact.VirtualSizeBytes { return nil, operation.MarkNotApplied(NewError( ErrInvalid, "disk_smaller_than_migration", "target disk is smaller than the peer migration artifact", nil, )) } initialCapacity := finalCapacity if uint64(artifact.SizeBytes) > initialCapacity { initialCapacity = uint64(artifact.SizeBytes) } pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged)) if err != nil { return nil, createVMStorageCloneStage.wrap(translateLibvirtError("lookup managed storage pool", err)) } defer pool.Free() definition := libvirtxml.StorageVolume{ Name: volumeName, Capacity: &libvirtxml.StorageVolumeSize{Value: initialCapacity, Unit: "bytes"}, Target: &libvirtxml.StorageVolumeTarget{Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"}}, } document, err := definition.Marshal() if err != nil { return nil, createVMStorageCloneStage.wrap(err) } volume, err := pool.StorageVolCreateXML(document, 0) if err != nil { absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName) return nil, createVMStorageCloneStage.wrap(deterministicArtifactCreateFailure( "create peer migration volume", err, inspectErr, !absent, )) } stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK) if err != nil { return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume( pool, volume, volumeName, err, "roll back peer migration volume", )) } transfer := &storageStreamTransfer{stream: stream} if err := volume.Upload(stream, 0, uint64(artifact.SizeBytes), 0); err != nil { cause := transfer.abort(err) return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure( pool, volume, volumeName, cause, "roll back peer migration volume", )) } file, err := os.Open(path) if err != nil { cause := transfer.abort(err) return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure( pool, volume, volumeName, cause, "roll back peer migration volume", )) } _, copyErr := io.Copy(&storageStreamWriter{ctx: ctx, transfer: transfer}, file) closeErr := file.Close() if copyErr != nil || closeErr != nil { cause := transfer.abort(errors.Join(copyErr, closeErr)) return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure( pool, volume, volumeName, cause, "roll back peer migration volume", )) } if err := transfer.finish(ctx); err != nil { return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure( pool, volume, volumeName, err, "roll back peer migration volume", )) } if err := pool.Refresh(0); err != nil { return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume( pool, volume, volumeName, err, "roll back peer migration volume", )) } info, err := validateStandaloneQCOW2Volume(volume, 0) if err != nil || info.Capacity != artifact.VirtualSizeBytes { if err == nil { err = NewError(ErrConflict, "peer_migration_capacity_mismatch", "peer migration qcow2 capacity does not match its manifest", nil) } return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume( pool, volume, volumeName, err, "roll back peer migration volume", )) } if finalCapacity != artifact.VirtualSizeBytes { if err := establishNewRootVolumeCapacity(volume, finalCapacity); err != nil { return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume( pool, volume, volumeName, err, "roll back peer migration volume", )) } } return volume, nil } func hashRegularFile(ctx context.Context, path string) (int64, string, error) { file, err := os.Open(path) if err != nil { return 0, "", err } defer file.Close() info, err := file.Stat() if err != nil || !info.Mode().IsRegular() || info.Size() < 1 { return 0, "", errors.Join(err, errors.New("artifact is not a non-empty regular file")) } hash := sha256.New() buffer := make([]byte, 4<<20) for { if err := ctx.Err(); err != nil { return 0, "", err } read, readErr := file.Read(buffer) if read > 0 { if _, err := hash.Write(buffer[:read]); err != nil { return 0, "", err } } if errors.Is(readErr, io.EOF) { break } if readErr != nil { return 0, "", readErr } } return info.Size(), hex.EncodeToString(hash.Sum(nil)), nil }