You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
678 lines
27 KiB
Go
678 lines
27 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
const resourceResizeGracefulShutdownTimeout = 10 * time.Minute
|
|
|
|
const (
|
|
resourceResizeOriginalPowerRunning = "running"
|
|
resourceResizeOriginalPowerStopped = "stopped"
|
|
resourceResizeOriginalAutostartEnabled = "enabled"
|
|
resourceResizeOriginalAutostartDisabled = "disabled"
|
|
)
|
|
|
|
type resizeResourceValues struct {
|
|
VCPUs uint
|
|
MemoryMiB uint64
|
|
DiskGiB uint64
|
|
BandwidthMbps uint64
|
|
}
|
|
|
|
func resizeOriginalPower(wasRunning bool) string {
|
|
if wasRunning {
|
|
return resourceResizeOriginalPowerRunning
|
|
}
|
|
return resourceResizeOriginalPowerStopped
|
|
}
|
|
|
|
func resizeWasOriginallyRunning(metadata *domainMetadataXML) (bool, bool) {
|
|
if metadata == nil {
|
|
return false, false
|
|
}
|
|
switch metadata.ResourcesOriginalPower {
|
|
case resourceResizeOriginalPowerRunning:
|
|
return true, true
|
|
case resourceResizeOriginalPowerStopped:
|
|
return false, true
|
|
default:
|
|
return false, false
|
|
}
|
|
}
|
|
|
|
func resizeOriginalAutostart(enabled bool) string {
|
|
if enabled {
|
|
return resourceResizeOriginalAutostartEnabled
|
|
}
|
|
return resourceResizeOriginalAutostartDisabled
|
|
}
|
|
|
|
func resizeWasOriginallyAutostart(metadata *domainMetadataXML) (bool, bool) {
|
|
if metadata == nil {
|
|
return false, false
|
|
}
|
|
switch metadata.ResourcesOriginalAutostart {
|
|
case resourceResizeOriginalAutostartEnabled:
|
|
return true, true
|
|
case resourceResizeOriginalAutostartDisabled:
|
|
return false, true
|
|
default:
|
|
return false, false
|
|
}
|
|
}
|
|
|
|
func resizeFromValues(request ResizeResourcesRequest) resizeResourceValues {
|
|
return resizeResourceValues{
|
|
VCPUs: request.FromVCPUs, MemoryMiB: request.FromMemoryMiB,
|
|
DiskGiB: request.FromDiskGiB, BandwidthMbps: request.FromBandwidthMbps,
|
|
}
|
|
}
|
|
|
|
func resizeTargetValues(request ResizeResourcesRequest) resizeResourceValues {
|
|
return resizeResourceValues{
|
|
VCPUs: request.TargetVCPUs, MemoryMiB: request.TargetMemoryMiB,
|
|
DiskGiB: request.TargetDiskGiB, BandwidthMbps: request.TargetBandwidthMbps,
|
|
}
|
|
}
|
|
|
|
func expectedResizeFromValues(expected *operation.ExpectedState) resizeResourceValues {
|
|
return resizeResourceValues{
|
|
VCPUs: expected.FromVCPUs, MemoryMiB: expected.FromMemoryMiB,
|
|
DiskGiB: expected.FromDiskGiB, BandwidthMbps: expected.FromBandwidthMbps,
|
|
}
|
|
}
|
|
|
|
func expectedResizeTargetValues(expected *operation.ExpectedState) resizeResourceValues {
|
|
return resizeResourceValues{
|
|
VCPUs: expected.TargetVCPUs, MemoryMiB: expected.TargetMemoryMiB,
|
|
DiskGiB: expected.TargetDiskGiB, BandwidthMbps: expected.TargetBandwidthMbps,
|
|
}
|
|
}
|
|
|
|
// resourceResizeLifecycleDomain intentionally has no force-stop method. A
|
|
// paid upgrade may request a guest ACPI shutdown, but it can never silently
|
|
// turn that authorization into DestroyFlags when the guest does not respond.
|
|
type resourceResizeLifecycleDomain interface {
|
|
gracefulShutdownDomainControl
|
|
Create() error
|
|
GetState() (libvirt.DomainState, int, error)
|
|
}
|
|
|
|
func stopForResourceResize(ctx context.Context, domain resourceResizeLifecycleDomain,
|
|
timeout time.Duration, beforeShutdown func()) (bool, error) {
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return false, translateLibvirtError("read domain state before resource resize", err)
|
|
}
|
|
mapped := mapDomainState(state)
|
|
switch mapped {
|
|
case VMStateRunning, VMStateBlocked:
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return false, translateLibvirtError("confirm active domain before resource resize", err)
|
|
}
|
|
if !active {
|
|
return false, NewError(ErrConflict, "vm_state_changed",
|
|
"virtual machine power state changed before resource resize", nil)
|
|
}
|
|
// The shutdown request is the first host-side mutation. Cross the
|
|
// mutation boundary before calling libvirt so a lost response remains
|
|
// fenced and is resolved only by the original operation's inspection.
|
|
if beforeShutdown != nil {
|
|
beforeShutdown()
|
|
}
|
|
if err := requestGracefulDomainShutoff(ctx, domain, timeout); err != nil {
|
|
var backend *Error
|
|
if errors.As(err, &backend) && backend.Code == "force_required" {
|
|
return true, operation.MarkUncertain(NewError(ErrConflict,
|
|
"upgrade_shutdown_timeout",
|
|
"guest did not stop after the automatic graceful shutdown request; no force power-off was attempted",
|
|
err))
|
|
}
|
|
if errors.As(err, &backend) && backend.Code == "graceful_shutdown_request_failed" {
|
|
return true, operation.MarkUncertain(NewError(ErrUnavailable,
|
|
"upgrade_shutdown_request_failed",
|
|
"automatic graceful shutdown could not be confirmed; no force power-off was attempted",
|
|
err))
|
|
}
|
|
return true, operation.MarkUncertain(NewError(ErrUnavailable,
|
|
"upgrade_shutdown_ambiguous",
|
|
"automatic graceful shutdown result could not be confirmed; no force power-off was attempted",
|
|
err))
|
|
}
|
|
return true, nil
|
|
case VMStateShutoff, VMStateCrashed:
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return false, translateLibvirtError("confirm inactive domain before resource resize", err)
|
|
}
|
|
if active {
|
|
return false, NewError(ErrConflict, "vm_state_changed",
|
|
"virtual machine power state changed before resource resize", nil)
|
|
}
|
|
return false, nil
|
|
default:
|
|
return false, NewError(ErrConflict, "vm_state_not_upgradeable",
|
|
"virtual machine must be running normally or shut off before resource resize", nil)
|
|
}
|
|
}
|
|
|
|
func restoreAfterResourceResize(domain resourceResizeLifecycleDomain,
|
|
wasRunning bool) (VMStateName, error) {
|
|
if !wasRunning {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return VMStateUnknown, operation.MarkUncertain(
|
|
translateLibvirtError("confirm inactive resized domain", err))
|
|
}
|
|
if active {
|
|
return VMStateUnknown, operation.MarkUncertain(NewError(ErrConflict,
|
|
"vm_state_changed",
|
|
"virtual machine became active during the stopped resource resize", nil))
|
|
}
|
|
return VMStateShutoff, nil
|
|
}
|
|
|
|
startErr := domain.Create()
|
|
active, inspectErr := domain.IsActive()
|
|
if inspectErr != nil {
|
|
return VMStateUnknown, operation.MarkUncertain(NewError(ErrUnavailable,
|
|
"upgrade_restart_ambiguous", "cold-start result could not be confirmed",
|
|
errors.Join(startErr,
|
|
translateLibvirtError("confirm resized domain cold start", inspectErr))))
|
|
}
|
|
if !active {
|
|
cause := startErr
|
|
if cause == nil {
|
|
cause = NewError(ErrConflict, "upgrade_restart_unconfirmed",
|
|
"resized domain remained shut off after the cold-start request", nil)
|
|
}
|
|
return VMStateUnknown, operation.MarkUncertain(NewError(ErrUnavailable,
|
|
"upgrade_restart_unconfirmed",
|
|
"new resources were committed but the automatic cold start was not confirmed",
|
|
cause))
|
|
}
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil {
|
|
return VMStateUnknown, operation.MarkUncertain(NewError(ErrUnavailable,
|
|
"upgrade_restart_ambiguous", "cold-start state could not be confirmed",
|
|
errors.Join(startErr,
|
|
translateLibvirtError("read resized domain cold-start state", stateErr))))
|
|
}
|
|
mapped := mapDomainState(state)
|
|
if mapped != VMStateRunning && mapped != VMStateBlocked {
|
|
return VMStateUnknown, operation.MarkUncertain(NewError(ErrConflict,
|
|
"upgrade_restart_unconfirmed",
|
|
"new resources were committed but the domain did not reach a running state",
|
|
startErr))
|
|
}
|
|
return mapped, nil
|
|
}
|
|
|
|
func restoreOriginalPowerAfterPreResizeFailure(domain resourceResizeLifecycleDomain,
|
|
wasRunning, resourceMutationStarted bool, resultErr error) error {
|
|
if resultErr == nil || !wasRunning || resourceMutationStarted {
|
|
return resultErr
|
|
}
|
|
if _, err := restoreAfterResourceResize(domain, true); err != nil {
|
|
return operation.MarkUncertain(errors.Join(resultErr, err))
|
|
}
|
|
return resultErr
|
|
}
|
|
|
|
func (l *Libvirt) ResizeResources(ctx context.Context, uuid string, request ResizeResourcesRequest) (_ ResizeResourcesResult, resultErr error) {
|
|
mutationStarted := false
|
|
resourceMutationStarted := false
|
|
defer func() {
|
|
if !mutationStarted {
|
|
resultErr = operation.MarkNotApplied(resultErr)
|
|
}
|
|
}()
|
|
if err := ValidateResizeResources(request); err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
expected, err := expectedMutationState(ctx, operation.ExpectedResizeResources)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
if !resizeRequestMatchesExpected(
|
|
request, expected.FromVCPUs, expected.TargetVCPUs,
|
|
expected.FromMemoryMiB, expected.TargetMemoryMiB,
|
|
expected.FromDiskGiB, expected.TargetDiskGiB,
|
|
expected.FromBandwidthMbps, expected.TargetBandwidthMbps,
|
|
) {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "reconciliation_descriptor_mismatch", "resource resize request does not match its durable reconciliation descriptor", nil)
|
|
}
|
|
|
|
domain, err := l.domainByUUID(ctx, uuid)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
defer domain.Free()
|
|
document, definition, metadata, pool, root, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
defer pool.Free()
|
|
defer root.Free()
|
|
if len(metadata.Snapshots) != 0 {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "snapshots_present", "resource resize is blocked while managed snapshots exist", nil)
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "disk_cleanup_pending", "resource resize is blocked while system disk cleanup is pending", nil)
|
|
}
|
|
if metadata.RescueTemplateID != "" {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "rescue_active", "resource resize is blocked while rescue boot is staged or active", nil)
|
|
}
|
|
sourceRootVolumeName := metadata.RootVolumeName
|
|
sourceTransitionID := metadata.ResourcesTransitionID
|
|
rootInfo, err := validateQCOW2Volume(root, request.FromDiskGiB*gib)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
if !managedDefinitionMatchesResources(definition, metadata, rootInfo.Capacity, resizeFromValues(request)) {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed", "host resources no longer match the requested source values", nil)
|
|
}
|
|
autostart, err := domain.GetAutostart()
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, translateLibvirtError("read resource resize domain autostart", err)
|
|
}
|
|
if _, err := resizeManagedDomainDocument(document, resizeTargetValues(request),
|
|
expected.TransitionID, false, autostart); err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
wasRunning, err := stopForResourceResize(ctx, domain,
|
|
resourceResizeGracefulShutdownTimeout, func() { mutationStarted = true })
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
defer func() {
|
|
resultErr = restoreOriginalPowerAfterPreResizeFailure(
|
|
domain, wasRunning, resourceMutationStarted, resultErr)
|
|
}()
|
|
if err := ctx.Err(); err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
|
|
// The graceful wait creates a deliberate race window. Re-read every
|
|
// resource that will be mutated and refuse to use the stale preflight if an
|
|
// administrator or another host-side tool changed the definition meanwhile.
|
|
if err := requirePersistentWithoutSnapshots(domain); err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
document, metadata, err = requireManagedDomain(domain)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
definition, _, err = parseManagedDomain(document)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
if len(metadata.Snapshots) != 0 {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "snapshots_present", "resource resize is blocked while managed snapshots exist", nil)
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "disk_cleanup_pending", "resource resize is blocked while system disk cleanup is pending", nil)
|
|
}
|
|
if metadata.RescueTemplateID != "" {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "rescue_active", "resource resize is blocked while rescue boot is staged or active", nil)
|
|
}
|
|
if metadata.RootVolumeName != sourceRootVolumeName ||
|
|
metadata.ResourcesTransitionID != sourceTransitionID {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed",
|
|
"managed resource identity changed during resource resize preflight", nil)
|
|
}
|
|
currentAutostart, err := domain.GetAutostart()
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, translateLibvirtError(
|
|
"recheck resource resize domain autostart", err)
|
|
}
|
|
if currentAutostart != autostart {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed",
|
|
"domain autostart changed during resource resize preflight", nil)
|
|
}
|
|
rootDisk, err := managedRootDisk(definition)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
rootPath, pathErr := root.GetPath()
|
|
if pathErr != nil || rootDisk.Source == nil || rootDisk.Source.File == nil ||
|
|
filepath.Clean(rootDisk.Source.File.File) != filepath.Clean(rootPath) {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed",
|
|
"managed system disk changed during resource resize preflight", pathErr)
|
|
}
|
|
rootInfo, err = validateQCOW2Volume(root, request.FromDiskGiB*gib)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
if !managedDefinitionMatchesResources(definition, metadata, rootInfo.Capacity, resizeFromValues(request)) {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed", "host resources changed during resource resize preflight", nil)
|
|
}
|
|
updatedXML, err := resizeManagedDomainDocument(
|
|
document, resizeTargetValues(request), expected.TransitionID,
|
|
wasRunning, autostart)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, translateLibvirtError(
|
|
"confirm stopped domain before resource resize", err)
|
|
}
|
|
if active {
|
|
return ResizeResourcesResult{}, NewError(ErrConflict, "vm_state_changed",
|
|
"virtual machine became active before resource resize", nil)
|
|
}
|
|
|
|
if request.TargetDiskGiB > request.FromDiskGiB {
|
|
mutationStarted = true
|
|
resourceMutationStarted = true
|
|
if err := root.Resize(request.TargetDiskGiB*gib, 0); err != nil {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(mutationLibvirtError("grow managed system disk", err))
|
|
}
|
|
if _, err := validateQCOW2Volume(root, request.TargetDiskGiB*gib); err != nil {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(err)
|
|
}
|
|
}
|
|
|
|
mutationStarted = true
|
|
resourceMutationStarted = true
|
|
defined, err := l.conn.DomainDefineXML(updatedXML)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(mutationLibvirtError("persist resized VM resources", err))
|
|
}
|
|
defer defined.Free()
|
|
setAutostartErr := defined.SetAutostart(autostart)
|
|
observedAutostart, inspectAutostartErr := defined.GetAutostart()
|
|
var autostartErr error
|
|
if inspectAutostartErr != nil || observedAutostart != autostart {
|
|
cause := errors.Join(setAutostartErr, inspectAutostartErr)
|
|
if cause == nil {
|
|
cause = NewError(ErrConflict, "resource_resize_autostart_mismatch",
|
|
"resized domain autostart did not match its original value", nil)
|
|
}
|
|
autostartErr = operation.MarkUncertain(NewError(ErrUnavailable,
|
|
"resource_resize_autostart_unconfirmed",
|
|
"new resources were committed but domain autostart preservation was not confirmed",
|
|
cause))
|
|
}
|
|
observedXML, observedMetadata, err := requireManagedDomain(defined)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(err)
|
|
}
|
|
observedDefinition, _, err := parseManagedDomain(observedXML)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(err)
|
|
}
|
|
observedRootInfo, err := validateQCOW2Volume(root, request.TargetDiskGiB*gib)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(err)
|
|
}
|
|
if observedMetadata.ResourcesTransitionID != expected.TransitionID ||
|
|
observedMetadata.ResourcesOriginalPower != resizeOriginalPower(wasRunning) ||
|
|
observedMetadata.ResourcesOriginalAutostart != resizeOriginalAutostart(autostart) ||
|
|
!managedDefinitionMatchesResources(observedDefinition, observedMetadata, observedRootInfo.Capacity, resizeTargetValues(request)) {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(NewError(ErrConflict, "resource_resize_observation_failed", "resized resources did not expose the committed transition", nil))
|
|
}
|
|
status, err := restoreAfterResourceResize(defined, wasRunning)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, err
|
|
}
|
|
if wasRunning {
|
|
liveXML, err := defined.GetXMLDesc(0)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(
|
|
translateLibvirtError("read live resized domain definition", err))
|
|
}
|
|
liveMatches, err := resizeLiveDocumentMatchesExpected(
|
|
liveXML, &expected, observedRootInfo.Capacity, true, autostart)
|
|
if err != nil {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(err)
|
|
}
|
|
if !liveMatches {
|
|
return ResizeResourcesResult{}, operation.MarkUncertain(NewError(
|
|
ErrConflict, "resource_resize_live_observation_failed",
|
|
"cold-started domain did not expose the committed resources", nil))
|
|
}
|
|
}
|
|
if autostartErr != nil {
|
|
return ResizeResourcesResult{}, autostartErr
|
|
}
|
|
return resizeResourcesResult(uuid, expected, status, wasRunning, true), nil
|
|
}
|
|
|
|
func resizeManagedDomainDocument(document string, target resizeResourceValues,
|
|
transitionID string, wasRunning, originalAutostart bool) (string, error) {
|
|
return mutateDomainDocument(document, func(definition *libvirtxml.Domain, metadata *domainMetadataXML) error {
|
|
if definition.VCPU == nil || definition.Memory == nil ||
|
|
!managedNetworkLayoutMatches(definition, metadata) {
|
|
return NewError(ErrConflict, "resource_layout_changed", "managed VM resource definition is incomplete", nil)
|
|
}
|
|
if definition.VCPUs != nil || definition.MaximumMemory != nil ||
|
|
(definition.CPU != nil && definition.CPU.Topology != nil) {
|
|
return NewError(ErrConflict, "resource_layout_changed", "managed VM resource layout does not support stopped resize", nil)
|
|
}
|
|
definition.VCPU.Value = target.VCPUs
|
|
if definition.VCPU.Current != 0 {
|
|
definition.VCPU.Current = target.VCPUs
|
|
}
|
|
definition.Memory.Value = uint(target.MemoryMiB)
|
|
definition.Memory.Unit = "MiB"
|
|
if definition.CurrentMemory != nil {
|
|
definition.CurrentMemory.Value = uint(target.MemoryMiB)
|
|
definition.CurrentMemory.Unit = "MiB"
|
|
}
|
|
setDomainInterfaceBandwidth(definition, target.BandwidthMbps)
|
|
metadata.DiskGiB = target.DiskGiB
|
|
metadata.BandwidthMbps = target.BandwidthMbps
|
|
metadata.ResourcesTransitionID = transitionID
|
|
metadata.ResourcesOriginalPower = resizeOriginalPower(wasRunning)
|
|
metadata.ResourcesOriginalAutostart = resizeOriginalAutostart(originalAutostart)
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func managedDefinitionMatchesResources(definition *libvirtxml.Domain, metadata *domainMetadataXML, rootCapacity uint64, values resizeResourceValues) bool {
|
|
if definition == nil || metadata == nil || definition.VCPU == nil || definition.Memory == nil ||
|
|
!managedNetworkLayoutMatches(definition, metadata) ||
|
|
definition.VCPUs != nil || definition.MaximumMemory != nil ||
|
|
(definition.CPU != nil && definition.CPU.Topology != nil) {
|
|
return false
|
|
}
|
|
if definition.VCPU.Value != values.VCPUs ||
|
|
(definition.VCPU.Current != 0 && definition.VCPU.Current != values.VCPUs) ||
|
|
!memoryValueMatchesMiB(definition.Memory.Value, definition.Memory.Unit, values.MemoryMiB) ||
|
|
(definition.CurrentMemory != nil && !memoryValueMatchesMiB(
|
|
definition.CurrentMemory.Value, definition.CurrentMemory.Unit, values.MemoryMiB,
|
|
)) {
|
|
return false
|
|
}
|
|
return rootCapacity == values.DiskGiB*gib && metadata.DiskGiB == values.DiskGiB &&
|
|
metadata.BandwidthMbps == values.BandwidthMbps &&
|
|
domainInterfaceBandwidthMatches(definition, values.BandwidthMbps)
|
|
}
|
|
|
|
func memoryValueMatchesMiB(value uint, unit string, expected uint64) bool {
|
|
switch strings.ToLower(unit) {
|
|
case "b", "bytes":
|
|
return uint64(value) == expected*1024*1024
|
|
case "kb", "kib", "":
|
|
return uint64(value) == expected*1024
|
|
case "mb", "mib":
|
|
return uint64(value) == expected
|
|
case "gb", "gib":
|
|
return uint64(value)*1024 == expected
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func setDomainInterfaceBandwidth(definition *libvirtxml.Domain, bandwidthMbps uint64) {
|
|
rate := bandwidthRateKiBPerSecond(bandwidthMbps)
|
|
for index := range definition.Devices.Interfaces {
|
|
definition.Devices.Interfaces[index].Bandwidth = &libvirtxml.DomainInterfaceBandwidth{
|
|
Inbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
|
|
Outbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
|
|
}
|
|
}
|
|
}
|
|
|
|
func domainInterfaceBandwidthMatches(definition *libvirtxml.Domain, bandwidthMbps uint64) bool {
|
|
if definition == nil || definition.Devices == nil || len(definition.Devices.Interfaces) == 0 {
|
|
return false
|
|
}
|
|
rate := bandwidthRateKiBPerSecond(bandwidthMbps)
|
|
for _, iface := range definition.Devices.Interfaces {
|
|
bandwidth := iface.Bandwidth
|
|
if bandwidth == nil || bandwidth.Inbound == nil || bandwidth.Outbound == nil ||
|
|
bandwidth.Inbound.Average == nil || bandwidth.Inbound.Peak == nil ||
|
|
bandwidth.Outbound.Average == nil || bandwidth.Outbound.Peak == nil ||
|
|
*bandwidth.Inbound.Average != rate || *bandwidth.Inbound.Peak != rate ||
|
|
*bandwidth.Outbound.Average != rate || *bandwidth.Outbound.Peak != rate {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func bandwidthRateKiBPerSecond(bandwidthMbps uint64) int {
|
|
return int((bandwidthMbps*1_000_000 + 8191) / 8192)
|
|
}
|
|
|
|
func resizeResourcesResult(uuid string, expected operation.ExpectedState,
|
|
status VMStateName, originallyRunning, powerStateRestored bool) ResizeResourcesResult {
|
|
return ResizeResourcesResult{
|
|
UUID: uuid,
|
|
FromVCPUs: expected.FromVCPUs, TargetVCPUs: expected.TargetVCPUs, AppliedVCPUs: expected.TargetVCPUs,
|
|
FromMemoryMiB: expected.FromMemoryMiB, TargetMemoryMiB: expected.TargetMemoryMiB, AppliedMemoryMiB: expected.TargetMemoryMiB,
|
|
FromDiskGiB: expected.FromDiskGiB, TargetDiskGiB: expected.TargetDiskGiB, AppliedDiskGiB: expected.TargetDiskGiB,
|
|
FromBandwidthMbps: expected.FromBandwidthMbps, TargetBandwidthMbps: expected.TargetBandwidthMbps, AppliedBandwidthMbps: expected.TargetBandwidthMbps,
|
|
Status: status, OriginallyRunning: originallyRunning,
|
|
PowerStateRestored: powerStateRestored, ResourcesResized: true,
|
|
}
|
|
}
|
|
|
|
func resizeLiveDocumentMatchesExpected(document string,
|
|
expected *operation.ExpectedState, rootCapacity uint64,
|
|
originallyRunning, originalAutostart bool) (bool, error) {
|
|
definition, metadata, err := parseManagedDomain(document)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return metadata.ResourcesTransitionID == expected.TransitionID &&
|
|
metadata.ResourcesOriginalPower == resizeOriginalPower(originallyRunning) &&
|
|
metadata.ResourcesOriginalAutostart == resizeOriginalAutostart(originalAutostart) &&
|
|
managedDefinitionMatchesResources(definition, metadata, rootCapacity,
|
|
expectedResizeTargetValues(expected)), nil
|
|
}
|
|
|
|
func reconcileResizeObservation(expected *operation.ExpectedState, marker string, definition *libvirtxml.Domain,
|
|
metadata *domainMetadataXML, rootCapacity uint64) operation.Resolution {
|
|
targetMatches := managedDefinitionMatchesResources(definition, metadata, rootCapacity, expectedResizeTargetValues(expected))
|
|
if marker == expected.TransitionID && targetMatches {
|
|
return operation.Resolution{Resolved: true, Succeeded: true}
|
|
}
|
|
fromMatches := managedDefinitionMatchesResources(definition, metadata, rootCapacity, expectedResizeFromValues(expected))
|
|
if marker != expected.TransitionID && fromMatches {
|
|
// An uncertain resize may already have delivered ACPI shutdown. The
|
|
// guest can still stop after this inspection, so source resources alone
|
|
// are not proof that PHP may refund and release the lifecycle fence.
|
|
return operation.Resolution{}
|
|
}
|
|
return resolvedExpectedFailure("resources_partially_resized", "host evidence shows a partial or conflicting resource resize")
|
|
}
|
|
|
|
func reconcileResizePowerObservation(originallyRunning, active,
|
|
liveTargetMatches bool) operation.Resolution {
|
|
if originallyRunning && !active {
|
|
return resolvedExpectedFailure("resources_partially_resized",
|
|
"persistent resources committed but the required cold restart has not completed")
|
|
}
|
|
if !originallyRunning && active {
|
|
return resolvedExpectedFailure("resources_partially_resized",
|
|
"persistent resources committed but the originally stopped domain became active")
|
|
}
|
|
if active && !liveTargetMatches {
|
|
return resolvedExpectedFailure("resources_partially_resized",
|
|
"persistent resources committed but the active domain does not expose the target resources")
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: true}
|
|
}
|
|
|
|
func (l *Libvirt) reconcileResizeResources(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
_, definition, metadata, pool, root, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
return expectedEvidenceError("resources", err)
|
|
}
|
|
defer pool.Free()
|
|
defer root.Free()
|
|
rootInfo, err := validateQCOW2Volume(root, 0)
|
|
if err != nil {
|
|
return expectedEvidenceError("resources", err)
|
|
}
|
|
classification := reconcileResizeObservation(record.ExpectedState, metadata.ResourcesTransitionID, definition, metadata, rootInfo.Capacity)
|
|
if !classification.Resolved || !classification.Succeeded {
|
|
return classification, nil
|
|
}
|
|
originallyRunning, validOriginalPower := resizeWasOriginallyRunning(metadata)
|
|
originalAutostart, validOriginalAutostart := resizeWasOriginallyAutostart(metadata)
|
|
if !validOriginalPower || !validOriginalAutostart {
|
|
return resolvedExpectedFailure("resources_partially_resized",
|
|
"resized resources do not contain valid original lifecycle markers"), nil
|
|
}
|
|
observedAutostart, err := domain.GetAutostart()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError(
|
|
"reconcile resized domain autostart", err)
|
|
}
|
|
if observedAutostart != originalAutostart {
|
|
return resolvedExpectedFailure("resources_partially_resized",
|
|
"persistent resources committed but domain autostart was not preserved"), nil
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError(
|
|
"reconcile resized domain activity", err)
|
|
}
|
|
liveMatches := false
|
|
if active {
|
|
liveXML, err := domain.GetXMLDesc(0)
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError(
|
|
"reconcile live resized domain definition", err)
|
|
}
|
|
liveMatches, err = resizeLiveDocumentMatchesExpected(
|
|
liveXML, record.ExpectedState, rootInfo.Capacity,
|
|
originallyRunning, originalAutostart)
|
|
if err != nil {
|
|
return expectedEvidenceError("resources", err)
|
|
}
|
|
}
|
|
powerClassification := reconcileResizePowerObservation(
|
|
originallyRunning, active, liveMatches)
|
|
if !powerClassification.Succeeded {
|
|
return powerClassification, nil
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, translateLibvirtError("reconcile resized domain state", err)
|
|
}
|
|
result := resizeResourcesResult(record.ResourceID, *record.ExpectedState,
|
|
mapDomainState(state), originallyRunning, true)
|
|
return resolvedExpectedSuccess(result)
|
|
}
|