//go:build libvirt package hypervisor import ( "context" "errors" "path/filepath" "strings" "time" "git.luoli.live/root/virt/agent/operation" "libvirt.org/go/libvirt" "libvirt.org/go/libvirtxml" ) const resourceResizeGracefulShutdownTimeout = 10 * time.Minute const ( resourceResizeOriginalPowerRunning = "running" resourceResizeOriginalPowerStopped = "stopped" resourceResizeOriginalAutostartEnabled = "enabled" resourceResizeOriginalAutostartDisabled = "disabled" ) type resizeResourceValues struct { VCPUs uint MemoryMiB uint64 DiskGiB uint64 BandwidthMbps uint64 } func resizeOriginalPower(wasRunning bool) string { if wasRunning { return resourceResizeOriginalPowerRunning } return resourceResizeOriginalPowerStopped } func resizeWasOriginallyRunning(metadata *domainMetadataXML) (bool, bool) { if metadata == nil { return false, false } switch metadata.ResourcesOriginalPower { case resourceResizeOriginalPowerRunning: return true, true case resourceResizeOriginalPowerStopped: return false, true default: return false, false } } func resizeOriginalAutostart(enabled bool) string { if enabled { return resourceResizeOriginalAutostartEnabled } return resourceResizeOriginalAutostartDisabled } func resizeWasOriginallyAutostart(metadata *domainMetadataXML) (bool, bool) { if metadata == nil { return false, false } switch metadata.ResourcesOriginalAutostart { case resourceResizeOriginalAutostartEnabled: return true, true case resourceResizeOriginalAutostartDisabled: return false, true default: return false, false } } func resizeFromValues(request ResizeResourcesRequest) resizeResourceValues { return resizeResourceValues{ VCPUs: request.FromVCPUs, MemoryMiB: request.FromMemoryMiB, DiskGiB: request.FromDiskGiB, BandwidthMbps: request.FromBandwidthMbps, } } func resizeTargetValues(request ResizeResourcesRequest) resizeResourceValues { return resizeResourceValues{ VCPUs: request.TargetVCPUs, MemoryMiB: request.TargetMemoryMiB, DiskGiB: request.TargetDiskGiB, BandwidthMbps: request.TargetBandwidthMbps, } } func expectedResizeFromValues(expected *operation.ExpectedState) resizeResourceValues { return resizeResourceValues{ VCPUs: expected.FromVCPUs, MemoryMiB: expected.FromMemoryMiB, DiskGiB: expected.FromDiskGiB, BandwidthMbps: expected.FromBandwidthMbps, } } func expectedResizeTargetValues(expected *operation.ExpectedState) resizeResourceValues { return resizeResourceValues{ VCPUs: expected.TargetVCPUs, MemoryMiB: expected.TargetMemoryMiB, DiskGiB: expected.TargetDiskGiB, BandwidthMbps: expected.TargetBandwidthMbps, } } // resourceResizeLifecycleDomain intentionally has no force-stop method. A // paid upgrade may request a guest ACPI shutdown, but it can never silently // turn that authorization into DestroyFlags when the guest does not respond. type resourceResizeLifecycleDomain interface { gracefulShutdownDomainControl Create() error GetState() (libvirt.DomainState, int, error) } func stopForResourceResize(ctx context.Context, domain resourceResizeLifecycleDomain, timeout time.Duration, beforeShutdown func()) (bool, error) { state, _, err := domain.GetState() if err != nil { return false, translateLibvirtError("read domain state before resource resize", err) } mapped := mapDomainState(state) switch mapped { case VMStateRunning, VMStateBlocked: active, err := domain.IsActive() if err != nil { return false, translateLibvirtError("confirm active domain before resource resize", err) } if !active { return false, NewError(ErrConflict, "vm_state_changed", "virtual machine power state changed before resource resize", nil) } // The shutdown request is the first host-side mutation. Cross the // mutation boundary before calling libvirt so a lost response remains // fenced and is resolved only by the original operation's inspection. if beforeShutdown != nil { beforeShutdown() } if err := requestGracefulDomainShutoff(ctx, domain, timeout); err != nil { var backend *Error if errors.As(err, &backend) && backend.Code == "force_required" { return true, operation.MarkUncertain(NewError(ErrConflict, "upgrade_shutdown_timeout", "guest did not stop after the automatic graceful shutdown request; no force power-off was attempted", err)) } if errors.As(err, &backend) && backend.Code == "graceful_shutdown_request_failed" { return true, operation.MarkUncertain(NewError(ErrUnavailable, "upgrade_shutdown_request_failed", "automatic graceful shutdown could not be confirmed; no force power-off was attempted", err)) } return true, operation.MarkUncertain(NewError(ErrUnavailable, "upgrade_shutdown_ambiguous", "automatic graceful shutdown result could not be confirmed; no force power-off was attempted", err)) } return true, nil case VMStateShutoff, VMStateCrashed: active, err := domain.IsActive() if err != nil { return false, translateLibvirtError("confirm inactive domain before resource resize", err) } if active { return false, NewError(ErrConflict, "vm_state_changed", "virtual machine power state changed before resource resize", nil) } return false, nil default: return false, NewError(ErrConflict, "vm_state_not_upgradeable", "virtual machine must be running normally or shut off before resource resize", nil) } } func restoreAfterResourceResize(domain resourceResizeLifecycleDomain, wasRunning bool, start func() error) (VMStateName, error) { if !wasRunning { active, err := domain.IsActive() if err != nil { return VMStateUnknown, operation.MarkUncertain( translateLibvirtError("confirm inactive resized domain", err)) } if active { return VMStateUnknown, operation.MarkUncertain(NewError(ErrConflict, "vm_state_changed", "virtual machine became active during the stopped resource resize", nil)) } return VMStateShutoff, nil } if start == nil { return VMStateUnknown, operation.MarkUncertain(NewError(ErrUnavailable, "public_guard_unavailable", "cold-start guard is unavailable", nil)) } startErr := start() active, inspectErr := domain.IsActive() if inspectErr != nil { return VMStateUnknown, operation.MarkUncertain(NewError(ErrUnavailable, "upgrade_restart_ambiguous", "cold-start result could not be confirmed", errors.Join(startErr, translateLibvirtError("confirm resized domain cold start", inspectErr)))) } if !active { cause := startErr if cause == nil { cause = NewError(ErrConflict, "upgrade_restart_unconfirmed", "resized domain remained shut off after the cold-start request", nil) } return VMStateUnknown, operation.MarkUncertain(NewError(ErrUnavailable, "upgrade_restart_unconfirmed", "new resources were committed but the automatic cold start was not confirmed", cause)) } state, _, stateErr := domain.GetState() if stateErr != nil { return VMStateUnknown, operation.MarkUncertain(NewError(ErrUnavailable, "upgrade_restart_ambiguous", "cold-start state could not be confirmed", errors.Join(startErr, translateLibvirtError("read resized domain cold-start state", stateErr)))) } mapped := mapDomainState(state) if mapped != VMStateRunning && mapped != VMStateBlocked { return VMStateUnknown, operation.MarkUncertain(NewError(ErrConflict, "upgrade_restart_unconfirmed", "new resources were committed but the domain did not reach a running state", startErr)) } return mapped, nil } func restoreOriginalPowerAfterPreResizeFailure(domain resourceResizeLifecycleDomain, wasRunning, resourceMutationStarted bool, resultErr error, start func() error) error { if resultErr == nil || !wasRunning || resourceMutationStarted { return resultErr } if _, err := restoreAfterResourceResize(domain, true, start); err != nil { return operation.MarkUncertain(errors.Join(resultErr, err)) } return resultErr } func (l *Libvirt) ResizeResources(ctx context.Context, uuid string, request ResizeResourcesRequest) (_ ResizeResourcesResult, resultErr error) { mutationStarted := false resourceMutationStarted := false defer func() { if !mutationStarted { resultErr = operation.MarkNotApplied(resultErr) } }() if err := ValidateResizeResources(request); err != nil { return ResizeResourcesResult{}, err } expected, err := expectedMutationState(ctx, operation.ExpectedResizeResources) if err != nil { return ResizeResourcesResult{}, err } if !resizeRequestMatchesExpected( request, expected.FromVCPUs, expected.TargetVCPUs, expected.FromMemoryMiB, expected.TargetMemoryMiB, expected.FromDiskGiB, expected.TargetDiskGiB, expected.FromBandwidthMbps, expected.TargetBandwidthMbps, ) { return ResizeResourcesResult{}, NewError(ErrConflict, "reconciliation_descriptor_mismatch", "resource resize request does not match its durable reconciliation descriptor", nil) } domain, err := l.domainByUUID(ctx, uuid) if err != nil { return ResizeResourcesResult{}, err } defer domain.Free() document, definition, metadata, pool, root, err := l.managedRootContext(domain) if err != nil { return ResizeResourcesResult{}, err } defer pool.Free() defer root.Free() if len(metadata.Snapshots) != 0 { return ResizeResourcesResult{}, NewError(ErrConflict, "snapshots_present", "resource resize is blocked while managed snapshots exist", nil) } if metadata.PendingCleanupVolume != "" { return ResizeResourcesResult{}, NewError(ErrConflict, "disk_cleanup_pending", "resource resize is blocked while system disk cleanup is pending", nil) } if metadata.RescueTemplateID != "" { return ResizeResourcesResult{}, NewError(ErrConflict, "rescue_active", "resource resize is blocked while rescue boot is staged or active", nil) } sourceRootVolumeName := metadata.RootVolumeName sourceTransitionID := metadata.ResourcesTransitionID rootInfo, err := validateQCOW2Volume(root, request.FromDiskGiB*gib) if err != nil { return ResizeResourcesResult{}, err } if !managedDefinitionMatchesResources(definition, metadata, rootInfo.Capacity, resizeFromValues(request)) { return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed", "host resources no longer match the requested source values", nil) } autostart, err := domain.GetAutostart() if err != nil { return ResizeResourcesResult{}, translateLibvirtError("read resource resize domain autostart", err) } if _, err := resizeManagedDomainDocument(document, resizeTargetValues(request), expected.TransitionID, false, autostart); err != nil { return ResizeResourcesResult{}, err } if err := ctx.Err(); err != nil { return ResizeResourcesResult{}, err } wasRunning, err := stopForResourceResize(ctx, domain, resourceResizeGracefulShutdownTimeout, func() { mutationStarted = true }) if err != nil { return ResizeResourcesResult{}, err } defer func() { resultErr = restoreOriginalPowerAfterPreResizeFailure( domain, wasRunning, resourceMutationStarted, resultErr, func() error { return l.startManagedDomain(ctx, domain, "restart domain after resource resize preflight failure") }, ) }() if err := ctx.Err(); err != nil { return ResizeResourcesResult{}, err } // The graceful wait creates a deliberate race window. Re-read every // resource that will be mutated and refuse to use the stale preflight if an // administrator or another host-side tool changed the definition meanwhile. if err := requirePersistentWithoutSnapshots(domain); err != nil { return ResizeResourcesResult{}, err } document, metadata, err = requireManagedDomain(domain) if err != nil { return ResizeResourcesResult{}, err } definition, _, err = parseManagedDomain(document) if err != nil { return ResizeResourcesResult{}, err } if len(metadata.Snapshots) != 0 { return ResizeResourcesResult{}, NewError(ErrConflict, "snapshots_present", "resource resize is blocked while managed snapshots exist", nil) } if metadata.PendingCleanupVolume != "" { return ResizeResourcesResult{}, NewError(ErrConflict, "disk_cleanup_pending", "resource resize is blocked while system disk cleanup is pending", nil) } if metadata.RescueTemplateID != "" { return ResizeResourcesResult{}, NewError(ErrConflict, "rescue_active", "resource resize is blocked while rescue boot is staged or active", nil) } if metadata.RootVolumeName != sourceRootVolumeName || metadata.ResourcesTransitionID != sourceTransitionID { return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed", "managed resource identity changed during resource resize preflight", nil) } currentAutostart, err := domain.GetAutostart() if err != nil { return ResizeResourcesResult{}, translateLibvirtError( "recheck resource resize domain autostart", err) } if currentAutostart != autostart { return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed", "domain autostart changed during resource resize preflight", nil) } rootDisk, err := managedRootDisk(definition) if err != nil { return ResizeResourcesResult{}, err } rootPath, pathErr := root.GetPath() if pathErr != nil || rootDisk.Source == nil || rootDisk.Source.File == nil || filepath.Clean(rootDisk.Source.File.File) != filepath.Clean(rootPath) { return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed", "managed system disk changed during resource resize preflight", pathErr) } rootInfo, err = validateQCOW2Volume(root, request.FromDiskGiB*gib) if err != nil { return ResizeResourcesResult{}, err } if !managedDefinitionMatchesResources(definition, metadata, rootInfo.Capacity, resizeFromValues(request)) { return ResizeResourcesResult{}, NewError(ErrConflict, "resource_state_changed", "host resources changed during resource resize preflight", nil) } updatedXML, err := resizeManagedDomainDocument( document, resizeTargetValues(request), expected.TransitionID, wasRunning, autostart) if err != nil { return ResizeResourcesResult{}, err } active, err := domain.IsActive() if err != nil { return ResizeResourcesResult{}, translateLibvirtError( "confirm stopped domain before resource resize", err) } if active { return ResizeResourcesResult{}, NewError(ErrConflict, "vm_state_changed", "virtual machine became active before resource resize", nil) } if request.TargetDiskGiB > request.FromDiskGiB { mutationStarted = true resourceMutationStarted = true if err := root.Resize(request.TargetDiskGiB*gib, 0); err != nil { return ResizeResourcesResult{}, operation.MarkUncertain(mutationLibvirtError("grow managed system disk", err)) } if _, err := validateQCOW2Volume(root, request.TargetDiskGiB*gib); err != nil { return ResizeResourcesResult{}, operation.MarkUncertain(err) } } mutationStarted = true resourceMutationStarted = true defined, err := l.conn.DomainDefineXML(updatedXML) if err != nil { return ResizeResourcesResult{}, operation.MarkUncertain(mutationLibvirtError("persist resized VM resources", err)) } defer defined.Free() setAutostartErr := defined.SetAutostart(autostart) observedAutostart, inspectAutostartErr := defined.GetAutostart() var autostartErr error if inspectAutostartErr != nil || observedAutostart != autostart { cause := errors.Join(setAutostartErr, inspectAutostartErr) if cause == nil { cause = NewError(ErrConflict, "resource_resize_autostart_mismatch", "resized domain autostart did not match its original value", nil) } autostartErr = operation.MarkUncertain(NewError(ErrUnavailable, "resource_resize_autostart_unconfirmed", "new resources were committed but domain autostart preservation was not confirmed", cause)) } observedXML, observedMetadata, err := requireManagedDomain(defined) if err != nil { return ResizeResourcesResult{}, operation.MarkUncertain(err) } observedDefinition, _, err := parseManagedDomain(observedXML) if err != nil { return ResizeResourcesResult{}, operation.MarkUncertain(err) } observedRootInfo, err := validateQCOW2Volume(root, request.TargetDiskGiB*gib) if err != nil { return ResizeResourcesResult{}, operation.MarkUncertain(err) } if observedMetadata.ResourcesTransitionID != expected.TransitionID || observedMetadata.ResourcesOriginalPower != resizeOriginalPower(wasRunning) || observedMetadata.ResourcesOriginalAutostart != resizeOriginalAutostart(autostart) || !managedDefinitionMatchesResources(observedDefinition, observedMetadata, observedRootInfo.Capacity, resizeTargetValues(request)) { return ResizeResourcesResult{}, operation.MarkUncertain(NewError(ErrConflict, "resource_resize_observation_failed", "resized resources did not expose the committed transition", nil)) } status, err := restoreAfterResourceResize(defined, wasRunning, func() error { return l.startManagedDomain(ctx, defined, "cold start resized domain") }) if err != nil { return ResizeResourcesResult{}, err } if wasRunning { liveXML, err := defined.GetXMLDesc(0) if err != nil { return ResizeResourcesResult{}, operation.MarkUncertain( translateLibvirtError("read live resized domain definition", err)) } liveMatches, err := resizeLiveDocumentMatchesExpected( liveXML, &expected, observedRootInfo.Capacity, true, autostart) if err != nil { return ResizeResourcesResult{}, operation.MarkUncertain(err) } if !liveMatches { return ResizeResourcesResult{}, operation.MarkUncertain(NewError( ErrConflict, "resource_resize_live_observation_failed", "cold-started domain did not expose the committed resources", nil)) } } if autostartErr != nil { return ResizeResourcesResult{}, autostartErr } return resizeResourcesResult(uuid, expected, status, wasRunning, true), nil } func resizeManagedDomainDocument(document string, target resizeResourceValues, transitionID string, wasRunning, originalAutostart bool) (string, error) { return mutateDomainDocument(document, func(definition *libvirtxml.Domain, metadata *domainMetadataXML) error { if definition.VCPU == nil || definition.Memory == nil || !managedNetworkLayoutMatches(definition, metadata) { return NewError(ErrConflict, "resource_layout_changed", "managed VM resource definition is incomplete", nil) } if definition.VCPUs != nil || definition.MaximumMemory != nil || (definition.CPU != nil && definition.CPU.Topology != nil) { return NewError(ErrConflict, "resource_layout_changed", "managed VM resource layout does not support stopped resize", nil) } definition.VCPU.Value = target.VCPUs if definition.VCPU.Current != 0 { definition.VCPU.Current = target.VCPUs } definition.Memory.Value = uint(target.MemoryMiB) definition.Memory.Unit = "MiB" if definition.CurrentMemory != nil { definition.CurrentMemory.Value = uint(target.MemoryMiB) definition.CurrentMemory.Unit = "MiB" } setDomainInterfaceBandwidth(definition, target.BandwidthMbps) metadata.DiskGiB = target.DiskGiB metadata.BandwidthMbps = target.BandwidthMbps metadata.ResourcesTransitionID = transitionID metadata.ResourcesOriginalPower = resizeOriginalPower(wasRunning) metadata.ResourcesOriginalAutostart = resizeOriginalAutostart(originalAutostart) return nil }) } func managedDefinitionMatchesResources(definition *libvirtxml.Domain, metadata *domainMetadataXML, rootCapacity uint64, values resizeResourceValues) bool { if definition == nil || metadata == nil || definition.VCPU == nil || definition.Memory == nil || !managedNetworkLayoutMatches(definition, metadata) || definition.VCPUs != nil || definition.MaximumMemory != nil || (definition.CPU != nil && definition.CPU.Topology != nil) { return false } if definition.VCPU.Value != values.VCPUs || (definition.VCPU.Current != 0 && definition.VCPU.Current != values.VCPUs) || !memoryValueMatchesMiB(definition.Memory.Value, definition.Memory.Unit, values.MemoryMiB) || (definition.CurrentMemory != nil && !memoryValueMatchesMiB( definition.CurrentMemory.Value, definition.CurrentMemory.Unit, values.MemoryMiB, )) { return false } return rootCapacity == values.DiskGiB*gib && metadata.DiskGiB == values.DiskGiB && metadata.BandwidthMbps == values.BandwidthMbps && domainInterfaceBandwidthMatches(definition, values.BandwidthMbps) } func memoryValueMatchesMiB(value uint, unit string, expected uint64) bool { switch strings.ToLower(unit) { case "b", "bytes": return uint64(value) == expected*1024*1024 case "kb", "kib", "": return uint64(value) == expected*1024 case "mb", "mib": return uint64(value) == expected case "gb", "gib": return uint64(value)*1024 == expected default: return false } } func setDomainInterfaceBandwidth(definition *libvirtxml.Domain, bandwidthMbps uint64) { rate := bandwidthRateKiBPerSecond(bandwidthMbps) for index := range definition.Devices.Interfaces { definition.Devices.Interfaces[index].Bandwidth = &libvirtxml.DomainInterfaceBandwidth{ Inbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate}, Outbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate}, } } } func domainInterfaceBandwidthMatches(definition *libvirtxml.Domain, bandwidthMbps uint64) bool { if definition == nil || definition.Devices == nil || len(definition.Devices.Interfaces) == 0 { return false } rate := bandwidthRateKiBPerSecond(bandwidthMbps) for _, iface := range definition.Devices.Interfaces { bandwidth := iface.Bandwidth if bandwidth == nil || bandwidth.Inbound == nil || bandwidth.Outbound == nil || bandwidth.Inbound.Average == nil || bandwidth.Inbound.Peak == nil || bandwidth.Outbound.Average == nil || bandwidth.Outbound.Peak == nil || *bandwidth.Inbound.Average != rate || *bandwidth.Inbound.Peak != rate || *bandwidth.Outbound.Average != rate || *bandwidth.Outbound.Peak != rate { return false } } return true } func bandwidthRateKiBPerSecond(bandwidthMbps uint64) int { return int((bandwidthMbps*1_000_000 + 8191) / 8192) } func resizeResourcesResult(uuid string, expected operation.ExpectedState, status VMStateName, originallyRunning, powerStateRestored bool) ResizeResourcesResult { return ResizeResourcesResult{ UUID: uuid, FromVCPUs: expected.FromVCPUs, TargetVCPUs: expected.TargetVCPUs, AppliedVCPUs: expected.TargetVCPUs, FromMemoryMiB: expected.FromMemoryMiB, TargetMemoryMiB: expected.TargetMemoryMiB, AppliedMemoryMiB: expected.TargetMemoryMiB, FromDiskGiB: expected.FromDiskGiB, TargetDiskGiB: expected.TargetDiskGiB, AppliedDiskGiB: expected.TargetDiskGiB, FromBandwidthMbps: expected.FromBandwidthMbps, TargetBandwidthMbps: expected.TargetBandwidthMbps, AppliedBandwidthMbps: expected.TargetBandwidthMbps, Status: status, OriginallyRunning: originallyRunning, PowerStateRestored: powerStateRestored, ResourcesResized: true, } } func resizeLiveDocumentMatchesExpected(document string, expected *operation.ExpectedState, rootCapacity uint64, originallyRunning, originalAutostart bool) (bool, error) { definition, metadata, err := parseManagedDomain(document) if err != nil { return false, err } return metadata.ResourcesTransitionID == expected.TransitionID && metadata.ResourcesOriginalPower == resizeOriginalPower(originallyRunning) && metadata.ResourcesOriginalAutostart == resizeOriginalAutostart(originalAutostart) && managedDefinitionMatchesResources(definition, metadata, rootCapacity, expectedResizeTargetValues(expected)), nil } func reconcileResizeObservation(expected *operation.ExpectedState, marker string, definition *libvirtxml.Domain, metadata *domainMetadataXML, rootCapacity uint64) operation.Resolution { targetMatches := managedDefinitionMatchesResources(definition, metadata, rootCapacity, expectedResizeTargetValues(expected)) if marker == expected.TransitionID && targetMatches { return operation.Resolution{Resolved: true, Succeeded: true} } fromMatches := managedDefinitionMatchesResources(definition, metadata, rootCapacity, expectedResizeFromValues(expected)) if marker != expected.TransitionID && fromMatches { // An uncertain resize may already have delivered ACPI shutdown. The // guest can still stop after this inspection, so source resources alone // are not proof that PHP may refund and release the lifecycle fence. return operation.Resolution{} } return resolvedExpectedFailure("resources_partially_resized", "host evidence shows a partial or conflicting resource resize") } func reconcileResizePowerObservation(originallyRunning, active, liveTargetMatches bool) operation.Resolution { if originallyRunning && !active { return resolvedExpectedFailure("resources_partially_resized", "persistent resources committed but the required cold restart has not completed") } if !originallyRunning && active { return resolvedExpectedFailure("resources_partially_resized", "persistent resources committed but the originally stopped domain became active") } if active && !liveTargetMatches { return resolvedExpectedFailure("resources_partially_resized", "persistent resources committed but the active domain does not expose the target resources") } return operation.Resolution{Resolved: true, Succeeded: true} } func (l *Libvirt) reconcileResizeResources(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) { _, definition, metadata, pool, root, err := l.managedRootContext(domain) if err != nil { return expectedEvidenceError("resources", err) } defer pool.Free() defer root.Free() rootInfo, err := validateQCOW2Volume(root, 0) if err != nil { return expectedEvidenceError("resources", err) } classification := reconcileResizeObservation(record.ExpectedState, metadata.ResourcesTransitionID, definition, metadata, rootInfo.Capacity) if !classification.Resolved || !classification.Succeeded { return classification, nil } originallyRunning, validOriginalPower := resizeWasOriginallyRunning(metadata) originalAutostart, validOriginalAutostart := resizeWasOriginallyAutostart(metadata) if !validOriginalPower || !validOriginalAutostart { return resolvedExpectedFailure("resources_partially_resized", "resized resources do not contain valid original lifecycle markers"), nil } observedAutostart, err := domain.GetAutostart() if err != nil { return operation.Resolution{}, translateLibvirtError( "reconcile resized domain autostart", err) } if observedAutostart != originalAutostart { return resolvedExpectedFailure("resources_partially_resized", "persistent resources committed but domain autostart was not preserved"), nil } active, err := domain.IsActive() if err != nil { return operation.Resolution{}, translateLibvirtError( "reconcile resized domain activity", err) } liveMatches := false if active { liveXML, err := domain.GetXMLDesc(0) if err != nil { return operation.Resolution{}, translateLibvirtError( "reconcile live resized domain definition", err) } liveMatches, err = resizeLiveDocumentMatchesExpected( liveXML, record.ExpectedState, rootInfo.Capacity, originallyRunning, originalAutostart) if err != nil { return expectedEvidenceError("resources", err) } } powerClassification := reconcileResizePowerObservation( originallyRunning, active, liveMatches) if !powerClassification.Succeeded { return powerClassification, nil } state, _, err := domain.GetState() if err != nil { return operation.Resolution{}, translateLibvirtError("reconcile resized domain state", err) } result := resizeResourcesResult(record.ResourceID, *record.ExpectedState, mapDomainState(state), originallyRunning, true) return resolvedExpectedSuccess(result) }