You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_resources_test.go

388 lines
15 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"encoding/json"
"errors"
"testing"
"time"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
type fakeResourceResizeDomain struct {
states []libvirt.DomainState
stateErrors []error
active []bool
activeErrors []error
shutdownErr error
createErr error
shutdownCalls int
createCalls int
mutationStarted *bool
mutationAtShutdown bool
}
func (f *fakeResourceResizeDomain) GetState() (libvirt.DomainState, int, error) {
state := libvirt.DOMAIN_NOSTATE
if len(f.states) != 0 {
state = f.states[0]
f.states = f.states[1:]
}
var err error
if len(f.stateErrors) != 0 {
err = f.stateErrors[0]
f.stateErrors = f.stateErrors[1:]
}
return state, 0, err
}
func (f *fakeResourceResizeDomain) IsActive() (bool, error) {
var active bool
if len(f.active) != 0 {
active = f.active[0]
f.active = f.active[1:]
}
var err error
if len(f.activeErrors) != 0 {
err = f.activeErrors[0]
f.activeErrors = f.activeErrors[1:]
}
return active, err
}
func (f *fakeResourceResizeDomain) ShutdownFlags(libvirt.DomainShutdownFlags) error {
f.shutdownCalls++
if f.mutationStarted != nil {
f.mutationAtShutdown = *f.mutationStarted
}
return f.shutdownErr
}
func (f *fakeResourceResizeDomain) Create() error {
f.createCalls++
return f.createErr
}
func TestResourceResizePowerLifecycleUsesGracefulShutdownAndRestoresOriginalState(t *testing.T) {
t.Run("running guest is stopped after mutation boundary and cold started", func(t *testing.T) {
mutationStarted := false
domain := &fakeResourceResizeDomain{
states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING, libvirt.DOMAIN_RUNNING},
active: []bool{true, true, false, true},
mutationStarted: &mutationStarted,
}
wasRunning, err := stopForResourceResize(context.Background(), domain,
time.Second, func() { mutationStarted = true })
if err != nil || !wasRunning || domain.shutdownCalls != 1 ||
!domain.mutationAtShutdown {
t.Fatalf("stop result running=%v shutdown=%d boundary=%v err=%v",
wasRunning, domain.shutdownCalls, domain.mutationAtShutdown, err)
}
status, err := restoreAfterResourceResize(domain, wasRunning)
if err != nil || status != VMStateRunning || domain.createCalls != 1 {
t.Fatalf("restore status=%q create=%d err=%v", status, domain.createCalls, err)
}
})
t.Run("inactive guest remains inactive", func(t *testing.T) {
domain := &fakeResourceResizeDomain{
states: []libvirt.DomainState{libvirt.DOMAIN_SHUTOFF},
active: []bool{false, false},
}
wasRunning, err := stopForResourceResize(context.Background(), domain,
time.Second, nil)
if err != nil || wasRunning || domain.shutdownCalls != 0 {
t.Fatalf("stop result running=%v shutdown=%d err=%v",
wasRunning, domain.shutdownCalls, err)
}
status, err := restoreAfterResourceResize(domain, wasRunning)
if err != nil || status != VMStateShutoff || domain.createCalls != 0 {
t.Fatalf("restore status=%q create=%d err=%v", status, domain.createCalls, err)
}
})
}
func TestResourceResizePowerLifecycleNeverForcesAndKeepsUncertainResultsFenced(t *testing.T) {
t.Run("guest that ignores ACPI shutdown times out without a force path", func(t *testing.T) {
mutationStarted := false
domain := &fakeResourceResizeDomain{
states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING},
active: []bool{true, true, true, true},
mutationStarted: &mutationStarted,
}
wasRunning, err := stopForResourceResize(context.Background(), domain,
time.Nanosecond, func() { mutationStarted = true })
var backend *Error
if !wasRunning || domain.shutdownCalls != 1 || domain.createCalls != 0 ||
!operation.IsUncertain(err) || !errors.As(err, &backend) ||
backend.Code != "upgrade_shutdown_timeout" {
t.Fatalf("timeout running=%v shutdown=%d create=%d err=%v",
wasRunning, domain.shutdownCalls, domain.createCalls, err)
}
})
t.Run("lost start response accepted only when running is observed", func(t *testing.T) {
cause := errors.New("start response lost")
domain := &fakeResourceResizeDomain{
states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING},
active: []bool{true}, createErr: cause,
}
status, err := restoreAfterResourceResize(domain, true)
if err != nil || status != VMStateRunning || domain.createCalls != 1 {
t.Fatalf("confirmed start status=%q calls=%d err=%v",
status, domain.createCalls, err)
}
})
t.Run("unconfirmed cold start remains uncertain", func(t *testing.T) {
cause := errors.New("start failed")
domain := &fakeResourceResizeDomain{active: []bool{false}, createErr: cause}
_, err := restoreAfterResourceResize(domain, true)
if domain.createCalls != 1 || !operation.IsUncertain(err) || !errors.Is(err, cause) {
t.Fatalf("unconfirmed start calls=%d err=%v", domain.createCalls, err)
}
})
t.Run("paused guest is rejected before any lifecycle mutation", func(t *testing.T) {
domain := &fakeResourceResizeDomain{states: []libvirt.DomainState{libvirt.DOMAIN_PAUSED}}
wasRunning, err := stopForResourceResize(context.Background(), domain,
time.Second, func() { t.Fatal("mutation boundary crossed for paused guest") })
var backend *Error
if wasRunning || domain.shutdownCalls != 0 || !errors.As(err, &backend) ||
backend.Code != "vm_state_not_upgradeable" || operation.IsUncertain(err) {
t.Fatalf("paused result running=%v shutdown=%d err=%v",
wasRunning, domain.shutdownCalls, err)
}
})
}
func TestResourceResizeRestoresRunningGuestWhenPreResizeWorkFails(t *testing.T) {
primary := errors.New("resource preflight changed")
domain := &fakeResourceResizeDomain{
states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING},
active: []bool{true},
}
err := restoreOriginalPowerAfterPreResizeFailure(domain, true, false, primary)
if !errors.Is(err, primary) || domain.createCalls != 1 {
t.Fatalf("pre-resize restore calls=%d err=%v", domain.createCalls, err)
}
afterMutation := &fakeResourceResizeDomain{}
err = restoreOriginalPowerAfterPreResizeFailure(afterMutation, true, true, primary)
if !errors.Is(err, primary) || afterMutation.createCalls != 0 {
t.Fatalf("post-mutation restore calls=%d err=%v", afterMutation.createCalls, err)
}
}
func TestResizeManagedDomainDocumentUpdatesStoppedResourcesAndMarker(t *testing.T) {
document, err := BuildDomainXML(validDomainDefinition())
if err != nil {
t.Fatal(err)
}
normalized, _, err := parseManagedDomain(document)
if err != nil {
t.Fatal(err)
}
normalized.VCPU.Current = normalized.VCPU.Value
normalized.CurrentMemory = &libvirtxml.DomainCurrentMemory{
Value: normalized.Memory.Value * 1024,
Unit: "KiB",
}
document, err = normalized.Marshal()
if err != nil {
t.Fatal(err)
}
target := resizeResourceValues{VCPUs: 4, MemoryMiB: 4096, DiskGiB: 80, BandwidthMbps: 250}
updated, err := resizeManagedDomainDocument(
document, target, "operation-resize-resources", true, true)
if err != nil {
t.Fatal(err)
}
definition, metadata, err := parseManagedDomain(updated)
if err != nil {
t.Fatal(err)
}
if metadata.ResourcesTransitionID != "operation-resize-resources" ||
metadata.ResourcesOriginalPower != resourceResizeOriginalPowerRunning ||
metadata.ResourcesOriginalAutostart != resourceResizeOriginalAutostartEnabled ||
!managedDefinitionMatchesResources(definition, metadata, target.DiskGiB*gib, target) {
t.Fatalf("resized definition did not match target: metadata=%#v xml=%s", metadata, updated)
}
if definition.VCPU.Current != target.VCPUs || definition.CurrentMemory == nil ||
!memoryValueMatchesMiB(definition.CurrentMemory.Value, definition.CurrentMemory.Unit, target.MemoryMiB) {
t.Fatalf("normalized current CPU/memory were not moved to the target: %#v", definition)
}
if managedDefinitionMatchesResources(definition, metadata, 40*gib, target) {
t.Fatal("metadata-only target was accepted without the actual volume capacity")
}
stoppedUpdated, err := resizeManagedDomainDocument(
document, target, "operation-resize-stopped", false, false)
if err != nil {
t.Fatal(err)
}
_, stoppedMetadata, err := parseManagedDomain(stoppedUpdated)
if err != nil {
t.Fatal(err)
}
stoppedWasRunning, stoppedMarkerValid := resizeWasOriginallyRunning(stoppedMetadata)
stoppedAutostart, stoppedAutostartValid := resizeWasOriginallyAutostart(stoppedMetadata)
if stoppedMetadata.ResourcesOriginalPower != resourceResizeOriginalPowerStopped ||
stoppedMetadata.ResourcesOriginalAutostart != resourceResizeOriginalAutostartDisabled ||
stoppedWasRunning || !stoppedMarkerValid || stoppedAutostart || !stoppedAutostartValid {
t.Fatalf("stopped original-power marker was not explicit: %#v", stoppedMetadata)
}
if _, valid := resizeWasOriginallyRunning(&domainMetadataXML{}); valid {
t.Fatal("missing original-power marker was accepted as an originally stopped VM")
}
if _, valid := resizeWasOriginallyAutostart(&domainMetadataXML{}); valid {
t.Fatal("missing original-autostart marker was accepted as disabled autostart")
}
}
func TestReconcileResizeObservationKeepsSourceIndeterminateAndDistinguishesAppliedAndPartial(t *testing.T) {
document, err := BuildDomainXML(validDomainDefinition())
if err != nil {
t.Fatal(err)
}
fromDefinition, fromMetadata, err := parseManagedDomain(document)
if err != nil {
t.Fatal(err)
}
expected := &operation.ExpectedState{
Kind: operation.ExpectedResizeResources, TransitionID: "operation-resize-reconcile",
FromVCPUs: 2, TargetVCPUs: 4,
FromMemoryMiB: 2048, TargetMemoryMiB: 4096,
FromDiskGiB: 40, TargetDiskGiB: 80,
FromBandwidthMbps: 100, TargetBandwidthMbps: 200,
}
for name, marker := range map[string]string{
"no marker": "",
"prior transition": "operation-prior-resize",
} {
indeterminate := reconcileResizeObservation(expected, marker, fromDefinition, fromMetadata, 40*gib)
if indeterminate.Resolved {
t.Fatalf("%s source observation released a possibly delayed ACPI shutdown = %#v", name, indeterminate)
}
}
updated, err := resizeManagedDomainDocument(
document, expectedResizeTargetValues(expected), expected.TransitionID, true, true)
if err != nil {
t.Fatal(err)
}
targetDefinition, targetMetadata, err := parseManagedDomain(updated)
if err != nil {
t.Fatal(err)
}
applied := reconcileResizeObservation(expected, expected.TransitionID, targetDefinition, targetMetadata, 80*gib)
if !applied.Resolved || !applied.Succeeded {
t.Fatalf("applied observation = %#v", applied)
}
liveMatches, err := resizeLiveDocumentMatchesExpected(
updated, expected, 80*gib, true, true)
if err != nil || !liveMatches {
t.Fatalf("target live definition matches=%v err=%v", liveMatches, err)
}
liveMatches, err = resizeLiveDocumentMatchesExpected(
document, expected, 40*gib, true, true)
if err != nil || liveMatches {
t.Fatalf("source live definition matches=%v err=%v", liveMatches, err)
}
for name, partial := range map[string]operation.Resolution{
"volume only": reconcileResizeObservation(expected, "", fromDefinition, fromMetadata, 80*gib),
"definition only": reconcileResizeObservation(expected, expected.TransitionID, targetDefinition, targetMetadata, 40*gib),
"target without marker": reconcileResizeObservation(expected, "operation-other-resize", targetDefinition, targetMetadata, 80*gib),
} {
if !partial.Resolved || partial.Succeeded || partial.ErrorCode != "resources_partially_resized" {
t.Fatalf("%s partial observation = %#v", name, partial)
}
}
}
func TestResizeInspectionPublishesOnlyBoundedProjectionAndKeepsPartialFenced(t *testing.T) {
const uuid = "123e4567-e89b-42d3-a456-426614174000"
record := operation.Record{
OperationID: "operation-resize-inspection", ResourceID: uuid,
Action: "resize-vm", State: operation.StateFailed, EffectMayHaveApplied: true,
ExpectedState: &operation.ExpectedState{
Kind: operation.ExpectedResizeResources, TransitionID: "operation-resize-inspection",
FromVCPUs: 2, TargetVCPUs: 4,
FromMemoryMiB: 2048, TargetMemoryMiB: 4096,
FromDiskGiB: 40, TargetDiskGiB: 80,
FromBandwidthMbps: 100, TargetBandwidthMbps: 200,
},
}
result := ResizeResourcesResult{
UUID: uuid,
FromVCPUs: 2, TargetVCPUs: 4, AppliedVCPUs: 4,
FromMemoryMiB: 2048, TargetMemoryMiB: 4096, AppliedMemoryMiB: 4096,
FromDiskGiB: 40, TargetDiskGiB: 80, AppliedDiskGiB: 80,
FromBandwidthMbps: 100, TargetBandwidthMbps: 200, AppliedBandwidthMbps: 200,
Status: VMStateRunning, OriginallyRunning: true,
PowerStateRestored: true, ResourcesResized: true,
}
raw, err := json.Marshal(result)
if err != nil {
t.Fatal(err)
}
applied := buildOperationInspection(record, operation.Resolution{Resolved: true, Succeeded: true, Result: raw}, OperationAbortInspection{}, operationCleanupInspection{})
if applied.EffectResolution != InspectionEffectApplied || len(applied.EffectProjection) == 0 {
t.Fatalf("applied resize inspection = %#v", applied)
}
var projected ResizeResourcesResult
if err := json.Unmarshal(applied.EffectProjection, &projected); err != nil || !projected.ResourcesResized || projected.AppliedDiskGiB != 80 {
t.Fatalf("resize projection = %s err=%v", applied.EffectProjection, err)
}
partial := buildOperationInspection(record, operation.Resolution{
Resolved: true, Succeeded: false, ErrorCode: "resources_partially_resized",
}, OperationAbortInspection{}, operationCleanupInspection{})
if partial.EffectResolution != InspectionEffectPartial || partial.Outcome != InspectionOutcomeUncertain || len(partial.EffectProjection) != 0 {
t.Fatalf("partial resize inspection = %#v", partial)
}
invalidPower := result
invalidPower.OriginallyRunning = false
invalidPower.Status = VMStateRunning
invalidRaw, err := json.Marshal(invalidPower)
if err != nil {
t.Fatal(err)
}
invalid := buildOperationInspection(record, operation.Resolution{
Resolved: true, Succeeded: true, Result: invalidRaw,
}, OperationAbortInspection{}, operationCleanupInspection{})
if invalid.EffectResolution != InspectionEffectInconclusive ||
invalid.Outcome != InspectionOutcomeUncertain || len(invalid.EffectProjection) != 0 {
t.Fatalf("invalid power-state resize inspection = %#v", invalid)
}
}
func TestReconcileResizePowerObservationRequiresOriginalRunningStateRestored(t *testing.T) {
for name, observation := range map[string]operation.Resolution{
"restart pending": reconcileResizePowerObservation(true, false, false),
"stale live source": reconcileResizePowerObservation(true, true, false),
"originally stopped now live": reconcileResizePowerObservation(false, true, true),
} {
if !observation.Resolved || observation.Succeeded ||
observation.ErrorCode != "resources_partially_resized" {
t.Fatalf("%s observation = %#v", name, observation)
}
}
for name, observation := range map[string]operation.Resolution{
"running target": reconcileResizePowerObservation(true, true, true),
"originally stopped": reconcileResizePowerObservation(false, false, false),
} {
if !observation.Resolved || !observation.Succeeded {
t.Fatalf("%s observation = %#v", name, observation)
}
}
}