You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
388 lines
15 KiB
Go
388 lines
15 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
type fakeResourceResizeDomain struct {
|
|
states []libvirt.DomainState
|
|
stateErrors []error
|
|
active []bool
|
|
activeErrors []error
|
|
shutdownErr error
|
|
createErr error
|
|
shutdownCalls int
|
|
createCalls int
|
|
mutationStarted *bool
|
|
mutationAtShutdown bool
|
|
}
|
|
|
|
func (f *fakeResourceResizeDomain) GetState() (libvirt.DomainState, int, error) {
|
|
state := libvirt.DOMAIN_NOSTATE
|
|
if len(f.states) != 0 {
|
|
state = f.states[0]
|
|
f.states = f.states[1:]
|
|
}
|
|
var err error
|
|
if len(f.stateErrors) != 0 {
|
|
err = f.stateErrors[0]
|
|
f.stateErrors = f.stateErrors[1:]
|
|
}
|
|
return state, 0, err
|
|
}
|
|
|
|
func (f *fakeResourceResizeDomain) IsActive() (bool, error) {
|
|
var active bool
|
|
if len(f.active) != 0 {
|
|
active = f.active[0]
|
|
f.active = f.active[1:]
|
|
}
|
|
var err error
|
|
if len(f.activeErrors) != 0 {
|
|
err = f.activeErrors[0]
|
|
f.activeErrors = f.activeErrors[1:]
|
|
}
|
|
return active, err
|
|
}
|
|
|
|
func (f *fakeResourceResizeDomain) ShutdownFlags(libvirt.DomainShutdownFlags) error {
|
|
f.shutdownCalls++
|
|
if f.mutationStarted != nil {
|
|
f.mutationAtShutdown = *f.mutationStarted
|
|
}
|
|
return f.shutdownErr
|
|
}
|
|
|
|
func (f *fakeResourceResizeDomain) Create() error {
|
|
f.createCalls++
|
|
return f.createErr
|
|
}
|
|
|
|
func TestResourceResizePowerLifecycleUsesGracefulShutdownAndRestoresOriginalState(t *testing.T) {
|
|
t.Run("running guest is stopped after mutation boundary and cold started", func(t *testing.T) {
|
|
mutationStarted := false
|
|
domain := &fakeResourceResizeDomain{
|
|
states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING, libvirt.DOMAIN_RUNNING},
|
|
active: []bool{true, true, false, true},
|
|
mutationStarted: &mutationStarted,
|
|
}
|
|
wasRunning, err := stopForResourceResize(context.Background(), domain,
|
|
time.Second, func() { mutationStarted = true })
|
|
if err != nil || !wasRunning || domain.shutdownCalls != 1 ||
|
|
!domain.mutationAtShutdown {
|
|
t.Fatalf("stop result running=%v shutdown=%d boundary=%v err=%v",
|
|
wasRunning, domain.shutdownCalls, domain.mutationAtShutdown, err)
|
|
}
|
|
status, err := restoreAfterResourceResize(domain, wasRunning, domain.Create)
|
|
if err != nil || status != VMStateRunning || domain.createCalls != 1 {
|
|
t.Fatalf("restore status=%q create=%d err=%v", status, domain.createCalls, err)
|
|
}
|
|
})
|
|
|
|
t.Run("inactive guest remains inactive", func(t *testing.T) {
|
|
domain := &fakeResourceResizeDomain{
|
|
states: []libvirt.DomainState{libvirt.DOMAIN_SHUTOFF},
|
|
active: []bool{false, false},
|
|
}
|
|
wasRunning, err := stopForResourceResize(context.Background(), domain,
|
|
time.Second, nil)
|
|
if err != nil || wasRunning || domain.shutdownCalls != 0 {
|
|
t.Fatalf("stop result running=%v shutdown=%d err=%v",
|
|
wasRunning, domain.shutdownCalls, err)
|
|
}
|
|
status, err := restoreAfterResourceResize(domain, wasRunning, domain.Create)
|
|
if err != nil || status != VMStateShutoff || domain.createCalls != 0 {
|
|
t.Fatalf("restore status=%q create=%d err=%v", status, domain.createCalls, err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestResourceResizePowerLifecycleNeverForcesAndKeepsUncertainResultsFenced(t *testing.T) {
|
|
t.Run("guest that ignores ACPI shutdown times out without a force path", func(t *testing.T) {
|
|
mutationStarted := false
|
|
domain := &fakeResourceResizeDomain{
|
|
states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING},
|
|
active: []bool{true, true, true, true},
|
|
mutationStarted: &mutationStarted,
|
|
}
|
|
wasRunning, err := stopForResourceResize(context.Background(), domain,
|
|
time.Nanosecond, func() { mutationStarted = true })
|
|
var backend *Error
|
|
if !wasRunning || domain.shutdownCalls != 1 || domain.createCalls != 0 ||
|
|
!operation.IsUncertain(err) || !errors.As(err, &backend) ||
|
|
backend.Code != "upgrade_shutdown_timeout" {
|
|
t.Fatalf("timeout running=%v shutdown=%d create=%d err=%v",
|
|
wasRunning, domain.shutdownCalls, domain.createCalls, err)
|
|
}
|
|
})
|
|
|
|
t.Run("lost start response accepted only when running is observed", func(t *testing.T) {
|
|
cause := errors.New("start response lost")
|
|
domain := &fakeResourceResizeDomain{
|
|
states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING},
|
|
active: []bool{true}, createErr: cause,
|
|
}
|
|
status, err := restoreAfterResourceResize(domain, true, domain.Create)
|
|
if err != nil || status != VMStateRunning || domain.createCalls != 1 {
|
|
t.Fatalf("confirmed start status=%q calls=%d err=%v",
|
|
status, domain.createCalls, err)
|
|
}
|
|
})
|
|
|
|
t.Run("unconfirmed cold start remains uncertain", func(t *testing.T) {
|
|
cause := errors.New("start failed")
|
|
domain := &fakeResourceResizeDomain{active: []bool{false}, createErr: cause}
|
|
_, err := restoreAfterResourceResize(domain, true, domain.Create)
|
|
if domain.createCalls != 1 || !operation.IsUncertain(err) || !errors.Is(err, cause) {
|
|
t.Fatalf("unconfirmed start calls=%d err=%v", domain.createCalls, err)
|
|
}
|
|
})
|
|
|
|
t.Run("paused guest is rejected before any lifecycle mutation", func(t *testing.T) {
|
|
domain := &fakeResourceResizeDomain{states: []libvirt.DomainState{libvirt.DOMAIN_PAUSED}}
|
|
wasRunning, err := stopForResourceResize(context.Background(), domain,
|
|
time.Second, func() { t.Fatal("mutation boundary crossed for paused guest") })
|
|
var backend *Error
|
|
if wasRunning || domain.shutdownCalls != 0 || !errors.As(err, &backend) ||
|
|
backend.Code != "vm_state_not_upgradeable" || operation.IsUncertain(err) {
|
|
t.Fatalf("paused result running=%v shutdown=%d err=%v",
|
|
wasRunning, domain.shutdownCalls, err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestResourceResizeRestoresRunningGuestWhenPreResizeWorkFails(t *testing.T) {
|
|
primary := errors.New("resource preflight changed")
|
|
domain := &fakeResourceResizeDomain{
|
|
states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING},
|
|
active: []bool{true},
|
|
}
|
|
err := restoreOriginalPowerAfterPreResizeFailure(domain, true, false, primary, domain.Create)
|
|
if !errors.Is(err, primary) || domain.createCalls != 1 {
|
|
t.Fatalf("pre-resize restore calls=%d err=%v", domain.createCalls, err)
|
|
}
|
|
|
|
afterMutation := &fakeResourceResizeDomain{}
|
|
err = restoreOriginalPowerAfterPreResizeFailure(afterMutation, true, true, primary, afterMutation.Create)
|
|
if !errors.Is(err, primary) || afterMutation.createCalls != 0 {
|
|
t.Fatalf("post-mutation restore calls=%d err=%v", afterMutation.createCalls, err)
|
|
}
|
|
}
|
|
|
|
func TestResizeManagedDomainDocumentUpdatesStoppedResourcesAndMarker(t *testing.T) {
|
|
document, err := BuildDomainXML(validDomainDefinition())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
normalized, _, err := parseManagedDomain(document)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
normalized.VCPU.Current = normalized.VCPU.Value
|
|
normalized.CurrentMemory = &libvirtxml.DomainCurrentMemory{
|
|
Value: normalized.Memory.Value * 1024,
|
|
Unit: "KiB",
|
|
}
|
|
document, err = normalized.Marshal()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
target := resizeResourceValues{VCPUs: 4, MemoryMiB: 4096, DiskGiB: 80, BandwidthMbps: 250}
|
|
updated, err := resizeManagedDomainDocument(
|
|
document, target, "operation-resize-resources", true, true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
definition, metadata, err := parseManagedDomain(updated)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if metadata.ResourcesTransitionID != "operation-resize-resources" ||
|
|
metadata.ResourcesOriginalPower != resourceResizeOriginalPowerRunning ||
|
|
metadata.ResourcesOriginalAutostart != resourceResizeOriginalAutostartEnabled ||
|
|
!managedDefinitionMatchesResources(definition, metadata, target.DiskGiB*gib, target) {
|
|
t.Fatalf("resized definition did not match target: metadata=%#v xml=%s", metadata, updated)
|
|
}
|
|
if definition.VCPU.Current != target.VCPUs || definition.CurrentMemory == nil ||
|
|
!memoryValueMatchesMiB(definition.CurrentMemory.Value, definition.CurrentMemory.Unit, target.MemoryMiB) {
|
|
t.Fatalf("normalized current CPU/memory were not moved to the target: %#v", definition)
|
|
}
|
|
if managedDefinitionMatchesResources(definition, metadata, 40*gib, target) {
|
|
t.Fatal("metadata-only target was accepted without the actual volume capacity")
|
|
}
|
|
|
|
stoppedUpdated, err := resizeManagedDomainDocument(
|
|
document, target, "operation-resize-stopped", false, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, stoppedMetadata, err := parseManagedDomain(stoppedUpdated)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stoppedWasRunning, stoppedMarkerValid := resizeWasOriginallyRunning(stoppedMetadata)
|
|
stoppedAutostart, stoppedAutostartValid := resizeWasOriginallyAutostart(stoppedMetadata)
|
|
if stoppedMetadata.ResourcesOriginalPower != resourceResizeOriginalPowerStopped ||
|
|
stoppedMetadata.ResourcesOriginalAutostart != resourceResizeOriginalAutostartDisabled ||
|
|
stoppedWasRunning || !stoppedMarkerValid || stoppedAutostart || !stoppedAutostartValid {
|
|
t.Fatalf("stopped original-power marker was not explicit: %#v", stoppedMetadata)
|
|
}
|
|
if _, valid := resizeWasOriginallyRunning(&domainMetadataXML{}); valid {
|
|
t.Fatal("missing original-power marker was accepted as an originally stopped VM")
|
|
}
|
|
if _, valid := resizeWasOriginallyAutostart(&domainMetadataXML{}); valid {
|
|
t.Fatal("missing original-autostart marker was accepted as disabled autostart")
|
|
}
|
|
}
|
|
|
|
func TestReconcileResizeObservationKeepsSourceIndeterminateAndDistinguishesAppliedAndPartial(t *testing.T) {
|
|
document, err := BuildDomainXML(validDomainDefinition())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fromDefinition, fromMetadata, err := parseManagedDomain(document)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedResizeResources, TransitionID: "operation-resize-reconcile",
|
|
FromVCPUs: 2, TargetVCPUs: 4,
|
|
FromMemoryMiB: 2048, TargetMemoryMiB: 4096,
|
|
FromDiskGiB: 40, TargetDiskGiB: 80,
|
|
FromBandwidthMbps: 100, TargetBandwidthMbps: 200,
|
|
}
|
|
|
|
for name, marker := range map[string]string{
|
|
"no marker": "",
|
|
"prior transition": "operation-prior-resize",
|
|
} {
|
|
indeterminate := reconcileResizeObservation(expected, marker, fromDefinition, fromMetadata, 40*gib)
|
|
if indeterminate.Resolved {
|
|
t.Fatalf("%s source observation released a possibly delayed ACPI shutdown = %#v", name, indeterminate)
|
|
}
|
|
}
|
|
|
|
updated, err := resizeManagedDomainDocument(
|
|
document, expectedResizeTargetValues(expected), expected.TransitionID, true, true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
targetDefinition, targetMetadata, err := parseManagedDomain(updated)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
applied := reconcileResizeObservation(expected, expected.TransitionID, targetDefinition, targetMetadata, 80*gib)
|
|
if !applied.Resolved || !applied.Succeeded {
|
|
t.Fatalf("applied observation = %#v", applied)
|
|
}
|
|
liveMatches, err := resizeLiveDocumentMatchesExpected(
|
|
updated, expected, 80*gib, true, true)
|
|
if err != nil || !liveMatches {
|
|
t.Fatalf("target live definition matches=%v err=%v", liveMatches, err)
|
|
}
|
|
liveMatches, err = resizeLiveDocumentMatchesExpected(
|
|
document, expected, 40*gib, true, true)
|
|
if err != nil || liveMatches {
|
|
t.Fatalf("source live definition matches=%v err=%v", liveMatches, err)
|
|
}
|
|
|
|
for name, partial := range map[string]operation.Resolution{
|
|
"volume only": reconcileResizeObservation(expected, "", fromDefinition, fromMetadata, 80*gib),
|
|
"definition only": reconcileResizeObservation(expected, expected.TransitionID, targetDefinition, targetMetadata, 40*gib),
|
|
"target without marker": reconcileResizeObservation(expected, "operation-other-resize", targetDefinition, targetMetadata, 80*gib),
|
|
} {
|
|
if !partial.Resolved || partial.Succeeded || partial.ErrorCode != "resources_partially_resized" {
|
|
t.Fatalf("%s partial observation = %#v", name, partial)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResizeInspectionPublishesOnlyBoundedProjectionAndKeepsPartialFenced(t *testing.T) {
|
|
const uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
record := operation.Record{
|
|
OperationID: "operation-resize-inspection", ResourceID: uuid,
|
|
Action: "resize-vm", State: operation.StateFailed, EffectMayHaveApplied: true,
|
|
ExpectedState: &operation.ExpectedState{
|
|
Kind: operation.ExpectedResizeResources, TransitionID: "operation-resize-inspection",
|
|
FromVCPUs: 2, TargetVCPUs: 4,
|
|
FromMemoryMiB: 2048, TargetMemoryMiB: 4096,
|
|
FromDiskGiB: 40, TargetDiskGiB: 80,
|
|
FromBandwidthMbps: 100, TargetBandwidthMbps: 200,
|
|
},
|
|
}
|
|
result := ResizeResourcesResult{
|
|
UUID: uuid,
|
|
FromVCPUs: 2, TargetVCPUs: 4, AppliedVCPUs: 4,
|
|
FromMemoryMiB: 2048, TargetMemoryMiB: 4096, AppliedMemoryMiB: 4096,
|
|
FromDiskGiB: 40, TargetDiskGiB: 80, AppliedDiskGiB: 80,
|
|
FromBandwidthMbps: 100, TargetBandwidthMbps: 200, AppliedBandwidthMbps: 200,
|
|
Status: VMStateRunning, OriginallyRunning: true,
|
|
PowerStateRestored: true, ResourcesResized: true,
|
|
}
|
|
raw, err := json.Marshal(result)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
applied := buildOperationInspection(record, operation.Resolution{Resolved: true, Succeeded: true, Result: raw}, OperationAbortInspection{}, operationCleanupInspection{})
|
|
if applied.EffectResolution != InspectionEffectApplied || len(applied.EffectProjection) == 0 {
|
|
t.Fatalf("applied resize inspection = %#v", applied)
|
|
}
|
|
var projected ResizeResourcesResult
|
|
if err := json.Unmarshal(applied.EffectProjection, &projected); err != nil || !projected.ResourcesResized || projected.AppliedDiskGiB != 80 {
|
|
t.Fatalf("resize projection = %s err=%v", applied.EffectProjection, err)
|
|
}
|
|
|
|
partial := buildOperationInspection(record, operation.Resolution{
|
|
Resolved: true, Succeeded: false, ErrorCode: "resources_partially_resized",
|
|
}, OperationAbortInspection{}, operationCleanupInspection{})
|
|
if partial.EffectResolution != InspectionEffectPartial || partial.Outcome != InspectionOutcomeUncertain || len(partial.EffectProjection) != 0 {
|
|
t.Fatalf("partial resize inspection = %#v", partial)
|
|
}
|
|
|
|
invalidPower := result
|
|
invalidPower.OriginallyRunning = false
|
|
invalidPower.Status = VMStateRunning
|
|
invalidRaw, err := json.Marshal(invalidPower)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
invalid := buildOperationInspection(record, operation.Resolution{
|
|
Resolved: true, Succeeded: true, Result: invalidRaw,
|
|
}, OperationAbortInspection{}, operationCleanupInspection{})
|
|
if invalid.EffectResolution != InspectionEffectInconclusive ||
|
|
invalid.Outcome != InspectionOutcomeUncertain || len(invalid.EffectProjection) != 0 {
|
|
t.Fatalf("invalid power-state resize inspection = %#v", invalid)
|
|
}
|
|
}
|
|
|
|
func TestReconcileResizePowerObservationRequiresOriginalRunningStateRestored(t *testing.T) {
|
|
for name, observation := range map[string]operation.Resolution{
|
|
"restart pending": reconcileResizePowerObservation(true, false, false),
|
|
"stale live source": reconcileResizePowerObservation(true, true, false),
|
|
"originally stopped now live": reconcileResizePowerObservation(false, true, true),
|
|
} {
|
|
if !observation.Resolved || observation.Succeeded ||
|
|
observation.ErrorCode != "resources_partially_resized" {
|
|
t.Fatalf("%s observation = %#v", name, observation)
|
|
}
|
|
}
|
|
for name, observation := range map[string]operation.Resolution{
|
|
"running target": reconcileResizePowerObservation(true, true, true),
|
|
"originally stopped": reconcileResizePowerObservation(false, false, false),
|
|
} {
|
|
if !observation.Resolved || !observation.Succeeded {
|
|
t.Fatalf("%s observation = %#v", name, observation)
|
|
}
|
|
}
|
|
}
|