//go:build libvirt package hypervisor import ( "context" "encoding/json" "errors" "testing" "time" "git.luoli.live/root/virt/agent/operation" "libvirt.org/go/libvirt" "libvirt.org/go/libvirtxml" ) type fakeResourceResizeDomain struct { states []libvirt.DomainState stateErrors []error active []bool activeErrors []error shutdownErr error createErr error shutdownCalls int createCalls int mutationStarted *bool mutationAtShutdown bool } func (f *fakeResourceResizeDomain) GetState() (libvirt.DomainState, int, error) { state := libvirt.DOMAIN_NOSTATE if len(f.states) != 0 { state = f.states[0] f.states = f.states[1:] } var err error if len(f.stateErrors) != 0 { err = f.stateErrors[0] f.stateErrors = f.stateErrors[1:] } return state, 0, err } func (f *fakeResourceResizeDomain) IsActive() (bool, error) { var active bool if len(f.active) != 0 { active = f.active[0] f.active = f.active[1:] } var err error if len(f.activeErrors) != 0 { err = f.activeErrors[0] f.activeErrors = f.activeErrors[1:] } return active, err } func (f *fakeResourceResizeDomain) ShutdownFlags(libvirt.DomainShutdownFlags) error { f.shutdownCalls++ if f.mutationStarted != nil { f.mutationAtShutdown = *f.mutationStarted } return f.shutdownErr } func (f *fakeResourceResizeDomain) Create() error { f.createCalls++ return f.createErr } func TestResourceResizePowerLifecycleUsesGracefulShutdownAndRestoresOriginalState(t *testing.T) { t.Run("running guest is stopped after mutation boundary and cold started", func(t *testing.T) { mutationStarted := false domain := &fakeResourceResizeDomain{ states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING, libvirt.DOMAIN_RUNNING}, active: []bool{true, true, false, true}, mutationStarted: &mutationStarted, } wasRunning, err := stopForResourceResize(context.Background(), domain, time.Second, func() { mutationStarted = true }) if err != nil || !wasRunning || domain.shutdownCalls != 1 || !domain.mutationAtShutdown { t.Fatalf("stop result running=%v shutdown=%d boundary=%v err=%v", wasRunning, domain.shutdownCalls, domain.mutationAtShutdown, err) } status, err := restoreAfterResourceResize(domain, wasRunning, domain.Create) if err != nil || status != VMStateRunning || domain.createCalls != 1 { t.Fatalf("restore status=%q create=%d err=%v", status, domain.createCalls, err) } }) t.Run("inactive guest remains inactive", func(t *testing.T) { domain := &fakeResourceResizeDomain{ states: []libvirt.DomainState{libvirt.DOMAIN_SHUTOFF}, active: []bool{false, false}, } wasRunning, err := stopForResourceResize(context.Background(), domain, time.Second, nil) if err != nil || wasRunning || domain.shutdownCalls != 0 { t.Fatalf("stop result running=%v shutdown=%d err=%v", wasRunning, domain.shutdownCalls, err) } status, err := restoreAfterResourceResize(domain, wasRunning, domain.Create) if err != nil || status != VMStateShutoff || domain.createCalls != 0 { t.Fatalf("restore status=%q create=%d err=%v", status, domain.createCalls, err) } }) } func TestResourceResizePowerLifecycleNeverForcesAndKeepsUncertainResultsFenced(t *testing.T) { t.Run("guest that ignores ACPI shutdown times out without a force path", func(t *testing.T) { mutationStarted := false domain := &fakeResourceResizeDomain{ states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING}, active: []bool{true, true, true, true}, mutationStarted: &mutationStarted, } wasRunning, err := stopForResourceResize(context.Background(), domain, time.Nanosecond, func() { mutationStarted = true }) var backend *Error if !wasRunning || domain.shutdownCalls != 1 || domain.createCalls != 0 || !operation.IsUncertain(err) || !errors.As(err, &backend) || backend.Code != "upgrade_shutdown_timeout" { t.Fatalf("timeout running=%v shutdown=%d create=%d err=%v", wasRunning, domain.shutdownCalls, domain.createCalls, err) } }) t.Run("lost start response accepted only when running is observed", func(t *testing.T) { cause := errors.New("start response lost") domain := &fakeResourceResizeDomain{ states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING}, active: []bool{true}, createErr: cause, } status, err := restoreAfterResourceResize(domain, true, domain.Create) if err != nil || status != VMStateRunning || domain.createCalls != 1 { t.Fatalf("confirmed start status=%q calls=%d err=%v", status, domain.createCalls, err) } }) t.Run("unconfirmed cold start remains uncertain", func(t *testing.T) { cause := errors.New("start failed") domain := &fakeResourceResizeDomain{active: []bool{false}, createErr: cause} _, err := restoreAfterResourceResize(domain, true, domain.Create) if domain.createCalls != 1 || !operation.IsUncertain(err) || !errors.Is(err, cause) { t.Fatalf("unconfirmed start calls=%d err=%v", domain.createCalls, err) } }) t.Run("paused guest is rejected before any lifecycle mutation", func(t *testing.T) { domain := &fakeResourceResizeDomain{states: []libvirt.DomainState{libvirt.DOMAIN_PAUSED}} wasRunning, err := stopForResourceResize(context.Background(), domain, time.Second, func() { t.Fatal("mutation boundary crossed for paused guest") }) var backend *Error if wasRunning || domain.shutdownCalls != 0 || !errors.As(err, &backend) || backend.Code != "vm_state_not_upgradeable" || operation.IsUncertain(err) { t.Fatalf("paused result running=%v shutdown=%d err=%v", wasRunning, domain.shutdownCalls, err) } }) } func TestResourceResizeRestoresRunningGuestWhenPreResizeWorkFails(t *testing.T) { primary := errors.New("resource preflight changed") domain := &fakeResourceResizeDomain{ states: []libvirt.DomainState{libvirt.DOMAIN_RUNNING}, active: []bool{true}, } err := restoreOriginalPowerAfterPreResizeFailure(domain, true, false, primary, domain.Create) if !errors.Is(err, primary) || domain.createCalls != 1 { t.Fatalf("pre-resize restore calls=%d err=%v", domain.createCalls, err) } afterMutation := &fakeResourceResizeDomain{} err = restoreOriginalPowerAfterPreResizeFailure(afterMutation, true, true, primary, afterMutation.Create) if !errors.Is(err, primary) || afterMutation.createCalls != 0 { t.Fatalf("post-mutation restore calls=%d err=%v", afterMutation.createCalls, err) } } func TestResizeManagedDomainDocumentUpdatesStoppedResourcesAndMarker(t *testing.T) { document, err := BuildDomainXML(validDomainDefinition()) if err != nil { t.Fatal(err) } normalized, _, err := parseManagedDomain(document) if err != nil { t.Fatal(err) } normalized.VCPU.Current = normalized.VCPU.Value normalized.CurrentMemory = &libvirtxml.DomainCurrentMemory{ Value: normalized.Memory.Value * 1024, Unit: "KiB", } document, err = normalized.Marshal() if err != nil { t.Fatal(err) } target := resizeResourceValues{VCPUs: 4, MemoryMiB: 4096, DiskGiB: 80, BandwidthMbps: 250} updated, err := resizeManagedDomainDocument( document, target, "operation-resize-resources", true, true) if err != nil { t.Fatal(err) } definition, metadata, err := parseManagedDomain(updated) if err != nil { t.Fatal(err) } if metadata.ResourcesTransitionID != "operation-resize-resources" || metadata.ResourcesOriginalPower != resourceResizeOriginalPowerRunning || metadata.ResourcesOriginalAutostart != resourceResizeOriginalAutostartEnabled || !managedDefinitionMatchesResources(definition, metadata, target.DiskGiB*gib, target) { t.Fatalf("resized definition did not match target: metadata=%#v xml=%s", metadata, updated) } if definition.VCPU.Current != target.VCPUs || definition.CurrentMemory == nil || !memoryValueMatchesMiB(definition.CurrentMemory.Value, definition.CurrentMemory.Unit, target.MemoryMiB) { t.Fatalf("normalized current CPU/memory were not moved to the target: %#v", definition) } if managedDefinitionMatchesResources(definition, metadata, 40*gib, target) { t.Fatal("metadata-only target was accepted without the actual volume capacity") } stoppedUpdated, err := resizeManagedDomainDocument( document, target, "operation-resize-stopped", false, false) if err != nil { t.Fatal(err) } _, stoppedMetadata, err := parseManagedDomain(stoppedUpdated) if err != nil { t.Fatal(err) } stoppedWasRunning, stoppedMarkerValid := resizeWasOriginallyRunning(stoppedMetadata) stoppedAutostart, stoppedAutostartValid := resizeWasOriginallyAutostart(stoppedMetadata) if stoppedMetadata.ResourcesOriginalPower != resourceResizeOriginalPowerStopped || stoppedMetadata.ResourcesOriginalAutostart != resourceResizeOriginalAutostartDisabled || stoppedWasRunning || !stoppedMarkerValid || stoppedAutostart || !stoppedAutostartValid { t.Fatalf("stopped original-power marker was not explicit: %#v", stoppedMetadata) } if _, valid := resizeWasOriginallyRunning(&domainMetadataXML{}); valid { t.Fatal("missing original-power marker was accepted as an originally stopped VM") } if _, valid := resizeWasOriginallyAutostart(&domainMetadataXML{}); valid { t.Fatal("missing original-autostart marker was accepted as disabled autostart") } } func TestReconcileResizeObservationKeepsSourceIndeterminateAndDistinguishesAppliedAndPartial(t *testing.T) { document, err := BuildDomainXML(validDomainDefinition()) if err != nil { t.Fatal(err) } fromDefinition, fromMetadata, err := parseManagedDomain(document) if err != nil { t.Fatal(err) } expected := &operation.ExpectedState{ Kind: operation.ExpectedResizeResources, TransitionID: "operation-resize-reconcile", FromVCPUs: 2, TargetVCPUs: 4, FromMemoryMiB: 2048, TargetMemoryMiB: 4096, FromDiskGiB: 40, TargetDiskGiB: 80, FromBandwidthMbps: 100, TargetBandwidthMbps: 200, } for name, marker := range map[string]string{ "no marker": "", "prior transition": "operation-prior-resize", } { indeterminate := reconcileResizeObservation(expected, marker, fromDefinition, fromMetadata, 40*gib) if indeterminate.Resolved { t.Fatalf("%s source observation released a possibly delayed ACPI shutdown = %#v", name, indeterminate) } } updated, err := resizeManagedDomainDocument( document, expectedResizeTargetValues(expected), expected.TransitionID, true, true) if err != nil { t.Fatal(err) } targetDefinition, targetMetadata, err := parseManagedDomain(updated) if err != nil { t.Fatal(err) } applied := reconcileResizeObservation(expected, expected.TransitionID, targetDefinition, targetMetadata, 80*gib) if !applied.Resolved || !applied.Succeeded { t.Fatalf("applied observation = %#v", applied) } liveMatches, err := resizeLiveDocumentMatchesExpected( updated, expected, 80*gib, true, true) if err != nil || !liveMatches { t.Fatalf("target live definition matches=%v err=%v", liveMatches, err) } liveMatches, err = resizeLiveDocumentMatchesExpected( document, expected, 40*gib, true, true) if err != nil || liveMatches { t.Fatalf("source live definition matches=%v err=%v", liveMatches, err) } for name, partial := range map[string]operation.Resolution{ "volume only": reconcileResizeObservation(expected, "", fromDefinition, fromMetadata, 80*gib), "definition only": reconcileResizeObservation(expected, expected.TransitionID, targetDefinition, targetMetadata, 40*gib), "target without marker": reconcileResizeObservation(expected, "operation-other-resize", targetDefinition, targetMetadata, 80*gib), } { if !partial.Resolved || partial.Succeeded || partial.ErrorCode != "resources_partially_resized" { t.Fatalf("%s partial observation = %#v", name, partial) } } } func TestResizeInspectionPublishesOnlyBoundedProjectionAndKeepsPartialFenced(t *testing.T) { const uuid = "123e4567-e89b-42d3-a456-426614174000" record := operation.Record{ OperationID: "operation-resize-inspection", ResourceID: uuid, Action: "resize-vm", State: operation.StateFailed, EffectMayHaveApplied: true, ExpectedState: &operation.ExpectedState{ Kind: operation.ExpectedResizeResources, TransitionID: "operation-resize-inspection", FromVCPUs: 2, TargetVCPUs: 4, FromMemoryMiB: 2048, TargetMemoryMiB: 4096, FromDiskGiB: 40, TargetDiskGiB: 80, FromBandwidthMbps: 100, TargetBandwidthMbps: 200, }, } result := ResizeResourcesResult{ UUID: uuid, FromVCPUs: 2, TargetVCPUs: 4, AppliedVCPUs: 4, FromMemoryMiB: 2048, TargetMemoryMiB: 4096, AppliedMemoryMiB: 4096, FromDiskGiB: 40, TargetDiskGiB: 80, AppliedDiskGiB: 80, FromBandwidthMbps: 100, TargetBandwidthMbps: 200, AppliedBandwidthMbps: 200, Status: VMStateRunning, OriginallyRunning: true, PowerStateRestored: true, ResourcesResized: true, } raw, err := json.Marshal(result) if err != nil { t.Fatal(err) } applied := buildOperationInspection(record, operation.Resolution{Resolved: true, Succeeded: true, Result: raw}, OperationAbortInspection{}, operationCleanupInspection{}) if applied.EffectResolution != InspectionEffectApplied || len(applied.EffectProjection) == 0 { t.Fatalf("applied resize inspection = %#v", applied) } var projected ResizeResourcesResult if err := json.Unmarshal(applied.EffectProjection, &projected); err != nil || !projected.ResourcesResized || projected.AppliedDiskGiB != 80 { t.Fatalf("resize projection = %s err=%v", applied.EffectProjection, err) } partial := buildOperationInspection(record, operation.Resolution{ Resolved: true, Succeeded: false, ErrorCode: "resources_partially_resized", }, OperationAbortInspection{}, operationCleanupInspection{}) if partial.EffectResolution != InspectionEffectPartial || partial.Outcome != InspectionOutcomeUncertain || len(partial.EffectProjection) != 0 { t.Fatalf("partial resize inspection = %#v", partial) } invalidPower := result invalidPower.OriginallyRunning = false invalidPower.Status = VMStateRunning invalidRaw, err := json.Marshal(invalidPower) if err != nil { t.Fatal(err) } invalid := buildOperationInspection(record, operation.Resolution{ Resolved: true, Succeeded: true, Result: invalidRaw, }, OperationAbortInspection{}, operationCleanupInspection{}) if invalid.EffectResolution != InspectionEffectInconclusive || invalid.Outcome != InspectionOutcomeUncertain || len(invalid.EffectProjection) != 0 { t.Fatalf("invalid power-state resize inspection = %#v", invalid) } } func TestReconcileResizePowerObservationRequiresOriginalRunningStateRestored(t *testing.T) { for name, observation := range map[string]operation.Resolution{ "restart pending": reconcileResizePowerObservation(true, false, false), "stale live source": reconcileResizePowerObservation(true, true, false), "originally stopped now live": reconcileResizePowerObservation(false, true, true), } { if !observation.Resolved || observation.Succeeded || observation.ErrorCode != "resources_partially_resized" { t.Fatalf("%s observation = %#v", name, observation) } } for name, observation := range map[string]operation.Resolution{ "running target": reconcileResizePowerObservation(true, true, true), "originally stopped": reconcileResizePowerObservation(false, false, false), } { if !observation.Resolved || !observation.Succeeded { t.Fatalf("%s observation = %#v", name, observation) } } }