//go:build libvirt package hypervisor import ( "context" "errors" "fmt" "reflect" "sync/atomic" "testing" "time" "git.luoli.live/root/virt/agent/operation" "libvirt.org/go/libvirt" ) type managedLibvirtTestBackend struct { Unsupported ping func(context.Context) error createVM func(context.Context, CreateVMRequest) (VMMetadata, error) resizeResources func(context.Context, string, ResizeResourcesRequest) (ResizeResourcesResult, error) } func (b *managedLibvirtTestBackend) ResizeResources(ctx context.Context, uuid string, request ResizeResourcesRequest) (ResizeResourcesResult, error) { if b.resizeResources != nil { return b.resizeResources(ctx, uuid, request) } return ResizeResourcesResult{}, nil } func (b *managedLibvirtTestBackend) Ping(ctx context.Context) error { if b.ping != nil { return b.ping(ctx) } return nil } func (b *managedLibvirtTestBackend) CreateVM(ctx context.Context, request CreateVMRequest) (VMMetadata, error) { if b.createVM != nil { return b.createVM(ctx, request) } return VMMetadata{}, nil } func newManagedLibvirtTestSession(generation uint64, backend Hypervisor, closed *atomic.Int32) *managedLibvirtSession { return &managedLibvirtSession{ generation: generation, backend: backend, drained: make(chan struct{}), closeFn: func() error { closed.Add(1) return nil }, } } func waitForManagedLibvirtGeneration(t *testing.T, manager *managedLibvirt, generation uint64) *managedLibvirtSession { t.Helper() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { manager.mu.Lock() session := manager.current manager.mu.Unlock() if session != nil && session.generation == generation && !session.dead.Load() { return session } time.Sleep(time.Millisecond) } t.Fatalf("libvirt generation %d was not published", generation) return nil } func waitForAtomicValue(t *testing.T, value *atomic.Int32, expected int32) { t.Helper() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if value.Load() == expected { return } time.Sleep(time.Millisecond) } t.Fatalf("value = %d, want %d", value.Load(), expected) } func TestManagedLibvirtInitialFailureReconnects(t *testing.T) { var attempts atomic.Int32 var closed atomic.Int32 manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { if attempts.Add(1) == 1 { return nil, NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", errors.New("daemon down")) } return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil }, func(int) time.Duration { return time.Millisecond }) t.Cleanup(func() { _ = manager.Close() }) manager.start() waitForManagedLibvirtGeneration(t, manager, 2) if err := manager.Ping(context.Background()); err != nil { t.Fatalf("ping after reconnect: %v", err) } if attempts.Load() != 2 { t.Fatalf("connection attempts = %d, want 2", attempts.Load()) } } func TestManagedLibvirtDelegatesResourceResizeAsMutation(t *testing.T) { var closed atomic.Int32 called := false request := ResizeResourcesRequest{OperationID: "operation-resize-wrapper", ConfirmShutdown: true} backend := &managedLibvirtTestBackend{resizeResources: func(_ context.Context, uuid string, got ResizeResourcesRequest) (ResizeResourcesResult, error) { called = uuid == "vm-resize-wrapper" && reflect.DeepEqual(got, request) return ResizeResourcesResult{UUID: uuid, ResourcesResized: true}, nil }} manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { return newManagedLibvirtTestSession(generation, backend, &closed), nil }, func(int) time.Duration { return time.Millisecond }) t.Cleanup(func() { _ = manager.Close() }) manager.start() waitForManagedLibvirtGeneration(t, manager, 1) result, err := manager.ResizeResources(context.Background(), "vm-resize-wrapper", request) if err != nil || !called || result.UUID != "vm-resize-wrapper" || !result.ResourcesResized { t.Fatalf("managed resize result=%#v called=%v err=%v", result, called, err) } } func TestManagedLibvirtReconnectPublishesNewSessionBeforeOldCallDrains(t *testing.T) { started := make(chan struct{}) release := make(chan struct{}) var firstClosed atomic.Int32 var secondClosed atomic.Int32 var secondPings atomic.Int32 var attempts atomic.Int32 firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) { close(started) <-release return VMMetadata{UUID: "first-result"}, nil }} secondBackend := &managedLibvirtTestBackend{ping: func(context.Context) error { secondPings.Add(1) return nil }} manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { if attempts.Add(1) == 1 { return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil } return newManagedLibvirtTestSession(generation, secondBackend, &secondClosed), nil }, func(int) time.Duration { return time.Millisecond }) t.Cleanup(func() { select { case <-release: default: close(release) } _ = manager.Close() }) manager.start() firstSession := waitForManagedLibvirtGeneration(t, manager, 1) callDone := make(chan error, 1) go func() { _, err := manager.CreateVM(context.Background(), CreateVMRequest{}) callDone <- err }() select { case <-started: case <-time.After(time.Second): t.Fatal("first generation call did not start") } manager.markConnectionLost(firstSession) secondSession := waitForManagedLibvirtGeneration(t, manager, 2) if secondSession == firstSession { t.Fatal("reconnect reused the retired session") } if err := manager.Ping(context.Background()); err != nil { t.Fatalf("new generation ping: %v", err) } if secondPings.Load() != 1 { t.Fatalf("new calls did not use the published generation: pings = %d", secondPings.Load()) } if firstClosed.Load() != 0 { t.Fatal("old connection closed while an in-flight call still held it") } close(release) if err := <-callDone; err != nil { t.Fatalf("late result from old generation: %v", err) } waitForAtomicValue(t, &firstClosed, 1) // A delayed callback from the retired connection must not evict the // replacement generation or start another reconnect. manager.markConnectionLost(firstSession) if current := waitForManagedLibvirtGeneration(t, manager, 2); current != secondSession { t.Fatal("stale generation replaced the current session") } } func TestManagedLibvirtDisconnectedMutationWasNotDispatched(t *testing.T) { manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) { return nil, errors.New("not used") }, nil) t.Cleanup(func() { _ = manager.Close() }) _, mutationErr := manager.CreateVM(context.Background(), CreateVMRequest{}) if !operation.IsNotApplied(mutationErr) || operation.IsUncertain(mutationErr) { t.Fatalf("disconnected mutation error = %v, want explicit not-applied", mutationErr) } if !errors.Is(mutationErr, ErrUnavailable) { t.Fatalf("disconnected mutation error = %v, want unavailable cause", mutationErr) } readErr := manager.Ping(context.Background()) if operation.IsUncertain(readErr) { t.Fatalf("read-only ping error = %v, must not be marked uncertain", readErr) } if !errors.Is(readErr, ErrUnavailable) { t.Fatalf("read-only ping error = %v, want unavailable", readErr) } } func TestManagedLibvirtAcquireContextFailureWasNotDispatched(t *testing.T) { var closed atomic.Int32 var calls atomic.Int32 backend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) { calls.Add(1) return VMMetadata{}, nil }} manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { return newManagedLibvirtTestSession(generation, backend, &closed), nil }, func(int) time.Duration { return time.Millisecond }) t.Cleanup(func() { _ = manager.Close() }) manager.start() waitForManagedLibvirtGeneration(t, manager, 1) canceled, cancel := context.WithCancel(context.Background()) cancel() deadline, deadlineCancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) defer deadlineCancel() for _, test := range []struct { name string ctx context.Context want error }{ {name: "canceled", ctx: canceled, want: context.Canceled}, {name: "deadline", ctx: deadline, want: context.DeadlineExceeded}, } { t.Run(test.name, func(t *testing.T) { _, err := manager.CreateVM(test.ctx, CreateVMRequest{}) if !errors.Is(err, test.want) || !operation.IsNotApplied(err) || operation.IsUncertain(err) { t.Fatalf("acquire failure = %v, want not-applied %v", err, test.want) } }) } if calls.Load() != 0 { t.Fatalf("backend mutation calls = %d, want 0", calls.Load()) } } func TestManagedLibvirtTransportErrorRetiresSessionAndReconnects(t *testing.T) { var attempts atomic.Int32 var firstClosed atomic.Int32 var secondClosed atomic.Int32 firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) { return VMMetadata{}, libvirt.Error{Code: libvirt.ERR_RPC, Message: "connection lost"} }} manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { if attempts.Add(1) == 1 { return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil } return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &secondClosed), nil }, func(int) time.Duration { return time.Millisecond }) t.Cleanup(func() { _ = manager.Close() }) manager.start() waitForManagedLibvirtGeneration(t, manager, 1) _, err := manager.CreateVM(context.Background(), CreateVMRequest{}) if !operation.IsUncertain(err) { t.Fatalf("transport failure = %v, want uncertain", err) } waitForManagedLibvirtGeneration(t, manager, 2) waitForAtomicValue(t, &firstClosed, 1) if err := manager.Ping(context.Background()); err != nil { t.Fatalf("ping after transport reconnect: %v", err) } } func TestManagedLibvirtPreservesExplicitPreDispatchNotAppliedTransportFailure(t *testing.T) { var attempts atomic.Int32 var firstClosed atomic.Int32 var secondClosed atomic.Int32 preDispatch := operation.MarkNotApplied(libvirt.Error{Code: libvirt.ERR_RPC, Message: "connection lost before dispatch"}) firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) { return VMMetadata{}, preDispatch }} manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { if attempts.Add(1) == 1 { return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil } return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &secondClosed), nil }, func(int) time.Duration { return time.Millisecond }) t.Cleanup(func() { _ = manager.Close() }) manager.start() waitForManagedLibvirtGeneration(t, manager, 1) _, err := manager.CreateVM(context.Background(), CreateVMRequest{}) if !operation.IsNotApplied(err) || operation.IsUncertain(err) || !errors.Is(err, preDispatch) { t.Fatalf("pre-dispatch transport failure = %v", err) } waitForManagedLibvirtGeneration(t, manager, 2) waitForAtomicValue(t, &firstClosed, 1) if err := manager.Ping(context.Background()); err != nil { t.Fatalf("ping after pre-dispatch transport reconnect: %v", err) } } func TestManagedLibvirtCloseWaitsForReferencesAndStopsReconnect(t *testing.T) { started := make(chan struct{}) release := make(chan struct{}) var attempts atomic.Int32 var closed atomic.Int32 backend := &managedLibvirtTestBackend{ping: func(context.Context) error { close(started) <-release return nil }} manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { attempts.Add(1) return newManagedLibvirtTestSession(generation, backend, &closed), nil }, func(int) time.Duration { return time.Millisecond }) manager.start() waitForManagedLibvirtGeneration(t, manager, 1) pingDone := make(chan error, 1) go func() { pingDone <- manager.Ping(context.Background()) }() select { case <-started: case <-time.After(time.Second): t.Fatal("ping did not acquire the session") } closeDone := make(chan error, 1) go func() { closeDone <- manager.Close() }() select { case err := <-closeDone: t.Fatalf("Close returned before the active reference drained: %v", err) case <-time.After(20 * time.Millisecond): } if closed.Load() != 0 { t.Fatal("session closed before its active reference drained") } close(release) if err := <-pingDone; err != nil { t.Fatalf("in-flight ping: %v", err) } select { case err := <-closeDone: if err != nil { t.Fatalf("Close: %v", err) } case <-time.After(time.Second): t.Fatal("Close did not finish after the reference drained") } if closed.Load() != 1 { t.Fatalf("session close count = %d, want 1", closed.Load()) } manager.signalReconnect() time.Sleep(20 * time.Millisecond) if attempts.Load() != 1 { t.Fatalf("connection attempts after Close = %d, want 1", attempts.Load()) } } func TestManagedLibvirtCloseWaitsForBlockingStartup(t *testing.T) { entered := make(chan struct{}) release := make(chan struct{}) startDone := make(chan struct{}) cleanupErr := errors.New("startup session cleanup failed") var closed atomic.Int32 manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { close(entered) <-release session := newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed) session.closeFn = func() error { closed.Add(1) return cleanupErr } return session, nil }, func(int) time.Duration { return time.Millisecond }) go func() { manager.start() close(startDone) }() select { case <-entered: case <-time.After(time.Second): t.Fatal("startup factory did not begin") } closeDone := make(chan error, 1) go func() { closeDone <- manager.Close() }() select { case err := <-closeDone: t.Fatalf("Close returned while startup was still blocked: %v", err) case <-time.After(20 * time.Millisecond): } close(release) select { case <-startDone: case <-time.After(time.Second): t.Fatal("startup did not finish after factory release") } select { case err := <-closeDone: if !errors.Is(err, cleanupErr) { t.Fatalf("Close error = %v, want startup cleanup error", err) } case <-time.After(time.Second): t.Fatal("Close did not wait for startup cleanup") } if closed.Load() != 1 { t.Fatalf("startup session close count = %d, want 1", closed.Load()) } } func TestManagedLibvirtStartAfterCloseDoesNotConnect(t *testing.T) { var attempts atomic.Int32 manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { attempts.Add(1) var closed atomic.Int32 return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil }, func(int) time.Duration { return time.Millisecond }) if err := manager.Close(); err != nil { t.Fatalf("Close: %v", err) } manager.start() if attempts.Load() != 0 { t.Fatalf("connection attempts after Close = %d, want 0", attempts.Load()) } } func TestLibvirtEventBroadcasterAdvancesGeneration(t *testing.T) { broadcaster := newLibvirtEventBroadcaster() generation, wake := broadcaster.snapshot() broadcaster.notify() if err := broadcaster.wait(context.Background(), generation, wake); err != nil { t.Fatalf("wait after notify: %v", err) } nextGeneration, _ := broadcaster.snapshot() if nextGeneration != generation+1 { t.Fatalf("generation = %d, want %d", nextGeneration, generation+1) } } func TestLibvirtConnectionCleanupUsesReverseRegistrationOrder(t *testing.T) { var calls []string cleanup := &libvirtConnectionCleanup{ closeRegistered: true, jobID: 22, deregister: func(id int) error { calls = append(calls, "event:"+fmt.Sprint(id)) return nil }, unregisterClose: func() error { calls = append(calls, "close-callback") return nil }, closeConnection: func() error { calls = append(calls, "connection") return nil }, } if err := cleanup.run(); err != nil { t.Fatalf("cleanup: %v", err) } want := []string{"event:22", "close-callback", "connection"} if !reflect.DeepEqual(calls, want) { t.Fatalf("cleanup order = %v, want %v", calls, want) } } func TestLibvirtConnectionCleanupHandlesPartialRegistration(t *testing.T) { var calls []string cleanup := &libvirtConnectionCleanup{ closeRegistered: true, jobID: -1, deregister: func(id int) error { calls = append(calls, "event:"+fmt.Sprint(id)) return nil }, unregisterClose: func() error { calls = append(calls, "close-callback") return nil }, closeConnection: func() error { calls = append(calls, "connection") return nil }, } if err := cleanup.run(); err != nil { t.Fatalf("cleanup: %v", err) } want := []string{"close-callback", "connection"} if !reflect.DeepEqual(calls, want) { t.Fatalf("partial cleanup order = %v, want %v", calls, want) } } func TestManagedLibvirtIgnoresExpectedDeadSessionCleanupError(t *testing.T) { manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) { return nil, errors.New("not used") }, nil) deadSession := &managedLibvirtSession{closeFn: func() error { return fmt.Errorf("unregister stale callback: %w", libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) }} deadSession.dead.Store(true) manager.cleanupSession(deadSession) if len(manager.cleanupErrs) != 0 { t.Fatalf("dead connection cleanup errors were retained: %v", manager.cleanupErrs) } liveSession := &managedLibvirtSession{closeFn: func() error { return errors.New("close failed") }} manager.cleanupSession(liveSession) if len(manager.cleanupErrs) != 1 { t.Fatalf("live connection cleanup errors = %v, want one retained error", manager.cleanupErrs) } mixedSession := &managedLibvirtSession{closeFn: func() error { return errors.Join(libvirt.Error{Code: libvirt.ERR_NO_CONNECT}, errors.New("unexpected cleanup failure")) }} mixedSession.dead.Store(true) manager.cleanupSession(mixedSession) if len(manager.cleanupErrs) != 2 { t.Fatalf("mixed dead connection cleanup errors = %v, want unexpected error retained", manager.cleanupErrs) } }