You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_connection_test.go

538 lines
18 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"errors"
"fmt"
"reflect"
"sync/atomic"
"testing"
"time"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
)
type managedLibvirtTestBackend struct {
Unsupported
ping func(context.Context) error
createVM func(context.Context, CreateVMRequest) (VMMetadata, error)
resizeResources func(context.Context, string, ResizeResourcesRequest) (ResizeResourcesResult, error)
}
func (b *managedLibvirtTestBackend) ResizeResources(ctx context.Context, uuid string, request ResizeResourcesRequest) (ResizeResourcesResult, error) {
if b.resizeResources != nil {
return b.resizeResources(ctx, uuid, request)
}
return ResizeResourcesResult{}, nil
}
func (b *managedLibvirtTestBackend) Ping(ctx context.Context) error {
if b.ping != nil {
return b.ping(ctx)
}
return nil
}
func (b *managedLibvirtTestBackend) CreateVM(ctx context.Context, request CreateVMRequest) (VMMetadata, error) {
if b.createVM != nil {
return b.createVM(ctx, request)
}
return VMMetadata{}, nil
}
func newManagedLibvirtTestSession(generation uint64, backend Hypervisor, closed *atomic.Int32) *managedLibvirtSession {
return &managedLibvirtSession{
generation: generation,
backend: backend,
drained: make(chan struct{}),
closeFn: func() error {
closed.Add(1)
return nil
},
}
}
func waitForManagedLibvirtGeneration(t *testing.T, manager *managedLibvirt, generation uint64) *managedLibvirtSession {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
manager.mu.Lock()
session := manager.current
manager.mu.Unlock()
if session != nil && session.generation == generation && !session.dead.Load() {
return session
}
time.Sleep(time.Millisecond)
}
t.Fatalf("libvirt generation %d was not published", generation)
return nil
}
func waitForAtomicValue(t *testing.T, value *atomic.Int32, expected int32) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if value.Load() == expected {
return
}
time.Sleep(time.Millisecond)
}
t.Fatalf("value = %d, want %d", value.Load(), expected)
}
func TestManagedLibvirtInitialFailureReconnects(t *testing.T) {
var attempts atomic.Int32
var closed atomic.Int32
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
if attempts.Add(1) == 1 {
return nil, NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", errors.New("daemon down"))
}
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil
}, func(int) time.Duration { return time.Millisecond })
t.Cleanup(func() { _ = manager.Close() })
manager.start()
waitForManagedLibvirtGeneration(t, manager, 2)
if err := manager.Ping(context.Background()); err != nil {
t.Fatalf("ping after reconnect: %v", err)
}
if attempts.Load() != 2 {
t.Fatalf("connection attempts = %d, want 2", attempts.Load())
}
}
func TestManagedLibvirtDelegatesResourceResizeAsMutation(t *testing.T) {
var closed atomic.Int32
called := false
request := ResizeResourcesRequest{OperationID: "operation-resize-wrapper", ConfirmShutdown: true}
backend := &managedLibvirtTestBackend{resizeResources: func(_ context.Context, uuid string, got ResizeResourcesRequest) (ResizeResourcesResult, error) {
called = uuid == "vm-resize-wrapper" && reflect.DeepEqual(got, request)
return ResizeResourcesResult{UUID: uuid, ResourcesResized: true}, nil
}}
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
return newManagedLibvirtTestSession(generation, backend, &closed), nil
}, func(int) time.Duration { return time.Millisecond })
t.Cleanup(func() { _ = manager.Close() })
manager.start()
waitForManagedLibvirtGeneration(t, manager, 1)
result, err := manager.ResizeResources(context.Background(), "vm-resize-wrapper", request)
if err != nil || !called || result.UUID != "vm-resize-wrapper" || !result.ResourcesResized {
t.Fatalf("managed resize result=%#v called=%v err=%v", result, called, err)
}
}
func TestManagedLibvirtReconnectPublishesNewSessionBeforeOldCallDrains(t *testing.T) {
started := make(chan struct{})
release := make(chan struct{})
var firstClosed atomic.Int32
var secondClosed atomic.Int32
var secondPings atomic.Int32
var attempts atomic.Int32
firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
close(started)
<-release
return VMMetadata{UUID: "first-result"}, nil
}}
secondBackend := &managedLibvirtTestBackend{ping: func(context.Context) error {
secondPings.Add(1)
return nil
}}
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
if attempts.Add(1) == 1 {
return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil
}
return newManagedLibvirtTestSession(generation, secondBackend, &secondClosed), nil
}, func(int) time.Duration { return time.Millisecond })
t.Cleanup(func() {
select {
case <-release:
default:
close(release)
}
_ = manager.Close()
})
manager.start()
firstSession := waitForManagedLibvirtGeneration(t, manager, 1)
callDone := make(chan error, 1)
go func() {
_, err := manager.CreateVM(context.Background(), CreateVMRequest{})
callDone <- err
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("first generation call did not start")
}
manager.markConnectionLost(firstSession)
secondSession := waitForManagedLibvirtGeneration(t, manager, 2)
if secondSession == firstSession {
t.Fatal("reconnect reused the retired session")
}
if err := manager.Ping(context.Background()); err != nil {
t.Fatalf("new generation ping: %v", err)
}
if secondPings.Load() != 1 {
t.Fatalf("new calls did not use the published generation: pings = %d", secondPings.Load())
}
if firstClosed.Load() != 0 {
t.Fatal("old connection closed while an in-flight call still held it")
}
close(release)
if err := <-callDone; err != nil {
t.Fatalf("late result from old generation: %v", err)
}
waitForAtomicValue(t, &firstClosed, 1)
// A delayed callback from the retired connection must not evict the
// replacement generation or start another reconnect.
manager.markConnectionLost(firstSession)
if current := waitForManagedLibvirtGeneration(t, manager, 2); current != secondSession {
t.Fatal("stale generation replaced the current session")
}
}
func TestManagedLibvirtDisconnectedMutationWasNotDispatched(t *testing.T) {
manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) {
return nil, errors.New("not used")
}, nil)
t.Cleanup(func() { _ = manager.Close() })
_, mutationErr := manager.CreateVM(context.Background(), CreateVMRequest{})
if !operation.IsNotApplied(mutationErr) || operation.IsUncertain(mutationErr) {
t.Fatalf("disconnected mutation error = %v, want explicit not-applied", mutationErr)
}
if !errors.Is(mutationErr, ErrUnavailable) {
t.Fatalf("disconnected mutation error = %v, want unavailable cause", mutationErr)
}
readErr := manager.Ping(context.Background())
if operation.IsUncertain(readErr) {
t.Fatalf("read-only ping error = %v, must not be marked uncertain", readErr)
}
if !errors.Is(readErr, ErrUnavailable) {
t.Fatalf("read-only ping error = %v, want unavailable", readErr)
}
}
func TestManagedLibvirtAcquireContextFailureWasNotDispatched(t *testing.T) {
var closed atomic.Int32
var calls atomic.Int32
backend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
calls.Add(1)
return VMMetadata{}, nil
}}
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
return newManagedLibvirtTestSession(generation, backend, &closed), nil
}, func(int) time.Duration { return time.Millisecond })
t.Cleanup(func() { _ = manager.Close() })
manager.start()
waitForManagedLibvirtGeneration(t, manager, 1)
canceled, cancel := context.WithCancel(context.Background())
cancel()
deadline, deadlineCancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
defer deadlineCancel()
for _, test := range []struct {
name string
ctx context.Context
want error
}{
{name: "canceled", ctx: canceled, want: context.Canceled},
{name: "deadline", ctx: deadline, want: context.DeadlineExceeded},
} {
t.Run(test.name, func(t *testing.T) {
_, err := manager.CreateVM(test.ctx, CreateVMRequest{})
if !errors.Is(err, test.want) || !operation.IsNotApplied(err) || operation.IsUncertain(err) {
t.Fatalf("acquire failure = %v, want not-applied %v", err, test.want)
}
})
}
if calls.Load() != 0 {
t.Fatalf("backend mutation calls = %d, want 0", calls.Load())
}
}
func TestManagedLibvirtTransportErrorRetiresSessionAndReconnects(t *testing.T) {
var attempts atomic.Int32
var firstClosed atomic.Int32
var secondClosed atomic.Int32
firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
return VMMetadata{}, libvirt.Error{Code: libvirt.ERR_RPC, Message: "connection lost"}
}}
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
if attempts.Add(1) == 1 {
return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil
}
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &secondClosed), nil
}, func(int) time.Duration { return time.Millisecond })
t.Cleanup(func() { _ = manager.Close() })
manager.start()
waitForManagedLibvirtGeneration(t, manager, 1)
_, err := manager.CreateVM(context.Background(), CreateVMRequest{})
if !operation.IsUncertain(err) {
t.Fatalf("transport failure = %v, want uncertain", err)
}
waitForManagedLibvirtGeneration(t, manager, 2)
waitForAtomicValue(t, &firstClosed, 1)
if err := manager.Ping(context.Background()); err != nil {
t.Fatalf("ping after transport reconnect: %v", err)
}
}
func TestManagedLibvirtPreservesExplicitPreDispatchNotAppliedTransportFailure(t *testing.T) {
var attempts atomic.Int32
var firstClosed atomic.Int32
var secondClosed atomic.Int32
preDispatch := operation.MarkNotApplied(libvirt.Error{Code: libvirt.ERR_RPC, Message: "connection lost before dispatch"})
firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
return VMMetadata{}, preDispatch
}}
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
if attempts.Add(1) == 1 {
return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil
}
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &secondClosed), nil
}, func(int) time.Duration { return time.Millisecond })
t.Cleanup(func() { _ = manager.Close() })
manager.start()
waitForManagedLibvirtGeneration(t, manager, 1)
_, err := manager.CreateVM(context.Background(), CreateVMRequest{})
if !operation.IsNotApplied(err) || operation.IsUncertain(err) || !errors.Is(err, preDispatch) {
t.Fatalf("pre-dispatch transport failure = %v", err)
}
waitForManagedLibvirtGeneration(t, manager, 2)
waitForAtomicValue(t, &firstClosed, 1)
if err := manager.Ping(context.Background()); err != nil {
t.Fatalf("ping after pre-dispatch transport reconnect: %v", err)
}
}
func TestManagedLibvirtCloseWaitsForReferencesAndStopsReconnect(t *testing.T) {
started := make(chan struct{})
release := make(chan struct{})
var attempts atomic.Int32
var closed atomic.Int32
backend := &managedLibvirtTestBackend{ping: func(context.Context) error {
close(started)
<-release
return nil
}}
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
attempts.Add(1)
return newManagedLibvirtTestSession(generation, backend, &closed), nil
}, func(int) time.Duration { return time.Millisecond })
manager.start()
waitForManagedLibvirtGeneration(t, manager, 1)
pingDone := make(chan error, 1)
go func() { pingDone <- manager.Ping(context.Background()) }()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("ping did not acquire the session")
}
closeDone := make(chan error, 1)
go func() { closeDone <- manager.Close() }()
select {
case err := <-closeDone:
t.Fatalf("Close returned before the active reference drained: %v", err)
case <-time.After(20 * time.Millisecond):
}
if closed.Load() != 0 {
t.Fatal("session closed before its active reference drained")
}
close(release)
if err := <-pingDone; err != nil {
t.Fatalf("in-flight ping: %v", err)
}
select {
case err := <-closeDone:
if err != nil {
t.Fatalf("Close: %v", err)
}
case <-time.After(time.Second):
t.Fatal("Close did not finish after the reference drained")
}
if closed.Load() != 1 {
t.Fatalf("session close count = %d, want 1", closed.Load())
}
manager.signalReconnect()
time.Sleep(20 * time.Millisecond)
if attempts.Load() != 1 {
t.Fatalf("connection attempts after Close = %d, want 1", attempts.Load())
}
}
func TestManagedLibvirtCloseWaitsForBlockingStartup(t *testing.T) {
entered := make(chan struct{})
release := make(chan struct{})
startDone := make(chan struct{})
cleanupErr := errors.New("startup session cleanup failed")
var closed atomic.Int32
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
close(entered)
<-release
session := newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed)
session.closeFn = func() error {
closed.Add(1)
return cleanupErr
}
return session, nil
}, func(int) time.Duration { return time.Millisecond })
go func() {
manager.start()
close(startDone)
}()
select {
case <-entered:
case <-time.After(time.Second):
t.Fatal("startup factory did not begin")
}
closeDone := make(chan error, 1)
go func() { closeDone <- manager.Close() }()
select {
case err := <-closeDone:
t.Fatalf("Close returned while startup was still blocked: %v", err)
case <-time.After(20 * time.Millisecond):
}
close(release)
select {
case <-startDone:
case <-time.After(time.Second):
t.Fatal("startup did not finish after factory release")
}
select {
case err := <-closeDone:
if !errors.Is(err, cleanupErr) {
t.Fatalf("Close error = %v, want startup cleanup error", err)
}
case <-time.After(time.Second):
t.Fatal("Close did not wait for startup cleanup")
}
if closed.Load() != 1 {
t.Fatalf("startup session close count = %d, want 1", closed.Load())
}
}
func TestManagedLibvirtStartAfterCloseDoesNotConnect(t *testing.T) {
var attempts atomic.Int32
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
attempts.Add(1)
var closed atomic.Int32
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil
}, func(int) time.Duration { return time.Millisecond })
if err := manager.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
manager.start()
if attempts.Load() != 0 {
t.Fatalf("connection attempts after Close = %d, want 0", attempts.Load())
}
}
func TestLibvirtEventBroadcasterAdvancesGeneration(t *testing.T) {
broadcaster := newLibvirtEventBroadcaster()
generation, wake := broadcaster.snapshot()
broadcaster.notify()
if err := broadcaster.wait(context.Background(), generation, wake); err != nil {
t.Fatalf("wait after notify: %v", err)
}
nextGeneration, _ := broadcaster.snapshot()
if nextGeneration != generation+1 {
t.Fatalf("generation = %d, want %d", nextGeneration, generation+1)
}
}
func TestLibvirtConnectionCleanupUsesReverseRegistrationOrder(t *testing.T) {
var calls []string
cleanup := &libvirtConnectionCleanup{
closeRegistered: true,
jobID: 22,
deregister: func(id int) error {
calls = append(calls, "event:"+fmt.Sprint(id))
return nil
},
unregisterClose: func() error {
calls = append(calls, "close-callback")
return nil
},
closeConnection: func() error {
calls = append(calls, "connection")
return nil
},
}
if err := cleanup.run(); err != nil {
t.Fatalf("cleanup: %v", err)
}
want := []string{"event:22", "close-callback", "connection"}
if !reflect.DeepEqual(calls, want) {
t.Fatalf("cleanup order = %v, want %v", calls, want)
}
}
func TestLibvirtConnectionCleanupHandlesPartialRegistration(t *testing.T) {
var calls []string
cleanup := &libvirtConnectionCleanup{
closeRegistered: true,
jobID: -1,
deregister: func(id int) error {
calls = append(calls, "event:"+fmt.Sprint(id))
return nil
},
unregisterClose: func() error {
calls = append(calls, "close-callback")
return nil
},
closeConnection: func() error {
calls = append(calls, "connection")
return nil
},
}
if err := cleanup.run(); err != nil {
t.Fatalf("cleanup: %v", err)
}
want := []string{"close-callback", "connection"}
if !reflect.DeepEqual(calls, want) {
t.Fatalf("partial cleanup order = %v, want %v", calls, want)
}
}
func TestManagedLibvirtIgnoresExpectedDeadSessionCleanupError(t *testing.T) {
manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) {
return nil, errors.New("not used")
}, nil)
deadSession := &managedLibvirtSession{closeFn: func() error {
return fmt.Errorf("unregister stale callback: %w", libvirt.Error{Code: libvirt.ERR_INVALID_CONN})
}}
deadSession.dead.Store(true)
manager.cleanupSession(deadSession)
if len(manager.cleanupErrs) != 0 {
t.Fatalf("dead connection cleanup errors were retained: %v", manager.cleanupErrs)
}
liveSession := &managedLibvirtSession{closeFn: func() error { return errors.New("close failed") }}
manager.cleanupSession(liveSession)
if len(manager.cleanupErrs) != 1 {
t.Fatalf("live connection cleanup errors = %v, want one retained error", manager.cleanupErrs)
}
mixedSession := &managedLibvirtSession{closeFn: func() error {
return errors.Join(libvirt.Error{Code: libvirt.ERR_NO_CONNECT}, errors.New("unexpected cleanup failure"))
}}
mixedSession.dead.Store(true)
manager.cleanupSession(mixedSession)
if len(manager.cleanupErrs) != 2 {
t.Fatalf("mixed dead connection cleanup errors = %v, want unexpected error retained", manager.cleanupErrs)
}
}