You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
443 lines
14 KiB
Go
443 lines
14 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
)
|
|
|
|
type managedLibvirtTestBackend struct {
|
|
Unsupported
|
|
ping func(context.Context) error
|
|
createVM func(context.Context, CreateVMRequest) (VMMetadata, error)
|
|
}
|
|
|
|
func (b *managedLibvirtTestBackend) Ping(ctx context.Context) error {
|
|
if b.ping != nil {
|
|
return b.ping(ctx)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *managedLibvirtTestBackend) CreateVM(ctx context.Context, request CreateVMRequest) (VMMetadata, error) {
|
|
if b.createVM != nil {
|
|
return b.createVM(ctx, request)
|
|
}
|
|
return VMMetadata{}, nil
|
|
}
|
|
|
|
func newManagedLibvirtTestSession(generation uint64, backend Hypervisor, closed *atomic.Int32) *managedLibvirtSession {
|
|
return &managedLibvirtSession{
|
|
generation: generation,
|
|
backend: backend,
|
|
drained: make(chan struct{}),
|
|
closeFn: func() error {
|
|
closed.Add(1)
|
|
return nil
|
|
},
|
|
}
|
|
}
|
|
|
|
func waitForManagedLibvirtGeneration(t *testing.T, manager *managedLibvirt, generation uint64) *managedLibvirtSession {
|
|
t.Helper()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
manager.mu.Lock()
|
|
session := manager.current
|
|
manager.mu.Unlock()
|
|
if session != nil && session.generation == generation && !session.dead.Load() {
|
|
return session
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
t.Fatalf("libvirt generation %d was not published", generation)
|
|
return nil
|
|
}
|
|
|
|
func waitForAtomicValue(t *testing.T, value *atomic.Int32, expected int32) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if value.Load() == expected {
|
|
return
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
t.Fatalf("value = %d, want %d", value.Load(), expected)
|
|
}
|
|
|
|
func TestManagedLibvirtInitialFailureReconnects(t *testing.T) {
|
|
var attempts atomic.Int32
|
|
var closed atomic.Int32
|
|
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
|
if attempts.Add(1) == 1 {
|
|
return nil, NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", errors.New("daemon down"))
|
|
}
|
|
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil
|
|
}, func(int) time.Duration { return time.Millisecond })
|
|
t.Cleanup(func() { _ = manager.Close() })
|
|
|
|
manager.start()
|
|
waitForManagedLibvirtGeneration(t, manager, 2)
|
|
if err := manager.Ping(context.Background()); err != nil {
|
|
t.Fatalf("ping after reconnect: %v", err)
|
|
}
|
|
if attempts.Load() != 2 {
|
|
t.Fatalf("connection attempts = %d, want 2", attempts.Load())
|
|
}
|
|
}
|
|
|
|
func TestManagedLibvirtReconnectPublishesNewSessionBeforeOldCallDrains(t *testing.T) {
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var firstClosed atomic.Int32
|
|
var secondClosed atomic.Int32
|
|
var secondPings atomic.Int32
|
|
var attempts atomic.Int32
|
|
|
|
firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
|
|
close(started)
|
|
<-release
|
|
return VMMetadata{UUID: "first-result"}, nil
|
|
}}
|
|
secondBackend := &managedLibvirtTestBackend{ping: func(context.Context) error {
|
|
secondPings.Add(1)
|
|
return nil
|
|
}}
|
|
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
|
if attempts.Add(1) == 1 {
|
|
return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil
|
|
}
|
|
return newManagedLibvirtTestSession(generation, secondBackend, &secondClosed), nil
|
|
}, func(int) time.Duration { return time.Millisecond })
|
|
t.Cleanup(func() {
|
|
select {
|
|
case <-release:
|
|
default:
|
|
close(release)
|
|
}
|
|
_ = manager.Close()
|
|
})
|
|
manager.start()
|
|
firstSession := waitForManagedLibvirtGeneration(t, manager, 1)
|
|
|
|
callDone := make(chan error, 1)
|
|
go func() {
|
|
_, err := manager.CreateVM(context.Background(), CreateVMRequest{})
|
|
callDone <- err
|
|
}()
|
|
select {
|
|
case <-started:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("first generation call did not start")
|
|
}
|
|
|
|
manager.markConnectionLost(firstSession)
|
|
secondSession := waitForManagedLibvirtGeneration(t, manager, 2)
|
|
if secondSession == firstSession {
|
|
t.Fatal("reconnect reused the retired session")
|
|
}
|
|
if err := manager.Ping(context.Background()); err != nil {
|
|
t.Fatalf("new generation ping: %v", err)
|
|
}
|
|
if secondPings.Load() != 1 {
|
|
t.Fatalf("new calls did not use the published generation: pings = %d", secondPings.Load())
|
|
}
|
|
if firstClosed.Load() != 0 {
|
|
t.Fatal("old connection closed while an in-flight call still held it")
|
|
}
|
|
|
|
close(release)
|
|
if err := <-callDone; err != nil {
|
|
t.Fatalf("late result from old generation: %v", err)
|
|
}
|
|
waitForAtomicValue(t, &firstClosed, 1)
|
|
|
|
// A delayed callback from the retired connection must not evict the
|
|
// replacement generation or start another reconnect.
|
|
manager.markConnectionLost(firstSession)
|
|
if current := waitForManagedLibvirtGeneration(t, manager, 2); current != secondSession {
|
|
t.Fatal("stale generation replaced the current session")
|
|
}
|
|
}
|
|
|
|
func TestManagedLibvirtDisconnectedMutationWasNotDispatched(t *testing.T) {
|
|
manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) {
|
|
return nil, errors.New("not used")
|
|
}, nil)
|
|
t.Cleanup(func() { _ = manager.Close() })
|
|
|
|
_, mutationErr := manager.CreateVM(context.Background(), CreateVMRequest{})
|
|
if operation.IsUncertain(mutationErr) {
|
|
t.Fatalf("disconnected mutation error = %v, backend was not invoked", mutationErr)
|
|
}
|
|
if !errors.Is(mutationErr, ErrUnavailable) {
|
|
t.Fatalf("disconnected mutation error = %v, want unavailable cause", mutationErr)
|
|
}
|
|
readErr := manager.Ping(context.Background())
|
|
if operation.IsUncertain(readErr) {
|
|
t.Fatalf("read-only ping error = %v, must not be marked uncertain", readErr)
|
|
}
|
|
if !errors.Is(readErr, ErrUnavailable) {
|
|
t.Fatalf("read-only ping error = %v, want unavailable", readErr)
|
|
}
|
|
}
|
|
|
|
func TestManagedLibvirtTransportErrorRetiresSessionAndReconnects(t *testing.T) {
|
|
var attempts atomic.Int32
|
|
var firstClosed atomic.Int32
|
|
var secondClosed atomic.Int32
|
|
firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
|
|
return VMMetadata{}, libvirt.Error{Code: libvirt.ERR_RPC, Message: "connection lost"}
|
|
}}
|
|
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
|
if attempts.Add(1) == 1 {
|
|
return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil
|
|
}
|
|
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &secondClosed), nil
|
|
}, func(int) time.Duration { return time.Millisecond })
|
|
t.Cleanup(func() { _ = manager.Close() })
|
|
manager.start()
|
|
waitForManagedLibvirtGeneration(t, manager, 1)
|
|
|
|
_, err := manager.CreateVM(context.Background(), CreateVMRequest{})
|
|
if !operation.IsUncertain(err) {
|
|
t.Fatalf("transport failure = %v, want uncertain", err)
|
|
}
|
|
waitForManagedLibvirtGeneration(t, manager, 2)
|
|
waitForAtomicValue(t, &firstClosed, 1)
|
|
if err := manager.Ping(context.Background()); err != nil {
|
|
t.Fatalf("ping after transport reconnect: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestManagedLibvirtCloseWaitsForReferencesAndStopsReconnect(t *testing.T) {
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var attempts atomic.Int32
|
|
var closed atomic.Int32
|
|
backend := &managedLibvirtTestBackend{ping: func(context.Context) error {
|
|
close(started)
|
|
<-release
|
|
return nil
|
|
}}
|
|
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
|
attempts.Add(1)
|
|
return newManagedLibvirtTestSession(generation, backend, &closed), nil
|
|
}, func(int) time.Duration { return time.Millisecond })
|
|
manager.start()
|
|
waitForManagedLibvirtGeneration(t, manager, 1)
|
|
|
|
pingDone := make(chan error, 1)
|
|
go func() { pingDone <- manager.Ping(context.Background()) }()
|
|
select {
|
|
case <-started:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("ping did not acquire the session")
|
|
}
|
|
closeDone := make(chan error, 1)
|
|
go func() { closeDone <- manager.Close() }()
|
|
select {
|
|
case err := <-closeDone:
|
|
t.Fatalf("Close returned before the active reference drained: %v", err)
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
if closed.Load() != 0 {
|
|
t.Fatal("session closed before its active reference drained")
|
|
}
|
|
|
|
close(release)
|
|
if err := <-pingDone; err != nil {
|
|
t.Fatalf("in-flight ping: %v", err)
|
|
}
|
|
select {
|
|
case err := <-closeDone:
|
|
if err != nil {
|
|
t.Fatalf("Close: %v", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("Close did not finish after the reference drained")
|
|
}
|
|
if closed.Load() != 1 {
|
|
t.Fatalf("session close count = %d, want 1", closed.Load())
|
|
}
|
|
manager.signalReconnect()
|
|
time.Sleep(20 * time.Millisecond)
|
|
if attempts.Load() != 1 {
|
|
t.Fatalf("connection attempts after Close = %d, want 1", attempts.Load())
|
|
}
|
|
}
|
|
|
|
func TestManagedLibvirtCloseWaitsForBlockingStartup(t *testing.T) {
|
|
entered := make(chan struct{})
|
|
release := make(chan struct{})
|
|
startDone := make(chan struct{})
|
|
cleanupErr := errors.New("startup session cleanup failed")
|
|
var closed atomic.Int32
|
|
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
|
close(entered)
|
|
<-release
|
|
session := newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed)
|
|
session.closeFn = func() error {
|
|
closed.Add(1)
|
|
return cleanupErr
|
|
}
|
|
return session, nil
|
|
}, func(int) time.Duration { return time.Millisecond })
|
|
go func() {
|
|
manager.start()
|
|
close(startDone)
|
|
}()
|
|
select {
|
|
case <-entered:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("startup factory did not begin")
|
|
}
|
|
|
|
closeDone := make(chan error, 1)
|
|
go func() { closeDone <- manager.Close() }()
|
|
select {
|
|
case err := <-closeDone:
|
|
t.Fatalf("Close returned while startup was still blocked: %v", err)
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
close(release)
|
|
select {
|
|
case <-startDone:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("startup did not finish after factory release")
|
|
}
|
|
select {
|
|
case err := <-closeDone:
|
|
if !errors.Is(err, cleanupErr) {
|
|
t.Fatalf("Close error = %v, want startup cleanup error", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("Close did not wait for startup cleanup")
|
|
}
|
|
if closed.Load() != 1 {
|
|
t.Fatalf("startup session close count = %d, want 1", closed.Load())
|
|
}
|
|
}
|
|
|
|
func TestManagedLibvirtStartAfterCloseDoesNotConnect(t *testing.T) {
|
|
var attempts atomic.Int32
|
|
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
|
attempts.Add(1)
|
|
var closed atomic.Int32
|
|
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil
|
|
}, func(int) time.Duration { return time.Millisecond })
|
|
if err := manager.Close(); err != nil {
|
|
t.Fatalf("Close: %v", err)
|
|
}
|
|
manager.start()
|
|
if attempts.Load() != 0 {
|
|
t.Fatalf("connection attempts after Close = %d, want 0", attempts.Load())
|
|
}
|
|
}
|
|
|
|
func TestLibvirtEventBroadcasterAdvancesGeneration(t *testing.T) {
|
|
broadcaster := newLibvirtEventBroadcaster()
|
|
generation, wake := broadcaster.snapshot()
|
|
broadcaster.notify()
|
|
if err := broadcaster.wait(context.Background(), generation, wake); err != nil {
|
|
t.Fatalf("wait after notify: %v", err)
|
|
}
|
|
nextGeneration, _ := broadcaster.snapshot()
|
|
if nextGeneration != generation+1 {
|
|
t.Fatalf("generation = %d, want %d", nextGeneration, generation+1)
|
|
}
|
|
}
|
|
|
|
func TestLibvirtConnectionCleanupUsesReverseRegistrationOrder(t *testing.T) {
|
|
var calls []string
|
|
cleanup := &libvirtConnectionCleanup{
|
|
closeRegistered: true,
|
|
jobID: 22,
|
|
deregister: func(id int) error {
|
|
calls = append(calls, "event:"+fmt.Sprint(id))
|
|
return nil
|
|
},
|
|
unregisterClose: func() error {
|
|
calls = append(calls, "close-callback")
|
|
return nil
|
|
},
|
|
closeConnection: func() error {
|
|
calls = append(calls, "connection")
|
|
return nil
|
|
},
|
|
}
|
|
if err := cleanup.run(); err != nil {
|
|
t.Fatalf("cleanup: %v", err)
|
|
}
|
|
want := []string{"event:22", "close-callback", "connection"}
|
|
if !reflect.DeepEqual(calls, want) {
|
|
t.Fatalf("cleanup order = %v, want %v", calls, want)
|
|
}
|
|
}
|
|
|
|
func TestLibvirtConnectionCleanupHandlesPartialRegistration(t *testing.T) {
|
|
var calls []string
|
|
cleanup := &libvirtConnectionCleanup{
|
|
closeRegistered: true,
|
|
jobID: -1,
|
|
deregister: func(id int) error {
|
|
calls = append(calls, "event:"+fmt.Sprint(id))
|
|
return nil
|
|
},
|
|
unregisterClose: func() error {
|
|
calls = append(calls, "close-callback")
|
|
return nil
|
|
},
|
|
closeConnection: func() error {
|
|
calls = append(calls, "connection")
|
|
return nil
|
|
},
|
|
}
|
|
if err := cleanup.run(); err != nil {
|
|
t.Fatalf("cleanup: %v", err)
|
|
}
|
|
want := []string{"close-callback", "connection"}
|
|
if !reflect.DeepEqual(calls, want) {
|
|
t.Fatalf("partial cleanup order = %v, want %v", calls, want)
|
|
}
|
|
}
|
|
|
|
func TestManagedLibvirtIgnoresExpectedDeadSessionCleanupError(t *testing.T) {
|
|
manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) {
|
|
return nil, errors.New("not used")
|
|
}, nil)
|
|
deadSession := &managedLibvirtSession{closeFn: func() error {
|
|
return fmt.Errorf("unregister stale callback: %w", libvirt.Error{Code: libvirt.ERR_INVALID_CONN})
|
|
}}
|
|
deadSession.dead.Store(true)
|
|
manager.cleanupSession(deadSession)
|
|
if len(manager.cleanupErrs) != 0 {
|
|
t.Fatalf("dead connection cleanup errors were retained: %v", manager.cleanupErrs)
|
|
}
|
|
|
|
liveSession := &managedLibvirtSession{closeFn: func() error { return errors.New("close failed") }}
|
|
manager.cleanupSession(liveSession)
|
|
if len(manager.cleanupErrs) != 1 {
|
|
t.Fatalf("live connection cleanup errors = %v, want one retained error", manager.cleanupErrs)
|
|
}
|
|
|
|
mixedSession := &managedLibvirtSession{closeFn: func() error {
|
|
return errors.Join(libvirt.Error{Code: libvirt.ERR_NO_CONNECT}, errors.New("unexpected cleanup failure"))
|
|
}}
|
|
mixedSession.dead.Store(true)
|
|
manager.cleanupSession(mixedSession)
|
|
if len(manager.cleanupErrs) != 2 {
|
|
t.Fatalf("mixed dead connection cleanup errors = %v, want unexpected error retained", manager.cleanupErrs)
|
|
}
|
|
}
|