fix(agent): recover libvirt connections and operations
parent
a73c2695c8
commit
1bfc8b7513
@ -0,0 +1,756 @@
|
||||
//go:build libvirt
|
||||
|
||||
package hypervisor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.luoli.live/root/virt/agent/operation"
|
||||
"libvirt.org/go/libvirt"
|
||||
)
|
||||
|
||||
const (
|
||||
libvirtReconnectInitialDelay = time.Second
|
||||
libvirtReconnectMaximumDelay = 30 * time.Second
|
||||
libvirtCleanupErrorLimit = 16
|
||||
)
|
||||
|
||||
// libvirtJobEvents wakes backup waiters when JOB_COMPLETED arrives. It is a
|
||||
// hint, not a result: callers must still use GetJobStats to establish the
|
||||
// terminal state. Closing a connection also advances the generation so a
|
||||
// waiter can promptly observe the lost session.
|
||||
var libvirtJobEvents = newLibvirtEventBroadcaster()
|
||||
|
||||
type managedLibvirtSession struct {
|
||||
generation uint64
|
||||
backend Hypervisor
|
||||
closeFn func() error
|
||||
|
||||
dead atomic.Bool
|
||||
|
||||
// refs and retired are protected by managedLibvirt.mu. drained is closed
|
||||
// exactly once after retirement when the last public call releases its
|
||||
// immutable session reference.
|
||||
refs int
|
||||
retired bool
|
||||
drained chan struct{}
|
||||
}
|
||||
|
||||
type libvirtConnectionCleanup struct {
|
||||
closeRegistered bool
|
||||
lifecycleID int
|
||||
jobID int
|
||||
deregister func(int) error
|
||||
unregisterClose func() error
|
||||
closeConnection func() error
|
||||
}
|
||||
|
||||
func (cleanup *libvirtConnectionCleanup) run() error {
|
||||
var cleanupErrors []error
|
||||
// Registrations are unwound in strict reverse order: JOB_COMPLETED,
|
||||
// lifecycle, close callback, then the connection reference itself.
|
||||
if cleanup.jobID >= 0 {
|
||||
if err := cleanup.deregister(cleanup.jobID); err != nil {
|
||||
cleanupErrors = append(cleanupErrors, fmt.Errorf("deregister completed job events: %w", err))
|
||||
}
|
||||
}
|
||||
if cleanup.lifecycleID >= 0 {
|
||||
if err := cleanup.deregister(cleanup.lifecycleID); err != nil {
|
||||
cleanupErrors = append(cleanupErrors, fmt.Errorf("deregister lifecycle events: %w", err))
|
||||
}
|
||||
}
|
||||
if cleanup.closeRegistered {
|
||||
if err := cleanup.unregisterClose(); err != nil {
|
||||
cleanupErrors = append(cleanupErrors, fmt.Errorf("unregister connection close callback: %w", err))
|
||||
}
|
||||
}
|
||||
if err := cleanup.closeConnection(); err != nil {
|
||||
cleanupErrors = append(cleanupErrors, fmt.Errorf("close libvirt connection: %w", err))
|
||||
}
|
||||
return errors.Join(cleanupErrors...)
|
||||
}
|
||||
|
||||
type managedLibvirtSessionFactory func(uint64) (*managedLibvirtSession, error)
|
||||
|
||||
type managedLibvirt struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
mu sync.Mutex
|
||||
current *managedLibvirtSession
|
||||
nextGeneration uint64
|
||||
closing bool
|
||||
lastConnectErr error
|
||||
|
||||
currentGeneration atomic.Uint64
|
||||
reconnect chan struct{}
|
||||
recovery chan struct{}
|
||||
|
||||
factory managedLibvirtSessionFactory
|
||||
retryDelay func(int) time.Duration
|
||||
|
||||
workerWG sync.WaitGroup
|
||||
retiredWG sync.WaitGroup
|
||||
|
||||
cleanupMu sync.Mutex
|
||||
cleanupErrs []error
|
||||
startOnce sync.Once
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func openManagedLibvirt(uri string, local LocalConfig, clientVersion uint32, traffic *trafficStore) *managedLibvirt {
|
||||
var manager *managedLibvirt
|
||||
manager = newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
||||
return manager.openSession(uri, local, clientVersion, traffic, generation)
|
||||
}, defaultLibvirtReconnectDelay)
|
||||
manager.start()
|
||||
return manager
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) start() {
|
||||
// Preserve the useful startup behavior of attempting the local socket once,
|
||||
// while allowing the Agent to start and report unavailable if the daemon is
|
||||
// temporarily down. Recovery gets one initial wakeup in either case.
|
||||
m.startOnce.Do(func() {
|
||||
// Register the whole startup/reconnect lifecycle before the first
|
||||
// potentially blocking connection attempt. Close uses the same mutex
|
||||
// before Wait, which prevents an Add-after-Wait race.
|
||||
m.mu.Lock()
|
||||
if m.closing {
|
||||
m.mu.Unlock()
|
||||
return
|
||||
}
|
||||
m.workerWG.Add(1)
|
||||
m.mu.Unlock()
|
||||
|
||||
_ = m.connectAndPublish()
|
||||
m.signalRecovery()
|
||||
go m.reconnectLoop()
|
||||
if !m.hasUsableSession() {
|
||||
m.signalReconnect()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func newManagedLibvirt(factory managedLibvirtSessionFactory, retryDelay func(int) time.Duration) *managedLibvirt {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
if retryDelay == nil {
|
||||
retryDelay = defaultLibvirtReconnectDelay
|
||||
}
|
||||
return &managedLibvirt{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
reconnect: make(chan struct{}, 1),
|
||||
recovery: make(chan struct{}, 1),
|
||||
factory: factory,
|
||||
retryDelay: retryDelay,
|
||||
}
|
||||
}
|
||||
|
||||
func defaultLibvirtReconnectDelay(failures int) time.Duration {
|
||||
if failures <= 1 {
|
||||
return libvirtReconnectInitialDelay
|
||||
}
|
||||
delay := libvirtReconnectInitialDelay
|
||||
for attempt := 1; attempt < failures && delay < libvirtReconnectMaximumDelay; attempt++ {
|
||||
delay *= 2
|
||||
if delay > libvirtReconnectMaximumDelay {
|
||||
return libvirtReconnectMaximumDelay
|
||||
}
|
||||
}
|
||||
return delay
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) openSession(uri string, local LocalConfig, clientVersion uint32, traffic *trafficStore, generation uint64) (*managedLibvirtSession, error) {
|
||||
conn, err := libvirt.NewConnect(uri)
|
||||
if err != nil {
|
||||
return nil, translateLibvirtError("connect", err)
|
||||
}
|
||||
session := &managedLibvirtSession{
|
||||
generation: generation,
|
||||
drained: make(chan struct{}),
|
||||
}
|
||||
cleanup := &libvirtConnectionCleanup{
|
||||
lifecycleID: -1,
|
||||
jobID: -1,
|
||||
deregister: conn.DomainEventDeregister,
|
||||
unregisterClose: func() error {
|
||||
return conn.UnregisterCloseCallback()
|
||||
},
|
||||
closeConnection: func() error {
|
||||
_, err := conn.Close()
|
||||
return err
|
||||
},
|
||||
}
|
||||
session.closeFn = cleanup.run
|
||||
failSetup := func(action string, cause error) (*managedLibvirtSession, error) {
|
||||
return nil, errors.Join(translateLibvirtError(action, cause), session.closeFn())
|
||||
}
|
||||
|
||||
if err := conn.RegisterCloseCallback(func(_ *libvirt.Connect, _ libvirt.ConnectCloseReason) {
|
||||
// The callback can run on libvirt's event thread. It only marks this
|
||||
// immutable generation dead and performs non-blocking wakeups; reconnect,
|
||||
// callback deregistration, and Close all happen on manager workers.
|
||||
session.dead.Store(true)
|
||||
if !m.isCurrentGeneration(session) {
|
||||
return
|
||||
}
|
||||
libvirtJobEvents.notify()
|
||||
storageStreamEvents.notify()
|
||||
m.signalRecovery()
|
||||
m.signalReconnect()
|
||||
}); err != nil {
|
||||
return failSetup("register connection close callback", err)
|
||||
}
|
||||
cleanup.closeRegistered = true
|
||||
|
||||
cleanup.lifecycleID, err = conn.DomainEventLifecycleRegister(nil, func(_ *libvirt.Connect, _ *libvirt.Domain, _ *libvirt.DomainEventLifecycle) {
|
||||
m.signalRecoveryFor(session)
|
||||
})
|
||||
if err != nil {
|
||||
return failSetup("register domain lifecycle events", err)
|
||||
}
|
||||
cleanup.jobID, err = conn.DomainEventJobCompletedRegister(nil, func(_ *libvirt.Connect, _ *libvirt.Domain, _ *libvirt.DomainEventJobCompleted) {
|
||||
if !m.isCurrentGeneration(session) {
|
||||
return
|
||||
}
|
||||
libvirtJobEvents.notify()
|
||||
m.signalRecovery()
|
||||
})
|
||||
if err != nil {
|
||||
return failSetup("register completed job events", err)
|
||||
}
|
||||
|
||||
daemonVersion, err := conn.GetLibVersion()
|
||||
if err != nil {
|
||||
return failSetup("get daemon libvirt version", err)
|
||||
}
|
||||
session.backend = &Libvirt{
|
||||
conn: conn,
|
||||
local: local,
|
||||
traffic: traffic,
|
||||
guestAgentResponseTimeoutSupported: supportsGuestAgentResponseTimeout(clientVersion, daemonVersion),
|
||||
}
|
||||
|
||||
alive, err := conn.IsAlive()
|
||||
if err != nil || !alive || session.dead.Load() {
|
||||
if err != nil {
|
||||
return failSetup("verify libvirt connection", err)
|
||||
}
|
||||
return nil, errors.Join(NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", nil), session.closeFn())
|
||||
}
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) reconnectLoop() {
|
||||
defer m.workerWG.Done()
|
||||
failures := 0
|
||||
for {
|
||||
select {
|
||||
case <-m.ctx.Done():
|
||||
return
|
||||
case <-m.reconnect:
|
||||
}
|
||||
|
||||
for {
|
||||
if m.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
m.detachDeadSession()
|
||||
if m.hasUsableSession() {
|
||||
failures = 0
|
||||
break
|
||||
}
|
||||
if err := m.connectAndPublish(); err == nil {
|
||||
failures = 0
|
||||
break
|
||||
}
|
||||
failures++
|
||||
timer := time.NewTimer(m.retryDelay(failures))
|
||||
select {
|
||||
case <-m.ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return
|
||||
case <-m.reconnect:
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) connectAndPublish() error {
|
||||
m.mu.Lock()
|
||||
if m.closing {
|
||||
m.mu.Unlock()
|
||||
return context.Canceled
|
||||
}
|
||||
m.nextGeneration++
|
||||
generation := m.nextGeneration
|
||||
m.mu.Unlock()
|
||||
|
||||
session, err := m.factory(generation)
|
||||
if err != nil {
|
||||
m.mu.Lock()
|
||||
m.lastConnectErr = err
|
||||
m.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
if session == nil || session.backend == nil || session.closeFn == nil {
|
||||
err = errors.New("libvirt session factory returned an incomplete session")
|
||||
if session != nil && session.closeFn != nil {
|
||||
m.cleanupSession(session)
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.lastConnectErr = err
|
||||
m.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
if session.drained == nil {
|
||||
session.drained = make(chan struct{})
|
||||
}
|
||||
session.generation = generation
|
||||
|
||||
m.mu.Lock()
|
||||
if m.closing || m.current != nil || session.dead.Load() {
|
||||
m.mu.Unlock()
|
||||
m.cleanupSession(session)
|
||||
if session.dead.Load() {
|
||||
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", nil)
|
||||
}
|
||||
return context.Canceled
|
||||
}
|
||||
m.current = session
|
||||
m.currentGeneration.Store(session.generation)
|
||||
if session.dead.Load() {
|
||||
m.current = nil
|
||||
m.currentGeneration.Store(0)
|
||||
m.retireSessionLocked(session)
|
||||
m.mu.Unlock()
|
||||
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", nil)
|
||||
}
|
||||
m.lastConnectErr = nil
|
||||
m.mu.Unlock()
|
||||
m.signalRecovery()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) hasUsableSession() bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return !m.closing && m.current != nil && !m.current.dead.Load()
|
||||
}
|
||||
|
||||
// detachDeadSession runs only outside libvirt callbacks. It atomically stops
|
||||
// new acquisitions, then retires the old generation after existing calls have
|
||||
// released their references.
|
||||
func (m *managedLibvirt) detachDeadSession() bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.current == nil || !m.current.dead.Load() {
|
||||
return false
|
||||
}
|
||||
session := m.current
|
||||
m.current = nil
|
||||
m.currentGeneration.Store(0)
|
||||
m.retireSessionLocked(session)
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) retireSessionLocked(session *managedLibvirtSession) {
|
||||
if session == nil || session.retired {
|
||||
return
|
||||
}
|
||||
session.retired = true
|
||||
if session.refs == 0 {
|
||||
close(session.drained)
|
||||
}
|
||||
m.retiredWG.Add(1)
|
||||
go func() {
|
||||
defer m.retiredWG.Done()
|
||||
<-session.drained
|
||||
m.cleanupSession(session)
|
||||
}()
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) cleanupSession(session *managedLibvirtSession) {
|
||||
if session == nil || session.closeFn == nil {
|
||||
return
|
||||
}
|
||||
if err := session.closeFn(); err != nil {
|
||||
// Once libvirt has told us this generation is dead, INVALID_CONN,
|
||||
// NO_CONNECT, and RPC errors are expected while unregistering callbacks
|
||||
// and releasing the stale handle. They must not poison a later clean
|
||||
// shutdown after the manager has successfully reconnected.
|
||||
if session.dead.Load() && onlyLibvirtConnectionFailures(err) {
|
||||
return
|
||||
}
|
||||
m.cleanupMu.Lock()
|
||||
if len(m.cleanupErrs) < libvirtCleanupErrorLimit {
|
||||
m.cleanupErrs = append(m.cleanupErrs, err)
|
||||
} else {
|
||||
m.cleanupErrs[libvirtCleanupErrorLimit-1] = err
|
||||
}
|
||||
m.cleanupMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) acquire(ctx context.Context) (*managedLibvirtSession, func(), error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
m.mu.Lock()
|
||||
if m.closing || m.current == nil || m.current.dead.Load() {
|
||||
cause := m.lastConnectErr
|
||||
m.mu.Unlock()
|
||||
return nil, nil, NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", cause)
|
||||
}
|
||||
session := m.current
|
||||
session.refs++
|
||||
m.mu.Unlock()
|
||||
|
||||
var once sync.Once
|
||||
release := func() {
|
||||
once.Do(func() {
|
||||
m.mu.Lock()
|
||||
session.refs--
|
||||
if session.retired && session.refs == 0 {
|
||||
close(session.drained)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
})
|
||||
}
|
||||
return session, release, nil
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) isCurrentGeneration(session *managedLibvirtSession) bool {
|
||||
return session != nil && m.currentGeneration.Load() == session.generation
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) signalRecoveryFor(session *managedLibvirtSession) {
|
||||
if m.isCurrentGeneration(session) {
|
||||
m.signalRecovery()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) signalRecovery() {
|
||||
select {
|
||||
case m.recovery <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) signalReconnect() {
|
||||
select {
|
||||
case m.reconnect <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) markConnectionLost(session *managedLibvirtSession) {
|
||||
if session == nil {
|
||||
return
|
||||
}
|
||||
session.dead.Store(true)
|
||||
if !m.isCurrentGeneration(session) {
|
||||
return
|
||||
}
|
||||
libvirtJobEvents.notify()
|
||||
storageStreamEvents.notify()
|
||||
m.signalRecovery()
|
||||
m.signalReconnect()
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) RecoveryEvents() <-chan struct{} { return m.recovery }
|
||||
|
||||
func (m *managedLibvirt) Close() error {
|
||||
m.closeOnce.Do(func() {
|
||||
m.mu.Lock()
|
||||
m.closing = true
|
||||
m.currentGeneration.Store(0)
|
||||
if m.current != nil {
|
||||
session := m.current
|
||||
m.current = nil
|
||||
m.retireSessionLocked(session)
|
||||
}
|
||||
m.cancel()
|
||||
m.mu.Unlock()
|
||||
|
||||
m.workerWG.Wait()
|
||||
m.retiredWG.Wait()
|
||||
m.cleanupMu.Lock()
|
||||
m.closeErr = errors.Join(m.cleanupErrs...)
|
||||
m.cleanupMu.Unlock()
|
||||
})
|
||||
return m.closeErr
|
||||
}
|
||||
|
||||
func isLibvirtConnectionFailure(err error) bool {
|
||||
var libvirtErr libvirt.Error
|
||||
if errors.As(err, &libvirtErr) {
|
||||
switch libvirtErr.Code {
|
||||
case libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_RPC:
|
||||
return true
|
||||
}
|
||||
}
|
||||
var publicError *Error
|
||||
return errors.As(err, &publicError) && publicError.Code == "libvirt_unavailable"
|
||||
}
|
||||
|
||||
func onlyLibvirtConnectionFailures(err error) bool {
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
if joined, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
causes := joined.Unwrap()
|
||||
if len(causes) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, cause := range causes {
|
||||
if !onlyLibvirtConnectionFailures(cause) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
|
||||
if cause := wrapped.Unwrap(); cause != nil {
|
||||
return onlyLibvirtConnectionFailures(cause)
|
||||
}
|
||||
}
|
||||
return isLibvirtConnectionFailure(err)
|
||||
}
|
||||
|
||||
func managedCall[T any](m *managedLibvirt, ctx context.Context, mutation bool, call func(Hypervisor) (T, error)) (T, error) {
|
||||
var zero T
|
||||
session, release, err := m.acquire(ctx)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
defer release()
|
||||
value, err := call(session.backend)
|
||||
if err == nil {
|
||||
return value, nil
|
||||
}
|
||||
if isLibvirtConnectionFailure(err) {
|
||||
m.markConnectionLost(session)
|
||||
if mutation && !operation.IsUncertain(err) {
|
||||
err = operation.MarkUncertain(err)
|
||||
}
|
||||
}
|
||||
return value, err
|
||||
}
|
||||
|
||||
func managedCallError(m *managedLibvirt, ctx context.Context, mutation bool, call func(Hypervisor) error) error {
|
||||
_, err := managedCall(m, ctx, mutation, func(backend Hypervisor) (struct{}, error) {
|
||||
return struct{}{}, call(backend)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) Ping(ctx context.Context) error {
|
||||
return managedCallError(m, ctx, false, func(h Hypervisor) error { return h.Ping(ctx) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) Capabilities(ctx context.Context) (Capabilities, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) (Capabilities, error) { return h.Capabilities(ctx) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ConfigureStorage(ctx context.Context, request StorageConfigurationRequest) (StorageConfigurationResult, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (StorageConfigurationResult, error) {
|
||||
return h.ConfigureStorage(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ListVMs(ctx context.Context) ([]VMMetadata, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) ([]VMMetadata, error) { return h.ListVMs(ctx) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) GetVMState(ctx context.Context, uuid string) (VMState, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) (VMState, error) { return h.GetVMState(ctx, uuid) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) GetVMMetadata(ctx context.Context, uuid string) (VMMetadata, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) (VMMetadata, error) { return h.GetVMMetadata(ctx, uuid) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) CreateVM(ctx context.Context, request CreateVMRequest) (VMMetadata, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) { return h.CreateVM(ctx, request) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) DeleteVM(ctx context.Context, uuid string, request DeleteVMRequest) error {
|
||||
return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteVM(ctx, uuid, request) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) Power(ctx context.Context, uuid string, request PowerRequest) (VMState, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (VMState, error) { return h.Power(ctx, uuid, request) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) UpdateSettings(ctx context.Context, uuid string, request SettingsRequest) (VMMetadata, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) {
|
||||
return h.UpdateSettings(ctx, uuid, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ResolveNetworkSelector(ctx context.Context, selector string) (LocalNetworkSource, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) (LocalNetworkSource, error) {
|
||||
return h.ResolveNetworkSelector(ctx, selector)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ChangeNetwork(ctx context.Context, uuid string, request ChangeNetworkRequest) (VMMetadata, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) {
|
||||
return h.ChangeNetwork(ctx, uuid, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) SetNetworkState(ctx context.Context, uuid string, enabled bool, request NetworkStateRequest) error {
|
||||
return managedCallError(m, ctx, true, func(h Hypervisor) error {
|
||||
return h.SetNetworkState(ctx, uuid, enabled, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) SetServiceState(ctx context.Context, uuid string, enabled bool, request ServiceStateRequest) (VMState, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (VMState, error) {
|
||||
return h.SetServiceState(ctx, uuid, enabled, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) Reinstall(ctx context.Context, uuid string, request ReinstallRequest) (VMMetadata, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) {
|
||||
return h.Reinstall(ctx, uuid, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) GetMetrics(ctx context.Context, uuid string) (Metrics, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) (Metrics, error) { return h.GetMetrics(ctx, uuid) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) GetDailyTraffic(ctx context.Context, uuid string, days int) ([]DailyTraffic, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) ([]DailyTraffic, error) {
|
||||
return h.GetDailyTraffic(ctx, uuid, days)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) GetConsole(ctx context.Context, uuid string) (ConsoleSession, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) (ConsoleSession, error) { return h.GetConsole(ctx, uuid) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ListSnapshots(ctx context.Context, uuid string) ([]Snapshot, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) ([]Snapshot, error) { return h.ListSnapshots(ctx, uuid) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) CreateSnapshot(ctx context.Context, uuid string, request SnapshotRequest) (Snapshot, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (Snapshot, error) {
|
||||
return h.CreateSnapshot(ctx, uuid, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) DeleteSnapshot(ctx context.Context, uuid, key string, request SnapshotMutationRequest) (SnapshotMutationResult, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (SnapshotMutationResult, error) {
|
||||
return h.DeleteSnapshot(ctx, uuid, key, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) RestoreSnapshot(ctx context.Context, uuid, key string, request SnapshotMutationRequest) (SnapshotMutationResult, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (SnapshotMutationResult, error) {
|
||||
return h.RestoreSnapshot(ctx, uuid, key, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) GetFirewall(ctx context.Context, uuid string) (FirewallPolicy, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) (FirewallPolicy, error) { return h.GetFirewall(ctx, uuid) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ApplyFirewall(ctx context.Context, uuid string, policy FirewallPolicy) (FirewallPolicy, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (FirewallPolicy, error) {
|
||||
return h.ApplyFirewall(ctx, uuid, policy)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) GetRescue(ctx context.Context, uuid string) (RescueState, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) (RescueState, error) { return h.GetRescue(ctx, uuid) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) EnterRescue(ctx context.Context, uuid string, request RescueRequest) (RescueState, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (RescueState, error) {
|
||||
return h.EnterRescue(ctx, uuid, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ForceStartRescue(ctx context.Context, uuid string, request ForceRescueStartRequest) (RescueState, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (RescueState, error) {
|
||||
return h.ForceStartRescue(ctx, uuid, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ExitRescue(ctx context.Context, uuid string, request ExitRescueRequest) (RescueState, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (RescueState, error) {
|
||||
return h.ExitRescue(ctx, uuid, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ListTemplates(ctx context.Context) ([]ImageTemplate, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) ([]ImageTemplate, error) { return h.ListTemplates(ctx) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ImportTemplate(ctx context.Context, request ImportTemplateRequest) (ImageTemplate, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (ImageTemplate, error) {
|
||||
return h.ImportTemplate(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) UpdateTemplate(ctx context.Context, id string, request UpdateTemplateRequest) (ImageTemplate, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (ImageTemplate, error) {
|
||||
return h.UpdateTemplate(ctx, id, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) DeleteTemplate(ctx context.Context, id string) error {
|
||||
return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteTemplate(ctx, id) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ListRescueTemplates(ctx context.Context) ([]RescueTemplate, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) ([]RescueTemplate, error) { return h.ListRescueTemplates(ctx) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ImportRescueTemplate(ctx context.Context, request ImportRescueTemplateRequest) (RescueTemplate, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (RescueTemplate, error) {
|
||||
return h.ImportRescueTemplate(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) DeleteRescueTemplate(ctx context.Context, id string) error {
|
||||
return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteRescueTemplate(ctx, id) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ListUserISOs(ctx context.Context, ownerID int64) ([]UserISO, error) {
|
||||
return managedCall(m, ctx, false, func(h Hypervisor) ([]UserISO, error) { return h.ListUserISOs(ctx, ownerID) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) ImportUserISO(ctx context.Context, request ImportUserISORequest) (UserISO, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (UserISO, error) { return h.ImportUserISO(ctx, request) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) DeleteUserISO(ctx context.Context, id string, ownerID int64) error {
|
||||
return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteUserISO(ctx, id, ownerID) })
|
||||
}
|
||||
|
||||
func (m *managedLibvirt) Reconcile(ctx context.Context, record operation.Record) (operation.Resolution, error) {
|
||||
return managedCall(m, ctx, true, func(h Hypervisor) (operation.Resolution, error) {
|
||||
return h.Reconcile(ctx, record)
|
||||
})
|
||||
}
|
||||
|
||||
var _ Hypervisor = (*managedLibvirt)(nil)
|
||||
var _ RecoveryEventSource = (*managedLibvirt)(nil)
|
||||
@ -0,0 +1,472 @@
|
||||
//go:build libvirt
|
||||
|
||||
package hypervisor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.luoli.live/root/virt/agent/operation"
|
||||
"libvirt.org/go/libvirt"
|
||||
)
|
||||
|
||||
type managedLibvirtTestBackend struct {
|
||||
Unsupported
|
||||
ping func(context.Context) error
|
||||
createVM func(context.Context, CreateVMRequest) (VMMetadata, error)
|
||||
}
|
||||
|
||||
func (b *managedLibvirtTestBackend) Ping(ctx context.Context) error {
|
||||
if b.ping != nil {
|
||||
return b.ping(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *managedLibvirtTestBackend) CreateVM(ctx context.Context, request CreateVMRequest) (VMMetadata, error) {
|
||||
if b.createVM != nil {
|
||||
return b.createVM(ctx, request)
|
||||
}
|
||||
return VMMetadata{}, nil
|
||||
}
|
||||
|
||||
func newManagedLibvirtTestSession(generation uint64, backend Hypervisor, closed *atomic.Int32) *managedLibvirtSession {
|
||||
return &managedLibvirtSession{
|
||||
generation: generation,
|
||||
backend: backend,
|
||||
drained: make(chan struct{}),
|
||||
closeFn: func() error {
|
||||
closed.Add(1)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func waitForManagedLibvirtGeneration(t *testing.T, manager *managedLibvirt, generation uint64) *managedLibvirtSession {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
manager.mu.Lock()
|
||||
session := manager.current
|
||||
manager.mu.Unlock()
|
||||
if session != nil && session.generation == generation && !session.dead.Load() {
|
||||
return session
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatalf("libvirt generation %d was not published", generation)
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitForAtomicValue(t *testing.T, value *atomic.Int32, expected int32) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if value.Load() == expected {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatalf("value = %d, want %d", value.Load(), expected)
|
||||
}
|
||||
|
||||
func drainManagedLibvirtEvents(events <-chan struct{}) {
|
||||
for {
|
||||
select {
|
||||
case <-events:
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedLibvirtInitialFailureSignalsAndReconnects(t *testing.T) {
|
||||
var attempts atomic.Int32
|
||||
var closed atomic.Int32
|
||||
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
||||
if attempts.Add(1) == 1 {
|
||||
return nil, NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", errors.New("daemon down"))
|
||||
}
|
||||
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil
|
||||
}, func(int) time.Duration { return time.Millisecond })
|
||||
t.Cleanup(func() { _ = manager.Close() })
|
||||
|
||||
manager.start()
|
||||
select {
|
||||
case <-manager.RecoveryEvents():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("initial connection attempt did not signal recovery readiness")
|
||||
}
|
||||
waitForManagedLibvirtGeneration(t, manager, 2)
|
||||
select {
|
||||
case <-manager.RecoveryEvents():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("successful reconnect did not signal recovery")
|
||||
}
|
||||
if err := manager.Ping(context.Background()); err != nil {
|
||||
t.Fatalf("ping after reconnect: %v", err)
|
||||
}
|
||||
if attempts.Load() != 2 {
|
||||
t.Fatalf("connection attempts = %d, want 2", attempts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedLibvirtReconnectPublishesNewSessionBeforeOldCallDrains(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var firstClosed atomic.Int32
|
||||
var secondClosed atomic.Int32
|
||||
var secondPings atomic.Int32
|
||||
var attempts atomic.Int32
|
||||
|
||||
firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
|
||||
close(started)
|
||||
<-release
|
||||
return VMMetadata{UUID: "first-result"}, nil
|
||||
}}
|
||||
secondBackend := &managedLibvirtTestBackend{ping: func(context.Context) error {
|
||||
secondPings.Add(1)
|
||||
return nil
|
||||
}}
|
||||
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
||||
if attempts.Add(1) == 1 {
|
||||
return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil
|
||||
}
|
||||
return newManagedLibvirtTestSession(generation, secondBackend, &secondClosed), nil
|
||||
}, func(int) time.Duration { return time.Millisecond })
|
||||
t.Cleanup(func() {
|
||||
select {
|
||||
case <-release:
|
||||
default:
|
||||
close(release)
|
||||
}
|
||||
_ = manager.Close()
|
||||
})
|
||||
manager.start()
|
||||
firstSession := waitForManagedLibvirtGeneration(t, manager, 1)
|
||||
drainManagedLibvirtEvents(manager.RecoveryEvents())
|
||||
|
||||
callDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := manager.CreateVM(context.Background(), CreateVMRequest{})
|
||||
callDone <- err
|
||||
}()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first generation call did not start")
|
||||
}
|
||||
|
||||
manager.markConnectionLost(firstSession)
|
||||
secondSession := waitForManagedLibvirtGeneration(t, manager, 2)
|
||||
if secondSession == firstSession {
|
||||
t.Fatal("reconnect reused the retired session")
|
||||
}
|
||||
if err := manager.Ping(context.Background()); err != nil {
|
||||
t.Fatalf("new generation ping: %v", err)
|
||||
}
|
||||
if secondPings.Load() != 1 {
|
||||
t.Fatalf("new calls did not use the published generation: pings = %d", secondPings.Load())
|
||||
}
|
||||
if firstClosed.Load() != 0 {
|
||||
t.Fatal("old connection closed while an in-flight call still held it")
|
||||
}
|
||||
|
||||
close(release)
|
||||
if err := <-callDone; err != nil {
|
||||
t.Fatalf("late result from old generation: %v", err)
|
||||
}
|
||||
waitForAtomicValue(t, &firstClosed, 1)
|
||||
|
||||
// A delayed callback from the retired connection must not wake recovery or
|
||||
// evict the replacement generation.
|
||||
drainManagedLibvirtEvents(manager.RecoveryEvents())
|
||||
manager.signalRecoveryFor(firstSession)
|
||||
manager.markConnectionLost(firstSession)
|
||||
select {
|
||||
case <-manager.RecoveryEvents():
|
||||
t.Fatal("stale generation emitted a recovery event")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
if current := waitForManagedLibvirtGeneration(t, manager, 2); current != secondSession {
|
||||
t.Fatal("stale generation replaced the current session")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedLibvirtDisconnectedMutationWasNotDispatched(t *testing.T) {
|
||||
manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) {
|
||||
return nil, errors.New("not used")
|
||||
}, nil)
|
||||
t.Cleanup(func() { _ = manager.Close() })
|
||||
|
||||
_, mutationErr := manager.CreateVM(context.Background(), CreateVMRequest{})
|
||||
if operation.IsUncertain(mutationErr) {
|
||||
t.Fatalf("disconnected mutation error = %v, backend was not invoked", mutationErr)
|
||||
}
|
||||
if !errors.Is(mutationErr, ErrUnavailable) {
|
||||
t.Fatalf("disconnected mutation error = %v, want unavailable cause", mutationErr)
|
||||
}
|
||||
readErr := manager.Ping(context.Background())
|
||||
if operation.IsUncertain(readErr) {
|
||||
t.Fatalf("read-only ping error = %v, must not be marked uncertain", readErr)
|
||||
}
|
||||
if !errors.Is(readErr, ErrUnavailable) {
|
||||
t.Fatalf("read-only ping error = %v, want unavailable", readErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedLibvirtTransportErrorRetiresSessionAndReconnects(t *testing.T) {
|
||||
var attempts atomic.Int32
|
||||
var firstClosed atomic.Int32
|
||||
var secondClosed atomic.Int32
|
||||
firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
|
||||
return VMMetadata{}, libvirt.Error{Code: libvirt.ERR_RPC, Message: "connection lost"}
|
||||
}}
|
||||
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
||||
if attempts.Add(1) == 1 {
|
||||
return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil
|
||||
}
|
||||
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &secondClosed), nil
|
||||
}, func(int) time.Duration { return time.Millisecond })
|
||||
t.Cleanup(func() { _ = manager.Close() })
|
||||
manager.start()
|
||||
waitForManagedLibvirtGeneration(t, manager, 1)
|
||||
|
||||
_, err := manager.CreateVM(context.Background(), CreateVMRequest{})
|
||||
if !operation.IsUncertain(err) {
|
||||
t.Fatalf("transport failure = %v, want uncertain", err)
|
||||
}
|
||||
waitForManagedLibvirtGeneration(t, manager, 2)
|
||||
waitForAtomicValue(t, &firstClosed, 1)
|
||||
if err := manager.Ping(context.Background()); err != nil {
|
||||
t.Fatalf("ping after transport reconnect: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedLibvirtCloseWaitsForReferencesAndStopsReconnect(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var attempts atomic.Int32
|
||||
var closed atomic.Int32
|
||||
backend := &managedLibvirtTestBackend{ping: func(context.Context) error {
|
||||
close(started)
|
||||
<-release
|
||||
return nil
|
||||
}}
|
||||
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
||||
attempts.Add(1)
|
||||
return newManagedLibvirtTestSession(generation, backend, &closed), nil
|
||||
}, func(int) time.Duration { return time.Millisecond })
|
||||
manager.start()
|
||||
waitForManagedLibvirtGeneration(t, manager, 1)
|
||||
|
||||
pingDone := make(chan error, 1)
|
||||
go func() { pingDone <- manager.Ping(context.Background()) }()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("ping did not acquire the session")
|
||||
}
|
||||
closeDone := make(chan error, 1)
|
||||
go func() { closeDone <- manager.Close() }()
|
||||
select {
|
||||
case err := <-closeDone:
|
||||
t.Fatalf("Close returned before the active reference drained: %v", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
if closed.Load() != 0 {
|
||||
t.Fatal("session closed before its active reference drained")
|
||||
}
|
||||
|
||||
close(release)
|
||||
if err := <-pingDone; err != nil {
|
||||
t.Fatalf("in-flight ping: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-closeDone:
|
||||
if err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Close did not finish after the reference drained")
|
||||
}
|
||||
if closed.Load() != 1 {
|
||||
t.Fatalf("session close count = %d, want 1", closed.Load())
|
||||
}
|
||||
manager.signalReconnect()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if attempts.Load() != 1 {
|
||||
t.Fatalf("connection attempts after Close = %d, want 1", attempts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedLibvirtCloseWaitsForBlockingStartup(t *testing.T) {
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
startDone := make(chan struct{})
|
||||
cleanupErr := errors.New("startup session cleanup failed")
|
||||
var closed atomic.Int32
|
||||
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
session := newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed)
|
||||
session.closeFn = func() error {
|
||||
closed.Add(1)
|
||||
return cleanupErr
|
||||
}
|
||||
return session, nil
|
||||
}, func(int) time.Duration { return time.Millisecond })
|
||||
go func() {
|
||||
manager.start()
|
||||
close(startDone)
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("startup factory did not begin")
|
||||
}
|
||||
|
||||
closeDone := make(chan error, 1)
|
||||
go func() { closeDone <- manager.Close() }()
|
||||
select {
|
||||
case err := <-closeDone:
|
||||
t.Fatalf("Close returned while startup was still blocked: %v", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
close(release)
|
||||
select {
|
||||
case <-startDone:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("startup did not finish after factory release")
|
||||
}
|
||||
select {
|
||||
case err := <-closeDone:
|
||||
if !errors.Is(err, cleanupErr) {
|
||||
t.Fatalf("Close error = %v, want startup cleanup error", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Close did not wait for startup cleanup")
|
||||
}
|
||||
if closed.Load() != 1 {
|
||||
t.Fatalf("startup session close count = %d, want 1", closed.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedLibvirtStartAfterCloseDoesNotConnect(t *testing.T) {
|
||||
var attempts atomic.Int32
|
||||
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
|
||||
attempts.Add(1)
|
||||
var closed atomic.Int32
|
||||
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil
|
||||
}, func(int) time.Duration { return time.Millisecond })
|
||||
if err := manager.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
manager.start()
|
||||
if attempts.Load() != 0 {
|
||||
t.Fatalf("connection attempts after Close = %d, want 0", attempts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibvirtEventBroadcasterAdvancesGeneration(t *testing.T) {
|
||||
broadcaster := newLibvirtEventBroadcaster()
|
||||
generation, wake := broadcaster.snapshot()
|
||||
broadcaster.notify()
|
||||
if err := broadcaster.wait(context.Background(), generation, wake); err != nil {
|
||||
t.Fatalf("wait after notify: %v", err)
|
||||
}
|
||||
nextGeneration, _ := broadcaster.snapshot()
|
||||
if nextGeneration != generation+1 {
|
||||
t.Fatalf("generation = %d, want %d", nextGeneration, generation+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibvirtConnectionCleanupUsesReverseRegistrationOrder(t *testing.T) {
|
||||
var calls []string
|
||||
cleanup := &libvirtConnectionCleanup{
|
||||
closeRegistered: true,
|
||||
lifecycleID: 11,
|
||||
jobID: 22,
|
||||
deregister: func(id int) error {
|
||||
calls = append(calls, "event:"+fmt.Sprint(id))
|
||||
return nil
|
||||
},
|
||||
unregisterClose: func() error {
|
||||
calls = append(calls, "close-callback")
|
||||
return nil
|
||||
},
|
||||
closeConnection: func() error {
|
||||
calls = append(calls, "connection")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
if err := cleanup.run(); err != nil {
|
||||
t.Fatalf("cleanup: %v", err)
|
||||
}
|
||||
want := []string{"event:22", "event:11", "close-callback", "connection"}
|
||||
if !reflect.DeepEqual(calls, want) {
|
||||
t.Fatalf("cleanup order = %v, want %v", calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibvirtConnectionCleanupHandlesPartialRegistration(t *testing.T) {
|
||||
var calls []string
|
||||
cleanup := &libvirtConnectionCleanup{
|
||||
closeRegistered: true,
|
||||
lifecycleID: 11,
|
||||
jobID: -1,
|
||||
deregister: func(id int) error {
|
||||
calls = append(calls, "event:"+fmt.Sprint(id))
|
||||
return nil
|
||||
},
|
||||
unregisterClose: func() error {
|
||||
calls = append(calls, "close-callback")
|
||||
return nil
|
||||
},
|
||||
closeConnection: func() error {
|
||||
calls = append(calls, "connection")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
if err := cleanup.run(); err != nil {
|
||||
t.Fatalf("cleanup: %v", err)
|
||||
}
|
||||
want := []string{"event:11", "close-callback", "connection"}
|
||||
if !reflect.DeepEqual(calls, want) {
|
||||
t.Fatalf("partial cleanup order = %v, want %v", calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedLibvirtIgnoresExpectedDeadSessionCleanupError(t *testing.T) {
|
||||
manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) {
|
||||
return nil, errors.New("not used")
|
||||
}, nil)
|
||||
deadSession := &managedLibvirtSession{closeFn: func() error {
|
||||
return fmt.Errorf("unregister stale callback: %w", libvirt.Error{Code: libvirt.ERR_INVALID_CONN})
|
||||
}}
|
||||
deadSession.dead.Store(true)
|
||||
manager.cleanupSession(deadSession)
|
||||
if len(manager.cleanupErrs) != 0 {
|
||||
t.Fatalf("dead connection cleanup errors were retained: %v", manager.cleanupErrs)
|
||||
}
|
||||
|
||||
liveSession := &managedLibvirtSession{closeFn: func() error { return errors.New("close failed") }}
|
||||
manager.cleanupSession(liveSession)
|
||||
if len(manager.cleanupErrs) != 1 {
|
||||
t.Fatalf("live connection cleanup errors = %v, want one retained error", manager.cleanupErrs)
|
||||
}
|
||||
|
||||
mixedSession := &managedLibvirtSession{closeFn: func() error {
|
||||
return errors.Join(libvirt.Error{Code: libvirt.ERR_NO_CONNECT}, errors.New("unexpected cleanup failure"))
|
||||
}}
|
||||
mixedSession.dead.Store(true)
|
||||
manager.cleanupSession(mixedSession)
|
||||
if len(manager.cleanupErrs) != 2 {
|
||||
t.Fatalf("mixed dead connection cleanup errors = %v, want unexpected error retained", manager.cleanupErrs)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,259 @@
|
||||
package operation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultRecoverySweepInterval = 5 * time.Minute
|
||||
defaultRecoveryItemTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// RecoveryReconciler inspects one durable operation. Implementations may only
|
||||
// inspect host evidence or continue an operation-owned, idempotent cleanup
|
||||
// tail. The Coordinator never receives or reconstructs the original mutation.
|
||||
type RecoveryReconciler func(context.Context, Record) (Resolution, error)
|
||||
|
||||
// RecoveryObserver receives bounded operation metadata plus the reconciliation
|
||||
// result. Callers decide how to rate-limit and sanitize local diagnostics. A
|
||||
// zero Record identifies a failure to read the durable candidate index.
|
||||
type RecoveryObserver func(Record, error)
|
||||
|
||||
// RecoveryCoordinator serially scans the durable recovery index. Startup,
|
||||
// libvirt events, successful reconnects, and a periodic safety sweep all use
|
||||
// the same coalesced path, so events can accelerate reconciliation but can
|
||||
// never replace Journal evidence.
|
||||
type RecoveryCoordinator struct {
|
||||
journal *Journal
|
||||
reconcile RecoveryReconciler
|
||||
external <-chan struct{}
|
||||
wake chan struct{}
|
||||
sweepInterval time.Duration
|
||||
itemTimeout time.Duration
|
||||
|
||||
lifecycleMu sync.Mutex
|
||||
started bool
|
||||
stopped bool
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
ready chan struct{}
|
||||
readyOnce sync.Once
|
||||
observer RecoveryObserver
|
||||
}
|
||||
|
||||
// SetObserver installs diagnostics before Start. Replacing it while the worker
|
||||
// is active would create a data race and is therefore rejected.
|
||||
func (c *RecoveryCoordinator) SetObserver(observer RecoveryObserver) error {
|
||||
if c == nil {
|
||||
return errors.New("recovery coordinator is required")
|
||||
}
|
||||
c.lifecycleMu.Lock()
|
||||
defer c.lifecycleMu.Unlock()
|
||||
if c.started || c.stopped {
|
||||
return errors.New("recovery observer must be set before start")
|
||||
}
|
||||
c.observer = observer
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewRecoveryCoordinator(journal *Journal, reconcile RecoveryReconciler,
|
||||
external <-chan struct{}) (*RecoveryCoordinator, error) {
|
||||
return newRecoveryCoordinator(
|
||||
journal, reconcile, external,
|
||||
defaultRecoverySweepInterval, defaultRecoveryItemTimeout,
|
||||
)
|
||||
}
|
||||
|
||||
func newRecoveryCoordinator(journal *Journal, reconcile RecoveryReconciler,
|
||||
external <-chan struct{}, sweepInterval, itemTimeout time.Duration) (*RecoveryCoordinator, error) {
|
||||
if journal == nil || reconcile == nil {
|
||||
return nil, errors.New("operation journal and recovery reconciler are required")
|
||||
}
|
||||
if sweepInterval <= 0 || itemTimeout <= 0 {
|
||||
return nil, errors.New("recovery timing must be positive")
|
||||
}
|
||||
return &RecoveryCoordinator{
|
||||
journal: journal, reconcile: reconcile, external: external,
|
||||
wake: make(chan struct{}, 1), sweepInterval: sweepInterval,
|
||||
itemTimeout: itemTimeout, done: make(chan struct{}), ready: make(chan struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Start launches one recovery worker. With a managed event source only an
|
||||
// external initial-status signal may open the readiness gate; without one the
|
||||
// Journal is scanned immediately. The full pass covers restarts even when
|
||||
// every earlier libvirt event and completed-job statistic was lost.
|
||||
func (c *RecoveryCoordinator) Start(parent context.Context) error {
|
||||
if parent == nil {
|
||||
return errors.New("recovery parent context is required")
|
||||
}
|
||||
c.lifecycleMu.Lock()
|
||||
defer c.lifecycleMu.Unlock()
|
||||
if c.stopped {
|
||||
return errors.New("recovery coordinator is stopped")
|
||||
}
|
||||
if c.started {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
c.cancel = cancel
|
||||
c.started = true
|
||||
go c.run(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wake is safe for event-forwarding goroutines and coalesces repeated events.
|
||||
func (c *RecoveryCoordinator) Wake() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case c.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Ready closes after the first candidate pass finishes. Unresolved evidence
|
||||
// and per-item timeouts still count as a finished pass because their records
|
||||
// remain durably indexed. Main can wait on Ready before accepting new mutation
|
||||
// requests, preventing fresh work from overtaking restart reconciliation.
|
||||
func (c *RecoveryCoordinator) Ready() <-chan struct{} {
|
||||
if c == nil {
|
||||
closed := make(chan struct{})
|
||||
close(closed)
|
||||
return closed
|
||||
}
|
||||
return c.ready
|
||||
}
|
||||
|
||||
// Stop prevents future scans and cancels the current per-operation context.
|
||||
// Wait must complete before closing libvirt or the Journal. If a native call
|
||||
// ignores cancellation, Wait deliberately remains blocked until that call
|
||||
// really returns, preserving the same resource-lifetime rule as mutations.
|
||||
func (c *RecoveryCoordinator) Stop() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.lifecycleMu.Lock()
|
||||
if c.stopped {
|
||||
c.lifecycleMu.Unlock()
|
||||
return
|
||||
}
|
||||
c.stopped = true
|
||||
if c.cancel != nil {
|
||||
c.cancel()
|
||||
}
|
||||
if !c.started {
|
||||
close(c.done)
|
||||
c.markReady()
|
||||
}
|
||||
c.lifecycleMu.Unlock()
|
||||
}
|
||||
|
||||
func (c *RecoveryCoordinator) Wait(ctx context.Context) error {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
if ctx == nil {
|
||||
return errors.New("recovery wait context is required")
|
||||
}
|
||||
c.lifecycleMu.Lock()
|
||||
started := c.started
|
||||
stopped := c.stopped
|
||||
done := c.done
|
||||
c.lifecycleMu.Unlock()
|
||||
if !started && !stopped {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *RecoveryCoordinator) run(ctx context.Context) {
|
||||
defer close(c.done)
|
||||
defer c.markReady()
|
||||
ticker := time.NewTicker(c.sweepInterval)
|
||||
defer ticker.Stop()
|
||||
external := c.external
|
||||
externalReadyRequired := external != nil
|
||||
initialPending := true
|
||||
|
||||
// A nil event source is useful for backends whose connection is already
|
||||
// known-ready. A managed source emits an initial-status signal after its
|
||||
// first connection attempt. Periodic or internal wakes may still make safe
|
||||
// read-only attempts while that signal is pending, but only a successful
|
||||
// external pass may open the admission gate. A failed index read is not a
|
||||
// completed pass and cannot open it in either mode.
|
||||
if !externalReadyRequired && c.reconcileAll(ctx) == nil {
|
||||
c.markReady()
|
||||
initialPending = false
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
passErr := c.reconcileAll(ctx)
|
||||
if initialPending && !externalReadyRequired && passErr == nil {
|
||||
c.markReady()
|
||||
initialPending = false
|
||||
}
|
||||
case <-c.wake:
|
||||
passErr := c.reconcileAll(ctx)
|
||||
if initialPending && !externalReadyRequired && passErr == nil {
|
||||
c.markReady()
|
||||
initialPending = false
|
||||
}
|
||||
case _, ok := <-external:
|
||||
if !ok {
|
||||
external = nil
|
||||
continue
|
||||
}
|
||||
passErr := c.reconcileAll(ctx)
|
||||
if initialPending && passErr == nil {
|
||||
c.markReady()
|
||||
initialPending = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *RecoveryCoordinator) markReady() {
|
||||
c.readyOnce.Do(func() { close(c.ready) })
|
||||
}
|
||||
|
||||
func (c *RecoveryCoordinator) reconcileAll(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
records, err := c.journal.ListRecoveryCandidates()
|
||||
if err != nil {
|
||||
// A corrupt indexed candidate is fail-closed by the Journal. Leave the
|
||||
// index untouched for an operator to repair; a later signal retries it.
|
||||
c.observe(Record{}, err)
|
||||
return err
|
||||
}
|
||||
for _, record := range records {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
itemCtx, cancel := context.WithTimeout(ctx, c.itemTimeout)
|
||||
result, reconcileErr := c.journal.Reconcile(itemCtx, record.OperationID, c.reconcile)
|
||||
cancel()
|
||||
c.observe(result, reconcileErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *RecoveryCoordinator) observe(record Record, err error) {
|
||||
if c.observer != nil {
|
||||
c.observer(cloneRecord(record), err)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,513 @@
|
||||
package operation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
func recoveryTestRecord(id, resource, action string, state State) Record {
|
||||
now := time.Unix(1_800_000_000, 0).UTC()
|
||||
return Record{
|
||||
OperationID: id,
|
||||
ResourceID: resource,
|
||||
Action: action,
|
||||
Fingerprint: strings.Repeat("a", 64),
|
||||
Destructive: true,
|
||||
State: state,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
func recoveryCandidateIDs(t *testing.T, journal *Journal) []string {
|
||||
t.Helper()
|
||||
records, err := journal.ListRecoveryCandidates()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := make([]string, 0, len(records))
|
||||
for _, record := range records {
|
||||
ids = append(ids, record.OperationID)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestSchemaV2MigrationBuildsRecoveryIndexWithoutDecodingCorruptHistoryOnLaterStarts(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "operations.db")
|
||||
journal, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := journal.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
database, err := bolt.Open(path, 0o600, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = database.Update(func(tx *bolt.Tx) error {
|
||||
if err := tx.Bucket(metadataBucket).Put(schemaVersionKey, previousSchemaVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.DeleteBucket(recoveryBucket); err != nil {
|
||||
return err
|
||||
}
|
||||
operations := tx.Bucket(operationsBucket)
|
||||
inflight := tx.Bucket(inflightBucket)
|
||||
records := []Record{
|
||||
recoveryTestRecord("operation-v2-uncertain", "vm-v2-uncertain", "delete-vm", StateUncertain),
|
||||
recoveryTestRecord("operation-v2-dispatch", "vm-v2-dispatch", "delete-vm", StateDispatching),
|
||||
recoveryTestRecord("operation-v2-reserved", "vm-v2-reserved", "delete-vm", StateReserved),
|
||||
recoveryTestRecord("operation-v2-complete", "vm-v2-complete", "delete-vm", StateSucceeded),
|
||||
}
|
||||
failedCreate := recoveryTestRecord("operation-v2-create", "vm-v2-create", "create-vm", StateFailed)
|
||||
failedCreate.Error = &PublicError{Code: "backend_error", Message: "creation result is not proven"}
|
||||
records = append(records, failedCreate)
|
||||
pendingCleanup := recoveryTestRecord("operation-v2-cleanup", "vm-v2-cleanup", "reinstall-vm", StateSucceeded)
|
||||
pendingCleanup.Result = json.RawMessage(`{"reinstall_committed":true,"cleanup_pending":true}`)
|
||||
records = append(records, pendingCleanup)
|
||||
for _, record := range records {
|
||||
if err := putRecord(operations, record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := inflight.Put([]byte("operation-v2-dispatch"), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inflight.Put([]byte("operation-v2-reserved"), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return operations.Put([]byte("operation-corrupt-tombstone"), []byte("not-json"))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reopened, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("migrate schema v2: %v", err)
|
||||
}
|
||||
want := []string{
|
||||
"operation-v2-cleanup",
|
||||
"operation-v2-create",
|
||||
"operation-v2-dispatch",
|
||||
"operation-v2-uncertain",
|
||||
}
|
||||
if got := recoveryCandidateIDs(t, reopened); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("recovery candidates = %v, want %v", got, want)
|
||||
}
|
||||
dispatched, err := reopened.Get("operation-v2-dispatch")
|
||||
if err != nil || dispatched.State != StateUncertain {
|
||||
t.Fatalf("recovered dispatch = %#v err=%v", dispatched, err)
|
||||
}
|
||||
reserved, err := reopened.Get("operation-v2-reserved")
|
||||
if err != nil || reserved.State != StateFailed || reserved.Error == nil || reserved.Error.Code != "operation_not_dispatched" {
|
||||
t.Fatalf("recovered reservation = %#v err=%v", reserved, err)
|
||||
}
|
||||
if err := reopened.db.View(func(tx *bolt.Tx) error {
|
||||
if got := tx.Bucket(metadataBucket).Get(schemaVersionKey); string(got) != string(schemaVersion) {
|
||||
return fmt.Errorf("schema = %q", got)
|
||||
}
|
||||
if tx.Bucket(recoveryBucket) == nil {
|
||||
return errors.New("recovery bucket was not created")
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := reopened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Schema v3 restarts read only the two active indexes. The unrelated corrupt
|
||||
// permanent history row remains unusable without blocking startup.
|
||||
reopened, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen schema v3: %v", err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
if got := recoveryCandidateIDs(t, reopened); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("reopened recovery candidates = %v, want %v", got, want)
|
||||
}
|
||||
if _, err := reopened.Get("operation-corrupt-tombstone"); err == nil {
|
||||
t.Fatal("corrupt permanent record unexpectedly decoded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryIndexExcludesLiveDispatchAndTracksUncertainResolution(t *testing.T) {
|
||||
journal := testJournal(t)
|
||||
spec := testSpec("operation-index-live", "vm-index-live", `{}`)
|
||||
if _, _, err := journal.Reserve(spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := journal.markDispatching(spec.OperationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recoveryCandidateIDs(t, journal); len(got) != 0 {
|
||||
t.Fatalf("live dispatch entered recovery index: %v", got)
|
||||
}
|
||||
if _, err := journal.markUncertain(spec.OperationID, errors.New("connection lost")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recoveryCandidateIDs(t, journal); !reflect.DeepEqual(got, []string{spec.OperationID}) {
|
||||
t.Fatalf("uncertain recovery candidates = %v", got)
|
||||
}
|
||||
reconciled, err := journal.Reconcile(context.Background(), spec.OperationID,
|
||||
func(context.Context, Record) (Resolution, error) {
|
||||
return Resolution{Resolved: true, Succeeded: true, Result: json.RawMessage(`{"absent":true}`)}, nil
|
||||
})
|
||||
if err != nil || reconciled.State != StateSucceeded {
|
||||
t.Fatalf("reconcile = %#v err=%v", reconciled, err)
|
||||
}
|
||||
if got := recoveryCandidateIDs(t, journal); len(got) != 0 {
|
||||
t.Fatalf("resolved operation retained in recovery index: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingCleanupEvidenceIsNotCompactedBeforeRecovery(t *testing.T) {
|
||||
journal := testJournal(t)
|
||||
old := time.Unix(1_500_000_000, 0).UTC()
|
||||
journal.SetClockForTesting(func() time.Time { return old })
|
||||
spec := Spec{
|
||||
OperationID: "operation-retained-cleanup", ResourceID: "123e4567-e89b-42d3-a456-426614174099",
|
||||
Action: "reinstall-vm", Fingerprint: strings.Repeat("b", 64), Destructive: true,
|
||||
HeavyIO: true,
|
||||
}
|
||||
record, _, err := journal.Execute(context.Background(), spec,
|
||||
func(context.Context) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"reinstall_committed":true,"cleanup_pending":true}`), nil
|
||||
})
|
||||
if err != nil || !succeededCleanupCanReconcile(record) {
|
||||
t.Fatalf("pending cleanup record = %#v err=%v", record, err)
|
||||
}
|
||||
|
||||
late := old.Add(operationDetailRetention + time.Hour)
|
||||
journal.SetClockForTesting(func() time.Time { return late })
|
||||
if accepted, err := journal.UseNonce("retainCleanupEvidence1", late, late.Add(time.Minute)); err != nil || !accepted {
|
||||
t.Fatalf("trigger retention accepted=%v err=%v", accepted, err)
|
||||
}
|
||||
retained, err := journal.Get(spec.OperationID)
|
||||
if err != nil || retained.Tombstone || !succeededCleanupCanReconcile(retained) ||
|
||||
len(retained.Result) == 0 || len(retained.LockKeys) == 0 || !retained.HeavyIO {
|
||||
t.Fatalf("unfinished cleanup evidence was compacted: %#v err=%v", retained, err)
|
||||
}
|
||||
if got := recoveryCandidateIDs(t, journal); !reflect.DeepEqual(got, []string{spec.OperationID}) {
|
||||
t.Fatalf("retained cleanup candidates = %v", got)
|
||||
}
|
||||
|
||||
resolved, err := journal.Reconcile(context.Background(), spec.OperationID,
|
||||
func(context.Context, Record) (Resolution, error) {
|
||||
return Resolution{Resolved: true, Succeeded: true,
|
||||
Result: json.RawMessage(`{"reinstall_committed":true,"cleanup_pending":false}`)}, nil
|
||||
})
|
||||
if err != nil || resolved.State != StateSucceeded || succeededCleanupCanReconcile(resolved) {
|
||||
t.Fatalf("resolved cleanup = %#v err=%v", resolved, err)
|
||||
}
|
||||
if got := recoveryCandidateIDs(t, journal); len(got) != 0 {
|
||||
t.Fatalf("resolved cleanup retained in recovery index: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func makeRecoveryCandidate(t *testing.T, journal *Journal, id, resource string) {
|
||||
t.Helper()
|
||||
_, _, err := journal.Execute(context.Background(), testSpec(id, resource, `{}`),
|
||||
func(context.Context) (json.RawMessage, error) {
|
||||
return nil, MarkUncertain(errors.New("lost result"))
|
||||
})
|
||||
if err == nil || !IsUncertain(err) {
|
||||
t.Fatalf("create recovery candidate err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForRecoveryRecord(t *testing.T, journal *Journal, operationID string,
|
||||
want State, timeout time.Duration) Record {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
record, err := journal.Get(operationID)
|
||||
if err == nil && record.State == want {
|
||||
return record
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("operation %s did not reach %s: record=%#v err=%v", operationID, want, record, err)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryCoordinatorScansAtStartupAndCoalescesEventWakeups(t *testing.T) {
|
||||
journal := testJournal(t)
|
||||
makeRecoveryCandidate(t, journal, "operation-recovery-event", "vm-recovery-event")
|
||||
external := make(chan struct{}, 1)
|
||||
first := make(chan struct{})
|
||||
var calls atomic.Int32
|
||||
var active atomic.Int32
|
||||
var maximum atomic.Int32
|
||||
var resolve atomic.Bool
|
||||
reconcile := func(_ context.Context, record Record) (Resolution, error) {
|
||||
if record.OperationID != "operation-recovery-event" || record.Action != "delete-vm" {
|
||||
t.Errorf("unexpected recovery record: %#v", record)
|
||||
}
|
||||
current := active.Add(1)
|
||||
defer active.Add(-1)
|
||||
for {
|
||||
previous := maximum.Load()
|
||||
if current <= previous || maximum.CompareAndSwap(previous, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if calls.Add(1) == 1 {
|
||||
close(first)
|
||||
}
|
||||
if !resolve.Load() {
|
||||
return Resolution{}, nil
|
||||
}
|
||||
return Resolution{Resolved: true, Succeeded: true, Result: json.RawMessage(`{"absent":true}`)}, nil
|
||||
}
|
||||
coordinator, err := newRecoveryCoordinator(journal, reconcile, external, time.Hour, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
if err := coordinator.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-coordinator.Ready():
|
||||
t.Fatal("coordinator became ready before the initial connection signal")
|
||||
default:
|
||||
}
|
||||
coordinator.Wake()
|
||||
select {
|
||||
case <-first:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pre-connection recovery attempt did not run")
|
||||
}
|
||||
select {
|
||||
case <-coordinator.Ready():
|
||||
t.Fatal("internal wake opened readiness before the initial connection signal")
|
||||
default:
|
||||
}
|
||||
external <- struct{}{}
|
||||
select {
|
||||
case <-coordinator.Ready():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("external initial-status pass did not open readiness")
|
||||
}
|
||||
record, err := journal.Get("operation-recovery-event")
|
||||
if err != nil || record.State != StateUncertain {
|
||||
t.Fatalf("unresolved initial pass did not retain candidate: %#v err=%v", record, err)
|
||||
}
|
||||
resolve.Store(true)
|
||||
for index := 0; index < 100; index++ {
|
||||
coordinator.Wake()
|
||||
select {
|
||||
case external <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
waitForRecoveryRecord(t, journal, "operation-recovery-event", StateSucceeded, time.Second)
|
||||
coordinator.Stop()
|
||||
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer waitCancel()
|
||||
if err := coordinator.Wait(waitCtx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if maximum.Load() != 1 {
|
||||
t.Fatalf("recovery callbacks ran concurrently: max=%d", maximum.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryCoordinatorReadyFailsClosedOnUnreadableIndex(t *testing.T) {
|
||||
journal := testJournal(t)
|
||||
if err := journal.db.Update(func(tx *bolt.Tx) error {
|
||||
return tx.Bucket(recoveryBucket).Put([]byte("operation-missing-from-history"), nil)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reconcilerCalled atomic.Bool
|
||||
coordinator, err := newRecoveryCoordinator(journal,
|
||||
func(context.Context, Record) (Resolution, error) {
|
||||
reconcilerCalled.Store(true)
|
||||
return Resolution{}, nil
|
||||
}, nil, time.Hour, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observed := make(chan error, 1)
|
||||
if err := coordinator.SetObserver(func(record Record, err error) {
|
||||
if record.OperationID == "" && err != nil {
|
||||
select {
|
||||
case observed <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := coordinator.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err := <-observed:
|
||||
if !strings.Contains(err.Error(), "missing operation") {
|
||||
t.Fatalf("unexpected index error: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("recovery index failure was not observed")
|
||||
}
|
||||
if reconcilerCalled.Load() {
|
||||
t.Fatal("reconciler ran despite corrupt recovery index")
|
||||
}
|
||||
select {
|
||||
case <-coordinator.Ready():
|
||||
t.Fatal("unreadable recovery index opened readiness")
|
||||
default:
|
||||
}
|
||||
if err := journal.db.Update(func(tx *bolt.Tx) error {
|
||||
return tx.Bucket(recoveryBucket).Delete([]byte("operation-missing-from-history"))
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
coordinator.Wake()
|
||||
select {
|
||||
case <-coordinator.Ready():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("repaired recovery index did not open readiness")
|
||||
}
|
||||
coordinator.Stop()
|
||||
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer waitCancel()
|
||||
if err := coordinator.Wait(waitCtx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryCoordinatorStopBeforeStartClosesLifecycleChannels(t *testing.T) {
|
||||
journal := testJournal(t)
|
||||
coordinator, err := newRecoveryCoordinator(journal,
|
||||
func(context.Context, Record) (Resolution, error) { return Resolution{}, nil },
|
||||
nil, time.Hour, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
coordinator.Stop()
|
||||
select {
|
||||
case <-coordinator.Ready():
|
||||
default:
|
||||
t.Fatal("Stop before Start left Ready open")
|
||||
}
|
||||
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer waitCancel()
|
||||
if err := coordinator.Wait(waitCtx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := coordinator.Start(context.Background()); err == nil {
|
||||
t.Fatal("stopped coordinator unexpectedly restarted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryCoordinatorPeriodicSweepAndPerItemTimeout(t *testing.T) {
|
||||
journal := testJournal(t)
|
||||
makeRecoveryCandidate(t, journal, "operation-a-timeout", "vm-a-timeout")
|
||||
makeRecoveryCandidate(t, journal, "operation-b-resolved", "vm-b-resolved")
|
||||
var timedOut atomic.Int32
|
||||
reconcile := func(ctx context.Context, record Record) (Resolution, error) {
|
||||
if record.OperationID == "operation-a-timeout" {
|
||||
<-ctx.Done()
|
||||
timedOut.Add(1)
|
||||
return Resolution{}, ctx.Err()
|
||||
}
|
||||
return Resolution{Resolved: true, Succeeded: true, Result: json.RawMessage(`{"absent":true}`)}, nil
|
||||
}
|
||||
coordinator, err := newRecoveryCoordinator(journal, reconcile, nil, 15*time.Millisecond, 10*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
if err := coordinator.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForRecoveryRecord(t, journal, "operation-b-resolved", StateSucceeded, time.Second)
|
||||
if timedOut.Load() == 0 {
|
||||
t.Fatal("per-item timeout did not cancel the first recovery attempt")
|
||||
}
|
||||
firstCount := timedOut.Load()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for timedOut.Load() <= firstCount && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if timedOut.Load() <= firstCount {
|
||||
t.Fatal("periodic safety sweep did not retry the unresolved candidate")
|
||||
}
|
||||
coordinator.Stop()
|
||||
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer waitCancel()
|
||||
if err := coordinator.Wait(waitCtx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryCoordinatorWaitDoesNotOutrunUncooperativeReconcile(t *testing.T) {
|
||||
journal := testJournal(t)
|
||||
makeRecoveryCandidate(t, journal, "operation-recovery-shutdown", "vm-recovery-shutdown")
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
coordinator, err := newRecoveryCoordinator(journal,
|
||||
func(context.Context, Record) (Resolution, error) {
|
||||
close(started)
|
||||
<-release
|
||||
return Resolution{}, nil
|
||||
}, nil, time.Hour, 10*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := coordinator.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("reconcile did not start")
|
||||
}
|
||||
select {
|
||||
case <-coordinator.Ready():
|
||||
t.Fatal("coordinator became ready before the first scan completed")
|
||||
default:
|
||||
}
|
||||
coordinator.Stop()
|
||||
short, shortCancel := context.WithTimeout(context.Background(), 15*time.Millisecond)
|
||||
defer shortCancel()
|
||||
if err := coordinator.Wait(short); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("Wait returned before native-equivalent call completed: %v", err)
|
||||
}
|
||||
close(release)
|
||||
select {
|
||||
case <-coordinator.Ready():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("coordinator did not become ready after the first scan completed")
|
||||
}
|
||||
long, longCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer longCancel()
|
||||
if err := coordinator.Wait(long); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue