fix(agent): recover libvirt connections and operations

go
codex 1 month ago
parent a73c2695c8
commit 1bfc8b7513

@ -53,6 +53,29 @@ func main() {
if err != nil {
log.Fatalf("initialize Agent request dispatcher: %v", err)
}
var recoveryEvents <-chan struct{}
if source, ok := hv.(hypervisor.RecoveryEventSource); ok {
recoveryEvents = source.RecoveryEvents()
}
recovery, err := operation.NewRecoveryCoordinator(journal, hv.Reconcile, recoveryEvents)
if err != nil {
log.Fatalf("initialize operation recovery: %v", err)
}
if err := recovery.SetObserver(func(record operation.Record, reconcileErr error) {
if reconcileErr == nil {
if record.OperationID != "" {
log.Printf("operation recovery resolved: operation_id=%s action=%s status=%s", record.OperationID, record.Action, record.State)
}
return
}
if record.OperationID == "" {
log.Printf("operation recovery index scan failed; retry is scheduled: %v", reconcileErr)
return
}
log.Printf("operation recovery remains pending: operation_id=%s action=%s status=%s", record.OperationID, record.Action, record.State)
}); err != nil {
log.Fatalf("configure operation recovery diagnostics: %v", err)
}
client, err := control.NewClient(
cfg.PanelURL, cfg.AgentID, []byte(cfg.AgentSecret), handler, console, cfg.MaxBodyBytes, cfg.OperationTimeout(),
)
@ -61,14 +84,30 @@ func main() {
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
log.Printf("virt host agent %s connecting to panel as Agent %s", api.Version, cfg.AgentID)
runErr := client.Run(ctx)
if err := recovery.Start(ctx); err != nil {
log.Fatalf("start operation recovery: %v", err)
}
select {
case <-recovery.Ready():
case <-ctx.Done():
}
var runErr error
if ctx.Err() == nil {
log.Printf("virt host agent %s connecting to panel as Agent %s", api.Version, cfg.AgentID)
runErr = client.Run(ctx)
}
stop()
recovery.Stop()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := handler.Shutdown(shutdownCtx); err != nil {
log.Printf("Agent handlers did not drain safely: %v", err)
os.Exit(1)
}
if err := recovery.Wait(shutdownCtx); err != nil {
log.Printf("Agent operation recovery did not drain safely: %v", err)
os.Exit(1)
}
if err := journal.Wait(shutdownCtx); err != nil {
log.Printf("Agent operation workers did not drain safely: %v", err)
os.Exit(1)

@ -96,27 +96,11 @@ func OpenLibvirt(uri string, local LocalConfig) (Hypervisor, error) {
if err != nil {
return nil, translateLibvirtError("get local libvirt version", err)
}
conn, err := libvirt.NewConnect(uri)
if err != nil {
return nil, translateLibvirtError("connect", err)
}
daemonVersion, err := conn.GetLibVersion()
if err != nil {
_, _ = conn.Close()
return nil, translateLibvirtError("get daemon libvirt version", err)
}
guestAgentResponseTimeoutSupported := supportsGuestAgentResponseTimeout(clientVersion, daemonVersion)
traffic, err := openTrafficStore(local.TrafficStatePath)
if err != nil {
_, _ = conn.Close()
return nil, NewError(ErrUnavailable, "traffic_state_unavailable", "durable traffic state could not be opened", err)
}
return &Libvirt{
conn: conn,
local: local,
traffic: traffic,
guestAgentResponseTimeoutSupported: guestAgentResponseTimeoutSupported,
}, nil
return openManagedLibvirt(uri, local, clientVersion, traffic), nil
}
func (l *Libvirt) Close() error {
@ -1518,7 +1502,7 @@ func translateLibvirtError(action string, err error) error {
return NewError(ErrInvalid, "invalid_libvirt_request", action+" was rejected as invalid", err)
case libvirt.ERR_OPERATION_UNSUPPORTED:
return NewError(ErrUnsupported, "unsupported", action+" is not supported", err)
case libvirt.ERR_NO_CONNECT, libvirt.ERR_AUTH_FAILED:
case libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_AUTH_FAILED:
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", err)
}
}
@ -1529,7 +1513,7 @@ func mutationLibvirtError(action string, err error) error {
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) {
switch libvirtErr.Code {
case libvirt.ERR_SYSTEM_ERROR, libvirt.ERR_RPC, libvirt.ERR_NO_CONNECT, libvirt.ERR_OPERATION_TIMEOUT:
case libvirt.ERR_SYSTEM_ERROR, libvirt.ERR_RPC, libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_OPERATION_TIMEOUT:
return operation.MarkUncertain(translateLibvirtError(action, err))
}
}

@ -0,0 +1,756 @@
//go:build libvirt
package hypervisor
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
)
const (
libvirtReconnectInitialDelay = time.Second
libvirtReconnectMaximumDelay = 30 * time.Second
libvirtCleanupErrorLimit = 16
)
// libvirtJobEvents wakes backup waiters when JOB_COMPLETED arrives. It is a
// hint, not a result: callers must still use GetJobStats to establish the
// terminal state. Closing a connection also advances the generation so a
// waiter can promptly observe the lost session.
var libvirtJobEvents = newLibvirtEventBroadcaster()
type managedLibvirtSession struct {
generation uint64
backend Hypervisor
closeFn func() error
dead atomic.Bool
// refs and retired are protected by managedLibvirt.mu. drained is closed
// exactly once after retirement when the last public call releases its
// immutable session reference.
refs int
retired bool
drained chan struct{}
}
type libvirtConnectionCleanup struct {
closeRegistered bool
lifecycleID int
jobID int
deregister func(int) error
unregisterClose func() error
closeConnection func() error
}
func (cleanup *libvirtConnectionCleanup) run() error {
var cleanupErrors []error
// Registrations are unwound in strict reverse order: JOB_COMPLETED,
// lifecycle, close callback, then the connection reference itself.
if cleanup.jobID >= 0 {
if err := cleanup.deregister(cleanup.jobID); err != nil {
cleanupErrors = append(cleanupErrors, fmt.Errorf("deregister completed job events: %w", err))
}
}
if cleanup.lifecycleID >= 0 {
if err := cleanup.deregister(cleanup.lifecycleID); err != nil {
cleanupErrors = append(cleanupErrors, fmt.Errorf("deregister lifecycle events: %w", err))
}
}
if cleanup.closeRegistered {
if err := cleanup.unregisterClose(); err != nil {
cleanupErrors = append(cleanupErrors, fmt.Errorf("unregister connection close callback: %w", err))
}
}
if err := cleanup.closeConnection(); err != nil {
cleanupErrors = append(cleanupErrors, fmt.Errorf("close libvirt connection: %w", err))
}
return errors.Join(cleanupErrors...)
}
type managedLibvirtSessionFactory func(uint64) (*managedLibvirtSession, error)
type managedLibvirt struct {
ctx context.Context
cancel context.CancelFunc
mu sync.Mutex
current *managedLibvirtSession
nextGeneration uint64
closing bool
lastConnectErr error
currentGeneration atomic.Uint64
reconnect chan struct{}
recovery chan struct{}
factory managedLibvirtSessionFactory
retryDelay func(int) time.Duration
workerWG sync.WaitGroup
retiredWG sync.WaitGroup
cleanupMu sync.Mutex
cleanupErrs []error
startOnce sync.Once
closeOnce sync.Once
closeErr error
}
func openManagedLibvirt(uri string, local LocalConfig, clientVersion uint32, traffic *trafficStore) *managedLibvirt {
var manager *managedLibvirt
manager = newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
return manager.openSession(uri, local, clientVersion, traffic, generation)
}, defaultLibvirtReconnectDelay)
manager.start()
return manager
}
func (m *managedLibvirt) start() {
// Preserve the useful startup behavior of attempting the local socket once,
// while allowing the Agent to start and report unavailable if the daemon is
// temporarily down. Recovery gets one initial wakeup in either case.
m.startOnce.Do(func() {
// Register the whole startup/reconnect lifecycle before the first
// potentially blocking connection attempt. Close uses the same mutex
// before Wait, which prevents an Add-after-Wait race.
m.mu.Lock()
if m.closing {
m.mu.Unlock()
return
}
m.workerWG.Add(1)
m.mu.Unlock()
_ = m.connectAndPublish()
m.signalRecovery()
go m.reconnectLoop()
if !m.hasUsableSession() {
m.signalReconnect()
}
})
}
func newManagedLibvirt(factory managedLibvirtSessionFactory, retryDelay func(int) time.Duration) *managedLibvirt {
ctx, cancel := context.WithCancel(context.Background())
if retryDelay == nil {
retryDelay = defaultLibvirtReconnectDelay
}
return &managedLibvirt{
ctx: ctx,
cancel: cancel,
reconnect: make(chan struct{}, 1),
recovery: make(chan struct{}, 1),
factory: factory,
retryDelay: retryDelay,
}
}
func defaultLibvirtReconnectDelay(failures int) time.Duration {
if failures <= 1 {
return libvirtReconnectInitialDelay
}
delay := libvirtReconnectInitialDelay
for attempt := 1; attempt < failures && delay < libvirtReconnectMaximumDelay; attempt++ {
delay *= 2
if delay > libvirtReconnectMaximumDelay {
return libvirtReconnectMaximumDelay
}
}
return delay
}
func (m *managedLibvirt) openSession(uri string, local LocalConfig, clientVersion uint32, traffic *trafficStore, generation uint64) (*managedLibvirtSession, error) {
conn, err := libvirt.NewConnect(uri)
if err != nil {
return nil, translateLibvirtError("connect", err)
}
session := &managedLibvirtSession{
generation: generation,
drained: make(chan struct{}),
}
cleanup := &libvirtConnectionCleanup{
lifecycleID: -1,
jobID: -1,
deregister: conn.DomainEventDeregister,
unregisterClose: func() error {
return conn.UnregisterCloseCallback()
},
closeConnection: func() error {
_, err := conn.Close()
return err
},
}
session.closeFn = cleanup.run
failSetup := func(action string, cause error) (*managedLibvirtSession, error) {
return nil, errors.Join(translateLibvirtError(action, cause), session.closeFn())
}
if err := conn.RegisterCloseCallback(func(_ *libvirt.Connect, _ libvirt.ConnectCloseReason) {
// The callback can run on libvirt's event thread. It only marks this
// immutable generation dead and performs non-blocking wakeups; reconnect,
// callback deregistration, and Close all happen on manager workers.
session.dead.Store(true)
if !m.isCurrentGeneration(session) {
return
}
libvirtJobEvents.notify()
storageStreamEvents.notify()
m.signalRecovery()
m.signalReconnect()
}); err != nil {
return failSetup("register connection close callback", err)
}
cleanup.closeRegistered = true
cleanup.lifecycleID, err = conn.DomainEventLifecycleRegister(nil, func(_ *libvirt.Connect, _ *libvirt.Domain, _ *libvirt.DomainEventLifecycle) {
m.signalRecoveryFor(session)
})
if err != nil {
return failSetup("register domain lifecycle events", err)
}
cleanup.jobID, err = conn.DomainEventJobCompletedRegister(nil, func(_ *libvirt.Connect, _ *libvirt.Domain, _ *libvirt.DomainEventJobCompleted) {
if !m.isCurrentGeneration(session) {
return
}
libvirtJobEvents.notify()
m.signalRecovery()
})
if err != nil {
return failSetup("register completed job events", err)
}
daemonVersion, err := conn.GetLibVersion()
if err != nil {
return failSetup("get daemon libvirt version", err)
}
session.backend = &Libvirt{
conn: conn,
local: local,
traffic: traffic,
guestAgentResponseTimeoutSupported: supportsGuestAgentResponseTimeout(clientVersion, daemonVersion),
}
alive, err := conn.IsAlive()
if err != nil || !alive || session.dead.Load() {
if err != nil {
return failSetup("verify libvirt connection", err)
}
return nil, errors.Join(NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", nil), session.closeFn())
}
return session, nil
}
func (m *managedLibvirt) reconnectLoop() {
defer m.workerWG.Done()
failures := 0
for {
select {
case <-m.ctx.Done():
return
case <-m.reconnect:
}
for {
if m.ctx.Err() != nil {
return
}
m.detachDeadSession()
if m.hasUsableSession() {
failures = 0
break
}
if err := m.connectAndPublish(); err == nil {
failures = 0
break
}
failures++
timer := time.NewTimer(m.retryDelay(failures))
select {
case <-m.ctx.Done():
if !timer.Stop() {
<-timer.C
}
return
case <-m.reconnect:
if !timer.Stop() {
<-timer.C
}
case <-timer.C:
}
}
}
}
func (m *managedLibvirt) connectAndPublish() error {
m.mu.Lock()
if m.closing {
m.mu.Unlock()
return context.Canceled
}
m.nextGeneration++
generation := m.nextGeneration
m.mu.Unlock()
session, err := m.factory(generation)
if err != nil {
m.mu.Lock()
m.lastConnectErr = err
m.mu.Unlock()
return err
}
if session == nil || session.backend == nil || session.closeFn == nil {
err = errors.New("libvirt session factory returned an incomplete session")
if session != nil && session.closeFn != nil {
m.cleanupSession(session)
}
m.mu.Lock()
m.lastConnectErr = err
m.mu.Unlock()
return err
}
if session.drained == nil {
session.drained = make(chan struct{})
}
session.generation = generation
m.mu.Lock()
if m.closing || m.current != nil || session.dead.Load() {
m.mu.Unlock()
m.cleanupSession(session)
if session.dead.Load() {
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", nil)
}
return context.Canceled
}
m.current = session
m.currentGeneration.Store(session.generation)
if session.dead.Load() {
m.current = nil
m.currentGeneration.Store(0)
m.retireSessionLocked(session)
m.mu.Unlock()
return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", nil)
}
m.lastConnectErr = nil
m.mu.Unlock()
m.signalRecovery()
return nil
}
func (m *managedLibvirt) hasUsableSession() bool {
m.mu.Lock()
defer m.mu.Unlock()
return !m.closing && m.current != nil && !m.current.dead.Load()
}
// detachDeadSession runs only outside libvirt callbacks. It atomically stops
// new acquisitions, then retires the old generation after existing calls have
// released their references.
func (m *managedLibvirt) detachDeadSession() bool {
m.mu.Lock()
defer m.mu.Unlock()
if m.current == nil || !m.current.dead.Load() {
return false
}
session := m.current
m.current = nil
m.currentGeneration.Store(0)
m.retireSessionLocked(session)
return true
}
func (m *managedLibvirt) retireSessionLocked(session *managedLibvirtSession) {
if session == nil || session.retired {
return
}
session.retired = true
if session.refs == 0 {
close(session.drained)
}
m.retiredWG.Add(1)
go func() {
defer m.retiredWG.Done()
<-session.drained
m.cleanupSession(session)
}()
}
func (m *managedLibvirt) cleanupSession(session *managedLibvirtSession) {
if session == nil || session.closeFn == nil {
return
}
if err := session.closeFn(); err != nil {
// Once libvirt has told us this generation is dead, INVALID_CONN,
// NO_CONNECT, and RPC errors are expected while unregistering callbacks
// and releasing the stale handle. They must not poison a later clean
// shutdown after the manager has successfully reconnected.
if session.dead.Load() && onlyLibvirtConnectionFailures(err) {
return
}
m.cleanupMu.Lock()
if len(m.cleanupErrs) < libvirtCleanupErrorLimit {
m.cleanupErrs = append(m.cleanupErrs, err)
} else {
m.cleanupErrs[libvirtCleanupErrorLimit-1] = err
}
m.cleanupMu.Unlock()
}
}
func (m *managedLibvirt) acquire(ctx context.Context) (*managedLibvirtSession, func(), error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
m.mu.Lock()
if m.closing || m.current == nil || m.current.dead.Load() {
cause := m.lastConnectErr
m.mu.Unlock()
return nil, nil, NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", cause)
}
session := m.current
session.refs++
m.mu.Unlock()
var once sync.Once
release := func() {
once.Do(func() {
m.mu.Lock()
session.refs--
if session.retired && session.refs == 0 {
close(session.drained)
}
m.mu.Unlock()
})
}
return session, release, nil
}
func (m *managedLibvirt) isCurrentGeneration(session *managedLibvirtSession) bool {
return session != nil && m.currentGeneration.Load() == session.generation
}
func (m *managedLibvirt) signalRecoveryFor(session *managedLibvirtSession) {
if m.isCurrentGeneration(session) {
m.signalRecovery()
}
}
func (m *managedLibvirt) signalRecovery() {
select {
case m.recovery <- struct{}{}:
default:
}
}
func (m *managedLibvirt) signalReconnect() {
select {
case m.reconnect <- struct{}{}:
default:
}
}
func (m *managedLibvirt) markConnectionLost(session *managedLibvirtSession) {
if session == nil {
return
}
session.dead.Store(true)
if !m.isCurrentGeneration(session) {
return
}
libvirtJobEvents.notify()
storageStreamEvents.notify()
m.signalRecovery()
m.signalReconnect()
}
func (m *managedLibvirt) RecoveryEvents() <-chan struct{} { return m.recovery }
func (m *managedLibvirt) Close() error {
m.closeOnce.Do(func() {
m.mu.Lock()
m.closing = true
m.currentGeneration.Store(0)
if m.current != nil {
session := m.current
m.current = nil
m.retireSessionLocked(session)
}
m.cancel()
m.mu.Unlock()
m.workerWG.Wait()
m.retiredWG.Wait()
m.cleanupMu.Lock()
m.closeErr = errors.Join(m.cleanupErrs...)
m.cleanupMu.Unlock()
})
return m.closeErr
}
func isLibvirtConnectionFailure(err error) bool {
var libvirtErr libvirt.Error
if errors.As(err, &libvirtErr) {
switch libvirtErr.Code {
case libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_RPC:
return true
}
}
var publicError *Error
return errors.As(err, &publicError) && publicError.Code == "libvirt_unavailable"
}
func onlyLibvirtConnectionFailures(err error) bool {
if err == nil {
return true
}
if joined, ok := err.(interface{ Unwrap() []error }); ok {
causes := joined.Unwrap()
if len(causes) == 0 {
return false
}
for _, cause := range causes {
if !onlyLibvirtConnectionFailures(cause) {
return false
}
}
return true
}
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
if cause := wrapped.Unwrap(); cause != nil {
return onlyLibvirtConnectionFailures(cause)
}
}
return isLibvirtConnectionFailure(err)
}
func managedCall[T any](m *managedLibvirt, ctx context.Context, mutation bool, call func(Hypervisor) (T, error)) (T, error) {
var zero T
session, release, err := m.acquire(ctx)
if err != nil {
return zero, err
}
defer release()
value, err := call(session.backend)
if err == nil {
return value, nil
}
if isLibvirtConnectionFailure(err) {
m.markConnectionLost(session)
if mutation && !operation.IsUncertain(err) {
err = operation.MarkUncertain(err)
}
}
return value, err
}
func managedCallError(m *managedLibvirt, ctx context.Context, mutation bool, call func(Hypervisor) error) error {
_, err := managedCall(m, ctx, mutation, func(backend Hypervisor) (struct{}, error) {
return struct{}{}, call(backend)
})
return err
}
func (m *managedLibvirt) Ping(ctx context.Context) error {
return managedCallError(m, ctx, false, func(h Hypervisor) error { return h.Ping(ctx) })
}
func (m *managedLibvirt) Capabilities(ctx context.Context) (Capabilities, error) {
return managedCall(m, ctx, false, func(h Hypervisor) (Capabilities, error) { return h.Capabilities(ctx) })
}
func (m *managedLibvirt) ConfigureStorage(ctx context.Context, request StorageConfigurationRequest) (StorageConfigurationResult, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (StorageConfigurationResult, error) {
return h.ConfigureStorage(ctx, request)
})
}
func (m *managedLibvirt) ListVMs(ctx context.Context) ([]VMMetadata, error) {
return managedCall(m, ctx, false, func(h Hypervisor) ([]VMMetadata, error) { return h.ListVMs(ctx) })
}
func (m *managedLibvirt) GetVMState(ctx context.Context, uuid string) (VMState, error) {
return managedCall(m, ctx, false, func(h Hypervisor) (VMState, error) { return h.GetVMState(ctx, uuid) })
}
func (m *managedLibvirt) GetVMMetadata(ctx context.Context, uuid string) (VMMetadata, error) {
return managedCall(m, ctx, false, func(h Hypervisor) (VMMetadata, error) { return h.GetVMMetadata(ctx, uuid) })
}
func (m *managedLibvirt) CreateVM(ctx context.Context, request CreateVMRequest) (VMMetadata, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) { return h.CreateVM(ctx, request) })
}
func (m *managedLibvirt) DeleteVM(ctx context.Context, uuid string, request DeleteVMRequest) error {
return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteVM(ctx, uuid, request) })
}
func (m *managedLibvirt) Power(ctx context.Context, uuid string, request PowerRequest) (VMState, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (VMState, error) { return h.Power(ctx, uuid, request) })
}
func (m *managedLibvirt) UpdateSettings(ctx context.Context, uuid string, request SettingsRequest) (VMMetadata, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) {
return h.UpdateSettings(ctx, uuid, request)
})
}
func (m *managedLibvirt) ResolveNetworkSelector(ctx context.Context, selector string) (LocalNetworkSource, error) {
return managedCall(m, ctx, false, func(h Hypervisor) (LocalNetworkSource, error) {
return h.ResolveNetworkSelector(ctx, selector)
})
}
func (m *managedLibvirt) ChangeNetwork(ctx context.Context, uuid string, request ChangeNetworkRequest) (VMMetadata, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) {
return h.ChangeNetwork(ctx, uuid, request)
})
}
func (m *managedLibvirt) SetNetworkState(ctx context.Context, uuid string, enabled bool, request NetworkStateRequest) error {
return managedCallError(m, ctx, true, func(h Hypervisor) error {
return h.SetNetworkState(ctx, uuid, enabled, request)
})
}
func (m *managedLibvirt) SetServiceState(ctx context.Context, uuid string, enabled bool, request ServiceStateRequest) (VMState, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (VMState, error) {
return h.SetServiceState(ctx, uuid, enabled, request)
})
}
func (m *managedLibvirt) Reinstall(ctx context.Context, uuid string, request ReinstallRequest) (VMMetadata, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) {
return h.Reinstall(ctx, uuid, request)
})
}
func (m *managedLibvirt) GetMetrics(ctx context.Context, uuid string) (Metrics, error) {
return managedCall(m, ctx, false, func(h Hypervisor) (Metrics, error) { return h.GetMetrics(ctx, uuid) })
}
func (m *managedLibvirt) GetDailyTraffic(ctx context.Context, uuid string, days int) ([]DailyTraffic, error) {
return managedCall(m, ctx, false, func(h Hypervisor) ([]DailyTraffic, error) {
return h.GetDailyTraffic(ctx, uuid, days)
})
}
func (m *managedLibvirt) GetConsole(ctx context.Context, uuid string) (ConsoleSession, error) {
return managedCall(m, ctx, false, func(h Hypervisor) (ConsoleSession, error) { return h.GetConsole(ctx, uuid) })
}
func (m *managedLibvirt) ListSnapshots(ctx context.Context, uuid string) ([]Snapshot, error) {
return managedCall(m, ctx, false, func(h Hypervisor) ([]Snapshot, error) { return h.ListSnapshots(ctx, uuid) })
}
func (m *managedLibvirt) CreateSnapshot(ctx context.Context, uuid string, request SnapshotRequest) (Snapshot, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (Snapshot, error) {
return h.CreateSnapshot(ctx, uuid, request)
})
}
func (m *managedLibvirt) DeleteSnapshot(ctx context.Context, uuid, key string, request SnapshotMutationRequest) (SnapshotMutationResult, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (SnapshotMutationResult, error) {
return h.DeleteSnapshot(ctx, uuid, key, request)
})
}
func (m *managedLibvirt) RestoreSnapshot(ctx context.Context, uuid, key string, request SnapshotMutationRequest) (SnapshotMutationResult, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (SnapshotMutationResult, error) {
return h.RestoreSnapshot(ctx, uuid, key, request)
})
}
func (m *managedLibvirt) GetFirewall(ctx context.Context, uuid string) (FirewallPolicy, error) {
return managedCall(m, ctx, false, func(h Hypervisor) (FirewallPolicy, error) { return h.GetFirewall(ctx, uuid) })
}
func (m *managedLibvirt) ApplyFirewall(ctx context.Context, uuid string, policy FirewallPolicy) (FirewallPolicy, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (FirewallPolicy, error) {
return h.ApplyFirewall(ctx, uuid, policy)
})
}
func (m *managedLibvirt) GetRescue(ctx context.Context, uuid string) (RescueState, error) {
return managedCall(m, ctx, false, func(h Hypervisor) (RescueState, error) { return h.GetRescue(ctx, uuid) })
}
func (m *managedLibvirt) EnterRescue(ctx context.Context, uuid string, request RescueRequest) (RescueState, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (RescueState, error) {
return h.EnterRescue(ctx, uuid, request)
})
}
func (m *managedLibvirt) ForceStartRescue(ctx context.Context, uuid string, request ForceRescueStartRequest) (RescueState, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (RescueState, error) {
return h.ForceStartRescue(ctx, uuid, request)
})
}
func (m *managedLibvirt) ExitRescue(ctx context.Context, uuid string, request ExitRescueRequest) (RescueState, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (RescueState, error) {
return h.ExitRescue(ctx, uuid, request)
})
}
func (m *managedLibvirt) ListTemplates(ctx context.Context) ([]ImageTemplate, error) {
return managedCall(m, ctx, false, func(h Hypervisor) ([]ImageTemplate, error) { return h.ListTemplates(ctx) })
}
func (m *managedLibvirt) ImportTemplate(ctx context.Context, request ImportTemplateRequest) (ImageTemplate, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (ImageTemplate, error) {
return h.ImportTemplate(ctx, request)
})
}
func (m *managedLibvirt) UpdateTemplate(ctx context.Context, id string, request UpdateTemplateRequest) (ImageTemplate, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (ImageTemplate, error) {
return h.UpdateTemplate(ctx, id, request)
})
}
func (m *managedLibvirt) DeleteTemplate(ctx context.Context, id string) error {
return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteTemplate(ctx, id) })
}
func (m *managedLibvirt) ListRescueTemplates(ctx context.Context) ([]RescueTemplate, error) {
return managedCall(m, ctx, false, func(h Hypervisor) ([]RescueTemplate, error) { return h.ListRescueTemplates(ctx) })
}
func (m *managedLibvirt) ImportRescueTemplate(ctx context.Context, request ImportRescueTemplateRequest) (RescueTemplate, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (RescueTemplate, error) {
return h.ImportRescueTemplate(ctx, request)
})
}
func (m *managedLibvirt) DeleteRescueTemplate(ctx context.Context, id string) error {
return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteRescueTemplate(ctx, id) })
}
func (m *managedLibvirt) ListUserISOs(ctx context.Context, ownerID int64) ([]UserISO, error) {
return managedCall(m, ctx, false, func(h Hypervisor) ([]UserISO, error) { return h.ListUserISOs(ctx, ownerID) })
}
func (m *managedLibvirt) ImportUserISO(ctx context.Context, request ImportUserISORequest) (UserISO, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (UserISO, error) { return h.ImportUserISO(ctx, request) })
}
func (m *managedLibvirt) DeleteUserISO(ctx context.Context, id string, ownerID int64) error {
return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteUserISO(ctx, id, ownerID) })
}
func (m *managedLibvirt) Reconcile(ctx context.Context, record operation.Record) (operation.Resolution, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (operation.Resolution, error) {
return h.Reconcile(ctx, record)
})
}
var _ Hypervisor = (*managedLibvirt)(nil)
var _ RecoveryEventSource = (*managedLibvirt)(nil)

@ -0,0 +1,472 @@
//go:build libvirt
package hypervisor
import (
"context"
"errors"
"fmt"
"reflect"
"sync/atomic"
"testing"
"time"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
)
type managedLibvirtTestBackend struct {
Unsupported
ping func(context.Context) error
createVM func(context.Context, CreateVMRequest) (VMMetadata, error)
}
func (b *managedLibvirtTestBackend) Ping(ctx context.Context) error {
if b.ping != nil {
return b.ping(ctx)
}
return nil
}
func (b *managedLibvirtTestBackend) CreateVM(ctx context.Context, request CreateVMRequest) (VMMetadata, error) {
if b.createVM != nil {
return b.createVM(ctx, request)
}
return VMMetadata{}, nil
}
func newManagedLibvirtTestSession(generation uint64, backend Hypervisor, closed *atomic.Int32) *managedLibvirtSession {
return &managedLibvirtSession{
generation: generation,
backend: backend,
drained: make(chan struct{}),
closeFn: func() error {
closed.Add(1)
return nil
},
}
}
func waitForManagedLibvirtGeneration(t *testing.T, manager *managedLibvirt, generation uint64) *managedLibvirtSession {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
manager.mu.Lock()
session := manager.current
manager.mu.Unlock()
if session != nil && session.generation == generation && !session.dead.Load() {
return session
}
time.Sleep(time.Millisecond)
}
t.Fatalf("libvirt generation %d was not published", generation)
return nil
}
func waitForAtomicValue(t *testing.T, value *atomic.Int32, expected int32) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if value.Load() == expected {
return
}
time.Sleep(time.Millisecond)
}
t.Fatalf("value = %d, want %d", value.Load(), expected)
}
func drainManagedLibvirtEvents(events <-chan struct{}) {
for {
select {
case <-events:
default:
return
}
}
}
func TestManagedLibvirtInitialFailureSignalsAndReconnects(t *testing.T) {
var attempts atomic.Int32
var closed atomic.Int32
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
if attempts.Add(1) == 1 {
return nil, NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", errors.New("daemon down"))
}
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil
}, func(int) time.Duration { return time.Millisecond })
t.Cleanup(func() { _ = manager.Close() })
manager.start()
select {
case <-manager.RecoveryEvents():
case <-time.After(time.Second):
t.Fatal("initial connection attempt did not signal recovery readiness")
}
waitForManagedLibvirtGeneration(t, manager, 2)
select {
case <-manager.RecoveryEvents():
case <-time.After(time.Second):
t.Fatal("successful reconnect did not signal recovery")
}
if err := manager.Ping(context.Background()); err != nil {
t.Fatalf("ping after reconnect: %v", err)
}
if attempts.Load() != 2 {
t.Fatalf("connection attempts = %d, want 2", attempts.Load())
}
}
func TestManagedLibvirtReconnectPublishesNewSessionBeforeOldCallDrains(t *testing.T) {
started := make(chan struct{})
release := make(chan struct{})
var firstClosed atomic.Int32
var secondClosed atomic.Int32
var secondPings atomic.Int32
var attempts atomic.Int32
firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
close(started)
<-release
return VMMetadata{UUID: "first-result"}, nil
}}
secondBackend := &managedLibvirtTestBackend{ping: func(context.Context) error {
secondPings.Add(1)
return nil
}}
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
if attempts.Add(1) == 1 {
return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil
}
return newManagedLibvirtTestSession(generation, secondBackend, &secondClosed), nil
}, func(int) time.Duration { return time.Millisecond })
t.Cleanup(func() {
select {
case <-release:
default:
close(release)
}
_ = manager.Close()
})
manager.start()
firstSession := waitForManagedLibvirtGeneration(t, manager, 1)
drainManagedLibvirtEvents(manager.RecoveryEvents())
callDone := make(chan error, 1)
go func() {
_, err := manager.CreateVM(context.Background(), CreateVMRequest{})
callDone <- err
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("first generation call did not start")
}
manager.markConnectionLost(firstSession)
secondSession := waitForManagedLibvirtGeneration(t, manager, 2)
if secondSession == firstSession {
t.Fatal("reconnect reused the retired session")
}
if err := manager.Ping(context.Background()); err != nil {
t.Fatalf("new generation ping: %v", err)
}
if secondPings.Load() != 1 {
t.Fatalf("new calls did not use the published generation: pings = %d", secondPings.Load())
}
if firstClosed.Load() != 0 {
t.Fatal("old connection closed while an in-flight call still held it")
}
close(release)
if err := <-callDone; err != nil {
t.Fatalf("late result from old generation: %v", err)
}
waitForAtomicValue(t, &firstClosed, 1)
// A delayed callback from the retired connection must not wake recovery or
// evict the replacement generation.
drainManagedLibvirtEvents(manager.RecoveryEvents())
manager.signalRecoveryFor(firstSession)
manager.markConnectionLost(firstSession)
select {
case <-manager.RecoveryEvents():
t.Fatal("stale generation emitted a recovery event")
case <-time.After(20 * time.Millisecond):
}
if current := waitForManagedLibvirtGeneration(t, manager, 2); current != secondSession {
t.Fatal("stale generation replaced the current session")
}
}
func TestManagedLibvirtDisconnectedMutationWasNotDispatched(t *testing.T) {
manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) {
return nil, errors.New("not used")
}, nil)
t.Cleanup(func() { _ = manager.Close() })
_, mutationErr := manager.CreateVM(context.Background(), CreateVMRequest{})
if operation.IsUncertain(mutationErr) {
t.Fatalf("disconnected mutation error = %v, backend was not invoked", mutationErr)
}
if !errors.Is(mutationErr, ErrUnavailable) {
t.Fatalf("disconnected mutation error = %v, want unavailable cause", mutationErr)
}
readErr := manager.Ping(context.Background())
if operation.IsUncertain(readErr) {
t.Fatalf("read-only ping error = %v, must not be marked uncertain", readErr)
}
if !errors.Is(readErr, ErrUnavailable) {
t.Fatalf("read-only ping error = %v, want unavailable", readErr)
}
}
func TestManagedLibvirtTransportErrorRetiresSessionAndReconnects(t *testing.T) {
var attempts atomic.Int32
var firstClosed atomic.Int32
var secondClosed atomic.Int32
firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) {
return VMMetadata{}, libvirt.Error{Code: libvirt.ERR_RPC, Message: "connection lost"}
}}
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
if attempts.Add(1) == 1 {
return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil
}
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &secondClosed), nil
}, func(int) time.Duration { return time.Millisecond })
t.Cleanup(func() { _ = manager.Close() })
manager.start()
waitForManagedLibvirtGeneration(t, manager, 1)
_, err := manager.CreateVM(context.Background(), CreateVMRequest{})
if !operation.IsUncertain(err) {
t.Fatalf("transport failure = %v, want uncertain", err)
}
waitForManagedLibvirtGeneration(t, manager, 2)
waitForAtomicValue(t, &firstClosed, 1)
if err := manager.Ping(context.Background()); err != nil {
t.Fatalf("ping after transport reconnect: %v", err)
}
}
func TestManagedLibvirtCloseWaitsForReferencesAndStopsReconnect(t *testing.T) {
started := make(chan struct{})
release := make(chan struct{})
var attempts atomic.Int32
var closed atomic.Int32
backend := &managedLibvirtTestBackend{ping: func(context.Context) error {
close(started)
<-release
return nil
}}
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
attempts.Add(1)
return newManagedLibvirtTestSession(generation, backend, &closed), nil
}, func(int) time.Duration { return time.Millisecond })
manager.start()
waitForManagedLibvirtGeneration(t, manager, 1)
pingDone := make(chan error, 1)
go func() { pingDone <- manager.Ping(context.Background()) }()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("ping did not acquire the session")
}
closeDone := make(chan error, 1)
go func() { closeDone <- manager.Close() }()
select {
case err := <-closeDone:
t.Fatalf("Close returned before the active reference drained: %v", err)
case <-time.After(20 * time.Millisecond):
}
if closed.Load() != 0 {
t.Fatal("session closed before its active reference drained")
}
close(release)
if err := <-pingDone; err != nil {
t.Fatalf("in-flight ping: %v", err)
}
select {
case err := <-closeDone:
if err != nil {
t.Fatalf("Close: %v", err)
}
case <-time.After(time.Second):
t.Fatal("Close did not finish after the reference drained")
}
if closed.Load() != 1 {
t.Fatalf("session close count = %d, want 1", closed.Load())
}
manager.signalReconnect()
time.Sleep(20 * time.Millisecond)
if attempts.Load() != 1 {
t.Fatalf("connection attempts after Close = %d, want 1", attempts.Load())
}
}
func TestManagedLibvirtCloseWaitsForBlockingStartup(t *testing.T) {
entered := make(chan struct{})
release := make(chan struct{})
startDone := make(chan struct{})
cleanupErr := errors.New("startup session cleanup failed")
var closed atomic.Int32
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
close(entered)
<-release
session := newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed)
session.closeFn = func() error {
closed.Add(1)
return cleanupErr
}
return session, nil
}, func(int) time.Duration { return time.Millisecond })
go func() {
manager.start()
close(startDone)
}()
select {
case <-entered:
case <-time.After(time.Second):
t.Fatal("startup factory did not begin")
}
closeDone := make(chan error, 1)
go func() { closeDone <- manager.Close() }()
select {
case err := <-closeDone:
t.Fatalf("Close returned while startup was still blocked: %v", err)
case <-time.After(20 * time.Millisecond):
}
close(release)
select {
case <-startDone:
case <-time.After(time.Second):
t.Fatal("startup did not finish after factory release")
}
select {
case err := <-closeDone:
if !errors.Is(err, cleanupErr) {
t.Fatalf("Close error = %v, want startup cleanup error", err)
}
case <-time.After(time.Second):
t.Fatal("Close did not wait for startup cleanup")
}
if closed.Load() != 1 {
t.Fatalf("startup session close count = %d, want 1", closed.Load())
}
}
func TestManagedLibvirtStartAfterCloseDoesNotConnect(t *testing.T) {
var attempts atomic.Int32
manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) {
attempts.Add(1)
var closed atomic.Int32
return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil
}, func(int) time.Duration { return time.Millisecond })
if err := manager.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
manager.start()
if attempts.Load() != 0 {
t.Fatalf("connection attempts after Close = %d, want 0", attempts.Load())
}
}
func TestLibvirtEventBroadcasterAdvancesGeneration(t *testing.T) {
broadcaster := newLibvirtEventBroadcaster()
generation, wake := broadcaster.snapshot()
broadcaster.notify()
if err := broadcaster.wait(context.Background(), generation, wake); err != nil {
t.Fatalf("wait after notify: %v", err)
}
nextGeneration, _ := broadcaster.snapshot()
if nextGeneration != generation+1 {
t.Fatalf("generation = %d, want %d", nextGeneration, generation+1)
}
}
func TestLibvirtConnectionCleanupUsesReverseRegistrationOrder(t *testing.T) {
var calls []string
cleanup := &libvirtConnectionCleanup{
closeRegistered: true,
lifecycleID: 11,
jobID: 22,
deregister: func(id int) error {
calls = append(calls, "event:"+fmt.Sprint(id))
return nil
},
unregisterClose: func() error {
calls = append(calls, "close-callback")
return nil
},
closeConnection: func() error {
calls = append(calls, "connection")
return nil
},
}
if err := cleanup.run(); err != nil {
t.Fatalf("cleanup: %v", err)
}
want := []string{"event:22", "event:11", "close-callback", "connection"}
if !reflect.DeepEqual(calls, want) {
t.Fatalf("cleanup order = %v, want %v", calls, want)
}
}
func TestLibvirtConnectionCleanupHandlesPartialRegistration(t *testing.T) {
var calls []string
cleanup := &libvirtConnectionCleanup{
closeRegistered: true,
lifecycleID: 11,
jobID: -1,
deregister: func(id int) error {
calls = append(calls, "event:"+fmt.Sprint(id))
return nil
},
unregisterClose: func() error {
calls = append(calls, "close-callback")
return nil
},
closeConnection: func() error {
calls = append(calls, "connection")
return nil
},
}
if err := cleanup.run(); err != nil {
t.Fatalf("cleanup: %v", err)
}
want := []string{"event:11", "close-callback", "connection"}
if !reflect.DeepEqual(calls, want) {
t.Fatalf("partial cleanup order = %v, want %v", calls, want)
}
}
func TestManagedLibvirtIgnoresExpectedDeadSessionCleanupError(t *testing.T) {
manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) {
return nil, errors.New("not used")
}, nil)
deadSession := &managedLibvirtSession{closeFn: func() error {
return fmt.Errorf("unregister stale callback: %w", libvirt.Error{Code: libvirt.ERR_INVALID_CONN})
}}
deadSession.dead.Store(true)
manager.cleanupSession(deadSession)
if len(manager.cleanupErrs) != 0 {
t.Fatalf("dead connection cleanup errors were retained: %v", manager.cleanupErrs)
}
liveSession := &managedLibvirtSession{closeFn: func() error { return errors.New("close failed") }}
manager.cleanupSession(liveSession)
if len(manager.cleanupErrs) != 1 {
t.Fatalf("live connection cleanup errors = %v, want one retained error", manager.cleanupErrs)
}
mixedSession := &managedLibvirtSession{closeFn: func() error {
return errors.Join(libvirt.Error{Code: libvirt.ERR_NO_CONNECT}, errors.New("unexpected cleanup failure"))
}}
mixedSession.dead.Store(true)
manager.cleanupSession(mixedSession)
if len(manager.cleanupErrs) != 2 {
t.Fatalf("mixed dead connection cleanup errors = %v, want unexpected error retained", manager.cleanupErrs)
}
}

@ -223,10 +223,17 @@ type snapshotBackupJobFake struct {
abortErr error
abortCalls int
getJobStatsCalls []libvirt.DomainGetJobStatsFlags
getJobStatsCall chan struct{}
}
func (f *snapshotBackupJobFake) GetJobStats(flags libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) {
f.getJobStatsCalls = append(f.getJobStatsCalls, flags)
if f.getJobStatsCall != nil {
select {
case f.getJobStatsCall <- struct{}{}:
default:
}
}
if flags == 0 {
if f.activeErr != nil {
return nil, f.activeErr
@ -374,6 +381,94 @@ func TestWaitForBackupTerminalDoesNotAbortNormalCompletion(t *testing.T) {
}
}
func TestWaitForBackupTerminalWakesOnJobCompletedEvent(t *testing.T) {
calls := make(chan struct{}, 4)
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)},
getJobStatsCall: calls,
}
timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second}
type result struct {
terminal libvirt.DomainJobType
err error
}
done := make(chan result, 1)
ctx, cancel := context.WithCancel(context.Background())
defer func() {
cancel()
libvirtJobEvents.notify()
}()
go func() {
terminal, err := waitForBackupTerminalWithTiming(ctx, job, timing)
done <- result{terminal: terminal, err: err}
}()
select {
case <-calls:
case <-time.After(time.Second):
t.Fatal("backup waiter did not inspect the active job")
}
libvirtJobEvents.notify()
select {
case <-calls:
case <-time.After(time.Second):
t.Fatal("spurious JOB_COMPLETED event did not trigger GetJobStats")
}
select {
case result := <-done:
t.Fatalf("event was treated as terminal evidence: terminal=%v err=%v", result.terminal, result.err)
case <-time.After(20 * time.Millisecond):
}
libvirtJobEvents.notify()
select {
case result := <-done:
if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_COMPLETED {
t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err)
}
case <-time.After(time.Second):
t.Fatal("JOB_COMPLETED event did not wake backup waiter")
}
}
func TestAbortBackupWaitWakesOnJobCompletedEvent(t *testing.T) {
calls := make(chan struct{}, 2)
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{
activeBackupJob(), activeBackupJob(), noActiveBackupJob(),
},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)},
getJobStatsCall: calls,
}
timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second}
type result struct {
terminal libvirt.DomainJobType
err error
}
done := make(chan result, 1)
defer libvirtJobEvents.notify()
go func() {
terminal, err := abortBackupAndWaitForTerminal(job, context.DeadlineExceeded, timing)
done <- result{terminal: terminal, err: err}
}()
for call := 0; call < 2; call++ {
select {
case <-calls:
case <-time.After(time.Second):
t.Fatal("abort waiter did not inspect the active job")
}
}
libvirtJobEvents.notify()
select {
case result := <-done:
if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_CANCELLED {
t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err)
}
case <-time.After(time.Second):
t.Fatal("JOB_COMPLETED event did not wake abort confirmation")
}
}
func TestInspectBackupTerminalAcceptsAllBackupTerminalStates(t *testing.T) {
for _, kind := range []libvirt.DomainJobType{
libvirt.DOMAIN_JOB_COMPLETED,

@ -346,6 +346,7 @@ func abortBackupAndWaitForTerminal(domain snapshotBackupJob, cause error,
defer timer.Stop()
var lastInspectErr error
for {
_, jobWake := libvirtJobEvents.snapshot()
terminal, done, inspectErr = inspectBackupTerminal(domain)
if done {
// A retained terminal result is stronger evidence than an error from
@ -356,6 +357,7 @@ func abortBackupAndWaitForTerminal(domain snapshotBackupJob, cause error,
lastInspectErr = inspectErr
}
select {
case <-jobWake:
case <-ticker.C:
case <-timer.C:
// Close the boundary race between the last polling tick and the
@ -383,6 +385,7 @@ func waitForBackupTerminalWithTiming(ctx context.Context, domain snapshotBackupJ
ticker := time.NewTicker(timing.pollInterval)
defer ticker.Stop()
for {
_, jobWake := libvirtJobEvents.snapshot()
terminal, done, err := inspectBackupTerminal(domain)
if err != nil {
return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(err)
@ -396,6 +399,7 @@ func waitForBackupTerminalWithTiming(ctx context.Context, domain snapshotBackupJ
select {
case <-ctx.Done():
return abortBackupAndWaitForTerminal(domain, ctx.Err(), timing)
case <-jobWake:
case <-ticker.C:
}
}

@ -25,7 +25,7 @@ var (
libvirtEventLoopOnce sync.Once
libvirtEventLoopMu sync.RWMutex
libvirtEventLoopErr error
storageStreamEvents = newStreamEventBroadcaster()
storageStreamEvents = newStorageStreamEventBroadcaster()
)
// startLibvirtEventLoop must run before the first libvirt connection is
@ -74,23 +74,26 @@ func currentLibvirtEventLoopError() error {
return libvirtEventLoopErr
}
type streamEventBroadcaster struct {
// libvirtEventBroadcaster is a coalescing generation signal shared by stream
// readiness and domain job completion. Events carry no durable evidence; they
// only tell waiters to re-read the corresponding libvirt state.
type libvirtEventBroadcaster struct {
mu sync.Mutex
generation uint64
wake chan struct{}
}
func newStreamEventBroadcaster() *streamEventBroadcaster {
return &streamEventBroadcaster{wake: make(chan struct{})}
func newLibvirtEventBroadcaster() *libvirtEventBroadcaster {
return &libvirtEventBroadcaster{wake: make(chan struct{})}
}
func (b *streamEventBroadcaster) snapshot() (uint64, <-chan struct{}) {
func (b *libvirtEventBroadcaster) snapshot() (uint64, <-chan struct{}) {
b.mu.Lock()
defer b.mu.Unlock()
return b.generation, b.wake
}
func (b *streamEventBroadcaster) notify() {
func (b *libvirtEventBroadcaster) notify() {
b.mu.Lock()
close(b.wake)
b.generation++
@ -98,19 +101,32 @@ func (b *streamEventBroadcaster) notify() {
b.mu.Unlock()
}
func (b *streamEventBroadcaster) wait(ctx context.Context, generation uint64, wake <-chan struct{}) error {
func (b *libvirtEventBroadcaster) wait(ctx context.Context, generation uint64, wake <-chan struct{}) error {
b.mu.Lock()
changed := b.generation != generation
b.mu.Unlock()
if changed {
return currentStreamEventLoopError()
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-wake:
return currentStreamEventLoopError()
return nil
}
}
type storageStreamEventBroadcaster struct{ *libvirtEventBroadcaster }
func newStorageStreamEventBroadcaster() *storageStreamEventBroadcaster {
return &storageStreamEventBroadcaster{libvirtEventBroadcaster: newLibvirtEventBroadcaster()}
}
func (b *storageStreamEventBroadcaster) wait(ctx context.Context, generation uint64, wake <-chan struct{}) error {
if err := b.libvirtEventBroadcaster.wait(ctx, generation, wake); err != nil {
return err
}
return currentStreamEventLoopError()
}
func currentStreamEventLoopError() error {

@ -559,6 +559,13 @@ type Hypervisor interface {
Reconcile(context.Context, operation.Record) (operation.Resolution, error)
}
// RecoveryEventSource is implemented by backends that can coalesce host-side
// state changes into reconciliation wakeups. Events are hints only; callers
// must always recover from the durable operation journal.
type RecoveryEventSource interface {
RecoveryEvents() <-chan struct{}
}
// Unsupported is embeddable in focused adapters and tests. Every method has
// a stable, capability-gated failure instead of silently succeeding.
type Unsupported struct{}

@ -57,11 +57,13 @@ var (
metadataBucket = []byte("metadata")
operationsBucket = []byte("operations")
inflightBucket = []byte("inflight")
recoveryBucket = []byte("recovery")
operationExpiryBucket = []byte("operation_expiry")
noncesBucket = []byte("nonces")
nonceExpiryBucket = []byte("nonce_expiry")
schemaVersionKey = []byte("schema_version")
schemaVersion = []byte("2")
previousSchemaVersion = []byte("2")
schemaVersion = []byte("3")
)
// ExpectedState is the non-secret, durable evidence contract used after a
@ -349,6 +351,40 @@ func (j *Journal) Get(operationID string) (Record, error) {
return cloneRecord(record), err
}
// ListRecoveryCandidates returns the durable operations whose original
// mutation must never be replayed, but whose observable host state can still
// be reconciled. The dedicated index keeps this proportional to unfinished
// work instead of the permanent operation tombstone history.
func (j *Journal) ListRecoveryCandidates() ([]Record, error) {
records := make([]Record, 0)
err := j.db.View(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
recovery := tx.Bucket(recoveryBucket)
if operations == nil || recovery == nil {
return errors.New("operation recovery buckets are unavailable")
}
return recovery.ForEach(func(key, _ []byte) error {
encoded := operations.Get(key)
if encoded == nil {
return fmt.Errorf("recovery index references missing operation %q", key)
}
record, err := decodeRecord(encoded)
if err != nil {
return fmt.Errorf("decode recovery operation %q: %w", key, err)
}
if record.OperationID != string(key) {
return fmt.Errorf("recovery index key %q does not match operation %q", key, record.OperationID)
}
if !recoveryIndexEligible(record) {
return fmt.Errorf("recovery index references resolved operation %q", key)
}
records = append(records, cloneRecord(record))
return nil
})
})
return records, err
}
// Reserve durably binds operation_id to one exact request fingerprint and its
// non-secret reconciliation descriptor. Records are never deleted, so an old
// destructive operation_id can never become valid again after housekeeping.
@ -362,7 +398,8 @@ func (j *Journal) Reserve(spec Spec) (Record, bool, error) {
err := j.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(operationsBucket)
inflight := tx.Bucket(inflightBucket)
if bucket == nil || inflight == nil {
recovery := tx.Bucket(recoveryBucket)
if bucket == nil || inflight == nil || recovery == nil {
return errors.New("operation buckets are unavailable")
}
if err := pruneExpiredOperationDetails(tx, j.currentTime()); err != nil {
@ -391,7 +428,10 @@ func (j *Journal) Reserve(spec Spec) (Record, bool, error) {
if err := putRecord(bucket, record); err != nil {
return err
}
return inflight.Put([]byte(record.OperationID), nil)
if err := inflight.Put([]byte(record.OperationID), nil); err != nil {
return err
}
return syncRecoveryIndex(recovery, record)
})
return cloneRecord(record), existed, err
}
@ -686,6 +726,9 @@ func (j *Journal) Reconcile(ctx context.Context, operationID string, fn func(con
}
func recordCanReconcile(record Record) bool {
if record.Tombstone {
return false
}
if record.State == StateUncertain || record.State == StateDispatching {
return true
}
@ -703,6 +746,25 @@ func recordCanReconcile(record Record) bool {
return !confirmedAbsent
}
func syncRecoveryIndex(recovery *bolt.Bucket, record Record) error {
if recovery == nil {
return errors.New("operation recovery bucket is unavailable")
}
key := []byte(record.OperationID)
if recoveryIndexEligible(record) {
return recovery.Put(key, nil)
}
return recovery.Delete(key)
}
// recoveryIndexEligible excludes an operation that is still executing in this
// process. Its real worker owns the admission leases and will record a late
// result. On restart initialize first converts its durable inflight record to
// uncertain, after which it becomes a recovery candidate.
func recoveryIndexEligible(record Record) bool {
return record.State != StateDispatching && recordCanReconcile(record)
}
// Older Agents could finalize a committed reinstall while its deterministic
// old-root cleanup was still pending. Permit the same operation's reconcile
// endpoint to replace that result after the cleanup tail is confirmed.
@ -757,11 +819,10 @@ func (j *Journal) initialize() error {
if err != nil {
return fmt.Errorf("create journal metadata bucket: %w", err)
}
if current := metadata.Get(schemaVersionKey); current == nil {
if err := metadata.Put(schemaVersionKey, schemaVersion); err != nil {
return err
}
} else if string(current) != string(schemaVersion) {
current := metadata.Get(schemaVersionKey)
fresh := current == nil
migrating := string(current) == string(previousSchemaVersion)
if !fresh && !migrating && string(current) != string(schemaVersion) {
return fmt.Errorf("unsupported operation journal schema %q", current)
}
operations, err := tx.CreateBucketIfNotExists(operationsBucket)
@ -782,6 +843,16 @@ func (j *Journal) initialize() error {
if err != nil {
return fmt.Errorf("create inflight operation bucket: %w", err)
}
recovery := tx.Bucket(recoveryBucket)
if recovery == nil {
if !fresh && !migrating {
return errors.New("operation recovery index is unavailable")
}
recovery, err = tx.CreateBucket(recoveryBucket)
if err != nil {
return fmt.Errorf("create operation recovery bucket: %w", err)
}
}
now := j.currentTime()
var recovered []Record
var stale [][]byte
@ -805,6 +876,9 @@ func (j *Journal) initialize() error {
record.Result = nil
record.Error = &PublicError{Code: "result_uncertain", Message: "agent restarted after dispatch; reconciliation is required"}
default:
if err := syncRecoveryIndex(recovery, record); err != nil {
return err
}
stale = append(stale, append([]byte(nil), key...))
return nil
}
@ -818,6 +892,9 @@ func (j *Journal) initialize() error {
if err := putRecord(operations, record); err != nil {
return err
}
if err := syncRecoveryIndex(recovery, record); err != nil {
return err
}
if terminalState(record.State) {
if err := operationExpiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
return err
@ -832,7 +909,26 @@ func (j *Journal) initialize() error {
return err
}
}
return pruneExpiredOperationDetails(tx, now)
if migrating {
// Schema v2 did not retain uncertain operations in its inflight
// index. Rebuild the new recovery index exactly once. A corrupt
// permanent history row that was not in an active index remains an
// unusable tombstone, but cannot prevent the Agent from recovering
// every other valid operation. Corrupt inflight rows fail above.
if err := operations.ForEach(func(key, value []byte) error {
record, decodeErr := decodeRecord(value)
if decodeErr != nil || record.OperationID != string(key) {
return recovery.Delete(key)
}
return syncRecoveryIndex(recovery, record)
}); err != nil {
return fmt.Errorf("migrate operation recovery index: %w", err)
}
}
if err := pruneExpiredOperationDetails(tx, now); err != nil {
return err
}
return metadata.Put(schemaVersionKey, schemaVersion)
})
}
@ -984,7 +1080,8 @@ func (j *Journal) updateRecord(operationID string, mutate func(*Record)) (Record
err := j.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(operationsBucket)
inflight := tx.Bucket(inflightBucket)
if bucket == nil || inflight == nil {
recovery := tx.Bucket(recoveryBucket)
if bucket == nil || inflight == nil || recovery == nil {
return errors.New("operation buckets are unavailable")
}
value := bucket.Get([]byte(operationID))
@ -999,6 +1096,9 @@ func (j *Journal) updateRecord(operationID string, mutate func(*Record)) (Record
if err := putRecord(bucket, record); err != nil {
return err
}
if err := syncRecoveryIndex(recovery, record); err != nil {
return err
}
switch record.State {
case StateReserved, StateDispatching:
if err := inflight.Put([]byte(record.OperationID), nil); err != nil {
@ -1085,8 +1185,9 @@ func (j *Journal) prepareAdmission(spec Spec) (Record, error) {
err := j.db.Update(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
inflight := tx.Bucket(inflightBucket)
recovery := tx.Bucket(recoveryBucket)
expiry := tx.Bucket(operationExpiryBucket)
if operations == nil || inflight == nil || expiry == nil {
if operations == nil || inflight == nil || recovery == nil || expiry == nil {
return errors.New("operation buckets are unavailable")
}
value := operations.Get([]byte(spec.OperationID))
@ -1121,6 +1222,9 @@ func (j *Journal) prepareAdmission(spec Spec) (Record, error) {
if err := putRecord(operations, existing); err != nil {
return err
}
if err := syncRecoveryIndex(recovery, existing); err != nil {
return err
}
if err := inflight.Put([]byte(existing.OperationID), nil); err != nil {
return err
}
@ -1507,7 +1611,8 @@ func terminalState(state State) bool {
func pruneExpiredOperationDetails(tx *bolt.Tx, now time.Time) error {
operations := tx.Bucket(operationsBucket)
expiry := tx.Bucket(operationExpiryBucket)
if operations == nil || expiry == nil {
recovery := tx.Bucket(recoveryBucket)
if operations == nil || expiry == nil || recovery == nil {
return errors.New("operation retention buckets are unavailable")
}
cutoff := now.Add(-operationDetailRetention).UnixNano()
@ -1525,18 +1630,28 @@ func pruneExpiredOperationDetails(tx *bolt.Tx, now time.Time) error {
}
indexedAt := int64(binary.BigEndian.Uint64(key[:8]))
if terminalState(record.State) && !record.Tombstone && record.UpdatedAt.UnixNano() == indexedAt {
record.Tombstone = true
record.Result = nil
record.ExpectedState = nil
record.LockKeys = nil
record.HeavyIO = false
if record.Error != nil {
record.Error = &PublicError{Code: record.Error.Code, Message: "operation details expired; operation_id remains permanently reserved"}
}
if err := putRecord(operations, record); err != nil {
return err
// A publicly terminal create can still lack the absence proof
// needed for a safe refund, and a committed reinstall can retain
// an operation-owned cleanup tail. Keep their evidence until the
// same operation is reconciled; only then does a new retention
// period begin from its final UpdatedAt.
if !recordCanReconcile(record) {
record.Tombstone = true
record.Result = nil
record.ExpectedState = nil
record.LockKeys = nil
record.HeavyIO = false
if record.Error != nil {
record.Error = &PublicError{Code: record.Error.Code, Message: "operation details expired; operation_id remains permanently reserved"}
}
if err := putRecord(operations, record); err != nil {
return err
}
}
}
if err := syncRecoveryIndex(recovery, record); err != nil {
return err
}
}
if err := cursor.Delete(); err != nil {
return err

@ -0,0 +1,259 @@
package operation
import (
"context"
"errors"
"sync"
"time"
)
const (
defaultRecoverySweepInterval = 5 * time.Minute
defaultRecoveryItemTimeout = 30 * time.Second
)
// RecoveryReconciler inspects one durable operation. Implementations may only
// inspect host evidence or continue an operation-owned, idempotent cleanup
// tail. The Coordinator never receives or reconstructs the original mutation.
type RecoveryReconciler func(context.Context, Record) (Resolution, error)
// RecoveryObserver receives bounded operation metadata plus the reconciliation
// result. Callers decide how to rate-limit and sanitize local diagnostics. A
// zero Record identifies a failure to read the durable candidate index.
type RecoveryObserver func(Record, error)
// RecoveryCoordinator serially scans the durable recovery index. Startup,
// libvirt events, successful reconnects, and a periodic safety sweep all use
// the same coalesced path, so events can accelerate reconciliation but can
// never replace Journal evidence.
type RecoveryCoordinator struct {
journal *Journal
reconcile RecoveryReconciler
external <-chan struct{}
wake chan struct{}
sweepInterval time.Duration
itemTimeout time.Duration
lifecycleMu sync.Mutex
started bool
stopped bool
cancel context.CancelFunc
done chan struct{}
ready chan struct{}
readyOnce sync.Once
observer RecoveryObserver
}
// SetObserver installs diagnostics before Start. Replacing it while the worker
// is active would create a data race and is therefore rejected.
func (c *RecoveryCoordinator) SetObserver(observer RecoveryObserver) error {
if c == nil {
return errors.New("recovery coordinator is required")
}
c.lifecycleMu.Lock()
defer c.lifecycleMu.Unlock()
if c.started || c.stopped {
return errors.New("recovery observer must be set before start")
}
c.observer = observer
return nil
}
func NewRecoveryCoordinator(journal *Journal, reconcile RecoveryReconciler,
external <-chan struct{}) (*RecoveryCoordinator, error) {
return newRecoveryCoordinator(
journal, reconcile, external,
defaultRecoverySweepInterval, defaultRecoveryItemTimeout,
)
}
func newRecoveryCoordinator(journal *Journal, reconcile RecoveryReconciler,
external <-chan struct{}, sweepInterval, itemTimeout time.Duration) (*RecoveryCoordinator, error) {
if journal == nil || reconcile == nil {
return nil, errors.New("operation journal and recovery reconciler are required")
}
if sweepInterval <= 0 || itemTimeout <= 0 {
return nil, errors.New("recovery timing must be positive")
}
return &RecoveryCoordinator{
journal: journal, reconcile: reconcile, external: external,
wake: make(chan struct{}, 1), sweepInterval: sweepInterval,
itemTimeout: itemTimeout, done: make(chan struct{}), ready: make(chan struct{}),
}, nil
}
// Start launches one recovery worker. With a managed event source only an
// external initial-status signal may open the readiness gate; without one the
// Journal is scanned immediately. The full pass covers restarts even when
// every earlier libvirt event and completed-job statistic was lost.
func (c *RecoveryCoordinator) Start(parent context.Context) error {
if parent == nil {
return errors.New("recovery parent context is required")
}
c.lifecycleMu.Lock()
defer c.lifecycleMu.Unlock()
if c.stopped {
return errors.New("recovery coordinator is stopped")
}
if c.started {
return nil
}
ctx, cancel := context.WithCancel(parent)
c.cancel = cancel
c.started = true
go c.run(ctx)
return nil
}
// Wake is safe for event-forwarding goroutines and coalesces repeated events.
func (c *RecoveryCoordinator) Wake() {
if c == nil {
return
}
select {
case c.wake <- struct{}{}:
default:
}
}
// Ready closes after the first candidate pass finishes. Unresolved evidence
// and per-item timeouts still count as a finished pass because their records
// remain durably indexed. Main can wait on Ready before accepting new mutation
// requests, preventing fresh work from overtaking restart reconciliation.
func (c *RecoveryCoordinator) Ready() <-chan struct{} {
if c == nil {
closed := make(chan struct{})
close(closed)
return closed
}
return c.ready
}
// Stop prevents future scans and cancels the current per-operation context.
// Wait must complete before closing libvirt or the Journal. If a native call
// ignores cancellation, Wait deliberately remains blocked until that call
// really returns, preserving the same resource-lifetime rule as mutations.
func (c *RecoveryCoordinator) Stop() {
if c == nil {
return
}
c.lifecycleMu.Lock()
if c.stopped {
c.lifecycleMu.Unlock()
return
}
c.stopped = true
if c.cancel != nil {
c.cancel()
}
if !c.started {
close(c.done)
c.markReady()
}
c.lifecycleMu.Unlock()
}
func (c *RecoveryCoordinator) Wait(ctx context.Context) error {
if c == nil {
return nil
}
if ctx == nil {
return errors.New("recovery wait context is required")
}
c.lifecycleMu.Lock()
started := c.started
stopped := c.stopped
done := c.done
c.lifecycleMu.Unlock()
if !started && !stopped {
return nil
}
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (c *RecoveryCoordinator) run(ctx context.Context) {
defer close(c.done)
defer c.markReady()
ticker := time.NewTicker(c.sweepInterval)
defer ticker.Stop()
external := c.external
externalReadyRequired := external != nil
initialPending := true
// A nil event source is useful for backends whose connection is already
// known-ready. A managed source emits an initial-status signal after its
// first connection attempt. Periodic or internal wakes may still make safe
// read-only attempts while that signal is pending, but only a successful
// external pass may open the admission gate. A failed index read is not a
// completed pass and cannot open it in either mode.
if !externalReadyRequired && c.reconcileAll(ctx) == nil {
c.markReady()
initialPending = false
}
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
passErr := c.reconcileAll(ctx)
if initialPending && !externalReadyRequired && passErr == nil {
c.markReady()
initialPending = false
}
case <-c.wake:
passErr := c.reconcileAll(ctx)
if initialPending && !externalReadyRequired && passErr == nil {
c.markReady()
initialPending = false
}
case _, ok := <-external:
if !ok {
external = nil
continue
}
passErr := c.reconcileAll(ctx)
if initialPending && passErr == nil {
c.markReady()
initialPending = false
}
}
}
}
func (c *RecoveryCoordinator) markReady() {
c.readyOnce.Do(func() { close(c.ready) })
}
func (c *RecoveryCoordinator) reconcileAll(ctx context.Context) error {
if ctx.Err() != nil {
return ctx.Err()
}
records, err := c.journal.ListRecoveryCandidates()
if err != nil {
// A corrupt indexed candidate is fail-closed by the Journal. Leave the
// index untouched for an operator to repair; a later signal retries it.
c.observe(Record{}, err)
return err
}
for _, record := range records {
if ctx.Err() != nil {
return ctx.Err()
}
itemCtx, cancel := context.WithTimeout(ctx, c.itemTimeout)
result, reconcileErr := c.journal.Reconcile(itemCtx, record.OperationID, c.reconcile)
cancel()
c.observe(result, reconcileErr)
}
return nil
}
func (c *RecoveryCoordinator) observe(record Record, err error) {
if c.observer != nil {
c.observer(cloneRecord(record), err)
}
}

@ -0,0 +1,513 @@
package operation
import (
"context"
"encoding/json"
"errors"
"fmt"
"path/filepath"
"reflect"
"sort"
"strings"
"sync/atomic"
"testing"
"time"
bolt "go.etcd.io/bbolt"
)
func recoveryTestRecord(id, resource, action string, state State) Record {
now := time.Unix(1_800_000_000, 0).UTC()
return Record{
OperationID: id,
ResourceID: resource,
Action: action,
Fingerprint: strings.Repeat("a", 64),
Destructive: true,
State: state,
CreatedAt: now,
UpdatedAt: now,
}
}
func recoveryCandidateIDs(t *testing.T, journal *Journal) []string {
t.Helper()
records, err := journal.ListRecoveryCandidates()
if err != nil {
t.Fatal(err)
}
ids := make([]string, 0, len(records))
for _, record := range records {
ids = append(ids, record.OperationID)
}
sort.Strings(ids)
return ids
}
func TestSchemaV2MigrationBuildsRecoveryIndexWithoutDecodingCorruptHistoryOnLaterStarts(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
journal, err := Open(path)
if err != nil {
t.Fatal(err)
}
if err := journal.Close(); err != nil {
t.Fatal(err)
}
database, err := bolt.Open(path, 0o600, nil)
if err != nil {
t.Fatal(err)
}
err = database.Update(func(tx *bolt.Tx) error {
if err := tx.Bucket(metadataBucket).Put(schemaVersionKey, previousSchemaVersion); err != nil {
return err
}
if err := tx.DeleteBucket(recoveryBucket); err != nil {
return err
}
operations := tx.Bucket(operationsBucket)
inflight := tx.Bucket(inflightBucket)
records := []Record{
recoveryTestRecord("operation-v2-uncertain", "vm-v2-uncertain", "delete-vm", StateUncertain),
recoveryTestRecord("operation-v2-dispatch", "vm-v2-dispatch", "delete-vm", StateDispatching),
recoveryTestRecord("operation-v2-reserved", "vm-v2-reserved", "delete-vm", StateReserved),
recoveryTestRecord("operation-v2-complete", "vm-v2-complete", "delete-vm", StateSucceeded),
}
failedCreate := recoveryTestRecord("operation-v2-create", "vm-v2-create", "create-vm", StateFailed)
failedCreate.Error = &PublicError{Code: "backend_error", Message: "creation result is not proven"}
records = append(records, failedCreate)
pendingCleanup := recoveryTestRecord("operation-v2-cleanup", "vm-v2-cleanup", "reinstall-vm", StateSucceeded)
pendingCleanup.Result = json.RawMessage(`{"reinstall_committed":true,"cleanup_pending":true}`)
records = append(records, pendingCleanup)
for _, record := range records {
if err := putRecord(operations, record); err != nil {
return err
}
}
if err := inflight.Put([]byte("operation-v2-dispatch"), nil); err != nil {
return err
}
if err := inflight.Put([]byte("operation-v2-reserved"), nil); err != nil {
return err
}
return operations.Put([]byte("operation-corrupt-tombstone"), []byte("not-json"))
})
if err != nil {
t.Fatal(err)
}
if err := database.Close(); err != nil {
t.Fatal(err)
}
reopened, err := Open(path)
if err != nil {
t.Fatalf("migrate schema v2: %v", err)
}
want := []string{
"operation-v2-cleanup",
"operation-v2-create",
"operation-v2-dispatch",
"operation-v2-uncertain",
}
if got := recoveryCandidateIDs(t, reopened); !reflect.DeepEqual(got, want) {
t.Fatalf("recovery candidates = %v, want %v", got, want)
}
dispatched, err := reopened.Get("operation-v2-dispatch")
if err != nil || dispatched.State != StateUncertain {
t.Fatalf("recovered dispatch = %#v err=%v", dispatched, err)
}
reserved, err := reopened.Get("operation-v2-reserved")
if err != nil || reserved.State != StateFailed || reserved.Error == nil || reserved.Error.Code != "operation_not_dispatched" {
t.Fatalf("recovered reservation = %#v err=%v", reserved, err)
}
if err := reopened.db.View(func(tx *bolt.Tx) error {
if got := tx.Bucket(metadataBucket).Get(schemaVersionKey); string(got) != string(schemaVersion) {
return fmt.Errorf("schema = %q", got)
}
if tx.Bucket(recoveryBucket) == nil {
return errors.New("recovery bucket was not created")
}
return nil
}); err != nil {
t.Fatal(err)
}
if err := reopened.Close(); err != nil {
t.Fatal(err)
}
// Schema v3 restarts read only the two active indexes. The unrelated corrupt
// permanent history row remains unusable without blocking startup.
reopened, err = Open(path)
if err != nil {
t.Fatalf("reopen schema v3: %v", err)
}
defer reopened.Close()
if got := recoveryCandidateIDs(t, reopened); !reflect.DeepEqual(got, want) {
t.Fatalf("reopened recovery candidates = %v, want %v", got, want)
}
if _, err := reopened.Get("operation-corrupt-tombstone"); err == nil {
t.Fatal("corrupt permanent record unexpectedly decoded")
}
}
func TestRecoveryIndexExcludesLiveDispatchAndTracksUncertainResolution(t *testing.T) {
journal := testJournal(t)
spec := testSpec("operation-index-live", "vm-index-live", `{}`)
if _, _, err := journal.Reserve(spec); err != nil {
t.Fatal(err)
}
if _, err := journal.markDispatching(spec.OperationID); err != nil {
t.Fatal(err)
}
if got := recoveryCandidateIDs(t, journal); len(got) != 0 {
t.Fatalf("live dispatch entered recovery index: %v", got)
}
if _, err := journal.markUncertain(spec.OperationID, errors.New("connection lost")); err != nil {
t.Fatal(err)
}
if got := recoveryCandidateIDs(t, journal); !reflect.DeepEqual(got, []string{spec.OperationID}) {
t.Fatalf("uncertain recovery candidates = %v", got)
}
reconciled, err := journal.Reconcile(context.Background(), spec.OperationID,
func(context.Context, Record) (Resolution, error) {
return Resolution{Resolved: true, Succeeded: true, Result: json.RawMessage(`{"absent":true}`)}, nil
})
if err != nil || reconciled.State != StateSucceeded {
t.Fatalf("reconcile = %#v err=%v", reconciled, err)
}
if got := recoveryCandidateIDs(t, journal); len(got) != 0 {
t.Fatalf("resolved operation retained in recovery index: %v", got)
}
}
func TestPendingCleanupEvidenceIsNotCompactedBeforeRecovery(t *testing.T) {
journal := testJournal(t)
old := time.Unix(1_500_000_000, 0).UTC()
journal.SetClockForTesting(func() time.Time { return old })
spec := Spec{
OperationID: "operation-retained-cleanup", ResourceID: "123e4567-e89b-42d3-a456-426614174099",
Action: "reinstall-vm", Fingerprint: strings.Repeat("b", 64), Destructive: true,
HeavyIO: true,
}
record, _, err := journal.Execute(context.Background(), spec,
func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"reinstall_committed":true,"cleanup_pending":true}`), nil
})
if err != nil || !succeededCleanupCanReconcile(record) {
t.Fatalf("pending cleanup record = %#v err=%v", record, err)
}
late := old.Add(operationDetailRetention + time.Hour)
journal.SetClockForTesting(func() time.Time { return late })
if accepted, err := journal.UseNonce("retainCleanupEvidence1", late, late.Add(time.Minute)); err != nil || !accepted {
t.Fatalf("trigger retention accepted=%v err=%v", accepted, err)
}
retained, err := journal.Get(spec.OperationID)
if err != nil || retained.Tombstone || !succeededCleanupCanReconcile(retained) ||
len(retained.Result) == 0 || len(retained.LockKeys) == 0 || !retained.HeavyIO {
t.Fatalf("unfinished cleanup evidence was compacted: %#v err=%v", retained, err)
}
if got := recoveryCandidateIDs(t, journal); !reflect.DeepEqual(got, []string{spec.OperationID}) {
t.Fatalf("retained cleanup candidates = %v", got)
}
resolved, err := journal.Reconcile(context.Background(), spec.OperationID,
func(context.Context, Record) (Resolution, error) {
return Resolution{Resolved: true, Succeeded: true,
Result: json.RawMessage(`{"reinstall_committed":true,"cleanup_pending":false}`)}, nil
})
if err != nil || resolved.State != StateSucceeded || succeededCleanupCanReconcile(resolved) {
t.Fatalf("resolved cleanup = %#v err=%v", resolved, err)
}
if got := recoveryCandidateIDs(t, journal); len(got) != 0 {
t.Fatalf("resolved cleanup retained in recovery index: %v", got)
}
}
func makeRecoveryCandidate(t *testing.T, journal *Journal, id, resource string) {
t.Helper()
_, _, err := journal.Execute(context.Background(), testSpec(id, resource, `{}`),
func(context.Context) (json.RawMessage, error) {
return nil, MarkUncertain(errors.New("lost result"))
})
if err == nil || !IsUncertain(err) {
t.Fatalf("create recovery candidate err=%v", err)
}
}
func waitForRecoveryRecord(t *testing.T, journal *Journal, operationID string,
want State, timeout time.Duration) Record {
t.Helper()
deadline := time.Now().Add(timeout)
for {
record, err := journal.Get(operationID)
if err == nil && record.State == want {
return record
}
if time.Now().After(deadline) {
t.Fatalf("operation %s did not reach %s: record=%#v err=%v", operationID, want, record, err)
}
time.Sleep(time.Millisecond)
}
}
func TestRecoveryCoordinatorScansAtStartupAndCoalescesEventWakeups(t *testing.T) {
journal := testJournal(t)
makeRecoveryCandidate(t, journal, "operation-recovery-event", "vm-recovery-event")
external := make(chan struct{}, 1)
first := make(chan struct{})
var calls atomic.Int32
var active atomic.Int32
var maximum atomic.Int32
var resolve atomic.Bool
reconcile := func(_ context.Context, record Record) (Resolution, error) {
if record.OperationID != "operation-recovery-event" || record.Action != "delete-vm" {
t.Errorf("unexpected recovery record: %#v", record)
}
current := active.Add(1)
defer active.Add(-1)
for {
previous := maximum.Load()
if current <= previous || maximum.CompareAndSwap(previous, current) {
break
}
}
if calls.Add(1) == 1 {
close(first)
}
if !resolve.Load() {
return Resolution{}, nil
}
return Resolution{Resolved: true, Succeeded: true, Result: json.RawMessage(`{"absent":true}`)}, nil
}
coordinator, err := newRecoveryCoordinator(journal, reconcile, external, time.Hour, time.Second)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := coordinator.Start(ctx); err != nil {
t.Fatal(err)
}
select {
case <-coordinator.Ready():
t.Fatal("coordinator became ready before the initial connection signal")
default:
}
coordinator.Wake()
select {
case <-first:
case <-time.After(time.Second):
t.Fatal("pre-connection recovery attempt did not run")
}
select {
case <-coordinator.Ready():
t.Fatal("internal wake opened readiness before the initial connection signal")
default:
}
external <- struct{}{}
select {
case <-coordinator.Ready():
case <-time.After(time.Second):
t.Fatal("external initial-status pass did not open readiness")
}
record, err := journal.Get("operation-recovery-event")
if err != nil || record.State != StateUncertain {
t.Fatalf("unresolved initial pass did not retain candidate: %#v err=%v", record, err)
}
resolve.Store(true)
for index := 0; index < 100; index++ {
coordinator.Wake()
select {
case external <- struct{}{}:
default:
}
}
waitForRecoveryRecord(t, journal, "operation-recovery-event", StateSucceeded, time.Second)
coordinator.Stop()
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
defer waitCancel()
if err := coordinator.Wait(waitCtx); err != nil {
t.Fatal(err)
}
if maximum.Load() != 1 {
t.Fatalf("recovery callbacks ran concurrently: max=%d", maximum.Load())
}
}
func TestRecoveryCoordinatorReadyFailsClosedOnUnreadableIndex(t *testing.T) {
journal := testJournal(t)
if err := journal.db.Update(func(tx *bolt.Tx) error {
return tx.Bucket(recoveryBucket).Put([]byte("operation-missing-from-history"), nil)
}); err != nil {
t.Fatal(err)
}
var reconcilerCalled atomic.Bool
coordinator, err := newRecoveryCoordinator(journal,
func(context.Context, Record) (Resolution, error) {
reconcilerCalled.Store(true)
return Resolution{}, nil
}, nil, time.Hour, time.Second)
if err != nil {
t.Fatal(err)
}
observed := make(chan error, 1)
if err := coordinator.SetObserver(func(record Record, err error) {
if record.OperationID == "" && err != nil {
select {
case observed <- err:
default:
}
}
}); err != nil {
t.Fatal(err)
}
if err := coordinator.Start(context.Background()); err != nil {
t.Fatal(err)
}
select {
case err := <-observed:
if !strings.Contains(err.Error(), "missing operation") {
t.Fatalf("unexpected index error: %v", err)
}
case <-time.After(time.Second):
t.Fatal("recovery index failure was not observed")
}
if reconcilerCalled.Load() {
t.Fatal("reconciler ran despite corrupt recovery index")
}
select {
case <-coordinator.Ready():
t.Fatal("unreadable recovery index opened readiness")
default:
}
if err := journal.db.Update(func(tx *bolt.Tx) error {
return tx.Bucket(recoveryBucket).Delete([]byte("operation-missing-from-history"))
}); err != nil {
t.Fatal(err)
}
coordinator.Wake()
select {
case <-coordinator.Ready():
case <-time.After(time.Second):
t.Fatal("repaired recovery index did not open readiness")
}
coordinator.Stop()
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
defer waitCancel()
if err := coordinator.Wait(waitCtx); err != nil {
t.Fatal(err)
}
}
func TestRecoveryCoordinatorStopBeforeStartClosesLifecycleChannels(t *testing.T) {
journal := testJournal(t)
coordinator, err := newRecoveryCoordinator(journal,
func(context.Context, Record) (Resolution, error) { return Resolution{}, nil },
nil, time.Hour, time.Second)
if err != nil {
t.Fatal(err)
}
coordinator.Stop()
select {
case <-coordinator.Ready():
default:
t.Fatal("Stop before Start left Ready open")
}
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
defer waitCancel()
if err := coordinator.Wait(waitCtx); err != nil {
t.Fatal(err)
}
if err := coordinator.Start(context.Background()); err == nil {
t.Fatal("stopped coordinator unexpectedly restarted")
}
}
func TestRecoveryCoordinatorPeriodicSweepAndPerItemTimeout(t *testing.T) {
journal := testJournal(t)
makeRecoveryCandidate(t, journal, "operation-a-timeout", "vm-a-timeout")
makeRecoveryCandidate(t, journal, "operation-b-resolved", "vm-b-resolved")
var timedOut atomic.Int32
reconcile := func(ctx context.Context, record Record) (Resolution, error) {
if record.OperationID == "operation-a-timeout" {
<-ctx.Done()
timedOut.Add(1)
return Resolution{}, ctx.Err()
}
return Resolution{Resolved: true, Succeeded: true, Result: json.RawMessage(`{"absent":true}`)}, nil
}
coordinator, err := newRecoveryCoordinator(journal, reconcile, nil, 15*time.Millisecond, 10*time.Millisecond)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := coordinator.Start(ctx); err != nil {
t.Fatal(err)
}
waitForRecoveryRecord(t, journal, "operation-b-resolved", StateSucceeded, time.Second)
if timedOut.Load() == 0 {
t.Fatal("per-item timeout did not cancel the first recovery attempt")
}
firstCount := timedOut.Load()
deadline := time.Now().Add(time.Second)
for timedOut.Load() <= firstCount && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
if timedOut.Load() <= firstCount {
t.Fatal("periodic safety sweep did not retry the unresolved candidate")
}
coordinator.Stop()
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
defer waitCancel()
if err := coordinator.Wait(waitCtx); err != nil {
t.Fatal(err)
}
}
func TestRecoveryCoordinatorWaitDoesNotOutrunUncooperativeReconcile(t *testing.T) {
journal := testJournal(t)
makeRecoveryCandidate(t, journal, "operation-recovery-shutdown", "vm-recovery-shutdown")
started := make(chan struct{})
release := make(chan struct{})
coordinator, err := newRecoveryCoordinator(journal,
func(context.Context, Record) (Resolution, error) {
close(started)
<-release
return Resolution{}, nil
}, nil, time.Hour, 10*time.Millisecond)
if err != nil {
t.Fatal(err)
}
if err := coordinator.Start(context.Background()); err != nil {
t.Fatal(err)
}
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("reconcile did not start")
}
select {
case <-coordinator.Ready():
t.Fatal("coordinator became ready before the first scan completed")
default:
}
coordinator.Stop()
short, shortCancel := context.WithTimeout(context.Background(), 15*time.Millisecond)
defer shortCancel()
if err := coordinator.Wait(short); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Wait returned before native-equivalent call completed: %v", err)
}
close(release)
select {
case <-coordinator.Ready():
case <-time.After(time.Second):
t.Fatal("coordinator did not become ready after the first scan completed")
}
long, longCancel := context.WithTimeout(context.Background(), time.Second)
defer longCancel()
if err := coordinator.Wait(long); err != nil {
t.Fatal(err)
}
}

@ -148,10 +148,14 @@ Agent 把原始 `name` 保存为 Virt Cloud 的 `display-name` 域 metadata,
创建快照请求的 `consistency_mode=filesystem` 表示“优先文件系统一致”:Agent 先确认本地 libvirt 客户端和守护进程均支持 5.10.0 引入的 Guest Agent 响应上限;版本较旧时直接改用 `crash`,版本查询失败则返回基础设施错误。确认支持后,Agent 用只读 `guest-ping` 检查 QEMU Guest Agent,再在冻结前通过 libvirt 把冻结和解冻命令的响应上限设置为 `guest_agent_timeout_seconds`(默认 15 秒)。当 libvirt 明确报告 Guest Agent 未配置、未响应、探测超时,或驱动不支持设置响应上限时,也自动改用 `crash`。显式请求 `crash` 时不探测也不冻结 Guest。若 QGA 探测和超时设置均成功,但后续冻结、解冻或 libvirt 通信失败,不得伪装成安全降级;结果不明确时仍必须保留 `uncertain`。该响应上限只约束 Guest Agent 命令,不是后续 push backup 的超时。成功响应的 `consistency` 以及宿主机和面板快照元数据必须记录实际采用的模式,协调时只允许 `filesystem` 请求单向降级为 `crash`,绝不允许反向提升。
在线快照继续使用 libvirt push backup:`BackupBegin` 启动后以 `GetJobStats` 等待终态。请求超时时,只有当前活动任务仍明确标识为该 backup 时才调用一次 `AbortJob`,随后继续等待最多 30 秒,直到观察到 `COMPLETED`、`FAILED` 或 `CANCELLED`;无法确认终态时保持 `uncertain`。已经终态、任务类型不明或出现其他 libvirt job 时不得调用 `AbortJob`。快照恢复、快照删除、创建、重装和删除虚机均不使用 `AbortJob`。
在线快照继续使用 libvirt push backup:`BackupBegin` 启动后以 `GetJobStats` 等待终态。`JOB_COMPLETED` 事件只会唤醒一次新的 `GetJobStats` 核对,事件本身不构成成功或失败证据;固定间隔轮询仍作为事件丢失时的兜底。请求超时时,只有当前活动任务仍明确标识为该 backup 时才调用一次 `AbortJob`,随后继续等待最多 30 秒,直到观察到 `COMPLETED`、`FAILED` 或 `CANCELLED`;无法确认终态时保持 `uncertain`。已经终态、任务类型不明或出现其他 libvirt job 时不得调用 `AbortJob`。快照恢复、快照删除、创建、重装和删除虚机均不使用 `AbortJob`。
存储上传和下载使用 `VIR_STREAM_NONBLOCK` 与 libvirt event loop。请求 context 取消、读写失败或校验失败时必须先移除事件回调并 `StreamAbort`;只有完整成功路径才 `StreamFinish`。创建、重装、删除等没有通用 libvirt 取消 API 的操作仍依赖晚到结果与状态核对,不能把 HTTP 超时当作底层操作已中止。
Agent 在打开 `qemu:///system` 前注册并持续运行 libvirt 默认 event loop。每个连接 generation 都注册全域 lifecycle、`JOB_COMPLETED` 和 connection close callback;stream readable/writable 使用同一个 event loop。回调只发送有界、合并的核对唤醒,不保存回调中的临时 domain/connection 句柄,也不在事件线程中调用 libvirt 做核对或重连。
连接异常关闭后,Agent 在后台按有界退避建立新 generation 并重新注册全部回调;新连接完整就绪并发布后,才允许新请求使用并扫描 Journal 的全部可核对记录。旧 generation 会等其在途调用返回后再释放。Agent 启动执行同一扫描,并以低频周期扫描作为事件丢失的兜底。事件、completed job stats 和连接内存状态都不替代 Journal;Agent 或 libvirt daemon 重启导致事件缺失时,仍按原 `operation_id` 与宿主机状态核对,绝不重放原变更。
本表只描述已注册的 HTTP 路由;对应 libvirt 实现未落地或本机依赖不支持时,`features.*` 必须为 false 并返回 501,不得假成功。公网池的销售库存仍只在 PHP;Agent 仅在 VM 创建/换网时验证签名的 pool key、CIDR、IP、gateway、DNS 和面板指定的网络接入。网络值只接受 `bridge:<名称>` 或 `network:<名称>`;名称以字母或数字开头,后续仅允许字母、数字、点、下划线和短横线,bridge 为 1~15 字符,network 为 1~64 字符。Agent 验证并使用宿主机已存在且活动的网络,不创建或修改系统网络。
## operation_id、指纹与串行
@ -164,7 +168,7 @@ SHA256(UPPERCASE_METHOD + "\n" + EXACT_TARGET + "\n" + EXACT_BODY_BYTES)
Agent 在调用 libvirt 前将 `operation_id`、VM UUID/资源 ID、动作、指纹、是否破坏性和状态原子持久化。状态为 `reserved`、`dispatching`、`succeeded`、`failed`、`uncertain`。nonce TTL 至少为签名时钟偏差的两倍,到期边界仍保留,避免未来时间戳签名仍有效时释放 nonce。
持久层是 bbolt 数据库;每次状态变化只更新对应记录,nonce 由独立的到期时间索引清理,不扫描或重写完整 operation 历史。独立 inflight 索引使 Agent 启动只解码待恢复操作,不遍历永久 tombstone。终态结果和 expected-state 详情保留 90 天后压缩,释放页面供后续事务复用;`operation_id`、请求指纹、资源、动作和终态永久保留为 tombstone,任何清理都不得让旧的破坏性标识重新可用。`update-settings`、`change-network`、网络 suspend/resume、服务 suspend/resume 与 `apply-firewall` 在同一派发事务中额外保存无密钥 expected-state(只含 transition id、公开网络意图、解析后的本机网络 source、布尔状态或防火墙摘要;绝不保存密码、SSH key、token 或原始正文)。设置/网络/防火墙写入相同的宿主机 transition marker;重启后的 reconcile 只读比较 marker 与实际状态,换网还必须匹配持久化的 bridge/libvirt-network source,完整匹配才成功,明确未提交或半提交则失败并要求前向处置,绝不自动重放。
持久层是 bbolt 数据库;每次状态变化只更新对应记录,nonce 由独立的到期时间索引清理,不扫描或重写完整 operation 历史。inflight 索引只负责把进程中断时的 `reserved`/`dispatching` 恢复为 `not_dispatched`/`uncertain`;独立 recovery 索引只列出可核对记录,因此启动、重连和事件唤醒都不遍历永久 tombstone。已完成且不再需要核对的结果和 expected-state 详情保留 90 天后压缩,释放页面供后续事务复用;仍可核对的记录必须保留 expected-state、结果和资源租约元数据,直到取得终态。`operation_id`、请求指纹、资源、动作和终态永久保留为 tombstone,任何清理都不得让旧的破坏性标识重新可用。`update-settings`、`change-network`、网络 suspend/resume、服务 suspend/resume 与 `apply-firewall` 在同一派发事务中额外保存无密钥 expected-state(只含 transition id、公开网络意图、解析后的本机网络 source、布尔状态或防火墙摘要;绝不保存密码、SSH key、token 或原始正文)。设置/网络/防火墙写入相同的宿主机 transition marker;重启后的 reconcile 只读比较 marker 与实际状态,换网还必须匹配持久化的 bridge/libvirt-network source,完整匹配才成功,明确未提交或半提交则失败并要求前向处置,绝不自动重放。
- 首次请求保留标识并写盘;同标识同指纹返回已有或进行中的记录。
- 同标识不同指纹返回 409,绝不执行第二个请求。

@ -127,7 +127,7 @@ Agent 的 capabilities 响应仍使用 `network_selectors` 字段,但它是自
Agent 的 systemd 单元保持 `ProtectSystem=strict`。`journal_path` 必须位于已授权的 `/var/lib/virt-agent`;若确需另一个状态目录,应由运维通过 systemd drop-in 精确增加对应 `ReadWritePaths`。存储池内容仍只由独立的 libvirt/QEMU 服务通过 `qemu:///system` 管理,不能把受管存储池目录授权给 Agent。删除回查需要清理域定义消失后残留的 UUID 专属 VARS 文件,因此样例 unit 仅额外授权标准 `/var/lib/libvirt/qemu/nvram`;若 `nvram_directory` 使用其他路径,必须在 drop-in 中用该实际目录替换,并保持授权范围只覆盖 NVRAM 目录。capabilities 只确认该目录存在,不冒充对 systemd mount namespace 写权限的验证;正式上线仍须用授权的创建/删除验收确认权限。
`journal_path` 是 bbolt 数据库,不是可编辑 JSON;同目录的 `.traffic.json` 是流量核对状态,两者都必须纳入一致性备份和恢复。存储映射由 libvirt 的持久 pool 定义保存,没有 `.storage.json`。nonce 通过独立到期索引清理,不会重写 operation 历史;到期时刻本身仍视为占用,只清理严格早于当前时刻的 nonce。独立 inflight 索引使启动只恢复 `reserved`/`dispatching`,不会扫描永久历史。终态结果详情保留 90 天后压缩,释放的 bbolt 页面会被后续写入复用,但 operation_id、请求指纹、资源、动作和终态会永久保留为防重放 tombstone。禁止以“清理历史”为由删除这些 tombstone。当前为全新 Go Agent 格式,不导入旧 Python 数据或早期 JSON 原型 journal。
`journal_path` 是 bbolt 数据库,不是可编辑 JSON;同目录的 `.traffic.json` 是流量核对状态,两者都必须纳入一致性备份和恢复。存储映射由 libvirt 的持久 pool 定义保存,没有 `.storage.json`。nonce 通过独立到期索引清理,不会重写 operation 历史;到期时刻本身仍视为占用,只清理严格早于当前时刻的 nonce。inflight 索引只恢复进程中断时的 `reserved`/`dispatching`,recovery 索引保存所有仍可核对的 operation;正常启动、libvirt 重连和事件唤醒只读取这些索引,不扫描永久历史。从 journal schema v2 首次升级到 v3 时会在一个 bbolt 事务内扫描现有 operation 一次以回填 recovery 索引,完成后不再重复全历史扫描。已完成且不再需要核对的结果详情保留 90 天后压缩;仍待核对的记录保留 expected-state、结果和资源租约元数据,直到取得终态。operation_id、请求指纹、资源、动作和终态会永久保留为防重放 tombstone,禁止以“清理历史”为由删除。当前为全新 Go Agent 格式,不导入旧 Python 数据或早期 JSON 原型 journal。
配置必须存在、可读、是单个严格 JSON 对象并通过内容校验。Agent **不会**检查文件 owner、group 或 mode,也不会因 `0600`、`0640`、`0644` 等可读权限差异拒绝启动;文档中的收紧权限只是运维建议。操作系统本身无权读取、JSON 损坏、字段未知或配置值无效仍会明确启动失败;这些是可用性/内容错误,不是权限位审计。旧的 `listen_address`、`shared_secret`、`tls_cert_file` 和 `tls_key_file` 会作为未知字段被拒绝;Agent 不启动入站监听。
@ -322,6 +322,7 @@ location = /agent/console {
- Q35 UEFI 启动且 Secure Boot 关闭;不同宿主机固件路径通过 capabilities 选择。
- 云镜像启动脚本只执行一次,并覆盖成功、非零退出、重启及 Agent 断线。
- 同一 `operation_id` 重试不重复创建/删除;派发后丢失响应进入 `uncertain` 并可核对。
- 在明确授权的非生产宿主机重启实际启用的 `virtqemud` 或 `libvirtd`,确认 Agent 检测连接关闭、后台重连并重新注册 lifecycle、`JOB_COMPLETED` 和 close callback;健康检查恢复后只按原 `operation_id` 扫描并核对,不重放变更。再重启 Agent,确认同一批未完成 operation 会从 Journal recovery 索引重新扫描。
- 并发购买不超卖 CPU/内存/磁盘/IP;同一 VM 变更串行。
- RX+TX 额度用尽只断网;下一账期恢复;最终到期只断网;三天宽限结束后删除定义与面板受管磁盘。
- MariaDB、Redis、SMTP、支付回调、大文件上传、在线更新恢复点和 PHP-FPM 重启。

@ -70,6 +70,8 @@ Unix 时间戳
4. Agent 明确失败且确认没有副作用时,PHP 才能执行相应前向补偿。发送后断线、超时或进程中断一律进入 `uncertain`。
5. `uncertain` 只能结合 Agent 操作记录和按 UUID 观察到的宿主机状态对账。运行中状态不证明先前关机/重启未执行;破坏性操作不得自动重放或直接退款。
Agent 在连接前注册并持续运行 libvirt event loop;每个连接 generation 注册 lifecycle、`JOB_COMPLETED` 和 close callback。事件只合并唤醒核对,不能作为操作结果证据。Agent 启动或 libvirt 重连成功后从 Journal recovery 索引扫描未完成 operation;即使事件和 completed job stats 因 Agent/libvirt daemon 重启丢失,也只按原 `operation_id` 和宿主机现状核对,不重新提交原请求。
取消尚未派发的 PHP 任务只会阻止后续领取。已经跨过 Agent 派发边界的任务不能通过删除面板记录来假定撤销。
## 虚拟机、镜像与启动脚本
@ -104,4 +106,4 @@ helper 只允许干净工作树 fast-forward,按持久检查点停止服务、
## 上线边界
生产前必须实测 Agent WSS/HMAC、防重放、断线后对账、宿主机重启、UEFI、cloud-init、快照/救援/重装、反向控制台撤销、流量跨 QEMU 重启、到期断网、宽限删除、HTTPS 镜像安全和并发购买。测试不得对未明确授权的宿主机做创建、写配置、改网络或修改虚拟机操作。
生产前必须实测 Agent WSS/HMAC、防重放、断线后对账、Agent 与实际 libvirt daemon 分别重启后的连接恢复和 Journal 扫描、宿主机重启、UEFI、cloud-init、快照/救援/重装、反向控制台撤销、流量跨 QEMU 重启、到期断网、宽限删除、HTTPS 镜像安全和并发购买。测试不得对未明确授权的宿主机做创建、写配置、改网络或修改虚拟机操作。

@ -24,7 +24,7 @@ go test ./...
go test -race ./...
```
Go 测试至少覆盖:配置内容校验但不检查 owner/mode、HTTPS `panel_url`、数字 Agent ID、旧 listener/TLS 字段被拒绝、WSS 子协议与认证、严格控制帧、HMAC 正文摘要、完整时间窗(含未来偏差与到期边界)、nonce 防重放、不同请求复用 `operation_id`、多资源键严格串行、不相交资源并行、重 I/O 容量限制、派发前锁等待超时、派发后超时仍保留租约、派发前/后故障注入、仅按 inflight 索引重启恢复、超过 16 MiB 的长期 journal 重启、nonce 到期索引清理、operation tombstone 保留、五类 expected-state/transition marker 只读核对、换网 source 反例、libvirt typed error、UEFI XML、启动脚本一次性 runner 和反向控制台。不带 tag 的 portable 测试只能验证缺失 adapter 时的明确失败;生产交付必须通过 `-tags libvirt` 的编译和测试。PHP 契约测试还必须证明 PHP→Gateway 只使用字面 loopback 固定路径,不再存储或请求宿主机 endpoint。
Go 测试至少覆盖:配置内容校验但不检查 owner/mode、HTTPS `panel_url`、数字 Agent ID、旧 listener/TLS 字段被拒绝、WSS 子协议与认证、严格控制帧、HMAC 正文摘要、完整时间窗(含未来偏差与到期边界)、nonce 防重放、不同请求复用 `operation_id`、多资源键严格串行、不相交资源并行、重 I/O 容量限制、派发前锁等待超时、派发后超时仍保留租约、派发前/后故障注入、inflight 与 recovery 索引重启恢复、长期待核对记录不被压缩、超过 16 MiB 的永久 journal 重启、nonce 到期索引清理、operation tombstone 保留、五类 expected-state/transition marker 只读核对、换网 source 反例、libvirt typed error、UEFI XML、启动脚本一次性 runner 和反向控制台。libvirt 定向测试还必须覆盖 event loop 早于连接、每个 generation 注册与逆序释放 lifecycle/`JOB_COMPLETED`/close callback、回调非阻塞合并、注册期间断线、旧 generation 延迟回调、新旧 session 租约切换、退避重连与关停,以及启动/重连 recovery 扫描不会重放 mutation。`JOB_COMPLETED` 只能提前唤醒 `GetJobStats`,没有事件时仍须靠轮询确认终态。不带 tag 的 portable 测试只能验证缺失 adapter 时的明确失败;生产交付必须通过 `-tags libvirt` 的编译和测试。PHP 契约测试还必须证明 PHP→Gateway 只使用字面 loopback 固定路径,不再存储或请求宿主机 endpoint。
PHP:
@ -88,9 +88,10 @@ test ! -e gateway/cmd/virt-cloud-console-relay
1. Agent 收到请求前失败:PHP 可按同一标识安全重试。
2. Agent 持久写入派发边界后、调用 libvirt 前失败:恢复后仍使用原 `operation_id`,不得新建操作。
3. libvirt 调用已返回但响应丢失:任务进入 `uncertain`,通过 UUID 和本地操作日志核对。
4. Agent 或面板 worker 重启:操作日志、PHP 租约和 reservation 不丢失。
4. Agent 或面板 worker 重启:操作日志、PHP 租约和 reservation 不丢失;Agent 启动扫描 recovery 索引并核对所有未完成 operation。
5. 同一 `operation_id` + 相同摘要:返回既有结果;同一标识 + 不同摘要:409 拒绝。
6. 同一 VM 或共享卷、过滤器、目录资源的并发变更:严格串行;不相交资源可在容量上限内并行。锁或重 I/O 槽等待超时必须保持 `not_dispatched`,派发后 HTTP 超时必须继续持有全部租约直到真实调用返回。
7. 在授权的非生产环境分别中断并重启实际使用的 libvirt daemon 与 Agent:close callback 只唤醒后台重连,重连后重新注册全部事件并扫描 Journal;旧连接的迟到事件不能影响新连接,扫描不得重放原 mutation。
运行中状态不能证明关机/重启未发生;持久域消失不能在未核对磁盘清单时自动视为完整删除。任何未知删除都不得释放 IP、库存或退款。
@ -153,6 +154,9 @@ systemd-analyze:
read-only host capability check:
authorized host mutation test:
browser/payment/SMTP test:
event/reconnect unit tests:
authorized libvirt daemon restart test:
Agent restart recovery scan test:
known skips or risks:
```
@ -484,3 +488,17 @@ artifact inspection: stripped ELF 64-bit x86-64;revision=118a5957a854114521e9d
build path / key marker scan: 未发现本机用户路径、临时构建目录、私钥标记或项目 Git 远端;产物先在临时目录构建和验证,再更新 release
live deployment: 未执行;产物尚未在真实 Linux 宿主机运行,未替换线上 Agent 或重启服务
```
### 2026-10-18 libvirt 连接、事件与 Journal 恢复
```text
source: 本记录所在 go 分支源码提交,基于 a73c269;Go 1.27.1、libvirt 12.8.0,macOS arm64
connection generations: 默认 event loop 在首次连接前注册并持续运行;每个 qemu:///system 连接代注册 connection close、全域 lifecycle 和 JOB_COMPLETED;断线后后台退避重连并重新注册;新代完整发布后接收新调用,旧代等全部在途调用真实返回后再逆序注销 callback 和关闭
event semantics: callback 仅做代际校验和有界合并唤醒;JOB_COMPLETED 只促使快照等待器重新读取 GetJobStats,500ms 轮询保留;connection close 同时唤醒 stream/job waiter;事件、completed job stats 和临时连接状态均不作为操作结果证据
recovery: Journal schema v2 自动事务升级到 v3,新增 recovery 索引;启动、成功重连、lifecycle/JOB_COMPLETED/close 事件及 5 分钟周期兜底均扫描可核对 operation;每项默认 30 秒,串行取得原资源租约,只读核对或继续同 operation_id 的幂等 cleanup tail,绝不重放原 mutation;未解决记录继续保留索引及证据,不受 90 天详情压缩影响
dispatch result: 无可用 session 且 backend 明确未调用时返回确定的 libvirt_unavailable;真实调用后遇到 RPC、INVALID_CONN 或 NO_CONNECT 时 mutation 保持 uncertain 并触发重连
validation: Agent go mod verify、go mod tidy -diff、gofmt 和 git diff --check 通过;默认、libvirt、libvirt+libvirt_dlopen 三套 go test 通过;默认与 libvirt+libvirt_dlopen 全量 race 通过;默认与 libvirt+libvirt_dlopen vet 通过;operation/hypervisor 定向测试覆盖 v2→v3 迁移、恢复索引、关停、事件合并、每项超时、连接代切换、迟到 callback、注册清理顺序、JOB_COMPLETED 非证据语义和 AbortJob 终态确认;macOS 链接器仅报告重复 -ldl/-lvirt warning
upgrade: 这是 Agent 本地 journal 的自动升级,不需要修改面板数据库;首次打开 v2 journal 会扫描一次现有 operation 回填 recovery 索引,后续启动只读取 inflight/recovery 索引
known limits: 尚未在真实 Debian/libvirt 宿主机执行 virtqemud/libvirtd 重启、Agent 重启和 callback 重注册验收;libvirt-go v1.12007.0 的 stream callback free-hook 限制仍与第二阶段记录相同
live deployment: 未执行;未替换线上 Agent、重启服务、修改宿主机配置或操作真实 VM
```

Loading…
Cancel
Save