diff --git a/agent/cmd/virt-agent/main.go b/agent/cmd/virt-agent/main.go index 5361b4ed..e2706dc8 100644 --- a/agent/cmd/virt-agent/main.go +++ b/agent/cmd/virt-agent/main.go @@ -53,6 +53,29 @@ func main() { if err != nil { log.Fatalf("initialize Agent request dispatcher: %v", err) } + var recoveryEvents <-chan struct{} + if source, ok := hv.(hypervisor.RecoveryEventSource); ok { + recoveryEvents = source.RecoveryEvents() + } + recovery, err := operation.NewRecoveryCoordinator(journal, hv.Reconcile, recoveryEvents) + if err != nil { + log.Fatalf("initialize operation recovery: %v", err) + } + if err := recovery.SetObserver(func(record operation.Record, reconcileErr error) { + if reconcileErr == nil { + if record.OperationID != "" { + log.Printf("operation recovery resolved: operation_id=%s action=%s status=%s", record.OperationID, record.Action, record.State) + } + return + } + if record.OperationID == "" { + log.Printf("operation recovery index scan failed; retry is scheduled: %v", reconcileErr) + return + } + log.Printf("operation recovery remains pending: operation_id=%s action=%s status=%s", record.OperationID, record.Action, record.State) + }); err != nil { + log.Fatalf("configure operation recovery diagnostics: %v", err) + } client, err := control.NewClient( cfg.PanelURL, cfg.AgentID, []byte(cfg.AgentSecret), handler, console, cfg.MaxBodyBytes, cfg.OperationTimeout(), ) @@ -61,14 +84,30 @@ func main() { } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - log.Printf("virt host agent %s connecting to panel as Agent %s", api.Version, cfg.AgentID) - runErr := client.Run(ctx) + if err := recovery.Start(ctx); err != nil { + log.Fatalf("start operation recovery: %v", err) + } + select { + case <-recovery.Ready(): + case <-ctx.Done(): + } + var runErr error + if ctx.Err() == nil { + log.Printf("virt host agent %s connecting to panel as Agent %s", api.Version, cfg.AgentID) + runErr = client.Run(ctx) + } + stop() + recovery.Stop() shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if err := handler.Shutdown(shutdownCtx); err != nil { log.Printf("Agent handlers did not drain safely: %v", err) os.Exit(1) } + if err := recovery.Wait(shutdownCtx); err != nil { + log.Printf("Agent operation recovery did not drain safely: %v", err) + os.Exit(1) + } if err := journal.Wait(shutdownCtx); err != nil { log.Printf("Agent operation workers did not drain safely: %v", err) os.Exit(1) diff --git a/agent/hypervisor/libvirt_adapter.go b/agent/hypervisor/libvirt_adapter.go index 42d4ff94..1209f11d 100644 --- a/agent/hypervisor/libvirt_adapter.go +++ b/agent/hypervisor/libvirt_adapter.go @@ -96,27 +96,11 @@ func OpenLibvirt(uri string, local LocalConfig) (Hypervisor, error) { if err != nil { return nil, translateLibvirtError("get local libvirt version", err) } - conn, err := libvirt.NewConnect(uri) - if err != nil { - return nil, translateLibvirtError("connect", err) - } - daemonVersion, err := conn.GetLibVersion() - if err != nil { - _, _ = conn.Close() - return nil, translateLibvirtError("get daemon libvirt version", err) - } - guestAgentResponseTimeoutSupported := supportsGuestAgentResponseTimeout(clientVersion, daemonVersion) traffic, err := openTrafficStore(local.TrafficStatePath) if err != nil { - _, _ = conn.Close() return nil, NewError(ErrUnavailable, "traffic_state_unavailable", "durable traffic state could not be opened", err) } - return &Libvirt{ - conn: conn, - local: local, - traffic: traffic, - guestAgentResponseTimeoutSupported: guestAgentResponseTimeoutSupported, - }, nil + return openManagedLibvirt(uri, local, clientVersion, traffic), nil } func (l *Libvirt) Close() error { @@ -1518,7 +1502,7 @@ func translateLibvirtError(action string, err error) error { return NewError(ErrInvalid, "invalid_libvirt_request", action+" was rejected as invalid", err) case libvirt.ERR_OPERATION_UNSUPPORTED: return NewError(ErrUnsupported, "unsupported", action+" is not supported", err) - case libvirt.ERR_NO_CONNECT, libvirt.ERR_AUTH_FAILED: + case libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_AUTH_FAILED: return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", err) } } @@ -1529,7 +1513,7 @@ func mutationLibvirtError(action string, err error) error { var libvirtErr libvirt.Error if errors.As(err, &libvirtErr) { switch libvirtErr.Code { - case libvirt.ERR_SYSTEM_ERROR, libvirt.ERR_RPC, libvirt.ERR_NO_CONNECT, libvirt.ERR_OPERATION_TIMEOUT: + case libvirt.ERR_SYSTEM_ERROR, libvirt.ERR_RPC, libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_OPERATION_TIMEOUT: return operation.MarkUncertain(translateLibvirtError(action, err)) } } diff --git a/agent/hypervisor/libvirt_connection.go b/agent/hypervisor/libvirt_connection.go new file mode 100644 index 00000000..aeb48cfa --- /dev/null +++ b/agent/hypervisor/libvirt_connection.go @@ -0,0 +1,756 @@ +//go:build libvirt + +package hypervisor + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "git.luoli.live/root/virt/agent/operation" + "libvirt.org/go/libvirt" +) + +const ( + libvirtReconnectInitialDelay = time.Second + libvirtReconnectMaximumDelay = 30 * time.Second + libvirtCleanupErrorLimit = 16 +) + +// libvirtJobEvents wakes backup waiters when JOB_COMPLETED arrives. It is a +// hint, not a result: callers must still use GetJobStats to establish the +// terminal state. Closing a connection also advances the generation so a +// waiter can promptly observe the lost session. +var libvirtJobEvents = newLibvirtEventBroadcaster() + +type managedLibvirtSession struct { + generation uint64 + backend Hypervisor + closeFn func() error + + dead atomic.Bool + + // refs and retired are protected by managedLibvirt.mu. drained is closed + // exactly once after retirement when the last public call releases its + // immutable session reference. + refs int + retired bool + drained chan struct{} +} + +type libvirtConnectionCleanup struct { + closeRegistered bool + lifecycleID int + jobID int + deregister func(int) error + unregisterClose func() error + closeConnection func() error +} + +func (cleanup *libvirtConnectionCleanup) run() error { + var cleanupErrors []error + // Registrations are unwound in strict reverse order: JOB_COMPLETED, + // lifecycle, close callback, then the connection reference itself. + if cleanup.jobID >= 0 { + if err := cleanup.deregister(cleanup.jobID); err != nil { + cleanupErrors = append(cleanupErrors, fmt.Errorf("deregister completed job events: %w", err)) + } + } + if cleanup.lifecycleID >= 0 { + if err := cleanup.deregister(cleanup.lifecycleID); err != nil { + cleanupErrors = append(cleanupErrors, fmt.Errorf("deregister lifecycle events: %w", err)) + } + } + if cleanup.closeRegistered { + if err := cleanup.unregisterClose(); err != nil { + cleanupErrors = append(cleanupErrors, fmt.Errorf("unregister connection close callback: %w", err)) + } + } + if err := cleanup.closeConnection(); err != nil { + cleanupErrors = append(cleanupErrors, fmt.Errorf("close libvirt connection: %w", err)) + } + return errors.Join(cleanupErrors...) +} + +type managedLibvirtSessionFactory func(uint64) (*managedLibvirtSession, error) + +type managedLibvirt struct { + ctx context.Context + cancel context.CancelFunc + + mu sync.Mutex + current *managedLibvirtSession + nextGeneration uint64 + closing bool + lastConnectErr error + + currentGeneration atomic.Uint64 + reconnect chan struct{} + recovery chan struct{} + + factory managedLibvirtSessionFactory + retryDelay func(int) time.Duration + + workerWG sync.WaitGroup + retiredWG sync.WaitGroup + + cleanupMu sync.Mutex + cleanupErrs []error + startOnce sync.Once + closeOnce sync.Once + closeErr error +} + +func openManagedLibvirt(uri string, local LocalConfig, clientVersion uint32, traffic *trafficStore) *managedLibvirt { + var manager *managedLibvirt + manager = newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { + return manager.openSession(uri, local, clientVersion, traffic, generation) + }, defaultLibvirtReconnectDelay) + manager.start() + return manager +} + +func (m *managedLibvirt) start() { + // Preserve the useful startup behavior of attempting the local socket once, + // while allowing the Agent to start and report unavailable if the daemon is + // temporarily down. Recovery gets one initial wakeup in either case. + m.startOnce.Do(func() { + // Register the whole startup/reconnect lifecycle before the first + // potentially blocking connection attempt. Close uses the same mutex + // before Wait, which prevents an Add-after-Wait race. + m.mu.Lock() + if m.closing { + m.mu.Unlock() + return + } + m.workerWG.Add(1) + m.mu.Unlock() + + _ = m.connectAndPublish() + m.signalRecovery() + go m.reconnectLoop() + if !m.hasUsableSession() { + m.signalReconnect() + } + }) +} + +func newManagedLibvirt(factory managedLibvirtSessionFactory, retryDelay func(int) time.Duration) *managedLibvirt { + ctx, cancel := context.WithCancel(context.Background()) + if retryDelay == nil { + retryDelay = defaultLibvirtReconnectDelay + } + return &managedLibvirt{ + ctx: ctx, + cancel: cancel, + reconnect: make(chan struct{}, 1), + recovery: make(chan struct{}, 1), + factory: factory, + retryDelay: retryDelay, + } +} + +func defaultLibvirtReconnectDelay(failures int) time.Duration { + if failures <= 1 { + return libvirtReconnectInitialDelay + } + delay := libvirtReconnectInitialDelay + for attempt := 1; attempt < failures && delay < libvirtReconnectMaximumDelay; attempt++ { + delay *= 2 + if delay > libvirtReconnectMaximumDelay { + return libvirtReconnectMaximumDelay + } + } + return delay +} + +func (m *managedLibvirt) openSession(uri string, local LocalConfig, clientVersion uint32, traffic *trafficStore, generation uint64) (*managedLibvirtSession, error) { + conn, err := libvirt.NewConnect(uri) + if err != nil { + return nil, translateLibvirtError("connect", err) + } + session := &managedLibvirtSession{ + generation: generation, + drained: make(chan struct{}), + } + cleanup := &libvirtConnectionCleanup{ + lifecycleID: -1, + jobID: -1, + deregister: conn.DomainEventDeregister, + unregisterClose: func() error { + return conn.UnregisterCloseCallback() + }, + closeConnection: func() error { + _, err := conn.Close() + return err + }, + } + session.closeFn = cleanup.run + failSetup := func(action string, cause error) (*managedLibvirtSession, error) { + return nil, errors.Join(translateLibvirtError(action, cause), session.closeFn()) + } + + if err := conn.RegisterCloseCallback(func(_ *libvirt.Connect, _ libvirt.ConnectCloseReason) { + // The callback can run on libvirt's event thread. It only marks this + // immutable generation dead and performs non-blocking wakeups; reconnect, + // callback deregistration, and Close all happen on manager workers. + session.dead.Store(true) + if !m.isCurrentGeneration(session) { + return + } + libvirtJobEvents.notify() + storageStreamEvents.notify() + m.signalRecovery() + m.signalReconnect() + }); err != nil { + return failSetup("register connection close callback", err) + } + cleanup.closeRegistered = true + + cleanup.lifecycleID, err = conn.DomainEventLifecycleRegister(nil, func(_ *libvirt.Connect, _ *libvirt.Domain, _ *libvirt.DomainEventLifecycle) { + m.signalRecoveryFor(session) + }) + if err != nil { + return failSetup("register domain lifecycle events", err) + } + cleanup.jobID, err = conn.DomainEventJobCompletedRegister(nil, func(_ *libvirt.Connect, _ *libvirt.Domain, _ *libvirt.DomainEventJobCompleted) { + if !m.isCurrentGeneration(session) { + return + } + libvirtJobEvents.notify() + m.signalRecovery() + }) + if err != nil { + return failSetup("register completed job events", err) + } + + daemonVersion, err := conn.GetLibVersion() + if err != nil { + return failSetup("get daemon libvirt version", err) + } + session.backend = &Libvirt{ + conn: conn, + local: local, + traffic: traffic, + guestAgentResponseTimeoutSupported: supportsGuestAgentResponseTimeout(clientVersion, daemonVersion), + } + + alive, err := conn.IsAlive() + if err != nil || !alive || session.dead.Load() { + if err != nil { + return failSetup("verify libvirt connection", err) + } + return nil, errors.Join(NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", nil), session.closeFn()) + } + return session, nil +} + +func (m *managedLibvirt) reconnectLoop() { + defer m.workerWG.Done() + failures := 0 + for { + select { + case <-m.ctx.Done(): + return + case <-m.reconnect: + } + + for { + if m.ctx.Err() != nil { + return + } + m.detachDeadSession() + if m.hasUsableSession() { + failures = 0 + break + } + if err := m.connectAndPublish(); err == nil { + failures = 0 + break + } + failures++ + timer := time.NewTimer(m.retryDelay(failures)) + select { + case <-m.ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return + case <-m.reconnect: + if !timer.Stop() { + <-timer.C + } + case <-timer.C: + } + } + } +} + +func (m *managedLibvirt) connectAndPublish() error { + m.mu.Lock() + if m.closing { + m.mu.Unlock() + return context.Canceled + } + m.nextGeneration++ + generation := m.nextGeneration + m.mu.Unlock() + + session, err := m.factory(generation) + if err != nil { + m.mu.Lock() + m.lastConnectErr = err + m.mu.Unlock() + return err + } + if session == nil || session.backend == nil || session.closeFn == nil { + err = errors.New("libvirt session factory returned an incomplete session") + if session != nil && session.closeFn != nil { + m.cleanupSession(session) + } + m.mu.Lock() + m.lastConnectErr = err + m.mu.Unlock() + return err + } + if session.drained == nil { + session.drained = make(chan struct{}) + } + session.generation = generation + + m.mu.Lock() + if m.closing || m.current != nil || session.dead.Load() { + m.mu.Unlock() + m.cleanupSession(session) + if session.dead.Load() { + return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", nil) + } + return context.Canceled + } + m.current = session + m.currentGeneration.Store(session.generation) + if session.dead.Load() { + m.current = nil + m.currentGeneration.Store(0) + m.retireSessionLocked(session) + m.mu.Unlock() + return NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", nil) + } + m.lastConnectErr = nil + m.mu.Unlock() + m.signalRecovery() + return nil +} + +func (m *managedLibvirt) hasUsableSession() bool { + m.mu.Lock() + defer m.mu.Unlock() + return !m.closing && m.current != nil && !m.current.dead.Load() +} + +// detachDeadSession runs only outside libvirt callbacks. It atomically stops +// new acquisitions, then retires the old generation after existing calls have +// released their references. +func (m *managedLibvirt) detachDeadSession() bool { + m.mu.Lock() + defer m.mu.Unlock() + if m.current == nil || !m.current.dead.Load() { + return false + } + session := m.current + m.current = nil + m.currentGeneration.Store(0) + m.retireSessionLocked(session) + return true +} + +func (m *managedLibvirt) retireSessionLocked(session *managedLibvirtSession) { + if session == nil || session.retired { + return + } + session.retired = true + if session.refs == 0 { + close(session.drained) + } + m.retiredWG.Add(1) + go func() { + defer m.retiredWG.Done() + <-session.drained + m.cleanupSession(session) + }() +} + +func (m *managedLibvirt) cleanupSession(session *managedLibvirtSession) { + if session == nil || session.closeFn == nil { + return + } + if err := session.closeFn(); err != nil { + // Once libvirt has told us this generation is dead, INVALID_CONN, + // NO_CONNECT, and RPC errors are expected while unregistering callbacks + // and releasing the stale handle. They must not poison a later clean + // shutdown after the manager has successfully reconnected. + if session.dead.Load() && onlyLibvirtConnectionFailures(err) { + return + } + m.cleanupMu.Lock() + if len(m.cleanupErrs) < libvirtCleanupErrorLimit { + m.cleanupErrs = append(m.cleanupErrs, err) + } else { + m.cleanupErrs[libvirtCleanupErrorLimit-1] = err + } + m.cleanupMu.Unlock() + } +} + +func (m *managedLibvirt) acquire(ctx context.Context) (*managedLibvirtSession, func(), error) { + if err := ctx.Err(); err != nil { + return nil, nil, err + } + m.mu.Lock() + if m.closing || m.current == nil || m.current.dead.Load() { + cause := m.lastConnectErr + m.mu.Unlock() + return nil, nil, NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", cause) + } + session := m.current + session.refs++ + m.mu.Unlock() + + var once sync.Once + release := func() { + once.Do(func() { + m.mu.Lock() + session.refs-- + if session.retired && session.refs == 0 { + close(session.drained) + } + m.mu.Unlock() + }) + } + return session, release, nil +} + +func (m *managedLibvirt) isCurrentGeneration(session *managedLibvirtSession) bool { + return session != nil && m.currentGeneration.Load() == session.generation +} + +func (m *managedLibvirt) signalRecoveryFor(session *managedLibvirtSession) { + if m.isCurrentGeneration(session) { + m.signalRecovery() + } +} + +func (m *managedLibvirt) signalRecovery() { + select { + case m.recovery <- struct{}{}: + default: + } +} + +func (m *managedLibvirt) signalReconnect() { + select { + case m.reconnect <- struct{}{}: + default: + } +} + +func (m *managedLibvirt) markConnectionLost(session *managedLibvirtSession) { + if session == nil { + return + } + session.dead.Store(true) + if !m.isCurrentGeneration(session) { + return + } + libvirtJobEvents.notify() + storageStreamEvents.notify() + m.signalRecovery() + m.signalReconnect() +} + +func (m *managedLibvirt) RecoveryEvents() <-chan struct{} { return m.recovery } + +func (m *managedLibvirt) Close() error { + m.closeOnce.Do(func() { + m.mu.Lock() + m.closing = true + m.currentGeneration.Store(0) + if m.current != nil { + session := m.current + m.current = nil + m.retireSessionLocked(session) + } + m.cancel() + m.mu.Unlock() + + m.workerWG.Wait() + m.retiredWG.Wait() + m.cleanupMu.Lock() + m.closeErr = errors.Join(m.cleanupErrs...) + m.cleanupMu.Unlock() + }) + return m.closeErr +} + +func isLibvirtConnectionFailure(err error) bool { + var libvirtErr libvirt.Error + if errors.As(err, &libvirtErr) { + switch libvirtErr.Code { + case libvirt.ERR_INVALID_CONN, libvirt.ERR_NO_CONNECT, libvirt.ERR_RPC: + return true + } + } + var publicError *Error + return errors.As(err, &publicError) && publicError.Code == "libvirt_unavailable" +} + +func onlyLibvirtConnectionFailures(err error) bool { + if err == nil { + return true + } + if joined, ok := err.(interface{ Unwrap() []error }); ok { + causes := joined.Unwrap() + if len(causes) == 0 { + return false + } + for _, cause := range causes { + if !onlyLibvirtConnectionFailures(cause) { + return false + } + } + return true + } + if wrapped, ok := err.(interface{ Unwrap() error }); ok { + if cause := wrapped.Unwrap(); cause != nil { + return onlyLibvirtConnectionFailures(cause) + } + } + return isLibvirtConnectionFailure(err) +} + +func managedCall[T any](m *managedLibvirt, ctx context.Context, mutation bool, call func(Hypervisor) (T, error)) (T, error) { + var zero T + session, release, err := m.acquire(ctx) + if err != nil { + return zero, err + } + defer release() + value, err := call(session.backend) + if err == nil { + return value, nil + } + if isLibvirtConnectionFailure(err) { + m.markConnectionLost(session) + if mutation && !operation.IsUncertain(err) { + err = operation.MarkUncertain(err) + } + } + return value, err +} + +func managedCallError(m *managedLibvirt, ctx context.Context, mutation bool, call func(Hypervisor) error) error { + _, err := managedCall(m, ctx, mutation, func(backend Hypervisor) (struct{}, error) { + return struct{}{}, call(backend) + }) + return err +} + +func (m *managedLibvirt) Ping(ctx context.Context) error { + return managedCallError(m, ctx, false, func(h Hypervisor) error { return h.Ping(ctx) }) +} + +func (m *managedLibvirt) Capabilities(ctx context.Context) (Capabilities, error) { + return managedCall(m, ctx, false, func(h Hypervisor) (Capabilities, error) { return h.Capabilities(ctx) }) +} + +func (m *managedLibvirt) ConfigureStorage(ctx context.Context, request StorageConfigurationRequest) (StorageConfigurationResult, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (StorageConfigurationResult, error) { + return h.ConfigureStorage(ctx, request) + }) +} + +func (m *managedLibvirt) ListVMs(ctx context.Context) ([]VMMetadata, error) { + return managedCall(m, ctx, false, func(h Hypervisor) ([]VMMetadata, error) { return h.ListVMs(ctx) }) +} + +func (m *managedLibvirt) GetVMState(ctx context.Context, uuid string) (VMState, error) { + return managedCall(m, ctx, false, func(h Hypervisor) (VMState, error) { return h.GetVMState(ctx, uuid) }) +} + +func (m *managedLibvirt) GetVMMetadata(ctx context.Context, uuid string) (VMMetadata, error) { + return managedCall(m, ctx, false, func(h Hypervisor) (VMMetadata, error) { return h.GetVMMetadata(ctx, uuid) }) +} + +func (m *managedLibvirt) CreateVM(ctx context.Context, request CreateVMRequest) (VMMetadata, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) { return h.CreateVM(ctx, request) }) +} + +func (m *managedLibvirt) DeleteVM(ctx context.Context, uuid string, request DeleteVMRequest) error { + return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteVM(ctx, uuid, request) }) +} + +func (m *managedLibvirt) Power(ctx context.Context, uuid string, request PowerRequest) (VMState, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (VMState, error) { return h.Power(ctx, uuid, request) }) +} + +func (m *managedLibvirt) UpdateSettings(ctx context.Context, uuid string, request SettingsRequest) (VMMetadata, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) { + return h.UpdateSettings(ctx, uuid, request) + }) +} + +func (m *managedLibvirt) ResolveNetworkSelector(ctx context.Context, selector string) (LocalNetworkSource, error) { + return managedCall(m, ctx, false, func(h Hypervisor) (LocalNetworkSource, error) { + return h.ResolveNetworkSelector(ctx, selector) + }) +} + +func (m *managedLibvirt) ChangeNetwork(ctx context.Context, uuid string, request ChangeNetworkRequest) (VMMetadata, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) { + return h.ChangeNetwork(ctx, uuid, request) + }) +} + +func (m *managedLibvirt) SetNetworkState(ctx context.Context, uuid string, enabled bool, request NetworkStateRequest) error { + return managedCallError(m, ctx, true, func(h Hypervisor) error { + return h.SetNetworkState(ctx, uuid, enabled, request) + }) +} + +func (m *managedLibvirt) SetServiceState(ctx context.Context, uuid string, enabled bool, request ServiceStateRequest) (VMState, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (VMState, error) { + return h.SetServiceState(ctx, uuid, enabled, request) + }) +} + +func (m *managedLibvirt) Reinstall(ctx context.Context, uuid string, request ReinstallRequest) (VMMetadata, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (VMMetadata, error) { + return h.Reinstall(ctx, uuid, request) + }) +} + +func (m *managedLibvirt) GetMetrics(ctx context.Context, uuid string) (Metrics, error) { + return managedCall(m, ctx, false, func(h Hypervisor) (Metrics, error) { return h.GetMetrics(ctx, uuid) }) +} + +func (m *managedLibvirt) GetDailyTraffic(ctx context.Context, uuid string, days int) ([]DailyTraffic, error) { + return managedCall(m, ctx, false, func(h Hypervisor) ([]DailyTraffic, error) { + return h.GetDailyTraffic(ctx, uuid, days) + }) +} + +func (m *managedLibvirt) GetConsole(ctx context.Context, uuid string) (ConsoleSession, error) { + return managedCall(m, ctx, false, func(h Hypervisor) (ConsoleSession, error) { return h.GetConsole(ctx, uuid) }) +} + +func (m *managedLibvirt) ListSnapshots(ctx context.Context, uuid string) ([]Snapshot, error) { + return managedCall(m, ctx, false, func(h Hypervisor) ([]Snapshot, error) { return h.ListSnapshots(ctx, uuid) }) +} + +func (m *managedLibvirt) CreateSnapshot(ctx context.Context, uuid string, request SnapshotRequest) (Snapshot, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (Snapshot, error) { + return h.CreateSnapshot(ctx, uuid, request) + }) +} + +func (m *managedLibvirt) DeleteSnapshot(ctx context.Context, uuid, key string, request SnapshotMutationRequest) (SnapshotMutationResult, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (SnapshotMutationResult, error) { + return h.DeleteSnapshot(ctx, uuid, key, request) + }) +} + +func (m *managedLibvirt) RestoreSnapshot(ctx context.Context, uuid, key string, request SnapshotMutationRequest) (SnapshotMutationResult, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (SnapshotMutationResult, error) { + return h.RestoreSnapshot(ctx, uuid, key, request) + }) +} + +func (m *managedLibvirt) GetFirewall(ctx context.Context, uuid string) (FirewallPolicy, error) { + return managedCall(m, ctx, false, func(h Hypervisor) (FirewallPolicy, error) { return h.GetFirewall(ctx, uuid) }) +} + +func (m *managedLibvirt) ApplyFirewall(ctx context.Context, uuid string, policy FirewallPolicy) (FirewallPolicy, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (FirewallPolicy, error) { + return h.ApplyFirewall(ctx, uuid, policy) + }) +} + +func (m *managedLibvirt) GetRescue(ctx context.Context, uuid string) (RescueState, error) { + return managedCall(m, ctx, false, func(h Hypervisor) (RescueState, error) { return h.GetRescue(ctx, uuid) }) +} + +func (m *managedLibvirt) EnterRescue(ctx context.Context, uuid string, request RescueRequest) (RescueState, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (RescueState, error) { + return h.EnterRescue(ctx, uuid, request) + }) +} + +func (m *managedLibvirt) ForceStartRescue(ctx context.Context, uuid string, request ForceRescueStartRequest) (RescueState, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (RescueState, error) { + return h.ForceStartRescue(ctx, uuid, request) + }) +} + +func (m *managedLibvirt) ExitRescue(ctx context.Context, uuid string, request ExitRescueRequest) (RescueState, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (RescueState, error) { + return h.ExitRescue(ctx, uuid, request) + }) +} + +func (m *managedLibvirt) ListTemplates(ctx context.Context) ([]ImageTemplate, error) { + return managedCall(m, ctx, false, func(h Hypervisor) ([]ImageTemplate, error) { return h.ListTemplates(ctx) }) +} + +func (m *managedLibvirt) ImportTemplate(ctx context.Context, request ImportTemplateRequest) (ImageTemplate, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (ImageTemplate, error) { + return h.ImportTemplate(ctx, request) + }) +} + +func (m *managedLibvirt) UpdateTemplate(ctx context.Context, id string, request UpdateTemplateRequest) (ImageTemplate, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (ImageTemplate, error) { + return h.UpdateTemplate(ctx, id, request) + }) +} + +func (m *managedLibvirt) DeleteTemplate(ctx context.Context, id string) error { + return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteTemplate(ctx, id) }) +} + +func (m *managedLibvirt) ListRescueTemplates(ctx context.Context) ([]RescueTemplate, error) { + return managedCall(m, ctx, false, func(h Hypervisor) ([]RescueTemplate, error) { return h.ListRescueTemplates(ctx) }) +} + +func (m *managedLibvirt) ImportRescueTemplate(ctx context.Context, request ImportRescueTemplateRequest) (RescueTemplate, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (RescueTemplate, error) { + return h.ImportRescueTemplate(ctx, request) + }) +} + +func (m *managedLibvirt) DeleteRescueTemplate(ctx context.Context, id string) error { + return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteRescueTemplate(ctx, id) }) +} + +func (m *managedLibvirt) ListUserISOs(ctx context.Context, ownerID int64) ([]UserISO, error) { + return managedCall(m, ctx, false, func(h Hypervisor) ([]UserISO, error) { return h.ListUserISOs(ctx, ownerID) }) +} + +func (m *managedLibvirt) ImportUserISO(ctx context.Context, request ImportUserISORequest) (UserISO, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (UserISO, error) { return h.ImportUserISO(ctx, request) }) +} + +func (m *managedLibvirt) DeleteUserISO(ctx context.Context, id string, ownerID int64) error { + return managedCallError(m, ctx, true, func(h Hypervisor) error { return h.DeleteUserISO(ctx, id, ownerID) }) +} + +func (m *managedLibvirt) Reconcile(ctx context.Context, record operation.Record) (operation.Resolution, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (operation.Resolution, error) { + return h.Reconcile(ctx, record) + }) +} + +var _ Hypervisor = (*managedLibvirt)(nil) +var _ RecoveryEventSource = (*managedLibvirt)(nil) diff --git a/agent/hypervisor/libvirt_connection_test.go b/agent/hypervisor/libvirt_connection_test.go new file mode 100644 index 00000000..fa812fb8 --- /dev/null +++ b/agent/hypervisor/libvirt_connection_test.go @@ -0,0 +1,472 @@ +//go:build libvirt + +package hypervisor + +import ( + "context" + "errors" + "fmt" + "reflect" + "sync/atomic" + "testing" + "time" + + "git.luoli.live/root/virt/agent/operation" + "libvirt.org/go/libvirt" +) + +type managedLibvirtTestBackend struct { + Unsupported + ping func(context.Context) error + createVM func(context.Context, CreateVMRequest) (VMMetadata, error) +} + +func (b *managedLibvirtTestBackend) Ping(ctx context.Context) error { + if b.ping != nil { + return b.ping(ctx) + } + return nil +} + +func (b *managedLibvirtTestBackend) CreateVM(ctx context.Context, request CreateVMRequest) (VMMetadata, error) { + if b.createVM != nil { + return b.createVM(ctx, request) + } + return VMMetadata{}, nil +} + +func newManagedLibvirtTestSession(generation uint64, backend Hypervisor, closed *atomic.Int32) *managedLibvirtSession { + return &managedLibvirtSession{ + generation: generation, + backend: backend, + drained: make(chan struct{}), + closeFn: func() error { + closed.Add(1) + return nil + }, + } +} + +func waitForManagedLibvirtGeneration(t *testing.T, manager *managedLibvirt, generation uint64) *managedLibvirtSession { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + manager.mu.Lock() + session := manager.current + manager.mu.Unlock() + if session != nil && session.generation == generation && !session.dead.Load() { + return session + } + time.Sleep(time.Millisecond) + } + t.Fatalf("libvirt generation %d was not published", generation) + return nil +} + +func waitForAtomicValue(t *testing.T, value *atomic.Int32, expected int32) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if value.Load() == expected { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("value = %d, want %d", value.Load(), expected) +} + +func drainManagedLibvirtEvents(events <-chan struct{}) { + for { + select { + case <-events: + default: + return + } + } +} + +func TestManagedLibvirtInitialFailureSignalsAndReconnects(t *testing.T) { + var attempts atomic.Int32 + var closed atomic.Int32 + manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { + if attempts.Add(1) == 1 { + return nil, NewError(ErrUnavailable, "libvirt_unavailable", "libvirt is unavailable", errors.New("daemon down")) + } + return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil + }, func(int) time.Duration { return time.Millisecond }) + t.Cleanup(func() { _ = manager.Close() }) + + manager.start() + select { + case <-manager.RecoveryEvents(): + case <-time.After(time.Second): + t.Fatal("initial connection attempt did not signal recovery readiness") + } + waitForManagedLibvirtGeneration(t, manager, 2) + select { + case <-manager.RecoveryEvents(): + case <-time.After(time.Second): + t.Fatal("successful reconnect did not signal recovery") + } + if err := manager.Ping(context.Background()); err != nil { + t.Fatalf("ping after reconnect: %v", err) + } + if attempts.Load() != 2 { + t.Fatalf("connection attempts = %d, want 2", attempts.Load()) + } +} + +func TestManagedLibvirtReconnectPublishesNewSessionBeforeOldCallDrains(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + var firstClosed atomic.Int32 + var secondClosed atomic.Int32 + var secondPings atomic.Int32 + var attempts atomic.Int32 + + firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) { + close(started) + <-release + return VMMetadata{UUID: "first-result"}, nil + }} + secondBackend := &managedLibvirtTestBackend{ping: func(context.Context) error { + secondPings.Add(1) + return nil + }} + manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { + if attempts.Add(1) == 1 { + return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil + } + return newManagedLibvirtTestSession(generation, secondBackend, &secondClosed), nil + }, func(int) time.Duration { return time.Millisecond }) + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + _ = manager.Close() + }) + manager.start() + firstSession := waitForManagedLibvirtGeneration(t, manager, 1) + drainManagedLibvirtEvents(manager.RecoveryEvents()) + + callDone := make(chan error, 1) + go func() { + _, err := manager.CreateVM(context.Background(), CreateVMRequest{}) + callDone <- err + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first generation call did not start") + } + + manager.markConnectionLost(firstSession) + secondSession := waitForManagedLibvirtGeneration(t, manager, 2) + if secondSession == firstSession { + t.Fatal("reconnect reused the retired session") + } + if err := manager.Ping(context.Background()); err != nil { + t.Fatalf("new generation ping: %v", err) + } + if secondPings.Load() != 1 { + t.Fatalf("new calls did not use the published generation: pings = %d", secondPings.Load()) + } + if firstClosed.Load() != 0 { + t.Fatal("old connection closed while an in-flight call still held it") + } + + close(release) + if err := <-callDone; err != nil { + t.Fatalf("late result from old generation: %v", err) + } + waitForAtomicValue(t, &firstClosed, 1) + + // A delayed callback from the retired connection must not wake recovery or + // evict the replacement generation. + drainManagedLibvirtEvents(manager.RecoveryEvents()) + manager.signalRecoveryFor(firstSession) + manager.markConnectionLost(firstSession) + select { + case <-manager.RecoveryEvents(): + t.Fatal("stale generation emitted a recovery event") + case <-time.After(20 * time.Millisecond): + } + if current := waitForManagedLibvirtGeneration(t, manager, 2); current != secondSession { + t.Fatal("stale generation replaced the current session") + } +} + +func TestManagedLibvirtDisconnectedMutationWasNotDispatched(t *testing.T) { + manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) { + return nil, errors.New("not used") + }, nil) + t.Cleanup(func() { _ = manager.Close() }) + + _, mutationErr := manager.CreateVM(context.Background(), CreateVMRequest{}) + if operation.IsUncertain(mutationErr) { + t.Fatalf("disconnected mutation error = %v, backend was not invoked", mutationErr) + } + if !errors.Is(mutationErr, ErrUnavailable) { + t.Fatalf("disconnected mutation error = %v, want unavailable cause", mutationErr) + } + readErr := manager.Ping(context.Background()) + if operation.IsUncertain(readErr) { + t.Fatalf("read-only ping error = %v, must not be marked uncertain", readErr) + } + if !errors.Is(readErr, ErrUnavailable) { + t.Fatalf("read-only ping error = %v, want unavailable", readErr) + } +} + +func TestManagedLibvirtTransportErrorRetiresSessionAndReconnects(t *testing.T) { + var attempts atomic.Int32 + var firstClosed atomic.Int32 + var secondClosed atomic.Int32 + firstBackend := &managedLibvirtTestBackend{createVM: func(context.Context, CreateVMRequest) (VMMetadata, error) { + return VMMetadata{}, libvirt.Error{Code: libvirt.ERR_RPC, Message: "connection lost"} + }} + manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { + if attempts.Add(1) == 1 { + return newManagedLibvirtTestSession(generation, firstBackend, &firstClosed), nil + } + return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &secondClosed), nil + }, func(int) time.Duration { return time.Millisecond }) + t.Cleanup(func() { _ = manager.Close() }) + manager.start() + waitForManagedLibvirtGeneration(t, manager, 1) + + _, err := manager.CreateVM(context.Background(), CreateVMRequest{}) + if !operation.IsUncertain(err) { + t.Fatalf("transport failure = %v, want uncertain", err) + } + waitForManagedLibvirtGeneration(t, manager, 2) + waitForAtomicValue(t, &firstClosed, 1) + if err := manager.Ping(context.Background()); err != nil { + t.Fatalf("ping after transport reconnect: %v", err) + } +} + +func TestManagedLibvirtCloseWaitsForReferencesAndStopsReconnect(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + var attempts atomic.Int32 + var closed atomic.Int32 + backend := &managedLibvirtTestBackend{ping: func(context.Context) error { + close(started) + <-release + return nil + }} + manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { + attempts.Add(1) + return newManagedLibvirtTestSession(generation, backend, &closed), nil + }, func(int) time.Duration { return time.Millisecond }) + manager.start() + waitForManagedLibvirtGeneration(t, manager, 1) + + pingDone := make(chan error, 1) + go func() { pingDone <- manager.Ping(context.Background()) }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("ping did not acquire the session") + } + closeDone := make(chan error, 1) + go func() { closeDone <- manager.Close() }() + select { + case err := <-closeDone: + t.Fatalf("Close returned before the active reference drained: %v", err) + case <-time.After(20 * time.Millisecond): + } + if closed.Load() != 0 { + t.Fatal("session closed before its active reference drained") + } + + close(release) + if err := <-pingDone; err != nil { + t.Fatalf("in-flight ping: %v", err) + } + select { + case err := <-closeDone: + if err != nil { + t.Fatalf("Close: %v", err) + } + case <-time.After(time.Second): + t.Fatal("Close did not finish after the reference drained") + } + if closed.Load() != 1 { + t.Fatalf("session close count = %d, want 1", closed.Load()) + } + manager.signalReconnect() + time.Sleep(20 * time.Millisecond) + if attempts.Load() != 1 { + t.Fatalf("connection attempts after Close = %d, want 1", attempts.Load()) + } +} + +func TestManagedLibvirtCloseWaitsForBlockingStartup(t *testing.T) { + entered := make(chan struct{}) + release := make(chan struct{}) + startDone := make(chan struct{}) + cleanupErr := errors.New("startup session cleanup failed") + var closed atomic.Int32 + manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { + close(entered) + <-release + session := newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed) + session.closeFn = func() error { + closed.Add(1) + return cleanupErr + } + return session, nil + }, func(int) time.Duration { return time.Millisecond }) + go func() { + manager.start() + close(startDone) + }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("startup factory did not begin") + } + + closeDone := make(chan error, 1) + go func() { closeDone <- manager.Close() }() + select { + case err := <-closeDone: + t.Fatalf("Close returned while startup was still blocked: %v", err) + case <-time.After(20 * time.Millisecond): + } + close(release) + select { + case <-startDone: + case <-time.After(time.Second): + t.Fatal("startup did not finish after factory release") + } + select { + case err := <-closeDone: + if !errors.Is(err, cleanupErr) { + t.Fatalf("Close error = %v, want startup cleanup error", err) + } + case <-time.After(time.Second): + t.Fatal("Close did not wait for startup cleanup") + } + if closed.Load() != 1 { + t.Fatalf("startup session close count = %d, want 1", closed.Load()) + } +} + +func TestManagedLibvirtStartAfterCloseDoesNotConnect(t *testing.T) { + var attempts atomic.Int32 + manager := newManagedLibvirt(func(generation uint64) (*managedLibvirtSession, error) { + attempts.Add(1) + var closed atomic.Int32 + return newManagedLibvirtTestSession(generation, &managedLibvirtTestBackend{}, &closed), nil + }, func(int) time.Duration { return time.Millisecond }) + if err := manager.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + manager.start() + if attempts.Load() != 0 { + t.Fatalf("connection attempts after Close = %d, want 0", attempts.Load()) + } +} + +func TestLibvirtEventBroadcasterAdvancesGeneration(t *testing.T) { + broadcaster := newLibvirtEventBroadcaster() + generation, wake := broadcaster.snapshot() + broadcaster.notify() + if err := broadcaster.wait(context.Background(), generation, wake); err != nil { + t.Fatalf("wait after notify: %v", err) + } + nextGeneration, _ := broadcaster.snapshot() + if nextGeneration != generation+1 { + t.Fatalf("generation = %d, want %d", nextGeneration, generation+1) + } +} + +func TestLibvirtConnectionCleanupUsesReverseRegistrationOrder(t *testing.T) { + var calls []string + cleanup := &libvirtConnectionCleanup{ + closeRegistered: true, + lifecycleID: 11, + jobID: 22, + deregister: func(id int) error { + calls = append(calls, "event:"+fmt.Sprint(id)) + return nil + }, + unregisterClose: func() error { + calls = append(calls, "close-callback") + return nil + }, + closeConnection: func() error { + calls = append(calls, "connection") + return nil + }, + } + if err := cleanup.run(); err != nil { + t.Fatalf("cleanup: %v", err) + } + want := []string{"event:22", "event:11", "close-callback", "connection"} + if !reflect.DeepEqual(calls, want) { + t.Fatalf("cleanup order = %v, want %v", calls, want) + } +} + +func TestLibvirtConnectionCleanupHandlesPartialRegistration(t *testing.T) { + var calls []string + cleanup := &libvirtConnectionCleanup{ + closeRegistered: true, + lifecycleID: 11, + jobID: -1, + deregister: func(id int) error { + calls = append(calls, "event:"+fmt.Sprint(id)) + return nil + }, + unregisterClose: func() error { + calls = append(calls, "close-callback") + return nil + }, + closeConnection: func() error { + calls = append(calls, "connection") + return nil + }, + } + if err := cleanup.run(); err != nil { + t.Fatalf("cleanup: %v", err) + } + want := []string{"event:11", "close-callback", "connection"} + if !reflect.DeepEqual(calls, want) { + t.Fatalf("partial cleanup order = %v, want %v", calls, want) + } +} + +func TestManagedLibvirtIgnoresExpectedDeadSessionCleanupError(t *testing.T) { + manager := newManagedLibvirt(func(uint64) (*managedLibvirtSession, error) { + return nil, errors.New("not used") + }, nil) + deadSession := &managedLibvirtSession{closeFn: func() error { + return fmt.Errorf("unregister stale callback: %w", libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) + }} + deadSession.dead.Store(true) + manager.cleanupSession(deadSession) + if len(manager.cleanupErrs) != 0 { + t.Fatalf("dead connection cleanup errors were retained: %v", manager.cleanupErrs) + } + + liveSession := &managedLibvirtSession{closeFn: func() error { return errors.New("close failed") }} + manager.cleanupSession(liveSession) + if len(manager.cleanupErrs) != 1 { + t.Fatalf("live connection cleanup errors = %v, want one retained error", manager.cleanupErrs) + } + + mixedSession := &managedLibvirtSession{closeFn: func() error { + return errors.Join(libvirt.Error{Code: libvirt.ERR_NO_CONNECT}, errors.New("unexpected cleanup failure")) + }} + mixedSession.dead.Store(true) + manager.cleanupSession(mixedSession) + if len(manager.cleanupErrs) != 2 { + t.Fatalf("mixed dead connection cleanup errors = %v, want unexpected error retained", manager.cleanupErrs) + } +} diff --git a/agent/hypervisor/libvirt_snapshot_fallback_test.go b/agent/hypervisor/libvirt_snapshot_fallback_test.go index 83bea5ac..1b355eaf 100644 --- a/agent/hypervisor/libvirt_snapshot_fallback_test.go +++ b/agent/hypervisor/libvirt_snapshot_fallback_test.go @@ -223,10 +223,17 @@ type snapshotBackupJobFake struct { abortErr error abortCalls int getJobStatsCalls []libvirt.DomainGetJobStatsFlags + getJobStatsCall chan struct{} } func (f *snapshotBackupJobFake) GetJobStats(flags libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) { f.getJobStatsCalls = append(f.getJobStatsCalls, flags) + if f.getJobStatsCall != nil { + select { + case f.getJobStatsCall <- struct{}{}: + default: + } + } if flags == 0 { if f.activeErr != nil { return nil, f.activeErr @@ -374,6 +381,94 @@ func TestWaitForBackupTerminalDoesNotAbortNormalCompletion(t *testing.T) { } } +func TestWaitForBackupTerminalWakesOnJobCompletedEvent(t *testing.T) { + calls := make(chan struct{}, 4) + job := &snapshotBackupJobFake{ + active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()}, + completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)}, + getJobStatsCall: calls, + } + timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second} + type result struct { + terminal libvirt.DomainJobType + err error + } + done := make(chan result, 1) + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + libvirtJobEvents.notify() + }() + go func() { + terminal, err := waitForBackupTerminalWithTiming(ctx, job, timing) + done <- result{terminal: terminal, err: err} + }() + select { + case <-calls: + case <-time.After(time.Second): + t.Fatal("backup waiter did not inspect the active job") + } + libvirtJobEvents.notify() + select { + case <-calls: + case <-time.After(time.Second): + t.Fatal("spurious JOB_COMPLETED event did not trigger GetJobStats") + } + select { + case result := <-done: + t.Fatalf("event was treated as terminal evidence: terminal=%v err=%v", result.terminal, result.err) + case <-time.After(20 * time.Millisecond): + } + + libvirtJobEvents.notify() + select { + case result := <-done: + if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_COMPLETED { + t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err) + } + case <-time.After(time.Second): + t.Fatal("JOB_COMPLETED event did not wake backup waiter") + } +} + +func TestAbortBackupWaitWakesOnJobCompletedEvent(t *testing.T) { + calls := make(chan struct{}, 2) + job := &snapshotBackupJobFake{ + active: []*libvirt.DomainJobInfo{ + activeBackupJob(), activeBackupJob(), noActiveBackupJob(), + }, + completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)}, + getJobStatsCall: calls, + } + timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second} + type result struct { + terminal libvirt.DomainJobType + err error + } + done := make(chan result, 1) + defer libvirtJobEvents.notify() + go func() { + terminal, err := abortBackupAndWaitForTerminal(job, context.DeadlineExceeded, timing) + done <- result{terminal: terminal, err: err} + }() + for call := 0; call < 2; call++ { + select { + case <-calls: + case <-time.After(time.Second): + t.Fatal("abort waiter did not inspect the active job") + } + } + libvirtJobEvents.notify() + select { + case result := <-done: + if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_CANCELLED { + t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err) + } + case <-time.After(time.Second): + t.Fatal("JOB_COMPLETED event did not wake abort confirmation") + } +} + func TestInspectBackupTerminalAcceptsAllBackupTerminalStates(t *testing.T) { for _, kind := range []libvirt.DomainJobType{ libvirt.DOMAIN_JOB_COMPLETED, diff --git a/agent/hypervisor/libvirt_snapshot_features.go b/agent/hypervisor/libvirt_snapshot_features.go index 3e808f15..2dcafde7 100644 --- a/agent/hypervisor/libvirt_snapshot_features.go +++ b/agent/hypervisor/libvirt_snapshot_features.go @@ -346,6 +346,7 @@ func abortBackupAndWaitForTerminal(domain snapshotBackupJob, cause error, defer timer.Stop() var lastInspectErr error for { + _, jobWake := libvirtJobEvents.snapshot() terminal, done, inspectErr = inspectBackupTerminal(domain) if done { // A retained terminal result is stronger evidence than an error from @@ -356,6 +357,7 @@ func abortBackupAndWaitForTerminal(domain snapshotBackupJob, cause error, lastInspectErr = inspectErr } select { + case <-jobWake: case <-ticker.C: case <-timer.C: // Close the boundary race between the last polling tick and the @@ -383,6 +385,7 @@ func waitForBackupTerminalWithTiming(ctx context.Context, domain snapshotBackupJ ticker := time.NewTicker(timing.pollInterval) defer ticker.Stop() for { + _, jobWake := libvirtJobEvents.snapshot() terminal, done, err := inspectBackupTerminal(domain) if err != nil { return libvirt.DOMAIN_JOB_NONE, operation.MarkUncertain(err) @@ -396,6 +399,7 @@ func waitForBackupTerminalWithTiming(ctx context.Context, domain snapshotBackupJ select { case <-ctx.Done(): return abortBackupAndWaitForTerminal(domain, ctx.Err(), timing) + case <-jobWake: case <-ticker.C: } } diff --git a/agent/hypervisor/libvirt_stream.go b/agent/hypervisor/libvirt_stream.go index 529f7519..c76dd465 100644 --- a/agent/hypervisor/libvirt_stream.go +++ b/agent/hypervisor/libvirt_stream.go @@ -25,7 +25,7 @@ var ( libvirtEventLoopOnce sync.Once libvirtEventLoopMu sync.RWMutex libvirtEventLoopErr error - storageStreamEvents = newStreamEventBroadcaster() + storageStreamEvents = newStorageStreamEventBroadcaster() ) // startLibvirtEventLoop must run before the first libvirt connection is @@ -74,23 +74,26 @@ func currentLibvirtEventLoopError() error { return libvirtEventLoopErr } -type streamEventBroadcaster struct { +// libvirtEventBroadcaster is a coalescing generation signal shared by stream +// readiness and domain job completion. Events carry no durable evidence; they +// only tell waiters to re-read the corresponding libvirt state. +type libvirtEventBroadcaster struct { mu sync.Mutex generation uint64 wake chan struct{} } -func newStreamEventBroadcaster() *streamEventBroadcaster { - return &streamEventBroadcaster{wake: make(chan struct{})} +func newLibvirtEventBroadcaster() *libvirtEventBroadcaster { + return &libvirtEventBroadcaster{wake: make(chan struct{})} } -func (b *streamEventBroadcaster) snapshot() (uint64, <-chan struct{}) { +func (b *libvirtEventBroadcaster) snapshot() (uint64, <-chan struct{}) { b.mu.Lock() defer b.mu.Unlock() return b.generation, b.wake } -func (b *streamEventBroadcaster) notify() { +func (b *libvirtEventBroadcaster) notify() { b.mu.Lock() close(b.wake) b.generation++ @@ -98,19 +101,32 @@ func (b *streamEventBroadcaster) notify() { b.mu.Unlock() } -func (b *streamEventBroadcaster) wait(ctx context.Context, generation uint64, wake <-chan struct{}) error { +func (b *libvirtEventBroadcaster) wait(ctx context.Context, generation uint64, wake <-chan struct{}) error { b.mu.Lock() changed := b.generation != generation b.mu.Unlock() if changed { - return currentStreamEventLoopError() + return nil } select { case <-ctx.Done(): return ctx.Err() case <-wake: - return currentStreamEventLoopError() + return nil + } +} + +type storageStreamEventBroadcaster struct{ *libvirtEventBroadcaster } + +func newStorageStreamEventBroadcaster() *storageStreamEventBroadcaster { + return &storageStreamEventBroadcaster{libvirtEventBroadcaster: newLibvirtEventBroadcaster()} +} + +func (b *storageStreamEventBroadcaster) wait(ctx context.Context, generation uint64, wake <-chan struct{}) error { + if err := b.libvirtEventBroadcaster.wait(ctx, generation, wake); err != nil { + return err } + return currentStreamEventLoopError() } func currentStreamEventLoopError() error { diff --git a/agent/hypervisor/types.go b/agent/hypervisor/types.go index 0d66b295..a5a3d700 100644 --- a/agent/hypervisor/types.go +++ b/agent/hypervisor/types.go @@ -559,6 +559,13 @@ type Hypervisor interface { Reconcile(context.Context, operation.Record) (operation.Resolution, error) } +// RecoveryEventSource is implemented by backends that can coalesce host-side +// state changes into reconciliation wakeups. Events are hints only; callers +// must always recover from the durable operation journal. +type RecoveryEventSource interface { + RecoveryEvents() <-chan struct{} +} + // Unsupported is embeddable in focused adapters and tests. Every method has // a stable, capability-gated failure instead of silently succeeding. type Unsupported struct{} diff --git a/agent/operation/journal.go b/agent/operation/journal.go index 6f1f8a7f..3de5fb69 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -57,11 +57,13 @@ var ( metadataBucket = []byte("metadata") operationsBucket = []byte("operations") inflightBucket = []byte("inflight") + recoveryBucket = []byte("recovery") operationExpiryBucket = []byte("operation_expiry") noncesBucket = []byte("nonces") nonceExpiryBucket = []byte("nonce_expiry") schemaVersionKey = []byte("schema_version") - schemaVersion = []byte("2") + previousSchemaVersion = []byte("2") + schemaVersion = []byte("3") ) // ExpectedState is the non-secret, durable evidence contract used after a @@ -349,6 +351,40 @@ func (j *Journal) Get(operationID string) (Record, error) { return cloneRecord(record), err } +// ListRecoveryCandidates returns the durable operations whose original +// mutation must never be replayed, but whose observable host state can still +// be reconciled. The dedicated index keeps this proportional to unfinished +// work instead of the permanent operation tombstone history. +func (j *Journal) ListRecoveryCandidates() ([]Record, error) { + records := make([]Record, 0) + err := j.db.View(func(tx *bolt.Tx) error { + operations := tx.Bucket(operationsBucket) + recovery := tx.Bucket(recoveryBucket) + if operations == nil || recovery == nil { + return errors.New("operation recovery buckets are unavailable") + } + return recovery.ForEach(func(key, _ []byte) error { + encoded := operations.Get(key) + if encoded == nil { + return fmt.Errorf("recovery index references missing operation %q", key) + } + record, err := decodeRecord(encoded) + if err != nil { + return fmt.Errorf("decode recovery operation %q: %w", key, err) + } + if record.OperationID != string(key) { + return fmt.Errorf("recovery index key %q does not match operation %q", key, record.OperationID) + } + if !recoveryIndexEligible(record) { + return fmt.Errorf("recovery index references resolved operation %q", key) + } + records = append(records, cloneRecord(record)) + return nil + }) + }) + return records, err +} + // Reserve durably binds operation_id to one exact request fingerprint and its // non-secret reconciliation descriptor. Records are never deleted, so an old // destructive operation_id can never become valid again after housekeeping. @@ -362,7 +398,8 @@ func (j *Journal) Reserve(spec Spec) (Record, bool, error) { err := j.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket(operationsBucket) inflight := tx.Bucket(inflightBucket) - if bucket == nil || inflight == nil { + recovery := tx.Bucket(recoveryBucket) + if bucket == nil || inflight == nil || recovery == nil { return errors.New("operation buckets are unavailable") } if err := pruneExpiredOperationDetails(tx, j.currentTime()); err != nil { @@ -391,7 +428,10 @@ func (j *Journal) Reserve(spec Spec) (Record, bool, error) { if err := putRecord(bucket, record); err != nil { return err } - return inflight.Put([]byte(record.OperationID), nil) + if err := inflight.Put([]byte(record.OperationID), nil); err != nil { + return err + } + return syncRecoveryIndex(recovery, record) }) return cloneRecord(record), existed, err } @@ -686,6 +726,9 @@ func (j *Journal) Reconcile(ctx context.Context, operationID string, fn func(con } func recordCanReconcile(record Record) bool { + if record.Tombstone { + return false + } if record.State == StateUncertain || record.State == StateDispatching { return true } @@ -703,6 +746,25 @@ func recordCanReconcile(record Record) bool { return !confirmedAbsent } +func syncRecoveryIndex(recovery *bolt.Bucket, record Record) error { + if recovery == nil { + return errors.New("operation recovery bucket is unavailable") + } + key := []byte(record.OperationID) + if recoveryIndexEligible(record) { + return recovery.Put(key, nil) + } + return recovery.Delete(key) +} + +// recoveryIndexEligible excludes an operation that is still executing in this +// process. Its real worker owns the admission leases and will record a late +// result. On restart initialize first converts its durable inflight record to +// uncertain, after which it becomes a recovery candidate. +func recoveryIndexEligible(record Record) bool { + return record.State != StateDispatching && recordCanReconcile(record) +} + // Older Agents could finalize a committed reinstall while its deterministic // old-root cleanup was still pending. Permit the same operation's reconcile // endpoint to replace that result after the cleanup tail is confirmed. @@ -757,11 +819,10 @@ func (j *Journal) initialize() error { if err != nil { return fmt.Errorf("create journal metadata bucket: %w", err) } - if current := metadata.Get(schemaVersionKey); current == nil { - if err := metadata.Put(schemaVersionKey, schemaVersion); err != nil { - return err - } - } else if string(current) != string(schemaVersion) { + current := metadata.Get(schemaVersionKey) + fresh := current == nil + migrating := string(current) == string(previousSchemaVersion) + if !fresh && !migrating && string(current) != string(schemaVersion) { return fmt.Errorf("unsupported operation journal schema %q", current) } operations, err := tx.CreateBucketIfNotExists(operationsBucket) @@ -782,6 +843,16 @@ func (j *Journal) initialize() error { if err != nil { return fmt.Errorf("create inflight operation bucket: %w", err) } + recovery := tx.Bucket(recoveryBucket) + if recovery == nil { + if !fresh && !migrating { + return errors.New("operation recovery index is unavailable") + } + recovery, err = tx.CreateBucket(recoveryBucket) + if err != nil { + return fmt.Errorf("create operation recovery bucket: %w", err) + } + } now := j.currentTime() var recovered []Record var stale [][]byte @@ -805,6 +876,9 @@ func (j *Journal) initialize() error { record.Result = nil record.Error = &PublicError{Code: "result_uncertain", Message: "agent restarted after dispatch; reconciliation is required"} default: + if err := syncRecoveryIndex(recovery, record); err != nil { + return err + } stale = append(stale, append([]byte(nil), key...)) return nil } @@ -818,6 +892,9 @@ func (j *Journal) initialize() error { if err := putRecord(operations, record); err != nil { return err } + if err := syncRecoveryIndex(recovery, record); err != nil { + return err + } if terminalState(record.State) { if err := operationExpiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil { return err @@ -832,7 +909,26 @@ func (j *Journal) initialize() error { return err } } - return pruneExpiredOperationDetails(tx, now) + if migrating { + // Schema v2 did not retain uncertain operations in its inflight + // index. Rebuild the new recovery index exactly once. A corrupt + // permanent history row that was not in an active index remains an + // unusable tombstone, but cannot prevent the Agent from recovering + // every other valid operation. Corrupt inflight rows fail above. + if err := operations.ForEach(func(key, value []byte) error { + record, decodeErr := decodeRecord(value) + if decodeErr != nil || record.OperationID != string(key) { + return recovery.Delete(key) + } + return syncRecoveryIndex(recovery, record) + }); err != nil { + return fmt.Errorf("migrate operation recovery index: %w", err) + } + } + if err := pruneExpiredOperationDetails(tx, now); err != nil { + return err + } + return metadata.Put(schemaVersionKey, schemaVersion) }) } @@ -984,7 +1080,8 @@ func (j *Journal) updateRecord(operationID string, mutate func(*Record)) (Record err := j.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket(operationsBucket) inflight := tx.Bucket(inflightBucket) - if bucket == nil || inflight == nil { + recovery := tx.Bucket(recoveryBucket) + if bucket == nil || inflight == nil || recovery == nil { return errors.New("operation buckets are unavailable") } value := bucket.Get([]byte(operationID)) @@ -999,6 +1096,9 @@ func (j *Journal) updateRecord(operationID string, mutate func(*Record)) (Record if err := putRecord(bucket, record); err != nil { return err } + if err := syncRecoveryIndex(recovery, record); err != nil { + return err + } switch record.State { case StateReserved, StateDispatching: if err := inflight.Put([]byte(record.OperationID), nil); err != nil { @@ -1085,8 +1185,9 @@ func (j *Journal) prepareAdmission(spec Spec) (Record, error) { err := j.db.Update(func(tx *bolt.Tx) error { operations := tx.Bucket(operationsBucket) inflight := tx.Bucket(inflightBucket) + recovery := tx.Bucket(recoveryBucket) expiry := tx.Bucket(operationExpiryBucket) - if operations == nil || inflight == nil || expiry == nil { + if operations == nil || inflight == nil || recovery == nil || expiry == nil { return errors.New("operation buckets are unavailable") } value := operations.Get([]byte(spec.OperationID)) @@ -1121,6 +1222,9 @@ func (j *Journal) prepareAdmission(spec Spec) (Record, error) { if err := putRecord(operations, existing); err != nil { return err } + if err := syncRecoveryIndex(recovery, existing); err != nil { + return err + } if err := inflight.Put([]byte(existing.OperationID), nil); err != nil { return err } @@ -1507,7 +1611,8 @@ func terminalState(state State) bool { func pruneExpiredOperationDetails(tx *bolt.Tx, now time.Time) error { operations := tx.Bucket(operationsBucket) expiry := tx.Bucket(operationExpiryBucket) - if operations == nil || expiry == nil { + recovery := tx.Bucket(recoveryBucket) + if operations == nil || expiry == nil || recovery == nil { return errors.New("operation retention buckets are unavailable") } cutoff := now.Add(-operationDetailRetention).UnixNano() @@ -1525,18 +1630,28 @@ func pruneExpiredOperationDetails(tx *bolt.Tx, now time.Time) error { } indexedAt := int64(binary.BigEndian.Uint64(key[:8])) if terminalState(record.State) && !record.Tombstone && record.UpdatedAt.UnixNano() == indexedAt { - record.Tombstone = true - record.Result = nil - record.ExpectedState = nil - record.LockKeys = nil - record.HeavyIO = false - if record.Error != nil { - record.Error = &PublicError{Code: record.Error.Code, Message: "operation details expired; operation_id remains permanently reserved"} - } - if err := putRecord(operations, record); err != nil { - return err + // A publicly terminal create can still lack the absence proof + // needed for a safe refund, and a committed reinstall can retain + // an operation-owned cleanup tail. Keep their evidence until the + // same operation is reconciled; only then does a new retention + // period begin from its final UpdatedAt. + if !recordCanReconcile(record) { + record.Tombstone = true + record.Result = nil + record.ExpectedState = nil + record.LockKeys = nil + record.HeavyIO = false + if record.Error != nil { + record.Error = &PublicError{Code: record.Error.Code, Message: "operation details expired; operation_id remains permanently reserved"} + } + if err := putRecord(operations, record); err != nil { + return err + } } } + if err := syncRecoveryIndex(recovery, record); err != nil { + return err + } } if err := cursor.Delete(); err != nil { return err diff --git a/agent/operation/recovery.go b/agent/operation/recovery.go new file mode 100644 index 00000000..a617130e --- /dev/null +++ b/agent/operation/recovery.go @@ -0,0 +1,259 @@ +package operation + +import ( + "context" + "errors" + "sync" + "time" +) + +const ( + defaultRecoverySweepInterval = 5 * time.Minute + defaultRecoveryItemTimeout = 30 * time.Second +) + +// RecoveryReconciler inspects one durable operation. Implementations may only +// inspect host evidence or continue an operation-owned, idempotent cleanup +// tail. The Coordinator never receives or reconstructs the original mutation. +type RecoveryReconciler func(context.Context, Record) (Resolution, error) + +// RecoveryObserver receives bounded operation metadata plus the reconciliation +// result. Callers decide how to rate-limit and sanitize local diagnostics. A +// zero Record identifies a failure to read the durable candidate index. +type RecoveryObserver func(Record, error) + +// RecoveryCoordinator serially scans the durable recovery index. Startup, +// libvirt events, successful reconnects, and a periodic safety sweep all use +// the same coalesced path, so events can accelerate reconciliation but can +// never replace Journal evidence. +type RecoveryCoordinator struct { + journal *Journal + reconcile RecoveryReconciler + external <-chan struct{} + wake chan struct{} + sweepInterval time.Duration + itemTimeout time.Duration + + lifecycleMu sync.Mutex + started bool + stopped bool + cancel context.CancelFunc + done chan struct{} + ready chan struct{} + readyOnce sync.Once + observer RecoveryObserver +} + +// SetObserver installs diagnostics before Start. Replacing it while the worker +// is active would create a data race and is therefore rejected. +func (c *RecoveryCoordinator) SetObserver(observer RecoveryObserver) error { + if c == nil { + return errors.New("recovery coordinator is required") + } + c.lifecycleMu.Lock() + defer c.lifecycleMu.Unlock() + if c.started || c.stopped { + return errors.New("recovery observer must be set before start") + } + c.observer = observer + return nil +} + +func NewRecoveryCoordinator(journal *Journal, reconcile RecoveryReconciler, + external <-chan struct{}) (*RecoveryCoordinator, error) { + return newRecoveryCoordinator( + journal, reconcile, external, + defaultRecoverySweepInterval, defaultRecoveryItemTimeout, + ) +} + +func newRecoveryCoordinator(journal *Journal, reconcile RecoveryReconciler, + external <-chan struct{}, sweepInterval, itemTimeout time.Duration) (*RecoveryCoordinator, error) { + if journal == nil || reconcile == nil { + return nil, errors.New("operation journal and recovery reconciler are required") + } + if sweepInterval <= 0 || itemTimeout <= 0 { + return nil, errors.New("recovery timing must be positive") + } + return &RecoveryCoordinator{ + journal: journal, reconcile: reconcile, external: external, + wake: make(chan struct{}, 1), sweepInterval: sweepInterval, + itemTimeout: itemTimeout, done: make(chan struct{}), ready: make(chan struct{}), + }, nil +} + +// Start launches one recovery worker. With a managed event source only an +// external initial-status signal may open the readiness gate; without one the +// Journal is scanned immediately. The full pass covers restarts even when +// every earlier libvirt event and completed-job statistic was lost. +func (c *RecoveryCoordinator) Start(parent context.Context) error { + if parent == nil { + return errors.New("recovery parent context is required") + } + c.lifecycleMu.Lock() + defer c.lifecycleMu.Unlock() + if c.stopped { + return errors.New("recovery coordinator is stopped") + } + if c.started { + return nil + } + ctx, cancel := context.WithCancel(parent) + c.cancel = cancel + c.started = true + go c.run(ctx) + return nil +} + +// Wake is safe for event-forwarding goroutines and coalesces repeated events. +func (c *RecoveryCoordinator) Wake() { + if c == nil { + return + } + select { + case c.wake <- struct{}{}: + default: + } +} + +// Ready closes after the first candidate pass finishes. Unresolved evidence +// and per-item timeouts still count as a finished pass because their records +// remain durably indexed. Main can wait on Ready before accepting new mutation +// requests, preventing fresh work from overtaking restart reconciliation. +func (c *RecoveryCoordinator) Ready() <-chan struct{} { + if c == nil { + closed := make(chan struct{}) + close(closed) + return closed + } + return c.ready +} + +// Stop prevents future scans and cancels the current per-operation context. +// Wait must complete before closing libvirt or the Journal. If a native call +// ignores cancellation, Wait deliberately remains blocked until that call +// really returns, preserving the same resource-lifetime rule as mutations. +func (c *RecoveryCoordinator) Stop() { + if c == nil { + return + } + c.lifecycleMu.Lock() + if c.stopped { + c.lifecycleMu.Unlock() + return + } + c.stopped = true + if c.cancel != nil { + c.cancel() + } + if !c.started { + close(c.done) + c.markReady() + } + c.lifecycleMu.Unlock() +} + +func (c *RecoveryCoordinator) Wait(ctx context.Context) error { + if c == nil { + return nil + } + if ctx == nil { + return errors.New("recovery wait context is required") + } + c.lifecycleMu.Lock() + started := c.started + stopped := c.stopped + done := c.done + c.lifecycleMu.Unlock() + if !started && !stopped { + return nil + } + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (c *RecoveryCoordinator) run(ctx context.Context) { + defer close(c.done) + defer c.markReady() + ticker := time.NewTicker(c.sweepInterval) + defer ticker.Stop() + external := c.external + externalReadyRequired := external != nil + initialPending := true + + // A nil event source is useful for backends whose connection is already + // known-ready. A managed source emits an initial-status signal after its + // first connection attempt. Periodic or internal wakes may still make safe + // read-only attempts while that signal is pending, but only a successful + // external pass may open the admission gate. A failed index read is not a + // completed pass and cannot open it in either mode. + if !externalReadyRequired && c.reconcileAll(ctx) == nil { + c.markReady() + initialPending = false + } + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + passErr := c.reconcileAll(ctx) + if initialPending && !externalReadyRequired && passErr == nil { + c.markReady() + initialPending = false + } + case <-c.wake: + passErr := c.reconcileAll(ctx) + if initialPending && !externalReadyRequired && passErr == nil { + c.markReady() + initialPending = false + } + case _, ok := <-external: + if !ok { + external = nil + continue + } + passErr := c.reconcileAll(ctx) + if initialPending && passErr == nil { + c.markReady() + initialPending = false + } + } + } +} + +func (c *RecoveryCoordinator) markReady() { + c.readyOnce.Do(func() { close(c.ready) }) +} + +func (c *RecoveryCoordinator) reconcileAll(ctx context.Context) error { + if ctx.Err() != nil { + return ctx.Err() + } + records, err := c.journal.ListRecoveryCandidates() + if err != nil { + // A corrupt indexed candidate is fail-closed by the Journal. Leave the + // index untouched for an operator to repair; a later signal retries it. + c.observe(Record{}, err) + return err + } + for _, record := range records { + if ctx.Err() != nil { + return ctx.Err() + } + itemCtx, cancel := context.WithTimeout(ctx, c.itemTimeout) + result, reconcileErr := c.journal.Reconcile(itemCtx, record.OperationID, c.reconcile) + cancel() + c.observe(result, reconcileErr) + } + return nil +} + +func (c *RecoveryCoordinator) observe(record Record, err error) { + if c.observer != nil { + c.observer(cloneRecord(record), err) + } +} diff --git a/agent/operation/recovery_test.go b/agent/operation/recovery_test.go new file mode 100644 index 00000000..bfd52cd2 --- /dev/null +++ b/agent/operation/recovery_test.go @@ -0,0 +1,513 @@ +package operation + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "reflect" + "sort" + "strings" + "sync/atomic" + "testing" + "time" + + bolt "go.etcd.io/bbolt" +) + +func recoveryTestRecord(id, resource, action string, state State) Record { + now := time.Unix(1_800_000_000, 0).UTC() + return Record{ + OperationID: id, + ResourceID: resource, + Action: action, + Fingerprint: strings.Repeat("a", 64), + Destructive: true, + State: state, + CreatedAt: now, + UpdatedAt: now, + } +} + +func recoveryCandidateIDs(t *testing.T, journal *Journal) []string { + t.Helper() + records, err := journal.ListRecoveryCandidates() + if err != nil { + t.Fatal(err) + } + ids := make([]string, 0, len(records)) + for _, record := range records { + ids = append(ids, record.OperationID) + } + sort.Strings(ids) + return ids +} + +func TestSchemaV2MigrationBuildsRecoveryIndexWithoutDecodingCorruptHistoryOnLaterStarts(t *testing.T) { + path := filepath.Join(t.TempDir(), "operations.db") + journal, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := journal.Close(); err != nil { + t.Fatal(err) + } + + database, err := bolt.Open(path, 0o600, nil) + if err != nil { + t.Fatal(err) + } + err = database.Update(func(tx *bolt.Tx) error { + if err := tx.Bucket(metadataBucket).Put(schemaVersionKey, previousSchemaVersion); err != nil { + return err + } + if err := tx.DeleteBucket(recoveryBucket); err != nil { + return err + } + operations := tx.Bucket(operationsBucket) + inflight := tx.Bucket(inflightBucket) + records := []Record{ + recoveryTestRecord("operation-v2-uncertain", "vm-v2-uncertain", "delete-vm", StateUncertain), + recoveryTestRecord("operation-v2-dispatch", "vm-v2-dispatch", "delete-vm", StateDispatching), + recoveryTestRecord("operation-v2-reserved", "vm-v2-reserved", "delete-vm", StateReserved), + recoveryTestRecord("operation-v2-complete", "vm-v2-complete", "delete-vm", StateSucceeded), + } + failedCreate := recoveryTestRecord("operation-v2-create", "vm-v2-create", "create-vm", StateFailed) + failedCreate.Error = &PublicError{Code: "backend_error", Message: "creation result is not proven"} + records = append(records, failedCreate) + pendingCleanup := recoveryTestRecord("operation-v2-cleanup", "vm-v2-cleanup", "reinstall-vm", StateSucceeded) + pendingCleanup.Result = json.RawMessage(`{"reinstall_committed":true,"cleanup_pending":true}`) + records = append(records, pendingCleanup) + for _, record := range records { + if err := putRecord(operations, record); err != nil { + return err + } + } + if err := inflight.Put([]byte("operation-v2-dispatch"), nil); err != nil { + return err + } + if err := inflight.Put([]byte("operation-v2-reserved"), nil); err != nil { + return err + } + return operations.Put([]byte("operation-corrupt-tombstone"), []byte("not-json")) + }) + if err != nil { + t.Fatal(err) + } + if err := database.Close(); err != nil { + t.Fatal(err) + } + + reopened, err := Open(path) + if err != nil { + t.Fatalf("migrate schema v2: %v", err) + } + want := []string{ + "operation-v2-cleanup", + "operation-v2-create", + "operation-v2-dispatch", + "operation-v2-uncertain", + } + if got := recoveryCandidateIDs(t, reopened); !reflect.DeepEqual(got, want) { + t.Fatalf("recovery candidates = %v, want %v", got, want) + } + dispatched, err := reopened.Get("operation-v2-dispatch") + if err != nil || dispatched.State != StateUncertain { + t.Fatalf("recovered dispatch = %#v err=%v", dispatched, err) + } + reserved, err := reopened.Get("operation-v2-reserved") + if err != nil || reserved.State != StateFailed || reserved.Error == nil || reserved.Error.Code != "operation_not_dispatched" { + t.Fatalf("recovered reservation = %#v err=%v", reserved, err) + } + if err := reopened.db.View(func(tx *bolt.Tx) error { + if got := tx.Bucket(metadataBucket).Get(schemaVersionKey); string(got) != string(schemaVersion) { + return fmt.Errorf("schema = %q", got) + } + if tx.Bucket(recoveryBucket) == nil { + return errors.New("recovery bucket was not created") + } + return nil + }); err != nil { + t.Fatal(err) + } + if err := reopened.Close(); err != nil { + t.Fatal(err) + } + + // Schema v3 restarts read only the two active indexes. The unrelated corrupt + // permanent history row remains unusable without blocking startup. + reopened, err = Open(path) + if err != nil { + t.Fatalf("reopen schema v3: %v", err) + } + defer reopened.Close() + if got := recoveryCandidateIDs(t, reopened); !reflect.DeepEqual(got, want) { + t.Fatalf("reopened recovery candidates = %v, want %v", got, want) + } + if _, err := reopened.Get("operation-corrupt-tombstone"); err == nil { + t.Fatal("corrupt permanent record unexpectedly decoded") + } +} + +func TestRecoveryIndexExcludesLiveDispatchAndTracksUncertainResolution(t *testing.T) { + journal := testJournal(t) + spec := testSpec("operation-index-live", "vm-index-live", `{}`) + if _, _, err := journal.Reserve(spec); err != nil { + t.Fatal(err) + } + if _, err := journal.markDispatching(spec.OperationID); err != nil { + t.Fatal(err) + } + if got := recoveryCandidateIDs(t, journal); len(got) != 0 { + t.Fatalf("live dispatch entered recovery index: %v", got) + } + if _, err := journal.markUncertain(spec.OperationID, errors.New("connection lost")); err != nil { + t.Fatal(err) + } + if got := recoveryCandidateIDs(t, journal); !reflect.DeepEqual(got, []string{spec.OperationID}) { + t.Fatalf("uncertain recovery candidates = %v", got) + } + reconciled, err := journal.Reconcile(context.Background(), spec.OperationID, + func(context.Context, Record) (Resolution, error) { + return Resolution{Resolved: true, Succeeded: true, Result: json.RawMessage(`{"absent":true}`)}, nil + }) + if err != nil || reconciled.State != StateSucceeded { + t.Fatalf("reconcile = %#v err=%v", reconciled, err) + } + if got := recoveryCandidateIDs(t, journal); len(got) != 0 { + t.Fatalf("resolved operation retained in recovery index: %v", got) + } +} + +func TestPendingCleanupEvidenceIsNotCompactedBeforeRecovery(t *testing.T) { + journal := testJournal(t) + old := time.Unix(1_500_000_000, 0).UTC() + journal.SetClockForTesting(func() time.Time { return old }) + spec := Spec{ + OperationID: "operation-retained-cleanup", ResourceID: "123e4567-e89b-42d3-a456-426614174099", + Action: "reinstall-vm", Fingerprint: strings.Repeat("b", 64), Destructive: true, + HeavyIO: true, + } + record, _, err := journal.Execute(context.Background(), spec, + func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"reinstall_committed":true,"cleanup_pending":true}`), nil + }) + if err != nil || !succeededCleanupCanReconcile(record) { + t.Fatalf("pending cleanup record = %#v err=%v", record, err) + } + + late := old.Add(operationDetailRetention + time.Hour) + journal.SetClockForTesting(func() time.Time { return late }) + if accepted, err := journal.UseNonce("retainCleanupEvidence1", late, late.Add(time.Minute)); err != nil || !accepted { + t.Fatalf("trigger retention accepted=%v err=%v", accepted, err) + } + retained, err := journal.Get(spec.OperationID) + if err != nil || retained.Tombstone || !succeededCleanupCanReconcile(retained) || + len(retained.Result) == 0 || len(retained.LockKeys) == 0 || !retained.HeavyIO { + t.Fatalf("unfinished cleanup evidence was compacted: %#v err=%v", retained, err) + } + if got := recoveryCandidateIDs(t, journal); !reflect.DeepEqual(got, []string{spec.OperationID}) { + t.Fatalf("retained cleanup candidates = %v", got) + } + + resolved, err := journal.Reconcile(context.Background(), spec.OperationID, + func(context.Context, Record) (Resolution, error) { + return Resolution{Resolved: true, Succeeded: true, + Result: json.RawMessage(`{"reinstall_committed":true,"cleanup_pending":false}`)}, nil + }) + if err != nil || resolved.State != StateSucceeded || succeededCleanupCanReconcile(resolved) { + t.Fatalf("resolved cleanup = %#v err=%v", resolved, err) + } + if got := recoveryCandidateIDs(t, journal); len(got) != 0 { + t.Fatalf("resolved cleanup retained in recovery index: %v", got) + } +} + +func makeRecoveryCandidate(t *testing.T, journal *Journal, id, resource string) { + t.Helper() + _, _, err := journal.Execute(context.Background(), testSpec(id, resource, `{}`), + func(context.Context) (json.RawMessage, error) { + return nil, MarkUncertain(errors.New("lost result")) + }) + if err == nil || !IsUncertain(err) { + t.Fatalf("create recovery candidate err=%v", err) + } +} + +func waitForRecoveryRecord(t *testing.T, journal *Journal, operationID string, + want State, timeout time.Duration) Record { + t.Helper() + deadline := time.Now().Add(timeout) + for { + record, err := journal.Get(operationID) + if err == nil && record.State == want { + return record + } + if time.Now().After(deadline) { + t.Fatalf("operation %s did not reach %s: record=%#v err=%v", operationID, want, record, err) + } + time.Sleep(time.Millisecond) + } +} + +func TestRecoveryCoordinatorScansAtStartupAndCoalescesEventWakeups(t *testing.T) { + journal := testJournal(t) + makeRecoveryCandidate(t, journal, "operation-recovery-event", "vm-recovery-event") + external := make(chan struct{}, 1) + first := make(chan struct{}) + var calls atomic.Int32 + var active atomic.Int32 + var maximum atomic.Int32 + var resolve atomic.Bool + reconcile := func(_ context.Context, record Record) (Resolution, error) { + if record.OperationID != "operation-recovery-event" || record.Action != "delete-vm" { + t.Errorf("unexpected recovery record: %#v", record) + } + current := active.Add(1) + defer active.Add(-1) + for { + previous := maximum.Load() + if current <= previous || maximum.CompareAndSwap(previous, current) { + break + } + } + if calls.Add(1) == 1 { + close(first) + } + if !resolve.Load() { + return Resolution{}, nil + } + return Resolution{Resolved: true, Succeeded: true, Result: json.RawMessage(`{"absent":true}`)}, nil + } + coordinator, err := newRecoveryCoordinator(journal, reconcile, external, time.Hour, time.Second) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := coordinator.Start(ctx); err != nil { + t.Fatal(err) + } + select { + case <-coordinator.Ready(): + t.Fatal("coordinator became ready before the initial connection signal") + default: + } + coordinator.Wake() + select { + case <-first: + case <-time.After(time.Second): + t.Fatal("pre-connection recovery attempt did not run") + } + select { + case <-coordinator.Ready(): + t.Fatal("internal wake opened readiness before the initial connection signal") + default: + } + external <- struct{}{} + select { + case <-coordinator.Ready(): + case <-time.After(time.Second): + t.Fatal("external initial-status pass did not open readiness") + } + record, err := journal.Get("operation-recovery-event") + if err != nil || record.State != StateUncertain { + t.Fatalf("unresolved initial pass did not retain candidate: %#v err=%v", record, err) + } + resolve.Store(true) + for index := 0; index < 100; index++ { + coordinator.Wake() + select { + case external <- struct{}{}: + default: + } + } + waitForRecoveryRecord(t, journal, "operation-recovery-event", StateSucceeded, time.Second) + coordinator.Stop() + waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second) + defer waitCancel() + if err := coordinator.Wait(waitCtx); err != nil { + t.Fatal(err) + } + if maximum.Load() != 1 { + t.Fatalf("recovery callbacks ran concurrently: max=%d", maximum.Load()) + } +} + +func TestRecoveryCoordinatorReadyFailsClosedOnUnreadableIndex(t *testing.T) { + journal := testJournal(t) + if err := journal.db.Update(func(tx *bolt.Tx) error { + return tx.Bucket(recoveryBucket).Put([]byte("operation-missing-from-history"), nil) + }); err != nil { + t.Fatal(err) + } + var reconcilerCalled atomic.Bool + coordinator, err := newRecoveryCoordinator(journal, + func(context.Context, Record) (Resolution, error) { + reconcilerCalled.Store(true) + return Resolution{}, nil + }, nil, time.Hour, time.Second) + if err != nil { + t.Fatal(err) + } + observed := make(chan error, 1) + if err := coordinator.SetObserver(func(record Record, err error) { + if record.OperationID == "" && err != nil { + select { + case observed <- err: + default: + } + } + }); err != nil { + t.Fatal(err) + } + if err := coordinator.Start(context.Background()); err != nil { + t.Fatal(err) + } + select { + case err := <-observed: + if !strings.Contains(err.Error(), "missing operation") { + t.Fatalf("unexpected index error: %v", err) + } + case <-time.After(time.Second): + t.Fatal("recovery index failure was not observed") + } + if reconcilerCalled.Load() { + t.Fatal("reconciler ran despite corrupt recovery index") + } + select { + case <-coordinator.Ready(): + t.Fatal("unreadable recovery index opened readiness") + default: + } + if err := journal.db.Update(func(tx *bolt.Tx) error { + return tx.Bucket(recoveryBucket).Delete([]byte("operation-missing-from-history")) + }); err != nil { + t.Fatal(err) + } + coordinator.Wake() + select { + case <-coordinator.Ready(): + case <-time.After(time.Second): + t.Fatal("repaired recovery index did not open readiness") + } + coordinator.Stop() + waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second) + defer waitCancel() + if err := coordinator.Wait(waitCtx); err != nil { + t.Fatal(err) + } +} + +func TestRecoveryCoordinatorStopBeforeStartClosesLifecycleChannels(t *testing.T) { + journal := testJournal(t) + coordinator, err := newRecoveryCoordinator(journal, + func(context.Context, Record) (Resolution, error) { return Resolution{}, nil }, + nil, time.Hour, time.Second) + if err != nil { + t.Fatal(err) + } + coordinator.Stop() + select { + case <-coordinator.Ready(): + default: + t.Fatal("Stop before Start left Ready open") + } + waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second) + defer waitCancel() + if err := coordinator.Wait(waitCtx); err != nil { + t.Fatal(err) + } + if err := coordinator.Start(context.Background()); err == nil { + t.Fatal("stopped coordinator unexpectedly restarted") + } +} + +func TestRecoveryCoordinatorPeriodicSweepAndPerItemTimeout(t *testing.T) { + journal := testJournal(t) + makeRecoveryCandidate(t, journal, "operation-a-timeout", "vm-a-timeout") + makeRecoveryCandidate(t, journal, "operation-b-resolved", "vm-b-resolved") + var timedOut atomic.Int32 + reconcile := func(ctx context.Context, record Record) (Resolution, error) { + if record.OperationID == "operation-a-timeout" { + <-ctx.Done() + timedOut.Add(1) + return Resolution{}, ctx.Err() + } + return Resolution{Resolved: true, Succeeded: true, Result: json.RawMessage(`{"absent":true}`)}, nil + } + coordinator, err := newRecoveryCoordinator(journal, reconcile, nil, 15*time.Millisecond, 10*time.Millisecond) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := coordinator.Start(ctx); err != nil { + t.Fatal(err) + } + waitForRecoveryRecord(t, journal, "operation-b-resolved", StateSucceeded, time.Second) + if timedOut.Load() == 0 { + t.Fatal("per-item timeout did not cancel the first recovery attempt") + } + firstCount := timedOut.Load() + deadline := time.Now().Add(time.Second) + for timedOut.Load() <= firstCount && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if timedOut.Load() <= firstCount { + t.Fatal("periodic safety sweep did not retry the unresolved candidate") + } + coordinator.Stop() + waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second) + defer waitCancel() + if err := coordinator.Wait(waitCtx); err != nil { + t.Fatal(err) + } +} + +func TestRecoveryCoordinatorWaitDoesNotOutrunUncooperativeReconcile(t *testing.T) { + journal := testJournal(t) + makeRecoveryCandidate(t, journal, "operation-recovery-shutdown", "vm-recovery-shutdown") + started := make(chan struct{}) + release := make(chan struct{}) + coordinator, err := newRecoveryCoordinator(journal, + func(context.Context, Record) (Resolution, error) { + close(started) + <-release + return Resolution{}, nil + }, nil, time.Hour, 10*time.Millisecond) + if err != nil { + t.Fatal(err) + } + if err := coordinator.Start(context.Background()); err != nil { + t.Fatal(err) + } + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("reconcile did not start") + } + select { + case <-coordinator.Ready(): + t.Fatal("coordinator became ready before the first scan completed") + default: + } + coordinator.Stop() + short, shortCancel := context.WithTimeout(context.Background(), 15*time.Millisecond) + defer shortCancel() + if err := coordinator.Wait(short); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Wait returned before native-equivalent call completed: %v", err) + } + close(release) + select { + case <-coordinator.Ready(): + case <-time.After(time.Second): + t.Fatal("coordinator did not become ready after the first scan completed") + } + long, longCancel := context.WithTimeout(context.Background(), time.Second) + defer longCancel() + if err := coordinator.Wait(long); err != nil { + t.Fatal(err) + } +} diff --git a/docs/AGENT_API.md b/docs/AGENT_API.md index fa2aae1c..99fc8a3e 100644 --- a/docs/AGENT_API.md +++ b/docs/AGENT_API.md @@ -148,10 +148,14 @@ Agent 把原始 `name` 保存为 Virt Cloud 的 `display-name` 域 metadata, 创建快照请求的 `consistency_mode=filesystem` 表示“优先文件系统一致”:Agent 先确认本地 libvirt 客户端和守护进程均支持 5.10.0 引入的 Guest Agent 响应上限;版本较旧时直接改用 `crash`,版本查询失败则返回基础设施错误。确认支持后,Agent 用只读 `guest-ping` 检查 QEMU Guest Agent,再在冻结前通过 libvirt 把冻结和解冻命令的响应上限设置为 `guest_agent_timeout_seconds`(默认 15 秒)。当 libvirt 明确报告 Guest Agent 未配置、未响应、探测超时,或驱动不支持设置响应上限时,也自动改用 `crash`。显式请求 `crash` 时不探测也不冻结 Guest。若 QGA 探测和超时设置均成功,但后续冻结、解冻或 libvirt 通信失败,不得伪装成安全降级;结果不明确时仍必须保留 `uncertain`。该响应上限只约束 Guest Agent 命令,不是后续 push backup 的超时。成功响应的 `consistency` 以及宿主机和面板快照元数据必须记录实际采用的模式,协调时只允许 `filesystem` 请求单向降级为 `crash`,绝不允许反向提升。 -在线快照继续使用 libvirt push backup:`BackupBegin` 启动后以 `GetJobStats` 等待终态。请求超时时,只有当前活动任务仍明确标识为该 backup 时才调用一次 `AbortJob`,随后继续等待最多 30 秒,直到观察到 `COMPLETED`、`FAILED` 或 `CANCELLED`;无法确认终态时保持 `uncertain`。已经终态、任务类型不明或出现其他 libvirt job 时不得调用 `AbortJob`。快照恢复、快照删除、创建、重装和删除虚机均不使用 `AbortJob`。 +在线快照继续使用 libvirt push backup:`BackupBegin` 启动后以 `GetJobStats` 等待终态。`JOB_COMPLETED` 事件只会唤醒一次新的 `GetJobStats` 核对,事件本身不构成成功或失败证据;固定间隔轮询仍作为事件丢失时的兜底。请求超时时,只有当前活动任务仍明确标识为该 backup 时才调用一次 `AbortJob`,随后继续等待最多 30 秒,直到观察到 `COMPLETED`、`FAILED` 或 `CANCELLED`;无法确认终态时保持 `uncertain`。已经终态、任务类型不明或出现其他 libvirt job 时不得调用 `AbortJob`。快照恢复、快照删除、创建、重装和删除虚机均不使用 `AbortJob`。 存储上传和下载使用 `VIR_STREAM_NONBLOCK` 与 libvirt event loop。请求 context 取消、读写失败或校验失败时必须先移除事件回调并 `StreamAbort`;只有完整成功路径才 `StreamFinish`。创建、重装、删除等没有通用 libvirt 取消 API 的操作仍依赖晚到结果与状态核对,不能把 HTTP 超时当作底层操作已中止。 +Agent 在打开 `qemu:///system` 前注册并持续运行 libvirt 默认 event loop。每个连接 generation 都注册全域 lifecycle、`JOB_COMPLETED` 和 connection close callback;stream readable/writable 使用同一个 event loop。回调只发送有界、合并的核对唤醒,不保存回调中的临时 domain/connection 句柄,也不在事件线程中调用 libvirt 做核对或重连。 + +连接异常关闭后,Agent 在后台按有界退避建立新 generation 并重新注册全部回调;新连接完整就绪并发布后,才允许新请求使用并扫描 Journal 的全部可核对记录。旧 generation 会等其在途调用返回后再释放。Agent 启动执行同一扫描,并以低频周期扫描作为事件丢失的兜底。事件、completed job stats 和连接内存状态都不替代 Journal;Agent 或 libvirt daemon 重启导致事件缺失时,仍按原 `operation_id` 与宿主机状态核对,绝不重放原变更。 + 本表只描述已注册的 HTTP 路由;对应 libvirt 实现未落地或本机依赖不支持时,`features.*` 必须为 false 并返回 501,不得假成功。公网池的销售库存仍只在 PHP;Agent 仅在 VM 创建/换网时验证签名的 pool key、CIDR、IP、gateway、DNS 和面板指定的网络接入。网络值只接受 `bridge:<名称>` 或 `network:<名称>`;名称以字母或数字开头,后续仅允许字母、数字、点、下划线和短横线,bridge 为 1~15 字符,network 为 1~64 字符。Agent 验证并使用宿主机已存在且活动的网络,不创建或修改系统网络。 ## operation_id、指纹与串行 @@ -164,7 +168,7 @@ SHA256(UPPERCASE_METHOD + "\n" + EXACT_TARGET + "\n" + EXACT_BODY_BYTES) Agent 在调用 libvirt 前将 `operation_id`、VM UUID/资源 ID、动作、指纹、是否破坏性和状态原子持久化。状态为 `reserved`、`dispatching`、`succeeded`、`failed`、`uncertain`。nonce TTL 至少为签名时钟偏差的两倍,到期边界仍保留,避免未来时间戳签名仍有效时释放 nonce。 -持久层是 bbolt 数据库;每次状态变化只更新对应记录,nonce 由独立的到期时间索引清理,不扫描或重写完整 operation 历史。独立 inflight 索引使 Agent 启动只解码待恢复操作,不遍历永久 tombstone。终态结果和 expected-state 详情保留 90 天后压缩,释放页面供后续事务复用;`operation_id`、请求指纹、资源、动作和终态永久保留为 tombstone,任何清理都不得让旧的破坏性标识重新可用。`update-settings`、`change-network`、网络 suspend/resume、服务 suspend/resume 与 `apply-firewall` 在同一派发事务中额外保存无密钥 expected-state(只含 transition id、公开网络意图、解析后的本机网络 source、布尔状态或防火墙摘要;绝不保存密码、SSH key、token 或原始正文)。设置/网络/防火墙写入相同的宿主机 transition marker;重启后的 reconcile 只读比较 marker 与实际状态,换网还必须匹配持久化的 bridge/libvirt-network source,完整匹配才成功,明确未提交或半提交则失败并要求前向处置,绝不自动重放。 +持久层是 bbolt 数据库;每次状态变化只更新对应记录,nonce 由独立的到期时间索引清理,不扫描或重写完整 operation 历史。inflight 索引只负责把进程中断时的 `reserved`/`dispatching` 恢复为 `not_dispatched`/`uncertain`;独立 recovery 索引只列出可核对记录,因此启动、重连和事件唤醒都不遍历永久 tombstone。已完成且不再需要核对的结果和 expected-state 详情保留 90 天后压缩,释放页面供后续事务复用;仍可核对的记录必须保留 expected-state、结果和资源租约元数据,直到取得终态。`operation_id`、请求指纹、资源、动作和终态永久保留为 tombstone,任何清理都不得让旧的破坏性标识重新可用。`update-settings`、`change-network`、网络 suspend/resume、服务 suspend/resume 与 `apply-firewall` 在同一派发事务中额外保存无密钥 expected-state(只含 transition id、公开网络意图、解析后的本机网络 source、布尔状态或防火墙摘要;绝不保存密码、SSH key、token 或原始正文)。设置/网络/防火墙写入相同的宿主机 transition marker;重启后的 reconcile 只读比较 marker 与实际状态,换网还必须匹配持久化的 bridge/libvirt-network source,完整匹配才成功,明确未提交或半提交则失败并要求前向处置,绝不自动重放。 - 首次请求保留标识并写盘;同标识同指纹返回已有或进行中的记录。 - 同标识不同指纹返回 409,绝不执行第二个请求。 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index e857030b..9e7895af 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -127,7 +127,7 @@ Agent 的 capabilities 响应仍使用 `network_selectors` 字段,但它是自 Agent 的 systemd 单元保持 `ProtectSystem=strict`。`journal_path` 必须位于已授权的 `/var/lib/virt-agent`;若确需另一个状态目录,应由运维通过 systemd drop-in 精确增加对应 `ReadWritePaths`。存储池内容仍只由独立的 libvirt/QEMU 服务通过 `qemu:///system` 管理,不能把受管存储池目录授权给 Agent。删除回查需要清理域定义消失后残留的 UUID 专属 VARS 文件,因此样例 unit 仅额外授权标准 `/var/lib/libvirt/qemu/nvram`;若 `nvram_directory` 使用其他路径,必须在 drop-in 中用该实际目录替换,并保持授权范围只覆盖 NVRAM 目录。capabilities 只确认该目录存在,不冒充对 systemd mount namespace 写权限的验证;正式上线仍须用授权的创建/删除验收确认权限。 -`journal_path` 是 bbolt 数据库,不是可编辑 JSON;同目录的 `.traffic.json` 是流量核对状态,两者都必须纳入一致性备份和恢复。存储映射由 libvirt 的持久 pool 定义保存,没有 `.storage.json`。nonce 通过独立到期索引清理,不会重写 operation 历史;到期时刻本身仍视为占用,只清理严格早于当前时刻的 nonce。独立 inflight 索引使启动只恢复 `reserved`/`dispatching`,不会扫描永久历史。终态结果详情保留 90 天后压缩,释放的 bbolt 页面会被后续写入复用,但 operation_id、请求指纹、资源、动作和终态会永久保留为防重放 tombstone。禁止以“清理历史”为由删除这些 tombstone。当前为全新 Go Agent 格式,不导入旧 Python 数据或早期 JSON 原型 journal。 +`journal_path` 是 bbolt 数据库,不是可编辑 JSON;同目录的 `.traffic.json` 是流量核对状态,两者都必须纳入一致性备份和恢复。存储映射由 libvirt 的持久 pool 定义保存,没有 `.storage.json`。nonce 通过独立到期索引清理,不会重写 operation 历史;到期时刻本身仍视为占用,只清理严格早于当前时刻的 nonce。inflight 索引只恢复进程中断时的 `reserved`/`dispatching`,recovery 索引保存所有仍可核对的 operation;正常启动、libvirt 重连和事件唤醒只读取这些索引,不扫描永久历史。从 journal schema v2 首次升级到 v3 时会在一个 bbolt 事务内扫描现有 operation 一次以回填 recovery 索引,完成后不再重复全历史扫描。已完成且不再需要核对的结果详情保留 90 天后压缩;仍待核对的记录保留 expected-state、结果和资源租约元数据,直到取得终态。operation_id、请求指纹、资源、动作和终态会永久保留为防重放 tombstone,禁止以“清理历史”为由删除。当前为全新 Go Agent 格式,不导入旧 Python 数据或早期 JSON 原型 journal。 配置必须存在、可读、是单个严格 JSON 对象并通过内容校验。Agent **不会**检查文件 owner、group 或 mode,也不会因 `0600`、`0640`、`0644` 等可读权限差异拒绝启动;文档中的收紧权限只是运维建议。操作系统本身无权读取、JSON 损坏、字段未知或配置值无效仍会明确启动失败;这些是可用性/内容错误,不是权限位审计。旧的 `listen_address`、`shared_secret`、`tls_cert_file` 和 `tls_key_file` 会作为未知字段被拒绝;Agent 不启动入站监听。 @@ -322,6 +322,7 @@ location = /agent/console { - Q35 UEFI 启动且 Secure Boot 关闭;不同宿主机固件路径通过 capabilities 选择。 - 云镜像启动脚本只执行一次,并覆盖成功、非零退出、重启及 Agent 断线。 - 同一 `operation_id` 重试不重复创建/删除;派发后丢失响应进入 `uncertain` 并可核对。 +- 在明确授权的非生产宿主机重启实际启用的 `virtqemud` 或 `libvirtd`,确认 Agent 检测连接关闭、后台重连并重新注册 lifecycle、`JOB_COMPLETED` 和 close callback;健康检查恢复后只按原 `operation_id` 扫描并核对,不重放变更。再重启 Agent,确认同一批未完成 operation 会从 Journal recovery 索引重新扫描。 - 并发购买不超卖 CPU/内存/磁盘/IP;同一 VM 变更串行。 - RX+TX 额度用尽只断网;下一账期恢复;最终到期只断网;三天宽限结束后删除定义与面板受管磁盘。 - MariaDB、Redis、SMTP、支付回调、大文件上传、在线更新恢复点和 PHP-FPM 重启。 diff --git a/docs/INTEGRATION_PLAN.md b/docs/INTEGRATION_PLAN.md index ad230462..38c8f734 100644 --- a/docs/INTEGRATION_PLAN.md +++ b/docs/INTEGRATION_PLAN.md @@ -70,6 +70,8 @@ Unix 时间戳 4. Agent 明确失败且确认没有副作用时,PHP 才能执行相应前向补偿。发送后断线、超时或进程中断一律进入 `uncertain`。 5. `uncertain` 只能结合 Agent 操作记录和按 UUID 观察到的宿主机状态对账。运行中状态不证明先前关机/重启未执行;破坏性操作不得自动重放或直接退款。 +Agent 在连接前注册并持续运行 libvirt event loop;每个连接 generation 注册 lifecycle、`JOB_COMPLETED` 和 close callback。事件只合并唤醒核对,不能作为操作结果证据。Agent 启动或 libvirt 重连成功后从 Journal recovery 索引扫描未完成 operation;即使事件和 completed job stats 因 Agent/libvirt daemon 重启丢失,也只按原 `operation_id` 和宿主机现状核对,不重新提交原请求。 + 取消尚未派发的 PHP 任务只会阻止后续领取。已经跨过 Agent 派发边界的任务不能通过删除面板记录来假定撤销。 ## 虚拟机、镜像与启动脚本 @@ -104,4 +106,4 @@ helper 只允许干净工作树 fast-forward,按持久检查点停止服务、 ## 上线边界 -生产前必须实测 Agent WSS/HMAC、防重放、断线后对账、宿主机重启、UEFI、cloud-init、快照/救援/重装、反向控制台撤销、流量跨 QEMU 重启、到期断网、宽限删除、HTTPS 镜像安全和并发购买。测试不得对未明确授权的宿主机做创建、写配置、改网络或修改虚拟机操作。 +生产前必须实测 Agent WSS/HMAC、防重放、断线后对账、Agent 与实际 libvirt daemon 分别重启后的连接恢复和 Journal 扫描、宿主机重启、UEFI、cloud-init、快照/救援/重装、反向控制台撤销、流量跨 QEMU 重启、到期断网、宽限删除、HTTPS 镜像安全和并发购买。测试不得对未明确授权的宿主机做创建、写配置、改网络或修改虚拟机操作。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 17c92104..72bcc4ec 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -24,7 +24,7 @@ go test ./... go test -race ./... ``` -Go 测试至少覆盖:配置内容校验但不检查 owner/mode、HTTPS `panel_url`、数字 Agent ID、旧 listener/TLS 字段被拒绝、WSS 子协议与认证、严格控制帧、HMAC 正文摘要、完整时间窗(含未来偏差与到期边界)、nonce 防重放、不同请求复用 `operation_id`、多资源键严格串行、不相交资源并行、重 I/O 容量限制、派发前锁等待超时、派发后超时仍保留租约、派发前/后故障注入、仅按 inflight 索引重启恢复、超过 16 MiB 的长期 journal 重启、nonce 到期索引清理、operation tombstone 保留、五类 expected-state/transition marker 只读核对、换网 source 反例、libvirt typed error、UEFI XML、启动脚本一次性 runner 和反向控制台。不带 tag 的 portable 测试只能验证缺失 adapter 时的明确失败;生产交付必须通过 `-tags libvirt` 的编译和测试。PHP 契约测试还必须证明 PHP→Gateway 只使用字面 loopback 固定路径,不再存储或请求宿主机 endpoint。 +Go 测试至少覆盖:配置内容校验但不检查 owner/mode、HTTPS `panel_url`、数字 Agent ID、旧 listener/TLS 字段被拒绝、WSS 子协议与认证、严格控制帧、HMAC 正文摘要、完整时间窗(含未来偏差与到期边界)、nonce 防重放、不同请求复用 `operation_id`、多资源键严格串行、不相交资源并行、重 I/O 容量限制、派发前锁等待超时、派发后超时仍保留租约、派发前/后故障注入、inflight 与 recovery 索引重启恢复、长期待核对记录不被压缩、超过 16 MiB 的永久 journal 重启、nonce 到期索引清理、operation tombstone 保留、五类 expected-state/transition marker 只读核对、换网 source 反例、libvirt typed error、UEFI XML、启动脚本一次性 runner 和反向控制台。libvirt 定向测试还必须覆盖 event loop 早于连接、每个 generation 注册与逆序释放 lifecycle/`JOB_COMPLETED`/close callback、回调非阻塞合并、注册期间断线、旧 generation 延迟回调、新旧 session 租约切换、退避重连与关停,以及启动/重连 recovery 扫描不会重放 mutation。`JOB_COMPLETED` 只能提前唤醒 `GetJobStats`,没有事件时仍须靠轮询确认终态。不带 tag 的 portable 测试只能验证缺失 adapter 时的明确失败;生产交付必须通过 `-tags libvirt` 的编译和测试。PHP 契约测试还必须证明 PHP→Gateway 只使用字面 loopback 固定路径,不再存储或请求宿主机 endpoint。 PHP: @@ -88,9 +88,10 @@ test ! -e gateway/cmd/virt-cloud-console-relay 1. Agent 收到请求前失败:PHP 可按同一标识安全重试。 2. Agent 持久写入派发边界后、调用 libvirt 前失败:恢复后仍使用原 `operation_id`,不得新建操作。 3. libvirt 调用已返回但响应丢失:任务进入 `uncertain`,通过 UUID 和本地操作日志核对。 -4. Agent 或面板 worker 重启:操作日志、PHP 租约和 reservation 不丢失。 +4. Agent 或面板 worker 重启:操作日志、PHP 租约和 reservation 不丢失;Agent 启动扫描 recovery 索引并核对所有未完成 operation。 5. 同一 `operation_id` + 相同摘要:返回既有结果;同一标识 + 不同摘要:409 拒绝。 6. 同一 VM 或共享卷、过滤器、目录资源的并发变更:严格串行;不相交资源可在容量上限内并行。锁或重 I/O 槽等待超时必须保持 `not_dispatched`,派发后 HTTP 超时必须继续持有全部租约直到真实调用返回。 +7. 在授权的非生产环境分别中断并重启实际使用的 libvirt daemon 与 Agent:close callback 只唤醒后台重连,重连后重新注册全部事件并扫描 Journal;旧连接的迟到事件不能影响新连接,扫描不得重放原 mutation。 运行中状态不能证明关机/重启未发生;持久域消失不能在未核对磁盘清单时自动视为完整删除。任何未知删除都不得释放 IP、库存或退款。 @@ -153,6 +154,9 @@ systemd-analyze: read-only host capability check: authorized host mutation test: browser/payment/SMTP test: +event/reconnect unit tests: +authorized libvirt daemon restart test: +Agent restart recovery scan test: known skips or risks: ``` @@ -484,3 +488,17 @@ artifact inspection: stripped ELF 64-bit x86-64;revision=118a5957a854114521e9d build path / key marker scan: 未发现本机用户路径、临时构建目录、私钥标记或项目 Git 远端;产物先在临时目录构建和验证,再更新 release live deployment: 未执行;产物尚未在真实 Linux 宿主机运行,未替换线上 Agent 或重启服务 ``` + +### 2026-10-18 libvirt 连接、事件与 Journal 恢复 + +```text +source: 本记录所在 go 分支源码提交,基于 a73c269;Go 1.27.1、libvirt 12.8.0,macOS arm64 +connection generations: 默认 event loop 在首次连接前注册并持续运行;每个 qemu:///system 连接代注册 connection close、全域 lifecycle 和 JOB_COMPLETED;断线后后台退避重连并重新注册;新代完整发布后接收新调用,旧代等全部在途调用真实返回后再逆序注销 callback 和关闭 +event semantics: callback 仅做代际校验和有界合并唤醒;JOB_COMPLETED 只促使快照等待器重新读取 GetJobStats,500ms 轮询保留;connection close 同时唤醒 stream/job waiter;事件、completed job stats 和临时连接状态均不作为操作结果证据 +recovery: Journal schema v2 自动事务升级到 v3,新增 recovery 索引;启动、成功重连、lifecycle/JOB_COMPLETED/close 事件及 5 分钟周期兜底均扫描可核对 operation;每项默认 30 秒,串行取得原资源租约,只读核对或继续同 operation_id 的幂等 cleanup tail,绝不重放原 mutation;未解决记录继续保留索引及证据,不受 90 天详情压缩影响 +dispatch result: 无可用 session 且 backend 明确未调用时返回确定的 libvirt_unavailable;真实调用后遇到 RPC、INVALID_CONN 或 NO_CONNECT 时 mutation 保持 uncertain 并触发重连 +validation: Agent go mod verify、go mod tidy -diff、gofmt 和 git diff --check 通过;默认、libvirt、libvirt+libvirt_dlopen 三套 go test 通过;默认与 libvirt+libvirt_dlopen 全量 race 通过;默认与 libvirt+libvirt_dlopen vet 通过;operation/hypervisor 定向测试覆盖 v2→v3 迁移、恢复索引、关停、事件合并、每项超时、连接代切换、迟到 callback、注册清理顺序、JOB_COMPLETED 非证据语义和 AbortJob 终态确认;macOS 链接器仅报告重复 -ldl/-lvirt warning +upgrade: 这是 Agent 本地 journal 的自动升级,不需要修改面板数据库;首次打开 v2 journal 会扫描一次现有 operation 回填 recovery 索引,后续启动只读取 inflight/recovery 索引 +known limits: 尚未在真实 Debian/libvirt 宿主机执行 virtqemud/libvirtd 重启、Agent 重启和 callback 重注册验收;libvirt-go v1.12007.0 的 stream callback free-hook 限制仍与第二阶段记录相同 +live deployment: 未执行;未替换线上 Agent、重启服务、修改宿主机配置或操作真实 VM +```