You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/cmd/virt-agent/main_test.go

511 lines
18 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/worker"
)
type fakeWorkerStateCleaner struct {
mu sync.Mutex
calls []string
failOnce bool
}
type fakeRecoveredWorkerState struct {
fakeWorkerStateCleaner
receipts []worker.Response
terminals []worker.TerminalEvidence
receiptErr error
terminalErr error
}
func TestPublicGuardPreflightModeIsIsolatedFromOperationWorkerMode(t *testing.T) {
for name, test := range map[string]struct {
preflight bool
worker bool
journal string
receipt string
wantErr bool
}{
"ordinary": {},
"worker": {worker: true, journal: "/tmp/journal.db", receipt: "/tmp/completion.json"},
"preflight": {preflight: true},
"preflight worker": {preflight: true, worker: true, wantErr: true},
"preflight worker paths": {preflight: true, journal: "/tmp/journal.db", wantErr: true},
"paths without worker mode": {journal: "/tmp/journal.db", receipt: "/tmp/completion.json", wantErr: true},
} {
t.Run(name, func(t *testing.T) {
err := validateRuntimeMode(test.preflight, test.worker, test.journal, test.receipt)
if (err != nil) != test.wantErr {
t.Fatalf("validate runtime mode error = %v, wantErr=%v", err, test.wantErr)
}
})
}
}
func TestPublicGuardPreflightSourceCannotOpenRuntimeSubsystems(t *testing.T) {
preflightPath := filepath.Join("public_guard_preflight.go")
raw, err := os.ReadFile(preflightPath)
if err != nil {
t.Fatal(err)
}
source := string(raw)
for _, forbidden := range []string{
`"net/http"`, `agent/operation`, `agent/hypervisor`, `agent/control`,
`operation.Open`, `hypervisor.Open`, `control.NewClient`, `http.`,
} {
if strings.Contains(source, forbidden) {
t.Fatalf("preflight source reaches forbidden runtime subsystem %q", forbidden)
}
}
mainRaw, err := os.ReadFile("main.go")
if err != nil {
t.Fatal(err)
}
mainSource := string(mainRaw)
branch := strings.Index(mainSource, "if *publicGuardPreflight {")
journal := strings.Index(mainSource, "operation.OpenWithHeavyIOConcurrency")
if branch < 0 || journal < 0 || branch >= journal {
t.Fatalf("preflight branch must return before Journal initialization: branch=%d journal=%d", branch, journal)
}
}
func TestIsolatedWorkerLifetimeAllowsPortableBackupDeadline(t *testing.T) {
for _, path := range []string{
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/backups",
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/restore-backup",
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/recover-backup",
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/peer-migration/prepare",
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/peer-migration/receive",
} {
if got := isolatedWorkerLifetime("POST", path); got != 24*time.Hour+30*time.Second {
t.Fatalf("backup worker lifetime for %q = %s", path, got)
}
}
for _, test := range []struct {
method string
path string
}{
{method: "GET", path: "/v1/vms/123e4567-e89b-42d3-a456-426614174000/backups"},
{method: "POST", path: "/v1/vms/123e4567-e89b-42d3-a456-426614174000/backups/extra"},
{method: "POST", path: "/v1/vms/123e4567-e89b-42d3-a456-426614174000/settings"},
{method: "POST", path: "/v1/vms/123e4567-e89b-42d3-a456-426614174000/peer-migration/rollback"},
} {
if got := isolatedWorkerLifetime(test.method, test.path); got != 2*time.Hour+30*time.Second {
t.Fatalf("ordinary worker lifetime for %s %q = %s", test.method, test.path, got)
}
}
}
func TestRunControlStopsWhenPeerMigrationListenerFails(t *testing.T) {
journalFailures := make(chan error)
peerFailures := make(chan error, 1)
peerFailures <- errors.New("listener failed")
close(peerFailures)
runStopped := make(chan struct{})
err := runControlUntilStopped(
context.Background(), journalFailures,
func(ctx context.Context) error {
<-ctx.Done()
close(runStopped)
return ctx.Err()
},
peerFailures,
)
if err == nil || !strings.Contains(err.Error(), "PeerMigration listener failed") {
t.Fatalf("unexpected listener failure result: %v", err)
}
select {
case <-runStopped:
case <-time.After(time.Second):
t.Fatal("control channel was not canceled after listener failure")
}
}
func reopenDispatchingJournal(t *testing.T, path string, spec operation.Spec) *operation.Journal {
t.Helper()
j, err := operation.Open(path)
if err != nil {
t.Fatal(err)
}
started := make(chan struct{})
release := make(chan struct{})
_, replayed, done, err := j.ExecuteBackground(
context.Background(), context.Background(), time.Minute, spec,
func(context.Context) (json.RawMessage, error) {
close(started)
<-release
return json.RawMessage(`{"state":"running"}`), nil
},
)
if err != nil || replayed || done == nil {
_ = j.Close()
t.Fatalf("seed dispatching operation replayed=%v done=%v err=%v", replayed, done != nil, err)
}
<-started
if err := j.Close(); err != nil {
t.Fatal(err)
}
close(release)
<-done
reopened, err := operation.Open(path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = reopened.Close() })
record, err := reopened.Get(spec.OperationID)
if err != nil || record.State != operation.StateDispatching {
t.Fatalf("reopened dispatching record = %#v err=%v", record, err)
}
return reopened
}
func writeLegacyTerminalWorkerJournal(t *testing.T, operationDir string, spec operation.Spec) {
t.Helper()
if err := os.Mkdir(operationDir, 0o700); err != nil {
t.Fatal(err)
}
child, err := operation.Open(filepath.Join(operationDir, "journal.db"))
if err != nil {
t.Fatal(err)
}
if _, _, err := child.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"state":"running"}`), nil
}); err != nil {
_ = child.Close()
t.Fatal(err)
}
if err := child.Close(); err != nil {
t.Fatal(err)
}
}
func (s *fakeRecoveredWorkerState) PendingReceipts() ([]worker.Response, error) {
return append([]worker.Response(nil), s.receipts...), s.receiptErr
}
func (s *fakeRecoveredWorkerState) PendingJournalTerminals() ([]worker.TerminalEvidence, error) {
return append([]worker.TerminalEvidence(nil), s.terminals...), s.terminalErr
}
func (c *fakeWorkerStateCleaner) Cleanup(operationID string) error {
c.mu.Lock()
defer c.mu.Unlock()
c.calls = append(c.calls, operationID)
if c.failOnce {
c.failOnce = false
return errors.New("injected cleanup failure")
}
return nil
}
func (c *fakeWorkerStateCleaner) CleanupCompleted(operationID, _ string) error {
return c.Cleanup(operationID)
}
func terminalOperation(t *testing.T, journal *operation.Journal, operationID string) operation.CompletionReceipt {
t.Helper()
spec := operation.Spec{
OperationID: operationID,
ResourceID: "vm-completion-cleanup",
Action: "delete-vm",
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/vm-completion-cleanup", []byte(`{}`)),
Destructive: true,
}
if _, _, err := journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"deleted":true}`), nil
}); err != nil {
t.Fatal(err)
}
claimed, err := journal.ClaimPendingCompletions(1)
if err != nil || len(claimed) != 1 {
t.Fatalf("claim completion = %#v err=%v", claimed, err)
}
return claimed[0]
}
func TestCompletionAckDoesNotDependOnDisposableWorkerCleanup(t *testing.T) {
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
defer journal.Close()
receipt := terminalOperation(t, journal, "operation-ack-cleanup")
cleaner := &fakeWorkerStateCleaner{failOnce: true}
outbox := &workerCompletionOutbox{journal: journal, cleaner: cleaner}
if err := outbox.AcknowledgeCompletion(receipt.EventID); err != nil {
t.Fatal(err)
}
if pending, err := journal.PendingCompletionOperationIDs(); err != nil || len(pending) != 0 {
t.Fatalf("acked completion remains pending: %#v err=%v", pending, err)
}
record, err := journal.Get(receipt.OperationID)
if err != nil || record.State != operation.StateSucceeded {
t.Fatalf("ACK changed permanent operation tombstone: %#v err=%v", record, err)
}
cleaner.mu.Lock()
calls := append([]string(nil), cleaner.calls...)
cleaner.mu.Unlock()
if len(calls) != 1 || calls[0] != receipt.OperationID {
t.Fatalf("cleanup calls = %#v", calls)
}
if err := outbox.AcknowledgeCompletion(receipt.EventID); err != nil {
t.Fatalf("duplicate ACK was not idempotent: %v", err)
}
}
func TestRecoveredWorkerCleanupDoesNotDeleteMainCompletion(t *testing.T) {
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
defer journal.Close()
receipt := terminalOperation(t, journal, "operation-recovered-helper-cleanup")
state := &fakeRecoveredWorkerState{}
if err := cleanupRecoveredWorkerState(state, map[string]struct{}{receipt.OperationID: {}}); err != nil {
t.Fatal(err)
}
state.mu.Lock()
calls := append([]string(nil), state.calls...)
state.mu.Unlock()
if len(calls) != 1 || calls[0] != receipt.OperationID {
t.Fatalf("recovered worker cleanup calls = %#v", calls)
}
if pending, err := journal.PendingCompletionOperationIDs(); err != nil {
t.Fatal(err)
} else if _, exists := pending[receipt.OperationID]; !exists {
t.Fatal("disposable helper cleanup deleted the main completion outbox")
}
record, err := journal.Get(receipt.OperationID)
if err != nil || record.State != operation.StateSucceeded {
t.Fatalf("helper cleanup changed operation tombstone: %#v err=%v", record, err)
}
}
func TestRecoveryPreservesUnadoptedWorkerEvidence(t *testing.T) {
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
defer journal.Close()
receipt := terminalOperation(t, journal, "operation-unadopted-evidence")
state := &fakeRecoveredWorkerState{terminals: []worker.TerminalEvidence{{
Record: operation.Record{
OperationID: receipt.OperationID, ResourceID: receipt.ResourceID,
Action: "power-reset", Fingerprint: receipt.RequestFingerprint,
State: operation.StateSucceeded,
},
}}}
if err := recoverInterruptedWorkerState(journal, state); err != nil {
t.Fatal(err)
}
state.mu.Lock()
calls := append([]string(nil), state.calls...)
state.mu.Unlock()
if len(calls) != 0 {
t.Fatalf("unadopted worker evidence was cleaned: %#v", calls)
}
}
func TestRecoveryScanErrorStopsBeforeWorkerCleanup(t *testing.T) {
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
defer journal.Close()
state := &fakeRecoveredWorkerState{terminalErr: errors.New("worker journal is temporarily locked")}
if err := recoverInterruptedWorkerState(journal, state); err == nil || !strings.Contains(err.Error(), "temporarily locked") {
t.Fatalf("recovery scan error = %v", err)
}
state.mu.Lock()
calls := append([]string(nil), state.calls...)
state.mu.Unlock()
if len(calls) != 0 {
t.Fatalf("cleanup ran after incomplete recovery scan: %#v", calls)
}
}
func TestRecoveryAdoptsTerminalChildJournalBeforeCompletionPublication(t *testing.T) {
temp := t.TempDir()
resource := "123e4567-e89b-42d3-a456-426614174010"
target := "/v1/vms/" + resource + "/power"
body := []byte(`{"operation_id":"operation-child-terminal","action":"reboot"}`)
spec := operation.Spec{
OperationID: "operation-child-terminal", ResourceID: resource, Action: "power-reboot",
Fingerprint: operation.Fingerprint("POST", target, body), Destructive: true,
}
journal := reopenDispatchingJournal(t, filepath.Join(temp, "main.db"), spec)
executor, err := worker.NewSubprocessExecutor(
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
)
if err != nil {
t.Fatal(err)
}
operationDir := filepath.Join(temp, "workers", spec.OperationID)
writeLegacyTerminalWorkerJournal(t, operationDir, spec)
if err := recoverInterruptedWorkerState(journal, executor); err != nil {
t.Fatal(err)
}
record, err := journal.Get(spec.OperationID)
if err != nil || record.State != operation.StateSucceeded || string(record.Result) != `{"state":"running"}` ||
record.EffectMayHaveApplied || record.ExecutionFenced {
t.Fatalf("adopted terminal record = %#v err=%v", record, err)
}
if _, err := os.Stat(operationDir); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("adopted worker directory was not cleaned: %v", err)
}
}
func TestCorruptRecoveryIdentityDoesNotManufactureSuccessOrBlockStartup(t *testing.T) {
temp := t.TempDir()
resource := "123e4567-e89b-42d3-a456-426614174011"
target := "/v1/vms/" + resource + "/power"
body := []byte(`{"operation_id":"operation-corrupt-recovery","action":"reset"}`)
spec := operation.Spec{
OperationID: "operation-corrupt-recovery", ResourceID: resource, Action: "power-reset",
Fingerprint: operation.Fingerprint("POST", target, body), Destructive: true,
}
journal := reopenDispatchingJournal(t, filepath.Join(temp, "main.db"), spec)
executor, err := worker.NewSubprocessExecutor(
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
)
if err != nil {
t.Fatal(err)
}
operationDir := filepath.Join(temp, "workers", spec.OperationID)
writeLegacyTerminalWorkerJournal(t, operationDir, spec)
if err := os.WriteFile(filepath.Join(operationDir, "request-identity.json"), []byte(`{"operation_id":"wrong"}`), 0o600); err != nil {
t.Fatal(err)
}
if err := recoverInterruptedWorkerState(journal, executor); err != nil {
t.Fatalf("permanently invalid evidence blocked startup: %v", err)
}
record, err := journal.Get(spec.OperationID)
if err != nil || record.State != operation.StateFailed || record.Error == nil ||
record.Error.Code != worker.ResultUnconfirmed || !record.EffectMayHaveApplied || !record.ExecutionFenced {
t.Fatalf("corrupt evidence terminal = %#v err=%v", record, err)
}
if _, err := os.Stat(operationDir); err != nil {
t.Fatalf("corrupt worker evidence was destroyed: %v", err)
}
outbox := &workerCompletionOutbox{journal: journal, cleaner: executor}
claimed, err := outbox.ClaimPendingCompletions(1, nil)
if err != nil || len(claimed) != 1 || claimed[0].OperationID != spec.OperationID {
t.Fatalf("claim unconfirmed completion = %#v err=%v", claimed, err)
}
if _, err := os.Stat(operationDir); err != nil {
t.Fatalf("completion claim destroyed unadopted worker evidence: %v", err)
}
if err := outbox.AcknowledgeCompletion(claimed[0].EventID); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(operationDir); err != nil {
t.Fatalf("completion ACK destroyed unadopted worker evidence: %v", err)
}
}
func TestTransientRecoveryScanErrorLeavesMainDispatchingForRetry(t *testing.T) {
temp := t.TempDir()
resource := "123e4567-e89b-42d3-a456-426614174012"
spec := operation.Spec{
OperationID: "operation-transient-scan", ResourceID: resource, Action: "power-reboot",
Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+resource+"/power", []byte(`{"action":"reboot"}`)),
Destructive: true,
}
journal := reopenDispatchingJournal(t, filepath.Join(temp, "main.db"), spec)
state := &fakeRecoveredWorkerState{terminalErr: errors.New("worker journal is temporarily locked")}
if err := recoverInterruptedWorkerState(journal, state); err == nil {
t.Fatal("transient recovery scan error was ignored")
}
record, err := journal.Get(spec.OperationID)
if err != nil || record.State != operation.StateDispatching {
t.Fatalf("transient scan irreversibly terminalized main record = %#v err=%v", record, err)
}
state.mu.Lock()
calls := append([]string(nil), state.calls...)
state.mu.Unlock()
if len(calls) != 0 {
t.Fatalf("worker evidence was cleaned after transient scan error: %#v", calls)
}
}
func TestCompletionRemainsDeliverableUntilAcknowledged(t *testing.T) {
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
defer journal.Close()
now := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC)
journal.SetClockForTesting(func() time.Time { return now })
receipt := terminalOperation(t, journal, "operation-durable-delivery")
now = now.Add(25 * time.Hour)
cleaner := &fakeWorkerStateCleaner{failOnce: true}
outbox := &workerCompletionOutbox{journal: journal, cleaner: cleaner}
claimed, err := outbox.ClaimPendingCompletions(1, nil)
if err != nil || len(claimed) != 1 || claimed[0].EventID != receipt.EventID {
t.Fatalf("25-hour claim = %#v err=%v", claimed, err)
}
now = now.Add(365 * 24 * time.Hour)
claimed, err = outbox.ClaimPendingCompletions(1, nil)
if err != nil || len(claimed) != 1 || claimed[0].EventID != receipt.EventID {
t.Fatalf("long-lived claim = %#v err=%v", claimed, err)
}
cleaner.mu.Lock()
calls := append([]string(nil), cleaner.calls...)
cleaner.mu.Unlock()
if len(calls) != 2 || calls[0] != receipt.OperationID || calls[1] != receipt.OperationID {
t.Fatalf("delivery cleanup calls = %#v", calls)
}
if pending, err := journal.PendingCompletionOperationIDs(); err != nil {
t.Fatal(err)
} else if _, exists := pending[receipt.OperationID]; !exists {
t.Fatal("disposable delivery cleanup deleted the main completion outbox")
}
if err := outbox.AcknowledgeCompletion(receipt.EventID); err != nil {
t.Fatal(err)
}
cleaner.mu.Lock()
calls = append([]string(nil), cleaner.calls...)
cleaner.mu.Unlock()
if len(calls) != 3 || calls[2] != receipt.OperationID {
t.Fatalf("ACK cleanup calls = %#v", calls)
}
if pending, err := journal.PendingCompletionOperationIDs(); err != nil || len(pending) != 0 {
t.Fatalf("ACK left completion pending: %#v err=%v", pending, err)
}
}
func TestRunControlUntilStoppedCancelsOnBackgroundFinalizationFailure(t *testing.T) {
failures := make(chan error, 1)
failures <- errors.New("injected journal write failure")
stopped := make(chan struct{})
err := runControlUntilStopped(context.Background(), failures, func(ctx context.Context) error {
<-ctx.Done()
close(stopped)
return ctx.Err()
})
if err == nil || !strings.Contains(err.Error(), "durable background operation finalization failed") {
t.Fatalf("run result = %v", err)
}
select {
case <-stopped:
default:
t.Fatal("control channel was not stopped before restart")
}
}