//go:build libvirt package hypervisor import ( "context" "errors" "reflect" "testing" "time" "git.luoli.live/root/virt/agent/operation" "libvirt.org/go/libvirt" ) type snapshotGuestAgentFake struct { probeErr error responseTimeoutErr error freezeErr error thawErr error probeCalls int responseTimeoutCalls int freezeCalls int thawCalls int command string timeout libvirt.DomainQemuAgentCommandTimeout responseTimeout int callOrder []string } func (f *snapshotGuestAgentFake) QemuAgentCommand(command string, timeout libvirt.DomainQemuAgentCommandTimeout, _ uint32) (string, error) { f.probeCalls++ f.command = command f.timeout = timeout f.callOrder = append(f.callOrder, "ping") return `{"return":{}}`, f.probeErr } func (f *snapshotGuestAgentFake) AgentSetResponseTimeout(timeout int, _ uint32) error { f.responseTimeoutCalls++ f.responseTimeout = timeout f.callOrder = append(f.callOrder, "set-timeout") return f.responseTimeoutErr } func (f *snapshotGuestAgentFake) FSFreeze(_ []string, _ uint32) error { f.freezeCalls++ f.callOrder = append(f.callOrder, "freeze") return f.freezeErr } func (f *snapshotGuestAgentFake) FSThaw(_ []string, _ uint32) error { f.thawCalls++ f.callOrder = append(f.callOrder, "thaw") return f.thawErr } const testGuestAgentResponseTimeout = 15 func TestSnapshotConsistencyMatchesRequest(t *testing.T) { tests := []struct { requested string actual string allowed bool }{ {requested: "filesystem", actual: "filesystem", allowed: true}, {requested: "filesystem", actual: "crash", allowed: true}, {requested: "crash", actual: "crash", allowed: true}, {requested: "crash", actual: "filesystem", allowed: false}, {requested: "filesystem", actual: "unknown", allowed: false}, } for _, test := range tests { if got := snapshotConsistencyMatchesRequest(test.requested, test.actual); got != test.allowed { t.Fatalf("snapshot consistency %s -> %s allowed=%v, want %v", test.requested, test.actual, got, test.allowed) } } } func TestPrepareSnapshotConsistencySkipsProbeForExplicitCrash(t *testing.T) { agent := &snapshotGuestAgentFake{} actual, frozen, err := prepareSnapshotConsistency(agent, "crash", testGuestAgentResponseTimeout, true) if err != nil || actual != "crash" || frozen || agent.probeCalls != 0 || agent.responseTimeoutCalls != 0 || agent.freezeCalls != 0 { t.Fatalf("explicit crash mode unexpectedly used guest agent: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent) } } func TestPrepareSnapshotConsistencyFallsBackWhenGuestAgentIsUnavailable(t *testing.T) { for _, test := range []struct { name string code libvirt.ErrorNumber }{ {name: "unresponsive", code: libvirt.ERR_AGENT_UNRESPONSIVE}, {name: "not-configured", code: libvirt.ERR_ARGUMENT_UNSUPPORTED}, {name: "probe-timeout", code: libvirt.ERR_AGENT_COMMAND_TIMEOUT}, } { t.Run(test.name, func(t *testing.T) { agent := &snapshotGuestAgentFake{probeErr: libvirt.Error{Code: test.code}} actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true) if err != nil || actual != "crash" || frozen { t.Fatalf("unavailable guest agent did not fall back to crash consistency: actual=%q frozen=%v err=%v", actual, frozen, err) } if agent.probeCalls != 1 || agent.responseTimeoutCalls != 0 || agent.freezeCalls != 0 || agent.thawCalls != 0 || agent.command != snapshotGuestAgentPing || agent.timeout != 5 { t.Fatalf("unexpected guest agent fallback calls: %+v", agent) } }) } } func TestPrepareSnapshotConsistencyFallsBackWhenResponseTimeoutUnsupported(t *testing.T) { for _, code := range []libvirt.ErrorNumber{ libvirt.ERR_NO_SUPPORT, libvirt.ERR_OPERATION_UNSUPPORTED, libvirt.ERR_ARGUMENT_UNSUPPORTED, } { agent := &snapshotGuestAgentFake{responseTimeoutErr: libvirt.Error{Code: code}} actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true) if err != nil || actual != "crash" || frozen { t.Fatalf("unsupported response timeout did not fall back to crash consistency: code=%v actual=%q frozen=%v err=%v", code, actual, frozen, err) } if agent.responseTimeout != testGuestAgentResponseTimeout || agent.freezeCalls != 0 || !reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout"}) { t.Fatalf("unexpected unsupported-timeout call sequence: %+v", agent) } } } func TestGuestAgentResponseTimeoutVersionGate(t *testing.T) { for _, test := range []struct { name string clientVersion uint32 daemonVersion uint32 want bool }{ {name: "both-supported", clientVersion: 5_010_000, daemonVersion: 5_010_000, want: true}, {name: "newer-supported", clientVersion: 12_008_000, daemonVersion: 10_000_000, want: true}, {name: "client-too-old", clientVersion: 5_009_000, daemonVersion: 5_010_000}, {name: "daemon-too-old", clientVersion: 5_010_000, daemonVersion: 5_009_000}, {name: "unknown-client", daemonVersion: 5_010_000}, {name: "unknown-daemon", clientVersion: 5_010_000}, } { t.Run(test.name, func(t *testing.T) { if got := supportsGuestAgentResponseTimeout(test.clientVersion, test.daemonVersion); got != test.want { t.Fatalf("version gate=%v, want %v", got, test.want) } }) } } func TestPrepareSnapshotConsistencyFallsBackWhenResponseTimeoutVersionIsUnsupported(t *testing.T) { agent := &snapshotGuestAgentFake{} actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, false) if err != nil || actual != "crash" || frozen { t.Fatalf("unsupported response-timeout version did not fall back: actual=%q frozen=%v err=%v", actual, frozen, err) } if len(agent.callOrder) != 0 { t.Fatalf("unsupported response-timeout version used freeze/thaw APIs: %+v", agent) } } func TestPrepareSnapshotConsistencyRejectsResponseTimeoutInfrastructureFailure(t *testing.T) { agent := &snapshotGuestAgentFake{responseTimeoutErr: libvirt.Error{Code: libvirt.ERR_RPC}} actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true) if err == nil || actual != "" || frozen || agent.freezeCalls != 0 { t.Fatalf("response-timeout infrastructure failure was incorrectly downgraded: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent) } if !reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout"}) { t.Fatalf("response-timeout failure used the guest agent again: %+v", agent) } } func TestPrepareSnapshotConsistencyRejectsInfrastructureProbeFailure(t *testing.T) { agent := &snapshotGuestAgentFake{probeErr: libvirt.Error{Code: libvirt.ERR_RPC}} actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true) if err == nil || actual != "" || frozen || agent.freezeCalls != 0 || agent.thawCalls != 0 { t.Fatalf("infrastructure failure was incorrectly downgraded: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent) } } func TestPrepareSnapshotConsistencyFreezesResponsiveGuest(t *testing.T) { agent := &snapshotGuestAgentFake{} actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true) if err != nil || actual != "filesystem" || !frozen || agent.probeCalls != 1 || agent.responseTimeoutCalls != 1 || agent.freezeCalls != 1 || agent.thawCalls != 0 { t.Fatalf("responsive guest was not frozen: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent) } if agent.responseTimeout != testGuestAgentResponseTimeout || !reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout", "freeze"}) { t.Fatalf("guest-agent timeout was not applied before freeze: %+v", agent) } } func TestPrepareSnapshotConsistencyKeepsFreezeFailureDeterministicAfterThaw(t *testing.T) { agent := &snapshotGuestAgentFake{freezeErr: libvirt.Error{Code: libvirt.ERR_AGENT_COMMAND_FAILED}} actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true) var public *Error if err == nil || actual != "" || frozen || !errors.As(err, &public) || public.Code != "snapshot_filesystem_freeze_failed" || operation.IsUncertain(err) { t.Fatalf("freeze failure was not preserved as a deterministic error: actual=%q frozen=%v err=%v", actual, frozen, err) } if agent.thawCalls != 1 { t.Fatalf("freeze failure did not attempt a compensating thaw: %+v", agent) } if !reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout", "freeze", "thaw"}) { t.Fatalf("unexpected freeze failure call sequence: %+v", agent) } } func TestPrepareSnapshotConsistencyMarksFailedThawUncertain(t *testing.T) { agent := &snapshotGuestAgentFake{ freezeErr: libvirt.Error{Code: libvirt.ERR_AGENT_COMMAND_TIMEOUT}, thawErr: libvirt.Error{Code: libvirt.ERR_AGENT_UNRESPONSIVE}, } _, _, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true) if err == nil || !operation.IsUncertain(err) || agent.thawCalls != 1 { t.Fatalf("ambiguous freeze without confirmed thaw was not uncertain: err=%v calls=%+v", err, agent) } } type snapshotBackupJobFake struct { active []*libvirt.DomainJobInfo completed []*libvirt.DomainJobInfo activeErr error completedErr error abortErr error abortCalls int getJobStatsCalls []libvirt.DomainGetJobStatsFlags getJobStatsCall chan struct{} } func (f *snapshotBackupJobFake) GetJobStats(flags libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) { f.getJobStatsCalls = append(f.getJobStatsCalls, flags) if f.getJobStatsCall != nil { select { case f.getJobStatsCall <- struct{}{}: default: } } if flags == 0 { if f.activeErr != nil { return nil, f.activeErr } return nextSnapshotJobInfo(&f.active), nil } if flags == libvirt.DOMAIN_JOB_STATS_COMPLETED|libvirt.DOMAIN_JOB_STATS_KEEP_COMPLETED { if f.completedErr != nil { return nil, f.completedErr } return nextSnapshotJobInfo(&f.completed), nil } return nil, errors.New("unexpected job stats flags") } func (f *snapshotBackupJobFake) AbortJob() error { f.abortCalls++ return f.abortErr } func nextSnapshotJobInfo(queue *[]*libvirt.DomainJobInfo) *libvirt.DomainJobInfo { if len(*queue) == 0 { return &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_BOUNDED, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_BACKUP} } value := (*queue)[0] if len(*queue) > 1 { *queue = (*queue)[1:] } return value } func activeBackupJob() *libvirt.DomainJobInfo { return &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_BOUNDED, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_BACKUP} } func noActiveBackupJob() *libvirt.DomainJobInfo { return &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE} } func terminalBackupJob(kind libvirt.DomainJobType) *libvirt.DomainJobInfo { return &libvirt.DomainJobInfo{Type: kind, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_BACKUP} } func fastSnapshotBackupWaitTiming() snapshotBackupWaitTiming { return snapshotBackupWaitTiming{pollInterval: time.Millisecond, abortConfirmationTimeout: 30 * time.Millisecond} } func TestWaitForBackupTerminalConfirmsCancellationAfterAbort(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() job := &snapshotBackupJobFake{ active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()}, completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)}, } terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming()) if err != nil || terminal != libvirt.DOMAIN_JOB_CANCELLED || job.abortCalls != 1 { t.Fatalf("cancelled backup was not confirmed after abort: terminal=%v err=%v aborts=%d calls=%v", terminal, err, job.abortCalls, job.getJobStatsCalls) } } func TestWaitForBackupTerminalAcceptsCompletionRacingAbortError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() job := &snapshotBackupJobFake{ active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()}, completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)}, abortErr: libvirt.Error{Code: libvirt.ERR_OPERATION_INVALID}, } terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming()) if err != nil || terminal != libvirt.DOMAIN_JOB_COMPLETED || job.abortCalls != 1 { t.Fatalf("completion evidence did not win an abort race: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls) } } func TestWaitForBackupTerminalLeavesUnconfirmedAbortUncertain(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() job := &snapshotBackupJobFake{active: []*libvirt.DomainJobInfo{activeBackupJob()}} timing := snapshotBackupWaitTiming{pollInterval: time.Millisecond, abortConfirmationTimeout: 8 * time.Millisecond} terminal, err := waitForBackupTerminalWithTiming(ctx, job, timing) if terminal != libvirt.DOMAIN_JOB_NONE || err == nil || !operation.IsUncertain(err) || job.abortCalls != 1 { t.Fatalf("unconfirmed abort did not remain uncertain: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls) } var public *Error if !errors.As(err, &public) || public.Code != "snapshot_abort_unconfirmed" { t.Fatalf("unconfirmed abort lost its stable public code: %v", err) } } func TestWaitForBackupTerminalDoesNotAbortAlreadyCompletedBackupAfterCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() job := &snapshotBackupJobFake{active: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)}} terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming()) if err != nil || terminal != libvirt.DOMAIN_JOB_COMPLETED || job.abortCalls != 0 { t.Fatalf("completed backup was aborted: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls) } } func TestWaitForBackupTerminalDoesNotAbortForeignOrUnknownJob(t *testing.T) { for _, test := range []struct { name string job *libvirt.DomainJobInfo }{ { name: "foreign", job: &libvirt.DomainJobInfo{ Type: libvirt.DOMAIN_JOB_BOUNDED, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_MIGRATION_OUT, }, }, {name: "operation-unknown", job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_BOUNDED}}, } { t.Run(test.name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() job := &snapshotBackupJobFake{active: []*libvirt.DomainJobInfo{activeBackupJob(), test.job}} terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming()) if terminal != libvirt.DOMAIN_JOB_NONE || err == nil || !operation.IsUncertain(err) || job.abortCalls != 0 { t.Fatalf("foreign or unknown job was not protected: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls) } }) } } func TestInspectBackupTerminalUsesRetainedEvidenceAfterActiveQueryFailure(t *testing.T) { job := &snapshotBackupJobFake{ activeErr: libvirt.Error{Code: libvirt.ERR_RPC}, completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)}, } terminal, done, err := inspectBackupTerminal(job) if err != nil || !done || terminal != libvirt.DOMAIN_JOB_COMPLETED { t.Fatalf("retained terminal evidence was lost after active query failure: terminal=%v done=%v err=%v", terminal, done, err) } } func TestWaitForBackupTerminalDoesNotAbortNormalCompletion(t *testing.T) { job := &snapshotBackupJobFake{ active: []*libvirt.DomainJobInfo{activeBackupJob(), noActiveBackupJob()}, completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)}, } terminal, err := waitForBackupTerminalWithTiming(context.Background(), job, fastSnapshotBackupWaitTiming()) if err != nil || terminal != libvirt.DOMAIN_JOB_COMPLETED || job.abortCalls != 0 { t.Fatalf("normal completion unexpectedly aborted: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls) } } func TestWaitForBackupTerminalWakesOnJobCompletedEvent(t *testing.T) { calls := make(chan struct{}, 4) job := &snapshotBackupJobFake{ active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()}, completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)}, getJobStatsCall: calls, } timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second} type result struct { terminal libvirt.DomainJobType err error } done := make(chan result, 1) ctx, cancel := context.WithCancel(context.Background()) defer func() { cancel() libvirtJobEvents.notify() }() go func() { terminal, err := waitForBackupTerminalWithTiming(ctx, job, timing) done <- result{terminal: terminal, err: err} }() select { case <-calls: case <-time.After(time.Second): t.Fatal("backup waiter did not inspect the active job") } libvirtJobEvents.notify() select { case <-calls: case <-time.After(time.Second): t.Fatal("spurious JOB_COMPLETED event did not trigger GetJobStats") } select { case result := <-done: t.Fatalf("event was treated as terminal evidence: terminal=%v err=%v", result.terminal, result.err) case <-time.After(20 * time.Millisecond): } libvirtJobEvents.notify() select { case result := <-done: if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_COMPLETED { t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err) } case <-time.After(time.Second): t.Fatal("JOB_COMPLETED event did not wake backup waiter") } } func TestAbortBackupWaitWakesOnJobCompletedEvent(t *testing.T) { calls := make(chan struct{}, 2) job := &snapshotBackupJobFake{ active: []*libvirt.DomainJobInfo{ activeBackupJob(), activeBackupJob(), noActiveBackupJob(), }, completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)}, getJobStatsCall: calls, } timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second} type result struct { terminal libvirt.DomainJobType err error } done := make(chan result, 1) defer libvirtJobEvents.notify() go func() { terminal, err := abortBackupAndWaitForTerminal(job, context.DeadlineExceeded, timing) done <- result{terminal: terminal, err: err} }() for call := 0; call < 2; call++ { select { case <-calls: case <-time.After(time.Second): t.Fatal("abort waiter did not inspect the active job") } } libvirtJobEvents.notify() select { case result := <-done: if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_CANCELLED { t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err) } case <-time.After(time.Second): t.Fatal("JOB_COMPLETED event did not wake abort confirmation") } } func TestInspectBackupTerminalAcceptsAllBackupTerminalStates(t *testing.T) { for _, kind := range []libvirt.DomainJobType{ libvirt.DOMAIN_JOB_COMPLETED, libvirt.DOMAIN_JOB_FAILED, libvirt.DOMAIN_JOB_CANCELLED, } { job := &snapshotBackupJobFake{ active: []*libvirt.DomainJobInfo{noActiveBackupJob()}, completed: []*libvirt.DomainJobInfo{terminalBackupJob(kind)}, } terminal, done, err := inspectBackupTerminal(job) if err != nil || !done || terminal != kind { t.Fatalf("backup terminal state %v was not accepted: terminal=%v done=%v err=%v", kind, terminal, done, err) } } } func TestWaitForBackupTerminalRejectsForeignTerminalEvidence(t *testing.T) { job := &snapshotBackupJobFake{ active: []*libvirt.DomainJobInfo{noActiveBackupJob()}, completed: []*libvirt.DomainJobInfo{{ Type: libvirt.DOMAIN_JOB_COMPLETED, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_MIGRATION_OUT, }}, } _, err := waitForBackupTerminalWithTiming(context.Background(), job, fastSnapshotBackupWaitTiming()) if err == nil || !operation.IsUncertain(err) || job.abortCalls != 0 { t.Fatalf("foreign terminal evidence was accepted: err=%v aborts=%d", err, job.abortCalls) } } func TestAbortBackupAfterThawFailureWaitsForTerminalAndRemainsUncertain(t *testing.T) { thawErr := libvirt.Error{Code: libvirt.ERR_AGENT_COMMAND_TIMEOUT} job := &snapshotBackupJobFake{ active: []*libvirt.DomainJobInfo{activeBackupJob(), noActiveBackupJob()}, completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)}, } terminal, err := abortBackupAfterThawFailure(job, thawErr, fastSnapshotBackupWaitTiming()) if terminal != libvirt.DOMAIN_JOB_CANCELLED || err == nil || !operation.IsUncertain(err) || job.abortCalls != 1 { t.Fatalf("thaw failure did not wait for terminal cancellation: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls) } }