You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
514 lines
20 KiB
Go
514 lines
20 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
)
|
|
|
|
type snapshotGuestAgentFake struct {
|
|
probeErr error
|
|
responseTimeoutErr error
|
|
freezeErr error
|
|
thawErr error
|
|
probeCalls int
|
|
responseTimeoutCalls int
|
|
freezeCalls int
|
|
thawCalls int
|
|
command string
|
|
timeout libvirt.DomainQemuAgentCommandTimeout
|
|
responseTimeout int
|
|
callOrder []string
|
|
}
|
|
|
|
func (f *snapshotGuestAgentFake) QemuAgentCommand(command string,
|
|
timeout libvirt.DomainQemuAgentCommandTimeout, _ uint32) (string, error) {
|
|
f.probeCalls++
|
|
f.command = command
|
|
f.timeout = timeout
|
|
f.callOrder = append(f.callOrder, "ping")
|
|
return `{"return":{}}`, f.probeErr
|
|
}
|
|
|
|
func (f *snapshotGuestAgentFake) AgentSetResponseTimeout(timeout int, _ uint32) error {
|
|
f.responseTimeoutCalls++
|
|
f.responseTimeout = timeout
|
|
f.callOrder = append(f.callOrder, "set-timeout")
|
|
return f.responseTimeoutErr
|
|
}
|
|
|
|
func (f *snapshotGuestAgentFake) FSFreeze(_ []string, _ uint32) error {
|
|
f.freezeCalls++
|
|
f.callOrder = append(f.callOrder, "freeze")
|
|
return f.freezeErr
|
|
}
|
|
|
|
func (f *snapshotGuestAgentFake) FSThaw(_ []string, _ uint32) error {
|
|
f.thawCalls++
|
|
f.callOrder = append(f.callOrder, "thaw")
|
|
return f.thawErr
|
|
}
|
|
|
|
const testGuestAgentResponseTimeout = 15
|
|
|
|
func TestSnapshotConsistencyMatchesRequest(t *testing.T) {
|
|
tests := []struct {
|
|
requested string
|
|
actual string
|
|
allowed bool
|
|
}{
|
|
{requested: "filesystem", actual: "filesystem", allowed: true},
|
|
{requested: "filesystem", actual: "crash", allowed: true},
|
|
{requested: "crash", actual: "crash", allowed: true},
|
|
{requested: "crash", actual: "filesystem", allowed: false},
|
|
{requested: "filesystem", actual: "unknown", allowed: false},
|
|
}
|
|
for _, test := range tests {
|
|
if got := snapshotConsistencyMatchesRequest(test.requested, test.actual); got != test.allowed {
|
|
t.Fatalf("snapshot consistency %s -> %s allowed=%v, want %v", test.requested, test.actual, got, test.allowed)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPrepareSnapshotConsistencySkipsProbeForExplicitCrash(t *testing.T) {
|
|
agent := &snapshotGuestAgentFake{}
|
|
actual, frozen, err := prepareSnapshotConsistency(agent, "crash", testGuestAgentResponseTimeout, true)
|
|
if err != nil || actual != "crash" || frozen || agent.probeCalls != 0 || agent.responseTimeoutCalls != 0 || agent.freezeCalls != 0 {
|
|
t.Fatalf("explicit crash mode unexpectedly used guest agent: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent)
|
|
}
|
|
}
|
|
|
|
func TestPrepareSnapshotConsistencyFallsBackWhenGuestAgentIsUnavailable(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
code libvirt.ErrorNumber
|
|
}{
|
|
{name: "unresponsive", code: libvirt.ERR_AGENT_UNRESPONSIVE},
|
|
{name: "not-configured", code: libvirt.ERR_ARGUMENT_UNSUPPORTED},
|
|
{name: "probe-timeout", code: libvirt.ERR_AGENT_COMMAND_TIMEOUT},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
agent := &snapshotGuestAgentFake{probeErr: libvirt.Error{Code: test.code}}
|
|
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
|
|
if err != nil || actual != "crash" || frozen {
|
|
t.Fatalf("unavailable guest agent did not fall back to crash consistency: actual=%q frozen=%v err=%v", actual, frozen, err)
|
|
}
|
|
if agent.probeCalls != 1 || agent.responseTimeoutCalls != 0 || agent.freezeCalls != 0 || agent.thawCalls != 0 || agent.command != snapshotGuestAgentPing || agent.timeout != 5 {
|
|
t.Fatalf("unexpected guest agent fallback calls: %+v", agent)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPrepareSnapshotConsistencyFallsBackWhenResponseTimeoutUnsupported(t *testing.T) {
|
|
for _, code := range []libvirt.ErrorNumber{
|
|
libvirt.ERR_NO_SUPPORT,
|
|
libvirt.ERR_OPERATION_UNSUPPORTED,
|
|
libvirt.ERR_ARGUMENT_UNSUPPORTED,
|
|
} {
|
|
agent := &snapshotGuestAgentFake{responseTimeoutErr: libvirt.Error{Code: code}}
|
|
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
|
|
if err != nil || actual != "crash" || frozen {
|
|
t.Fatalf("unsupported response timeout did not fall back to crash consistency: code=%v actual=%q frozen=%v err=%v", code, actual, frozen, err)
|
|
}
|
|
if agent.responseTimeout != testGuestAgentResponseTimeout || agent.freezeCalls != 0 ||
|
|
!reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout"}) {
|
|
t.Fatalf("unexpected unsupported-timeout call sequence: %+v", agent)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGuestAgentResponseTimeoutVersionGate(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
clientVersion uint32
|
|
daemonVersion uint32
|
|
want bool
|
|
}{
|
|
{name: "both-supported", clientVersion: 5_010_000, daemonVersion: 5_010_000, want: true},
|
|
{name: "newer-supported", clientVersion: 12_008_000, daemonVersion: 10_000_000, want: true},
|
|
{name: "client-too-old", clientVersion: 5_009_000, daemonVersion: 5_010_000},
|
|
{name: "daemon-too-old", clientVersion: 5_010_000, daemonVersion: 5_009_000},
|
|
{name: "unknown-client", daemonVersion: 5_010_000},
|
|
{name: "unknown-daemon", clientVersion: 5_010_000},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if got := supportsGuestAgentResponseTimeout(test.clientVersion, test.daemonVersion); got != test.want {
|
|
t.Fatalf("version gate=%v, want %v", got, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPrepareSnapshotConsistencyFallsBackWhenResponseTimeoutVersionIsUnsupported(t *testing.T) {
|
|
agent := &snapshotGuestAgentFake{}
|
|
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, false)
|
|
if err != nil || actual != "crash" || frozen {
|
|
t.Fatalf("unsupported response-timeout version did not fall back: actual=%q frozen=%v err=%v", actual, frozen, err)
|
|
}
|
|
if len(agent.callOrder) != 0 {
|
|
t.Fatalf("unsupported response-timeout version used freeze/thaw APIs: %+v", agent)
|
|
}
|
|
}
|
|
|
|
func TestPrepareSnapshotConsistencyRejectsResponseTimeoutInfrastructureFailure(t *testing.T) {
|
|
agent := &snapshotGuestAgentFake{responseTimeoutErr: libvirt.Error{Code: libvirt.ERR_RPC}}
|
|
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
|
|
if err == nil || actual != "" || frozen || agent.freezeCalls != 0 {
|
|
t.Fatalf("response-timeout infrastructure failure was incorrectly downgraded: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent)
|
|
}
|
|
if !reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout"}) {
|
|
t.Fatalf("response-timeout failure used the guest agent again: %+v", agent)
|
|
}
|
|
}
|
|
|
|
func TestPrepareSnapshotConsistencyRejectsInfrastructureProbeFailure(t *testing.T) {
|
|
agent := &snapshotGuestAgentFake{probeErr: libvirt.Error{Code: libvirt.ERR_RPC}}
|
|
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
|
|
if err == nil || actual != "" || frozen || agent.freezeCalls != 0 || agent.thawCalls != 0 {
|
|
t.Fatalf("infrastructure failure was incorrectly downgraded: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent)
|
|
}
|
|
}
|
|
|
|
func TestPrepareSnapshotConsistencyFreezesResponsiveGuest(t *testing.T) {
|
|
agent := &snapshotGuestAgentFake{}
|
|
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
|
|
if err != nil || actual != "filesystem" || !frozen || agent.probeCalls != 1 || agent.responseTimeoutCalls != 1 || agent.freezeCalls != 1 || agent.thawCalls != 0 {
|
|
t.Fatalf("responsive guest was not frozen: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent)
|
|
}
|
|
if agent.responseTimeout != testGuestAgentResponseTimeout ||
|
|
!reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout", "freeze"}) {
|
|
t.Fatalf("guest-agent timeout was not applied before freeze: %+v", agent)
|
|
}
|
|
}
|
|
|
|
func TestPrepareSnapshotConsistencyKeepsFreezeFailureDeterministicAfterThaw(t *testing.T) {
|
|
agent := &snapshotGuestAgentFake{freezeErr: libvirt.Error{Code: libvirt.ERR_AGENT_COMMAND_FAILED}}
|
|
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
|
|
var public *Error
|
|
if err == nil || actual != "" || frozen || !errors.As(err, &public) || public.Code != "snapshot_filesystem_freeze_failed" || operation.IsUncertain(err) {
|
|
t.Fatalf("freeze failure was not preserved as a deterministic error: actual=%q frozen=%v err=%v", actual, frozen, err)
|
|
}
|
|
if agent.thawCalls != 1 {
|
|
t.Fatalf("freeze failure did not attempt a compensating thaw: %+v", agent)
|
|
}
|
|
if !reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout", "freeze", "thaw"}) {
|
|
t.Fatalf("unexpected freeze failure call sequence: %+v", agent)
|
|
}
|
|
}
|
|
|
|
func TestPrepareSnapshotConsistencyMarksFailedThawUncertain(t *testing.T) {
|
|
agent := &snapshotGuestAgentFake{
|
|
freezeErr: libvirt.Error{Code: libvirt.ERR_AGENT_COMMAND_TIMEOUT},
|
|
thawErr: libvirt.Error{Code: libvirt.ERR_AGENT_UNRESPONSIVE},
|
|
}
|
|
_, _, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
|
|
if err == nil || !operation.IsUncertain(err) || agent.thawCalls != 1 {
|
|
t.Fatalf("ambiguous freeze without confirmed thaw was not uncertain: err=%v calls=%+v", err, agent)
|
|
}
|
|
}
|
|
|
|
type snapshotBackupJobFake struct {
|
|
active []*libvirt.DomainJobInfo
|
|
completed []*libvirt.DomainJobInfo
|
|
activeErr error
|
|
completedErr error
|
|
abortErr error
|
|
abortCalls int
|
|
getJobStatsCalls []libvirt.DomainGetJobStatsFlags
|
|
getJobStatsCall chan struct{}
|
|
}
|
|
|
|
func (f *snapshotBackupJobFake) GetJobStats(flags libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) {
|
|
f.getJobStatsCalls = append(f.getJobStatsCalls, flags)
|
|
if f.getJobStatsCall != nil {
|
|
select {
|
|
case f.getJobStatsCall <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
if flags == 0 {
|
|
if f.activeErr != nil {
|
|
return nil, f.activeErr
|
|
}
|
|
return nextSnapshotJobInfo(&f.active), nil
|
|
}
|
|
if flags == libvirt.DOMAIN_JOB_STATS_COMPLETED|libvirt.DOMAIN_JOB_STATS_KEEP_COMPLETED {
|
|
if f.completedErr != nil {
|
|
return nil, f.completedErr
|
|
}
|
|
return nextSnapshotJobInfo(&f.completed), nil
|
|
}
|
|
return nil, errors.New("unexpected job stats flags")
|
|
}
|
|
|
|
func (f *snapshotBackupJobFake) AbortJob() error {
|
|
f.abortCalls++
|
|
return f.abortErr
|
|
}
|
|
|
|
func nextSnapshotJobInfo(queue *[]*libvirt.DomainJobInfo) *libvirt.DomainJobInfo {
|
|
if len(*queue) == 0 {
|
|
return &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_BOUNDED, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_BACKUP}
|
|
}
|
|
value := (*queue)[0]
|
|
if len(*queue) > 1 {
|
|
*queue = (*queue)[1:]
|
|
}
|
|
return value
|
|
}
|
|
|
|
func activeBackupJob() *libvirt.DomainJobInfo {
|
|
return &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_BOUNDED, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_BACKUP}
|
|
}
|
|
|
|
func noActiveBackupJob() *libvirt.DomainJobInfo {
|
|
return &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}
|
|
}
|
|
|
|
func terminalBackupJob(kind libvirt.DomainJobType) *libvirt.DomainJobInfo {
|
|
return &libvirt.DomainJobInfo{Type: kind, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_BACKUP}
|
|
}
|
|
|
|
func fastSnapshotBackupWaitTiming() snapshotBackupWaitTiming {
|
|
return snapshotBackupWaitTiming{pollInterval: time.Millisecond, abortConfirmationTimeout: 30 * time.Millisecond}
|
|
}
|
|
|
|
func TestWaitForBackupTerminalConfirmsCancellationAfterAbort(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
job := &snapshotBackupJobFake{
|
|
active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()},
|
|
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)},
|
|
}
|
|
terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming())
|
|
if err != nil || terminal != libvirt.DOMAIN_JOB_CANCELLED || job.abortCalls != 1 {
|
|
t.Fatalf("cancelled backup was not confirmed after abort: terminal=%v err=%v aborts=%d calls=%v", terminal, err, job.abortCalls, job.getJobStatsCalls)
|
|
}
|
|
}
|
|
|
|
func TestWaitForBackupTerminalAcceptsCompletionRacingAbortError(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
job := &snapshotBackupJobFake{
|
|
active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()},
|
|
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)},
|
|
abortErr: libvirt.Error{Code: libvirt.ERR_OPERATION_INVALID},
|
|
}
|
|
terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming())
|
|
if err != nil || terminal != libvirt.DOMAIN_JOB_COMPLETED || job.abortCalls != 1 {
|
|
t.Fatalf("completion evidence did not win an abort race: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
|
|
}
|
|
}
|
|
|
|
func TestWaitForBackupTerminalLeavesUnconfirmedAbortUncertain(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
job := &snapshotBackupJobFake{active: []*libvirt.DomainJobInfo{activeBackupJob()}}
|
|
timing := snapshotBackupWaitTiming{pollInterval: time.Millisecond, abortConfirmationTimeout: 8 * time.Millisecond}
|
|
terminal, err := waitForBackupTerminalWithTiming(ctx, job, timing)
|
|
if terminal != libvirt.DOMAIN_JOB_NONE || err == nil || !operation.IsUncertain(err) || job.abortCalls != 1 {
|
|
t.Fatalf("unconfirmed abort did not remain uncertain: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
|
|
}
|
|
var public *Error
|
|
if !errors.As(err, &public) || public.Code != "snapshot_abort_unconfirmed" {
|
|
t.Fatalf("unconfirmed abort lost its stable public code: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestWaitForBackupTerminalDoesNotAbortAlreadyCompletedBackupAfterCancellation(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
job := &snapshotBackupJobFake{active: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)}}
|
|
terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming())
|
|
if err != nil || terminal != libvirt.DOMAIN_JOB_COMPLETED || job.abortCalls != 0 {
|
|
t.Fatalf("completed backup was aborted: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
|
|
}
|
|
}
|
|
|
|
func TestWaitForBackupTerminalDoesNotAbortForeignOrUnknownJob(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
job *libvirt.DomainJobInfo
|
|
}{
|
|
{
|
|
name: "foreign",
|
|
job: &libvirt.DomainJobInfo{
|
|
Type: libvirt.DOMAIN_JOB_BOUNDED, OperationSet: true,
|
|
Operation: libvirt.DOMAIN_JOB_OPERATION_MIGRATION_OUT,
|
|
},
|
|
},
|
|
{name: "operation-unknown", job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_BOUNDED}},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
job := &snapshotBackupJobFake{active: []*libvirt.DomainJobInfo{activeBackupJob(), test.job}}
|
|
terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming())
|
|
if terminal != libvirt.DOMAIN_JOB_NONE || err == nil || !operation.IsUncertain(err) || job.abortCalls != 0 {
|
|
t.Fatalf("foreign or unknown job was not protected: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestInspectBackupTerminalUsesRetainedEvidenceAfterActiveQueryFailure(t *testing.T) {
|
|
job := &snapshotBackupJobFake{
|
|
activeErr: libvirt.Error{Code: libvirt.ERR_RPC},
|
|
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)},
|
|
}
|
|
terminal, done, err := inspectBackupTerminal(job)
|
|
if err != nil || !done || terminal != libvirt.DOMAIN_JOB_COMPLETED {
|
|
t.Fatalf("retained terminal evidence was lost after active query failure: terminal=%v done=%v err=%v", terminal, done, err)
|
|
}
|
|
}
|
|
|
|
func TestWaitForBackupTerminalDoesNotAbortNormalCompletion(t *testing.T) {
|
|
job := &snapshotBackupJobFake{
|
|
active: []*libvirt.DomainJobInfo{activeBackupJob(), noActiveBackupJob()},
|
|
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)},
|
|
}
|
|
terminal, err := waitForBackupTerminalWithTiming(context.Background(), job, fastSnapshotBackupWaitTiming())
|
|
if err != nil || terminal != libvirt.DOMAIN_JOB_COMPLETED || job.abortCalls != 0 {
|
|
t.Fatalf("normal completion unexpectedly aborted: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
|
|
}
|
|
}
|
|
|
|
func TestWaitForBackupTerminalWakesOnJobCompletedEvent(t *testing.T) {
|
|
calls := make(chan struct{}, 4)
|
|
job := &snapshotBackupJobFake{
|
|
active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()},
|
|
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)},
|
|
getJobStatsCall: calls,
|
|
}
|
|
timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second}
|
|
type result struct {
|
|
terminal libvirt.DomainJobType
|
|
err error
|
|
}
|
|
done := make(chan result, 1)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer func() {
|
|
cancel()
|
|
libvirtJobEvents.notify()
|
|
}()
|
|
go func() {
|
|
terminal, err := waitForBackupTerminalWithTiming(ctx, job, timing)
|
|
done <- result{terminal: terminal, err: err}
|
|
}()
|
|
select {
|
|
case <-calls:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("backup waiter did not inspect the active job")
|
|
}
|
|
libvirtJobEvents.notify()
|
|
select {
|
|
case <-calls:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("spurious JOB_COMPLETED event did not trigger GetJobStats")
|
|
}
|
|
select {
|
|
case result := <-done:
|
|
t.Fatalf("event was treated as terminal evidence: terminal=%v err=%v", result.terminal, result.err)
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
|
|
libvirtJobEvents.notify()
|
|
select {
|
|
case result := <-done:
|
|
if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_COMPLETED {
|
|
t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("JOB_COMPLETED event did not wake backup waiter")
|
|
}
|
|
}
|
|
|
|
func TestAbortBackupWaitWakesOnJobCompletedEvent(t *testing.T) {
|
|
calls := make(chan struct{}, 2)
|
|
job := &snapshotBackupJobFake{
|
|
active: []*libvirt.DomainJobInfo{
|
|
activeBackupJob(), activeBackupJob(), noActiveBackupJob(),
|
|
},
|
|
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)},
|
|
getJobStatsCall: calls,
|
|
}
|
|
timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second}
|
|
type result struct {
|
|
terminal libvirt.DomainJobType
|
|
err error
|
|
}
|
|
done := make(chan result, 1)
|
|
defer libvirtJobEvents.notify()
|
|
go func() {
|
|
terminal, err := abortBackupAndWaitForTerminal(job, context.DeadlineExceeded, timing)
|
|
done <- result{terminal: terminal, err: err}
|
|
}()
|
|
for call := 0; call < 2; call++ {
|
|
select {
|
|
case <-calls:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("abort waiter did not inspect the active job")
|
|
}
|
|
}
|
|
libvirtJobEvents.notify()
|
|
select {
|
|
case result := <-done:
|
|
if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_CANCELLED {
|
|
t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("JOB_COMPLETED event did not wake abort confirmation")
|
|
}
|
|
}
|
|
|
|
func TestInspectBackupTerminalAcceptsAllBackupTerminalStates(t *testing.T) {
|
|
for _, kind := range []libvirt.DomainJobType{
|
|
libvirt.DOMAIN_JOB_COMPLETED,
|
|
libvirt.DOMAIN_JOB_FAILED,
|
|
libvirt.DOMAIN_JOB_CANCELLED,
|
|
} {
|
|
job := &snapshotBackupJobFake{
|
|
active: []*libvirt.DomainJobInfo{noActiveBackupJob()},
|
|
completed: []*libvirt.DomainJobInfo{terminalBackupJob(kind)},
|
|
}
|
|
terminal, done, err := inspectBackupTerminal(job)
|
|
if err != nil || !done || terminal != kind {
|
|
t.Fatalf("backup terminal state %v was not accepted: terminal=%v done=%v err=%v", kind, terminal, done, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestWaitForBackupTerminalRejectsForeignTerminalEvidence(t *testing.T) {
|
|
job := &snapshotBackupJobFake{
|
|
active: []*libvirt.DomainJobInfo{noActiveBackupJob()},
|
|
completed: []*libvirt.DomainJobInfo{{
|
|
Type: libvirt.DOMAIN_JOB_COMPLETED, OperationSet: true,
|
|
Operation: libvirt.DOMAIN_JOB_OPERATION_MIGRATION_OUT,
|
|
}},
|
|
}
|
|
_, err := waitForBackupTerminalWithTiming(context.Background(), job, fastSnapshotBackupWaitTiming())
|
|
if err == nil || !operation.IsUncertain(err) || job.abortCalls != 0 {
|
|
t.Fatalf("foreign terminal evidence was accepted: err=%v aborts=%d", err, job.abortCalls)
|
|
}
|
|
}
|
|
|
|
func TestAbortBackupAfterThawFailureWaitsForTerminalAndRemainsUncertain(t *testing.T) {
|
|
thawErr := libvirt.Error{Code: libvirt.ERR_AGENT_COMMAND_TIMEOUT}
|
|
job := &snapshotBackupJobFake{
|
|
active: []*libvirt.DomainJobInfo{activeBackupJob(), noActiveBackupJob()},
|
|
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)},
|
|
}
|
|
terminal, err := abortBackupAfterThawFailure(job, thawErr, fastSnapshotBackupWaitTiming())
|
|
if terminal != libvirt.DOMAIN_JOB_CANCELLED || err == nil || !operation.IsUncertain(err) || job.abortCalls != 1 {
|
|
t.Fatalf("thaw failure did not wait for terminal cancellation: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
|
|
}
|
|
}
|