You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_snapshot_fallback_t...

514 lines
20 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"errors"
"reflect"
"testing"
"time"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
)
type snapshotGuestAgentFake struct {
probeErr error
responseTimeoutErr error
freezeErr error
thawErr error
probeCalls int
responseTimeoutCalls int
freezeCalls int
thawCalls int
command string
timeout libvirt.DomainQemuAgentCommandTimeout
responseTimeout int
callOrder []string
}
func (f *snapshotGuestAgentFake) QemuAgentCommand(command string,
timeout libvirt.DomainQemuAgentCommandTimeout, _ uint32) (string, error) {
f.probeCalls++
f.command = command
f.timeout = timeout
f.callOrder = append(f.callOrder, "ping")
return `{"return":{}}`, f.probeErr
}
func (f *snapshotGuestAgentFake) AgentSetResponseTimeout(timeout int, _ uint32) error {
f.responseTimeoutCalls++
f.responseTimeout = timeout
f.callOrder = append(f.callOrder, "set-timeout")
return f.responseTimeoutErr
}
func (f *snapshotGuestAgentFake) FSFreeze(_ []string, _ uint32) error {
f.freezeCalls++
f.callOrder = append(f.callOrder, "freeze")
return f.freezeErr
}
func (f *snapshotGuestAgentFake) FSThaw(_ []string, _ uint32) error {
f.thawCalls++
f.callOrder = append(f.callOrder, "thaw")
return f.thawErr
}
const testGuestAgentResponseTimeout = 15
func TestSnapshotConsistencyMatchesRequest(t *testing.T) {
tests := []struct {
requested string
actual string
allowed bool
}{
{requested: "filesystem", actual: "filesystem", allowed: true},
{requested: "filesystem", actual: "crash", allowed: true},
{requested: "crash", actual: "crash", allowed: true},
{requested: "crash", actual: "filesystem", allowed: false},
{requested: "filesystem", actual: "unknown", allowed: false},
}
for _, test := range tests {
if got := snapshotConsistencyMatchesRequest(test.requested, test.actual); got != test.allowed {
t.Fatalf("snapshot consistency %s -> %s allowed=%v, want %v", test.requested, test.actual, got, test.allowed)
}
}
}
func TestPrepareSnapshotConsistencySkipsProbeForExplicitCrash(t *testing.T) {
agent := &snapshotGuestAgentFake{}
actual, frozen, err := prepareSnapshotConsistency(agent, "crash", testGuestAgentResponseTimeout, true)
if err != nil || actual != "crash" || frozen || agent.probeCalls != 0 || agent.responseTimeoutCalls != 0 || agent.freezeCalls != 0 {
t.Fatalf("explicit crash mode unexpectedly used guest agent: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent)
}
}
func TestPrepareSnapshotConsistencyFallsBackWhenGuestAgentIsUnavailable(t *testing.T) {
for _, test := range []struct {
name string
code libvirt.ErrorNumber
}{
{name: "unresponsive", code: libvirt.ERR_AGENT_UNRESPONSIVE},
{name: "not-configured", code: libvirt.ERR_ARGUMENT_UNSUPPORTED},
{name: "probe-timeout", code: libvirt.ERR_AGENT_COMMAND_TIMEOUT},
} {
t.Run(test.name, func(t *testing.T) {
agent := &snapshotGuestAgentFake{probeErr: libvirt.Error{Code: test.code}}
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
if err != nil || actual != "crash" || frozen {
t.Fatalf("unavailable guest agent did not fall back to crash consistency: actual=%q frozen=%v err=%v", actual, frozen, err)
}
if agent.probeCalls != 1 || agent.responseTimeoutCalls != 0 || agent.freezeCalls != 0 || agent.thawCalls != 0 || agent.command != snapshotGuestAgentPing || agent.timeout != 5 {
t.Fatalf("unexpected guest agent fallback calls: %+v", agent)
}
})
}
}
func TestPrepareSnapshotConsistencyFallsBackWhenResponseTimeoutUnsupported(t *testing.T) {
for _, code := range []libvirt.ErrorNumber{
libvirt.ERR_NO_SUPPORT,
libvirt.ERR_OPERATION_UNSUPPORTED,
libvirt.ERR_ARGUMENT_UNSUPPORTED,
} {
agent := &snapshotGuestAgentFake{responseTimeoutErr: libvirt.Error{Code: code}}
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
if err != nil || actual != "crash" || frozen {
t.Fatalf("unsupported response timeout did not fall back to crash consistency: code=%v actual=%q frozen=%v err=%v", code, actual, frozen, err)
}
if agent.responseTimeout != testGuestAgentResponseTimeout || agent.freezeCalls != 0 ||
!reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout"}) {
t.Fatalf("unexpected unsupported-timeout call sequence: %+v", agent)
}
}
}
func TestGuestAgentResponseTimeoutVersionGate(t *testing.T) {
for _, test := range []struct {
name string
clientVersion uint32
daemonVersion uint32
want bool
}{
{name: "both-supported", clientVersion: 5_010_000, daemonVersion: 5_010_000, want: true},
{name: "newer-supported", clientVersion: 12_008_000, daemonVersion: 10_000_000, want: true},
{name: "client-too-old", clientVersion: 5_009_000, daemonVersion: 5_010_000},
{name: "daemon-too-old", clientVersion: 5_010_000, daemonVersion: 5_009_000},
{name: "unknown-client", daemonVersion: 5_010_000},
{name: "unknown-daemon", clientVersion: 5_010_000},
} {
t.Run(test.name, func(t *testing.T) {
if got := supportsGuestAgentResponseTimeout(test.clientVersion, test.daemonVersion); got != test.want {
t.Fatalf("version gate=%v, want %v", got, test.want)
}
})
}
}
func TestPrepareSnapshotConsistencyFallsBackWhenResponseTimeoutVersionIsUnsupported(t *testing.T) {
agent := &snapshotGuestAgentFake{}
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, false)
if err != nil || actual != "crash" || frozen {
t.Fatalf("unsupported response-timeout version did not fall back: actual=%q frozen=%v err=%v", actual, frozen, err)
}
if len(agent.callOrder) != 0 {
t.Fatalf("unsupported response-timeout version used freeze/thaw APIs: %+v", agent)
}
}
func TestPrepareSnapshotConsistencyRejectsResponseTimeoutInfrastructureFailure(t *testing.T) {
agent := &snapshotGuestAgentFake{responseTimeoutErr: libvirt.Error{Code: libvirt.ERR_RPC}}
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
if err == nil || actual != "" || frozen || agent.freezeCalls != 0 {
t.Fatalf("response-timeout infrastructure failure was incorrectly downgraded: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent)
}
if !reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout"}) {
t.Fatalf("response-timeout failure used the guest agent again: %+v", agent)
}
}
func TestPrepareSnapshotConsistencyRejectsInfrastructureProbeFailure(t *testing.T) {
agent := &snapshotGuestAgentFake{probeErr: libvirt.Error{Code: libvirt.ERR_RPC}}
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
if err == nil || actual != "" || frozen || agent.freezeCalls != 0 || agent.thawCalls != 0 {
t.Fatalf("infrastructure failure was incorrectly downgraded: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent)
}
}
func TestPrepareSnapshotConsistencyFreezesResponsiveGuest(t *testing.T) {
agent := &snapshotGuestAgentFake{}
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
if err != nil || actual != "filesystem" || !frozen || agent.probeCalls != 1 || agent.responseTimeoutCalls != 1 || agent.freezeCalls != 1 || agent.thawCalls != 0 {
t.Fatalf("responsive guest was not frozen: actual=%q frozen=%v err=%v calls=%+v", actual, frozen, err, agent)
}
if agent.responseTimeout != testGuestAgentResponseTimeout ||
!reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout", "freeze"}) {
t.Fatalf("guest-agent timeout was not applied before freeze: %+v", agent)
}
}
func TestPrepareSnapshotConsistencyKeepsFreezeFailureDeterministicAfterThaw(t *testing.T) {
agent := &snapshotGuestAgentFake{freezeErr: libvirt.Error{Code: libvirt.ERR_AGENT_COMMAND_FAILED}}
actual, frozen, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
var public *Error
if err == nil || actual != "" || frozen || !errors.As(err, &public) || public.Code != "snapshot_filesystem_freeze_failed" || operation.IsUncertain(err) {
t.Fatalf("freeze failure was not preserved as a deterministic error: actual=%q frozen=%v err=%v", actual, frozen, err)
}
if agent.thawCalls != 1 {
t.Fatalf("freeze failure did not attempt a compensating thaw: %+v", agent)
}
if !reflect.DeepEqual(agent.callOrder, []string{"ping", "set-timeout", "freeze", "thaw"}) {
t.Fatalf("unexpected freeze failure call sequence: %+v", agent)
}
}
func TestPrepareSnapshotConsistencyMarksFailedThawUncertain(t *testing.T) {
agent := &snapshotGuestAgentFake{
freezeErr: libvirt.Error{Code: libvirt.ERR_AGENT_COMMAND_TIMEOUT},
thawErr: libvirt.Error{Code: libvirt.ERR_AGENT_UNRESPONSIVE},
}
_, _, err := prepareSnapshotConsistency(agent, "filesystem", testGuestAgentResponseTimeout, true)
if err == nil || !operation.IsUncertain(err) || agent.thawCalls != 1 {
t.Fatalf("ambiguous freeze without confirmed thaw was not uncertain: err=%v calls=%+v", err, agent)
}
}
type snapshotBackupJobFake struct {
active []*libvirt.DomainJobInfo
completed []*libvirt.DomainJobInfo
activeErr error
completedErr error
abortErr error
abortCalls int
getJobStatsCalls []libvirt.DomainGetJobStatsFlags
getJobStatsCall chan struct{}
}
func (f *snapshotBackupJobFake) GetJobStats(flags libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) {
f.getJobStatsCalls = append(f.getJobStatsCalls, flags)
if f.getJobStatsCall != nil {
select {
case f.getJobStatsCall <- struct{}{}:
default:
}
}
if flags == 0 {
if f.activeErr != nil {
return nil, f.activeErr
}
return nextSnapshotJobInfo(&f.active), nil
}
if flags == libvirt.DOMAIN_JOB_STATS_COMPLETED|libvirt.DOMAIN_JOB_STATS_KEEP_COMPLETED {
if f.completedErr != nil {
return nil, f.completedErr
}
return nextSnapshotJobInfo(&f.completed), nil
}
return nil, errors.New("unexpected job stats flags")
}
func (f *snapshotBackupJobFake) AbortJob() error {
f.abortCalls++
return f.abortErr
}
func nextSnapshotJobInfo(queue *[]*libvirt.DomainJobInfo) *libvirt.DomainJobInfo {
if len(*queue) == 0 {
return &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_BOUNDED, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_BACKUP}
}
value := (*queue)[0]
if len(*queue) > 1 {
*queue = (*queue)[1:]
}
return value
}
func activeBackupJob() *libvirt.DomainJobInfo {
return &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_BOUNDED, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_BACKUP}
}
func noActiveBackupJob() *libvirt.DomainJobInfo {
return &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}
}
func terminalBackupJob(kind libvirt.DomainJobType) *libvirt.DomainJobInfo {
return &libvirt.DomainJobInfo{Type: kind, OperationSet: true, Operation: libvirt.DOMAIN_JOB_OPERATION_BACKUP}
}
func fastSnapshotBackupWaitTiming() snapshotBackupWaitTiming {
return snapshotBackupWaitTiming{pollInterval: time.Millisecond, abortConfirmationTimeout: 30 * time.Millisecond}
}
func TestWaitForBackupTerminalConfirmsCancellationAfterAbort(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)},
}
terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming())
if err != nil || terminal != libvirt.DOMAIN_JOB_CANCELLED || job.abortCalls != 1 {
t.Fatalf("cancelled backup was not confirmed after abort: terminal=%v err=%v aborts=%d calls=%v", terminal, err, job.abortCalls, job.getJobStatsCalls)
}
}
func TestWaitForBackupTerminalAcceptsCompletionRacingAbortError(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)},
abortErr: libvirt.Error{Code: libvirt.ERR_OPERATION_INVALID},
}
terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming())
if err != nil || terminal != libvirt.DOMAIN_JOB_COMPLETED || job.abortCalls != 1 {
t.Fatalf("completion evidence did not win an abort race: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
}
}
func TestWaitForBackupTerminalLeavesUnconfirmedAbortUncertain(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
job := &snapshotBackupJobFake{active: []*libvirt.DomainJobInfo{activeBackupJob()}}
timing := snapshotBackupWaitTiming{pollInterval: time.Millisecond, abortConfirmationTimeout: 8 * time.Millisecond}
terminal, err := waitForBackupTerminalWithTiming(ctx, job, timing)
if terminal != libvirt.DOMAIN_JOB_NONE || err == nil || !operation.IsUncertain(err) || job.abortCalls != 1 {
t.Fatalf("unconfirmed abort did not remain uncertain: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
}
var public *Error
if !errors.As(err, &public) || public.Code != "snapshot_abort_unconfirmed" {
t.Fatalf("unconfirmed abort lost its stable public code: %v", err)
}
}
func TestWaitForBackupTerminalDoesNotAbortAlreadyCompletedBackupAfterCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
job := &snapshotBackupJobFake{active: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)}}
terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming())
if err != nil || terminal != libvirt.DOMAIN_JOB_COMPLETED || job.abortCalls != 0 {
t.Fatalf("completed backup was aborted: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
}
}
func TestWaitForBackupTerminalDoesNotAbortForeignOrUnknownJob(t *testing.T) {
for _, test := range []struct {
name string
job *libvirt.DomainJobInfo
}{
{
name: "foreign",
job: &libvirt.DomainJobInfo{
Type: libvirt.DOMAIN_JOB_BOUNDED, OperationSet: true,
Operation: libvirt.DOMAIN_JOB_OPERATION_MIGRATION_OUT,
},
},
{name: "operation-unknown", job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_BOUNDED}},
} {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
job := &snapshotBackupJobFake{active: []*libvirt.DomainJobInfo{activeBackupJob(), test.job}}
terminal, err := waitForBackupTerminalWithTiming(ctx, job, fastSnapshotBackupWaitTiming())
if terminal != libvirt.DOMAIN_JOB_NONE || err == nil || !operation.IsUncertain(err) || job.abortCalls != 0 {
t.Fatalf("foreign or unknown job was not protected: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
}
})
}
}
func TestInspectBackupTerminalUsesRetainedEvidenceAfterActiveQueryFailure(t *testing.T) {
job := &snapshotBackupJobFake{
activeErr: libvirt.Error{Code: libvirt.ERR_RPC},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)},
}
terminal, done, err := inspectBackupTerminal(job)
if err != nil || !done || terminal != libvirt.DOMAIN_JOB_COMPLETED {
t.Fatalf("retained terminal evidence was lost after active query failure: terminal=%v done=%v err=%v", terminal, done, err)
}
}
func TestWaitForBackupTerminalDoesNotAbortNormalCompletion(t *testing.T) {
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{activeBackupJob(), noActiveBackupJob()},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)},
}
terminal, err := waitForBackupTerminalWithTiming(context.Background(), job, fastSnapshotBackupWaitTiming())
if err != nil || terminal != libvirt.DOMAIN_JOB_COMPLETED || job.abortCalls != 0 {
t.Fatalf("normal completion unexpectedly aborted: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
}
}
func TestWaitForBackupTerminalWakesOnJobCompletedEvent(t *testing.T) {
calls := make(chan struct{}, 4)
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{activeBackupJob(), activeBackupJob(), noActiveBackupJob()},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_COMPLETED)},
getJobStatsCall: calls,
}
timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second}
type result struct {
terminal libvirt.DomainJobType
err error
}
done := make(chan result, 1)
ctx, cancel := context.WithCancel(context.Background())
defer func() {
cancel()
libvirtJobEvents.notify()
}()
go func() {
terminal, err := waitForBackupTerminalWithTiming(ctx, job, timing)
done <- result{terminal: terminal, err: err}
}()
select {
case <-calls:
case <-time.After(time.Second):
t.Fatal("backup waiter did not inspect the active job")
}
libvirtJobEvents.notify()
select {
case <-calls:
case <-time.After(time.Second):
t.Fatal("spurious JOB_COMPLETED event did not trigger GetJobStats")
}
select {
case result := <-done:
t.Fatalf("event was treated as terminal evidence: terminal=%v err=%v", result.terminal, result.err)
case <-time.After(20 * time.Millisecond):
}
libvirtJobEvents.notify()
select {
case result := <-done:
if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_COMPLETED {
t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err)
}
case <-time.After(time.Second):
t.Fatal("JOB_COMPLETED event did not wake backup waiter")
}
}
func TestAbortBackupWaitWakesOnJobCompletedEvent(t *testing.T) {
calls := make(chan struct{}, 2)
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{
activeBackupJob(), activeBackupJob(), noActiveBackupJob(),
},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)},
getJobStatsCall: calls,
}
timing := snapshotBackupWaitTiming{pollInterval: time.Minute, abortConfirmationTimeout: time.Second}
type result struct {
terminal libvirt.DomainJobType
err error
}
done := make(chan result, 1)
defer libvirtJobEvents.notify()
go func() {
terminal, err := abortBackupAndWaitForTerminal(job, context.DeadlineExceeded, timing)
done <- result{terminal: terminal, err: err}
}()
for call := 0; call < 2; call++ {
select {
case <-calls:
case <-time.After(time.Second):
t.Fatal("abort waiter did not inspect the active job")
}
}
libvirtJobEvents.notify()
select {
case result := <-done:
if result.err != nil || result.terminal != libvirt.DOMAIN_JOB_CANCELLED {
t.Fatalf("event wake result: terminal=%v err=%v", result.terminal, result.err)
}
case <-time.After(time.Second):
t.Fatal("JOB_COMPLETED event did not wake abort confirmation")
}
}
func TestInspectBackupTerminalAcceptsAllBackupTerminalStates(t *testing.T) {
for _, kind := range []libvirt.DomainJobType{
libvirt.DOMAIN_JOB_COMPLETED,
libvirt.DOMAIN_JOB_FAILED,
libvirt.DOMAIN_JOB_CANCELLED,
} {
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{noActiveBackupJob()},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(kind)},
}
terminal, done, err := inspectBackupTerminal(job)
if err != nil || !done || terminal != kind {
t.Fatalf("backup terminal state %v was not accepted: terminal=%v done=%v err=%v", kind, terminal, done, err)
}
}
}
func TestWaitForBackupTerminalRejectsForeignTerminalEvidence(t *testing.T) {
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{noActiveBackupJob()},
completed: []*libvirt.DomainJobInfo{{
Type: libvirt.DOMAIN_JOB_COMPLETED, OperationSet: true,
Operation: libvirt.DOMAIN_JOB_OPERATION_MIGRATION_OUT,
}},
}
_, err := waitForBackupTerminalWithTiming(context.Background(), job, fastSnapshotBackupWaitTiming())
if err == nil || !operation.IsUncertain(err) || job.abortCalls != 0 {
t.Fatalf("foreign terminal evidence was accepted: err=%v aborts=%d", err, job.abortCalls)
}
}
func TestAbortBackupAfterThawFailureWaitsForTerminalAndRemainsUncertain(t *testing.T) {
thawErr := libvirt.Error{Code: libvirt.ERR_AGENT_COMMAND_TIMEOUT}
job := &snapshotBackupJobFake{
active: []*libvirt.DomainJobInfo{activeBackupJob(), noActiveBackupJob()},
completed: []*libvirt.DomainJobInfo{terminalBackupJob(libvirt.DOMAIN_JOB_CANCELLED)},
}
terminal, err := abortBackupAfterThawFailure(job, thawErr, fastSnapshotBackupWaitTiming())
if terminal != libvirt.DOMAIN_JOB_CANCELLED || err == nil || !operation.IsUncertain(err) || job.abortCalls != 1 {
t.Fatalf("thaw failure did not wait for terminal cancellation: terminal=%v err=%v aborts=%d", terminal, err, job.abortCalls)
}
}