You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
511 lines
18 KiB
Go
511 lines
18 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"git.luoli.live/root/virt/agent/worker"
|
|
)
|
|
|
|
type fakeWorkerStateCleaner struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
failOnce bool
|
|
}
|
|
|
|
type fakeRecoveredWorkerState struct {
|
|
fakeWorkerStateCleaner
|
|
receipts []worker.Response
|
|
terminals []worker.TerminalEvidence
|
|
receiptErr error
|
|
terminalErr error
|
|
}
|
|
|
|
func TestPublicGuardPreflightModeIsIsolatedFromOperationWorkerMode(t *testing.T) {
|
|
for name, test := range map[string]struct {
|
|
preflight bool
|
|
worker bool
|
|
journal string
|
|
receipt string
|
|
wantErr bool
|
|
}{
|
|
"ordinary": {},
|
|
"worker": {worker: true, journal: "/tmp/journal.db", receipt: "/tmp/completion.json"},
|
|
"preflight": {preflight: true},
|
|
"preflight worker": {preflight: true, worker: true, wantErr: true},
|
|
"preflight worker paths": {preflight: true, journal: "/tmp/journal.db", wantErr: true},
|
|
"paths without worker mode": {journal: "/tmp/journal.db", receipt: "/tmp/completion.json", wantErr: true},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
err := validateRuntimeMode(test.preflight, test.worker, test.journal, test.receipt)
|
|
if (err != nil) != test.wantErr {
|
|
t.Fatalf("validate runtime mode error = %v, wantErr=%v", err, test.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPublicGuardPreflightSourceCannotOpenRuntimeSubsystems(t *testing.T) {
|
|
preflightPath := filepath.Join("public_guard_preflight.go")
|
|
raw, err := os.ReadFile(preflightPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
source := string(raw)
|
|
for _, forbidden := range []string{
|
|
`"net/http"`, `agent/operation`, `agent/hypervisor`, `agent/control`,
|
|
`operation.Open`, `hypervisor.Open`, `control.NewClient`, `http.`,
|
|
} {
|
|
if strings.Contains(source, forbidden) {
|
|
t.Fatalf("preflight source reaches forbidden runtime subsystem %q", forbidden)
|
|
}
|
|
}
|
|
mainRaw, err := os.ReadFile("main.go")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mainSource := string(mainRaw)
|
|
branch := strings.Index(mainSource, "if *publicGuardPreflight {")
|
|
journal := strings.Index(mainSource, "operation.OpenWithHeavyIOConcurrency")
|
|
if branch < 0 || journal < 0 || branch >= journal {
|
|
t.Fatalf("preflight branch must return before Journal initialization: branch=%d journal=%d", branch, journal)
|
|
}
|
|
}
|
|
|
|
func TestIsolatedWorkerLifetimeAllowsPortableBackupDeadline(t *testing.T) {
|
|
for _, path := range []string{
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/backups",
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/restore-backup",
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/recover-backup",
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/peer-migration/prepare",
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/peer-migration/receive",
|
|
} {
|
|
if got := isolatedWorkerLifetime("POST", path); got != 24*time.Hour+30*time.Second {
|
|
t.Fatalf("backup worker lifetime for %q = %s", path, got)
|
|
}
|
|
}
|
|
for _, test := range []struct {
|
|
method string
|
|
path string
|
|
}{
|
|
{method: "GET", path: "/v1/vms/123e4567-e89b-42d3-a456-426614174000/backups"},
|
|
{method: "POST", path: "/v1/vms/123e4567-e89b-42d3-a456-426614174000/backups/extra"},
|
|
{method: "POST", path: "/v1/vms/123e4567-e89b-42d3-a456-426614174000/settings"},
|
|
{method: "POST", path: "/v1/vms/123e4567-e89b-42d3-a456-426614174000/peer-migration/rollback"},
|
|
} {
|
|
if got := isolatedWorkerLifetime(test.method, test.path); got != 2*time.Hour+30*time.Second {
|
|
t.Fatalf("ordinary worker lifetime for %s %q = %s", test.method, test.path, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunControlStopsWhenPeerMigrationListenerFails(t *testing.T) {
|
|
journalFailures := make(chan error)
|
|
peerFailures := make(chan error, 1)
|
|
peerFailures <- errors.New("listener failed")
|
|
close(peerFailures)
|
|
runStopped := make(chan struct{})
|
|
err := runControlUntilStopped(
|
|
context.Background(), journalFailures,
|
|
func(ctx context.Context) error {
|
|
<-ctx.Done()
|
|
close(runStopped)
|
|
return ctx.Err()
|
|
},
|
|
peerFailures,
|
|
)
|
|
if err == nil || !strings.Contains(err.Error(), "PeerMigration listener failed") {
|
|
t.Fatalf("unexpected listener failure result: %v", err)
|
|
}
|
|
select {
|
|
case <-runStopped:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("control channel was not canceled after listener failure")
|
|
}
|
|
}
|
|
|
|
func reopenDispatchingJournal(t *testing.T, path string, spec operation.Spec) *operation.Journal {
|
|
t.Helper()
|
|
j, err := operation.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
_, replayed, done, err := j.ExecuteBackground(
|
|
context.Background(), context.Background(), time.Minute, spec,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
close(started)
|
|
<-release
|
|
return json.RawMessage(`{"state":"running"}`), nil
|
|
},
|
|
)
|
|
if err != nil || replayed || done == nil {
|
|
_ = j.Close()
|
|
t.Fatalf("seed dispatching operation replayed=%v done=%v err=%v", replayed, done != nil, err)
|
|
}
|
|
<-started
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
close(release)
|
|
<-done
|
|
|
|
reopened, err := operation.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = reopened.Close() })
|
|
record, err := reopened.Get(spec.OperationID)
|
|
if err != nil || record.State != operation.StateDispatching {
|
|
t.Fatalf("reopened dispatching record = %#v err=%v", record, err)
|
|
}
|
|
return reopened
|
|
}
|
|
|
|
func writeLegacyTerminalWorkerJournal(t *testing.T, operationDir string, spec operation.Spec) {
|
|
t.Helper()
|
|
if err := os.Mkdir(operationDir, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
child, err := operation.Open(filepath.Join(operationDir, "journal.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := child.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"state":"running"}`), nil
|
|
}); err != nil {
|
|
_ = child.Close()
|
|
t.Fatal(err)
|
|
}
|
|
if err := child.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (s *fakeRecoveredWorkerState) PendingReceipts() ([]worker.Response, error) {
|
|
return append([]worker.Response(nil), s.receipts...), s.receiptErr
|
|
}
|
|
|
|
func (s *fakeRecoveredWorkerState) PendingJournalTerminals() ([]worker.TerminalEvidence, error) {
|
|
return append([]worker.TerminalEvidence(nil), s.terminals...), s.terminalErr
|
|
}
|
|
|
|
func (c *fakeWorkerStateCleaner) Cleanup(operationID string) error {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.calls = append(c.calls, operationID)
|
|
if c.failOnce {
|
|
c.failOnce = false
|
|
return errors.New("injected cleanup failure")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *fakeWorkerStateCleaner) CleanupCompleted(operationID, _ string) error {
|
|
return c.Cleanup(operationID)
|
|
}
|
|
|
|
func terminalOperation(t *testing.T, journal *operation.Journal, operationID string) operation.CompletionReceipt {
|
|
t.Helper()
|
|
spec := operation.Spec{
|
|
OperationID: operationID,
|
|
ResourceID: "vm-completion-cleanup",
|
|
Action: "delete-vm",
|
|
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/vm-completion-cleanup", []byte(`{}`)),
|
|
Destructive: true,
|
|
}
|
|
if _, _, err := journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"deleted":true}`), nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, err := journal.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 {
|
|
t.Fatalf("claim completion = %#v err=%v", claimed, err)
|
|
}
|
|
return claimed[0]
|
|
}
|
|
|
|
func TestCompletionAckDoesNotDependOnDisposableWorkerCleanup(t *testing.T) {
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer journal.Close()
|
|
receipt := terminalOperation(t, journal, "operation-ack-cleanup")
|
|
cleaner := &fakeWorkerStateCleaner{failOnce: true}
|
|
outbox := &workerCompletionOutbox{journal: journal, cleaner: cleaner}
|
|
|
|
if err := outbox.AcknowledgeCompletion(receipt.EventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if pending, err := journal.PendingCompletionOperationIDs(); err != nil || len(pending) != 0 {
|
|
t.Fatalf("acked completion remains pending: %#v err=%v", pending, err)
|
|
}
|
|
record, err := journal.Get(receipt.OperationID)
|
|
if err != nil || record.State != operation.StateSucceeded {
|
|
t.Fatalf("ACK changed permanent operation tombstone: %#v err=%v", record, err)
|
|
}
|
|
cleaner.mu.Lock()
|
|
calls := append([]string(nil), cleaner.calls...)
|
|
cleaner.mu.Unlock()
|
|
if len(calls) != 1 || calls[0] != receipt.OperationID {
|
|
t.Fatalf("cleanup calls = %#v", calls)
|
|
}
|
|
if err := outbox.AcknowledgeCompletion(receipt.EventID); err != nil {
|
|
t.Fatalf("duplicate ACK was not idempotent: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRecoveredWorkerCleanupDoesNotDeleteMainCompletion(t *testing.T) {
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer journal.Close()
|
|
receipt := terminalOperation(t, journal, "operation-recovered-helper-cleanup")
|
|
state := &fakeRecoveredWorkerState{}
|
|
if err := cleanupRecoveredWorkerState(state, map[string]struct{}{receipt.OperationID: {}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state.mu.Lock()
|
|
calls := append([]string(nil), state.calls...)
|
|
state.mu.Unlock()
|
|
if len(calls) != 1 || calls[0] != receipt.OperationID {
|
|
t.Fatalf("recovered worker cleanup calls = %#v", calls)
|
|
}
|
|
if pending, err := journal.PendingCompletionOperationIDs(); err != nil {
|
|
t.Fatal(err)
|
|
} else if _, exists := pending[receipt.OperationID]; !exists {
|
|
t.Fatal("disposable helper cleanup deleted the main completion outbox")
|
|
}
|
|
record, err := journal.Get(receipt.OperationID)
|
|
if err != nil || record.State != operation.StateSucceeded {
|
|
t.Fatalf("helper cleanup changed operation tombstone: %#v err=%v", record, err)
|
|
}
|
|
}
|
|
|
|
func TestRecoveryPreservesUnadoptedWorkerEvidence(t *testing.T) {
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer journal.Close()
|
|
receipt := terminalOperation(t, journal, "operation-unadopted-evidence")
|
|
state := &fakeRecoveredWorkerState{terminals: []worker.TerminalEvidence{{
|
|
Record: operation.Record{
|
|
OperationID: receipt.OperationID, ResourceID: receipt.ResourceID,
|
|
Action: "power-reset", Fingerprint: receipt.RequestFingerprint,
|
|
State: operation.StateSucceeded,
|
|
},
|
|
}}}
|
|
|
|
if err := recoverInterruptedWorkerState(journal, state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state.mu.Lock()
|
|
calls := append([]string(nil), state.calls...)
|
|
state.mu.Unlock()
|
|
if len(calls) != 0 {
|
|
t.Fatalf("unadopted worker evidence was cleaned: %#v", calls)
|
|
}
|
|
}
|
|
|
|
func TestRecoveryScanErrorStopsBeforeWorkerCleanup(t *testing.T) {
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer journal.Close()
|
|
state := &fakeRecoveredWorkerState{terminalErr: errors.New("worker journal is temporarily locked")}
|
|
if err := recoverInterruptedWorkerState(journal, state); err == nil || !strings.Contains(err.Error(), "temporarily locked") {
|
|
t.Fatalf("recovery scan error = %v", err)
|
|
}
|
|
state.mu.Lock()
|
|
calls := append([]string(nil), state.calls...)
|
|
state.mu.Unlock()
|
|
if len(calls) != 0 {
|
|
t.Fatalf("cleanup ran after incomplete recovery scan: %#v", calls)
|
|
}
|
|
}
|
|
|
|
func TestRecoveryAdoptsTerminalChildJournalBeforeCompletionPublication(t *testing.T) {
|
|
temp := t.TempDir()
|
|
resource := "123e4567-e89b-42d3-a456-426614174010"
|
|
target := "/v1/vms/" + resource + "/power"
|
|
body := []byte(`{"operation_id":"operation-child-terminal","action":"reboot"}`)
|
|
spec := operation.Spec{
|
|
OperationID: "operation-child-terminal", ResourceID: resource, Action: "power-reboot",
|
|
Fingerprint: operation.Fingerprint("POST", target, body), Destructive: true,
|
|
}
|
|
journal := reopenDispatchingJournal(t, filepath.Join(temp, "main.db"), spec)
|
|
executor, err := worker.NewSubprocessExecutor(
|
|
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
operationDir := filepath.Join(temp, "workers", spec.OperationID)
|
|
writeLegacyTerminalWorkerJournal(t, operationDir, spec)
|
|
|
|
if err := recoverInterruptedWorkerState(journal, executor); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record, err := journal.Get(spec.OperationID)
|
|
if err != nil || record.State != operation.StateSucceeded || string(record.Result) != `{"state":"running"}` ||
|
|
record.EffectMayHaveApplied || record.ExecutionFenced {
|
|
t.Fatalf("adopted terminal record = %#v err=%v", record, err)
|
|
}
|
|
if _, err := os.Stat(operationDir); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("adopted worker directory was not cleaned: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCorruptRecoveryIdentityDoesNotManufactureSuccessOrBlockStartup(t *testing.T) {
|
|
temp := t.TempDir()
|
|
resource := "123e4567-e89b-42d3-a456-426614174011"
|
|
target := "/v1/vms/" + resource + "/power"
|
|
body := []byte(`{"operation_id":"operation-corrupt-recovery","action":"reset"}`)
|
|
spec := operation.Spec{
|
|
OperationID: "operation-corrupt-recovery", ResourceID: resource, Action: "power-reset",
|
|
Fingerprint: operation.Fingerprint("POST", target, body), Destructive: true,
|
|
}
|
|
journal := reopenDispatchingJournal(t, filepath.Join(temp, "main.db"), spec)
|
|
executor, err := worker.NewSubprocessExecutor(
|
|
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
operationDir := filepath.Join(temp, "workers", spec.OperationID)
|
|
writeLegacyTerminalWorkerJournal(t, operationDir, spec)
|
|
if err := os.WriteFile(filepath.Join(operationDir, "request-identity.json"), []byte(`{"operation_id":"wrong"}`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err := recoverInterruptedWorkerState(journal, executor); err != nil {
|
|
t.Fatalf("permanently invalid evidence blocked startup: %v", err)
|
|
}
|
|
record, err := journal.Get(spec.OperationID)
|
|
if err != nil || record.State != operation.StateFailed || record.Error == nil ||
|
|
record.Error.Code != worker.ResultUnconfirmed || !record.EffectMayHaveApplied || !record.ExecutionFenced {
|
|
t.Fatalf("corrupt evidence terminal = %#v err=%v", record, err)
|
|
}
|
|
if _, err := os.Stat(operationDir); err != nil {
|
|
t.Fatalf("corrupt worker evidence was destroyed: %v", err)
|
|
}
|
|
outbox := &workerCompletionOutbox{journal: journal, cleaner: executor}
|
|
claimed, err := outbox.ClaimPendingCompletions(1, nil)
|
|
if err != nil || len(claimed) != 1 || claimed[0].OperationID != spec.OperationID {
|
|
t.Fatalf("claim unconfirmed completion = %#v err=%v", claimed, err)
|
|
}
|
|
if _, err := os.Stat(operationDir); err != nil {
|
|
t.Fatalf("completion claim destroyed unadopted worker evidence: %v", err)
|
|
}
|
|
if err := outbox.AcknowledgeCompletion(claimed[0].EventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(operationDir); err != nil {
|
|
t.Fatalf("completion ACK destroyed unadopted worker evidence: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTransientRecoveryScanErrorLeavesMainDispatchingForRetry(t *testing.T) {
|
|
temp := t.TempDir()
|
|
resource := "123e4567-e89b-42d3-a456-426614174012"
|
|
spec := operation.Spec{
|
|
OperationID: "operation-transient-scan", ResourceID: resource, Action: "power-reboot",
|
|
Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+resource+"/power", []byte(`{"action":"reboot"}`)),
|
|
Destructive: true,
|
|
}
|
|
journal := reopenDispatchingJournal(t, filepath.Join(temp, "main.db"), spec)
|
|
state := &fakeRecoveredWorkerState{terminalErr: errors.New("worker journal is temporarily locked")}
|
|
if err := recoverInterruptedWorkerState(journal, state); err == nil {
|
|
t.Fatal("transient recovery scan error was ignored")
|
|
}
|
|
record, err := journal.Get(spec.OperationID)
|
|
if err != nil || record.State != operation.StateDispatching {
|
|
t.Fatalf("transient scan irreversibly terminalized main record = %#v err=%v", record, err)
|
|
}
|
|
state.mu.Lock()
|
|
calls := append([]string(nil), state.calls...)
|
|
state.mu.Unlock()
|
|
if len(calls) != 0 {
|
|
t.Fatalf("worker evidence was cleaned after transient scan error: %#v", calls)
|
|
}
|
|
}
|
|
|
|
func TestCompletionRemainsDeliverableUntilAcknowledged(t *testing.T) {
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer journal.Close()
|
|
now := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC)
|
|
journal.SetClockForTesting(func() time.Time { return now })
|
|
receipt := terminalOperation(t, journal, "operation-durable-delivery")
|
|
now = now.Add(25 * time.Hour)
|
|
cleaner := &fakeWorkerStateCleaner{failOnce: true}
|
|
outbox := &workerCompletionOutbox{journal: journal, cleaner: cleaner}
|
|
claimed, err := outbox.ClaimPendingCompletions(1, nil)
|
|
if err != nil || len(claimed) != 1 || claimed[0].EventID != receipt.EventID {
|
|
t.Fatalf("25-hour claim = %#v err=%v", claimed, err)
|
|
}
|
|
now = now.Add(365 * 24 * time.Hour)
|
|
claimed, err = outbox.ClaimPendingCompletions(1, nil)
|
|
if err != nil || len(claimed) != 1 || claimed[0].EventID != receipt.EventID {
|
|
t.Fatalf("long-lived claim = %#v err=%v", claimed, err)
|
|
}
|
|
cleaner.mu.Lock()
|
|
calls := append([]string(nil), cleaner.calls...)
|
|
cleaner.mu.Unlock()
|
|
if len(calls) != 2 || calls[0] != receipt.OperationID || calls[1] != receipt.OperationID {
|
|
t.Fatalf("delivery cleanup calls = %#v", calls)
|
|
}
|
|
if pending, err := journal.PendingCompletionOperationIDs(); err != nil {
|
|
t.Fatal(err)
|
|
} else if _, exists := pending[receipt.OperationID]; !exists {
|
|
t.Fatal("disposable delivery cleanup deleted the main completion outbox")
|
|
}
|
|
if err := outbox.AcknowledgeCompletion(receipt.EventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cleaner.mu.Lock()
|
|
calls = append([]string(nil), cleaner.calls...)
|
|
cleaner.mu.Unlock()
|
|
if len(calls) != 3 || calls[2] != receipt.OperationID {
|
|
t.Fatalf("ACK cleanup calls = %#v", calls)
|
|
}
|
|
if pending, err := journal.PendingCompletionOperationIDs(); err != nil || len(pending) != 0 {
|
|
t.Fatalf("ACK left completion pending: %#v err=%v", pending, err)
|
|
}
|
|
}
|
|
|
|
func TestRunControlUntilStoppedCancelsOnBackgroundFinalizationFailure(t *testing.T) {
|
|
failures := make(chan error, 1)
|
|
failures <- errors.New("injected journal write failure")
|
|
stopped := make(chan struct{})
|
|
err := runControlUntilStopped(context.Background(), failures, func(ctx context.Context) error {
|
|
<-ctx.Done()
|
|
close(stopped)
|
|
return ctx.Err()
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "durable background operation finalization failed") {
|
|
t.Fatalf("run result = %v", err)
|
|
}
|
|
select {
|
|
case <-stopped:
|
|
default:
|
|
t.Fatal("control channel was not stopped before restart")
|
|
}
|
|
}
|