You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/operation/journal_test.go

971 lines
35 KiB
Go

package operation
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
bolt "go.etcd.io/bbolt"
)
type testFailure struct {
code string
message string
effect bool
safe bool
fence bool
}
func (e testFailure) Error() string { return e.message }
func (e testFailure) PublicCode() string { return e.code }
func (e testFailure) PublicMessage() string { return e.message }
func (e testFailure) FailureSemantics() (bool, bool) {
return e.effect, e.safe
}
func (e testFailure) RequiresExecutionFence() bool { return e.fence }
func openTestJournal(t *testing.T) *Journal {
t.Helper()
j, err := Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = j.Close() })
return j
}
func testSpec(id, resource string) Spec {
return Spec{
OperationID: id,
ResourceID: resource,
Action: "delete-vm",
Fingerprint: Fingerprint("DELETE", "/v1/vms/"+resource, []byte(`{}`)),
Destructive: true,
}
}
func storageConfigurationSpec(id string) Spec {
return Spec{
OperationID: id,
ResourceID: "host-storage",
Action: "configure-storage",
Fingerprint: Fingerprint("POST", "/v1/storage/configuration", []byte(id)),
Destructive: true,
LockKeys: []string{"storage-config"},
HeavyIO: true,
}
}
func TestDefaultResourceLockKeyUsesCurrentNamespace(t *testing.T) {
key := defaultResourceLockKey("operation-resource-key", "opaque-resource", "delete-vm")
if len(key) != len("resource-hash:")+64 || key[:len("resource-hash:")] != "resource-hash:" {
t.Fatalf("default resource key = %q", key)
}
if err := validateLockKey(key); err != nil {
t.Fatalf("current hashed resource key rejected: %v", err)
}
if err := validateLockKey("legacy:" + key[len("resource-hash:"):]); err == nil {
t.Fatal("retired legacy resource key was accepted")
}
}
func TestStorageConfigurationTerminalFailureDoesNotFenceNewOperation(t *testing.T) {
for _, test := range []struct {
name string
executionFenced bool
}{
{name: "effect unknown"},
{name: "execution fenced", executionFenced: true},
} {
t.Run(test.name, func(t *testing.T) {
j := openTestJournal(t)
first := storageConfigurationSpec("operation-storage-failed")
var calls atomic.Int32
record, replayed, err := j.Execute(context.Background(), first, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return nil, testFailure{
code: "storage_configuration_failed", message: "storage configuration failed",
effect: true, safe: false, fence: test.executionFenced,
}
})
if err == nil || replayed || record.State != StateFailed || !record.EffectMayHaveApplied ||
record.SafeToResubmit || record.ExecutionFenced != test.executionFenced {
t.Fatalf("first storage result = %#v replayed=%v err=%v", record, replayed, err)
}
replayedRecord, replayed, err := j.Execute(context.Background(), first, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return nil, errors.New("same operation must not run again")
})
if err != nil || !replayed || replayedRecord.State != StateFailed || calls.Load() != 1 {
t.Fatalf("same operation replay = %#v replayed=%v calls=%d err=%v", replayedRecord, replayed, calls.Load(), err)
}
successor := storageConfigurationSpec("operation-storage-successor")
succeeded, replayed, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return json.RawMessage(`{"configured":true}`), nil
})
if err != nil || replayed || succeeded.State != StateSucceeded || calls.Load() != 2 {
t.Fatalf("independent successor = %#v replayed=%v calls=%d err=%v", succeeded, replayed, calls.Load(), err)
}
})
}
}
func TestStorageConfigurationAdmissionRemainsSerializedWhileRunning(t *testing.T) {
j := openTestJournal(t)
firstStarted := make(chan struct{})
releaseFirst := make(chan struct{})
secondStarted := make(chan struct{})
firstDone := make(chan error, 1)
secondDone := make(chan error, 1)
go func() {
_, _, err := j.Execute(context.Background(), storageConfigurationSpec("operation-storage-running"), func(context.Context) (json.RawMessage, error) {
close(firstStarted)
<-releaseFirst
return nil, testFailure{
code: "storage_configuration_failed", message: "storage configuration failed",
effect: true, safe: false,
}
})
firstDone <- err
}()
<-firstStarted
go func() {
_, _, err := j.Execute(context.Background(), storageConfigurationSpec("operation-storage-waiting"), func(context.Context) (json.RawMessage, error) {
close(secondStarted)
return json.RawMessage(`{"configured":true}`), nil
})
secondDone <- err
}()
deadline := time.Now().Add(time.Second)
for {
select {
case <-secondStarted:
t.Fatal("second storage mutation crossed admission while the first was running")
default:
}
record, err := j.Get("operation-storage-waiting")
if err == nil && record.State == StateReserved {
break
}
if time.Now().After(deadline) {
t.Fatalf("second storage operation did not reach serialized admission: %#v err=%v", record, err)
}
time.Sleep(time.Millisecond)
}
close(releaseFirst)
if err := <-firstDone; err == nil {
t.Fatal("first storage mutation unexpectedly succeeded")
}
if err := <-secondDone; err != nil {
t.Fatalf("second storage mutation remained blocked after terminal failure: %v", err)
}
}
func TestOperationIDIsPermanentTombstoneAndMutationRunsOnce(t *testing.T) {
j := openTestJournal(t)
spec := testSpec("operation-single-dispatch", "vm-single")
var calls atomic.Int32
first, replayed, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return json.RawMessage(`{"deleted":true}`), nil
})
if err != nil || replayed || first.State != StateSucceeded {
t.Fatalf("first execution = %#v replayed=%v err=%v", first, replayed, err)
}
second, replayed, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return nil, errors.New("must not run")
})
if err != nil || !replayed || second.State != StateSucceeded || calls.Load() != 1 {
t.Fatalf("replay = %#v replayed=%v calls=%d err=%v", second, replayed, calls.Load(), err)
}
conflict := spec
conflict.Fingerprint = Fingerprint("DELETE", "/v1/vms/vm-single", []byte(`{"different":true}`))
if _, _, err := j.Execute(context.Background(), conflict, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return nil, nil
}); !errors.Is(err, ErrFingerprint) || calls.Load() != 1 {
t.Fatalf("conflicting replay err=%v calls=%d", err, calls.Load())
}
}
func TestCanceledExecutionKeepsAdmissionUntilWorkerReturns(t *testing.T) {
j := openTestJournal(t)
resource := "vm-reap-order"
first := testSpec("operation-reap-first", resource)
started := make(chan struct{})
release := make(chan struct{})
workerReturned := make(chan struct{})
ctx, cancel := context.WithCancel(context.Background())
firstDone := make(chan error, 1)
go func() {
_, _, err := j.Execute(ctx, first, func(context.Context) (json.RawMessage, error) {
close(started)
<-release
close(workerReturned)
return nil, context.DeadlineExceeded
})
firstDone <- err
}()
<-started
cancel()
second := testSpec("operation-reap-second", resource)
secondStarted := make(chan struct{})
secondDone := make(chan error, 1)
go func() {
_, _, err := j.Execute(context.Background(), second, func(context.Context) (json.RawMessage, error) {
close(secondStarted)
return json.RawMessage(`{"deleted":true}`), nil
})
secondDone <- err
}()
select {
case <-secondStarted:
t.Fatal("successor crossed dispatch before the first worker returned")
case <-time.After(40 * time.Millisecond):
}
close(release)
<-workerReturned
if err := <-firstDone; !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("first error = %v", err)
}
if err := <-secondDone; !errors.Is(err, ErrResourceBlocked) {
t.Fatalf("ambiguous first result did not durably fence successor: %v", err)
}
select {
case <-secondStarted:
t.Fatal("fenced successor unexpectedly invoked its mutation")
default:
}
}
func TestBackgroundWorkerDeadlineStartsAfterAdmission(t *testing.T) {
j := openTestJournal(t)
resource := "vm-background-deadline"
firstStarted := make(chan struct{})
releaseFirst := make(chan struct{})
firstDone := make(chan error, 1)
go func() {
_, _, err := j.Execute(context.Background(), testSpec("operation-deadline-first", resource), func(context.Context) (json.RawMessage, error) {
close(firstStarted)
<-releaseFirst
return json.RawMessage(`{"done":true}`), nil
})
firstDone <- err
}()
<-firstStarted
go func() {
time.Sleep(60 * time.Millisecond)
close(releaseFirst)
}()
admissionCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
secondID := "operation-deadline-second"
record, replayed, done, err := j.ExecuteBackground(
admissionCtx, context.Background(), 30*time.Millisecond,
testSpec(secondID, resource),
func(ctx context.Context) (json.RawMessage, error) {
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("execution budget expired before worker start: %w", err)
}
return json.RawMessage(`{"done":true}`), nil
},
)
if err != nil || replayed || done == nil || record.State != StateDispatching {
t.Fatalf("background start = %#v replayed=%v done=%v err=%v", record, replayed, done != nil, err)
}
<-done
if err := <-firstDone; err != nil {
t.Fatalf("first execution: %v", err)
}
terminal, err := j.Get(secondID)
if err != nil || terminal.State != StateSucceeded {
t.Fatalf("second terminal = %#v err=%v", terminal, err)
}
}
func TestBackgroundFinalizationFailureForcesRestartRecoveryWithoutReplay(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
j, err := Open(path)
if err != nil {
t.Fatal(err)
}
spec := testSpec("operation-background-persist-failure", "vm-background-persist-failure")
started := make(chan struct{})
release := make(chan struct{})
var calls atomic.Int32
record, replayed, done, err := j.ExecuteBackground(
context.Background(), context.Background(), time.Minute, spec,
func(context.Context) (json.RawMessage, error) {
calls.Add(1)
close(started)
<-release
return json.RawMessage(`{"done":true}`), nil
},
)
if err != nil || replayed || done == nil || record.State != StateDispatching {
t.Fatalf("background start = %#v replayed=%v done=%v err=%v", record, replayed, done != nil, err)
}
<-started
if err := j.Close(); err != nil {
t.Fatal(err)
}
close(release)
<-done
select {
case backgroundErr := <-j.BackgroundFailures():
if !strings.Contains(backgroundErr.Error(), "finalize background operation "+spec.OperationID) {
t.Fatalf("background failure = %v", backgroundErr)
}
case <-time.After(time.Second):
t.Fatal("background persistence failure was silently discarded")
}
reopened, err := Open(path)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
finalized, err := reopened.FinalizeInterruptedOperations([]InterruptedReceipt{{
OperationID: spec.OperationID, Fingerprint: spec.Fingerprint,
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
}})
if err != nil || len(finalized) != 1 || finalized[0] != spec.OperationID {
t.Fatalf("restart finalization = %v err=%v", finalized, err)
}
terminal, err := reopened.Get(spec.OperationID)
if err != nil || terminal.State != StateSucceeded || calls.Load() != 1 {
t.Fatalf("restart terminal = %#v calls=%d err=%v", terminal, calls.Load(), err)
}
}
func TestBackgroundFencePersistenceFailureForcesUnconfirmedRestart(t *testing.T) {
oldDeadline := workerReapDeadline
workerReapDeadline = 25 * time.Millisecond
t.Cleanup(func() { workerReapDeadline = oldDeadline })
path := filepath.Join(t.TempDir(), "operations.db")
j, err := Open(path)
if err != nil {
t.Fatal(err)
}
spec := testSpec("operation-background-fence-failure", "vm-background-fence-failure")
workerParent, cancelWorker := context.WithCancel(context.Background())
started := make(chan struct{})
release := make(chan struct{})
var calls atomic.Int32
record, replayed, done, err := j.ExecuteBackground(
context.Background(), workerParent, time.Minute, spec,
func(context.Context) (json.RawMessage, error) {
calls.Add(1)
close(started)
<-release
return json.RawMessage(`{"late":true}`), nil
},
)
if err != nil || replayed || done == nil || record.State != StateDispatching {
t.Fatalf("background start = %#v replayed=%v done=%v err=%v", record, replayed, done != nil, err)
}
<-started
if err := j.Close(); err != nil {
t.Fatal(err)
}
cancelWorker()
<-done
select {
case backgroundErr := <-j.BackgroundFailures():
if !strings.Contains(backgroundErr.Error(), "fence background operation "+spec.OperationID) {
t.Fatalf("background failure = %v", backgroundErr)
}
case <-time.After(time.Second):
t.Fatal("background fence persistence failure was silently discarded")
}
close(release)
reopened, err := Open(path)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
finalized, err := reopened.FinalizeInterruptedOperations(nil)
if err != nil || len(finalized) != 1 || finalized[0] != spec.OperationID {
t.Fatalf("restart finalization = %v err=%v", finalized, err)
}
terminal, err := reopened.Get(spec.OperationID)
if err != nil || terminal.State != StateFailed || terminal.Error == nil ||
terminal.Error.Code != "result_unconfirmed" || !terminal.ExecutionFenced || calls.Load() != 1 {
t.Fatalf("restart terminal = %#v calls=%d err=%v", terminal, calls.Load(), err)
}
}
func TestUnreapedExecutionBecomesTerminalDurableFence(t *testing.T) {
oldDeadline := workerReapDeadline
workerReapDeadline = 25 * time.Millisecond
t.Cleanup(func() { workerReapDeadline = oldDeadline })
j := openTestJournal(t)
resource := "vm-unreaped"
ctx, cancel := context.WithCancel(context.Background())
started := make(chan struct{})
release := make(chan struct{})
defer close(release)
go func() {
<-started
cancel()
}()
record, _, err := j.Execute(ctx, testSpec("operation-unreaped-worker", resource), func(context.Context) (json.RawMessage, error) {
close(started)
<-release
return nil, nil
})
if !errors.Is(err, ErrExecutionNotReaped) || record.State != StateFailed || !record.ExecutionFenced ||
!record.EffectMayHaveApplied || record.SafeToResubmit {
t.Fatalf("unreaped result = %#v err=%v", record, err)
}
called := false
blocked, _, blockErr := j.Execute(context.Background(), testSpec("operation-after-unreaped", resource), func(context.Context) (json.RawMessage, error) {
called = true
return nil, nil
})
if !errors.Is(blockErr, ErrResourceBlocked) || called || !WasNotDispatched(blocked) {
t.Fatalf("durable fence = %#v called=%v err=%v", blocked, called, blockErr)
}
}
func TestRestartNeverReplaysReservedOrDispatchingMutation(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
j, err := Open(path)
if err != nil {
t.Fatal(err)
}
reserved := testSpec("operation-restart-reserved", "vm-reserved")
if _, _, err := j.Reserve(reserved); err != nil {
t.Fatal(err)
}
dispatching := testSpec("operation-restart-dispatched", "vm-dispatched")
if _, _, err := j.Reserve(dispatching); err != nil {
t.Fatal(err)
}
if _, err := j.markDispatching(dispatching.OperationID); err != nil {
t.Fatal(err)
}
if err := j.Close(); err != nil {
t.Fatal(err)
}
j, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer j.Close()
storedReserved, err := j.Get(reserved.OperationID)
if err != nil || !WasNotDispatched(storedReserved) || !storedReserved.SafeToResubmit || storedReserved.EffectMayHaveApplied {
t.Fatalf("reserved restart = %#v err=%v", storedReserved, err)
}
finalized, err := j.FinalizeInterruptedOperations(nil)
if err != nil || len(finalized) != 1 || finalized[0] != dispatching.OperationID {
t.Fatalf("dispatching finalization = %v err=%v", finalized, err)
}
storedDispatching, err := j.Get(dispatching.OperationID)
if err != nil || storedDispatching.State != StateFailed || storedDispatching.Error == nil ||
storedDispatching.Error.Code != "result_unconfirmed" || !storedDispatching.ExecutionFenced ||
!storedDispatching.EffectMayHaveApplied || storedDispatching.SafeToResubmit {
t.Fatalf("dispatching restart = %#v err=%v", storedDispatching, err)
}
}
func TestRestartImportsExactImmutableWorkerReceipt(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
j, err := Open(path)
if err != nil {
t.Fatal(err)
}
spec := testSpec("operation-receipt-import", "vm-receipt")
if _, _, err := j.Reserve(spec); err != nil {
t.Fatal(err)
}
if _, err := j.markDispatching(spec.OperationID); err != nil {
t.Fatal(err)
}
if err := j.Close(); err != nil {
t.Fatal(err)
}
j, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer j.Close()
want := json.RawMessage(`{"deleted":true}`)
finalized, err := j.FinalizeInterruptedOperations([]InterruptedReceipt{{
OperationID: spec.OperationID,
Fingerprint: spec.Fingerprint,
Status: "succeeded",
Result: want,
}})
if err != nil || len(finalized) != 1 {
t.Fatalf("receipt finalization = %v err=%v", finalized, err)
}
record, err := j.Get(spec.OperationID)
if err != nil || record.State != StateSucceeded || string(record.Result) != string(want) {
t.Fatalf("imported receipt = %#v err=%v", record, err)
}
if again, err := j.FinalizeInterruptedOperations(nil); err != nil || len(again) != 0 {
t.Fatalf("second finalization = %v err=%v", again, err)
}
}
func TestCompletionReceiptIsImmutableAndAckOnlyClearsOutbox(t *testing.T) {
j := openTestJournal(t)
now := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC)
j.SetClockForTesting(func() time.Time { return now })
spec := testSpec("operation-completion-event", "vm-completion")
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"deleted":true}`), nil
}); err != nil {
t.Fatal(err)
}
claimed, err := j.ClaimPendingCompletions(1)
if err != nil || len(claimed) != 1 {
t.Fatalf("completion claim = %#v err=%v", claimed, err)
}
receipt := claimed[0]
if receipt.Status != "succeeded" || receipt.Result == nil || string(receipt.Result) != `{"deleted":true}` {
t.Fatalf("completion receipt = %#v", receipt)
}
if err := j.AcknowledgeCompletion(receipt.EventID); err != nil {
t.Fatal(err)
}
now = now.Add(48 * time.Hour)
if pending, err := j.ClaimPendingCompletions(1); err != nil || len(pending) != 0 {
t.Fatalf("acked completion remained pending: %#v err=%v", pending, err)
}
record, err := j.Get(spec.OperationID)
if err != nil || record.State != StateSucceeded || string(record.Result) != `{"deleted":true}` {
t.Fatalf("ACK rewrote operation: %#v err=%v", record, err)
}
var persisted CompletionReceipt
err = j.db.View(func(tx *bolt.Tx) error {
encoded := tx.Bucket(completionReceiptsBucket).Get([]byte(receipt.EventID))
return json.Unmarshal(encoded, &persisted)
})
if err != nil || persisted.EventID != receipt.EventID || string(persisted.Result) != string(receipt.Result) {
t.Fatalf("immutable receipt = %#v err=%v", persisted, err)
}
}
func TestCompletionOutboxHasFiniteAttemptsAndBackoff(t *testing.T) {
j := openTestJournal(t)
now := time.Date(2026, 10, 19, 1, 0, 0, 0, time.UTC)
j.SetClockForTesting(func() time.Time { return now })
if _, _, err := j.Execute(context.Background(), testSpec("operation-bounded-outbox", "vm-outbox"), func(context.Context) (json.RawMessage, error) {
return nil, testFailure{code: "injected_failure", message: "injected", effect: false, safe: true}
}); err == nil {
t.Fatal("injected failure unexpectedly succeeded")
}
for attempt := uint32(0); attempt < completionDeliveryMaxAttempts; attempt++ {
claimed, err := j.ClaimPendingCompletions(1)
if err != nil || len(claimed) != 1 {
t.Fatalf("attempt %d claim = %#v err=%v", attempt+1, claimed, err)
}
if immediate, err := j.ClaimPendingCompletions(1); err != nil || len(immediate) != 0 {
t.Fatalf("attempt %d ignored backoff: %#v err=%v", attempt+1, immediate, err)
}
now = now.Add(completionDeliveryInterval)
}
if exhausted, err := j.ClaimPendingCompletions(1); err != nil || len(exhausted) != 0 {
t.Fatalf("exhausted outbox = %#v err=%v", exhausted, err)
}
}
func TestCompletionOutboxDoesNotChargeInflightACKWindowTwice(t *testing.T) {
j := openTestJournal(t)
now := time.Date(2026, 10, 19, 1, 30, 0, 0, time.UTC)
j.SetClockForTesting(func() time.Time { return now })
if _, _, err := j.Execute(context.Background(), testSpec("operation-ack-window", "vm-ack-window"), func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"done":true}`), nil
}); err != nil {
t.Fatal(err)
}
first, err := j.ClaimPendingCompletions(1)
if err != nil || len(first) != 1 {
t.Fatalf("first claim = %#v err=%v", first, err)
}
now = now.Add(completionDeliveryInterval)
excluded, _, err := j.ClaimPendingCompletionsWithExpiredExcluding(1, []string{first[0].EventID})
if err != nil || len(excluded) != 0 {
t.Fatalf("inflight event was reclaimed: %#v err=%v", excluded, err)
}
retry, err := j.ClaimPendingCompletions(1)
if err != nil || len(retry) != 1 || retry[0].EventID != first[0].EventID {
t.Fatalf("excluded claim consumed an attempt/backoff: %#v err=%v", retry, err)
}
}
func TestFenceResolutionIsAuditedIdempotentAndDoesNotRewriteTerminalEvidence(t *testing.T) {
j := openTestJournal(t)
now := time.Date(2026, 10, 19, 2, 0, 0, 0, time.UTC)
j.SetClockForTesting(func() time.Time { return now })
spec := testSpec("operation-effect-fence", "vm-effect-fence")
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"partial":true}`), testFailure{
code: "injected_effect", message: "effect may have applied", effect: true, safe: false,
}
}); err == nil {
t.Fatal("effectful failure unexpectedly succeeded")
}
claimed, err := j.ClaimPendingCompletions(1)
if err != nil || len(claimed) != 1 {
t.Fatalf("claim original completion = %#v err=%v", claimed, err)
}
eventID := claimed[0].EventID
var operationBefore, receiptBefore []byte
if err := j.db.View(func(tx *bolt.Tx) error {
operationBefore = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(spec.OperationID))...)
receiptBefore = append([]byte(nil), tx.Bucket(completionReceiptsBucket).Get([]byte(eventID))...)
return nil
}); err != nil {
t.Fatal(err)
}
request := FenceResolutionRequest{
ResolutionID: "0123456789abcdef0123456789abcdef",
Resolution: FenceResolutionNotApplied, InspectionCheckedAt: now.Unix(),
ReasonSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
resolution, replayed, err := j.ResolveResourceFence(spec.OperationID, 42, request)
if err != nil || replayed || resolution.OperationID != spec.OperationID || resolution.ResolvedBy != 42 {
t.Fatalf("resolve fence = %#v replayed=%v err=%v", resolution, replayed, err)
}
replayedResolution, replayed, err := j.ResolveResourceFence(spec.OperationID, 42, request)
if err != nil || !replayed || replayedResolution != resolution {
t.Fatalf("idempotent resolve = %#v replayed=%v err=%v", replayedResolution, replayed, err)
}
conflict := request
conflict.Resolution = FenceResolutionApplied
if _, _, err := j.ResolveResourceFence(spec.OperationID, 42, conflict); !errors.Is(err, ErrFenceResolutionConflict) {
t.Fatalf("conflicting resolution err=%v", err)
}
stored, err := j.GetFenceResolution(spec.OperationID)
if err != nil || stored != resolution {
t.Fatalf("stored fence resolution = %#v err=%v", stored, err)
}
var operationAfter, receiptAfter []byte
if err := j.db.View(func(tx *bolt.Tx) error {
operationAfter = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(spec.OperationID))...)
receiptAfter = append([]byte(nil), tx.Bucket(completionReceiptsBucket).Get([]byte(eventID))...)
return nil
}); err != nil {
t.Fatal(err)
}
if !bytes.Equal(operationBefore, operationAfter) || !bytes.Equal(receiptBefore, receiptAfter) {
t.Fatal("fence resolution rewrote the terminal operation or immutable completion receipt")
}
var calls atomic.Int32
successor := testSpec("operation-after-fence", spec.ResourceID)
if record, _, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return json.RawMessage(`{"accepted":true}`), nil
}); err != nil || record.State != StateSucceeded || calls.Load() != 1 {
t.Fatalf("successor after resolution = %#v calls=%d err=%v", record, calls.Load(), err)
}
}
func TestFenceResolutionRequiresFreshInspectionAndActiveBlocker(t *testing.T) {
j := openTestJournal(t)
now := time.Date(2026, 10, 19, 3, 0, 0, 0, time.UTC)
j.SetClockForTesting(func() time.Time { return now })
request := FenceResolutionRequest{
ResolutionID: "fedcba9876543210fedcba9876543210",
Resolution: FenceResolutionApplied, InspectionCheckedAt: now.Add(-3 * time.Minute).Unix(),
ReasonSHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
}
if _, _, err := j.ResolveResourceFence("operation-no-fence", 7, request); !errors.Is(err, ErrFenceInspectionStale) {
t.Fatalf("stale inspection err=%v", err)
}
request.InspectionCheckedAt = now.Unix()
if _, _, err := j.ResolveResourceFence("operation-no-fence", 7, request); !errors.Is(err, ErrNotFound) {
t.Fatalf("missing operation err=%v", err)
}
if _, _, err := j.Execute(context.Background(), testSpec("operation-clean-result", "vm-clean"), func(context.Context) (json.RawMessage, error) {
return nil, testFailure{code: "clean_failure", message: "not applied", effect: false, safe: true}
}); err == nil {
t.Fatal("seed clean failure unexpectedly succeeded")
}
if _, _, err := j.ResolveResourceFence("operation-clean-result", 7, request); !errors.Is(err, ErrFenceNotResolvable) {
t.Fatalf("non-effectful failure err=%v", err)
}
}
func TestSuccessfulForceSuccessorOnlyRemovesPredecessorBlockerIndex(t *testing.T) {
j := openTestJournal(t)
predecessor := testSpec("operation-graceful-predecessor", "vm-force-successor")
if _, _, err := j.Execute(context.Background(), predecessor, func(context.Context) (json.RawMessage, error) {
return nil, testFailure{code: "force_required", message: "explicit force required", effect: true, safe: false}
}); err == nil {
t.Fatal("graceful predecessor unexpectedly succeeded")
}
claimed, err := j.ClaimPendingCompletions(1)
if err != nil || len(claimed) != 1 {
t.Fatalf("claim predecessor completion = %#v err=%v", claimed, err)
}
var operationBefore, receiptBefore []byte
if err := j.db.View(func(tx *bolt.Tx) error {
operationBefore = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(predecessor.OperationID))...)
receiptBefore = append([]byte(nil), tx.Bucket(completionReceiptsBucket).Get([]byte(claimed[0].EventID))...)
return nil
}); err != nil {
t.Fatal(err)
}
successor := testSpec("operation-force-successor", predecessor.ResourceID)
successor.Action = "force-delete-vm"
successor.Fingerprint = Fingerprint("DELETE", "/v1/vms/"+predecessor.ResourceID+"?force=1", []byte(`{}`))
successor.SupersedesOperationID = predecessor.OperationID
if record, _, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"deleted":true}`), nil
}); err != nil || record.State != StateSucceeded {
t.Fatalf("force successor = %#v err=%v", record, err)
}
var operationAfter, receiptAfter []byte
if err := j.db.View(func(tx *bolt.Tx) error {
operationAfter = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(predecessor.OperationID))...)
receiptAfter = append([]byte(nil), tx.Bucket(completionReceiptsBucket).Get([]byte(claimed[0].EventID))...)
return nil
}); err != nil {
t.Fatal(err)
}
if !bytes.Equal(operationBefore, operationAfter) || !bytes.Equal(receiptBefore, receiptAfter) {
t.Fatal("force successor rewrote predecessor terminal evidence")
}
followup := testSpec("operation-after-force-successor", predecessor.ResourceID)
if record, _, err := j.Execute(context.Background(), followup, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"accepted":true}`), nil
}); err != nil || record.State != StateSucceeded {
t.Fatalf("predecessor blocker remained after force successor: %#v err=%v", record, err)
}
}
func TestInitializeRemovesLegacyStorageBlockerAndPreservesVMFence(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
j, err := Open(path)
if err != nil {
t.Fatal(err)
}
storage := storageConfigurationSpec("operation-legacy-storage")
if _, _, err := j.Execute(context.Background(), storage, func(context.Context) (json.RawMessage, error) {
return nil, testFailure{
code: "storage_configuration_failed", message: "storage effect unknown", effect: true, safe: false,
}
}); err == nil {
t.Fatal("legacy storage seed unexpectedly succeeded")
}
vm := testSpec("operation-legacy-vm-fence", "vm-legacy-fence")
if _, _, err := j.Execute(context.Background(), vm, func(context.Context) (json.RawMessage, error) {
return nil, testFailure{code: "vm_effect_unknown", message: "VM effect unknown", effect: true, safe: false}
}); err == nil {
t.Fatal("VM fence seed unexpectedly succeeded")
}
storageBlockerKey := blockerEntryKey("storage-config", storage.OperationID)
vmLockKeys := effectiveLockKeys(vm.OperationID, vm.ResourceID, vm.Action, vm.LockKeys)
vmBlockerKey := blockerEntryKey(vmLockKeys[0], vm.OperationID)
storageReceiptKey := []byte(completionEventID(storage.OperationID))
var operationBefore, receiptBefore []byte
if err := j.db.Update(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
blockers := tx.Bucket(resourceBlockersBucket)
receipts := tx.Bucket(completionReceiptsBucket)
if err := blockers.Put(storageBlockerKey, nil); err != nil {
return err
}
operationBefore = append([]byte(nil), operations.Get([]byte(storage.OperationID))...)
receiptBefore = append([]byte(nil), receipts.Get(storageReceiptKey)...)
if len(operationBefore) == 0 || len(receiptBefore) == 0 {
return errors.New("storage operation evidence was not persisted")
}
return nil
}); err != nil {
t.Fatal(err)
}
if err := j.Close(); err != nil {
t.Fatal(err)
}
j, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer j.Close()
if err := j.db.View(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
blockers := tx.Bucket(resourceBlockersBucket)
receipts := tx.Bucket(completionReceiptsBucket)
metadata := tx.Bucket(metadataBucket)
if string(metadata.Get(schemaVersionKey)) != string(schemaVersion) {
return fmt.Errorf("schema version changed during blocker repair")
}
cursor := blockers.Cursor()
if key, _ := cursor.Seek(storageBlockerKey); bytes.Equal(key, storageBlockerKey) {
return errors.New("legacy storage blocker survived restart repair")
}
if key, _ := cursor.Seek(vmBlockerKey); !bytes.Equal(key, vmBlockerKey) {
return errors.New("VM blocker was removed during storage repair")
}
if !bytes.Equal(operationBefore, operations.Get([]byte(storage.OperationID))) ||
!bytes.Equal(receiptBefore, receipts.Get(storageReceiptKey)) {
return errors.New("storage blocker repair rewrote immutable operation evidence")
}
return nil
}); err != nil {
t.Fatal(err)
}
successor := storageConfigurationSpec("operation-after-storage-repair")
if record, replayed, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"configured":true}`), nil
}); err != nil || replayed || record.State != StateSucceeded {
t.Fatalf("storage successor after restart repair = %#v replayed=%v err=%v", record, replayed, err)
}
vmCalled := false
if _, _, err := j.Execute(context.Background(), testSpec("operation-after-vm-fence", vm.ResourceID), func(context.Context) (json.RawMessage, error) {
vmCalled = true
return nil, nil
}); !errors.Is(err, ErrResourceBlocked) || vmCalled {
t.Fatalf("VM fence changed during storage repair: called=%v err=%v", vmCalled, err)
}
}
func TestInitializeRejectsBrokenResourceBlockerReferencesAtomically(t *testing.T) {
for _, test := range []struct {
name string
seed func(*bolt.Tx) error
}{
{
name: "missing operation",
seed: func(tx *bolt.Tx) error {
return tx.Bucket(resourceBlockersBucket).Put(
blockerEntryKey("storage-config", "operation-zz-missing-reference"), nil,
)
},
},
{
name: "damaged operation",
seed: func(tx *bolt.Tx) error {
operationID := "operation-zz-damaged-reference"
if err := tx.Bucket(operationsBucket).Put([]byte(operationID), []byte(`{"record":`)); err != nil {
return err
}
return tx.Bucket(resourceBlockersBucket).Put(blockerEntryKey("storage-config", operationID), nil)
},
},
} {
t.Run(test.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
j, err := Open(path)
if err != nil {
t.Fatal(err)
}
storage := storageConfigurationSpec("operation-valid-storage-reference")
if _, _, err := j.Execute(context.Background(), storage, func(context.Context) (json.RawMessage, error) {
return nil, testFailure{code: "storage_configuration_failed", message: "unknown", effect: true}
}); err == nil {
t.Fatal("storage seed unexpectedly succeeded")
}
storageBlockerKey := blockerEntryKey("storage-config", storage.OperationID)
if err := j.db.Update(func(tx *bolt.Tx) error {
if err := tx.Bucket(resourceBlockersBucket).Put(storageBlockerKey, nil); err != nil {
return err
}
return test.seed(tx)
}); err != nil {
t.Fatal(err)
}
if err := j.Close(); err != nil {
t.Fatal(err)
}
if reopened, err := Open(path); err == nil {
_ = reopened.Close()
t.Fatal("broken blocker reference was accepted")
}
db, err := bolt.Open(path, 0o600, nil)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := db.View(func(tx *bolt.Tx) error {
cursor := tx.Bucket(resourceBlockersBucket).Cursor()
key, _ := cursor.Seek(storageBlockerKey)
if !bytes.Equal(key, storageBlockerKey) {
return errors.New("failed startup partially deleted the valid storage blocker")
}
return nil
}); err != nil {
t.Fatal(err)
}
})
}
}
func TestFreshSchemaRejectsOldJournalWithoutMigration(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
db, err := bolt.Open(path, 0o600, nil)
if err != nil {
t.Fatal(err)
}
if err := db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucket(metadataBucket)
if err != nil {
return err
}
return bucket.Put(schemaVersionKey, []byte("4"))
}); err != nil {
t.Fatal(err)
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
if journal, err := Open(path); err == nil {
_ = journal.Close()
t.Fatal("old schema was silently migrated")
}
}
func TestFreshSchemaRejectsUnversionedExistingJournal(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
db, err := bolt.Open(path, 0o600, nil)
if err != nil {
t.Fatal(err)
}
if err := db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucket([]byte("legacy_operations"))
return err
}); err != nil {
t.Fatal(err)
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
if journal, err := Open(path); err == nil {
_ = journal.Close()
t.Fatal("unversioned existing journal was silently initialized")
}
}