You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2398 lines
92 KiB
Go
2398 lines
92 KiB
Go
package operation
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/publicnetwork"
|
|
bolt "go.etcd.io/bbolt"
|
|
)
|
|
|
|
type testFailure struct {
|
|
code string
|
|
message string
|
|
effect bool
|
|
safe bool
|
|
fence bool
|
|
}
|
|
|
|
func (e testFailure) Error() string { return e.message }
|
|
func (e testFailure) PublicCode() string { return e.code }
|
|
func (e testFailure) PublicMessage() string { return e.message }
|
|
func (e testFailure) FailureSemantics() (bool, bool) {
|
|
return e.effect, e.safe
|
|
}
|
|
func (e testFailure) RequiresExecutionFence() bool { return e.fence }
|
|
|
|
func openTestJournal(t *testing.T) *Journal {
|
|
t.Helper()
|
|
j, err := Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = j.Close() })
|
|
return j
|
|
}
|
|
|
|
func testSpec(id, resource string) Spec {
|
|
return Spec{
|
|
OperationID: id,
|
|
ResourceID: resource,
|
|
Action: "delete-vm",
|
|
Fingerprint: Fingerprint("DELETE", "/v1/vms/"+resource, []byte(`{}`)),
|
|
Destructive: true,
|
|
}
|
|
}
|
|
|
|
func storageConfigurationSpec(id string) Spec {
|
|
return Spec{
|
|
OperationID: id,
|
|
ResourceID: "host-storage",
|
|
Action: "configure-storage",
|
|
Fingerprint: Fingerprint("POST", "/v1/storage/configuration", []byte(id)),
|
|
Destructive: true,
|
|
LockKeys: []string{"storage-config"},
|
|
HeavyIO: true,
|
|
}
|
|
}
|
|
|
|
func TestDefaultResourceLockKeyUsesCurrentNamespace(t *testing.T) {
|
|
key := defaultResourceLockKey("operation-resource-key", "opaque-resource", "delete-vm")
|
|
if len(key) != len("resource-hash:")+64 || key[:len("resource-hash:")] != "resource-hash:" {
|
|
t.Fatalf("default resource key = %q", key)
|
|
}
|
|
if err := validateLockKey(key); err != nil {
|
|
t.Fatalf("current hashed resource key rejected: %v", err)
|
|
}
|
|
if err := validateLockKey("legacy:" + key[len("resource-hash:"):]); err == nil {
|
|
t.Fatal("retired legacy resource key was accepted")
|
|
}
|
|
}
|
|
|
|
func TestStorageConfigurationTerminalFailureDoesNotFenceNewOperation(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
executionFenced bool
|
|
}{
|
|
{name: "effect unknown"},
|
|
{name: "execution fenced", executionFenced: true},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
first := storageConfigurationSpec("operation-storage-failed")
|
|
var calls atomic.Int32
|
|
record, replayed, err := j.Execute(context.Background(), first, func(context.Context) (json.RawMessage, error) {
|
|
calls.Add(1)
|
|
return nil, testFailure{
|
|
code: "storage_configuration_failed", message: "storage configuration failed",
|
|
effect: true, safe: false, fence: test.executionFenced,
|
|
}
|
|
})
|
|
if err == nil || replayed || record.State != StateFailed || !record.EffectMayHaveApplied ||
|
|
record.SafeToResubmit || record.ExecutionFenced != test.executionFenced {
|
|
t.Fatalf("first storage result = %#v replayed=%v err=%v", record, replayed, err)
|
|
}
|
|
|
|
replayedRecord, replayed, err := j.Execute(context.Background(), first, func(context.Context) (json.RawMessage, error) {
|
|
calls.Add(1)
|
|
return nil, errors.New("same operation must not run again")
|
|
})
|
|
if err != nil || !replayed || replayedRecord.State != StateFailed || calls.Load() != 1 {
|
|
t.Fatalf("same operation replay = %#v replayed=%v calls=%d err=%v", replayedRecord, replayed, calls.Load(), err)
|
|
}
|
|
|
|
successor := storageConfigurationSpec("operation-storage-successor")
|
|
succeeded, replayed, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
|
|
calls.Add(1)
|
|
return json.RawMessage(`{"configured":true}`), nil
|
|
})
|
|
if err != nil || replayed || succeeded.State != StateSucceeded || calls.Load() != 2 {
|
|
t.Fatalf("independent successor = %#v replayed=%v calls=%d err=%v", succeeded, replayed, calls.Load(), err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMarkNotAppliedPersistsExplicitNoEffectSemantics(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
spec := testSpec("operation-no-effect-failure", "vm-no-effect")
|
|
|
|
record, replayed, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return nil, MarkNotApplied(testFailure{
|
|
code: "snapshot_in_use", message: "snapshot volume is referenced by a domain",
|
|
effect: true, safe: false,
|
|
})
|
|
})
|
|
if err == nil || replayed || record.State != StateFailed || record.Error == nil ||
|
|
record.Error.Code != "snapshot_in_use" || record.EffectMayHaveApplied || !record.SafeToResubmit || record.ExecutionFenced {
|
|
t.Fatalf("explicit no-effect result = %#v replayed=%v err=%v", record, replayed, err)
|
|
}
|
|
}
|
|
|
|
func TestMarkNotAppliedNeverDowngradesUncertainFailure(t *testing.T) {
|
|
err := MarkNotApplied(MarkUncertain(errors.New("mutation boundary crossed")))
|
|
if !IsUncertain(err) {
|
|
t.Fatalf("uncertain failure was downgraded: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOperationFailureMarkersPrecedeBareContextSemantics(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
err error
|
|
uncertain bool
|
|
notApplied bool
|
|
}{
|
|
{name: "bare deadline", err: context.DeadlineExceeded, uncertain: true},
|
|
{name: "bare cancellation", err: context.Canceled, uncertain: true},
|
|
{name: "pre-dispatch deadline", err: MarkNotApplied(context.DeadlineExceeded), notApplied: true},
|
|
{name: "pre-dispatch cancellation", err: MarkNotApplied(context.Canceled), notApplied: true},
|
|
{
|
|
name: "not applied wins over joined bare deadline",
|
|
err: errors.Join(MarkNotApplied(errors.New("pre-dispatch failure")), context.DeadlineExceeded), notApplied: true,
|
|
},
|
|
{
|
|
name: "explicit uncertainty wins over not applied",
|
|
err: MarkUncertain(MarkNotApplied(context.DeadlineExceeded)), uncertain: true, notApplied: true,
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if got := IsUncertain(test.err); got != test.uncertain {
|
|
t.Fatalf("IsUncertain(%v) = %v, want %v", test.err, got, test.uncertain)
|
|
}
|
|
if got := IsNotApplied(test.err); got != test.notApplied {
|
|
t.Fatalf("IsNotApplied(%v) = %v, want %v", test.err, got, test.notApplied)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMarkNotAppliedContextFailurePersistsNoEffectSemantics(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
spec := testSpec("operation-no-effect-deadline", "vm-no-effect-deadline")
|
|
|
|
record, replayed, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return nil, MarkNotApplied(context.DeadlineExceeded)
|
|
})
|
|
if !errors.Is(err, context.DeadlineExceeded) || replayed || record.State != StateFailed ||
|
|
record.EffectMayHaveApplied || !record.SafeToResubmit {
|
|
t.Fatalf("pre-dispatch deadline result = %#v replayed=%v err=%v", record, replayed, err)
|
|
}
|
|
}
|
|
|
|
func TestStorageConfigurationAdmissionRemainsSerializedWhileRunning(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
firstStarted := make(chan struct{})
|
|
releaseFirst := make(chan struct{})
|
|
secondStarted := make(chan struct{})
|
|
firstDone := make(chan error, 1)
|
|
secondDone := make(chan error, 1)
|
|
|
|
go func() {
|
|
_, _, err := j.Execute(context.Background(), storageConfigurationSpec("operation-storage-running"), func(context.Context) (json.RawMessage, error) {
|
|
close(firstStarted)
|
|
<-releaseFirst
|
|
return nil, testFailure{
|
|
code: "storage_configuration_failed", message: "storage configuration failed",
|
|
effect: true, safe: false,
|
|
}
|
|
})
|
|
firstDone <- err
|
|
}()
|
|
<-firstStarted
|
|
|
|
go func() {
|
|
_, _, err := j.Execute(context.Background(), storageConfigurationSpec("operation-storage-waiting"), func(context.Context) (json.RawMessage, error) {
|
|
close(secondStarted)
|
|
return json.RawMessage(`{"configured":true}`), nil
|
|
})
|
|
secondDone <- err
|
|
}()
|
|
deadline := time.Now().Add(time.Second)
|
|
for {
|
|
select {
|
|
case <-secondStarted:
|
|
t.Fatal("second storage mutation crossed admission while the first was running")
|
|
default:
|
|
}
|
|
record, err := j.Get("operation-storage-waiting")
|
|
if err == nil && record.State == StateReserved {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("second storage operation did not reach serialized admission: %#v err=%v", record, err)
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
|
|
close(releaseFirst)
|
|
if err := <-firstDone; err == nil {
|
|
t.Fatal("first storage mutation unexpectedly succeeded")
|
|
}
|
|
if err := <-secondDone; err != nil {
|
|
t.Fatalf("second storage mutation remained blocked after terminal failure: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOperationIDIsPermanentTombstoneAndMutationRunsOnce(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
spec := testSpec("operation-single-dispatch", "vm-single")
|
|
var calls atomic.Int32
|
|
|
|
first, replayed, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
calls.Add(1)
|
|
return json.RawMessage(`{"deleted":true}`), nil
|
|
})
|
|
if err != nil || replayed || first.State != StateSucceeded {
|
|
t.Fatalf("first execution = %#v replayed=%v err=%v", first, replayed, err)
|
|
}
|
|
second, replayed, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
calls.Add(1)
|
|
return nil, errors.New("must not run")
|
|
})
|
|
if err != nil || !replayed || second.State != StateSucceeded || calls.Load() != 1 {
|
|
t.Fatalf("replay = %#v replayed=%v calls=%d err=%v", second, replayed, calls.Load(), err)
|
|
}
|
|
|
|
conflict := spec
|
|
conflict.Fingerprint = Fingerprint("DELETE", "/v1/vms/vm-single", []byte(`{"different":true}`))
|
|
if _, _, err := j.Execute(context.Background(), conflict, func(context.Context) (json.RawMessage, error) {
|
|
calls.Add(1)
|
|
return nil, nil
|
|
}); !errors.Is(err, ErrFingerprint) || calls.Load() != 1 {
|
|
t.Fatalf("conflicting replay err=%v calls=%d", err, calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestCanceledExecutionKeepsAdmissionUntilWorkerReturns(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
resource := "vm-reap-order"
|
|
first := testSpec("operation-reap-first", resource)
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
workerReturned := make(chan struct{})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
firstDone := make(chan error, 1)
|
|
go func() {
|
|
_, _, err := j.Execute(ctx, first, func(context.Context) (json.RawMessage, error) {
|
|
close(started)
|
|
<-release
|
|
close(workerReturned)
|
|
return nil, context.DeadlineExceeded
|
|
})
|
|
firstDone <- err
|
|
}()
|
|
<-started
|
|
cancel()
|
|
|
|
second := testSpec("operation-reap-second", resource)
|
|
secondStarted := make(chan struct{})
|
|
secondDone := make(chan error, 1)
|
|
go func() {
|
|
_, _, err := j.Execute(context.Background(), second, func(context.Context) (json.RawMessage, error) {
|
|
close(secondStarted)
|
|
return json.RawMessage(`{"deleted":true}`), nil
|
|
})
|
|
secondDone <- err
|
|
}()
|
|
select {
|
|
case <-secondStarted:
|
|
t.Fatal("successor crossed dispatch before the first worker returned")
|
|
case <-time.After(40 * time.Millisecond):
|
|
}
|
|
close(release)
|
|
<-workerReturned
|
|
if err := <-firstDone; !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("first error = %v", err)
|
|
}
|
|
if err := <-secondDone; !errors.Is(err, ErrResourceBlocked) {
|
|
t.Fatalf("ambiguous first result did not durably fence successor: %v", err)
|
|
}
|
|
select {
|
|
case <-secondStarted:
|
|
t.Fatal("fenced successor unexpectedly invoked its mutation")
|
|
default:
|
|
}
|
|
}
|
|
|
|
func TestBackgroundWorkerDeadlineStartsAfterAdmission(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
resource := "vm-background-deadline"
|
|
firstStarted := make(chan struct{})
|
|
releaseFirst := make(chan struct{})
|
|
firstDone := make(chan error, 1)
|
|
go func() {
|
|
_, _, err := j.Execute(context.Background(), testSpec("operation-deadline-first", resource), func(context.Context) (json.RawMessage, error) {
|
|
close(firstStarted)
|
|
<-releaseFirst
|
|
return json.RawMessage(`{"done":true}`), nil
|
|
})
|
|
firstDone <- err
|
|
}()
|
|
<-firstStarted
|
|
go func() {
|
|
time.Sleep(60 * time.Millisecond)
|
|
close(releaseFirst)
|
|
}()
|
|
|
|
admissionCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
secondID := "operation-deadline-second"
|
|
record, replayed, done, err := j.ExecuteBackground(
|
|
admissionCtx, context.Background(), 30*time.Millisecond,
|
|
testSpec(secondID, resource),
|
|
func(ctx context.Context) (json.RawMessage, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, fmt.Errorf("execution budget expired before worker start: %w", err)
|
|
}
|
|
return json.RawMessage(`{"done":true}`), nil
|
|
},
|
|
)
|
|
if err != nil || replayed || done == nil || record.State != StateDispatching {
|
|
t.Fatalf("background start = %#v replayed=%v done=%v err=%v", record, replayed, done != nil, err)
|
|
}
|
|
<-done
|
|
if err := <-firstDone; err != nil {
|
|
t.Fatalf("first execution: %v", err)
|
|
}
|
|
terminal, err := j.Get(secondID)
|
|
if err != nil || terminal.State != StateSucceeded {
|
|
t.Fatalf("second terminal = %#v err=%v", terminal, err)
|
|
}
|
|
}
|
|
|
|
func TestBackgroundFinalizationFailureForcesRestartRecoveryWithoutReplay(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "operations.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := testSpec("operation-background-persist-failure", "vm-background-persist-failure")
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var calls atomic.Int32
|
|
record, replayed, done, err := j.ExecuteBackground(
|
|
context.Background(), context.Background(), time.Minute, spec,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
calls.Add(1)
|
|
close(started)
|
|
<-release
|
|
return json.RawMessage(`{"done":true}`), nil
|
|
},
|
|
)
|
|
if err != nil || replayed || done == nil || record.State != StateDispatching {
|
|
t.Fatalf("background start = %#v replayed=%v done=%v err=%v", record, replayed, done != nil, err)
|
|
}
|
|
<-started
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
close(release)
|
|
<-done
|
|
select {
|
|
case backgroundErr := <-j.BackgroundFailures():
|
|
if !strings.Contains(backgroundErr.Error(), "finalize background operation "+spec.OperationID) {
|
|
t.Fatalf("background failure = %v", backgroundErr)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("background persistence failure was silently discarded")
|
|
}
|
|
|
|
reopened, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer reopened.Close()
|
|
finalized, err := reopened.FinalizeInterruptedOperations([]InterruptedReceipt{{
|
|
OperationID: spec.OperationID, Fingerprint: spec.Fingerprint,
|
|
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
|
|
}})
|
|
if err != nil || len(finalized) != 1 || finalized[0] != spec.OperationID {
|
|
t.Fatalf("restart finalization = %v err=%v", finalized, err)
|
|
}
|
|
terminal, err := reopened.Get(spec.OperationID)
|
|
if err != nil || terminal.State != StateSucceeded || calls.Load() != 1 {
|
|
t.Fatalf("restart terminal = %#v calls=%d err=%v", terminal, calls.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestBackgroundFencePersistenceFailureForcesUnconfirmedRestart(t *testing.T) {
|
|
oldDeadline := workerReapDeadline
|
|
workerReapDeadline = 25 * time.Millisecond
|
|
t.Cleanup(func() { workerReapDeadline = oldDeadline })
|
|
|
|
path := filepath.Join(t.TempDir(), "operations.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := testSpec("operation-background-fence-failure", "vm-background-fence-failure")
|
|
workerParent, cancelWorker := context.WithCancel(context.Background())
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var calls atomic.Int32
|
|
record, replayed, done, err := j.ExecuteBackground(
|
|
context.Background(), workerParent, time.Minute, spec,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
calls.Add(1)
|
|
close(started)
|
|
<-release
|
|
return json.RawMessage(`{"late":true}`), nil
|
|
},
|
|
)
|
|
if err != nil || replayed || done == nil || record.State != StateDispatching {
|
|
t.Fatalf("background start = %#v replayed=%v done=%v err=%v", record, replayed, done != nil, err)
|
|
}
|
|
<-started
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cancelWorker()
|
|
<-done
|
|
select {
|
|
case backgroundErr := <-j.BackgroundFailures():
|
|
if !strings.Contains(backgroundErr.Error(), "fence background operation "+spec.OperationID) {
|
|
t.Fatalf("background failure = %v", backgroundErr)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("background fence persistence failure was silently discarded")
|
|
}
|
|
close(release)
|
|
|
|
reopened, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer reopened.Close()
|
|
finalized, err := reopened.FinalizeInterruptedOperations(nil)
|
|
if err != nil || len(finalized) != 1 || finalized[0] != spec.OperationID {
|
|
t.Fatalf("restart finalization = %v err=%v", finalized, err)
|
|
}
|
|
terminal, err := reopened.Get(spec.OperationID)
|
|
if err != nil || terminal.State != StateFailed || terminal.Error == nil ||
|
|
terminal.Error.Code != "result_unconfirmed" || !terminal.ExecutionFenced || calls.Load() != 1 {
|
|
t.Fatalf("restart terminal = %#v calls=%d err=%v", terminal, calls.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestUnreapedExecutionBecomesTerminalDurableFence(t *testing.T) {
|
|
oldDeadline := workerReapDeadline
|
|
workerReapDeadline = 25 * time.Millisecond
|
|
t.Cleanup(func() { workerReapDeadline = oldDeadline })
|
|
|
|
j := openTestJournal(t)
|
|
resource := "vm-unreaped"
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
defer close(release)
|
|
go func() {
|
|
<-started
|
|
cancel()
|
|
}()
|
|
record, _, err := j.Execute(ctx, testSpec("operation-unreaped-worker", resource), func(context.Context) (json.RawMessage, error) {
|
|
close(started)
|
|
<-release
|
|
return nil, nil
|
|
})
|
|
if !errors.Is(err, ErrExecutionNotReaped) || record.State != StateFailed || !record.ExecutionFenced ||
|
|
!record.EffectMayHaveApplied || record.SafeToResubmit {
|
|
t.Fatalf("unreaped result = %#v err=%v", record, err)
|
|
}
|
|
|
|
called := false
|
|
blocked, _, blockErr := j.Execute(context.Background(), testSpec("operation-after-unreaped", resource), func(context.Context) (json.RawMessage, error) {
|
|
called = true
|
|
return nil, nil
|
|
})
|
|
if !errors.Is(blockErr, ErrResourceBlocked) || called || !WasNotDispatched(blocked) {
|
|
t.Fatalf("durable fence = %#v called=%v err=%v", blocked, called, blockErr)
|
|
}
|
|
}
|
|
|
|
func TestRestartNeverReplaysReservedOrDispatchingMutation(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "operations.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reserved := testSpec("operation-restart-reserved", "vm-reserved")
|
|
if _, _, err := j.Reserve(reserved); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dispatching := testSpec("operation-restart-dispatched", "vm-dispatched")
|
|
if _, _, err := j.Reserve(dispatching); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := j.markDispatching(dispatching.OperationID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
j, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer j.Close()
|
|
storedReserved, err := j.Get(reserved.OperationID)
|
|
if err != nil || !WasNotDispatched(storedReserved) || !storedReserved.SafeToResubmit || storedReserved.EffectMayHaveApplied {
|
|
t.Fatalf("reserved restart = %#v err=%v", storedReserved, err)
|
|
}
|
|
finalized, err := j.FinalizeInterruptedOperations(nil)
|
|
if err != nil || len(finalized) != 1 || finalized[0] != dispatching.OperationID {
|
|
t.Fatalf("dispatching finalization = %v err=%v", finalized, err)
|
|
}
|
|
storedDispatching, err := j.Get(dispatching.OperationID)
|
|
if err != nil || storedDispatching.State != StateFailed || storedDispatching.Error == nil ||
|
|
storedDispatching.Error.Code != "result_unconfirmed" || !storedDispatching.ExecutionFenced ||
|
|
!storedDispatching.EffectMayHaveApplied || storedDispatching.SafeToResubmit {
|
|
t.Fatalf("dispatching restart = %#v err=%v", storedDispatching, err)
|
|
}
|
|
}
|
|
|
|
func TestRestartImportsExactImmutableWorkerReceipt(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "operations.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := testSpec("operation-receipt-import", "vm-receipt")
|
|
if _, _, err := j.Reserve(spec); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := j.markDispatching(spec.OperationID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
j, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer j.Close()
|
|
want := json.RawMessage(`{"deleted":true}`)
|
|
finalized, err := j.FinalizeInterruptedOperations([]InterruptedReceipt{{
|
|
OperationID: spec.OperationID,
|
|
Fingerprint: spec.Fingerprint,
|
|
Status: "succeeded",
|
|
Result: want,
|
|
}})
|
|
if err != nil || len(finalized) != 1 {
|
|
t.Fatalf("receipt finalization = %v err=%v", finalized, err)
|
|
}
|
|
record, err := j.Get(spec.OperationID)
|
|
if err != nil || record.State != StateSucceeded || string(record.Result) != string(want) {
|
|
t.Fatalf("imported receipt = %#v err=%v", record, err)
|
|
}
|
|
if again, err := j.FinalizeInterruptedOperations(nil); err != nil || len(again) != 0 {
|
|
t.Fatalf("second finalization = %v err=%v", again, err)
|
|
}
|
|
}
|
|
|
|
func TestReadTerminalRecordRecoversCommittedWorkerJournalWithoutMutation(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "worker-journal.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := testSpec("operation-worker-terminal", "vm-worker-terminal")
|
|
want := json.RawMessage(`{"deleted":true}`)
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return want, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
record, err := ReadTerminalRecord(path, spec.OperationID)
|
|
if err != nil || record.State != StateSucceeded || record.OperationID != spec.OperationID ||
|
|
record.Action != spec.Action || record.ResourceID != spec.ResourceID ||
|
|
record.Fingerprint != spec.Fingerprint || string(record.Result) != string(want) {
|
|
t.Fatalf("terminal record = %#v err=%v", record, err)
|
|
}
|
|
if _, err := ReadTerminalRecord(path, "operation-worker-other"); !errors.Is(err, ErrTerminalEvidencePending) {
|
|
t.Fatalf("missing operation evidence error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestReadTerminalRecordRejectsNonTerminalAndCorruptEvidence(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "worker-journal.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := testSpec("operation-worker-pending", "vm-worker-pending")
|
|
if _, _, err := j.Reserve(spec); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := ReadTerminalRecord(path, spec.OperationID); !errors.Is(err, ErrTerminalEvidencePending) {
|
|
t.Fatalf("non-terminal evidence error = %v", err)
|
|
}
|
|
|
|
corrupt := filepath.Join(t.TempDir(), "corrupt.db")
|
|
if err := os.WriteFile(corrupt, []byte("not a bolt database"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := ReadTerminalRecord(corrupt, spec.OperationID); !errors.Is(err, ErrTerminalEvidenceInvalid) {
|
|
t.Fatalf("corrupt evidence error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestReadTerminalRecordSurfacesTemporaryJournalLock(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "worker-journal.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer j.Close()
|
|
oldTimeout := terminalEvidenceOpenTimeout
|
|
terminalEvidenceOpenTimeout = 20 * time.Millisecond
|
|
t.Cleanup(func() { terminalEvidenceOpenTimeout = oldTimeout })
|
|
|
|
if _, err := ReadTerminalRecord(path, "operation-worker-locked"); err == nil ||
|
|
errors.Is(err, ErrTerminalEvidenceInvalid) || errors.Is(err, ErrTerminalEvidencePending) {
|
|
t.Fatalf("temporary Journal lock was not surfaced for startup retry: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCompletionReceiptIsImmutableAndAckOnlyClearsOutbox(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
spec := testSpec("operation-completion-event", "vm-completion")
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"deleted":true}`), nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 {
|
|
t.Fatalf("completion claim = %#v err=%v", claimed, err)
|
|
}
|
|
receipt := claimed[0]
|
|
if receipt.Status != "succeeded" || receipt.Result == nil || string(receipt.Result) != `{"deleted":true}` ||
|
|
receipt.Action != spec.Action || receipt.ResourceID != spec.ResourceID ||
|
|
receipt.RequestFingerprint != spec.Fingerprint {
|
|
t.Fatalf("completion receipt = %#v", receipt)
|
|
}
|
|
if err := j.AcknowledgeCompletion(receipt.EventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(48 * time.Hour)
|
|
if pending, err := j.ClaimPendingCompletions(1); err != nil || len(pending) != 0 {
|
|
t.Fatalf("acked completion remained pending: %#v err=%v", pending, err)
|
|
}
|
|
record, err := j.Get(spec.OperationID)
|
|
if err != nil || record.State != StateSucceeded || string(record.Result) != `{"deleted":true}` {
|
|
t.Fatalf("ACK rewrote operation: %#v err=%v", record, err)
|
|
}
|
|
var persisted CompletionReceipt
|
|
err = j.db.View(func(tx *bolt.Tx) error {
|
|
encoded := tx.Bucket(completionReceiptsBucket).Get([]byte(receipt.EventID))
|
|
return json.Unmarshal(encoded, &persisted)
|
|
})
|
|
if err != nil || persisted.EventID != receipt.EventID || string(persisted.Result) != string(receipt.Result) {
|
|
t.Fatalf("immutable receipt = %#v err=%v", persisted, err)
|
|
}
|
|
}
|
|
|
|
func TestFailedCompletionReceiptRetainsCauseCode(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
spec := testSpec("operation-completion-cause", "vm-completion-cause")
|
|
if _, _, err := j.Reserve(spec); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := j.finishWithSemantics(
|
|
spec.OperationID,
|
|
"operation_failed",
|
|
"operation failed",
|
|
nil,
|
|
"backup_disk_transition_active",
|
|
false,
|
|
true,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 {
|
|
t.Fatalf("completion claim = %#v err=%v", claimed, err)
|
|
}
|
|
if claimed[0].ErrorCode != "operation_failed" ||
|
|
claimed[0].CauseCode != "backup_disk_transition_active" {
|
|
t.Fatalf("failed completion receipt = %#v", claimed[0])
|
|
}
|
|
}
|
|
|
|
func TestInspectionRecoversAckedCompletionReceiptAfterDetailPrune(t *testing.T) {
|
|
j, spec, eventID, want := pruneAckedSettingsOperation(t)
|
|
|
|
plain, err := j.Get(spec.OperationID)
|
|
if err != nil || !plain.Tombstone || len(plain.Result) != 0 {
|
|
t.Fatalf("pruned Journal record = %#v err=%v", plain, err)
|
|
}
|
|
var persistedBefore []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
persistedBefore = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(spec.OperationID))...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
inspected, err := j.GetForInspection(spec.OperationID)
|
|
if err != nil || !inspected.Tombstone || !inspected.InspectionReceiptRecovered ||
|
|
inspected.InspectionEvidenceUnavailable || string(inspected.Result) != string(want) {
|
|
t.Fatalf("receipt-backed inspection record = %#v err=%v", inspected, err)
|
|
}
|
|
if pending, err := j.ClaimPendingCompletions(1); err != nil || len(pending) != 0 {
|
|
t.Fatalf("rejected/ACKed completion returned to outbox: %#v err=%v", pending, err)
|
|
}
|
|
var persistedAfter, receipt []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
persistedAfter = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(spec.OperationID))...)
|
|
receipt = append([]byte(nil), tx.Bucket(completionReceiptsBucket).Get([]byte(eventID))...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(persistedBefore, persistedAfter) || len(receipt) == 0 {
|
|
t.Fatal("inspection rewrote the Journal or lost the immutable receipt")
|
|
}
|
|
again, err := j.Get(spec.OperationID)
|
|
if err != nil || len(again.Result) != 0 || again.InspectionReceiptRecovered ||
|
|
again.InspectionEvidenceUnavailable {
|
|
t.Fatalf("in-memory recovery leaked into persistent Journal read: %#v err=%v", again, err)
|
|
}
|
|
}
|
|
|
|
func TestInspectionRejectsTamperedCompletionReceipt(t *testing.T) {
|
|
tests := map[string]func(*CompletionReceipt){
|
|
"event id": func(receipt *CompletionReceipt) {
|
|
receipt.EventID = completionEventID("operation-different-event")
|
|
},
|
|
"operation id": func(receipt *CompletionReceipt) {
|
|
receipt.OperationID = "operation-different-identity"
|
|
},
|
|
"action": func(receipt *CompletionReceipt) {
|
|
receipt.Action = "change-network"
|
|
},
|
|
"resource id": func(receipt *CompletionReceipt) {
|
|
receipt.ResourceID = "123e4567-e89b-42d3-a456-426614174999"
|
|
},
|
|
"request fingerprint": func(receipt *CompletionReceipt) {
|
|
receipt.RequestFingerprint = strings.Repeat("a", 64)
|
|
},
|
|
"status": func(receipt *CompletionReceipt) {
|
|
receipt.Status = "failed"
|
|
},
|
|
"effect": func(receipt *CompletionReceipt) {
|
|
receipt.EffectMayHaveApplied = true
|
|
},
|
|
"safe to resubmit": func(receipt *CompletionReceipt) {
|
|
receipt.SafeToResubmit = true
|
|
},
|
|
"error": func(receipt *CompletionReceipt) {
|
|
receipt.ErrorCode = "operation_failed"
|
|
},
|
|
}
|
|
for name, tamper := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
j, spec, eventID, _ := pruneAckedSettingsOperation(t)
|
|
if err := j.db.Update(func(tx *bolt.Tx) error {
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
var receipt CompletionReceipt
|
|
if err := json.Unmarshal(receipts.Get([]byte(eventID)), &receipt); err != nil {
|
|
return err
|
|
}
|
|
tamper(&receipt)
|
|
encoded, err := json.Marshal(receipt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return receipts.Put([]byte(eventID), encoded)
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
inspected, err := j.GetForInspection(spec.OperationID)
|
|
if err != nil || inspected.InspectionReceiptRecovered ||
|
|
!inspected.InspectionEvidenceUnavailable || len(inspected.Result) != 0 {
|
|
t.Fatalf("tampered receipt inspection record = %#v err=%v", inspected, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func pruneAckedSettingsOperation(t *testing.T) (*Journal, Spec, string, json.RawMessage) {
|
|
t.Helper()
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
resourceID := "123e4567-e89b-42d3-a456-426614174000"
|
|
spec := Spec{
|
|
OperationID: "operation-settings-receipt-prune",
|
|
ResourceID: resourceID,
|
|
Action: "update-settings",
|
|
Fingerprint: Fingerprint("POST", "/v1/vms/"+resourceID+"/settings", []byte(`{"hostname":"safe.example"}`)),
|
|
Destructive: true,
|
|
}
|
|
want := json.RawMessage(`{"uuid":"123e4567-e89b-42d3-a456-426614174000","name":"vm-safe","status":"running","hostname":"safe.example","settings_changed":true,"internal_note":"must-not-leak"}`)
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return want, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 {
|
|
t.Fatalf("completion claim = %#v err=%v", claimed, err)
|
|
}
|
|
eventID := completionEventID(spec.OperationID)
|
|
if claimed[0].EventID != eventID {
|
|
t.Fatalf("completion event ID = %q, want %q", claimed[0].EventID, eventID)
|
|
}
|
|
// The Agent intentionally cannot distinguish an applied/late completion
|
|
// from a Gateway "rejected" disposition: all produce the same ACK.
|
|
if err := j.AcknowledgeCompletion(eventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(operationDetailRetention + time.Hour)
|
|
if accepted, err := j.UseNonce("nonce-prune-acked-receipt", now, now.Add(time.Minute)); err != nil || !accepted {
|
|
t.Fatalf("trigger operation pruning accepted=%v err=%v", accepted, err)
|
|
}
|
|
return j, spec, eventID, want
|
|
}
|
|
|
|
func TestCompletionOutboxRetriesLegacyExpiredRecordUntilACK(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 19, 1, 0, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
spec := testSpec("operation-durable-outbox", "vm-outbox")
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "injected_failure", message: "injected", effect: false, safe: true}
|
|
}); err == nil {
|
|
t.Fatal("injected failure unexpectedly succeeded")
|
|
}
|
|
eventID := completionEventID(spec.OperationID)
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
var legacyReceipt map[string]json.RawMessage
|
|
if unmarshalErr := json.Unmarshal(receipts.Get([]byte(eventID)), &legacyReceipt); unmarshalErr != nil {
|
|
return unmarshalErr
|
|
}
|
|
delete(legacyReceipt, "action")
|
|
delete(legacyReceipt, "resource_id")
|
|
delete(legacyReceipt, "request_fingerprint")
|
|
legacyReceiptEncoded, marshalErr := json.Marshal(legacyReceipt)
|
|
if marshalErr != nil {
|
|
return marshalErr
|
|
}
|
|
if putErr := receipts.Put([]byte(eventID), legacyReceiptEncoded); putErr != nil {
|
|
return putErr
|
|
}
|
|
legacy, marshalErr := json.Marshal(struct {
|
|
OperationID string `json:"operation_id"`
|
|
Attempts uint32 `json:"attempts"`
|
|
NextAttemptAt time.Time `json:"next_attempt_at"`
|
|
ExpiresAt time.Time `json:"expires_at"`
|
|
}{
|
|
OperationID: spec.OperationID, Attempts: 12,
|
|
NextAttemptAt: now, ExpiresAt: now.Add(24 * time.Hour),
|
|
})
|
|
if marshalErr != nil {
|
|
return marshalErr
|
|
}
|
|
return tx.Bucket(completionOutboxBucket).Put([]byte(eventID), legacy)
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
now = now.Add(25 * time.Hour)
|
|
claimed, expired, err := j.ClaimPendingCompletionsWithExpired(1)
|
|
if err != nil || len(claimed) != 1 || claimed[0].EventID != eventID || len(expired) != 0 {
|
|
t.Fatalf("legacy expired claim = %#v expired=%#v err=%v", claimed, expired, err)
|
|
}
|
|
if claimed[0].Action != spec.Action || claimed[0].ResourceID != spec.ResourceID ||
|
|
claimed[0].RequestFingerprint != spec.Fingerprint {
|
|
t.Fatalf("legacy receipt identity was not enriched from Journal: %#v", claimed[0])
|
|
}
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
var persisted map[string]json.RawMessage
|
|
if unmarshalErr := json.Unmarshal(tx.Bucket(completionReceiptsBucket).Get([]byte(eventID)), &persisted); unmarshalErr != nil {
|
|
return unmarshalErr
|
|
}
|
|
if _, exists := persisted["action"]; exists {
|
|
return errors.New("legacy immutable receipt was rewritten")
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if immediate, err := j.ClaimPendingCompletions(1); err != nil || len(immediate) != 0 {
|
|
t.Fatalf("claim ignored delivery interval: %#v err=%v", immediate, err)
|
|
}
|
|
|
|
now = now.Add(365 * 24 * time.Hour)
|
|
claimed, expired, err = j.ClaimPendingCompletionsWithExpired(1)
|
|
if err != nil || len(claimed) != 1 || claimed[0].EventID != eventID || len(expired) != 0 {
|
|
t.Fatalf("long-lived claim = %#v expired=%#v err=%v", claimed, expired, err)
|
|
}
|
|
if err := j.AcknowledgeCompletion(eventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(completionDeliveryInterval)
|
|
if afterACK, err := j.ClaimPendingCompletions(1); err != nil || len(afterACK) != 0 {
|
|
t.Fatalf("ACK did not remove outbox event: %#v err=%v", afterACK, err)
|
|
}
|
|
}
|
|
|
|
func TestCompletionOutboxQuarantinesPoisonAndContinuesDelivery(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
errorCode string
|
|
poison func(*bolt.Bucket, []byte, []byte) error
|
|
}{
|
|
{
|
|
name: "identity mismatch",
|
|
errorCode: "completion_identity_mismatch",
|
|
poison: func(receipts *bolt.Bucket, key, original []byte) error {
|
|
var receipt CompletionReceipt
|
|
if err := json.Unmarshal(original, &receipt); err != nil {
|
|
return err
|
|
}
|
|
receipt.Action = "power-start"
|
|
encoded, err := json.Marshal(receipt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return receipts.Put(key, encoded)
|
|
},
|
|
},
|
|
{
|
|
name: "invalid receipt JSON",
|
|
errorCode: "completion_receipt_invalid",
|
|
poison: func(receipts *bolt.Bucket, key, _ []byte) error {
|
|
return receipts.Put(key, []byte(`{"id":`))
|
|
},
|
|
},
|
|
{
|
|
name: "missing receipt",
|
|
errorCode: "completion_receipt_missing",
|
|
poison: func(receipts *bolt.Bucket, key, _ []byte) error {
|
|
return receipts.Delete(key)
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "operations.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Date(2026, 10, 19, 1, 15, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
|
|
first := testSpec("operation-poison-first", "vm-poison-first")
|
|
second := testSpec("operation-poison-second", "vm-poison-second")
|
|
poisonSpec, healthySpec := first, second
|
|
if completionEventID(poisonSpec.OperationID) > completionEventID(healthySpec.OperationID) {
|
|
poisonSpec, healthySpec = healthySpec, poisonSpec
|
|
}
|
|
for _, spec := range []Spec{poisonSpec, healthySpec} {
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"done":true}`), nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
poisonEventID := completionEventID(poisonSpec.OperationID)
|
|
healthyEventID := completionEventID(healthySpec.OperationID)
|
|
var originalReceipt, poisonedReceipt []byte
|
|
if err := j.db.Update(func(tx *bolt.Tx) error {
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
originalReceipt = append([]byte(nil), receipts.Get([]byte(poisonEventID))...)
|
|
if len(originalReceipt) == 0 || len(outbox.Get([]byte(poisonEventID))) == 0 {
|
|
return errors.New("poison fixture is missing durable completion state")
|
|
}
|
|
if err := test.poison(receipts, []byte(poisonEventID), originalReceipt); err != nil {
|
|
return err
|
|
}
|
|
poisonedReceipt = append([]byte(nil), receipts.Get([]byte(poisonEventID))...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
claimed, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 || claimed[0].EventID != healthyEventID {
|
|
t.Fatalf("claim after poison = %#v err=%v", claimed, err)
|
|
}
|
|
incident, found, err := j.CompletionDeliveryIncident(poisonSpec.OperationID)
|
|
if err != nil || !found || incident.OperationID != poisonSpec.OperationID ||
|
|
incident.EventID != poisonEventID || incident.ErrorCode != test.errorCode ||
|
|
incident.FirstObservedAt.IsZero() || !hexDigestPattern.MatchString(incident.EvidenceSHA256) {
|
|
t.Fatalf("completion incident = %#v found=%v err=%v", incident, found, err)
|
|
}
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
if len(tx.Bucket(completionOutboxBucket).Get([]byte(poisonEventID))) == 0 {
|
|
return errors.New("poison completion outbox entry was deleted")
|
|
}
|
|
if !bytes.Equal(tx.Bucket(completionReceiptsBucket).Get([]byte(poisonEventID)), poisonedReceipt) {
|
|
return errors.New("poison completion receipt was rewritten or deleted")
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer j.Close()
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
persisted, found, err := j.CompletionDeliveryIncident(poisonSpec.OperationID)
|
|
if err != nil || !found || persisted.OperationID != incident.OperationID || persisted.EventID != incident.EventID ||
|
|
persisted.ErrorCode != incident.ErrorCode || persisted.EvidenceSHA256 != incident.EvidenceSHA256 ||
|
|
!persisted.FirstObservedAt.Equal(incident.FirstObservedAt) {
|
|
t.Fatalf("persisted completion incident = %#v found=%v err=%v; want %#v", persisted, found, err, incident)
|
|
}
|
|
|
|
if err := j.db.Update(func(tx *bolt.Tx) error {
|
|
return tx.Bucket(completionReceiptsBucket).Put([]byte(poisonEventID), originalReceipt)
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(completionDeliveryInterval)
|
|
claimed, err = j.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 || claimed[0].EventID != poisonEventID {
|
|
t.Fatalf("claim after repairing poison = %#v err=%v", claimed, err)
|
|
}
|
|
if incident, found, err := j.CompletionDeliveryIncident(poisonSpec.OperationID); err != nil || found {
|
|
t.Fatalf("repaired completion remains quarantined: %#v found=%v err=%v", incident, found, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCompletionQuarantineAssociatesDeterministicEventAfterDoubleCorruption(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 19, 1, 20, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
spec := testSpec("operation-double-corruption", "vm-double-corruption")
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"done":true}`), nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
eventID := completionEventID(spec.OperationID)
|
|
var originalOutbox, originalReceipt []byte
|
|
if err := j.db.Update(func(tx *bolt.Tx) error {
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
originalOutbox = append([]byte(nil), outbox.Get([]byte(eventID))...)
|
|
originalReceipt = append([]byte(nil), receipts.Get([]byte(eventID))...)
|
|
if len(originalOutbox) == 0 || len(originalReceipt) == 0 {
|
|
return errors.New("double-corruption fixture is missing completion state")
|
|
}
|
|
if err := outbox.Put([]byte(eventID), []byte(`{"operation_id":`)); err != nil {
|
|
return err
|
|
}
|
|
return receipts.Put([]byte(eventID), []byte(`{"id":`))
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if claimed, err := j.ClaimPendingCompletions(1); err != nil || len(claimed) != 0 {
|
|
t.Fatalf("double-corrupt completion was delivered: %#v err=%v", claimed, err)
|
|
}
|
|
incident, found, err := j.CompletionDeliveryIncident(spec.OperationID)
|
|
if err != nil || !found || incident.OperationID != spec.OperationID ||
|
|
incident.EventID != eventID || incident.ErrorCode != CompletionDeliveryErrorOutboxInvalid {
|
|
t.Fatalf("derived double-corruption incident = %#v found=%v err=%v", incident, found, err)
|
|
}
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
encoded := tx.Bucket(completionQuarantineBucket).Get([]byte(eventID))
|
|
var persisted CompletionDeliveryIncident
|
|
if json.Unmarshal(encoded, &persisted) != nil || persisted.OperationID != "" {
|
|
return errors.New("derived incident lookup rewrote quarantine evidence")
|
|
}
|
|
if tx.Bucket(completionQuarantineOperationsBucket).Get([]byte(spec.OperationID)) != nil {
|
|
return errors.New("derived incident lookup invented a reverse index")
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err := j.db.Update(func(tx *bolt.Tx) error {
|
|
if err := tx.Bucket(completionOutboxBucket).Put([]byte(eventID), originalOutbox); err != nil {
|
|
return err
|
|
}
|
|
return tx.Bucket(completionReceiptsBucket).Put([]byte(eventID), originalReceipt)
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(completionDeliveryInterval)
|
|
claimed, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 || claimed[0].EventID != eventID {
|
|
t.Fatalf("repaired double-corrupt completion = %#v err=%v", claimed, err)
|
|
}
|
|
if incident, found, err := j.CompletionDeliveryIncident(spec.OperationID); err != nil || found {
|
|
t.Fatalf("repaired double-corrupt incident remains: %#v found=%v err=%v", incident, found, err)
|
|
}
|
|
}
|
|
|
|
func TestCompletionOutboxQuarantinesUndeliverableResultAndContinuesDelivery(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
result json.RawMessage
|
|
}{
|
|
{
|
|
name: "recursive sensitive key",
|
|
result: json.RawMessage(`{"safe":{"nested":{"root_password":"must-not-leak"}}}`),
|
|
},
|
|
{
|
|
name: "scalar result",
|
|
result: json.RawMessage(`"unexpected"`),
|
|
},
|
|
{
|
|
name: "result exceeds safe recursion depth",
|
|
result: json.RawMessage(
|
|
strings.Repeat(`{"nested":`, 18) + `true` + strings.Repeat(`}`, 18),
|
|
),
|
|
},
|
|
{
|
|
name: "result makes completion HTTP body exceed one MiB",
|
|
result: json.RawMessage(
|
|
`{"data":"` + strings.Repeat("x", 1<<20) + `"}`,
|
|
),
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 19, 1, 25, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
|
|
first := testSpec("operation-result-poison-first", "vm-result-poison-first")
|
|
second := testSpec("operation-result-poison-second", "vm-result-poison-second")
|
|
poisonSpec, healthySpec := first, second
|
|
if completionEventID(poisonSpec.OperationID) > completionEventID(healthySpec.OperationID) {
|
|
poisonSpec, healthySpec = healthySpec, poisonSpec
|
|
}
|
|
if _, _, err := j.Execute(context.Background(), poisonSpec, func(context.Context) (json.RawMessage, error) {
|
|
return test.result, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := j.Execute(context.Background(), healthySpec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"done":true}`), nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
poisonEventID := completionEventID(poisonSpec.OperationID)
|
|
healthyEventID := completionEventID(healthySpec.OperationID)
|
|
claimed, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 || claimed[0].EventID != healthyEventID {
|
|
t.Fatalf("claim after undeliverable result = %#v err=%v", claimed, err)
|
|
}
|
|
incident, found, err := j.CompletionDeliveryIncident(poisonSpec.OperationID)
|
|
if err != nil || !found || incident.OperationID != poisonSpec.OperationID ||
|
|
incident.EventID != poisonEventID || incident.ErrorCode != "completion_receipt_invalid" ||
|
|
incident.FirstObservedAt.IsZero() || !hexDigestPattern.MatchString(incident.EvidenceSHA256) {
|
|
t.Fatalf("undeliverable result incident = %#v found=%v err=%v", incident, found, err)
|
|
}
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
if len(tx.Bucket(completionOutboxBucket).Get([]byte(poisonEventID))) == 0 {
|
|
return errors.New("undeliverable completion outbox entry was deleted")
|
|
}
|
|
if len(tx.Bucket(completionReceiptsBucket).Get([]byte(poisonEventID))) == 0 {
|
|
return errors.New("undeliverable completion receipt was deleted")
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCompletionOutboxDoesNotChargeInflightACKWindowTwice(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 19, 1, 30, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
if _, _, err := j.Execute(context.Background(), testSpec("operation-ack-window", "vm-ack-window"), func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"done":true}`), nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(first) != 1 {
|
|
t.Fatalf("first claim = %#v err=%v", first, err)
|
|
}
|
|
now = now.Add(completionDeliveryInterval)
|
|
excluded, _, err := j.ClaimPendingCompletionsWithExpiredExcluding(1, []string{first[0].EventID})
|
|
if err != nil || len(excluded) != 0 {
|
|
t.Fatalf("inflight event was reclaimed: %#v err=%v", excluded, err)
|
|
}
|
|
retry, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(retry) != 1 || retry[0].EventID != first[0].EventID {
|
|
t.Fatalf("excluded claim consumed an attempt/backoff: %#v err=%v", retry, err)
|
|
}
|
|
}
|
|
|
|
func TestFenceResolutionIsAuditedIdempotentAndDoesNotRewriteTerminalEvidence(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 19, 2, 0, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
spec := testSpec("operation-effect-fence", "vm-effect-fence")
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"partial":true}`), testFailure{
|
|
code: "injected_effect", message: "effect may have applied", effect: true, safe: false,
|
|
}
|
|
}); err == nil {
|
|
t.Fatal("effectful failure unexpectedly succeeded")
|
|
}
|
|
if required, err := j.FenceResolutionRequired(spec.OperationID); err != nil || !required {
|
|
t.Fatalf("unresolved effect fence required=%v err=%v", required, err)
|
|
}
|
|
if state, err := j.FenceState(spec.OperationID); err != nil || state != FenceStateActive {
|
|
t.Fatalf("unresolved effect fence state=%q err=%v", state, err)
|
|
}
|
|
claimed, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 {
|
|
t.Fatalf("claim original completion = %#v err=%v", claimed, err)
|
|
}
|
|
eventID := claimed[0].EventID
|
|
var operationBefore, receiptBefore []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
operationBefore = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(spec.OperationID))...)
|
|
receiptBefore = append([]byte(nil), tx.Bucket(completionReceiptsBucket).Get([]byte(eventID))...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request := FenceResolutionRequest{
|
|
ResolutionID: "0123456789abcdef0123456789abcdef",
|
|
Resolution: FenceResolutionNotApplied, InspectionCheckedAt: now.Unix(),
|
|
ReasonSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint,
|
|
}
|
|
missingIdentity := request
|
|
missingIdentity.RequestFingerprint = ""
|
|
if _, _, err := j.ResolveResourceFence(spec.OperationID, 42, missingIdentity); err == nil {
|
|
t.Fatal("fence resolution accepted a missing operation identity")
|
|
}
|
|
mismatchedIdentity := request
|
|
mismatchedIdentity.ResourceID = "vm-other-effect-fence"
|
|
if _, _, err := j.ResolveResourceFence(spec.OperationID, 42, mismatchedIdentity); !errors.Is(err, ErrFenceIdentityConflict) {
|
|
t.Fatalf("mismatched fence identity err=%v", err)
|
|
}
|
|
if state, err := j.FenceState(spec.OperationID); err != nil || state != FenceStateActive {
|
|
t.Fatalf("identity conflict released fence state=%q err=%v", state, err)
|
|
}
|
|
resolution, replayed, err := j.ResolveResourceFence(spec.OperationID, 42, request)
|
|
if err != nil || replayed || resolution.OperationID != spec.OperationID || resolution.ResolvedBy != 42 {
|
|
t.Fatalf("resolve fence = %#v replayed=%v err=%v", resolution, replayed, err)
|
|
}
|
|
replayedResolution, replayed, err := j.ResolveResourceFence(spec.OperationID, 42, request)
|
|
if err != nil || !replayed || replayedResolution != resolution {
|
|
t.Fatalf("idempotent resolve = %#v replayed=%v err=%v", replayedResolution, replayed, err)
|
|
}
|
|
now = now.Add(3 * time.Minute)
|
|
replayedResolution, replayed, err = j.ResolveResourceFence(spec.OperationID, 42, request)
|
|
if err != nil || !replayed || replayedResolution != resolution {
|
|
t.Fatalf("stale exact replay = %#v replayed=%v err=%v", replayedResolution, replayed, err)
|
|
}
|
|
if required, err := j.FenceResolutionRequired(spec.OperationID); err != nil || required {
|
|
t.Fatalf("resolved effect fence required=%v err=%v", required, err)
|
|
}
|
|
if state, err := j.FenceState(spec.OperationID); err != nil || state != FenceStateResolved {
|
|
t.Fatalf("resolved effect fence state=%q err=%v", state, err)
|
|
}
|
|
conflict := request
|
|
conflict.Resolution = FenceResolutionApplied
|
|
if _, _, err := j.ResolveResourceFence(spec.OperationID, 42, conflict); !errors.Is(err, ErrFenceResolutionConflict) {
|
|
t.Fatalf("conflicting resolution err=%v", err)
|
|
}
|
|
stored, err := j.GetFenceResolution(spec.OperationID)
|
|
if err != nil || stored != resolution {
|
|
t.Fatalf("stored fence resolution = %#v err=%v", stored, err)
|
|
}
|
|
var operationAfter, receiptAfter []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
operationAfter = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(spec.OperationID))...)
|
|
receiptAfter = append([]byte(nil), tx.Bucket(completionReceiptsBucket).Get([]byte(eventID))...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(operationBefore, operationAfter) || !bytes.Equal(receiptBefore, receiptAfter) {
|
|
t.Fatal("fence resolution rewrote the terminal operation or immutable completion receipt")
|
|
}
|
|
var calls atomic.Int32
|
|
successor := testSpec("operation-after-fence", spec.ResourceID)
|
|
if record, _, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
|
|
calls.Add(1)
|
|
return json.RawMessage(`{"accepted":true}`), nil
|
|
}); err != nil || record.State != StateSucceeded || calls.Load() != 1 {
|
|
t.Fatalf("successor after resolution = %#v calls=%d err=%v", record, calls.Load(), err)
|
|
}
|
|
now = now.Add(operationDetailRetention + time.Hour)
|
|
if accepted, err := j.UseNonce("nonce-prune-resolved-operation", now, now.Add(time.Minute)); err != nil || !accepted {
|
|
t.Fatalf("trigger resolved-operation pruning accepted=%v err=%v", accepted, err)
|
|
}
|
|
pruned, err := j.Get(spec.OperationID)
|
|
if err != nil || !pruned.Tombstone || len(pruned.Result) != 0 {
|
|
t.Fatalf("resolved effectful operation was not tombstoned: %#v err=%v", pruned, err)
|
|
}
|
|
}
|
|
|
|
func TestExpiredUnresolvedFenceRetainsPruneIndexUntilLateResolution(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 19, 2, 30, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
spec := testSpec("operation-late-retention-fence", "vm-late-retention-fence")
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"partial":true}`), testFailure{
|
|
code: "injected_effect", message: "effect may have applied", effect: true, safe: false,
|
|
}
|
|
}); err == nil {
|
|
t.Fatal("effectful failure unexpectedly succeeded")
|
|
}
|
|
now = now.Add(operationDetailRetention + time.Hour)
|
|
if accepted, err := j.UseNonce("nonce-prune-unresolved-fence", now, now.Add(time.Minute)); err != nil || !accepted {
|
|
t.Fatalf("trigger unresolved pruning accepted=%v err=%v", accepted, err)
|
|
}
|
|
unresolved, err := j.Get(spec.OperationID)
|
|
if err != nil || unresolved.Tombstone {
|
|
t.Fatalf("active old fence lost full record: %#v err=%v", unresolved, err)
|
|
}
|
|
request := FenceResolutionRequest{
|
|
ResolutionID: "abcdefabcdefabcdefabcdefabcdefab",
|
|
Resolution: FenceResolutionNotApplied, InspectionCheckedAt: now.Unix(),
|
|
ReasonSHA256: "abababababababababababababababababababababababababababababababab",
|
|
Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint,
|
|
}
|
|
if _, replayed, err := j.ResolveResourceFence(spec.OperationID, 42, request); err != nil || replayed {
|
|
t.Fatalf("late fence resolution replayed=%v err=%v", replayed, err)
|
|
}
|
|
if accepted, err := j.UseNonce("nonce-prune-late-resolution", now.Add(time.Second), now.Add(time.Minute)); err != nil || !accepted {
|
|
t.Fatalf("trigger late-resolution pruning accepted=%v err=%v", accepted, err)
|
|
}
|
|
pruned, err := j.Get(spec.OperationID)
|
|
if err != nil || !pruned.Tombstone || len(pruned.Result) != 0 {
|
|
t.Fatalf("late-resolved old operation was not tombstoned: %#v err=%v", pruned, err)
|
|
}
|
|
}
|
|
|
|
func TestFenceResolutionRequiresFreshInspectionAndActiveBlocker(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 19, 3, 0, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
request := FenceResolutionRequest{
|
|
ResolutionID: "fedcba9876543210fedcba9876543210",
|
|
Resolution: FenceResolutionApplied, InspectionCheckedAt: now.Add(-3 * time.Minute).Unix(),
|
|
ReasonSHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
Action: "delete-vm", ResourceID: "vm-no-fence",
|
|
RequestFingerprint: Fingerprint("DELETE", "/v1/vms/vm-no-fence", []byte(`{}`)),
|
|
}
|
|
if _, _, err := j.ResolveResourceFence("operation-no-fence", 7, request); !errors.Is(err, ErrFenceInspectionStale) {
|
|
t.Fatalf("stale inspection err=%v", err)
|
|
}
|
|
request.InspectionCheckedAt = now.Unix()
|
|
if _, _, err := j.ResolveResourceFence("operation-no-fence", 7, request); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("missing operation err=%v", err)
|
|
}
|
|
cleanSpec := testSpec("operation-clean-result", "vm-clean")
|
|
if _, _, err := j.Execute(context.Background(), cleanSpec, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "clean_failure", message: "not applied", effect: false, safe: true}
|
|
}); err == nil {
|
|
t.Fatal("seed clean failure unexpectedly succeeded")
|
|
}
|
|
request.Action = cleanSpec.Action
|
|
request.ResourceID = cleanSpec.ResourceID
|
|
request.RequestFingerprint = cleanSpec.Fingerprint
|
|
if _, _, err := j.ResolveResourceFence("operation-clean-result", 7, request); !errors.Is(err, ErrFenceNotResolvable) {
|
|
t.Fatalf("non-effectful failure err=%v", err)
|
|
}
|
|
storage := storageConfigurationSpec("operation-storage-no-cross-task-fence")
|
|
if _, _, err := j.Execute(context.Background(), storage, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "storage_failed", message: "storage failed", effect: true}
|
|
}); err == nil {
|
|
t.Fatal("seed storage failure unexpectedly succeeded")
|
|
}
|
|
if required, err := j.FenceResolutionRequired(storage.OperationID); err != nil || required {
|
|
t.Fatalf("non-blocking storage fence required=%v err=%v", required, err)
|
|
}
|
|
if state, err := j.FenceState(storage.OperationID); err != nil || state != FenceStateNotRequired {
|
|
t.Fatalf("non-blocking storage fence state=%q err=%v", state, err)
|
|
}
|
|
}
|
|
|
|
func TestSystemFenceResolutionUsesZeroAuditActorButRejectsNegativeActors(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 19, 3, 30, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
spec := testSpec("operation-system-fence", "vm-system-fence")
|
|
spec.Action = "delete-vm"
|
|
spec.Fingerprint = Fingerprint("DELETE", "/v1/vms/vm-system-fence", []byte(`{}`))
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "result_unconfirmed", message: "effect may have applied", effect: true}
|
|
}); err == nil {
|
|
t.Fatal("seed effectful failure unexpectedly succeeded")
|
|
}
|
|
request := FenceResolutionRequest{
|
|
ResolutionID: "01230123012301230123012301230123",
|
|
Resolution: FenceResolutionApplied, InspectionCheckedAt: now.Unix(),
|
|
ReasonSHA256: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
|
|
Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint,
|
|
}
|
|
resolved, replayed, err := j.ResolveResourceFence(spec.OperationID, 0, request)
|
|
if err != nil || replayed || resolved.ResolvedBy != 0 {
|
|
t.Fatalf("system fence resolution = %#v replayed=%v err=%v", resolved, replayed, err)
|
|
}
|
|
|
|
disallowed := testSpec("operation-system-delete-fence", "vm-system-delete-fence")
|
|
disallowed.Action = "future-unknown"
|
|
disallowed.Fingerprint = Fingerprint("POST", "/v1/future/vm-system-delete-fence", []byte(`{}`))
|
|
if _, _, err := j.Execute(context.Background(), disallowed, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "result_unconfirmed", message: "effect may have applied", effect: true}
|
|
}); err == nil {
|
|
t.Fatal("seed system-disallowed failure unexpectedly succeeded")
|
|
}
|
|
request.ResolutionID = "23012301230123012301230123012301"
|
|
request.Action = disallowed.Action
|
|
request.ResourceID = disallowed.ResourceID
|
|
request.RequestFingerprint = disallowed.Fingerprint
|
|
if _, _, err := j.ResolveResourceFence(disallowed.OperationID, 0, request); !errors.Is(err, ErrFenceNotResolvable) {
|
|
t.Fatalf("system actor resolved unknown-action fence: %v", err)
|
|
}
|
|
var successorCalls atomic.Int32
|
|
successor := testSpec("operation-after-system-delete-fence", disallowed.ResourceID)
|
|
if _, _, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
|
|
successorCalls.Add(1)
|
|
return json.RawMessage(`{"deleted":true}`), nil
|
|
}); !errors.Is(err, ErrResourceBlocked) || successorCalls.Load() != 0 {
|
|
t.Fatalf("rejected system resolution released blocker: calls=%d err=%v", successorCalls.Load(), err)
|
|
}
|
|
|
|
other := testSpec("operation-negative-fence", "vm-negative-fence")
|
|
if _, _, err := j.Execute(context.Background(), other, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "result_unconfirmed", message: "effect may have applied", effect: true}
|
|
}); err == nil {
|
|
t.Fatal("seed negative actor failure unexpectedly succeeded")
|
|
}
|
|
request.ResolutionID = "32103210321032103210321032103210"
|
|
if _, _, err := j.ResolveResourceFence(other.OperationID, -1, request); err == nil {
|
|
t.Fatal("negative fence resolution actor was accepted")
|
|
}
|
|
}
|
|
|
|
func TestSystemCreateFenceResolutionOnlyAcceptsNotApplied(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
now := time.Date(2026, 10, 21, 2, 0, 0, 0, time.UTC)
|
|
j.SetClockForTesting(func() time.Time { return now })
|
|
newCreate := func(operationID, resourceID string) Spec {
|
|
return Spec{
|
|
OperationID: operationID,
|
|
ResourceID: resourceID,
|
|
Action: "create-vm",
|
|
Fingerprint: Fingerprint("POST", "/v1/vms", []byte(`{"operation_id":"`+operationID+`"}`)),
|
|
Destructive: true,
|
|
}
|
|
}
|
|
seedUnknown := func(spec Spec) {
|
|
t.Helper()
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "result_unconfirmed", message: "create result is unknown", effect: true}
|
|
}); err == nil {
|
|
t.Fatal("seed create failure unexpectedly succeeded")
|
|
}
|
|
}
|
|
requestFor := func(spec Spec, resolutionID, resolution string) FenceResolutionRequest {
|
|
return FenceResolutionRequest{
|
|
ResolutionID: resolutionID,
|
|
Resolution: resolution,
|
|
InspectionCheckedAt: now.Unix(),
|
|
ReasonSHA256: "abababababababababababababababababababababababababababababababab",
|
|
Action: spec.Action,
|
|
ResourceID: spec.ResourceID,
|
|
RequestFingerprint: spec.Fingerprint,
|
|
}
|
|
}
|
|
|
|
systemSpec := newCreate("operation-system-create", "vm-system-create")
|
|
seedUnknown(systemSpec)
|
|
applied := requestFor(
|
|
systemSpec,
|
|
"10101010101010101010101010101010",
|
|
FenceResolutionApplied,
|
|
)
|
|
if _, _, err := j.ResolveResourceFence(systemSpec.OperationID, 0, applied); !errors.Is(err, ErrFenceNotResolvable) {
|
|
t.Fatalf("system create applied resolution err=%v", err)
|
|
}
|
|
if state, err := j.FenceState(systemSpec.OperationID); err != nil || state != FenceStateActive {
|
|
t.Fatalf("rejected system create resolution released fence state=%q err=%v", state, err)
|
|
}
|
|
notApplied := applied
|
|
notApplied.ResolutionID = "20202020202020202020202020202020"
|
|
notApplied.Resolution = FenceResolutionNotApplied
|
|
resolved, replayed, err := j.ResolveResourceFence(systemSpec.OperationID, 0, notApplied)
|
|
if err != nil || replayed || resolved.Resolution != FenceResolutionNotApplied || resolved.ResolvedBy != 0 {
|
|
t.Fatalf("system create not-applied resolution=%#v replayed=%v err=%v", resolved, replayed, err)
|
|
}
|
|
|
|
manualSpec := newCreate("operation-manual-create", "vm-manual-create")
|
|
seedUnknown(manualSpec)
|
|
manualApplied := requestFor(
|
|
manualSpec,
|
|
"30303030303030303030303030303030",
|
|
FenceResolutionApplied,
|
|
)
|
|
resolved, replayed, err = j.ResolveResourceFence(manualSpec.OperationID, 42, manualApplied)
|
|
if err != nil || replayed || resolved.Resolution != FenceResolutionApplied || resolved.ResolvedBy != 42 {
|
|
t.Fatalf("manual create applied resolution=%#v replayed=%v err=%v", resolved, replayed, err)
|
|
}
|
|
|
|
fencedSpec := newCreate("operation-system-create-unreaped", "vm-system-create-unreaped")
|
|
if _, _, err := j.Execute(context.Background(), fencedSpec, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{
|
|
code: "worker_reap_unconfirmed", message: "worker may still be running",
|
|
effect: true, fence: true,
|
|
}
|
|
}); err == nil {
|
|
t.Fatal("seed execution-fenced create unexpectedly succeeded")
|
|
}
|
|
fencedRequest := requestFor(
|
|
fencedSpec,
|
|
"40404040404040404040404040404040",
|
|
FenceResolutionNotApplied,
|
|
)
|
|
if _, _, err := j.ResolveResourceFence(fencedSpec.OperationID, 0, fencedRequest); !errors.Is(err, ErrFenceNotResolvable) {
|
|
t.Fatalf("system execution-fenced create resolution err=%v", err)
|
|
}
|
|
if state, err := j.FenceState(fencedSpec.OperationID); err != nil || state != FenceStateActive {
|
|
t.Fatalf("execution-fenced create was released state=%q err=%v", state, err)
|
|
}
|
|
}
|
|
|
|
func TestSystemRecoveryActionAllowlistIsExplicit(t *testing.T) {
|
|
allowed := []string{
|
|
"create-vm",
|
|
"power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy",
|
|
"reinstall-vm",
|
|
"enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended",
|
|
"change-network", "network-suspend", "network-resume",
|
|
"update-settings", "resize-vm", "apply-firewall",
|
|
"apply-vpc-network:0123456789abcdef0123456789abcdef",
|
|
"remove-vpc-network:0123456789abcdef0123456789abcdef",
|
|
"apply-vpc-port", "remove-vpc-port",
|
|
"delete-vm", "force-delete-vm",
|
|
"service-suspend",
|
|
"prepare-peer-migration:0123456789abcdef0123456789abcdef",
|
|
"receive-peer-migration:0123456789abcdef0123456789abcdef",
|
|
"cleanup-peer-migration-target:0123456789abcdef0123456789abcdef",
|
|
"activate-peer-migration:0123456789abcdef0123456789abcdef",
|
|
"rollback-peer-migration:0123456789abcdef0123456789abcdef",
|
|
"create-snapshot:0123456789abcdef0123456789abcdef:filesystem",
|
|
"create-snapshot:0123456789abcdef0123456789abcdef:crash",
|
|
"delete-snapshot:0123456789abcdef0123456789abcdef",
|
|
"restore-snapshot:0123456789abcdef0123456789abcdef",
|
|
"import-template", "update-template", "delete-template",
|
|
"import-rescue-template", "delete-rescue-template",
|
|
"import-user-iso", "delete-user-iso",
|
|
}
|
|
for _, action := range allowed {
|
|
if !SystemRecoveryActionAllowed(action) {
|
|
t.Errorf("reviewed action %q was rejected", action)
|
|
}
|
|
}
|
|
|
|
denied := []string{
|
|
"", "configure-storage", "future-unknown", "power-future",
|
|
"exit-rescue:future",
|
|
"create-snapshot:0123456789abcdef0123456789abcdef:future",
|
|
"create-snapshot:0123456789ABCDEF0123456789ABCDEF:filesystem",
|
|
"delete-snapshot:0123456789abcdef0123456789abcde",
|
|
"restore-snapshot:0123456789abcdef0123456789abcdef:extra",
|
|
"import-template:future",
|
|
"apply-vpc-network:0123456789abcdef0123456789abcde",
|
|
"remove-vpc-network:0123456789ABCDEF0123456789ABCDEF",
|
|
"apply-vpc-port:future", "remove-vpc-port:future",
|
|
}
|
|
for _, action := range denied {
|
|
if SystemRecoveryActionAllowed(action) {
|
|
t.Errorf("unreviewed action %q was accepted", action)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResizeExpectedStateRequiresCompleteMonotonicSourceAndTarget(t *testing.T) {
|
|
expected := ExpectedState{
|
|
Kind: ExpectedResizeResources, TransitionID: "operation-resize-expected",
|
|
FromVCPUs: 2, TargetVCPUs: 4,
|
|
FromMemoryMiB: 2048, TargetMemoryMiB: 4096,
|
|
FromDiskGiB: 40, TargetDiskGiB: 80,
|
|
FromBandwidthMbps: 100, TargetBandwidthMbps: 200,
|
|
}
|
|
if err := validateExpectedState("resize-vm", expected.TransitionID, expected); err != nil {
|
|
t.Fatalf("valid resize expected state rejected: %v", err)
|
|
}
|
|
for name, mutate := range map[string]func(*ExpectedState){
|
|
"wrong action": func(value *ExpectedState) { value.Kind = ExpectedUpdateSettings },
|
|
"CPU shrink": func(value *ExpectedState) { value.TargetVCPUs = 1 },
|
|
"disk shrink": func(value *ExpectedState) { value.TargetDiskGiB = 20 },
|
|
"unchanged": func(value *ExpectedState) {
|
|
value.TargetVCPUs, value.TargetMemoryMiB, value.TargetDiskGiB, value.TargetBandwidthMbps =
|
|
value.FromVCPUs, value.FromMemoryMiB, value.FromDiskGiB, value.FromBandwidthMbps
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
candidate := expected
|
|
mutate(&candidate)
|
|
if err := validateExpectedState("resize-vm", candidate.TransitionID, candidate); err == nil {
|
|
t.Fatal("invalid resize expected state was accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVPCExpectedStateBindsActionKeyRevisionAndDigest(t *testing.T) {
|
|
const (
|
|
operationID = "operation-vpc-expected"
|
|
key = "0123456789abcdef0123456789abcdef"
|
|
digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
|
)
|
|
expected := ExpectedState{
|
|
Kind: ExpectedVPCNetwork, TransitionID: operationID,
|
|
VPCKey: key, VPCRevision: 2, VPCDigest: digest, VPCPresent: true,
|
|
}
|
|
if err := validateExpectedState("apply-vpc-network:"+key, operationID, expected); err != nil {
|
|
t.Fatalf("valid VPC expected state rejected: %v", err)
|
|
}
|
|
for name, mutate := range map[string]func(*ExpectedState){
|
|
"wrong key": func(value *ExpectedState) { value.VPCKey = "1123456789abcdef0123456789abcdef" },
|
|
"missing revision": func(value *ExpectedState) { value.VPCRevision = 0 },
|
|
"invalid digest": func(value *ExpectedState) { value.VPCDigest = "bad" },
|
|
"wrong presence": func(value *ExpectedState) { value.VPCPresent = false },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
candidate := expected
|
|
mutate(&candidate)
|
|
if err := validateExpectedState("apply-vpc-network:"+key, operationID, candidate); err == nil {
|
|
t.Fatal("invalid VPC expected state was accepted")
|
|
}
|
|
})
|
|
}
|
|
expected.VPCPresent = false
|
|
if err := validateExpectedState("remove-vpc-network:"+key, operationID, expected); err != nil {
|
|
t.Fatalf("valid VPC removal expected state rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestVPCPortExpectedStateBindsCompleteRoutedIdentity(t *testing.T) {
|
|
const (
|
|
operationID = "operation-vpc-port-expected"
|
|
vpcKey = "0123456789abcdef0123456789abcdef"
|
|
portKey = "1123456789abcdef0123456789abcdef"
|
|
digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
|
)
|
|
expected := ExpectedState{
|
|
Kind: ExpectedVPCPort, TransitionID: operationID, VPCPresent: true,
|
|
VPCKey: vpcKey, VPCPortKey: portKey, VPCRevision: 7, VPCDigest: digest,
|
|
VPCCIDR: "10.60.0.0/24", VPCGateway: "10.60.0.1",
|
|
VPCPrivateIP: "10.60.0.9", VPCMAC: "02:00:00:00:00:09",
|
|
VPCBridge: "vpc-0123456789ab",
|
|
}
|
|
if err := validateExpectedState("apply-vpc-port", operationID, expected); err != nil {
|
|
t.Fatalf("valid VPC port expected state rejected: %v", err)
|
|
}
|
|
for name, mutate := range map[string]func(*ExpectedState){
|
|
"wrong action": func(value *ExpectedState) { value.VPCPresent = false },
|
|
"missing revision": func(value *ExpectedState) { value.VPCRevision = 0 },
|
|
"noncanonical CIDR": func(value *ExpectedState) { value.VPCCIDR = "10.60.0.9/24" },
|
|
"wrong gateway": func(value *ExpectedState) { value.VPCGateway = "10.60.0.2" },
|
|
"gateway address": func(value *ExpectedState) { value.VPCPrivateIP = "10.60.0.1" },
|
|
"multicast MAC": func(value *ExpectedState) { value.VPCMAC = "03:00:00:00:00:09" },
|
|
"upper-case MAC": func(value *ExpectedState) { value.VPCMAC = "02:00:00:00:00:AA" },
|
|
"invalid bridge": func(value *ExpectedState) { value.VPCBridge = "bad/bridge" },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
candidate := expected
|
|
mutate(&candidate)
|
|
if err := validateExpectedState("apply-vpc-port", operationID, candidate); err == nil {
|
|
t.Fatal("invalid VPC port expected state was accepted")
|
|
}
|
|
})
|
|
}
|
|
expected.VPCPresent = false
|
|
if err := validateExpectedState("remove-vpc-port", operationID, expected); err != nil {
|
|
t.Fatalf("valid VPC port removal expected state rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateLikeNetworkExpectedStateBindsPublicSourceAndOptionalVPC(t *testing.T) {
|
|
const (
|
|
operationID = "operation-create-like-network"
|
|
migrationID = "0123456789abcdef0123456789abcdef"
|
|
vpcKey = "1123456789abcdef0123456789abcdef"
|
|
portKey = "2123456789abcdef0123456789abcdef"
|
|
digest = "3123456789abcdef0123456789abcdef3123456789abcdef0123456789abcdef"
|
|
)
|
|
expected := ExpectedState{
|
|
Kind: ExpectedCreateLikeNetwork, TransitionID: operationID,
|
|
NetworkSourceType: "bridge", NetworkSourceName: "br0",
|
|
PublicNetwork: publicnetwork.Intent{
|
|
Contract: 1, NetworkSelector: "bridge:br0",
|
|
IPv4: publicnetwork.AddressIntent{
|
|
PoolKey: migrationID, PoolCIDR: "8.8.8.0/24",
|
|
AddressCIDR: "8.8.8.10/24", Gateway: "8.8.8.1",
|
|
},
|
|
IPv6: publicnetwork.AddressIntent{
|
|
PoolKey: "abcdef0123456789abcdef0123456789",
|
|
PoolCIDR: "2404:c140:156:3a::/64", AddressCIDR: "2404:c140:156:3a::1000/64",
|
|
Gateway: "2404:c140:156::1", GatewayOnLink: true,
|
|
},
|
|
DNSServers: []string{"1.1.1.1", "2606:4700:4700::1111"},
|
|
},
|
|
}
|
|
restoreAction := "restore-backup:" + migrationID
|
|
if err := validateExpectedState(restoreAction, operationID, expected); err != nil {
|
|
t.Fatalf("valid restore expected state rejected: %v", err)
|
|
}
|
|
receiveAction := "receive-peer-migration:" + migrationID
|
|
if err := validateExpectedState(receiveAction, operationID, expected); err != nil {
|
|
t.Fatalf("valid public-only migration expected state rejected: %v", err)
|
|
}
|
|
|
|
withVPC := expected
|
|
withVPC.VPCPresent = true
|
|
withVPC.VPCKey = vpcKey
|
|
withVPC.VPCPortKey = portKey
|
|
withVPC.VPCRevision = 7
|
|
withVPC.VPCDigest = digest
|
|
withVPC.VPCCIDR = "10.60.0.0/24"
|
|
withVPC.VPCGateway = "10.60.0.1"
|
|
withVPC.VPCPrivateIP = "10.60.0.9"
|
|
withVPC.VPCMAC = "02:00:00:00:00:09"
|
|
withVPC.VPCBridge = "vc0123456789ab"
|
|
if err := validateExpectedState(receiveAction, operationID, withVPC); err != nil {
|
|
t.Fatalf("valid VPC migration expected state rejected: %v", err)
|
|
}
|
|
|
|
for name, candidate := range map[string]ExpectedState{
|
|
"restore cannot carry VPC": withVPC,
|
|
"source must match selector": func() ExpectedState {
|
|
value := expected
|
|
value.NetworkSourceName = "br1"
|
|
return value
|
|
}(),
|
|
"public network must remain canonical": func() ExpectedState {
|
|
value := expected
|
|
value.PublicNetwork.IPv6.AddressCIDR = "2404:C140:156:3A::1000/64"
|
|
return value
|
|
}(),
|
|
"absent VPC cannot retain identity": func() ExpectedState {
|
|
value := expected
|
|
value.VPCKey = vpcKey
|
|
return value
|
|
}(),
|
|
"guest identity is outside this projection": func() ExpectedState {
|
|
value := expected
|
|
value.Hostname = "forbidden.example.com"
|
|
return value
|
|
}(),
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
action := receiveAction
|
|
if name == "restore cannot carry VPC" {
|
|
action = restoreAction
|
|
}
|
|
if err := validateExpectedState(action, operationID, candidate); err == nil {
|
|
t.Fatal("invalid create-like network expected state was accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCreateLikeNetworkExpectedStatePersistsWithoutRequestSecrets(t *testing.T) {
|
|
const (
|
|
operationID = "operation-create-like-secrecy"
|
|
migrationID = "0123456789abcdef0123456789abcdef"
|
|
)
|
|
j := openTestJournal(t)
|
|
expected := &ExpectedState{
|
|
Kind: ExpectedCreateLikeNetwork, TransitionID: operationID,
|
|
NetworkSourceType: "bridge", NetworkSourceName: "br0",
|
|
PublicNetwork: publicnetwork.Intent{
|
|
Contract: 1, NetworkSelector: "bridge:br0",
|
|
IPv4: publicnetwork.AddressIntent{
|
|
PoolKey: migrationID, PoolCIDR: "8.8.8.0/24",
|
|
AddressCIDR: "8.8.8.10/24", Gateway: "8.8.8.1",
|
|
},
|
|
IPv6: publicnetwork.AddressIntent{
|
|
PoolKey: "abcdef0123456789abcdef0123456789",
|
|
PoolCIDR: "2404:c140:156:3a::/64", AddressCIDR: "2404:c140:156:3a::1000/64",
|
|
Gateway: "2404:c140:156::1", GatewayOnLink: true,
|
|
},
|
|
DNSServers: []string{"1.1.1.1", "2606:4700:4700::1111"},
|
|
},
|
|
}
|
|
sensitiveRequestBody := []byte(`{
|
|
"source_url":"signed-source-url-secret",
|
|
"ticket":"migration-ticket-secret",
|
|
"root_password":"guest-password-secret",
|
|
"guest_ssh_public_key":"ssh-public-key-secret",
|
|
"startup_script":"startup-script-secret"
|
|
}`)
|
|
spec := Spec{
|
|
OperationID: operationID, ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "restore-backup:" + migrationID,
|
|
Fingerprint: Fingerprint("POST", "/v1/vms/123e4567-e89b-42d3-a456-426614174000/restore-backup", sensitiveRequestBody),
|
|
Destructive: true, ExpectedState: expected,
|
|
}
|
|
if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"restored":true}`), nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var encoded []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
encoded = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(operationID))...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, required := range []string{
|
|
`"expected_state"`, `"kind":"create-like-network"`,
|
|
`"network_selector":"bridge:br0"`, `"network_source_name":"br0"`,
|
|
`"address_cidr":"2404:c140:156:3a::1000/64"`,
|
|
} {
|
|
if !bytes.Contains(encoded, []byte(required)) {
|
|
t.Fatalf("persisted expected state is missing %s: %s", required, encoded)
|
|
}
|
|
}
|
|
for _, forbidden := range []string{
|
|
`"source_url"`, `"ticket"`, `"root_password"`, `"guest_ssh_public_key"`,
|
|
`"startup_script"`, `"hostname"`, "signed-source-url-secret", "migration-ticket-secret",
|
|
"guest-password-secret", "ssh-public-key-secret", "startup-script-secret",
|
|
} {
|
|
if bytes.Contains(encoded, []byte(forbidden)) {
|
|
t.Fatalf("persisted expected state leaked forbidden request data %q: %s", forbidden, encoded)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSuccessfulForceSuccessorOnlyRemovesPredecessorBlockerIndex(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
predecessor := testSpec("operation-graceful-predecessor", "vm-force-successor")
|
|
if _, _, err := j.Execute(context.Background(), predecessor, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "force_required", message: "explicit force required", effect: true, safe: false}
|
|
}); err == nil {
|
|
t.Fatal("graceful predecessor unexpectedly succeeded")
|
|
}
|
|
claimed, err := j.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 {
|
|
t.Fatalf("claim predecessor completion = %#v err=%v", claimed, err)
|
|
}
|
|
var operationBefore, receiptBefore []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
operationBefore = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(predecessor.OperationID))...)
|
|
receiptBefore = append([]byte(nil), tx.Bucket(completionReceiptsBucket).Get([]byte(claimed[0].EventID))...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
successor := testSpec("operation-force-successor", predecessor.ResourceID)
|
|
successor.Action = "force-delete-vm"
|
|
successor.Fingerprint = Fingerprint("DELETE", "/v1/vms/"+predecessor.ResourceID+"?force=1", []byte(`{}`))
|
|
successor.SupersedesOperationID = predecessor.OperationID
|
|
if record, _, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"deleted":true}`), nil
|
|
}); err != nil || record.State != StateSucceeded {
|
|
t.Fatalf("force successor = %#v err=%v", record, err)
|
|
}
|
|
var operationAfter, receiptAfter []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
operationAfter = append([]byte(nil), tx.Bucket(operationsBucket).Get([]byte(predecessor.OperationID))...)
|
|
receiptAfter = append([]byte(nil), tx.Bucket(completionReceiptsBucket).Get([]byte(claimed[0].EventID))...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(operationBefore, operationAfter) || !bytes.Equal(receiptBefore, receiptAfter) {
|
|
t.Fatal("force successor rewrote predecessor terminal evidence")
|
|
}
|
|
followup := testSpec("operation-after-force-successor", predecessor.ResourceID)
|
|
if record, _, err := j.Execute(context.Background(), followup, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"accepted":true}`), nil
|
|
}); err != nil || record.State != StateSucceeded {
|
|
t.Fatalf("predecessor blocker remained after force successor: %#v err=%v", record, err)
|
|
}
|
|
}
|
|
|
|
func TestSuccessfulVPCRepairSupersedesOnlyExactFencedIntent(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
const (
|
|
vpcKey = "0123456789abcdef0123456789abcdef"
|
|
digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
|
)
|
|
makeSpec := func(id string) Spec {
|
|
return Spec{
|
|
OperationID: id,
|
|
ResourceID: "vpc:" + vpcKey,
|
|
Action: "apply-vpc-network:" + vpcKey,
|
|
Fingerprint: Fingerprint("POST", "/v1/vpc/networks/"+vpcKey+"/apply/repair", []byte(id)),
|
|
Destructive: true,
|
|
ExpectedState: &ExpectedState{
|
|
Kind: ExpectedVPCNetwork, TransitionID: id,
|
|
VPCKey: vpcKey, VPCRevision: 7,
|
|
VPCDigest: digest, VPCPresent: true,
|
|
},
|
|
LockKeys: []string{"catalog:vpc-host/state", "catalog:vpc/" + vpcKey},
|
|
}
|
|
}
|
|
predecessor := makeSpec("operation-vpc-repair-predecessor")
|
|
if _, _, err := j.Execute(context.Background(), predecessor,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{
|
|
code: "vpc_reconcile_unconfirmed", message: "VPC state is inconsistent",
|
|
effect: true, safe: false, fence: true,
|
|
}
|
|
}); err == nil {
|
|
t.Fatal("VPC predecessor unexpectedly succeeded")
|
|
}
|
|
prior, err := j.Get(predecessor.OperationID)
|
|
if err != nil || !prior.ExecutionFenced {
|
|
t.Fatalf("VPC predecessor is not fenced: %#v err=%v", prior, err)
|
|
}
|
|
var before []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
before = append([]byte(nil), tx.Bucket(operationsBucket).Get(
|
|
[]byte(predecessor.OperationID),
|
|
)...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
successor := makeSpec("operation-vpc-repair-successor")
|
|
successor.SupersedesOperationID = predecessor.OperationID
|
|
if record, _, err := j.Execute(context.Background(), successor,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"applied":true}`), nil
|
|
}); err != nil || record.State != StateSucceeded {
|
|
t.Fatalf("VPC repair successor = %#v err=%v", record, err)
|
|
}
|
|
var after []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
after = append([]byte(nil), tx.Bucket(operationsBucket).Get(
|
|
[]byte(predecessor.OperationID),
|
|
)...)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(before, after) {
|
|
t.Fatal("VPC repair rewrote predecessor terminal evidence")
|
|
}
|
|
|
|
followup := makeSpec("operation-vpc-after-repair")
|
|
followup.ExpectedState.VPCRevision = 8
|
|
followup.ExpectedState.VPCDigest = strings.Repeat("b", 64)
|
|
if _, _, err := j.Execute(context.Background(), followup,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"applied":true}`), nil
|
|
}); err != nil {
|
|
t.Fatalf("VPC predecessor blocker survived repair: %v", err)
|
|
}
|
|
|
|
invalid := makeSpec("operation-vpc-repair-wrong-intent")
|
|
invalid.ExpectedState.VPCDigest = strings.Repeat("c", 64)
|
|
invalid.SupersedesOperationID = predecessor.OperationID
|
|
if _, _, err := j.Execute(context.Background(), invalid,
|
|
func(context.Context) (json.RawMessage, error) { return nil, nil }); !errors.Is(err, ErrInvalidSupersession) {
|
|
t.Fatalf("VPC repair accepted changed intent: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSuccessfulVPCRepairClosesExactUnknownRepairChain(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
const (
|
|
vpcKey = "1123456789abcdef0123456789abcdef"
|
|
digest = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"
|
|
)
|
|
makeSpec := func(id string) Spec {
|
|
return Spec{
|
|
OperationID: id,
|
|
ResourceID: "vpc:" + vpcKey,
|
|
Action: "apply-vpc-network:" + vpcKey,
|
|
Fingerprint: Fingerprint(
|
|
"POST", "/v1/vpc/networks/"+vpcKey+"/apply/repair",
|
|
[]byte(id),
|
|
),
|
|
Destructive: true,
|
|
ExpectedState: &ExpectedState{
|
|
Kind: ExpectedVPCNetwork, TransitionID: id,
|
|
VPCKey: vpcKey, VPCRevision: 9,
|
|
VPCDigest: digest, VPCPresent: true,
|
|
},
|
|
LockKeys: []string{
|
|
"catalog:vpc-host/state", "catalog:vpc/" + vpcKey,
|
|
},
|
|
}
|
|
}
|
|
failUnknown := func(spec Spec) {
|
|
if _, _, err := j.Execute(context.Background(), spec,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{
|
|
code: "vpc_reconcile_unconfirmed",
|
|
message: "repair result was not confirmed",
|
|
effect: true, safe: false, fence: true,
|
|
}
|
|
}); err == nil {
|
|
t.Fatalf("VPC generation %s unexpectedly succeeded", spec.OperationID)
|
|
}
|
|
}
|
|
|
|
original := makeSpec("operation-vpc-chain-original")
|
|
failUnknown(original)
|
|
firstRepair := makeSpec("operation-vpc-chain-repair-one")
|
|
firstRepair.SupersedesOperationID = original.OperationID
|
|
failUnknown(firstRepair)
|
|
|
|
var originalBefore, repairBefore []byte
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(operationsBucket)
|
|
originalBefore = append(
|
|
[]byte(nil), bucket.Get([]byte(original.OperationID))...,
|
|
)
|
|
repairBefore = append(
|
|
[]byte(nil), bucket.Get([]byte(firstRepair.OperationID))...,
|
|
)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
secondRepair := makeSpec("operation-vpc-chain-repair-two")
|
|
secondRepair.SupersedesOperationID = firstRepair.OperationID
|
|
if record, _, err := j.Execute(
|
|
context.Background(), secondRepair,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"applied":true}`), nil
|
|
},
|
|
); err != nil || record.State != StateSucceeded {
|
|
t.Fatalf("second repair = %#v err=%v", record, err)
|
|
}
|
|
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(operationsBucket)
|
|
if !bytes.Equal(
|
|
originalBefore, bucket.Get([]byte(original.OperationID)),
|
|
) || !bytes.Equal(
|
|
repairBefore, bucket.Get([]byte(firstRepair.OperationID)),
|
|
) {
|
|
t.Fatal("successful repair rewrote predecessor evidence")
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
followup := makeSpec("operation-vpc-chain-followup")
|
|
followup.ExpectedState.VPCRevision = 10
|
|
followup.ExpectedState.VPCDigest = strings.Repeat("d", 64)
|
|
if _, _, err := j.Execute(
|
|
context.Background(), followup,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"applied":true}`), nil
|
|
},
|
|
); err != nil {
|
|
t.Fatalf("repair chain retained a stale blocker: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPeerMigrationTargetCleanupMayCrossOnlyItsExactReceiveFence(t *testing.T) {
|
|
j := openTestJournal(t)
|
|
migrationID := "0123456789abcdef0123456789abcdef"
|
|
predecessor := testSpec("operation-peer-receive-residue", "vm-peer-residue")
|
|
predecessor.Action = "receive-peer-migration:" + migrationID
|
|
predecessor.Fingerprint = Fingerprint("POST", "/v1/vms/"+predecessor.ResourceID+"/peer-migration/receive", []byte(`{}`))
|
|
if _, _, err := j.Execute(context.Background(), predecessor, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "result_unconfirmed", message: "receive result unknown", effect: true, safe: false}
|
|
}); err == nil {
|
|
t.Fatal("peer receive predecessor unexpectedly succeeded")
|
|
}
|
|
|
|
successor := testSpec("operation-peer-target-cleanup", predecessor.ResourceID)
|
|
successor.Action = "cleanup-peer-migration-target:" + migrationID
|
|
successor.Fingerprint = Fingerprint("POST", "/v1/vms/"+predecessor.ResourceID+"/peer-migration/cleanup-target", []byte(`{}`))
|
|
successor.SupersedesOperationID = predecessor.OperationID
|
|
if record, _, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"deleted":true,"managed_artifacts_absent":true,"forced":false}`), nil
|
|
}); err != nil || record.State != StateSucceeded {
|
|
t.Fatalf("peer target cleanup successor = %#v err=%v", record, err)
|
|
}
|
|
|
|
// Cleanup is an evidence-producing successor, not the effect verdict for
|
|
// the original receive. Keep that receive fence active until the panel
|
|
// performs one fresh absence inspection and resolves it as not applied.
|
|
if state, err := j.FenceState(predecessor.OperationID); err != nil || state != FenceStateActive {
|
|
t.Fatalf("peer receive fence state=%q err=%v", state, err)
|
|
}
|
|
followup := testSpec("operation-peer-after-cleanup", predecessor.ResourceID)
|
|
if _, _, err := j.Execute(context.Background(), followup, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"accepted":true}`), nil
|
|
}); !errors.Is(err, ErrResourceBlocked) {
|
|
t.Fatalf("original receive fence was released before reconciliation: %v", err)
|
|
}
|
|
|
|
wrong := testSpec("operation-peer-wrong-cleanup", predecessor.ResourceID)
|
|
wrong.Action = "cleanup-peer-migration-target:fedcba9876543210fedcba9876543210"
|
|
wrong.SupersedesOperationID = predecessor.OperationID
|
|
if _, _, err := j.Execute(context.Background(), wrong, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"deleted":true}`), nil
|
|
}); !errors.Is(err, ErrInvalidSupersession) {
|
|
t.Fatalf("mismatched migration cleanup supersession err=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestInitializeRemovesLegacyStorageBlockerAndPreservesVMFence(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "operations.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
storage := storageConfigurationSpec("operation-legacy-storage")
|
|
if _, _, err := j.Execute(context.Background(), storage, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{
|
|
code: "storage_configuration_failed", message: "storage effect unknown", effect: true, safe: false,
|
|
}
|
|
}); err == nil {
|
|
t.Fatal("legacy storage seed unexpectedly succeeded")
|
|
}
|
|
vm := testSpec("operation-legacy-vm-fence", "vm-legacy-fence")
|
|
if _, _, err := j.Execute(context.Background(), vm, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "vm_effect_unknown", message: "VM effect unknown", effect: true, safe: false}
|
|
}); err == nil {
|
|
t.Fatal("VM fence seed unexpectedly succeeded")
|
|
}
|
|
|
|
storageBlockerKey := blockerEntryKey("storage-config", storage.OperationID)
|
|
vmLockKeys := effectiveLockKeys(vm.OperationID, vm.ResourceID, vm.Action, vm.LockKeys)
|
|
vmBlockerKey := blockerEntryKey(vmLockKeys[0], vm.OperationID)
|
|
storageReceiptKey := []byte(completionEventID(storage.OperationID))
|
|
var operationBefore, receiptBefore []byte
|
|
if err := j.db.Update(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
if err := blockers.Put(storageBlockerKey, nil); err != nil {
|
|
return err
|
|
}
|
|
operationBefore = append([]byte(nil), operations.Get([]byte(storage.OperationID))...)
|
|
receiptBefore = append([]byte(nil), receipts.Get(storageReceiptKey)...)
|
|
if len(operationBefore) == 0 || len(receiptBefore) == 0 {
|
|
return errors.New("storage operation evidence was not persisted")
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
j, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer j.Close()
|
|
if err := j.db.View(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
metadata := tx.Bucket(metadataBucket)
|
|
if string(metadata.Get(schemaVersionKey)) != string(schemaVersion) {
|
|
return fmt.Errorf("schema version changed during blocker repair")
|
|
}
|
|
cursor := blockers.Cursor()
|
|
if key, _ := cursor.Seek(storageBlockerKey); bytes.Equal(key, storageBlockerKey) {
|
|
return errors.New("legacy storage blocker survived restart repair")
|
|
}
|
|
if key, _ := cursor.Seek(vmBlockerKey); !bytes.Equal(key, vmBlockerKey) {
|
|
return errors.New("VM blocker was removed during storage repair")
|
|
}
|
|
if !bytes.Equal(operationBefore, operations.Get([]byte(storage.OperationID))) ||
|
|
!bytes.Equal(receiptBefore, receipts.Get(storageReceiptKey)) {
|
|
return errors.New("storage blocker repair rewrote immutable operation evidence")
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
successor := storageConfigurationSpec("operation-after-storage-repair")
|
|
if record, replayed, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"configured":true}`), nil
|
|
}); err != nil || replayed || record.State != StateSucceeded {
|
|
t.Fatalf("storage successor after restart repair = %#v replayed=%v err=%v", record, replayed, err)
|
|
}
|
|
vmCalled := false
|
|
if _, _, err := j.Execute(context.Background(), testSpec("operation-after-vm-fence", vm.ResourceID), func(context.Context) (json.RawMessage, error) {
|
|
vmCalled = true
|
|
return nil, nil
|
|
}); !errors.Is(err, ErrResourceBlocked) || vmCalled {
|
|
t.Fatalf("VM fence changed during storage repair: called=%v err=%v", vmCalled, err)
|
|
}
|
|
}
|
|
|
|
func TestInitializeRejectsBrokenResourceBlockerReferencesAtomically(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
seed func(*bolt.Tx) error
|
|
}{
|
|
{
|
|
name: "missing operation",
|
|
seed: func(tx *bolt.Tx) error {
|
|
return tx.Bucket(resourceBlockersBucket).Put(
|
|
blockerEntryKey("storage-config", "operation-zz-missing-reference"), nil,
|
|
)
|
|
},
|
|
},
|
|
{
|
|
name: "damaged operation",
|
|
seed: func(tx *bolt.Tx) error {
|
|
operationID := "operation-zz-damaged-reference"
|
|
if err := tx.Bucket(operationsBucket).Put([]byte(operationID), []byte(`{"record":`)); err != nil {
|
|
return err
|
|
}
|
|
return tx.Bucket(resourceBlockersBucket).Put(blockerEntryKey("storage-config", operationID), nil)
|
|
},
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "operations.db")
|
|
j, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
storage := storageConfigurationSpec("operation-valid-storage-reference")
|
|
if _, _, err := j.Execute(context.Background(), storage, func(context.Context) (json.RawMessage, error) {
|
|
return nil, testFailure{code: "storage_configuration_failed", message: "unknown", effect: true}
|
|
}); err == nil {
|
|
t.Fatal("storage seed unexpectedly succeeded")
|
|
}
|
|
storageBlockerKey := blockerEntryKey("storage-config", storage.OperationID)
|
|
if err := j.db.Update(func(tx *bolt.Tx) error {
|
|
if err := tx.Bucket(resourceBlockersBucket).Put(storageBlockerKey, nil); err != nil {
|
|
return err
|
|
}
|
|
return test.seed(tx)
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if reopened, err := Open(path); err == nil {
|
|
_ = reopened.Close()
|
|
t.Fatal("broken blocker reference was accepted")
|
|
}
|
|
db, err := bolt.Open(path, 0o600, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
if err := db.View(func(tx *bolt.Tx) error {
|
|
cursor := tx.Bucket(resourceBlockersBucket).Cursor()
|
|
key, _ := cursor.Seek(storageBlockerKey)
|
|
if !bytes.Equal(key, storageBlockerKey) {
|
|
return errors.New("failed startup partially deleted the valid storage blocker")
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFreshSchemaRejectsOldJournalWithoutMigration(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "operations.db")
|
|
db, err := bolt.Open(path, 0o600, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Update(func(tx *bolt.Tx) error {
|
|
bucket, err := tx.CreateBucket(metadataBucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bucket.Put(schemaVersionKey, []byte("6"))
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if journal, err := Open(path); err == nil {
|
|
_ = journal.Close()
|
|
t.Fatal("old schema was silently migrated")
|
|
} else if !strings.Contains(err.Error(), "removing the old test journal before upgrade") {
|
|
t.Fatalf("old schema rejection did not include clean-break deployment policy: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestFreshSchemaRejectsUnversionedExistingJournal(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "operations.db")
|
|
db, err := bolt.Open(path, 0o600, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Update(func(tx *bolt.Tx) error {
|
|
_, err := tx.CreateBucket([]byte("legacy_operations"))
|
|
return err
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if journal, err := Open(path); err == nil {
|
|
_ = journal.Close()
|
|
t.Fatal("unversioned existing journal was silently initialized")
|
|
}
|
|
}
|