You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/operation/journal.go

3584 lines
128 KiB
Go

package operation
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"net/netip"
"os"
"path/filepath"
"regexp"
"slices"
"sort"
"strings"
"sync"
"time"
"git.luoli.live/root/virt/agent/publicnetwork"
bolt "go.etcd.io/bbolt"
)
type State string
const (
StateReserved State = "reserved"
StateDispatching State = "dispatching"
StateSucceeded State = "succeeded"
StateFailed State = "failed"
)
const (
ExpectedUpdateSettings = "update-settings"
ExpectedResizeResources = "resize-resources"
ExpectedChangeNetwork = "change-network"
ExpectedNetworkState = "network-state"
ExpectedServiceState = "service-state"
ExpectedPeerMigrationState = "peer-migration-state"
ExpectedFirewall = "firewall"
ExpectedStorageConfiguration = "storage-configuration"
ExpectedVPCNetwork = "vpc-network"
ExpectedVPCPort = "vpc-port"
ExpectedCreateLikeNetwork = "create-like-network"
operationDetailRetention = 90 * 24 * time.Hour
)
var (
ErrNotFound = errors.New("operation not found")
ErrFingerprint = errors.New("operation_id payload conflict")
ErrOperationStarting = errors.New("operation dispatch is already starting")
ErrBusy = errors.New("operation execution capacity is exhausted")
ErrResourceBlocked = errors.New("resource has an unresolved operation")
ErrInvalidSupersession = errors.New("operation cannot supersede the requested predecessor")
ErrInspectionInconclusive = errors.New("read-only inspection did not confirm a terminal outcome")
ErrExecutionNotReaped = errors.New("isolated mutation process did not exit after forced termination")
ErrFenceNotResolvable = errors.New("operation does not hold an effect fence")
ErrFenceIdentityConflict = errors.New("fence resolution identity does not match operation")
ErrFenceResolutionConflict = errors.New("operation effect fence already has a different resolution")
ErrFenceInspectionStale = errors.New("operation effect fence resolution requires a fresh inspection")
ErrTerminalEvidencePending = errors.New("worker journal has no terminal operation evidence")
ErrTerminalEvidenceInvalid = errors.New("worker journal terminal evidence is invalid")
operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`)
operationActionPattern = regexp.MustCompile(`^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$`)
operationResourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`)
hexDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
shortHexIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
noncePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`)
networkSourcePattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,128}$`)
causeCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
canonicalUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
resourcePartPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,192}$`)
resourceKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
fenceResolutionIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
completionSensitiveKeyPattern = regexp.MustCompile(`(?i)password|secret|(?:public|private).?key|ssh.?key|passphrase|credential|(?:(?:access|refresh|auth|api|bearer|session|source).?token)|^token$`)
metadataBucket = []byte("metadata")
operationsBucket = []byte("operations")
inflightBucket = []byte("inflight")
resourceBlockersBucket = []byte("resource_blockers")
operationExpiryBucket = []byte("operation_expiry")
completionReceiptsBucket = []byte("completion_receipts")
completionOutboxBucket = []byte("completion_outbox")
completionQuarantineBucket = []byte("completion_quarantine")
completionQuarantineOperationsBucket = []byte("completion_quarantine_operations")
fenceResolutionsBucket = []byte("fence_resolutions")
fenceResolutionIDsBucket = []byte("fence_resolution_ids")
noncesBucket = []byte("nonces")
nonceExpiryBucket = []byte("nonce_expiry")
schemaVersionKey = []byte("schema_version")
schemaVersion = []byte("7")
)
// ExpectedState is the non-secret, durable evidence contract used after a
// crash. It intentionally has no password, SSH key, token, file path, or raw
// request field. The exact request remains bound separately by Fingerprint.
type ExpectedState struct {
Kind string `json:"kind"`
TransitionID string `json:"transition_id"`
HostnameChanged bool `json:"hostname_changed,omitempty"`
Hostname string `json:"hostname,omitempty"`
RootPasswordChanged bool `json:"root_password_changed,omitempty"`
SSHKeyChanged bool `json:"ssh_key_changed,omitempty"`
FromVCPUs uint `json:"from_vcpu,omitempty"`
TargetVCPUs uint `json:"target_vcpu,omitempty"`
FromMemoryMiB uint64 `json:"from_memory_mib,omitempty"`
TargetMemoryMiB uint64 `json:"target_memory_mib,omitempty"`
FromDiskGiB uint64 `json:"from_disk_gib,omitempty"`
TargetDiskGiB uint64 `json:"target_disk_gib,omitempty"`
FromBandwidthMbps uint64 `json:"from_bandwidth_mbps,omitempty"`
TargetBandwidthMbps uint64 `json:"target_bandwidth_mbps,omitempty"`
NetworkSourceType string `json:"network_source_type,omitempty"`
NetworkSourceName string `json:"network_source_name,omitempty"`
PublicNetwork publicnetwork.Intent `json:"public_network,omitempty"`
Enabled bool `json:"enabled,omitempty"`
StartIfSuspended bool `json:"start_if_suspended,omitempty"`
FirewallRevision uint64 `json:"firewall_revision,omitempty"`
FirewallDigest string `json:"firewall_digest,omitempty"`
ManagedPathDigest string `json:"managed_path_digest,omitempty"`
TemplatePathDigest string `json:"template_path_digest,omitempty"`
SeedPathDigest string `json:"seed_path_digest,omitempty"`
UserISOPathDigest string `json:"user_iso_path_digest,omitempty"`
RescuePathDigest string `json:"rescue_path_digest,omitempty"`
VPCKey string `json:"vpc_key,omitempty"`
VPCRevision uint64 `json:"vpc_revision,omitempty"`
VPCDigest string `json:"vpc_digest,omitempty"`
VPCPresent bool `json:"vpc_present,omitempty"`
VPCPortKey string `json:"vpc_port_key,omitempty"`
VPCCIDR string `json:"vpc_cidr,omitempty"`
VPCGateway string `json:"vpc_gateway,omitempty"`
VPCPrivateIP string `json:"vpc_private_ip,omitempty"`
VPCMAC string `json:"vpc_mac,omitempty"`
VPCBridge string `json:"vpc_bridge,omitempty"`
}
type PublicError struct {
Code string `json:"code"`
Message string `json:"message"`
}
type Record struct {
OperationID string `json:"operation_id"`
ResourceID string `json:"resource_id,omitempty"`
Action string `json:"action"`
Fingerprint string `json:"fingerprint"`
Destructive bool `json:"destructive"`
State State `json:"status"`
Result json.RawMessage `json:"result,omitempty"`
Error *PublicError `json:"error,omitempty"`
CauseCode string `json:"cause_code,omitempty"`
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
SafeToResubmit bool `json:"safe_to_resubmit"`
ExecutionFenced bool `json:"execution_fenced,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Tombstone bool `json:"tombstone,omitempty"`
// ExpectedState is persisted in the private journal but deliberately never
// serialized on the HTTP API.
ExpectedState *ExpectedState `json:"-"`
// LockKeys and HeavyIO are private admission metadata. They are persisted so
// restart terminalization can retain the exact affected-resource fence, but
// they are never exposed by the Agent API.
LockKeys []string `json:"-"`
HeavyIO bool `json:"-"`
// SupersedesOperationID is valid only for an explicitly reviewed successor:
// force-delete-vm, plan-bound cleanup, or an exact VPC repair generation.
SupersedesOperationID string `json:"-"`
// InspectionReceiptRecovered and InspectionEvidenceUnavailable are
// ephemeral read-path markers. They are never persisted or exposed by the
// Agent API; the hypervisor inspector uses them to distinguish strictly
// validated completion evidence from a missing or inconsistent receipt.
InspectionReceiptRecovered bool `json:"-"`
InspectionEvidenceUnavailable bool `json:"-"`
}
type Spec struct {
OperationID string
ResourceID string
Action string
Fingerprint string
Destructive bool
ExpectedState *ExpectedState
LockKeys []string
HeavyIO bool
SupersedesOperationID string
}
// ValidOperationIdentity validates the immutable non-secret identity copied
// from a Journal record into inspections and completion deliveries.
func ValidOperationIdentity(action, resourceID, requestFingerprint string) bool {
return len(action) <= 128 && operationActionPattern.MatchString(action) &&
len(resourceID) <= 255 && operationResourceIDPattern.MatchString(resourceID) &&
hexDigestPattern.MatchString(requestFingerprint)
}
const (
completionDeliveryInterval = 30 * time.Second
completionPanelMaximumBody = 1 << 20
fenceInspectionMaximumAge = 2 * time.Minute
fenceInspectionFutureSkew = 5 * time.Second
)
const (
CompletionDeliveryStateQuarantined = "quarantined"
CompletionDeliveryErrorOutboxInvalid = "completion_outbox_invalid"
CompletionDeliveryErrorReceiptMissing = "completion_receipt_missing"
CompletionDeliveryErrorReceiptInvalid = "completion_receipt_invalid"
CompletionDeliveryErrorOperationMissing = "completion_operation_missing"
CompletionDeliveryErrorOperationInvalid = "completion_operation_invalid"
CompletionDeliveryErrorIdentityMismatch = "completion_identity_mismatch"
)
var workerReapDeadline = 3 * time.Second
var terminalEvidenceOpenTimeout = 2 * time.Second
// CompletionReceipt is immutable once an operation first reaches a terminal
// state. Delivery timing is kept in a separate outbox record so an ACK can
// never rewrite the operation or its receipt.
type CompletionReceipt struct {
Version int `json:"version,omitempty"`
EventID string `json:"id"`
OperationID string `json:"operation_id"`
Action string `json:"action"`
ResourceID string `json:"resource_id"`
RequestFingerprint string `json:"request_fingerprint"`
Status string `json:"status"`
Result json.RawMessage `json:"result"`
ErrorCode string `json:"error_code,omitempty"`
CauseCode string `json:"cause_code,omitempty"`
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
SafeToResubmit bool `json:"safe_to_resubmit"`
CreatedAt time.Time `json:"-"`
}
const (
FenceResolutionApplied = "applied"
FenceResolutionNotApplied = "not_applied"
FenceStateNotRequired FenceState = "not_required"
FenceStateActive FenceState = "active"
FenceStateResolved FenceState = "resolved"
)
// FenceState distinguishes an operation that never owned a cross-task fence
// from an active blocker and one already released by immutable audit metadata.
// Callers must not collapse resolved into not_required: doing so could let a
// second coordinator accept evidence that conflicts with the stored verdict.
type FenceState string
// FenceResolution is immutable audit metadata for a super-administrator's
// explicit conclusion after a fresh, read-only host inspection. It releases
// only Journal admission blockers; it never changes the original operation or
// completion receipt and never authorizes replay of that mutation.
type FenceResolution struct {
OperationID string `json:"operation_id"`
ResolutionID string `json:"resolution_id"`
Resolution string `json:"resolution"`
InspectionCheckedAt int64 `json:"inspection_checked_at"`
ReasonSHA256 string `json:"reason_sha256"`
ResolvedAt int64 `json:"resolved_at"`
ResolvedBy int64 `json:"resolved_by"`
}
type FenceResolutionRequest struct {
ResolutionID string `json:"resolution_id"`
Resolution string `json:"resolution"`
InspectionCheckedAt int64 `json:"inspection_checked_at"`
ReasonSHA256 string `json:"reason_sha256"`
Action string `json:"action"`
ResourceID string `json:"resource_id"`
RequestFingerprint string `json:"request_fingerprint"`
}
// InterruptedReceipt is the non-secret terminal result recovered from an
// isolated worker directory after the parent Agent restarts.
type InterruptedReceipt struct {
OperationID string
Fingerprint string
Status string
Result json.RawMessage
ErrorCode string
ErrorMessage string
EffectMayHaveApplied bool
SafeToResubmit bool
}
type completionOutboxRecord struct {
OperationID string `json:"operation_id"`
NextAttemptAt time.Time `json:"next_attempt_at"`
}
// CompletionDeliveryIncident is durable, non-secret audit metadata for one
// outbox entry that could not be reconstructed into the immutable completion
// wire contract. The receipt and outbox bytes remain untouched. A later claim
// revalidates them and removes this incident automatically after repair.
type CompletionDeliveryIncident struct {
EventID string `json:"event_id,omitempty"`
EventKeySHA256 string `json:"event_key_sha256"`
OperationID string `json:"operation_id,omitempty"`
ErrorCode string `json:"error_code"`
EvidenceSHA256 string `json:"evidence_sha256"`
FirstObservedAt time.Time `json:"first_observed_at"`
}
type Result struct {
Value json.RawMessage
}
type Resolution struct {
Resolved bool
Succeeded bool
Result json.RawMessage
ErrorCode string
ErrorMessage string
}
type UncertainError struct {
Cause error
}
func (e *UncertainError) Error() string {
if e.Cause == nil {
return "operation result is unconfirmed"
}
return fmt.Sprintf("operation result is unconfirmed: %v", e.Cause)
}
func (e *UncertainError) Unwrap() error { return e.Cause }
func MarkUncertain(err error) error { return &UncertainError{Cause: err} }
func hasUncertainMarker(err error) bool {
var target *UncertainError
return errors.As(err, &target)
}
func IsUncertain(err error) bool {
// Explicit uncertainty wins. Explicit NotApplied then proves that the
// caller never crossed its mutation boundary, even when the underlying
// pre-dispatch wait ended through context cancellation. Only an otherwise
// unclassified context failure is ambiguous.
if hasUncertainMarker(err) {
return true
}
if IsNotApplied(err) {
return false
}
return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
}
// NotAppliedError records that the caller failed before its exact mutation
// boundary. The wrapped error keeps its public code/message, while the Journal
// can durably distinguish a retry-safe precondition failure from an ordinary
// backend error whose effect is unknown.
type NotAppliedError struct {
Cause error
}
func (e *NotAppliedError) Error() string {
if e == nil || e.Cause == nil {
return "operation was not applied"
}
return e.Cause.Error()
}
func (e *NotAppliedError) Unwrap() error {
if e == nil {
return nil
}
return e.Cause
}
func (*NotAppliedError) FailureSemantics() (bool, bool) { return false, true }
// MarkNotApplied must only be used before a caller-specific mutation boundary.
// An already-uncertain error is never downgraded, even if it is accidentally
// passed through a precondition classification helper.
func MarkNotApplied(err error) error {
if err == nil || hasUncertainMarker(err) {
return err
}
return &NotAppliedError{Cause: err}
}
func IsNotApplied(err error) bool {
var target *NotAppliedError
return errors.As(err, &target)
}
type recordAlias Record
type storedRecord struct {
Record recordAlias `json:"record"`
ExpectedState *ExpectedState `json:"expected_state,omitempty"`
LockKeys []string `json:"lock_keys,omitempty"`
HeavyIO bool `json:"heavy_io,omitempty"`
SupersedesOperationID string `json:"supersedes_operation_id,omitempty"`
}
type resourceLock struct {
semaphore chan struct{}
references int
}
type Journal struct {
path string
db *bolt.DB
clockMu sync.RWMutex
now func() time.Time
resourceMu sync.Mutex
resources map[string]*resourceLock
admissionClaimMu sync.Mutex
admissionClaims map[string]struct{}
executionSlots chan struct{}
heavyIOSlots chan struct{}
workerWG sync.WaitGroup
// backgroundFailures carries the first durable finalization failure to the
// process supervisor. ExecuteBackground has already crossed the dispatch
// boundary when these failures occur, so the mutation must never be
// replayed in-process. The main Agent exits cleanly and startup recovery
// imports an immutable helper receipt, or terminally records an unconfirmed
// result when no receipt exists.
backgroundFailures chan error
}
const (
maxConcurrentExecutions = 32
DefaultHeavyIOConcurrency = 4
maxAdmissionResourceKeys = 16
maximumAdmissionKeyLength = 255
)
func Open(path string) (*Journal, error) {
return OpenWithHeavyIOConcurrency(path, DefaultHeavyIOConcurrency)
}
func OpenWithHeavyIOConcurrency(path string, heavyIOConcurrency int) (*Journal, error) {
if strings.TrimSpace(path) == "" {
return nil, errors.New("journal path is required")
}
if heavyIOConcurrency < 1 {
return nil, errors.New("heavy I/O concurrency must be positive")
}
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
return nil, fmt.Errorf("create operation journal directory: %w", err)
}
db, err := bolt.Open(path, 0o600, &bolt.Options{Timeout: 2 * time.Second})
if err != nil {
return nil, fmt.Errorf("open operation journal database: %w", err)
}
j := &Journal{
path: path, db: db, now: time.Now, resources: make(map[string]*resourceLock),
admissionClaims: make(map[string]struct{}),
executionSlots: make(chan struct{}, maxConcurrentExecutions),
heavyIOSlots: make(chan struct{}, heavyIOConcurrency),
backgroundFailures: make(chan error, 1),
}
if err := j.initialize(); err != nil {
_ = db.Close()
return nil, err
}
return j, nil
}
func (j *Journal) Close() error {
if j == nil || j.db == nil {
return nil
}
return j.db.Close()
}
// Wait blocks until every real mutation and late-result finalizer has stopped.
// Callers must stop API admission before invoking it so no new worker can be
// added concurrently with the wait.
func (j *Journal) Wait(ctx context.Context) error {
done := make(chan struct{})
go func() {
j.workerWG.Wait()
close(done)
}()
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// BackgroundFailures reports an unrecoverable failure to durably finalize an
// already-dispatched background operation. Callers must stop request
// admission and restart the Agent; FinalizeInterruptedOperations then settles
// the dispatching record without invoking the mutation again.
func (j *Journal) BackgroundFailures() <-chan error {
return j.backgroundFailures
}
func (j *Journal) reportBackgroundFailure(operationID, phase string, err error) {
if err == nil {
return
}
failure := fmt.Errorf("%s background operation %s: %w", phase, operationID, err)
select {
case j.backgroundFailures <- failure:
default:
// The first failure is sufficient to force a supervised restart. Never
// block finalizers while the process is already shutting down.
}
}
// UseNonce uses two Bolt buckets: direct lookup is O(1), while the ordered
// expiry index makes cleanup proportional only to expired entries. It never
// rewrites or decodes the operation history.
func (j *Journal) UseNonce(nonce string, now, expiresAt time.Time) (bool, error) {
if !noncePattern.MatchString(nonce) {
return false, errors.New("invalid nonce")
}
if !expiresAt.After(now) {
return false, errors.New("nonce expiry must be in the future")
}
accepted := false
err := j.db.Update(func(tx *bolt.Tx) error {
nonces := tx.Bucket(noncesBucket)
expiry := tx.Bucket(nonceExpiryBucket)
if nonces == nil || expiry == nil {
return errors.New("nonce buckets are unavailable")
}
if err := pruneExpiredOperationDetails(tx, now); err != nil {
return err
}
pruneExpiredNonces(nonces, expiry, now)
key := []byte(nonce)
if encoded := nonces.Get(key); encoded != nil {
current, err := decodeExpiry(encoded)
if err != nil {
return err
}
if !current.Before(now) {
return nil
}
if err := expiry.Delete(expiryIndexKey(current, nonce)); err != nil {
return err
}
}
encoded := encodeExpiry(expiresAt.UTC())
if err := nonces.Put(key, encoded); err != nil {
return err
}
if err := expiry.Put(expiryIndexKey(expiresAt.UTC(), nonce), nil); err != nil {
return err
}
accepted = true
return nil
})
return accepted, err
}
func Fingerprint(method, target string, body []byte) string {
sum := sha256.Sum256([]byte(strings.ToUpper(method) + "\n" + target + "\n" + string(body)))
return hex.EncodeToString(sum[:])
}
func (j *Journal) SetClockForTesting(now func() time.Time) {
j.clockMu.Lock()
defer j.clockMu.Unlock()
j.now = now
}
func (j *Journal) currentTime() time.Time {
j.clockMu.RLock()
defer j.clockMu.RUnlock()
return j.now().UTC()
}
func (j *Journal) Get(operationID string) (Record, error) {
var record Record
err := j.db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(operationsBucket)
if bucket == nil {
return errors.New("operation bucket is unavailable")
}
value := bucket.Get([]byte(operationID))
if value == nil {
return ErrNotFound
}
decoded, err := decodeRecord(value)
if err != nil {
return err
}
record = decoded
return nil
})
return cloneRecord(record), err
}
// ReadTerminalRecord inspects an isolated worker Journal without initializing,
// migrating, or otherwise writing it. It is used only during parent startup,
// after a worker may have committed its terminal operation row but before it
// managed to publish completion.json.
func ReadTerminalRecord(path, operationID string) (Record, error) {
if strings.TrimSpace(path) == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path ||
!operationIDPattern.MatchString(operationID) {
return Record{}, ErrTerminalEvidenceInvalid
}
db, err := bolt.Open(path, 0o600, &bolt.Options{ReadOnly: true, Timeout: terminalEvidenceOpenTimeout})
if err != nil {
if invalidTerminalEvidenceDatabaseError(err) {
return Record{}, fmt.Errorf("%w: %v", ErrTerminalEvidenceInvalid, err)
}
return Record{}, err
}
defer db.Close()
var record Record
err = db.View(func(tx *bolt.Tx) error {
metadata := tx.Bucket(metadataBucket)
operations := tx.Bucket(operationsBucket)
if metadata == nil || operations == nil || !bytes.Equal(metadata.Get(schemaVersionKey), schemaVersion) {
return ErrTerminalEvidenceInvalid
}
encoded := operations.Get([]byte(operationID))
if encoded == nil {
return ErrTerminalEvidencePending
}
decoded, decodeErr := decodeRecord(encoded)
if decodeErr != nil {
return fmt.Errorf("%w: %v", ErrTerminalEvidenceInvalid, decodeErr)
}
if !terminalState(decoded.State) {
return ErrTerminalEvidencePending
}
record = decoded
return nil
})
if err != nil {
if invalidTerminalEvidenceDatabaseError(err) {
return Record{}, fmt.Errorf("%w: %v", ErrTerminalEvidenceInvalid, err)
}
return Record{}, err
}
return cloneRecord(record), nil
}
func invalidTerminalEvidenceDatabaseError(err error) bool {
return errors.Is(err, bolt.ErrInvalid) || errors.Is(err, bolt.ErrVersionMismatch) || errors.Is(err, bolt.ErrChecksum)
}
// GetForInspection returns the immutable Journal record and, when retention
// has removed its result (or the terminal result was empty), reconstructs only
// an in-memory copy from the operation's immutable completion receipt. Receipt
// anomalies are represented on the returned copy so inspection can stay
// inconclusive; this method never rewrites the Journal, receipt, or outbox.
func (j *Journal) GetForInspection(operationID string) (Record, error) {
var record Record
err := j.db.View(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
if operations == nil {
return errors.New("operation bucket is unavailable")
}
value := operations.Get([]byte(operationID))
if value == nil {
return ErrNotFound
}
decoded, err := decodeRecord(value)
if err != nil {
return err
}
record = decoded
if !terminalState(record.State) || (!record.Tombstone && len(record.Result) != 0) {
return nil
}
// Treat every receipt anomaly as incomplete evidence. The caller still
// receives the immutable operation identity and terminal status, but no
// receipt bytes are trusted or persisted back into the Journal.
record.InspectionEvidenceUnavailable = true
receipts := tx.Bucket(completionReceiptsBucket)
if receipts == nil {
return nil
}
eventID := completionEventID(operationID)
encoded := receipts.Get([]byte(eventID))
if encoded == nil {
return nil
}
var receipt CompletionReceipt
if err := json.Unmarshal(encoded, &receipt); err != nil ||
!completionReceiptMatchesRecordForInspection(eventID, operationID, receipt, record) {
return nil
}
record.Result = append(json.RawMessage(nil), receipt.Result...)
record.InspectionReceiptRecovered = true
record.InspectionEvidenceUnavailable = false
return nil
})
return cloneRecord(record), err
}
func completionReceiptMatchesRecordForInspection(eventID, requestedOperationID string,
receipt CompletionReceipt, record Record) bool {
if record.OperationID != requestedOperationID || eventID != completionEventID(requestedOperationID) ||
receipt.EventID != eventID || receipt.OperationID != requestedOperationID ||
!ValidOperationIdentity(record.Action, record.ResourceID, record.Fingerprint) ||
!ValidOperationIdentity(receipt.Action, receipt.ResourceID, receipt.RequestFingerprint) ||
receipt.Action != record.Action || receipt.ResourceID != record.ResourceID ||
receipt.RequestFingerprint != record.Fingerprint ||
receipt.EffectMayHaveApplied != record.EffectMayHaveApplied ||
receipt.SafeToResubmit != record.SafeToResubmit ||
!completionCauseMatchesRecord(receipt, record) ||
len(receipt.Result) == 0 || !json.Valid(receipt.Result) {
return false
}
recordErrorCode := ""
if record.Error != nil {
recordErrorCode = record.Error.Code
}
switch record.State {
case StateSucceeded:
return receipt.Status == "succeeded" && receipt.ErrorCode == "" &&
recordErrorCode == "" && !receipt.EffectMayHaveApplied && !receipt.SafeToResubmit
case StateFailed:
return receipt.Status == "failed" && receipt.ErrorCode != "" &&
validCauseCode(receipt.ErrorCode) && receipt.ErrorCode == recordErrorCode
default:
return false
}
}
func ValidateFenceResolutionRequest(request FenceResolutionRequest) error {
if !fenceResolutionIDPattern.MatchString(request.ResolutionID) {
return errors.New("resolution_id must be 32 lowercase hexadecimal characters")
}
if request.Resolution != FenceResolutionApplied && request.Resolution != FenceResolutionNotApplied {
return errors.New("resolution must be applied or not_applied")
}
if request.InspectionCheckedAt < 1 {
return errors.New("inspection_checked_at must be a positive Unix timestamp")
}
if !hexDigestPattern.MatchString(request.ReasonSHA256) {
return errors.New("reason_sha256 must be a 64-character lowercase SHA-256 digest")
}
if !ValidOperationIdentity(request.Action, request.ResourceID, request.RequestFingerprint) {
return errors.New("fence resolution operation identity is invalid")
}
return nil
}
// GetFenceResolution reads immutable, bounded audit metadata without touching
// operation state or host resources.
func (j *Journal) GetFenceResolution(operationID string) (FenceResolution, error) {
if !operationIDPattern.MatchString(operationID) {
return FenceResolution{}, ErrNotFound
}
var resolution FenceResolution
err := j.db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(fenceResolutionsBucket)
if bucket == nil {
return errors.New("fence resolution bucket is unavailable")
}
encoded := bucket.Get([]byte(operationID))
if encoded == nil {
return ErrNotFound
}
if err := json.Unmarshal(encoded, &resolution); err != nil || resolution.OperationID != operationID {
return fmt.Errorf("decode fence resolution %q", operationID)
}
return nil
})
return resolution, err
}
// FenceResolutionRequired reports whether the immutable operation currently
// owns every admission-index entry for an unresolved resource fence. A stored
// resolution, a tombstone, and non-blocking kinds such as configure-storage
// always return false. Inconsistent indexes fail closed instead of telling a
// caller to skip the fence protocol.
func (j *Journal) FenceState(operationID string) (FenceState, error) {
if !operationIDPattern.MatchString(operationID) {
return "", ErrNotFound
}
state := FenceStateNotRequired
err := j.db.View(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
blockers := tx.Bucket(resourceBlockersBucket)
resolutions := tx.Bucket(fenceResolutionsBucket)
if operations == nil || blockers == nil || resolutions == nil {
return errors.New("operation fence buckets are unavailable")
}
encoded := operations.Get([]byte(operationID))
if encoded == nil {
return ErrNotFound
}
record, err := decodeRecord(encoded)
if err != nil {
return err
}
if encodedResolution := resolutions.Get([]byte(operationID)); encodedResolution != nil {
var resolution FenceResolution
if err := json.Unmarshal(encodedResolution, &resolution); err != nil ||
resolution.OperationID != operationID {
return fmt.Errorf("decode fence resolution %q", operationID)
}
state = FenceStateResolved
return nil
}
if !blocksResources(record) {
return nil
}
for _, lockKey := range effectiveLockKeys(
record.OperationID,
record.ResourceID,
record.Action,
record.LockKeys,
) {
key := blockerEntryKey(lockKey, operationID)
foundKey, _ := blockers.Cursor().Seek(key)
if !bytes.Equal(foundKey, key) {
return fmt.Errorf(
"operation %q is missing its resource blocker index",
operationID,
)
}
}
state = FenceStateActive
return nil
})
return state, err
}
func (j *Journal) FenceResolutionRequired(operationID string) (bool, error) {
state, err := j.FenceState(operationID)
return state == FenceStateActive, err
}
// ResolveResourceFence records one explicit, audited effect conclusion and
// removes only this operation's admission-index entries. resolvedBy zero is
// reserved for the API's operation-bound system_recovery actor; human callers
// must retain their positive user ID. It never mutates the operation record or
// completion receipt and never invokes a host mutation.
func (j *Journal) ResolveResourceFence(operationID string, resolvedBy int64,
request FenceResolutionRequest) (FenceResolution, bool, error) {
if !operationIDPattern.MatchString(operationID) {
return FenceResolution{}, false, ErrNotFound
}
if resolvedBy < 0 {
return FenceResolution{}, false, errors.New("resolved_by must be a non-negative actor ID")
}
if err := ValidateFenceResolutionRequest(request); err != nil {
return FenceResolution{}, false, err
}
now := j.currentTime().UTC()
var resolution FenceResolution
replayed := false
err := j.db.Update(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
blockers := tx.Bucket(resourceBlockersBucket)
resolutions := tx.Bucket(fenceResolutionsBucket)
resolutionIDs := tx.Bucket(fenceResolutionIDsBucket)
if operations == nil || blockers == nil || resolutions == nil || resolutionIDs == nil {
return errors.New("operation fence buckets are unavailable")
}
encodedRecord := operations.Get([]byte(operationID))
if encodedRecord == nil {
checkedAt := time.Unix(request.InspectionCheckedAt, 0).UTC()
if checkedAt.Before(now.Add(-fenceInspectionMaximumAge)) || checkedAt.After(now.Add(fenceInspectionFutureSkew)) {
return ErrFenceInspectionStale
}
return ErrNotFound
}
record, err := decodeRecord(encodedRecord)
if err != nil {
return err
}
if !fenceResolutionIdentityMatches(request, record) {
return ErrFenceIdentityConflict
}
// The zero audit actor is a deliberately narrow internal capability.
// Only an action/resolution pair explicitly granted by the embedded
// contract may use it; human administrators retain their positive
// audited identity.
if resolvedBy == 0 && !SystemRecoveryResolutionAllowed(record.Action, request.Resolution) {
return ErrFenceNotResolvable
}
// An unreaped create worker could still create the resource after the
// inspection. Its fence therefore remains active even though the
// contract permits system not_applied for a safely reaped create.
if resolvedBy == 0 && IsCreateLikeAction(record.Action) && record.ExecutionFenced {
return ErrFenceNotResolvable
}
if encoded := resolutions.Get([]byte(operationID)); encoded != nil {
if err := json.Unmarshal(encoded, &resolution); err != nil || resolution.OperationID != operationID {
return fmt.Errorf("decode fence resolution %q", operationID)
}
if resolution.ResolutionID != request.ResolutionID || resolution.Resolution != request.Resolution ||
resolution.InspectionCheckedAt != request.InspectionCheckedAt || resolution.ReasonSHA256 != request.ReasonSHA256 ||
resolution.ResolvedBy != resolvedBy {
return ErrFenceResolutionConflict
}
replayed = true
return nil
}
if owner := resolutionIDs.Get([]byte(request.ResolutionID)); owner != nil {
return ErrFenceResolutionConflict
}
checkedAt := time.Unix(request.InspectionCheckedAt, 0).UTC()
if checkedAt.Before(now.Add(-fenceInspectionMaximumAge)) || checkedAt.After(now.Add(fenceInspectionFutureSkew)) {
return ErrFenceInspectionStale
}
if !blocksResources(record) {
return ErrFenceNotResolvable
}
lockKeys := effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys)
for _, lockKey := range lockKeys {
key := blockerEntryKey(lockKey, operationID)
cursor := blockers.Cursor()
foundKey, _ := cursor.Seek(key)
if !bytes.Equal(foundKey, key) {
return ErrFenceNotResolvable
}
}
resolution = FenceResolution{
OperationID: operationID, ResolutionID: request.ResolutionID,
Resolution: request.Resolution, InspectionCheckedAt: request.InspectionCheckedAt,
ReasonSHA256: request.ReasonSHA256, ResolvedAt: now.Unix(), ResolvedBy: resolvedBy,
}
resolutionEncoded, err := json.Marshal(resolution)
if err != nil {
return err
}
if err := resolutions.Put([]byte(operationID), resolutionEncoded); err != nil {
return err
}
if err := resolutionIDs.Put([]byte(request.ResolutionID), []byte(operationID)); err != nil {
return err
}
for _, lockKey := range lockKeys {
if err := blockers.Delete(blockerEntryKey(lockKey, operationID)); err != nil {
return err
}
}
return nil
})
return resolution, replayed, err
}
func fenceResolutionIdentityMatches(request FenceResolutionRequest, record Record) bool {
return request.Action == record.Action && request.ResourceID == record.ResourceID &&
request.RequestFingerprint == record.Fingerprint
}
// ClaimPendingCompletions returns a bounded batch and durably schedules every
// returned event's next delivery. A completion remains in the outbox until an
// ACK removes it; delivery age and attempt count never discard it.
func (j *Journal) ClaimPendingCompletions(limit int) ([]CompletionReceipt, error) {
claimed, _, err := j.ClaimPendingCompletionsWithExpired(limit)
return claimed, err
}
// ClaimPendingCompletionsWithExpired retains its historical return shape for
// callers compiled against the bounded-outbox API. Completion delivery no
// longer expires, so the second result is always empty.
func (j *Journal) ClaimPendingCompletionsWithExpired(limit int) ([]CompletionReceipt, []string, error) {
return j.ClaimPendingCompletionsWithExpiredExcluding(limit, nil)
}
// ClaimPendingCompletionsWithExpiredExcluding leaves events already in the
// current control session's ACK window untouched. An in-flight receipt cannot
// move its durable resend time merely because that time arrived while the
// Gateway was still processing the first delivery. The historical "expired"
// result is retained for API compatibility and is always empty.
func (j *Journal) ClaimPendingCompletionsWithExpiredExcluding(limit int, excluded []string) ([]CompletionReceipt, []string, error) {
if limit < 1 || limit > 128 {
return nil, nil, errors.New("completion claim limit must be between 1 and 128")
}
excludedEvents := make(map[string]struct{}, len(excluded))
for _, eventID := range excluded {
if !validCompletionEventID(eventID) {
return nil, nil, errors.New("invalid excluded completion event ID")
}
excludedEvents[eventID] = struct{}{}
}
claimed := make([]CompletionReceipt, 0, limit)
newIncidents := make([]CompletionDeliveryIncident, 0, 1)
err := j.db.Update(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
receipts := tx.Bucket(completionReceiptsBucket)
outbox := tx.Bucket(completionOutboxBucket)
quarantine := tx.Bucket(completionQuarantineBucket)
quarantineOperations := tx.Bucket(completionQuarantineOperationsBucket)
if operations == nil || receipts == nil || outbox == nil || quarantine == nil || quarantineOperations == nil {
return errors.New("completion buckets are unavailable")
}
now := j.currentTime()
cursor := outbox.Cursor()
for key, encoded := cursor.First(); key != nil && len(claimed) < limit; key, encoded = cursor.Next() {
receiptEncoded := receipts.Get(key)
operationID := completionIncidentOperationID(key, encoded, receiptEncoded)
var pending completionOutboxRecord
// Older records may contain attempts and expires_at fields. JSON
// decoding deliberately ignores them so an upgrade resumes delivery
// even when the legacy deadline or attempt cap was already reached.
if err := json.Unmarshal(encoded, &pending); err != nil ||
!validCompletionEventID(string(key)) ||
!operationIDPattern.MatchString(pending.OperationID) ||
string(key) != completionEventID(pending.OperationID) || pending.NextAttemptAt.IsZero() {
incident, changed, incidentErr := quarantineCompletionDelivery(
quarantine, quarantineOperations, key, operationID,
CompletionDeliveryErrorOutboxInvalid, now, encoded, receiptEncoded,
)
if incidentErr != nil {
return incidentErr
}
if changed {
newIncidents = append(newIncidents, incident)
}
continue
}
operationID = pending.OperationID
if _, excluded := excludedEvents[string(key)]; excluded {
continue
}
if now.Before(pending.NextAttemptAt) {
continue
}
if receiptEncoded == nil {
incident, changed, incidentErr := quarantineCompletionDelivery(
quarantine, quarantineOperations, key, operationID,
CompletionDeliveryErrorReceiptMissing, now, encoded,
)
if incidentErr != nil {
return incidentErr
}
if changed {
newIncidents = append(newIncidents, incident)
}
continue
}
var receipt CompletionReceipt
if err := json.Unmarshal(receiptEncoded, &receipt); err != nil || receipt.EventID != string(key) || receipt.OperationID != pending.OperationID {
incident, changed, incidentErr := quarantineCompletionDelivery(
quarantine, quarantineOperations, key, operationID,
CompletionDeliveryErrorReceiptInvalid, now, encoded, receiptEncoded,
)
if incidentErr != nil {
return incidentErr
}
if changed {
newIncidents = append(newIncidents, incident)
}
continue
}
recordEncoded := operations.Get([]byte(pending.OperationID))
if recordEncoded == nil {
incident, changed, incidentErr := quarantineCompletionDelivery(
quarantine, quarantineOperations, key, operationID,
CompletionDeliveryErrorOperationMissing, now, encoded, receiptEncoded,
)
if incidentErr != nil {
return incidentErr
}
if changed {
newIncidents = append(newIncidents, incident)
}
continue
}
record, err := decodeRecord(recordEncoded)
if err != nil {
incident, changed, incidentErr := quarantineCompletionDelivery(
quarantine, quarantineOperations, key, operationID,
CompletionDeliveryErrorOperationInvalid, now, encoded, receiptEncoded, recordEncoded,
)
if incidentErr != nil {
return incidentErr
}
if changed {
newIncidents = append(newIncidents, incident)
}
continue
}
if err := bindCompletionReceiptIdentity(&receipt, record); err != nil {
incident, changed, incidentErr := quarantineCompletionDelivery(
quarantine, quarantineOperations, key, operationID,
CompletionDeliveryErrorIdentityMismatch, now, encoded, receiptEncoded, recordEncoded,
)
if incidentErr != nil {
return incidentErr
}
if changed {
newIncidents = append(newIncidents, incident)
}
continue
}
if !completionReceiptMatchesRecord(receipt, record) {
incident, changed, incidentErr := quarantineCompletionDelivery(
quarantine, quarantineOperations, key, operationID,
CompletionDeliveryErrorReceiptInvalid, now, encoded, receiptEncoded, recordEncoded,
)
if incidentErr != nil {
return incidentErr
}
if changed {
newIncidents = append(newIncidents, incident)
}
continue
}
if err := clearCompletionDeliveryIncident(
quarantine, quarantineOperations, key,
); err != nil {
return err
}
pending.NextAttemptAt = now.Add(completionDeliveryInterval)
updated, err := json.Marshal(pending)
if err != nil {
return err
}
if err := outbox.Put(key, updated); err != nil {
return err
}
receipt.Result = append(json.RawMessage(nil), receipt.Result...)
claimed = append(claimed, receipt)
}
return nil
})
if err == nil {
for _, incident := range newIncidents {
log.Printf(
"completion_delivery phase=quarantined operation_id=%s event_id=%s event_key_sha256=%s error_code=%s",
incident.OperationID, incident.EventID, incident.EventKeySHA256, incident.ErrorCode,
)
}
}
return claimed, []string{}, err
}
func bindCompletionReceiptIdentity(receipt *CompletionReceipt, record Record) error {
if receipt == nil || !ValidOperationIdentity(record.Action, record.ResourceID, record.Fingerprint) {
return errors.New("Journal operation identity is invalid")
}
if receipt.Action == "" && receipt.ResourceID == "" && receipt.RequestFingerprint == "" {
// Compatibility for immutable receipts created before operation identity
// was added to the wire contract. The returned copy is enriched from the
// same immutable Journal record; the stored receipt bytes are untouched.
receipt.Action = record.Action
receipt.ResourceID = record.ResourceID
receipt.RequestFingerprint = record.Fingerprint
return nil
}
if !ValidOperationIdentity(receipt.Action, receipt.ResourceID, receipt.RequestFingerprint) ||
receipt.Action != record.Action || receipt.ResourceID != record.ResourceID ||
receipt.RequestFingerprint != record.Fingerprint {
return errors.New("completion identity does not match Journal operation")
}
return nil
}
func completionIncidentOperationID(eventKey, outboxEncoded, receiptEncoded []byte) string {
// Prefer an internally consistent immutable receipt over the mutable retry
// marker. This still recovers the affected operation when only the outbox
// bytes or key were damaged, without letting an arbitrary operation_id in a
// malformed receipt surface on the administrator inspection path.
var receiptIdentity struct {
EventID string `json:"id"`
OperationID string `json:"operation_id"`
}
receiptIdentityValid := json.Unmarshal(receiptEncoded, &receiptIdentity) == nil &&
operationIDPattern.MatchString(receiptIdentity.OperationID) &&
receiptIdentity.EventID == completionEventID(receiptIdentity.OperationID)
var pendingIdentity struct {
OperationID string `json:"operation_id"`
}
pendingIdentityValid := json.Unmarshal(outboxEncoded, &pendingIdentity) == nil &&
operationIDPattern.MatchString(pendingIdentity.OperationID)
if receiptIdentityValid && receiptIdentity.EventID == string(eventKey) {
return receiptIdentity.OperationID
}
if pendingIdentityValid && completionEventID(pendingIdentity.OperationID) == string(eventKey) {
return pendingIdentity.OperationID
}
if receiptIdentityValid {
return receiptIdentity.OperationID
}
if pendingIdentityValid {
return pendingIdentity.OperationID
}
// A valid event key alone is not reversible to an operation ID. It remains
// part of the durable evidence digest for an unassociated Agent incident.
return ""
}
func validCompletionDeliveryErrorCode(code string) bool {
switch code {
case CompletionDeliveryErrorOutboxInvalid,
CompletionDeliveryErrorReceiptMissing,
CompletionDeliveryErrorReceiptInvalid,
CompletionDeliveryErrorOperationMissing,
CompletionDeliveryErrorOperationInvalid,
CompletionDeliveryErrorIdentityMismatch:
return true
default:
return false
}
}
func completionDeliveryDigest(eventKey []byte, errorCode string, evidence ...[]byte) string {
hash := sha256.New()
writePart := func(value []byte) {
var size [8]byte
binary.BigEndian.PutUint64(size[:], uint64(len(value)))
_, _ = hash.Write(size[:])
_, _ = hash.Write(value)
}
writePart([]byte("completion-delivery-incident-v1"))
writePart(eventKey)
writePart([]byte(errorCode))
for _, value := range evidence {
writePart(value)
}
return hex.EncodeToString(hash.Sum(nil))
}
func quarantineCompletionDelivery(quarantine, operationIndex *bolt.Bucket, eventKey []byte,
operationID, errorCode string, observedAt time.Time, evidence ...[]byte) (CompletionDeliveryIncident, bool, error) {
if quarantine == nil || operationIndex == nil {
return CompletionDeliveryIncident{}, false, errors.New("completion quarantine buckets are unavailable")
}
if !validCompletionDeliveryErrorCode(errorCode) {
return CompletionDeliveryIncident{}, false, errors.New("invalid completion delivery incident code")
}
if !operationIDPattern.MatchString(operationID) {
operationID = ""
}
observedAt = observedAt.UTC()
if observedAt.IsZero() {
return CompletionDeliveryIncident{}, false, errors.New("completion delivery incident time is invalid")
}
eventID := ""
if validCompletionEventID(string(eventKey)) {
eventID = string(eventKey)
}
eventKeySum := sha256.Sum256(eventKey)
incident := CompletionDeliveryIncident{
EventID: eventID,
EventKeySHA256: hex.EncodeToString(eventKeySum[:]),
OperationID: operationID,
ErrorCode: errorCode,
EvidenceSHA256: completionDeliveryDigest(eventKey, errorCode, evidence...),
FirstObservedAt: observedAt,
}
changed := true
if encoded := quarantine.Get(eventKey); encoded != nil {
var existing CompletionDeliveryIncident
if json.Unmarshal(encoded, &existing) == nil &&
existing.EventID == incident.EventID &&
existing.EventKeySHA256 == incident.EventKeySHA256 &&
existing.OperationID == incident.OperationID &&
existing.ErrorCode == incident.ErrorCode &&
existing.EvidenceSHA256 == incident.EvidenceSHA256 &&
!existing.FirstObservedAt.IsZero() {
incident.FirstObservedAt = existing.FirstObservedAt.UTC()
changed = false
}
}
// Remove every stale reverse-index entry for this raw event key. This also
// repairs an index left behind by corrupt incident JSON or by a changed
// best-effort operation association.
indexCursor := operationIndex.Cursor()
for key, value := indexCursor.First(); key != nil; key, value = indexCursor.Next() {
if bytes.Equal(value, eventKey) && (operationID == "" || string(key) != operationID) {
if err := indexCursor.Delete(); err != nil {
return CompletionDeliveryIncident{}, false, err
}
}
}
encoded, err := json.Marshal(incident)
if err != nil {
return CompletionDeliveryIncident{}, false, err
}
if err := quarantine.Put(eventKey, encoded); err != nil {
return CompletionDeliveryIncident{}, false, err
}
if operationID != "" {
indexed := operationIndex.Get([]byte(operationID))
if !bytes.Equal(indexed, eventKey) {
changed = true
}
if err := operationIndex.Put([]byte(operationID), eventKey); err != nil {
return CompletionDeliveryIncident{}, false, err
}
}
return incident, changed, nil
}
func clearCompletionDeliveryIncident(quarantine, operationIndex *bolt.Bucket, eventKey []byte) error {
if quarantine == nil || operationIndex == nil {
return errors.New("completion quarantine buckets are unavailable")
}
if err := quarantine.Delete(eventKey); err != nil {
return err
}
cursor := operationIndex.Cursor()
for key, value := cursor.First(); key != nil; key, value = cursor.Next() {
if bytes.Equal(value, eventKey) {
if err := cursor.Delete(); err != nil {
return err
}
}
}
return nil
}
// CompletionDeliveryIncident returns a persistent delivery-only incident for
// administrator inspection. It never changes the immutable operation status,
// receipt, or outbox entry.
func (j *Journal) CompletionDeliveryIncident(operationID string) (CompletionDeliveryIncident, bool, error) {
if !operationIDPattern.MatchString(operationID) {
return CompletionDeliveryIncident{}, false, ErrNotFound
}
var incident CompletionDeliveryIncident
found := false
err := j.db.View(func(tx *bolt.Tx) error {
quarantine := tx.Bucket(completionQuarantineBucket)
operationIndex := tx.Bucket(completionQuarantineOperationsBucket)
if quarantine == nil || operationIndex == nil {
return errors.New("completion quarantine buckets are unavailable")
}
eventKey := operationIndex.Get([]byte(operationID))
derivedAssociation := false
if eventKey == nil {
// The event ID is a deterministic digest of the operation ID. If both
// the outbox and immutable receipt JSON are damaged, quarantine cannot
// recover an operation ID to populate the reverse index. The exact
// deterministic key still safely associates that detected corruption
// with this requested operation without scanning either bucket.
eventKey = []byte(completionEventID(operationID))
if quarantine.Get(eventKey) == nil {
return nil
}
derivedAssociation = true
}
encoded := quarantine.Get(eventKey)
if encoded == nil {
return errors.New("completion quarantine operation index is inconsistent")
}
if err := json.Unmarshal(encoded, &incident); err != nil {
return fmt.Errorf("decode completion delivery incident: %w", err)
}
eventKeySum := sha256.Sum256(eventKey)
operationMatches := incident.OperationID == operationID ||
(derivedAssociation && incident.OperationID == "" &&
incident.EventID == string(eventKey))
if !operationMatches ||
!validCompletionDeliveryErrorCode(incident.ErrorCode) ||
!hexDigestPattern.MatchString(incident.EventKeySHA256) ||
incident.EventKeySHA256 != hex.EncodeToString(eventKeySum[:]) ||
!hexDigestPattern.MatchString(incident.EvidenceSHA256) ||
incident.FirstObservedAt.IsZero() ||
(incident.EventID != "" && (incident.EventID != string(eventKey) || !validCompletionEventID(incident.EventID))) {
return errors.New("completion delivery incident is invalid")
}
if derivedAssociation {
// Bind only the returned copy. Repairing the receipt/outbox must still
// clear the original quarantine entry through its immutable event key;
// inspection must never rewrite corruption evidence.
incident.OperationID = operationID
}
found = true
return nil
})
return incident, found, err
}
func completionReceiptMatchesRecord(receipt CompletionReceipt, record Record) bool {
if receipt.EventID != completionEventID(record.OperationID) ||
receipt.OperationID != record.OperationID ||
!ValidOperationIdentity(receipt.Action, receipt.ResourceID, receipt.RequestFingerprint) ||
receipt.Action != record.Action || receipt.ResourceID != record.ResourceID ||
receipt.RequestFingerprint != record.Fingerprint ||
receipt.EffectMayHaveApplied != record.EffectMayHaveApplied ||
receipt.SafeToResubmit != record.SafeToResubmit ||
!completionCauseMatchesRecord(receipt, record) ||
!completionResultSafe(receipt.Result) || !completionDeliveryFitsPanelLimit(receipt) {
return false
}
recordErrorCode := ""
if record.Error != nil {
recordErrorCode = record.Error.Code
}
switch record.State {
case StateSucceeded:
if receipt.Status != "succeeded" || receipt.ErrorCode != "" || recordErrorCode != "" ||
receipt.EffectMayHaveApplied || receipt.SafeToResubmit {
return false
}
case StateFailed:
if receipt.Status != "failed" || receipt.ErrorCode == "" ||
!validCauseCode(receipt.ErrorCode) || receipt.ErrorCode != recordErrorCode ||
(receipt.EffectMayHaveApplied && receipt.SafeToResubmit) {
return false
}
default:
return false
}
// Retention intentionally removes the Journal result while preserving the
// immutable receipt. All other records must still carry the exact JSON value
// that was copied into that receipt at terminalization.
if record.Tombstone {
return true
}
return completionResultsEqual(receipt.Result, record.Result)
}
func completionCauseMatchesRecord(receipt CompletionReceipt, record Record) bool {
switch receipt.Version {
case 0:
// Legacy v1 receipts predate cause_code. They remain deliverable after
// an Agent upgrade, but cannot recover the diagnostic that was omitted.
return receipt.CauseCode == ""
case 2:
return (receipt.CauseCode == "" || validCauseCode(receipt.CauseCode)) &&
receipt.CauseCode == record.CauseCode
default:
return false
}
}
func completionDeliveryFitsPanelLimit(receipt CompletionReceipt) bool {
// The Gateway adds agent_id before POSTing to PHP. Marshal the exact public
// envelope with the longest valid agent ID so every claimed receipt is
// guaranteed to fit the panel's authenticated 1 MiB request limit.
encoded, err := json.Marshal(struct {
Type string `json:"type"`
ID string `json:"id"`
AgentID string `json:"agent_id"`
OperationID string `json:"operation_id"`
Action string `json:"action"`
ResourceID string `json:"resource_id"`
RequestFingerprint string `json:"request_fingerprint"`
Status string `json:"status"`
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
SafeToResubmit bool `json:"safe_to_resubmit"`
ErrorCode string `json:"error_code"`
CauseCode string `json:"cause_code,omitempty"`
Result json.RawMessage `json:"result"`
}{
Type: "completion", ID: receipt.EventID, AgentID: "9999999999999999999",
OperationID: receipt.OperationID, Action: receipt.Action, ResourceID: receipt.ResourceID,
RequestFingerprint: receipt.RequestFingerprint, Status: receipt.Status,
EffectMayHaveApplied: receipt.EffectMayHaveApplied, SafeToResubmit: receipt.SafeToResubmit,
ErrorCode: receipt.ErrorCode, CauseCode: receipt.CauseCode, Result: receipt.Result,
})
return err == nil && len(encoded) <= completionPanelMaximumBody
}
func completionResultsEqual(receipt, record json.RawMessage) bool {
if len(record) == 0 {
return bytes.Equal(bytes.TrimSpace(receipt), []byte("null"))
}
if !json.Valid(receipt) || !json.Valid(record) {
return false
}
var compactReceipt bytes.Buffer
var compactRecord bytes.Buffer
if json.Compact(&compactReceipt, receipt) != nil || json.Compact(&compactRecord, record) != nil {
return false
}
return bytes.Equal(compactReceipt.Bytes(), compactRecord.Bytes())
}
func completionResultSafe(raw json.RawMessage) bool {
if len(raw) == 0 || !json.Valid(raw) {
return false
}
var value any
if err := json.Unmarshal(raw, &value); err != nil {
return false
}
if value == nil {
return true
}
switch value.(type) {
case map[string]any, []any:
return completionResultValueSafe(value, 0)
default:
return false
}
}
func completionResultValueSafe(value any, depth int) bool {
if depth > 16 {
return false
}
switch typed := value.(type) {
case map[string]any:
for key, nested := range typed {
if completionSensitiveKeyPattern.MatchString(key) {
return false
}
if _, ok := nested.(map[string]any); ok && !completionResultValueSafe(nested, depth+1) {
return false
}
if _, ok := nested.([]any); ok && !completionResultValueSafe(nested, depth+1) {
return false
}
}
case []any:
for _, nested := range typed {
if _, ok := nested.(map[string]any); ok && !completionResultValueSafe(nested, depth+1) {
return false
}
if _, ok := nested.([]any); ok && !completionResultValueSafe(nested, depth+1) {
return false
}
}
default:
return false
}
return true
}
// CompletionOperationID resolves a still-pending delivery marker without
// changing it. It lets the Agent clean disposable worker state independently
// before durably consuming an ACK; the lookup never changes the receipt or
// its delivery marker.
func (j *Journal) CompletionOperationID(eventID string) (string, error) {
if !validCompletionEventID(eventID) {
return "", errors.New("invalid completion event ID")
}
var operationID string
err := j.db.View(func(tx *bolt.Tx) error {
outbox := tx.Bucket(completionOutboxBucket)
if outbox == nil {
return errors.New("completion outbox is unavailable")
}
encoded := outbox.Get([]byte(eventID))
if encoded == nil {
return nil
}
var pending completionOutboxRecord
if err := json.Unmarshal(encoded, &pending); err != nil || !operationIDPattern.MatchString(pending.OperationID) {
return fmt.Errorf("decode completion outbox event %q", eventID)
}
operationID = pending.OperationID
return nil
})
return operationID, err
}
// PendingCompletionOperationIDs returns operation IDs whose terminal receipt
// still awaits an ACK. Disposable worker directories are no longer coupled to
// this set once their result has been imported into the main Journal.
func (j *Journal) PendingCompletionOperationIDs() (map[string]struct{}, error) {
operationIDs := make(map[string]struct{})
err := j.db.View(func(tx *bolt.Tx) error {
outbox := tx.Bucket(completionOutboxBucket)
if outbox == nil {
return errors.New("completion outbox is unavailable")
}
return outbox.ForEach(func(key, encoded []byte) error {
var pending completionOutboxRecord
if err := json.Unmarshal(encoded, &pending); err != nil || !operationIDPattern.MatchString(pending.OperationID) {
return fmt.Errorf("decode completion outbox event %q", key)
}
operationIDs[pending.OperationID] = struct{}{}
return nil
})
})
return operationIDs, err
}
// AcknowledgeCompletion only removes the delivery marker. It deliberately
// cannot update the operation or immutable completion receipt.
func (j *Journal) AcknowledgeCompletion(eventID string) error {
if !validCompletionEventID(eventID) {
return errors.New("invalid completion event ID")
}
return j.db.Update(func(tx *bolt.Tx) error {
outbox := tx.Bucket(completionOutboxBucket)
if outbox == nil {
return errors.New("completion outbox is unavailable")
}
return outbox.Delete([]byte(eventID))
})
}
// FinalizeInterruptedOperations is called exactly once during Agent startup,
// before request admission. It imports already-written worker receipts and
// terminally fails every other dispatching row without spawning a worker.
func (j *Journal) FinalizeInterruptedOperations(workerReceipts []InterruptedReceipt) ([]string, error) {
byOperation := make(map[string]InterruptedReceipt, len(workerReceipts))
for _, receipt := range workerReceipts {
if !operationIDPattern.MatchString(receipt.OperationID) || !hexDigestPattern.MatchString(receipt.Fingerprint) ||
(receipt.Status != "succeeded" && receipt.Status != "failed") ||
(receipt.SafeToResubmit && receipt.EffectMayHaveApplied) ||
(len(receipt.Result) != 0 && !json.Valid(receipt.Result)) {
return nil, errors.New("invalid interrupted worker receipt")
}
if receipt.Status == "failed" && (receipt.ErrorCode == "" || !validCauseCode(receipt.ErrorCode)) {
return nil, errors.New("invalid interrupted worker failure")
}
if _, duplicate := byOperation[receipt.OperationID]; duplicate {
return nil, errors.New("duplicate interrupted worker receipt")
}
byOperation[receipt.OperationID] = receipt
}
finalized := make([]string, 0)
err := j.db.Update(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
inflight := tx.Bucket(inflightBucket)
blockers := tx.Bucket(resourceBlockersBucket)
expiry := tx.Bucket(operationExpiryBucket)
receipts := tx.Bucket(completionReceiptsBucket)
outbox := tx.Bucket(completionOutboxBucket)
if operations == nil || inflight == nil || blockers == nil || expiry == nil || receipts == nil || outbox == nil {
return errors.New("operation buckets are unavailable")
}
now := j.currentTime()
var keys [][]byte
if err := inflight.ForEach(func(key, _ []byte) error {
keys = append(keys, append([]byte(nil), key...))
return nil
}); err != nil {
return err
}
for _, key := range keys {
encoded := operations.Get(key)
if encoded == nil {
return fmt.Errorf("inflight index references missing operation %q", key)
}
record, err := decodeRecord(encoded)
if err != nil {
return err
}
if record.State != StateDispatching {
continue
}
previous := cloneRecord(record)
receipt, found := byOperation[record.OperationID]
if found && receipt.Fingerprint == record.Fingerprint {
record.Result = append(json.RawMessage(nil), receipt.Result...)
if receipt.Status == "succeeded" {
record.State = StateSucceeded
record.Error = nil
record.CauseCode = ""
record.EffectMayHaveApplied = false
record.SafeToResubmit = false
} else {
record.State = StateFailed
record.Error = &PublicError{Code: receipt.ErrorCode, Message: receipt.ErrorMessage}
record.CauseCode = receipt.ErrorCode
record.EffectMayHaveApplied = receipt.EffectMayHaveApplied
record.SafeToResubmit = receipt.SafeToResubmit
}
record.ExecutionFenced = false
} else {
record.State = StateFailed
record.Result = nil
record.Error = &PublicError{Code: "result_unconfirmed", Message: "agent restarted after dispatch; the operation was not replayed"}
record.CauseCode = ""
record.EffectMayHaveApplied = true
record.SafeToResubmit = false
record.ExecutionFenced = true
}
record.UpdatedAt = now
if err := putRecord(operations, record); err != nil {
return err
}
if err := syncCompletionReceipt(receipts, outbox, previous, record); err != nil {
return err
}
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
return err
}
if err := inflight.Delete(key); err != nil {
return err
}
if err := expiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
return err
}
finalized = append(finalized, record.OperationID)
}
return nil
})
return finalized, err
}
func completionEventID(operationID string) string {
digest := sha256.Sum256([]byte("operation-completion\n" + operationID))
return base64.RawURLEncoding.EncodeToString(digest[:])
}
func validCompletionEventID(value string) bool {
decoded, err := base64.RawURLEncoding.DecodeString(value)
return err == nil && len(decoded) == sha256.Size && base64.RawURLEncoding.EncodeToString(decoded) == value
}
func syncCompletionReceipt(receipts, outbox *bolt.Bucket, previous, current Record) error {
if receipts == nil || outbox == nil {
return errors.New("completion buckets are unavailable")
}
if !terminalState(current.State) || terminalState(previous.State) {
return nil
}
if !ValidOperationIdentity(current.Action, current.ResourceID, current.Fingerprint) {
return errors.New("terminal operation identity is invalid")
}
eventID := completionEventID(current.OperationID)
key := []byte(eventID)
if receipts.Get(key) != nil {
return nil
}
status := "failed"
errorCode := ""
if current.State == StateSucceeded {
status = "succeeded"
} else if current.Error != nil {
errorCode = current.Error.Code
}
receipt := CompletionReceipt{
Version: 2,
EventID: eventID, OperationID: current.OperationID,
Action: current.Action, ResourceID: current.ResourceID, RequestFingerprint: current.Fingerprint,
Status: status,
Result: append(json.RawMessage(nil), current.Result...), ErrorCode: errorCode,
CauseCode: current.CauseCode,
EffectMayHaveApplied: current.EffectMayHaveApplied,
SafeToResubmit: current.SafeToResubmit, CreatedAt: current.UpdatedAt.UTC(),
}
encoded, err := json.Marshal(receipt)
if err != nil {
return err
}
if err := receipts.Put(key, encoded); err != nil {
return err
}
pending, err := json.Marshal(completionOutboxRecord{
OperationID: current.OperationID, NextAttemptAt: current.UpdatedAt.UTC(),
})
if err != nil {
return err
}
return outbox.Put(key, pending)
}
// Reserve durably binds operation_id to one exact request fingerprint and its
// non-secret execution descriptor. Records are never deleted, so an old
// destructive operation_id can never become valid again after housekeeping.
func (j *Journal) Reserve(spec Spec) (Record, bool, error) {
if err := validateSpec(spec); err != nil {
return Record{}, false, err
}
lockKeys := effectiveLockKeys(spec.OperationID, spec.ResourceID, spec.Action, spec.LockKeys)
var record Record
existed := false
err := j.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(operationsBucket)
inflight := tx.Bucket(inflightBucket)
blockers := tx.Bucket(resourceBlockersBucket)
if bucket == nil || inflight == nil || blockers == nil {
return errors.New("operation buckets are unavailable")
}
if err := pruneExpiredOperationDetails(tx, j.currentTime()); err != nil {
return err
}
if value := bucket.Get([]byte(spec.OperationID)); value != nil {
existing, err := decodeRecord(value)
if err != nil {
return err
}
existed = true
if existing.Fingerprint != spec.Fingerprint || existing.Action != spec.Action || existing.ResourceID != spec.ResourceID ||
existing.SupersedesOperationID != spec.SupersedesOperationID {
return ErrFingerprint
}
record = existing
return nil
}
now := j.currentTime()
record = Record{
OperationID: spec.OperationID, ResourceID: spec.ResourceID,
Action: spec.Action, Fingerprint: spec.Fingerprint,
Destructive: spec.Destructive, State: StateReserved,
CreatedAt: now, UpdatedAt: now, ExpectedState: cloneExpectedState(spec.ExpectedState),
LockKeys: append([]string(nil), lockKeys...), HeavyIO: spec.HeavyIO,
SupersedesOperationID: spec.SupersedesOperationID,
}
if err := putRecord(bucket, record); err != nil {
return err
}
if err := inflight.Put([]byte(record.OperationID), nil); err != nil {
return err
}
return nil
})
return cloneRecord(record), existed, err
}
// WasNotDispatched reports the terminal 0-call outcome. Clean-break Agents do
// not reopen this record; a new request must use a new operation identifier.
func WasNotDispatched(record Record) bool {
return !record.Tombstone && record.State == StateFailed && record.Error != nil &&
record.Error.Code == "operation_not_dispatched"
}
// Execute persists the dispatch boundary before invoking fn. Once that
// boundary is crossed the function is invoked at most once. A deadline
// waits for the supervised worker to return before releasing admission. If the
// worker cannot be reaped within the secondary hard limit, the resource is
// durably fenced and any late result is ignored.
func (j *Journal) Execute(ctx context.Context, spec Spec, fn func(context.Context) (json.RawMessage, error)) (Record, bool, error) {
record, existed, err := j.Reserve(spec)
if err != nil {
return Record{}, existed, err
}
if existed && record.State != StateReserved {
return record, true, nil
}
releaseClaim, claimed := j.claimAdmission(spec.OperationID)
if !claimed {
return j.admissionReplay(spec.OperationID)
}
defer releaseClaim()
record, err = j.prepareAdmission(spec)
if err != nil {
return record, existed, err
}
if record.State != StateReserved {
return record, true, nil
}
releaseAdmission, err := j.acquireAdmission(ctx, record)
if err != nil {
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
releaseClaim()
if failErr != nil {
return failed, existed, failErr
}
return failed, existed, err
}
if err := ctx.Err(); err != nil {
releaseAdmission()
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
releaseClaim()
if failErr != nil {
return failed, existed, failErr
}
return failed, existed, err
}
if err := j.verifyAdmissionUnblocked(spec); err != nil {
releaseAdmission()
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
releaseClaim()
if failErr != nil {
return failed, existed, failErr
}
return failed, existed, err
}
record, err = j.markDispatching(spec.OperationID)
releaseClaim()
if err != nil {
releaseAdmission()
return record, existed, err
}
if record.State != StateDispatching {
releaseAdmission()
return record, true, nil
}
type outcome struct {
result json.RawMessage
err error
}
done := make(chan outcome, 1)
mutationContext := context.WithValue(ctx, expectedStateContextKey{}, cloneExpectedState(record.ExpectedState))
go func() {
result, callErr := fn(mutationContext)
done <- outcome{result: result, err: callErr}
}()
select {
case completed := <-done:
terminal, persistErr := j.finish(spec.OperationID, completed.result, completed.err)
releaseAdmission()
if persistErr != nil {
return terminal, existed, persistErr
}
return terminal, existed, completed.err
case <-ctx.Done():
reapTimer := time.NewTimer(workerReapDeadline)
defer reapTimer.Stop()
select {
case completed := <-done:
terminal, persistErr := j.finish(spec.OperationID, completed.result, completed.err)
releaseAdmission()
if persistErr != nil {
return terminal, existed, persistErr
}
return terminal, existed, completed.err
case <-reapTimer.C:
terminal, persistErr := j.markExecutionFenced(spec.OperationID)
releaseAdmission()
if persistErr != nil {
return terminal, existed, persistErr
}
return terminal, existed, ErrExecutionNotReaped
}
}
}
// ExecuteBackground persists the dispatch boundary and starts one bounded,
// tracked worker before returning. A nil done channel means an existing
// non-reserved record was replayed and no new call was started.
func (j *Journal) ExecuteBackground(
dispatchCtx, workerParent context.Context,
workerTimeout time.Duration,
spec Spec,
fn func(context.Context) (json.RawMessage, error),
) (Record, bool, <-chan struct{}, error) {
if workerParent == nil || workerTimeout <= 0 {
return Record{}, false, nil, errors.New("background worker deadline is invalid")
}
record, existed, err := j.Reserve(spec)
if err != nil {
return Record{}, existed, nil, err
}
if existed && record.State != StateReserved {
return record, true, nil, nil
}
releaseClaim, claimed := j.claimAdmission(spec.OperationID)
if !claimed {
replayed, _, replayErr := j.admissionReplay(spec.OperationID)
return replayed, true, nil, replayErr
}
defer releaseClaim()
record, err = j.prepareAdmission(spec)
if err != nil {
return record, existed, nil, err
}
if record.State != StateReserved {
return record, true, nil, nil
}
releaseAdmission, err := j.acquireAdmission(dispatchCtx, record)
if err != nil {
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
releaseClaim()
if failErr != nil {
return failed, existed, nil, failErr
}
return failed, existed, nil, err
}
if err := dispatchCtx.Err(); err != nil {
releaseAdmission()
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
releaseClaim()
if failErr != nil {
return failed, existed, nil, failErr
}
return failed, existed, nil, err
}
if err := j.verifyAdmissionUnblocked(spec); err != nil {
releaseAdmission()
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
releaseClaim()
if failErr != nil {
return failed, existed, nil, failErr
}
return failed, existed, nil, err
}
record, err = j.markDispatching(spec.OperationID)
releaseClaim()
if err != nil {
releaseAdmission()
return record, existed, nil, err
}
if record.State != StateDispatching {
releaseAdmission()
return record, true, nil, nil
}
// Start the execution budget only after admission and the durable dispatch
// boundary. Time spent waiting for a resource or capacity slot must never
// consume the helper's bounded execution window.
workerCtx, workerCancel := context.WithTimeout(workerParent, workerTimeout)
type outcome struct {
result json.RawMessage
err error
}
callDone := make(chan outcome, 1)
done := make(chan struct{})
mutationContext := context.WithValue(workerCtx, expectedStateContextKey{}, cloneExpectedState(record.ExpectedState))
go func() {
result, callErr := fn(mutationContext)
callDone <- outcome{result: result, err: callErr}
}()
j.workerWG.Add(1)
go func() {
defer j.workerWG.Done()
defer close(done)
defer releaseAdmission()
defer workerCancel()
select {
case completed := <-callDone:
if _, persistErr := j.finish(spec.OperationID, completed.result, completed.err); persistErr != nil {
j.reportBackgroundFailure(spec.OperationID, "finalize", persistErr)
}
case <-workerCtx.Done():
reapTimer := time.NewTimer(workerReapDeadline)
select {
case completed := <-callDone:
if _, persistErr := j.finish(spec.OperationID, completed.result, completed.err); persistErr != nil {
j.reportBackgroundFailure(spec.OperationID, "finalize", persistErr)
}
case <-reapTimer.C:
if _, persistErr := j.markExecutionFenced(spec.OperationID); persistErr != nil {
j.reportBackgroundFailure(spec.OperationID, "fence", persistErr)
}
}
if !reapTimer.Stop() {
select {
case <-reapTimer.C:
default:
}
}
}
}()
return record, existed, done, nil
}
// ExpectedStateFromContext returns the descriptor atomically persisted with
// the dispatch record. Mutation implementations use it to write a matching
// host-side transition marker before reporting success.
func ExpectedStateFromContext(ctx context.Context) (ExpectedState, bool) {
expected, ok := ctx.Value(expectedStateContextKey{}).(*ExpectedState)
if !ok || expected == nil {
return ExpectedState{}, false
}
return *cloneExpectedState(expected), true
}
// RequiresExplicitForceDelete identifies the terminal graceful result which
// remains open for fresh operator inspection and resource conflict blocking.
// Only a separately acknowledged force-delete-vm operation may cross this
// boundary; inspection itself never changes the record.
func RequiresExplicitForceDelete(record Record) bool {
return !record.Tombstone && record.State == StateFailed && record.Error != nil &&
record.Error.Code == "force_required" &&
(record.Action == "delete-vm" || record.Action == "service-suspend")
}
func (j *Journal) initialize() error {
return j.db.Update(func(tx *bolt.Tx) error {
metadata := tx.Bucket(metadataBucket)
fresh := metadata == nil
var err error
if fresh {
hasExistingBuckets := false
if err := tx.ForEach(func(_ []byte, _ *bolt.Bucket) error {
hasExistingBuckets = true
return nil
}); err != nil {
return err
}
if hasExistingBuckets {
return errors.New("operation journal has no supported schema; automatic migration is disabled")
}
metadata, err = tx.CreateBucket(metadataBucket)
if err != nil {
return fmt.Errorf("create journal metadata bucket: %w", err)
}
} else {
current := metadata.Get(schemaVersionKey)
if string(current) != string(schemaVersion) {
return fmt.Errorf("unsupported operation journal schema %q; clean-break deployment requires stopping the agent and removing the old test journal before upgrade", current)
}
}
operations, err := tx.CreateBucketIfNotExists(operationsBucket)
if err != nil {
return fmt.Errorf("create operation bucket: %w", err)
}
if _, err := tx.CreateBucketIfNotExists(noncesBucket); err != nil {
return fmt.Errorf("create nonce bucket: %w", err)
}
if _, err := tx.CreateBucketIfNotExists(nonceExpiryBucket); err != nil {
return fmt.Errorf("create nonce expiry bucket: %w", err)
}
operationExpiry, err := tx.CreateBucketIfNotExists(operationExpiryBucket)
if err != nil {
return fmt.Errorf("create operation expiry bucket: %w", err)
}
completionReceipts, err := tx.CreateBucketIfNotExists(completionReceiptsBucket)
if err != nil {
return fmt.Errorf("create completion receipt bucket: %w", err)
}
completionOutbox, err := tx.CreateBucketIfNotExists(completionOutboxBucket)
if err != nil {
return fmt.Errorf("create completion outbox bucket: %w", err)
}
if _, err := tx.CreateBucketIfNotExists(completionQuarantineBucket); err != nil {
return fmt.Errorf("create completion quarantine bucket: %w", err)
}
if _, err := tx.CreateBucketIfNotExists(completionQuarantineOperationsBucket); err != nil {
return fmt.Errorf("create completion quarantine operation index: %w", err)
}
if _, err := tx.CreateBucketIfNotExists(fenceResolutionsBucket); err != nil {
return fmt.Errorf("create fence resolution bucket: %w", err)
}
if _, err := tx.CreateBucketIfNotExists(fenceResolutionIDsBucket); err != nil {
return fmt.Errorf("create fence resolution identity bucket: %w", err)
}
inflight, err := tx.CreateBucketIfNotExists(inflightBucket)
if err != nil {
return fmt.Errorf("create inflight operation bucket: %w", err)
}
blockers := tx.Bucket(resourceBlockersBucket)
if blockers == nil {
if !fresh {
return errors.New("operation resource blocker index is unavailable")
}
blockers, err = tx.CreateBucket(resourceBlockersBucket)
if err != nil {
return fmt.Errorf("create operation resource blocker bucket: %w", err)
}
}
// resource_blockers is a derived admission index. Host storage
// configuration keeps immutable failure evidence, but a terminal attempt
// must never prevent an administrator from submitting a new, independently
// identified configuration request. Remove entries written by the earlier
// schema-v6 policy before this process admits any request. The surrounding
// Bolt transaction makes the repair atomic, and the operation plus its
// immutable completion receipt remain untouched.
if err := removeStorageConfigurationBlockers(operations, blockers); err != nil {
return err
}
now := j.currentTime()
var recovered []Record
var stale [][]byte
if err := inflight.ForEach(func(key, _ []byte) error {
value := operations.Get(key)
if value == nil {
stale = append(stale, append([]byte(nil), key...))
return nil
}
record, err := decodeRecord(value)
if err != nil {
return fmt.Errorf("decode operation %q: %w", key, err)
}
switch record.State {
case StateReserved:
record.State = StateFailed
record.Result = notDispatchedEvidence(record.Action)
record.Error = &PublicError{Code: "operation_not_dispatched", Message: "agent restarted before the dispatch boundary"}
record.EffectMayHaveApplied = false
record.SafeToResubmit = true
case StateDispatching:
// The parent process may have crashed after its isolated worker
// durably wrote a receipt. Leave this row in the inflight index until
// FinalizeInterruptedOperations imports that receipt or records
// result_unconfirmed. The mutation is never started again.
return nil
default:
stale = append(stale, append([]byte(nil), key...))
return nil
}
record.UpdatedAt = now
recovered = append(recovered, record)
return nil
}); err != nil {
return err
}
for _, record := range recovered {
previous := cloneRecord(record)
previous.State = StateDispatching
if record.Error != nil && record.Error.Code == "operation_not_dispatched" {
previous.State = StateReserved
}
if err := putRecord(operations, record); err != nil {
return err
}
if err := syncCompletionReceipt(completionReceipts, completionOutbox, previous, record); err != nil {
return err
}
if err := syncResourceBlockers(blockers, nil, &record); err != nil {
return err
}
if terminalState(record.State) {
if err := operationExpiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
return err
}
}
if err := inflight.Delete([]byte(record.OperationID)); err != nil {
return err
}
}
for _, key := range stale {
if err := inflight.Delete(key); err != nil {
return err
}
}
if err := pruneExpiredOperationDetails(tx, now); err != nil {
return err
}
return metadata.Put(schemaVersionKey, schemaVersion)
})
}
func (j *Journal) markDispatching(operationID string) (Record, error) {
return j.updateRecord(operationID, func(record *Record) {
if record.State == StateReserved {
record.State = StateDispatching
record.UpdatedAt = j.currentTime()
}
})
}
func (j *Journal) finish(operationID string, result json.RawMessage, callErr error) (Record, error) {
if IsUncertain(callErr) {
return j.markResultUnconfirmed(operationID, callErr)
}
if callErr != nil {
code := "backend_error"
message := callErr.Error()
effectMayHaveApplied := true
safeToResubmit := false
var public interface {
PublicCode() string
PublicMessage() string
}
if errors.As(callErr, &public) {
if public.PublicCode() != "" {
code = public.PublicCode()
}
if public.PublicMessage() != "" {
message = public.PublicMessage()
}
}
var semantics interface{ FailureSemantics() (bool, bool) }
if errors.As(callErr, &semantics) {
effectMayHaveApplied, safeToResubmit = semantics.FailureSemantics()
}
executionFenced := false
var fence interface{ RequiresExecutionFence() bool }
if errors.As(callErr, &fence) {
executionFenced = fence.RequiresExecutionFence()
}
return j.finishWithSemanticsAndFence(operationID, code, message, result, code, effectMayHaveApplied, safeToResubmit, executionFenced)
}
return j.updateRecord(operationID, func(record *Record) {
record.State = StateSucceeded
record.Result = append(json.RawMessage(nil), result...)
record.Error = nil
record.EffectMayHaveApplied = false
record.SafeToResubmit = false
record.ExecutionFenced = false
record.CauseCode = ""
record.UpdatedAt = j.currentTime()
})
}
func (j *Journal) finishWithPublicError(operationID, code, message string) (Record, error) {
return j.finishWithPublicErrorAndResult(operationID, code, message, nil)
}
func (j *Journal) finishWithPublicErrorAndResult(operationID, code, message string, result json.RawMessage) (Record, error) {
return j.finishWithCause(operationID, code, message, result, "")
}
func (j *Journal) finishWithCause(operationID, code, message string, result json.RawMessage, causeCode string) (Record, error) {
return j.finishWithSemantics(operationID, code, message, result, causeCode, true, false)
}
func (j *Journal) finishWithSemantics(operationID, code, message string, result json.RawMessage, causeCode string,
effectMayHaveApplied, safeToResubmit bool) (Record, error) {
return j.finishWithSemanticsAndFence(operationID, code, message, result, causeCode, effectMayHaveApplied, safeToResubmit, false)
}
func (j *Journal) finishWithSemanticsAndFence(operationID, code, message string, result json.RawMessage, causeCode string,
effectMayHaveApplied, safeToResubmit, executionFenced bool) (Record, error) {
return j.updateRecord(operationID, func(record *Record) {
record.State = StateFailed
record.Result = append(json.RawMessage(nil), result...)
record.Error = &PublicError{Code: code, Message: message}
record.EffectMayHaveApplied = effectMayHaveApplied
record.SafeToResubmit = safeToResubmit
record.ExecutionFenced = executionFenced
preserveCauseCode(record, causeCode)
record.UpdatedAt = j.currentTime()
})
}
func (j *Journal) failNotDispatchedIfReserved(operationID string) (Record, error) {
return j.updateRecord(operationID, func(record *Record) {
if record.State != StateReserved {
return
}
record.State = StateFailed
record.Result = notDispatchedEvidence(record.Action)
record.Error = &PublicError{Code: "operation_not_dispatched", Message: "operation did not cross the dispatch boundary"}
record.EffectMayHaveApplied = false
record.SafeToResubmit = true
record.ExecutionFenced = false
record.UpdatedAt = j.currentTime()
})
}
// FailNotDispatched terminally records that a reserved operation never
// crossed the dispatch boundary. It is safe for admission failures only.
func (j *Journal) FailNotDispatched(operationID string) (Record, error) {
return j.failNotDispatchedIfReserved(operationID)
}
func notDispatchedEvidence(action string) json.RawMessage {
if IsCreateLikeAction(action) {
return json.RawMessage(`{"not_dispatched":true,"creation_absent":true,"managed_artifacts_absent":true}`)
}
return json.RawMessage(`{"not_dispatched":true}`)
}
func (j *Journal) markResultUnconfirmed(operationID string, cause error) (Record, error) {
return j.updateRecord(operationID, func(record *Record) {
if terminalState(record.State) {
return
}
message := "operation result could not be confirmed before the execution deadline"
if summary := publicMessageFromError(cause); summary != "" {
message += ": " + summary
}
record.State = StateFailed
record.Result = nil
record.Error = &PublicError{Code: "result_unconfirmed", Message: message}
record.EffectMayHaveApplied = true
record.SafeToResubmit = false
record.ExecutionFenced = false
preserveCauseCode(record, publicCodeFromError(cause))
record.UpdatedAt = j.currentTime()
})
}
func (j *Journal) markExecutionFenced(operationID string) (Record, error) {
return j.updateRecord(operationID, func(record *Record) {
if terminalState(record.State) {
return
}
record.State = StateFailed
record.Result = nil
record.Error = &PublicError{
Code: "worker_reap_unconfirmed",
Message: "isolated mutation process did not exit after forced termination; this resource is fenced",
}
record.EffectMayHaveApplied = true
record.SafeToResubmit = false
record.ExecutionFenced = true
record.UpdatedAt = j.currentTime()
})
}
func preserveCauseCode(record *Record, code string) {
if record.CauseCode == "" && validCauseCode(code) {
record.CauseCode = code
}
}
func validCauseCode(code string) bool {
return len(code) <= 64 && causeCodePattern.MatchString(code)
}
func publicCodeFromError(err error) string {
if err == nil {
return ""
}
var public interface{ PublicCode() string }
if !errors.As(err, &public) {
return ""
}
code := public.PublicCode()
if !validCauseCode(code) {
return ""
}
return code
}
func publicMessageFromError(err error) string {
if err == nil {
return ""
}
var public interface{ PublicMessage() string }
if !errors.As(err, &public) {
return ""
}
return public.PublicMessage()
}
func (j *Journal) updateRecord(operationID string, mutate func(*Record)) (Record, error) {
return j.updateRecordWhen(operationID, nil, mutate)
}
// updateRecordWhen checks eligibility and applies a mutation in the same Bolt
// write transaction. A nil predicate preserves the ordinary update behavior.
func (j *Journal) updateRecordWhen(operationID string, eligible func(Record) bool,
mutate func(*Record)) (Record, error) {
var result Record
err := j.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(operationsBucket)
inflight := tx.Bucket(inflightBucket)
blockers := tx.Bucket(resourceBlockersBucket)
expiry := tx.Bucket(operationExpiryBucket)
receipts := tx.Bucket(completionReceiptsBucket)
outbox := tx.Bucket(completionOutboxBucket)
if bucket == nil || inflight == nil || blockers == nil || expiry == nil || receipts == nil || outbox == nil {
return errors.New("operation buckets are unavailable")
}
value := bucket.Get([]byte(operationID))
if value == nil {
return ErrNotFound
}
record, err := decodeRecord(value)
if err != nil {
return err
}
if eligible != nil && !eligible(record) {
result = record
return nil
}
previous := cloneRecord(record)
mutate(&record)
if err := putRecord(bucket, record); err != nil {
return err
}
if err := syncCompletionReceipt(receipts, outbox, previous, record); err != nil {
return err
}
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
return err
}
if record.State == StateSucceeded && record.SupersedesOperationID != "" &&
(record.Action == "force-delete-vm" || isVPCAction(record.Action)) {
if err := closeSuccessfulSupersessionPredecessor(bucket, blockers, record); err != nil {
return err
}
}
switch record.State {
case StateReserved, StateDispatching:
if err := inflight.Put([]byte(record.OperationID), nil); err != nil {
return err
}
default:
if err := inflight.Delete([]byte(record.OperationID)); err != nil {
return err
}
}
if terminalState(record.State) && !record.Tombstone {
if err := expiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
return err
}
}
result = record
return nil
})
return cloneRecord(result), err
}
// closeSuccessfulSupersessionPredecessor runs in the same Bolt transaction
// which marks the reviewed successor successful. It removes only the prior
// blocker index entries; terminal evidence remains byte-for-byte immutable.
func closeSuccessfulSupersessionPredecessor(operations, blockers *bolt.Bucket, successor Record) error {
encoded := operations.Get([]byte(successor.SupersedesOperationID))
if encoded == nil {
return ErrInvalidSupersession
}
predecessor, err := decodeRecord(encoded)
if err != nil {
return err
}
lockKeys := effectiveLockKeys(successor.OperationID, successor.ResourceID, successor.Action, successor.LockKeys)
spec := Spec{
OperationID: successor.OperationID, ResourceID: successor.ResourceID,
Action: successor.Action, ExpectedState: successor.ExpectedState,
SupersedesOperationID: successor.SupersedesOperationID,
}
if validForceDeleteSupersession(spec, lockKeys, predecessor) {
for _, lockKey := range effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys) {
if err := blockers.Delete(blockerEntryKey(lockKey, predecessor.OperationID)); err != nil {
return err
}
}
return nil
}
if !isVPCAction(successor.Action) {
return ErrInvalidSupersession
}
chain, err := vpcSupersessionChain(operations, spec, lockKeys)
if err != nil {
return err
}
for _, prior := range chain {
for _, lockKey := range effectiveLockKeys(prior.OperationID, prior.ResourceID, prior.Action, prior.LockKeys) {
if err := blockers.Delete(blockerEntryKey(lockKey, prior.OperationID)); err != nil {
return err
}
}
}
return nil
}
func (j *Journal) acquire(ctx context.Context, key string) (func(), error) {
if err := ctx.Err(); err != nil {
return nil, err
}
j.resourceMu.Lock()
lock, ok := j.resources[key]
if !ok {
lock = &resourceLock{semaphore: make(chan struct{}, 1)}
j.resources[key] = lock
}
lock.references++
j.resourceMu.Unlock()
select {
case lock.semaphore <- struct{}{}:
var once sync.Once
release := func() {
once.Do(func() {
<-lock.semaphore
j.releaseResourceReference(key, lock)
})
}
if err := ctx.Err(); err != nil {
release()
return nil, err
}
return release, nil
case <-ctx.Done():
j.releaseResourceReference(key, lock)
return nil, ctx.Err()
}
}
func (j *Journal) claimAdmission(operationID string) (func(), bool) {
j.admissionClaimMu.Lock()
if _, exists := j.admissionClaims[operationID]; exists {
j.admissionClaimMu.Unlock()
return nil, false
}
j.admissionClaims[operationID] = struct{}{}
j.admissionClaimMu.Unlock()
var once sync.Once
return func() {
once.Do(func() {
j.admissionClaimMu.Lock()
delete(j.admissionClaims, operationID)
j.admissionClaimMu.Unlock()
})
}, true
}
func blockerEntryKey(lockKey, operationID string) []byte {
return []byte(lockKey + "\x00" + operationID)
}
func blockerEntryPrefix(lockKey string) []byte {
return []byte(lockKey + "\x00")
}
func effectfulTerminalFailure(record Record) bool {
return !record.Tombstone && record.State == StateFailed && record.EffectMayHaveApplied
}
func blocksResources(record Record) bool {
// Fixed storage-pool registration is monotonic: a successor validates and
// reuses matching pools, and never deletes or rebinds an existing pool. Keep
// the immutable failed operation as evidence, but do not let it fence a new,
// explicitly identified configure-storage request.
return effectfulTerminalFailure(record) && record.Action != "configure-storage"
}
// removeStorageConfigurationBlockers removes only derived blocker-index
// entries owned by host storage configuration operations. Every reference is
// validated before any key is deleted so a missing, malformed, or stale
// non-storage reference still fails Agent startup instead of silently dropping
// a safety fence.
func removeStorageConfigurationBlockers(operations, blockers *bolt.Bucket) error {
if operations == nil || blockers == nil {
return errors.New("operation resource blocker index is unavailable")
}
removable := make([][]byte, 0)
cursor := blockers.Cursor()
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
separator := bytes.IndexByte(key, 0)
if separator <= 0 || separator != bytes.LastIndexByte(key, 0) || separator == len(key)-1 {
return fmt.Errorf("invalid operation resource blocker key %q", key)
}
lockKey := string(key[:separator])
operationID := string(key[separator+1:])
if err := validateLockKey(lockKey); err != nil || !operationIDPattern.MatchString(operationID) {
return fmt.Errorf("invalid operation resource blocker key %q", key)
}
encoded := operations.Get([]byte(operationID))
if encoded == nil {
return fmt.Errorf("resource blocker index references missing operation %q", operationID)
}
record, err := decodeRecord(encoded)
if err != nil {
return fmt.Errorf("decode resource blocker %q: %w", operationID, err)
}
if record.OperationID != operationID {
return fmt.Errorf("resource blocker index references mismatched operation %q", operationID)
}
matchingKey := false
for _, expected := range effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys) {
if expected == lockKey {
matchingKey = true
break
}
}
if !matchingKey {
return fmt.Errorf("resource blocker index references unexpected lock %q for operation %q", lockKey, operationID)
}
if record.Action == "configure-storage" {
if !effectfulTerminalFailure(record) {
return fmt.Errorf("resource blocker index references non-blocking operation %q", operationID)
}
removable = append(removable, append([]byte(nil), key...))
continue
}
if !blocksResources(record) {
return fmt.Errorf("resource blocker index references resolved operation %q", operationID)
}
}
for _, key := range removable {
if err := blockers.Delete(key); err != nil {
return fmt.Errorf("delete retired storage configuration blocker %q: %w", key, err)
}
}
return nil
}
func syncResourceBlockers(blockers *bolt.Bucket, previous, current *Record) error {
if blockers == nil {
return errors.New("operation resource blocker bucket is unavailable")
}
if previous != nil && blocksResources(*previous) {
for _, key := range effectiveLockKeys(previous.OperationID, previous.ResourceID, previous.Action, previous.LockKeys) {
if err := blockers.Delete(blockerEntryKey(key, previous.OperationID)); err != nil {
return err
}
}
}
if current != nil && blocksResources(*current) {
for _, key := range effectiveLockKeys(current.OperationID, current.ResourceID, current.Action, current.LockKeys) {
if err := blockers.Put(blockerEntryKey(key, current.OperationID), nil); err != nil {
return err
}
}
}
return nil
}
func lockKeysOverlap(left, right []string) bool {
leftSet := make(map[string]struct{}, len(left))
for _, key := range left {
leftSet[key] = struct{}{}
}
for _, key := range right {
if _, ok := leftSet[key]; ok {
return true
}
}
return false
}
func validForceDeleteSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID ||
spec.Action != "force-delete-vm" || predecessor.OperationID != spec.SupersedesOperationID ||
predecessor.ResourceID != spec.ResourceID || (predecessor.Action != "delete-vm" && predecessor.Action != "service-suspend") ||
predecessor.State != StateFailed || predecessor.Error == nil || predecessor.Error.Code != "force_required" {
return false
}
predecessorKeys := effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys)
return lockKeysOverlap(lockKeys, predecessorKeys)
}
func validPeerMigrationTargetCleanupSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID ||
!cleanupPeerTargetActionPattern.MatchString(spec.Action) ||
predecessor.OperationID != spec.SupersedesOperationID ||
predecessor.ResourceID != spec.ResourceID ||
predecessor.State != StateFailed || !predecessor.EffectMayHaveApplied ||
predecessor.SafeToResubmit || predecessor.ExecutionFenced ||
!receivePeerActionPattern.MatchString(predecessor.Action) ||
strings.TrimPrefix(spec.Action, "cleanup-peer-migration-target:") !=
strings.TrimPrefix(predecessor.Action, "receive-peer-migration:") {
return false
}
predecessorKeys := effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys)
return lockKeysOverlap(lockKeys, predecessorKeys)
}
func validBackupWorkspaceCleanupSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID ||
!cleanupBackupActionPattern.MatchString(spec.Action) ||
predecessor.OperationID != spec.SupersedesOperationID ||
predecessor.ResourceID != spec.ResourceID ||
predecessor.State != StateSucceeded || predecessor.ExecutionFenced ||
!createBackupActionPattern.MatchString(predecessor.Action) {
return false
}
backupKey := strings.TrimPrefix(spec.Action, "cleanup-backup-workspace:")
priorParts := strings.Split(predecessor.Action, ":")
if len(priorParts) != 3 || priorParts[1] != backupKey {
return false
}
var result struct {
CleanupPending bool `json:"cleanup_pending"`
}
if json.Unmarshal(predecessor.Result, &result) != nil || !result.CleanupPending {
return false
}
predecessorKeys := effectiveLockKeys(
predecessor.OperationID, predecessor.ResourceID,
predecessor.Action, predecessor.LockKeys,
)
return lockKeysOverlap(lockKeys, predecessorKeys)
}
func isVPCAction(action string) bool {
return strings.HasPrefix(action, "apply-vpc-network:") ||
strings.HasPrefix(action, "remove-vpc-network:") ||
action == "apply-vpc-port" || action == "remove-vpc-port"
}
func sameVPCRepairIntent(current, predecessor *ExpectedState) bool {
if current == nil || predecessor == nil || current.Kind != predecessor.Kind ||
current.VPCKey != predecessor.VPCKey ||
current.VPCRevision != predecessor.VPCRevision ||
current.VPCDigest != predecessor.VPCDigest ||
current.VPCPresent != predecessor.VPCPresent {
return false
}
if current.Kind == ExpectedVPCNetwork {
return true
}
return current.Kind == ExpectedVPCPort &&
current.VPCPortKey == predecessor.VPCPortKey &&
current.VPCCIDR == predecessor.VPCCIDR &&
current.VPCGateway == predecessor.VPCGateway &&
current.VPCPrivateIP == predecessor.VPCPrivateIP &&
current.VPCMAC == predecessor.VPCMAC &&
current.VPCBridge == predecessor.VPCBridge
}
func validVPCRepairSupersession(spec Spec, lockKeys []string,
predecessor Record) bool {
if spec.SupersedesOperationID == "" ||
spec.SupersedesOperationID == spec.OperationID ||
predecessor.OperationID != spec.SupersedesOperationID ||
predecessor.ResourceID != spec.ResourceID ||
predecessor.Action != spec.Action || !isVPCAction(spec.Action) ||
predecessor.State != StateFailed || !predecessor.EffectMayHaveApplied ||
predecessor.SafeToResubmit || !predecessor.ExecutionFenced ||
predecessor.Tombstone ||
!sameVPCRepairIntent(spec.ExpectedState, predecessor.ExpectedState) {
return false
}
predecessorKeys := effectiveLockKeys(
predecessor.OperationID, predecessor.ResourceID,
predecessor.Action, predecessor.LockKeys,
)
return lockKeysOverlap(lockKeys, predecessorKeys)
}
// ValidVPCRepairSupersession exposes the same fail-closed predecessor check to
// the authenticated API before a repair is dispatched. Journal admission
// repeats it under the Bolt transaction, so this preflight is not authority.
func ValidVPCRepairSupersession(spec Spec, predecessor Record) bool {
lockKeys, err := normalizeLockKeys(spec.LockKeys)
return err == nil && validVPCRepairSupersession(
spec, lockKeys, predecessor,
)
}
func validExplicitSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
return validForceDeleteSupersession(spec, lockKeys, predecessor) ||
validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor) ||
validBackupWorkspaceCleanupSupersession(spec, lockKeys, predecessor) ||
validVPCRepairSupersession(spec, lockKeys, predecessor)
}
const maxVPCSupersessionDepth = 32
// vpcSupersessionChain validates every durable predecessor generation. A
// repair that itself lost its response may be followed by another exact
// repair, but it may never skip a changed intent or grow without a bound.
func vpcSupersessionChain(operations *bolt.Bucket, spec Spec,
lockKeys []string) ([]Record, error) {
if operations == nil || !isVPCAction(spec.Action) ||
spec.SupersedesOperationID == "" {
return nil, ErrInvalidSupersession
}
currentSpec := spec
currentKeys := lockKeys
seen := map[string]struct{}{spec.OperationID: {}}
chain := make([]Record, 0, 2)
for len(chain) < maxVPCSupersessionDepth {
predecessorID := currentSpec.SupersedesOperationID
if predecessorID == "" {
return chain, nil
}
if _, exists := seen[predecessorID]; exists {
return nil, ErrInvalidSupersession
}
seen[predecessorID] = struct{}{}
encoded := operations.Get([]byte(predecessorID))
if encoded == nil {
return nil, ErrInvalidSupersession
}
predecessor, err := decodeRecord(encoded)
if err != nil || !validVPCRepairSupersession(
currentSpec, currentKeys, predecessor,
) {
return nil, ErrInvalidSupersession
}
chain = append(chain, predecessor)
if predecessor.SupersedesOperationID == "" {
return chain, nil
}
currentSpec = Spec{
OperationID: predecessor.OperationID,
ResourceID: predecessor.ResourceID,
Action: predecessor.Action,
ExpectedState: predecessor.ExpectedState,
SupersedesOperationID: predecessor.SupersedesOperationID,
}
currentKeys = effectiveLockKeys(
predecessor.OperationID, predecessor.ResourceID,
predecessor.Action, predecessor.LockKeys,
)
}
return nil, ErrInvalidSupersession
}
// findResourceBlocker validates any explicit force-delete predecessor and
// reads only the compact blocker index for unresolved resource conflicts.
func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, bool, error) {
operations := tx.Bucket(operationsBucket)
blockers := tx.Bucket(resourceBlockersBucket)
if operations == nil || blockers == nil {
return Record{}, false, errors.New("operation resource blocker index is unavailable")
}
supersededOperationIDs := map[string]struct{}{}
if spec.SupersedesOperationID != "" {
encoded := operations.Get([]byte(spec.SupersedesOperationID))
if encoded == nil {
return Record{}, false, ErrInvalidSupersession
}
predecessor, err := decodeRecord(encoded)
if err != nil || !validExplicitSupersession(spec, lockKeys, predecessor) {
return Record{}, false, ErrInvalidSupersession
}
if isVPCAction(spec.Action) {
chain, chainErr := vpcSupersessionChain(
operations, spec, lockKeys,
)
if chainErr != nil {
return Record{}, false, chainErr
}
for _, prior := range chain {
supersededOperationIDs[prior.OperationID] = struct{}{}
}
} else {
supersededOperationIDs[predecessor.OperationID] = struct{}{}
}
}
for _, lockKey := range lockKeys {
prefix := blockerEntryPrefix(lockKey)
cursor := blockers.Cursor()
for key, _ := cursor.Seek(prefix); key != nil && bytes.HasPrefix(key, prefix); key, _ = cursor.Next() {
operationID := string(key[len(prefix):])
_, explicitlySuperseded := supersededOperationIDs[operationID]
if operationID == spec.OperationID || explicitlySuperseded {
continue
}
encoded := operations.Get([]byte(operationID))
if encoded == nil {
return Record{}, false, fmt.Errorf("resource blocker index references missing operation %q", operationID)
}
blocker, err := decodeRecord(encoded)
if err != nil {
return Record{}, false, fmt.Errorf("decode resource blocker %q: %w", operationID, err)
}
if !blocksResources(blocker) {
return Record{}, false, fmt.Errorf("resource blocker index references resolved operation %q", operationID)
}
return cloneRecord(blocker), true, nil
}
}
return Record{}, false, nil
}
func (j *Journal) verifyAdmissionUnblocked(spec Spec) error {
lockKeys := effectiveLockKeys(spec.OperationID, spec.ResourceID, spec.Action, spec.LockKeys)
return j.db.View(func(tx *bolt.Tx) error {
_, blocked, err := findResourceBlocker(tx, spec, lockKeys)
if err != nil {
return err
}
if blocked {
return ErrResourceBlocked
}
return nil
})
}
// prepareAdmission runs only while this process owns the operation's admission
// claim. Terminal records are immutable and are never reopened.
func (j *Journal) prepareAdmission(spec Spec) (Record, error) {
lockKeys := effectiveLockKeys(spec.OperationID, spec.ResourceID, spec.Action, spec.LockKeys)
var record Record
err := j.db.Update(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
inflight := tx.Bucket(inflightBucket)
if operations == nil || inflight == nil {
return errors.New("operation buckets are unavailable")
}
value := operations.Get([]byte(spec.OperationID))
if value == nil {
return ErrNotFound
}
existing, err := decodeRecord(value)
if err != nil {
return err
}
if existing.Fingerprint != spec.Fingerprint || existing.Action != spec.Action || existing.ResourceID != spec.ResourceID ||
existing.SupersedesOperationID != spec.SupersedesOperationID {
return ErrFingerprint
}
if existing.State != StateReserved {
record = existing
return nil
}
existing.Destructive = spec.Destructive
existing.ExpectedState = cloneExpectedState(spec.ExpectedState)
existing.LockKeys = append([]string(nil), lockKeys...)
existing.HeavyIO = spec.HeavyIO
existing.SupersedesOperationID = spec.SupersedesOperationID
if err := putRecord(operations, existing); err != nil {
return err
}
if err := inflight.Put([]byte(existing.OperationID), nil); err != nil {
return err
}
record = existing
return nil
})
return cloneRecord(record), err
}
func (j *Journal) admissionReplay(operationID string) (Record, bool, error) {
record, err := j.Get(operationID)
if err != nil {
return Record{}, true, err
}
switch record.State {
case StateReserved:
return record, true, ErrOperationStarting
default:
return record, true, nil
}
}
func (j *Journal) acquireAdmission(ctx context.Context, record Record) (func(), error) {
keys := effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys)
releases := make([]func(), 0, len(keys)+2)
releaseAll := func() {
for index := len(releases) - 1; index >= 0; index-- {
releases[index]()
}
}
for _, key := range keys {
release, err := j.acquire(ctx, key)
if err != nil {
releaseAll()
return nil, err
}
releases = append(releases, release)
}
if record.HeavyIO || defaultHeavyIOAction(record.Action) {
release, err := acquireCapacity(ctx, j.heavyIOSlots)
if err != nil {
releaseAll()
return nil, err
}
releases = append(releases, release)
}
releaseExecution, err := j.acquireExecution(ctx)
if err != nil {
releaseAll()
return nil, err
}
releases = append(releases, releaseExecution)
if err := ctx.Err(); err != nil {
releaseAll()
return nil, err
}
var once sync.Once
return func() { once.Do(releaseAll) }, nil
}
func acquireCapacity(ctx context.Context, slots chan struct{}) (func(), error) {
if err := ctx.Err(); err != nil {
return nil, err
}
select {
case slots <- struct{}{}:
var once sync.Once
release := func() { once.Do(func() { <-slots }) }
if err := ctx.Err(); err != nil {
release()
return nil, err
}
return release, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (j *Journal) acquireExecution(ctx context.Context) (func(), error) {
if err := ctx.Err(); err != nil {
return nil, err
}
select {
case j.executionSlots <- struct{}{}:
var once sync.Once
release := func() {
once.Do(func() { <-j.executionSlots })
}
if err := ctx.Err(); err != nil {
release()
return nil, err
}
return release, nil
default:
return nil, ErrBusy
}
}
func (j *Journal) releaseResourceReference(key string, lock *resourceLock) {
j.resourceMu.Lock()
defer j.resourceMu.Unlock()
lock.references--
if lock.references == 0 && len(lock.semaphore) == 0 && j.resources[key] == lock {
delete(j.resources, key)
}
}
func effectiveLockKeys(operationID, resourceID, action string, configured []string) []string {
if len(configured) != 0 {
keys, _ := normalizeLockKeys(configured)
return keys
}
return []string{defaultResourceLockKey(operationID, resourceID, action)}
}
func defaultResourceLockKey(operationID, resourceID, action string) string {
canonicalResource := strings.ToLower(resourceID)
if canonicalUUIDPattern.MatchString(canonicalResource) {
return "domain:" + canonicalResource
}
if action == "configure-storage" {
return "storage-config"
}
for prefix, kind := range map[string]string{
"template:": "template", "rescue-template:": "rescue-template",
} {
if strings.HasPrefix(resourceID, prefix) {
return "catalog:" + kind + "/" + strings.TrimPrefix(resourceID, prefix)
}
}
if strings.HasPrefix(resourceID, "user-iso:") {
return "catalog:user-iso/" + strings.TrimPrefix(resourceID, "user-iso:")
}
if resourceID == "" {
return "operation:" + operationID
}
digest := sha256.Sum256([]byte(resourceID))
return "resource-hash:" + hex.EncodeToString(digest[:])
}
func defaultHeavyIOAction(action string) bool {
return action == "configure-storage" || IsCreateLikeAction(action) || action == "reinstall-vm" ||
strings.HasPrefix(action, "create-backup:") ||
action == "import-template" || action == "import-rescue-template" || action == "import-user-iso" ||
strings.HasPrefix(action, "create-snapshot:") || strings.HasPrefix(action, "restore-snapshot:")
}
func normalizeLockKeys(keys []string) ([]string, error) {
if len(keys) > maxAdmissionResourceKeys {
return nil, fmt.Errorf("operation has more than %d resource keys", maxAdmissionResourceKeys)
}
unique := make(map[string]struct{}, len(keys))
for _, key := range keys {
if err := validateLockKey(key); err != nil {
return nil, err
}
unique[key] = struct{}{}
}
result := make([]string, 0, len(unique))
for key := range unique {
result = append(result, key)
}
sort.Strings(result)
return result, nil
}
func validateLockKey(key string) error {
if key == "" || len(key) > maximumAdmissionKeyLength || strings.TrimSpace(key) != key || strings.ContainsRune(key, 0) {
return errors.New("invalid operation resource key")
}
if key == "storage-config" {
return nil
}
prefix, value, ok := strings.Cut(key, ":")
if !ok || value == "" {
return errors.New("invalid operation resource key")
}
switch prefix {
case "domain":
if !canonicalUUIDPattern.MatchString(value) {
return errors.New("domain resource key must contain a canonical lowercase UUID")
}
case "volume", "catalog":
left, right, found := strings.Cut(value, "/")
if !found || strings.Contains(right, "/") || !resourceKindPattern.MatchString(left) || !resourcePartPattern.MatchString(right) {
return errors.New("invalid operation resource key")
}
case "nwfilter":
if !resourcePartPattern.MatchString(value) {
return errors.New("invalid nwfilter resource key")
}
case "network-identity", "resource-hash":
if !hexDigestPattern.MatchString(value) {
return errors.New("invalid hashed operation resource key")
}
case "operation":
if !operationIDPattern.MatchString(value) {
return errors.New("invalid operation resource key")
}
default:
return errors.New("unsupported operation resource key")
}
return nil
}
func validateSpec(spec Spec) error {
if !operationIDPattern.MatchString(spec.OperationID) {
return errors.New("operation_id must be 8-80 URL-safe characters")
}
if !ValidOperationIdentity(spec.Action, spec.ResourceID, spec.Fingerprint) {
return errors.New("operation action, resource_id, or request fingerprint is invalid")
}
if _, err := normalizeLockKeys(spec.LockKeys); err != nil {
return err
}
if spec.SupersedesOperationID != "" {
if !operationIDPattern.MatchString(spec.SupersedesOperationID) ||
spec.SupersedesOperationID == spec.OperationID ||
(spec.Action != "force-delete-vm" &&
!cleanupPeerTargetActionPattern.MatchString(spec.Action) &&
!cleanupBackupActionPattern.MatchString(spec.Action) &&
!isVPCAction(spec.Action)) {
return ErrInvalidSupersession
}
}
if spec.ExpectedState != nil {
if err := validateExpectedState(spec.Action, spec.OperationID, *spec.ExpectedState); err != nil {
return err
}
}
return nil
}
func validateExpectedState(action, operationID string, expected ExpectedState) error {
if expected.TransitionID != operationID {
return errors.New("expected state transition_id must match operation_id")
}
if len(expected.Hostname) > 253 || len(expected.NetworkSourceType) > 16 || len(expected.NetworkSourceName) > 128 {
return errors.New("expected state contains an oversized value")
}
values := []string{expected.Hostname, expected.NetworkSourceType, expected.NetworkSourceName, expected.VPCKey, expected.VPCDigest, expected.VPCPortKey, expected.VPCCIDR, expected.VPCGateway, expected.VPCPrivateIP, expected.VPCMAC, expected.VPCBridge}
values = append(values, expected.PublicNetwork.NetworkSelector,
expected.PublicNetwork.IPv4.PoolKey, expected.PublicNetwork.IPv4.PoolCIDR,
expected.PublicNetwork.IPv4.AddressCIDR, expected.PublicNetwork.IPv4.Gateway,
expected.PublicNetwork.IPv6.PoolKey, expected.PublicNetwork.IPv6.PoolCIDR,
expected.PublicNetwork.IPv6.AddressCIDR, expected.PublicNetwork.IPv6.Gateway)
values = append(values, expected.PublicNetwork.DNSServers...)
for _, value := range values {
if strings.ContainsRune(value, 0) {
return errors.New("expected state contains NUL")
}
}
switch expected.Kind {
case ExpectedUpdateSettings:
if action != "update-settings" || (!expected.HostnameChanged && !expected.RootPasswordChanged && !expected.SSHKeyChanged) {
return errors.New("invalid update-settings expected state")
}
case ExpectedResizeResources:
if action != "resize-vm" || !validExpectedResizeResources(expected) {
return errors.New("invalid resize-resources expected state")
}
case ExpectedChangeNetwork:
if action != "change-network" || !validExpectedPublicNetwork(expected.PublicNetwork) ||
(expected.NetworkSourceType != "bridge" && expected.NetworkSourceType != "network") || !networkSourcePattern.MatchString(expected.NetworkSourceName) {
return errors.New("invalid change-network expected state")
}
case ExpectedNetworkState:
if (action != "network-suspend" && action != "network-resume") || expected.Enabled != (action == "network-resume") {
return errors.New("invalid network-state expected state")
}
case ExpectedServiceState:
if (action != "service-suspend" && action != "service-resume") || expected.Enabled != (action == "service-resume") || (!expected.Enabled && expected.StartIfSuspended) {
return errors.New("invalid service-state expected state")
}
case ExpectedPeerMigrationState:
if !strings.HasPrefix(action, "activate-peer-migration:") &&
!strings.HasPrefix(action, "rollback-peer-migration:") {
return errors.New("invalid peer-migration-state expected state")
}
case ExpectedFirewall:
if action != "apply-firewall" || expected.FirewallRevision == 0 || !hexDigestPattern.MatchString(expected.FirewallDigest) {
return errors.New("invalid firewall expected state")
}
case ExpectedStorageConfiguration:
if action != "configure-storage" ||
!hexDigestPattern.MatchString(expected.ManagedPathDigest) ||
!hexDigestPattern.MatchString(expected.TemplatePathDigest) ||
!hexDigestPattern.MatchString(expected.SeedPathDigest) ||
!hexDigestPattern.MatchString(expected.UserISOPathDigest) ||
!hexDigestPattern.MatchString(expected.RescuePathDigest) {
return errors.New("invalid storage-configuration expected state")
}
case ExpectedVPCNetwork:
applyAction := "apply-vpc-network:" + expected.VPCKey
removeAction := "remove-vpc-network:" + expected.VPCKey
if !shortHexIDPattern.MatchString(expected.VPCKey) || expected.VPCRevision == 0 ||
!hexDigestPattern.MatchString(expected.VPCDigest) ||
(expected.VPCPresent && action != applyAction) ||
(!expected.VPCPresent && action != removeAction) {
return errors.New("invalid VPC network expected state")
}
case ExpectedVPCPort:
if !shortHexIDPattern.MatchString(expected.VPCKey) ||
!shortHexIDPattern.MatchString(expected.VPCPortKey) ||
expected.VPCRevision == 0 || !hexDigestPattern.MatchString(expected.VPCDigest) ||
!validExpectedVPCPort(expected) ||
(expected.VPCPresent && action != "apply-vpc-port") ||
(!expected.VPCPresent && action != "remove-vpc-port") {
return errors.New("invalid VPC port expected state")
}
case ExpectedCreateLikeNetwork:
if !validExpectedCreateLikeNetwork(action, expected) {
return errors.New("invalid create-like network expected state")
}
default:
return errors.New("unsupported expected state kind")
}
return nil
}
func validExpectedCreateLikeNetwork(action string, expected ExpectedState) bool {
if (!restoreBackupActionPattern.MatchString(action) &&
!receivePeerActionPattern.MatchString(action)) ||
!validExpectedPublicNetwork(expected.PublicNetwork) ||
!validExpectedNetworkSource(expected) ||
!createLikeExpectedStateFieldsAreNarrow(expected) {
return false
}
if restoreBackupActionPattern.MatchString(action) {
return !expected.VPCPresent && expectedVPCPortFieldsAreEmpty(expected)
}
if !expected.VPCPresent {
return expectedVPCPortFieldsAreEmpty(expected)
}
return shortHexIDPattern.MatchString(expected.VPCKey) &&
shortHexIDPattern.MatchString(expected.VPCPortKey) &&
expected.VPCRevision > 0 && hexDigestPattern.MatchString(expected.VPCDigest) &&
validExpectedVPCPort(expected)
}
func validExpectedNetworkSource(expected ExpectedState) bool {
if (expected.NetworkSourceType != "bridge" && expected.NetworkSourceType != "network") ||
!networkSourcePattern.MatchString(expected.NetworkSourceName) {
return false
}
return expected.PublicNetwork.NetworkSelector ==
expected.NetworkSourceType+":"+expected.NetworkSourceName
}
func expectedVPCPortFieldsAreEmpty(expected ExpectedState) bool {
return expected.VPCKey == "" && expected.VPCRevision == 0 && expected.VPCDigest == "" &&
expected.VPCPortKey == "" && expected.VPCCIDR == "" && expected.VPCGateway == "" &&
expected.VPCPrivateIP == "" && expected.VPCMAC == "" && expected.VPCBridge == ""
}
func createLikeExpectedStateFieldsAreNarrow(expected ExpectedState) bool {
return !expected.HostnameChanged && expected.Hostname == "" &&
!expected.RootPasswordChanged && !expected.SSHKeyChanged &&
expected.FromVCPUs == 0 && expected.TargetVCPUs == 0 &&
expected.FromMemoryMiB == 0 && expected.TargetMemoryMiB == 0 &&
expected.FromDiskGiB == 0 && expected.TargetDiskGiB == 0 &&
expected.FromBandwidthMbps == 0 && expected.TargetBandwidthMbps == 0 &&
!expected.Enabled && !expected.StartIfSuspended &&
expected.FirewallRevision == 0 && expected.FirewallDigest == "" &&
expected.ManagedPathDigest == "" && expected.TemplatePathDigest == "" &&
expected.SeedPathDigest == "" && expected.UserISOPathDigest == "" &&
expected.RescuePathDigest == ""
}
func validExpectedVPCPort(expected ExpectedState) bool {
prefix, err := netip.ParsePrefix(expected.VPCCIDR)
if err != nil || !prefix.Addr().Is4() || !prefix.Addr().IsPrivate() ||
prefix != prefix.Masked() || prefix.String() != expected.VPCCIDR ||
prefix.Bits() < 16 || prefix.Bits() > 28 {
return false
}
gateway, gatewayErr := netip.ParseAddr(expected.VPCGateway)
address, addressErr := netip.ParseAddr(expected.VPCPrivateIP)
if gatewayErr != nil || addressErr != nil || !gateway.Is4() || !address.Is4() ||
gateway != prefix.Addr().Next() || !prefix.Contains(address) ||
address == prefix.Addr() || address == gateway ||
address == expectedVPCLastAddress(prefix) {
return false
}
hardware, err := net.ParseMAC(expected.VPCMAC)
if err != nil || len(hardware) != 6 || hardware[0]&1 != 0 ||
hardware.String() != expected.VPCMAC {
return false
}
zeroMAC := true
for _, value := range hardware {
zeroMAC = zeroMAC && value == 0
}
return !zeroMAC && networkSourcePattern.MatchString(expected.VPCBridge)
}
func expectedVPCLastAddress(prefix netip.Prefix) netip.Addr {
bytes := prefix.Masked().Addr().As4()
value := binary.BigEndian.Uint32(bytes[:]) | (^uint32(0) >> prefix.Bits())
var result [4]byte
binary.BigEndian.PutUint32(result[:], value)
return netip.AddrFrom4(result)
}
func validExpectedResizeResources(expected ExpectedState) bool {
if expected.FromVCPUs < 1 || expected.FromVCPUs > 512 ||
expected.TargetVCPUs < expected.FromVCPUs || expected.TargetVCPUs > 512 ||
expected.FromMemoryMiB < 128 || expected.FromMemoryMiB > 16*1024*1024 ||
expected.TargetMemoryMiB < expected.FromMemoryMiB || expected.TargetMemoryMiB > 16*1024*1024 ||
expected.FromDiskGiB < 1 || expected.FromDiskGiB > 4096 ||
expected.TargetDiskGiB < expected.FromDiskGiB || expected.TargetDiskGiB > 4096 ||
expected.FromBandwidthMbps < 1 || expected.FromBandwidthMbps > 100000 ||
expected.TargetBandwidthMbps < expected.FromBandwidthMbps || expected.TargetBandwidthMbps > 100000 {
return false
}
return expected.TargetVCPUs != expected.FromVCPUs ||
expected.TargetMemoryMiB != expected.FromMemoryMiB ||
expected.TargetDiskGiB != expected.FromDiskGiB ||
expected.TargetBandwidthMbps != expected.FromBandwidthMbps
}
func validateRecord(record Record) error {
if err := validateSpec(Spec{
OperationID: record.OperationID, ResourceID: record.ResourceID,
Action: record.Action, Fingerprint: record.Fingerprint, ExpectedState: record.ExpectedState,
LockKeys: record.LockKeys, HeavyIO: record.HeavyIO,
SupersedesOperationID: record.SupersedesOperationID,
}); err != nil {
return err
}
switch record.State {
case StateReserved, StateDispatching, StateSucceeded, StateFailed:
default:
return fmt.Errorf("invalid state %q", record.State)
}
if record.EffectMayHaveApplied && record.SafeToResubmit {
return errors.New("an effectful failure cannot be safe to resubmit")
}
if record.State != StateFailed && (record.EffectMayHaveApplied || record.SafeToResubmit) {
return errors.New("failure semantics require a failed operation")
}
if record.CauseCode != "" && !validCauseCode(record.CauseCode) {
return errors.New("invalid operation cause_code")
}
return nil
}
func putRecord(bucket *bolt.Bucket, record Record) error {
if err := validateRecord(record); err != nil {
return err
}
encoded, err := json.Marshal(storedRecord{
Record: recordAlias(record), ExpectedState: cloneExpectedState(record.ExpectedState),
LockKeys: append([]string(nil), record.LockKeys...), HeavyIO: record.HeavyIO,
SupersedesOperationID: record.SupersedesOperationID,
})
if err != nil {
return fmt.Errorf("encode operation record: %w", err)
}
return bucket.Put([]byte(record.OperationID), encoded)
}
func decodeRecord(encoded []byte) (Record, error) {
var stored storedRecord
if err := json.Unmarshal(encoded, &stored); err != nil {
return Record{}, fmt.Errorf("decode operation record: %w", err)
}
record := Record(stored.Record)
record.ExpectedState = cloneExpectedState(stored.ExpectedState)
record.LockKeys = append([]string(nil), stored.LockKeys...)
record.HeavyIO = stored.HeavyIO
record.SupersedesOperationID = stored.SupersedesOperationID
if err := validateRecord(record); err != nil {
return Record{}, err
}
return record, nil
}
func cloneRecord(record Record) Record {
copyRecord := record
copyRecord.Result = append(json.RawMessage(nil), record.Result...)
copyRecord.ExpectedState = cloneExpectedState(record.ExpectedState)
copyRecord.LockKeys = append([]string(nil), record.LockKeys...)
if record.Error != nil {
copyError := *record.Error
copyRecord.Error = &copyError
}
return copyRecord
}
func cloneExpectedState(expected *ExpectedState) *ExpectedState {
if expected == nil {
return nil
}
copyExpected := *expected
copyExpected.PublicNetwork.DNSServers = append([]string(nil), expected.PublicNetwork.DNSServers...)
return &copyExpected
}
func validExpectedPublicNetwork(value publicnetwork.Intent) bool {
canonical, err := value.Canonical()
return err == nil && canonical.Contract == value.Contract &&
canonical.NetworkSelector == value.NetworkSelector && canonical.IPv4 == value.IPv4 &&
canonical.IPv6 == value.IPv6 && slices.Equal(canonical.DNSServers, value.DNSServers)
}
func encodeExpiry(value time.Time) []byte {
encoded := make([]byte, 8)
binary.BigEndian.PutUint64(encoded, uint64(value.UnixNano()))
return encoded
}
func decodeExpiry(encoded []byte) (time.Time, error) {
if len(encoded) != 8 {
return time.Time{}, errors.New("invalid nonce expiry record")
}
return time.Unix(0, int64(binary.BigEndian.Uint64(encoded))).UTC(), nil
}
func expiryIndexKey(expiresAt time.Time, nonce string) []byte {
result := make([]byte, 8+len(nonce))
copy(result, encodeExpiry(expiresAt))
copy(result[8:], nonce)
return result
}
func operationExpiryKey(terminalAt time.Time, operationID string) []byte {
return expiryIndexKey(terminalAt, operationID)
}
func terminalState(state State) bool {
return state == StateSucceeded || state == StateFailed
}
func pruneExpiredOperationDetails(tx *bolt.Tx, now time.Time) error {
operations := tx.Bucket(operationsBucket)
expiry := tx.Bucket(operationExpiryBucket)
blockers := tx.Bucket(resourceBlockersBucket)
resolutions := tx.Bucket(fenceResolutionsBucket)
if operations == nil || expiry == nil || blockers == nil || resolutions == nil {
return errors.New("operation retention buckets are unavailable")
}
cutoff := now.Add(-operationDetailRetention).UnixNano()
cursor := expiry.Cursor()
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
if len(key) < 9 || int64(binary.BigEndian.Uint64(key[:8])) > cutoff {
break
}
operationID := string(key[8:])
retainExpiry := false
encoded := operations.Get([]byte(operationID))
if encoded != nil {
record, err := decodeRecord(encoded)
if err != nil {
return err
}
indexedAt := int64(binary.BigEndian.Uint64(key[:8]))
if terminalState(record.State) && !record.Tombstone && record.UpdatedAt.UnixNano() == indexedAt {
// Effectful failures remain durable resource fences. Other terminal
// details may expire, while the operation ID tombstone remains. A
// durable fence resolution makes the formerly effectful record safe
// to compact without rewriting its immutable resolution audit.
if !blocksResources(record) || resolutions.Get([]byte(operationID)) != nil {
previous := cloneRecord(record)
record.Tombstone = true
record.Result = nil
record.ExpectedState = nil
record.LockKeys = nil
record.HeavyIO = false
if record.Error != nil {
record.Error = &PublicError{Code: record.Error.Code, Message: "operation details expired; operation_id remains permanently reserved"}
}
if err := putRecord(operations, record); err != nil {
return err
}
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
return err
}
} else {
// Keep the expiry index while an old effectful failure still
// owns its fence. A later resolution leaves the immutable
// operation untouched, so this retained key is what lets the
// next bounded prune compact the full record.
retainExpiry = true
}
}
}
if !retainExpiry {
if err := cursor.Delete(); err != nil {
return err
}
}
}
return nil
}
func pruneExpiredNonces(nonces, expiry *bolt.Bucket, now time.Time) {
cursor := expiry.Cursor()
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
if len(key) < 9 || int64(binary.BigEndian.Uint64(key[:8])) >= now.UnixNano() {
break
}
nonce := key[8:]
if encoded := nonces.Get(nonce); encoded != nil && len(encoded) == 8 && string(encoded) == string(key[:8]) {
_ = nonces.Delete(nonce)
}
_ = cursor.Delete()
}
}
type expectedStateContextKey struct{}