You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
3584 lines
128 KiB
Go
3584 lines
128 KiB
Go
package operation
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"net"
|
|
"net/netip"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/publicnetwork"
|
|
bolt "go.etcd.io/bbolt"
|
|
)
|
|
|
|
type State string
|
|
|
|
const (
|
|
StateReserved State = "reserved"
|
|
StateDispatching State = "dispatching"
|
|
StateSucceeded State = "succeeded"
|
|
StateFailed State = "failed"
|
|
)
|
|
|
|
const (
|
|
ExpectedUpdateSettings = "update-settings"
|
|
ExpectedResizeResources = "resize-resources"
|
|
ExpectedChangeNetwork = "change-network"
|
|
ExpectedNetworkState = "network-state"
|
|
ExpectedServiceState = "service-state"
|
|
ExpectedPeerMigrationState = "peer-migration-state"
|
|
ExpectedFirewall = "firewall"
|
|
ExpectedStorageConfiguration = "storage-configuration"
|
|
ExpectedVPCNetwork = "vpc-network"
|
|
ExpectedVPCPort = "vpc-port"
|
|
ExpectedCreateLikeNetwork = "create-like-network"
|
|
operationDetailRetention = 90 * 24 * time.Hour
|
|
)
|
|
|
|
var (
|
|
ErrNotFound = errors.New("operation not found")
|
|
ErrFingerprint = errors.New("operation_id payload conflict")
|
|
ErrOperationStarting = errors.New("operation dispatch is already starting")
|
|
ErrBusy = errors.New("operation execution capacity is exhausted")
|
|
ErrResourceBlocked = errors.New("resource has an unresolved operation")
|
|
ErrInvalidSupersession = errors.New("operation cannot supersede the requested predecessor")
|
|
ErrInspectionInconclusive = errors.New("read-only inspection did not confirm a terminal outcome")
|
|
ErrExecutionNotReaped = errors.New("isolated mutation process did not exit after forced termination")
|
|
ErrFenceNotResolvable = errors.New("operation does not hold an effect fence")
|
|
ErrFenceIdentityConflict = errors.New("fence resolution identity does not match operation")
|
|
ErrFenceResolutionConflict = errors.New("operation effect fence already has a different resolution")
|
|
ErrFenceInspectionStale = errors.New("operation effect fence resolution requires a fresh inspection")
|
|
ErrTerminalEvidencePending = errors.New("worker journal has no terminal operation evidence")
|
|
ErrTerminalEvidenceInvalid = errors.New("worker journal terminal evidence is invalid")
|
|
operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`)
|
|
operationActionPattern = regexp.MustCompile(`^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$`)
|
|
operationResourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`)
|
|
hexDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
|
shortHexIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
|
noncePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`)
|
|
networkSourcePattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,128}$`)
|
|
causeCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
|
|
canonicalUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
|
|
resourcePartPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,192}$`)
|
|
resourceKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
|
|
fenceResolutionIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
|
|
completionSensitiveKeyPattern = regexp.MustCompile(`(?i)password|secret|(?:public|private).?key|ssh.?key|passphrase|credential|(?:(?:access|refresh|auth|api|bearer|session|source).?token)|^token$`)
|
|
metadataBucket = []byte("metadata")
|
|
operationsBucket = []byte("operations")
|
|
inflightBucket = []byte("inflight")
|
|
resourceBlockersBucket = []byte("resource_blockers")
|
|
operationExpiryBucket = []byte("operation_expiry")
|
|
completionReceiptsBucket = []byte("completion_receipts")
|
|
completionOutboxBucket = []byte("completion_outbox")
|
|
completionQuarantineBucket = []byte("completion_quarantine")
|
|
completionQuarantineOperationsBucket = []byte("completion_quarantine_operations")
|
|
fenceResolutionsBucket = []byte("fence_resolutions")
|
|
fenceResolutionIDsBucket = []byte("fence_resolution_ids")
|
|
noncesBucket = []byte("nonces")
|
|
nonceExpiryBucket = []byte("nonce_expiry")
|
|
schemaVersionKey = []byte("schema_version")
|
|
schemaVersion = []byte("7")
|
|
)
|
|
|
|
// ExpectedState is the non-secret, durable evidence contract used after a
|
|
// crash. It intentionally has no password, SSH key, token, file path, or raw
|
|
// request field. The exact request remains bound separately by Fingerprint.
|
|
type ExpectedState struct {
|
|
Kind string `json:"kind"`
|
|
TransitionID string `json:"transition_id"`
|
|
|
|
HostnameChanged bool `json:"hostname_changed,omitempty"`
|
|
Hostname string `json:"hostname,omitempty"`
|
|
RootPasswordChanged bool `json:"root_password_changed,omitempty"`
|
|
SSHKeyChanged bool `json:"ssh_key_changed,omitempty"`
|
|
|
|
FromVCPUs uint `json:"from_vcpu,omitempty"`
|
|
TargetVCPUs uint `json:"target_vcpu,omitempty"`
|
|
FromMemoryMiB uint64 `json:"from_memory_mib,omitempty"`
|
|
TargetMemoryMiB uint64 `json:"target_memory_mib,omitempty"`
|
|
FromDiskGiB uint64 `json:"from_disk_gib,omitempty"`
|
|
TargetDiskGiB uint64 `json:"target_disk_gib,omitempty"`
|
|
FromBandwidthMbps uint64 `json:"from_bandwidth_mbps,omitempty"`
|
|
TargetBandwidthMbps uint64 `json:"target_bandwidth_mbps,omitempty"`
|
|
|
|
NetworkSourceType string `json:"network_source_type,omitempty"`
|
|
NetworkSourceName string `json:"network_source_name,omitempty"`
|
|
PublicNetwork publicnetwork.Intent `json:"public_network,omitempty"`
|
|
|
|
Enabled bool `json:"enabled,omitempty"`
|
|
StartIfSuspended bool `json:"start_if_suspended,omitempty"`
|
|
|
|
FirewallRevision uint64 `json:"firewall_revision,omitempty"`
|
|
FirewallDigest string `json:"firewall_digest,omitempty"`
|
|
|
|
ManagedPathDigest string `json:"managed_path_digest,omitempty"`
|
|
TemplatePathDigest string `json:"template_path_digest,omitempty"`
|
|
SeedPathDigest string `json:"seed_path_digest,omitempty"`
|
|
UserISOPathDigest string `json:"user_iso_path_digest,omitempty"`
|
|
RescuePathDigest string `json:"rescue_path_digest,omitempty"`
|
|
|
|
VPCKey string `json:"vpc_key,omitempty"`
|
|
VPCRevision uint64 `json:"vpc_revision,omitempty"`
|
|
VPCDigest string `json:"vpc_digest,omitempty"`
|
|
VPCPresent bool `json:"vpc_present,omitempty"`
|
|
VPCPortKey string `json:"vpc_port_key,omitempty"`
|
|
VPCCIDR string `json:"vpc_cidr,omitempty"`
|
|
VPCGateway string `json:"vpc_gateway,omitempty"`
|
|
VPCPrivateIP string `json:"vpc_private_ip,omitempty"`
|
|
VPCMAC string `json:"vpc_mac,omitempty"`
|
|
VPCBridge string `json:"vpc_bridge,omitempty"`
|
|
}
|
|
|
|
type PublicError struct {
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
type Record struct {
|
|
OperationID string `json:"operation_id"`
|
|
ResourceID string `json:"resource_id,omitempty"`
|
|
Action string `json:"action"`
|
|
Fingerprint string `json:"fingerprint"`
|
|
Destructive bool `json:"destructive"`
|
|
State State `json:"status"`
|
|
Result json.RawMessage `json:"result,omitempty"`
|
|
Error *PublicError `json:"error,omitempty"`
|
|
CauseCode string `json:"cause_code,omitempty"`
|
|
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
|
|
SafeToResubmit bool `json:"safe_to_resubmit"`
|
|
ExecutionFenced bool `json:"execution_fenced,omitempty"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
Tombstone bool `json:"tombstone,omitempty"`
|
|
|
|
// ExpectedState is persisted in the private journal but deliberately never
|
|
// serialized on the HTTP API.
|
|
ExpectedState *ExpectedState `json:"-"`
|
|
// LockKeys and HeavyIO are private admission metadata. They are persisted so
|
|
// restart terminalization can retain the exact affected-resource fence, but
|
|
// they are never exposed by the Agent API.
|
|
LockKeys []string `json:"-"`
|
|
HeavyIO bool `json:"-"`
|
|
// SupersedesOperationID is valid only for an explicitly reviewed successor:
|
|
// force-delete-vm, plan-bound cleanup, or an exact VPC repair generation.
|
|
SupersedesOperationID string `json:"-"`
|
|
// InspectionReceiptRecovered and InspectionEvidenceUnavailable are
|
|
// ephemeral read-path markers. They are never persisted or exposed by the
|
|
// Agent API; the hypervisor inspector uses them to distinguish strictly
|
|
// validated completion evidence from a missing or inconsistent receipt.
|
|
InspectionReceiptRecovered bool `json:"-"`
|
|
InspectionEvidenceUnavailable bool `json:"-"`
|
|
}
|
|
|
|
type Spec struct {
|
|
OperationID string
|
|
ResourceID string
|
|
Action string
|
|
Fingerprint string
|
|
Destructive bool
|
|
ExpectedState *ExpectedState
|
|
LockKeys []string
|
|
HeavyIO bool
|
|
SupersedesOperationID string
|
|
}
|
|
|
|
// ValidOperationIdentity validates the immutable non-secret identity copied
|
|
// from a Journal record into inspections and completion deliveries.
|
|
func ValidOperationIdentity(action, resourceID, requestFingerprint string) bool {
|
|
return len(action) <= 128 && operationActionPattern.MatchString(action) &&
|
|
len(resourceID) <= 255 && operationResourceIDPattern.MatchString(resourceID) &&
|
|
hexDigestPattern.MatchString(requestFingerprint)
|
|
}
|
|
|
|
const (
|
|
completionDeliveryInterval = 30 * time.Second
|
|
completionPanelMaximumBody = 1 << 20
|
|
fenceInspectionMaximumAge = 2 * time.Minute
|
|
fenceInspectionFutureSkew = 5 * time.Second
|
|
)
|
|
|
|
const (
|
|
CompletionDeliveryStateQuarantined = "quarantined"
|
|
|
|
CompletionDeliveryErrorOutboxInvalid = "completion_outbox_invalid"
|
|
CompletionDeliveryErrorReceiptMissing = "completion_receipt_missing"
|
|
CompletionDeliveryErrorReceiptInvalid = "completion_receipt_invalid"
|
|
CompletionDeliveryErrorOperationMissing = "completion_operation_missing"
|
|
CompletionDeliveryErrorOperationInvalid = "completion_operation_invalid"
|
|
CompletionDeliveryErrorIdentityMismatch = "completion_identity_mismatch"
|
|
)
|
|
|
|
var workerReapDeadline = 3 * time.Second
|
|
var terminalEvidenceOpenTimeout = 2 * time.Second
|
|
|
|
// CompletionReceipt is immutable once an operation first reaches a terminal
|
|
// state. Delivery timing is kept in a separate outbox record so an ACK can
|
|
// never rewrite the operation or its receipt.
|
|
type CompletionReceipt struct {
|
|
Version int `json:"version,omitempty"`
|
|
EventID string `json:"id"`
|
|
OperationID string `json:"operation_id"`
|
|
Action string `json:"action"`
|
|
ResourceID string `json:"resource_id"`
|
|
RequestFingerprint string `json:"request_fingerprint"`
|
|
Status string `json:"status"`
|
|
Result json.RawMessage `json:"result"`
|
|
ErrorCode string `json:"error_code,omitempty"`
|
|
CauseCode string `json:"cause_code,omitempty"`
|
|
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
|
|
SafeToResubmit bool `json:"safe_to_resubmit"`
|
|
CreatedAt time.Time `json:"-"`
|
|
}
|
|
|
|
const (
|
|
FenceResolutionApplied = "applied"
|
|
FenceResolutionNotApplied = "not_applied"
|
|
|
|
FenceStateNotRequired FenceState = "not_required"
|
|
FenceStateActive FenceState = "active"
|
|
FenceStateResolved FenceState = "resolved"
|
|
)
|
|
|
|
// FenceState distinguishes an operation that never owned a cross-task fence
|
|
// from an active blocker and one already released by immutable audit metadata.
|
|
// Callers must not collapse resolved into not_required: doing so could let a
|
|
// second coordinator accept evidence that conflicts with the stored verdict.
|
|
type FenceState string
|
|
|
|
// FenceResolution is immutable audit metadata for a super-administrator's
|
|
// explicit conclusion after a fresh, read-only host inspection. It releases
|
|
// only Journal admission blockers; it never changes the original operation or
|
|
// completion receipt and never authorizes replay of that mutation.
|
|
type FenceResolution struct {
|
|
OperationID string `json:"operation_id"`
|
|
ResolutionID string `json:"resolution_id"`
|
|
Resolution string `json:"resolution"`
|
|
InspectionCheckedAt int64 `json:"inspection_checked_at"`
|
|
ReasonSHA256 string `json:"reason_sha256"`
|
|
ResolvedAt int64 `json:"resolved_at"`
|
|
ResolvedBy int64 `json:"resolved_by"`
|
|
}
|
|
|
|
type FenceResolutionRequest struct {
|
|
ResolutionID string `json:"resolution_id"`
|
|
Resolution string `json:"resolution"`
|
|
InspectionCheckedAt int64 `json:"inspection_checked_at"`
|
|
ReasonSHA256 string `json:"reason_sha256"`
|
|
Action string `json:"action"`
|
|
ResourceID string `json:"resource_id"`
|
|
RequestFingerprint string `json:"request_fingerprint"`
|
|
}
|
|
|
|
// InterruptedReceipt is the non-secret terminal result recovered from an
|
|
// isolated worker directory after the parent Agent restarts.
|
|
type InterruptedReceipt struct {
|
|
OperationID string
|
|
Fingerprint string
|
|
Status string
|
|
Result json.RawMessage
|
|
ErrorCode string
|
|
ErrorMessage string
|
|
EffectMayHaveApplied bool
|
|
SafeToResubmit bool
|
|
}
|
|
|
|
type completionOutboxRecord struct {
|
|
OperationID string `json:"operation_id"`
|
|
NextAttemptAt time.Time `json:"next_attempt_at"`
|
|
}
|
|
|
|
// CompletionDeliveryIncident is durable, non-secret audit metadata for one
|
|
// outbox entry that could not be reconstructed into the immutable completion
|
|
// wire contract. The receipt and outbox bytes remain untouched. A later claim
|
|
// revalidates them and removes this incident automatically after repair.
|
|
type CompletionDeliveryIncident struct {
|
|
EventID string `json:"event_id,omitempty"`
|
|
EventKeySHA256 string `json:"event_key_sha256"`
|
|
OperationID string `json:"operation_id,omitempty"`
|
|
ErrorCode string `json:"error_code"`
|
|
EvidenceSHA256 string `json:"evidence_sha256"`
|
|
FirstObservedAt time.Time `json:"first_observed_at"`
|
|
}
|
|
|
|
type Result struct {
|
|
Value json.RawMessage
|
|
}
|
|
|
|
type Resolution struct {
|
|
Resolved bool
|
|
Succeeded bool
|
|
Result json.RawMessage
|
|
ErrorCode string
|
|
ErrorMessage string
|
|
}
|
|
|
|
type UncertainError struct {
|
|
Cause error
|
|
}
|
|
|
|
func (e *UncertainError) Error() string {
|
|
if e.Cause == nil {
|
|
return "operation result is unconfirmed"
|
|
}
|
|
return fmt.Sprintf("operation result is unconfirmed: %v", e.Cause)
|
|
}
|
|
|
|
func (e *UncertainError) Unwrap() error { return e.Cause }
|
|
|
|
func MarkUncertain(err error) error { return &UncertainError{Cause: err} }
|
|
|
|
func hasUncertainMarker(err error) bool {
|
|
var target *UncertainError
|
|
return errors.As(err, &target)
|
|
}
|
|
|
|
func IsUncertain(err error) bool {
|
|
// Explicit uncertainty wins. Explicit NotApplied then proves that the
|
|
// caller never crossed its mutation boundary, even when the underlying
|
|
// pre-dispatch wait ended through context cancellation. Only an otherwise
|
|
// unclassified context failure is ambiguous.
|
|
if hasUncertainMarker(err) {
|
|
return true
|
|
}
|
|
if IsNotApplied(err) {
|
|
return false
|
|
}
|
|
return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
|
|
}
|
|
|
|
// NotAppliedError records that the caller failed before its exact mutation
|
|
// boundary. The wrapped error keeps its public code/message, while the Journal
|
|
// can durably distinguish a retry-safe precondition failure from an ordinary
|
|
// backend error whose effect is unknown.
|
|
type NotAppliedError struct {
|
|
Cause error
|
|
}
|
|
|
|
func (e *NotAppliedError) Error() string {
|
|
if e == nil || e.Cause == nil {
|
|
return "operation was not applied"
|
|
}
|
|
return e.Cause.Error()
|
|
}
|
|
|
|
func (e *NotAppliedError) Unwrap() error {
|
|
if e == nil {
|
|
return nil
|
|
}
|
|
return e.Cause
|
|
}
|
|
|
|
func (*NotAppliedError) FailureSemantics() (bool, bool) { return false, true }
|
|
|
|
// MarkNotApplied must only be used before a caller-specific mutation boundary.
|
|
// An already-uncertain error is never downgraded, even if it is accidentally
|
|
// passed through a precondition classification helper.
|
|
func MarkNotApplied(err error) error {
|
|
if err == nil || hasUncertainMarker(err) {
|
|
return err
|
|
}
|
|
return &NotAppliedError{Cause: err}
|
|
}
|
|
|
|
func IsNotApplied(err error) bool {
|
|
var target *NotAppliedError
|
|
return errors.As(err, &target)
|
|
}
|
|
|
|
type recordAlias Record
|
|
|
|
type storedRecord struct {
|
|
Record recordAlias `json:"record"`
|
|
ExpectedState *ExpectedState `json:"expected_state,omitempty"`
|
|
LockKeys []string `json:"lock_keys,omitempty"`
|
|
HeavyIO bool `json:"heavy_io,omitempty"`
|
|
SupersedesOperationID string `json:"supersedes_operation_id,omitempty"`
|
|
}
|
|
|
|
type resourceLock struct {
|
|
semaphore chan struct{}
|
|
references int
|
|
}
|
|
|
|
type Journal struct {
|
|
path string
|
|
db *bolt.DB
|
|
|
|
clockMu sync.RWMutex
|
|
now func() time.Time
|
|
|
|
resourceMu sync.Mutex
|
|
resources map[string]*resourceLock
|
|
|
|
admissionClaimMu sync.Mutex
|
|
admissionClaims map[string]struct{}
|
|
|
|
executionSlots chan struct{}
|
|
heavyIOSlots chan struct{}
|
|
workerWG sync.WaitGroup
|
|
|
|
// backgroundFailures carries the first durable finalization failure to the
|
|
// process supervisor. ExecuteBackground has already crossed the dispatch
|
|
// boundary when these failures occur, so the mutation must never be
|
|
// replayed in-process. The main Agent exits cleanly and startup recovery
|
|
// imports an immutable helper receipt, or terminally records an unconfirmed
|
|
// result when no receipt exists.
|
|
backgroundFailures chan error
|
|
}
|
|
|
|
const (
|
|
maxConcurrentExecutions = 32
|
|
DefaultHeavyIOConcurrency = 4
|
|
maxAdmissionResourceKeys = 16
|
|
maximumAdmissionKeyLength = 255
|
|
)
|
|
|
|
func Open(path string) (*Journal, error) {
|
|
return OpenWithHeavyIOConcurrency(path, DefaultHeavyIOConcurrency)
|
|
}
|
|
|
|
func OpenWithHeavyIOConcurrency(path string, heavyIOConcurrency int) (*Journal, error) {
|
|
if strings.TrimSpace(path) == "" {
|
|
return nil, errors.New("journal path is required")
|
|
}
|
|
if heavyIOConcurrency < 1 {
|
|
return nil, errors.New("heavy I/O concurrency must be positive")
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
|
return nil, fmt.Errorf("create operation journal directory: %w", err)
|
|
}
|
|
db, err := bolt.Open(path, 0o600, &bolt.Options{Timeout: 2 * time.Second})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open operation journal database: %w", err)
|
|
}
|
|
j := &Journal{
|
|
path: path, db: db, now: time.Now, resources: make(map[string]*resourceLock),
|
|
admissionClaims: make(map[string]struct{}),
|
|
executionSlots: make(chan struct{}, maxConcurrentExecutions),
|
|
heavyIOSlots: make(chan struct{}, heavyIOConcurrency),
|
|
backgroundFailures: make(chan error, 1),
|
|
}
|
|
if err := j.initialize(); err != nil {
|
|
_ = db.Close()
|
|
return nil, err
|
|
}
|
|
return j, nil
|
|
}
|
|
|
|
func (j *Journal) Close() error {
|
|
if j == nil || j.db == nil {
|
|
return nil
|
|
}
|
|
return j.db.Close()
|
|
}
|
|
|
|
// Wait blocks until every real mutation and late-result finalizer has stopped.
|
|
// Callers must stop API admission before invoking it so no new worker can be
|
|
// added concurrently with the wait.
|
|
func (j *Journal) Wait(ctx context.Context) error {
|
|
done := make(chan struct{})
|
|
go func() {
|
|
j.workerWG.Wait()
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
// BackgroundFailures reports an unrecoverable failure to durably finalize an
|
|
// already-dispatched background operation. Callers must stop request
|
|
// admission and restart the Agent; FinalizeInterruptedOperations then settles
|
|
// the dispatching record without invoking the mutation again.
|
|
func (j *Journal) BackgroundFailures() <-chan error {
|
|
return j.backgroundFailures
|
|
}
|
|
|
|
func (j *Journal) reportBackgroundFailure(operationID, phase string, err error) {
|
|
if err == nil {
|
|
return
|
|
}
|
|
failure := fmt.Errorf("%s background operation %s: %w", phase, operationID, err)
|
|
select {
|
|
case j.backgroundFailures <- failure:
|
|
default:
|
|
// The first failure is sufficient to force a supervised restart. Never
|
|
// block finalizers while the process is already shutting down.
|
|
}
|
|
}
|
|
|
|
// UseNonce uses two Bolt buckets: direct lookup is O(1), while the ordered
|
|
// expiry index makes cleanup proportional only to expired entries. It never
|
|
// rewrites or decodes the operation history.
|
|
func (j *Journal) UseNonce(nonce string, now, expiresAt time.Time) (bool, error) {
|
|
if !noncePattern.MatchString(nonce) {
|
|
return false, errors.New("invalid nonce")
|
|
}
|
|
if !expiresAt.After(now) {
|
|
return false, errors.New("nonce expiry must be in the future")
|
|
}
|
|
accepted := false
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
nonces := tx.Bucket(noncesBucket)
|
|
expiry := tx.Bucket(nonceExpiryBucket)
|
|
if nonces == nil || expiry == nil {
|
|
return errors.New("nonce buckets are unavailable")
|
|
}
|
|
if err := pruneExpiredOperationDetails(tx, now); err != nil {
|
|
return err
|
|
}
|
|
pruneExpiredNonces(nonces, expiry, now)
|
|
key := []byte(nonce)
|
|
if encoded := nonces.Get(key); encoded != nil {
|
|
current, err := decodeExpiry(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !current.Before(now) {
|
|
return nil
|
|
}
|
|
if err := expiry.Delete(expiryIndexKey(current, nonce)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
encoded := encodeExpiry(expiresAt.UTC())
|
|
if err := nonces.Put(key, encoded); err != nil {
|
|
return err
|
|
}
|
|
if err := expiry.Put(expiryIndexKey(expiresAt.UTC(), nonce), nil); err != nil {
|
|
return err
|
|
}
|
|
accepted = true
|
|
return nil
|
|
})
|
|
return accepted, err
|
|
}
|
|
|
|
func Fingerprint(method, target string, body []byte) string {
|
|
sum := sha256.Sum256([]byte(strings.ToUpper(method) + "\n" + target + "\n" + string(body)))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func (j *Journal) SetClockForTesting(now func() time.Time) {
|
|
j.clockMu.Lock()
|
|
defer j.clockMu.Unlock()
|
|
j.now = now
|
|
}
|
|
|
|
func (j *Journal) currentTime() time.Time {
|
|
j.clockMu.RLock()
|
|
defer j.clockMu.RUnlock()
|
|
return j.now().UTC()
|
|
}
|
|
|
|
func (j *Journal) Get(operationID string) (Record, error) {
|
|
var record Record
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(operationsBucket)
|
|
if bucket == nil {
|
|
return errors.New("operation bucket is unavailable")
|
|
}
|
|
value := bucket.Get([]byte(operationID))
|
|
if value == nil {
|
|
return ErrNotFound
|
|
}
|
|
decoded, err := decodeRecord(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
record = decoded
|
|
return nil
|
|
})
|
|
return cloneRecord(record), err
|
|
}
|
|
|
|
// ReadTerminalRecord inspects an isolated worker Journal without initializing,
|
|
// migrating, or otherwise writing it. It is used only during parent startup,
|
|
// after a worker may have committed its terminal operation row but before it
|
|
// managed to publish completion.json.
|
|
func ReadTerminalRecord(path, operationID string) (Record, error) {
|
|
if strings.TrimSpace(path) == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path ||
|
|
!operationIDPattern.MatchString(operationID) {
|
|
return Record{}, ErrTerminalEvidenceInvalid
|
|
}
|
|
db, err := bolt.Open(path, 0o600, &bolt.Options{ReadOnly: true, Timeout: terminalEvidenceOpenTimeout})
|
|
if err != nil {
|
|
if invalidTerminalEvidenceDatabaseError(err) {
|
|
return Record{}, fmt.Errorf("%w: %v", ErrTerminalEvidenceInvalid, err)
|
|
}
|
|
return Record{}, err
|
|
}
|
|
defer db.Close()
|
|
|
|
var record Record
|
|
err = db.View(func(tx *bolt.Tx) error {
|
|
metadata := tx.Bucket(metadataBucket)
|
|
operations := tx.Bucket(operationsBucket)
|
|
if metadata == nil || operations == nil || !bytes.Equal(metadata.Get(schemaVersionKey), schemaVersion) {
|
|
return ErrTerminalEvidenceInvalid
|
|
}
|
|
encoded := operations.Get([]byte(operationID))
|
|
if encoded == nil {
|
|
return ErrTerminalEvidencePending
|
|
}
|
|
decoded, decodeErr := decodeRecord(encoded)
|
|
if decodeErr != nil {
|
|
return fmt.Errorf("%w: %v", ErrTerminalEvidenceInvalid, decodeErr)
|
|
}
|
|
if !terminalState(decoded.State) {
|
|
return ErrTerminalEvidencePending
|
|
}
|
|
record = decoded
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
if invalidTerminalEvidenceDatabaseError(err) {
|
|
return Record{}, fmt.Errorf("%w: %v", ErrTerminalEvidenceInvalid, err)
|
|
}
|
|
return Record{}, err
|
|
}
|
|
return cloneRecord(record), nil
|
|
}
|
|
|
|
func invalidTerminalEvidenceDatabaseError(err error) bool {
|
|
return errors.Is(err, bolt.ErrInvalid) || errors.Is(err, bolt.ErrVersionMismatch) || errors.Is(err, bolt.ErrChecksum)
|
|
}
|
|
|
|
// GetForInspection returns the immutable Journal record and, when retention
|
|
// has removed its result (or the terminal result was empty), reconstructs only
|
|
// an in-memory copy from the operation's immutable completion receipt. Receipt
|
|
// anomalies are represented on the returned copy so inspection can stay
|
|
// inconclusive; this method never rewrites the Journal, receipt, or outbox.
|
|
func (j *Journal) GetForInspection(operationID string) (Record, error) {
|
|
var record Record
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
if operations == nil {
|
|
return errors.New("operation bucket is unavailable")
|
|
}
|
|
value := operations.Get([]byte(operationID))
|
|
if value == nil {
|
|
return ErrNotFound
|
|
}
|
|
decoded, err := decodeRecord(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
record = decoded
|
|
if !terminalState(record.State) || (!record.Tombstone && len(record.Result) != 0) {
|
|
return nil
|
|
}
|
|
|
|
// Treat every receipt anomaly as incomplete evidence. The caller still
|
|
// receives the immutable operation identity and terminal status, but no
|
|
// receipt bytes are trusted or persisted back into the Journal.
|
|
record.InspectionEvidenceUnavailable = true
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
if receipts == nil {
|
|
return nil
|
|
}
|
|
eventID := completionEventID(operationID)
|
|
encoded := receipts.Get([]byte(eventID))
|
|
if encoded == nil {
|
|
return nil
|
|
}
|
|
var receipt CompletionReceipt
|
|
if err := json.Unmarshal(encoded, &receipt); err != nil ||
|
|
!completionReceiptMatchesRecordForInspection(eventID, operationID, receipt, record) {
|
|
return nil
|
|
}
|
|
record.Result = append(json.RawMessage(nil), receipt.Result...)
|
|
record.InspectionReceiptRecovered = true
|
|
record.InspectionEvidenceUnavailable = false
|
|
return nil
|
|
})
|
|
return cloneRecord(record), err
|
|
}
|
|
|
|
func completionReceiptMatchesRecordForInspection(eventID, requestedOperationID string,
|
|
receipt CompletionReceipt, record Record) bool {
|
|
if record.OperationID != requestedOperationID || eventID != completionEventID(requestedOperationID) ||
|
|
receipt.EventID != eventID || receipt.OperationID != requestedOperationID ||
|
|
!ValidOperationIdentity(record.Action, record.ResourceID, record.Fingerprint) ||
|
|
!ValidOperationIdentity(receipt.Action, receipt.ResourceID, receipt.RequestFingerprint) ||
|
|
receipt.Action != record.Action || receipt.ResourceID != record.ResourceID ||
|
|
receipt.RequestFingerprint != record.Fingerprint ||
|
|
receipt.EffectMayHaveApplied != record.EffectMayHaveApplied ||
|
|
receipt.SafeToResubmit != record.SafeToResubmit ||
|
|
!completionCauseMatchesRecord(receipt, record) ||
|
|
len(receipt.Result) == 0 || !json.Valid(receipt.Result) {
|
|
return false
|
|
}
|
|
|
|
recordErrorCode := ""
|
|
if record.Error != nil {
|
|
recordErrorCode = record.Error.Code
|
|
}
|
|
switch record.State {
|
|
case StateSucceeded:
|
|
return receipt.Status == "succeeded" && receipt.ErrorCode == "" &&
|
|
recordErrorCode == "" && !receipt.EffectMayHaveApplied && !receipt.SafeToResubmit
|
|
case StateFailed:
|
|
return receipt.Status == "failed" && receipt.ErrorCode != "" &&
|
|
validCauseCode(receipt.ErrorCode) && receipt.ErrorCode == recordErrorCode
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func ValidateFenceResolutionRequest(request FenceResolutionRequest) error {
|
|
if !fenceResolutionIDPattern.MatchString(request.ResolutionID) {
|
|
return errors.New("resolution_id must be 32 lowercase hexadecimal characters")
|
|
}
|
|
if request.Resolution != FenceResolutionApplied && request.Resolution != FenceResolutionNotApplied {
|
|
return errors.New("resolution must be applied or not_applied")
|
|
}
|
|
if request.InspectionCheckedAt < 1 {
|
|
return errors.New("inspection_checked_at must be a positive Unix timestamp")
|
|
}
|
|
if !hexDigestPattern.MatchString(request.ReasonSHA256) {
|
|
return errors.New("reason_sha256 must be a 64-character lowercase SHA-256 digest")
|
|
}
|
|
if !ValidOperationIdentity(request.Action, request.ResourceID, request.RequestFingerprint) {
|
|
return errors.New("fence resolution operation identity is invalid")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// GetFenceResolution reads immutable, bounded audit metadata without touching
|
|
// operation state or host resources.
|
|
func (j *Journal) GetFenceResolution(operationID string) (FenceResolution, error) {
|
|
if !operationIDPattern.MatchString(operationID) {
|
|
return FenceResolution{}, ErrNotFound
|
|
}
|
|
var resolution FenceResolution
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(fenceResolutionsBucket)
|
|
if bucket == nil {
|
|
return errors.New("fence resolution bucket is unavailable")
|
|
}
|
|
encoded := bucket.Get([]byte(operationID))
|
|
if encoded == nil {
|
|
return ErrNotFound
|
|
}
|
|
if err := json.Unmarshal(encoded, &resolution); err != nil || resolution.OperationID != operationID {
|
|
return fmt.Errorf("decode fence resolution %q", operationID)
|
|
}
|
|
return nil
|
|
})
|
|
return resolution, err
|
|
}
|
|
|
|
// FenceResolutionRequired reports whether the immutable operation currently
|
|
// owns every admission-index entry for an unresolved resource fence. A stored
|
|
// resolution, a tombstone, and non-blocking kinds such as configure-storage
|
|
// always return false. Inconsistent indexes fail closed instead of telling a
|
|
// caller to skip the fence protocol.
|
|
func (j *Journal) FenceState(operationID string) (FenceState, error) {
|
|
if !operationIDPattern.MatchString(operationID) {
|
|
return "", ErrNotFound
|
|
}
|
|
state := FenceStateNotRequired
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
resolutions := tx.Bucket(fenceResolutionsBucket)
|
|
if operations == nil || blockers == nil || resolutions == nil {
|
|
return errors.New("operation fence buckets are unavailable")
|
|
}
|
|
encoded := operations.Get([]byte(operationID))
|
|
if encoded == nil {
|
|
return ErrNotFound
|
|
}
|
|
record, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if encodedResolution := resolutions.Get([]byte(operationID)); encodedResolution != nil {
|
|
var resolution FenceResolution
|
|
if err := json.Unmarshal(encodedResolution, &resolution); err != nil ||
|
|
resolution.OperationID != operationID {
|
|
return fmt.Errorf("decode fence resolution %q", operationID)
|
|
}
|
|
state = FenceStateResolved
|
|
return nil
|
|
}
|
|
if !blocksResources(record) {
|
|
return nil
|
|
}
|
|
for _, lockKey := range effectiveLockKeys(
|
|
record.OperationID,
|
|
record.ResourceID,
|
|
record.Action,
|
|
record.LockKeys,
|
|
) {
|
|
key := blockerEntryKey(lockKey, operationID)
|
|
foundKey, _ := blockers.Cursor().Seek(key)
|
|
if !bytes.Equal(foundKey, key) {
|
|
return fmt.Errorf(
|
|
"operation %q is missing its resource blocker index",
|
|
operationID,
|
|
)
|
|
}
|
|
}
|
|
state = FenceStateActive
|
|
return nil
|
|
})
|
|
return state, err
|
|
}
|
|
|
|
func (j *Journal) FenceResolutionRequired(operationID string) (bool, error) {
|
|
state, err := j.FenceState(operationID)
|
|
return state == FenceStateActive, err
|
|
}
|
|
|
|
// ResolveResourceFence records one explicit, audited effect conclusion and
|
|
// removes only this operation's admission-index entries. resolvedBy zero is
|
|
// reserved for the API's operation-bound system_recovery actor; human callers
|
|
// must retain their positive user ID. It never mutates the operation record or
|
|
// completion receipt and never invokes a host mutation.
|
|
func (j *Journal) ResolveResourceFence(operationID string, resolvedBy int64,
|
|
request FenceResolutionRequest) (FenceResolution, bool, error) {
|
|
if !operationIDPattern.MatchString(operationID) {
|
|
return FenceResolution{}, false, ErrNotFound
|
|
}
|
|
if resolvedBy < 0 {
|
|
return FenceResolution{}, false, errors.New("resolved_by must be a non-negative actor ID")
|
|
}
|
|
if err := ValidateFenceResolutionRequest(request); err != nil {
|
|
return FenceResolution{}, false, err
|
|
}
|
|
now := j.currentTime().UTC()
|
|
var resolution FenceResolution
|
|
replayed := false
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
resolutions := tx.Bucket(fenceResolutionsBucket)
|
|
resolutionIDs := tx.Bucket(fenceResolutionIDsBucket)
|
|
if operations == nil || blockers == nil || resolutions == nil || resolutionIDs == nil {
|
|
return errors.New("operation fence buckets are unavailable")
|
|
}
|
|
encodedRecord := operations.Get([]byte(operationID))
|
|
if encodedRecord == nil {
|
|
checkedAt := time.Unix(request.InspectionCheckedAt, 0).UTC()
|
|
if checkedAt.Before(now.Add(-fenceInspectionMaximumAge)) || checkedAt.After(now.Add(fenceInspectionFutureSkew)) {
|
|
return ErrFenceInspectionStale
|
|
}
|
|
return ErrNotFound
|
|
}
|
|
record, err := decodeRecord(encodedRecord)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !fenceResolutionIdentityMatches(request, record) {
|
|
return ErrFenceIdentityConflict
|
|
}
|
|
// The zero audit actor is a deliberately narrow internal capability.
|
|
// Only an action/resolution pair explicitly granted by the embedded
|
|
// contract may use it; human administrators retain their positive
|
|
// audited identity.
|
|
if resolvedBy == 0 && !SystemRecoveryResolutionAllowed(record.Action, request.Resolution) {
|
|
return ErrFenceNotResolvable
|
|
}
|
|
// An unreaped create worker could still create the resource after the
|
|
// inspection. Its fence therefore remains active even though the
|
|
// contract permits system not_applied for a safely reaped create.
|
|
if resolvedBy == 0 && IsCreateLikeAction(record.Action) && record.ExecutionFenced {
|
|
return ErrFenceNotResolvable
|
|
}
|
|
if encoded := resolutions.Get([]byte(operationID)); encoded != nil {
|
|
if err := json.Unmarshal(encoded, &resolution); err != nil || resolution.OperationID != operationID {
|
|
return fmt.Errorf("decode fence resolution %q", operationID)
|
|
}
|
|
if resolution.ResolutionID != request.ResolutionID || resolution.Resolution != request.Resolution ||
|
|
resolution.InspectionCheckedAt != request.InspectionCheckedAt || resolution.ReasonSHA256 != request.ReasonSHA256 ||
|
|
resolution.ResolvedBy != resolvedBy {
|
|
return ErrFenceResolutionConflict
|
|
}
|
|
replayed = true
|
|
return nil
|
|
}
|
|
if owner := resolutionIDs.Get([]byte(request.ResolutionID)); owner != nil {
|
|
return ErrFenceResolutionConflict
|
|
}
|
|
checkedAt := time.Unix(request.InspectionCheckedAt, 0).UTC()
|
|
if checkedAt.Before(now.Add(-fenceInspectionMaximumAge)) || checkedAt.After(now.Add(fenceInspectionFutureSkew)) {
|
|
return ErrFenceInspectionStale
|
|
}
|
|
if !blocksResources(record) {
|
|
return ErrFenceNotResolvable
|
|
}
|
|
lockKeys := effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys)
|
|
for _, lockKey := range lockKeys {
|
|
key := blockerEntryKey(lockKey, operationID)
|
|
cursor := blockers.Cursor()
|
|
foundKey, _ := cursor.Seek(key)
|
|
if !bytes.Equal(foundKey, key) {
|
|
return ErrFenceNotResolvable
|
|
}
|
|
}
|
|
resolution = FenceResolution{
|
|
OperationID: operationID, ResolutionID: request.ResolutionID,
|
|
Resolution: request.Resolution, InspectionCheckedAt: request.InspectionCheckedAt,
|
|
ReasonSHA256: request.ReasonSHA256, ResolvedAt: now.Unix(), ResolvedBy: resolvedBy,
|
|
}
|
|
resolutionEncoded, err := json.Marshal(resolution)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := resolutions.Put([]byte(operationID), resolutionEncoded); err != nil {
|
|
return err
|
|
}
|
|
if err := resolutionIDs.Put([]byte(request.ResolutionID), []byte(operationID)); err != nil {
|
|
return err
|
|
}
|
|
for _, lockKey := range lockKeys {
|
|
if err := blockers.Delete(blockerEntryKey(lockKey, operationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
return resolution, replayed, err
|
|
}
|
|
|
|
func fenceResolutionIdentityMatches(request FenceResolutionRequest, record Record) bool {
|
|
return request.Action == record.Action && request.ResourceID == record.ResourceID &&
|
|
request.RequestFingerprint == record.Fingerprint
|
|
}
|
|
|
|
// ClaimPendingCompletions returns a bounded batch and durably schedules every
|
|
// returned event's next delivery. A completion remains in the outbox until an
|
|
// ACK removes it; delivery age and attempt count never discard it.
|
|
func (j *Journal) ClaimPendingCompletions(limit int) ([]CompletionReceipt, error) {
|
|
claimed, _, err := j.ClaimPendingCompletionsWithExpired(limit)
|
|
return claimed, err
|
|
}
|
|
|
|
// ClaimPendingCompletionsWithExpired retains its historical return shape for
|
|
// callers compiled against the bounded-outbox API. Completion delivery no
|
|
// longer expires, so the second result is always empty.
|
|
func (j *Journal) ClaimPendingCompletionsWithExpired(limit int) ([]CompletionReceipt, []string, error) {
|
|
return j.ClaimPendingCompletionsWithExpiredExcluding(limit, nil)
|
|
}
|
|
|
|
// ClaimPendingCompletionsWithExpiredExcluding leaves events already in the
|
|
// current control session's ACK window untouched. An in-flight receipt cannot
|
|
// move its durable resend time merely because that time arrived while the
|
|
// Gateway was still processing the first delivery. The historical "expired"
|
|
// result is retained for API compatibility and is always empty.
|
|
func (j *Journal) ClaimPendingCompletionsWithExpiredExcluding(limit int, excluded []string) ([]CompletionReceipt, []string, error) {
|
|
if limit < 1 || limit > 128 {
|
|
return nil, nil, errors.New("completion claim limit must be between 1 and 128")
|
|
}
|
|
excludedEvents := make(map[string]struct{}, len(excluded))
|
|
for _, eventID := range excluded {
|
|
if !validCompletionEventID(eventID) {
|
|
return nil, nil, errors.New("invalid excluded completion event ID")
|
|
}
|
|
excludedEvents[eventID] = struct{}{}
|
|
}
|
|
claimed := make([]CompletionReceipt, 0, limit)
|
|
newIncidents := make([]CompletionDeliveryIncident, 0, 1)
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
quarantine := tx.Bucket(completionQuarantineBucket)
|
|
quarantineOperations := tx.Bucket(completionQuarantineOperationsBucket)
|
|
if operations == nil || receipts == nil || outbox == nil || quarantine == nil || quarantineOperations == nil {
|
|
return errors.New("completion buckets are unavailable")
|
|
}
|
|
now := j.currentTime()
|
|
cursor := outbox.Cursor()
|
|
for key, encoded := cursor.First(); key != nil && len(claimed) < limit; key, encoded = cursor.Next() {
|
|
receiptEncoded := receipts.Get(key)
|
|
operationID := completionIncidentOperationID(key, encoded, receiptEncoded)
|
|
var pending completionOutboxRecord
|
|
// Older records may contain attempts and expires_at fields. JSON
|
|
// decoding deliberately ignores them so an upgrade resumes delivery
|
|
// even when the legacy deadline or attempt cap was already reached.
|
|
if err := json.Unmarshal(encoded, &pending); err != nil ||
|
|
!validCompletionEventID(string(key)) ||
|
|
!operationIDPattern.MatchString(pending.OperationID) ||
|
|
string(key) != completionEventID(pending.OperationID) || pending.NextAttemptAt.IsZero() {
|
|
incident, changed, incidentErr := quarantineCompletionDelivery(
|
|
quarantine, quarantineOperations, key, operationID,
|
|
CompletionDeliveryErrorOutboxInvalid, now, encoded, receiptEncoded,
|
|
)
|
|
if incidentErr != nil {
|
|
return incidentErr
|
|
}
|
|
if changed {
|
|
newIncidents = append(newIncidents, incident)
|
|
}
|
|
continue
|
|
}
|
|
operationID = pending.OperationID
|
|
if _, excluded := excludedEvents[string(key)]; excluded {
|
|
continue
|
|
}
|
|
if now.Before(pending.NextAttemptAt) {
|
|
continue
|
|
}
|
|
if receiptEncoded == nil {
|
|
incident, changed, incidentErr := quarantineCompletionDelivery(
|
|
quarantine, quarantineOperations, key, operationID,
|
|
CompletionDeliveryErrorReceiptMissing, now, encoded,
|
|
)
|
|
if incidentErr != nil {
|
|
return incidentErr
|
|
}
|
|
if changed {
|
|
newIncidents = append(newIncidents, incident)
|
|
}
|
|
continue
|
|
}
|
|
var receipt CompletionReceipt
|
|
if err := json.Unmarshal(receiptEncoded, &receipt); err != nil || receipt.EventID != string(key) || receipt.OperationID != pending.OperationID {
|
|
incident, changed, incidentErr := quarantineCompletionDelivery(
|
|
quarantine, quarantineOperations, key, operationID,
|
|
CompletionDeliveryErrorReceiptInvalid, now, encoded, receiptEncoded,
|
|
)
|
|
if incidentErr != nil {
|
|
return incidentErr
|
|
}
|
|
if changed {
|
|
newIncidents = append(newIncidents, incident)
|
|
}
|
|
continue
|
|
}
|
|
recordEncoded := operations.Get([]byte(pending.OperationID))
|
|
if recordEncoded == nil {
|
|
incident, changed, incidentErr := quarantineCompletionDelivery(
|
|
quarantine, quarantineOperations, key, operationID,
|
|
CompletionDeliveryErrorOperationMissing, now, encoded, receiptEncoded,
|
|
)
|
|
if incidentErr != nil {
|
|
return incidentErr
|
|
}
|
|
if changed {
|
|
newIncidents = append(newIncidents, incident)
|
|
}
|
|
continue
|
|
}
|
|
record, err := decodeRecord(recordEncoded)
|
|
if err != nil {
|
|
incident, changed, incidentErr := quarantineCompletionDelivery(
|
|
quarantine, quarantineOperations, key, operationID,
|
|
CompletionDeliveryErrorOperationInvalid, now, encoded, receiptEncoded, recordEncoded,
|
|
)
|
|
if incidentErr != nil {
|
|
return incidentErr
|
|
}
|
|
if changed {
|
|
newIncidents = append(newIncidents, incident)
|
|
}
|
|
continue
|
|
}
|
|
if err := bindCompletionReceiptIdentity(&receipt, record); err != nil {
|
|
incident, changed, incidentErr := quarantineCompletionDelivery(
|
|
quarantine, quarantineOperations, key, operationID,
|
|
CompletionDeliveryErrorIdentityMismatch, now, encoded, receiptEncoded, recordEncoded,
|
|
)
|
|
if incidentErr != nil {
|
|
return incidentErr
|
|
}
|
|
if changed {
|
|
newIncidents = append(newIncidents, incident)
|
|
}
|
|
continue
|
|
}
|
|
if !completionReceiptMatchesRecord(receipt, record) {
|
|
incident, changed, incidentErr := quarantineCompletionDelivery(
|
|
quarantine, quarantineOperations, key, operationID,
|
|
CompletionDeliveryErrorReceiptInvalid, now, encoded, receiptEncoded, recordEncoded,
|
|
)
|
|
if incidentErr != nil {
|
|
return incidentErr
|
|
}
|
|
if changed {
|
|
newIncidents = append(newIncidents, incident)
|
|
}
|
|
continue
|
|
}
|
|
if err := clearCompletionDeliveryIncident(
|
|
quarantine, quarantineOperations, key,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
pending.NextAttemptAt = now.Add(completionDeliveryInterval)
|
|
updated, err := json.Marshal(pending)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := outbox.Put(key, updated); err != nil {
|
|
return err
|
|
}
|
|
receipt.Result = append(json.RawMessage(nil), receipt.Result...)
|
|
claimed = append(claimed, receipt)
|
|
}
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
for _, incident := range newIncidents {
|
|
log.Printf(
|
|
"completion_delivery phase=quarantined operation_id=%s event_id=%s event_key_sha256=%s error_code=%s",
|
|
incident.OperationID, incident.EventID, incident.EventKeySHA256, incident.ErrorCode,
|
|
)
|
|
}
|
|
}
|
|
return claimed, []string{}, err
|
|
}
|
|
|
|
func bindCompletionReceiptIdentity(receipt *CompletionReceipt, record Record) error {
|
|
if receipt == nil || !ValidOperationIdentity(record.Action, record.ResourceID, record.Fingerprint) {
|
|
return errors.New("Journal operation identity is invalid")
|
|
}
|
|
if receipt.Action == "" && receipt.ResourceID == "" && receipt.RequestFingerprint == "" {
|
|
// Compatibility for immutable receipts created before operation identity
|
|
// was added to the wire contract. The returned copy is enriched from the
|
|
// same immutable Journal record; the stored receipt bytes are untouched.
|
|
receipt.Action = record.Action
|
|
receipt.ResourceID = record.ResourceID
|
|
receipt.RequestFingerprint = record.Fingerprint
|
|
return nil
|
|
}
|
|
if !ValidOperationIdentity(receipt.Action, receipt.ResourceID, receipt.RequestFingerprint) ||
|
|
receipt.Action != record.Action || receipt.ResourceID != record.ResourceID ||
|
|
receipt.RequestFingerprint != record.Fingerprint {
|
|
return errors.New("completion identity does not match Journal operation")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func completionIncidentOperationID(eventKey, outboxEncoded, receiptEncoded []byte) string {
|
|
// Prefer an internally consistent immutable receipt over the mutable retry
|
|
// marker. This still recovers the affected operation when only the outbox
|
|
// bytes or key were damaged, without letting an arbitrary operation_id in a
|
|
// malformed receipt surface on the administrator inspection path.
|
|
var receiptIdentity struct {
|
|
EventID string `json:"id"`
|
|
OperationID string `json:"operation_id"`
|
|
}
|
|
receiptIdentityValid := json.Unmarshal(receiptEncoded, &receiptIdentity) == nil &&
|
|
operationIDPattern.MatchString(receiptIdentity.OperationID) &&
|
|
receiptIdentity.EventID == completionEventID(receiptIdentity.OperationID)
|
|
|
|
var pendingIdentity struct {
|
|
OperationID string `json:"operation_id"`
|
|
}
|
|
pendingIdentityValid := json.Unmarshal(outboxEncoded, &pendingIdentity) == nil &&
|
|
operationIDPattern.MatchString(pendingIdentity.OperationID)
|
|
|
|
if receiptIdentityValid && receiptIdentity.EventID == string(eventKey) {
|
|
return receiptIdentity.OperationID
|
|
}
|
|
if pendingIdentityValid && completionEventID(pendingIdentity.OperationID) == string(eventKey) {
|
|
return pendingIdentity.OperationID
|
|
}
|
|
if receiptIdentityValid {
|
|
return receiptIdentity.OperationID
|
|
}
|
|
if pendingIdentityValid {
|
|
return pendingIdentity.OperationID
|
|
}
|
|
|
|
// A valid event key alone is not reversible to an operation ID. It remains
|
|
// part of the durable evidence digest for an unassociated Agent incident.
|
|
return ""
|
|
}
|
|
|
|
func validCompletionDeliveryErrorCode(code string) bool {
|
|
switch code {
|
|
case CompletionDeliveryErrorOutboxInvalid,
|
|
CompletionDeliveryErrorReceiptMissing,
|
|
CompletionDeliveryErrorReceiptInvalid,
|
|
CompletionDeliveryErrorOperationMissing,
|
|
CompletionDeliveryErrorOperationInvalid,
|
|
CompletionDeliveryErrorIdentityMismatch:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func completionDeliveryDigest(eventKey []byte, errorCode string, evidence ...[]byte) string {
|
|
hash := sha256.New()
|
|
writePart := func(value []byte) {
|
|
var size [8]byte
|
|
binary.BigEndian.PutUint64(size[:], uint64(len(value)))
|
|
_, _ = hash.Write(size[:])
|
|
_, _ = hash.Write(value)
|
|
}
|
|
writePart([]byte("completion-delivery-incident-v1"))
|
|
writePart(eventKey)
|
|
writePart([]byte(errorCode))
|
|
for _, value := range evidence {
|
|
writePart(value)
|
|
}
|
|
return hex.EncodeToString(hash.Sum(nil))
|
|
}
|
|
|
|
func quarantineCompletionDelivery(quarantine, operationIndex *bolt.Bucket, eventKey []byte,
|
|
operationID, errorCode string, observedAt time.Time, evidence ...[]byte) (CompletionDeliveryIncident, bool, error) {
|
|
if quarantine == nil || operationIndex == nil {
|
|
return CompletionDeliveryIncident{}, false, errors.New("completion quarantine buckets are unavailable")
|
|
}
|
|
if !validCompletionDeliveryErrorCode(errorCode) {
|
|
return CompletionDeliveryIncident{}, false, errors.New("invalid completion delivery incident code")
|
|
}
|
|
if !operationIDPattern.MatchString(operationID) {
|
|
operationID = ""
|
|
}
|
|
observedAt = observedAt.UTC()
|
|
if observedAt.IsZero() {
|
|
return CompletionDeliveryIncident{}, false, errors.New("completion delivery incident time is invalid")
|
|
}
|
|
eventID := ""
|
|
if validCompletionEventID(string(eventKey)) {
|
|
eventID = string(eventKey)
|
|
}
|
|
eventKeySum := sha256.Sum256(eventKey)
|
|
incident := CompletionDeliveryIncident{
|
|
EventID: eventID,
|
|
EventKeySHA256: hex.EncodeToString(eventKeySum[:]),
|
|
OperationID: operationID,
|
|
ErrorCode: errorCode,
|
|
EvidenceSHA256: completionDeliveryDigest(eventKey, errorCode, evidence...),
|
|
FirstObservedAt: observedAt,
|
|
}
|
|
|
|
changed := true
|
|
if encoded := quarantine.Get(eventKey); encoded != nil {
|
|
var existing CompletionDeliveryIncident
|
|
if json.Unmarshal(encoded, &existing) == nil &&
|
|
existing.EventID == incident.EventID &&
|
|
existing.EventKeySHA256 == incident.EventKeySHA256 &&
|
|
existing.OperationID == incident.OperationID &&
|
|
existing.ErrorCode == incident.ErrorCode &&
|
|
existing.EvidenceSHA256 == incident.EvidenceSHA256 &&
|
|
!existing.FirstObservedAt.IsZero() {
|
|
incident.FirstObservedAt = existing.FirstObservedAt.UTC()
|
|
changed = false
|
|
}
|
|
}
|
|
|
|
// Remove every stale reverse-index entry for this raw event key. This also
|
|
// repairs an index left behind by corrupt incident JSON or by a changed
|
|
// best-effort operation association.
|
|
indexCursor := operationIndex.Cursor()
|
|
for key, value := indexCursor.First(); key != nil; key, value = indexCursor.Next() {
|
|
if bytes.Equal(value, eventKey) && (operationID == "" || string(key) != operationID) {
|
|
if err := indexCursor.Delete(); err != nil {
|
|
return CompletionDeliveryIncident{}, false, err
|
|
}
|
|
}
|
|
}
|
|
|
|
encoded, err := json.Marshal(incident)
|
|
if err != nil {
|
|
return CompletionDeliveryIncident{}, false, err
|
|
}
|
|
if err := quarantine.Put(eventKey, encoded); err != nil {
|
|
return CompletionDeliveryIncident{}, false, err
|
|
}
|
|
if operationID != "" {
|
|
indexed := operationIndex.Get([]byte(operationID))
|
|
if !bytes.Equal(indexed, eventKey) {
|
|
changed = true
|
|
}
|
|
if err := operationIndex.Put([]byte(operationID), eventKey); err != nil {
|
|
return CompletionDeliveryIncident{}, false, err
|
|
}
|
|
}
|
|
return incident, changed, nil
|
|
}
|
|
|
|
func clearCompletionDeliveryIncident(quarantine, operationIndex *bolt.Bucket, eventKey []byte) error {
|
|
if quarantine == nil || operationIndex == nil {
|
|
return errors.New("completion quarantine buckets are unavailable")
|
|
}
|
|
if err := quarantine.Delete(eventKey); err != nil {
|
|
return err
|
|
}
|
|
cursor := operationIndex.Cursor()
|
|
for key, value := cursor.First(); key != nil; key, value = cursor.Next() {
|
|
if bytes.Equal(value, eventKey) {
|
|
if err := cursor.Delete(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CompletionDeliveryIncident returns a persistent delivery-only incident for
|
|
// administrator inspection. It never changes the immutable operation status,
|
|
// receipt, or outbox entry.
|
|
func (j *Journal) CompletionDeliveryIncident(operationID string) (CompletionDeliveryIncident, bool, error) {
|
|
if !operationIDPattern.MatchString(operationID) {
|
|
return CompletionDeliveryIncident{}, false, ErrNotFound
|
|
}
|
|
var incident CompletionDeliveryIncident
|
|
found := false
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
quarantine := tx.Bucket(completionQuarantineBucket)
|
|
operationIndex := tx.Bucket(completionQuarantineOperationsBucket)
|
|
if quarantine == nil || operationIndex == nil {
|
|
return errors.New("completion quarantine buckets are unavailable")
|
|
}
|
|
eventKey := operationIndex.Get([]byte(operationID))
|
|
derivedAssociation := false
|
|
if eventKey == nil {
|
|
// The event ID is a deterministic digest of the operation ID. If both
|
|
// the outbox and immutable receipt JSON are damaged, quarantine cannot
|
|
// recover an operation ID to populate the reverse index. The exact
|
|
// deterministic key still safely associates that detected corruption
|
|
// with this requested operation without scanning either bucket.
|
|
eventKey = []byte(completionEventID(operationID))
|
|
if quarantine.Get(eventKey) == nil {
|
|
return nil
|
|
}
|
|
derivedAssociation = true
|
|
}
|
|
encoded := quarantine.Get(eventKey)
|
|
if encoded == nil {
|
|
return errors.New("completion quarantine operation index is inconsistent")
|
|
}
|
|
if err := json.Unmarshal(encoded, &incident); err != nil {
|
|
return fmt.Errorf("decode completion delivery incident: %w", err)
|
|
}
|
|
eventKeySum := sha256.Sum256(eventKey)
|
|
operationMatches := incident.OperationID == operationID ||
|
|
(derivedAssociation && incident.OperationID == "" &&
|
|
incident.EventID == string(eventKey))
|
|
if !operationMatches ||
|
|
!validCompletionDeliveryErrorCode(incident.ErrorCode) ||
|
|
!hexDigestPattern.MatchString(incident.EventKeySHA256) ||
|
|
incident.EventKeySHA256 != hex.EncodeToString(eventKeySum[:]) ||
|
|
!hexDigestPattern.MatchString(incident.EvidenceSHA256) ||
|
|
incident.FirstObservedAt.IsZero() ||
|
|
(incident.EventID != "" && (incident.EventID != string(eventKey) || !validCompletionEventID(incident.EventID))) {
|
|
return errors.New("completion delivery incident is invalid")
|
|
}
|
|
if derivedAssociation {
|
|
// Bind only the returned copy. Repairing the receipt/outbox must still
|
|
// clear the original quarantine entry through its immutable event key;
|
|
// inspection must never rewrite corruption evidence.
|
|
incident.OperationID = operationID
|
|
}
|
|
found = true
|
|
return nil
|
|
})
|
|
return incident, found, err
|
|
}
|
|
|
|
func completionReceiptMatchesRecord(receipt CompletionReceipt, record Record) bool {
|
|
if receipt.EventID != completionEventID(record.OperationID) ||
|
|
receipt.OperationID != record.OperationID ||
|
|
!ValidOperationIdentity(receipt.Action, receipt.ResourceID, receipt.RequestFingerprint) ||
|
|
receipt.Action != record.Action || receipt.ResourceID != record.ResourceID ||
|
|
receipt.RequestFingerprint != record.Fingerprint ||
|
|
receipt.EffectMayHaveApplied != record.EffectMayHaveApplied ||
|
|
receipt.SafeToResubmit != record.SafeToResubmit ||
|
|
!completionCauseMatchesRecord(receipt, record) ||
|
|
!completionResultSafe(receipt.Result) || !completionDeliveryFitsPanelLimit(receipt) {
|
|
return false
|
|
}
|
|
|
|
recordErrorCode := ""
|
|
if record.Error != nil {
|
|
recordErrorCode = record.Error.Code
|
|
}
|
|
switch record.State {
|
|
case StateSucceeded:
|
|
if receipt.Status != "succeeded" || receipt.ErrorCode != "" || recordErrorCode != "" ||
|
|
receipt.EffectMayHaveApplied || receipt.SafeToResubmit {
|
|
return false
|
|
}
|
|
case StateFailed:
|
|
if receipt.Status != "failed" || receipt.ErrorCode == "" ||
|
|
!validCauseCode(receipt.ErrorCode) || receipt.ErrorCode != recordErrorCode ||
|
|
(receipt.EffectMayHaveApplied && receipt.SafeToResubmit) {
|
|
return false
|
|
}
|
|
default:
|
|
return false
|
|
}
|
|
|
|
// Retention intentionally removes the Journal result while preserving the
|
|
// immutable receipt. All other records must still carry the exact JSON value
|
|
// that was copied into that receipt at terminalization.
|
|
if record.Tombstone {
|
|
return true
|
|
}
|
|
return completionResultsEqual(receipt.Result, record.Result)
|
|
}
|
|
|
|
func completionCauseMatchesRecord(receipt CompletionReceipt, record Record) bool {
|
|
switch receipt.Version {
|
|
case 0:
|
|
// Legacy v1 receipts predate cause_code. They remain deliverable after
|
|
// an Agent upgrade, but cannot recover the diagnostic that was omitted.
|
|
return receipt.CauseCode == ""
|
|
case 2:
|
|
return (receipt.CauseCode == "" || validCauseCode(receipt.CauseCode)) &&
|
|
receipt.CauseCode == record.CauseCode
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func completionDeliveryFitsPanelLimit(receipt CompletionReceipt) bool {
|
|
// The Gateway adds agent_id before POSTing to PHP. Marshal the exact public
|
|
// envelope with the longest valid agent ID so every claimed receipt is
|
|
// guaranteed to fit the panel's authenticated 1 MiB request limit.
|
|
encoded, err := json.Marshal(struct {
|
|
Type string `json:"type"`
|
|
ID string `json:"id"`
|
|
AgentID string `json:"agent_id"`
|
|
OperationID string `json:"operation_id"`
|
|
Action string `json:"action"`
|
|
ResourceID string `json:"resource_id"`
|
|
RequestFingerprint string `json:"request_fingerprint"`
|
|
Status string `json:"status"`
|
|
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
|
|
SafeToResubmit bool `json:"safe_to_resubmit"`
|
|
ErrorCode string `json:"error_code"`
|
|
CauseCode string `json:"cause_code,omitempty"`
|
|
Result json.RawMessage `json:"result"`
|
|
}{
|
|
Type: "completion", ID: receipt.EventID, AgentID: "9999999999999999999",
|
|
OperationID: receipt.OperationID, Action: receipt.Action, ResourceID: receipt.ResourceID,
|
|
RequestFingerprint: receipt.RequestFingerprint, Status: receipt.Status,
|
|
EffectMayHaveApplied: receipt.EffectMayHaveApplied, SafeToResubmit: receipt.SafeToResubmit,
|
|
ErrorCode: receipt.ErrorCode, CauseCode: receipt.CauseCode, Result: receipt.Result,
|
|
})
|
|
return err == nil && len(encoded) <= completionPanelMaximumBody
|
|
}
|
|
|
|
func completionResultsEqual(receipt, record json.RawMessage) bool {
|
|
if len(record) == 0 {
|
|
return bytes.Equal(bytes.TrimSpace(receipt), []byte("null"))
|
|
}
|
|
if !json.Valid(receipt) || !json.Valid(record) {
|
|
return false
|
|
}
|
|
var compactReceipt bytes.Buffer
|
|
var compactRecord bytes.Buffer
|
|
if json.Compact(&compactReceipt, receipt) != nil || json.Compact(&compactRecord, record) != nil {
|
|
return false
|
|
}
|
|
return bytes.Equal(compactReceipt.Bytes(), compactRecord.Bytes())
|
|
}
|
|
|
|
func completionResultSafe(raw json.RawMessage) bool {
|
|
if len(raw) == 0 || !json.Valid(raw) {
|
|
return false
|
|
}
|
|
var value any
|
|
if err := json.Unmarshal(raw, &value); err != nil {
|
|
return false
|
|
}
|
|
if value == nil {
|
|
return true
|
|
}
|
|
switch value.(type) {
|
|
case map[string]any, []any:
|
|
return completionResultValueSafe(value, 0)
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func completionResultValueSafe(value any, depth int) bool {
|
|
if depth > 16 {
|
|
return false
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
for key, nested := range typed {
|
|
if completionSensitiveKeyPattern.MatchString(key) {
|
|
return false
|
|
}
|
|
if _, ok := nested.(map[string]any); ok && !completionResultValueSafe(nested, depth+1) {
|
|
return false
|
|
}
|
|
if _, ok := nested.([]any); ok && !completionResultValueSafe(nested, depth+1) {
|
|
return false
|
|
}
|
|
}
|
|
case []any:
|
|
for _, nested := range typed {
|
|
if _, ok := nested.(map[string]any); ok && !completionResultValueSafe(nested, depth+1) {
|
|
return false
|
|
}
|
|
if _, ok := nested.([]any); ok && !completionResultValueSafe(nested, depth+1) {
|
|
return false
|
|
}
|
|
}
|
|
default:
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// CompletionOperationID resolves a still-pending delivery marker without
|
|
// changing it. It lets the Agent clean disposable worker state independently
|
|
// before durably consuming an ACK; the lookup never changes the receipt or
|
|
// its delivery marker.
|
|
func (j *Journal) CompletionOperationID(eventID string) (string, error) {
|
|
if !validCompletionEventID(eventID) {
|
|
return "", errors.New("invalid completion event ID")
|
|
}
|
|
var operationID string
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if outbox == nil {
|
|
return errors.New("completion outbox is unavailable")
|
|
}
|
|
encoded := outbox.Get([]byte(eventID))
|
|
if encoded == nil {
|
|
return nil
|
|
}
|
|
var pending completionOutboxRecord
|
|
if err := json.Unmarshal(encoded, &pending); err != nil || !operationIDPattern.MatchString(pending.OperationID) {
|
|
return fmt.Errorf("decode completion outbox event %q", eventID)
|
|
}
|
|
operationID = pending.OperationID
|
|
return nil
|
|
})
|
|
return operationID, err
|
|
}
|
|
|
|
// PendingCompletionOperationIDs returns operation IDs whose terminal receipt
|
|
// still awaits an ACK. Disposable worker directories are no longer coupled to
|
|
// this set once their result has been imported into the main Journal.
|
|
func (j *Journal) PendingCompletionOperationIDs() (map[string]struct{}, error) {
|
|
operationIDs := make(map[string]struct{})
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if outbox == nil {
|
|
return errors.New("completion outbox is unavailable")
|
|
}
|
|
return outbox.ForEach(func(key, encoded []byte) error {
|
|
var pending completionOutboxRecord
|
|
if err := json.Unmarshal(encoded, &pending); err != nil || !operationIDPattern.MatchString(pending.OperationID) {
|
|
return fmt.Errorf("decode completion outbox event %q", key)
|
|
}
|
|
operationIDs[pending.OperationID] = struct{}{}
|
|
return nil
|
|
})
|
|
})
|
|
return operationIDs, err
|
|
}
|
|
|
|
// AcknowledgeCompletion only removes the delivery marker. It deliberately
|
|
// cannot update the operation or immutable completion receipt.
|
|
func (j *Journal) AcknowledgeCompletion(eventID string) error {
|
|
if !validCompletionEventID(eventID) {
|
|
return errors.New("invalid completion event ID")
|
|
}
|
|
return j.db.Update(func(tx *bolt.Tx) error {
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if outbox == nil {
|
|
return errors.New("completion outbox is unavailable")
|
|
}
|
|
return outbox.Delete([]byte(eventID))
|
|
})
|
|
}
|
|
|
|
// FinalizeInterruptedOperations is called exactly once during Agent startup,
|
|
// before request admission. It imports already-written worker receipts and
|
|
// terminally fails every other dispatching row without spawning a worker.
|
|
func (j *Journal) FinalizeInterruptedOperations(workerReceipts []InterruptedReceipt) ([]string, error) {
|
|
byOperation := make(map[string]InterruptedReceipt, len(workerReceipts))
|
|
for _, receipt := range workerReceipts {
|
|
if !operationIDPattern.MatchString(receipt.OperationID) || !hexDigestPattern.MatchString(receipt.Fingerprint) ||
|
|
(receipt.Status != "succeeded" && receipt.Status != "failed") ||
|
|
(receipt.SafeToResubmit && receipt.EffectMayHaveApplied) ||
|
|
(len(receipt.Result) != 0 && !json.Valid(receipt.Result)) {
|
|
return nil, errors.New("invalid interrupted worker receipt")
|
|
}
|
|
if receipt.Status == "failed" && (receipt.ErrorCode == "" || !validCauseCode(receipt.ErrorCode)) {
|
|
return nil, errors.New("invalid interrupted worker failure")
|
|
}
|
|
if _, duplicate := byOperation[receipt.OperationID]; duplicate {
|
|
return nil, errors.New("duplicate interrupted worker receipt")
|
|
}
|
|
byOperation[receipt.OperationID] = receipt
|
|
}
|
|
finalized := make([]string, 0)
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
inflight := tx.Bucket(inflightBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
expiry := tx.Bucket(operationExpiryBucket)
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if operations == nil || inflight == nil || blockers == nil || expiry == nil || receipts == nil || outbox == nil {
|
|
return errors.New("operation buckets are unavailable")
|
|
}
|
|
now := j.currentTime()
|
|
var keys [][]byte
|
|
if err := inflight.ForEach(func(key, _ []byte) error {
|
|
keys = append(keys, append([]byte(nil), key...))
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
for _, key := range keys {
|
|
encoded := operations.Get(key)
|
|
if encoded == nil {
|
|
return fmt.Errorf("inflight index references missing operation %q", key)
|
|
}
|
|
record, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if record.State != StateDispatching {
|
|
continue
|
|
}
|
|
previous := cloneRecord(record)
|
|
receipt, found := byOperation[record.OperationID]
|
|
if found && receipt.Fingerprint == record.Fingerprint {
|
|
record.Result = append(json.RawMessage(nil), receipt.Result...)
|
|
if receipt.Status == "succeeded" {
|
|
record.State = StateSucceeded
|
|
record.Error = nil
|
|
record.CauseCode = ""
|
|
record.EffectMayHaveApplied = false
|
|
record.SafeToResubmit = false
|
|
} else {
|
|
record.State = StateFailed
|
|
record.Error = &PublicError{Code: receipt.ErrorCode, Message: receipt.ErrorMessage}
|
|
record.CauseCode = receipt.ErrorCode
|
|
record.EffectMayHaveApplied = receipt.EffectMayHaveApplied
|
|
record.SafeToResubmit = receipt.SafeToResubmit
|
|
}
|
|
record.ExecutionFenced = false
|
|
} else {
|
|
record.State = StateFailed
|
|
record.Result = nil
|
|
record.Error = &PublicError{Code: "result_unconfirmed", Message: "agent restarted after dispatch; the operation was not replayed"}
|
|
record.CauseCode = ""
|
|
record.EffectMayHaveApplied = true
|
|
record.SafeToResubmit = false
|
|
record.ExecutionFenced = true
|
|
}
|
|
record.UpdatedAt = now
|
|
if err := putRecord(operations, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncCompletionReceipt(receipts, outbox, previous, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
|
|
return err
|
|
}
|
|
if err := inflight.Delete(key); err != nil {
|
|
return err
|
|
}
|
|
if err := expiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
finalized = append(finalized, record.OperationID)
|
|
}
|
|
return nil
|
|
})
|
|
return finalized, err
|
|
}
|
|
|
|
func completionEventID(operationID string) string {
|
|
digest := sha256.Sum256([]byte("operation-completion\n" + operationID))
|
|
return base64.RawURLEncoding.EncodeToString(digest[:])
|
|
}
|
|
|
|
func validCompletionEventID(value string) bool {
|
|
decoded, err := base64.RawURLEncoding.DecodeString(value)
|
|
return err == nil && len(decoded) == sha256.Size && base64.RawURLEncoding.EncodeToString(decoded) == value
|
|
}
|
|
|
|
func syncCompletionReceipt(receipts, outbox *bolt.Bucket, previous, current Record) error {
|
|
if receipts == nil || outbox == nil {
|
|
return errors.New("completion buckets are unavailable")
|
|
}
|
|
if !terminalState(current.State) || terminalState(previous.State) {
|
|
return nil
|
|
}
|
|
if !ValidOperationIdentity(current.Action, current.ResourceID, current.Fingerprint) {
|
|
return errors.New("terminal operation identity is invalid")
|
|
}
|
|
eventID := completionEventID(current.OperationID)
|
|
key := []byte(eventID)
|
|
if receipts.Get(key) != nil {
|
|
return nil
|
|
}
|
|
status := "failed"
|
|
errorCode := ""
|
|
if current.State == StateSucceeded {
|
|
status = "succeeded"
|
|
} else if current.Error != nil {
|
|
errorCode = current.Error.Code
|
|
}
|
|
receipt := CompletionReceipt{
|
|
Version: 2,
|
|
EventID: eventID, OperationID: current.OperationID,
|
|
Action: current.Action, ResourceID: current.ResourceID, RequestFingerprint: current.Fingerprint,
|
|
Status: status,
|
|
Result: append(json.RawMessage(nil), current.Result...), ErrorCode: errorCode,
|
|
CauseCode: current.CauseCode,
|
|
EffectMayHaveApplied: current.EffectMayHaveApplied,
|
|
SafeToResubmit: current.SafeToResubmit, CreatedAt: current.UpdatedAt.UTC(),
|
|
}
|
|
encoded, err := json.Marshal(receipt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := receipts.Put(key, encoded); err != nil {
|
|
return err
|
|
}
|
|
pending, err := json.Marshal(completionOutboxRecord{
|
|
OperationID: current.OperationID, NextAttemptAt: current.UpdatedAt.UTC(),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return outbox.Put(key, pending)
|
|
}
|
|
|
|
// Reserve durably binds operation_id to one exact request fingerprint and its
|
|
// non-secret execution descriptor. Records are never deleted, so an old
|
|
// destructive operation_id can never become valid again after housekeeping.
|
|
func (j *Journal) Reserve(spec Spec) (Record, bool, error) {
|
|
if err := validateSpec(spec); err != nil {
|
|
return Record{}, false, err
|
|
}
|
|
lockKeys := effectiveLockKeys(spec.OperationID, spec.ResourceID, spec.Action, spec.LockKeys)
|
|
var record Record
|
|
existed := false
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(operationsBucket)
|
|
inflight := tx.Bucket(inflightBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
if bucket == nil || inflight == nil || blockers == nil {
|
|
return errors.New("operation buckets are unavailable")
|
|
}
|
|
if err := pruneExpiredOperationDetails(tx, j.currentTime()); err != nil {
|
|
return err
|
|
}
|
|
if value := bucket.Get([]byte(spec.OperationID)); value != nil {
|
|
existing, err := decodeRecord(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
existed = true
|
|
if existing.Fingerprint != spec.Fingerprint || existing.Action != spec.Action || existing.ResourceID != spec.ResourceID ||
|
|
existing.SupersedesOperationID != spec.SupersedesOperationID {
|
|
return ErrFingerprint
|
|
}
|
|
record = existing
|
|
return nil
|
|
}
|
|
now := j.currentTime()
|
|
record = Record{
|
|
OperationID: spec.OperationID, ResourceID: spec.ResourceID,
|
|
Action: spec.Action, Fingerprint: spec.Fingerprint,
|
|
Destructive: spec.Destructive, State: StateReserved,
|
|
CreatedAt: now, UpdatedAt: now, ExpectedState: cloneExpectedState(spec.ExpectedState),
|
|
LockKeys: append([]string(nil), lockKeys...), HeavyIO: spec.HeavyIO,
|
|
SupersedesOperationID: spec.SupersedesOperationID,
|
|
}
|
|
if err := putRecord(bucket, record); err != nil {
|
|
return err
|
|
}
|
|
if err := inflight.Put([]byte(record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
return cloneRecord(record), existed, err
|
|
}
|
|
|
|
// WasNotDispatched reports the terminal 0-call outcome. Clean-break Agents do
|
|
// not reopen this record; a new request must use a new operation identifier.
|
|
func WasNotDispatched(record Record) bool {
|
|
return !record.Tombstone && record.State == StateFailed && record.Error != nil &&
|
|
record.Error.Code == "operation_not_dispatched"
|
|
}
|
|
|
|
// Execute persists the dispatch boundary before invoking fn. Once that
|
|
// boundary is crossed the function is invoked at most once. A deadline
|
|
// waits for the supervised worker to return before releasing admission. If the
|
|
// worker cannot be reaped within the secondary hard limit, the resource is
|
|
// durably fenced and any late result is ignored.
|
|
func (j *Journal) Execute(ctx context.Context, spec Spec, fn func(context.Context) (json.RawMessage, error)) (Record, bool, error) {
|
|
record, existed, err := j.Reserve(spec)
|
|
if err != nil {
|
|
return Record{}, existed, err
|
|
}
|
|
if existed && record.State != StateReserved {
|
|
return record, true, nil
|
|
}
|
|
releaseClaim, claimed := j.claimAdmission(spec.OperationID)
|
|
if !claimed {
|
|
return j.admissionReplay(spec.OperationID)
|
|
}
|
|
defer releaseClaim()
|
|
record, err = j.prepareAdmission(spec)
|
|
if err != nil {
|
|
return record, existed, err
|
|
}
|
|
if record.State != StateReserved {
|
|
return record, true, nil
|
|
}
|
|
|
|
releaseAdmission, err := j.acquireAdmission(ctx, record)
|
|
if err != nil {
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, failErr
|
|
}
|
|
return failed, existed, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
releaseAdmission()
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, failErr
|
|
}
|
|
return failed, existed, err
|
|
}
|
|
if err := j.verifyAdmissionUnblocked(spec); err != nil {
|
|
releaseAdmission()
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, failErr
|
|
}
|
|
return failed, existed, err
|
|
}
|
|
|
|
record, err = j.markDispatching(spec.OperationID)
|
|
releaseClaim()
|
|
if err != nil {
|
|
releaseAdmission()
|
|
return record, existed, err
|
|
}
|
|
if record.State != StateDispatching {
|
|
releaseAdmission()
|
|
return record, true, nil
|
|
}
|
|
|
|
type outcome struct {
|
|
result json.RawMessage
|
|
err error
|
|
}
|
|
done := make(chan outcome, 1)
|
|
mutationContext := context.WithValue(ctx, expectedStateContextKey{}, cloneExpectedState(record.ExpectedState))
|
|
go func() {
|
|
result, callErr := fn(mutationContext)
|
|
done <- outcome{result: result, err: callErr}
|
|
}()
|
|
|
|
select {
|
|
case completed := <-done:
|
|
terminal, persistErr := j.finish(spec.OperationID, completed.result, completed.err)
|
|
releaseAdmission()
|
|
if persistErr != nil {
|
|
return terminal, existed, persistErr
|
|
}
|
|
return terminal, existed, completed.err
|
|
case <-ctx.Done():
|
|
reapTimer := time.NewTimer(workerReapDeadline)
|
|
defer reapTimer.Stop()
|
|
select {
|
|
case completed := <-done:
|
|
terminal, persistErr := j.finish(spec.OperationID, completed.result, completed.err)
|
|
releaseAdmission()
|
|
if persistErr != nil {
|
|
return terminal, existed, persistErr
|
|
}
|
|
return terminal, existed, completed.err
|
|
case <-reapTimer.C:
|
|
terminal, persistErr := j.markExecutionFenced(spec.OperationID)
|
|
releaseAdmission()
|
|
if persistErr != nil {
|
|
return terminal, existed, persistErr
|
|
}
|
|
return terminal, existed, ErrExecutionNotReaped
|
|
}
|
|
}
|
|
}
|
|
|
|
// ExecuteBackground persists the dispatch boundary and starts one bounded,
|
|
// tracked worker before returning. A nil done channel means an existing
|
|
// non-reserved record was replayed and no new call was started.
|
|
func (j *Journal) ExecuteBackground(
|
|
dispatchCtx, workerParent context.Context,
|
|
workerTimeout time.Duration,
|
|
spec Spec,
|
|
fn func(context.Context) (json.RawMessage, error),
|
|
) (Record, bool, <-chan struct{}, error) {
|
|
if workerParent == nil || workerTimeout <= 0 {
|
|
return Record{}, false, nil, errors.New("background worker deadline is invalid")
|
|
}
|
|
record, existed, err := j.Reserve(spec)
|
|
if err != nil {
|
|
return Record{}, existed, nil, err
|
|
}
|
|
if existed && record.State != StateReserved {
|
|
return record, true, nil, nil
|
|
}
|
|
releaseClaim, claimed := j.claimAdmission(spec.OperationID)
|
|
if !claimed {
|
|
replayed, _, replayErr := j.admissionReplay(spec.OperationID)
|
|
return replayed, true, nil, replayErr
|
|
}
|
|
defer releaseClaim()
|
|
record, err = j.prepareAdmission(spec)
|
|
if err != nil {
|
|
return record, existed, nil, err
|
|
}
|
|
if record.State != StateReserved {
|
|
return record, true, nil, nil
|
|
}
|
|
releaseAdmission, err := j.acquireAdmission(dispatchCtx, record)
|
|
if err != nil {
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, nil, failErr
|
|
}
|
|
return failed, existed, nil, err
|
|
}
|
|
if err := dispatchCtx.Err(); err != nil {
|
|
releaseAdmission()
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, nil, failErr
|
|
}
|
|
return failed, existed, nil, err
|
|
}
|
|
if err := j.verifyAdmissionUnblocked(spec); err != nil {
|
|
releaseAdmission()
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, nil, failErr
|
|
}
|
|
return failed, existed, nil, err
|
|
}
|
|
record, err = j.markDispatching(spec.OperationID)
|
|
releaseClaim()
|
|
if err != nil {
|
|
releaseAdmission()
|
|
return record, existed, nil, err
|
|
}
|
|
if record.State != StateDispatching {
|
|
releaseAdmission()
|
|
return record, true, nil, nil
|
|
}
|
|
// Start the execution budget only after admission and the durable dispatch
|
|
// boundary. Time spent waiting for a resource or capacity slot must never
|
|
// consume the helper's bounded execution window.
|
|
workerCtx, workerCancel := context.WithTimeout(workerParent, workerTimeout)
|
|
type outcome struct {
|
|
result json.RawMessage
|
|
err error
|
|
}
|
|
callDone := make(chan outcome, 1)
|
|
done := make(chan struct{})
|
|
mutationContext := context.WithValue(workerCtx, expectedStateContextKey{}, cloneExpectedState(record.ExpectedState))
|
|
go func() {
|
|
result, callErr := fn(mutationContext)
|
|
callDone <- outcome{result: result, err: callErr}
|
|
}()
|
|
j.workerWG.Add(1)
|
|
go func() {
|
|
defer j.workerWG.Done()
|
|
defer close(done)
|
|
defer releaseAdmission()
|
|
defer workerCancel()
|
|
select {
|
|
case completed := <-callDone:
|
|
if _, persistErr := j.finish(spec.OperationID, completed.result, completed.err); persistErr != nil {
|
|
j.reportBackgroundFailure(spec.OperationID, "finalize", persistErr)
|
|
}
|
|
case <-workerCtx.Done():
|
|
reapTimer := time.NewTimer(workerReapDeadline)
|
|
select {
|
|
case completed := <-callDone:
|
|
if _, persistErr := j.finish(spec.OperationID, completed.result, completed.err); persistErr != nil {
|
|
j.reportBackgroundFailure(spec.OperationID, "finalize", persistErr)
|
|
}
|
|
case <-reapTimer.C:
|
|
if _, persistErr := j.markExecutionFenced(spec.OperationID); persistErr != nil {
|
|
j.reportBackgroundFailure(spec.OperationID, "fence", persistErr)
|
|
}
|
|
}
|
|
if !reapTimer.Stop() {
|
|
select {
|
|
case <-reapTimer.C:
|
|
default:
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
return record, existed, done, nil
|
|
}
|
|
|
|
// ExpectedStateFromContext returns the descriptor atomically persisted with
|
|
// the dispatch record. Mutation implementations use it to write a matching
|
|
// host-side transition marker before reporting success.
|
|
func ExpectedStateFromContext(ctx context.Context) (ExpectedState, bool) {
|
|
expected, ok := ctx.Value(expectedStateContextKey{}).(*ExpectedState)
|
|
if !ok || expected == nil {
|
|
return ExpectedState{}, false
|
|
}
|
|
return *cloneExpectedState(expected), true
|
|
}
|
|
|
|
// RequiresExplicitForceDelete identifies the terminal graceful result which
|
|
// remains open for fresh operator inspection and resource conflict blocking.
|
|
// Only a separately acknowledged force-delete-vm operation may cross this
|
|
// boundary; inspection itself never changes the record.
|
|
func RequiresExplicitForceDelete(record Record) bool {
|
|
return !record.Tombstone && record.State == StateFailed && record.Error != nil &&
|
|
record.Error.Code == "force_required" &&
|
|
(record.Action == "delete-vm" || record.Action == "service-suspend")
|
|
}
|
|
|
|
func (j *Journal) initialize() error {
|
|
return j.db.Update(func(tx *bolt.Tx) error {
|
|
metadata := tx.Bucket(metadataBucket)
|
|
fresh := metadata == nil
|
|
var err error
|
|
if fresh {
|
|
hasExistingBuckets := false
|
|
if err := tx.ForEach(func(_ []byte, _ *bolt.Bucket) error {
|
|
hasExistingBuckets = true
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if hasExistingBuckets {
|
|
return errors.New("operation journal has no supported schema; automatic migration is disabled")
|
|
}
|
|
metadata, err = tx.CreateBucket(metadataBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create journal metadata bucket: %w", err)
|
|
}
|
|
} else {
|
|
current := metadata.Get(schemaVersionKey)
|
|
if string(current) != string(schemaVersion) {
|
|
return fmt.Errorf("unsupported operation journal schema %q; clean-break deployment requires stopping the agent and removing the old test journal before upgrade", current)
|
|
}
|
|
}
|
|
operations, err := tx.CreateBucketIfNotExists(operationsBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create operation bucket: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(noncesBucket); err != nil {
|
|
return fmt.Errorf("create nonce bucket: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(nonceExpiryBucket); err != nil {
|
|
return fmt.Errorf("create nonce expiry bucket: %w", err)
|
|
}
|
|
operationExpiry, err := tx.CreateBucketIfNotExists(operationExpiryBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create operation expiry bucket: %w", err)
|
|
}
|
|
completionReceipts, err := tx.CreateBucketIfNotExists(completionReceiptsBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create completion receipt bucket: %w", err)
|
|
}
|
|
completionOutbox, err := tx.CreateBucketIfNotExists(completionOutboxBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create completion outbox bucket: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(completionQuarantineBucket); err != nil {
|
|
return fmt.Errorf("create completion quarantine bucket: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(completionQuarantineOperationsBucket); err != nil {
|
|
return fmt.Errorf("create completion quarantine operation index: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(fenceResolutionsBucket); err != nil {
|
|
return fmt.Errorf("create fence resolution bucket: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(fenceResolutionIDsBucket); err != nil {
|
|
return fmt.Errorf("create fence resolution identity bucket: %w", err)
|
|
}
|
|
inflight, err := tx.CreateBucketIfNotExists(inflightBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create inflight operation bucket: %w", err)
|
|
}
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
if blockers == nil {
|
|
if !fresh {
|
|
return errors.New("operation resource blocker index is unavailable")
|
|
}
|
|
blockers, err = tx.CreateBucket(resourceBlockersBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create operation resource blocker bucket: %w", err)
|
|
}
|
|
}
|
|
// resource_blockers is a derived admission index. Host storage
|
|
// configuration keeps immutable failure evidence, but a terminal attempt
|
|
// must never prevent an administrator from submitting a new, independently
|
|
// identified configuration request. Remove entries written by the earlier
|
|
// schema-v6 policy before this process admits any request. The surrounding
|
|
// Bolt transaction makes the repair atomic, and the operation plus its
|
|
// immutable completion receipt remain untouched.
|
|
if err := removeStorageConfigurationBlockers(operations, blockers); err != nil {
|
|
return err
|
|
}
|
|
now := j.currentTime()
|
|
var recovered []Record
|
|
var stale [][]byte
|
|
if err := inflight.ForEach(func(key, _ []byte) error {
|
|
value := operations.Get(key)
|
|
if value == nil {
|
|
stale = append(stale, append([]byte(nil), key...))
|
|
return nil
|
|
}
|
|
record, err := decodeRecord(value)
|
|
if err != nil {
|
|
return fmt.Errorf("decode operation %q: %w", key, err)
|
|
}
|
|
switch record.State {
|
|
case StateReserved:
|
|
record.State = StateFailed
|
|
record.Result = notDispatchedEvidence(record.Action)
|
|
record.Error = &PublicError{Code: "operation_not_dispatched", Message: "agent restarted before the dispatch boundary"}
|
|
record.EffectMayHaveApplied = false
|
|
record.SafeToResubmit = true
|
|
case StateDispatching:
|
|
// The parent process may have crashed after its isolated worker
|
|
// durably wrote a receipt. Leave this row in the inflight index until
|
|
// FinalizeInterruptedOperations imports that receipt or records
|
|
// result_unconfirmed. The mutation is never started again.
|
|
return nil
|
|
default:
|
|
stale = append(stale, append([]byte(nil), key...))
|
|
return nil
|
|
}
|
|
record.UpdatedAt = now
|
|
recovered = append(recovered, record)
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
for _, record := range recovered {
|
|
previous := cloneRecord(record)
|
|
previous.State = StateDispatching
|
|
if record.Error != nil && record.Error.Code == "operation_not_dispatched" {
|
|
previous.State = StateReserved
|
|
}
|
|
if err := putRecord(operations, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncCompletionReceipt(completionReceipts, completionOutbox, previous, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncResourceBlockers(blockers, nil, &record); err != nil {
|
|
return err
|
|
}
|
|
if terminalState(record.State) {
|
|
if err := operationExpiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := inflight.Delete([]byte(record.OperationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, key := range stale {
|
|
if err := inflight.Delete(key); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := pruneExpiredOperationDetails(tx, now); err != nil {
|
|
return err
|
|
}
|
|
return metadata.Put(schemaVersionKey, schemaVersion)
|
|
})
|
|
}
|
|
|
|
func (j *Journal) markDispatching(operationID string) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
if record.State == StateReserved {
|
|
record.State = StateDispatching
|
|
record.UpdatedAt = j.currentTime()
|
|
}
|
|
})
|
|
}
|
|
|
|
func (j *Journal) finish(operationID string, result json.RawMessage, callErr error) (Record, error) {
|
|
if IsUncertain(callErr) {
|
|
return j.markResultUnconfirmed(operationID, callErr)
|
|
}
|
|
if callErr != nil {
|
|
code := "backend_error"
|
|
message := callErr.Error()
|
|
effectMayHaveApplied := true
|
|
safeToResubmit := false
|
|
var public interface {
|
|
PublicCode() string
|
|
PublicMessage() string
|
|
}
|
|
if errors.As(callErr, &public) {
|
|
if public.PublicCode() != "" {
|
|
code = public.PublicCode()
|
|
}
|
|
if public.PublicMessage() != "" {
|
|
message = public.PublicMessage()
|
|
}
|
|
}
|
|
var semantics interface{ FailureSemantics() (bool, bool) }
|
|
if errors.As(callErr, &semantics) {
|
|
effectMayHaveApplied, safeToResubmit = semantics.FailureSemantics()
|
|
}
|
|
executionFenced := false
|
|
var fence interface{ RequiresExecutionFence() bool }
|
|
if errors.As(callErr, &fence) {
|
|
executionFenced = fence.RequiresExecutionFence()
|
|
}
|
|
return j.finishWithSemanticsAndFence(operationID, code, message, result, code, effectMayHaveApplied, safeToResubmit, executionFenced)
|
|
}
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
record.State = StateSucceeded
|
|
record.Result = append(json.RawMessage(nil), result...)
|
|
record.Error = nil
|
|
record.EffectMayHaveApplied = false
|
|
record.SafeToResubmit = false
|
|
record.ExecutionFenced = false
|
|
record.CauseCode = ""
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
func (j *Journal) finishWithPublicError(operationID, code, message string) (Record, error) {
|
|
return j.finishWithPublicErrorAndResult(operationID, code, message, nil)
|
|
}
|
|
|
|
func (j *Journal) finishWithPublicErrorAndResult(operationID, code, message string, result json.RawMessage) (Record, error) {
|
|
return j.finishWithCause(operationID, code, message, result, "")
|
|
}
|
|
|
|
func (j *Journal) finishWithCause(operationID, code, message string, result json.RawMessage, causeCode string) (Record, error) {
|
|
return j.finishWithSemantics(operationID, code, message, result, causeCode, true, false)
|
|
}
|
|
|
|
func (j *Journal) finishWithSemantics(operationID, code, message string, result json.RawMessage, causeCode string,
|
|
effectMayHaveApplied, safeToResubmit bool) (Record, error) {
|
|
return j.finishWithSemanticsAndFence(operationID, code, message, result, causeCode, effectMayHaveApplied, safeToResubmit, false)
|
|
}
|
|
|
|
func (j *Journal) finishWithSemanticsAndFence(operationID, code, message string, result json.RawMessage, causeCode string,
|
|
effectMayHaveApplied, safeToResubmit, executionFenced bool) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
record.State = StateFailed
|
|
record.Result = append(json.RawMessage(nil), result...)
|
|
record.Error = &PublicError{Code: code, Message: message}
|
|
record.EffectMayHaveApplied = effectMayHaveApplied
|
|
record.SafeToResubmit = safeToResubmit
|
|
record.ExecutionFenced = executionFenced
|
|
preserveCauseCode(record, causeCode)
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
func (j *Journal) failNotDispatchedIfReserved(operationID string) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
if record.State != StateReserved {
|
|
return
|
|
}
|
|
record.State = StateFailed
|
|
record.Result = notDispatchedEvidence(record.Action)
|
|
record.Error = &PublicError{Code: "operation_not_dispatched", Message: "operation did not cross the dispatch boundary"}
|
|
record.EffectMayHaveApplied = false
|
|
record.SafeToResubmit = true
|
|
record.ExecutionFenced = false
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
// FailNotDispatched terminally records that a reserved operation never
|
|
// crossed the dispatch boundary. It is safe for admission failures only.
|
|
func (j *Journal) FailNotDispatched(operationID string) (Record, error) {
|
|
return j.failNotDispatchedIfReserved(operationID)
|
|
}
|
|
|
|
func notDispatchedEvidence(action string) json.RawMessage {
|
|
if IsCreateLikeAction(action) {
|
|
return json.RawMessage(`{"not_dispatched":true,"creation_absent":true,"managed_artifacts_absent":true}`)
|
|
}
|
|
return json.RawMessage(`{"not_dispatched":true}`)
|
|
}
|
|
|
|
func (j *Journal) markResultUnconfirmed(operationID string, cause error) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
if terminalState(record.State) {
|
|
return
|
|
}
|
|
message := "operation result could not be confirmed before the execution deadline"
|
|
if summary := publicMessageFromError(cause); summary != "" {
|
|
message += ": " + summary
|
|
}
|
|
record.State = StateFailed
|
|
record.Result = nil
|
|
record.Error = &PublicError{Code: "result_unconfirmed", Message: message}
|
|
record.EffectMayHaveApplied = true
|
|
record.SafeToResubmit = false
|
|
record.ExecutionFenced = false
|
|
preserveCauseCode(record, publicCodeFromError(cause))
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
func (j *Journal) markExecutionFenced(operationID string) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
if terminalState(record.State) {
|
|
return
|
|
}
|
|
record.State = StateFailed
|
|
record.Result = nil
|
|
record.Error = &PublicError{
|
|
Code: "worker_reap_unconfirmed",
|
|
Message: "isolated mutation process did not exit after forced termination; this resource is fenced",
|
|
}
|
|
record.EffectMayHaveApplied = true
|
|
record.SafeToResubmit = false
|
|
record.ExecutionFenced = true
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
func preserveCauseCode(record *Record, code string) {
|
|
if record.CauseCode == "" && validCauseCode(code) {
|
|
record.CauseCode = code
|
|
}
|
|
}
|
|
|
|
func validCauseCode(code string) bool {
|
|
return len(code) <= 64 && causeCodePattern.MatchString(code)
|
|
}
|
|
|
|
func publicCodeFromError(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
var public interface{ PublicCode() string }
|
|
if !errors.As(err, &public) {
|
|
return ""
|
|
}
|
|
code := public.PublicCode()
|
|
if !validCauseCode(code) {
|
|
return ""
|
|
}
|
|
return code
|
|
}
|
|
|
|
func publicMessageFromError(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
var public interface{ PublicMessage() string }
|
|
if !errors.As(err, &public) {
|
|
return ""
|
|
}
|
|
return public.PublicMessage()
|
|
}
|
|
|
|
func (j *Journal) updateRecord(operationID string, mutate func(*Record)) (Record, error) {
|
|
return j.updateRecordWhen(operationID, nil, mutate)
|
|
}
|
|
|
|
// updateRecordWhen checks eligibility and applies a mutation in the same Bolt
|
|
// write transaction. A nil predicate preserves the ordinary update behavior.
|
|
func (j *Journal) updateRecordWhen(operationID string, eligible func(Record) bool,
|
|
mutate func(*Record)) (Record, error) {
|
|
var result Record
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(operationsBucket)
|
|
inflight := tx.Bucket(inflightBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
expiry := tx.Bucket(operationExpiryBucket)
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if bucket == nil || inflight == nil || blockers == nil || expiry == nil || receipts == nil || outbox == nil {
|
|
return errors.New("operation buckets are unavailable")
|
|
}
|
|
value := bucket.Get([]byte(operationID))
|
|
if value == nil {
|
|
return ErrNotFound
|
|
}
|
|
record, err := decodeRecord(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if eligible != nil && !eligible(record) {
|
|
result = record
|
|
return nil
|
|
}
|
|
previous := cloneRecord(record)
|
|
mutate(&record)
|
|
if err := putRecord(bucket, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncCompletionReceipt(receipts, outbox, previous, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
|
|
return err
|
|
}
|
|
if record.State == StateSucceeded && record.SupersedesOperationID != "" &&
|
|
(record.Action == "force-delete-vm" || isVPCAction(record.Action)) {
|
|
if err := closeSuccessfulSupersessionPredecessor(bucket, blockers, record); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
switch record.State {
|
|
case StateReserved, StateDispatching:
|
|
if err := inflight.Put([]byte(record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
if err := inflight.Delete([]byte(record.OperationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if terminalState(record.State) && !record.Tombstone {
|
|
if err := expiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
result = record
|
|
return nil
|
|
})
|
|
return cloneRecord(result), err
|
|
}
|
|
|
|
// closeSuccessfulSupersessionPredecessor runs in the same Bolt transaction
|
|
// which marks the reviewed successor successful. It removes only the prior
|
|
// blocker index entries; terminal evidence remains byte-for-byte immutable.
|
|
func closeSuccessfulSupersessionPredecessor(operations, blockers *bolt.Bucket, successor Record) error {
|
|
encoded := operations.Get([]byte(successor.SupersedesOperationID))
|
|
if encoded == nil {
|
|
return ErrInvalidSupersession
|
|
}
|
|
predecessor, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
lockKeys := effectiveLockKeys(successor.OperationID, successor.ResourceID, successor.Action, successor.LockKeys)
|
|
spec := Spec{
|
|
OperationID: successor.OperationID, ResourceID: successor.ResourceID,
|
|
Action: successor.Action, ExpectedState: successor.ExpectedState,
|
|
SupersedesOperationID: successor.SupersedesOperationID,
|
|
}
|
|
if validForceDeleteSupersession(spec, lockKeys, predecessor) {
|
|
for _, lockKey := range effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys) {
|
|
if err := blockers.Delete(blockerEntryKey(lockKey, predecessor.OperationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
if !isVPCAction(successor.Action) {
|
|
return ErrInvalidSupersession
|
|
}
|
|
chain, err := vpcSupersessionChain(operations, spec, lockKeys)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, prior := range chain {
|
|
for _, lockKey := range effectiveLockKeys(prior.OperationID, prior.ResourceID, prior.Action, prior.LockKeys) {
|
|
if err := blockers.Delete(blockerEntryKey(lockKey, prior.OperationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (j *Journal) acquire(ctx context.Context, key string) (func(), error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
j.resourceMu.Lock()
|
|
lock, ok := j.resources[key]
|
|
if !ok {
|
|
lock = &resourceLock{semaphore: make(chan struct{}, 1)}
|
|
j.resources[key] = lock
|
|
}
|
|
lock.references++
|
|
j.resourceMu.Unlock()
|
|
|
|
select {
|
|
case lock.semaphore <- struct{}{}:
|
|
var once sync.Once
|
|
release := func() {
|
|
once.Do(func() {
|
|
<-lock.semaphore
|
|
j.releaseResourceReference(key, lock)
|
|
})
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
release()
|
|
return nil, err
|
|
}
|
|
return release, nil
|
|
case <-ctx.Done():
|
|
j.releaseResourceReference(key, lock)
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (j *Journal) claimAdmission(operationID string) (func(), bool) {
|
|
j.admissionClaimMu.Lock()
|
|
if _, exists := j.admissionClaims[operationID]; exists {
|
|
j.admissionClaimMu.Unlock()
|
|
return nil, false
|
|
}
|
|
j.admissionClaims[operationID] = struct{}{}
|
|
j.admissionClaimMu.Unlock()
|
|
|
|
var once sync.Once
|
|
return func() {
|
|
once.Do(func() {
|
|
j.admissionClaimMu.Lock()
|
|
delete(j.admissionClaims, operationID)
|
|
j.admissionClaimMu.Unlock()
|
|
})
|
|
}, true
|
|
}
|
|
|
|
func blockerEntryKey(lockKey, operationID string) []byte {
|
|
return []byte(lockKey + "\x00" + operationID)
|
|
}
|
|
|
|
func blockerEntryPrefix(lockKey string) []byte {
|
|
return []byte(lockKey + "\x00")
|
|
}
|
|
|
|
func effectfulTerminalFailure(record Record) bool {
|
|
return !record.Tombstone && record.State == StateFailed && record.EffectMayHaveApplied
|
|
}
|
|
|
|
func blocksResources(record Record) bool {
|
|
// Fixed storage-pool registration is monotonic: a successor validates and
|
|
// reuses matching pools, and never deletes or rebinds an existing pool. Keep
|
|
// the immutable failed operation as evidence, but do not let it fence a new,
|
|
// explicitly identified configure-storage request.
|
|
return effectfulTerminalFailure(record) && record.Action != "configure-storage"
|
|
}
|
|
|
|
// removeStorageConfigurationBlockers removes only derived blocker-index
|
|
// entries owned by host storage configuration operations. Every reference is
|
|
// validated before any key is deleted so a missing, malformed, or stale
|
|
// non-storage reference still fails Agent startup instead of silently dropping
|
|
// a safety fence.
|
|
func removeStorageConfigurationBlockers(operations, blockers *bolt.Bucket) error {
|
|
if operations == nil || blockers == nil {
|
|
return errors.New("operation resource blocker index is unavailable")
|
|
}
|
|
removable := make([][]byte, 0)
|
|
cursor := blockers.Cursor()
|
|
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
|
|
separator := bytes.IndexByte(key, 0)
|
|
if separator <= 0 || separator != bytes.LastIndexByte(key, 0) || separator == len(key)-1 {
|
|
return fmt.Errorf("invalid operation resource blocker key %q", key)
|
|
}
|
|
lockKey := string(key[:separator])
|
|
operationID := string(key[separator+1:])
|
|
if err := validateLockKey(lockKey); err != nil || !operationIDPattern.MatchString(operationID) {
|
|
return fmt.Errorf("invalid operation resource blocker key %q", key)
|
|
}
|
|
encoded := operations.Get([]byte(operationID))
|
|
if encoded == nil {
|
|
return fmt.Errorf("resource blocker index references missing operation %q", operationID)
|
|
}
|
|
record, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return fmt.Errorf("decode resource blocker %q: %w", operationID, err)
|
|
}
|
|
if record.OperationID != operationID {
|
|
return fmt.Errorf("resource blocker index references mismatched operation %q", operationID)
|
|
}
|
|
matchingKey := false
|
|
for _, expected := range effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys) {
|
|
if expected == lockKey {
|
|
matchingKey = true
|
|
break
|
|
}
|
|
}
|
|
if !matchingKey {
|
|
return fmt.Errorf("resource blocker index references unexpected lock %q for operation %q", lockKey, operationID)
|
|
}
|
|
if record.Action == "configure-storage" {
|
|
if !effectfulTerminalFailure(record) {
|
|
return fmt.Errorf("resource blocker index references non-blocking operation %q", operationID)
|
|
}
|
|
removable = append(removable, append([]byte(nil), key...))
|
|
continue
|
|
}
|
|
if !blocksResources(record) {
|
|
return fmt.Errorf("resource blocker index references resolved operation %q", operationID)
|
|
}
|
|
}
|
|
for _, key := range removable {
|
|
if err := blockers.Delete(key); err != nil {
|
|
return fmt.Errorf("delete retired storage configuration blocker %q: %w", key, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func syncResourceBlockers(blockers *bolt.Bucket, previous, current *Record) error {
|
|
if blockers == nil {
|
|
return errors.New("operation resource blocker bucket is unavailable")
|
|
}
|
|
if previous != nil && blocksResources(*previous) {
|
|
for _, key := range effectiveLockKeys(previous.OperationID, previous.ResourceID, previous.Action, previous.LockKeys) {
|
|
if err := blockers.Delete(blockerEntryKey(key, previous.OperationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if current != nil && blocksResources(*current) {
|
|
for _, key := range effectiveLockKeys(current.OperationID, current.ResourceID, current.Action, current.LockKeys) {
|
|
if err := blockers.Put(blockerEntryKey(key, current.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func lockKeysOverlap(left, right []string) bool {
|
|
leftSet := make(map[string]struct{}, len(left))
|
|
for _, key := range left {
|
|
leftSet[key] = struct{}{}
|
|
}
|
|
for _, key := range right {
|
|
if _, ok := leftSet[key]; ok {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func validForceDeleteSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
|
|
if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID ||
|
|
spec.Action != "force-delete-vm" || predecessor.OperationID != spec.SupersedesOperationID ||
|
|
predecessor.ResourceID != spec.ResourceID || (predecessor.Action != "delete-vm" && predecessor.Action != "service-suspend") ||
|
|
predecessor.State != StateFailed || predecessor.Error == nil || predecessor.Error.Code != "force_required" {
|
|
return false
|
|
}
|
|
predecessorKeys := effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys)
|
|
return lockKeysOverlap(lockKeys, predecessorKeys)
|
|
}
|
|
|
|
func validPeerMigrationTargetCleanupSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
|
|
if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID ||
|
|
!cleanupPeerTargetActionPattern.MatchString(spec.Action) ||
|
|
predecessor.OperationID != spec.SupersedesOperationID ||
|
|
predecessor.ResourceID != spec.ResourceID ||
|
|
predecessor.State != StateFailed || !predecessor.EffectMayHaveApplied ||
|
|
predecessor.SafeToResubmit || predecessor.ExecutionFenced ||
|
|
!receivePeerActionPattern.MatchString(predecessor.Action) ||
|
|
strings.TrimPrefix(spec.Action, "cleanup-peer-migration-target:") !=
|
|
strings.TrimPrefix(predecessor.Action, "receive-peer-migration:") {
|
|
return false
|
|
}
|
|
predecessorKeys := effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys)
|
|
return lockKeysOverlap(lockKeys, predecessorKeys)
|
|
}
|
|
|
|
func validBackupWorkspaceCleanupSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
|
|
if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID ||
|
|
!cleanupBackupActionPattern.MatchString(spec.Action) ||
|
|
predecessor.OperationID != spec.SupersedesOperationID ||
|
|
predecessor.ResourceID != spec.ResourceID ||
|
|
predecessor.State != StateSucceeded || predecessor.ExecutionFenced ||
|
|
!createBackupActionPattern.MatchString(predecessor.Action) {
|
|
return false
|
|
}
|
|
backupKey := strings.TrimPrefix(spec.Action, "cleanup-backup-workspace:")
|
|
priorParts := strings.Split(predecessor.Action, ":")
|
|
if len(priorParts) != 3 || priorParts[1] != backupKey {
|
|
return false
|
|
}
|
|
var result struct {
|
|
CleanupPending bool `json:"cleanup_pending"`
|
|
}
|
|
if json.Unmarshal(predecessor.Result, &result) != nil || !result.CleanupPending {
|
|
return false
|
|
}
|
|
predecessorKeys := effectiveLockKeys(
|
|
predecessor.OperationID, predecessor.ResourceID,
|
|
predecessor.Action, predecessor.LockKeys,
|
|
)
|
|
return lockKeysOverlap(lockKeys, predecessorKeys)
|
|
}
|
|
|
|
func isVPCAction(action string) bool {
|
|
return strings.HasPrefix(action, "apply-vpc-network:") ||
|
|
strings.HasPrefix(action, "remove-vpc-network:") ||
|
|
action == "apply-vpc-port" || action == "remove-vpc-port"
|
|
}
|
|
|
|
func sameVPCRepairIntent(current, predecessor *ExpectedState) bool {
|
|
if current == nil || predecessor == nil || current.Kind != predecessor.Kind ||
|
|
current.VPCKey != predecessor.VPCKey ||
|
|
current.VPCRevision != predecessor.VPCRevision ||
|
|
current.VPCDigest != predecessor.VPCDigest ||
|
|
current.VPCPresent != predecessor.VPCPresent {
|
|
return false
|
|
}
|
|
if current.Kind == ExpectedVPCNetwork {
|
|
return true
|
|
}
|
|
return current.Kind == ExpectedVPCPort &&
|
|
current.VPCPortKey == predecessor.VPCPortKey &&
|
|
current.VPCCIDR == predecessor.VPCCIDR &&
|
|
current.VPCGateway == predecessor.VPCGateway &&
|
|
current.VPCPrivateIP == predecessor.VPCPrivateIP &&
|
|
current.VPCMAC == predecessor.VPCMAC &&
|
|
current.VPCBridge == predecessor.VPCBridge
|
|
}
|
|
|
|
func validVPCRepairSupersession(spec Spec, lockKeys []string,
|
|
predecessor Record) bool {
|
|
if spec.SupersedesOperationID == "" ||
|
|
spec.SupersedesOperationID == spec.OperationID ||
|
|
predecessor.OperationID != spec.SupersedesOperationID ||
|
|
predecessor.ResourceID != spec.ResourceID ||
|
|
predecessor.Action != spec.Action || !isVPCAction(spec.Action) ||
|
|
predecessor.State != StateFailed || !predecessor.EffectMayHaveApplied ||
|
|
predecessor.SafeToResubmit || !predecessor.ExecutionFenced ||
|
|
predecessor.Tombstone ||
|
|
!sameVPCRepairIntent(spec.ExpectedState, predecessor.ExpectedState) {
|
|
return false
|
|
}
|
|
predecessorKeys := effectiveLockKeys(
|
|
predecessor.OperationID, predecessor.ResourceID,
|
|
predecessor.Action, predecessor.LockKeys,
|
|
)
|
|
return lockKeysOverlap(lockKeys, predecessorKeys)
|
|
}
|
|
|
|
// ValidVPCRepairSupersession exposes the same fail-closed predecessor check to
|
|
// the authenticated API before a repair is dispatched. Journal admission
|
|
// repeats it under the Bolt transaction, so this preflight is not authority.
|
|
func ValidVPCRepairSupersession(spec Spec, predecessor Record) bool {
|
|
lockKeys, err := normalizeLockKeys(spec.LockKeys)
|
|
return err == nil && validVPCRepairSupersession(
|
|
spec, lockKeys, predecessor,
|
|
)
|
|
}
|
|
|
|
func validExplicitSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
|
|
return validForceDeleteSupersession(spec, lockKeys, predecessor) ||
|
|
validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor) ||
|
|
validBackupWorkspaceCleanupSupersession(spec, lockKeys, predecessor) ||
|
|
validVPCRepairSupersession(spec, lockKeys, predecessor)
|
|
}
|
|
|
|
const maxVPCSupersessionDepth = 32
|
|
|
|
// vpcSupersessionChain validates every durable predecessor generation. A
|
|
// repair that itself lost its response may be followed by another exact
|
|
// repair, but it may never skip a changed intent or grow without a bound.
|
|
func vpcSupersessionChain(operations *bolt.Bucket, spec Spec,
|
|
lockKeys []string) ([]Record, error) {
|
|
if operations == nil || !isVPCAction(spec.Action) ||
|
|
spec.SupersedesOperationID == "" {
|
|
return nil, ErrInvalidSupersession
|
|
}
|
|
currentSpec := spec
|
|
currentKeys := lockKeys
|
|
seen := map[string]struct{}{spec.OperationID: {}}
|
|
chain := make([]Record, 0, 2)
|
|
for len(chain) < maxVPCSupersessionDepth {
|
|
predecessorID := currentSpec.SupersedesOperationID
|
|
if predecessorID == "" {
|
|
return chain, nil
|
|
}
|
|
if _, exists := seen[predecessorID]; exists {
|
|
return nil, ErrInvalidSupersession
|
|
}
|
|
seen[predecessorID] = struct{}{}
|
|
encoded := operations.Get([]byte(predecessorID))
|
|
if encoded == nil {
|
|
return nil, ErrInvalidSupersession
|
|
}
|
|
predecessor, err := decodeRecord(encoded)
|
|
if err != nil || !validVPCRepairSupersession(
|
|
currentSpec, currentKeys, predecessor,
|
|
) {
|
|
return nil, ErrInvalidSupersession
|
|
}
|
|
chain = append(chain, predecessor)
|
|
if predecessor.SupersedesOperationID == "" {
|
|
return chain, nil
|
|
}
|
|
currentSpec = Spec{
|
|
OperationID: predecessor.OperationID,
|
|
ResourceID: predecessor.ResourceID,
|
|
Action: predecessor.Action,
|
|
ExpectedState: predecessor.ExpectedState,
|
|
SupersedesOperationID: predecessor.SupersedesOperationID,
|
|
}
|
|
currentKeys = effectiveLockKeys(
|
|
predecessor.OperationID, predecessor.ResourceID,
|
|
predecessor.Action, predecessor.LockKeys,
|
|
)
|
|
}
|
|
return nil, ErrInvalidSupersession
|
|
}
|
|
|
|
// findResourceBlocker validates any explicit force-delete predecessor and
|
|
// reads only the compact blocker index for unresolved resource conflicts.
|
|
func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, bool, error) {
|
|
operations := tx.Bucket(operationsBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
if operations == nil || blockers == nil {
|
|
return Record{}, false, errors.New("operation resource blocker index is unavailable")
|
|
}
|
|
supersededOperationIDs := map[string]struct{}{}
|
|
if spec.SupersedesOperationID != "" {
|
|
encoded := operations.Get([]byte(spec.SupersedesOperationID))
|
|
if encoded == nil {
|
|
return Record{}, false, ErrInvalidSupersession
|
|
}
|
|
predecessor, err := decodeRecord(encoded)
|
|
if err != nil || !validExplicitSupersession(spec, lockKeys, predecessor) {
|
|
return Record{}, false, ErrInvalidSupersession
|
|
}
|
|
if isVPCAction(spec.Action) {
|
|
chain, chainErr := vpcSupersessionChain(
|
|
operations, spec, lockKeys,
|
|
)
|
|
if chainErr != nil {
|
|
return Record{}, false, chainErr
|
|
}
|
|
for _, prior := range chain {
|
|
supersededOperationIDs[prior.OperationID] = struct{}{}
|
|
}
|
|
} else {
|
|
supersededOperationIDs[predecessor.OperationID] = struct{}{}
|
|
}
|
|
}
|
|
for _, lockKey := range lockKeys {
|
|
prefix := blockerEntryPrefix(lockKey)
|
|
cursor := blockers.Cursor()
|
|
for key, _ := cursor.Seek(prefix); key != nil && bytes.HasPrefix(key, prefix); key, _ = cursor.Next() {
|
|
operationID := string(key[len(prefix):])
|
|
_, explicitlySuperseded := supersededOperationIDs[operationID]
|
|
if operationID == spec.OperationID || explicitlySuperseded {
|
|
continue
|
|
}
|
|
encoded := operations.Get([]byte(operationID))
|
|
if encoded == nil {
|
|
return Record{}, false, fmt.Errorf("resource blocker index references missing operation %q", operationID)
|
|
}
|
|
blocker, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return Record{}, false, fmt.Errorf("decode resource blocker %q: %w", operationID, err)
|
|
}
|
|
if !blocksResources(blocker) {
|
|
return Record{}, false, fmt.Errorf("resource blocker index references resolved operation %q", operationID)
|
|
}
|
|
return cloneRecord(blocker), true, nil
|
|
}
|
|
}
|
|
return Record{}, false, nil
|
|
}
|
|
|
|
func (j *Journal) verifyAdmissionUnblocked(spec Spec) error {
|
|
lockKeys := effectiveLockKeys(spec.OperationID, spec.ResourceID, spec.Action, spec.LockKeys)
|
|
return j.db.View(func(tx *bolt.Tx) error {
|
|
_, blocked, err := findResourceBlocker(tx, spec, lockKeys)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if blocked {
|
|
return ErrResourceBlocked
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// prepareAdmission runs only while this process owns the operation's admission
|
|
// claim. Terminal records are immutable and are never reopened.
|
|
func (j *Journal) prepareAdmission(spec Spec) (Record, error) {
|
|
lockKeys := effectiveLockKeys(spec.OperationID, spec.ResourceID, spec.Action, spec.LockKeys)
|
|
var record Record
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
inflight := tx.Bucket(inflightBucket)
|
|
if operations == nil || inflight == nil {
|
|
return errors.New("operation buckets are unavailable")
|
|
}
|
|
value := operations.Get([]byte(spec.OperationID))
|
|
if value == nil {
|
|
return ErrNotFound
|
|
}
|
|
existing, err := decodeRecord(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if existing.Fingerprint != spec.Fingerprint || existing.Action != spec.Action || existing.ResourceID != spec.ResourceID ||
|
|
existing.SupersedesOperationID != spec.SupersedesOperationID {
|
|
return ErrFingerprint
|
|
}
|
|
if existing.State != StateReserved {
|
|
record = existing
|
|
return nil
|
|
}
|
|
existing.Destructive = spec.Destructive
|
|
existing.ExpectedState = cloneExpectedState(spec.ExpectedState)
|
|
existing.LockKeys = append([]string(nil), lockKeys...)
|
|
existing.HeavyIO = spec.HeavyIO
|
|
existing.SupersedesOperationID = spec.SupersedesOperationID
|
|
if err := putRecord(operations, existing); err != nil {
|
|
return err
|
|
}
|
|
if err := inflight.Put([]byte(existing.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
record = existing
|
|
return nil
|
|
})
|
|
return cloneRecord(record), err
|
|
}
|
|
|
|
func (j *Journal) admissionReplay(operationID string) (Record, bool, error) {
|
|
record, err := j.Get(operationID)
|
|
if err != nil {
|
|
return Record{}, true, err
|
|
}
|
|
switch record.State {
|
|
case StateReserved:
|
|
return record, true, ErrOperationStarting
|
|
default:
|
|
return record, true, nil
|
|
}
|
|
}
|
|
|
|
func (j *Journal) acquireAdmission(ctx context.Context, record Record) (func(), error) {
|
|
keys := effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys)
|
|
releases := make([]func(), 0, len(keys)+2)
|
|
releaseAll := func() {
|
|
for index := len(releases) - 1; index >= 0; index-- {
|
|
releases[index]()
|
|
}
|
|
}
|
|
for _, key := range keys {
|
|
release, err := j.acquire(ctx, key)
|
|
if err != nil {
|
|
releaseAll()
|
|
return nil, err
|
|
}
|
|
releases = append(releases, release)
|
|
}
|
|
if record.HeavyIO || defaultHeavyIOAction(record.Action) {
|
|
release, err := acquireCapacity(ctx, j.heavyIOSlots)
|
|
if err != nil {
|
|
releaseAll()
|
|
return nil, err
|
|
}
|
|
releases = append(releases, release)
|
|
}
|
|
releaseExecution, err := j.acquireExecution(ctx)
|
|
if err != nil {
|
|
releaseAll()
|
|
return nil, err
|
|
}
|
|
releases = append(releases, releaseExecution)
|
|
if err := ctx.Err(); err != nil {
|
|
releaseAll()
|
|
return nil, err
|
|
}
|
|
var once sync.Once
|
|
return func() { once.Do(releaseAll) }, nil
|
|
}
|
|
|
|
func acquireCapacity(ctx context.Context, slots chan struct{}) (func(), error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
select {
|
|
case slots <- struct{}{}:
|
|
var once sync.Once
|
|
release := func() { once.Do(func() { <-slots }) }
|
|
if err := ctx.Err(); err != nil {
|
|
release()
|
|
return nil, err
|
|
}
|
|
return release, nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (j *Journal) acquireExecution(ctx context.Context) (func(), error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
select {
|
|
case j.executionSlots <- struct{}{}:
|
|
var once sync.Once
|
|
release := func() {
|
|
once.Do(func() { <-j.executionSlots })
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
release()
|
|
return nil, err
|
|
}
|
|
return release, nil
|
|
default:
|
|
return nil, ErrBusy
|
|
}
|
|
}
|
|
|
|
func (j *Journal) releaseResourceReference(key string, lock *resourceLock) {
|
|
j.resourceMu.Lock()
|
|
defer j.resourceMu.Unlock()
|
|
lock.references--
|
|
if lock.references == 0 && len(lock.semaphore) == 0 && j.resources[key] == lock {
|
|
delete(j.resources, key)
|
|
}
|
|
}
|
|
|
|
func effectiveLockKeys(operationID, resourceID, action string, configured []string) []string {
|
|
if len(configured) != 0 {
|
|
keys, _ := normalizeLockKeys(configured)
|
|
return keys
|
|
}
|
|
return []string{defaultResourceLockKey(operationID, resourceID, action)}
|
|
}
|
|
|
|
func defaultResourceLockKey(operationID, resourceID, action string) string {
|
|
canonicalResource := strings.ToLower(resourceID)
|
|
if canonicalUUIDPattern.MatchString(canonicalResource) {
|
|
return "domain:" + canonicalResource
|
|
}
|
|
if action == "configure-storage" {
|
|
return "storage-config"
|
|
}
|
|
for prefix, kind := range map[string]string{
|
|
"template:": "template", "rescue-template:": "rescue-template",
|
|
} {
|
|
if strings.HasPrefix(resourceID, prefix) {
|
|
return "catalog:" + kind + "/" + strings.TrimPrefix(resourceID, prefix)
|
|
}
|
|
}
|
|
if strings.HasPrefix(resourceID, "user-iso:") {
|
|
return "catalog:user-iso/" + strings.TrimPrefix(resourceID, "user-iso:")
|
|
}
|
|
if resourceID == "" {
|
|
return "operation:" + operationID
|
|
}
|
|
digest := sha256.Sum256([]byte(resourceID))
|
|
return "resource-hash:" + hex.EncodeToString(digest[:])
|
|
}
|
|
|
|
func defaultHeavyIOAction(action string) bool {
|
|
return action == "configure-storage" || IsCreateLikeAction(action) || action == "reinstall-vm" ||
|
|
strings.HasPrefix(action, "create-backup:") ||
|
|
action == "import-template" || action == "import-rescue-template" || action == "import-user-iso" ||
|
|
strings.HasPrefix(action, "create-snapshot:") || strings.HasPrefix(action, "restore-snapshot:")
|
|
}
|
|
|
|
func normalizeLockKeys(keys []string) ([]string, error) {
|
|
if len(keys) > maxAdmissionResourceKeys {
|
|
return nil, fmt.Errorf("operation has more than %d resource keys", maxAdmissionResourceKeys)
|
|
}
|
|
unique := make(map[string]struct{}, len(keys))
|
|
for _, key := range keys {
|
|
if err := validateLockKey(key); err != nil {
|
|
return nil, err
|
|
}
|
|
unique[key] = struct{}{}
|
|
}
|
|
result := make([]string, 0, len(unique))
|
|
for key := range unique {
|
|
result = append(result, key)
|
|
}
|
|
sort.Strings(result)
|
|
return result, nil
|
|
}
|
|
|
|
func validateLockKey(key string) error {
|
|
if key == "" || len(key) > maximumAdmissionKeyLength || strings.TrimSpace(key) != key || strings.ContainsRune(key, 0) {
|
|
return errors.New("invalid operation resource key")
|
|
}
|
|
if key == "storage-config" {
|
|
return nil
|
|
}
|
|
prefix, value, ok := strings.Cut(key, ":")
|
|
if !ok || value == "" {
|
|
return errors.New("invalid operation resource key")
|
|
}
|
|
switch prefix {
|
|
case "domain":
|
|
if !canonicalUUIDPattern.MatchString(value) {
|
|
return errors.New("domain resource key must contain a canonical lowercase UUID")
|
|
}
|
|
case "volume", "catalog":
|
|
left, right, found := strings.Cut(value, "/")
|
|
if !found || strings.Contains(right, "/") || !resourceKindPattern.MatchString(left) || !resourcePartPattern.MatchString(right) {
|
|
return errors.New("invalid operation resource key")
|
|
}
|
|
case "nwfilter":
|
|
if !resourcePartPattern.MatchString(value) {
|
|
return errors.New("invalid nwfilter resource key")
|
|
}
|
|
case "network-identity", "resource-hash":
|
|
if !hexDigestPattern.MatchString(value) {
|
|
return errors.New("invalid hashed operation resource key")
|
|
}
|
|
case "operation":
|
|
if !operationIDPattern.MatchString(value) {
|
|
return errors.New("invalid operation resource key")
|
|
}
|
|
default:
|
|
return errors.New("unsupported operation resource key")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateSpec(spec Spec) error {
|
|
if !operationIDPattern.MatchString(spec.OperationID) {
|
|
return errors.New("operation_id must be 8-80 URL-safe characters")
|
|
}
|
|
if !ValidOperationIdentity(spec.Action, spec.ResourceID, spec.Fingerprint) {
|
|
return errors.New("operation action, resource_id, or request fingerprint is invalid")
|
|
}
|
|
if _, err := normalizeLockKeys(spec.LockKeys); err != nil {
|
|
return err
|
|
}
|
|
if spec.SupersedesOperationID != "" {
|
|
if !operationIDPattern.MatchString(spec.SupersedesOperationID) ||
|
|
spec.SupersedesOperationID == spec.OperationID ||
|
|
(spec.Action != "force-delete-vm" &&
|
|
!cleanupPeerTargetActionPattern.MatchString(spec.Action) &&
|
|
!cleanupBackupActionPattern.MatchString(spec.Action) &&
|
|
!isVPCAction(spec.Action)) {
|
|
return ErrInvalidSupersession
|
|
}
|
|
}
|
|
if spec.ExpectedState != nil {
|
|
if err := validateExpectedState(spec.Action, spec.OperationID, *spec.ExpectedState); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateExpectedState(action, operationID string, expected ExpectedState) error {
|
|
if expected.TransitionID != operationID {
|
|
return errors.New("expected state transition_id must match operation_id")
|
|
}
|
|
if len(expected.Hostname) > 253 || len(expected.NetworkSourceType) > 16 || len(expected.NetworkSourceName) > 128 {
|
|
return errors.New("expected state contains an oversized value")
|
|
}
|
|
values := []string{expected.Hostname, expected.NetworkSourceType, expected.NetworkSourceName, expected.VPCKey, expected.VPCDigest, expected.VPCPortKey, expected.VPCCIDR, expected.VPCGateway, expected.VPCPrivateIP, expected.VPCMAC, expected.VPCBridge}
|
|
values = append(values, expected.PublicNetwork.NetworkSelector,
|
|
expected.PublicNetwork.IPv4.PoolKey, expected.PublicNetwork.IPv4.PoolCIDR,
|
|
expected.PublicNetwork.IPv4.AddressCIDR, expected.PublicNetwork.IPv4.Gateway,
|
|
expected.PublicNetwork.IPv6.PoolKey, expected.PublicNetwork.IPv6.PoolCIDR,
|
|
expected.PublicNetwork.IPv6.AddressCIDR, expected.PublicNetwork.IPv6.Gateway)
|
|
values = append(values, expected.PublicNetwork.DNSServers...)
|
|
for _, value := range values {
|
|
if strings.ContainsRune(value, 0) {
|
|
return errors.New("expected state contains NUL")
|
|
}
|
|
}
|
|
switch expected.Kind {
|
|
case ExpectedUpdateSettings:
|
|
if action != "update-settings" || (!expected.HostnameChanged && !expected.RootPasswordChanged && !expected.SSHKeyChanged) {
|
|
return errors.New("invalid update-settings expected state")
|
|
}
|
|
case ExpectedResizeResources:
|
|
if action != "resize-vm" || !validExpectedResizeResources(expected) {
|
|
return errors.New("invalid resize-resources expected state")
|
|
}
|
|
case ExpectedChangeNetwork:
|
|
if action != "change-network" || !validExpectedPublicNetwork(expected.PublicNetwork) ||
|
|
(expected.NetworkSourceType != "bridge" && expected.NetworkSourceType != "network") || !networkSourcePattern.MatchString(expected.NetworkSourceName) {
|
|
return errors.New("invalid change-network expected state")
|
|
}
|
|
case ExpectedNetworkState:
|
|
if (action != "network-suspend" && action != "network-resume") || expected.Enabled != (action == "network-resume") {
|
|
return errors.New("invalid network-state expected state")
|
|
}
|
|
case ExpectedServiceState:
|
|
if (action != "service-suspend" && action != "service-resume") || expected.Enabled != (action == "service-resume") || (!expected.Enabled && expected.StartIfSuspended) {
|
|
return errors.New("invalid service-state expected state")
|
|
}
|
|
case ExpectedPeerMigrationState:
|
|
if !strings.HasPrefix(action, "activate-peer-migration:") &&
|
|
!strings.HasPrefix(action, "rollback-peer-migration:") {
|
|
return errors.New("invalid peer-migration-state expected state")
|
|
}
|
|
case ExpectedFirewall:
|
|
if action != "apply-firewall" || expected.FirewallRevision == 0 || !hexDigestPattern.MatchString(expected.FirewallDigest) {
|
|
return errors.New("invalid firewall expected state")
|
|
}
|
|
case ExpectedStorageConfiguration:
|
|
if action != "configure-storage" ||
|
|
!hexDigestPattern.MatchString(expected.ManagedPathDigest) ||
|
|
!hexDigestPattern.MatchString(expected.TemplatePathDigest) ||
|
|
!hexDigestPattern.MatchString(expected.SeedPathDigest) ||
|
|
!hexDigestPattern.MatchString(expected.UserISOPathDigest) ||
|
|
!hexDigestPattern.MatchString(expected.RescuePathDigest) {
|
|
return errors.New("invalid storage-configuration expected state")
|
|
}
|
|
case ExpectedVPCNetwork:
|
|
applyAction := "apply-vpc-network:" + expected.VPCKey
|
|
removeAction := "remove-vpc-network:" + expected.VPCKey
|
|
if !shortHexIDPattern.MatchString(expected.VPCKey) || expected.VPCRevision == 0 ||
|
|
!hexDigestPattern.MatchString(expected.VPCDigest) ||
|
|
(expected.VPCPresent && action != applyAction) ||
|
|
(!expected.VPCPresent && action != removeAction) {
|
|
return errors.New("invalid VPC network expected state")
|
|
}
|
|
case ExpectedVPCPort:
|
|
if !shortHexIDPattern.MatchString(expected.VPCKey) ||
|
|
!shortHexIDPattern.MatchString(expected.VPCPortKey) ||
|
|
expected.VPCRevision == 0 || !hexDigestPattern.MatchString(expected.VPCDigest) ||
|
|
!validExpectedVPCPort(expected) ||
|
|
(expected.VPCPresent && action != "apply-vpc-port") ||
|
|
(!expected.VPCPresent && action != "remove-vpc-port") {
|
|
return errors.New("invalid VPC port expected state")
|
|
}
|
|
case ExpectedCreateLikeNetwork:
|
|
if !validExpectedCreateLikeNetwork(action, expected) {
|
|
return errors.New("invalid create-like network expected state")
|
|
}
|
|
default:
|
|
return errors.New("unsupported expected state kind")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validExpectedCreateLikeNetwork(action string, expected ExpectedState) bool {
|
|
if (!restoreBackupActionPattern.MatchString(action) &&
|
|
!receivePeerActionPattern.MatchString(action)) ||
|
|
!validExpectedPublicNetwork(expected.PublicNetwork) ||
|
|
!validExpectedNetworkSource(expected) ||
|
|
!createLikeExpectedStateFieldsAreNarrow(expected) {
|
|
return false
|
|
}
|
|
if restoreBackupActionPattern.MatchString(action) {
|
|
return !expected.VPCPresent && expectedVPCPortFieldsAreEmpty(expected)
|
|
}
|
|
if !expected.VPCPresent {
|
|
return expectedVPCPortFieldsAreEmpty(expected)
|
|
}
|
|
return shortHexIDPattern.MatchString(expected.VPCKey) &&
|
|
shortHexIDPattern.MatchString(expected.VPCPortKey) &&
|
|
expected.VPCRevision > 0 && hexDigestPattern.MatchString(expected.VPCDigest) &&
|
|
validExpectedVPCPort(expected)
|
|
}
|
|
|
|
func validExpectedNetworkSource(expected ExpectedState) bool {
|
|
if (expected.NetworkSourceType != "bridge" && expected.NetworkSourceType != "network") ||
|
|
!networkSourcePattern.MatchString(expected.NetworkSourceName) {
|
|
return false
|
|
}
|
|
return expected.PublicNetwork.NetworkSelector ==
|
|
expected.NetworkSourceType+":"+expected.NetworkSourceName
|
|
}
|
|
|
|
func expectedVPCPortFieldsAreEmpty(expected ExpectedState) bool {
|
|
return expected.VPCKey == "" && expected.VPCRevision == 0 && expected.VPCDigest == "" &&
|
|
expected.VPCPortKey == "" && expected.VPCCIDR == "" && expected.VPCGateway == "" &&
|
|
expected.VPCPrivateIP == "" && expected.VPCMAC == "" && expected.VPCBridge == ""
|
|
}
|
|
|
|
func createLikeExpectedStateFieldsAreNarrow(expected ExpectedState) bool {
|
|
return !expected.HostnameChanged && expected.Hostname == "" &&
|
|
!expected.RootPasswordChanged && !expected.SSHKeyChanged &&
|
|
expected.FromVCPUs == 0 && expected.TargetVCPUs == 0 &&
|
|
expected.FromMemoryMiB == 0 && expected.TargetMemoryMiB == 0 &&
|
|
expected.FromDiskGiB == 0 && expected.TargetDiskGiB == 0 &&
|
|
expected.FromBandwidthMbps == 0 && expected.TargetBandwidthMbps == 0 &&
|
|
!expected.Enabled && !expected.StartIfSuspended &&
|
|
expected.FirewallRevision == 0 && expected.FirewallDigest == "" &&
|
|
expected.ManagedPathDigest == "" && expected.TemplatePathDigest == "" &&
|
|
expected.SeedPathDigest == "" && expected.UserISOPathDigest == "" &&
|
|
expected.RescuePathDigest == ""
|
|
}
|
|
|
|
func validExpectedVPCPort(expected ExpectedState) bool {
|
|
prefix, err := netip.ParsePrefix(expected.VPCCIDR)
|
|
if err != nil || !prefix.Addr().Is4() || !prefix.Addr().IsPrivate() ||
|
|
prefix != prefix.Masked() || prefix.String() != expected.VPCCIDR ||
|
|
prefix.Bits() < 16 || prefix.Bits() > 28 {
|
|
return false
|
|
}
|
|
gateway, gatewayErr := netip.ParseAddr(expected.VPCGateway)
|
|
address, addressErr := netip.ParseAddr(expected.VPCPrivateIP)
|
|
if gatewayErr != nil || addressErr != nil || !gateway.Is4() || !address.Is4() ||
|
|
gateway != prefix.Addr().Next() || !prefix.Contains(address) ||
|
|
address == prefix.Addr() || address == gateway ||
|
|
address == expectedVPCLastAddress(prefix) {
|
|
return false
|
|
}
|
|
hardware, err := net.ParseMAC(expected.VPCMAC)
|
|
if err != nil || len(hardware) != 6 || hardware[0]&1 != 0 ||
|
|
hardware.String() != expected.VPCMAC {
|
|
return false
|
|
}
|
|
zeroMAC := true
|
|
for _, value := range hardware {
|
|
zeroMAC = zeroMAC && value == 0
|
|
}
|
|
return !zeroMAC && networkSourcePattern.MatchString(expected.VPCBridge)
|
|
}
|
|
|
|
func expectedVPCLastAddress(prefix netip.Prefix) netip.Addr {
|
|
bytes := prefix.Masked().Addr().As4()
|
|
value := binary.BigEndian.Uint32(bytes[:]) | (^uint32(0) >> prefix.Bits())
|
|
var result [4]byte
|
|
binary.BigEndian.PutUint32(result[:], value)
|
|
return netip.AddrFrom4(result)
|
|
}
|
|
|
|
func validExpectedResizeResources(expected ExpectedState) bool {
|
|
if expected.FromVCPUs < 1 || expected.FromVCPUs > 512 ||
|
|
expected.TargetVCPUs < expected.FromVCPUs || expected.TargetVCPUs > 512 ||
|
|
expected.FromMemoryMiB < 128 || expected.FromMemoryMiB > 16*1024*1024 ||
|
|
expected.TargetMemoryMiB < expected.FromMemoryMiB || expected.TargetMemoryMiB > 16*1024*1024 ||
|
|
expected.FromDiskGiB < 1 || expected.FromDiskGiB > 4096 ||
|
|
expected.TargetDiskGiB < expected.FromDiskGiB || expected.TargetDiskGiB > 4096 ||
|
|
expected.FromBandwidthMbps < 1 || expected.FromBandwidthMbps > 100000 ||
|
|
expected.TargetBandwidthMbps < expected.FromBandwidthMbps || expected.TargetBandwidthMbps > 100000 {
|
|
return false
|
|
}
|
|
return expected.TargetVCPUs != expected.FromVCPUs ||
|
|
expected.TargetMemoryMiB != expected.FromMemoryMiB ||
|
|
expected.TargetDiskGiB != expected.FromDiskGiB ||
|
|
expected.TargetBandwidthMbps != expected.FromBandwidthMbps
|
|
}
|
|
|
|
func validateRecord(record Record) error {
|
|
if err := validateSpec(Spec{
|
|
OperationID: record.OperationID, ResourceID: record.ResourceID,
|
|
Action: record.Action, Fingerprint: record.Fingerprint, ExpectedState: record.ExpectedState,
|
|
LockKeys: record.LockKeys, HeavyIO: record.HeavyIO,
|
|
SupersedesOperationID: record.SupersedesOperationID,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
switch record.State {
|
|
case StateReserved, StateDispatching, StateSucceeded, StateFailed:
|
|
default:
|
|
return fmt.Errorf("invalid state %q", record.State)
|
|
}
|
|
if record.EffectMayHaveApplied && record.SafeToResubmit {
|
|
return errors.New("an effectful failure cannot be safe to resubmit")
|
|
}
|
|
if record.State != StateFailed && (record.EffectMayHaveApplied || record.SafeToResubmit) {
|
|
return errors.New("failure semantics require a failed operation")
|
|
}
|
|
if record.CauseCode != "" && !validCauseCode(record.CauseCode) {
|
|
return errors.New("invalid operation cause_code")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func putRecord(bucket *bolt.Bucket, record Record) error {
|
|
if err := validateRecord(record); err != nil {
|
|
return err
|
|
}
|
|
encoded, err := json.Marshal(storedRecord{
|
|
Record: recordAlias(record), ExpectedState: cloneExpectedState(record.ExpectedState),
|
|
LockKeys: append([]string(nil), record.LockKeys...), HeavyIO: record.HeavyIO,
|
|
SupersedesOperationID: record.SupersedesOperationID,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("encode operation record: %w", err)
|
|
}
|
|
return bucket.Put([]byte(record.OperationID), encoded)
|
|
}
|
|
|
|
func decodeRecord(encoded []byte) (Record, error) {
|
|
var stored storedRecord
|
|
if err := json.Unmarshal(encoded, &stored); err != nil {
|
|
return Record{}, fmt.Errorf("decode operation record: %w", err)
|
|
}
|
|
record := Record(stored.Record)
|
|
record.ExpectedState = cloneExpectedState(stored.ExpectedState)
|
|
record.LockKeys = append([]string(nil), stored.LockKeys...)
|
|
record.HeavyIO = stored.HeavyIO
|
|
record.SupersedesOperationID = stored.SupersedesOperationID
|
|
if err := validateRecord(record); err != nil {
|
|
return Record{}, err
|
|
}
|
|
return record, nil
|
|
}
|
|
|
|
func cloneRecord(record Record) Record {
|
|
copyRecord := record
|
|
copyRecord.Result = append(json.RawMessage(nil), record.Result...)
|
|
copyRecord.ExpectedState = cloneExpectedState(record.ExpectedState)
|
|
copyRecord.LockKeys = append([]string(nil), record.LockKeys...)
|
|
if record.Error != nil {
|
|
copyError := *record.Error
|
|
copyRecord.Error = ©Error
|
|
}
|
|
return copyRecord
|
|
}
|
|
|
|
func cloneExpectedState(expected *ExpectedState) *ExpectedState {
|
|
if expected == nil {
|
|
return nil
|
|
}
|
|
copyExpected := *expected
|
|
copyExpected.PublicNetwork.DNSServers = append([]string(nil), expected.PublicNetwork.DNSServers...)
|
|
return ©Expected
|
|
}
|
|
|
|
func validExpectedPublicNetwork(value publicnetwork.Intent) bool {
|
|
canonical, err := value.Canonical()
|
|
return err == nil && canonical.Contract == value.Contract &&
|
|
canonical.NetworkSelector == value.NetworkSelector && canonical.IPv4 == value.IPv4 &&
|
|
canonical.IPv6 == value.IPv6 && slices.Equal(canonical.DNSServers, value.DNSServers)
|
|
}
|
|
|
|
func encodeExpiry(value time.Time) []byte {
|
|
encoded := make([]byte, 8)
|
|
binary.BigEndian.PutUint64(encoded, uint64(value.UnixNano()))
|
|
return encoded
|
|
}
|
|
|
|
func decodeExpiry(encoded []byte) (time.Time, error) {
|
|
if len(encoded) != 8 {
|
|
return time.Time{}, errors.New("invalid nonce expiry record")
|
|
}
|
|
return time.Unix(0, int64(binary.BigEndian.Uint64(encoded))).UTC(), nil
|
|
}
|
|
|
|
func expiryIndexKey(expiresAt time.Time, nonce string) []byte {
|
|
result := make([]byte, 8+len(nonce))
|
|
copy(result, encodeExpiry(expiresAt))
|
|
copy(result[8:], nonce)
|
|
return result
|
|
}
|
|
|
|
func operationExpiryKey(terminalAt time.Time, operationID string) []byte {
|
|
return expiryIndexKey(terminalAt, operationID)
|
|
}
|
|
|
|
func terminalState(state State) bool {
|
|
return state == StateSucceeded || state == StateFailed
|
|
}
|
|
|
|
func pruneExpiredOperationDetails(tx *bolt.Tx, now time.Time) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
expiry := tx.Bucket(operationExpiryBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
resolutions := tx.Bucket(fenceResolutionsBucket)
|
|
if operations == nil || expiry == nil || blockers == nil || resolutions == nil {
|
|
return errors.New("operation retention buckets are unavailable")
|
|
}
|
|
cutoff := now.Add(-operationDetailRetention).UnixNano()
|
|
cursor := expiry.Cursor()
|
|
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
|
|
if len(key) < 9 || int64(binary.BigEndian.Uint64(key[:8])) > cutoff {
|
|
break
|
|
}
|
|
operationID := string(key[8:])
|
|
retainExpiry := false
|
|
encoded := operations.Get([]byte(operationID))
|
|
if encoded != nil {
|
|
record, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
indexedAt := int64(binary.BigEndian.Uint64(key[:8]))
|
|
if terminalState(record.State) && !record.Tombstone && record.UpdatedAt.UnixNano() == indexedAt {
|
|
// Effectful failures remain durable resource fences. Other terminal
|
|
// details may expire, while the operation ID tombstone remains. A
|
|
// durable fence resolution makes the formerly effectful record safe
|
|
// to compact without rewriting its immutable resolution audit.
|
|
if !blocksResources(record) || resolutions.Get([]byte(operationID)) != nil {
|
|
previous := cloneRecord(record)
|
|
record.Tombstone = true
|
|
record.Result = nil
|
|
record.ExpectedState = nil
|
|
record.LockKeys = nil
|
|
record.HeavyIO = false
|
|
if record.Error != nil {
|
|
record.Error = &PublicError{Code: record.Error.Code, Message: "operation details expired; operation_id remains permanently reserved"}
|
|
}
|
|
if err := putRecord(operations, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
// Keep the expiry index while an old effectful failure still
|
|
// owns its fence. A later resolution leaves the immutable
|
|
// operation untouched, so this retained key is what lets the
|
|
// next bounded prune compact the full record.
|
|
retainExpiry = true
|
|
}
|
|
}
|
|
}
|
|
if !retainExpiry {
|
|
if err := cursor.Delete(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func pruneExpiredNonces(nonces, expiry *bolt.Bucket, now time.Time) {
|
|
cursor := expiry.Cursor()
|
|
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
|
|
if len(key) < 9 || int64(binary.BigEndian.Uint64(key[:8])) >= now.UnixNano() {
|
|
break
|
|
}
|
|
nonce := key[8:]
|
|
if encoded := nonces.Get(nonce); encoded != nil && len(encoded) == 8 && string(encoded) == string(key[:8]) {
|
|
_ = nonces.Delete(nonce)
|
|
}
|
|
_ = cursor.Delete()
|
|
}
|
|
}
|
|
|
|
type expectedStateContextKey struct{}
|