You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2370 lines
81 KiB
Go
2370 lines
81 KiB
Go
package operation
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
bolt "go.etcd.io/bbolt"
|
|
)
|
|
|
|
type State string
|
|
|
|
const (
|
|
StateReserved State = "reserved"
|
|
StateDispatching State = "dispatching"
|
|
StateSucceeded State = "succeeded"
|
|
StateFailed State = "failed"
|
|
)
|
|
|
|
const (
|
|
ExpectedUpdateSettings = "update-settings"
|
|
ExpectedChangeNetwork = "change-network"
|
|
ExpectedNetworkState = "network-state"
|
|
ExpectedServiceState = "service-state"
|
|
ExpectedFirewall = "firewall"
|
|
ExpectedStorageConfiguration = "storage-configuration"
|
|
operationDetailRetention = 90 * 24 * time.Hour
|
|
)
|
|
|
|
var (
|
|
ErrNotFound = errors.New("operation not found")
|
|
ErrFingerprint = errors.New("operation_id payload conflict")
|
|
ErrOperationStarting = errors.New("operation dispatch is already starting")
|
|
ErrBusy = errors.New("operation execution capacity is exhausted")
|
|
ErrResourceBlocked = errors.New("resource has an unresolved operation")
|
|
ErrInvalidSupersession = errors.New("operation cannot supersede the requested predecessor")
|
|
ErrInspectionInconclusive = errors.New("read-only inspection did not confirm a terminal outcome")
|
|
ErrExecutionNotReaped = errors.New("isolated mutation process did not exit after forced termination")
|
|
ErrFenceNotResolvable = errors.New("operation does not hold an effect fence")
|
|
ErrFenceResolutionConflict = errors.New("operation effect fence already has a different resolution")
|
|
ErrFenceInspectionStale = errors.New("operation effect fence resolution requires a fresh inspection")
|
|
operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`)
|
|
hexDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
|
noncePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`)
|
|
networkSourcePattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,128}$`)
|
|
causeCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
|
|
canonicalUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
|
|
resourcePartPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,192}$`)
|
|
resourceKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
|
|
fenceResolutionIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
|
|
|
|
metadataBucket = []byte("metadata")
|
|
operationsBucket = []byte("operations")
|
|
inflightBucket = []byte("inflight")
|
|
resourceBlockersBucket = []byte("resource_blockers")
|
|
operationExpiryBucket = []byte("operation_expiry")
|
|
completionReceiptsBucket = []byte("completion_receipts")
|
|
completionOutboxBucket = []byte("completion_outbox")
|
|
fenceResolutionsBucket = []byte("fence_resolutions")
|
|
fenceResolutionIDsBucket = []byte("fence_resolution_ids")
|
|
noncesBucket = []byte("nonces")
|
|
nonceExpiryBucket = []byte("nonce_expiry")
|
|
schemaVersionKey = []byte("schema_version")
|
|
schemaVersion = []byte("6")
|
|
)
|
|
|
|
// ExpectedState is the non-secret, durable evidence contract used after a
|
|
// crash. It intentionally has no password, SSH key, token, file path, or raw
|
|
// request field. The exact request remains bound separately by Fingerprint.
|
|
type ExpectedState struct {
|
|
Kind string `json:"kind"`
|
|
TransitionID string `json:"transition_id"`
|
|
|
|
HostnameChanged bool `json:"hostname_changed,omitempty"`
|
|
Hostname string `json:"hostname,omitempty"`
|
|
RootPasswordChanged bool `json:"root_password_changed,omitempty"`
|
|
SSHKeyChanged bool `json:"ssh_key_changed,omitempty"`
|
|
|
|
NetworkSelector string `json:"network,omitempty"`
|
|
NetworkSourceType string `json:"network_source_type,omitempty"`
|
|
NetworkSourceName string `json:"network_source_name,omitempty"`
|
|
PublicPoolKey string `json:"public_pool_key,omitempty"`
|
|
PublicPoolCIDR string `json:"public_pool_cidr,omitempty"`
|
|
PublicIPCIDR string `json:"public_ip_cidr,omitempty"`
|
|
Gateway string `json:"gateway,omitempty"`
|
|
DNSServers []string `json:"dns_servers,omitempty"`
|
|
|
|
Enabled bool `json:"enabled,omitempty"`
|
|
StartIfSuspended bool `json:"start_if_suspended,omitempty"`
|
|
|
|
FirewallRevision uint64 `json:"firewall_revision,omitempty"`
|
|
FirewallDigest string `json:"firewall_digest,omitempty"`
|
|
|
|
ManagedPathDigest string `json:"managed_path_digest,omitempty"`
|
|
TemplatePathDigest string `json:"template_path_digest,omitempty"`
|
|
SeedPathDigest string `json:"seed_path_digest,omitempty"`
|
|
UserISOPathDigest string `json:"user_iso_path_digest,omitempty"`
|
|
RescuePathDigest string `json:"rescue_path_digest,omitempty"`
|
|
}
|
|
|
|
type PublicError struct {
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
type Record struct {
|
|
OperationID string `json:"operation_id"`
|
|
ResourceID string `json:"resource_id,omitempty"`
|
|
Action string `json:"action"`
|
|
Fingerprint string `json:"fingerprint"`
|
|
Destructive bool `json:"destructive"`
|
|
State State `json:"status"`
|
|
Result json.RawMessage `json:"result,omitempty"`
|
|
Error *PublicError `json:"error,omitempty"`
|
|
CauseCode string `json:"cause_code,omitempty"`
|
|
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
|
|
SafeToResubmit bool `json:"safe_to_resubmit"`
|
|
ExecutionFenced bool `json:"execution_fenced,omitempty"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
Tombstone bool `json:"tombstone,omitempty"`
|
|
|
|
// ExpectedState is persisted in the private journal but deliberately never
|
|
// serialized on the HTTP API.
|
|
ExpectedState *ExpectedState `json:"-"`
|
|
// LockKeys and HeavyIO are private admission metadata. They are persisted so
|
|
// restart terminalization can retain the exact affected-resource fence, but
|
|
// they are never exposed by the Agent API.
|
|
LockKeys []string `json:"-"`
|
|
HeavyIO bool `json:"-"`
|
|
// SupersedesOperationID is only valid for an explicitly acknowledged
|
|
// force-delete-vm successor of one matching graceful/force_required record.
|
|
SupersedesOperationID string `json:"-"`
|
|
}
|
|
|
|
type Spec struct {
|
|
OperationID string
|
|
ResourceID string
|
|
Action string
|
|
Fingerprint string
|
|
Destructive bool
|
|
ExpectedState *ExpectedState
|
|
LockKeys []string
|
|
HeavyIO bool
|
|
SupersedesOperationID string
|
|
}
|
|
|
|
const (
|
|
completionDeliveryMaxAttempts = uint32(12)
|
|
completionDeliveryLifetime = 24 * time.Hour
|
|
completionDeliveryInterval = 30 * time.Second
|
|
fenceInspectionMaximumAge = 2 * time.Minute
|
|
fenceInspectionFutureSkew = 5 * time.Second
|
|
)
|
|
|
|
var workerReapDeadline = 3 * time.Second
|
|
|
|
// CompletionReceipt is immutable once an operation first reaches a terminal
|
|
// state. Delivery attempts are kept in a separate bounded outbox record so an
|
|
// ACK can never rewrite the operation or its receipt.
|
|
type CompletionReceipt struct {
|
|
EventID string `json:"id"`
|
|
OperationID string `json:"operation_id"`
|
|
Status string `json:"status"`
|
|
Result json.RawMessage `json:"result"`
|
|
ErrorCode string `json:"error_code,omitempty"`
|
|
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
|
|
SafeToResubmit bool `json:"safe_to_resubmit"`
|
|
CreatedAt time.Time `json:"-"`
|
|
}
|
|
|
|
const (
|
|
FenceResolutionApplied = "applied"
|
|
FenceResolutionNotApplied = "not_applied"
|
|
)
|
|
|
|
// FenceResolution is immutable audit metadata for a super-administrator's
|
|
// explicit conclusion after a fresh, read-only host inspection. It releases
|
|
// only Journal admission blockers; it never changes the original operation or
|
|
// completion receipt and never authorizes replay of that mutation.
|
|
type FenceResolution struct {
|
|
OperationID string `json:"operation_id"`
|
|
ResolutionID string `json:"resolution_id"`
|
|
Resolution string `json:"resolution"`
|
|
InspectionCheckedAt int64 `json:"inspection_checked_at"`
|
|
ReasonSHA256 string `json:"reason_sha256"`
|
|
ResolvedAt int64 `json:"resolved_at"`
|
|
ResolvedBy int64 `json:"resolved_by"`
|
|
}
|
|
|
|
type FenceResolutionRequest struct {
|
|
ResolutionID string `json:"resolution_id"`
|
|
Resolution string `json:"resolution"`
|
|
InspectionCheckedAt int64 `json:"inspection_checked_at"`
|
|
ReasonSHA256 string `json:"reason_sha256"`
|
|
}
|
|
|
|
// InterruptedReceipt is the non-secret terminal result recovered from an
|
|
// isolated worker directory after the parent Agent restarts.
|
|
type InterruptedReceipt struct {
|
|
OperationID string
|
|
Fingerprint string
|
|
Status string
|
|
Result json.RawMessage
|
|
ErrorCode string
|
|
ErrorMessage string
|
|
EffectMayHaveApplied bool
|
|
SafeToResubmit bool
|
|
}
|
|
|
|
type completionOutboxRecord struct {
|
|
OperationID string `json:"operation_id"`
|
|
Attempts uint32 `json:"attempts"`
|
|
NextAttemptAt time.Time `json:"next_attempt_at"`
|
|
ExpiresAt time.Time `json:"expires_at"`
|
|
}
|
|
|
|
type Result struct {
|
|
Value json.RawMessage
|
|
}
|
|
|
|
type Resolution struct {
|
|
Resolved bool
|
|
Succeeded bool
|
|
Result json.RawMessage
|
|
ErrorCode string
|
|
ErrorMessage string
|
|
}
|
|
|
|
type UncertainError struct {
|
|
Cause error
|
|
}
|
|
|
|
func (e *UncertainError) Error() string {
|
|
if e.Cause == nil {
|
|
return "operation result is unconfirmed"
|
|
}
|
|
return fmt.Sprintf("operation result is unconfirmed: %v", e.Cause)
|
|
}
|
|
|
|
func (e *UncertainError) Unwrap() error { return e.Cause }
|
|
|
|
func MarkUncertain(err error) error { return &UncertainError{Cause: err} }
|
|
|
|
func IsUncertain(err error) bool {
|
|
var target *UncertainError
|
|
return errors.As(err, &target) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
|
|
}
|
|
|
|
type recordAlias Record
|
|
|
|
type storedRecord struct {
|
|
Record recordAlias `json:"record"`
|
|
ExpectedState *ExpectedState `json:"expected_state,omitempty"`
|
|
LockKeys []string `json:"lock_keys,omitempty"`
|
|
HeavyIO bool `json:"heavy_io,omitempty"`
|
|
SupersedesOperationID string `json:"supersedes_operation_id,omitempty"`
|
|
}
|
|
|
|
type resourceLock struct {
|
|
semaphore chan struct{}
|
|
references int
|
|
}
|
|
|
|
type Journal struct {
|
|
path string
|
|
db *bolt.DB
|
|
|
|
clockMu sync.RWMutex
|
|
now func() time.Time
|
|
|
|
resourceMu sync.Mutex
|
|
resources map[string]*resourceLock
|
|
|
|
admissionClaimMu sync.Mutex
|
|
admissionClaims map[string]struct{}
|
|
|
|
executionSlots chan struct{}
|
|
heavyIOSlots chan struct{}
|
|
workerWG sync.WaitGroup
|
|
|
|
// backgroundFailures carries the first durable finalization failure to the
|
|
// process supervisor. ExecuteBackground has already crossed the dispatch
|
|
// boundary when these failures occur, so the mutation must never be
|
|
// replayed in-process. The main Agent exits cleanly and startup recovery
|
|
// imports an immutable helper receipt, or terminally records an unconfirmed
|
|
// result when no receipt exists.
|
|
backgroundFailures chan error
|
|
}
|
|
|
|
const (
|
|
maxConcurrentExecutions = 32
|
|
DefaultHeavyIOConcurrency = 4
|
|
maxAdmissionResourceKeys = 16
|
|
maximumAdmissionKeyLength = 255
|
|
)
|
|
|
|
func Open(path string) (*Journal, error) {
|
|
return OpenWithHeavyIOConcurrency(path, DefaultHeavyIOConcurrency)
|
|
}
|
|
|
|
func OpenWithHeavyIOConcurrency(path string, heavyIOConcurrency int) (*Journal, error) {
|
|
if strings.TrimSpace(path) == "" {
|
|
return nil, errors.New("journal path is required")
|
|
}
|
|
if heavyIOConcurrency < 1 {
|
|
return nil, errors.New("heavy I/O concurrency must be positive")
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
|
return nil, fmt.Errorf("create operation journal directory: %w", err)
|
|
}
|
|
db, err := bolt.Open(path, 0o600, &bolt.Options{Timeout: 2 * time.Second})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open operation journal database: %w", err)
|
|
}
|
|
j := &Journal{
|
|
path: path, db: db, now: time.Now, resources: make(map[string]*resourceLock),
|
|
admissionClaims: make(map[string]struct{}),
|
|
executionSlots: make(chan struct{}, maxConcurrentExecutions),
|
|
heavyIOSlots: make(chan struct{}, heavyIOConcurrency),
|
|
backgroundFailures: make(chan error, 1),
|
|
}
|
|
if err := j.initialize(); err != nil {
|
|
_ = db.Close()
|
|
return nil, err
|
|
}
|
|
return j, nil
|
|
}
|
|
|
|
func (j *Journal) Close() error {
|
|
if j == nil || j.db == nil {
|
|
return nil
|
|
}
|
|
return j.db.Close()
|
|
}
|
|
|
|
// Wait blocks until every real mutation and late-result finalizer has stopped.
|
|
// Callers must stop API admission before invoking it so no new worker can be
|
|
// added concurrently with the wait.
|
|
func (j *Journal) Wait(ctx context.Context) error {
|
|
done := make(chan struct{})
|
|
go func() {
|
|
j.workerWG.Wait()
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
// BackgroundFailures reports an unrecoverable failure to durably finalize an
|
|
// already-dispatched background operation. Callers must stop request
|
|
// admission and restart the Agent; FinalizeInterruptedOperations then settles
|
|
// the dispatching record without invoking the mutation again.
|
|
func (j *Journal) BackgroundFailures() <-chan error {
|
|
return j.backgroundFailures
|
|
}
|
|
|
|
func (j *Journal) reportBackgroundFailure(operationID, phase string, err error) {
|
|
if err == nil {
|
|
return
|
|
}
|
|
failure := fmt.Errorf("%s background operation %s: %w", phase, operationID, err)
|
|
select {
|
|
case j.backgroundFailures <- failure:
|
|
default:
|
|
// The first failure is sufficient to force a supervised restart. Never
|
|
// block finalizers while the process is already shutting down.
|
|
}
|
|
}
|
|
|
|
// UseNonce uses two Bolt buckets: direct lookup is O(1), while the ordered
|
|
// expiry index makes cleanup proportional only to expired entries. It never
|
|
// rewrites or decodes the operation history.
|
|
func (j *Journal) UseNonce(nonce string, now, expiresAt time.Time) (bool, error) {
|
|
if !noncePattern.MatchString(nonce) {
|
|
return false, errors.New("invalid nonce")
|
|
}
|
|
if !expiresAt.After(now) {
|
|
return false, errors.New("nonce expiry must be in the future")
|
|
}
|
|
accepted := false
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
nonces := tx.Bucket(noncesBucket)
|
|
expiry := tx.Bucket(nonceExpiryBucket)
|
|
if nonces == nil || expiry == nil {
|
|
return errors.New("nonce buckets are unavailable")
|
|
}
|
|
if err := pruneExpiredOperationDetails(tx, now); err != nil {
|
|
return err
|
|
}
|
|
pruneExpiredNonces(nonces, expiry, now)
|
|
key := []byte(nonce)
|
|
if encoded := nonces.Get(key); encoded != nil {
|
|
current, err := decodeExpiry(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !current.Before(now) {
|
|
return nil
|
|
}
|
|
if err := expiry.Delete(expiryIndexKey(current, nonce)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
encoded := encodeExpiry(expiresAt.UTC())
|
|
if err := nonces.Put(key, encoded); err != nil {
|
|
return err
|
|
}
|
|
if err := expiry.Put(expiryIndexKey(expiresAt.UTC(), nonce), nil); err != nil {
|
|
return err
|
|
}
|
|
accepted = true
|
|
return nil
|
|
})
|
|
return accepted, err
|
|
}
|
|
|
|
func Fingerprint(method, target string, body []byte) string {
|
|
sum := sha256.Sum256([]byte(strings.ToUpper(method) + "\n" + target + "\n" + string(body)))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func (j *Journal) SetClockForTesting(now func() time.Time) {
|
|
j.clockMu.Lock()
|
|
defer j.clockMu.Unlock()
|
|
j.now = now
|
|
}
|
|
|
|
func (j *Journal) currentTime() time.Time {
|
|
j.clockMu.RLock()
|
|
defer j.clockMu.RUnlock()
|
|
return j.now().UTC()
|
|
}
|
|
|
|
func (j *Journal) Get(operationID string) (Record, error) {
|
|
var record Record
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(operationsBucket)
|
|
if bucket == nil {
|
|
return errors.New("operation bucket is unavailable")
|
|
}
|
|
value := bucket.Get([]byte(operationID))
|
|
if value == nil {
|
|
return ErrNotFound
|
|
}
|
|
decoded, err := decodeRecord(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
record = decoded
|
|
return nil
|
|
})
|
|
return cloneRecord(record), err
|
|
}
|
|
|
|
func ValidateFenceResolutionRequest(request FenceResolutionRequest) error {
|
|
if !fenceResolutionIDPattern.MatchString(request.ResolutionID) {
|
|
return errors.New("resolution_id must be 32 lowercase hexadecimal characters")
|
|
}
|
|
if request.Resolution != FenceResolutionApplied && request.Resolution != FenceResolutionNotApplied {
|
|
return errors.New("resolution must be applied or not_applied")
|
|
}
|
|
if request.InspectionCheckedAt < 1 {
|
|
return errors.New("inspection_checked_at must be a positive Unix timestamp")
|
|
}
|
|
if !hexDigestPattern.MatchString(request.ReasonSHA256) {
|
|
return errors.New("reason_sha256 must be a 64-character lowercase SHA-256 digest")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// GetFenceResolution reads immutable, bounded audit metadata without touching
|
|
// operation state or host resources.
|
|
func (j *Journal) GetFenceResolution(operationID string) (FenceResolution, error) {
|
|
if !operationIDPattern.MatchString(operationID) {
|
|
return FenceResolution{}, ErrNotFound
|
|
}
|
|
var resolution FenceResolution
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(fenceResolutionsBucket)
|
|
if bucket == nil {
|
|
return errors.New("fence resolution bucket is unavailable")
|
|
}
|
|
encoded := bucket.Get([]byte(operationID))
|
|
if encoded == nil {
|
|
return ErrNotFound
|
|
}
|
|
if err := json.Unmarshal(encoded, &resolution); err != nil || resolution.OperationID != operationID {
|
|
return fmt.Errorf("decode fence resolution %q", operationID)
|
|
}
|
|
return nil
|
|
})
|
|
return resolution, err
|
|
}
|
|
|
|
// ResolveResourceFence records one explicit, audited effect conclusion and
|
|
// removes only this operation's admission-index entries. It never mutates the
|
|
// operation record or completion receipt and never invokes a host mutation.
|
|
func (j *Journal) ResolveResourceFence(operationID string, resolvedBy int64,
|
|
request FenceResolutionRequest) (FenceResolution, bool, error) {
|
|
if !operationIDPattern.MatchString(operationID) {
|
|
return FenceResolution{}, false, ErrNotFound
|
|
}
|
|
if resolvedBy < 1 {
|
|
return FenceResolution{}, false, errors.New("resolved_by must be a positive user ID")
|
|
}
|
|
if err := ValidateFenceResolutionRequest(request); err != nil {
|
|
return FenceResolution{}, false, err
|
|
}
|
|
now := j.currentTime().UTC()
|
|
var resolution FenceResolution
|
|
replayed := false
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
resolutions := tx.Bucket(fenceResolutionsBucket)
|
|
resolutionIDs := tx.Bucket(fenceResolutionIDsBucket)
|
|
if operations == nil || blockers == nil || resolutions == nil || resolutionIDs == nil {
|
|
return errors.New("operation fence buckets are unavailable")
|
|
}
|
|
if encoded := resolutions.Get([]byte(operationID)); encoded != nil {
|
|
if err := json.Unmarshal(encoded, &resolution); err != nil || resolution.OperationID != operationID {
|
|
return fmt.Errorf("decode fence resolution %q", operationID)
|
|
}
|
|
if resolution.ResolutionID != request.ResolutionID || resolution.Resolution != request.Resolution ||
|
|
resolution.InspectionCheckedAt != request.InspectionCheckedAt || resolution.ReasonSHA256 != request.ReasonSHA256 ||
|
|
resolution.ResolvedBy != resolvedBy {
|
|
return ErrFenceResolutionConflict
|
|
}
|
|
replayed = true
|
|
return nil
|
|
}
|
|
if owner := resolutionIDs.Get([]byte(request.ResolutionID)); owner != nil {
|
|
return ErrFenceResolutionConflict
|
|
}
|
|
checkedAt := time.Unix(request.InspectionCheckedAt, 0).UTC()
|
|
if checkedAt.Before(now.Add(-fenceInspectionMaximumAge)) || checkedAt.After(now.Add(fenceInspectionFutureSkew)) {
|
|
return ErrFenceInspectionStale
|
|
}
|
|
encoded := operations.Get([]byte(operationID))
|
|
if encoded == nil {
|
|
return ErrNotFound
|
|
}
|
|
record, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !blocksResources(record) {
|
|
return ErrFenceNotResolvable
|
|
}
|
|
lockKeys := effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys)
|
|
for _, lockKey := range lockKeys {
|
|
key := blockerEntryKey(lockKey, operationID)
|
|
cursor := blockers.Cursor()
|
|
foundKey, _ := cursor.Seek(key)
|
|
if !bytes.Equal(foundKey, key) {
|
|
return ErrFenceNotResolvable
|
|
}
|
|
}
|
|
resolution = FenceResolution{
|
|
OperationID: operationID, ResolutionID: request.ResolutionID,
|
|
Resolution: request.Resolution, InspectionCheckedAt: request.InspectionCheckedAt,
|
|
ReasonSHA256: request.ReasonSHA256, ResolvedAt: now.Unix(), ResolvedBy: resolvedBy,
|
|
}
|
|
resolutionEncoded, err := json.Marshal(resolution)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := resolutions.Put([]byte(operationID), resolutionEncoded); err != nil {
|
|
return err
|
|
}
|
|
if err := resolutionIDs.Put([]byte(request.ResolutionID), []byte(operationID)); err != nil {
|
|
return err
|
|
}
|
|
for _, lockKey := range lockKeys {
|
|
if err := blockers.Delete(blockerEntryKey(lockKey, operationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
return resolution, replayed, err
|
|
}
|
|
|
|
// ClaimPendingCompletions returns a bounded batch and durably charges one
|
|
// delivery attempt for every returned event. Expired or exhausted events are
|
|
// removed from the outbox; their immutable receipts remain available for
|
|
// diagnostics and duplicate ACKs remain harmless.
|
|
func (j *Journal) ClaimPendingCompletions(limit int) ([]CompletionReceipt, error) {
|
|
claimed, _, err := j.ClaimPendingCompletionsWithExpired(limit)
|
|
return claimed, err
|
|
}
|
|
|
|
// ClaimPendingCompletionsWithExpired additionally returns the operation IDs
|
|
// whose bounded delivery window ended during this claim. The Agent uses those
|
|
// IDs only to remove disposable worker directories; operation tombstones and
|
|
// immutable completion receipts remain in the main journal.
|
|
func (j *Journal) ClaimPendingCompletionsWithExpired(limit int) ([]CompletionReceipt, []string, error) {
|
|
return j.ClaimPendingCompletionsWithExpiredExcluding(limit, nil)
|
|
}
|
|
|
|
// ClaimPendingCompletionsWithExpiredExcluding leaves events already in the
|
|
// current control session's ACK window untouched. An in-flight receipt cannot
|
|
// consume another durable attempt merely because its resend time arrived
|
|
// while the Gateway was still processing the first delivery.
|
|
func (j *Journal) ClaimPendingCompletionsWithExpiredExcluding(limit int, excluded []string) ([]CompletionReceipt, []string, error) {
|
|
if limit < 1 || limit > 128 {
|
|
return nil, nil, errors.New("completion claim limit must be between 1 and 128")
|
|
}
|
|
excludedEvents := make(map[string]struct{}, len(excluded))
|
|
for _, eventID := range excluded {
|
|
if !validCompletionEventID(eventID) {
|
|
return nil, nil, errors.New("invalid excluded completion event ID")
|
|
}
|
|
excludedEvents[eventID] = struct{}{}
|
|
}
|
|
claimed := make([]CompletionReceipt, 0, limit)
|
|
expired := make([]string, 0)
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if receipts == nil || outbox == nil {
|
|
return errors.New("completion buckets are unavailable")
|
|
}
|
|
now := j.currentTime()
|
|
cursor := outbox.Cursor()
|
|
for key, encoded := cursor.First(); key != nil && len(claimed) < limit; key, encoded = cursor.Next() {
|
|
var pending completionOutboxRecord
|
|
if err := json.Unmarshal(encoded, &pending); err != nil || pending.OperationID == "" ||
|
|
pending.NextAttemptAt.IsZero() || pending.ExpiresAt.IsZero() {
|
|
return fmt.Errorf("decode completion outbox event %q", key)
|
|
}
|
|
if _, excluded := excludedEvents[string(key)]; excluded {
|
|
continue
|
|
}
|
|
if !now.Before(pending.ExpiresAt) || pending.Attempts >= completionDeliveryMaxAttempts {
|
|
expired = append(expired, pending.OperationID)
|
|
if err := cursor.Delete(); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
if now.Before(pending.NextAttemptAt) {
|
|
continue
|
|
}
|
|
receiptEncoded := receipts.Get(key)
|
|
if receiptEncoded == nil {
|
|
return fmt.Errorf("completion outbox references missing receipt %q", key)
|
|
}
|
|
var receipt CompletionReceipt
|
|
if err := json.Unmarshal(receiptEncoded, &receipt); err != nil || receipt.EventID != string(key) || receipt.OperationID != pending.OperationID {
|
|
return fmt.Errorf("decode completion receipt %q", key)
|
|
}
|
|
pending.Attempts++
|
|
pending.NextAttemptAt = now.Add(completionDeliveryInterval)
|
|
updated, err := json.Marshal(pending)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := outbox.Put(key, updated); err != nil {
|
|
return err
|
|
}
|
|
receipt.Result = append(json.RawMessage(nil), receipt.Result...)
|
|
claimed = append(claimed, receipt)
|
|
}
|
|
return nil
|
|
})
|
|
return claimed, expired, err
|
|
}
|
|
|
|
// CompletionOperationID resolves a still-pending delivery marker without
|
|
// changing it. It lets the Agent remove disposable worker state before
|
|
// durably consuming an ACK, so a cleanup failure can be retried safely.
|
|
func (j *Journal) CompletionOperationID(eventID string) (string, error) {
|
|
if !validCompletionEventID(eventID) {
|
|
return "", errors.New("invalid completion event ID")
|
|
}
|
|
var operationID string
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if outbox == nil {
|
|
return errors.New("completion outbox is unavailable")
|
|
}
|
|
encoded := outbox.Get([]byte(eventID))
|
|
if encoded == nil {
|
|
return nil
|
|
}
|
|
var pending completionOutboxRecord
|
|
if err := json.Unmarshal(encoded, &pending); err != nil || !operationIDPattern.MatchString(pending.OperationID) {
|
|
return fmt.Errorf("decode completion outbox event %q", eventID)
|
|
}
|
|
operationID = pending.OperationID
|
|
return nil
|
|
})
|
|
return operationID, err
|
|
}
|
|
|
|
// PendingCompletionOperationIDs returns the worker directories that must be
|
|
// retained across restart until their terminal receipt is ACKed or expires.
|
|
func (j *Journal) PendingCompletionOperationIDs() (map[string]struct{}, error) {
|
|
operationIDs := make(map[string]struct{})
|
|
err := j.db.View(func(tx *bolt.Tx) error {
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if outbox == nil {
|
|
return errors.New("completion outbox is unavailable")
|
|
}
|
|
return outbox.ForEach(func(key, encoded []byte) error {
|
|
var pending completionOutboxRecord
|
|
if err := json.Unmarshal(encoded, &pending); err != nil || !operationIDPattern.MatchString(pending.OperationID) {
|
|
return fmt.Errorf("decode completion outbox event %q", key)
|
|
}
|
|
operationIDs[pending.OperationID] = struct{}{}
|
|
return nil
|
|
})
|
|
})
|
|
return operationIDs, err
|
|
}
|
|
|
|
// AcknowledgeCompletion only removes the delivery marker. It deliberately
|
|
// cannot update the operation or immutable completion receipt.
|
|
func (j *Journal) AcknowledgeCompletion(eventID string) error {
|
|
if !validCompletionEventID(eventID) {
|
|
return errors.New("invalid completion event ID")
|
|
}
|
|
return j.db.Update(func(tx *bolt.Tx) error {
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if outbox == nil {
|
|
return errors.New("completion outbox is unavailable")
|
|
}
|
|
return outbox.Delete([]byte(eventID))
|
|
})
|
|
}
|
|
|
|
// FinalizeInterruptedOperations is called exactly once during Agent startup,
|
|
// before request admission. It imports already-written worker receipts and
|
|
// terminally fails every other dispatching row without spawning a worker.
|
|
func (j *Journal) FinalizeInterruptedOperations(workerReceipts []InterruptedReceipt) ([]string, error) {
|
|
byOperation := make(map[string]InterruptedReceipt, len(workerReceipts))
|
|
for _, receipt := range workerReceipts {
|
|
if !operationIDPattern.MatchString(receipt.OperationID) || !hexDigestPattern.MatchString(receipt.Fingerprint) ||
|
|
(receipt.Status != "succeeded" && receipt.Status != "failed") ||
|
|
(receipt.SafeToResubmit && receipt.EffectMayHaveApplied) ||
|
|
(len(receipt.Result) != 0 && !json.Valid(receipt.Result)) {
|
|
return nil, errors.New("invalid interrupted worker receipt")
|
|
}
|
|
if receipt.Status == "failed" && (receipt.ErrorCode == "" || !validCauseCode(receipt.ErrorCode)) {
|
|
return nil, errors.New("invalid interrupted worker failure")
|
|
}
|
|
if _, duplicate := byOperation[receipt.OperationID]; duplicate {
|
|
return nil, errors.New("duplicate interrupted worker receipt")
|
|
}
|
|
byOperation[receipt.OperationID] = receipt
|
|
}
|
|
finalized := make([]string, 0)
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
inflight := tx.Bucket(inflightBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
expiry := tx.Bucket(operationExpiryBucket)
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if operations == nil || inflight == nil || blockers == nil || expiry == nil || receipts == nil || outbox == nil {
|
|
return errors.New("operation buckets are unavailable")
|
|
}
|
|
now := j.currentTime()
|
|
var keys [][]byte
|
|
if err := inflight.ForEach(func(key, _ []byte) error {
|
|
keys = append(keys, append([]byte(nil), key...))
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
for _, key := range keys {
|
|
encoded := operations.Get(key)
|
|
if encoded == nil {
|
|
return fmt.Errorf("inflight index references missing operation %q", key)
|
|
}
|
|
record, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if record.State != StateDispatching {
|
|
continue
|
|
}
|
|
previous := cloneRecord(record)
|
|
receipt, found := byOperation[record.OperationID]
|
|
if found && receipt.Fingerprint == record.Fingerprint {
|
|
record.Result = append(json.RawMessage(nil), receipt.Result...)
|
|
if receipt.Status == "succeeded" {
|
|
record.State = StateSucceeded
|
|
record.Error = nil
|
|
record.CauseCode = ""
|
|
record.EffectMayHaveApplied = false
|
|
record.SafeToResubmit = false
|
|
} else {
|
|
record.State = StateFailed
|
|
record.Error = &PublicError{Code: receipt.ErrorCode, Message: receipt.ErrorMessage}
|
|
record.CauseCode = receipt.ErrorCode
|
|
record.EffectMayHaveApplied = receipt.EffectMayHaveApplied
|
|
record.SafeToResubmit = receipt.SafeToResubmit
|
|
}
|
|
record.ExecutionFenced = false
|
|
} else {
|
|
record.State = StateFailed
|
|
record.Result = nil
|
|
record.Error = &PublicError{Code: "result_unconfirmed", Message: "agent restarted after dispatch; the operation was not replayed"}
|
|
record.CauseCode = ""
|
|
record.EffectMayHaveApplied = true
|
|
record.SafeToResubmit = false
|
|
record.ExecutionFenced = true
|
|
}
|
|
record.UpdatedAt = now
|
|
if err := putRecord(operations, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncCompletionReceipt(receipts, outbox, previous, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
|
|
return err
|
|
}
|
|
if err := inflight.Delete(key); err != nil {
|
|
return err
|
|
}
|
|
if err := expiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
finalized = append(finalized, record.OperationID)
|
|
}
|
|
return nil
|
|
})
|
|
return finalized, err
|
|
}
|
|
|
|
func completionEventID(operationID string) string {
|
|
digest := sha256.Sum256([]byte("operation-completion\n" + operationID))
|
|
return base64.RawURLEncoding.EncodeToString(digest[:])
|
|
}
|
|
|
|
func validCompletionEventID(value string) bool {
|
|
decoded, err := base64.RawURLEncoding.DecodeString(value)
|
|
return err == nil && len(decoded) == sha256.Size && base64.RawURLEncoding.EncodeToString(decoded) == value
|
|
}
|
|
|
|
func syncCompletionReceipt(receipts, outbox *bolt.Bucket, previous, current Record) error {
|
|
if receipts == nil || outbox == nil {
|
|
return errors.New("completion buckets are unavailable")
|
|
}
|
|
if !terminalState(current.State) || terminalState(previous.State) {
|
|
return nil
|
|
}
|
|
eventID := completionEventID(current.OperationID)
|
|
key := []byte(eventID)
|
|
if receipts.Get(key) != nil {
|
|
return nil
|
|
}
|
|
status := "failed"
|
|
errorCode := ""
|
|
if current.State == StateSucceeded {
|
|
status = "succeeded"
|
|
} else if current.Error != nil {
|
|
errorCode = current.Error.Code
|
|
}
|
|
receipt := CompletionReceipt{
|
|
EventID: eventID, OperationID: current.OperationID, Status: status,
|
|
Result: append(json.RawMessage(nil), current.Result...), ErrorCode: errorCode,
|
|
EffectMayHaveApplied: current.EffectMayHaveApplied,
|
|
SafeToResubmit: current.SafeToResubmit, CreatedAt: current.UpdatedAt.UTC(),
|
|
}
|
|
encoded, err := json.Marshal(receipt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := receipts.Put(key, encoded); err != nil {
|
|
return err
|
|
}
|
|
pending, err := json.Marshal(completionOutboxRecord{
|
|
OperationID: current.OperationID, NextAttemptAt: current.UpdatedAt.UTC(),
|
|
ExpiresAt: current.UpdatedAt.UTC().Add(completionDeliveryLifetime),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return outbox.Put(key, pending)
|
|
}
|
|
|
|
// Reserve durably binds operation_id to one exact request fingerprint and its
|
|
// non-secret execution descriptor. Records are never deleted, so an old
|
|
// destructive operation_id can never become valid again after housekeeping.
|
|
func (j *Journal) Reserve(spec Spec) (Record, bool, error) {
|
|
if err := validateSpec(spec); err != nil {
|
|
return Record{}, false, err
|
|
}
|
|
lockKeys := effectiveLockKeys(spec.OperationID, spec.ResourceID, spec.Action, spec.LockKeys)
|
|
var record Record
|
|
existed := false
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(operationsBucket)
|
|
inflight := tx.Bucket(inflightBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
if bucket == nil || inflight == nil || blockers == nil {
|
|
return errors.New("operation buckets are unavailable")
|
|
}
|
|
if err := pruneExpiredOperationDetails(tx, j.currentTime()); err != nil {
|
|
return err
|
|
}
|
|
if value := bucket.Get([]byte(spec.OperationID)); value != nil {
|
|
existing, err := decodeRecord(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
existed = true
|
|
if existing.Fingerprint != spec.Fingerprint || existing.Action != spec.Action || existing.ResourceID != spec.ResourceID ||
|
|
existing.SupersedesOperationID != spec.SupersedesOperationID {
|
|
return ErrFingerprint
|
|
}
|
|
record = existing
|
|
return nil
|
|
}
|
|
now := j.currentTime()
|
|
record = Record{
|
|
OperationID: spec.OperationID, ResourceID: spec.ResourceID,
|
|
Action: spec.Action, Fingerprint: spec.Fingerprint,
|
|
Destructive: spec.Destructive, State: StateReserved,
|
|
CreatedAt: now, UpdatedAt: now, ExpectedState: cloneExpectedState(spec.ExpectedState),
|
|
LockKeys: append([]string(nil), lockKeys...), HeavyIO: spec.HeavyIO,
|
|
SupersedesOperationID: spec.SupersedesOperationID,
|
|
}
|
|
if err := putRecord(bucket, record); err != nil {
|
|
return err
|
|
}
|
|
if err := inflight.Put([]byte(record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
return cloneRecord(record), existed, err
|
|
}
|
|
|
|
// WasNotDispatched reports the terminal 0-call outcome. Clean-break Agents do
|
|
// not reopen this record; a new request must use a new operation identifier.
|
|
func WasNotDispatched(record Record) bool {
|
|
return !record.Tombstone && record.State == StateFailed && record.Error != nil &&
|
|
record.Error.Code == "operation_not_dispatched"
|
|
}
|
|
|
|
// Execute persists the dispatch boundary before invoking fn. Once that
|
|
// boundary is crossed the function is invoked at most once. A deadline
|
|
// waits for the supervised worker to return before releasing admission. If the
|
|
// worker cannot be reaped within the secondary hard limit, the resource is
|
|
// durably fenced and any late result is ignored.
|
|
func (j *Journal) Execute(ctx context.Context, spec Spec, fn func(context.Context) (json.RawMessage, error)) (Record, bool, error) {
|
|
record, existed, err := j.Reserve(spec)
|
|
if err != nil {
|
|
return Record{}, existed, err
|
|
}
|
|
if existed && record.State != StateReserved {
|
|
return record, true, nil
|
|
}
|
|
releaseClaim, claimed := j.claimAdmission(spec.OperationID)
|
|
if !claimed {
|
|
return j.admissionReplay(spec.OperationID)
|
|
}
|
|
defer releaseClaim()
|
|
record, err = j.prepareAdmission(spec)
|
|
if err != nil {
|
|
return record, existed, err
|
|
}
|
|
if record.State != StateReserved {
|
|
return record, true, nil
|
|
}
|
|
|
|
releaseAdmission, err := j.acquireAdmission(ctx, record)
|
|
if err != nil {
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, failErr
|
|
}
|
|
return failed, existed, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
releaseAdmission()
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, failErr
|
|
}
|
|
return failed, existed, err
|
|
}
|
|
if err := j.verifyAdmissionUnblocked(spec); err != nil {
|
|
releaseAdmission()
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, failErr
|
|
}
|
|
return failed, existed, err
|
|
}
|
|
|
|
record, err = j.markDispatching(spec.OperationID)
|
|
releaseClaim()
|
|
if err != nil {
|
|
releaseAdmission()
|
|
return record, existed, err
|
|
}
|
|
if record.State != StateDispatching {
|
|
releaseAdmission()
|
|
return record, true, nil
|
|
}
|
|
|
|
type outcome struct {
|
|
result json.RawMessage
|
|
err error
|
|
}
|
|
done := make(chan outcome, 1)
|
|
mutationContext := context.WithValue(ctx, expectedStateContextKey{}, cloneExpectedState(record.ExpectedState))
|
|
go func() {
|
|
result, callErr := fn(mutationContext)
|
|
done <- outcome{result: result, err: callErr}
|
|
}()
|
|
|
|
select {
|
|
case completed := <-done:
|
|
terminal, persistErr := j.finish(spec.OperationID, completed.result, completed.err)
|
|
releaseAdmission()
|
|
if persistErr != nil {
|
|
return terminal, existed, persistErr
|
|
}
|
|
return terminal, existed, completed.err
|
|
case <-ctx.Done():
|
|
reapTimer := time.NewTimer(workerReapDeadline)
|
|
defer reapTimer.Stop()
|
|
select {
|
|
case completed := <-done:
|
|
terminal, persistErr := j.finish(spec.OperationID, completed.result, completed.err)
|
|
releaseAdmission()
|
|
if persistErr != nil {
|
|
return terminal, existed, persistErr
|
|
}
|
|
return terminal, existed, completed.err
|
|
case <-reapTimer.C:
|
|
terminal, persistErr := j.markExecutionFenced(spec.OperationID)
|
|
releaseAdmission()
|
|
if persistErr != nil {
|
|
return terminal, existed, persistErr
|
|
}
|
|
return terminal, existed, ErrExecutionNotReaped
|
|
}
|
|
}
|
|
}
|
|
|
|
// ExecuteBackground persists the dispatch boundary and starts one bounded,
|
|
// tracked worker before returning. A nil done channel means an existing
|
|
// non-reserved record was replayed and no new call was started.
|
|
func (j *Journal) ExecuteBackground(
|
|
dispatchCtx, workerParent context.Context,
|
|
workerTimeout time.Duration,
|
|
spec Spec,
|
|
fn func(context.Context) (json.RawMessage, error),
|
|
) (Record, bool, <-chan struct{}, error) {
|
|
if workerParent == nil || workerTimeout <= 0 {
|
|
return Record{}, false, nil, errors.New("background worker deadline is invalid")
|
|
}
|
|
record, existed, err := j.Reserve(spec)
|
|
if err != nil {
|
|
return Record{}, existed, nil, err
|
|
}
|
|
if existed && record.State != StateReserved {
|
|
return record, true, nil, nil
|
|
}
|
|
releaseClaim, claimed := j.claimAdmission(spec.OperationID)
|
|
if !claimed {
|
|
replayed, _, replayErr := j.admissionReplay(spec.OperationID)
|
|
return replayed, true, nil, replayErr
|
|
}
|
|
defer releaseClaim()
|
|
record, err = j.prepareAdmission(spec)
|
|
if err != nil {
|
|
return record, existed, nil, err
|
|
}
|
|
if record.State != StateReserved {
|
|
return record, true, nil, nil
|
|
}
|
|
releaseAdmission, err := j.acquireAdmission(dispatchCtx, record)
|
|
if err != nil {
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, nil, failErr
|
|
}
|
|
return failed, existed, nil, err
|
|
}
|
|
if err := dispatchCtx.Err(); err != nil {
|
|
releaseAdmission()
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, nil, failErr
|
|
}
|
|
return failed, existed, nil, err
|
|
}
|
|
if err := j.verifyAdmissionUnblocked(spec); err != nil {
|
|
releaseAdmission()
|
|
failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID)
|
|
releaseClaim()
|
|
if failErr != nil {
|
|
return failed, existed, nil, failErr
|
|
}
|
|
return failed, existed, nil, err
|
|
}
|
|
record, err = j.markDispatching(spec.OperationID)
|
|
releaseClaim()
|
|
if err != nil {
|
|
releaseAdmission()
|
|
return record, existed, nil, err
|
|
}
|
|
if record.State != StateDispatching {
|
|
releaseAdmission()
|
|
return record, true, nil, nil
|
|
}
|
|
// Start the execution budget only after admission and the durable dispatch
|
|
// boundary. Time spent waiting for a resource or capacity slot must never
|
|
// consume the helper's bounded execution window.
|
|
workerCtx, workerCancel := context.WithTimeout(workerParent, workerTimeout)
|
|
type outcome struct {
|
|
result json.RawMessage
|
|
err error
|
|
}
|
|
callDone := make(chan outcome, 1)
|
|
done := make(chan struct{})
|
|
mutationContext := context.WithValue(workerCtx, expectedStateContextKey{}, cloneExpectedState(record.ExpectedState))
|
|
go func() {
|
|
result, callErr := fn(mutationContext)
|
|
callDone <- outcome{result: result, err: callErr}
|
|
}()
|
|
j.workerWG.Add(1)
|
|
go func() {
|
|
defer j.workerWG.Done()
|
|
defer close(done)
|
|
defer releaseAdmission()
|
|
defer workerCancel()
|
|
select {
|
|
case completed := <-callDone:
|
|
if _, persistErr := j.finish(spec.OperationID, completed.result, completed.err); persistErr != nil {
|
|
j.reportBackgroundFailure(spec.OperationID, "finalize", persistErr)
|
|
}
|
|
case <-workerCtx.Done():
|
|
reapTimer := time.NewTimer(workerReapDeadline)
|
|
select {
|
|
case completed := <-callDone:
|
|
if _, persistErr := j.finish(spec.OperationID, completed.result, completed.err); persistErr != nil {
|
|
j.reportBackgroundFailure(spec.OperationID, "finalize", persistErr)
|
|
}
|
|
case <-reapTimer.C:
|
|
if _, persistErr := j.markExecutionFenced(spec.OperationID); persistErr != nil {
|
|
j.reportBackgroundFailure(spec.OperationID, "fence", persistErr)
|
|
}
|
|
}
|
|
if !reapTimer.Stop() {
|
|
select {
|
|
case <-reapTimer.C:
|
|
default:
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
return record, existed, done, nil
|
|
}
|
|
|
|
// ExpectedStateFromContext returns the descriptor atomically persisted with
|
|
// the dispatch record. Mutation implementations use it to write a matching
|
|
// host-side transition marker before reporting success.
|
|
func ExpectedStateFromContext(ctx context.Context) (ExpectedState, bool) {
|
|
expected, ok := ctx.Value(expectedStateContextKey{}).(*ExpectedState)
|
|
if !ok || expected == nil {
|
|
return ExpectedState{}, false
|
|
}
|
|
return *cloneExpectedState(expected), true
|
|
}
|
|
|
|
// RequiresExplicitForceDelete identifies the terminal graceful result which
|
|
// remains open for fresh operator inspection and resource conflict blocking.
|
|
// Only a separately acknowledged force-delete-vm operation may cross this
|
|
// boundary; inspection itself never changes the record.
|
|
func RequiresExplicitForceDelete(record Record) bool {
|
|
return !record.Tombstone && record.State == StateFailed && record.Error != nil &&
|
|
record.Error.Code == "force_required" &&
|
|
(record.Action == "delete-vm" || record.Action == "service-suspend")
|
|
}
|
|
|
|
func (j *Journal) initialize() error {
|
|
return j.db.Update(func(tx *bolt.Tx) error {
|
|
metadata := tx.Bucket(metadataBucket)
|
|
fresh := metadata == nil
|
|
var err error
|
|
if fresh {
|
|
hasExistingBuckets := false
|
|
if err := tx.ForEach(func(_ []byte, _ *bolt.Bucket) error {
|
|
hasExistingBuckets = true
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if hasExistingBuckets {
|
|
return errors.New("operation journal has no supported schema; automatic migration is disabled")
|
|
}
|
|
metadata, err = tx.CreateBucket(metadataBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create journal metadata bucket: %w", err)
|
|
}
|
|
} else {
|
|
current := metadata.Get(schemaVersionKey)
|
|
if string(current) != string(schemaVersion) {
|
|
return fmt.Errorf("unsupported operation journal schema %q", current)
|
|
}
|
|
}
|
|
operations, err := tx.CreateBucketIfNotExists(operationsBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create operation bucket: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(noncesBucket); err != nil {
|
|
return fmt.Errorf("create nonce bucket: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(nonceExpiryBucket); err != nil {
|
|
return fmt.Errorf("create nonce expiry bucket: %w", err)
|
|
}
|
|
operationExpiry, err := tx.CreateBucketIfNotExists(operationExpiryBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create operation expiry bucket: %w", err)
|
|
}
|
|
completionReceipts, err := tx.CreateBucketIfNotExists(completionReceiptsBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create completion receipt bucket: %w", err)
|
|
}
|
|
completionOutbox, err := tx.CreateBucketIfNotExists(completionOutboxBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create completion outbox bucket: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(fenceResolutionsBucket); err != nil {
|
|
return fmt.Errorf("create fence resolution bucket: %w", err)
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(fenceResolutionIDsBucket); err != nil {
|
|
return fmt.Errorf("create fence resolution identity bucket: %w", err)
|
|
}
|
|
inflight, err := tx.CreateBucketIfNotExists(inflightBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create inflight operation bucket: %w", err)
|
|
}
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
if blockers == nil {
|
|
if !fresh {
|
|
return errors.New("operation resource blocker index is unavailable")
|
|
}
|
|
blockers, err = tx.CreateBucket(resourceBlockersBucket)
|
|
if err != nil {
|
|
return fmt.Errorf("create operation resource blocker bucket: %w", err)
|
|
}
|
|
}
|
|
// resource_blockers is a derived admission index. Host storage
|
|
// configuration keeps immutable failure evidence, but a terminal attempt
|
|
// must never prevent an administrator from submitting a new, independently
|
|
// identified configuration request. Remove entries written by the earlier
|
|
// schema-v6 policy before this process admits any request. The surrounding
|
|
// Bolt transaction makes the repair atomic, and the operation plus its
|
|
// immutable completion receipt remain untouched.
|
|
if err := removeStorageConfigurationBlockers(operations, blockers); err != nil {
|
|
return err
|
|
}
|
|
now := j.currentTime()
|
|
var recovered []Record
|
|
var stale [][]byte
|
|
if err := inflight.ForEach(func(key, _ []byte) error {
|
|
value := operations.Get(key)
|
|
if value == nil {
|
|
stale = append(stale, append([]byte(nil), key...))
|
|
return nil
|
|
}
|
|
record, err := decodeRecord(value)
|
|
if err != nil {
|
|
return fmt.Errorf("decode operation %q: %w", key, err)
|
|
}
|
|
switch record.State {
|
|
case StateReserved:
|
|
record.State = StateFailed
|
|
record.Result = notDispatchedEvidence(record.Action)
|
|
record.Error = &PublicError{Code: "operation_not_dispatched", Message: "agent restarted before the dispatch boundary"}
|
|
record.EffectMayHaveApplied = false
|
|
record.SafeToResubmit = true
|
|
case StateDispatching:
|
|
// The parent process may have crashed after its isolated worker
|
|
// durably wrote a receipt. Leave this row in the inflight index until
|
|
// FinalizeInterruptedOperations imports that receipt or records
|
|
// result_unconfirmed. The mutation is never started again.
|
|
return nil
|
|
default:
|
|
stale = append(stale, append([]byte(nil), key...))
|
|
return nil
|
|
}
|
|
record.UpdatedAt = now
|
|
recovered = append(recovered, record)
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
for _, record := range recovered {
|
|
previous := cloneRecord(record)
|
|
previous.State = StateDispatching
|
|
if record.Error != nil && record.Error.Code == "operation_not_dispatched" {
|
|
previous.State = StateReserved
|
|
}
|
|
if err := putRecord(operations, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncCompletionReceipt(completionReceipts, completionOutbox, previous, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncResourceBlockers(blockers, nil, &record); err != nil {
|
|
return err
|
|
}
|
|
if terminalState(record.State) {
|
|
if err := operationExpiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := inflight.Delete([]byte(record.OperationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, key := range stale {
|
|
if err := inflight.Delete(key); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := pruneExpiredOperationDetails(tx, now); err != nil {
|
|
return err
|
|
}
|
|
return metadata.Put(schemaVersionKey, schemaVersion)
|
|
})
|
|
}
|
|
|
|
func (j *Journal) markDispatching(operationID string) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
if record.State == StateReserved {
|
|
record.State = StateDispatching
|
|
record.UpdatedAt = j.currentTime()
|
|
}
|
|
})
|
|
}
|
|
|
|
func (j *Journal) finish(operationID string, result json.RawMessage, callErr error) (Record, error) {
|
|
if IsUncertain(callErr) {
|
|
return j.markResultUnconfirmed(operationID, callErr)
|
|
}
|
|
if callErr != nil {
|
|
code := "backend_error"
|
|
message := callErr.Error()
|
|
effectMayHaveApplied := true
|
|
safeToResubmit := false
|
|
var public interface {
|
|
PublicCode() string
|
|
PublicMessage() string
|
|
}
|
|
if errors.As(callErr, &public) {
|
|
if public.PublicCode() != "" {
|
|
code = public.PublicCode()
|
|
}
|
|
if public.PublicMessage() != "" {
|
|
message = public.PublicMessage()
|
|
}
|
|
}
|
|
var semantics interface{ FailureSemantics() (bool, bool) }
|
|
if errors.As(callErr, &semantics) {
|
|
effectMayHaveApplied, safeToResubmit = semantics.FailureSemantics()
|
|
}
|
|
executionFenced := false
|
|
var fence interface{ RequiresExecutionFence() bool }
|
|
if errors.As(callErr, &fence) {
|
|
executionFenced = fence.RequiresExecutionFence()
|
|
}
|
|
return j.finishWithSemanticsAndFence(operationID, code, message, result, code, effectMayHaveApplied, safeToResubmit, executionFenced)
|
|
}
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
record.State = StateSucceeded
|
|
record.Result = append(json.RawMessage(nil), result...)
|
|
record.Error = nil
|
|
record.EffectMayHaveApplied = false
|
|
record.SafeToResubmit = false
|
|
record.ExecutionFenced = false
|
|
record.CauseCode = ""
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
func (j *Journal) finishWithPublicError(operationID, code, message string) (Record, error) {
|
|
return j.finishWithPublicErrorAndResult(operationID, code, message, nil)
|
|
}
|
|
|
|
func (j *Journal) finishWithPublicErrorAndResult(operationID, code, message string, result json.RawMessage) (Record, error) {
|
|
return j.finishWithCause(operationID, code, message, result, "")
|
|
}
|
|
|
|
func (j *Journal) finishWithCause(operationID, code, message string, result json.RawMessage, causeCode string) (Record, error) {
|
|
return j.finishWithSemantics(operationID, code, message, result, causeCode, true, false)
|
|
}
|
|
|
|
func (j *Journal) finishWithSemantics(operationID, code, message string, result json.RawMessage, causeCode string,
|
|
effectMayHaveApplied, safeToResubmit bool) (Record, error) {
|
|
return j.finishWithSemanticsAndFence(operationID, code, message, result, causeCode, effectMayHaveApplied, safeToResubmit, false)
|
|
}
|
|
|
|
func (j *Journal) finishWithSemanticsAndFence(operationID, code, message string, result json.RawMessage, causeCode string,
|
|
effectMayHaveApplied, safeToResubmit, executionFenced bool) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
record.State = StateFailed
|
|
record.Result = append(json.RawMessage(nil), result...)
|
|
record.Error = &PublicError{Code: code, Message: message}
|
|
record.EffectMayHaveApplied = effectMayHaveApplied
|
|
record.SafeToResubmit = safeToResubmit
|
|
record.ExecutionFenced = executionFenced
|
|
preserveCauseCode(record, causeCode)
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
func (j *Journal) failNotDispatchedIfReserved(operationID string) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
if record.State != StateReserved {
|
|
return
|
|
}
|
|
record.State = StateFailed
|
|
record.Result = notDispatchedEvidence(record.Action)
|
|
record.Error = &PublicError{Code: "operation_not_dispatched", Message: "operation did not cross the dispatch boundary"}
|
|
record.EffectMayHaveApplied = false
|
|
record.SafeToResubmit = true
|
|
record.ExecutionFenced = false
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
// FailNotDispatched terminally records that a reserved operation never
|
|
// crossed the dispatch boundary. It is safe for admission failures only.
|
|
func (j *Journal) FailNotDispatched(operationID string) (Record, error) {
|
|
return j.failNotDispatchedIfReserved(operationID)
|
|
}
|
|
|
|
func notDispatchedEvidence(action string) json.RawMessage {
|
|
if action == "create-vm" {
|
|
return json.RawMessage(`{"not_dispatched":true,"creation_absent":true,"managed_artifacts_absent":true}`)
|
|
}
|
|
return json.RawMessage(`{"not_dispatched":true}`)
|
|
}
|
|
|
|
func (j *Journal) markResultUnconfirmed(operationID string, cause error) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
if terminalState(record.State) {
|
|
return
|
|
}
|
|
message := "operation result could not be confirmed before the execution deadline"
|
|
if summary := publicMessageFromError(cause); summary != "" {
|
|
message += ": " + summary
|
|
}
|
|
record.State = StateFailed
|
|
record.Result = nil
|
|
record.Error = &PublicError{Code: "result_unconfirmed", Message: message}
|
|
record.EffectMayHaveApplied = true
|
|
record.SafeToResubmit = false
|
|
record.ExecutionFenced = false
|
|
preserveCauseCode(record, publicCodeFromError(cause))
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
func (j *Journal) markExecutionFenced(operationID string) (Record, error) {
|
|
return j.updateRecord(operationID, func(record *Record) {
|
|
if terminalState(record.State) {
|
|
return
|
|
}
|
|
record.State = StateFailed
|
|
record.Result = nil
|
|
record.Error = &PublicError{
|
|
Code: "worker_reap_unconfirmed",
|
|
Message: "isolated mutation process did not exit after forced termination; this resource is fenced",
|
|
}
|
|
record.EffectMayHaveApplied = true
|
|
record.SafeToResubmit = false
|
|
record.ExecutionFenced = true
|
|
record.UpdatedAt = j.currentTime()
|
|
})
|
|
}
|
|
|
|
func preserveCauseCode(record *Record, code string) {
|
|
if record.CauseCode == "" && validCauseCode(code) {
|
|
record.CauseCode = code
|
|
}
|
|
}
|
|
|
|
func validCauseCode(code string) bool {
|
|
return len(code) <= 64 && causeCodePattern.MatchString(code)
|
|
}
|
|
|
|
func publicCodeFromError(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
var public interface{ PublicCode() string }
|
|
if !errors.As(err, &public) {
|
|
return ""
|
|
}
|
|
code := public.PublicCode()
|
|
if !validCauseCode(code) {
|
|
return ""
|
|
}
|
|
return code
|
|
}
|
|
|
|
func publicMessageFromError(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
var public interface{ PublicMessage() string }
|
|
if !errors.As(err, &public) {
|
|
return ""
|
|
}
|
|
return public.PublicMessage()
|
|
}
|
|
|
|
func (j *Journal) updateRecord(operationID string, mutate func(*Record)) (Record, error) {
|
|
return j.updateRecordWhen(operationID, nil, mutate)
|
|
}
|
|
|
|
// updateRecordWhen checks eligibility and applies a mutation in the same Bolt
|
|
// write transaction. A nil predicate preserves the ordinary update behavior.
|
|
func (j *Journal) updateRecordWhen(operationID string, eligible func(Record) bool,
|
|
mutate func(*Record)) (Record, error) {
|
|
var result Record
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(operationsBucket)
|
|
inflight := tx.Bucket(inflightBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
expiry := tx.Bucket(operationExpiryBucket)
|
|
receipts := tx.Bucket(completionReceiptsBucket)
|
|
outbox := tx.Bucket(completionOutboxBucket)
|
|
if bucket == nil || inflight == nil || blockers == nil || expiry == nil || receipts == nil || outbox == nil {
|
|
return errors.New("operation buckets are unavailable")
|
|
}
|
|
value := bucket.Get([]byte(operationID))
|
|
if value == nil {
|
|
return ErrNotFound
|
|
}
|
|
record, err := decodeRecord(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if eligible != nil && !eligible(record) {
|
|
result = record
|
|
return nil
|
|
}
|
|
previous := cloneRecord(record)
|
|
mutate(&record)
|
|
if err := putRecord(bucket, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncCompletionReceipt(receipts, outbox, previous, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
|
|
return err
|
|
}
|
|
if record.State == StateSucceeded && record.Action == "force-delete-vm" && record.SupersedesOperationID != "" {
|
|
if err := closeForceDeletePredecessor(bucket, blockers, record); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
switch record.State {
|
|
case StateReserved, StateDispatching:
|
|
if err := inflight.Put([]byte(record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
if err := inflight.Delete([]byte(record.OperationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if terminalState(record.State) && !record.Tombstone {
|
|
if err := expiry.Put(operationExpiryKey(record.UpdatedAt, record.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
result = record
|
|
return nil
|
|
})
|
|
return cloneRecord(result), err
|
|
}
|
|
|
|
// closeForceDeletePredecessor runs in the same Bolt transaction which marks
|
|
// the explicit force operation succeeded. It removes only the predecessor's
|
|
// blocker index entries; the failed operation and its completion receipt stay
|
|
// byte-for-byte immutable, with the successor retaining the audit link.
|
|
func closeForceDeletePredecessor(operations, blockers *bolt.Bucket, successor Record) error {
|
|
encoded := operations.Get([]byte(successor.SupersedesOperationID))
|
|
if encoded == nil {
|
|
return ErrInvalidSupersession
|
|
}
|
|
predecessor, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
lockKeys := effectiveLockKeys(successor.OperationID, successor.ResourceID, successor.Action, successor.LockKeys)
|
|
if !validForceDeleteSupersession(Spec{
|
|
OperationID: successor.OperationID, ResourceID: successor.ResourceID,
|
|
Action: successor.Action, SupersedesOperationID: successor.SupersedesOperationID,
|
|
}, lockKeys, predecessor) {
|
|
return ErrInvalidSupersession
|
|
}
|
|
for _, lockKey := range effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys) {
|
|
if err := blockers.Delete(blockerEntryKey(lockKey, predecessor.OperationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (j *Journal) acquire(ctx context.Context, key string) (func(), error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
j.resourceMu.Lock()
|
|
lock, ok := j.resources[key]
|
|
if !ok {
|
|
lock = &resourceLock{semaphore: make(chan struct{}, 1)}
|
|
j.resources[key] = lock
|
|
}
|
|
lock.references++
|
|
j.resourceMu.Unlock()
|
|
|
|
select {
|
|
case lock.semaphore <- struct{}{}:
|
|
var once sync.Once
|
|
release := func() {
|
|
once.Do(func() {
|
|
<-lock.semaphore
|
|
j.releaseResourceReference(key, lock)
|
|
})
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
release()
|
|
return nil, err
|
|
}
|
|
return release, nil
|
|
case <-ctx.Done():
|
|
j.releaseResourceReference(key, lock)
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (j *Journal) claimAdmission(operationID string) (func(), bool) {
|
|
j.admissionClaimMu.Lock()
|
|
if _, exists := j.admissionClaims[operationID]; exists {
|
|
j.admissionClaimMu.Unlock()
|
|
return nil, false
|
|
}
|
|
j.admissionClaims[operationID] = struct{}{}
|
|
j.admissionClaimMu.Unlock()
|
|
|
|
var once sync.Once
|
|
return func() {
|
|
once.Do(func() {
|
|
j.admissionClaimMu.Lock()
|
|
delete(j.admissionClaims, operationID)
|
|
j.admissionClaimMu.Unlock()
|
|
})
|
|
}, true
|
|
}
|
|
|
|
func blockerEntryKey(lockKey, operationID string) []byte {
|
|
return []byte(lockKey + "\x00" + operationID)
|
|
}
|
|
|
|
func blockerEntryPrefix(lockKey string) []byte {
|
|
return []byte(lockKey + "\x00")
|
|
}
|
|
|
|
func effectfulTerminalFailure(record Record) bool {
|
|
return !record.Tombstone && record.State == StateFailed && record.EffectMayHaveApplied
|
|
}
|
|
|
|
func blocksResources(record Record) bool {
|
|
// Fixed storage-pool registration is monotonic: a successor validates and
|
|
// reuses matching pools, and never deletes or rebinds an existing pool. Keep
|
|
// the immutable failed operation as evidence, but do not let it fence a new,
|
|
// explicitly identified configure-storage request.
|
|
return effectfulTerminalFailure(record) && record.Action != "configure-storage"
|
|
}
|
|
|
|
// removeStorageConfigurationBlockers removes only derived blocker-index
|
|
// entries owned by host storage configuration operations. Every reference is
|
|
// validated before any key is deleted so a missing, malformed, or stale
|
|
// non-storage reference still fails Agent startup instead of silently dropping
|
|
// a safety fence.
|
|
func removeStorageConfigurationBlockers(operations, blockers *bolt.Bucket) error {
|
|
if operations == nil || blockers == nil {
|
|
return errors.New("operation resource blocker index is unavailable")
|
|
}
|
|
removable := make([][]byte, 0)
|
|
cursor := blockers.Cursor()
|
|
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
|
|
separator := bytes.IndexByte(key, 0)
|
|
if separator <= 0 || separator != bytes.LastIndexByte(key, 0) || separator == len(key)-1 {
|
|
return fmt.Errorf("invalid operation resource blocker key %q", key)
|
|
}
|
|
lockKey := string(key[:separator])
|
|
operationID := string(key[separator+1:])
|
|
if err := validateLockKey(lockKey); err != nil || !operationIDPattern.MatchString(operationID) {
|
|
return fmt.Errorf("invalid operation resource blocker key %q", key)
|
|
}
|
|
encoded := operations.Get([]byte(operationID))
|
|
if encoded == nil {
|
|
return fmt.Errorf("resource blocker index references missing operation %q", operationID)
|
|
}
|
|
record, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return fmt.Errorf("decode resource blocker %q: %w", operationID, err)
|
|
}
|
|
if record.OperationID != operationID {
|
|
return fmt.Errorf("resource blocker index references mismatched operation %q", operationID)
|
|
}
|
|
matchingKey := false
|
|
for _, expected := range effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys) {
|
|
if expected == lockKey {
|
|
matchingKey = true
|
|
break
|
|
}
|
|
}
|
|
if !matchingKey {
|
|
return fmt.Errorf("resource blocker index references unexpected lock %q for operation %q", lockKey, operationID)
|
|
}
|
|
if record.Action == "configure-storage" {
|
|
if !effectfulTerminalFailure(record) {
|
|
return fmt.Errorf("resource blocker index references non-blocking operation %q", operationID)
|
|
}
|
|
removable = append(removable, append([]byte(nil), key...))
|
|
continue
|
|
}
|
|
if !blocksResources(record) {
|
|
return fmt.Errorf("resource blocker index references resolved operation %q", operationID)
|
|
}
|
|
}
|
|
for _, key := range removable {
|
|
if err := blockers.Delete(key); err != nil {
|
|
return fmt.Errorf("delete retired storage configuration blocker %q: %w", key, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func syncResourceBlockers(blockers *bolt.Bucket, previous, current *Record) error {
|
|
if blockers == nil {
|
|
return errors.New("operation resource blocker bucket is unavailable")
|
|
}
|
|
if previous != nil && blocksResources(*previous) {
|
|
for _, key := range effectiveLockKeys(previous.OperationID, previous.ResourceID, previous.Action, previous.LockKeys) {
|
|
if err := blockers.Delete(blockerEntryKey(key, previous.OperationID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if current != nil && blocksResources(*current) {
|
|
for _, key := range effectiveLockKeys(current.OperationID, current.ResourceID, current.Action, current.LockKeys) {
|
|
if err := blockers.Put(blockerEntryKey(key, current.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func lockKeysOverlap(left, right []string) bool {
|
|
leftSet := make(map[string]struct{}, len(left))
|
|
for _, key := range left {
|
|
leftSet[key] = struct{}{}
|
|
}
|
|
for _, key := range right {
|
|
if _, ok := leftSet[key]; ok {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func validForceDeleteSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
|
|
if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID ||
|
|
spec.Action != "force-delete-vm" || predecessor.OperationID != spec.SupersedesOperationID ||
|
|
predecessor.ResourceID != spec.ResourceID || (predecessor.Action != "delete-vm" && predecessor.Action != "service-suspend") ||
|
|
predecessor.State != StateFailed || predecessor.Error == nil || predecessor.Error.Code != "force_required" {
|
|
return false
|
|
}
|
|
predecessorKeys := effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys)
|
|
return lockKeysOverlap(lockKeys, predecessorKeys)
|
|
}
|
|
|
|
// findResourceBlocker validates any explicit force-delete predecessor and
|
|
// reads only the compact blocker index for unresolved resource conflicts.
|
|
func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, bool, error) {
|
|
operations := tx.Bucket(operationsBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
if operations == nil || blockers == nil {
|
|
return Record{}, false, errors.New("operation resource blocker index is unavailable")
|
|
}
|
|
supersededOperationID := ""
|
|
if spec.SupersedesOperationID != "" {
|
|
encoded := operations.Get([]byte(spec.SupersedesOperationID))
|
|
if encoded == nil {
|
|
return Record{}, false, ErrInvalidSupersession
|
|
}
|
|
predecessor, err := decodeRecord(encoded)
|
|
if err != nil || !validForceDeleteSupersession(spec, lockKeys, predecessor) {
|
|
return Record{}, false, ErrInvalidSupersession
|
|
}
|
|
supersededOperationID = predecessor.OperationID
|
|
}
|
|
for _, lockKey := range lockKeys {
|
|
prefix := blockerEntryPrefix(lockKey)
|
|
cursor := blockers.Cursor()
|
|
for key, _ := cursor.Seek(prefix); key != nil && bytes.HasPrefix(key, prefix); key, _ = cursor.Next() {
|
|
operationID := string(key[len(prefix):])
|
|
if operationID == spec.OperationID || operationID == supersededOperationID {
|
|
continue
|
|
}
|
|
encoded := operations.Get([]byte(operationID))
|
|
if encoded == nil {
|
|
return Record{}, false, fmt.Errorf("resource blocker index references missing operation %q", operationID)
|
|
}
|
|
blocker, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return Record{}, false, fmt.Errorf("decode resource blocker %q: %w", operationID, err)
|
|
}
|
|
if !blocksResources(blocker) {
|
|
return Record{}, false, fmt.Errorf("resource blocker index references resolved operation %q", operationID)
|
|
}
|
|
return cloneRecord(blocker), true, nil
|
|
}
|
|
}
|
|
return Record{}, false, nil
|
|
}
|
|
|
|
func (j *Journal) verifyAdmissionUnblocked(spec Spec) error {
|
|
lockKeys := effectiveLockKeys(spec.OperationID, spec.ResourceID, spec.Action, spec.LockKeys)
|
|
return j.db.View(func(tx *bolt.Tx) error {
|
|
_, blocked, err := findResourceBlocker(tx, spec, lockKeys)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if blocked {
|
|
return ErrResourceBlocked
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// prepareAdmission runs only while this process owns the operation's admission
|
|
// claim. Terminal records are immutable and are never reopened.
|
|
func (j *Journal) prepareAdmission(spec Spec) (Record, error) {
|
|
lockKeys := effectiveLockKeys(spec.OperationID, spec.ResourceID, spec.Action, spec.LockKeys)
|
|
var record Record
|
|
err := j.db.Update(func(tx *bolt.Tx) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
inflight := tx.Bucket(inflightBucket)
|
|
if operations == nil || inflight == nil {
|
|
return errors.New("operation buckets are unavailable")
|
|
}
|
|
value := operations.Get([]byte(spec.OperationID))
|
|
if value == nil {
|
|
return ErrNotFound
|
|
}
|
|
existing, err := decodeRecord(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if existing.Fingerprint != spec.Fingerprint || existing.Action != spec.Action || existing.ResourceID != spec.ResourceID ||
|
|
existing.SupersedesOperationID != spec.SupersedesOperationID {
|
|
return ErrFingerprint
|
|
}
|
|
if existing.State != StateReserved {
|
|
record = existing
|
|
return nil
|
|
}
|
|
existing.Destructive = spec.Destructive
|
|
existing.ExpectedState = cloneExpectedState(spec.ExpectedState)
|
|
existing.LockKeys = append([]string(nil), lockKeys...)
|
|
existing.HeavyIO = spec.HeavyIO
|
|
existing.SupersedesOperationID = spec.SupersedesOperationID
|
|
if err := putRecord(operations, existing); err != nil {
|
|
return err
|
|
}
|
|
if err := inflight.Put([]byte(existing.OperationID), nil); err != nil {
|
|
return err
|
|
}
|
|
record = existing
|
|
return nil
|
|
})
|
|
return cloneRecord(record), err
|
|
}
|
|
|
|
func (j *Journal) admissionReplay(operationID string) (Record, bool, error) {
|
|
record, err := j.Get(operationID)
|
|
if err != nil {
|
|
return Record{}, true, err
|
|
}
|
|
switch record.State {
|
|
case StateReserved:
|
|
return record, true, ErrOperationStarting
|
|
default:
|
|
return record, true, nil
|
|
}
|
|
}
|
|
|
|
func (j *Journal) acquireAdmission(ctx context.Context, record Record) (func(), error) {
|
|
keys := effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys)
|
|
releases := make([]func(), 0, len(keys)+2)
|
|
releaseAll := func() {
|
|
for index := len(releases) - 1; index >= 0; index-- {
|
|
releases[index]()
|
|
}
|
|
}
|
|
for _, key := range keys {
|
|
release, err := j.acquire(ctx, key)
|
|
if err != nil {
|
|
releaseAll()
|
|
return nil, err
|
|
}
|
|
releases = append(releases, release)
|
|
}
|
|
if record.HeavyIO || defaultHeavyIOAction(record.Action) {
|
|
release, err := acquireCapacity(ctx, j.heavyIOSlots)
|
|
if err != nil {
|
|
releaseAll()
|
|
return nil, err
|
|
}
|
|
releases = append(releases, release)
|
|
}
|
|
releaseExecution, err := j.acquireExecution(ctx)
|
|
if err != nil {
|
|
releaseAll()
|
|
return nil, err
|
|
}
|
|
releases = append(releases, releaseExecution)
|
|
if err := ctx.Err(); err != nil {
|
|
releaseAll()
|
|
return nil, err
|
|
}
|
|
var once sync.Once
|
|
return func() { once.Do(releaseAll) }, nil
|
|
}
|
|
|
|
func acquireCapacity(ctx context.Context, slots chan struct{}) (func(), error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
select {
|
|
case slots <- struct{}{}:
|
|
var once sync.Once
|
|
release := func() { once.Do(func() { <-slots }) }
|
|
if err := ctx.Err(); err != nil {
|
|
release()
|
|
return nil, err
|
|
}
|
|
return release, nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (j *Journal) acquireExecution(ctx context.Context) (func(), error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
select {
|
|
case j.executionSlots <- struct{}{}:
|
|
var once sync.Once
|
|
release := func() {
|
|
once.Do(func() { <-j.executionSlots })
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
release()
|
|
return nil, err
|
|
}
|
|
return release, nil
|
|
default:
|
|
return nil, ErrBusy
|
|
}
|
|
}
|
|
|
|
func (j *Journal) releaseResourceReference(key string, lock *resourceLock) {
|
|
j.resourceMu.Lock()
|
|
defer j.resourceMu.Unlock()
|
|
lock.references--
|
|
if lock.references == 0 && len(lock.semaphore) == 0 && j.resources[key] == lock {
|
|
delete(j.resources, key)
|
|
}
|
|
}
|
|
|
|
func effectiveLockKeys(operationID, resourceID, action string, configured []string) []string {
|
|
if len(configured) != 0 {
|
|
keys, _ := normalizeLockKeys(configured)
|
|
return keys
|
|
}
|
|
return []string{defaultResourceLockKey(operationID, resourceID, action)}
|
|
}
|
|
|
|
func defaultResourceLockKey(operationID, resourceID, action string) string {
|
|
canonicalResource := strings.ToLower(resourceID)
|
|
if canonicalUUIDPattern.MatchString(canonicalResource) {
|
|
return "domain:" + canonicalResource
|
|
}
|
|
if action == "configure-storage" {
|
|
return "storage-config"
|
|
}
|
|
for prefix, kind := range map[string]string{
|
|
"template:": "template", "rescue-template:": "rescue-template",
|
|
} {
|
|
if strings.HasPrefix(resourceID, prefix) {
|
|
return "catalog:" + kind + "/" + strings.TrimPrefix(resourceID, prefix)
|
|
}
|
|
}
|
|
if strings.HasPrefix(resourceID, "user-iso:") {
|
|
return "catalog:user-iso/" + strings.TrimPrefix(resourceID, "user-iso:")
|
|
}
|
|
if resourceID == "" {
|
|
return "operation:" + operationID
|
|
}
|
|
digest := sha256.Sum256([]byte(resourceID))
|
|
return "resource-hash:" + hex.EncodeToString(digest[:])
|
|
}
|
|
|
|
func defaultHeavyIOAction(action string) bool {
|
|
return action == "configure-storage" || action == "create-vm" || action == "reinstall-vm" ||
|
|
action == "import-template" || action == "import-rescue-template" || action == "import-user-iso" ||
|
|
strings.HasPrefix(action, "create-snapshot:") || strings.HasPrefix(action, "restore-snapshot:")
|
|
}
|
|
|
|
func normalizeLockKeys(keys []string) ([]string, error) {
|
|
if len(keys) > maxAdmissionResourceKeys {
|
|
return nil, fmt.Errorf("operation has more than %d resource keys", maxAdmissionResourceKeys)
|
|
}
|
|
unique := make(map[string]struct{}, len(keys))
|
|
for _, key := range keys {
|
|
if err := validateLockKey(key); err != nil {
|
|
return nil, err
|
|
}
|
|
unique[key] = struct{}{}
|
|
}
|
|
result := make([]string, 0, len(unique))
|
|
for key := range unique {
|
|
result = append(result, key)
|
|
}
|
|
sort.Strings(result)
|
|
return result, nil
|
|
}
|
|
|
|
func validateLockKey(key string) error {
|
|
if key == "" || len(key) > maximumAdmissionKeyLength || strings.TrimSpace(key) != key || strings.ContainsRune(key, 0) {
|
|
return errors.New("invalid operation resource key")
|
|
}
|
|
if key == "storage-config" {
|
|
return nil
|
|
}
|
|
prefix, value, ok := strings.Cut(key, ":")
|
|
if !ok || value == "" {
|
|
return errors.New("invalid operation resource key")
|
|
}
|
|
switch prefix {
|
|
case "domain":
|
|
if !canonicalUUIDPattern.MatchString(value) {
|
|
return errors.New("domain resource key must contain a canonical lowercase UUID")
|
|
}
|
|
case "volume", "catalog":
|
|
left, right, found := strings.Cut(value, "/")
|
|
if !found || strings.Contains(right, "/") || !resourceKindPattern.MatchString(left) || !resourcePartPattern.MatchString(right) {
|
|
return errors.New("invalid operation resource key")
|
|
}
|
|
case "nwfilter":
|
|
if !resourcePartPattern.MatchString(value) {
|
|
return errors.New("invalid nwfilter resource key")
|
|
}
|
|
case "network-identity", "resource-hash":
|
|
if !hexDigestPattern.MatchString(value) {
|
|
return errors.New("invalid hashed operation resource key")
|
|
}
|
|
case "operation":
|
|
if !operationIDPattern.MatchString(value) {
|
|
return errors.New("invalid operation resource key")
|
|
}
|
|
default:
|
|
return errors.New("unsupported operation resource key")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateSpec(spec Spec) error {
|
|
if !operationIDPattern.MatchString(spec.OperationID) {
|
|
return errors.New("operation_id must be 8-80 URL-safe characters")
|
|
}
|
|
if strings.TrimSpace(spec.Action) == "" || len(spec.Action) > 128 {
|
|
return errors.New("action must contain 1-128 characters")
|
|
}
|
|
if len(spec.ResourceID) > 255 {
|
|
return errors.New("resource_id is too long")
|
|
}
|
|
if !hexDigestPattern.MatchString(spec.Fingerprint) {
|
|
return errors.New("fingerprint must be lowercase SHA-256 hex")
|
|
}
|
|
if _, err := normalizeLockKeys(spec.LockKeys); err != nil {
|
|
return err
|
|
}
|
|
if spec.SupersedesOperationID != "" {
|
|
if !operationIDPattern.MatchString(spec.SupersedesOperationID) || spec.SupersedesOperationID == spec.OperationID || spec.Action != "force-delete-vm" {
|
|
return ErrInvalidSupersession
|
|
}
|
|
}
|
|
if spec.ExpectedState != nil {
|
|
if err := validateExpectedState(spec.Action, spec.OperationID, *spec.ExpectedState); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateExpectedState(action, operationID string, expected ExpectedState) error {
|
|
if expected.TransitionID != operationID {
|
|
return errors.New("expected state transition_id must match operation_id")
|
|
}
|
|
if len(expected.Hostname) > 253 || len(expected.NetworkSelector) > 128 || len(expected.NetworkSourceType) > 16 || len(expected.NetworkSourceName) > 128 || len(expected.PublicPoolKey) > 128 ||
|
|
len(expected.PublicPoolCIDR) > 64 || len(expected.PublicIPCIDR) > 64 || len(expected.Gateway) > 64 || len(expected.DNSServers) > 4 {
|
|
return errors.New("expected state contains an oversized value")
|
|
}
|
|
values := []string{expected.Hostname, expected.NetworkSelector, expected.NetworkSourceType, expected.NetworkSourceName, expected.PublicPoolKey, expected.PublicPoolCIDR, expected.PublicIPCIDR, expected.Gateway}
|
|
values = append(values, expected.DNSServers...)
|
|
for _, value := range values {
|
|
if strings.ContainsRune(value, 0) {
|
|
return errors.New("expected state contains NUL")
|
|
}
|
|
}
|
|
switch expected.Kind {
|
|
case ExpectedUpdateSettings:
|
|
if action != "update-settings" || (!expected.HostnameChanged && !expected.RootPasswordChanged && !expected.SSHKeyChanged) {
|
|
return errors.New("invalid update-settings expected state")
|
|
}
|
|
case ExpectedChangeNetwork:
|
|
if action != "change-network" || expected.NetworkSelector == "" || expected.PublicPoolKey == "" || expected.PublicPoolCIDR == "" ||
|
|
expected.PublicIPCIDR == "" || expected.Gateway == "" || len(expected.DNSServers) == 0 ||
|
|
(expected.NetworkSourceType != "bridge" && expected.NetworkSourceType != "network") || !networkSourcePattern.MatchString(expected.NetworkSourceName) {
|
|
return errors.New("invalid change-network expected state")
|
|
}
|
|
case ExpectedNetworkState:
|
|
if (action != "network-suspend" && action != "network-resume") || expected.Enabled != (action == "network-resume") {
|
|
return errors.New("invalid network-state expected state")
|
|
}
|
|
case ExpectedServiceState:
|
|
if (action != "service-suspend" && action != "service-resume") || expected.Enabled != (action == "service-resume") || (!expected.Enabled && expected.StartIfSuspended) {
|
|
return errors.New("invalid service-state expected state")
|
|
}
|
|
case ExpectedFirewall:
|
|
if action != "apply-firewall" || expected.FirewallRevision == 0 || !hexDigestPattern.MatchString(expected.FirewallDigest) {
|
|
return errors.New("invalid firewall expected state")
|
|
}
|
|
case ExpectedStorageConfiguration:
|
|
if action != "configure-storage" ||
|
|
!hexDigestPattern.MatchString(expected.ManagedPathDigest) ||
|
|
!hexDigestPattern.MatchString(expected.TemplatePathDigest) ||
|
|
!hexDigestPattern.MatchString(expected.SeedPathDigest) ||
|
|
!hexDigestPattern.MatchString(expected.UserISOPathDigest) ||
|
|
!hexDigestPattern.MatchString(expected.RescuePathDigest) {
|
|
return errors.New("invalid storage-configuration expected state")
|
|
}
|
|
default:
|
|
return errors.New("unsupported expected state kind")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateRecord(record Record) error {
|
|
if err := validateSpec(Spec{
|
|
OperationID: record.OperationID, ResourceID: record.ResourceID,
|
|
Action: record.Action, Fingerprint: record.Fingerprint, ExpectedState: record.ExpectedState,
|
|
LockKeys: record.LockKeys, HeavyIO: record.HeavyIO,
|
|
SupersedesOperationID: record.SupersedesOperationID,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
switch record.State {
|
|
case StateReserved, StateDispatching, StateSucceeded, StateFailed:
|
|
default:
|
|
return fmt.Errorf("invalid state %q", record.State)
|
|
}
|
|
if record.EffectMayHaveApplied && record.SafeToResubmit {
|
|
return errors.New("an effectful failure cannot be safe to resubmit")
|
|
}
|
|
if record.State != StateFailed && (record.EffectMayHaveApplied || record.SafeToResubmit) {
|
|
return errors.New("failure semantics require a failed operation")
|
|
}
|
|
if record.CauseCode != "" && !validCauseCode(record.CauseCode) {
|
|
return errors.New("invalid operation cause_code")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func putRecord(bucket *bolt.Bucket, record Record) error {
|
|
if err := validateRecord(record); err != nil {
|
|
return err
|
|
}
|
|
encoded, err := json.Marshal(storedRecord{
|
|
Record: recordAlias(record), ExpectedState: cloneExpectedState(record.ExpectedState),
|
|
LockKeys: append([]string(nil), record.LockKeys...), HeavyIO: record.HeavyIO,
|
|
SupersedesOperationID: record.SupersedesOperationID,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("encode operation record: %w", err)
|
|
}
|
|
return bucket.Put([]byte(record.OperationID), encoded)
|
|
}
|
|
|
|
func decodeRecord(encoded []byte) (Record, error) {
|
|
var stored storedRecord
|
|
if err := json.Unmarshal(encoded, &stored); err != nil {
|
|
return Record{}, fmt.Errorf("decode operation record: %w", err)
|
|
}
|
|
record := Record(stored.Record)
|
|
record.ExpectedState = cloneExpectedState(stored.ExpectedState)
|
|
record.LockKeys = append([]string(nil), stored.LockKeys...)
|
|
record.HeavyIO = stored.HeavyIO
|
|
record.SupersedesOperationID = stored.SupersedesOperationID
|
|
if err := validateRecord(record); err != nil {
|
|
return Record{}, err
|
|
}
|
|
return record, nil
|
|
}
|
|
|
|
func cloneRecord(record Record) Record {
|
|
copyRecord := record
|
|
copyRecord.Result = append(json.RawMessage(nil), record.Result...)
|
|
copyRecord.ExpectedState = cloneExpectedState(record.ExpectedState)
|
|
copyRecord.LockKeys = append([]string(nil), record.LockKeys...)
|
|
if record.Error != nil {
|
|
copyError := *record.Error
|
|
copyRecord.Error = ©Error
|
|
}
|
|
return copyRecord
|
|
}
|
|
|
|
func cloneExpectedState(expected *ExpectedState) *ExpectedState {
|
|
if expected == nil {
|
|
return nil
|
|
}
|
|
copyExpected := *expected
|
|
copyExpected.DNSServers = append([]string(nil), expected.DNSServers...)
|
|
return ©Expected
|
|
}
|
|
|
|
func encodeExpiry(value time.Time) []byte {
|
|
encoded := make([]byte, 8)
|
|
binary.BigEndian.PutUint64(encoded, uint64(value.UnixNano()))
|
|
return encoded
|
|
}
|
|
|
|
func decodeExpiry(encoded []byte) (time.Time, error) {
|
|
if len(encoded) != 8 {
|
|
return time.Time{}, errors.New("invalid nonce expiry record")
|
|
}
|
|
return time.Unix(0, int64(binary.BigEndian.Uint64(encoded))).UTC(), nil
|
|
}
|
|
|
|
func expiryIndexKey(expiresAt time.Time, nonce string) []byte {
|
|
result := make([]byte, 8+len(nonce))
|
|
copy(result, encodeExpiry(expiresAt))
|
|
copy(result[8:], nonce)
|
|
return result
|
|
}
|
|
|
|
func operationExpiryKey(terminalAt time.Time, operationID string) []byte {
|
|
return expiryIndexKey(terminalAt, operationID)
|
|
}
|
|
|
|
func terminalState(state State) bool {
|
|
return state == StateSucceeded || state == StateFailed
|
|
}
|
|
|
|
func pruneExpiredOperationDetails(tx *bolt.Tx, now time.Time) error {
|
|
operations := tx.Bucket(operationsBucket)
|
|
expiry := tx.Bucket(operationExpiryBucket)
|
|
blockers := tx.Bucket(resourceBlockersBucket)
|
|
if operations == nil || expiry == nil || blockers == nil {
|
|
return errors.New("operation retention buckets are unavailable")
|
|
}
|
|
cutoff := now.Add(-operationDetailRetention).UnixNano()
|
|
cursor := expiry.Cursor()
|
|
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
|
|
if len(key) < 9 || int64(binary.BigEndian.Uint64(key[:8])) > cutoff {
|
|
break
|
|
}
|
|
operationID := string(key[8:])
|
|
encoded := operations.Get([]byte(operationID))
|
|
if encoded != nil {
|
|
record, err := decodeRecord(encoded)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
indexedAt := int64(binary.BigEndian.Uint64(key[:8]))
|
|
if terminalState(record.State) && !record.Tombstone && record.UpdatedAt.UnixNano() == indexedAt {
|
|
// Effectful failures remain durable resource fences. Other terminal
|
|
// details may expire, while the operation ID tombstone remains.
|
|
if !blocksResources(record) {
|
|
previous := cloneRecord(record)
|
|
record.Tombstone = true
|
|
record.Result = nil
|
|
record.ExpectedState = nil
|
|
record.LockKeys = nil
|
|
record.HeavyIO = false
|
|
if record.Error != nil {
|
|
record.Error = &PublicError{Code: record.Error.Code, Message: "operation details expired; operation_id remains permanently reserved"}
|
|
}
|
|
if err := putRecord(operations, record); err != nil {
|
|
return err
|
|
}
|
|
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if err := cursor.Delete(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func pruneExpiredNonces(nonces, expiry *bolt.Bucket, now time.Time) {
|
|
cursor := expiry.Cursor()
|
|
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
|
|
if len(key) < 9 || int64(binary.BigEndian.Uint64(key[:8])) >= now.UnixNano() {
|
|
break
|
|
}
|
|
nonce := key[8:]
|
|
if encoded := nonces.Get(nonce); encoded != nil && len(encoded) == 8 && string(encoded) == string(key[:8]) {
|
|
_ = nonces.Delete(nonce)
|
|
}
|
|
_ = cursor.Delete()
|
|
}
|
|
}
|
|
|
|
type expectedStateContextKey struct{}
|