package api import ( "bytes" "context" "crypto/sha256" "encoding/json" "errors" "fmt" "io" "log" "net/http" "net/url" "regexp" "strconv" "strings" "sync" "time" "unicode/utf8" "git.luoli.live/root/virt/agent/auth" consolebroker "git.luoli.live/root/virt/agent/console" "git.luoli.live/root/virt/agent/hypervisor" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/vpcrouting" "git.luoli.live/root/virt/agent/worker" ) const Version = "0.1.0" var ( apiUUIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$`) resourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`) operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`) stableCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`) sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) catalogIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`) sourceTokenPattern = regexp.MustCompile(`^[A-Za-z0-9._~-]{32,512}$`) ) const ( catalogImportTimeout = 2 * time.Hour backupOperationTimeout = 24 * time.Hour storageConfigurationWorkerTimeout = 120 * time.Second defaultAdmissionTimeout = 2 * time.Second maxAsyncImports = 2 ) type Server struct { Authenticator *auth.Authenticator Journal *operation.Journal Hypervisor hypervisor.Hypervisor Console *consolebroker.Broker MaxBodyBytes int64 OperationTimeout time.Duration AdmissionTimeout time.Duration MutationExecutor worker.Executor asyncMu sync.Mutex asyncRunning map[string]struct{} asyncSlots chan struct{} asyncWG sync.WaitGroup lifecycleMu sync.Mutex closing bool requestWG sync.WaitGroup backgroundCtx context.Context backgroundCancel context.CancelFunc } type trustedWorkerRequestKey struct{} type trustedSupersededOperationKey struct{} // SetMutationExecutor enables the clean-break process-isolated mutation path. // It must be called during startup before request admission begins. func (s *Server) SetMutationExecutor(executor worker.Executor) error { if executor == nil { return errors.New("mutation executor is required") } s.MutationExecutor = executor return nil } // ServeTrustedWorkerRequest dispatches a request whose bounded envelope was // already authenticated and validated by the parent Agent. Hidden workers do // not have access to the panel secret through HTTP headers and cannot recurse // because they never install a MutationExecutor. func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, superseded ...operation.Record) { setResponseHeaders(w) if !s.beginRequest() { writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down") return } defer s.requestWG.Done() if int64(len(body)) > s.MaxBodyBytes || actor.OperationID == "" { writePreDispatchError(w, http.StatusBadRequest, "invalid_worker_request", "worker request is invalid") return } trusted := context.WithValue(r.Context(), trustedWorkerRequestKey{}, struct{}{}) if len(superseded) > 0 { prior := superseded[0] trusted = context.WithValue(trusted, trustedSupersededOperationKey{}, prior) } s.route(w, r.WithContext(trusted), body, actor) } func trustedSupersededOperation(ctx context.Context, operationID string) (operation.Record, bool) { prior, ok := ctx.Value(trustedSupersededOperationKey{}).(operation.Record) return prior, ok && prior.OperationID == operationID } func isVPCRepairTarget(target string) bool { return strings.HasSuffix(target, "/repair") && (strings.Contains(target, "/v1/vpc/networks/") || strings.Contains(target, "/vpc-port/")) } type errorBody struct { Code string `json:"code"` Message string `json:"message"` Retryable bool `json:"retryable"` DispatchState string `json:"dispatch_state,omitempty"` Operation *operation.Record `json:"operation,omitempty"` } type errorEnvelope struct { Error errorBody `json:"error"` } type operationResponse struct { OperationID string `json:"operation_id"` Action string `json:"action"` ResourceID string `json:"resource_id"` RequestFingerprint string `json:"request_fingerprint"` Status operation.State `json:"status"` Replayed bool `json:"replayed"` Result json.RawMessage `json:"result,omitempty"` Error *operation.PublicError `json:"error,omitempty"` CauseCode string `json:"cause_code,omitempty"` EffectMayHaveApplied bool `json:"effect_may_have_applied"` SafeToResubmit bool `json:"safe_to_resubmit"` Retryable bool `json:"retryable"` } type fenceResolutionResponse struct { operation.FenceResolution Replayed bool `json:"replayed"` } func NewServer( authenticator *auth.Authenticator, journal *operation.Journal, hv hypervisor.Hypervisor, console *consolebroker.Broker, maxBodyBytes int64, operationTimeout time.Duration, ) (*Server, error) { if authenticator == nil || journal == nil || hv == nil || console == nil { return nil, errors.New("authenticator, journal, hypervisor, and console broker are required") } if maxBodyBytes < 1024 { return nil, errors.New("max body size must be at least 1024 bytes") } if operationTimeout <= 0 { return nil, errors.New("operation timeout must be positive") } backgroundCtx, backgroundCancel := context.WithCancel(context.Background()) return &Server{ Authenticator: authenticator, Journal: journal, Hypervisor: hv, Console: console, MaxBodyBytes: maxBodyBytes, OperationTimeout: operationTimeout, AdmissionTimeout: min(operationTimeout, defaultAdmissionTimeout), asyncRunning: make(map[string]struct{}), asyncSlots: make(chan struct{}, maxAsyncImports), backgroundCtx: backgroundCtx, backgroundCancel: backgroundCancel, }, nil } func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { setResponseHeaders(w) if !s.beginRequest() { writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down") return } defer s.requestWG.Done() if !strings.HasPrefix(r.URL.Path, "/v1/") || strings.HasSuffix(r.URL.Path, "/") { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return } body, err := readBoundedBody(w, r, s.MaxBodyBytes) if err != nil { var tooLarge *http.MaxBytesError if errors.As(err, &tooLarge) { writePreDispatchError(w, http.StatusRequestEntityTooLarge, "body_too_large", "request body exceeds configured limit") } else { writePreDispatchError(w, http.StatusBadRequest, "invalid_body", "could not read request body") } return } actor, err := s.Authenticator.Authenticate(r, body) if err != nil { if errors.Is(err, auth.ErrNonceStore) { writePreDispatchError(w, http.StatusServiceUnavailable, "nonce_store_unavailable", "authentication state is unavailable") return } if errors.Is(err, auth.ErrReplay) { // The matching nonce may belong to an earlier request that already // crossed the durable operation boundary. A replay rejection is not // proof that the operation was never dispatched. writeError(w, http.StatusUnauthorized, "replayed_request", "request nonce has already been used", nil) return } writePreDispatchError(w, http.StatusUnauthorized, "unauthorized", "request authentication failed") return } s.route(w, r, body, actor) } func (s *Server) route(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor) { segments := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") if len(segments) < 2 || segments[0] != "v1" { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return } if actor.SystemRecovery && !systemRecoveryRequestAllowed(actor, r.Method, segments) { writePreDispatchError(w, http.StatusForbidden, "forbidden", "system recovery actor is not authorized for this endpoint") return } switch segments[1] { case "health": if len(segments) != 2 || r.Method != http.MethodGet { methodOrNotFound(w, r, len(segments) == 2, http.MethodGet) return } if err := s.Hypervisor.Ping(r.Context()); err != nil { writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, map[string]any{"status": "ok", "service": "virt-host-agent", "version": Version}) case "capabilities": if len(segments) != 2 || r.Method != http.MethodGet { methodOrNotFound(w, r, len(segments) == 2, http.MethodGet) return } result, err := s.Hypervisor.Capabilities(r.Context()) if err != nil { writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, result) case "storage": s.routeStorage(w, r, segments, body, actor) case "vpc": s.routeVPC(w, r, segments, body, actor) case "vms": s.routeVMs(w, r, segments, body, actor) case "templates": s.routeTemplates(w, r, segments, body, actor) case "rescue-templates": s.routeRescueTemplates(w, r, segments, body, actor) case "user-isos": s.routeUserISOs(w, r, segments, body, actor) case "operations": s.routeOperations(w, r, segments, body, actor) default: writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } } func (s *Server) routeVPC(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { repairPath := len(segments) == 6 && segments[5] == "repair" if (len(segments) != 5 && !repairPath) || segments[2] != "networks" || !catalogIDPattern.MatchString(segments[3]) || (segments[4] != "apply" && segments[4] != "remove") { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return } if !requireSuperAdminManage(w, actor) { return } if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } vpcKey := segments[3] switch segments[4] { case "apply": request, ok := decodeRequired[vpcrouting.ApplyRequest](w, body) if !ok { return } if request.Network.VPCKey != vpcKey { writePreDispatchError(w, http.StatusUnprocessableEntity, "vpc_key_mismatch", "body vpc_key must match the request target") return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = operationID normalized, err := vpcrouting.ValidateApplyRequest(request) if err != nil { writePreDispatchMappedError(w, hypervisor.NewError(hypervisor.ErrInvalid, publicErrorCode(err, "invalid_vpc_projection"), publicErrorMessage(err, "VPC projection is invalid"), err)) return } request = normalized if request.VPCRepair != repairPath || (request.VPCRepair != (request.SupersedesOperationID != "")) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation") return } action := "apply-vpc-network:" + vpcKey expected := &operation.ExpectedState{ Kind: operation.ExpectedVPCNetwork, VPCKey: vpcKey, VPCRevision: request.Network.DesiredRevision, VPCDigest: request.DesiredDigest, VPCPresent: true, } admission := hypervisor.VPCNetworkAdmission(vpcKey) supersedes, ok := s.authorizeVPCRepair( w, r, operationID, request.SupersedesOperationID, "vpc:"+vpcKey, action, expected, admission, repairPath, ) if !ok { return } s.runMutationExpected(w, r, body, actor, operationID, "vpc:"+vpcKey, action, expected, true, http.StatusOK, admission, func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyVPCNetwork(ctx, request) }, supersedes...) case "remove": request, ok := decodeRequired[vpcrouting.RemoveRequest](w, body) if !ok { return } if request.VPCKey != vpcKey { writePreDispatchError(w, http.StatusUnprocessableEntity, "vpc_key_mismatch", "body vpc_key must match the request target") return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = operationID if err := vpcrouting.ValidateRemoveRequest(request); err != nil { writePreDispatchMappedError(w, hypervisor.NewError(hypervisor.ErrInvalid, publicErrorCode(err, "invalid_vpc_remove"), publicErrorMessage(err, "VPC removal request is invalid"), err)) return } if request.VPCRepair != repairPath || (request.VPCRepair != (request.SupersedesOperationID != "")) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation") return } action := "remove-vpc-network:" + vpcKey expected := &operation.ExpectedState{ Kind: operation.ExpectedVPCNetwork, VPCKey: vpcKey, VPCRevision: request.ExpectedRevision, VPCDigest: request.ExpectedDigest, VPCPresent: false, } admission := hypervisor.VPCNetworkAdmission(vpcKey) supersedes, ok := s.authorizeVPCRepair( w, r, operationID, request.SupersedesOperationID, "vpc:"+vpcKey, action, expected, admission, repairPath, ) if !ok { return } s.runMutationExpected(w, r, body, actor, operationID, "vpc:"+vpcKey, action, expected, true, http.StatusOK, admission, func(ctx context.Context) (any, error) { return s.Hypervisor.RemoveVPCNetwork(ctx, request) }, supersedes...) } } func (s *Server) authorizeVPCRepair(w http.ResponseWriter, r *http.Request, operationID, predecessorID, resourceID, action string, expected *operation.ExpectedState, admission hypervisor.MutationAdmission, repair bool) ([]string, bool) { if !repair { return nil, true } if !operationIDPattern.MatchString(predecessorID) || operationID == predecessorID { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair_predecessor", "VPC repair requires a different valid predecessor operation") return nil, false } trustedPrior, trusted := trustedSupersededOperation( r.Context(), predecessorID, ) existing, existingErr := s.Journal.Get(operationID) if existingErr == nil { if existing.Action != action || existing.ResourceID != resourceID || existing.SupersedesOperationID != predecessorID { writePreDispatchError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request") return nil, false } } else if !errors.Is(existingErr, operation.ErrNotFound) { writeMappedError(w, existingErr, nil) return nil, false } else { prior := trustedPrior var err error if !trusted { prior, err = s.Journal.Get(predecessorID) } spec := operation.Spec{ OperationID: operationID, ResourceID: resourceID, Action: action, ExpectedState: expected, LockKeys: append([]string(nil), admission.LockKeys...), SupersedesOperationID: predecessorID, } if err != nil || !operation.ValidVPCRepairSupersession(spec, prior) { writePreDispatchError(w, http.StatusConflict, "vpc_repair_not_authorized", "VPC repair no longer matches one exact fenced predecessor") return nil, false } if !trusted { _, incident, incidentErr := s.Journal.CompletionDeliveryIncident( predecessorID, ) if incidentErr != nil { writeMappedError(w, incidentErr, nil) return nil, false } if incident { writePreDispatchError(w, http.StatusConflict, "vpc_repair_not_authorized", "VPC repair is blocked by a completion-delivery incident") return nil, false } } } if trusted { return nil, true } return []string{predecessorID}, true } func (s *Server) routeStorage(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if len(segments) != 3 || segments[2] != "configuration" { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return } if !requireSuperAdminManage(w, actor) { return } if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } request, ok := decodeRequired[hypervisor.StorageConfigurationRequest](w, body) if !ok { return } if err := hypervisor.ValidateStorageConfigurationRequest(request); err != nil { writePreDispatchMappedError(w, err) return } digests := hypervisor.StorageConfigurationPathDigests(request) expected := &operation.ExpectedState{ Kind: operation.ExpectedStorageConfiguration, ManagedPathDigest: digests[hypervisor.StorageRoleManaged], TemplatePathDigest: digests[hypervisor.StorageRoleTemplate], SeedPathDigest: digests[hypervisor.StorageRoleSeed], UserISOPathDigest: digests[hypervisor.StorageRoleUserISO], RescuePathDigest: digests[hypervisor.StorageRoleRescue], } s.runMutationExpected(w, r, body, actor, request.OperationID, "host-storage", "configure-storage", expected, true, http.StatusOK, hypervisor.StorageConfigurationAdmission(), func(ctx context.Context) (any, error) { return s.Hypervisor.ConfigureStorage(ctx, request) }) } func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if len(segments) == 2 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListVMs(r.Context()) writeNamedList(w, "vms", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.CreateVMRequest](w, body) if !ok { return } normalized, err := hypervisor.NormalizeStartupScript(request.StartupScript) if err != nil { writePreDispatchMappedError(w, err) return } request.StartupScript = normalized resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID if request.UUID == "" { request.UUID = operationUUID(resolvedOperationID) } request.DomainName, err = hypervisor.LibvirtDomainName(actor.UserID, request.UUID) if err != nil { writePreDispatchMappedError(w, err) return } if err := hypervisor.ValidateCreateVM(request); err != nil { writePreDispatchMappedError(w, err) return } s.runMutation(w, r, body, actor, request.OperationID, request.UUID, "create-vm", true, http.StatusCreated, hypervisor.CreateVMAdmission(request), func(ctx context.Context) (any, error) { return s.Hypervisor.CreateVM(ctx, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) < 3 || !apiUUIDPattern.MatchString(segments[2]) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_uuid", "VM UUID must be canonical") return } uuid := strings.ToLower(segments[2]) if len(segments) == 3 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid) writeResult(w, result, err) case http.MethodDelete: if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.DeleteVMRequest](w, body) if !ok { return } if !request.RemoveManagedDisks { writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true") return } s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-vm", true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.DeleteVM(ctx, uuid, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete) } return } switch segments[3] { case "backups": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.BackupRequest](w, body) if !ok { return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID if err := hypervisor.ValidateBackupRequest(request); err != nil { writePreDispatchMappedError(w, err) return } action := "create-backup:" + request.BackupKey + ":" + request.ConsistencyMode s.runAsyncMutationWithDeadline( w, r, body, actor, request.OperationID, uuid, action, false, hypervisor.CreateBackupAdmission(uuid, request), backupOperationTimeout, func(ctx context.Context) (any, error) { return s.Hypervisor.CreateBackup(ctx, uuid, request) }, ) return } case "restore-backup": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.RestoreBackupRequest](w, body) if !ok { return } if request.UUID != "" && !strings.EqualFold(request.UUID, uuid) { writePreDispatchError(w, http.StatusUnprocessableEntity, "uuid_mismatch", "body UUID must match the restore target") return } request.UUID = uuid resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID derivedDomainName, err := hypervisor.LibvirtDomainName(actor.UserID, uuid) if err != nil { writePreDispatchMappedError(w, err) return } if request.RequestedDomainName != "" && request.RequestedDomainName != derivedDomainName { writePreDispatchError(w, http.StatusUnprocessableEntity, "domain_name_mismatch", "domain_name must match the Agent-derived ownership name") return } request.DomainName = derivedDomainName if err := hypervisor.ValidateRestoreBackupRequest(request); err != nil { writePreDispatchMappedError(w, err) return } expected, err := s.createLikeNetworkExpectedState(r.Context(), request.CreateVMRequest, nil) if err != nil { writePreDispatchMappedError(w, err) return } action := "restore-backup:" + request.BackupKey s.runAsyncMutationWithDeadlineExpected( w, r, body, actor, request.OperationID, uuid, action, expected, true, hypervisor.RestoreBackupAdmission(request), backupOperationTimeout, func(ctx context.Context) (any, error) { return s.Hypervisor.RestoreBackup(ctx, request) }, ) return } case "recover-backup": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.RecoverBackupRequest](w, body) if !ok { return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID if err := hypervisor.ValidateRecoverBackupRequest(request); err != nil { writePreDispatchMappedError(w, err) return } action := "recover-backup:" + request.BackupKey + ":" + request.SourceSHA256 s.runAsyncMutationWithDeadline( w, r, body, actor, request.OperationID, uuid, action, true, hypervisor.RecoverBackupAdmission(uuid, request), backupOperationTimeout, func(ctx context.Context) (any, error) { return s.Hypervisor.RecoverBackup(ctx, uuid, request) }, ) return } case "peer-migration": if len(segments) != 5 || !requireManage(w, actor) { if len(segments) != 5 { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } return } switch segments[4] { case "prepare": if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } request, ok := decodeRequired[hypervisor.PreparePeerMigrationRequest](w, body) if !ok { return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = operationID if err := hypervisor.ValidatePreparePeerMigrationRequest(request); err != nil { writePreDispatchMappedError(w, err) return } s.runAsyncMutationWithDeadline(w, r, body, actor, operationID, uuid, "prepare-peer-migration:"+request.MigrationID, true, hypervisor.PreparePeerMigrationAdmission(uuid, request), backupOperationTimeout, func(ctx context.Context) (any, error) { return s.Hypervisor.PreparePeerMigration(ctx, uuid, request) }) case "receive": if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } request, ok := decodeRequired[hypervisor.ReceivePeerMigrationRequest](w, body) if !ok { return } if request.UUID != "" && !strings.EqualFold(request.UUID, uuid) { writePreDispatchError(w, http.StatusUnprocessableEntity, "uuid_mismatch", "body UUID must match the migration target") return } request.UUID = uuid operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = operationID request.DomainName, err = hypervisor.LibvirtDomainName(actor.UserID, uuid) if err != nil { writePreDispatchMappedError(w, err) return } if err := hypervisor.ValidateReceivePeerMigrationRequest(request); err != nil { writePreDispatchMappedError(w, err) return } expected, err := s.createLikeNetworkExpectedState(r.Context(), request.CreateVMRequest, request.VPCPort) if err != nil { writePreDispatchMappedError(w, err) return } s.runAsyncMutationWithDeadlineExpected(w, r, body, actor, operationID, uuid, "receive-peer-migration:"+request.MigrationID, expected, true, hypervisor.ReceivePeerMigrationAdmission(request), backupOperationTimeout, func(ctx context.Context) (any, error) { return s.Hypervisor.ReceivePeerMigration(ctx, request) }) case "rollback": if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } request, ok := decodeRequired[hypervisor.RollbackPeerMigrationRequest](w, body) if !ok { return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = operationID if err := hypervisor.ValidateRollbackPeerMigrationRequest(request); err != nil { writePreDispatchMappedError(w, err) return } expected := &operation.ExpectedState{ Kind: operation.ExpectedPeerMigrationState, Enabled: request.SourceAutostartEnabled, StartIfSuspended: request.SourceWasRunning, } s.runMutationExpected(w, r, body, actor, operationID, uuid, "rollback-peer-migration:"+request.MigrationID, expected, true, http.StatusOK, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.RollbackPeerMigration(ctx, uuid, request) }) case "activate": if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } request, ok := decodeRequired[hypervisor.ActivatePeerMigrationRequest](w, body) if !ok { return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = operationID if err := hypervisor.ValidateActivatePeerMigrationRequest(request); err != nil { writePreDispatchMappedError(w, err) return } expected := &operation.ExpectedState{ Kind: operation.ExpectedPeerMigrationState, Enabled: request.TargetAutostartEnabled, StartIfSuspended: request.TargetWasRunning, } s.runMutationExpected(w, r, body, actor, operationID, uuid, "activate-peer-migration:"+request.MigrationID, expected, true, http.StatusOK, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ActivatePeerMigration(ctx, uuid, request) }) case "cleanup": if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } request, ok := decodeRequired[hypervisor.CleanupPeerMigrationRequest](w, body) if !ok { return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = operationID if err := hypervisor.ValidateCleanupPeerMigrationRequest(request); err != nil { writePreDispatchMappedError(w, err) return } s.runMutation(w, r, body, actor, operationID, uuid, "delete-vm", true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.CleanupPeerMigration(ctx, uuid, request) }) case "cleanup-target": if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } request, ok := decodeRequired[hypervisor.CleanupPeerMigrationTargetRequest](w, body) if !ok { return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = operationID if err := hypervisor.ValidateCleanupPeerMigrationTargetRequest(request); err != nil { writePreDispatchMappedError(w, err) return } action := "cleanup-peer-migration-target:" + request.MigrationID existingReplay := false trustedPrior, trustedPriorFound := trustedSupersededOperation( r.Context(), request.PriorOperationID, ) existing, existingErr := s.Journal.Get(operationID) if existingErr == nil { existingReplay = existing.Action == action && existing.ResourceID == uuid && existing.SupersedesOperationID == request.PriorOperationID } else if !errors.Is(existingErr, operation.ErrNotFound) { writeMappedError(w, existingErr, nil) return } if !existingReplay { prior := trustedPrior var priorErr error if !trustedPriorFound { prior, priorErr = s.Journal.Get(request.PriorOperationID) } if priorErr != nil || prior.ResourceID != uuid || prior.Action != "receive-peer-migration:"+request.MigrationID || prior.State != operation.StateFailed || !prior.EffectMayHaveApplied || prior.SafeToResubmit || prior.ExecutionFenced { writePreDispatchError(w, http.StatusConflict, "peer_migration_target_cleanup_not_authorized", "target cleanup requires the exact safely reaped failed receive operation") return } if !trustedPriorFound { _, incidentFound, incidentErr := s.Journal.CompletionDeliveryIncident(prior.OperationID) if incidentErr != nil { writeMappedError(w, incidentErr, nil) return } if incidentFound { writePreDispatchError(w, http.StatusConflict, "peer_migration_target_cleanup_not_authorized", "target cleanup is blocked by a completion-delivery incident") return } } ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, prior) cancel() if inspectErr != nil { writeMappedError(w, inspectErr, nil) return } if !peerMigrationTargetCleanupInspectionMatches(prior, request.CleanupPlanID, inspection) { writePreDispatchError(w, http.StatusConflict, "peer_migration_cleanup_plan_changed", "target cleanup plan is stale or no longer safe") return } request.PriorOperation = prior } supersedes := []string{request.PriorOperationID} if trustedPriorFound { // The isolated worker has its own Journal. The parent Journal // already admitted this exact supersession and retains the audit // link; the child receives the immutable predecessor separately. supersedes = nil } s.runMutationExpected(w, r, body, actor, operationID, uuid, action, nil, true, http.StatusOK, hypervisor.CleanupPeerMigrationTargetAdmission(uuid, request.MigrationID), func(ctx context.Context) (any, error) { return s.Hypervisor.CleanupPeerMigrationTarget(ctx, uuid, request) }, supersedes...) default: writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } return case "force-delete": if len(segments) == 4 && r.Method == http.MethodPost { if !requireSuperAdminPrivilegedOperation(w, actor) { return } request, ok := decodeRequired[hypervisor.ForceDeleteVMRequest](w, body) if !ok { return } if !request.RemoveManagedDisks { writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true") return } if !request.AckDataLoss { writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true") return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } if !operationIDPattern.MatchString(request.PriorOperationID) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_prior_operation_id", "prior_operation_id must contain 8-80 URL-safe characters") return } if operationID == request.PriorOperationID { writePreDispatchError(w, http.StatusUnprocessableEntity, "force_operation_id_reused", "force delete requires a new operation_id") return } existingReplay := false trustedPrior, trustedPriorFound := trustedSupersededOperation( r.Context(), request.PriorOperationID, ) existing, existingErr := s.Journal.Get(operationID) if existingErr == nil { existingReplay = existing.Action == "force-delete-vm" && existing.ResourceID == uuid && existing.SupersedesOperationID == request.PriorOperationID } else if !errors.Is(existingErr, operation.ErrNotFound) { writeMappedError(w, existingErr, nil) return } if !existingReplay { prior := trustedPrior var priorErr error if !trustedPriorFound { prior, priorErr = s.Journal.Get(request.PriorOperationID) } priorActionAllowed := priorErr == nil && (prior.Action == "delete-vm" || prior.Action == "service-suspend") if !priorActionAllowed || prior.ResourceID != uuid || prior.State != operation.StateFailed || prior.Error == nil || prior.Error.Code != "force_required" { writePreDispatchError(w, http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful stop to require force") return } } supersedes := []string{request.PriorOperationID} if trustedPriorFound { // The parent Journal already authorized and records the force // successor. The isolated worker Journal receives the immutable // predecessor separately and must not try to load it locally. supersedes = nil } s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "force-delete-vm", nil, true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ForceDeleteVM(ctx, uuid, request) }, supersedes...) return } case "state": if len(segments) == 4 && r.Method == http.MethodGet { result, err := s.Hypervisor.GetVMState(r.Context(), uuid) writeResult(w, result, err) return } case "metadata": if len(segments) == 4 && r.Method == http.MethodGet { result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid) writeResult(w, result, err) return } case "actions": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.PowerRequest](w, body) if !ok { return } if !validPowerAction(request.Action) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_power_action", "action must be start, shutdown, reboot, reset, or destroy") return } s.runMutation(w, r, body, actor, request.OperationID, uuid, "power-"+string(request.Action), true, http.StatusOK, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.Power(ctx, uuid, request) }) return } case "settings": if len(segments) == 4 && (r.Method == http.MethodPatch || r.Method == http.MethodPost) { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.SettingsRequest](w, body) if !ok { return } if request.Hostname == nil && request.RootPassword == nil && request.SSHPublicKey == nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "empty_settings", "at least one setting is required") return } if !request.ConfirmRestart { writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true") return } expected := &operation.ExpectedState{ Kind: operation.ExpectedUpdateSettings, HostnameChanged: request.Hostname != nil, RootPasswordChanged: request.RootPassword != nil, SSHKeyChanged: request.SSHPublicKey != nil, } if request.Hostname != nil { expected.Hostname = *request.Hostname } s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "update-settings", expected, false, http.StatusOK, hypervisor.SeedMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateSettings(ctx, uuid, request) }) return } case "resources": if len(segments) == 4 { if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ResizeResourcesRequest](w, body) if !ok { return } if err := hypervisor.ValidateResizeResources(request); err != nil { writePreDispatchMappedError(w, err) return } expected := &operation.ExpectedState{ Kind: operation.ExpectedResizeResources, FromVCPUs: request.FromVCPUs, TargetVCPUs: request.TargetVCPUs, FromMemoryMiB: request.FromMemoryMiB, TargetMemoryMiB: request.TargetMemoryMiB, FromDiskGiB: request.FromDiskGiB, TargetDiskGiB: request.TargetDiskGiB, FromBandwidthMbps: request.FromBandwidthMbps, TargetBandwidthMbps: request.TargetBandwidthMbps, } s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "resize-vm", expected, true, http.StatusOK, hypervisor.ResizeResourcesAdmission(uuid, request), func(ctx context.Context) (any, error) { return s.Hypervisor.ResizeResources(ctx, uuid, request) }) return } case "network": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ChangeNetworkRequest](w, body) if !ok { return } if !request.ConfirmRestart { writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true") return } if err := hypervisor.ValidatePublicNetworkIntent(request.PublicNetwork); err != nil { writePreDispatchMappedError(w, err) return } var source hypervisor.LocalNetworkSource var err error if s.MutationExecutor != nil { source, err = hypervisor.ParseNetworkSelector(request.PublicNetwork.NetworkSelector) } else { source, err = s.Hypervisor.ResolveNetworkSelector(r.Context(), request.PublicNetwork.NetworkSelector) } if err != nil { writePreDispatchMappedError(w, err) return } expected := &operation.ExpectedState{ Kind: operation.ExpectedChangeNetwork, PublicNetwork: request.PublicNetwork, NetworkSourceType: source.Type, NetworkSourceName: source.Source, } s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "change-network", expected, true, http.StatusAccepted, hypervisor.ChangeNetworkAdmission(uuid, request.PublicNetwork), func(ctx context.Context) (any, error) { return s.Hypervisor.ChangeNetwork(ctx, uuid, request) }) return } if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.NetworkStateRequest](w, body) if !ok { return } enabled := segments[4] == "resume" action := "network-" + segments[4] expected := &operation.ExpectedState{Kind: operation.ExpectedNetworkState, Enabled: enabled} s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, action, expected, false, http.StatusOK, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { err := s.Hypervisor.SetNetworkState(ctx, uuid, enabled, request) return map[string]bool{"network_enabled": enabled}, err }) return } case "vpc-port": repairPath := len(segments) == 5 && segments[4] == "repair" if (len(segments) == 4 || repairPath) && (r.Method == http.MethodPost || r.Method == http.MethodDelete) { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.VPCPortRequest](w, body) if !ok { return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = operationID request, err = hypervisor.NormalizeVPCPortRequest(request) if err != nil { writePreDispatchMappedError(w, err) return } if request.VPCRepair != repairPath || (request.VPCRepair != (request.SupersedesOperationID != "")) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation") return } bridge, _ := vpcrouting.BridgeName(request.VPCKey) present := r.Method == http.MethodPost action := "apply-vpc-port" if !present { action = "remove-vpc-port" } expected := &operation.ExpectedState{ Kind: operation.ExpectedVPCPort, VPCPresent: present, VPCPortKey: request.PortKey, VPCKey: request.VPCKey, VPCPrivateIP: request.PrivateIP, VPCCIDR: request.CIDR, VPCGateway: request.Gateway, VPCMAC: request.MAC, VPCBridge: bridge, VPCRevision: request.HostRevision, VPCDigest: request.HostDigest, } admission := hypervisor.VPCPortAdmission(uuid, request) supersedes, ok := s.authorizeVPCRepair( w, r, operationID, request.SupersedesOperationID, uuid, action, expected, admission, repairPath, ) if !ok { return } s.runMutationExpected(w, r, body, actor, operationID, uuid, action, expected, true, http.StatusOK, admission, func(ctx context.Context) (any, error) { if present { return s.Hypervisor.ApplyVPCPort(ctx, uuid, request) } return s.Hypervisor.RemoveVPCPort(ctx, uuid, request) }, supersedes...) return } case "service": if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.ServiceStateRequest](w, body) if !ok { return } enabled := segments[4] == "resume" expected := &operation.ExpectedState{Kind: operation.ExpectedServiceState, Enabled: enabled, StartIfSuspended: request.StartIfSuspended} s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "service-"+segments[4], expected, true, http.StatusOK, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.SetServiceState(ctx, uuid, enabled, request) }) return } case "reinstall": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ReinstallRequest](w, body) if !ok { return } if err := hypervisor.ValidateTemplateSnapshot(request.TemplateID, request.TemplateRevision, request.TemplateGenerationToken); err != nil { writePreDispatchMappedError(w, err) return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID s.runMutation(w, r, body, actor, request.OperationID, uuid, "reinstall-vm", true, http.StatusAccepted, hypervisor.ReinstallVMAdmission(uuid, request), func(ctx context.Context) (any, error) { return s.Hypervisor.Reinstall(ctx, uuid, request) }) return } case "metrics": if len(segments) == 4 && r.Method == http.MethodGet { result, err := s.Hypervisor.GetMetrics(r.Context(), uuid) writeResult(w, result, err) return } case "console-ticket": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } target, err := s.Hypervisor.GetConsole(r.Context(), uuid) if err != nil { writeMappedError(w, err, nil) return } if target.Protocol != "vnc" { writeError(w, http.StatusNotImplemented, "unsupported_console", "only VNC console targets are supported", nil) return } ticket, err := s.Console.Issue(target.Address) if err != nil { writeError(w, http.StatusServiceUnavailable, "console_unavailable", "console target is unavailable", nil) return } writeJSON(w, http.StatusCreated, ticket) return } case "snapshots": s.routeSnapshots(w, r, segments, body, actor, uuid) return case "firewall": if len(segments) == 4 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.GetFirewall(r.Context(), uuid) writeResult(w, result, err) case http.MethodPut: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.FirewallPolicy](w, body) if !ok { return } operationID := request.OperationID request, err := hypervisor.NormalizeFirewallPolicy(request) if err != nil { writePreDispatchMappedError(w, err) return } expected := &operation.ExpectedState{Kind: operation.ExpectedFirewall, FirewallRevision: request.Revision, FirewallDigest: request.PolicyDigest} s.runMutationExpected(w, r, body, actor, operationID, uuid, "apply-firewall", expected, false, http.StatusOK, hypervisor.FirewallMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyFirewall(ctx, uuid, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPut) } return } case "rescue": s.routeRescue(w, r, segments, body, actor, uuid) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func peerMigrationTargetCleanupInspectionMatches(prior operation.Record, planID string, inspection hypervisor.OperationInspection) bool { if inspection.OperationID != prior.OperationID || inspection.Action != prior.Action || inspection.ResourceID != prior.ResourceID || inspection.RequestFingerprint != prior.Fingerprint || inspection.OperationStatus != prior.State || inspection.CheckedAt <= 0 || inspection.SummaryCode != hypervisor.InspectionSummaryCleanupRequired || inspection.EffectResolution != hypervisor.InspectionEffectPartial || inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" || inspection.AbortJob.Available || !inspection.Cleanup.Available || !strings.EqualFold(inspection.Cleanup.PlanID, planID) || len(inspection.Cleanup.Steps) == 0 { return false } for _, step := range inspection.Cleanup.Steps { switch step { case hypervisor.CleanupStepDeleteManagedVolumes, hypervisor.CleanupStepDeleteSeedVolume, hypervisor.CleanupStepUndefineNetworkFilter, hypervisor.CleanupStepDeleteNVRAM, hypervisor.CleanupStepRemoveTrafficState: default: return false } } return true } func backupWorkspaceCleanupInspectionMatches(prior operation.Record, planID string, inspection hypervisor.OperationInspection) bool { if inspection.OperationID != prior.OperationID || inspection.Action != prior.Action || inspection.ResourceID != prior.ResourceID || inspection.RequestFingerprint != prior.Fingerprint || inspection.OperationStatus != prior.State || inspection.CheckedAt <= 0 || inspection.Outcome != hypervisor.InspectionOutcomeSucceeded || inspection.EffectResolution != hypervisor.InspectionEffectApplied || inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" || inspection.AbortJob.Available { return false } if !inspection.Cleanup.Available { return planID == "" && inspection.SummaryCode == hypervisor.InspectionSummarySucceeded } if inspection.SummaryCode != hypervisor.InspectionSummaryCleanupRequired || planID == "" || !strings.EqualFold(inspection.Cleanup.PlanID, planID) || len(inspection.Cleanup.Steps) == 0 { return false } for _, step := range inspection.Cleanup.Steps { switch step { case hypervisor.CleanupStepDeleteSnapshotVolume, hypervisor.CleanupStepRemoveSnapshotMetadata: default: return false } } return true } func (s *Server) routeSnapshots(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) { if len(segments) == 4 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListSnapshots(r.Context(), uuid) writeNamedList(w, "snapshots", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.SnapshotRequest](w, body) if !ok { return } if request.SnapshotKey == "" { request.SnapshotKey = request.SnapshotID } if !catalogIDPattern.MatchString(request.SnapshotKey) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot", "snapshot key is invalid") return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID action := "create-snapshot:" + request.SnapshotKey + ":" + request.ConsistencyMode s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusCreated, hypervisor.CreateSnapshotAdmission(uuid, request.SnapshotKey), func(ctx context.Context) (any, error) { return s.Hypervisor.CreateSnapshot(ctx, uuid, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) < 5 || !resourceIDPattern.MatchString(segments[4]) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot_id", "snapshot ID is invalid") return } snapshotID := segments[4] if len(segments) == 5 { switch r.Method { case http.MethodGet: snapshots, err := s.Hypervisor.ListSnapshots(r.Context(), uuid) if err != nil { writeMappedError(w, err, nil) return } for _, snapshot := range snapshots { if snapshot.ID == snapshotID { writeJSON(w, http.StatusOK, snapshot) return } } writeError(w, http.StatusNotFound, "snapshot_not_found", "snapshot was not found", nil) case http.MethodDelete: if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body) if !ok { return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-snapshot:"+snapshotID, true, http.StatusOK, hypervisor.DeleteSnapshotAdmission(uuid, snapshotID), func(ctx context.Context) (any, error) { return s.Hypervisor.DeleteSnapshot(ctx, uuid, snapshotID, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete) } return } if len(segments) == 6 && segments[5] == "restore" && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body) if !ok { return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID s.runMutation(w, r, body, actor, request.OperationID, uuid, "restore-snapshot:"+snapshotID, true, http.StatusAccepted, hypervisor.RestoreSnapshotAdmission(uuid, snapshotID, request.OperationID), func(ctx context.Context) (any, error) { return s.Hypervisor.RestoreSnapshot(ctx, uuid, snapshotID, request) }) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) routeRescue(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) { if len(segments) == 4 && r.Method == http.MethodGet { result, err := s.Hypervisor.GetRescue(r.Context(), uuid) writeResult(w, result, err) return } if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "enter" || segments[4] == "exit" || segments[4] == "force-start") { if !requireManage(w, actor) { return } if segments[4] == "enter" { request, ok := decodeRequired[hypervisor.RescueRequest](w, body) if !ok { return } if err := hypervisor.ValidateRescueTemplateSnapshot( request.TemplateID, request.Revision, request.GenerationToken, ); err != nil { writePreDispatchMappedError(w, err) return } s.runMutation(w, r, body, actor, request.OperationID, uuid, "enter-rescue", true, http.StatusAccepted, hypervisor.EnterRescueAdmission(uuid, request.TemplateID), func(ctx context.Context) (any, error) { return s.Hypervisor.EnterRescue(ctx, uuid, request) }) } else if segments[4] == "force-start" { request, ok := decodeRequired[hypervisor.ForceRescueStartRequest](w, body) if !ok { return } if !request.AckDataLoss { writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true") return } s.runMutation(w, r, body, actor, request.OperationID, uuid, "force-start-rescue", true, http.StatusAccepted, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ForceStartRescue(ctx, uuid, request) }) } else { request, ok := decodeRequired[hypervisor.ExitRescueRequest](w, body) if !ok { return } if !request.AckDataLoss { writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true") return } if request.NetworkSuspended == nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "network_state_required", "network_suspended must be provided") return } action := "exit-rescue:connected" if *request.NetworkSuspended { action = "exit-rescue:suspended" } s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusAccepted, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ExitRescue(ctx, uuid, request) }) } return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) routeTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if !actor.IsAdmin && !actor.IsSuperAdmin { writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required") return } if len(segments) == 2 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListTemplates(r.Context()) writeNamedList(w, "templates", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ImportTemplateRequest](w, body) if !ok { return } if err := hypervisor.ValidateTemplateSnapshot(request.ID, request.Revision, request.GenerationToken); err != nil { writePreDispatchMappedError(w, err) return } if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil { writePreDispatchMappedError(w, err) return } s.runAsyncMutation(w, r, body, actor, request.OperationID, "template:"+request.ID, "import-template", false, hypervisor.ImportTemplateAdmission(request), func(ctx context.Context) (any, error) { return s.Hypervisor.ImportTemplate(ctx, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) == 3 && resourceIDPattern.MatchString(segments[2]) { templateID := segments[2] switch r.Method { case http.MethodPatch: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.UpdateTemplateRequest](w, body) if !ok { return } if strings.TrimSpace(request.Name) == "" || len(request.Name) > 120 { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_template_name", "template name must contain 1-120 characters") return } s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "update-template", false, http.StatusOK, hypervisor.UpdateTemplateAdmission(templateID, request), func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateTemplate(ctx, templateID, request) }) case http.MethodDelete: if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.MutationRequest](w, body) if !ok { return } s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "delete-template", true, http.StatusOK, hypervisor.DeleteTemplateAdmission(templateID), func(ctx context.Context) (any, error) { err := s.Hypervisor.DeleteTemplate(ctx, templateID) return catalogDeleteResult("template", templateID, 0, err == nil), err }) default: methodOrNotFound(w, r, true, http.MethodPatch, http.MethodDelete) } return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) routeRescueTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if !actor.IsAdmin && !actor.IsSuperAdmin { writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required") return } if len(segments) == 2 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListRescueTemplates(r.Context()) writeNamedList(w, "templates", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ImportRescueTemplateRequest](w, body) if !ok { return } if err := hypervisor.ValidateRescueTemplateSnapshot(request.ID, request.Revision, request.GenerationToken); err != nil { writePreDispatchMappedError(w, err) return } if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil { writePreDispatchMappedError(w, err) return } s.runAsyncMutation(w, r, body, actor, request.OperationID, "rescue-template:"+request.ID, "import-rescue-template", false, hypervisor.ImportRescueTemplateAdmission(request), func(ctx context.Context) (any, error) { return s.Hypervisor.ImportRescueTemplate(ctx, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.MutationRequest](w, body) if !ok { return } id := segments[2] s.runMutation(w, r, body, actor, request.OperationID, "rescue-template:"+id, "delete-rescue-template", true, http.StatusOK, hypervisor.DeleteRescueTemplateAdmission(id), func(ctx context.Context) (any, error) { err := s.Hypervisor.DeleteRescueTemplate(ctx, id) return catalogDeleteResult("rescue_template", id, 0, err == nil), err }) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) routeUserISOs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { ownerID, err := ownerFromRequest(r, actor) if err != nil { writePreDispatchError(w, http.StatusForbidden, "forbidden", err.Error()) return } if len(segments) == 2 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListUserISOs(r.Context(), ownerID) writeNamedList(w, "user_isos", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ImportUserISORequest](w, body) if !ok { return } if request.OwnerID == 0 { request.OwnerID = ownerID } if request.OwnerID != ownerID && !actor.IsAdmin && !actor.IsSuperAdmin { writePreDispatchError(w, http.StatusForbidden, "forbidden", "cannot import ISO for another owner") return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } if request.ID == "" { request.ID = operationCatalogID(operationID) } if !catalogIDPattern.MatchString(request.ID) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_user_iso_id", "id must contain 32 lowercase hexadecimal characters") return } if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil { writePreDispatchMappedError(w, err) return } s.runAsyncMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", request.OwnerID, request.ID), "import-user-iso", false, hypervisor.ImportUserISOAdmission(request), func(ctx context.Context) (any, error) { return s.Hypervisor.ImportUserISO(ctx, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.MutationRequest](w, body) if !ok { return } isoID := segments[2] s.runMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", ownerID, isoID), "delete-user-iso", true, http.StatusOK, hypervisor.DeleteUserISOAdmission(ownerID, isoID), func(ctx context.Context) (any, error) { err := s.Hypervisor.DeleteUserISO(ctx, isoID, ownerID) return catalogDeleteResult("user_iso", isoID, ownerID, err == nil), err }) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func catalogDeleteResult(resourceType, id string, ownerID int64, deleted bool) hypervisor.CatalogDeleteResult { return hypervisor.CatalogDeleteResult{ ResourceType: resourceType, ID: id, OwnerID: ownerID, Deleted: deleted, ManagedImageAbsent: deleted, ManagedMetadataAbsent: deleted, ManagedArtifactsAbsent: deleted, } } func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if len(segments) < 3 || !resourceIDPattern.MatchString(segments[2]) { writeError(w, http.StatusNotFound, "operation_not_found", "operation was not found", nil) return } operationID := segments[2] if len(segments) == 3 && r.Method == http.MethodGet { record, err := s.Journal.Get(operationID) if err != nil { writeMappedError(w, err, nil) return } if record.Action == "configure-storage" && !requireSuperAdminManage(w, actor) { return } writeJSON(w, http.StatusOK, record) return } if len(segments) == 4 && segments[3] == "inspect" && r.Method == http.MethodGet { if !requireSuperAdminInspection(w, actor, operationID) { return } stored, err := s.Journal.GetForInspection(operationID) if err != nil { writeMappedError(w, err, nil) return } if !requireSystemRecoveryRecord(w, actor, stored) { return } incident, incidentFound, err := s.Journal.CompletionDeliveryIncident(operationID) if err != nil { writeMappedError(w, err, nil) return } var inspection hypervisor.OperationInspection if incidentFound { // Delivery corruption is already sufficient to make the panel view // inconclusive. Do not make reporting that durable Agent incident // depend on a live host read which may itself time out or fail. inspection = hypervisor.OperationInspection{ OperationStatus: stored.State, CheckedAt: time.Now().UTC().Unix(), } if stored.Error != nil { inspection.ErrorCode = stored.Error.Code } applyCompletionDeliveryIncident(&inspection, incident) } else { ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) defer cancel() inspection, err = s.Hypervisor.InspectOperation(ctx, stored) if err != nil { writeMappedError(w, err, nil) return } } applyExecutionFencedCreateInspection(&inspection, stored) // Identity always comes from the immutable Journal record, never from a // hypervisor adapter or its observation of current host state. inspection.OperationID = stored.OperationID inspection.Action = stored.Action inspection.ResourceID = stored.ResourceID inspection.RequestFingerprint = stored.Fingerprint inspection.FenceState, err = s.Journal.FenceState(operationID) if err != nil { writeMappedError(w, err, nil) return } inspection.FenceResolutionRequired = inspection.FenceState == operation.FenceStateActive writeJSON(w, http.StatusOK, inspection) return } if len(segments) == 4 && segments[3] == "cleanup-backup-workspace" && r.Method == http.MethodPost { if !requireSuperAdminPrivilegedOperation(w, actor) { return } if actor.OperationID == "" || actor.OperationID != operationID { writePreDispatchError(w, http.StatusBadRequest, "operation_id_mismatch", "actor operation_id must match the cleanup operation") return } request, ok := decodeRequired[hypervisor.CleanupBackupWorkspaceRequest](w, body) if !ok { return } if err := hypervisor.ValidateCleanupBackupWorkspaceRequest(request); err != nil { writePreDispatchMappedError(w, err) return } action := "" existingReplay := false trustedPrior, trustedPriorFound := trustedSupersededOperation( r.Context(), request.PriorOperationID, ) existing, existingErr := s.Journal.Get(operationID) if existingErr == nil { action = existing.Action existingReplay = strings.HasPrefix(action, "cleanup-backup-workspace:") && existing.SupersedesOperationID == request.PriorOperationID } else if !errors.Is(existingErr, operation.ErrNotFound) { writeMappedError(w, existingErr, nil) return } prior := trustedPrior if !existingReplay { var priorErr error if !trustedPriorFound { prior, priorErr = s.Journal.Get(request.PriorOperationID) } parts := strings.Split(prior.Action, ":") if priorErr != nil || len(parts) != 3 || parts[0] != "create-backup" || !catalogIDPattern.MatchString(parts[1]) || prior.State != operation.StateSucceeded || prior.ResourceID == "" || prior.ExecutionFenced { writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup requires the exact successful backup operation") return } action = "cleanup-backup-workspace:" + parts[1] if !trustedPriorFound { _, incidentFound, incidentErr := s.Journal.CompletionDeliveryIncident(prior.OperationID) if incidentErr != nil { writeMappedError(w, incidentErr, nil) return } if incidentFound { writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup is blocked by a completion-delivery incident") return } } ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, prior) cancel() if inspectErr != nil { writeMappedError(w, inspectErr, nil) return } if !backupWorkspaceCleanupInspectionMatches( prior, request.CleanupPlanID, inspection, ) { writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_plan_changed", "cleanup plan is stale or no longer safe") return } request.PriorOperation = prior } parts := strings.Split(action, ":") if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) { writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup action identity is invalid") return } resourceID := prior.ResourceID if existingReplay { resourceID = existing.ResourceID } supersedes := []string{request.PriorOperationID} if trustedPriorFound { supersedes = nil } s.runMutationExpected(w, r, body, actor, operationID, resourceID, action, nil, true, http.StatusOK, hypervisor.CleanupBackupWorkspaceAdmission(resourceID, parts[1]), func(ctx context.Context) (any, error) { return s.Hypervisor.CleanupBackupWorkspace(ctx, resourceID, request) }, supersedes...) return } if len(segments) == 4 && segments[3] == "fence-resolution" && r.Method == http.MethodGet { if !requireSuperAdminInspection(w, actor, operationID) { return } stored, err := s.Journal.Get(operationID) if err != nil { writeMappedError(w, err, nil) return } if !requireSystemRecoveryRecord(w, actor, stored) { return } resolution, err := s.Journal.GetFenceResolution(operationID) if errors.Is(err, operation.ErrNotFound) { writeError(w, http.StatusNotFound, "fence_resolution_not_found", "operation fence resolution was not found", nil) return } if err != nil { writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, fenceResolutionResponse{FenceResolution: resolution, Replayed: false}) return } if len(segments) == 4 && segments[3] == "resolve-fence" && r.Method == http.MethodPost { if !requireSuperAdminFenceResolution(w, actor, operationID) { return } if actor.OperationID == "" || actor.OperationID != operationID { writeError(w, http.StatusBadRequest, "operation_id_mismatch", "actor operation_id must match the fenced operation", nil) return } stored, err := s.Journal.Get(operationID) if err != nil { writeMappedError(w, err, nil) return } if !requireSystemRecoveryRecord(w, actor, stored) { return } request, ok := decodeRequired[operation.FenceResolutionRequest](w, body) if !ok { return } if err := operation.ValidateFenceResolutionRequest(request); err != nil { writeError(w, http.StatusBadRequest, "invalid_fence_resolution", err.Error(), nil) return } if actor.SystemRecovery && actor.UserID == 0 && operation.IsCreateLikeAction(stored.Action) { if request.Resolution != operation.FenceResolutionNotApplied { writeError(w, http.StatusConflict, "operation_fence_not_resolvable", "automatic create recovery can only confirm that creation was not applied", nil) return } if stored.ExecutionFenced { writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery cannot conclude while the original worker may still be running", nil) return } fenceState, err := s.Journal.FenceState(operationID) if err != nil { writeMappedError(w, err, nil) return } if fenceState == operation.FenceStateActive { _, incidentFound, err := s.Journal.CompletionDeliveryIncident(operationID) if err != nil { writeMappedError(w, err, nil) return } if incidentFound { writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil) return } ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, stored) cancel() if inspectErr != nil { writeMappedError(w, inspectErr, nil) return } if !systemCreateAbsenceConfirmed(stored, request, inspection) { writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil) return } } } resolution, replayed, err := s.Journal.ResolveResourceFence(operationID, actor.UserID, request) switch { case errors.Is(err, operation.ErrNotFound): writeError(w, http.StatusNotFound, "operation_not_found", "operation was not found", nil) return case errors.Is(err, operation.ErrFenceNotResolvable): writeError(w, http.StatusConflict, "operation_fence_not_resolvable", "operation does not hold an active effect fence", nil) return case errors.Is(err, operation.ErrFenceIdentityConflict): writeError(w, http.StatusConflict, "operation_identity_conflict", "fence resolution identity does not match the immutable operation", nil) return case errors.Is(err, operation.ErrFenceResolutionConflict): writeError(w, http.StatusConflict, "operation_fence_resolution_conflict", "operation fence already has a different audit resolution", nil) return case errors.Is(err, operation.ErrFenceInspectionStale): writeError(w, http.StatusConflict, "fence_inspection_stale", "operation fence resolution requires a fresh read-only inspection", nil) return case err != nil: writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, fenceResolutionResponse{FenceResolution: resolution, Replayed: replayed}) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) createLikeNetworkExpectedState( ctx context.Context, request hypervisor.CreateVMRequest, port *hypervisor.VPCPortMetadata, ) (*operation.ExpectedState, error) { var source hypervisor.LocalNetworkSource var err error if s.MutationExecutor != nil { source, err = hypervisor.ParseNetworkSelector(request.PublicNetwork.NetworkSelector) } else { source, err = s.Hypervisor.ResolveNetworkSelector(ctx, request.PublicNetwork.NetworkSelector) } if err != nil { return nil, err } expected := &operation.ExpectedState{ Kind: operation.ExpectedCreateLikeNetwork, NetworkSourceType: source.Type, NetworkSourceName: source.Source, PublicNetwork: request.PublicNetwork, VPCPresent: port != nil, } if port != nil { expected.VPCKey = port.VPCKey expected.VPCRevision = port.HostRevision expected.VPCDigest = port.HostDigest expected.VPCPortKey = port.PortKey expected.VPCCIDR = port.CIDR expected.VPCGateway = port.Gateway expected.VPCPrivateIP = port.PrivateIP expected.VPCMAC = port.MAC expected.VPCBridge = port.Bridge } return expected, nil } func (s *Server) runMutation( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, bodyOperationID, resourceID, action string, destructive bool, successStatus int, admission hypervisor.MutationAdmission, call func(context.Context) (any, error), ) { s.runMutationExpected(w, r, body, actor, bodyOperationID, resourceID, action, nil, destructive, successStatus, admission, call) } func (s *Server) runMutationExpected( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, bodyOperationID, resourceID, action string, expectedState *operation.ExpectedState, destructive bool, successStatus int, admission hypervisor.MutationAdmission, call func(context.Context) (any, error), supersedesOperationID ...string, ) { operationID, err := resolveOperationID(actor.OperationID, bodyOperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } if expectedState != nil { copyExpected := *expectedState copyExpected.TransitionID = operationID copyExpected.PublicNetwork.DNSServers = append([]string(nil), expectedState.PublicNetwork.DNSServers...) expectedState = ©Expected } supersedes := "" if len(supersedesOperationID) != 0 { supersedes = supersedesOperationID[0] } spec := operation.Spec{ OperationID: operationID, ResourceID: resourceID, Action: action, Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body), Destructive: destructive, ExpectedState: expectedState, LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO, SupersedesOperationID: supersedes, } if s.MutationExecutor != nil { s.runIsolatedMutation(w, r, body, actor, spec, s.isolatedMutationDeadline(action)) return } if _, trusted := r.Context().Value(trustedWorkerRequestKey{}).(struct{}); !trusted { writePreDispatchError(w, http.StatusServiceUnavailable, "mutation_worker_unavailable", "isolated mutation worker is unavailable") return } ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) defer cancel() record, replayed, callErr := s.Journal.Execute(ctx, spec, func(ctx context.Context) (json.RawMessage, error) { result, err := call(ctx) if err != nil { return nil, err } encoded, err := json.Marshal(result) if err != nil { return nil, fmt.Errorf("encode operation result: %w", err) } return encoded, nil }) logOperationOutcome(record) if errors.Is(callErr, operation.ErrFingerprint) { writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil) return } if errors.Is(callErr, operation.ErrOperationStarting) { writeOperationStarting(w, record) return } if operation.WasNotDispatched(record) { writeOperationFailure(w, record, replayed) return } if callErr != nil { if record.State == operation.StateFailed { writeOperationFailure(w, record, replayed) return } writeMappedError(w, callErr, &record) return } if record.State == operation.StateFailed { writeOperationFailure(w, record, replayed) return } if record.State == operation.StateDispatching || record.State == operation.StateReserved { writeJSON(w, http.StatusAccepted, operationWire(record, replayed)) return } if replayed { successStatus = http.StatusOK } writeJSON(w, successStatus, operationWire(record, replayed)) } func (s *Server) runAsyncMutation( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, bodyOperationID, resourceID, action string, destructive bool, admission hypervisor.MutationAdmission, call func(context.Context) (any, error), ) { s.runAsyncMutationWithDeadline( w, r, body, actor, bodyOperationID, resourceID, action, destructive, admission, catalogImportTimeout, call, ) } func (s *Server) runAsyncMutationWithDeadline( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, bodyOperationID, resourceID, action string, destructive bool, admission hypervisor.MutationAdmission, deadline time.Duration, call func(context.Context) (any, error), ) { s.runAsyncMutationWithDeadlineExpected( w, r, body, actor, bodyOperationID, resourceID, action, nil, destructive, admission, deadline, call, ) } func (s *Server) runAsyncMutationWithDeadlineExpected( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, bodyOperationID, resourceID, action string, expectedState *operation.ExpectedState, destructive bool, admission hypervisor.MutationAdmission, deadline time.Duration, call func(context.Context) (any, error), ) { operationID, err := resolveOperationID(actor.OperationID, bodyOperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } if expectedState != nil { copyExpected := *expectedState copyExpected.TransitionID = operationID copyExpected.PublicNetwork.DNSServers = append([]string(nil), expectedState.PublicNetwork.DNSServers...) expectedState = ©Expected } spec := operation.Spec{ OperationID: operationID, ResourceID: resourceID, Action: action, Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body), Destructive: destructive, ExpectedState: expectedState, LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO, } if s.MutationExecutor != nil { s.runIsolatedMutation(w, r, body, actor, spec, deadline) return } if _, trusted := r.Context().Value(trustedWorkerRequestKey{}).(struct{}); !trusted { writePreDispatchError(w, http.StatusServiceUnavailable, "mutation_worker_unavailable", "isolated mutation worker is unavailable") return } record, replayed, err := s.Journal.Reserve(spec) if errors.Is(err, operation.ErrFingerprint) { writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil) return } if err != nil { writeMappedError(w, err, nil) return } canStart := record.State == operation.StateReserved if canStart && !s.claimAsync(operationID) { writeOperationStarting(w, record) return } if canStart { select { case s.asyncSlots <- struct{}{}: default: failed, failErr := s.Journal.FailNotDispatched(operationID) s.releaseAsync(operationID) if failErr != nil { writeMappedError(w, failErr, &record) return } writeOperationFailure(w, failed, false) return } dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout) started, _, done, startErr := s.Journal.ExecuteBackground( dispatchCtx, s.backgroundCtx, deadline, spec, func(ctx context.Context) (json.RawMessage, error) { result, callErr := call(ctx) if callErr != nil { return nil, callErr } return json.Marshal(result) }, ) dispatchCancel() if startErr != nil || done == nil { <-s.asyncSlots s.releaseAsync(operationID) if startErr != nil { if errors.Is(startErr, operation.ErrOperationStarting) { writeOperationStarting(w, started) return } if started.State == operation.StateFailed && started.Error != nil && started.Error.Code == "operation_not_dispatched" { writeOperationFailure(w, started, false) return } writeMappedError(w, startErr, &started) return } record = started } else { record = started s.asyncWG.Add(1) go func() { defer s.asyncWG.Done() defer func() { <-s.asyncSlots }() defer s.releaseAsync(operationID) <-done if final, getErr := s.Journal.Get(operationID); getErr == nil { logOperationOutcome(final) } }() } } writeJSON(w, http.StatusAccepted, operationWire(record, replayed)) } func (s *Server) runIsolatedMutation( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, spec operation.Spec, deadline time.Duration, ) { if deadline <= 0 { deadline = s.OperationTimeout } actor.OperationID = spec.OperationID request := worker.Request{ Version: worker.ProtocolVersion, Method: r.Method, RequestTarget: auth.RequestTarget(r), Body: string(body), Actor: actor, OperationID: spec.OperationID, Fingerprint: spec.Fingerprint, } if spec.SupersedesOperationID != "" { prior, err := s.Journal.Get(spec.SupersedesOperationID) if err != nil { writeMappedError(w, err, nil) return } request.SupersededOperation = &prior if isVPCRepairTarget(request.RequestTarget) { request.SupersededExpectedState = prior.ExpectedState } } dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout) record, replayed, done, err := s.Journal.ExecuteBackground( dispatchCtx, s.backgroundCtx, deadline, spec, func(callCtx context.Context) (json.RawMessage, error) { return s.MutationExecutor.Execute(callCtx, request) }, ) dispatchCancel() if done != nil { s.asyncWG.Add(1) go func(operationID string) { defer s.asyncWG.Done() <-done if final, getErr := s.Journal.Get(operationID); getErr == nil { logOperationOutcome(final) } }(spec.OperationID) } if errors.Is(err, operation.ErrFingerprint) { writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil) return } if errors.Is(err, operation.ErrOperationStarting) { writeOperationStarting(w, record) return } if operation.WasNotDispatched(record) { writeJSON(w, http.StatusOK, operationWire(record, replayed)) return } if err != nil { writeMappedError(w, err, &record) return } status := http.StatusAccepted if record.State == operation.StateSucceeded || record.State == operation.StateFailed { status = http.StatusOK } writeJSON(w, status, operationWire(record, replayed)) } func (s *Server) isolatedMutationDeadline(action string) time.Duration { deadline := s.OperationTimeout if action == "configure-storage" && deadline > storageConfigurationWorkerTimeout { return storageConfigurationWorkerTimeout } return deadline } func (s *Server) beginRequest() bool { s.lifecycleMu.Lock() defer s.lifecycleMu.Unlock() if s.closing { return false } s.requestWG.Add(1) return true } // Shutdown stops background imports and waits for every admitted API handler // and import watcher. The journal's own workers are drained separately before // libvirt or BoltDB are closed. func (s *Server) Shutdown(ctx context.Context) error { s.lifecycleMu.Lock() if !s.closing { s.closing = true s.backgroundCancel() } s.lifecycleMu.Unlock() if err := waitGroup(ctx, &s.requestWG); err != nil { return err } return waitGroup(ctx, &s.asyncWG) } func waitGroup(ctx context.Context, group *sync.WaitGroup) error { done := make(chan struct{}) go func() { group.Wait() close(done) }() select { case <-done: return nil case <-ctx.Done(): return ctx.Err() } } func (s *Server) claimAsync(operationID string) bool { s.asyncMu.Lock() defer s.asyncMu.Unlock() if s.asyncRunning == nil { s.asyncRunning = make(map[string]struct{}) } if _, exists := s.asyncRunning[operationID]; exists { return false } s.asyncRunning[operationID] = struct{}{} return true } func applyCompletionDeliveryIncident(inspection *hypervisor.OperationInspection, incident operation.CompletionDeliveryIncident) { if inspection == nil { return } inspection.Outcome = hypervisor.InspectionOutcomeUncertain inspection.SummaryCode = hypervisor.InspectionSummaryIncomplete inspection.EffectResolution = hypervisor.InspectionEffectInconclusive inspection.EffectProjection = nil inspection.AbortJob = hypervisor.OperationAbortInspection{} inspection.Cleanup = hypervisor.OperationCleanupPlan{} inspection.CompletionDeliveryState = operation.CompletionDeliveryStateQuarantined inspection.CompletionDeliveryErrorCode = incident.ErrorCode } func applyExecutionFencedCreateInspection(inspection *hypervisor.OperationInspection, record operation.Record) { if inspection == nil || !operation.IsCreateLikeAction(record.Action) || !record.ExecutionFenced { return } inspection.Outcome = hypervisor.InspectionOutcomeUncertain inspection.SummaryCode = hypervisor.InspectionSummaryIncomplete inspection.ErrorCode = "" if record.Error != nil { inspection.ErrorCode = record.Error.Code } inspection.EffectResolution = hypervisor.InspectionEffectInconclusive inspection.EffectProjection = nil inspection.AbortJob = hypervisor.OperationAbortInspection{} inspection.Cleanup = hypervisor.OperationCleanupPlan{} } // systemCreateAbsenceConfirmed accepts only a fresh, exact read-only host // observation that both the VM and all panel-managed creation artifacts are // absent. It does not infer absence from the original operation error and does // not authorize any create replay or host mutation. func systemCreateAbsenceConfirmed(record operation.Record, request operation.FenceResolutionRequest, inspection hypervisor.OperationInspection) bool { if !operation.IsCreateLikeAction(record.Action) || record.State != operation.StateFailed || inspection.OperationID != record.OperationID || inspection.Action != record.Action || inspection.ResourceID != record.ResourceID || inspection.RequestFingerprint != record.Fingerprint || inspection.OperationStatus != record.State || inspection.CheckedAt < request.InspectionCheckedAt || inspection.Outcome != hypervisor.InspectionOutcomeFailed || inspection.SummaryCode != hypervisor.InspectionSummaryFailed || inspection.ErrorCode != "creation_absent" || inspection.EffectResolution != hypervisor.InspectionEffectNotApplied || inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" || inspection.AbortJob.Available || inspection.Cleanup.Available || len(inspection.EffectProjection) == 0 || len(inspection.EffectProjection) > 1024 { return false } var projection map[string]json.RawMessage if err := json.Unmarshal(inspection.EffectProjection, &projection); err != nil || len(projection) != 2 { return false } var creationAbsent, managedArtifactsAbsent bool if err := json.Unmarshal(projection["creation_absent"], &creationAbsent); err != nil || !creationAbsent { return false } if err := json.Unmarshal(projection["managed_artifacts_absent"], &managedArtifactsAbsent); err != nil || !managedArtifactsAbsent { return false } return true } func (s *Server) releaseAsync(operationID string) { s.asyncMu.Lock() defer s.asyncMu.Unlock() delete(s.asyncRunning, operationID) } func operationWire(record operation.Record, replayed bool) operationResponse { return operationResponse{ OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID, RequestFingerprint: record.Fingerprint, Status: record.State, Replayed: replayed, Result: record.Result, Error: record.Error, CauseCode: record.CauseCode, EffectMayHaveApplied: record.EffectMayHaveApplied, SafeToResubmit: record.SafeToResubmit, Retryable: false, } } // Log only bounded identifiers and stable codes. Free-form backend errors, // request bodies, guest credentials, and host paths never enter this line. func logOperationOutcome(record operation.Record) { if record.State != operation.StateFailed { return } errorCode := "" if record.Error != nil && len(record.Error.Code) <= 64 && stableCodePattern.MatchString(record.Error.Code) { errorCode = record.Error.Code } log.Printf("Agent operation outcome operation_id=%s action=%s status=%s error_code=%s cause_code=%s", record.OperationID, record.Action, record.State, errorCode, record.CauseCode) } func resolveOperationID(actorID, bodyID string) (string, error) { if actorID == "" && bodyID == "" { return "", errors.New("operation_id is required in X-Virt-Actor or request body") } if actorID != "" && bodyID != "" && actorID != bodyID { return "", errors.New("actor and body operation_id values differ") } resolved := bodyID if actorID != "" { resolved = actorID } if !operationIDPattern.MatchString(resolved) { return "", errors.New("operation_id must contain 8-80 URL-safe characters") } return resolved, nil } func operationUUID(operationID string) string { digest := sha256.Sum256([]byte("virt-cloud-operation:" + operationID)) digest[6] = digest[6]&0x0f | 0x50 digest[8] = digest[8]&0x3f | 0x80 return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", digest[0:4], digest[4:6], digest[6:8], digest[8:10], digest[10:16]) } func operationCatalogID(operationID string) string { digest := sha256.Sum256([]byte("virt-cloud-catalog:" + operationID)) return fmt.Sprintf("%x", digest[:16]) } func validPowerAction(action hypervisor.PowerAction) bool { switch action { case hypervisor.PowerStart, hypervisor.PowerShutdown, hypervisor.PowerReboot, hypervisor.PowerReset, hypervisor.PowerDestroy: return true default: return false } } func validateImportSource(raw, token, digest string) error { parsed, err := url.Parse(raw) if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" { return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_url", "source_url must be an HTTPS URL without credentials or fragment", err) } if digest != "" && !sha256Pattern.MatchString(digest) { return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_sha256", "sha256 must be 64 lowercase hexadecimal characters", nil) } if token != "" && !sourceTokenPattern.MatchString(token) { return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_token", "source_token must contain 32-512 URL-safe characters", nil) } return nil } func ownerFromRequest(r *http.Request, actor auth.Actor) (int64, error) { value := r.URL.Query().Get("owner_id") if value == "" { return actor.UserID, nil } ownerID, err := strconv.ParseInt(value, 10, 64) if err != nil || ownerID <= 0 { return 0, errors.New("owner_id is invalid") } if ownerID != actor.UserID && !actor.IsAdmin && !actor.IsSuperAdmin { return 0, errors.New("cannot access another owner's ISO") } return ownerID, nil } func requireManage(w http.ResponseWriter, actor auth.Actor) bool { if !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to manage this resource") return false } return true } func requireSuperAdminManage(w http.ResponseWriter, actor auth.Actor) bool { if !actor.IsSuperAdmin || !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to configure host storage") return false } return true } func requireSuperAdminPrivilegedOperation(w http.ResponseWriter, actor auth.Actor) bool { if !actor.IsSuperAdmin || !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to perform this privileged operation") return false } return true } func requireSuperAdminInspection(w http.ResponseWriter, actor auth.Actor, operationID string) bool { if actor.SystemRecovery && actor.UserID == 0 && actor.OperationID == operationID { return true } if actor.UserID < 1 || !actor.IsSuperAdmin || !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to inspect operation state") return false } return true } func requireSuperAdminFenceResolution(w http.ResponseWriter, actor auth.Actor, operationID string) bool { if actor.SystemRecovery && actor.UserID == 0 && actor.OperationID == operationID { return true } if actor.UserID < 1 || !actor.IsSuperAdmin || !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to resolve operation effect fences") return false } return true } // Route shape and operation identity are necessary but not sufficient for // system recovery: authorization is completed only after reading the // immutable Journal action and applying the explicit Agent action allowlist. func requireSystemRecoveryRecord(w http.ResponseWriter, actor auth.Actor, record operation.Record) bool { if actor.SystemRecovery && !operation.SystemRecoveryActionAllowed(record.Action) { writePreDispatchError(w, http.StatusForbidden, "forbidden", "system recovery is not enabled for this operation kind") return false } return true } // systemRecoveryRequestAllowed is a deny-by-default capability boundary. The // signed system actor is bound to one operation and cannot reach health, // inventory, raw Journal records, catalog reads, or any mutation route. func systemRecoveryRequestAllowed(actor auth.Actor, method string, segments []string) bool { if !actor.SystemRecovery || actor.UserID != 0 || actor.OperationID == "" || len(segments) != 4 || segments[0] != "v1" || segments[1] != "operations" || segments[2] != actor.OperationID { return false } switch segments[3] { case "inspect", "fence-resolution": return method == http.MethodGet case "resolve-fence": return method == http.MethodPost default: return false } } func decodeRequired[T any](w http.ResponseWriter, body []byte) (T, bool) { var value T if len(body) == 0 { writePreDispatchError(w, http.StatusBadRequest, "invalid_json", "JSON request body is required") return value, false } if err := decodeStrict(body, &value); err != nil { writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error()) return value, false } return value, true } func decodeOptional[T any](w http.ResponseWriter, body []byte) (T, bool) { var value T if len(body) == 0 { return value, true } if err := decodeStrict(body, &value); err != nil { writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error()) return value, false } return value, true } func decodeStrict(body []byte, target any) error { if !utf8.Valid(body) { return errors.New("invalid JSON: body must be valid UTF-8") } decoder := json.NewDecoder(bytes.NewReader(body)) decoder.DisallowUnknownFields() if err := decoder.Decode(target); err != nil { return fmt.Errorf("invalid JSON: %w", err) } var extra any if err := decoder.Decode(&extra); err != io.EOF { return errors.New("invalid JSON: exactly one object is required") } return nil } func readBoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, error) { if r.Body == nil { return []byte{}, nil } r.Body = http.MaxBytesReader(w, r.Body, limit) defer r.Body.Close() return io.ReadAll(r.Body) } func writeResult(w http.ResponseWriter, value any, err error) { if err != nil { writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, value) } func writeNamedList[T any](w http.ResponseWriter, name string, value []T, err error) { if err != nil { writeMappedError(w, err, nil) return } if value == nil { value = make([]T, 0) } writeJSON(w, http.StatusOK, map[string]any{name: value}) } func writeMappedError(w http.ResponseWriter, err error, record *operation.Record) { status, code, message := classifyError(err) writeError(w, status, code, message, record) } // writePreDispatchMappedError is only for validation or dependency checks // completed before Journal.Execute/Reserve. The explicit marker lets the // panel distinguish a proven no-journal outcome from a lost response after a // mutation may already have crossed its durable dispatch boundary. func writePreDispatchMappedError(w http.ResponseWriter, err error) { status, code, message := classifyError(err) writePreDispatchError(w, status, code, message) } func publicErrorCode(err error, fallback string) string { var public interface{ PublicCode() string } if errors.As(err, &public) && stableCodePattern.MatchString(public.PublicCode()) { return public.PublicCode() } return fallback } func publicErrorMessage(err error, fallback string) string { var public interface{ PublicMessage() string } if errors.As(err, &public) && public.PublicMessage() != "" { return public.PublicMessage() } return fallback } // writeOperationStarting reports the active admission owner without exposing a // stale failed/not-dispatched record from before that owner acquired admission. // The projected reserved state is diagnostic only; it never authorizes a // caller to dispatch the mutation again. The journal remains authoritative. func writeOperationStarting(w http.ResponseWriter, record operation.Record) { record.State = operation.StateReserved record.Result = nil record.Error = nil record.CauseCode = "" record.Tombstone = false writeError(w, http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting", &record) } func classifyError(err error) (int, string, string) { if errors.Is(err, operation.ErrNotFound) { return http.StatusNotFound, "operation_not_found", "operation was not found" } if errors.Is(err, operation.ErrBusy) { return http.StatusServiceUnavailable, "agent_busy", "Agent mutation capacity is exhausted" } if errors.Is(err, operation.ErrOperationStarting) { return http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting" } if errors.Is(err, operation.ErrResourceBlocked) { return http.StatusConflict, "resource_operation_blocked", "resource is fenced by a prior failed operation whose effect may have applied" } if errors.Is(err, operation.ErrInvalidSupersession) { return http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful operation" } var backend *hypervisor.Error if errors.As(err, &backend) { status := statusForKind(backend.Kind) code := backend.Code if code == "" { code = "backend_error" } message := backend.Message if message == "" { message = http.StatusText(status) } return status, code, message } switch { case errors.Is(err, hypervisor.ErrNotFound): return http.StatusNotFound, "not_found", "resource was not found" case errors.Is(err, hypervisor.ErrConflict): return http.StatusConflict, "conflict", "resource conflict" case errors.Is(err, hypervisor.ErrUnsupported): return http.StatusNotImplemented, "unsupported", "operation is not supported" case errors.Is(err, hypervisor.ErrInvalid): return http.StatusUnprocessableEntity, "invalid_request", "request is invalid" case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): return http.StatusServiceUnavailable, "backend_timeout", "backend operation did not complete" default: return http.StatusServiceUnavailable, "backend_error", "backend operation failed" } } func statusForKind(kind error) int { switch kind { case hypervisor.ErrNotFound: return http.StatusNotFound case hypervisor.ErrConflict: return http.StatusConflict case hypervisor.ErrUnsupported: return http.StatusNotImplemented case hypervisor.ErrInvalid: return http.StatusUnprocessableEntity default: return http.StatusServiceUnavailable } } func writeOperationFailure(w http.ResponseWriter, record operation.Record, replayed bool) { status := http.StatusServiceUnavailable if record.Error != nil { switch record.Error.Code { case "unsupported": status = http.StatusNotImplemented case "not_found": status = http.StatusNotFound case "conflict": status = http.StatusConflict case "force_required", "force_delete_not_authorized": status = http.StatusConflict default: if strings.HasSuffix(record.Error.Code, "_not_found") { status = http.StatusNotFound } else if strings.HasPrefix(record.Error.Code, "invalid_") { status = http.StatusUnprocessableEntity } } } writeJSON(w, status, operationWire(record, replayed)) } func methodOrNotFound(w http.ResponseWriter, r *http.Request, exists bool, methods ...string) { if !exists { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return } w.Header().Set("Allow", strings.Join(methods, ", ")) writePreDispatchError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method is not allowed") } func writePreDispatchError(w http.ResponseWriter, status int, code, message string) { writeJSON(w, status, errorEnvelope{Error: errorBody{ Code: code, Message: message, Retryable: retryableError(status, code), DispatchState: "not_dispatched", }}) } func writeError(w http.ResponseWriter, status int, code, message string, record *operation.Record) { writeJSON(w, status, errorEnvelope{Error: errorBody{ Code: code, Message: message, Retryable: retryableError(status, code), Operation: record, }}) } func retryableError(status int, code string) bool { return false } func writeJSON(w http.ResponseWriter, status int, value any) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(status) _ = json.NewEncoder(w).Encode(value) } func setResponseHeaders(w http.ResponseWriter) { w.Header().Set("Cache-Control", "no-store") w.Header().Set("X-Content-Type-Options", "nosniff") }