You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2884 lines
103 KiB
Go
2884 lines
103 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
consolebroker "git.luoli.live/root/virt/agent/console"
|
|
"git.luoli.live/root/virt/agent/hypervisor"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"git.luoli.live/root/virt/agent/vpcrouting"
|
|
"git.luoli.live/root/virt/agent/worker"
|
|
)
|
|
|
|
const Version = "0.1.0"
|
|
|
|
var (
|
|
apiUUIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$`)
|
|
resourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`)
|
|
operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`)
|
|
stableCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
|
|
sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
|
catalogIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
|
sourceTokenPattern = regexp.MustCompile(`^[A-Za-z0-9._~-]{32,512}$`)
|
|
)
|
|
|
|
const (
|
|
catalogImportTimeout = 2 * time.Hour
|
|
backupOperationTimeout = 24 * time.Hour
|
|
storageConfigurationWorkerTimeout = 120 * time.Second
|
|
defaultAdmissionTimeout = 2 * time.Second
|
|
maxAsyncImports = 2
|
|
)
|
|
|
|
type Server struct {
|
|
Authenticator *auth.Authenticator
|
|
Journal *operation.Journal
|
|
Hypervisor hypervisor.Hypervisor
|
|
Console *consolebroker.Broker
|
|
MaxBodyBytes int64
|
|
OperationTimeout time.Duration
|
|
AdmissionTimeout time.Duration
|
|
MutationExecutor worker.Executor
|
|
asyncMu sync.Mutex
|
|
asyncRunning map[string]struct{}
|
|
asyncSlots chan struct{}
|
|
asyncWG sync.WaitGroup
|
|
lifecycleMu sync.Mutex
|
|
closing bool
|
|
requestWG sync.WaitGroup
|
|
backgroundCtx context.Context
|
|
backgroundCancel context.CancelFunc
|
|
networkObserverMu sync.RWMutex
|
|
networkObserver NetworkObservationReader
|
|
}
|
|
|
|
type trustedWorkerRequestKey struct{}
|
|
type trustedSupersededOperationKey struct{}
|
|
|
|
// SetMutationExecutor enables the clean-break process-isolated mutation path.
|
|
// It must be called during startup before request admission begins.
|
|
func (s *Server) SetMutationExecutor(executor worker.Executor) error {
|
|
if executor == nil {
|
|
return errors.New("mutation executor is required")
|
|
}
|
|
s.MutationExecutor = executor
|
|
return nil
|
|
}
|
|
|
|
// ServeTrustedWorkerRequest dispatches a request whose bounded envelope was
|
|
// already authenticated and validated by the parent Agent. Hidden workers do
|
|
// not have access to the panel secret through HTTP headers and cannot recurse
|
|
// because they never install a MutationExecutor.
|
|
func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor,
|
|
superseded ...operation.Record) {
|
|
setResponseHeaders(w)
|
|
if !s.beginRequest() {
|
|
writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down")
|
|
return
|
|
}
|
|
defer s.requestWG.Done()
|
|
if int64(len(body)) > s.MaxBodyBytes || actor.OperationID == "" {
|
|
writePreDispatchError(w, http.StatusBadRequest, "invalid_worker_request", "worker request is invalid")
|
|
return
|
|
}
|
|
trusted := context.WithValue(r.Context(), trustedWorkerRequestKey{}, struct{}{})
|
|
if len(superseded) > 0 {
|
|
prior := superseded[0]
|
|
trusted = context.WithValue(trusted, trustedSupersededOperationKey{}, prior)
|
|
}
|
|
s.route(w, r.WithContext(trusted), body, actor)
|
|
}
|
|
|
|
func trustedSupersededOperation(ctx context.Context, operationID string) (operation.Record, bool) {
|
|
prior, ok := ctx.Value(trustedSupersededOperationKey{}).(operation.Record)
|
|
return prior, ok && prior.OperationID == operationID
|
|
}
|
|
|
|
func isVPCRepairTarget(target string) bool {
|
|
return strings.HasSuffix(target, "/repair") &&
|
|
(strings.Contains(target, "/v1/vpc/networks/") ||
|
|
strings.Contains(target, "/vpc-port/"))
|
|
}
|
|
|
|
type errorBody struct {
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
Retryable bool `json:"retryable"`
|
|
DispatchState string `json:"dispatch_state,omitempty"`
|
|
Operation *operation.Record `json:"operation,omitempty"`
|
|
}
|
|
|
|
type errorEnvelope struct {
|
|
Error errorBody `json:"error"`
|
|
}
|
|
|
|
type operationResponse struct {
|
|
OperationID string `json:"operation_id"`
|
|
Action string `json:"action"`
|
|
ResourceID string `json:"resource_id"`
|
|
RequestFingerprint string `json:"request_fingerprint"`
|
|
Status operation.State `json:"status"`
|
|
Replayed bool `json:"replayed"`
|
|
Result json.RawMessage `json:"result,omitempty"`
|
|
Error *operation.PublicError `json:"error,omitempty"`
|
|
CauseCode string `json:"cause_code,omitempty"`
|
|
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
|
|
SafeToResubmit bool `json:"safe_to_resubmit"`
|
|
Retryable bool `json:"retryable"`
|
|
}
|
|
|
|
type fenceResolutionResponse struct {
|
|
operation.FenceResolution
|
|
Replayed bool `json:"replayed"`
|
|
}
|
|
|
|
func NewServer(
|
|
authenticator *auth.Authenticator,
|
|
journal *operation.Journal,
|
|
hv hypervisor.Hypervisor,
|
|
console *consolebroker.Broker,
|
|
maxBodyBytes int64,
|
|
operationTimeout time.Duration,
|
|
) (*Server, error) {
|
|
if authenticator == nil || journal == nil || hv == nil || console == nil {
|
|
return nil, errors.New("authenticator, journal, hypervisor, and console broker are required")
|
|
}
|
|
if maxBodyBytes < 1024 {
|
|
return nil, errors.New("max body size must be at least 1024 bytes")
|
|
}
|
|
if operationTimeout <= 0 {
|
|
return nil, errors.New("operation timeout must be positive")
|
|
}
|
|
backgroundCtx, backgroundCancel := context.WithCancel(context.Background())
|
|
return &Server{
|
|
Authenticator: authenticator, Journal: journal, Hypervisor: hv, Console: console,
|
|
MaxBodyBytes: maxBodyBytes, OperationTimeout: operationTimeout,
|
|
AdmissionTimeout: min(operationTimeout, defaultAdmissionTimeout),
|
|
asyncRunning: make(map[string]struct{}), asyncSlots: make(chan struct{}, maxAsyncImports),
|
|
backgroundCtx: backgroundCtx, backgroundCancel: backgroundCancel,
|
|
}, nil
|
|
}
|
|
|
|
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
setResponseHeaders(w)
|
|
if !s.beginRequest() {
|
|
writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down")
|
|
return
|
|
}
|
|
defer s.requestWG.Done()
|
|
if !strings.HasPrefix(r.URL.Path, "/v1/") || strings.HasSuffix(r.URL.Path, "/") {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
return
|
|
}
|
|
body, err := readBoundedBody(w, r, s.MaxBodyBytes)
|
|
if err != nil {
|
|
var tooLarge *http.MaxBytesError
|
|
if errors.As(err, &tooLarge) {
|
|
writePreDispatchError(w, http.StatusRequestEntityTooLarge, "body_too_large", "request body exceeds configured limit")
|
|
} else {
|
|
writePreDispatchError(w, http.StatusBadRequest, "invalid_body", "could not read request body")
|
|
}
|
|
return
|
|
}
|
|
actor, err := s.Authenticator.Authenticate(r, body)
|
|
if err != nil {
|
|
if errors.Is(err, auth.ErrNonceStore) {
|
|
writePreDispatchError(w, http.StatusServiceUnavailable, "nonce_store_unavailable", "authentication state is unavailable")
|
|
return
|
|
}
|
|
if errors.Is(err, auth.ErrReplay) {
|
|
// The matching nonce may belong to an earlier request that already
|
|
// crossed the durable operation boundary. A replay rejection is not
|
|
// proof that the operation was never dispatched.
|
|
writeError(w, http.StatusUnauthorized, "replayed_request", "request nonce has already been used", nil)
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusUnauthorized, "unauthorized", "request authentication failed")
|
|
return
|
|
}
|
|
s.route(w, r, body, actor)
|
|
}
|
|
|
|
func (s *Server) route(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor) {
|
|
segments := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/")
|
|
if len(segments) < 2 || segments[0] != "v1" {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
return
|
|
}
|
|
if actor.SystemRecovery && !systemRecoveryRequestAllowed(actor, r.Method, segments) {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "system recovery actor is not authorized for this endpoint")
|
|
return
|
|
}
|
|
switch segments[1] {
|
|
case "health":
|
|
if len(segments) != 2 || r.Method != http.MethodGet {
|
|
methodOrNotFound(w, r, len(segments) == 2, http.MethodGet)
|
|
return
|
|
}
|
|
if err := s.Hypervisor.Ping(r.Context()); err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"status": "ok", "service": "virt-host-agent", "version": Version})
|
|
case "capabilities":
|
|
if len(segments) != 2 || r.Method != http.MethodGet {
|
|
methodOrNotFound(w, r, len(segments) == 2, http.MethodGet)
|
|
return
|
|
}
|
|
result, err := s.Hypervisor.Capabilities(r.Context())
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, s.capabilitiesResponse(result))
|
|
case "network-observations":
|
|
s.routeNetworkObservations(w, r, segments, actor)
|
|
case "storage":
|
|
s.routeStorage(w, r, segments, body, actor)
|
|
case "vpc":
|
|
s.routeVPC(w, r, segments, body, actor)
|
|
case "vms":
|
|
s.routeVMs(w, r, segments, body, actor)
|
|
case "templates":
|
|
s.routeTemplates(w, r, segments, body, actor)
|
|
case "rescue-templates":
|
|
s.routeRescueTemplates(w, r, segments, body, actor)
|
|
case "user-isos":
|
|
s.routeUserISOs(w, r, segments, body, actor)
|
|
case "operations":
|
|
s.routeOperations(w, r, segments, body, actor)
|
|
default:
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
}
|
|
|
|
func (s *Server) routeVPC(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
repairPath := len(segments) == 6 && segments[5] == "repair"
|
|
if (len(segments) != 5 && !repairPath) || segments[2] != "networks" || !catalogIDPattern.MatchString(segments[3]) ||
|
|
(segments[4] != "apply" && segments[4] != "remove") {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
return
|
|
}
|
|
if !requireSuperAdminManage(w, actor) {
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
vpcKey := segments[3]
|
|
switch segments[4] {
|
|
case "apply":
|
|
request, ok := decodeRequired[vpcrouting.ApplyRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.Network.VPCKey != vpcKey {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "vpc_key_mismatch", "body vpc_key must match the request target")
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = operationID
|
|
normalized, err := vpcrouting.ValidateApplyRequest(request)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, hypervisor.NewError(hypervisor.ErrInvalid, publicErrorCode(err, "invalid_vpc_projection"), publicErrorMessage(err, "VPC projection is invalid"), err))
|
|
return
|
|
}
|
|
request = normalized
|
|
if request.VPCRepair != repairPath ||
|
|
(request.VPCRepair != (request.SupersedesOperationID != "")) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation")
|
|
return
|
|
}
|
|
action := "apply-vpc-network:" + vpcKey
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedVPCNetwork, VPCKey: vpcKey,
|
|
VPCRevision: request.Network.DesiredRevision,
|
|
VPCDigest: request.DesiredDigest, VPCPresent: true,
|
|
}
|
|
admission := hypervisor.VPCNetworkAdmission(vpcKey)
|
|
supersedes, ok := s.authorizeVPCRepair(
|
|
w, r, operationID, request.SupersedesOperationID,
|
|
"vpc:"+vpcKey, action, expected, admission, repairPath,
|
|
)
|
|
if !ok {
|
|
return
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, operationID, "vpc:"+vpcKey, action, expected, true, http.StatusOK,
|
|
admission,
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyVPCNetwork(ctx, request) }, supersedes...)
|
|
case "remove":
|
|
request, ok := decodeRequired[vpcrouting.RemoveRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.VPCKey != vpcKey {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "vpc_key_mismatch", "body vpc_key must match the request target")
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = operationID
|
|
if err := vpcrouting.ValidateRemoveRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, hypervisor.NewError(hypervisor.ErrInvalid, publicErrorCode(err, "invalid_vpc_remove"), publicErrorMessage(err, "VPC removal request is invalid"), err))
|
|
return
|
|
}
|
|
if request.VPCRepair != repairPath ||
|
|
(request.VPCRepair != (request.SupersedesOperationID != "")) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation")
|
|
return
|
|
}
|
|
action := "remove-vpc-network:" + vpcKey
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedVPCNetwork, VPCKey: vpcKey,
|
|
VPCRevision: request.ExpectedRevision,
|
|
VPCDigest: request.ExpectedDigest, VPCPresent: false,
|
|
}
|
|
admission := hypervisor.VPCNetworkAdmission(vpcKey)
|
|
supersedes, ok := s.authorizeVPCRepair(
|
|
w, r, operationID, request.SupersedesOperationID,
|
|
"vpc:"+vpcKey, action, expected, admission, repairPath,
|
|
)
|
|
if !ok {
|
|
return
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, operationID, "vpc:"+vpcKey, action, expected, true, http.StatusOK,
|
|
admission,
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.RemoveVPCNetwork(ctx, request) }, supersedes...)
|
|
}
|
|
}
|
|
|
|
func (s *Server) authorizeVPCRepair(w http.ResponseWriter, r *http.Request,
|
|
operationID, predecessorID, resourceID, action string,
|
|
expected *operation.ExpectedState, admission hypervisor.MutationAdmission,
|
|
repair bool) ([]string, bool) {
|
|
if !repair {
|
|
return nil, true
|
|
}
|
|
if !operationIDPattern.MatchString(predecessorID) ||
|
|
operationID == predecessorID {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair_predecessor", "VPC repair requires a different valid predecessor operation")
|
|
return nil, false
|
|
}
|
|
trustedPrior, trusted := trustedSupersededOperation(
|
|
r.Context(), predecessorID,
|
|
)
|
|
existing, existingErr := s.Journal.Get(operationID)
|
|
if existingErr == nil {
|
|
if existing.Action != action || existing.ResourceID != resourceID ||
|
|
existing.SupersedesOperationID != predecessorID {
|
|
writePreDispatchError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request")
|
|
return nil, false
|
|
}
|
|
} else if !errors.Is(existingErr, operation.ErrNotFound) {
|
|
writeMappedError(w, existingErr, nil)
|
|
return nil, false
|
|
} else {
|
|
prior := trustedPrior
|
|
var err error
|
|
if !trusted {
|
|
prior, err = s.Journal.Get(predecessorID)
|
|
}
|
|
spec := operation.Spec{
|
|
OperationID: operationID, ResourceID: resourceID,
|
|
Action: action, ExpectedState: expected,
|
|
LockKeys: append([]string(nil), admission.LockKeys...),
|
|
SupersedesOperationID: predecessorID,
|
|
}
|
|
if err != nil || !operation.ValidVPCRepairSupersession(spec, prior) {
|
|
writePreDispatchError(w, http.StatusConflict, "vpc_repair_not_authorized", "VPC repair no longer matches one exact fenced predecessor")
|
|
return nil, false
|
|
}
|
|
if !trusted {
|
|
_, incident, incidentErr := s.Journal.CompletionDeliveryIncident(
|
|
predecessorID,
|
|
)
|
|
if incidentErr != nil {
|
|
writeMappedError(w, incidentErr, nil)
|
|
return nil, false
|
|
}
|
|
if incident {
|
|
writePreDispatchError(w, http.StatusConflict, "vpc_repair_not_authorized", "VPC repair is blocked by a completion-delivery incident")
|
|
return nil, false
|
|
}
|
|
}
|
|
}
|
|
if trusted {
|
|
return nil, true
|
|
}
|
|
return []string{predecessorID}, true
|
|
}
|
|
|
|
func (s *Server) routeStorage(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if len(segments) != 3 || segments[2] != "configuration" {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
return
|
|
}
|
|
if !requireSuperAdminManage(w, actor) {
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.StorageConfigurationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateStorageConfigurationRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
digests := hypervisor.StorageConfigurationPathDigests(request)
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedStorageConfiguration,
|
|
ManagedPathDigest: digests[hypervisor.StorageRoleManaged],
|
|
TemplatePathDigest: digests[hypervisor.StorageRoleTemplate],
|
|
SeedPathDigest: digests[hypervisor.StorageRoleSeed],
|
|
UserISOPathDigest: digests[hypervisor.StorageRoleUserISO],
|
|
RescuePathDigest: digests[hypervisor.StorageRoleRescue],
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, "host-storage", "configure-storage", expected, true, http.StatusOK,
|
|
hypervisor.StorageConfigurationAdmission(),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ConfigureStorage(ctx, request) })
|
|
}
|
|
|
|
func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if len(segments) == 2 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListVMs(r.Context())
|
|
writeNamedList(w, "vms", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.CreateVMRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
normalized, err := hypervisor.NormalizeStartupScript(request.StartupScript)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
request.StartupScript = normalized
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
if request.UUID == "" {
|
|
request.UUID = operationUUID(resolvedOperationID)
|
|
}
|
|
request.DomainName, err = hypervisor.LibvirtDomainName(actor.UserID, request.UUID)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateCreateVM(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, request.UUID, "create-vm", true, http.StatusCreated,
|
|
hypervisor.CreateVMAdmission(request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.CreateVM(ctx, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) < 3 || !apiUUIDPattern.MatchString(segments[2]) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_uuid", "VM UUID must be canonical")
|
|
return
|
|
}
|
|
uuid := strings.ToLower(segments[2])
|
|
if len(segments) == 3 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
case http.MethodDelete:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.DeleteVMRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !request.RemoveManagedDisks {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true")
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-vm", true, http.StatusOK,
|
|
hypervisor.DeleteVMAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.DeleteVM(ctx, uuid, request)
|
|
})
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete)
|
|
}
|
|
return
|
|
}
|
|
|
|
switch segments[3] {
|
|
case "backup-readiness":
|
|
if len(segments) == 4 && r.Method == http.MethodGet {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
result, err := s.Hypervisor.GetBackupReadiness(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
return
|
|
}
|
|
case "backups":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.BackupRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
if err := hypervisor.ValidateBackupRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
action := "create-backup:" + request.BackupKey + ":" + request.ConsistencyMode
|
|
s.runAsyncMutationWithDeadline(
|
|
w, r, body, actor, request.OperationID, uuid, action, false,
|
|
hypervisor.CreateBackupAdmission(uuid, request), backupOperationTimeout,
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.CreateBackup(ctx, uuid, request) },
|
|
)
|
|
return
|
|
}
|
|
case "restore-backup":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.RestoreBackupRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.UUID != "" && !strings.EqualFold(request.UUID, uuid) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "uuid_mismatch", "body UUID must match the restore target")
|
|
return
|
|
}
|
|
request.UUID = uuid
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
derivedDomainName, err := hypervisor.LibvirtDomainName(actor.UserID, uuid)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
if request.RequestedDomainName != "" && request.RequestedDomainName != derivedDomainName {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "domain_name_mismatch", "domain_name must match the Agent-derived ownership name")
|
|
return
|
|
}
|
|
request.DomainName = derivedDomainName
|
|
if err := hypervisor.ValidateRestoreBackupRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
expected, err := s.createLikeNetworkExpectedState(r.Context(), request.CreateVMRequest, nil)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
action := "restore-backup:" + request.BackupKey
|
|
s.runAsyncMutationWithDeadlineExpected(
|
|
w, r, body, actor, request.OperationID, uuid, action, expected, true,
|
|
hypervisor.RestoreBackupAdmission(request), backupOperationTimeout,
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.RestoreBackup(ctx, request) },
|
|
)
|
|
return
|
|
}
|
|
case "recover-backup":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.RecoverBackupRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
if err := hypervisor.ValidateRecoverBackupRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
action := "recover-backup:" + request.BackupKey + ":" + request.SourceSHA256
|
|
s.runAsyncMutationWithDeadline(
|
|
w, r, body, actor, request.OperationID, uuid, action, true,
|
|
hypervisor.RecoverBackupAdmission(uuid, request), backupOperationTimeout,
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.RecoverBackup(ctx, uuid, request)
|
|
},
|
|
)
|
|
return
|
|
}
|
|
case "peer-migration":
|
|
if len(segments) != 5 || !requireManage(w, actor) {
|
|
if len(segments) != 5 {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
return
|
|
}
|
|
switch segments[4] {
|
|
case "prepare":
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.PreparePeerMigrationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = operationID
|
|
if err := hypervisor.ValidatePreparePeerMigrationRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runAsyncMutationWithDeadline(w, r, body, actor, operationID, uuid,
|
|
"prepare-peer-migration:"+request.MigrationID, true,
|
|
hypervisor.PreparePeerMigrationAdmission(uuid, request), backupOperationTimeout,
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.PreparePeerMigration(ctx, uuid, request)
|
|
})
|
|
case "receive":
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ReceivePeerMigrationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.UUID != "" && !strings.EqualFold(request.UUID, uuid) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "uuid_mismatch", "body UUID must match the migration target")
|
|
return
|
|
}
|
|
request.UUID = uuid
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = operationID
|
|
request.DomainName, err = hypervisor.LibvirtDomainName(actor.UserID, uuid)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateReceivePeerMigrationRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
expected, err := s.createLikeNetworkExpectedState(r.Context(), request.CreateVMRequest, request.VPCPort)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runAsyncMutationWithDeadlineExpected(w, r, body, actor, operationID, uuid,
|
|
"receive-peer-migration:"+request.MigrationID, expected, true,
|
|
hypervisor.ReceivePeerMigrationAdmission(request), backupOperationTimeout,
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.ReceivePeerMigration(ctx, request)
|
|
})
|
|
case "rollback":
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.RollbackPeerMigrationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = operationID
|
|
if err := hypervisor.ValidateRollbackPeerMigrationRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedPeerMigrationState, Enabled: request.SourceAutostartEnabled,
|
|
StartIfSuspended: request.SourceWasRunning,
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, operationID, uuid,
|
|
"rollback-peer-migration:"+request.MigrationID,
|
|
expected, true, http.StatusOK,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.RollbackPeerMigration(ctx, uuid, request)
|
|
})
|
|
case "activate":
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ActivatePeerMigrationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = operationID
|
|
if err := hypervisor.ValidateActivatePeerMigrationRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedPeerMigrationState,
|
|
Enabled: request.TargetAutostartEnabled,
|
|
StartIfSuspended: request.TargetWasRunning,
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, operationID, uuid,
|
|
"activate-peer-migration:"+request.MigrationID,
|
|
expected, true, http.StatusOK,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.ActivatePeerMigration(ctx, uuid, request)
|
|
})
|
|
case "cleanup":
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.CleanupPeerMigrationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = operationID
|
|
if err := hypervisor.ValidateCleanupPeerMigrationRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, operationID, uuid,
|
|
"delete-vm", true, http.StatusOK,
|
|
hypervisor.DeleteVMAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.CleanupPeerMigration(ctx, uuid, request)
|
|
})
|
|
case "cleanup-target":
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.CleanupPeerMigrationTargetRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = operationID
|
|
if err := hypervisor.ValidateCleanupPeerMigrationTargetRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
action := "cleanup-peer-migration-target:" + request.MigrationID
|
|
existingReplay := false
|
|
trustedPrior, trustedPriorFound := trustedSupersededOperation(
|
|
r.Context(), request.PriorOperationID,
|
|
)
|
|
existing, existingErr := s.Journal.Get(operationID)
|
|
if existingErr == nil {
|
|
existingReplay = existing.Action == action && existing.ResourceID == uuid &&
|
|
existing.SupersedesOperationID == request.PriorOperationID
|
|
} else if !errors.Is(existingErr, operation.ErrNotFound) {
|
|
writeMappedError(w, existingErr, nil)
|
|
return
|
|
}
|
|
if !existingReplay {
|
|
prior := trustedPrior
|
|
var priorErr error
|
|
if !trustedPriorFound {
|
|
prior, priorErr = s.Journal.Get(request.PriorOperationID)
|
|
}
|
|
if priorErr != nil || prior.ResourceID != uuid ||
|
|
prior.Action != "receive-peer-migration:"+request.MigrationID ||
|
|
prior.State != operation.StateFailed || !prior.EffectMayHaveApplied ||
|
|
prior.SafeToResubmit || prior.ExecutionFenced {
|
|
writePreDispatchError(w, http.StatusConflict, "peer_migration_target_cleanup_not_authorized", "target cleanup requires the exact safely reaped failed receive operation")
|
|
return
|
|
}
|
|
if !trustedPriorFound {
|
|
_, incidentFound, incidentErr := s.Journal.CompletionDeliveryIncident(prior.OperationID)
|
|
if incidentErr != nil {
|
|
writeMappedError(w, incidentErr, nil)
|
|
return
|
|
}
|
|
if incidentFound {
|
|
writePreDispatchError(w, http.StatusConflict, "peer_migration_target_cleanup_not_authorized", "target cleanup is blocked by a completion-delivery incident")
|
|
return
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, prior)
|
|
cancel()
|
|
if inspectErr != nil {
|
|
writeMappedError(w, inspectErr, nil)
|
|
return
|
|
}
|
|
if !peerMigrationTargetCleanupInspectionMatches(prior, request.CleanupPlanID, inspection) {
|
|
writePreDispatchError(w, http.StatusConflict, "peer_migration_cleanup_plan_changed", "target cleanup plan is stale or no longer safe")
|
|
return
|
|
}
|
|
request.PriorOperation = prior
|
|
}
|
|
supersedes := []string{request.PriorOperationID}
|
|
if trustedPriorFound {
|
|
// The isolated worker has its own Journal. The parent Journal
|
|
// already admitted this exact supersession and retains the audit
|
|
// link; the child receives the immutable predecessor separately.
|
|
supersedes = nil
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, operationID, uuid, action, nil,
|
|
true, http.StatusOK,
|
|
hypervisor.CleanupPeerMigrationTargetAdmission(uuid, request.MigrationID),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.CleanupPeerMigrationTarget(ctx, uuid, request)
|
|
}, supersedes...)
|
|
default:
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
return
|
|
case "force-delete":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireSuperAdminPrivilegedOperation(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ForceDeleteVMRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !request.RemoveManagedDisks {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true")
|
|
return
|
|
}
|
|
if !request.AckDataLoss {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true")
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
if !operationIDPattern.MatchString(request.PriorOperationID) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_prior_operation_id", "prior_operation_id must contain 8-80 URL-safe characters")
|
|
return
|
|
}
|
|
if operationID == request.PriorOperationID {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "force_operation_id_reused", "force delete requires a new operation_id")
|
|
return
|
|
}
|
|
existingReplay := false
|
|
trustedPrior, trustedPriorFound := trustedSupersededOperation(
|
|
r.Context(), request.PriorOperationID,
|
|
)
|
|
existing, existingErr := s.Journal.Get(operationID)
|
|
if existingErr == nil {
|
|
existingReplay = existing.Action == "force-delete-vm" && existing.ResourceID == uuid &&
|
|
existing.SupersedesOperationID == request.PriorOperationID
|
|
} else if !errors.Is(existingErr, operation.ErrNotFound) {
|
|
writeMappedError(w, existingErr, nil)
|
|
return
|
|
}
|
|
if !existingReplay {
|
|
prior := trustedPrior
|
|
var priorErr error
|
|
if !trustedPriorFound {
|
|
prior, priorErr = s.Journal.Get(request.PriorOperationID)
|
|
}
|
|
priorActionAllowed := priorErr == nil && (prior.Action == "delete-vm" || prior.Action == "service-suspend")
|
|
if !priorActionAllowed || prior.ResourceID != uuid ||
|
|
prior.State != operation.StateFailed || prior.Error == nil || prior.Error.Code != "force_required" {
|
|
writePreDispatchError(w, http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful stop to require force")
|
|
return
|
|
}
|
|
}
|
|
supersedes := []string{request.PriorOperationID}
|
|
if trustedPriorFound {
|
|
// The parent Journal already authorized and records the force
|
|
// successor. The isolated worker Journal receives the immutable
|
|
// predecessor separately and must not try to load it locally.
|
|
supersedes = nil
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "force-delete-vm", nil,
|
|
true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.ForceDeleteVM(ctx, uuid, request)
|
|
}, supersedes...)
|
|
return
|
|
}
|
|
case "state":
|
|
if len(segments) == 4 && r.Method == http.MethodGet {
|
|
result, err := s.Hypervisor.GetVMState(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
return
|
|
}
|
|
case "metadata":
|
|
if len(segments) == 4 && r.Method == http.MethodGet {
|
|
result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
return
|
|
}
|
|
case "actions":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.PowerRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !validPowerAction(request.Action) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_power_action", "action must be start, shutdown, reboot, reset, or destroy")
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "power-"+string(request.Action), true, http.StatusOK,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.Power(ctx, uuid, request) })
|
|
return
|
|
}
|
|
case "settings":
|
|
if len(segments) == 4 && (r.Method == http.MethodPatch || r.Method == http.MethodPost) {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.SettingsRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.Hostname == nil && request.RootPassword == nil && request.SSHPublicKey == nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "empty_settings", "at least one setting is required")
|
|
return
|
|
}
|
|
if !request.ConfirmRestart {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true")
|
|
return
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedUpdateSettings, HostnameChanged: request.Hostname != nil,
|
|
RootPasswordChanged: request.RootPassword != nil, SSHKeyChanged: request.SSHPublicKey != nil,
|
|
}
|
|
if request.Hostname != nil {
|
|
expected.Hostname = *request.Hostname
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "update-settings", expected, false, http.StatusOK,
|
|
hypervisor.SeedMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateSettings(ctx, uuid, request) })
|
|
return
|
|
}
|
|
case "resources":
|
|
if len(segments) == 4 {
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ResizeResourcesRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateResizeResources(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedResizeResources,
|
|
FromVCPUs: request.FromVCPUs,
|
|
TargetVCPUs: request.TargetVCPUs,
|
|
FromMemoryMiB: request.FromMemoryMiB,
|
|
TargetMemoryMiB: request.TargetMemoryMiB,
|
|
FromDiskGiB: request.FromDiskGiB,
|
|
TargetDiskGiB: request.TargetDiskGiB,
|
|
FromBandwidthMbps: request.FromBandwidthMbps,
|
|
TargetBandwidthMbps: request.TargetBandwidthMbps,
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "resize-vm", expected, true, http.StatusOK,
|
|
hypervisor.ResizeResourcesAdmission(uuid, request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ResizeResources(ctx, uuid, request) })
|
|
return
|
|
}
|
|
case "network":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ChangeNetworkRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !request.ConfirmRestart {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true")
|
|
return
|
|
}
|
|
if err := hypervisor.ValidatePublicNetworkIntent(request.PublicNetwork); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
var source hypervisor.LocalNetworkSource
|
|
var err error
|
|
if s.MutationExecutor != nil {
|
|
source, err = hypervisor.ParseNetworkSelector(request.PublicNetwork.NetworkSelector)
|
|
} else {
|
|
source, err = s.Hypervisor.ResolveNetworkSelector(r.Context(), request.PublicNetwork.NetworkSelector)
|
|
}
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedChangeNetwork, PublicNetwork: request.PublicNetwork,
|
|
NetworkSourceType: source.Type, NetworkSourceName: source.Source,
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "change-network", expected, true, http.StatusAccepted,
|
|
hypervisor.ChangeNetworkAdmission(uuid, request.PublicNetwork),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ChangeNetwork(ctx, uuid, request) })
|
|
return
|
|
}
|
|
if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.NetworkStateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
enabled := segments[4] == "resume"
|
|
action := "network-" + segments[4]
|
|
expected := &operation.ExpectedState{Kind: operation.ExpectedNetworkState, Enabled: enabled}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, action, expected, false, http.StatusOK,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
err := s.Hypervisor.SetNetworkState(ctx, uuid, enabled, request)
|
|
return map[string]bool{"network_enabled": enabled}, err
|
|
})
|
|
return
|
|
}
|
|
case "vpc-port":
|
|
repairPath := len(segments) == 5 && segments[4] == "repair"
|
|
if (len(segments) == 4 || repairPath) &&
|
|
(r.Method == http.MethodPost || r.Method == http.MethodDelete) {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.VPCPortRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = operationID
|
|
request, err = hypervisor.NormalizeVPCPortRequest(request)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
if request.VPCRepair != repairPath ||
|
|
(request.VPCRepair != (request.SupersedesOperationID != "")) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation")
|
|
return
|
|
}
|
|
bridge, _ := vpcrouting.BridgeName(request.VPCKey)
|
|
present := r.Method == http.MethodPost
|
|
action := "apply-vpc-port"
|
|
if !present {
|
|
action = "remove-vpc-port"
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedVPCPort, VPCPresent: present,
|
|
VPCPortKey: request.PortKey, VPCKey: request.VPCKey,
|
|
VPCPrivateIP: request.PrivateIP, VPCCIDR: request.CIDR,
|
|
VPCGateway: request.Gateway, VPCMAC: request.MAC,
|
|
VPCBridge: bridge, VPCRevision: request.HostRevision,
|
|
VPCDigest: request.HostDigest,
|
|
}
|
|
admission := hypervisor.VPCPortAdmission(uuid, request)
|
|
supersedes, ok := s.authorizeVPCRepair(
|
|
w, r, operationID, request.SupersedesOperationID,
|
|
uuid, action, expected, admission, repairPath,
|
|
)
|
|
if !ok {
|
|
return
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, operationID, uuid,
|
|
action, expected, true, http.StatusOK,
|
|
admission,
|
|
func(ctx context.Context) (any, error) {
|
|
if present {
|
|
return s.Hypervisor.ApplyVPCPort(ctx, uuid, request)
|
|
}
|
|
return s.Hypervisor.RemoveVPCPort(ctx, uuid, request)
|
|
}, supersedes...)
|
|
return
|
|
}
|
|
case "service":
|
|
if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.ServiceStateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
enabled := segments[4] == "resume"
|
|
expected := &operation.ExpectedState{Kind: operation.ExpectedServiceState, Enabled: enabled, StartIfSuspended: request.StartIfSuspended}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "service-"+segments[4], expected, true, http.StatusOK,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.SetServiceState(ctx, uuid, enabled, request)
|
|
})
|
|
return
|
|
}
|
|
case "reinstall":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ReinstallRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateTemplateSnapshot(request.TemplateID, request.TemplateRevision, request.TemplateGenerationToken); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "reinstall-vm", true, http.StatusAccepted,
|
|
hypervisor.ReinstallVMAdmission(uuid, request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.Reinstall(ctx, uuid, request) })
|
|
return
|
|
}
|
|
case "metrics":
|
|
if len(segments) == 4 && r.Method == http.MethodGet {
|
|
result, err := s.Hypervisor.GetMetrics(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
return
|
|
}
|
|
case "console-ticket":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
target, err := s.Hypervisor.GetConsole(r.Context(), uuid)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if target.Protocol != "vnc" {
|
|
writeError(w, http.StatusNotImplemented, "unsupported_console", "only VNC console targets are supported", nil)
|
|
return
|
|
}
|
|
ticket, err := s.Console.Issue(target.Address)
|
|
if err != nil {
|
|
writeError(w, http.StatusServiceUnavailable, "console_unavailable", "console target is unavailable", nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusCreated, ticket)
|
|
return
|
|
}
|
|
case "snapshots":
|
|
s.routeSnapshots(w, r, segments, body, actor, uuid)
|
|
return
|
|
case "firewall":
|
|
if len(segments) == 4 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.GetFirewall(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
case http.MethodPut:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.FirewallPolicy](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
operationID := request.OperationID
|
|
request, err := hypervisor.NormalizeFirewallPolicy(request)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
expected := &operation.ExpectedState{Kind: operation.ExpectedFirewall, FirewallRevision: request.Revision, FirewallDigest: request.PolicyDigest}
|
|
s.runMutationExpected(w, r, body, actor, operationID, uuid, "apply-firewall", expected, false, http.StatusOK,
|
|
hypervisor.FirewallMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyFirewall(ctx, uuid, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPut)
|
|
}
|
|
return
|
|
}
|
|
case "rescue":
|
|
s.routeRescue(w, r, segments, body, actor, uuid)
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func peerMigrationTargetCleanupInspectionMatches(prior operation.Record, planID string,
|
|
inspection hypervisor.OperationInspection) bool {
|
|
if inspection.OperationID != prior.OperationID || inspection.Action != prior.Action ||
|
|
inspection.ResourceID != prior.ResourceID || inspection.RequestFingerprint != prior.Fingerprint ||
|
|
inspection.OperationStatus != prior.State || inspection.CheckedAt <= 0 ||
|
|
inspection.SummaryCode != hypervisor.InspectionSummaryCleanupRequired ||
|
|
inspection.EffectResolution != hypervisor.InspectionEffectPartial ||
|
|
inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" ||
|
|
inspection.AbortJob.Available || !inspection.Cleanup.Available ||
|
|
!strings.EqualFold(inspection.Cleanup.PlanID, planID) || len(inspection.Cleanup.Steps) == 0 {
|
|
return false
|
|
}
|
|
for _, step := range inspection.Cleanup.Steps {
|
|
switch step {
|
|
case hypervisor.CleanupStepDeleteManagedVolumes,
|
|
hypervisor.CleanupStepDeleteSeedVolume,
|
|
hypervisor.CleanupStepUndefineNetworkFilter,
|
|
hypervisor.CleanupStepDeleteNVRAM,
|
|
hypervisor.CleanupStepRemoveTrafficState:
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func backupWorkspaceCleanupInspectionMatches(prior operation.Record, planID string,
|
|
inspection hypervisor.OperationInspection) bool {
|
|
if inspection.OperationID != prior.OperationID || inspection.Action != prior.Action ||
|
|
inspection.ResourceID != prior.ResourceID || inspection.RequestFingerprint != prior.Fingerprint ||
|
|
inspection.OperationStatus != prior.State || inspection.CheckedAt <= 0 ||
|
|
inspection.Outcome != hypervisor.InspectionOutcomeSucceeded ||
|
|
inspection.EffectResolution != hypervisor.InspectionEffectApplied ||
|
|
inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" ||
|
|
inspection.AbortJob.Available {
|
|
return false
|
|
}
|
|
if !inspection.Cleanup.Available {
|
|
return planID == "" &&
|
|
inspection.SummaryCode == hypervisor.InspectionSummarySucceeded
|
|
}
|
|
if inspection.SummaryCode != hypervisor.InspectionSummaryCleanupRequired ||
|
|
planID == "" || !strings.EqualFold(inspection.Cleanup.PlanID, planID) ||
|
|
len(inspection.Cleanup.Steps) == 0 {
|
|
return false
|
|
}
|
|
for _, step := range inspection.Cleanup.Steps {
|
|
switch step {
|
|
case hypervisor.CleanupStepDeleteSnapshotVolume,
|
|
hypervisor.CleanupStepRemoveSnapshotMetadata:
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (s *Server) routeSnapshots(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) {
|
|
if len(segments) == 4 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListSnapshots(r.Context(), uuid)
|
|
writeNamedList(w, "snapshots", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.SnapshotRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.SnapshotKey == "" {
|
|
request.SnapshotKey = request.SnapshotID
|
|
}
|
|
if !catalogIDPattern.MatchString(request.SnapshotKey) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot", "snapshot key is invalid")
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
action := "create-snapshot:" + request.SnapshotKey + ":" + request.ConsistencyMode
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusCreated,
|
|
hypervisor.CreateSnapshotAdmission(uuid, request.SnapshotKey),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.CreateSnapshot(ctx, uuid, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) < 5 || !resourceIDPattern.MatchString(segments[4]) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot_id", "snapshot ID is invalid")
|
|
return
|
|
}
|
|
snapshotID := segments[4]
|
|
if len(segments) == 5 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
snapshots, err := s.Hypervisor.ListSnapshots(r.Context(), uuid)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
for _, snapshot := range snapshots {
|
|
if snapshot.ID == snapshotID {
|
|
writeJSON(w, http.StatusOK, snapshot)
|
|
return
|
|
}
|
|
}
|
|
writeError(w, http.StatusNotFound, "snapshot_not_found", "snapshot was not found", nil)
|
|
case http.MethodDelete:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-snapshot:"+snapshotID, true, http.StatusOK,
|
|
hypervisor.DeleteSnapshotAdmission(uuid, snapshotID),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.DeleteSnapshot(ctx, uuid, snapshotID, request)
|
|
})
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) == 6 && segments[5] == "restore" && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "restore-snapshot:"+snapshotID, true, http.StatusAccepted,
|
|
hypervisor.RestoreSnapshotAdmission(uuid, snapshotID, request.OperationID),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.RestoreSnapshot(ctx, uuid, snapshotID, request)
|
|
})
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) routeRescue(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) {
|
|
if len(segments) == 4 && r.Method == http.MethodGet {
|
|
result, err := s.Hypervisor.GetRescue(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
return
|
|
}
|
|
if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "enter" || segments[4] == "exit" || segments[4] == "force-start") {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
if segments[4] == "enter" {
|
|
request, ok := decodeRequired[hypervisor.RescueRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateRescueTemplateSnapshot(
|
|
request.TemplateID,
|
|
request.Revision,
|
|
request.GenerationToken,
|
|
); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "enter-rescue", true, http.StatusAccepted,
|
|
hypervisor.EnterRescueAdmission(uuid, request.TemplateID),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.EnterRescue(ctx, uuid, request) })
|
|
} else if segments[4] == "force-start" {
|
|
request, ok := decodeRequired[hypervisor.ForceRescueStartRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !request.AckDataLoss {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true")
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "force-start-rescue", true, http.StatusAccepted,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ForceStartRescue(ctx, uuid, request) })
|
|
} else {
|
|
request, ok := decodeRequired[hypervisor.ExitRescueRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !request.AckDataLoss {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true")
|
|
return
|
|
}
|
|
if request.NetworkSuspended == nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "network_state_required", "network_suspended must be provided")
|
|
return
|
|
}
|
|
action := "exit-rescue:connected"
|
|
if *request.NetworkSuspended {
|
|
action = "exit-rescue:suspended"
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusAccepted,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ExitRescue(ctx, uuid, request) })
|
|
}
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) routeTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if !actor.IsAdmin && !actor.IsSuperAdmin {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required")
|
|
return
|
|
}
|
|
if len(segments) == 2 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListTemplates(r.Context())
|
|
writeNamedList(w, "templates", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ImportTemplateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateTemplateSnapshot(request.ID, request.Revision, request.GenerationToken); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runAsyncMutation(w, r, body, actor, request.OperationID, "template:"+request.ID, "import-template", false,
|
|
hypervisor.ImportTemplateAdmission(request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ImportTemplate(ctx, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) == 3 && resourceIDPattern.MatchString(segments[2]) {
|
|
templateID := segments[2]
|
|
switch r.Method {
|
|
case http.MethodPatch:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.UpdateTemplateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if strings.TrimSpace(request.Name) == "" || len(request.Name) > 120 {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_template_name", "template name must contain 1-120 characters")
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "update-template", false, http.StatusOK,
|
|
hypervisor.UpdateTemplateAdmission(templateID, request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateTemplate(ctx, templateID, request) })
|
|
case http.MethodDelete:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.MutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "delete-template", true, http.StatusOK,
|
|
hypervisor.DeleteTemplateAdmission(templateID),
|
|
func(ctx context.Context) (any, error) {
|
|
err := s.Hypervisor.DeleteTemplate(ctx, templateID)
|
|
return catalogDeleteResult("template", templateID, 0, err == nil), err
|
|
})
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodPatch, http.MethodDelete)
|
|
}
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) routeRescueTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if !actor.IsAdmin && !actor.IsSuperAdmin {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required")
|
|
return
|
|
}
|
|
if len(segments) == 2 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListRescueTemplates(r.Context())
|
|
writeNamedList(w, "templates", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ImportRescueTemplateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateRescueTemplateSnapshot(request.ID, request.Revision, request.GenerationToken); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runAsyncMutation(w, r, body, actor, request.OperationID, "rescue-template:"+request.ID, "import-rescue-template", false,
|
|
hypervisor.ImportRescueTemplateAdmission(request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ImportRescueTemplate(ctx, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.MutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
id := segments[2]
|
|
s.runMutation(w, r, body, actor, request.OperationID, "rescue-template:"+id, "delete-rescue-template", true, http.StatusOK,
|
|
hypervisor.DeleteRescueTemplateAdmission(id),
|
|
func(ctx context.Context) (any, error) {
|
|
err := s.Hypervisor.DeleteRescueTemplate(ctx, id)
|
|
return catalogDeleteResult("rescue_template", id, 0, err == nil), err
|
|
})
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) routeUserISOs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
ownerID, err := ownerFromRequest(r, actor)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", err.Error())
|
|
return
|
|
}
|
|
if len(segments) == 2 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListUserISOs(r.Context(), ownerID)
|
|
writeNamedList(w, "user_isos", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ImportUserISORequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.OwnerID == 0 {
|
|
request.OwnerID = ownerID
|
|
}
|
|
if request.OwnerID != ownerID && !actor.IsAdmin && !actor.IsSuperAdmin {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "cannot import ISO for another owner")
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
if request.ID == "" {
|
|
request.ID = operationCatalogID(operationID)
|
|
}
|
|
if !catalogIDPattern.MatchString(request.ID) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_user_iso_id", "id must contain 32 lowercase hexadecimal characters")
|
|
return
|
|
}
|
|
if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runAsyncMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", request.OwnerID, request.ID), "import-user-iso", false,
|
|
hypervisor.ImportUserISOAdmission(request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ImportUserISO(ctx, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.MutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
isoID := segments[2]
|
|
s.runMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", ownerID, isoID), "delete-user-iso", true, http.StatusOK,
|
|
hypervisor.DeleteUserISOAdmission(ownerID, isoID),
|
|
func(ctx context.Context) (any, error) {
|
|
err := s.Hypervisor.DeleteUserISO(ctx, isoID, ownerID)
|
|
return catalogDeleteResult("user_iso", isoID, ownerID, err == nil), err
|
|
})
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func catalogDeleteResult(resourceType, id string, ownerID int64, deleted bool) hypervisor.CatalogDeleteResult {
|
|
return hypervisor.CatalogDeleteResult{
|
|
ResourceType: resourceType, ID: id, OwnerID: ownerID, Deleted: deleted,
|
|
ManagedImageAbsent: deleted, ManagedMetadataAbsent: deleted, ManagedArtifactsAbsent: deleted,
|
|
}
|
|
}
|
|
|
|
func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if len(segments) < 3 || !resourceIDPattern.MatchString(segments[2]) {
|
|
writeError(w, http.StatusNotFound, "operation_not_found", "operation was not found", nil)
|
|
return
|
|
}
|
|
operationID := segments[2]
|
|
if len(segments) == 3 && r.Method == http.MethodGet {
|
|
record, err := s.Journal.Get(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if record.Action == "configure-storage" && !requireSuperAdminManage(w, actor) {
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, record)
|
|
return
|
|
}
|
|
if len(segments) == 4 && segments[3] == "inspect" && r.Method == http.MethodGet {
|
|
if !requireSuperAdminInspection(w, actor, operationID) {
|
|
return
|
|
}
|
|
stored, err := s.Journal.GetForInspection(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if !requireSystemRecoveryRecord(w, actor, stored) {
|
|
return
|
|
}
|
|
incident, incidentFound, err := s.Journal.CompletionDeliveryIncident(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
var inspection hypervisor.OperationInspection
|
|
if incidentFound {
|
|
// Delivery corruption is already sufficient to make the panel view
|
|
// inconclusive. Do not make reporting that durable Agent incident
|
|
// depend on a live host read which may itself time out or fail.
|
|
inspection = hypervisor.OperationInspection{
|
|
OperationStatus: stored.State,
|
|
CheckedAt: time.Now().UTC().Unix(),
|
|
}
|
|
if stored.Error != nil {
|
|
inspection.ErrorCode = stored.Error.Code
|
|
}
|
|
applyCompletionDeliveryIncident(&inspection, incident)
|
|
} else {
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
defer cancel()
|
|
inspection, err = s.Hypervisor.InspectOperation(ctx, stored)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
}
|
|
applyExecutionFencedCreateInspection(&inspection, stored)
|
|
// Identity always comes from the immutable Journal record, never from a
|
|
// hypervisor adapter or its observation of current host state.
|
|
inspection.OperationID = stored.OperationID
|
|
inspection.Action = stored.Action
|
|
inspection.ResourceID = stored.ResourceID
|
|
inspection.RequestFingerprint = stored.Fingerprint
|
|
inspection.FenceState, err = s.Journal.FenceState(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
inspection.FenceResolutionRequired =
|
|
inspection.FenceState == operation.FenceStateActive
|
|
writeJSON(w, http.StatusOK, inspection)
|
|
return
|
|
}
|
|
if len(segments) == 4 && segments[3] == "cleanup-backup-workspace" &&
|
|
r.Method == http.MethodPost {
|
|
if !requireSuperAdminPrivilegedOperation(w, actor) {
|
|
return
|
|
}
|
|
if actor.OperationID == "" || actor.OperationID != operationID {
|
|
writePreDispatchError(w, http.StatusBadRequest, "operation_id_mismatch", "actor operation_id must match the cleanup operation")
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.CleanupBackupWorkspaceRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateCleanupBackupWorkspaceRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
action := ""
|
|
existingReplay := false
|
|
trustedPrior, trustedPriorFound := trustedSupersededOperation(
|
|
r.Context(), request.PriorOperationID,
|
|
)
|
|
existing, existingErr := s.Journal.Get(operationID)
|
|
if existingErr == nil {
|
|
action = existing.Action
|
|
existingReplay = strings.HasPrefix(action, "cleanup-backup-workspace:") &&
|
|
existing.SupersedesOperationID == request.PriorOperationID
|
|
} else if !errors.Is(existingErr, operation.ErrNotFound) {
|
|
writeMappedError(w, existingErr, nil)
|
|
return
|
|
}
|
|
prior := trustedPrior
|
|
if !existingReplay {
|
|
var priorErr error
|
|
if !trustedPriorFound {
|
|
prior, priorErr = s.Journal.Get(request.PriorOperationID)
|
|
}
|
|
parts := strings.Split(prior.Action, ":")
|
|
if priorErr != nil || len(parts) != 3 || parts[0] != "create-backup" ||
|
|
!catalogIDPattern.MatchString(parts[1]) ||
|
|
prior.State != operation.StateSucceeded ||
|
|
prior.ResourceID == "" || prior.ExecutionFenced {
|
|
writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup requires the exact successful backup operation")
|
|
return
|
|
}
|
|
action = "cleanup-backup-workspace:" + parts[1]
|
|
if !trustedPriorFound {
|
|
_, incidentFound, incidentErr := s.Journal.CompletionDeliveryIncident(prior.OperationID)
|
|
if incidentErr != nil {
|
|
writeMappedError(w, incidentErr, nil)
|
|
return
|
|
}
|
|
if incidentFound {
|
|
writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup is blocked by a completion-delivery incident")
|
|
return
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, prior)
|
|
cancel()
|
|
if inspectErr != nil {
|
|
writeMappedError(w, inspectErr, nil)
|
|
return
|
|
}
|
|
if !backupWorkspaceCleanupInspectionMatches(
|
|
prior, request.CleanupPlanID, inspection,
|
|
) {
|
|
writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_plan_changed", "cleanup plan is stale or no longer safe")
|
|
return
|
|
}
|
|
request.PriorOperation = prior
|
|
}
|
|
parts := strings.Split(action, ":")
|
|
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) {
|
|
writePreDispatchError(w, http.StatusConflict, "backup_workspace_cleanup_not_authorized", "cleanup action identity is invalid")
|
|
return
|
|
}
|
|
resourceID := prior.ResourceID
|
|
if existingReplay {
|
|
resourceID = existing.ResourceID
|
|
}
|
|
supersedes := []string{request.PriorOperationID}
|
|
if trustedPriorFound {
|
|
supersedes = nil
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, operationID, resourceID,
|
|
action, nil, true, http.StatusOK,
|
|
hypervisor.CleanupBackupWorkspaceAdmission(resourceID, parts[1]),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.CleanupBackupWorkspace(ctx, resourceID, request)
|
|
}, supersedes...)
|
|
return
|
|
}
|
|
if len(segments) == 4 && segments[3] == "fence-resolution" && r.Method == http.MethodGet {
|
|
if !requireSuperAdminInspection(w, actor, operationID) {
|
|
return
|
|
}
|
|
stored, err := s.Journal.Get(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if !requireSystemRecoveryRecord(w, actor, stored) {
|
|
return
|
|
}
|
|
resolution, err := s.Journal.GetFenceResolution(operationID)
|
|
if errors.Is(err, operation.ErrNotFound) {
|
|
writeError(w, http.StatusNotFound, "fence_resolution_not_found", "operation fence resolution was not found", nil)
|
|
return
|
|
}
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, fenceResolutionResponse{FenceResolution: resolution, Replayed: false})
|
|
return
|
|
}
|
|
if len(segments) == 4 && segments[3] == "resolve-fence" && r.Method == http.MethodPost {
|
|
if !requireSuperAdminFenceResolution(w, actor, operationID) {
|
|
return
|
|
}
|
|
if actor.OperationID == "" || actor.OperationID != operationID {
|
|
writeError(w, http.StatusBadRequest, "operation_id_mismatch", "actor operation_id must match the fenced operation", nil)
|
|
return
|
|
}
|
|
stored, err := s.Journal.Get(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if !requireSystemRecoveryRecord(w, actor, stored) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[operation.FenceResolutionRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := operation.ValidateFenceResolutionRequest(request); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_fence_resolution", err.Error(), nil)
|
|
return
|
|
}
|
|
if actor.SystemRecovery && actor.UserID == 0 && operation.IsCreateLikeAction(stored.Action) {
|
|
if request.Resolution != operation.FenceResolutionNotApplied {
|
|
writeError(w, http.StatusConflict, "operation_fence_not_resolvable", "automatic create recovery can only confirm that creation was not applied", nil)
|
|
return
|
|
}
|
|
if stored.ExecutionFenced {
|
|
writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery cannot conclude while the original worker may still be running", nil)
|
|
return
|
|
}
|
|
fenceState, err := s.Journal.FenceState(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if fenceState == operation.FenceStateActive {
|
|
_, incidentFound, err := s.Journal.CompletionDeliveryIncident(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if incidentFound {
|
|
writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil)
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, stored)
|
|
cancel()
|
|
if inspectErr != nil {
|
|
writeMappedError(w, inspectErr, nil)
|
|
return
|
|
}
|
|
if !systemCreateAbsenceConfirmed(stored, request, inspection) {
|
|
writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
resolution, replayed, err := s.Journal.ResolveResourceFence(operationID, actor.UserID, request)
|
|
switch {
|
|
case errors.Is(err, operation.ErrNotFound):
|
|
writeError(w, http.StatusNotFound, "operation_not_found", "operation was not found", nil)
|
|
return
|
|
case errors.Is(err, operation.ErrFenceNotResolvable):
|
|
writeError(w, http.StatusConflict, "operation_fence_not_resolvable", "operation does not hold an active effect fence", nil)
|
|
return
|
|
case errors.Is(err, operation.ErrFenceIdentityConflict):
|
|
writeError(w, http.StatusConflict, "operation_identity_conflict", "fence resolution identity does not match the immutable operation", nil)
|
|
return
|
|
case errors.Is(err, operation.ErrFenceResolutionConflict):
|
|
writeError(w, http.StatusConflict, "operation_fence_resolution_conflict", "operation fence already has a different audit resolution", nil)
|
|
return
|
|
case errors.Is(err, operation.ErrFenceInspectionStale):
|
|
writeError(w, http.StatusConflict, "fence_inspection_stale", "operation fence resolution requires a fresh read-only inspection", nil)
|
|
return
|
|
case err != nil:
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, fenceResolutionResponse{FenceResolution: resolution, Replayed: replayed})
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) createLikeNetworkExpectedState(
|
|
ctx context.Context,
|
|
request hypervisor.CreateVMRequest,
|
|
port *hypervisor.VPCPortMetadata,
|
|
) (*operation.ExpectedState, error) {
|
|
var source hypervisor.LocalNetworkSource
|
|
var err error
|
|
if s.MutationExecutor != nil {
|
|
source, err = hypervisor.ParseNetworkSelector(request.PublicNetwork.NetworkSelector)
|
|
} else {
|
|
source, err = s.Hypervisor.ResolveNetworkSelector(ctx, request.PublicNetwork.NetworkSelector)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedCreateLikeNetwork,
|
|
NetworkSourceType: source.Type, NetworkSourceName: source.Source,
|
|
PublicNetwork: request.PublicNetwork,
|
|
VPCPresent: port != nil,
|
|
}
|
|
if port != nil {
|
|
expected.VPCKey = port.VPCKey
|
|
expected.VPCRevision = port.HostRevision
|
|
expected.VPCDigest = port.HostDigest
|
|
expected.VPCPortKey = port.PortKey
|
|
expected.VPCCIDR = port.CIDR
|
|
expected.VPCGateway = port.Gateway
|
|
expected.VPCPrivateIP = port.PrivateIP
|
|
expected.VPCMAC = port.MAC
|
|
expected.VPCBridge = port.Bridge
|
|
}
|
|
return expected, nil
|
|
}
|
|
|
|
func (s *Server) runMutation(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
body []byte,
|
|
actor auth.Actor,
|
|
bodyOperationID, resourceID, action string,
|
|
destructive bool,
|
|
successStatus int,
|
|
admission hypervisor.MutationAdmission,
|
|
call func(context.Context) (any, error),
|
|
) {
|
|
s.runMutationExpected(w, r, body, actor, bodyOperationID, resourceID, action, nil, destructive, successStatus, admission, call)
|
|
}
|
|
|
|
func (s *Server) runMutationExpected(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
body []byte,
|
|
actor auth.Actor,
|
|
bodyOperationID, resourceID, action string,
|
|
expectedState *operation.ExpectedState,
|
|
destructive bool,
|
|
successStatus int,
|
|
admission hypervisor.MutationAdmission,
|
|
call func(context.Context) (any, error),
|
|
supersedesOperationID ...string,
|
|
) {
|
|
operationID, err := resolveOperationID(actor.OperationID, bodyOperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
if expectedState != nil {
|
|
copyExpected := *expectedState
|
|
copyExpected.TransitionID = operationID
|
|
copyExpected.PublicNetwork.DNSServers = append([]string(nil), expectedState.PublicNetwork.DNSServers...)
|
|
expectedState = ©Expected
|
|
}
|
|
supersedes := ""
|
|
if len(supersedesOperationID) != 0 {
|
|
supersedes = supersedesOperationID[0]
|
|
}
|
|
spec := operation.Spec{
|
|
OperationID: operationID, ResourceID: resourceID, Action: action,
|
|
Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body),
|
|
Destructive: destructive, ExpectedState: expectedState,
|
|
LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO,
|
|
SupersedesOperationID: supersedes,
|
|
}
|
|
if s.MutationExecutor != nil {
|
|
s.runIsolatedMutation(w, r, body, actor, spec, s.isolatedMutationDeadline(action))
|
|
return
|
|
}
|
|
if _, trusted := r.Context().Value(trustedWorkerRequestKey{}).(struct{}); !trusted {
|
|
writePreDispatchError(w, http.StatusServiceUnavailable, "mutation_worker_unavailable", "isolated mutation worker is unavailable")
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
defer cancel()
|
|
record, replayed, callErr := s.Journal.Execute(ctx, spec, func(ctx context.Context) (json.RawMessage, error) {
|
|
result, err := call(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
encoded, err := json.Marshal(result)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("encode operation result: %w", err)
|
|
}
|
|
return encoded, nil
|
|
})
|
|
logOperationOutcome(record)
|
|
if errors.Is(callErr, operation.ErrFingerprint) {
|
|
writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil)
|
|
return
|
|
}
|
|
if errors.Is(callErr, operation.ErrOperationStarting) {
|
|
writeOperationStarting(w, record)
|
|
return
|
|
}
|
|
if operation.WasNotDispatched(record) {
|
|
writeOperationFailure(w, record, replayed)
|
|
return
|
|
}
|
|
if callErr != nil {
|
|
if record.State == operation.StateFailed {
|
|
writeOperationFailure(w, record, replayed)
|
|
return
|
|
}
|
|
writeMappedError(w, callErr, &record)
|
|
return
|
|
}
|
|
if record.State == operation.StateFailed {
|
|
writeOperationFailure(w, record, replayed)
|
|
return
|
|
}
|
|
if record.State == operation.StateDispatching || record.State == operation.StateReserved {
|
|
writeJSON(w, http.StatusAccepted, operationWire(record, replayed))
|
|
return
|
|
}
|
|
if replayed {
|
|
successStatus = http.StatusOK
|
|
}
|
|
writeJSON(w, successStatus, operationWire(record, replayed))
|
|
}
|
|
|
|
func (s *Server) runAsyncMutation(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
body []byte,
|
|
actor auth.Actor,
|
|
bodyOperationID, resourceID, action string,
|
|
destructive bool,
|
|
admission hypervisor.MutationAdmission,
|
|
call func(context.Context) (any, error),
|
|
) {
|
|
s.runAsyncMutationWithDeadline(
|
|
w, r, body, actor, bodyOperationID, resourceID, action,
|
|
destructive, admission, catalogImportTimeout, call,
|
|
)
|
|
}
|
|
|
|
func (s *Server) runAsyncMutationWithDeadline(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
body []byte,
|
|
actor auth.Actor,
|
|
bodyOperationID, resourceID, action string,
|
|
destructive bool,
|
|
admission hypervisor.MutationAdmission,
|
|
deadline time.Duration,
|
|
call func(context.Context) (any, error),
|
|
) {
|
|
s.runAsyncMutationWithDeadlineExpected(
|
|
w, r, body, actor, bodyOperationID, resourceID, action, nil,
|
|
destructive, admission, deadline, call,
|
|
)
|
|
}
|
|
|
|
func (s *Server) runAsyncMutationWithDeadlineExpected(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
body []byte,
|
|
actor auth.Actor,
|
|
bodyOperationID, resourceID, action string,
|
|
expectedState *operation.ExpectedState,
|
|
destructive bool,
|
|
admission hypervisor.MutationAdmission,
|
|
deadline time.Duration,
|
|
call func(context.Context) (any, error),
|
|
) {
|
|
operationID, err := resolveOperationID(actor.OperationID, bodyOperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
if expectedState != nil {
|
|
copyExpected := *expectedState
|
|
copyExpected.TransitionID = operationID
|
|
copyExpected.PublicNetwork.DNSServers = append([]string(nil), expectedState.PublicNetwork.DNSServers...)
|
|
expectedState = ©Expected
|
|
}
|
|
spec := operation.Spec{
|
|
OperationID: operationID, ResourceID: resourceID, Action: action,
|
|
Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body),
|
|
Destructive: destructive, ExpectedState: expectedState,
|
|
LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO,
|
|
}
|
|
if s.MutationExecutor != nil {
|
|
s.runIsolatedMutation(w, r, body, actor, spec, deadline)
|
|
return
|
|
}
|
|
if _, trusted := r.Context().Value(trustedWorkerRequestKey{}).(struct{}); !trusted {
|
|
writePreDispatchError(w, http.StatusServiceUnavailable, "mutation_worker_unavailable", "isolated mutation worker is unavailable")
|
|
return
|
|
}
|
|
record, replayed, err := s.Journal.Reserve(spec)
|
|
if errors.Is(err, operation.ErrFingerprint) {
|
|
writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil)
|
|
return
|
|
}
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
canStart := record.State == operation.StateReserved
|
|
if canStart && !s.claimAsync(operationID) {
|
|
writeOperationStarting(w, record)
|
|
return
|
|
}
|
|
if canStart {
|
|
select {
|
|
case s.asyncSlots <- struct{}{}:
|
|
default:
|
|
failed, failErr := s.Journal.FailNotDispatched(operationID)
|
|
s.releaseAsync(operationID)
|
|
if failErr != nil {
|
|
writeMappedError(w, failErr, &record)
|
|
return
|
|
}
|
|
writeOperationFailure(w, failed, false)
|
|
return
|
|
}
|
|
dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout)
|
|
started, _, done, startErr := s.Journal.ExecuteBackground(
|
|
dispatchCtx, s.backgroundCtx, deadline, spec,
|
|
func(ctx context.Context) (json.RawMessage, error) {
|
|
result, callErr := call(ctx)
|
|
if callErr != nil {
|
|
return nil, callErr
|
|
}
|
|
return json.Marshal(result)
|
|
},
|
|
)
|
|
dispatchCancel()
|
|
if startErr != nil || done == nil {
|
|
<-s.asyncSlots
|
|
s.releaseAsync(operationID)
|
|
if startErr != nil {
|
|
if errors.Is(startErr, operation.ErrOperationStarting) {
|
|
writeOperationStarting(w, started)
|
|
return
|
|
}
|
|
if started.State == operation.StateFailed && started.Error != nil && started.Error.Code == "operation_not_dispatched" {
|
|
writeOperationFailure(w, started, false)
|
|
return
|
|
}
|
|
writeMappedError(w, startErr, &started)
|
|
return
|
|
}
|
|
record = started
|
|
} else {
|
|
record = started
|
|
s.asyncWG.Add(1)
|
|
go func() {
|
|
defer s.asyncWG.Done()
|
|
defer func() { <-s.asyncSlots }()
|
|
defer s.releaseAsync(operationID)
|
|
<-done
|
|
if final, getErr := s.Journal.Get(operationID); getErr == nil {
|
|
logOperationOutcome(final)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
writeJSON(w, http.StatusAccepted, operationWire(record, replayed))
|
|
}
|
|
|
|
func (s *Server) runIsolatedMutation(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
body []byte,
|
|
actor auth.Actor,
|
|
spec operation.Spec,
|
|
deadline time.Duration,
|
|
) {
|
|
if deadline <= 0 {
|
|
deadline = s.OperationTimeout
|
|
}
|
|
actor.OperationID = spec.OperationID
|
|
request := worker.Request{
|
|
Version: worker.ProtocolVersion, Method: r.Method,
|
|
RequestTarget: auth.RequestTarget(r), Body: string(body), Actor: actor,
|
|
OperationID: spec.OperationID, Fingerprint: spec.Fingerprint,
|
|
}
|
|
if spec.SupersedesOperationID != "" {
|
|
prior, err := s.Journal.Get(spec.SupersedesOperationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
request.SupersededOperation = &prior
|
|
if isVPCRepairTarget(request.RequestTarget) {
|
|
request.SupersededExpectedState = prior.ExpectedState
|
|
}
|
|
}
|
|
dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout)
|
|
record, replayed, done, err := s.Journal.ExecuteBackground(
|
|
dispatchCtx, s.backgroundCtx, deadline, spec,
|
|
func(callCtx context.Context) (json.RawMessage, error) {
|
|
return s.MutationExecutor.Execute(callCtx, request)
|
|
},
|
|
)
|
|
dispatchCancel()
|
|
if done != nil {
|
|
s.asyncWG.Add(1)
|
|
go func(operationID string) {
|
|
defer s.asyncWG.Done()
|
|
<-done
|
|
if final, getErr := s.Journal.Get(operationID); getErr == nil {
|
|
logOperationOutcome(final)
|
|
}
|
|
}(spec.OperationID)
|
|
}
|
|
if errors.Is(err, operation.ErrFingerprint) {
|
|
writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil)
|
|
return
|
|
}
|
|
if errors.Is(err, operation.ErrOperationStarting) {
|
|
writeOperationStarting(w, record)
|
|
return
|
|
}
|
|
if operation.WasNotDispatched(record) {
|
|
writeJSON(w, http.StatusOK, operationWire(record, replayed))
|
|
return
|
|
}
|
|
if err != nil {
|
|
writeMappedError(w, err, &record)
|
|
return
|
|
}
|
|
status := http.StatusAccepted
|
|
if record.State == operation.StateSucceeded || record.State == operation.StateFailed {
|
|
status = http.StatusOK
|
|
}
|
|
writeJSON(w, status, operationWire(record, replayed))
|
|
}
|
|
|
|
func (s *Server) isolatedMutationDeadline(action string) time.Duration {
|
|
deadline := s.OperationTimeout
|
|
if action == "configure-storage" && deadline > storageConfigurationWorkerTimeout {
|
|
return storageConfigurationWorkerTimeout
|
|
}
|
|
return deadline
|
|
}
|
|
|
|
func (s *Server) beginRequest() bool {
|
|
s.lifecycleMu.Lock()
|
|
defer s.lifecycleMu.Unlock()
|
|
if s.closing {
|
|
return false
|
|
}
|
|
s.requestWG.Add(1)
|
|
return true
|
|
}
|
|
|
|
// Shutdown stops background imports and waits for every admitted API handler
|
|
// and import watcher. The journal's own workers are drained separately before
|
|
// libvirt or BoltDB are closed.
|
|
func (s *Server) Shutdown(ctx context.Context) error {
|
|
s.lifecycleMu.Lock()
|
|
if !s.closing {
|
|
s.closing = true
|
|
s.backgroundCancel()
|
|
}
|
|
s.lifecycleMu.Unlock()
|
|
if err := waitGroup(ctx, &s.requestWG); err != nil {
|
|
return err
|
|
}
|
|
return waitGroup(ctx, &s.asyncWG)
|
|
}
|
|
|
|
func waitGroup(ctx context.Context, group *sync.WaitGroup) error {
|
|
done := make(chan struct{})
|
|
go func() {
|
|
group.Wait()
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (s *Server) claimAsync(operationID string) bool {
|
|
s.asyncMu.Lock()
|
|
defer s.asyncMu.Unlock()
|
|
if s.asyncRunning == nil {
|
|
s.asyncRunning = make(map[string]struct{})
|
|
}
|
|
if _, exists := s.asyncRunning[operationID]; exists {
|
|
return false
|
|
}
|
|
s.asyncRunning[operationID] = struct{}{}
|
|
return true
|
|
}
|
|
|
|
func applyCompletionDeliveryIncident(inspection *hypervisor.OperationInspection,
|
|
incident operation.CompletionDeliveryIncident) {
|
|
if inspection == nil {
|
|
return
|
|
}
|
|
inspection.Outcome = hypervisor.InspectionOutcomeUncertain
|
|
inspection.SummaryCode = hypervisor.InspectionSummaryIncomplete
|
|
inspection.EffectResolution = hypervisor.InspectionEffectInconclusive
|
|
inspection.EffectProjection = nil
|
|
inspection.AbortJob = hypervisor.OperationAbortInspection{}
|
|
inspection.Cleanup = hypervisor.OperationCleanupPlan{}
|
|
inspection.CompletionDeliveryState = operation.CompletionDeliveryStateQuarantined
|
|
inspection.CompletionDeliveryErrorCode = incident.ErrorCode
|
|
}
|
|
|
|
func applyExecutionFencedCreateInspection(inspection *hypervisor.OperationInspection, record operation.Record) {
|
|
if inspection == nil || !operation.IsCreateLikeAction(record.Action) || !record.ExecutionFenced {
|
|
return
|
|
}
|
|
inspection.Outcome = hypervisor.InspectionOutcomeUncertain
|
|
inspection.SummaryCode = hypervisor.InspectionSummaryIncomplete
|
|
inspection.ErrorCode = ""
|
|
if record.Error != nil {
|
|
inspection.ErrorCode = record.Error.Code
|
|
}
|
|
inspection.EffectResolution = hypervisor.InspectionEffectInconclusive
|
|
inspection.EffectProjection = nil
|
|
inspection.AbortJob = hypervisor.OperationAbortInspection{}
|
|
inspection.Cleanup = hypervisor.OperationCleanupPlan{}
|
|
}
|
|
|
|
// systemCreateAbsenceConfirmed accepts only a fresh, exact read-only host
|
|
// observation that both the VM and all panel-managed creation artifacts are
|
|
// absent. It does not infer absence from the original operation error and does
|
|
// not authorize any create replay or host mutation.
|
|
func systemCreateAbsenceConfirmed(record operation.Record, request operation.FenceResolutionRequest,
|
|
inspection hypervisor.OperationInspection) bool {
|
|
if !operation.IsCreateLikeAction(record.Action) || record.State != operation.StateFailed ||
|
|
inspection.OperationID != record.OperationID || inspection.Action != record.Action ||
|
|
inspection.ResourceID != record.ResourceID || inspection.RequestFingerprint != record.Fingerprint ||
|
|
inspection.OperationStatus != record.State || inspection.CheckedAt < request.InspectionCheckedAt ||
|
|
inspection.Outcome != hypervisor.InspectionOutcomeFailed ||
|
|
inspection.SummaryCode != hypervisor.InspectionSummaryFailed ||
|
|
inspection.ErrorCode != "creation_absent" ||
|
|
inspection.EffectResolution != hypervisor.InspectionEffectNotApplied ||
|
|
inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" ||
|
|
inspection.AbortJob.Available || inspection.Cleanup.Available ||
|
|
len(inspection.EffectProjection) == 0 || len(inspection.EffectProjection) > 1024 {
|
|
return false
|
|
}
|
|
var projection map[string]json.RawMessage
|
|
if err := json.Unmarshal(inspection.EffectProjection, &projection); err != nil || len(projection) != 2 {
|
|
return false
|
|
}
|
|
var creationAbsent, managedArtifactsAbsent bool
|
|
if err := json.Unmarshal(projection["creation_absent"], &creationAbsent); err != nil || !creationAbsent {
|
|
return false
|
|
}
|
|
if err := json.Unmarshal(projection["managed_artifacts_absent"], &managedArtifactsAbsent); err != nil || !managedArtifactsAbsent {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (s *Server) releaseAsync(operationID string) {
|
|
s.asyncMu.Lock()
|
|
defer s.asyncMu.Unlock()
|
|
delete(s.asyncRunning, operationID)
|
|
}
|
|
|
|
func operationWire(record operation.Record, replayed bool) operationResponse {
|
|
return operationResponse{
|
|
OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID,
|
|
RequestFingerprint: record.Fingerprint, Status: record.State, Replayed: replayed,
|
|
Result: record.Result, Error: record.Error, CauseCode: record.CauseCode,
|
|
EffectMayHaveApplied: record.EffectMayHaveApplied,
|
|
SafeToResubmit: record.SafeToResubmit,
|
|
Retryable: false,
|
|
}
|
|
}
|
|
|
|
// Log only bounded identifiers and stable codes. Free-form backend errors,
|
|
// request bodies, guest credentials, and host paths never enter this line.
|
|
func logOperationOutcome(record operation.Record) {
|
|
if record.State != operation.StateFailed {
|
|
return
|
|
}
|
|
errorCode := ""
|
|
if record.Error != nil && len(record.Error.Code) <= 64 &&
|
|
stableCodePattern.MatchString(record.Error.Code) {
|
|
errorCode = record.Error.Code
|
|
}
|
|
log.Printf("Agent operation outcome operation_id=%s action=%s status=%s error_code=%s cause_code=%s",
|
|
record.OperationID, record.Action, record.State, errorCode, record.CauseCode)
|
|
}
|
|
|
|
func resolveOperationID(actorID, bodyID string) (string, error) {
|
|
if actorID == "" && bodyID == "" {
|
|
return "", errors.New("operation_id is required in X-Virt-Actor or request body")
|
|
}
|
|
if actorID != "" && bodyID != "" && actorID != bodyID {
|
|
return "", errors.New("actor and body operation_id values differ")
|
|
}
|
|
resolved := bodyID
|
|
if actorID != "" {
|
|
resolved = actorID
|
|
}
|
|
if !operationIDPattern.MatchString(resolved) {
|
|
return "", errors.New("operation_id must contain 8-80 URL-safe characters")
|
|
}
|
|
return resolved, nil
|
|
}
|
|
|
|
func operationUUID(operationID string) string {
|
|
digest := sha256.Sum256([]byte("virt-cloud-operation:" + operationID))
|
|
digest[6] = digest[6]&0x0f | 0x50
|
|
digest[8] = digest[8]&0x3f | 0x80
|
|
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
|
|
digest[0:4], digest[4:6], digest[6:8], digest[8:10], digest[10:16])
|
|
}
|
|
|
|
func operationCatalogID(operationID string) string {
|
|
digest := sha256.Sum256([]byte("virt-cloud-catalog:" + operationID))
|
|
return fmt.Sprintf("%x", digest[:16])
|
|
}
|
|
|
|
func validPowerAction(action hypervisor.PowerAction) bool {
|
|
switch action {
|
|
case hypervisor.PowerStart, hypervisor.PowerShutdown, hypervisor.PowerReboot, hypervisor.PowerReset, hypervisor.PowerDestroy:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func validateImportSource(raw, token, digest string) error {
|
|
parsed, err := url.Parse(raw)
|
|
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" {
|
|
return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_url", "source_url must be an HTTPS URL without credentials or fragment", err)
|
|
}
|
|
if digest != "" && !sha256Pattern.MatchString(digest) {
|
|
return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_sha256", "sha256 must be 64 lowercase hexadecimal characters", nil)
|
|
}
|
|
if token != "" && !sourceTokenPattern.MatchString(token) {
|
|
return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_token", "source_token must contain 32-512 URL-safe characters", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ownerFromRequest(r *http.Request, actor auth.Actor) (int64, error) {
|
|
value := r.URL.Query().Get("owner_id")
|
|
if value == "" {
|
|
return actor.UserID, nil
|
|
}
|
|
ownerID, err := strconv.ParseInt(value, 10, 64)
|
|
if err != nil || ownerID <= 0 {
|
|
return 0, errors.New("owner_id is invalid")
|
|
}
|
|
if ownerID != actor.UserID && !actor.IsAdmin && !actor.IsSuperAdmin {
|
|
return 0, errors.New("cannot access another owner's ISO")
|
|
}
|
|
return ownerID, nil
|
|
}
|
|
|
|
func requireManage(w http.ResponseWriter, actor auth.Actor) bool {
|
|
if !actor.CanManage {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to manage this resource")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func requireSuperAdminManage(w http.ResponseWriter, actor auth.Actor) bool {
|
|
if !actor.IsSuperAdmin || !actor.CanManage {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to configure host storage")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func requireSuperAdminPrivilegedOperation(w http.ResponseWriter, actor auth.Actor) bool {
|
|
if !actor.IsSuperAdmin || !actor.CanManage {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to perform this privileged operation")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func requireSuperAdminInspection(w http.ResponseWriter, actor auth.Actor, operationID string) bool {
|
|
if actor.SystemRecovery && actor.UserID == 0 && actor.OperationID == operationID {
|
|
return true
|
|
}
|
|
if actor.UserID < 1 || !actor.IsSuperAdmin || !actor.CanManage {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to inspect operation state")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func requireSuperAdminFenceResolution(w http.ResponseWriter, actor auth.Actor, operationID string) bool {
|
|
if actor.SystemRecovery && actor.UserID == 0 && actor.OperationID == operationID {
|
|
return true
|
|
}
|
|
if actor.UserID < 1 || !actor.IsSuperAdmin || !actor.CanManage {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to resolve operation effect fences")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Route shape and operation identity are necessary but not sufficient for
|
|
// system recovery: authorization is completed only after reading the
|
|
// immutable Journal action and applying the explicit Agent action allowlist.
|
|
func requireSystemRecoveryRecord(w http.ResponseWriter, actor auth.Actor, record operation.Record) bool {
|
|
if actor.SystemRecovery && !operation.SystemRecoveryActionAllowed(record.Action) {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "system recovery is not enabled for this operation kind")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// systemRecoveryRequestAllowed is a deny-by-default capability boundary. The
|
|
// signed system actor is bound to one operation and cannot reach health,
|
|
// inventory, raw Journal records, catalog reads, or any mutation route.
|
|
func systemRecoveryRequestAllowed(actor auth.Actor, method string, segments []string) bool {
|
|
if !actor.SystemRecovery || actor.UserID != 0 || actor.OperationID == "" ||
|
|
len(segments) != 4 || segments[0] != "v1" || segments[1] != "operations" ||
|
|
segments[2] != actor.OperationID {
|
|
return false
|
|
}
|
|
switch segments[3] {
|
|
case "inspect", "fence-resolution":
|
|
return method == http.MethodGet
|
|
case "resolve-fence":
|
|
return method == http.MethodPost
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func decodeRequired[T any](w http.ResponseWriter, body []byte) (T, bool) {
|
|
var value T
|
|
if len(body) == 0 {
|
|
writePreDispatchError(w, http.StatusBadRequest, "invalid_json", "JSON request body is required")
|
|
return value, false
|
|
}
|
|
if err := decodeStrict(body, &value); err != nil {
|
|
writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error())
|
|
return value, false
|
|
}
|
|
return value, true
|
|
}
|
|
|
|
func decodeOptional[T any](w http.ResponseWriter, body []byte) (T, bool) {
|
|
var value T
|
|
if len(body) == 0 {
|
|
return value, true
|
|
}
|
|
if err := decodeStrict(body, &value); err != nil {
|
|
writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error())
|
|
return value, false
|
|
}
|
|
return value, true
|
|
}
|
|
|
|
func decodeStrict(body []byte, target any) error {
|
|
if !utf8.Valid(body) {
|
|
return errors.New("invalid JSON: body must be valid UTF-8")
|
|
}
|
|
decoder := json.NewDecoder(bytes.NewReader(body))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(target); err != nil {
|
|
return fmt.Errorf("invalid JSON: %w", err)
|
|
}
|
|
var extra any
|
|
if err := decoder.Decode(&extra); err != io.EOF {
|
|
return errors.New("invalid JSON: exactly one object is required")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func readBoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, error) {
|
|
if r.Body == nil {
|
|
return []byte{}, nil
|
|
}
|
|
r.Body = http.MaxBytesReader(w, r.Body, limit)
|
|
defer r.Body.Close()
|
|
return io.ReadAll(r.Body)
|
|
}
|
|
|
|
func writeResult(w http.ResponseWriter, value any, err error) {
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, value)
|
|
}
|
|
|
|
func writeNamedList[T any](w http.ResponseWriter, name string, value []T, err error) {
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if value == nil {
|
|
value = make([]T, 0)
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{name: value})
|
|
}
|
|
|
|
func writeMappedError(w http.ResponseWriter, err error, record *operation.Record) {
|
|
status, code, message := classifyError(err)
|
|
writeError(w, status, code, message, record)
|
|
}
|
|
|
|
// writePreDispatchMappedError is only for validation or dependency checks
|
|
// completed before Journal.Execute/Reserve. The explicit marker lets the
|
|
// panel distinguish a proven no-journal outcome from a lost response after a
|
|
// mutation may already have crossed its durable dispatch boundary.
|
|
func writePreDispatchMappedError(w http.ResponseWriter, err error) {
|
|
status, code, message := classifyError(err)
|
|
writePreDispatchError(w, status, code, message)
|
|
}
|
|
|
|
func publicErrorCode(err error, fallback string) string {
|
|
var public interface{ PublicCode() string }
|
|
if errors.As(err, &public) && stableCodePattern.MatchString(public.PublicCode()) {
|
|
return public.PublicCode()
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func publicErrorMessage(err error, fallback string) string {
|
|
var public interface{ PublicMessage() string }
|
|
if errors.As(err, &public) && public.PublicMessage() != "" {
|
|
return public.PublicMessage()
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
// writeOperationStarting reports the active admission owner without exposing a
|
|
// stale failed/not-dispatched record from before that owner acquired admission.
|
|
// The projected reserved state is diagnostic only; it never authorizes a
|
|
// caller to dispatch the mutation again. The journal remains authoritative.
|
|
func writeOperationStarting(w http.ResponseWriter, record operation.Record) {
|
|
record.State = operation.StateReserved
|
|
record.Result = nil
|
|
record.Error = nil
|
|
record.CauseCode = ""
|
|
record.Tombstone = false
|
|
writeError(w, http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting", &record)
|
|
}
|
|
|
|
func classifyError(err error) (int, string, string) {
|
|
if errors.Is(err, operation.ErrNotFound) {
|
|
return http.StatusNotFound, "operation_not_found", "operation was not found"
|
|
}
|
|
if errors.Is(err, operation.ErrBusy) {
|
|
return http.StatusServiceUnavailable, "agent_busy", "Agent mutation capacity is exhausted"
|
|
}
|
|
if errors.Is(err, operation.ErrOperationStarting) {
|
|
return http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting"
|
|
}
|
|
if errors.Is(err, operation.ErrResourceBlocked) {
|
|
return http.StatusConflict, "resource_operation_blocked", "resource is fenced by a prior failed operation whose effect may have applied"
|
|
}
|
|
if errors.Is(err, operation.ErrInvalidSupersession) {
|
|
return http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful operation"
|
|
}
|
|
var backend *hypervisor.Error
|
|
if errors.As(err, &backend) {
|
|
status := statusForKind(backend.Kind)
|
|
code := backend.Code
|
|
if code == "" {
|
|
code = "backend_error"
|
|
}
|
|
message := backend.Message
|
|
if message == "" {
|
|
message = http.StatusText(status)
|
|
}
|
|
return status, code, message
|
|
}
|
|
switch {
|
|
case errors.Is(err, hypervisor.ErrNotFound):
|
|
return http.StatusNotFound, "not_found", "resource was not found"
|
|
case errors.Is(err, hypervisor.ErrConflict):
|
|
return http.StatusConflict, "conflict", "resource conflict"
|
|
case errors.Is(err, hypervisor.ErrUnsupported):
|
|
return http.StatusNotImplemented, "unsupported", "operation is not supported"
|
|
case errors.Is(err, hypervisor.ErrInvalid):
|
|
return http.StatusUnprocessableEntity, "invalid_request", "request is invalid"
|
|
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
|
|
return http.StatusServiceUnavailable, "backend_timeout", "backend operation did not complete"
|
|
default:
|
|
return http.StatusServiceUnavailable, "backend_error", "backend operation failed"
|
|
}
|
|
}
|
|
|
|
func statusForKind(kind error) int {
|
|
switch kind {
|
|
case hypervisor.ErrNotFound:
|
|
return http.StatusNotFound
|
|
case hypervisor.ErrConflict:
|
|
return http.StatusConflict
|
|
case hypervisor.ErrUnsupported:
|
|
return http.StatusNotImplemented
|
|
case hypervisor.ErrInvalid:
|
|
return http.StatusUnprocessableEntity
|
|
default:
|
|
return http.StatusServiceUnavailable
|
|
}
|
|
}
|
|
|
|
func writeOperationFailure(w http.ResponseWriter, record operation.Record, replayed bool) {
|
|
status := http.StatusServiceUnavailable
|
|
if record.Error != nil {
|
|
switch record.Error.Code {
|
|
case "unsupported":
|
|
status = http.StatusNotImplemented
|
|
case "not_found":
|
|
status = http.StatusNotFound
|
|
case "conflict":
|
|
status = http.StatusConflict
|
|
case "force_required", "force_delete_not_authorized":
|
|
status = http.StatusConflict
|
|
default:
|
|
if strings.HasSuffix(record.Error.Code, "_not_found") {
|
|
status = http.StatusNotFound
|
|
} else if strings.HasPrefix(record.Error.Code, "invalid_") {
|
|
status = http.StatusUnprocessableEntity
|
|
}
|
|
}
|
|
}
|
|
writeJSON(w, status, operationWire(record, replayed))
|
|
}
|
|
|
|
func methodOrNotFound(w http.ResponseWriter, r *http.Request, exists bool, methods ...string) {
|
|
if !exists {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
return
|
|
}
|
|
w.Header().Set("Allow", strings.Join(methods, ", "))
|
|
writePreDispatchError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method is not allowed")
|
|
}
|
|
|
|
func writePreDispatchError(w http.ResponseWriter, status int, code, message string) {
|
|
writeJSON(w, status, errorEnvelope{Error: errorBody{
|
|
Code: code, Message: message, Retryable: retryableError(status, code),
|
|
DispatchState: "not_dispatched",
|
|
}})
|
|
}
|
|
|
|
func writeError(w http.ResponseWriter, status int, code, message string, record *operation.Record) {
|
|
writeJSON(w, status, errorEnvelope{Error: errorBody{
|
|
Code: code, Message: message, Retryable: retryableError(status, code), Operation: record,
|
|
}})
|
|
}
|
|
|
|
func retryableError(status int, code string) bool {
|
|
return false
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, value any) {
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(value)
|
|
}
|
|
|
|
func setResponseHeaders(w http.ResponseWriter) {
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
}
|