You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1817 lines
63 KiB
Go
1817 lines
63 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/subtle"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
consolebroker "git.luoli.live/root/virt/agent/console"
|
|
"git.luoli.live/root/virt/agent/hypervisor"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
)
|
|
|
|
const Version = "0.1.0"
|
|
|
|
var (
|
|
apiUUIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$`)
|
|
resourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`)
|
|
operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`)
|
|
stableCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
|
|
sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
|
catalogIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
|
sourceTokenPattern = regexp.MustCompile(`^[A-Za-z0-9._~-]{32,512}$`)
|
|
)
|
|
|
|
const (
|
|
catalogImportTimeout = 2 * time.Hour
|
|
maxAsyncImports = 2
|
|
)
|
|
|
|
type Server struct {
|
|
Authenticator *auth.Authenticator
|
|
Journal *operation.Journal
|
|
Hypervisor hypervisor.Hypervisor
|
|
Console *consolebroker.Broker
|
|
MaxBodyBytes int64
|
|
OperationTimeout time.Duration
|
|
asyncMu sync.Mutex
|
|
asyncRunning map[string]struct{}
|
|
asyncSlots chan struct{}
|
|
asyncWG sync.WaitGroup
|
|
lifecycleMu sync.Mutex
|
|
closing bool
|
|
requestWG sync.WaitGroup
|
|
backgroundCtx context.Context
|
|
backgroundCancel context.CancelFunc
|
|
}
|
|
|
|
type errorBody struct {
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
Retryable bool `json:"retryable"`
|
|
DispatchState string `json:"dispatch_state,omitempty"`
|
|
Operation *operation.Record `json:"operation,omitempty"`
|
|
}
|
|
|
|
type errorEnvelope struct {
|
|
Error errorBody `json:"error"`
|
|
}
|
|
|
|
type operationResponse struct {
|
|
OperationID string `json:"operation_id"`
|
|
Status operation.State `json:"status"`
|
|
Replayed bool `json:"replayed"`
|
|
Result json.RawMessage `json:"result,omitempty"`
|
|
Error *operation.PublicError `json:"error,omitempty"`
|
|
CauseCode string `json:"cause_code,omitempty"`
|
|
Retryable bool `json:"retryable"`
|
|
}
|
|
|
|
type abortOperationJobRequest struct {
|
|
InspectionToken string `json:"inspection_token"`
|
|
}
|
|
|
|
type cleanupOperationRequest struct {
|
|
PlanID string `json:"plan_id"`
|
|
}
|
|
|
|
type confirmOperationInspectionRequest struct {
|
|
InspectionToken string `json:"inspection_token"`
|
|
}
|
|
|
|
type confirmOperationInspectionResponse struct {
|
|
Operation operation.Record `json:"operation"`
|
|
Inspection hypervisor.OperationInspection `json:"inspection"`
|
|
}
|
|
|
|
type inspectionInconclusiveEnvelope struct {
|
|
Error errorBody `json:"error"`
|
|
Inspection hypervisor.OperationInspection `json:"inspection"`
|
|
}
|
|
|
|
func NewServer(
|
|
authenticator *auth.Authenticator,
|
|
journal *operation.Journal,
|
|
hv hypervisor.Hypervisor,
|
|
console *consolebroker.Broker,
|
|
maxBodyBytes int64,
|
|
operationTimeout time.Duration,
|
|
) (*Server, error) {
|
|
if authenticator == nil || journal == nil || hv == nil || console == nil {
|
|
return nil, errors.New("authenticator, journal, hypervisor, and console broker are required")
|
|
}
|
|
if maxBodyBytes < 1024 {
|
|
return nil, errors.New("max body size must be at least 1024 bytes")
|
|
}
|
|
if operationTimeout <= 0 {
|
|
return nil, errors.New("operation timeout must be positive")
|
|
}
|
|
backgroundCtx, backgroundCancel := context.WithCancel(context.Background())
|
|
return &Server{
|
|
Authenticator: authenticator, Journal: journal, Hypervisor: hv, Console: console,
|
|
MaxBodyBytes: maxBodyBytes, OperationTimeout: operationTimeout,
|
|
asyncRunning: make(map[string]struct{}), asyncSlots: make(chan struct{}, maxAsyncImports),
|
|
backgroundCtx: backgroundCtx, backgroundCancel: backgroundCancel,
|
|
}, nil
|
|
}
|
|
|
|
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
setResponseHeaders(w)
|
|
if !s.beginRequest() {
|
|
writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down")
|
|
return
|
|
}
|
|
defer s.requestWG.Done()
|
|
if !strings.HasPrefix(r.URL.Path, "/v1/") || strings.HasSuffix(r.URL.Path, "/") {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
return
|
|
}
|
|
body, err := readBoundedBody(w, r, s.MaxBodyBytes)
|
|
if err != nil {
|
|
var tooLarge *http.MaxBytesError
|
|
if errors.As(err, &tooLarge) {
|
|
writePreDispatchError(w, http.StatusRequestEntityTooLarge, "body_too_large", "request body exceeds configured limit")
|
|
} else {
|
|
writePreDispatchError(w, http.StatusBadRequest, "invalid_body", "could not read request body")
|
|
}
|
|
return
|
|
}
|
|
actor, err := s.Authenticator.Authenticate(r, body)
|
|
if err != nil {
|
|
if errors.Is(err, auth.ErrNonceStore) {
|
|
writePreDispatchError(w, http.StatusServiceUnavailable, "nonce_store_unavailable", "authentication state is unavailable")
|
|
return
|
|
}
|
|
if errors.Is(err, auth.ErrReplay) {
|
|
// The matching nonce may belong to an earlier request that already
|
|
// crossed the durable operation boundary. A replay rejection is not
|
|
// proof that the operation was never dispatched.
|
|
writeError(w, http.StatusUnauthorized, "replayed_request", "request nonce has already been used", nil)
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusUnauthorized, "unauthorized", "request authentication failed")
|
|
return
|
|
}
|
|
s.route(w, r, body, actor)
|
|
}
|
|
|
|
func (s *Server) route(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor) {
|
|
segments := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/")
|
|
if len(segments) < 2 || segments[0] != "v1" {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
return
|
|
}
|
|
switch segments[1] {
|
|
case "health":
|
|
if len(segments) != 2 || r.Method != http.MethodGet {
|
|
methodOrNotFound(w, r, len(segments) == 2, http.MethodGet)
|
|
return
|
|
}
|
|
if err := s.Hypervisor.Ping(r.Context()); err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"status": "ok", "service": "virt-host-agent", "version": Version})
|
|
case "capabilities":
|
|
if len(segments) != 2 || r.Method != http.MethodGet {
|
|
methodOrNotFound(w, r, len(segments) == 2, http.MethodGet)
|
|
return
|
|
}
|
|
result, err := s.Hypervisor.Capabilities(r.Context())
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, result)
|
|
case "storage":
|
|
s.routeStorage(w, r, segments, body, actor)
|
|
case "vms":
|
|
s.routeVMs(w, r, segments, body, actor)
|
|
case "templates":
|
|
s.routeTemplates(w, r, segments, body, actor)
|
|
case "rescue-templates":
|
|
s.routeRescueTemplates(w, r, segments, body, actor)
|
|
case "user-isos":
|
|
s.routeUserISOs(w, r, segments, body, actor)
|
|
case "operations":
|
|
s.routeOperations(w, r, segments, body, actor)
|
|
default:
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
}
|
|
|
|
func (s *Server) routeStorage(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if len(segments) != 3 || segments[2] != "configuration" {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
return
|
|
}
|
|
if !requireSuperAdminManage(w, actor) {
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
methodOrNotFound(w, r, true, http.MethodPost)
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.StorageConfigurationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateStorageConfigurationRequest(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
digests := hypervisor.StorageConfigurationPathDigests(request)
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedStorageConfiguration,
|
|
ManagedPathDigest: digests[hypervisor.StorageRoleManaged],
|
|
TemplatePathDigest: digests[hypervisor.StorageRoleTemplate],
|
|
SeedPathDigest: digests[hypervisor.StorageRoleSeed],
|
|
UserISOPathDigest: digests[hypervisor.StorageRoleUserISO],
|
|
RescuePathDigest: digests[hypervisor.StorageRoleRescue],
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, "host-storage", "configure-storage", expected, true, http.StatusOK,
|
|
hypervisor.StorageConfigurationAdmission(),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ConfigureStorage(ctx, request) })
|
|
}
|
|
|
|
func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if len(segments) == 2 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListVMs(r.Context())
|
|
writeNamedList(w, "vms", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.CreateVMRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
normalized, err := hypervisor.NormalizeStartupScript(request.StartupScript)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
request.StartupScript = normalized
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
if request.UUID == "" {
|
|
request.UUID = operationUUID(resolvedOperationID)
|
|
}
|
|
request.DomainName, err = hypervisor.LibvirtDomainName(actor.UserID, request.UUID)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateCreateVM(request); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, request.UUID, "create-vm", true, http.StatusCreated,
|
|
hypervisor.CreateVMAdmission(request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.CreateVM(ctx, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) < 3 || !apiUUIDPattern.MatchString(segments[2]) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_uuid", "VM UUID must be canonical")
|
|
return
|
|
}
|
|
uuid := strings.ToLower(segments[2])
|
|
if len(segments) == 3 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
case http.MethodDelete:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.DeleteVMRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !request.RemoveManagedDisks {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true")
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-vm", true, http.StatusOK,
|
|
hypervisor.DeleteVMAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.DeleteVM(ctx, uuid, request)
|
|
})
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete)
|
|
}
|
|
return
|
|
}
|
|
|
|
switch segments[3] {
|
|
case "force-delete":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireSuperAdminOperationManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ForceDeleteVMRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !request.RemoveManagedDisks {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true")
|
|
return
|
|
}
|
|
if !request.AckDataLoss {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true")
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
if !operationIDPattern.MatchString(request.PriorOperationID) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_prior_operation_id", "prior_operation_id must contain 8-80 URL-safe characters")
|
|
return
|
|
}
|
|
if operationID == request.PriorOperationID {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "force_operation_id_reused", "force delete requires a new operation_id")
|
|
return
|
|
}
|
|
existingReplay := false
|
|
existing, existingErr := s.Journal.Get(operationID)
|
|
if existingErr == nil {
|
|
existingReplay = existing.Action == "force-delete-vm" && existing.ResourceID == uuid &&
|
|
existing.SupersedesOperationID == request.PriorOperationID
|
|
} else if !errors.Is(existingErr, operation.ErrNotFound) {
|
|
writeMappedError(w, existingErr, nil)
|
|
return
|
|
}
|
|
if !existingReplay {
|
|
prior, priorErr := s.Journal.Get(request.PriorOperationID)
|
|
priorActionAllowed := priorErr == nil && (prior.Action == "delete-vm" || prior.Action == "service-suspend")
|
|
if !priorActionAllowed || prior.ResourceID != uuid ||
|
|
prior.State != operation.StateFailed || prior.Error == nil || prior.Error.Code != "force_required" {
|
|
writePreDispatchError(w, http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful stop to require force")
|
|
return
|
|
}
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "force-delete-vm", nil,
|
|
true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.ForceDeleteVM(ctx, uuid, request)
|
|
}, request.PriorOperationID)
|
|
return
|
|
}
|
|
case "state":
|
|
if len(segments) == 4 && r.Method == http.MethodGet {
|
|
result, err := s.Hypervisor.GetVMState(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
return
|
|
}
|
|
case "metadata":
|
|
if len(segments) == 4 && r.Method == http.MethodGet {
|
|
result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
return
|
|
}
|
|
case "actions":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.PowerRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !validPowerAction(request.Action) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_power_action", "action must be start, shutdown, reboot, reset, or destroy")
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "power-"+string(request.Action), true, http.StatusOK,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.Power(ctx, uuid, request) })
|
|
return
|
|
}
|
|
case "settings":
|
|
if len(segments) == 4 && (r.Method == http.MethodPatch || r.Method == http.MethodPost) {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.SettingsRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.Hostname == nil && request.RootPassword == nil && request.SSHPublicKey == nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "empty_settings", "at least one setting is required")
|
|
return
|
|
}
|
|
if !request.ConfirmRestart {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true")
|
|
return
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedUpdateSettings, HostnameChanged: request.Hostname != nil,
|
|
RootPasswordChanged: request.RootPassword != nil, SSHKeyChanged: request.SSHPublicKey != nil,
|
|
}
|
|
if request.Hostname != nil {
|
|
expected.Hostname = *request.Hostname
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "update-settings", expected, false, http.StatusOK,
|
|
hypervisor.SeedMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateSettings(ctx, uuid, request) })
|
|
return
|
|
}
|
|
case "network":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ChangeNetworkRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !request.ConfirmRestart {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true")
|
|
return
|
|
}
|
|
if err := hypervisor.ValidatePublicNetworkIntent(
|
|
request.NetworkSelector, request.PublicPoolKey, request.PublicPoolCIDR,
|
|
request.PublicIPCIDR, request.Gateway, request.DNSServers,
|
|
); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
source, err := s.Hypervisor.ResolveNetworkSelector(r.Context(), request.NetworkSelector)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedChangeNetwork, NetworkSelector: request.NetworkSelector,
|
|
NetworkSourceType: source.Type, NetworkSourceName: source.Source,
|
|
PublicPoolKey: request.PublicPoolKey, PublicPoolCIDR: request.PublicPoolCIDR,
|
|
PublicIPCIDR: request.PublicIPCIDR, Gateway: request.Gateway,
|
|
DNSServers: append([]string(nil), request.DNSServers...),
|
|
}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "change-network", expected, true, http.StatusAccepted,
|
|
hypervisor.ChangeNetworkAdmission(uuid, request.PublicIPCIDR),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ChangeNetwork(ctx, uuid, request) })
|
|
return
|
|
}
|
|
if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.NetworkStateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
enabled := segments[4] == "resume"
|
|
action := "network-" + segments[4]
|
|
expected := &operation.ExpectedState{Kind: operation.ExpectedNetworkState, Enabled: enabled}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, action, expected, false, http.StatusOK,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
err := s.Hypervisor.SetNetworkState(ctx, uuid, enabled, request)
|
|
return map[string]bool{"network_enabled": enabled}, err
|
|
})
|
|
return
|
|
}
|
|
case "service":
|
|
if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.ServiceStateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
enabled := segments[4] == "resume"
|
|
expected := &operation.ExpectedState{Kind: operation.ExpectedServiceState, Enabled: enabled, StartIfSuspended: request.StartIfSuspended}
|
|
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "service-"+segments[4], expected, true, http.StatusOK,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.SetServiceState(ctx, uuid, enabled, request)
|
|
})
|
|
return
|
|
}
|
|
case "reinstall":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ReinstallRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateTemplateSnapshot(request.TemplateID, request.TemplateRevision, request.TemplateGenerationToken); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "reinstall-vm", true, http.StatusAccepted,
|
|
hypervisor.ReinstallVMAdmission(uuid, request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.Reinstall(ctx, uuid, request) })
|
|
return
|
|
}
|
|
case "metrics":
|
|
if len(segments) == 4 && r.Method == http.MethodGet {
|
|
result, err := s.Hypervisor.GetMetrics(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
return
|
|
}
|
|
case "traffic":
|
|
if len(segments) == 5 && segments[4] == "daily" && r.Method == http.MethodGet {
|
|
days, err := strconv.Atoi(r.URL.Query().Get("days"))
|
|
if err != nil || days < 1 || days > 30 {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_days", "days must be between 1 and 30")
|
|
return
|
|
}
|
|
result, err := s.Hypervisor.GetDailyTraffic(r.Context(), uuid, days)
|
|
writeNamedList(w, "traffic", result, err)
|
|
return
|
|
}
|
|
case "console-ticket":
|
|
if len(segments) == 4 && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
target, err := s.Hypervisor.GetConsole(r.Context(), uuid)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if target.Protocol != "vnc" {
|
|
writeError(w, http.StatusNotImplemented, "unsupported_console", "only VNC console targets are supported", nil)
|
|
return
|
|
}
|
|
ticket, err := s.Console.Issue(target.Address)
|
|
if err != nil {
|
|
writeError(w, http.StatusServiceUnavailable, "console_unavailable", "console target is unavailable", nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusCreated, ticket)
|
|
return
|
|
}
|
|
case "snapshots":
|
|
s.routeSnapshots(w, r, segments, body, actor, uuid)
|
|
return
|
|
case "firewall":
|
|
if len(segments) == 4 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.GetFirewall(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
case http.MethodPut:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.FirewallPolicy](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
operationID := request.OperationID
|
|
request, err := hypervisor.NormalizeFirewallPolicy(request)
|
|
if err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
expected := &operation.ExpectedState{Kind: operation.ExpectedFirewall, FirewallRevision: request.Revision, FirewallDigest: request.PolicyDigest}
|
|
s.runMutationExpected(w, r, body, actor, operationID, uuid, "apply-firewall", expected, false, http.StatusOK,
|
|
hypervisor.FirewallMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyFirewall(ctx, uuid, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPut)
|
|
}
|
|
return
|
|
}
|
|
case "rescue":
|
|
s.routeRescue(w, r, segments, body, actor, uuid)
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) routeSnapshots(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) {
|
|
if len(segments) == 4 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListSnapshots(r.Context(), uuid)
|
|
writeNamedList(w, "snapshots", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.SnapshotRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.SnapshotKey == "" {
|
|
request.SnapshotKey = request.SnapshotID
|
|
}
|
|
if !catalogIDPattern.MatchString(request.SnapshotKey) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot", "snapshot key is invalid")
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
action := "create-snapshot:" + request.SnapshotKey + ":" + request.ConsistencyMode
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusCreated,
|
|
hypervisor.CreateSnapshotAdmission(uuid, request.SnapshotKey),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.CreateSnapshot(ctx, uuid, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) < 5 || !resourceIDPattern.MatchString(segments[4]) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot_id", "snapshot ID is invalid")
|
|
return
|
|
}
|
|
snapshotID := segments[4]
|
|
if len(segments) == 5 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
snapshots, err := s.Hypervisor.ListSnapshots(r.Context(), uuid)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
for _, snapshot := range snapshots {
|
|
if snapshot.ID == snapshotID {
|
|
writeJSON(w, http.StatusOK, snapshot)
|
|
return
|
|
}
|
|
}
|
|
writeError(w, http.StatusNotFound, "snapshot_not_found", "snapshot was not found", nil)
|
|
case http.MethodDelete:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-snapshot:"+snapshotID, true, http.StatusOK,
|
|
hypervisor.DeleteSnapshotAdmission(uuid, snapshotID),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.DeleteSnapshot(ctx, uuid, snapshotID, request)
|
|
})
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) == 6 && segments[5] == "restore" && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
request.OperationID = resolvedOperationID
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "restore-snapshot:"+snapshotID, true, http.StatusAccepted,
|
|
hypervisor.RestoreSnapshotAdmission(uuid, snapshotID, request.OperationID),
|
|
func(ctx context.Context) (any, error) {
|
|
return s.Hypervisor.RestoreSnapshot(ctx, uuid, snapshotID, request)
|
|
})
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) routeRescue(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) {
|
|
if len(segments) == 4 && r.Method == http.MethodGet {
|
|
result, err := s.Hypervisor.GetRescue(r.Context(), uuid)
|
|
writeResult(w, result, err)
|
|
return
|
|
}
|
|
if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "enter" || segments[4] == "exit" || segments[4] == "force-start") {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
if segments[4] == "enter" {
|
|
request, ok := decodeRequired[hypervisor.RescueRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !catalogIDPattern.MatchString(request.TemplateID) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_template_id", "rescue template id is invalid")
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "enter-rescue", true, http.StatusAccepted,
|
|
hypervisor.EnterRescueAdmission(uuid, request.TemplateID),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.EnterRescue(ctx, uuid, request) })
|
|
} else if segments[4] == "force-start" {
|
|
request, ok := decodeRequired[hypervisor.ForceRescueStartRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !request.AckDataLoss {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true")
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, "force-start-rescue", true, http.StatusAccepted,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ForceStartRescue(ctx, uuid, request) })
|
|
} else {
|
|
request, ok := decodeRequired[hypervisor.ExitRescueRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.NetworkSuspended == nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "network_state_required", "network_suspended must be provided")
|
|
return
|
|
}
|
|
action := "exit-rescue:connected"
|
|
if *request.NetworkSuspended {
|
|
action = "exit-rescue:suspended"
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusAccepted,
|
|
hypervisor.DomainMutationAdmission(uuid),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ExitRescue(ctx, uuid, request) })
|
|
}
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) routeTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if !actor.IsAdmin && !actor.IsSuperAdmin {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required")
|
|
return
|
|
}
|
|
if len(segments) == 2 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListTemplates(r.Context())
|
|
writeNamedList(w, "templates", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ImportTemplateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := hypervisor.ValidateTemplateSnapshot(request.ID, request.Revision, request.GenerationToken); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runAsyncMutation(w, r, body, actor, request.OperationID, "template:"+request.ID, "import-template", false,
|
|
hypervisor.ImportTemplateAdmission(request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ImportTemplate(ctx, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) == 3 && resourceIDPattern.MatchString(segments[2]) {
|
|
templateID := segments[2]
|
|
switch r.Method {
|
|
case http.MethodPatch:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.UpdateTemplateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if strings.TrimSpace(request.Name) == "" || len(request.Name) > 120 {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_template_name", "template name must contain 1-120 characters")
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "update-template", false, http.StatusOK,
|
|
hypervisor.UpdateTemplateAdmission(templateID, request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateTemplate(ctx, templateID, request) })
|
|
case http.MethodDelete:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.MutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "delete-template", true, http.StatusOK,
|
|
hypervisor.DeleteTemplateAdmission(templateID),
|
|
func(ctx context.Context) (any, error) {
|
|
err := s.Hypervisor.DeleteTemplate(ctx, templateID)
|
|
return catalogDeleteResult("template", templateID, 0, err == nil), err
|
|
})
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodPatch, http.MethodDelete)
|
|
}
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) routeRescueTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if !actor.IsAdmin && !actor.IsSuperAdmin {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required")
|
|
return
|
|
}
|
|
if len(segments) == 2 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListRescueTemplates(r.Context())
|
|
writeNamedList(w, "templates", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ImportRescueTemplateRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
if request.ID == "" {
|
|
request.ID = operationCatalogID(operationID)
|
|
}
|
|
if !catalogIDPattern.MatchString(request.ID) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_rescue_template_id", "id must contain 32 lowercase hexadecimal characters")
|
|
return
|
|
}
|
|
if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runAsyncMutation(w, r, body, actor, request.OperationID, "rescue-template:"+request.ID, "import-rescue-template", false,
|
|
hypervisor.ImportRescueTemplateAdmission(request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ImportRescueTemplate(ctx, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.MutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
id := segments[2]
|
|
s.runMutation(w, r, body, actor, request.OperationID, "rescue-template:"+id, "delete-rescue-template", true, http.StatusOK,
|
|
hypervisor.DeleteRescueTemplateAdmission(id),
|
|
func(ctx context.Context) (any, error) {
|
|
err := s.Hypervisor.DeleteRescueTemplate(ctx, id)
|
|
return catalogDeleteResult("rescue_template", id, 0, err == nil), err
|
|
})
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func (s *Server) routeUserISOs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
ownerID, err := ownerFromRequest(r, actor)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", err.Error())
|
|
return
|
|
}
|
|
if len(segments) == 2 {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
result, err := s.Hypervisor.ListUserISOs(r.Context(), ownerID)
|
|
writeNamedList(w, "user_isos", result, err)
|
|
case http.MethodPost:
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[hypervisor.ImportUserISORequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.OwnerID == 0 {
|
|
request.OwnerID = ownerID
|
|
}
|
|
if request.OwnerID != ownerID && !actor.IsAdmin && !actor.IsSuperAdmin {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "cannot import ISO for another owner")
|
|
return
|
|
}
|
|
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
if request.ID == "" {
|
|
request.ID = operationCatalogID(operationID)
|
|
}
|
|
if !catalogIDPattern.MatchString(request.ID) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_user_iso_id", "id must contain 32 lowercase hexadecimal characters")
|
|
return
|
|
}
|
|
if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil {
|
|
writePreDispatchMappedError(w, err)
|
|
return
|
|
}
|
|
s.runAsyncMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", request.OwnerID, request.ID), "import-user-iso", false,
|
|
hypervisor.ImportUserISOAdmission(request),
|
|
func(ctx context.Context) (any, error) { return s.Hypervisor.ImportUserISO(ctx, request) })
|
|
default:
|
|
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
|
|
}
|
|
return
|
|
}
|
|
if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeOptional[hypervisor.MutationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
isoID := segments[2]
|
|
s.runMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", ownerID, isoID), "delete-user-iso", true, http.StatusOK,
|
|
hypervisor.DeleteUserISOAdmission(ownerID, isoID),
|
|
func(ctx context.Context) (any, error) {
|
|
err := s.Hypervisor.DeleteUserISO(ctx, isoID, ownerID)
|
|
return catalogDeleteResult("user_iso", isoID, ownerID, err == nil), err
|
|
})
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func catalogDeleteResult(resourceType, id string, ownerID int64, deleted bool) hypervisor.CatalogDeleteResult {
|
|
return hypervisor.CatalogDeleteResult{
|
|
ResourceType: resourceType, ID: id, OwnerID: ownerID, Deleted: deleted,
|
|
ManagedImageAbsent: deleted, ManagedMetadataAbsent: deleted, ManagedArtifactsAbsent: deleted,
|
|
}
|
|
}
|
|
|
|
func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
|
|
if len(segments) < 3 || !resourceIDPattern.MatchString(segments[2]) {
|
|
writeError(w, http.StatusNotFound, "operation_not_found", "operation was not found", nil)
|
|
return
|
|
}
|
|
operationID := segments[2]
|
|
if len(segments) == 3 && r.Method == http.MethodGet {
|
|
record, err := s.Journal.Get(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if record.Action == "configure-storage" && !requireSuperAdminManage(w, actor) {
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, record)
|
|
return
|
|
}
|
|
if len(segments) == 4 && segments[3] == "reconcile" && r.Method == http.MethodPost {
|
|
if !requireManage(w, actor) {
|
|
return
|
|
}
|
|
stored, err := s.Journal.Get(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if stored.Action == "configure-storage" && !requireSuperAdminManage(w, actor) {
|
|
return
|
|
}
|
|
if stored.State == operation.StateNeedsAttention {
|
|
writeError(w, http.StatusConflict, "operation_needs_attention",
|
|
"operation requires super-admin inspection and an explicit confirmed action", &stored)
|
|
return
|
|
}
|
|
if operation.RequiresExplicitForceDelete(stored) {
|
|
writeError(w, http.StatusConflict, "force_required",
|
|
"operation requires super-admin inspection and a separately acknowledged force delete or confirmed cleanup", &stored)
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
defer cancel()
|
|
record, err := s.Journal.Reconcile(ctx, operationID, s.Hypervisor.Reconcile)
|
|
logOperationOutcome(record)
|
|
if err != nil {
|
|
if errors.Is(err, operation.ErrOperationUncertain) || operation.IsUncertain(err) {
|
|
writeError(w, http.StatusConflict, "operation_uncertain", "operation remains uncertain", &record)
|
|
return
|
|
}
|
|
writeMappedError(w, err, &record)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, record)
|
|
return
|
|
}
|
|
if len(segments) == 4 && segments[3] == "inspect" && r.Method == http.MethodGet {
|
|
if !requireSuperAdminOperationManage(w, actor) {
|
|
return
|
|
}
|
|
stored, err := s.Journal.Get(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
defer cancel()
|
|
inspection, err := s.Hypervisor.InspectOperation(ctx, stored)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, inspection)
|
|
return
|
|
}
|
|
if len(segments) == 4 && segments[3] == "confirm-inspection" && r.Method == http.MethodPost {
|
|
if !requireSuperAdminOperationManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[confirmOperationInspectionRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !sha256Pattern.MatchString(request.InspectionToken) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_inspection_token", "inspection_token must be a lowercase SHA-256 digest")
|
|
return
|
|
}
|
|
stored, err := s.Journal.Get(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
defer cancel()
|
|
fresh := inconclusiveOperationInspection(stored)
|
|
inspected := false
|
|
record, err := s.Journal.Reconcile(ctx, operationID, func(callCtx context.Context, current operation.Record) (operation.Resolution, error) {
|
|
inspected = true
|
|
observed, inspectErr := s.Hypervisor.InspectOperation(callCtx, current)
|
|
if inspectErr != nil {
|
|
return operation.Resolution{}, operation.ErrInspectionInconclusive
|
|
}
|
|
fresh = observed
|
|
if !matchingInspectionConfirmation(request.InspectionToken, observed) {
|
|
return operation.Resolution{}, operation.ErrInspectionInconclusive
|
|
}
|
|
resolution := *observed.ReadOnlyResolution
|
|
resolution.Result = append([]byte(nil), observed.ReadOnlyResolution.Result...)
|
|
return resolution, nil
|
|
})
|
|
if !inspected && err == nil {
|
|
observed, inspectErr := s.Hypervisor.InspectOperation(ctx, record)
|
|
if inspectErr == nil {
|
|
fresh = observed
|
|
}
|
|
if inspectErr != nil || !matchingInspectionConfirmation(request.InspectionToken, fresh) {
|
|
err = operation.ErrInspectionInconclusive
|
|
}
|
|
}
|
|
if errors.Is(err, operation.ErrInspectionInconclusive) {
|
|
writeInspectionInconclusive(w, fresh)
|
|
return
|
|
}
|
|
if err != nil {
|
|
writeMappedError(w, err, &record)
|
|
return
|
|
}
|
|
fresh.OperationStatus = record.State
|
|
writeJSON(w, http.StatusOK, confirmOperationInspectionResponse{Operation: record, Inspection: fresh})
|
|
return
|
|
}
|
|
if len(segments) == 4 && segments[3] == "abort-job" && r.Method == http.MethodPost {
|
|
if !requireSuperAdminOperationManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[abortOperationJobRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !sha256Pattern.MatchString(request.InspectionToken) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_inspection_token", "inspection_token must be a lowercase SHA-256 digest")
|
|
return
|
|
}
|
|
stored, err := s.Journal.Get(operationID)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
defer cancel()
|
|
inspection, err := s.Hypervisor.AbortOperationJob(ctx, stored, request.InspectionToken)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, inspection)
|
|
return
|
|
}
|
|
if len(segments) == 4 && segments[3] == "cleanup" && r.Method == http.MethodPost {
|
|
if !requireSuperAdminOperationManage(w, actor) {
|
|
return
|
|
}
|
|
request, ok := decodeRequired[cleanupOperationRequest](w, body)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !sha256Pattern.MatchString(request.PlanID) {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_cleanup_plan", "plan_id must be a lowercase SHA-256 digest")
|
|
return
|
|
}
|
|
if _, err := s.Journal.Get(operationID); err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
defer cancel()
|
|
record, err := s.Journal.Reconcile(ctx, operationID, func(callCtx context.Context, current operation.Record) (operation.Resolution, error) {
|
|
return s.Hypervisor.CleanupOperation(callCtx, current, request.PlanID)
|
|
})
|
|
logOperationOutcome(record)
|
|
if err != nil {
|
|
if errors.Is(err, operation.ErrOperationUncertain) || operation.IsUncertain(err) {
|
|
writeError(w, http.StatusConflict, "operation_uncertain", "operation remains uncertain", nil)
|
|
return
|
|
}
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
inspection, err := s.Hypervisor.InspectOperation(ctx, record)
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, inspection)
|
|
return
|
|
}
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
}
|
|
|
|
func matchingInspectionConfirmation(expected string, inspection hypervisor.OperationInspection) bool {
|
|
return inspection.Confirmation.Available && inspection.ReadOnlyResolution != nil &&
|
|
inspection.ReadOnlyResolution.Resolved &&
|
|
subtle.ConstantTimeCompare([]byte(expected), []byte(inspection.Confirmation.InspectionToken)) == 1
|
|
}
|
|
|
|
func inconclusiveOperationInspection(record operation.Record) hypervisor.OperationInspection {
|
|
return hypervisor.OperationInspection{
|
|
OperationID: record.OperationID, OperationStatus: record.State,
|
|
CheckedAt: time.Now().UTC().Unix(), Outcome: hypervisor.InspectionOutcomeUncertain,
|
|
SummaryCode: hypervisor.InspectionSummaryIncomplete,
|
|
}
|
|
}
|
|
|
|
func writeInspectionInconclusive(w http.ResponseWriter, inspection hypervisor.OperationInspection) {
|
|
writeJSON(w, http.StatusConflict, inspectionInconclusiveEnvelope{
|
|
Error: errorBody{
|
|
Code: "inspection_inconclusive", Message: "fresh read-only evidence does not match the confirmed terminal inspection",
|
|
Retryable: false,
|
|
},
|
|
Inspection: inspection,
|
|
})
|
|
}
|
|
|
|
func (s *Server) runMutation(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
body []byte,
|
|
actor auth.Actor,
|
|
bodyOperationID, resourceID, action string,
|
|
destructive bool,
|
|
successStatus int,
|
|
admission hypervisor.MutationAdmission,
|
|
call func(context.Context) (any, error),
|
|
) {
|
|
s.runMutationExpected(w, r, body, actor, bodyOperationID, resourceID, action, nil, destructive, successStatus, admission, call)
|
|
}
|
|
|
|
func (s *Server) runMutationExpected(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
body []byte,
|
|
actor auth.Actor,
|
|
bodyOperationID, resourceID, action string,
|
|
expectedState *operation.ExpectedState,
|
|
destructive bool,
|
|
successStatus int,
|
|
admission hypervisor.MutationAdmission,
|
|
call func(context.Context) (any, error),
|
|
supersedesOperationID ...string,
|
|
) {
|
|
operationID, err := resolveOperationID(actor.OperationID, bodyOperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
if expectedState != nil {
|
|
copyExpected := *expectedState
|
|
copyExpected.TransitionID = operationID
|
|
copyExpected.DNSServers = append([]string(nil), expectedState.DNSServers...)
|
|
expectedState = ©Expected
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
defer cancel()
|
|
supersedes := ""
|
|
if len(supersedesOperationID) != 0 {
|
|
supersedes = supersedesOperationID[0]
|
|
}
|
|
record, replayed, callErr := s.Journal.Execute(ctx, operation.Spec{
|
|
OperationID: operationID, ResourceID: resourceID, Action: action,
|
|
Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body),
|
|
Destructive: destructive, ExpectedState: expectedState,
|
|
LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO,
|
|
SupersedesOperationID: supersedes,
|
|
}, func(ctx context.Context) (json.RawMessage, error) {
|
|
result, err := call(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
encoded, err := json.Marshal(result)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("encode operation result: %w", err)
|
|
}
|
|
return encoded, nil
|
|
})
|
|
logOperationOutcome(record)
|
|
if errors.Is(callErr, operation.ErrFingerprint) {
|
|
writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil)
|
|
return
|
|
}
|
|
if errors.Is(callErr, operation.ErrOperationStarting) {
|
|
writeOperationStarting(w, record)
|
|
return
|
|
}
|
|
if operation.CanRetryNotDispatched(record) {
|
|
writeNotDispatchedOperationError(w, callErr)
|
|
return
|
|
}
|
|
if callErr != nil {
|
|
if operation.IsUncertain(callErr) {
|
|
writeJSON(w, http.StatusAccepted, operationWire(record, replayed))
|
|
return
|
|
}
|
|
if errors.Is(callErr, operation.ErrOperationUncertain) {
|
|
writeError(w, http.StatusConflict, "operation_uncertain", "operation crossed the dispatch boundary; reconcile it instead of replaying", &record)
|
|
return
|
|
}
|
|
writeMappedError(w, callErr, &record)
|
|
return
|
|
}
|
|
if record.State == operation.StateFailed {
|
|
writeOperationFailure(w, record, replayed)
|
|
return
|
|
}
|
|
if record.State == operation.StateUncertain || record.State == operation.StateDispatching {
|
|
writeError(w, http.StatusConflict, "operation_uncertain", "operation requires reconciliation", &record)
|
|
return
|
|
}
|
|
if record.State == operation.StateNeedsAttention {
|
|
writeError(w, http.StatusConflict, "operation_needs_attention", "operation requires operator inspection", &record)
|
|
return
|
|
}
|
|
if replayed {
|
|
successStatus = http.StatusOK
|
|
}
|
|
writeJSON(w, successStatus, operationWire(record, replayed))
|
|
}
|
|
|
|
func (s *Server) runAsyncMutation(
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
body []byte,
|
|
actor auth.Actor,
|
|
bodyOperationID, resourceID, action string,
|
|
destructive bool,
|
|
admission hypervisor.MutationAdmission,
|
|
call func(context.Context) (any, error),
|
|
) {
|
|
operationID, err := resolveOperationID(actor.OperationID, bodyOperationID)
|
|
if err != nil {
|
|
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
|
|
return
|
|
}
|
|
spec := operation.Spec{
|
|
OperationID: operationID, ResourceID: resourceID, Action: action,
|
|
Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body),
|
|
Destructive: destructive,
|
|
LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO,
|
|
}
|
|
record, replayed, err := s.Journal.Reserve(spec)
|
|
if errors.Is(err, operation.ErrFingerprint) {
|
|
writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil)
|
|
return
|
|
}
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
canStart := record.State == operation.StateReserved || operation.CanRetryNotDispatched(record)
|
|
if canStart && !s.claimAsync(operationID) {
|
|
writeOperationStarting(w, record)
|
|
return
|
|
}
|
|
if canStart {
|
|
select {
|
|
case s.asyncSlots <- struct{}{}:
|
|
default:
|
|
_, failErr := s.Journal.FailNotDispatched(operationID)
|
|
s.releaseAsync(operationID)
|
|
if failErr != nil {
|
|
writeMappedError(w, failErr, &record)
|
|
return
|
|
}
|
|
writeNotDispatchedOperationError(w, operation.ErrBusy)
|
|
return
|
|
}
|
|
workerCtx, cancel := context.WithTimeout(s.backgroundCtx, catalogImportTimeout)
|
|
dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.OperationTimeout)
|
|
started, _, done, startErr := s.Journal.ExecuteBackground(
|
|
dispatchCtx, workerCtx, spec,
|
|
func(ctx context.Context) (json.RawMessage, error) {
|
|
result, callErr := call(ctx)
|
|
if callErr != nil {
|
|
return nil, callErr
|
|
}
|
|
return json.Marshal(result)
|
|
},
|
|
)
|
|
dispatchCancel()
|
|
if startErr != nil || done == nil {
|
|
cancel()
|
|
<-s.asyncSlots
|
|
s.releaseAsync(operationID)
|
|
if startErr != nil {
|
|
if errors.Is(startErr, operation.ErrOperationStarting) {
|
|
writeOperationStarting(w, started)
|
|
return
|
|
}
|
|
if started.State == operation.StateFailed && started.Error != nil && started.Error.Code == "operation_not_dispatched" {
|
|
writeNotDispatchedOperationError(w, startErr)
|
|
return
|
|
}
|
|
writeMappedError(w, startErr, &started)
|
|
return
|
|
}
|
|
record = started
|
|
} else {
|
|
record = started
|
|
s.asyncWG.Add(1)
|
|
go func() {
|
|
defer s.asyncWG.Done()
|
|
defer cancel()
|
|
defer func() { <-s.asyncSlots }()
|
|
defer s.releaseAsync(operationID)
|
|
<-done
|
|
if final, getErr := s.Journal.Get(operationID); getErr == nil {
|
|
logOperationOutcome(final)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
writeJSON(w, http.StatusAccepted, operationWire(record, replayed))
|
|
}
|
|
|
|
func (s *Server) beginRequest() bool {
|
|
s.lifecycleMu.Lock()
|
|
defer s.lifecycleMu.Unlock()
|
|
if s.closing {
|
|
return false
|
|
}
|
|
s.requestWG.Add(1)
|
|
return true
|
|
}
|
|
|
|
// Shutdown stops background imports and waits for every admitted API handler
|
|
// and import watcher. The journal's own workers are drained separately before
|
|
// libvirt or BoltDB are closed.
|
|
func (s *Server) Shutdown(ctx context.Context) error {
|
|
s.lifecycleMu.Lock()
|
|
if !s.closing {
|
|
s.closing = true
|
|
s.backgroundCancel()
|
|
}
|
|
s.lifecycleMu.Unlock()
|
|
if err := waitGroup(ctx, &s.requestWG); err != nil {
|
|
return err
|
|
}
|
|
return waitGroup(ctx, &s.asyncWG)
|
|
}
|
|
|
|
func waitGroup(ctx context.Context, group *sync.WaitGroup) error {
|
|
done := make(chan struct{})
|
|
go func() {
|
|
group.Wait()
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (s *Server) claimAsync(operationID string) bool {
|
|
s.asyncMu.Lock()
|
|
defer s.asyncMu.Unlock()
|
|
if s.asyncRunning == nil {
|
|
s.asyncRunning = make(map[string]struct{})
|
|
}
|
|
if _, exists := s.asyncRunning[operationID]; exists {
|
|
return false
|
|
}
|
|
s.asyncRunning[operationID] = struct{}{}
|
|
return true
|
|
}
|
|
|
|
func (s *Server) releaseAsync(operationID string) {
|
|
s.asyncMu.Lock()
|
|
defer s.asyncMu.Unlock()
|
|
delete(s.asyncRunning, operationID)
|
|
}
|
|
|
|
func operationWire(record operation.Record, replayed bool) operationResponse {
|
|
return operationResponse{
|
|
OperationID: record.OperationID, Status: record.State, Replayed: replayed,
|
|
Result: record.Result, Error: record.Error, CauseCode: record.CauseCode,
|
|
Retryable: record.State == operation.StateUncertain,
|
|
}
|
|
}
|
|
|
|
// Log only bounded identifiers and stable codes. Free-form backend errors,
|
|
// request bodies, guest credentials, and host paths never enter this line.
|
|
func logOperationOutcome(record operation.Record) {
|
|
if record.State != operation.StateFailed && record.State != operation.StateUncertain && record.State != operation.StateNeedsAttention {
|
|
return
|
|
}
|
|
errorCode := ""
|
|
if record.Error != nil && len(record.Error.Code) <= 64 &&
|
|
stableCodePattern.MatchString(record.Error.Code) {
|
|
errorCode = record.Error.Code
|
|
}
|
|
log.Printf("Agent operation outcome operation_id=%s action=%s status=%s error_code=%s cause_code=%s",
|
|
record.OperationID, record.Action, record.State, errorCode, record.CauseCode)
|
|
}
|
|
|
|
func resolveOperationID(actorID, bodyID string) (string, error) {
|
|
if actorID == "" && bodyID == "" {
|
|
return "", errors.New("operation_id is required in X-Virt-Actor or request body")
|
|
}
|
|
if actorID != "" && bodyID != "" && actorID != bodyID {
|
|
return "", errors.New("actor and body operation_id values differ")
|
|
}
|
|
resolved := bodyID
|
|
if actorID != "" {
|
|
resolved = actorID
|
|
}
|
|
if !operationIDPattern.MatchString(resolved) {
|
|
return "", errors.New("operation_id must contain 8-80 URL-safe characters")
|
|
}
|
|
return resolved, nil
|
|
}
|
|
|
|
func operationUUID(operationID string) string {
|
|
digest := sha256.Sum256([]byte("virt-cloud-operation:" + operationID))
|
|
digest[6] = digest[6]&0x0f | 0x50
|
|
digest[8] = digest[8]&0x3f | 0x80
|
|
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
|
|
digest[0:4], digest[4:6], digest[6:8], digest[8:10], digest[10:16])
|
|
}
|
|
|
|
func operationCatalogID(operationID string) string {
|
|
digest := sha256.Sum256([]byte("virt-cloud-catalog:" + operationID))
|
|
return fmt.Sprintf("%x", digest[:16])
|
|
}
|
|
|
|
func validPowerAction(action hypervisor.PowerAction) bool {
|
|
switch action {
|
|
case hypervisor.PowerStart, hypervisor.PowerShutdown, hypervisor.PowerReboot, hypervisor.PowerReset, hypervisor.PowerDestroy:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func validateImportSource(raw, token, digest string) error {
|
|
parsed, err := url.Parse(raw)
|
|
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" {
|
|
return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_url", "source_url must be an HTTPS URL without credentials or fragment", err)
|
|
}
|
|
if digest != "" && !sha256Pattern.MatchString(digest) {
|
|
return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_sha256", "sha256 must be 64 lowercase hexadecimal characters", nil)
|
|
}
|
|
if token != "" && !sourceTokenPattern.MatchString(token) {
|
|
return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_token", "source_token must contain 32-512 URL-safe characters", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ownerFromRequest(r *http.Request, actor auth.Actor) (int64, error) {
|
|
value := r.URL.Query().Get("owner_id")
|
|
if value == "" {
|
|
return actor.UserID, nil
|
|
}
|
|
ownerID, err := strconv.ParseInt(value, 10, 64)
|
|
if err != nil || ownerID <= 0 {
|
|
return 0, errors.New("owner_id is invalid")
|
|
}
|
|
if ownerID != actor.UserID && !actor.IsAdmin && !actor.IsSuperAdmin {
|
|
return 0, errors.New("cannot access another owner's ISO")
|
|
}
|
|
return ownerID, nil
|
|
}
|
|
|
|
func requireManage(w http.ResponseWriter, actor auth.Actor) bool {
|
|
if !actor.CanManage {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to manage this resource")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func requireSuperAdminManage(w http.ResponseWriter, actor auth.Actor) bool {
|
|
if !actor.IsSuperAdmin || !actor.CanManage {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to configure host storage")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func requireSuperAdminOperationManage(w http.ResponseWriter, actor auth.Actor) bool {
|
|
if !actor.IsSuperAdmin || !actor.CanManage {
|
|
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to manage operation recovery")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func decodeRequired[T any](w http.ResponseWriter, body []byte) (T, bool) {
|
|
var value T
|
|
if len(body) == 0 {
|
|
writePreDispatchError(w, http.StatusBadRequest, "invalid_json", "JSON request body is required")
|
|
return value, false
|
|
}
|
|
if err := decodeStrict(body, &value); err != nil {
|
|
writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error())
|
|
return value, false
|
|
}
|
|
return value, true
|
|
}
|
|
|
|
func decodeOptional[T any](w http.ResponseWriter, body []byte) (T, bool) {
|
|
var value T
|
|
if len(body) == 0 {
|
|
return value, true
|
|
}
|
|
if err := decodeStrict(body, &value); err != nil {
|
|
writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error())
|
|
return value, false
|
|
}
|
|
return value, true
|
|
}
|
|
|
|
func decodeStrict(body []byte, target any) error {
|
|
if !utf8.Valid(body) {
|
|
return errors.New("invalid JSON: body must be valid UTF-8")
|
|
}
|
|
decoder := json.NewDecoder(bytes.NewReader(body))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(target); err != nil {
|
|
return fmt.Errorf("invalid JSON: %w", err)
|
|
}
|
|
var extra any
|
|
if err := decoder.Decode(&extra); err != io.EOF {
|
|
return errors.New("invalid JSON: exactly one object is required")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func readBoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, error) {
|
|
if r.Body == nil {
|
|
return []byte{}, nil
|
|
}
|
|
r.Body = http.MaxBytesReader(w, r.Body, limit)
|
|
defer r.Body.Close()
|
|
return io.ReadAll(r.Body)
|
|
}
|
|
|
|
func writeResult(w http.ResponseWriter, value any, err error) {
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, value)
|
|
}
|
|
|
|
func writeNamedList[T any](w http.ResponseWriter, name string, value []T, err error) {
|
|
if err != nil {
|
|
writeMappedError(w, err, nil)
|
|
return
|
|
}
|
|
if value == nil {
|
|
value = make([]T, 0)
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{name: value})
|
|
}
|
|
|
|
func writeMappedError(w http.ResponseWriter, err error, record *operation.Record) {
|
|
status, code, message := classifyError(err)
|
|
writeError(w, status, code, message, record)
|
|
}
|
|
|
|
// writePreDispatchMappedError is only for validation or dependency checks
|
|
// completed before Journal.Execute/Reserve. The explicit marker lets the
|
|
// panel distinguish a proven no-journal outcome from a lost response after a
|
|
// mutation may already have crossed its durable dispatch boundary.
|
|
func writePreDispatchMappedError(w http.ResponseWriter, err error) {
|
|
status, code, message := classifyError(err)
|
|
writePreDispatchError(w, status, code, message)
|
|
}
|
|
|
|
func writeNotDispatchedOperationError(w http.ResponseWriter, err error) {
|
|
status, code, message := classifyError(err)
|
|
writeJSON(w, status, errorEnvelope{Error: errorBody{
|
|
Code: code, Message: message, Retryable: retryableError(status, code),
|
|
DispatchState: "not_dispatched",
|
|
}})
|
|
}
|
|
|
|
// writeOperationStarting reports the active admission owner without exposing a
|
|
// stale failed/not-dispatched record from before that owner began the retry.
|
|
// The projected reserved state tells the panel to keep polling the operation;
|
|
// the journal remains the source of truth and is changed only by that owner.
|
|
func writeOperationStarting(w http.ResponseWriter, record operation.Record) {
|
|
record.State = operation.StateReserved
|
|
record.Result = nil
|
|
record.Error = nil
|
|
record.CauseCode = ""
|
|
record.Tombstone = false
|
|
writeError(w, http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting", &record)
|
|
}
|
|
|
|
func classifyError(err error) (int, string, string) {
|
|
if errors.Is(err, operation.ErrNotFound) {
|
|
return http.StatusNotFound, "operation_not_found", "operation was not found"
|
|
}
|
|
if errors.Is(err, operation.ErrBusy) {
|
|
return http.StatusServiceUnavailable, "agent_busy", "Agent mutation capacity is exhausted"
|
|
}
|
|
if errors.Is(err, operation.ErrOperationStarting) {
|
|
return http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting"
|
|
}
|
|
if errors.Is(err, operation.ErrResourceBlocked) {
|
|
return http.StatusConflict, "resource_operation_blocked", "resource has an unresolved operation; inspect and reconcile it first"
|
|
}
|
|
if errors.Is(err, operation.ErrInvalidSupersession) {
|
|
return http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful operation"
|
|
}
|
|
var backend *hypervisor.Error
|
|
if errors.As(err, &backend) {
|
|
status := statusForKind(backend.Kind)
|
|
code := backend.Code
|
|
if code == "" {
|
|
code = "backend_error"
|
|
}
|
|
message := backend.Message
|
|
if message == "" {
|
|
message = http.StatusText(status)
|
|
}
|
|
return status, code, message
|
|
}
|
|
switch {
|
|
case errors.Is(err, hypervisor.ErrNotFound):
|
|
return http.StatusNotFound, "not_found", "resource was not found"
|
|
case errors.Is(err, hypervisor.ErrConflict):
|
|
return http.StatusConflict, "conflict", "resource conflict"
|
|
case errors.Is(err, hypervisor.ErrUnsupported):
|
|
return http.StatusNotImplemented, "unsupported", "operation is not supported"
|
|
case errors.Is(err, hypervisor.ErrInvalid):
|
|
return http.StatusUnprocessableEntity, "invalid_request", "request is invalid"
|
|
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
|
|
return http.StatusServiceUnavailable, "backend_timeout", "backend operation did not complete"
|
|
default:
|
|
return http.StatusServiceUnavailable, "backend_error", "backend operation failed"
|
|
}
|
|
}
|
|
|
|
func statusForKind(kind error) int {
|
|
switch kind {
|
|
case hypervisor.ErrNotFound:
|
|
return http.StatusNotFound
|
|
case hypervisor.ErrConflict:
|
|
return http.StatusConflict
|
|
case hypervisor.ErrUnsupported:
|
|
return http.StatusNotImplemented
|
|
case hypervisor.ErrInvalid:
|
|
return http.StatusUnprocessableEntity
|
|
default:
|
|
return http.StatusServiceUnavailable
|
|
}
|
|
}
|
|
|
|
func writeOperationFailure(w http.ResponseWriter, record operation.Record, replayed bool) {
|
|
status := http.StatusServiceUnavailable
|
|
if record.Error != nil {
|
|
switch record.Error.Code {
|
|
case "unsupported":
|
|
status = http.StatusNotImplemented
|
|
case "not_found":
|
|
status = http.StatusNotFound
|
|
case "conflict":
|
|
status = http.StatusConflict
|
|
case "force_required", "force_delete_not_authorized":
|
|
status = http.StatusConflict
|
|
default:
|
|
if strings.HasSuffix(record.Error.Code, "_not_found") {
|
|
status = http.StatusNotFound
|
|
} else if strings.HasPrefix(record.Error.Code, "invalid_") {
|
|
status = http.StatusUnprocessableEntity
|
|
}
|
|
}
|
|
}
|
|
writeJSON(w, status, operationWire(record, replayed))
|
|
}
|
|
|
|
func methodOrNotFound(w http.ResponseWriter, r *http.Request, exists bool, methods ...string) {
|
|
if !exists {
|
|
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
|
|
return
|
|
}
|
|
w.Header().Set("Allow", strings.Join(methods, ", "))
|
|
writePreDispatchError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method is not allowed")
|
|
}
|
|
|
|
func writePreDispatchError(w http.ResponseWriter, status int, code, message string) {
|
|
writeJSON(w, status, errorEnvelope{Error: errorBody{
|
|
Code: code, Message: message, Retryable: retryableError(status, code),
|
|
DispatchState: "not_dispatched",
|
|
}})
|
|
}
|
|
|
|
func writeError(w http.ResponseWriter, status int, code, message string, record *operation.Record) {
|
|
writeJSON(w, status, errorEnvelope{Error: errorBody{
|
|
Code: code, Message: message, Retryable: retryableError(status, code), Operation: record,
|
|
}})
|
|
}
|
|
|
|
func retryableError(status int, code string) bool {
|
|
if code == "unsupported" || code == "libvirt_adapter_not_built" {
|
|
return false
|
|
}
|
|
return status >= 500 || code == "operation_uncertain"
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, value any) {
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(value)
|
|
}
|
|
|
|
func setResponseHeaders(w http.ResponseWriter) {
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
}
|