You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/api/server.go

1794 lines
63 KiB
Go

package api
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"sync"
"time"
"unicode/utf8"
"git.luoli.live/root/virt/agent/auth"
consolebroker "git.luoli.live/root/virt/agent/console"
"git.luoli.live/root/virt/agent/hypervisor"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/worker"
)
const Version = "0.1.0"
var (
apiUUIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$`)
resourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`)
operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`)
stableCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
catalogIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
sourceTokenPattern = regexp.MustCompile(`^[A-Za-z0-9._~-]{32,512}$`)
)
const (
catalogImportTimeout = 2 * time.Hour
storageConfigurationWorkerTimeout = 120 * time.Second
defaultAdmissionTimeout = 2 * time.Second
maxAsyncImports = 2
)
type Server struct {
Authenticator *auth.Authenticator
Journal *operation.Journal
Hypervisor hypervisor.Hypervisor
Console *consolebroker.Broker
MaxBodyBytes int64
OperationTimeout time.Duration
AdmissionTimeout time.Duration
MutationExecutor worker.Executor
asyncMu sync.Mutex
asyncRunning map[string]struct{}
asyncSlots chan struct{}
asyncWG sync.WaitGroup
lifecycleMu sync.Mutex
closing bool
requestWG sync.WaitGroup
backgroundCtx context.Context
backgroundCancel context.CancelFunc
}
type trustedWorkerRequestKey struct{}
// SetMutationExecutor enables the clean-break process-isolated mutation path.
// It must be called during startup before request admission begins.
func (s *Server) SetMutationExecutor(executor worker.Executor) error {
if executor == nil {
return errors.New("mutation executor is required")
}
s.MutationExecutor = executor
return nil
}
// ServeTrustedWorkerRequest dispatches a request whose bounded envelope was
// already authenticated and validated by the parent Agent. Hidden workers do
// not have access to the panel secret through HTTP headers and cannot recurse
// because they never install a MutationExecutor.
func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor) {
setResponseHeaders(w)
if !s.beginRequest() {
writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down")
return
}
defer s.requestWG.Done()
if int64(len(body)) > s.MaxBodyBytes || actor.OperationID == "" {
writePreDispatchError(w, http.StatusBadRequest, "invalid_worker_request", "worker request is invalid")
return
}
trusted := context.WithValue(r.Context(), trustedWorkerRequestKey{}, struct{}{})
s.route(w, r.WithContext(trusted), body, actor)
}
type errorBody struct {
Code string `json:"code"`
Message string `json:"message"`
Retryable bool `json:"retryable"`
DispatchState string `json:"dispatch_state,omitempty"`
Operation *operation.Record `json:"operation,omitempty"`
}
type errorEnvelope struct {
Error errorBody `json:"error"`
}
type operationResponse struct {
OperationID string `json:"operation_id"`
Status operation.State `json:"status"`
Replayed bool `json:"replayed"`
Result json.RawMessage `json:"result,omitempty"`
Error *operation.PublicError `json:"error,omitempty"`
CauseCode string `json:"cause_code,omitempty"`
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
SafeToResubmit bool `json:"safe_to_resubmit"`
Retryable bool `json:"retryable"`
}
type fenceResolutionResponse struct {
operation.FenceResolution
Replayed bool `json:"replayed"`
}
func NewServer(
authenticator *auth.Authenticator,
journal *operation.Journal,
hv hypervisor.Hypervisor,
console *consolebroker.Broker,
maxBodyBytes int64,
operationTimeout time.Duration,
) (*Server, error) {
if authenticator == nil || journal == nil || hv == nil || console == nil {
return nil, errors.New("authenticator, journal, hypervisor, and console broker are required")
}
if maxBodyBytes < 1024 {
return nil, errors.New("max body size must be at least 1024 bytes")
}
if operationTimeout <= 0 {
return nil, errors.New("operation timeout must be positive")
}
backgroundCtx, backgroundCancel := context.WithCancel(context.Background())
return &Server{
Authenticator: authenticator, Journal: journal, Hypervisor: hv, Console: console,
MaxBodyBytes: maxBodyBytes, OperationTimeout: operationTimeout,
AdmissionTimeout: min(operationTimeout, defaultAdmissionTimeout),
asyncRunning: make(map[string]struct{}), asyncSlots: make(chan struct{}, maxAsyncImports),
backgroundCtx: backgroundCtx, backgroundCancel: backgroundCancel,
}, nil
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
setResponseHeaders(w)
if !s.beginRequest() {
writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down")
return
}
defer s.requestWG.Done()
if !strings.HasPrefix(r.URL.Path, "/v1/") || strings.HasSuffix(r.URL.Path, "/") {
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
return
}
body, err := readBoundedBody(w, r, s.MaxBodyBytes)
if err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
writePreDispatchError(w, http.StatusRequestEntityTooLarge, "body_too_large", "request body exceeds configured limit")
} else {
writePreDispatchError(w, http.StatusBadRequest, "invalid_body", "could not read request body")
}
return
}
actor, err := s.Authenticator.Authenticate(r, body)
if err != nil {
if errors.Is(err, auth.ErrNonceStore) {
writePreDispatchError(w, http.StatusServiceUnavailable, "nonce_store_unavailable", "authentication state is unavailable")
return
}
if errors.Is(err, auth.ErrReplay) {
// The matching nonce may belong to an earlier request that already
// crossed the durable operation boundary. A replay rejection is not
// proof that the operation was never dispatched.
writeError(w, http.StatusUnauthorized, "replayed_request", "request nonce has already been used", nil)
return
}
writePreDispatchError(w, http.StatusUnauthorized, "unauthorized", "request authentication failed")
return
}
s.route(w, r, body, actor)
}
func (s *Server) route(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor) {
segments := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/")
if len(segments) < 2 || segments[0] != "v1" {
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
return
}
switch segments[1] {
case "health":
if len(segments) != 2 || r.Method != http.MethodGet {
methodOrNotFound(w, r, len(segments) == 2, http.MethodGet)
return
}
if err := s.Hypervisor.Ping(r.Context()); err != nil {
writeMappedError(w, err, nil)
return
}
writeJSON(w, http.StatusOK, map[string]any{"status": "ok", "service": "virt-host-agent", "version": Version})
case "capabilities":
if len(segments) != 2 || r.Method != http.MethodGet {
methodOrNotFound(w, r, len(segments) == 2, http.MethodGet)
return
}
result, err := s.Hypervisor.Capabilities(r.Context())
if err != nil {
writeMappedError(w, err, nil)
return
}
writeJSON(w, http.StatusOK, result)
case "storage":
s.routeStorage(w, r, segments, body, actor)
case "vms":
s.routeVMs(w, r, segments, body, actor)
case "templates":
s.routeTemplates(w, r, segments, body, actor)
case "rescue-templates":
s.routeRescueTemplates(w, r, segments, body, actor)
case "user-isos":
s.routeUserISOs(w, r, segments, body, actor)
case "operations":
s.routeOperations(w, r, segments, body, actor)
default:
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
}
func (s *Server) routeStorage(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
if len(segments) != 3 || segments[2] != "configuration" {
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
return
}
if !requireSuperAdminManage(w, actor) {
return
}
if r.Method != http.MethodPost {
methodOrNotFound(w, r, true, http.MethodPost)
return
}
request, ok := decodeRequired[hypervisor.StorageConfigurationRequest](w, body)
if !ok {
return
}
if err := hypervisor.ValidateStorageConfigurationRequest(request); err != nil {
writePreDispatchMappedError(w, err)
return
}
digests := hypervisor.StorageConfigurationPathDigests(request)
expected := &operation.ExpectedState{
Kind: operation.ExpectedStorageConfiguration,
ManagedPathDigest: digests[hypervisor.StorageRoleManaged],
TemplatePathDigest: digests[hypervisor.StorageRoleTemplate],
SeedPathDigest: digests[hypervisor.StorageRoleSeed],
UserISOPathDigest: digests[hypervisor.StorageRoleUserISO],
RescuePathDigest: digests[hypervisor.StorageRoleRescue],
}
s.runMutationExpected(w, r, body, actor, request.OperationID, "host-storage", "configure-storage", expected, true, http.StatusOK,
hypervisor.StorageConfigurationAdmission(),
func(ctx context.Context) (any, error) { return s.Hypervisor.ConfigureStorage(ctx, request) })
}
func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
if len(segments) == 2 {
switch r.Method {
case http.MethodGet:
result, err := s.Hypervisor.ListVMs(r.Context())
writeNamedList(w, "vms", result, err)
case http.MethodPost:
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.CreateVMRequest](w, body)
if !ok {
return
}
normalized, err := hypervisor.NormalizeStartupScript(request.StartupScript)
if err != nil {
writePreDispatchMappedError(w, err)
return
}
request.StartupScript = normalized
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
request.OperationID = resolvedOperationID
if request.UUID == "" {
request.UUID = operationUUID(resolvedOperationID)
}
request.DomainName, err = hypervisor.LibvirtDomainName(actor.UserID, request.UUID)
if err != nil {
writePreDispatchMappedError(w, err)
return
}
if err := hypervisor.ValidateCreateVM(request); err != nil {
writePreDispatchMappedError(w, err)
return
}
s.runMutation(w, r, body, actor, request.OperationID, request.UUID, "create-vm", true, http.StatusCreated,
hypervisor.CreateVMAdmission(request),
func(ctx context.Context) (any, error) { return s.Hypervisor.CreateVM(ctx, request) })
default:
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
}
return
}
if len(segments) < 3 || !apiUUIDPattern.MatchString(segments[2]) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_uuid", "VM UUID must be canonical")
return
}
uuid := strings.ToLower(segments[2])
if len(segments) == 3 {
switch r.Method {
case http.MethodGet:
result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid)
writeResult(w, result, err)
case http.MethodDelete:
if !requireManage(w, actor) {
return
}
request, ok := decodeOptional[hypervisor.DeleteVMRequest](w, body)
if !ok {
return
}
if !request.RemoveManagedDisks {
writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true")
return
}
s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-vm", true, http.StatusOK,
hypervisor.DeleteVMAdmission(uuid),
func(ctx context.Context) (any, error) {
return s.Hypervisor.DeleteVM(ctx, uuid, request)
})
default:
methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete)
}
return
}
switch segments[3] {
case "force-delete":
if len(segments) == 4 && r.Method == http.MethodPost {
if !requireSuperAdminPrivilegedOperation(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.ForceDeleteVMRequest](w, body)
if !ok {
return
}
if !request.RemoveManagedDisks {
writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true")
return
}
if !request.AckDataLoss {
writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true")
return
}
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
if !operationIDPattern.MatchString(request.PriorOperationID) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_prior_operation_id", "prior_operation_id must contain 8-80 URL-safe characters")
return
}
if operationID == request.PriorOperationID {
writePreDispatchError(w, http.StatusUnprocessableEntity, "force_operation_id_reused", "force delete requires a new operation_id")
return
}
existingReplay := false
existing, existingErr := s.Journal.Get(operationID)
if existingErr == nil {
existingReplay = existing.Action == "force-delete-vm" && existing.ResourceID == uuid &&
existing.SupersedesOperationID == request.PriorOperationID
} else if !errors.Is(existingErr, operation.ErrNotFound) {
writeMappedError(w, existingErr, nil)
return
}
if !existingReplay {
prior, priorErr := s.Journal.Get(request.PriorOperationID)
priorActionAllowed := priorErr == nil && (prior.Action == "delete-vm" || prior.Action == "service-suspend")
if !priorActionAllowed || prior.ResourceID != uuid ||
prior.State != operation.StateFailed || prior.Error == nil || prior.Error.Code != "force_required" {
writePreDispatchError(w, http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful stop to require force")
return
}
}
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "force-delete-vm", nil,
true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid),
func(ctx context.Context) (any, error) {
return s.Hypervisor.ForceDeleteVM(ctx, uuid, request)
}, request.PriorOperationID)
return
}
case "state":
if len(segments) == 4 && r.Method == http.MethodGet {
result, err := s.Hypervisor.GetVMState(r.Context(), uuid)
writeResult(w, result, err)
return
}
case "metadata":
if len(segments) == 4 && r.Method == http.MethodGet {
result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid)
writeResult(w, result, err)
return
}
case "actions":
if len(segments) == 4 && r.Method == http.MethodPost {
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.PowerRequest](w, body)
if !ok {
return
}
if !validPowerAction(request.Action) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_power_action", "action must be start, shutdown, reboot, reset, or destroy")
return
}
s.runMutation(w, r, body, actor, request.OperationID, uuid, "power-"+string(request.Action), true, http.StatusOK,
hypervisor.DomainMutationAdmission(uuid),
func(ctx context.Context) (any, error) { return s.Hypervisor.Power(ctx, uuid, request) })
return
}
case "settings":
if len(segments) == 4 && (r.Method == http.MethodPatch || r.Method == http.MethodPost) {
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.SettingsRequest](w, body)
if !ok {
return
}
if request.Hostname == nil && request.RootPassword == nil && request.SSHPublicKey == nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "empty_settings", "at least one setting is required")
return
}
if !request.ConfirmRestart {
writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true")
return
}
expected := &operation.ExpectedState{
Kind: operation.ExpectedUpdateSettings, HostnameChanged: request.Hostname != nil,
RootPasswordChanged: request.RootPassword != nil, SSHKeyChanged: request.SSHPublicKey != nil,
}
if request.Hostname != nil {
expected.Hostname = *request.Hostname
}
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "update-settings", expected, false, http.StatusOK,
hypervisor.SeedMutationAdmission(uuid),
func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateSettings(ctx, uuid, request) })
return
}
case "network":
if len(segments) == 4 && r.Method == http.MethodPost {
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.ChangeNetworkRequest](w, body)
if !ok {
return
}
if !request.ConfirmRestart {
writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true")
return
}
if err := hypervisor.ValidatePublicNetworkIntent(
request.NetworkSelector, request.PublicPoolKey, request.PublicPoolCIDR,
request.PublicIPCIDR, request.Gateway, request.DNSServers,
); err != nil {
writePreDispatchMappedError(w, err)
return
}
var source hypervisor.LocalNetworkSource
var err error
if s.MutationExecutor != nil {
source, err = hypervisor.ParseNetworkSelector(request.NetworkSelector)
} else {
source, err = s.Hypervisor.ResolveNetworkSelector(r.Context(), request.NetworkSelector)
}
if err != nil {
writePreDispatchMappedError(w, err)
return
}
expected := &operation.ExpectedState{
Kind: operation.ExpectedChangeNetwork, NetworkSelector: request.NetworkSelector,
NetworkSourceType: source.Type, NetworkSourceName: source.Source,
PublicPoolKey: request.PublicPoolKey, PublicPoolCIDR: request.PublicPoolCIDR,
PublicIPCIDR: request.PublicIPCIDR, Gateway: request.Gateway,
DNSServers: append([]string(nil), request.DNSServers...),
}
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "change-network", expected, true, http.StatusAccepted,
hypervisor.ChangeNetworkAdmission(uuid, request.PublicIPCIDR),
func(ctx context.Context) (any, error) { return s.Hypervisor.ChangeNetwork(ctx, uuid, request) })
return
}
if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") {
if !requireManage(w, actor) {
return
}
request, ok := decodeOptional[hypervisor.NetworkStateRequest](w, body)
if !ok {
return
}
enabled := segments[4] == "resume"
action := "network-" + segments[4]
expected := &operation.ExpectedState{Kind: operation.ExpectedNetworkState, Enabled: enabled}
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, action, expected, false, http.StatusOK,
hypervisor.DomainMutationAdmission(uuid),
func(ctx context.Context) (any, error) {
err := s.Hypervisor.SetNetworkState(ctx, uuid, enabled, request)
return map[string]bool{"network_enabled": enabled}, err
})
return
}
case "service":
if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") {
if !requireManage(w, actor) {
return
}
request, ok := decodeOptional[hypervisor.ServiceStateRequest](w, body)
if !ok {
return
}
enabled := segments[4] == "resume"
expected := &operation.ExpectedState{Kind: operation.ExpectedServiceState, Enabled: enabled, StartIfSuspended: request.StartIfSuspended}
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "service-"+segments[4], expected, true, http.StatusOK,
hypervisor.DomainMutationAdmission(uuid),
func(ctx context.Context) (any, error) {
return s.Hypervisor.SetServiceState(ctx, uuid, enabled, request)
})
return
}
case "reinstall":
if len(segments) == 4 && r.Method == http.MethodPost {
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.ReinstallRequest](w, body)
if !ok {
return
}
if err := hypervisor.ValidateTemplateSnapshot(request.TemplateID, request.TemplateRevision, request.TemplateGenerationToken); err != nil {
writePreDispatchMappedError(w, err)
return
}
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
request.OperationID = resolvedOperationID
s.runMutation(w, r, body, actor, request.OperationID, uuid, "reinstall-vm", true, http.StatusAccepted,
hypervisor.ReinstallVMAdmission(uuid, request),
func(ctx context.Context) (any, error) { return s.Hypervisor.Reinstall(ctx, uuid, request) })
return
}
case "metrics":
if len(segments) == 4 && r.Method == http.MethodGet {
result, err := s.Hypervisor.GetMetrics(r.Context(), uuid)
writeResult(w, result, err)
return
}
case "traffic":
if len(segments) == 5 && segments[4] == "daily" && r.Method == http.MethodGet {
days, err := strconv.Atoi(r.URL.Query().Get("days"))
if err != nil || days < 1 || days > 30 {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_days", "days must be between 1 and 30")
return
}
result, err := s.Hypervisor.GetDailyTraffic(r.Context(), uuid, days)
writeNamedList(w, "traffic", result, err)
return
}
case "console-ticket":
if len(segments) == 4 && r.Method == http.MethodPost {
if !requireManage(w, actor) {
return
}
target, err := s.Hypervisor.GetConsole(r.Context(), uuid)
if err != nil {
writeMappedError(w, err, nil)
return
}
if target.Protocol != "vnc" {
writeError(w, http.StatusNotImplemented, "unsupported_console", "only VNC console targets are supported", nil)
return
}
ticket, err := s.Console.Issue(target.Address)
if err != nil {
writeError(w, http.StatusServiceUnavailable, "console_unavailable", "console target is unavailable", nil)
return
}
writeJSON(w, http.StatusCreated, ticket)
return
}
case "snapshots":
s.routeSnapshots(w, r, segments, body, actor, uuid)
return
case "firewall":
if len(segments) == 4 {
switch r.Method {
case http.MethodGet:
result, err := s.Hypervisor.GetFirewall(r.Context(), uuid)
writeResult(w, result, err)
case http.MethodPut:
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.FirewallPolicy](w, body)
if !ok {
return
}
operationID := request.OperationID
request, err := hypervisor.NormalizeFirewallPolicy(request)
if err != nil {
writePreDispatchMappedError(w, err)
return
}
expected := &operation.ExpectedState{Kind: operation.ExpectedFirewall, FirewallRevision: request.Revision, FirewallDigest: request.PolicyDigest}
s.runMutationExpected(w, r, body, actor, operationID, uuid, "apply-firewall", expected, false, http.StatusOK,
hypervisor.FirewallMutationAdmission(uuid),
func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyFirewall(ctx, uuid, request) })
default:
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPut)
}
return
}
case "rescue":
s.routeRescue(w, r, segments, body, actor, uuid)
return
}
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
func (s *Server) routeSnapshots(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) {
if len(segments) == 4 {
switch r.Method {
case http.MethodGet:
result, err := s.Hypervisor.ListSnapshots(r.Context(), uuid)
writeNamedList(w, "snapshots", result, err)
case http.MethodPost:
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.SnapshotRequest](w, body)
if !ok {
return
}
if request.SnapshotKey == "" {
request.SnapshotKey = request.SnapshotID
}
if !catalogIDPattern.MatchString(request.SnapshotKey) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot", "snapshot key is invalid")
return
}
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
request.OperationID = resolvedOperationID
action := "create-snapshot:" + request.SnapshotKey + ":" + request.ConsistencyMode
s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusCreated,
hypervisor.CreateSnapshotAdmission(uuid, request.SnapshotKey),
func(ctx context.Context) (any, error) { return s.Hypervisor.CreateSnapshot(ctx, uuid, request) })
default:
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
}
return
}
if len(segments) < 5 || !resourceIDPattern.MatchString(segments[4]) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot_id", "snapshot ID is invalid")
return
}
snapshotID := segments[4]
if len(segments) == 5 {
switch r.Method {
case http.MethodGet:
snapshots, err := s.Hypervisor.ListSnapshots(r.Context(), uuid)
if err != nil {
writeMappedError(w, err, nil)
return
}
for _, snapshot := range snapshots {
if snapshot.ID == snapshotID {
writeJSON(w, http.StatusOK, snapshot)
return
}
}
writeError(w, http.StatusNotFound, "snapshot_not_found", "snapshot was not found", nil)
case http.MethodDelete:
if !requireManage(w, actor) {
return
}
request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body)
if !ok {
return
}
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
request.OperationID = resolvedOperationID
s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-snapshot:"+snapshotID, true, http.StatusOK,
hypervisor.DeleteSnapshotAdmission(uuid, snapshotID),
func(ctx context.Context) (any, error) {
return s.Hypervisor.DeleteSnapshot(ctx, uuid, snapshotID, request)
})
default:
methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete)
}
return
}
if len(segments) == 6 && segments[5] == "restore" && r.Method == http.MethodPost {
if !requireManage(w, actor) {
return
}
request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body)
if !ok {
return
}
resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
request.OperationID = resolvedOperationID
s.runMutation(w, r, body, actor, request.OperationID, uuid, "restore-snapshot:"+snapshotID, true, http.StatusAccepted,
hypervisor.RestoreSnapshotAdmission(uuid, snapshotID, request.OperationID),
func(ctx context.Context) (any, error) {
return s.Hypervisor.RestoreSnapshot(ctx, uuid, snapshotID, request)
})
return
}
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
func (s *Server) routeRescue(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) {
if len(segments) == 4 && r.Method == http.MethodGet {
result, err := s.Hypervisor.GetRescue(r.Context(), uuid)
writeResult(w, result, err)
return
}
if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "enter" || segments[4] == "exit" || segments[4] == "force-start") {
if !requireManage(w, actor) {
return
}
if segments[4] == "enter" {
request, ok := decodeRequired[hypervisor.RescueRequest](w, body)
if !ok {
return
}
if !catalogIDPattern.MatchString(request.TemplateID) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_template_id", "rescue template id is invalid")
return
}
s.runMutation(w, r, body, actor, request.OperationID, uuid, "enter-rescue", true, http.StatusAccepted,
hypervisor.EnterRescueAdmission(uuid, request.TemplateID),
func(ctx context.Context) (any, error) { return s.Hypervisor.EnterRescue(ctx, uuid, request) })
} else if segments[4] == "force-start" {
request, ok := decodeRequired[hypervisor.ForceRescueStartRequest](w, body)
if !ok {
return
}
if !request.AckDataLoss {
writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true")
return
}
s.runMutation(w, r, body, actor, request.OperationID, uuid, "force-start-rescue", true, http.StatusAccepted,
hypervisor.DomainMutationAdmission(uuid),
func(ctx context.Context) (any, error) { return s.Hypervisor.ForceStartRescue(ctx, uuid, request) })
} else {
request, ok := decodeRequired[hypervisor.ExitRescueRequest](w, body)
if !ok {
return
}
if request.NetworkSuspended == nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "network_state_required", "network_suspended must be provided")
return
}
action := "exit-rescue:connected"
if *request.NetworkSuspended {
action = "exit-rescue:suspended"
}
s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusAccepted,
hypervisor.DomainMutationAdmission(uuid),
func(ctx context.Context) (any, error) { return s.Hypervisor.ExitRescue(ctx, uuid, request) })
}
return
}
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
func (s *Server) routeTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
if !actor.IsAdmin && !actor.IsSuperAdmin {
writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required")
return
}
if len(segments) == 2 {
switch r.Method {
case http.MethodGet:
result, err := s.Hypervisor.ListTemplates(r.Context())
writeNamedList(w, "templates", result, err)
case http.MethodPost:
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.ImportTemplateRequest](w, body)
if !ok {
return
}
if err := hypervisor.ValidateTemplateSnapshot(request.ID, request.Revision, request.GenerationToken); err != nil {
writePreDispatchMappedError(w, err)
return
}
if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil {
writePreDispatchMappedError(w, err)
return
}
s.runAsyncMutation(w, r, body, actor, request.OperationID, "template:"+request.ID, "import-template", false,
hypervisor.ImportTemplateAdmission(request),
func(ctx context.Context) (any, error) { return s.Hypervisor.ImportTemplate(ctx, request) })
default:
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
}
return
}
if len(segments) == 3 && resourceIDPattern.MatchString(segments[2]) {
templateID := segments[2]
switch r.Method {
case http.MethodPatch:
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.UpdateTemplateRequest](w, body)
if !ok {
return
}
if strings.TrimSpace(request.Name) == "" || len(request.Name) > 120 {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_template_name", "template name must contain 1-120 characters")
return
}
s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "update-template", false, http.StatusOK,
hypervisor.UpdateTemplateAdmission(templateID, request),
func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateTemplate(ctx, templateID, request) })
case http.MethodDelete:
if !requireManage(w, actor) {
return
}
request, ok := decodeOptional[hypervisor.MutationRequest](w, body)
if !ok {
return
}
s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "delete-template", true, http.StatusOK,
hypervisor.DeleteTemplateAdmission(templateID),
func(ctx context.Context) (any, error) {
err := s.Hypervisor.DeleteTemplate(ctx, templateID)
return catalogDeleteResult("template", templateID, 0, err == nil), err
})
default:
methodOrNotFound(w, r, true, http.MethodPatch, http.MethodDelete)
}
return
}
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
func (s *Server) routeRescueTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
if !actor.IsAdmin && !actor.IsSuperAdmin {
writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required")
return
}
if len(segments) == 2 {
switch r.Method {
case http.MethodGet:
result, err := s.Hypervisor.ListRescueTemplates(r.Context())
writeNamedList(w, "templates", result, err)
case http.MethodPost:
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.ImportRescueTemplateRequest](w, body)
if !ok {
return
}
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
if request.ID == "" {
request.ID = operationCatalogID(operationID)
}
if !catalogIDPattern.MatchString(request.ID) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_rescue_template_id", "id must contain 32 lowercase hexadecimal characters")
return
}
if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil {
writePreDispatchMappedError(w, err)
return
}
s.runAsyncMutation(w, r, body, actor, request.OperationID, "rescue-template:"+request.ID, "import-rescue-template", false,
hypervisor.ImportRescueTemplateAdmission(request),
func(ctx context.Context) (any, error) { return s.Hypervisor.ImportRescueTemplate(ctx, request) })
default:
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
}
return
}
if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete {
if !requireManage(w, actor) {
return
}
request, ok := decodeOptional[hypervisor.MutationRequest](w, body)
if !ok {
return
}
id := segments[2]
s.runMutation(w, r, body, actor, request.OperationID, "rescue-template:"+id, "delete-rescue-template", true, http.StatusOK,
hypervisor.DeleteRescueTemplateAdmission(id),
func(ctx context.Context) (any, error) {
err := s.Hypervisor.DeleteRescueTemplate(ctx, id)
return catalogDeleteResult("rescue_template", id, 0, err == nil), err
})
return
}
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
func (s *Server) routeUserISOs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
ownerID, err := ownerFromRequest(r, actor)
if err != nil {
writePreDispatchError(w, http.StatusForbidden, "forbidden", err.Error())
return
}
if len(segments) == 2 {
switch r.Method {
case http.MethodGet:
result, err := s.Hypervisor.ListUserISOs(r.Context(), ownerID)
writeNamedList(w, "user_isos", result, err)
case http.MethodPost:
if !requireManage(w, actor) {
return
}
request, ok := decodeRequired[hypervisor.ImportUserISORequest](w, body)
if !ok {
return
}
if request.OwnerID == 0 {
request.OwnerID = ownerID
}
if request.OwnerID != ownerID && !actor.IsAdmin && !actor.IsSuperAdmin {
writePreDispatchError(w, http.StatusForbidden, "forbidden", "cannot import ISO for another owner")
return
}
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
if request.ID == "" {
request.ID = operationCatalogID(operationID)
}
if !catalogIDPattern.MatchString(request.ID) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_user_iso_id", "id must contain 32 lowercase hexadecimal characters")
return
}
if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil {
writePreDispatchMappedError(w, err)
return
}
s.runAsyncMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", request.OwnerID, request.ID), "import-user-iso", false,
hypervisor.ImportUserISOAdmission(request),
func(ctx context.Context) (any, error) { return s.Hypervisor.ImportUserISO(ctx, request) })
default:
methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost)
}
return
}
if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete {
if !requireManage(w, actor) {
return
}
request, ok := decodeOptional[hypervisor.MutationRequest](w, body)
if !ok {
return
}
isoID := segments[2]
s.runMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", ownerID, isoID), "delete-user-iso", true, http.StatusOK,
hypervisor.DeleteUserISOAdmission(ownerID, isoID),
func(ctx context.Context) (any, error) {
err := s.Hypervisor.DeleteUserISO(ctx, isoID, ownerID)
return catalogDeleteResult("user_iso", isoID, ownerID, err == nil), err
})
return
}
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
func catalogDeleteResult(resourceType, id string, ownerID int64, deleted bool) hypervisor.CatalogDeleteResult {
return hypervisor.CatalogDeleteResult{
ResourceType: resourceType, ID: id, OwnerID: ownerID, Deleted: deleted,
ManagedImageAbsent: deleted, ManagedMetadataAbsent: deleted, ManagedArtifactsAbsent: deleted,
}
}
func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
if len(segments) < 3 || !resourceIDPattern.MatchString(segments[2]) {
writeError(w, http.StatusNotFound, "operation_not_found", "operation was not found", nil)
return
}
operationID := segments[2]
if len(segments) == 3 && r.Method == http.MethodGet {
record, err := s.Journal.Get(operationID)
if err != nil {
writeMappedError(w, err, nil)
return
}
if record.Action == "configure-storage" && !requireSuperAdminManage(w, actor) {
return
}
writeJSON(w, http.StatusOK, record)
return
}
if len(segments) == 4 && segments[3] == "inspect" && r.Method == http.MethodGet {
if !requireSuperAdminInspection(w, actor) {
return
}
stored, err := s.Journal.Get(operationID)
if err != nil {
writeMappedError(w, err, nil)
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
defer cancel()
inspection, err := s.Hypervisor.InspectOperation(ctx, stored)
if err != nil {
writeMappedError(w, err, nil)
return
}
writeJSON(w, http.StatusOK, inspection)
return
}
if len(segments) == 4 && segments[3] == "fence-resolution" && r.Method == http.MethodGet {
if !requireSuperAdminInspection(w, actor) {
return
}
resolution, err := s.Journal.GetFenceResolution(operationID)
if errors.Is(err, operation.ErrNotFound) {
writeError(w, http.StatusNotFound, "fence_resolution_not_found", "operation fence resolution was not found", nil)
return
}
if err != nil {
writeMappedError(w, err, nil)
return
}
writeJSON(w, http.StatusOK, fenceResolutionResponse{FenceResolution: resolution, Replayed: false})
return
}
if len(segments) == 4 && segments[3] == "resolve-fence" && r.Method == http.MethodPost {
if !requireSuperAdminFenceResolution(w, actor) {
return
}
if actor.OperationID == "" || actor.OperationID != operationID {
writeError(w, http.StatusBadRequest, "operation_id_mismatch", "actor operation_id must match the fenced operation", nil)
return
}
request, ok := decodeRequired[operation.FenceResolutionRequest](w, body)
if !ok {
return
}
if err := operation.ValidateFenceResolutionRequest(request); err != nil {
writeError(w, http.StatusBadRequest, "invalid_fence_resolution", err.Error(), nil)
return
}
resolution, replayed, err := s.Journal.ResolveResourceFence(operationID, actor.UserID, request)
switch {
case errors.Is(err, operation.ErrNotFound):
writeError(w, http.StatusNotFound, "operation_not_found", "operation was not found", nil)
return
case errors.Is(err, operation.ErrFenceNotResolvable):
writeError(w, http.StatusConflict, "operation_fence_not_resolvable", "operation does not hold an active effect fence", nil)
return
case errors.Is(err, operation.ErrFenceResolutionConflict):
writeError(w, http.StatusConflict, "operation_fence_resolution_conflict", "operation fence already has a different audit resolution", nil)
return
case errors.Is(err, operation.ErrFenceInspectionStale):
writeError(w, http.StatusConflict, "fence_inspection_stale", "operation fence resolution requires a fresh read-only inspection", nil)
return
case err != nil:
writeMappedError(w, err, nil)
return
}
writeJSON(w, http.StatusOK, fenceResolutionResponse{FenceResolution: resolution, Replayed: replayed})
return
}
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
func (s *Server) runMutation(
w http.ResponseWriter,
r *http.Request,
body []byte,
actor auth.Actor,
bodyOperationID, resourceID, action string,
destructive bool,
successStatus int,
admission hypervisor.MutationAdmission,
call func(context.Context) (any, error),
) {
s.runMutationExpected(w, r, body, actor, bodyOperationID, resourceID, action, nil, destructive, successStatus, admission, call)
}
func (s *Server) runMutationExpected(
w http.ResponseWriter,
r *http.Request,
body []byte,
actor auth.Actor,
bodyOperationID, resourceID, action string,
expectedState *operation.ExpectedState,
destructive bool,
successStatus int,
admission hypervisor.MutationAdmission,
call func(context.Context) (any, error),
supersedesOperationID ...string,
) {
operationID, err := resolveOperationID(actor.OperationID, bodyOperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
if expectedState != nil {
copyExpected := *expectedState
copyExpected.TransitionID = operationID
copyExpected.DNSServers = append([]string(nil), expectedState.DNSServers...)
expectedState = &copyExpected
}
supersedes := ""
if len(supersedesOperationID) != 0 {
supersedes = supersedesOperationID[0]
}
spec := operation.Spec{
OperationID: operationID, ResourceID: resourceID, Action: action,
Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body),
Destructive: destructive, ExpectedState: expectedState,
LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO,
SupersedesOperationID: supersedes,
}
if s.MutationExecutor != nil {
s.runIsolatedMutation(w, r, body, actor, spec, s.isolatedMutationDeadline(action))
return
}
if _, trusted := r.Context().Value(trustedWorkerRequestKey{}).(struct{}); !trusted {
writePreDispatchError(w, http.StatusServiceUnavailable, "mutation_worker_unavailable", "isolated mutation worker is unavailable")
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
defer cancel()
record, replayed, callErr := s.Journal.Execute(ctx, spec, func(ctx context.Context) (json.RawMessage, error) {
result, err := call(ctx)
if err != nil {
return nil, err
}
encoded, err := json.Marshal(result)
if err != nil {
return nil, fmt.Errorf("encode operation result: %w", err)
}
return encoded, nil
})
logOperationOutcome(record)
if errors.Is(callErr, operation.ErrFingerprint) {
writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil)
return
}
if errors.Is(callErr, operation.ErrOperationStarting) {
writeOperationStarting(w, record)
return
}
if operation.WasNotDispatched(record) {
writeOperationFailure(w, record, replayed)
return
}
if callErr != nil {
if record.State == operation.StateFailed {
writeOperationFailure(w, record, replayed)
return
}
writeMappedError(w, callErr, &record)
return
}
if record.State == operation.StateFailed {
writeOperationFailure(w, record, replayed)
return
}
if record.State == operation.StateDispatching || record.State == operation.StateReserved {
writeJSON(w, http.StatusAccepted, operationWire(record, replayed))
return
}
if replayed {
successStatus = http.StatusOK
}
writeJSON(w, successStatus, operationWire(record, replayed))
}
func (s *Server) runAsyncMutation(
w http.ResponseWriter,
r *http.Request,
body []byte,
actor auth.Actor,
bodyOperationID, resourceID, action string,
destructive bool,
admission hypervisor.MutationAdmission,
call func(context.Context) (any, error),
) {
operationID, err := resolveOperationID(actor.OperationID, bodyOperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
spec := operation.Spec{
OperationID: operationID, ResourceID: resourceID, Action: action,
Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body),
Destructive: destructive,
LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO,
}
if s.MutationExecutor != nil {
s.runIsolatedMutation(w, r, body, actor, spec, catalogImportTimeout)
return
}
if _, trusted := r.Context().Value(trustedWorkerRequestKey{}).(struct{}); !trusted {
writePreDispatchError(w, http.StatusServiceUnavailable, "mutation_worker_unavailable", "isolated mutation worker is unavailable")
return
}
record, replayed, err := s.Journal.Reserve(spec)
if errors.Is(err, operation.ErrFingerprint) {
writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil)
return
}
if err != nil {
writeMappedError(w, err, nil)
return
}
canStart := record.State == operation.StateReserved
if canStart && !s.claimAsync(operationID) {
writeOperationStarting(w, record)
return
}
if canStart {
select {
case s.asyncSlots <- struct{}{}:
default:
failed, failErr := s.Journal.FailNotDispatched(operationID)
s.releaseAsync(operationID)
if failErr != nil {
writeMappedError(w, failErr, &record)
return
}
writeOperationFailure(w, failed, false)
return
}
dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout)
started, _, done, startErr := s.Journal.ExecuteBackground(
dispatchCtx, s.backgroundCtx, catalogImportTimeout, spec,
func(ctx context.Context) (json.RawMessage, error) {
result, callErr := call(ctx)
if callErr != nil {
return nil, callErr
}
return json.Marshal(result)
},
)
dispatchCancel()
if startErr != nil || done == nil {
<-s.asyncSlots
s.releaseAsync(operationID)
if startErr != nil {
if errors.Is(startErr, operation.ErrOperationStarting) {
writeOperationStarting(w, started)
return
}
if started.State == operation.StateFailed && started.Error != nil && started.Error.Code == "operation_not_dispatched" {
writeOperationFailure(w, started, false)
return
}
writeMappedError(w, startErr, &started)
return
}
record = started
} else {
record = started
s.asyncWG.Add(1)
go func() {
defer s.asyncWG.Done()
defer func() { <-s.asyncSlots }()
defer s.releaseAsync(operationID)
<-done
if final, getErr := s.Journal.Get(operationID); getErr == nil {
logOperationOutcome(final)
}
}()
}
}
writeJSON(w, http.StatusAccepted, operationWire(record, replayed))
}
func (s *Server) runIsolatedMutation(
w http.ResponseWriter,
r *http.Request,
body []byte,
actor auth.Actor,
spec operation.Spec,
deadline time.Duration,
) {
if deadline <= 0 {
deadline = s.OperationTimeout
}
actor.OperationID = spec.OperationID
request := worker.Request{
Version: worker.ProtocolVersion, Method: r.Method,
RequestTarget: auth.RequestTarget(r), Body: string(body), Actor: actor,
OperationID: spec.OperationID, Fingerprint: spec.Fingerprint,
}
dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout)
record, replayed, done, err := s.Journal.ExecuteBackground(
dispatchCtx, s.backgroundCtx, deadline, spec,
func(callCtx context.Context) (json.RawMessage, error) {
return s.MutationExecutor.Execute(callCtx, request)
},
)
dispatchCancel()
if done != nil {
s.asyncWG.Add(1)
go func(operationID string) {
defer s.asyncWG.Done()
<-done
if final, getErr := s.Journal.Get(operationID); getErr == nil {
logOperationOutcome(final)
}
}(spec.OperationID)
}
if errors.Is(err, operation.ErrFingerprint) {
writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil)
return
}
if errors.Is(err, operation.ErrOperationStarting) {
writeOperationStarting(w, record)
return
}
if operation.WasNotDispatched(record) {
writeJSON(w, http.StatusOK, operationWire(record, replayed))
return
}
if err != nil {
writeMappedError(w, err, &record)
return
}
status := http.StatusAccepted
if record.State == operation.StateSucceeded || record.State == operation.StateFailed {
status = http.StatusOK
}
writeJSON(w, status, operationWire(record, replayed))
}
func (s *Server) isolatedMutationDeadline(action string) time.Duration {
deadline := s.OperationTimeout
if action == "configure-storage" && deadline > storageConfigurationWorkerTimeout {
return storageConfigurationWorkerTimeout
}
return deadline
}
func (s *Server) beginRequest() bool {
s.lifecycleMu.Lock()
defer s.lifecycleMu.Unlock()
if s.closing {
return false
}
s.requestWG.Add(1)
return true
}
// Shutdown stops background imports and waits for every admitted API handler
// and import watcher. The journal's own workers are drained separately before
// libvirt or BoltDB are closed.
func (s *Server) Shutdown(ctx context.Context) error {
s.lifecycleMu.Lock()
if !s.closing {
s.closing = true
s.backgroundCancel()
}
s.lifecycleMu.Unlock()
if err := waitGroup(ctx, &s.requestWG); err != nil {
return err
}
return waitGroup(ctx, &s.asyncWG)
}
func waitGroup(ctx context.Context, group *sync.WaitGroup) error {
done := make(chan struct{})
go func() {
group.Wait()
close(done)
}()
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (s *Server) claimAsync(operationID string) bool {
s.asyncMu.Lock()
defer s.asyncMu.Unlock()
if s.asyncRunning == nil {
s.asyncRunning = make(map[string]struct{})
}
if _, exists := s.asyncRunning[operationID]; exists {
return false
}
s.asyncRunning[operationID] = struct{}{}
return true
}
func (s *Server) releaseAsync(operationID string) {
s.asyncMu.Lock()
defer s.asyncMu.Unlock()
delete(s.asyncRunning, operationID)
}
func operationWire(record operation.Record, replayed bool) operationResponse {
return operationResponse{
OperationID: record.OperationID, Status: record.State, Replayed: replayed,
Result: record.Result, Error: record.Error, CauseCode: record.CauseCode,
EffectMayHaveApplied: record.EffectMayHaveApplied,
SafeToResubmit: record.SafeToResubmit,
Retryable: false,
}
}
// Log only bounded identifiers and stable codes. Free-form backend errors,
// request bodies, guest credentials, and host paths never enter this line.
func logOperationOutcome(record operation.Record) {
if record.State != operation.StateFailed {
return
}
errorCode := ""
if record.Error != nil && len(record.Error.Code) <= 64 &&
stableCodePattern.MatchString(record.Error.Code) {
errorCode = record.Error.Code
}
log.Printf("Agent operation outcome operation_id=%s action=%s status=%s error_code=%s cause_code=%s",
record.OperationID, record.Action, record.State, errorCode, record.CauseCode)
}
func resolveOperationID(actorID, bodyID string) (string, error) {
if actorID == "" && bodyID == "" {
return "", errors.New("operation_id is required in X-Virt-Actor or request body")
}
if actorID != "" && bodyID != "" && actorID != bodyID {
return "", errors.New("actor and body operation_id values differ")
}
resolved := bodyID
if actorID != "" {
resolved = actorID
}
if !operationIDPattern.MatchString(resolved) {
return "", errors.New("operation_id must contain 8-80 URL-safe characters")
}
return resolved, nil
}
func operationUUID(operationID string) string {
digest := sha256.Sum256([]byte("virt-cloud-operation:" + operationID))
digest[6] = digest[6]&0x0f | 0x50
digest[8] = digest[8]&0x3f | 0x80
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
digest[0:4], digest[4:6], digest[6:8], digest[8:10], digest[10:16])
}
func operationCatalogID(operationID string) string {
digest := sha256.Sum256([]byte("virt-cloud-catalog:" + operationID))
return fmt.Sprintf("%x", digest[:16])
}
func validPowerAction(action hypervisor.PowerAction) bool {
switch action {
case hypervisor.PowerStart, hypervisor.PowerShutdown, hypervisor.PowerReboot, hypervisor.PowerReset, hypervisor.PowerDestroy:
return true
default:
return false
}
}
func validateImportSource(raw, token, digest string) error {
parsed, err := url.Parse(raw)
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" {
return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_url", "source_url must be an HTTPS URL without credentials or fragment", err)
}
if digest != "" && !sha256Pattern.MatchString(digest) {
return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_sha256", "sha256 must be 64 lowercase hexadecimal characters", nil)
}
if token != "" && !sourceTokenPattern.MatchString(token) {
return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_token", "source_token must contain 32-512 URL-safe characters", nil)
}
return nil
}
func ownerFromRequest(r *http.Request, actor auth.Actor) (int64, error) {
value := r.URL.Query().Get("owner_id")
if value == "" {
return actor.UserID, nil
}
ownerID, err := strconv.ParseInt(value, 10, 64)
if err != nil || ownerID <= 0 {
return 0, errors.New("owner_id is invalid")
}
if ownerID != actor.UserID && !actor.IsAdmin && !actor.IsSuperAdmin {
return 0, errors.New("cannot access another owner's ISO")
}
return ownerID, nil
}
func requireManage(w http.ResponseWriter, actor auth.Actor) bool {
if !actor.CanManage {
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to manage this resource")
return false
}
return true
}
func requireSuperAdminManage(w http.ResponseWriter, actor auth.Actor) bool {
if !actor.IsSuperAdmin || !actor.CanManage {
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to configure host storage")
return false
}
return true
}
func requireSuperAdminPrivilegedOperation(w http.ResponseWriter, actor auth.Actor) bool {
if !actor.IsSuperAdmin || !actor.CanManage {
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to perform this privileged operation")
return false
}
return true
}
func requireSuperAdminInspection(w http.ResponseWriter, actor auth.Actor) bool {
if !actor.IsSuperAdmin || !actor.CanManage {
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to inspect operation state")
return false
}
return true
}
func requireSuperAdminFenceResolution(w http.ResponseWriter, actor auth.Actor) bool {
if !actor.IsSuperAdmin || !actor.CanManage {
writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to resolve operation effect fences")
return false
}
return true
}
func decodeRequired[T any](w http.ResponseWriter, body []byte) (T, bool) {
var value T
if len(body) == 0 {
writePreDispatchError(w, http.StatusBadRequest, "invalid_json", "JSON request body is required")
return value, false
}
if err := decodeStrict(body, &value); err != nil {
writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error())
return value, false
}
return value, true
}
func decodeOptional[T any](w http.ResponseWriter, body []byte) (T, bool) {
var value T
if len(body) == 0 {
return value, true
}
if err := decodeStrict(body, &value); err != nil {
writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error())
return value, false
}
return value, true
}
func decodeStrict(body []byte, target any) error {
if !utf8.Valid(body) {
return errors.New("invalid JSON: body must be valid UTF-8")
}
decoder := json.NewDecoder(bytes.NewReader(body))
decoder.DisallowUnknownFields()
if err := decoder.Decode(target); err != nil {
return fmt.Errorf("invalid JSON: %w", err)
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return errors.New("invalid JSON: exactly one object is required")
}
return nil
}
func readBoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, error) {
if r.Body == nil {
return []byte{}, nil
}
r.Body = http.MaxBytesReader(w, r.Body, limit)
defer r.Body.Close()
return io.ReadAll(r.Body)
}
func writeResult(w http.ResponseWriter, value any, err error) {
if err != nil {
writeMappedError(w, err, nil)
return
}
writeJSON(w, http.StatusOK, value)
}
func writeNamedList[T any](w http.ResponseWriter, name string, value []T, err error) {
if err != nil {
writeMappedError(w, err, nil)
return
}
if value == nil {
value = make([]T, 0)
}
writeJSON(w, http.StatusOK, map[string]any{name: value})
}
func writeMappedError(w http.ResponseWriter, err error, record *operation.Record) {
status, code, message := classifyError(err)
writeError(w, status, code, message, record)
}
// writePreDispatchMappedError is only for validation or dependency checks
// completed before Journal.Execute/Reserve. The explicit marker lets the
// panel distinguish a proven no-journal outcome from a lost response after a
// mutation may already have crossed its durable dispatch boundary.
func writePreDispatchMappedError(w http.ResponseWriter, err error) {
status, code, message := classifyError(err)
writePreDispatchError(w, status, code, message)
}
// writeOperationStarting reports the active admission owner without exposing a
// stale failed/not-dispatched record from before that owner acquired admission.
// The projected reserved state is diagnostic only; it never authorizes a
// caller to dispatch the mutation again. The journal remains authoritative.
func writeOperationStarting(w http.ResponseWriter, record operation.Record) {
record.State = operation.StateReserved
record.Result = nil
record.Error = nil
record.CauseCode = ""
record.Tombstone = false
writeError(w, http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting", &record)
}
func classifyError(err error) (int, string, string) {
if errors.Is(err, operation.ErrNotFound) {
return http.StatusNotFound, "operation_not_found", "operation was not found"
}
if errors.Is(err, operation.ErrBusy) {
return http.StatusServiceUnavailable, "agent_busy", "Agent mutation capacity is exhausted"
}
if errors.Is(err, operation.ErrOperationStarting) {
return http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting"
}
if errors.Is(err, operation.ErrResourceBlocked) {
return http.StatusConflict, "resource_operation_blocked", "resource is fenced by a prior failed operation whose effect may have applied"
}
if errors.Is(err, operation.ErrInvalidSupersession) {
return http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful operation"
}
var backend *hypervisor.Error
if errors.As(err, &backend) {
status := statusForKind(backend.Kind)
code := backend.Code
if code == "" {
code = "backend_error"
}
message := backend.Message
if message == "" {
message = http.StatusText(status)
}
return status, code, message
}
switch {
case errors.Is(err, hypervisor.ErrNotFound):
return http.StatusNotFound, "not_found", "resource was not found"
case errors.Is(err, hypervisor.ErrConflict):
return http.StatusConflict, "conflict", "resource conflict"
case errors.Is(err, hypervisor.ErrUnsupported):
return http.StatusNotImplemented, "unsupported", "operation is not supported"
case errors.Is(err, hypervisor.ErrInvalid):
return http.StatusUnprocessableEntity, "invalid_request", "request is invalid"
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
return http.StatusServiceUnavailable, "backend_timeout", "backend operation did not complete"
default:
return http.StatusServiceUnavailable, "backend_error", "backend operation failed"
}
}
func statusForKind(kind error) int {
switch kind {
case hypervisor.ErrNotFound:
return http.StatusNotFound
case hypervisor.ErrConflict:
return http.StatusConflict
case hypervisor.ErrUnsupported:
return http.StatusNotImplemented
case hypervisor.ErrInvalid:
return http.StatusUnprocessableEntity
default:
return http.StatusServiceUnavailable
}
}
func writeOperationFailure(w http.ResponseWriter, record operation.Record, replayed bool) {
status := http.StatusServiceUnavailable
if record.Error != nil {
switch record.Error.Code {
case "unsupported":
status = http.StatusNotImplemented
case "not_found":
status = http.StatusNotFound
case "conflict":
status = http.StatusConflict
case "force_required", "force_delete_not_authorized":
status = http.StatusConflict
default:
if strings.HasSuffix(record.Error.Code, "_not_found") {
status = http.StatusNotFound
} else if strings.HasPrefix(record.Error.Code, "invalid_") {
status = http.StatusUnprocessableEntity
}
}
}
writeJSON(w, status, operationWire(record, replayed))
}
func methodOrNotFound(w http.ResponseWriter, r *http.Request, exists bool, methods ...string) {
if !exists {
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
return
}
w.Header().Set("Allow", strings.Join(methods, ", "))
writePreDispatchError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method is not allowed")
}
func writePreDispatchError(w http.ResponseWriter, status int, code, message string) {
writeJSON(w, status, errorEnvelope{Error: errorBody{
Code: code, Message: message, Retryable: retryableError(status, code),
DispatchState: "not_dispatched",
}})
}
func writeError(w http.ResponseWriter, status int, code, message string, record *operation.Record) {
writeJSON(w, status, errorEnvelope{Error: errorBody{
Code: code, Message: message, Retryable: retryableError(status, code), Operation: record,
}})
}
func retryableError(status int, code string) bool {
return false
}
func writeJSON(w http.ResponseWriter, status int, value any) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(value)
}
func setResponseHeaders(w http.ResponseWriter) {
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("X-Content-Type-Options", "nosniff")
}