package api import ( "bytes" "context" "crypto/sha256" "encoding/json" "errors" "fmt" "io" "log" "net/http" "net/url" "regexp" "strconv" "strings" "sync" "time" "unicode/utf8" "git.luoli.live/root/virt/agent/auth" consolebroker "git.luoli.live/root/virt/agent/console" "git.luoli.live/root/virt/agent/hypervisor" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/worker" ) const Version = "0.1.0" var ( apiUUIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$`) resourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`) operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`) stableCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`) sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) catalogIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`) sourceTokenPattern = regexp.MustCompile(`^[A-Za-z0-9._~-]{32,512}$`) ) const ( catalogImportTimeout = 2 * time.Hour storageConfigurationWorkerTimeout = 120 * time.Second defaultAdmissionTimeout = 2 * time.Second maxAsyncImports = 2 ) type Server struct { Authenticator *auth.Authenticator Journal *operation.Journal Hypervisor hypervisor.Hypervisor Console *consolebroker.Broker MaxBodyBytes int64 OperationTimeout time.Duration AdmissionTimeout time.Duration MutationExecutor worker.Executor asyncMu sync.Mutex asyncRunning map[string]struct{} asyncSlots chan struct{} asyncWG sync.WaitGroup lifecycleMu sync.Mutex closing bool requestWG sync.WaitGroup backgroundCtx context.Context backgroundCancel context.CancelFunc } type trustedWorkerRequestKey struct{} // SetMutationExecutor enables the clean-break process-isolated mutation path. // It must be called during startup before request admission begins. func (s *Server) SetMutationExecutor(executor worker.Executor) error { if executor == nil { return errors.New("mutation executor is required") } s.MutationExecutor = executor return nil } // ServeTrustedWorkerRequest dispatches a request whose bounded envelope was // already authenticated and validated by the parent Agent. Hidden workers do // not have access to the panel secret through HTTP headers and cannot recurse // because they never install a MutationExecutor. func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor) { setResponseHeaders(w) if !s.beginRequest() { writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down") return } defer s.requestWG.Done() if int64(len(body)) > s.MaxBodyBytes || actor.OperationID == "" { writePreDispatchError(w, http.StatusBadRequest, "invalid_worker_request", "worker request is invalid") return } trusted := context.WithValue(r.Context(), trustedWorkerRequestKey{}, struct{}{}) s.route(w, r.WithContext(trusted), body, actor) } type errorBody struct { Code string `json:"code"` Message string `json:"message"` Retryable bool `json:"retryable"` DispatchState string `json:"dispatch_state,omitempty"` Operation *operation.Record `json:"operation,omitempty"` } type errorEnvelope struct { Error errorBody `json:"error"` } type operationResponse struct { OperationID string `json:"operation_id"` Status operation.State `json:"status"` Replayed bool `json:"replayed"` Result json.RawMessage `json:"result,omitempty"` Error *operation.PublicError `json:"error,omitempty"` CauseCode string `json:"cause_code,omitempty"` EffectMayHaveApplied bool `json:"effect_may_have_applied"` SafeToResubmit bool `json:"safe_to_resubmit"` Retryable bool `json:"retryable"` } type fenceResolutionResponse struct { operation.FenceResolution Replayed bool `json:"replayed"` } func NewServer( authenticator *auth.Authenticator, journal *operation.Journal, hv hypervisor.Hypervisor, console *consolebroker.Broker, maxBodyBytes int64, operationTimeout time.Duration, ) (*Server, error) { if authenticator == nil || journal == nil || hv == nil || console == nil { return nil, errors.New("authenticator, journal, hypervisor, and console broker are required") } if maxBodyBytes < 1024 { return nil, errors.New("max body size must be at least 1024 bytes") } if operationTimeout <= 0 { return nil, errors.New("operation timeout must be positive") } backgroundCtx, backgroundCancel := context.WithCancel(context.Background()) return &Server{ Authenticator: authenticator, Journal: journal, Hypervisor: hv, Console: console, MaxBodyBytes: maxBodyBytes, OperationTimeout: operationTimeout, AdmissionTimeout: min(operationTimeout, defaultAdmissionTimeout), asyncRunning: make(map[string]struct{}), asyncSlots: make(chan struct{}, maxAsyncImports), backgroundCtx: backgroundCtx, backgroundCancel: backgroundCancel, }, nil } func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { setResponseHeaders(w) if !s.beginRequest() { writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down") return } defer s.requestWG.Done() if !strings.HasPrefix(r.URL.Path, "/v1/") || strings.HasSuffix(r.URL.Path, "/") { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return } body, err := readBoundedBody(w, r, s.MaxBodyBytes) if err != nil { var tooLarge *http.MaxBytesError if errors.As(err, &tooLarge) { writePreDispatchError(w, http.StatusRequestEntityTooLarge, "body_too_large", "request body exceeds configured limit") } else { writePreDispatchError(w, http.StatusBadRequest, "invalid_body", "could not read request body") } return } actor, err := s.Authenticator.Authenticate(r, body) if err != nil { if errors.Is(err, auth.ErrNonceStore) { writePreDispatchError(w, http.StatusServiceUnavailable, "nonce_store_unavailable", "authentication state is unavailable") return } if errors.Is(err, auth.ErrReplay) { // The matching nonce may belong to an earlier request that already // crossed the durable operation boundary. A replay rejection is not // proof that the operation was never dispatched. writeError(w, http.StatusUnauthorized, "replayed_request", "request nonce has already been used", nil) return } writePreDispatchError(w, http.StatusUnauthorized, "unauthorized", "request authentication failed") return } s.route(w, r, body, actor) } func (s *Server) route(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor) { segments := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") if len(segments) < 2 || segments[0] != "v1" { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return } switch segments[1] { case "health": if len(segments) != 2 || r.Method != http.MethodGet { methodOrNotFound(w, r, len(segments) == 2, http.MethodGet) return } if err := s.Hypervisor.Ping(r.Context()); err != nil { writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, map[string]any{"status": "ok", "service": "virt-host-agent", "version": Version}) case "capabilities": if len(segments) != 2 || r.Method != http.MethodGet { methodOrNotFound(w, r, len(segments) == 2, http.MethodGet) return } result, err := s.Hypervisor.Capabilities(r.Context()) if err != nil { writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, result) case "storage": s.routeStorage(w, r, segments, body, actor) case "vms": s.routeVMs(w, r, segments, body, actor) case "templates": s.routeTemplates(w, r, segments, body, actor) case "rescue-templates": s.routeRescueTemplates(w, r, segments, body, actor) case "user-isos": s.routeUserISOs(w, r, segments, body, actor) case "operations": s.routeOperations(w, r, segments, body, actor) default: writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } } func (s *Server) routeStorage(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if len(segments) != 3 || segments[2] != "configuration" { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return } if !requireSuperAdminManage(w, actor) { return } if r.Method != http.MethodPost { methodOrNotFound(w, r, true, http.MethodPost) return } request, ok := decodeRequired[hypervisor.StorageConfigurationRequest](w, body) if !ok { return } if err := hypervisor.ValidateStorageConfigurationRequest(request); err != nil { writePreDispatchMappedError(w, err) return } digests := hypervisor.StorageConfigurationPathDigests(request) expected := &operation.ExpectedState{ Kind: operation.ExpectedStorageConfiguration, ManagedPathDigest: digests[hypervisor.StorageRoleManaged], TemplatePathDigest: digests[hypervisor.StorageRoleTemplate], SeedPathDigest: digests[hypervisor.StorageRoleSeed], UserISOPathDigest: digests[hypervisor.StorageRoleUserISO], RescuePathDigest: digests[hypervisor.StorageRoleRescue], } s.runMutationExpected(w, r, body, actor, request.OperationID, "host-storage", "configure-storage", expected, true, http.StatusOK, hypervisor.StorageConfigurationAdmission(), func(ctx context.Context) (any, error) { return s.Hypervisor.ConfigureStorage(ctx, request) }) } func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if len(segments) == 2 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListVMs(r.Context()) writeNamedList(w, "vms", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.CreateVMRequest](w, body) if !ok { return } normalized, err := hypervisor.NormalizeStartupScript(request.StartupScript) if err != nil { writePreDispatchMappedError(w, err) return } request.StartupScript = normalized resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID if request.UUID == "" { request.UUID = operationUUID(resolvedOperationID) } request.DomainName, err = hypervisor.LibvirtDomainName(actor.UserID, request.UUID) if err != nil { writePreDispatchMappedError(w, err) return } if err := hypervisor.ValidateCreateVM(request); err != nil { writePreDispatchMappedError(w, err) return } s.runMutation(w, r, body, actor, request.OperationID, request.UUID, "create-vm", true, http.StatusCreated, hypervisor.CreateVMAdmission(request), func(ctx context.Context) (any, error) { return s.Hypervisor.CreateVM(ctx, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) < 3 || !apiUUIDPattern.MatchString(segments[2]) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_uuid", "VM UUID must be canonical") return } uuid := strings.ToLower(segments[2]) if len(segments) == 3 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid) writeResult(w, result, err) case http.MethodDelete: if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.DeleteVMRequest](w, body) if !ok { return } if !request.RemoveManagedDisks { writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true") return } s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-vm", true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.DeleteVM(ctx, uuid, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete) } return } switch segments[3] { case "force-delete": if len(segments) == 4 && r.Method == http.MethodPost { if !requireSuperAdminPrivilegedOperation(w, actor) { return } request, ok := decodeRequired[hypervisor.ForceDeleteVMRequest](w, body) if !ok { return } if !request.RemoveManagedDisks { writePreDispatchError(w, http.StatusUnprocessableEntity, "managed_disk_cleanup_required", "remove_managed_disks must be true") return } if !request.AckDataLoss { writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true") return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } if !operationIDPattern.MatchString(request.PriorOperationID) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_prior_operation_id", "prior_operation_id must contain 8-80 URL-safe characters") return } if operationID == request.PriorOperationID { writePreDispatchError(w, http.StatusUnprocessableEntity, "force_operation_id_reused", "force delete requires a new operation_id") return } existingReplay := false existing, existingErr := s.Journal.Get(operationID) if existingErr == nil { existingReplay = existing.Action == "force-delete-vm" && existing.ResourceID == uuid && existing.SupersedesOperationID == request.PriorOperationID } else if !errors.Is(existingErr, operation.ErrNotFound) { writeMappedError(w, existingErr, nil) return } if !existingReplay { prior, priorErr := s.Journal.Get(request.PriorOperationID) priorActionAllowed := priorErr == nil && (prior.Action == "delete-vm" || prior.Action == "service-suspend") if !priorActionAllowed || prior.ResourceID != uuid || prior.State != operation.StateFailed || prior.Error == nil || prior.Error.Code != "force_required" { writePreDispatchError(w, http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful stop to require force") return } } s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "force-delete-vm", nil, true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ForceDeleteVM(ctx, uuid, request) }, request.PriorOperationID) return } case "state": if len(segments) == 4 && r.Method == http.MethodGet { result, err := s.Hypervisor.GetVMState(r.Context(), uuid) writeResult(w, result, err) return } case "metadata": if len(segments) == 4 && r.Method == http.MethodGet { result, err := s.Hypervisor.GetVMMetadata(r.Context(), uuid) writeResult(w, result, err) return } case "actions": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.PowerRequest](w, body) if !ok { return } if !validPowerAction(request.Action) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_power_action", "action must be start, shutdown, reboot, reset, or destroy") return } s.runMutation(w, r, body, actor, request.OperationID, uuid, "power-"+string(request.Action), true, http.StatusOK, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.Power(ctx, uuid, request) }) return } case "settings": if len(segments) == 4 && (r.Method == http.MethodPatch || r.Method == http.MethodPost) { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.SettingsRequest](w, body) if !ok { return } if request.Hostname == nil && request.RootPassword == nil && request.SSHPublicKey == nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "empty_settings", "at least one setting is required") return } if !request.ConfirmRestart { writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true") return } expected := &operation.ExpectedState{ Kind: operation.ExpectedUpdateSettings, HostnameChanged: request.Hostname != nil, RootPasswordChanged: request.RootPassword != nil, SSHKeyChanged: request.SSHPublicKey != nil, } if request.Hostname != nil { expected.Hostname = *request.Hostname } s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "update-settings", expected, false, http.StatusOK, hypervisor.SeedMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateSettings(ctx, uuid, request) }) return } case "network": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ChangeNetworkRequest](w, body) if !ok { return } if !request.ConfirmRestart { writePreDispatchError(w, http.StatusUnprocessableEntity, "restart_confirmation_required", "confirm_restart must be true") return } if err := hypervisor.ValidatePublicNetworkIntent( request.NetworkSelector, request.PublicPoolKey, request.PublicPoolCIDR, request.PublicIPCIDR, request.Gateway, request.DNSServers, ); err != nil { writePreDispatchMappedError(w, err) return } var source hypervisor.LocalNetworkSource var err error if s.MutationExecutor != nil { source, err = hypervisor.ParseNetworkSelector(request.NetworkSelector) } else { source, err = s.Hypervisor.ResolveNetworkSelector(r.Context(), request.NetworkSelector) } if err != nil { writePreDispatchMappedError(w, err) return } expected := &operation.ExpectedState{ Kind: operation.ExpectedChangeNetwork, NetworkSelector: request.NetworkSelector, NetworkSourceType: source.Type, NetworkSourceName: source.Source, PublicPoolKey: request.PublicPoolKey, PublicPoolCIDR: request.PublicPoolCIDR, PublicIPCIDR: request.PublicIPCIDR, Gateway: request.Gateway, DNSServers: append([]string(nil), request.DNSServers...), } s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "change-network", expected, true, http.StatusAccepted, hypervisor.ChangeNetworkAdmission(uuid, request.PublicIPCIDR), func(ctx context.Context) (any, error) { return s.Hypervisor.ChangeNetwork(ctx, uuid, request) }) return } if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.NetworkStateRequest](w, body) if !ok { return } enabled := segments[4] == "resume" action := "network-" + segments[4] expected := &operation.ExpectedState{Kind: operation.ExpectedNetworkState, Enabled: enabled} s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, action, expected, false, http.StatusOK, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { err := s.Hypervisor.SetNetworkState(ctx, uuid, enabled, request) return map[string]bool{"network_enabled": enabled}, err }) return } case "service": if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "suspend" || segments[4] == "resume") { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.ServiceStateRequest](w, body) if !ok { return } enabled := segments[4] == "resume" expected := &operation.ExpectedState{Kind: operation.ExpectedServiceState, Enabled: enabled, StartIfSuspended: request.StartIfSuspended} s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "service-"+segments[4], expected, true, http.StatusOK, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.SetServiceState(ctx, uuid, enabled, request) }) return } case "reinstall": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ReinstallRequest](w, body) if !ok { return } if err := hypervisor.ValidateTemplateSnapshot(request.TemplateID, request.TemplateRevision, request.TemplateGenerationToken); err != nil { writePreDispatchMappedError(w, err) return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID s.runMutation(w, r, body, actor, request.OperationID, uuid, "reinstall-vm", true, http.StatusAccepted, hypervisor.ReinstallVMAdmission(uuid, request), func(ctx context.Context) (any, error) { return s.Hypervisor.Reinstall(ctx, uuid, request) }) return } case "metrics": if len(segments) == 4 && r.Method == http.MethodGet { result, err := s.Hypervisor.GetMetrics(r.Context(), uuid) writeResult(w, result, err) return } case "traffic": if len(segments) == 5 && segments[4] == "daily" && r.Method == http.MethodGet { days, err := strconv.Atoi(r.URL.Query().Get("days")) if err != nil || days < 1 || days > 30 { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_days", "days must be between 1 and 30") return } result, err := s.Hypervisor.GetDailyTraffic(r.Context(), uuid, days) writeNamedList(w, "traffic", result, err) return } case "console-ticket": if len(segments) == 4 && r.Method == http.MethodPost { if !requireManage(w, actor) { return } target, err := s.Hypervisor.GetConsole(r.Context(), uuid) if err != nil { writeMappedError(w, err, nil) return } if target.Protocol != "vnc" { writeError(w, http.StatusNotImplemented, "unsupported_console", "only VNC console targets are supported", nil) return } ticket, err := s.Console.Issue(target.Address) if err != nil { writeError(w, http.StatusServiceUnavailable, "console_unavailable", "console target is unavailable", nil) return } writeJSON(w, http.StatusCreated, ticket) return } case "snapshots": s.routeSnapshots(w, r, segments, body, actor, uuid) return case "firewall": if len(segments) == 4 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.GetFirewall(r.Context(), uuid) writeResult(w, result, err) case http.MethodPut: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.FirewallPolicy](w, body) if !ok { return } operationID := request.OperationID request, err := hypervisor.NormalizeFirewallPolicy(request) if err != nil { writePreDispatchMappedError(w, err) return } expected := &operation.ExpectedState{Kind: operation.ExpectedFirewall, FirewallRevision: request.Revision, FirewallDigest: request.PolicyDigest} s.runMutationExpected(w, r, body, actor, operationID, uuid, "apply-firewall", expected, false, http.StatusOK, hypervisor.FirewallMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyFirewall(ctx, uuid, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPut) } return } case "rescue": s.routeRescue(w, r, segments, body, actor, uuid) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) routeSnapshots(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) { if len(segments) == 4 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListSnapshots(r.Context(), uuid) writeNamedList(w, "snapshots", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.SnapshotRequest](w, body) if !ok { return } if request.SnapshotKey == "" { request.SnapshotKey = request.SnapshotID } if !catalogIDPattern.MatchString(request.SnapshotKey) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot", "snapshot key is invalid") return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID action := "create-snapshot:" + request.SnapshotKey + ":" + request.ConsistencyMode s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusCreated, hypervisor.CreateSnapshotAdmission(uuid, request.SnapshotKey), func(ctx context.Context) (any, error) { return s.Hypervisor.CreateSnapshot(ctx, uuid, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) < 5 || !resourceIDPattern.MatchString(segments[4]) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_snapshot_id", "snapshot ID is invalid") return } snapshotID := segments[4] if len(segments) == 5 { switch r.Method { case http.MethodGet: snapshots, err := s.Hypervisor.ListSnapshots(r.Context(), uuid) if err != nil { writeMappedError(w, err, nil) return } for _, snapshot := range snapshots { if snapshot.ID == snapshotID { writeJSON(w, http.StatusOK, snapshot) return } } writeError(w, http.StatusNotFound, "snapshot_not_found", "snapshot was not found", nil) case http.MethodDelete: if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body) if !ok { return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID s.runMutation(w, r, body, actor, request.OperationID, uuid, "delete-snapshot:"+snapshotID, true, http.StatusOK, hypervisor.DeleteSnapshotAdmission(uuid, snapshotID), func(ctx context.Context) (any, error) { return s.Hypervisor.DeleteSnapshot(ctx, uuid, snapshotID, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodDelete) } return } if len(segments) == 6 && segments[5] == "restore" && r.Method == http.MethodPost { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.SnapshotMutationRequest](w, body) if !ok { return } resolvedOperationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } request.OperationID = resolvedOperationID s.runMutation(w, r, body, actor, request.OperationID, uuid, "restore-snapshot:"+snapshotID, true, http.StatusAccepted, hypervisor.RestoreSnapshotAdmission(uuid, snapshotID, request.OperationID), func(ctx context.Context) (any, error) { return s.Hypervisor.RestoreSnapshot(ctx, uuid, snapshotID, request) }) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) routeRescue(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) { if len(segments) == 4 && r.Method == http.MethodGet { result, err := s.Hypervisor.GetRescue(r.Context(), uuid) writeResult(w, result, err) return } if len(segments) == 5 && r.Method == http.MethodPost && (segments[4] == "enter" || segments[4] == "exit" || segments[4] == "force-start") { if !requireManage(w, actor) { return } if segments[4] == "enter" { request, ok := decodeRequired[hypervisor.RescueRequest](w, body) if !ok { return } if !catalogIDPattern.MatchString(request.TemplateID) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_template_id", "rescue template id is invalid") return } s.runMutation(w, r, body, actor, request.OperationID, uuid, "enter-rescue", true, http.StatusAccepted, hypervisor.EnterRescueAdmission(uuid, request.TemplateID), func(ctx context.Context) (any, error) { return s.Hypervisor.EnterRescue(ctx, uuid, request) }) } else if segments[4] == "force-start" { request, ok := decodeRequired[hypervisor.ForceRescueStartRequest](w, body) if !ok { return } if !request.AckDataLoss { writePreDispatchError(w, http.StatusUnprocessableEntity, "data_loss_ack_required", "ack_data_loss must be true") return } s.runMutation(w, r, body, actor, request.OperationID, uuid, "force-start-rescue", true, http.StatusAccepted, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ForceStartRescue(ctx, uuid, request) }) } else { request, ok := decodeRequired[hypervisor.ExitRescueRequest](w, body) if !ok { return } if request.NetworkSuspended == nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "network_state_required", "network_suspended must be provided") return } action := "exit-rescue:connected" if *request.NetworkSuspended { action = "exit-rescue:suspended" } s.runMutation(w, r, body, actor, request.OperationID, uuid, action, true, http.StatusAccepted, hypervisor.DomainMutationAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ExitRescue(ctx, uuid, request) }) } return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) routeTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if !actor.IsAdmin && !actor.IsSuperAdmin { writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required") return } if len(segments) == 2 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListTemplates(r.Context()) writeNamedList(w, "templates", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ImportTemplateRequest](w, body) if !ok { return } if err := hypervisor.ValidateTemplateSnapshot(request.ID, request.Revision, request.GenerationToken); err != nil { writePreDispatchMappedError(w, err) return } if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil { writePreDispatchMappedError(w, err) return } s.runAsyncMutation(w, r, body, actor, request.OperationID, "template:"+request.ID, "import-template", false, hypervisor.ImportTemplateAdmission(request), func(ctx context.Context) (any, error) { return s.Hypervisor.ImportTemplate(ctx, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) == 3 && resourceIDPattern.MatchString(segments[2]) { templateID := segments[2] switch r.Method { case http.MethodPatch: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.UpdateTemplateRequest](w, body) if !ok { return } if strings.TrimSpace(request.Name) == "" || len(request.Name) > 120 { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_template_name", "template name must contain 1-120 characters") return } s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "update-template", false, http.StatusOK, hypervisor.UpdateTemplateAdmission(templateID, request), func(ctx context.Context) (any, error) { return s.Hypervisor.UpdateTemplate(ctx, templateID, request) }) case http.MethodDelete: if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.MutationRequest](w, body) if !ok { return } s.runMutation(w, r, body, actor, request.OperationID, "template:"+templateID, "delete-template", true, http.StatusOK, hypervisor.DeleteTemplateAdmission(templateID), func(ctx context.Context) (any, error) { err := s.Hypervisor.DeleteTemplate(ctx, templateID) return catalogDeleteResult("template", templateID, 0, err == nil), err }) default: methodOrNotFound(w, r, true, http.MethodPatch, http.MethodDelete) } return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) routeRescueTemplates(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if !actor.IsAdmin && !actor.IsSuperAdmin { writePreDispatchError(w, http.StatusForbidden, "forbidden", "administrator privileges are required") return } if len(segments) == 2 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListRescueTemplates(r.Context()) writeNamedList(w, "templates", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ImportRescueTemplateRequest](w, body) if !ok { return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } if request.ID == "" { request.ID = operationCatalogID(operationID) } if !catalogIDPattern.MatchString(request.ID) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_rescue_template_id", "id must contain 32 lowercase hexadecimal characters") return } if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil { writePreDispatchMappedError(w, err) return } s.runAsyncMutation(w, r, body, actor, request.OperationID, "rescue-template:"+request.ID, "import-rescue-template", false, hypervisor.ImportRescueTemplateAdmission(request), func(ctx context.Context) (any, error) { return s.Hypervisor.ImportRescueTemplate(ctx, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.MutationRequest](w, body) if !ok { return } id := segments[2] s.runMutation(w, r, body, actor, request.OperationID, "rescue-template:"+id, "delete-rescue-template", true, http.StatusOK, hypervisor.DeleteRescueTemplateAdmission(id), func(ctx context.Context) (any, error) { err := s.Hypervisor.DeleteRescueTemplate(ctx, id) return catalogDeleteResult("rescue_template", id, 0, err == nil), err }) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) routeUserISOs(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { ownerID, err := ownerFromRequest(r, actor) if err != nil { writePreDispatchError(w, http.StatusForbidden, "forbidden", err.Error()) return } if len(segments) == 2 { switch r.Method { case http.MethodGet: result, err := s.Hypervisor.ListUserISOs(r.Context(), ownerID) writeNamedList(w, "user_isos", result, err) case http.MethodPost: if !requireManage(w, actor) { return } request, ok := decodeRequired[hypervisor.ImportUserISORequest](w, body) if !ok { return } if request.OwnerID == 0 { request.OwnerID = ownerID } if request.OwnerID != ownerID && !actor.IsAdmin && !actor.IsSuperAdmin { writePreDispatchError(w, http.StatusForbidden, "forbidden", "cannot import ISO for another owner") return } operationID, err := resolveOperationID(actor.OperationID, request.OperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } if request.ID == "" { request.ID = operationCatalogID(operationID) } if !catalogIDPattern.MatchString(request.ID) { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_user_iso_id", "id must contain 32 lowercase hexadecimal characters") return } if err := validateImportSource(request.SourceURL, request.SourceToken, request.SHA256); err != nil { writePreDispatchMappedError(w, err) return } s.runAsyncMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", request.OwnerID, request.ID), "import-user-iso", false, hypervisor.ImportUserISOAdmission(request), func(ctx context.Context) (any, error) { return s.Hypervisor.ImportUserISO(ctx, request) }) default: methodOrNotFound(w, r, true, http.MethodGet, http.MethodPost) } return } if len(segments) == 3 && catalogIDPattern.MatchString(segments[2]) && r.Method == http.MethodDelete { if !requireManage(w, actor) { return } request, ok := decodeOptional[hypervisor.MutationRequest](w, body) if !ok { return } isoID := segments[2] s.runMutation(w, r, body, actor, request.OperationID, fmt.Sprintf("user-iso:%d:%s", ownerID, isoID), "delete-user-iso", true, http.StatusOK, hypervisor.DeleteUserISOAdmission(ownerID, isoID), func(ctx context.Context) (any, error) { err := s.Hypervisor.DeleteUserISO(ctx, isoID, ownerID) return catalogDeleteResult("user_iso", isoID, ownerID, err == nil), err }) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func catalogDeleteResult(resourceType, id string, ownerID int64, deleted bool) hypervisor.CatalogDeleteResult { return hypervisor.CatalogDeleteResult{ ResourceType: resourceType, ID: id, OwnerID: ownerID, Deleted: deleted, ManagedImageAbsent: deleted, ManagedMetadataAbsent: deleted, ManagedArtifactsAbsent: deleted, } } func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if len(segments) < 3 || !resourceIDPattern.MatchString(segments[2]) { writeError(w, http.StatusNotFound, "operation_not_found", "operation was not found", nil) return } operationID := segments[2] if len(segments) == 3 && r.Method == http.MethodGet { record, err := s.Journal.Get(operationID) if err != nil { writeMappedError(w, err, nil) return } if record.Action == "configure-storage" && !requireSuperAdminManage(w, actor) { return } writeJSON(w, http.StatusOK, record) return } if len(segments) == 4 && segments[3] == "inspect" && r.Method == http.MethodGet { if !requireSuperAdminInspection(w, actor) { return } stored, err := s.Journal.Get(operationID) if err != nil { writeMappedError(w, err, nil) return } ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) defer cancel() inspection, err := s.Hypervisor.InspectOperation(ctx, stored) if err != nil { writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, inspection) return } if len(segments) == 4 && segments[3] == "fence-resolution" && r.Method == http.MethodGet { if !requireSuperAdminInspection(w, actor) { return } resolution, err := s.Journal.GetFenceResolution(operationID) if errors.Is(err, operation.ErrNotFound) { writeError(w, http.StatusNotFound, "fence_resolution_not_found", "operation fence resolution was not found", nil) return } if err != nil { writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, fenceResolutionResponse{FenceResolution: resolution, Replayed: false}) return } if len(segments) == 4 && segments[3] == "resolve-fence" && r.Method == http.MethodPost { if !requireSuperAdminFenceResolution(w, actor) { return } if actor.OperationID == "" || actor.OperationID != operationID { writeError(w, http.StatusBadRequest, "operation_id_mismatch", "actor operation_id must match the fenced operation", nil) return } request, ok := decodeRequired[operation.FenceResolutionRequest](w, body) if !ok { return } if err := operation.ValidateFenceResolutionRequest(request); err != nil { writeError(w, http.StatusBadRequest, "invalid_fence_resolution", err.Error(), nil) return } resolution, replayed, err := s.Journal.ResolveResourceFence(operationID, actor.UserID, request) switch { case errors.Is(err, operation.ErrNotFound): writeError(w, http.StatusNotFound, "operation_not_found", "operation was not found", nil) return case errors.Is(err, operation.ErrFenceNotResolvable): writeError(w, http.StatusConflict, "operation_fence_not_resolvable", "operation does not hold an active effect fence", nil) return case errors.Is(err, operation.ErrFenceResolutionConflict): writeError(w, http.StatusConflict, "operation_fence_resolution_conflict", "operation fence already has a different audit resolution", nil) return case errors.Is(err, operation.ErrFenceInspectionStale): writeError(w, http.StatusConflict, "fence_inspection_stale", "operation fence resolution requires a fresh read-only inspection", nil) return case err != nil: writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, fenceResolutionResponse{FenceResolution: resolution, Replayed: replayed}) return } writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } func (s *Server) runMutation( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, bodyOperationID, resourceID, action string, destructive bool, successStatus int, admission hypervisor.MutationAdmission, call func(context.Context) (any, error), ) { s.runMutationExpected(w, r, body, actor, bodyOperationID, resourceID, action, nil, destructive, successStatus, admission, call) } func (s *Server) runMutationExpected( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, bodyOperationID, resourceID, action string, expectedState *operation.ExpectedState, destructive bool, successStatus int, admission hypervisor.MutationAdmission, call func(context.Context) (any, error), supersedesOperationID ...string, ) { operationID, err := resolveOperationID(actor.OperationID, bodyOperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } if expectedState != nil { copyExpected := *expectedState copyExpected.TransitionID = operationID copyExpected.DNSServers = append([]string(nil), expectedState.DNSServers...) expectedState = ©Expected } supersedes := "" if len(supersedesOperationID) != 0 { supersedes = supersedesOperationID[0] } spec := operation.Spec{ OperationID: operationID, ResourceID: resourceID, Action: action, Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body), Destructive: destructive, ExpectedState: expectedState, LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO, SupersedesOperationID: supersedes, } if s.MutationExecutor != nil { s.runIsolatedMutation(w, r, body, actor, spec, s.isolatedMutationDeadline(action)) return } if _, trusted := r.Context().Value(trustedWorkerRequestKey{}).(struct{}); !trusted { writePreDispatchError(w, http.StatusServiceUnavailable, "mutation_worker_unavailable", "isolated mutation worker is unavailable") return } ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) defer cancel() record, replayed, callErr := s.Journal.Execute(ctx, spec, func(ctx context.Context) (json.RawMessage, error) { result, err := call(ctx) if err != nil { return nil, err } encoded, err := json.Marshal(result) if err != nil { return nil, fmt.Errorf("encode operation result: %w", err) } return encoded, nil }) logOperationOutcome(record) if errors.Is(callErr, operation.ErrFingerprint) { writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil) return } if errors.Is(callErr, operation.ErrOperationStarting) { writeOperationStarting(w, record) return } if operation.WasNotDispatched(record) { writeOperationFailure(w, record, replayed) return } if callErr != nil { if record.State == operation.StateFailed { writeOperationFailure(w, record, replayed) return } writeMappedError(w, callErr, &record) return } if record.State == operation.StateFailed { writeOperationFailure(w, record, replayed) return } if record.State == operation.StateDispatching || record.State == operation.StateReserved { writeJSON(w, http.StatusAccepted, operationWire(record, replayed)) return } if replayed { successStatus = http.StatusOK } writeJSON(w, successStatus, operationWire(record, replayed)) } func (s *Server) runAsyncMutation( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, bodyOperationID, resourceID, action string, destructive bool, admission hypervisor.MutationAdmission, call func(context.Context) (any, error), ) { operationID, err := resolveOperationID(actor.OperationID, bodyOperationID) if err != nil { writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) return } spec := operation.Spec{ OperationID: operationID, ResourceID: resourceID, Action: action, Fingerprint: operation.Fingerprint(r.Method, auth.RequestTarget(r), body), Destructive: destructive, LockKeys: append([]string(nil), admission.LockKeys...), HeavyIO: admission.HeavyIO, } if s.MutationExecutor != nil { s.runIsolatedMutation(w, r, body, actor, spec, catalogImportTimeout) return } if _, trusted := r.Context().Value(trustedWorkerRequestKey{}).(struct{}); !trusted { writePreDispatchError(w, http.StatusServiceUnavailable, "mutation_worker_unavailable", "isolated mutation worker is unavailable") return } record, replayed, err := s.Journal.Reserve(spec) if errors.Is(err, operation.ErrFingerprint) { writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil) return } if err != nil { writeMappedError(w, err, nil) return } canStart := record.State == operation.StateReserved if canStart && !s.claimAsync(operationID) { writeOperationStarting(w, record) return } if canStart { select { case s.asyncSlots <- struct{}{}: default: failed, failErr := s.Journal.FailNotDispatched(operationID) s.releaseAsync(operationID) if failErr != nil { writeMappedError(w, failErr, &record) return } writeOperationFailure(w, failed, false) return } dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout) started, _, done, startErr := s.Journal.ExecuteBackground( dispatchCtx, s.backgroundCtx, catalogImportTimeout, spec, func(ctx context.Context) (json.RawMessage, error) { result, callErr := call(ctx) if callErr != nil { return nil, callErr } return json.Marshal(result) }, ) dispatchCancel() if startErr != nil || done == nil { <-s.asyncSlots s.releaseAsync(operationID) if startErr != nil { if errors.Is(startErr, operation.ErrOperationStarting) { writeOperationStarting(w, started) return } if started.State == operation.StateFailed && started.Error != nil && started.Error.Code == "operation_not_dispatched" { writeOperationFailure(w, started, false) return } writeMappedError(w, startErr, &started) return } record = started } else { record = started s.asyncWG.Add(1) go func() { defer s.asyncWG.Done() defer func() { <-s.asyncSlots }() defer s.releaseAsync(operationID) <-done if final, getErr := s.Journal.Get(operationID); getErr == nil { logOperationOutcome(final) } }() } } writeJSON(w, http.StatusAccepted, operationWire(record, replayed)) } func (s *Server) runIsolatedMutation( w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, spec operation.Spec, deadline time.Duration, ) { if deadline <= 0 { deadline = s.OperationTimeout } actor.OperationID = spec.OperationID request := worker.Request{ Version: worker.ProtocolVersion, Method: r.Method, RequestTarget: auth.RequestTarget(r), Body: string(body), Actor: actor, OperationID: spec.OperationID, Fingerprint: spec.Fingerprint, } dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout) record, replayed, done, err := s.Journal.ExecuteBackground( dispatchCtx, s.backgroundCtx, deadline, spec, func(callCtx context.Context) (json.RawMessage, error) { return s.MutationExecutor.Execute(callCtx, request) }, ) dispatchCancel() if done != nil { s.asyncWG.Add(1) go func(operationID string) { defer s.asyncWG.Done() <-done if final, getErr := s.Journal.Get(operationID); getErr == nil { logOperationOutcome(final) } }(spec.OperationID) } if errors.Is(err, operation.ErrFingerprint) { writeError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request", nil) return } if errors.Is(err, operation.ErrOperationStarting) { writeOperationStarting(w, record) return } if operation.WasNotDispatched(record) { writeJSON(w, http.StatusOK, operationWire(record, replayed)) return } if err != nil { writeMappedError(w, err, &record) return } status := http.StatusAccepted if record.State == operation.StateSucceeded || record.State == operation.StateFailed { status = http.StatusOK } writeJSON(w, status, operationWire(record, replayed)) } func (s *Server) isolatedMutationDeadline(action string) time.Duration { deadline := s.OperationTimeout if action == "configure-storage" && deadline > storageConfigurationWorkerTimeout { return storageConfigurationWorkerTimeout } return deadline } func (s *Server) beginRequest() bool { s.lifecycleMu.Lock() defer s.lifecycleMu.Unlock() if s.closing { return false } s.requestWG.Add(1) return true } // Shutdown stops background imports and waits for every admitted API handler // and import watcher. The journal's own workers are drained separately before // libvirt or BoltDB are closed. func (s *Server) Shutdown(ctx context.Context) error { s.lifecycleMu.Lock() if !s.closing { s.closing = true s.backgroundCancel() } s.lifecycleMu.Unlock() if err := waitGroup(ctx, &s.requestWG); err != nil { return err } return waitGroup(ctx, &s.asyncWG) } func waitGroup(ctx context.Context, group *sync.WaitGroup) error { done := make(chan struct{}) go func() { group.Wait() close(done) }() select { case <-done: return nil case <-ctx.Done(): return ctx.Err() } } func (s *Server) claimAsync(operationID string) bool { s.asyncMu.Lock() defer s.asyncMu.Unlock() if s.asyncRunning == nil { s.asyncRunning = make(map[string]struct{}) } if _, exists := s.asyncRunning[operationID]; exists { return false } s.asyncRunning[operationID] = struct{}{} return true } func (s *Server) releaseAsync(operationID string) { s.asyncMu.Lock() defer s.asyncMu.Unlock() delete(s.asyncRunning, operationID) } func operationWire(record operation.Record, replayed bool) operationResponse { return operationResponse{ OperationID: record.OperationID, Status: record.State, Replayed: replayed, Result: record.Result, Error: record.Error, CauseCode: record.CauseCode, EffectMayHaveApplied: record.EffectMayHaveApplied, SafeToResubmit: record.SafeToResubmit, Retryable: false, } } // Log only bounded identifiers and stable codes. Free-form backend errors, // request bodies, guest credentials, and host paths never enter this line. func logOperationOutcome(record operation.Record) { if record.State != operation.StateFailed { return } errorCode := "" if record.Error != nil && len(record.Error.Code) <= 64 && stableCodePattern.MatchString(record.Error.Code) { errorCode = record.Error.Code } log.Printf("Agent operation outcome operation_id=%s action=%s status=%s error_code=%s cause_code=%s", record.OperationID, record.Action, record.State, errorCode, record.CauseCode) } func resolveOperationID(actorID, bodyID string) (string, error) { if actorID == "" && bodyID == "" { return "", errors.New("operation_id is required in X-Virt-Actor or request body") } if actorID != "" && bodyID != "" && actorID != bodyID { return "", errors.New("actor and body operation_id values differ") } resolved := bodyID if actorID != "" { resolved = actorID } if !operationIDPattern.MatchString(resolved) { return "", errors.New("operation_id must contain 8-80 URL-safe characters") } return resolved, nil } func operationUUID(operationID string) string { digest := sha256.Sum256([]byte("virt-cloud-operation:" + operationID)) digest[6] = digest[6]&0x0f | 0x50 digest[8] = digest[8]&0x3f | 0x80 return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", digest[0:4], digest[4:6], digest[6:8], digest[8:10], digest[10:16]) } func operationCatalogID(operationID string) string { digest := sha256.Sum256([]byte("virt-cloud-catalog:" + operationID)) return fmt.Sprintf("%x", digest[:16]) } func validPowerAction(action hypervisor.PowerAction) bool { switch action { case hypervisor.PowerStart, hypervisor.PowerShutdown, hypervisor.PowerReboot, hypervisor.PowerReset, hypervisor.PowerDestroy: return true default: return false } } func validateImportSource(raw, token, digest string) error { parsed, err := url.Parse(raw) if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" { return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_url", "source_url must be an HTTPS URL without credentials or fragment", err) } if digest != "" && !sha256Pattern.MatchString(digest) { return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_sha256", "sha256 must be 64 lowercase hexadecimal characters", nil) } if token != "" && !sourceTokenPattern.MatchString(token) { return hypervisor.NewError(hypervisor.ErrInvalid, "invalid_source_token", "source_token must contain 32-512 URL-safe characters", nil) } return nil } func ownerFromRequest(r *http.Request, actor auth.Actor) (int64, error) { value := r.URL.Query().Get("owner_id") if value == "" { return actor.UserID, nil } ownerID, err := strconv.ParseInt(value, 10, 64) if err != nil || ownerID <= 0 { return 0, errors.New("owner_id is invalid") } if ownerID != actor.UserID && !actor.IsAdmin && !actor.IsSuperAdmin { return 0, errors.New("cannot access another owner's ISO") } return ownerID, nil } func requireManage(w http.ResponseWriter, actor auth.Actor) bool { if !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to manage this resource") return false } return true } func requireSuperAdminManage(w http.ResponseWriter, actor auth.Actor) bool { if !actor.IsSuperAdmin || !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to configure host storage") return false } return true } func requireSuperAdminPrivilegedOperation(w http.ResponseWriter, actor auth.Actor) bool { if !actor.IsSuperAdmin || !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to perform this privileged operation") return false } return true } func requireSuperAdminInspection(w http.ResponseWriter, actor auth.Actor) bool { if !actor.IsSuperAdmin || !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to inspect operation state") return false } return true } func requireSuperAdminFenceResolution(w http.ResponseWriter, actor auth.Actor) bool { if !actor.IsSuperAdmin || !actor.CanManage { writePreDispatchError(w, http.StatusForbidden, "forbidden", "actor is not authorized to resolve operation effect fences") return false } return true } func decodeRequired[T any](w http.ResponseWriter, body []byte) (T, bool) { var value T if len(body) == 0 { writePreDispatchError(w, http.StatusBadRequest, "invalid_json", "JSON request body is required") return value, false } if err := decodeStrict(body, &value); err != nil { writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error()) return value, false } return value, true } func decodeOptional[T any](w http.ResponseWriter, body []byte) (T, bool) { var value T if len(body) == 0 { return value, true } if err := decodeStrict(body, &value); err != nil { writePreDispatchError(w, http.StatusBadRequest, "invalid_json", err.Error()) return value, false } return value, true } func decodeStrict(body []byte, target any) error { if !utf8.Valid(body) { return errors.New("invalid JSON: body must be valid UTF-8") } decoder := json.NewDecoder(bytes.NewReader(body)) decoder.DisallowUnknownFields() if err := decoder.Decode(target); err != nil { return fmt.Errorf("invalid JSON: %w", err) } var extra any if err := decoder.Decode(&extra); err != io.EOF { return errors.New("invalid JSON: exactly one object is required") } return nil } func readBoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, error) { if r.Body == nil { return []byte{}, nil } r.Body = http.MaxBytesReader(w, r.Body, limit) defer r.Body.Close() return io.ReadAll(r.Body) } func writeResult(w http.ResponseWriter, value any, err error) { if err != nil { writeMappedError(w, err, nil) return } writeJSON(w, http.StatusOK, value) } func writeNamedList[T any](w http.ResponseWriter, name string, value []T, err error) { if err != nil { writeMappedError(w, err, nil) return } if value == nil { value = make([]T, 0) } writeJSON(w, http.StatusOK, map[string]any{name: value}) } func writeMappedError(w http.ResponseWriter, err error, record *operation.Record) { status, code, message := classifyError(err) writeError(w, status, code, message, record) } // writePreDispatchMappedError is only for validation or dependency checks // completed before Journal.Execute/Reserve. The explicit marker lets the // panel distinguish a proven no-journal outcome from a lost response after a // mutation may already have crossed its durable dispatch boundary. func writePreDispatchMappedError(w http.ResponseWriter, err error) { status, code, message := classifyError(err) writePreDispatchError(w, status, code, message) } // writeOperationStarting reports the active admission owner without exposing a // stale failed/not-dispatched record from before that owner acquired admission. // The projected reserved state is diagnostic only; it never authorizes a // caller to dispatch the mutation again. The journal remains authoritative. func writeOperationStarting(w http.ResponseWriter, record operation.Record) { record.State = operation.StateReserved record.Result = nil record.Error = nil record.CauseCode = "" record.Tombstone = false writeError(w, http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting", &record) } func classifyError(err error) (int, string, string) { if errors.Is(err, operation.ErrNotFound) { return http.StatusNotFound, "operation_not_found", "operation was not found" } if errors.Is(err, operation.ErrBusy) { return http.StatusServiceUnavailable, "agent_busy", "Agent mutation capacity is exhausted" } if errors.Is(err, operation.ErrOperationStarting) { return http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting" } if errors.Is(err, operation.ErrResourceBlocked) { return http.StatusConflict, "resource_operation_blocked", "resource is fenced by a prior failed operation whose effect may have applied" } if errors.Is(err, operation.ErrInvalidSupersession) { return http.StatusConflict, "force_delete_not_authorized", "force delete requires the matching graceful operation" } var backend *hypervisor.Error if errors.As(err, &backend) { status := statusForKind(backend.Kind) code := backend.Code if code == "" { code = "backend_error" } message := backend.Message if message == "" { message = http.StatusText(status) } return status, code, message } switch { case errors.Is(err, hypervisor.ErrNotFound): return http.StatusNotFound, "not_found", "resource was not found" case errors.Is(err, hypervisor.ErrConflict): return http.StatusConflict, "conflict", "resource conflict" case errors.Is(err, hypervisor.ErrUnsupported): return http.StatusNotImplemented, "unsupported", "operation is not supported" case errors.Is(err, hypervisor.ErrInvalid): return http.StatusUnprocessableEntity, "invalid_request", "request is invalid" case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): return http.StatusServiceUnavailable, "backend_timeout", "backend operation did not complete" default: return http.StatusServiceUnavailable, "backend_error", "backend operation failed" } } func statusForKind(kind error) int { switch kind { case hypervisor.ErrNotFound: return http.StatusNotFound case hypervisor.ErrConflict: return http.StatusConflict case hypervisor.ErrUnsupported: return http.StatusNotImplemented case hypervisor.ErrInvalid: return http.StatusUnprocessableEntity default: return http.StatusServiceUnavailable } } func writeOperationFailure(w http.ResponseWriter, record operation.Record, replayed bool) { status := http.StatusServiceUnavailable if record.Error != nil { switch record.Error.Code { case "unsupported": status = http.StatusNotImplemented case "not_found": status = http.StatusNotFound case "conflict": status = http.StatusConflict case "force_required", "force_delete_not_authorized": status = http.StatusConflict default: if strings.HasSuffix(record.Error.Code, "_not_found") { status = http.StatusNotFound } else if strings.HasPrefix(record.Error.Code, "invalid_") { status = http.StatusUnprocessableEntity } } } writeJSON(w, status, operationWire(record, replayed)) } func methodOrNotFound(w http.ResponseWriter, r *http.Request, exists bool, methods ...string) { if !exists { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return } w.Header().Set("Allow", strings.Join(methods, ", ")) writePreDispatchError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method is not allowed") } func writePreDispatchError(w http.ResponseWriter, status int, code, message string) { writeJSON(w, status, errorEnvelope{Error: errorBody{ Code: code, Message: message, Retryable: retryableError(status, code), DispatchState: "not_dispatched", }}) } func writeError(w http.ResponseWriter, status int, code, message string, record *operation.Record) { writeJSON(w, status, errorEnvelope{Error: errorBody{ Code: code, Message: message, Retryable: retryableError(status, code), Operation: record, }}) } func retryableError(status int, code string) bool { return false } func writeJSON(w http.ResponseWriter, status int, value any) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(status) _ = json.NewEncoder(w).Encode(value) } func setResponseHeaders(w http.ResponseWriter) { w.Header().Set("Cache-Control", "no-store") w.Header().Set("X-Content-Type-Options", "nosniff") }