You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2167 lines
90 KiB
Go
2167 lines
90 KiB
Go
// Package api implements the versioned panel-to-agent HTTP boundary.
|
|
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"g.luoli.live/root/virt/agent/internal/auth"
|
|
"g.luoli.live/root/virt/agent/internal/capability"
|
|
"g.luoli.live/root/virt/agent/internal/config"
|
|
"g.luoli.live/root/virt/agent/internal/domaincreateplan"
|
|
"g.luoli.live/root/virt/agent/internal/domaincreatevalidate"
|
|
"g.luoli.live/root/virt/agent/internal/domainpreflight"
|
|
"g.luoli.live/root/virt/agent/internal/hypervisor"
|
|
"g.luoli.live/root/virt/agent/internal/operation"
|
|
"g.luoli.live/root/virt/agent/internal/startup"
|
|
)
|
|
|
|
type Options struct {
|
|
Config config.Config
|
|
Warnings []config.Warning
|
|
Store *operation.Store
|
|
Capabilities capability.Provider
|
|
Backend hypervisor.Backend
|
|
// HostBackendReady is explicit so a static provider cannot accidentally
|
|
// admit mutating operations in a production process. Tests may set it when
|
|
// they intentionally use a deterministic fake provider.
|
|
HostBackendReady bool
|
|
// TLSReady is set after the process successfully loads the configured
|
|
// certificate material. A path being configured is not proof that a remote
|
|
// listener can actually enforce mTLS.
|
|
TLSReady bool
|
|
Auth *auth.Verifier
|
|
Logger *slog.Logger
|
|
}
|
|
|
|
type Server struct {
|
|
cfg config.Config
|
|
warnings []config.Warning
|
|
store *operation.Store
|
|
caps capability.Provider
|
|
backend hypervisor.Backend
|
|
stateMu sync.RWMutex
|
|
hostReady bool
|
|
tlsReady bool
|
|
// readinessProbeMu serializes health-triggered host probes. Health is an
|
|
// unauthenticated liveness endpoint, so without a small single-flight
|
|
// boundary every remote poll could start a fresh virsh/libvirt probe.
|
|
readinessProbeMu sync.Mutex
|
|
readinessProbeAt time.Time
|
|
readinessProbeDone chan struct{}
|
|
auth *auth.Verifier
|
|
logger *slog.Logger
|
|
mux *http.ServeMux
|
|
}
|
|
|
|
// healthReadinessProbeTTL bounds how often the unauthenticated health route
|
|
// starts a new host capability probe. Mutating request gates continue to use
|
|
// refreshHostReadiness directly so they retain a fresh, fail-closed probe.
|
|
const healthReadinessProbeTTL = time.Second
|
|
|
|
type healthResponse struct {
|
|
Status string `json:"status"`
|
|
Readiness config.Readiness `json:"readiness"`
|
|
AgentID string `json:"agent_id,omitempty"`
|
|
ProtocolVersion string `json:"protocol_version"`
|
|
Warnings []config.Warning `json:"warnings,omitempty"`
|
|
Now time.Time `json:"now"`
|
|
}
|
|
|
|
type operationRequest struct {
|
|
OperationID string `json:"operation_id"`
|
|
IdempotencyKey string `json:"idempotency_key"`
|
|
Kind string `json:"kind"`
|
|
Target string `json:"target,omitempty"`
|
|
Payload json.RawMessage `json:"payload,omitempty"`
|
|
}
|
|
|
|
type errorResponse struct {
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
func New(opts Options) (*Server, error) {
|
|
if err := opts.Config.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
if opts.Capabilities == nil {
|
|
return nil, fmt.Errorf("capability provider is required")
|
|
}
|
|
if opts.Auth == nil {
|
|
if opts.Config.RequireSigned {
|
|
// A required verifier with no configured key must remain unavailable.
|
|
// Falling back to NewDisabled here would silently turn a required
|
|
// authenticated control plane into an unsigned one whenever the
|
|
// caller forgot to inject Auth.
|
|
verifier, err := auth.NewWithNonceConsumerForAgent(nil, true, opts.Store, opts.Config.AgentID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("configure request signing verifier: %w", err)
|
|
}
|
|
opts.Auth = verifier
|
|
} else {
|
|
opts.Auth = auth.NewDisabled()
|
|
}
|
|
}
|
|
if opts.Logger == nil {
|
|
opts.Logger = slog.New(slog.NewTextHandler(io.Discard, nil))
|
|
}
|
|
s := &Server{
|
|
cfg: opts.Config,
|
|
warnings: append([]config.Warning(nil), opts.Warnings...),
|
|
store: opts.Store,
|
|
caps: opts.Capabilities,
|
|
backend: opts.Backend,
|
|
hostReady: opts.HostBackendReady,
|
|
tlsReady: !opts.Config.RequireMTLS || opts.TLSReady,
|
|
auth: opts.Auth,
|
|
logger: opts.Logger,
|
|
mux: http.NewServeMux(),
|
|
}
|
|
s.routes()
|
|
return s, nil
|
|
}
|
|
|
|
func (s *Server) Handler() http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/v1/health" {
|
|
if !s.authorized(r) {
|
|
writeJSON(w, http.StatusUnauthorized, errorResponse{Error: "client authentication required"})
|
|
return
|
|
}
|
|
body, err := readBody(r, s.cfg.MaxBodyBytes)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusRequestEntityTooLarge, errorResponse{Error: "request body exceeds limit"})
|
|
return
|
|
}
|
|
r.Body = io.NopCloser(bytes.NewReader(body))
|
|
if err := s.auth.Verify(r, body, time.Now().UTC()); err != nil {
|
|
if errors.Is(err, auth.ErrUnavailable) {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "request signing is not ready"})
|
|
} else {
|
|
writeJSON(w, http.StatusUnauthorized, errorResponse{Error: "invalid signed request envelope"})
|
|
}
|
|
return
|
|
}
|
|
}
|
|
s.mux.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
func (s *Server) routes() {
|
|
s.mux.HandleFunc("GET /v1/health", s.health)
|
|
s.mux.HandleFunc("GET /v1/capabilities", s.capabilities)
|
|
s.mux.HandleFunc("POST /v1/operations", s.createOperation)
|
|
s.mux.HandleFunc("GET /v1/operations/{id}", s.getOperation)
|
|
s.mux.HandleFunc("POST /v1/operations/{id}/reconcile", s.reconcileOperation)
|
|
// A direct read-only probe is useful during enrollment and does not mutate
|
|
// host networking, domains, storage, or firewall state.
|
|
s.mux.HandleFunc("POST /v1/agent/test", s.agentTest)
|
|
}
|
|
|
|
func (s *Server) health(w http.ResponseWriter, r *http.Request) {
|
|
// Readiness is a live host fact, not only a startup result. A libvirt
|
|
// daemon or socket can disappear after the process starts; refresh it on
|
|
// health and write boundaries so monitoring does not report a stale green
|
|
// state and mutating requests fail closed.
|
|
s.refreshHostReadinessCached(r.Context())
|
|
readiness := s.readiness()
|
|
status := "ok"
|
|
if readiness != config.ReadinessReady {
|
|
status = "degraded"
|
|
}
|
|
warnings := append([]config.Warning(nil), s.warnings...)
|
|
if s.store == nil || !s.store.Writable() {
|
|
warnings = append(warnings, config.Warning{Code: "operation_journal_unavailable", Message: "operation writes are disabled until the local journal is available"})
|
|
}
|
|
if s.cfg.AgentID == "" {
|
|
// Keep this warning stable and non-sensitive for monitoring clients.
|
|
warnings = append(warnings, config.Warning{Code: "agent_identity_missing", Message: "agent identity is not configured; mutating requests remain disabled"})
|
|
}
|
|
if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) {
|
|
warnings = append(warnings, config.Warning{Code: "request_signing_not_ready", Message: "signed request verification is not ready; protected requests remain disabled"})
|
|
}
|
|
if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 {
|
|
warnings = append(warnings, config.Warning{Code: "client_certificate_not_pinned", Message: "no client certificate fingerprint is registered; protected requests remain disabled"})
|
|
}
|
|
if s.cfg.RequireMTLS && !s.tlsReady {
|
|
warnings = append(warnings, config.Warning{Code: "tls_listener_not_ready", Message: "TLS material is not loaded; protected writes remain disabled"})
|
|
}
|
|
writeJSON(w, http.StatusOK, healthResponse{
|
|
Status: status,
|
|
Readiness: readiness,
|
|
AgentID: s.cfg.AgentID,
|
|
ProtocolVersion: s.cfg.ProtocolVersion,
|
|
Warnings: warnings,
|
|
Now: time.Now().UTC(),
|
|
})
|
|
}
|
|
|
|
func (s *Server) readiness() config.Readiness {
|
|
s.stateMu.RLock()
|
|
hostReady := s.hostReady
|
|
tlsReady := s.tlsReady
|
|
s.stateMu.RUnlock()
|
|
if s.store == nil || !s.store.Writable() || s.cfg.AgentID == "" || !hostReady {
|
|
return config.ReadinessDegraded
|
|
}
|
|
if s.cfg.RequireMTLS && (!s.cfg.TLSComplete() || !tlsReady) {
|
|
return config.ReadinessDegraded
|
|
}
|
|
if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 {
|
|
return config.ReadinessDegraded
|
|
}
|
|
if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) {
|
|
return config.ReadinessDegraded
|
|
}
|
|
return config.ReadinessReady
|
|
}
|
|
|
|
// SetTLSReady publishes the result of loading certificate material after the
|
|
// server has been constructed. A failed load keeps liveness available while
|
|
// protected requests remain fail-closed.
|
|
func (s *Server) SetTLSReady(ready bool) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.stateMu.Lock()
|
|
defer s.stateMu.Unlock()
|
|
s.tlsReady = ready
|
|
}
|
|
|
|
// refreshHostReadiness performs a bounded, read-only capability probe. It is
|
|
// intentionally independent from operation execution: a failed probe never
|
|
// invokes a lifecycle or define command and only downgrades readiness.
|
|
func (s *Server) refreshHostReadiness(parent context.Context) bool {
|
|
if s == nil || s.backend == nil {
|
|
return s.hostBackendReady()
|
|
}
|
|
return s.probeHostReadiness(parent)
|
|
}
|
|
|
|
// refreshHostReadinessCached is used only by health. It coalesces concurrent
|
|
// requests into one bounded probe and briefly reuses its result, including a
|
|
// failed result. The latter matters for fail-closed behavior: an attacker
|
|
// must not be able to turn a failed host into an unbounded subprocess storm by
|
|
// polling the public health endpoint.
|
|
func (s *Server) refreshHostReadinessCached(parent context.Context) bool {
|
|
if s == nil || s.backend == nil {
|
|
return s.hostBackendReady()
|
|
}
|
|
if parent == nil {
|
|
parent = context.Background()
|
|
}
|
|
|
|
s.readinessProbeMu.Lock()
|
|
now := time.Now()
|
|
if !s.readinessProbeAt.IsZero() && now.Sub(s.readinessProbeAt) < healthReadinessProbeTTL {
|
|
s.readinessProbeMu.Unlock()
|
|
return s.hostBackendReady()
|
|
}
|
|
if done := s.readinessProbeDone; done != nil {
|
|
s.readinessProbeMu.Unlock()
|
|
select {
|
|
case <-done:
|
|
return s.hostBackendReady()
|
|
case <-parent.Done():
|
|
return s.hostBackendReady()
|
|
}
|
|
}
|
|
done := make(chan struct{})
|
|
s.readinessProbeDone = done
|
|
s.readinessProbeMu.Unlock()
|
|
|
|
ready := s.probeHostReadiness(parent)
|
|
s.readinessProbeMu.Lock()
|
|
s.readinessProbeAt = time.Now()
|
|
s.readinessProbeDone = nil
|
|
close(done)
|
|
s.readinessProbeMu.Unlock()
|
|
return ready
|
|
}
|
|
|
|
// probeHostReadiness performs the actual bounded, read-only backend probe.
|
|
// Callers choose whether to serialize/cache it based on the request boundary.
|
|
func (s *Server) probeHostReadiness(parent context.Context) bool {
|
|
if s == nil || s.backend == nil {
|
|
return s.hostBackendReady()
|
|
}
|
|
if parent == nil {
|
|
parent = context.Background()
|
|
}
|
|
timeout := s.cfg.RequestTimeout
|
|
if timeout <= 0 || timeout > 2*time.Second {
|
|
timeout = 2 * time.Second
|
|
}
|
|
ctx, cancel := context.WithTimeout(parent, timeout)
|
|
defer cancel()
|
|
_, err := s.backend.Probe(ctx)
|
|
s.stateMu.Lock()
|
|
s.hostReady = err == nil
|
|
ready := s.hostReady
|
|
s.stateMu.Unlock()
|
|
return ready
|
|
}
|
|
|
|
func (s *Server) hostBackendReady() bool {
|
|
if s == nil {
|
|
return false
|
|
}
|
|
s.stateMu.RLock()
|
|
defer s.stateMu.RUnlock()
|
|
return s.hostReady
|
|
}
|
|
|
|
func (s *Server) capabilities(w http.ResponseWriter, r *http.Request) {
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
snapshot, err := s.caps.Snapshot(ctx)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability probe failed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, snapshot)
|
|
}
|
|
|
|
func (s *Server) agentTest(w http.ResponseWriter, r *http.Request) {
|
|
if !s.writeAllowed(w, r.Context()) {
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
if err := s.caps.Probe(ctx); err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability probe failed"})
|
|
return
|
|
}
|
|
snapshot, err := s.caps.Snapshot(ctx)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability snapshot failed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, snapshot)
|
|
}
|
|
|
|
func (s *Server) createOperation(w http.ResponseWriter, r *http.Request) {
|
|
body, err := readBody(r, s.cfg.MaxBodyBytes)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid operation request body"})
|
|
return
|
|
}
|
|
var request operationRequest
|
|
if err := decodeJSON(body, &request); err != nil {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid operation JSON"})
|
|
return
|
|
}
|
|
if err := validateOperationHeaders(r, request, body, s.cfg.RequireSigned); err != nil {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid operation request headers"})
|
|
return
|
|
}
|
|
// Raw XML define is intentionally not a public Agent operation. The
|
|
// existing hypervisor backend can define an arbitrary caller-supplied XML
|
|
// document, while its read-back contract proves only UUID/firmware. Keep
|
|
// that legacy implementation unreachable until the transactional typed
|
|
// create executor has ownership markers, exact disk/seed read-back and
|
|
// fenced cleanup.
|
|
if request.Kind == "domain_define" {
|
|
writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "domain define is disabled until the typed create executor is available"})
|
|
return
|
|
}
|
|
if request.Kind != "capability_probe" && request.Kind != "domain_read" && request.Kind != "domain_observe" && request.Kind != "domain_power" && request.Kind != "domain_delete" && request.Kind != "seed_prepare" && request.Kind != "domain_preflight" && request.Kind != "domain_create_plan" && request.Kind != "domain_create_validate" {
|
|
writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "operation kind is not implemented by the agent"})
|
|
return
|
|
}
|
|
// Planning operations are deliberately the escape hatch from a stale host
|
|
// readiness result: their purpose is to obtain a fresh capability
|
|
// observation that can repair that result. They still require the complete
|
|
// control-plane gate (authenticated transport, agent identity and a
|
|
// durable journal), but must not require a successful Probe() first.
|
|
// Every other operation remains behind the normal host readiness gate.
|
|
if isPlanningKind(request.Kind) {
|
|
if !s.preflightAllowed(w) {
|
|
return
|
|
}
|
|
} else if !s.writeAllowed(w, r.Context()) {
|
|
return
|
|
}
|
|
if request.Kind == "domain_preflight" {
|
|
preflightRequest, parseErr := parseDomainPreflightRequest(request.Payload)
|
|
if parseErr != nil || !targetMatchesUUID(request.Target, preflightRequest.UUID) {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain preflight payload"})
|
|
return
|
|
}
|
|
}
|
|
if request.Kind == "domain_create_plan" {
|
|
planRequest, parseErr := parseDomainCreatePlanRequest(request.Payload)
|
|
if parseErr != nil || !targetMatchesUUID(request.Target, planRequest.Domain.UUID) {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain create plan payload"})
|
|
return
|
|
}
|
|
}
|
|
if request.Kind == "domain_create_validate" {
|
|
validatePayload, parseErr := parseDomainCreateValidatePayload(request.Payload)
|
|
if parseErr != nil || !targetMatchesUUID(request.Target, validatePayload.Domain.UUID) {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain create validate payload"})
|
|
return
|
|
}
|
|
}
|
|
if request.Kind == "domain_power" {
|
|
powerRequest, parseErr := parseDomainPowerRequest(request.Payload)
|
|
if parseErr != nil || !targetMatchesUUID(request.Target, powerRequest.UUID) {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain power payload"})
|
|
return
|
|
}
|
|
}
|
|
if request.Kind == "domain_delete" {
|
|
uuid, parseErr := parseDomainDeletePayload(request.Payload)
|
|
if parseErr != nil || !targetMatchesUUID(request.Target, uuid) {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"})
|
|
return
|
|
}
|
|
}
|
|
if request.Kind == "domain_read" {
|
|
uuid, parseErr := parseDomainReadPayload(request.Payload)
|
|
if parseErr != nil || !targetMatchesUUID(request.Target, uuid) {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain read payload"})
|
|
return
|
|
}
|
|
}
|
|
if request.Kind == "domain_observe" {
|
|
uuid, parseErr := parseDomainObservePayload(request.Payload)
|
|
if parseErr != nil || !targetMatchesUUID(request.Target, uuid) {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain observe payload"})
|
|
return
|
|
}
|
|
}
|
|
if request.Kind == "seed_prepare" {
|
|
seedRequest, parseErr := parseSeedPrepareRequest(request.Payload)
|
|
if parseErr != nil || !targetMatchesUUID(request.Target, seedRequest.ArtifactID) {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid seed prepare payload"})
|
|
return
|
|
}
|
|
}
|
|
journalPayload, err := safeJournalPayload(request.Kind, request.Payload)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "operation payload cannot be safely journaled"})
|
|
return
|
|
}
|
|
record, replay, err := s.store.Begin(operation.BeginRequest{
|
|
OperationID: request.OperationID,
|
|
IdempotencyKey: request.IdempotencyKey,
|
|
Kind: request.Kind,
|
|
Target: request.Target,
|
|
RequestHash: operation.HashPayload(body),
|
|
// The request hash above covers the complete body for idempotency, but
|
|
// the durable journal receives only a typed, non-secret intent. XML,
|
|
// network documents and startup scripts are never copied into a
|
|
// plaintext operation record merely because a request timed out.
|
|
Payload: journalPayload,
|
|
})
|
|
if err != nil {
|
|
status := http.StatusBadRequest
|
|
if errors.Is(err, operation.ErrConflict) {
|
|
status = http.StatusConflict
|
|
} else if errors.Is(err, operation.ErrNotWritable) {
|
|
status = http.StatusServiceUnavailable
|
|
}
|
|
message := "operation request was rejected"
|
|
if errors.Is(err, operation.ErrConflict) {
|
|
message = "operation identity conflicts with an existing record"
|
|
} else if errors.Is(err, operation.ErrNotWritable) {
|
|
message = "operation journal is not writable"
|
|
}
|
|
writeJSON(w, status, errorResponse{Error: message})
|
|
return
|
|
}
|
|
if replay {
|
|
writeJSON(w, http.StatusOK, record)
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
if _, err := s.store.Update(record.OperationID, operation.StatusRunning, nil, "", ""); err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, errorResponse{Error: "operation journal update failed"})
|
|
return
|
|
}
|
|
if request.Kind == "capability_probe" {
|
|
s.completeCapabilityProbe(w, record, ctx)
|
|
return
|
|
}
|
|
// domain_preflight owns its own typed nil-backend/unavailable result. Keep
|
|
// it ahead of the generic backend guard so the durable operation contains
|
|
// domain_preflight_host_unavailable rather than backend_unavailable.
|
|
if request.Kind == "domain_preflight" {
|
|
s.completeDomainPreflight(w, record, request, ctx)
|
|
return
|
|
}
|
|
if request.Kind == "domain_create_plan" {
|
|
s.completeDomainCreatePlan(w, record, request, ctx)
|
|
return
|
|
}
|
|
if request.Kind == "domain_create_validate" {
|
|
s.completeDomainCreateValidate(w, record, request, ctx)
|
|
return
|
|
}
|
|
if s.backend == nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "backend_unavailable", "host backend is not configured")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "host backend is not configured"})
|
|
return
|
|
}
|
|
if request.Kind == "domain_read" {
|
|
s.completeDomainRead(w, record, request, ctx)
|
|
return
|
|
}
|
|
if request.Kind == "domain_observe" {
|
|
s.completeDomainObserve(w, record, request, ctx)
|
|
return
|
|
}
|
|
if request.Kind == "domain_power" {
|
|
s.completeDomainPower(w, record, request, ctx)
|
|
return
|
|
}
|
|
if request.Kind == "domain_delete" {
|
|
s.completeDomainDelete(w, record, request, ctx)
|
|
return
|
|
}
|
|
if request.Kind == "seed_prepare" {
|
|
s.completeSeedPrepare(w, record, request, ctx)
|
|
return
|
|
}
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "operation_not_implemented", "operation kind is not implemented by the agent")
|
|
writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "operation kind is not implemented by the agent"})
|
|
}
|
|
|
|
func validateOperationHeaders(r *http.Request, request operationRequest, body []byte, required bool) error {
|
|
requestHash := strings.TrimSpace(r.Header.Get("X-Cloud-Request-Hash"))
|
|
requestID := strings.TrimSpace(r.Header.Get("X-Cloud-Request-ID"))
|
|
idempotency := strings.TrimSpace(r.Header.Get("X-Cloud-Idempotency-Key"))
|
|
if required && requestHash == "" {
|
|
return fmt.Errorf("request hash header is required")
|
|
}
|
|
if requestHash != "" && requestHash != operation.HashPayload(body) {
|
|
return fmt.Errorf("request hash header does not match body")
|
|
}
|
|
if requestID != "" && requestID != request.OperationID {
|
|
return fmt.Errorf("request ID header does not match operation")
|
|
}
|
|
if idempotency != "" && idempotency != request.IdempotencyKey {
|
|
return fmt.Errorf("idempotency header does not match operation")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) completeCapabilityProbe(w http.ResponseWriter, record operation.Record, ctx context.Context) {
|
|
if err := s.caps.Probe(ctx); err != nil {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, nil, "capability_probe_failed", "capability probe did not complete")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability probe failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
snapshot, err := s.caps.Snapshot(ctx)
|
|
if err != nil {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, nil, "capability_snapshot_failed", "capability snapshot did not complete")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability snapshot failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
result, _ := json.Marshal(snapshot)
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "capability_readback_confirmed"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, result, "operation_journal_update_failed", "capability probe was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
type domainReadPayload struct {
|
|
UUID string `json:"uuid"`
|
|
}
|
|
|
|
type domainPowerPayload struct {
|
|
UUID string `json:"uuid"`
|
|
Action string `json:"action"`
|
|
}
|
|
|
|
// domainDeletePayload is intentionally only an exact UUID. Disk cleanup,
|
|
// network teardown, and panel billing state are separate contracts and are
|
|
// not accepted by the host Agent's narrow undefine operation.
|
|
type domainDeletePayload struct {
|
|
UUID string `json:"uuid"`
|
|
}
|
|
|
|
type seedPreparePayload struct {
|
|
ArtifactID string `json:"artifact_id"`
|
|
InstanceID string `json:"instance_id"`
|
|
Hostname string `json:"hostname"`
|
|
NetworkConfig string `json:"network_config,omitempty"`
|
|
StartupScript string `json:"startup_script,omitempty"`
|
|
}
|
|
|
|
// These journal payloads are intentionally smaller than the request payloads.
|
|
// The operation hash still covers the complete request, but durable plaintext
|
|
// state keeps only identity and content digests for fields that can contain
|
|
// credentials, host paths, or tenant-provided scripts.
|
|
type seedPrepareJournalPayload struct {
|
|
ArtifactID string `json:"artifact_id"`
|
|
InstanceID string `json:"instance_id"`
|
|
Hostname string `json:"hostname"`
|
|
NetworkDigest string `json:"network_digest"`
|
|
ScriptDigest string `json:"script_digest"`
|
|
ScriptPresent bool `json:"script_present"`
|
|
}
|
|
|
|
type capabilityProbeJournalPayload struct{}
|
|
|
|
type domainReadJournalPayload struct {
|
|
UUID string `json:"uuid"`
|
|
}
|
|
|
|
type domainObserveResult struct {
|
|
UUID string `json:"uuid"`
|
|
XMLHash string `json:"xml_hash"`
|
|
State hypervisor.DomainPowerState `json:"state"`
|
|
Active bool `json:"active"`
|
|
ObservedAt time.Time `json:"observed_at"`
|
|
}
|
|
|
|
type domainDeleteJournalPayload struct {
|
|
UUID string `json:"uuid"`
|
|
}
|
|
|
|
type domainPowerJournalPayload struct {
|
|
UUID string `json:"uuid"`
|
|
Action string `json:"action"`
|
|
}
|
|
|
|
// domainCreateValidateResult combines the immutable seed/XML validation
|
|
// boundary with the fresh host capability observation. The embedded result
|
|
// is already redacted by domaincreatevalidate; capabilities are a typed,
|
|
// read-only summary. No source XML, script, network document, or artifact
|
|
// path is returned.
|
|
type domainCreateValidateResult struct {
|
|
domaincreatevalidate.Result
|
|
Capabilities domainpreflight.CapabilitySummary `json:"capabilities"`
|
|
}
|
|
|
|
// domainXMLResult is the durable, non-sensitive form of a domain read-back.
|
|
// The full XML can contain host paths, auth metadata, and storage details; the
|
|
// operation journal only needs a content identity for reconciliation. Callers
|
|
// that need the live XML must use a separately authorized read path rather than
|
|
// recovering it from a plaintext restart journal.
|
|
type domainXMLResult struct {
|
|
UUID string `json:"uuid"`
|
|
Revision uint64 `json:"revision,omitempty"`
|
|
XMLHash string `json:"xml_hash"`
|
|
}
|
|
|
|
func safeJournalPayload(kind string, raw json.RawMessage) (json.RawMessage, error) {
|
|
switch kind {
|
|
case "capability_probe":
|
|
var payload struct{}
|
|
if err := decodeJSON(raw, &payload); err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(capabilityProbeJournalPayload{})
|
|
case "domain_read":
|
|
uuid, err := parseDomainReadPayload(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(domainReadJournalPayload{UUID: uuid})
|
|
case "domain_observe":
|
|
uuid, err := parseDomainObservePayload(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(domainReadJournalPayload{UUID: uuid})
|
|
case "domain_power":
|
|
power, err := parseDomainPowerRequest(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(domainPowerJournalPayload{UUID: power.UUID, Action: string(power.Action)})
|
|
case "domain_delete":
|
|
uuid, err := parseDomainDeletePayload(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(domainDeleteJournalPayload{UUID: uuid})
|
|
case "seed_prepare":
|
|
payload, err := parseSeedPrepareRequest(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(seedPrepareJournalPayload{
|
|
ArtifactID: payload.ArtifactID,
|
|
InstanceID: payload.InstanceID,
|
|
Hostname: payload.Hostname,
|
|
NetworkDigest: operation.HashPayload([]byte(payload.NetworkConfig)),
|
|
ScriptDigest: operation.HashPayload([]byte(payload.StartupScript)),
|
|
ScriptPresent: payload.StartupScript != "",
|
|
})
|
|
case "domain_preflight":
|
|
payload, err := parseDomainPreflightRequest(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// The request type contains only scalar policy fields; unlike domain
|
|
// define/seed payloads it needs no digest-only projection.
|
|
return json.Marshal(payload)
|
|
case "domain_create_plan":
|
|
payload, err := parseDomainCreatePlanRequest(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
journal, err := domaincreateplan.PrepareJournal(payload)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(journal)
|
|
case "domain_create_validate":
|
|
payload, err := parseDomainCreateValidatePayload(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// The request is deliberately the already-redacted create-plan journal
|
|
// shape. Re-validate it before persisting so a caller cannot smuggle raw
|
|
// seed data or an inconsistent plan identity into the operation record.
|
|
return json.Marshal(payload)
|
|
default:
|
|
// Unknown operation kinds remain behind the capability gate. Store's
|
|
// generic sanitizer is a second defense for any future typed payload,
|
|
// but unknown requests must not be treated as an approved operation.
|
|
return append(json.RawMessage(nil), raw...), nil
|
|
}
|
|
}
|
|
|
|
func parseDomainPreflightRequest(raw json.RawMessage) (domainpreflight.Request, error) {
|
|
var payload domainpreflight.Request
|
|
if err := decodeJSON(raw, &payload); err != nil {
|
|
return domainpreflight.Request{}, err
|
|
}
|
|
return domainpreflight.ValidateRequest(payload)
|
|
}
|
|
|
|
func parseDomainCreatePlanRequest(raw json.RawMessage) (domaincreateplan.Request, error) {
|
|
var payload domaincreateplan.Request
|
|
if err := decodeJSON(raw, &payload); err != nil {
|
|
return domaincreateplan.Request{}, err
|
|
}
|
|
return domaincreateplan.ValidateRequest(payload)
|
|
}
|
|
|
|
// parseDomainCreateValidatePayload accepts only the redacted journal emitted
|
|
// by domain_create_plan. A validate operation must refer to a published,
|
|
// immutable seed artifact; accepting the original create request here would
|
|
// make the endpoint look as if it could publish or repair the artifact.
|
|
func parseDomainCreateValidatePayload(raw json.RawMessage) (domaincreateplan.JournalPayload, error) {
|
|
var payload domaincreateplan.JournalPayload
|
|
if err := decodeJSON(raw, &payload); err != nil {
|
|
return domaincreateplan.JournalPayload{}, err
|
|
}
|
|
if err := domaincreateplan.ValidateJournal(payload); err != nil {
|
|
return domaincreateplan.JournalPayload{}, err
|
|
}
|
|
return payload, nil
|
|
}
|
|
|
|
func parseSeedPrepareRequest(raw json.RawMessage) (seedPreparePayload, error) {
|
|
var payload seedPreparePayload
|
|
if err := decodeJSON(raw, &payload); err != nil {
|
|
return seedPreparePayload{}, err
|
|
}
|
|
artifactID, err := hypervisor.CanonicalUUID(payload.ArtifactID)
|
|
if err != nil {
|
|
return seedPreparePayload{}, err
|
|
}
|
|
payload.ArtifactID = artifactID
|
|
if err := startup.Validate(startup.SeedRequest{
|
|
Operation: startup.OperationCreate,
|
|
InstanceID: payload.InstanceID,
|
|
Hostname: payload.Hostname,
|
|
NetworkConfig: payload.NetworkConfig,
|
|
StartupScript: payload.StartupScript,
|
|
}); err != nil {
|
|
return seedPreparePayload{}, err
|
|
}
|
|
return payload, nil
|
|
}
|
|
|
|
func parseDomainPowerRequest(raw json.RawMessage) (hypervisor.PowerRequest, error) {
|
|
var payload domainPowerPayload
|
|
if err := decodeJSON(raw, &payload); err != nil {
|
|
return hypervisor.PowerRequest{}, err
|
|
}
|
|
canonical, err := hypervisor.CanonicalUUID(payload.UUID)
|
|
if err != nil {
|
|
return hypervisor.PowerRequest{}, err
|
|
}
|
|
action, err := hypervisor.ParsePowerAction(payload.Action)
|
|
if err != nil {
|
|
return hypervisor.PowerRequest{}, err
|
|
}
|
|
return hypervisor.PowerRequest{UUID: canonical, Action: action}, nil
|
|
}
|
|
|
|
func parseDomainDeletePayload(raw json.RawMessage) (string, error) {
|
|
var payload domainDeletePayload
|
|
if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" {
|
|
return "", fmt.Errorf("invalid domain delete payload")
|
|
}
|
|
return hypervisor.CanonicalUUID(payload.UUID)
|
|
}
|
|
|
|
func parseDomainReadPayload(raw json.RawMessage) (string, error) {
|
|
var payload domainReadPayload
|
|
if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" {
|
|
return "", fmt.Errorf("invalid domain read payload")
|
|
}
|
|
canonical, err := hypervisor.CanonicalUUID(payload.UUID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return canonical, nil
|
|
}
|
|
|
|
func parseDomainObservePayload(raw json.RawMessage) (string, error) {
|
|
return parseDomainReadPayload(raw)
|
|
}
|
|
|
|
func parseSeedJournalPayload(raw json.RawMessage) (seedPreparePayload, error) {
|
|
var journal seedPrepareJournalPayload
|
|
if err := decodeJSON(raw, &journal); err == nil && journal.ArtifactID != "" && journal.NetworkDigest != "" && journal.ScriptDigest != "" {
|
|
artifactID, canonicalErr := hypervisor.CanonicalUUID(journal.ArtifactID)
|
|
if canonicalErr != nil {
|
|
return seedPreparePayload{}, canonicalErr
|
|
}
|
|
if !validSHA256(journal.NetworkDigest) || !validSHA256(journal.ScriptDigest) {
|
|
return seedPreparePayload{}, fmt.Errorf("seed journal content hash is invalid")
|
|
}
|
|
if err := startup.Validate(startup.SeedRequest{Operation: startup.OperationCreate, InstanceID: journal.InstanceID, Hostname: journal.Hostname}); err != nil {
|
|
return seedPreparePayload{}, err
|
|
}
|
|
return seedPreparePayload{
|
|
ArtifactID: artifactID,
|
|
InstanceID: journal.InstanceID,
|
|
Hostname: journal.Hostname,
|
|
StartupScript: "[redacted]",
|
|
}, nil
|
|
}
|
|
return parseSeedPrepareRequest(raw)
|
|
}
|
|
|
|
func validSHA256(value string) bool {
|
|
if len(value) != sha256.Size*2 {
|
|
return false
|
|
}
|
|
_, err := hex.DecodeString(value)
|
|
return err == nil && strings.ToLower(value) == value
|
|
}
|
|
|
|
func seedManifestFor(files startup.SeedFiles) startup.SeedManifest {
|
|
payloads := map[string][]byte{
|
|
startup.UserDataFilename: files.UserData,
|
|
startup.MetaDataFilename: files.MetaData,
|
|
startup.NetworkConfigFilename: files.NetworkConfig,
|
|
}
|
|
manifest := startup.SeedManifest{
|
|
Version: startup.SeedManifestVersion,
|
|
Files: make(map[string]startup.SeedFileDigest, len(payloads)),
|
|
}
|
|
for name, data := range payloads {
|
|
digest := sha256.Sum256(data)
|
|
manifest.Files[name] = startup.SeedFileDigest{
|
|
Bytes: len(data),
|
|
SHA256: hex.EncodeToString(digest[:]),
|
|
}
|
|
}
|
|
return manifest
|
|
}
|
|
|
|
type seedReconcileResult struct {
|
|
ArtifactID string `json:"artifact_id"`
|
|
Manifest startup.SeedManifest `json:"manifest"`
|
|
RequestContentVerified bool `json:"request_content_verified"`
|
|
}
|
|
|
|
func marshalSeedReconcileResult(artifactID string, manifest startup.SeedManifest, requestContentVerified bool) ([]byte, error) {
|
|
return json.Marshal(seedReconcileResult{
|
|
ArtifactID: artifactID,
|
|
Manifest: manifest,
|
|
RequestContentVerified: requestContentVerified,
|
|
})
|
|
}
|
|
|
|
func persistedSeedManifest(raw json.RawMessage, artifactID string) (startup.SeedManifest, bool) {
|
|
if len(bytes.TrimSpace(raw)) == 0 {
|
|
return startup.SeedManifest{}, false
|
|
}
|
|
var result seedReconcileResult
|
|
if err := decodeJSON(raw, &result); err != nil || result.ArtifactID != artifactID || !result.RequestContentVerified {
|
|
return startup.SeedManifest{}, false
|
|
}
|
|
if result.Manifest.Version != startup.SeedManifestVersion || len(result.Manifest.Files) != 3 {
|
|
return startup.SeedManifest{}, false
|
|
}
|
|
return result.Manifest, true
|
|
}
|
|
|
|
func seedManifestsEqual(left, right startup.SeedManifest) bool {
|
|
if left.Version != right.Version || len(left.Files) != len(right.Files) {
|
|
return false
|
|
}
|
|
for name, digest := range right.Files {
|
|
if observed, ok := left.Files[name]; !ok || observed != digest {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func targetMatchesUUID(target, uuid string) bool {
|
|
target = strings.TrimSpace(target)
|
|
if target == "" {
|
|
return true
|
|
}
|
|
canonical, err := hypervisor.CanonicalUUID(target)
|
|
return err == nil && canonical == uuid
|
|
}
|
|
|
|
func marshalDomainXMLResult(definition hypervisor.Definition) ([]byte, error) {
|
|
return json.Marshal(domainXMLResult{
|
|
UUID: definition.UUID,
|
|
Revision: definition.Revision,
|
|
XMLHash: operation.HashPayload(definition.XML),
|
|
})
|
|
}
|
|
|
|
func (s *Server) completeDomainRead(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
|
|
var payload domainReadPayload
|
|
if err := decodeJSON(request.Payload, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_request", "domain read payload is invalid")
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain read payload"})
|
|
return
|
|
}
|
|
canonical, err := hypervisor.CanonicalUUID(payload.UUID)
|
|
if err != nil || !targetMatchesUUID(request.Target, canonical) {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_request", "domain read target does not match payload")
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain read payload"})
|
|
return
|
|
}
|
|
payload.UUID = canonical
|
|
definition, err := hypervisor.ReadbackDomain(ctx, s.backend, payload.UUID)
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
errorCode := "domain_read_failed"
|
|
if hypervisor.IsDefinitive(err) {
|
|
status = operation.StatusFailed
|
|
errorCode = "domain_read_rejected"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, nil, errorCode, "domain read-back did not complete")
|
|
if updateErr == nil {
|
|
if status == operation.StatusFailed {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, updated)
|
|
}
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain read-back failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "readback_confirmed"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "operation_journal_update_failed", "domain read-back was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
result, _ := marshalDomainXMLResult(definition)
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
// observeDomain performs a read-only XML and power-state observation for one
|
|
// explicit UUID. The XML itself never crosses the API or durable journal;
|
|
// callers receive only its content hash plus the independently read-back
|
|
// libvirt state. Keeping this helper shared by the request and reconcile paths
|
|
// prevents the latter from accidentally replaying a lifecycle operation.
|
|
func observeDomain(ctx context.Context, backend hypervisor.Backend, uuid string) (domainObserveResult, error) {
|
|
definition, err := hypervisor.ReadbackDomain(ctx, backend, uuid)
|
|
if err != nil {
|
|
return domainObserveResult{}, err
|
|
}
|
|
state, err := backend.ReadDomainPowerState(ctx, definition.UUID)
|
|
if err != nil {
|
|
return domainObserveResult{}, err
|
|
}
|
|
return domainObserveResult{
|
|
UUID: definition.UUID,
|
|
XMLHash: operation.HashPayload(definition.XML),
|
|
State: state,
|
|
Active: state.IsActive(),
|
|
ObservedAt: time.Now().UTC(),
|
|
}, nil
|
|
}
|
|
|
|
func (s *Server) completeDomainObserve(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
|
|
uuid, err := parseDomainObservePayload(request.Payload)
|
|
if err != nil || !targetMatchesUUID(request.Target, uuid) {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_observe_request", "domain observe payload is invalid")
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain observe payload"})
|
|
return
|
|
}
|
|
result, err := observeDomain(ctx, s.backend, uuid)
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
errorCode := "domain_observe_uncertain"
|
|
if hypervisor.IsDefinitive(err) {
|
|
status = operation.StatusFailed
|
|
errorCode = "domain_observe_rejected"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, nil, errorCode, "domain observation did not complete")
|
|
if updateErr == nil {
|
|
if status == operation.StatusFailed {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, updated)
|
|
}
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain observation failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
resultJSON, marshalErr := json.Marshal(result)
|
|
if marshalErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_observe_result_failed", "domain observation result could not be encoded")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain observation result could not be encoded"})
|
|
return
|
|
}
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_observe_confirmed"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain observation was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
func (s *Server) completeDomainPower(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
|
|
powerRequest, err := parseDomainPowerRequest(request.Payload)
|
|
if err != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_power_request", "domain power payload is invalid")
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain power payload"})
|
|
return
|
|
}
|
|
result, err := s.backend.PowerDomain(ctx, powerRequest.UUID, powerRequest.Action)
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
errorCode := "domain_power_uncertain"
|
|
if hypervisor.IsDefinitive(err) {
|
|
status = operation.StatusFailed
|
|
errorCode = "domain_power_rejected"
|
|
}
|
|
var resultJSON json.RawMessage
|
|
if result.State != "" {
|
|
resultJSON, _ = json.Marshal(result)
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, resultJSON, errorCode, "domain power action did not reach a confirmed desired state")
|
|
if updateErr == nil {
|
|
if status == operation.StatusFailed {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, updated)
|
|
}
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power action failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "power_readback_confirmed"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "operation_journal_update_failed", "domain power was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
resultJSON, _ := json.Marshal(result)
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
// completeDomainDelete performs only the typed, inactive-domain undefine
|
|
// boundary. The hypervisor implementation must prove absence with a targeted
|
|
// read-back before this method can persist success. Disk images, seed
|
|
// artifacts, firewall rules, and host networking are deliberately untouched;
|
|
// the panel's full VM cleanup contract is not represented here.
|
|
func (s *Server) completeDomainDelete(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
|
|
uuid, err := parseDomainDeletePayload(request.Payload)
|
|
if err != nil || !targetMatchesUUID(request.Target, uuid) {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_delete_request", "domain delete payload is invalid")
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"})
|
|
return
|
|
}
|
|
result, err := s.backend.DeleteDomain(ctx, uuid)
|
|
var resultJSON json.RawMessage
|
|
if result.UUID != "" {
|
|
resultJSON, _ = json.Marshal(result)
|
|
}
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
httpStatus := http.StatusServiceUnavailable
|
|
errorCode := "domain_delete_uncertain"
|
|
message := "domain undefine or absence read-back did not complete"
|
|
if errors.Is(err, hypervisor.ErrInvalidUUID) || errors.Is(err, hypervisor.ErrDomainActive) {
|
|
status = operation.StatusFailed
|
|
httpStatus = http.StatusUnprocessableEntity
|
|
if errors.Is(err, hypervisor.ErrDomainActive) {
|
|
errorCode = "domain_delete_active"
|
|
message = "domain must be inactive before undefine"
|
|
} else {
|
|
errorCode = "invalid_domain_delete_request"
|
|
message = "domain delete UUID is invalid"
|
|
}
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, resultJSON, errorCode, message)
|
|
if updateErr == nil {
|
|
writeJSON(w, httpStatus, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_delete_readback_confirmed"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain absence was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"})
|
|
return
|
|
}
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
func (s *Server) completeSeedPrepare(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
|
|
payload, err := parseSeedPrepareRequest(request.Payload)
|
|
if err != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_seed_request", "seed payload is invalid")
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid seed prepare payload"})
|
|
return
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "seed_prepare_canceled", "seed preparation was canceled before publication")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed preparation was canceled"})
|
|
return
|
|
}
|
|
files, err := startup.BuildCreateSeed(payload.InstanceID, payload.Hostname, payload.NetworkConfig, payload.StartupScript)
|
|
if err != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_seed_request", "seed payload is invalid")
|
|
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid seed prepare payload"})
|
|
return
|
|
}
|
|
target := filepath.Join(s.cfg.ArtifactRoot, payload.ArtifactID)
|
|
artifact, err := startup.WriteSeedDirectory(target, files)
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
errorCode := "seed_prepare_uncertain"
|
|
if errors.Is(err, startup.ErrInvalidSeed) || errors.Is(err, startup.ErrInvalidScript) || errors.Is(err, startup.ErrScriptCreateOnly) || errors.Is(err, startup.ErrArtifactPath) {
|
|
status = operation.StatusFailed
|
|
errorCode = "seed_prepare_rejected"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, nil, errorCode, "seed artifact publication was not confirmed")
|
|
if updateErr == nil {
|
|
if status == operation.StatusFailed {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, updated)
|
|
}
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed publication failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
result, _ := marshalSeedReconcileResult(payload.ArtifactID, artifact.Manifest, true)
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "seed_published"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, result, "operation_journal_update_failed", "seed artifact was published but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed operation result could not be committed"})
|
|
return
|
|
}
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed operation result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
// completeDomainPreflight performs a typed, read-only host capability check.
|
|
// It intentionally never calls DefineDomain: a successful preflight is only a
|
|
// capability/normalization decision, not evidence that storage or a domain was
|
|
// created.
|
|
func (s *Server) completeDomainPreflight(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
|
|
payload, err := parseDomainPreflightRequest(request.Payload)
|
|
if err != nil || !targetMatchesUUID(request.Target, payload.UUID) {
|
|
code := domainpreflight.Code(err)
|
|
if err == nil {
|
|
code = "domain_preflight_target_mismatch"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, "domain preflight request was rejected")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusBadRequest, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight request could not be journaled"})
|
|
}
|
|
return
|
|
}
|
|
result, err := domainpreflight.Run(ctx, s.backend, payload)
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
httpStatus := http.StatusServiceUnavailable
|
|
code := domainpreflight.Code(err)
|
|
message := "domain preflight capability observation did not complete"
|
|
if domainpreflight.IsDefinitive(err) {
|
|
status = operation.StatusFailed
|
|
httpStatus = http.StatusUnprocessableEntity
|
|
message = "domain preflight request was rejected"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
|
|
if updateErr == nil {
|
|
writeJSON(w, httpStatus, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
resultJSON, err := json.Marshal(result)
|
|
if err != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_preflight_result_failed", "domain preflight result could not be encoded")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight result could not be committed"})
|
|
return
|
|
}
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_preflight_confirmed"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain preflight was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight result could not be committed"})
|
|
return
|
|
}
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
// completeDomainCreatePlan performs a pure creation-plan observation. It
|
|
// validates the typed domain/seed/image intent, observes current host
|
|
// capabilities, and persists only redacted plan metadata. No seed directory,
|
|
// storage volume, domain definition, power state, or network object is
|
|
// touched here.
|
|
func (s *Server) completeDomainCreatePlan(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
|
|
payload, err := parseDomainCreatePlanRequest(request.Payload)
|
|
if err != nil || !targetMatchesUUID(request.Target, payload.Domain.UUID) {
|
|
code := domaincreateplan.Code(err)
|
|
if err == nil {
|
|
code = "domain_create_plan_target_mismatch"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, "domain create plan request was rejected")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusBadRequest, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan request could not be journaled"})
|
|
}
|
|
return
|
|
}
|
|
result, err := domaincreateplan.Build(ctx, s.backend, payload)
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
httpStatus := http.StatusServiceUnavailable
|
|
code := domaincreateplan.Code(err)
|
|
message := "domain create plan host observation did not complete"
|
|
if domaincreateplan.IsDefinitive(err) {
|
|
status = operation.StatusFailed
|
|
httpStatus = http.StatusUnprocessableEntity
|
|
message = "domain create plan request was rejected"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
|
|
if updateErr == nil {
|
|
writeJSON(w, httpStatus, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
resultJSON, err := json.Marshal(result)
|
|
if err != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_create_plan_result_failed", "domain create plan result could not be encoded")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan result could not be committed"})
|
|
return
|
|
}
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_create_plan_confirmed"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain create plan was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan result could not be committed"})
|
|
return
|
|
}
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
// completeDomainCreateValidate verifies an already-published immutable seed,
|
|
// then performs one read-only host capability observation. It deliberately
|
|
// stops before storage allocation, seed publication, domain definition,
|
|
// lifecycle, or network operations. The seed check comes first so a missing
|
|
// or tampered artifact is a definitive rejection even when libvirt is also
|
|
// unavailable.
|
|
func (s *Server) completeDomainCreateValidate(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
|
|
payload, err := parseDomainCreateValidatePayload(request.Payload)
|
|
if err != nil || !targetMatchesUUID(request.Target, payload.Domain.UUID) {
|
|
code := domainCreateValidateCode(err)
|
|
if err == nil {
|
|
code = "domain_create_validate_target_mismatch"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, "domain create validation request was rejected")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusBadRequest, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation request could not be journaled"})
|
|
}
|
|
return
|
|
}
|
|
|
|
validated, err := domaincreatevalidate.Validate(ctx, s.cfg.ArtifactRoot, payload)
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
httpStatus := http.StatusServiceUnavailable
|
|
code := domainCreateValidateCode(err)
|
|
message := "published seed validation did not complete"
|
|
if domaincreatevalidate.IsDefinitive(err) {
|
|
status = operation.StatusFailed
|
|
httpStatus = http.StatusUnprocessableEntity
|
|
message = "published seed or domain validation was rejected"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
|
|
if updateErr == nil {
|
|
writeJSON(w, httpStatus, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
|
|
// Keep the nil-backend case typed and uncertain rather than falling through
|
|
// to the generic mutating-backend error. The artifact has been verified, but
|
|
// the host capability boundary is not yet proven.
|
|
if s.backend == nil {
|
|
s.reconcileUncertain(w, record, nil, "domain_create_validate_host_unavailable", "host capability observation is unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
observation, err := domainpreflight.Run(ctx, s.backend, payload.Domain)
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
httpStatus := http.StatusServiceUnavailable
|
|
code := domainCreateValidateCode(err)
|
|
message := "host capability observation did not complete after seed validation"
|
|
if domainpreflight.IsDefinitive(err) {
|
|
status = operation.StatusFailed
|
|
httpStatus = http.StatusUnprocessableEntity
|
|
message = "host firmware policy rejected the validated domain"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
|
|
if updateErr == nil {
|
|
writeJSON(w, httpStatus, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
|
|
// Reuse the plan identity check so a fresh capability observation cannot be
|
|
// paired with a tampered journal or a different deterministic XML policy.
|
|
plan, err := domaincreateplan.BuildFromJournal(observation, payload)
|
|
if err != nil {
|
|
status := operation.StatusUncertain
|
|
httpStatus := http.StatusServiceUnavailable
|
|
code := domainCreateValidateCode(err)
|
|
message := "validated domain identity no longer matches host observation"
|
|
if domaincreateplan.IsDefinitive(err) {
|
|
status = operation.StatusFailed
|
|
httpStatus = http.StatusConflict
|
|
message = "validated domain identity was rejected"
|
|
}
|
|
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
|
|
if updateErr == nil {
|
|
writeJSON(w, httpStatus, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation failed and operation state is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
|
|
resultJSON, err := json.Marshal(domainCreateValidateResult{
|
|
Result: validated,
|
|
Capabilities: plan.Capabilities,
|
|
})
|
|
if err != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_create_validate_result_failed", "domain create validation result could not be encoded")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation result could not be committed"})
|
|
return
|
|
}
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_create_validate_confirmed"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain create validation was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation result could not be committed"})
|
|
return
|
|
}
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
// domainCreateValidateCode keeps errors from the plan/preflight packages in
|
|
// this operation's namespace. Stable codes are persisted; host error strings
|
|
// never cross the API boundary.
|
|
func domainCreateValidateCode(err error) string {
|
|
if err == nil {
|
|
return "domain_create_validate_failed"
|
|
}
|
|
code := domaincreatevalidate.Code(err)
|
|
if strings.HasPrefix(code, "domain_create_validate_") && code != "domain_create_validate_failed" {
|
|
return code
|
|
}
|
|
preflightCode := domainpreflight.Code(err)
|
|
if strings.HasPrefix(preflightCode, "domain_preflight_") && preflightCode != "domain_preflight_failed" {
|
|
return "domain_create_validate_" + strings.TrimPrefix(preflightCode, "domain_preflight_")
|
|
}
|
|
planCode := domaincreateplan.Code(err)
|
|
if strings.HasPrefix(planCode, "domain_create_plan_") && planCode != "domain_create_plan_failed" {
|
|
return "domain_create_validate_" + strings.TrimPrefix(planCode, "domain_create_plan_")
|
|
}
|
|
return "domain_create_validate_failed"
|
|
}
|
|
|
|
func (s *Server) getOperation(w http.ResponseWriter, r *http.Request) {
|
|
record, err := s.storeRecord(r.PathValue("id"))
|
|
if err != nil {
|
|
writeJSON(w, http.StatusNotFound, errorResponse{Error: "operation not found"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, record)
|
|
}
|
|
|
|
func (s *Server) reconcileOperation(w http.ResponseWriter, r *http.Request) {
|
|
record, err := s.storeRecord(r.PathValue("id"))
|
|
if err != nil {
|
|
writeJSON(w, http.StatusNotFound, errorResponse{Error: "operation not found"})
|
|
return
|
|
}
|
|
// Every reconcile handler below is a read-back operation. It may update
|
|
// the local journal, but it must never replay the original host mutation.
|
|
// Therefore it uses the control-plane gate only and deliberately does not
|
|
// require a fresh Probe()/hostReady bit. A failed/stale probe is exactly
|
|
// the state in which an operator needs to read back an already ambiguous
|
|
// operation. Lifecycle requests created through POST /operations remain
|
|
// behind writeAllowed and still fail closed while the host is degraded.
|
|
if !s.readbackAllowed(w) {
|
|
return
|
|
}
|
|
if record.Status != operation.StatusUncertain {
|
|
writeJSON(w, http.StatusOK, record)
|
|
return
|
|
}
|
|
switch record.Kind {
|
|
case "capability_probe":
|
|
s.reconcileCapabilityProbe(w, record, r)
|
|
case "domain_read":
|
|
s.reconcileDomainRead(w, record, r)
|
|
case "domain_observe":
|
|
s.reconcileDomainObserve(w, record, r)
|
|
case "domain_power":
|
|
s.reconcileDomainPower(w, record, r)
|
|
case "domain_delete":
|
|
s.reconcileDomainDelete(w, record, r)
|
|
case "seed_prepare":
|
|
s.reconcileSeedPrepare(w, record, r)
|
|
case "domain_preflight":
|
|
s.reconcileDomainPreflight(w, record, r)
|
|
case "domain_create_plan":
|
|
s.reconcileDomainCreatePlan(w, record, r)
|
|
case "domain_create_validate":
|
|
s.reconcileDomainCreateValidate(w, record, r)
|
|
default:
|
|
// No destructive replay is performed. An operation kind that has no
|
|
// typed read-back contract remains explicitly unresolved.
|
|
writeJSON(w, http.StatusConflict, errorResponse{Error: "operation requires backend-specific reconciliation"})
|
|
}
|
|
}
|
|
|
|
// reconcileDomainPreflight repeats only the typed capability observation. It
|
|
// never defines a domain or allocates storage/network resources, so a retry is
|
|
// safe even when the original HTTP response was lost.
|
|
func (s *Server) reconcileDomainPreflight(w http.ResponseWriter, record operation.Record, r *http.Request) {
|
|
payload, err := parseDomainPreflightRequest(record.Payload)
|
|
if err != nil || !targetMatchesUUID(record.Target, payload.UUID) {
|
|
s.reconcileInvalidJournal(w, record, domainpreflight.Code(err), "domain preflight journal payload is invalid")
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
result, err := domainpreflight.Run(ctx, s.backend, payload)
|
|
if err != nil {
|
|
if domainpreflight.IsDefinitive(err) {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainpreflight.Code(err), "domain preflight request was rejected")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight reconciliation failed"})
|
|
}
|
|
return
|
|
}
|
|
s.reconcileUncertain(w, record, record.Result, domainpreflight.Code(err), "domain preflight capability observation did not complete", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
resultJSON, marshalErr := json.Marshal(result)
|
|
if marshalErr != nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_preflight_result_failed", "domain preflight result could not be encoded", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
s.reconcileSuccess(w, record, "domain_preflight_confirmed", resultJSON)
|
|
}
|
|
|
|
// reconcileDomainCreatePlan repeats only the read-only capability observation
|
|
// and rebuilds the plan identity from the redacted journal. It never renders
|
|
// or republishes a seed and never calls a lifecycle/define operation.
|
|
func (s *Server) reconcileDomainCreatePlan(w http.ResponseWriter, record operation.Record, r *http.Request) {
|
|
var payload domaincreateplan.JournalPayload
|
|
if err := decodeJSON(record.Payload, &payload); err != nil || !targetMatchesUUID(record.Target, payload.Domain.UUID) {
|
|
s.reconcileInvalidJournal(w, record, "domain_create_plan_journal_invalid", "domain create plan journal payload is invalid")
|
|
return
|
|
}
|
|
if err := domaincreateplan.ValidateJournal(payload); err != nil {
|
|
s.reconcileInvalidJournal(w, record, domaincreateplan.Code(err), "domain create plan journal payload is invalid")
|
|
return
|
|
}
|
|
if s.backend == nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_create_plan_host_unavailable", "host backend is not configured for plan reconciliation", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
observation, err := domainpreflight.Run(ctx, s.backend, payload.Domain)
|
|
if err != nil {
|
|
if domainpreflight.IsDefinitive(err) {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domaincreateplan.Code(err), "domain create plan host policy was rejected")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan reconciliation failed"})
|
|
}
|
|
return
|
|
}
|
|
s.reconcileUncertain(w, record, record.Result, domaincreateplan.Code(err), "domain create plan host observation did not complete", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
result, err := domaincreateplan.BuildFromJournal(observation, payload)
|
|
if err != nil {
|
|
if domaincreateplan.IsDefinitive(err) {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domaincreateplan.Code(err), "domain create plan journal no longer proves the same intent")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusConflict, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan reconciliation failed"})
|
|
}
|
|
return
|
|
}
|
|
s.reconcileUncertain(w, record, record.Result, domaincreateplan.Code(err), "domain create plan reconciliation remains uncertain", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
resultJSON, marshalErr := json.Marshal(result)
|
|
if marshalErr != nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_create_plan_result_failed", "domain create plan result could not be encoded", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
s.reconcileSuccess(w, record, "domain_create_plan_confirmed", resultJSON)
|
|
}
|
|
|
|
// reconcileDomainCreateValidate repeats the immutable seed read-back before
|
|
// obtaining a fresh capability observation. It never republishes a seed,
|
|
// defines a domain, changes power, or configures networking.
|
|
func (s *Server) reconcileDomainCreateValidate(w http.ResponseWriter, record operation.Record, r *http.Request) {
|
|
var payload domaincreateplan.JournalPayload
|
|
if err := decodeJSON(record.Payload, &payload); err != nil || !targetMatchesUUID(record.Target, payload.Domain.UUID) {
|
|
s.reconcileInvalidJournal(w, record, "domain_create_validate_journal_invalid", "domain create validation journal payload is invalid")
|
|
return
|
|
}
|
|
if err := domaincreateplan.ValidateJournal(payload); err != nil {
|
|
s.reconcileInvalidJournal(w, record, domainCreateValidateCode(err), "domain create validation journal payload is invalid")
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
validated, err := domaincreatevalidate.Validate(ctx, s.cfg.ArtifactRoot, payload)
|
|
if err != nil {
|
|
if domaincreatevalidate.IsDefinitive(err) {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainCreateValidateCode(err), "published seed or domain validation was rejected")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation reconciliation failed"})
|
|
}
|
|
return
|
|
}
|
|
s.reconcileUncertain(w, record, record.Result, domainCreateValidateCode(err), "published seed validation did not complete", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if s.backend == nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_create_validate_host_unavailable", "host capability observation is unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
observation, err := domainpreflight.Run(ctx, s.backend, payload.Domain)
|
|
if err != nil {
|
|
if domainpreflight.IsDefinitive(err) {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainCreateValidateCode(err), "host firmware policy rejected the validated domain")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation reconciliation failed"})
|
|
}
|
|
return
|
|
}
|
|
s.reconcileUncertain(w, record, record.Result, domainCreateValidateCode(err), "host capability observation did not complete after seed validation", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
plan, err := domaincreateplan.BuildFromJournal(observation, payload)
|
|
if err != nil {
|
|
if domaincreateplan.IsDefinitive(err) {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainCreateValidateCode(err), "validated domain identity no longer matches host observation")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusConflict, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation reconciliation failed"})
|
|
}
|
|
return
|
|
}
|
|
s.reconcileUncertain(w, record, record.Result, domainCreateValidateCode(err), "domain create validation remains uncertain", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
resultJSON, marshalErr := json.Marshal(domainCreateValidateResult{
|
|
Result: validated,
|
|
Capabilities: plan.Capabilities,
|
|
})
|
|
if marshalErr != nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_create_validate_result_failed", "domain create validation result could not be encoded", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
s.reconcileSuccess(w, record, "domain_create_validate_confirmed", resultJSON)
|
|
}
|
|
|
|
// reconcileCapabilityProbe performs only the capability snapshot read-back.
|
|
// Probe is intentionally not called here: even though the current providers
|
|
// implement Probe as read-only, reconciliation must depend on the narrowest
|
|
// contract and never replay the original operation.
|
|
func (s *Server) reconcileCapabilityProbe(w http.ResponseWriter, record operation.Record, r *http.Request) {
|
|
if s.caps == nil {
|
|
s.reconcileUncertain(w, record, record.Result, "capability_reconcile_uncertain", "capability snapshot provider is unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if payload := bytes.TrimSpace(record.Payload); len(payload) != 0 {
|
|
var empty struct{}
|
|
if err := decodeJSON(payload, &empty); err != nil {
|
|
s.reconcileInvalidJournal(w, record, "capability_probe_payload_invalid", "capability probe journal payload is invalid")
|
|
return
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
snapshot, err := s.caps.Snapshot(ctx)
|
|
if err != nil {
|
|
s.reconcileUncertain(w, record, record.Result, "capability_reconcile_uncertain", "capability snapshot read-back did not complete", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if target := strings.TrimSpace(record.Target); target != "" && snapshot.AgentID != target {
|
|
result, _ := json.Marshal(snapshot)
|
|
s.reconcileUncertain(w, record, result, "capability_target_mismatch", "capability snapshot belongs to a different agent", http.StatusConflict)
|
|
return
|
|
}
|
|
result, _ := json.Marshal(snapshot)
|
|
s.reconcileSuccess(w, record, "capability_readback_confirmed", result)
|
|
}
|
|
|
|
// reconcileDomainRead performs a read-only dumpxml and never invokes a
|
|
// define/start/lifecycle method. A missing or policy-mismatched definition is
|
|
// left uncertain: the agent must not infer a destructive conclusion from one
|
|
// failed observation.
|
|
func (s *Server) reconcileDomainRead(w http.ResponseWriter, record operation.Record, r *http.Request) {
|
|
if s.backend == nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_read_reconcile_uncertain", "host backend is not configured for domain read-back", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
payload, err := parseDomainReadPayload(record.Payload)
|
|
if err != nil || !targetMatchesUUID(record.Target, payload) {
|
|
s.reconcileInvalidJournal(w, record, "invalid_domain_request", "domain read journal payload is invalid")
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
definition, err := hypervisor.ReadbackDomain(ctx, s.backend, payload)
|
|
if err != nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_read_reconcile_uncertain", "domain read-back did not complete", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
result, _ := marshalDomainXMLResult(definition)
|
|
s.reconcileSuccess(w, record, "domain_readback_confirmed", result)
|
|
}
|
|
|
|
// reconcileDomainObserve repeats only the typed XML/state read-back. It never
|
|
// invokes a lifecycle or define command, so an uncertain response cannot
|
|
// cause a host mutation when the panel retries reconciliation.
|
|
func (s *Server) reconcileDomainObserve(w http.ResponseWriter, record operation.Record, r *http.Request) {
|
|
if s.backend == nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_observe_reconcile_uncertain", "host backend is not configured for domain observation", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
uuid, err := parseDomainObservePayload(record.Payload)
|
|
if err != nil || !targetMatchesUUID(record.Target, uuid) {
|
|
s.reconcileInvalidJournal(w, record, "invalid_domain_observe_request", "domain observe journal payload is invalid")
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
result, err := observeDomain(ctx, s.backend, uuid)
|
|
if err != nil {
|
|
code := "domain_observe_reconcile_uncertain"
|
|
if hypervisor.IsDefinitive(err) {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, "domain_observe_rejected", "domain observation was rejected during reconciliation")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain observation reconciliation failed"})
|
|
}
|
|
return
|
|
}
|
|
s.reconcileUncertain(w, record, record.Result, code, "domain observation read-back did not complete", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
resultJSON, marshalErr := json.Marshal(result)
|
|
if marshalErr != nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_observe_result_failed", "domain observation result could not be encoded", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
s.reconcileSuccess(w, record, "domain_observe_confirmed", resultJSON)
|
|
}
|
|
|
|
// reconcileSeedPrepare verifies the already-published immutable seed and
|
|
// compares its manifest with a deterministic rendering of the journaled
|
|
// request. It never calls WriteSeedDirectory or otherwise republishes files.
|
|
func (s *Server) reconcileSeedPrepare(w http.ResponseWriter, record operation.Record, r *http.Request) {
|
|
payload, err := parseSeedJournalPayload(record.Payload)
|
|
if err != nil || !targetMatchesUUID(record.Target, payload.ArtifactID) {
|
|
s.reconcileInvalidJournal(w, record, "invalid_seed_request", "seed prepare journal payload is invalid")
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
if err := ctx.Err(); err != nil {
|
|
s.reconcileUncertain(w, record, nil, "seed_reconcile_uncertain", "seed artifact read-back was canceled", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
observed, err := startup.VerifySeedDirectory(filepath.Join(s.cfg.ArtifactRoot, payload.ArtifactID))
|
|
if err != nil {
|
|
s.reconcileUncertain(w, record, record.Result, "seed_reconcile_uncertain", "seed artifact read-back did not complete", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
requestContentVerified := false
|
|
if expected, ok := persistedSeedManifest(record.Result, payload.ArtifactID); ok {
|
|
if !seedManifestsEqual(observed, expected) {
|
|
result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, false)
|
|
s.reconcileUncertain(w, record, result, "seed_reconcile_mismatch", "seed artifact does not match the persisted request manifest", http.StatusConflict)
|
|
return
|
|
}
|
|
requestContentVerified = true
|
|
} else if payload.StartupScript != "[redacted]" {
|
|
expectedFiles, renderErr := startup.BuildCreateSeed(payload.InstanceID, payload.Hostname, payload.NetworkConfig, payload.StartupScript)
|
|
if renderErr != nil {
|
|
s.reconcileUncertain(w, record, nil, "seed_reconcile_mismatch", "seed request could not be rendered for comparison", http.StatusConflict)
|
|
return
|
|
}
|
|
expected := seedManifestFor(expectedFiles)
|
|
if !seedManifestsEqual(observed, expected) {
|
|
result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, false)
|
|
s.reconcileUncertain(w, record, result, "seed_reconcile_mismatch", "seed artifact does not match the requested content", http.StatusConflict)
|
|
return
|
|
}
|
|
requestContentVerified = true
|
|
} else if payload.StartupScript == "[redacted]" {
|
|
// A safe journal intentionally omits the request's network document and
|
|
// startup script. Once the response/result manifest is also missing,
|
|
// an artifact read-back proves only that *some* immutable seed exists;
|
|
// it cannot prove that it is the artifact requested by this operation.
|
|
// Never turn that integrity-only observation into a succeeded operation.
|
|
// Keep the observed manifest (with an explicit false flag) so an
|
|
// operator can inspect it while the operation remains reconcilable.
|
|
result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, false)
|
|
code := "seed_reconcile_content_unproven"
|
|
message := "seed artifact integrity was observed, but request content equivalence could not be proven"
|
|
if record.ErrorCode == "seed_reconcile_mismatch" {
|
|
// Preserve the stronger diagnostic for a previously observed
|
|
// mismatch, while retaining the non-successful uncertain state.
|
|
code = "seed_reconcile_mismatch"
|
|
message = "seed request content remains unavailable for comparison"
|
|
}
|
|
s.reconcileUncertain(w, record, result, code, message, http.StatusConflict)
|
|
return
|
|
}
|
|
// At this point the journal carried the original request fields (for
|
|
// example, a test/recovery record created before the digest-only projection
|
|
// was enabled), so the deterministic render above proved equivalence.
|
|
result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, requestContentVerified)
|
|
s.reconcileSuccess(w, record, "seed_readback_confirmed", result)
|
|
}
|
|
|
|
func (s *Server) reconcileSuccess(w http.ResponseWriter, record operation.Record, boundary string, result []byte) {
|
|
if _, err := s.store.SetCommitBoundary(record.OperationID, boundary); err != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, result, "operation_journal_update_failed", "read-back was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
func (s *Server) reconcileUncertain(w http.ResponseWriter, record operation.Record, result []byte, code, message string, status int) {
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusUncertain, result, code, message)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation reconciliation remains uncertain"})
|
|
return
|
|
}
|
|
writeJSON(w, status, updated)
|
|
}
|
|
|
|
func (s *Server) reconcileInvalidJournal(w http.ResponseWriter, record operation.Record, code, message string) {
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, message)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation journal payload is invalid"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
}
|
|
|
|
// reconcileDomainPower performs only a typed, read-only state probe. It never
|
|
// calls PowerDomain: an ambiguous lifecycle response must not be replayed just
|
|
// because a caller asked to reconcile it.
|
|
func (s *Server) reconcileDomainPower(w http.ResponseWriter, record operation.Record, r *http.Request) {
|
|
if s.backend == nil {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, record.Result, "domain_power_reconcile_uncertain", "host backend is not configured for state reconciliation")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power reconciliation is unavailable"})
|
|
}
|
|
return
|
|
}
|
|
powerRequest, err := parseDomainPowerRequest(record.Payload)
|
|
if err != nil || !targetMatchesUUID(record.Target, powerRequest.UUID) {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_power_request", "domain power journal payload is invalid")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusUnprocessableEntity, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power journal payload is invalid"})
|
|
}
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
state, err := s.backend.ReadDomainPowerState(ctx, powerRequest.UUID)
|
|
if err != nil {
|
|
// A failed read cannot prove that the requested outcome did not happen;
|
|
// keep the operation uncertain and require a later probe.
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, record.Result, "domain_power_reconcile_uncertain", "domain power state read-back did not complete")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power state reconciliation is uncertain"})
|
|
}
|
|
return
|
|
}
|
|
result := hypervisor.PowerResult{
|
|
UUID: powerRequest.UUID,
|
|
Action: powerRequest.Action,
|
|
State: state,
|
|
Active: state.IsActive(),
|
|
ObservedAt: time.Now().UTC(),
|
|
}
|
|
resultJSON, _ := json.Marshal(result)
|
|
if !hypervisor.PowerActionReconcileSatisfied(powerRequest.Action, state) {
|
|
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "domain_power_not_reconciled", "observed domain power state does not satisfy the requested action")
|
|
if updateErr == nil {
|
|
writeJSON(w, http.StatusConflict, updated)
|
|
} else {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power state remains uncertain"})
|
|
}
|
|
return
|
|
}
|
|
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "power_readback_confirmed"); boundaryErr != nil {
|
|
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain power was confirmed but operation state could not be committed")
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, updated)
|
|
}
|
|
|
|
// reconcileDomainDelete is strictly read-only. It observes whether the exact
|
|
// definition UUID is absent and never retries undefine after an ambiguous
|
|
// response. A present definition, an active guest, or a failed observation
|
|
// keeps the journal uncertain for a later operator-approved read-back.
|
|
func (s *Server) reconcileDomainDelete(w http.ResponseWriter, record operation.Record, r *http.Request) {
|
|
if s.backend == nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "host backend is not configured for domain deletion read-back", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
uuid, err := parseDomainDeletePayload(record.Payload)
|
|
if err != nil || !targetMatchesUUID(record.Target, uuid) {
|
|
s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal payload is invalid")
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
|
|
defer cancel()
|
|
observation, err := s.backend.InspectDomainForDeletion(ctx, uuid)
|
|
if err != nil {
|
|
if errors.Is(err, hypervisor.ErrInvalidUUID) {
|
|
s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal UUID is invalid")
|
|
return
|
|
}
|
|
s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "domain deletion absence read-back did not complete", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if observation.Present {
|
|
result, _ := json.Marshal(observation)
|
|
s.reconcileUncertain(w, record, result, "domain_delete_not_reconciled", "domain definition is still present; no undefine was replayed", http.StatusConflict)
|
|
return
|
|
}
|
|
result, marshalErr := json.Marshal(hypervisor.DomainDeleteResult{
|
|
UUID: uuid,
|
|
AlreadyAbsent: true,
|
|
Removed: true,
|
|
ObservedAt: observation.ObservedAt,
|
|
})
|
|
if marshalErr != nil {
|
|
s.reconcileUncertain(w, record, record.Result, "domain_delete_result_failed", "domain deletion result could not be encoded", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
s.reconcileSuccess(w, record, "domain_delete_readback_confirmed", result)
|
|
}
|
|
|
|
func (s *Server) storeRecord(id string) (operation.Record, error) {
|
|
if s.store == nil {
|
|
return operation.Record{}, operation.ErrNotFound
|
|
}
|
|
return s.store.Get(id)
|
|
}
|
|
|
|
func isPlanningKind(kind string) bool {
|
|
return kind == "domain_preflight" || kind == "domain_create_plan" || kind == "domain_create_validate"
|
|
}
|
|
|
|
func (s *Server) writeAllowed(w http.ResponseWriter, ctx context.Context) bool {
|
|
if s.refreshHostReadiness(ctx) && s.readiness() == config.ReadinessReady {
|
|
return true
|
|
}
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent is degraded and mutating operations are disabled"})
|
|
return false
|
|
}
|
|
|
|
// readbackAllowed is the control-plane gate for read-only planning,
|
|
// validation, and reconciliation operations. These operations may obtain a
|
|
// fresh capability/artifact/domain observation and commit that observation to
|
|
// the local journal, but they must not require a successful host Probe(). A
|
|
// failed/stale probe is precisely when an ambiguous operation needs a later
|
|
// read-back. Transport authentication is enforced by Handler; this method
|
|
// covers the remaining local control-plane invariants and keeps the durable
|
|
// journal mandatory.
|
|
func (s *Server) readbackAllowed(w http.ResponseWriter) bool {
|
|
return s.controlPlaneReadOnlyAllowed(w, "read-back")
|
|
}
|
|
|
|
func (s *Server) controlPlaneReadOnlyAllowed(w http.ResponseWriter, operation string) bool {
|
|
if s == nil || s.store == nil || !s.store.Writable() || s.cfg.AgentID == "" {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent control plane is degraded and " + operation + " is disabled"})
|
|
return false
|
|
}
|
|
s.stateMu.RLock()
|
|
tlsReady := s.tlsReady
|
|
s.stateMu.RUnlock()
|
|
if s.cfg.RequireMTLS && (!s.cfg.TLSComplete() || !tlsReady) {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent TLS is not ready for " + operation})
|
|
return false
|
|
}
|
|
if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent client certificate policy is not ready for " + operation})
|
|
return false
|
|
}
|
|
if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) {
|
|
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent request signing is not ready for " + operation})
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// preflightAllowed is kept as a narrow compatibility name for the create
|
|
// planning path. Both planning and reconciliation are read-only control-plane
|
|
// operations and intentionally share the same host-readiness-independent
|
|
// checks, while retaining the preflight-specific error wording.
|
|
func (s *Server) preflightAllowed(w http.ResponseWriter) bool {
|
|
return s.controlPlaneReadOnlyAllowed(w, "preflight")
|
|
}
|
|
|
|
func (s *Server) authorized(r *http.Request) bool {
|
|
if !s.cfg.RequireMTLS && s.cfg.IsLoopbackListen() {
|
|
return true
|
|
}
|
|
if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 {
|
|
return false
|
|
}
|
|
if !s.cfg.RequireMTLS {
|
|
return false
|
|
}
|
|
leaf := r.TLS.PeerCertificates[0]
|
|
digest := sha256.Sum256(leaf.Raw)
|
|
observed := hex.EncodeToString(digest[:])
|
|
for _, expected := range s.cfg.ClientCertFingerprints {
|
|
if strings.EqualFold(strings.TrimSpace(expected), observed) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// TLSConfig constructs a mutual-TLS configuration only when all configured
|
|
// files can be read. The caller may keep serving health in degraded mode when
|
|
// this returns an error.
|
|
func (s *Server) TLSConfig() (*tls.Config, error) {
|
|
if !s.cfg.TLSComplete() {
|
|
return nil, fmt.Errorf("TLS certificate and key are not configured")
|
|
}
|
|
cert, err := tls.LoadX509KeyPair(s.cfg.TLSCertFile, s.cfg.TLSKeyFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load agent TLS certificate: %w", err)
|
|
}
|
|
caData, err := os.ReadFile(s.cfg.TLSCAFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read agent TLS CA: %w", err)
|
|
}
|
|
pool := x509.NewCertPool()
|
|
if !pool.AppendCertsFromPEM(caData) {
|
|
return nil, fmt.Errorf("parse agent TLS CA")
|
|
}
|
|
return &tls.Config{
|
|
MinVersion: tls.VersionTLS13,
|
|
Certificates: []tls.Certificate{cert},
|
|
ClientCAs: pool,
|
|
ClientAuth: tls.RequireAndVerifyClientCert,
|
|
NextProtos: []string{"h2", "http/1.1"},
|
|
}, nil
|
|
}
|
|
|
|
func readBody(r *http.Request, max int64) ([]byte, error) {
|
|
if max <= 0 {
|
|
return nil, fmt.Errorf("request body limit is not configured")
|
|
}
|
|
data, err := io.ReadAll(io.LimitReader(r.Body, max+1))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read request body: %w", err)
|
|
}
|
|
if int64(len(data)) > max {
|
|
return nil, fmt.Errorf("request body exceeds limit")
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
func decodeJSON(data []byte, target any) error {
|
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(target); err != nil {
|
|
return err
|
|
}
|
|
var extra any
|
|
if err := decoder.Decode(&extra); err != io.EOF {
|
|
return fmt.Errorf("multiple JSON values")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, value any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(value)
|
|
}
|
|
|
|
// NewHTTPServer provides conservative timeout defaults for a caller that
|
|
// wants to run the agent directly from main or an integration test.
|
|
func (s *Server) NewHTTPServer() *http.Server {
|
|
return &http.Server{
|
|
Addr: s.cfg.ListenAddr,
|
|
Handler: s.Handler(),
|
|
ReadHeaderTimeout: 5 * time.Second,
|
|
ReadTimeout: s.cfg.RequestTimeout,
|
|
WriteTimeout: s.cfg.RequestTimeout,
|
|
IdleTimeout: 60 * time.Second,
|
|
MaxHeaderBytes: 32 << 10,
|
|
}
|
|
}
|
|
|
|
func remoteIP(r *http.Request) net.IP {
|
|
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return net.ParseIP(strings.Trim(host, "[]"))
|
|
}
|