You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/internal/api/server.go

2167 lines
90 KiB
Go

// Package api implements the versioned panel-to-agent HTTP boundary.
package api
import (
"bytes"
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
"g.luoli.live/root/virt/agent/internal/auth"
"g.luoli.live/root/virt/agent/internal/capability"
"g.luoli.live/root/virt/agent/internal/config"
"g.luoli.live/root/virt/agent/internal/domaincreateplan"
"g.luoli.live/root/virt/agent/internal/domaincreatevalidate"
"g.luoli.live/root/virt/agent/internal/domainpreflight"
"g.luoli.live/root/virt/agent/internal/hypervisor"
"g.luoli.live/root/virt/agent/internal/operation"
"g.luoli.live/root/virt/agent/internal/startup"
)
type Options struct {
Config config.Config
Warnings []config.Warning
Store *operation.Store
Capabilities capability.Provider
Backend hypervisor.Backend
// HostBackendReady is explicit so a static provider cannot accidentally
// admit mutating operations in a production process. Tests may set it when
// they intentionally use a deterministic fake provider.
HostBackendReady bool
// TLSReady is set after the process successfully loads the configured
// certificate material. A path being configured is not proof that a remote
// listener can actually enforce mTLS.
TLSReady bool
Auth *auth.Verifier
Logger *slog.Logger
}
type Server struct {
cfg config.Config
warnings []config.Warning
store *operation.Store
caps capability.Provider
backend hypervisor.Backend
stateMu sync.RWMutex
hostReady bool
tlsReady bool
// readinessProbeMu serializes health-triggered host probes. Health is an
// unauthenticated liveness endpoint, so without a small single-flight
// boundary every remote poll could start a fresh virsh/libvirt probe.
readinessProbeMu sync.Mutex
readinessProbeAt time.Time
readinessProbeDone chan struct{}
auth *auth.Verifier
logger *slog.Logger
mux *http.ServeMux
}
// healthReadinessProbeTTL bounds how often the unauthenticated health route
// starts a new host capability probe. Mutating request gates continue to use
// refreshHostReadiness directly so they retain a fresh, fail-closed probe.
const healthReadinessProbeTTL = time.Second
type healthResponse struct {
Status string `json:"status"`
Readiness config.Readiness `json:"readiness"`
AgentID string `json:"agent_id,omitempty"`
ProtocolVersion string `json:"protocol_version"`
Warnings []config.Warning `json:"warnings,omitempty"`
Now time.Time `json:"now"`
}
type operationRequest struct {
OperationID string `json:"operation_id"`
IdempotencyKey string `json:"idempotency_key"`
Kind string `json:"kind"`
Target string `json:"target,omitempty"`
Payload json.RawMessage `json:"payload,omitempty"`
}
type errorResponse struct {
Error string `json:"error"`
}
func New(opts Options) (*Server, error) {
if err := opts.Config.Validate(); err != nil {
return nil, err
}
if opts.Capabilities == nil {
return nil, fmt.Errorf("capability provider is required")
}
if opts.Auth == nil {
if opts.Config.RequireSigned {
// A required verifier with no configured key must remain unavailable.
// Falling back to NewDisabled here would silently turn a required
// authenticated control plane into an unsigned one whenever the
// caller forgot to inject Auth.
verifier, err := auth.NewWithNonceConsumerForAgent(nil, true, opts.Store, opts.Config.AgentID)
if err != nil {
return nil, fmt.Errorf("configure request signing verifier: %w", err)
}
opts.Auth = verifier
} else {
opts.Auth = auth.NewDisabled()
}
}
if opts.Logger == nil {
opts.Logger = slog.New(slog.NewTextHandler(io.Discard, nil))
}
s := &Server{
cfg: opts.Config,
warnings: append([]config.Warning(nil), opts.Warnings...),
store: opts.Store,
caps: opts.Capabilities,
backend: opts.Backend,
hostReady: opts.HostBackendReady,
tlsReady: !opts.Config.RequireMTLS || opts.TLSReady,
auth: opts.Auth,
logger: opts.Logger,
mux: http.NewServeMux(),
}
s.routes()
return s, nil
}
func (s *Server) Handler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/health" {
if !s.authorized(r) {
writeJSON(w, http.StatusUnauthorized, errorResponse{Error: "client authentication required"})
return
}
body, err := readBody(r, s.cfg.MaxBodyBytes)
if err != nil {
writeJSON(w, http.StatusRequestEntityTooLarge, errorResponse{Error: "request body exceeds limit"})
return
}
r.Body = io.NopCloser(bytes.NewReader(body))
if err := s.auth.Verify(r, body, time.Now().UTC()); err != nil {
if errors.Is(err, auth.ErrUnavailable) {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "request signing is not ready"})
} else {
writeJSON(w, http.StatusUnauthorized, errorResponse{Error: "invalid signed request envelope"})
}
return
}
}
s.mux.ServeHTTP(w, r)
})
}
func (s *Server) routes() {
s.mux.HandleFunc("GET /v1/health", s.health)
s.mux.HandleFunc("GET /v1/capabilities", s.capabilities)
s.mux.HandleFunc("POST /v1/operations", s.createOperation)
s.mux.HandleFunc("GET /v1/operations/{id}", s.getOperation)
s.mux.HandleFunc("POST /v1/operations/{id}/reconcile", s.reconcileOperation)
// A direct read-only probe is useful during enrollment and does not mutate
// host networking, domains, storage, or firewall state.
s.mux.HandleFunc("POST /v1/agent/test", s.agentTest)
}
func (s *Server) health(w http.ResponseWriter, r *http.Request) {
// Readiness is a live host fact, not only a startup result. A libvirt
// daemon or socket can disappear after the process starts; refresh it on
// health and write boundaries so monitoring does not report a stale green
// state and mutating requests fail closed.
s.refreshHostReadinessCached(r.Context())
readiness := s.readiness()
status := "ok"
if readiness != config.ReadinessReady {
status = "degraded"
}
warnings := append([]config.Warning(nil), s.warnings...)
if s.store == nil || !s.store.Writable() {
warnings = append(warnings, config.Warning{Code: "operation_journal_unavailable", Message: "operation writes are disabled until the local journal is available"})
}
if s.cfg.AgentID == "" {
// Keep this warning stable and non-sensitive for monitoring clients.
warnings = append(warnings, config.Warning{Code: "agent_identity_missing", Message: "agent identity is not configured; mutating requests remain disabled"})
}
if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) {
warnings = append(warnings, config.Warning{Code: "request_signing_not_ready", Message: "signed request verification is not ready; protected requests remain disabled"})
}
if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 {
warnings = append(warnings, config.Warning{Code: "client_certificate_not_pinned", Message: "no client certificate fingerprint is registered; protected requests remain disabled"})
}
if s.cfg.RequireMTLS && !s.tlsReady {
warnings = append(warnings, config.Warning{Code: "tls_listener_not_ready", Message: "TLS material is not loaded; protected writes remain disabled"})
}
writeJSON(w, http.StatusOK, healthResponse{
Status: status,
Readiness: readiness,
AgentID: s.cfg.AgentID,
ProtocolVersion: s.cfg.ProtocolVersion,
Warnings: warnings,
Now: time.Now().UTC(),
})
}
func (s *Server) readiness() config.Readiness {
s.stateMu.RLock()
hostReady := s.hostReady
tlsReady := s.tlsReady
s.stateMu.RUnlock()
if s.store == nil || !s.store.Writable() || s.cfg.AgentID == "" || !hostReady {
return config.ReadinessDegraded
}
if s.cfg.RequireMTLS && (!s.cfg.TLSComplete() || !tlsReady) {
return config.ReadinessDegraded
}
if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 {
return config.ReadinessDegraded
}
if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) {
return config.ReadinessDegraded
}
return config.ReadinessReady
}
// SetTLSReady publishes the result of loading certificate material after the
// server has been constructed. A failed load keeps liveness available while
// protected requests remain fail-closed.
func (s *Server) SetTLSReady(ready bool) {
if s == nil {
return
}
s.stateMu.Lock()
defer s.stateMu.Unlock()
s.tlsReady = ready
}
// refreshHostReadiness performs a bounded, read-only capability probe. It is
// intentionally independent from operation execution: a failed probe never
// invokes a lifecycle or define command and only downgrades readiness.
func (s *Server) refreshHostReadiness(parent context.Context) bool {
if s == nil || s.backend == nil {
return s.hostBackendReady()
}
return s.probeHostReadiness(parent)
}
// refreshHostReadinessCached is used only by health. It coalesces concurrent
// requests into one bounded probe and briefly reuses its result, including a
// failed result. The latter matters for fail-closed behavior: an attacker
// must not be able to turn a failed host into an unbounded subprocess storm by
// polling the public health endpoint.
func (s *Server) refreshHostReadinessCached(parent context.Context) bool {
if s == nil || s.backend == nil {
return s.hostBackendReady()
}
if parent == nil {
parent = context.Background()
}
s.readinessProbeMu.Lock()
now := time.Now()
if !s.readinessProbeAt.IsZero() && now.Sub(s.readinessProbeAt) < healthReadinessProbeTTL {
s.readinessProbeMu.Unlock()
return s.hostBackendReady()
}
if done := s.readinessProbeDone; done != nil {
s.readinessProbeMu.Unlock()
select {
case <-done:
return s.hostBackendReady()
case <-parent.Done():
return s.hostBackendReady()
}
}
done := make(chan struct{})
s.readinessProbeDone = done
s.readinessProbeMu.Unlock()
ready := s.probeHostReadiness(parent)
s.readinessProbeMu.Lock()
s.readinessProbeAt = time.Now()
s.readinessProbeDone = nil
close(done)
s.readinessProbeMu.Unlock()
return ready
}
// probeHostReadiness performs the actual bounded, read-only backend probe.
// Callers choose whether to serialize/cache it based on the request boundary.
func (s *Server) probeHostReadiness(parent context.Context) bool {
if s == nil || s.backend == nil {
return s.hostBackendReady()
}
if parent == nil {
parent = context.Background()
}
timeout := s.cfg.RequestTimeout
if timeout <= 0 || timeout > 2*time.Second {
timeout = 2 * time.Second
}
ctx, cancel := context.WithTimeout(parent, timeout)
defer cancel()
_, err := s.backend.Probe(ctx)
s.stateMu.Lock()
s.hostReady = err == nil
ready := s.hostReady
s.stateMu.Unlock()
return ready
}
func (s *Server) hostBackendReady() bool {
if s == nil {
return false
}
s.stateMu.RLock()
defer s.stateMu.RUnlock()
return s.hostReady
}
func (s *Server) capabilities(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
snapshot, err := s.caps.Snapshot(ctx)
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability probe failed"})
return
}
writeJSON(w, http.StatusOK, snapshot)
}
func (s *Server) agentTest(w http.ResponseWriter, r *http.Request) {
if !s.writeAllowed(w, r.Context()) {
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
if err := s.caps.Probe(ctx); err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability probe failed"})
return
}
snapshot, err := s.caps.Snapshot(ctx)
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability snapshot failed"})
return
}
writeJSON(w, http.StatusOK, snapshot)
}
func (s *Server) createOperation(w http.ResponseWriter, r *http.Request) {
body, err := readBody(r, s.cfg.MaxBodyBytes)
if err != nil {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid operation request body"})
return
}
var request operationRequest
if err := decodeJSON(body, &request); err != nil {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid operation JSON"})
return
}
if err := validateOperationHeaders(r, request, body, s.cfg.RequireSigned); err != nil {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid operation request headers"})
return
}
// Raw XML define is intentionally not a public Agent operation. The
// existing hypervisor backend can define an arbitrary caller-supplied XML
// document, while its read-back contract proves only UUID/firmware. Keep
// that legacy implementation unreachable until the transactional typed
// create executor has ownership markers, exact disk/seed read-back and
// fenced cleanup.
if request.Kind == "domain_define" {
writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "domain define is disabled until the typed create executor is available"})
return
}
if request.Kind != "capability_probe" && request.Kind != "domain_read" && request.Kind != "domain_observe" && request.Kind != "domain_power" && request.Kind != "domain_delete" && request.Kind != "seed_prepare" && request.Kind != "domain_preflight" && request.Kind != "domain_create_plan" && request.Kind != "domain_create_validate" {
writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "operation kind is not implemented by the agent"})
return
}
// Planning operations are deliberately the escape hatch from a stale host
// readiness result: their purpose is to obtain a fresh capability
// observation that can repair that result. They still require the complete
// control-plane gate (authenticated transport, agent identity and a
// durable journal), but must not require a successful Probe() first.
// Every other operation remains behind the normal host readiness gate.
if isPlanningKind(request.Kind) {
if !s.preflightAllowed(w) {
return
}
} else if !s.writeAllowed(w, r.Context()) {
return
}
if request.Kind == "domain_preflight" {
preflightRequest, parseErr := parseDomainPreflightRequest(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, preflightRequest.UUID) {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain preflight payload"})
return
}
}
if request.Kind == "domain_create_plan" {
planRequest, parseErr := parseDomainCreatePlanRequest(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, planRequest.Domain.UUID) {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain create plan payload"})
return
}
}
if request.Kind == "domain_create_validate" {
validatePayload, parseErr := parseDomainCreateValidatePayload(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, validatePayload.Domain.UUID) {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain create validate payload"})
return
}
}
if request.Kind == "domain_power" {
powerRequest, parseErr := parseDomainPowerRequest(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, powerRequest.UUID) {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain power payload"})
return
}
}
if request.Kind == "domain_delete" {
uuid, parseErr := parseDomainDeletePayload(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, uuid) {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"})
return
}
}
if request.Kind == "domain_read" {
uuid, parseErr := parseDomainReadPayload(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, uuid) {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain read payload"})
return
}
}
if request.Kind == "domain_observe" {
uuid, parseErr := parseDomainObservePayload(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, uuid) {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain observe payload"})
return
}
}
if request.Kind == "seed_prepare" {
seedRequest, parseErr := parseSeedPrepareRequest(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, seedRequest.ArtifactID) {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid seed prepare payload"})
return
}
}
journalPayload, err := safeJournalPayload(request.Kind, request.Payload)
if err != nil {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "operation payload cannot be safely journaled"})
return
}
record, replay, err := s.store.Begin(operation.BeginRequest{
OperationID: request.OperationID,
IdempotencyKey: request.IdempotencyKey,
Kind: request.Kind,
Target: request.Target,
RequestHash: operation.HashPayload(body),
// The request hash above covers the complete body for idempotency, but
// the durable journal receives only a typed, non-secret intent. XML,
// network documents and startup scripts are never copied into a
// plaintext operation record merely because a request timed out.
Payload: journalPayload,
})
if err != nil {
status := http.StatusBadRequest
if errors.Is(err, operation.ErrConflict) {
status = http.StatusConflict
} else if errors.Is(err, operation.ErrNotWritable) {
status = http.StatusServiceUnavailable
}
message := "operation request was rejected"
if errors.Is(err, operation.ErrConflict) {
message = "operation identity conflicts with an existing record"
} else if errors.Is(err, operation.ErrNotWritable) {
message = "operation journal is not writable"
}
writeJSON(w, status, errorResponse{Error: message})
return
}
if replay {
writeJSON(w, http.StatusOK, record)
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
if _, err := s.store.Update(record.OperationID, operation.StatusRunning, nil, "", ""); err != nil {
writeJSON(w, http.StatusInternalServerError, errorResponse{Error: "operation journal update failed"})
return
}
if request.Kind == "capability_probe" {
s.completeCapabilityProbe(w, record, ctx)
return
}
// domain_preflight owns its own typed nil-backend/unavailable result. Keep
// it ahead of the generic backend guard so the durable operation contains
// domain_preflight_host_unavailable rather than backend_unavailable.
if request.Kind == "domain_preflight" {
s.completeDomainPreflight(w, record, request, ctx)
return
}
if request.Kind == "domain_create_plan" {
s.completeDomainCreatePlan(w, record, request, ctx)
return
}
if request.Kind == "domain_create_validate" {
s.completeDomainCreateValidate(w, record, request, ctx)
return
}
if s.backend == nil {
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "backend_unavailable", "host backend is not configured")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "host backend is not configured"})
return
}
if request.Kind == "domain_read" {
s.completeDomainRead(w, record, request, ctx)
return
}
if request.Kind == "domain_observe" {
s.completeDomainObserve(w, record, request, ctx)
return
}
if request.Kind == "domain_power" {
s.completeDomainPower(w, record, request, ctx)
return
}
if request.Kind == "domain_delete" {
s.completeDomainDelete(w, record, request, ctx)
return
}
if request.Kind == "seed_prepare" {
s.completeSeedPrepare(w, record, request, ctx)
return
}
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "operation_not_implemented", "operation kind is not implemented by the agent")
writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "operation kind is not implemented by the agent"})
}
func validateOperationHeaders(r *http.Request, request operationRequest, body []byte, required bool) error {
requestHash := strings.TrimSpace(r.Header.Get("X-Cloud-Request-Hash"))
requestID := strings.TrimSpace(r.Header.Get("X-Cloud-Request-ID"))
idempotency := strings.TrimSpace(r.Header.Get("X-Cloud-Idempotency-Key"))
if required && requestHash == "" {
return fmt.Errorf("request hash header is required")
}
if requestHash != "" && requestHash != operation.HashPayload(body) {
return fmt.Errorf("request hash header does not match body")
}
if requestID != "" && requestID != request.OperationID {
return fmt.Errorf("request ID header does not match operation")
}
if idempotency != "" && idempotency != request.IdempotencyKey {
return fmt.Errorf("idempotency header does not match operation")
}
return nil
}
func (s *Server) completeCapabilityProbe(w http.ResponseWriter, record operation.Record, ctx context.Context) {
if err := s.caps.Probe(ctx); err != nil {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, nil, "capability_probe_failed", "capability probe did not complete")
if updateErr == nil {
writeJSON(w, http.StatusServiceUnavailable, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability probe failed and operation state is uncertain"})
}
return
}
snapshot, err := s.caps.Snapshot(ctx)
if err != nil {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, nil, "capability_snapshot_failed", "capability snapshot did not complete")
if updateErr == nil {
writeJSON(w, http.StatusServiceUnavailable, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability snapshot failed and operation state is uncertain"})
}
return
}
result, _ := json.Marshal(snapshot)
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "capability_readback_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, result, "operation_journal_update_failed", "capability probe was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
type domainReadPayload struct {
UUID string `json:"uuid"`
}
type domainPowerPayload struct {
UUID string `json:"uuid"`
Action string `json:"action"`
}
// domainDeletePayload is intentionally only an exact UUID. Disk cleanup,
// network teardown, and panel billing state are separate contracts and are
// not accepted by the host Agent's narrow undefine operation.
type domainDeletePayload struct {
UUID string `json:"uuid"`
}
type seedPreparePayload struct {
ArtifactID string `json:"artifact_id"`
InstanceID string `json:"instance_id"`
Hostname string `json:"hostname"`
NetworkConfig string `json:"network_config,omitempty"`
StartupScript string `json:"startup_script,omitempty"`
}
// These journal payloads are intentionally smaller than the request payloads.
// The operation hash still covers the complete request, but durable plaintext
// state keeps only identity and content digests for fields that can contain
// credentials, host paths, or tenant-provided scripts.
type seedPrepareJournalPayload struct {
ArtifactID string `json:"artifact_id"`
InstanceID string `json:"instance_id"`
Hostname string `json:"hostname"`
NetworkDigest string `json:"network_digest"`
ScriptDigest string `json:"script_digest"`
ScriptPresent bool `json:"script_present"`
}
type capabilityProbeJournalPayload struct{}
type domainReadJournalPayload struct {
UUID string `json:"uuid"`
}
type domainObserveResult struct {
UUID string `json:"uuid"`
XMLHash string `json:"xml_hash"`
State hypervisor.DomainPowerState `json:"state"`
Active bool `json:"active"`
ObservedAt time.Time `json:"observed_at"`
}
type domainDeleteJournalPayload struct {
UUID string `json:"uuid"`
}
type domainPowerJournalPayload struct {
UUID string `json:"uuid"`
Action string `json:"action"`
}
// domainCreateValidateResult combines the immutable seed/XML validation
// boundary with the fresh host capability observation. The embedded result
// is already redacted by domaincreatevalidate; capabilities are a typed,
// read-only summary. No source XML, script, network document, or artifact
// path is returned.
type domainCreateValidateResult struct {
domaincreatevalidate.Result
Capabilities domainpreflight.CapabilitySummary `json:"capabilities"`
}
// domainXMLResult is the durable, non-sensitive form of a domain read-back.
// The full XML can contain host paths, auth metadata, and storage details; the
// operation journal only needs a content identity for reconciliation. Callers
// that need the live XML must use a separately authorized read path rather than
// recovering it from a plaintext restart journal.
type domainXMLResult struct {
UUID string `json:"uuid"`
Revision uint64 `json:"revision,omitempty"`
XMLHash string `json:"xml_hash"`
}
func safeJournalPayload(kind string, raw json.RawMessage) (json.RawMessage, error) {
switch kind {
case "capability_probe":
var payload struct{}
if err := decodeJSON(raw, &payload); err != nil {
return nil, err
}
return json.Marshal(capabilityProbeJournalPayload{})
case "domain_read":
uuid, err := parseDomainReadPayload(raw)
if err != nil {
return nil, err
}
return json.Marshal(domainReadJournalPayload{UUID: uuid})
case "domain_observe":
uuid, err := parseDomainObservePayload(raw)
if err != nil {
return nil, err
}
return json.Marshal(domainReadJournalPayload{UUID: uuid})
case "domain_power":
power, err := parseDomainPowerRequest(raw)
if err != nil {
return nil, err
}
return json.Marshal(domainPowerJournalPayload{UUID: power.UUID, Action: string(power.Action)})
case "domain_delete":
uuid, err := parseDomainDeletePayload(raw)
if err != nil {
return nil, err
}
return json.Marshal(domainDeleteJournalPayload{UUID: uuid})
case "seed_prepare":
payload, err := parseSeedPrepareRequest(raw)
if err != nil {
return nil, err
}
return json.Marshal(seedPrepareJournalPayload{
ArtifactID: payload.ArtifactID,
InstanceID: payload.InstanceID,
Hostname: payload.Hostname,
NetworkDigest: operation.HashPayload([]byte(payload.NetworkConfig)),
ScriptDigest: operation.HashPayload([]byte(payload.StartupScript)),
ScriptPresent: payload.StartupScript != "",
})
case "domain_preflight":
payload, err := parseDomainPreflightRequest(raw)
if err != nil {
return nil, err
}
// The request type contains only scalar policy fields; unlike domain
// define/seed payloads it needs no digest-only projection.
return json.Marshal(payload)
case "domain_create_plan":
payload, err := parseDomainCreatePlanRequest(raw)
if err != nil {
return nil, err
}
journal, err := domaincreateplan.PrepareJournal(payload)
if err != nil {
return nil, err
}
return json.Marshal(journal)
case "domain_create_validate":
payload, err := parseDomainCreateValidatePayload(raw)
if err != nil {
return nil, err
}
// The request is deliberately the already-redacted create-plan journal
// shape. Re-validate it before persisting so a caller cannot smuggle raw
// seed data or an inconsistent plan identity into the operation record.
return json.Marshal(payload)
default:
// Unknown operation kinds remain behind the capability gate. Store's
// generic sanitizer is a second defense for any future typed payload,
// but unknown requests must not be treated as an approved operation.
return append(json.RawMessage(nil), raw...), nil
}
}
func parseDomainPreflightRequest(raw json.RawMessage) (domainpreflight.Request, error) {
var payload domainpreflight.Request
if err := decodeJSON(raw, &payload); err != nil {
return domainpreflight.Request{}, err
}
return domainpreflight.ValidateRequest(payload)
}
func parseDomainCreatePlanRequest(raw json.RawMessage) (domaincreateplan.Request, error) {
var payload domaincreateplan.Request
if err := decodeJSON(raw, &payload); err != nil {
return domaincreateplan.Request{}, err
}
return domaincreateplan.ValidateRequest(payload)
}
// parseDomainCreateValidatePayload accepts only the redacted journal emitted
// by domain_create_plan. A validate operation must refer to a published,
// immutable seed artifact; accepting the original create request here would
// make the endpoint look as if it could publish or repair the artifact.
func parseDomainCreateValidatePayload(raw json.RawMessage) (domaincreateplan.JournalPayload, error) {
var payload domaincreateplan.JournalPayload
if err := decodeJSON(raw, &payload); err != nil {
return domaincreateplan.JournalPayload{}, err
}
if err := domaincreateplan.ValidateJournal(payload); err != nil {
return domaincreateplan.JournalPayload{}, err
}
return payload, nil
}
func parseSeedPrepareRequest(raw json.RawMessage) (seedPreparePayload, error) {
var payload seedPreparePayload
if err := decodeJSON(raw, &payload); err != nil {
return seedPreparePayload{}, err
}
artifactID, err := hypervisor.CanonicalUUID(payload.ArtifactID)
if err != nil {
return seedPreparePayload{}, err
}
payload.ArtifactID = artifactID
if err := startup.Validate(startup.SeedRequest{
Operation: startup.OperationCreate,
InstanceID: payload.InstanceID,
Hostname: payload.Hostname,
NetworkConfig: payload.NetworkConfig,
StartupScript: payload.StartupScript,
}); err != nil {
return seedPreparePayload{}, err
}
return payload, nil
}
func parseDomainPowerRequest(raw json.RawMessage) (hypervisor.PowerRequest, error) {
var payload domainPowerPayload
if err := decodeJSON(raw, &payload); err != nil {
return hypervisor.PowerRequest{}, err
}
canonical, err := hypervisor.CanonicalUUID(payload.UUID)
if err != nil {
return hypervisor.PowerRequest{}, err
}
action, err := hypervisor.ParsePowerAction(payload.Action)
if err != nil {
return hypervisor.PowerRequest{}, err
}
return hypervisor.PowerRequest{UUID: canonical, Action: action}, nil
}
func parseDomainDeletePayload(raw json.RawMessage) (string, error) {
var payload domainDeletePayload
if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" {
return "", fmt.Errorf("invalid domain delete payload")
}
return hypervisor.CanonicalUUID(payload.UUID)
}
func parseDomainReadPayload(raw json.RawMessage) (string, error) {
var payload domainReadPayload
if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" {
return "", fmt.Errorf("invalid domain read payload")
}
canonical, err := hypervisor.CanonicalUUID(payload.UUID)
if err != nil {
return "", err
}
return canonical, nil
}
func parseDomainObservePayload(raw json.RawMessage) (string, error) {
return parseDomainReadPayload(raw)
}
func parseSeedJournalPayload(raw json.RawMessage) (seedPreparePayload, error) {
var journal seedPrepareJournalPayload
if err := decodeJSON(raw, &journal); err == nil && journal.ArtifactID != "" && journal.NetworkDigest != "" && journal.ScriptDigest != "" {
artifactID, canonicalErr := hypervisor.CanonicalUUID(journal.ArtifactID)
if canonicalErr != nil {
return seedPreparePayload{}, canonicalErr
}
if !validSHA256(journal.NetworkDigest) || !validSHA256(journal.ScriptDigest) {
return seedPreparePayload{}, fmt.Errorf("seed journal content hash is invalid")
}
if err := startup.Validate(startup.SeedRequest{Operation: startup.OperationCreate, InstanceID: journal.InstanceID, Hostname: journal.Hostname}); err != nil {
return seedPreparePayload{}, err
}
return seedPreparePayload{
ArtifactID: artifactID,
InstanceID: journal.InstanceID,
Hostname: journal.Hostname,
StartupScript: "[redacted]",
}, nil
}
return parseSeedPrepareRequest(raw)
}
func validSHA256(value string) bool {
if len(value) != sha256.Size*2 {
return false
}
_, err := hex.DecodeString(value)
return err == nil && strings.ToLower(value) == value
}
func seedManifestFor(files startup.SeedFiles) startup.SeedManifest {
payloads := map[string][]byte{
startup.UserDataFilename: files.UserData,
startup.MetaDataFilename: files.MetaData,
startup.NetworkConfigFilename: files.NetworkConfig,
}
manifest := startup.SeedManifest{
Version: startup.SeedManifestVersion,
Files: make(map[string]startup.SeedFileDigest, len(payloads)),
}
for name, data := range payloads {
digest := sha256.Sum256(data)
manifest.Files[name] = startup.SeedFileDigest{
Bytes: len(data),
SHA256: hex.EncodeToString(digest[:]),
}
}
return manifest
}
type seedReconcileResult struct {
ArtifactID string `json:"artifact_id"`
Manifest startup.SeedManifest `json:"manifest"`
RequestContentVerified bool `json:"request_content_verified"`
}
func marshalSeedReconcileResult(artifactID string, manifest startup.SeedManifest, requestContentVerified bool) ([]byte, error) {
return json.Marshal(seedReconcileResult{
ArtifactID: artifactID,
Manifest: manifest,
RequestContentVerified: requestContentVerified,
})
}
func persistedSeedManifest(raw json.RawMessage, artifactID string) (startup.SeedManifest, bool) {
if len(bytes.TrimSpace(raw)) == 0 {
return startup.SeedManifest{}, false
}
var result seedReconcileResult
if err := decodeJSON(raw, &result); err != nil || result.ArtifactID != artifactID || !result.RequestContentVerified {
return startup.SeedManifest{}, false
}
if result.Manifest.Version != startup.SeedManifestVersion || len(result.Manifest.Files) != 3 {
return startup.SeedManifest{}, false
}
return result.Manifest, true
}
func seedManifestsEqual(left, right startup.SeedManifest) bool {
if left.Version != right.Version || len(left.Files) != len(right.Files) {
return false
}
for name, digest := range right.Files {
if observed, ok := left.Files[name]; !ok || observed != digest {
return false
}
}
return true
}
func targetMatchesUUID(target, uuid string) bool {
target = strings.TrimSpace(target)
if target == "" {
return true
}
canonical, err := hypervisor.CanonicalUUID(target)
return err == nil && canonical == uuid
}
func marshalDomainXMLResult(definition hypervisor.Definition) ([]byte, error) {
return json.Marshal(domainXMLResult{
UUID: definition.UUID,
Revision: definition.Revision,
XMLHash: operation.HashPayload(definition.XML),
})
}
func (s *Server) completeDomainRead(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
var payload domainReadPayload
if err := decodeJSON(request.Payload, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" {
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_request", "domain read payload is invalid")
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain read payload"})
return
}
canonical, err := hypervisor.CanonicalUUID(payload.UUID)
if err != nil || !targetMatchesUUID(request.Target, canonical) {
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_request", "domain read target does not match payload")
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain read payload"})
return
}
payload.UUID = canonical
definition, err := hypervisor.ReadbackDomain(ctx, s.backend, payload.UUID)
if err != nil {
status := operation.StatusUncertain
errorCode := "domain_read_failed"
if hypervisor.IsDefinitive(err) {
status = operation.StatusFailed
errorCode = "domain_read_rejected"
}
updated, updateErr := s.store.Update(record.OperationID, status, nil, errorCode, "domain read-back did not complete")
if updateErr == nil {
if status == operation.StatusFailed {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, updated)
}
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain read-back failed and operation state is uncertain"})
}
return
}
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "readback_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "operation_journal_update_failed", "domain read-back was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
result, _ := marshalDomainXMLResult(definition)
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
// observeDomain performs a read-only XML and power-state observation for one
// explicit UUID. The XML itself never crosses the API or durable journal;
// callers receive only its content hash plus the independently read-back
// libvirt state. Keeping this helper shared by the request and reconcile paths
// prevents the latter from accidentally replaying a lifecycle operation.
func observeDomain(ctx context.Context, backend hypervisor.Backend, uuid string) (domainObserveResult, error) {
definition, err := hypervisor.ReadbackDomain(ctx, backend, uuid)
if err != nil {
return domainObserveResult{}, err
}
state, err := backend.ReadDomainPowerState(ctx, definition.UUID)
if err != nil {
return domainObserveResult{}, err
}
return domainObserveResult{
UUID: definition.UUID,
XMLHash: operation.HashPayload(definition.XML),
State: state,
Active: state.IsActive(),
ObservedAt: time.Now().UTC(),
}, nil
}
func (s *Server) completeDomainObserve(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
uuid, err := parseDomainObservePayload(request.Payload)
if err != nil || !targetMatchesUUID(request.Target, uuid) {
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_observe_request", "domain observe payload is invalid")
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain observe payload"})
return
}
result, err := observeDomain(ctx, s.backend, uuid)
if err != nil {
status := operation.StatusUncertain
errorCode := "domain_observe_uncertain"
if hypervisor.IsDefinitive(err) {
status = operation.StatusFailed
errorCode = "domain_observe_rejected"
}
updated, updateErr := s.store.Update(record.OperationID, status, nil, errorCode, "domain observation did not complete")
if updateErr == nil {
if status == operation.StatusFailed {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, updated)
}
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain observation failed and operation state is uncertain"})
}
return
}
resultJSON, marshalErr := json.Marshal(result)
if marshalErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_observe_result_failed", "domain observation result could not be encoded")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain observation result could not be encoded"})
return
}
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_observe_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain observation was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
func (s *Server) completeDomainPower(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
powerRequest, err := parseDomainPowerRequest(request.Payload)
if err != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_power_request", "domain power payload is invalid")
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain power payload"})
return
}
result, err := s.backend.PowerDomain(ctx, powerRequest.UUID, powerRequest.Action)
if err != nil {
status := operation.StatusUncertain
errorCode := "domain_power_uncertain"
if hypervisor.IsDefinitive(err) {
status = operation.StatusFailed
errorCode = "domain_power_rejected"
}
var resultJSON json.RawMessage
if result.State != "" {
resultJSON, _ = json.Marshal(result)
}
updated, updateErr := s.store.Update(record.OperationID, status, resultJSON, errorCode, "domain power action did not reach a confirmed desired state")
if updateErr == nil {
if status == operation.StatusFailed {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, updated)
}
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power action failed and operation state is uncertain"})
}
return
}
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "power_readback_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "operation_journal_update_failed", "domain power was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
resultJSON, _ := json.Marshal(result)
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
// completeDomainDelete performs only the typed, inactive-domain undefine
// boundary. The hypervisor implementation must prove absence with a targeted
// read-back before this method can persist success. Disk images, seed
// artifacts, firewall rules, and host networking are deliberately untouched;
// the panel's full VM cleanup contract is not represented here.
func (s *Server) completeDomainDelete(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
uuid, err := parseDomainDeletePayload(request.Payload)
if err != nil || !targetMatchesUUID(request.Target, uuid) {
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_delete_request", "domain delete payload is invalid")
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"})
return
}
result, err := s.backend.DeleteDomain(ctx, uuid)
var resultJSON json.RawMessage
if result.UUID != "" {
resultJSON, _ = json.Marshal(result)
}
if err != nil {
status := operation.StatusUncertain
httpStatus := http.StatusServiceUnavailable
errorCode := "domain_delete_uncertain"
message := "domain undefine or absence read-back did not complete"
if errors.Is(err, hypervisor.ErrInvalidUUID) || errors.Is(err, hypervisor.ErrDomainActive) {
status = operation.StatusFailed
httpStatus = http.StatusUnprocessableEntity
if errors.Is(err, hypervisor.ErrDomainActive) {
errorCode = "domain_delete_active"
message = "domain must be inactive before undefine"
} else {
errorCode = "invalid_domain_delete_request"
message = "domain delete UUID is invalid"
}
}
updated, updateErr := s.store.Update(record.OperationID, status, resultJSON, errorCode, message)
if updateErr == nil {
writeJSON(w, httpStatus, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete failed and operation state is uncertain"})
}
return
}
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_delete_readback_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain absence was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
func (s *Server) completeSeedPrepare(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
payload, err := parseSeedPrepareRequest(request.Payload)
if err != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_seed_request", "seed payload is invalid")
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid seed prepare payload"})
return
}
if err := ctx.Err(); err != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "seed_prepare_canceled", "seed preparation was canceled before publication")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed preparation was canceled"})
return
}
files, err := startup.BuildCreateSeed(payload.InstanceID, payload.Hostname, payload.NetworkConfig, payload.StartupScript)
if err != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_seed_request", "seed payload is invalid")
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid seed prepare payload"})
return
}
target := filepath.Join(s.cfg.ArtifactRoot, payload.ArtifactID)
artifact, err := startup.WriteSeedDirectory(target, files)
if err != nil {
status := operation.StatusUncertain
errorCode := "seed_prepare_uncertain"
if errors.Is(err, startup.ErrInvalidSeed) || errors.Is(err, startup.ErrInvalidScript) || errors.Is(err, startup.ErrScriptCreateOnly) || errors.Is(err, startup.ErrArtifactPath) {
status = operation.StatusFailed
errorCode = "seed_prepare_rejected"
}
updated, updateErr := s.store.Update(record.OperationID, status, nil, errorCode, "seed artifact publication was not confirmed")
if updateErr == nil {
if status == operation.StatusFailed {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, updated)
}
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed publication failed and operation state is uncertain"})
}
return
}
result, _ := marshalSeedReconcileResult(payload.ArtifactID, artifact.Manifest, true)
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "seed_published"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, result, "operation_journal_update_failed", "seed artifact was published but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed operation result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed operation result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
// completeDomainPreflight performs a typed, read-only host capability check.
// It intentionally never calls DefineDomain: a successful preflight is only a
// capability/normalization decision, not evidence that storage or a domain was
// created.
func (s *Server) completeDomainPreflight(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
payload, err := parseDomainPreflightRequest(request.Payload)
if err != nil || !targetMatchesUUID(request.Target, payload.UUID) {
code := domainpreflight.Code(err)
if err == nil {
code = "domain_preflight_target_mismatch"
}
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, "domain preflight request was rejected")
if updateErr == nil {
writeJSON(w, http.StatusBadRequest, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight request could not be journaled"})
}
return
}
result, err := domainpreflight.Run(ctx, s.backend, payload)
if err != nil {
status := operation.StatusUncertain
httpStatus := http.StatusServiceUnavailable
code := domainpreflight.Code(err)
message := "domain preflight capability observation did not complete"
if domainpreflight.IsDefinitive(err) {
status = operation.StatusFailed
httpStatus = http.StatusUnprocessableEntity
message = "domain preflight request was rejected"
}
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
if updateErr == nil {
writeJSON(w, httpStatus, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight failed and operation state is uncertain"})
}
return
}
resultJSON, err := json.Marshal(result)
if err != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_preflight_result_failed", "domain preflight result could not be encoded")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight result could not be committed"})
return
}
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_preflight_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain preflight was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
// completeDomainCreatePlan performs a pure creation-plan observation. It
// validates the typed domain/seed/image intent, observes current host
// capabilities, and persists only redacted plan metadata. No seed directory,
// storage volume, domain definition, power state, or network object is
// touched here.
func (s *Server) completeDomainCreatePlan(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
payload, err := parseDomainCreatePlanRequest(request.Payload)
if err != nil || !targetMatchesUUID(request.Target, payload.Domain.UUID) {
code := domaincreateplan.Code(err)
if err == nil {
code = "domain_create_plan_target_mismatch"
}
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, "domain create plan request was rejected")
if updateErr == nil {
writeJSON(w, http.StatusBadRequest, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan request could not be journaled"})
}
return
}
result, err := domaincreateplan.Build(ctx, s.backend, payload)
if err != nil {
status := operation.StatusUncertain
httpStatus := http.StatusServiceUnavailable
code := domaincreateplan.Code(err)
message := "domain create plan host observation did not complete"
if domaincreateplan.IsDefinitive(err) {
status = operation.StatusFailed
httpStatus = http.StatusUnprocessableEntity
message = "domain create plan request was rejected"
}
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
if updateErr == nil {
writeJSON(w, httpStatus, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan failed and operation state is uncertain"})
}
return
}
resultJSON, err := json.Marshal(result)
if err != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_create_plan_result_failed", "domain create plan result could not be encoded")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan result could not be committed"})
return
}
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_create_plan_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain create plan was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
// completeDomainCreateValidate verifies an already-published immutable seed,
// then performs one read-only host capability observation. It deliberately
// stops before storage allocation, seed publication, domain definition,
// lifecycle, or network operations. The seed check comes first so a missing
// or tampered artifact is a definitive rejection even when libvirt is also
// unavailable.
func (s *Server) completeDomainCreateValidate(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
payload, err := parseDomainCreateValidatePayload(request.Payload)
if err != nil || !targetMatchesUUID(request.Target, payload.Domain.UUID) {
code := domainCreateValidateCode(err)
if err == nil {
code = "domain_create_validate_target_mismatch"
}
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, "domain create validation request was rejected")
if updateErr == nil {
writeJSON(w, http.StatusBadRequest, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation request could not be journaled"})
}
return
}
validated, err := domaincreatevalidate.Validate(ctx, s.cfg.ArtifactRoot, payload)
if err != nil {
status := operation.StatusUncertain
httpStatus := http.StatusServiceUnavailable
code := domainCreateValidateCode(err)
message := "published seed validation did not complete"
if domaincreatevalidate.IsDefinitive(err) {
status = operation.StatusFailed
httpStatus = http.StatusUnprocessableEntity
message = "published seed or domain validation was rejected"
}
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
if updateErr == nil {
writeJSON(w, httpStatus, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation failed and operation state is uncertain"})
}
return
}
// Keep the nil-backend case typed and uncertain rather than falling through
// to the generic mutating-backend error. The artifact has been verified, but
// the host capability boundary is not yet proven.
if s.backend == nil {
s.reconcileUncertain(w, record, nil, "domain_create_validate_host_unavailable", "host capability observation is unavailable", http.StatusServiceUnavailable)
return
}
observation, err := domainpreflight.Run(ctx, s.backend, payload.Domain)
if err != nil {
status := operation.StatusUncertain
httpStatus := http.StatusServiceUnavailable
code := domainCreateValidateCode(err)
message := "host capability observation did not complete after seed validation"
if domainpreflight.IsDefinitive(err) {
status = operation.StatusFailed
httpStatus = http.StatusUnprocessableEntity
message = "host firmware policy rejected the validated domain"
}
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
if updateErr == nil {
writeJSON(w, httpStatus, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation failed and operation state is uncertain"})
}
return
}
// Reuse the plan identity check so a fresh capability observation cannot be
// paired with a tampered journal or a different deterministic XML policy.
plan, err := domaincreateplan.BuildFromJournal(observation, payload)
if err != nil {
status := operation.StatusUncertain
httpStatus := http.StatusServiceUnavailable
code := domainCreateValidateCode(err)
message := "validated domain identity no longer matches host observation"
if domaincreateplan.IsDefinitive(err) {
status = operation.StatusFailed
httpStatus = http.StatusConflict
message = "validated domain identity was rejected"
}
updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message)
if updateErr == nil {
writeJSON(w, httpStatus, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation failed and operation state is uncertain"})
}
return
}
resultJSON, err := json.Marshal(domainCreateValidateResult{
Result: validated,
Capabilities: plan.Capabilities,
})
if err != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_create_validate_result_failed", "domain create validation result could not be encoded")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation result could not be committed"})
return
}
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_create_validate_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain create validation was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
// domainCreateValidateCode keeps errors from the plan/preflight packages in
// this operation's namespace. Stable codes are persisted; host error strings
// never cross the API boundary.
func domainCreateValidateCode(err error) string {
if err == nil {
return "domain_create_validate_failed"
}
code := domaincreatevalidate.Code(err)
if strings.HasPrefix(code, "domain_create_validate_") && code != "domain_create_validate_failed" {
return code
}
preflightCode := domainpreflight.Code(err)
if strings.HasPrefix(preflightCode, "domain_preflight_") && preflightCode != "domain_preflight_failed" {
return "domain_create_validate_" + strings.TrimPrefix(preflightCode, "domain_preflight_")
}
planCode := domaincreateplan.Code(err)
if strings.HasPrefix(planCode, "domain_create_plan_") && planCode != "domain_create_plan_failed" {
return "domain_create_validate_" + strings.TrimPrefix(planCode, "domain_create_plan_")
}
return "domain_create_validate_failed"
}
func (s *Server) getOperation(w http.ResponseWriter, r *http.Request) {
record, err := s.storeRecord(r.PathValue("id"))
if err != nil {
writeJSON(w, http.StatusNotFound, errorResponse{Error: "operation not found"})
return
}
writeJSON(w, http.StatusOK, record)
}
func (s *Server) reconcileOperation(w http.ResponseWriter, r *http.Request) {
record, err := s.storeRecord(r.PathValue("id"))
if err != nil {
writeJSON(w, http.StatusNotFound, errorResponse{Error: "operation not found"})
return
}
// Every reconcile handler below is a read-back operation. It may update
// the local journal, but it must never replay the original host mutation.
// Therefore it uses the control-plane gate only and deliberately does not
// require a fresh Probe()/hostReady bit. A failed/stale probe is exactly
// the state in which an operator needs to read back an already ambiguous
// operation. Lifecycle requests created through POST /operations remain
// behind writeAllowed and still fail closed while the host is degraded.
if !s.readbackAllowed(w) {
return
}
if record.Status != operation.StatusUncertain {
writeJSON(w, http.StatusOK, record)
return
}
switch record.Kind {
case "capability_probe":
s.reconcileCapabilityProbe(w, record, r)
case "domain_read":
s.reconcileDomainRead(w, record, r)
case "domain_observe":
s.reconcileDomainObserve(w, record, r)
case "domain_power":
s.reconcileDomainPower(w, record, r)
case "domain_delete":
s.reconcileDomainDelete(w, record, r)
case "seed_prepare":
s.reconcileSeedPrepare(w, record, r)
case "domain_preflight":
s.reconcileDomainPreflight(w, record, r)
case "domain_create_plan":
s.reconcileDomainCreatePlan(w, record, r)
case "domain_create_validate":
s.reconcileDomainCreateValidate(w, record, r)
default:
// No destructive replay is performed. An operation kind that has no
// typed read-back contract remains explicitly unresolved.
writeJSON(w, http.StatusConflict, errorResponse{Error: "operation requires backend-specific reconciliation"})
}
}
// reconcileDomainPreflight repeats only the typed capability observation. It
// never defines a domain or allocates storage/network resources, so a retry is
// safe even when the original HTTP response was lost.
func (s *Server) reconcileDomainPreflight(w http.ResponseWriter, record operation.Record, r *http.Request) {
payload, err := parseDomainPreflightRequest(record.Payload)
if err != nil || !targetMatchesUUID(record.Target, payload.UUID) {
s.reconcileInvalidJournal(w, record, domainpreflight.Code(err), "domain preflight journal payload is invalid")
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
result, err := domainpreflight.Run(ctx, s.backend, payload)
if err != nil {
if domainpreflight.IsDefinitive(err) {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainpreflight.Code(err), "domain preflight request was rejected")
if updateErr == nil {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight reconciliation failed"})
}
return
}
s.reconcileUncertain(w, record, record.Result, domainpreflight.Code(err), "domain preflight capability observation did not complete", http.StatusServiceUnavailable)
return
}
resultJSON, marshalErr := json.Marshal(result)
if marshalErr != nil {
s.reconcileUncertain(w, record, record.Result, "domain_preflight_result_failed", "domain preflight result could not be encoded", http.StatusServiceUnavailable)
return
}
s.reconcileSuccess(w, record, "domain_preflight_confirmed", resultJSON)
}
// reconcileDomainCreatePlan repeats only the read-only capability observation
// and rebuilds the plan identity from the redacted journal. It never renders
// or republishes a seed and never calls a lifecycle/define operation.
func (s *Server) reconcileDomainCreatePlan(w http.ResponseWriter, record operation.Record, r *http.Request) {
var payload domaincreateplan.JournalPayload
if err := decodeJSON(record.Payload, &payload); err != nil || !targetMatchesUUID(record.Target, payload.Domain.UUID) {
s.reconcileInvalidJournal(w, record, "domain_create_plan_journal_invalid", "domain create plan journal payload is invalid")
return
}
if err := domaincreateplan.ValidateJournal(payload); err != nil {
s.reconcileInvalidJournal(w, record, domaincreateplan.Code(err), "domain create plan journal payload is invalid")
return
}
if s.backend == nil {
s.reconcileUncertain(w, record, record.Result, "domain_create_plan_host_unavailable", "host backend is not configured for plan reconciliation", http.StatusServiceUnavailable)
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
observation, err := domainpreflight.Run(ctx, s.backend, payload.Domain)
if err != nil {
if domainpreflight.IsDefinitive(err) {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domaincreateplan.Code(err), "domain create plan host policy was rejected")
if updateErr == nil {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan reconciliation failed"})
}
return
}
s.reconcileUncertain(w, record, record.Result, domaincreateplan.Code(err), "domain create plan host observation did not complete", http.StatusServiceUnavailable)
return
}
result, err := domaincreateplan.BuildFromJournal(observation, payload)
if err != nil {
if domaincreateplan.IsDefinitive(err) {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domaincreateplan.Code(err), "domain create plan journal no longer proves the same intent")
if updateErr == nil {
writeJSON(w, http.StatusConflict, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan reconciliation failed"})
}
return
}
s.reconcileUncertain(w, record, record.Result, domaincreateplan.Code(err), "domain create plan reconciliation remains uncertain", http.StatusServiceUnavailable)
return
}
resultJSON, marshalErr := json.Marshal(result)
if marshalErr != nil {
s.reconcileUncertain(w, record, record.Result, "domain_create_plan_result_failed", "domain create plan result could not be encoded", http.StatusServiceUnavailable)
return
}
s.reconcileSuccess(w, record, "domain_create_plan_confirmed", resultJSON)
}
// reconcileDomainCreateValidate repeats the immutable seed read-back before
// obtaining a fresh capability observation. It never republishes a seed,
// defines a domain, changes power, or configures networking.
func (s *Server) reconcileDomainCreateValidate(w http.ResponseWriter, record operation.Record, r *http.Request) {
var payload domaincreateplan.JournalPayload
if err := decodeJSON(record.Payload, &payload); err != nil || !targetMatchesUUID(record.Target, payload.Domain.UUID) {
s.reconcileInvalidJournal(w, record, "domain_create_validate_journal_invalid", "domain create validation journal payload is invalid")
return
}
if err := domaincreateplan.ValidateJournal(payload); err != nil {
s.reconcileInvalidJournal(w, record, domainCreateValidateCode(err), "domain create validation journal payload is invalid")
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
validated, err := domaincreatevalidate.Validate(ctx, s.cfg.ArtifactRoot, payload)
if err != nil {
if domaincreatevalidate.IsDefinitive(err) {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainCreateValidateCode(err), "published seed or domain validation was rejected")
if updateErr == nil {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation reconciliation failed"})
}
return
}
s.reconcileUncertain(w, record, record.Result, domainCreateValidateCode(err), "published seed validation did not complete", http.StatusServiceUnavailable)
return
}
if s.backend == nil {
s.reconcileUncertain(w, record, record.Result, "domain_create_validate_host_unavailable", "host capability observation is unavailable", http.StatusServiceUnavailable)
return
}
observation, err := domainpreflight.Run(ctx, s.backend, payload.Domain)
if err != nil {
if domainpreflight.IsDefinitive(err) {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainCreateValidateCode(err), "host firmware policy rejected the validated domain")
if updateErr == nil {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation reconciliation failed"})
}
return
}
s.reconcileUncertain(w, record, record.Result, domainCreateValidateCode(err), "host capability observation did not complete after seed validation", http.StatusServiceUnavailable)
return
}
plan, err := domaincreateplan.BuildFromJournal(observation, payload)
if err != nil {
if domaincreateplan.IsDefinitive(err) {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainCreateValidateCode(err), "validated domain identity no longer matches host observation")
if updateErr == nil {
writeJSON(w, http.StatusConflict, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation reconciliation failed"})
}
return
}
s.reconcileUncertain(w, record, record.Result, domainCreateValidateCode(err), "domain create validation remains uncertain", http.StatusServiceUnavailable)
return
}
resultJSON, marshalErr := json.Marshal(domainCreateValidateResult{
Result: validated,
Capabilities: plan.Capabilities,
})
if marshalErr != nil {
s.reconcileUncertain(w, record, record.Result, "domain_create_validate_result_failed", "domain create validation result could not be encoded", http.StatusServiceUnavailable)
return
}
s.reconcileSuccess(w, record, "domain_create_validate_confirmed", resultJSON)
}
// reconcileCapabilityProbe performs only the capability snapshot read-back.
// Probe is intentionally not called here: even though the current providers
// implement Probe as read-only, reconciliation must depend on the narrowest
// contract and never replay the original operation.
func (s *Server) reconcileCapabilityProbe(w http.ResponseWriter, record operation.Record, r *http.Request) {
if s.caps == nil {
s.reconcileUncertain(w, record, record.Result, "capability_reconcile_uncertain", "capability snapshot provider is unavailable", http.StatusServiceUnavailable)
return
}
if payload := bytes.TrimSpace(record.Payload); len(payload) != 0 {
var empty struct{}
if err := decodeJSON(payload, &empty); err != nil {
s.reconcileInvalidJournal(w, record, "capability_probe_payload_invalid", "capability probe journal payload is invalid")
return
}
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
snapshot, err := s.caps.Snapshot(ctx)
if err != nil {
s.reconcileUncertain(w, record, record.Result, "capability_reconcile_uncertain", "capability snapshot read-back did not complete", http.StatusServiceUnavailable)
return
}
if target := strings.TrimSpace(record.Target); target != "" && snapshot.AgentID != target {
result, _ := json.Marshal(snapshot)
s.reconcileUncertain(w, record, result, "capability_target_mismatch", "capability snapshot belongs to a different agent", http.StatusConflict)
return
}
result, _ := json.Marshal(snapshot)
s.reconcileSuccess(w, record, "capability_readback_confirmed", result)
}
// reconcileDomainRead performs a read-only dumpxml and never invokes a
// define/start/lifecycle method. A missing or policy-mismatched definition is
// left uncertain: the agent must not infer a destructive conclusion from one
// failed observation.
func (s *Server) reconcileDomainRead(w http.ResponseWriter, record operation.Record, r *http.Request) {
if s.backend == nil {
s.reconcileUncertain(w, record, record.Result, "domain_read_reconcile_uncertain", "host backend is not configured for domain read-back", http.StatusServiceUnavailable)
return
}
payload, err := parseDomainReadPayload(record.Payload)
if err != nil || !targetMatchesUUID(record.Target, payload) {
s.reconcileInvalidJournal(w, record, "invalid_domain_request", "domain read journal payload is invalid")
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
definition, err := hypervisor.ReadbackDomain(ctx, s.backend, payload)
if err != nil {
s.reconcileUncertain(w, record, record.Result, "domain_read_reconcile_uncertain", "domain read-back did not complete", http.StatusServiceUnavailable)
return
}
result, _ := marshalDomainXMLResult(definition)
s.reconcileSuccess(w, record, "domain_readback_confirmed", result)
}
// reconcileDomainObserve repeats only the typed XML/state read-back. It never
// invokes a lifecycle or define command, so an uncertain response cannot
// cause a host mutation when the panel retries reconciliation.
func (s *Server) reconcileDomainObserve(w http.ResponseWriter, record operation.Record, r *http.Request) {
if s.backend == nil {
s.reconcileUncertain(w, record, record.Result, "domain_observe_reconcile_uncertain", "host backend is not configured for domain observation", http.StatusServiceUnavailable)
return
}
uuid, err := parseDomainObservePayload(record.Payload)
if err != nil || !targetMatchesUUID(record.Target, uuid) {
s.reconcileInvalidJournal(w, record, "invalid_domain_observe_request", "domain observe journal payload is invalid")
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
result, err := observeDomain(ctx, s.backend, uuid)
if err != nil {
code := "domain_observe_reconcile_uncertain"
if hypervisor.IsDefinitive(err) {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, "domain_observe_rejected", "domain observation was rejected during reconciliation")
if updateErr == nil {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain observation reconciliation failed"})
}
return
}
s.reconcileUncertain(w, record, record.Result, code, "domain observation read-back did not complete", http.StatusServiceUnavailable)
return
}
resultJSON, marshalErr := json.Marshal(result)
if marshalErr != nil {
s.reconcileUncertain(w, record, record.Result, "domain_observe_result_failed", "domain observation result could not be encoded", http.StatusServiceUnavailable)
return
}
s.reconcileSuccess(w, record, "domain_observe_confirmed", resultJSON)
}
// reconcileSeedPrepare verifies the already-published immutable seed and
// compares its manifest with a deterministic rendering of the journaled
// request. It never calls WriteSeedDirectory or otherwise republishes files.
func (s *Server) reconcileSeedPrepare(w http.ResponseWriter, record operation.Record, r *http.Request) {
payload, err := parseSeedJournalPayload(record.Payload)
if err != nil || !targetMatchesUUID(record.Target, payload.ArtifactID) {
s.reconcileInvalidJournal(w, record, "invalid_seed_request", "seed prepare journal payload is invalid")
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
if err := ctx.Err(); err != nil {
s.reconcileUncertain(w, record, nil, "seed_reconcile_uncertain", "seed artifact read-back was canceled", http.StatusServiceUnavailable)
return
}
observed, err := startup.VerifySeedDirectory(filepath.Join(s.cfg.ArtifactRoot, payload.ArtifactID))
if err != nil {
s.reconcileUncertain(w, record, record.Result, "seed_reconcile_uncertain", "seed artifact read-back did not complete", http.StatusServiceUnavailable)
return
}
requestContentVerified := false
if expected, ok := persistedSeedManifest(record.Result, payload.ArtifactID); ok {
if !seedManifestsEqual(observed, expected) {
result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, false)
s.reconcileUncertain(w, record, result, "seed_reconcile_mismatch", "seed artifact does not match the persisted request manifest", http.StatusConflict)
return
}
requestContentVerified = true
} else if payload.StartupScript != "[redacted]" {
expectedFiles, renderErr := startup.BuildCreateSeed(payload.InstanceID, payload.Hostname, payload.NetworkConfig, payload.StartupScript)
if renderErr != nil {
s.reconcileUncertain(w, record, nil, "seed_reconcile_mismatch", "seed request could not be rendered for comparison", http.StatusConflict)
return
}
expected := seedManifestFor(expectedFiles)
if !seedManifestsEqual(observed, expected) {
result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, false)
s.reconcileUncertain(w, record, result, "seed_reconcile_mismatch", "seed artifact does not match the requested content", http.StatusConflict)
return
}
requestContentVerified = true
} else if payload.StartupScript == "[redacted]" {
// A safe journal intentionally omits the request's network document and
// startup script. Once the response/result manifest is also missing,
// an artifact read-back proves only that *some* immutable seed exists;
// it cannot prove that it is the artifact requested by this operation.
// Never turn that integrity-only observation into a succeeded operation.
// Keep the observed manifest (with an explicit false flag) so an
// operator can inspect it while the operation remains reconcilable.
result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, false)
code := "seed_reconcile_content_unproven"
message := "seed artifact integrity was observed, but request content equivalence could not be proven"
if record.ErrorCode == "seed_reconcile_mismatch" {
// Preserve the stronger diagnostic for a previously observed
// mismatch, while retaining the non-successful uncertain state.
code = "seed_reconcile_mismatch"
message = "seed request content remains unavailable for comparison"
}
s.reconcileUncertain(w, record, result, code, message, http.StatusConflict)
return
}
// At this point the journal carried the original request fields (for
// example, a test/recovery record created before the digest-only projection
// was enabled), so the deterministic render above proved equivalence.
result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, requestContentVerified)
s.reconcileSuccess(w, record, "seed_readback_confirmed", result)
}
func (s *Server) reconcileSuccess(w http.ResponseWriter, record operation.Record, boundary string, result []byte) {
if _, err := s.store.SetCommitBoundary(record.OperationID, boundary); err != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, result, "operation_journal_update_failed", "read-back was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
func (s *Server) reconcileUncertain(w http.ResponseWriter, record operation.Record, result []byte, code, message string, status int) {
updated, err := s.store.Update(record.OperationID, operation.StatusUncertain, result, code, message)
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation reconciliation remains uncertain"})
return
}
writeJSON(w, status, updated)
}
func (s *Server) reconcileInvalidJournal(w http.ResponseWriter, record operation.Record, code, message string) {
updated, err := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, message)
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation journal payload is invalid"})
return
}
writeJSON(w, http.StatusUnprocessableEntity, updated)
}
// reconcileDomainPower performs only a typed, read-only state probe. It never
// calls PowerDomain: an ambiguous lifecycle response must not be replayed just
// because a caller asked to reconcile it.
func (s *Server) reconcileDomainPower(w http.ResponseWriter, record operation.Record, r *http.Request) {
if s.backend == nil {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, record.Result, "domain_power_reconcile_uncertain", "host backend is not configured for state reconciliation")
if updateErr == nil {
writeJSON(w, http.StatusServiceUnavailable, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power reconciliation is unavailable"})
}
return
}
powerRequest, err := parseDomainPowerRequest(record.Payload)
if err != nil || !targetMatchesUUID(record.Target, powerRequest.UUID) {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_power_request", "domain power journal payload is invalid")
if updateErr == nil {
writeJSON(w, http.StatusUnprocessableEntity, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power journal payload is invalid"})
}
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
state, err := s.backend.ReadDomainPowerState(ctx, powerRequest.UUID)
if err != nil {
// A failed read cannot prove that the requested outcome did not happen;
// keep the operation uncertain and require a later probe.
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, record.Result, "domain_power_reconcile_uncertain", "domain power state read-back did not complete")
if updateErr == nil {
writeJSON(w, http.StatusServiceUnavailable, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power state reconciliation is uncertain"})
}
return
}
result := hypervisor.PowerResult{
UUID: powerRequest.UUID,
Action: powerRequest.Action,
State: state,
Active: state.IsActive(),
ObservedAt: time.Now().UTC(),
}
resultJSON, _ := json.Marshal(result)
if !hypervisor.PowerActionReconcileSatisfied(powerRequest.Action, state) {
updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "domain_power_not_reconciled", "observed domain power state does not satisfy the requested action")
if updateErr == nil {
writeJSON(w, http.StatusConflict, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power state remains uncertain"})
}
return
}
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "power_readback_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain power was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
// reconcileDomainDelete is strictly read-only. It observes whether the exact
// definition UUID is absent and never retries undefine after an ambiguous
// response. A present definition, an active guest, or a failed observation
// keeps the journal uncertain for a later operator-approved read-back.
func (s *Server) reconcileDomainDelete(w http.ResponseWriter, record operation.Record, r *http.Request) {
if s.backend == nil {
s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "host backend is not configured for domain deletion read-back", http.StatusServiceUnavailable)
return
}
uuid, err := parseDomainDeletePayload(record.Payload)
if err != nil || !targetMatchesUUID(record.Target, uuid) {
s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal payload is invalid")
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
observation, err := s.backend.InspectDomainForDeletion(ctx, uuid)
if err != nil {
if errors.Is(err, hypervisor.ErrInvalidUUID) {
s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal UUID is invalid")
return
}
s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "domain deletion absence read-back did not complete", http.StatusServiceUnavailable)
return
}
if observation.Present {
result, _ := json.Marshal(observation)
s.reconcileUncertain(w, record, result, "domain_delete_not_reconciled", "domain definition is still present; no undefine was replayed", http.StatusConflict)
return
}
result, marshalErr := json.Marshal(hypervisor.DomainDeleteResult{
UUID: uuid,
AlreadyAbsent: true,
Removed: true,
ObservedAt: observation.ObservedAt,
})
if marshalErr != nil {
s.reconcileUncertain(w, record, record.Result, "domain_delete_result_failed", "domain deletion result could not be encoded", http.StatusServiceUnavailable)
return
}
s.reconcileSuccess(w, record, "domain_delete_readback_confirmed", result)
}
func (s *Server) storeRecord(id string) (operation.Record, error) {
if s.store == nil {
return operation.Record{}, operation.ErrNotFound
}
return s.store.Get(id)
}
func isPlanningKind(kind string) bool {
return kind == "domain_preflight" || kind == "domain_create_plan" || kind == "domain_create_validate"
}
func (s *Server) writeAllowed(w http.ResponseWriter, ctx context.Context) bool {
if s.refreshHostReadiness(ctx) && s.readiness() == config.ReadinessReady {
return true
}
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent is degraded and mutating operations are disabled"})
return false
}
// readbackAllowed is the control-plane gate for read-only planning,
// validation, and reconciliation operations. These operations may obtain a
// fresh capability/artifact/domain observation and commit that observation to
// the local journal, but they must not require a successful host Probe(). A
// failed/stale probe is precisely when an ambiguous operation needs a later
// read-back. Transport authentication is enforced by Handler; this method
// covers the remaining local control-plane invariants and keeps the durable
// journal mandatory.
func (s *Server) readbackAllowed(w http.ResponseWriter) bool {
return s.controlPlaneReadOnlyAllowed(w, "read-back")
}
func (s *Server) controlPlaneReadOnlyAllowed(w http.ResponseWriter, operation string) bool {
if s == nil || s.store == nil || !s.store.Writable() || s.cfg.AgentID == "" {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent control plane is degraded and " + operation + " is disabled"})
return false
}
s.stateMu.RLock()
tlsReady := s.tlsReady
s.stateMu.RUnlock()
if s.cfg.RequireMTLS && (!s.cfg.TLSComplete() || !tlsReady) {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent TLS is not ready for " + operation})
return false
}
if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent client certificate policy is not ready for " + operation})
return false
}
if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent request signing is not ready for " + operation})
return false
}
return true
}
// preflightAllowed is kept as a narrow compatibility name for the create
// planning path. Both planning and reconciliation are read-only control-plane
// operations and intentionally share the same host-readiness-independent
// checks, while retaining the preflight-specific error wording.
func (s *Server) preflightAllowed(w http.ResponseWriter) bool {
return s.controlPlaneReadOnlyAllowed(w, "preflight")
}
func (s *Server) authorized(r *http.Request) bool {
if !s.cfg.RequireMTLS && s.cfg.IsLoopbackListen() {
return true
}
if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 {
return false
}
if !s.cfg.RequireMTLS {
return false
}
leaf := r.TLS.PeerCertificates[0]
digest := sha256.Sum256(leaf.Raw)
observed := hex.EncodeToString(digest[:])
for _, expected := range s.cfg.ClientCertFingerprints {
if strings.EqualFold(strings.TrimSpace(expected), observed) {
return true
}
}
return false
}
// TLSConfig constructs a mutual-TLS configuration only when all configured
// files can be read. The caller may keep serving health in degraded mode when
// this returns an error.
func (s *Server) TLSConfig() (*tls.Config, error) {
if !s.cfg.TLSComplete() {
return nil, fmt.Errorf("TLS certificate and key are not configured")
}
cert, err := tls.LoadX509KeyPair(s.cfg.TLSCertFile, s.cfg.TLSKeyFile)
if err != nil {
return nil, fmt.Errorf("load agent TLS certificate: %w", err)
}
caData, err := os.ReadFile(s.cfg.TLSCAFile)
if err != nil {
return nil, fmt.Errorf("read agent TLS CA: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caData) {
return nil, fmt.Errorf("parse agent TLS CA")
}
return &tls.Config{
MinVersion: tls.VersionTLS13,
Certificates: []tls.Certificate{cert},
ClientCAs: pool,
ClientAuth: tls.RequireAndVerifyClientCert,
NextProtos: []string{"h2", "http/1.1"},
}, nil
}
func readBody(r *http.Request, max int64) ([]byte, error) {
if max <= 0 {
return nil, fmt.Errorf("request body limit is not configured")
}
data, err := io.ReadAll(io.LimitReader(r.Body, max+1))
if err != nil {
return nil, fmt.Errorf("read request body: %w", err)
}
if int64(len(data)) > max {
return nil, fmt.Errorf("request body exceeds limit")
}
return data, nil
}
func decodeJSON(data []byte, target any) error {
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
if err := decoder.Decode(target); err != nil {
return err
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return fmt.Errorf("multiple JSON values")
}
return nil
}
func writeJSON(w http.ResponseWriter, status int, value any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(value)
}
// NewHTTPServer provides conservative timeout defaults for a caller that
// wants to run the agent directly from main or an integration test.
func (s *Server) NewHTTPServer() *http.Server {
return &http.Server{
Addr: s.cfg.ListenAddr,
Handler: s.Handler(),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: s.cfg.RequestTimeout,
WriteTimeout: s.cfg.RequestTimeout,
IdleTimeout: 60 * time.Second,
MaxHeaderBytes: 32 << 10,
}
}
func remoteIP(r *http.Request) net.IP {
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
return nil
}
return net.ParseIP(strings.Trim(host, "[]"))
}