// Package api implements the versioned panel-to-agent HTTP boundary. package api import ( "bytes" "context" "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/hex" "encoding/json" "errors" "fmt" "io" "log/slog" "net" "net/http" "os" "path/filepath" "strings" "sync" "time" "g.luoli.live/root/virt/agent/internal/auth" "g.luoli.live/root/virt/agent/internal/capability" "g.luoli.live/root/virt/agent/internal/config" "g.luoli.live/root/virt/agent/internal/domaincreateplan" "g.luoli.live/root/virt/agent/internal/domaincreatevalidate" "g.luoli.live/root/virt/agent/internal/domainpreflight" "g.luoli.live/root/virt/agent/internal/hypervisor" "g.luoli.live/root/virt/agent/internal/operation" "g.luoli.live/root/virt/agent/internal/startup" ) type Options struct { Config config.Config Warnings []config.Warning Store *operation.Store Capabilities capability.Provider Backend hypervisor.Backend // HostBackendReady is explicit so a static provider cannot accidentally // admit mutating operations in a production process. Tests may set it when // they intentionally use a deterministic fake provider. HostBackendReady bool // TLSReady is set after the process successfully loads the configured // certificate material. A path being configured is not proof that a remote // listener can actually enforce mTLS. TLSReady bool Auth *auth.Verifier Logger *slog.Logger } type Server struct { cfg config.Config warnings []config.Warning store *operation.Store caps capability.Provider backend hypervisor.Backend stateMu sync.RWMutex hostReady bool tlsReady bool // readinessProbeMu serializes health-triggered host probes. Health is an // unauthenticated liveness endpoint, so without a small single-flight // boundary every remote poll could start a fresh virsh/libvirt probe. readinessProbeMu sync.Mutex readinessProbeAt time.Time readinessProbeDone chan struct{} auth *auth.Verifier logger *slog.Logger mux *http.ServeMux } // healthReadinessProbeTTL bounds how often the unauthenticated health route // starts a new host capability probe. Mutating request gates continue to use // refreshHostReadiness directly so they retain a fresh, fail-closed probe. const healthReadinessProbeTTL = time.Second type healthResponse struct { Status string `json:"status"` Readiness config.Readiness `json:"readiness"` AgentID string `json:"agent_id,omitempty"` ProtocolVersion string `json:"protocol_version"` Warnings []config.Warning `json:"warnings,omitempty"` Now time.Time `json:"now"` } type operationRequest struct { OperationID string `json:"operation_id"` IdempotencyKey string `json:"idempotency_key"` Kind string `json:"kind"` Target string `json:"target,omitempty"` Payload json.RawMessage `json:"payload,omitempty"` } type errorResponse struct { Error string `json:"error"` } func New(opts Options) (*Server, error) { if err := opts.Config.Validate(); err != nil { return nil, err } if opts.Capabilities == nil { return nil, fmt.Errorf("capability provider is required") } if opts.Auth == nil { if opts.Config.RequireSigned { // A required verifier with no configured key must remain unavailable. // Falling back to NewDisabled here would silently turn a required // authenticated control plane into an unsigned one whenever the // caller forgot to inject Auth. verifier, err := auth.NewWithNonceConsumerForAgent(nil, true, opts.Store, opts.Config.AgentID) if err != nil { return nil, fmt.Errorf("configure request signing verifier: %w", err) } opts.Auth = verifier } else { opts.Auth = auth.NewDisabled() } } if opts.Logger == nil { opts.Logger = slog.New(slog.NewTextHandler(io.Discard, nil)) } s := &Server{ cfg: opts.Config, warnings: append([]config.Warning(nil), opts.Warnings...), store: opts.Store, caps: opts.Capabilities, backend: opts.Backend, hostReady: opts.HostBackendReady, tlsReady: !opts.Config.RequireMTLS || opts.TLSReady, auth: opts.Auth, logger: opts.Logger, mux: http.NewServeMux(), } s.routes() return s, nil } func (s *Server) Handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v1/health" { if !s.authorized(r) { writeJSON(w, http.StatusUnauthorized, errorResponse{Error: "client authentication required"}) return } body, err := readBody(r, s.cfg.MaxBodyBytes) if err != nil { writeJSON(w, http.StatusRequestEntityTooLarge, errorResponse{Error: "request body exceeds limit"}) return } r.Body = io.NopCloser(bytes.NewReader(body)) if err := s.auth.Verify(r, body, time.Now().UTC()); err != nil { if errors.Is(err, auth.ErrUnavailable) { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "request signing is not ready"}) } else { writeJSON(w, http.StatusUnauthorized, errorResponse{Error: "invalid signed request envelope"}) } return } } s.mux.ServeHTTP(w, r) }) } func (s *Server) routes() { s.mux.HandleFunc("GET /v1/health", s.health) s.mux.HandleFunc("GET /v1/capabilities", s.capabilities) s.mux.HandleFunc("POST /v1/operations", s.createOperation) s.mux.HandleFunc("GET /v1/operations/{id}", s.getOperation) s.mux.HandleFunc("POST /v1/operations/{id}/reconcile", s.reconcileOperation) // A direct read-only probe is useful during enrollment and does not mutate // host networking, domains, storage, or firewall state. s.mux.HandleFunc("POST /v1/agent/test", s.agentTest) } func (s *Server) health(w http.ResponseWriter, r *http.Request) { // Readiness is a live host fact, not only a startup result. A libvirt // daemon or socket can disappear after the process starts; refresh it on // health and write boundaries so monitoring does not report a stale green // state and mutating requests fail closed. s.refreshHostReadinessCached(r.Context()) readiness := s.readiness() status := "ok" if readiness != config.ReadinessReady { status = "degraded" } warnings := append([]config.Warning(nil), s.warnings...) if s.store == nil || !s.store.Writable() { warnings = append(warnings, config.Warning{Code: "operation_journal_unavailable", Message: "operation writes are disabled until the local journal is available"}) } if s.cfg.AgentID == "" { // Keep this warning stable and non-sensitive for monitoring clients. warnings = append(warnings, config.Warning{Code: "agent_identity_missing", Message: "agent identity is not configured; mutating requests remain disabled"}) } if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) { warnings = append(warnings, config.Warning{Code: "request_signing_not_ready", Message: "signed request verification is not ready; protected requests remain disabled"}) } if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 { warnings = append(warnings, config.Warning{Code: "client_certificate_not_pinned", Message: "no client certificate fingerprint is registered; protected requests remain disabled"}) } if s.cfg.RequireMTLS && !s.tlsReady { warnings = append(warnings, config.Warning{Code: "tls_listener_not_ready", Message: "TLS material is not loaded; protected writes remain disabled"}) } writeJSON(w, http.StatusOK, healthResponse{ Status: status, Readiness: readiness, AgentID: s.cfg.AgentID, ProtocolVersion: s.cfg.ProtocolVersion, Warnings: warnings, Now: time.Now().UTC(), }) } func (s *Server) readiness() config.Readiness { s.stateMu.RLock() hostReady := s.hostReady tlsReady := s.tlsReady s.stateMu.RUnlock() if s.store == nil || !s.store.Writable() || s.cfg.AgentID == "" || !hostReady { return config.ReadinessDegraded } if s.cfg.RequireMTLS && (!s.cfg.TLSComplete() || !tlsReady) { return config.ReadinessDegraded } if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 { return config.ReadinessDegraded } if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) { return config.ReadinessDegraded } return config.ReadinessReady } // SetTLSReady publishes the result of loading certificate material after the // server has been constructed. A failed load keeps liveness available while // protected requests remain fail-closed. func (s *Server) SetTLSReady(ready bool) { if s == nil { return } s.stateMu.Lock() defer s.stateMu.Unlock() s.tlsReady = ready } // refreshHostReadiness performs a bounded, read-only capability probe. It is // intentionally independent from operation execution: a failed probe never // invokes a lifecycle or define command and only downgrades readiness. func (s *Server) refreshHostReadiness(parent context.Context) bool { if s == nil || s.backend == nil { return s.hostBackendReady() } return s.probeHostReadiness(parent) } // refreshHostReadinessCached is used only by health. It coalesces concurrent // requests into one bounded probe and briefly reuses its result, including a // failed result. The latter matters for fail-closed behavior: an attacker // must not be able to turn a failed host into an unbounded subprocess storm by // polling the public health endpoint. func (s *Server) refreshHostReadinessCached(parent context.Context) bool { if s == nil || s.backend == nil { return s.hostBackendReady() } if parent == nil { parent = context.Background() } s.readinessProbeMu.Lock() now := time.Now() if !s.readinessProbeAt.IsZero() && now.Sub(s.readinessProbeAt) < healthReadinessProbeTTL { s.readinessProbeMu.Unlock() return s.hostBackendReady() } if done := s.readinessProbeDone; done != nil { s.readinessProbeMu.Unlock() select { case <-done: return s.hostBackendReady() case <-parent.Done(): return s.hostBackendReady() } } done := make(chan struct{}) s.readinessProbeDone = done s.readinessProbeMu.Unlock() ready := s.probeHostReadiness(parent) s.readinessProbeMu.Lock() s.readinessProbeAt = time.Now() s.readinessProbeDone = nil close(done) s.readinessProbeMu.Unlock() return ready } // probeHostReadiness performs the actual bounded, read-only backend probe. // Callers choose whether to serialize/cache it based on the request boundary. func (s *Server) probeHostReadiness(parent context.Context) bool { if s == nil || s.backend == nil { return s.hostBackendReady() } if parent == nil { parent = context.Background() } timeout := s.cfg.RequestTimeout if timeout <= 0 || timeout > 2*time.Second { timeout = 2 * time.Second } ctx, cancel := context.WithTimeout(parent, timeout) defer cancel() _, err := s.backend.Probe(ctx) s.stateMu.Lock() s.hostReady = err == nil ready := s.hostReady s.stateMu.Unlock() return ready } func (s *Server) hostBackendReady() bool { if s == nil { return false } s.stateMu.RLock() defer s.stateMu.RUnlock() return s.hostReady } func (s *Server) capabilities(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() snapshot, err := s.caps.Snapshot(ctx) if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability probe failed"}) return } writeJSON(w, http.StatusOK, snapshot) } func (s *Server) agentTest(w http.ResponseWriter, r *http.Request) { if !s.writeAllowed(w, r.Context()) { return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() if err := s.caps.Probe(ctx); err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability probe failed"}) return } snapshot, err := s.caps.Snapshot(ctx) if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability snapshot failed"}) return } writeJSON(w, http.StatusOK, snapshot) } func (s *Server) createOperation(w http.ResponseWriter, r *http.Request) { body, err := readBody(r, s.cfg.MaxBodyBytes) if err != nil { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid operation request body"}) return } var request operationRequest if err := decodeJSON(body, &request); err != nil { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid operation JSON"}) return } if err := validateOperationHeaders(r, request, body, s.cfg.RequireSigned); err != nil { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid operation request headers"}) return } // Raw XML define is intentionally not a public Agent operation. The // existing hypervisor backend can define an arbitrary caller-supplied XML // document, while its read-back contract proves only UUID/firmware. Keep // that legacy implementation unreachable until the transactional typed // create executor has ownership markers, exact disk/seed read-back and // fenced cleanup. if request.Kind == "domain_define" { writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "domain define is disabled until the typed create executor is available"}) return } if request.Kind != "capability_probe" && request.Kind != "domain_read" && request.Kind != "domain_observe" && request.Kind != "domain_power" && request.Kind != "domain_delete" && request.Kind != "seed_prepare" && request.Kind != "domain_preflight" && request.Kind != "domain_create_plan" && request.Kind != "domain_create_validate" { writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "operation kind is not implemented by the agent"}) return } // Planning operations are deliberately the escape hatch from a stale host // readiness result: their purpose is to obtain a fresh capability // observation that can repair that result. They still require the complete // control-plane gate (authenticated transport, agent identity and a // durable journal), but must not require a successful Probe() first. // Every other operation remains behind the normal host readiness gate. if isPlanningKind(request.Kind) { if !s.preflightAllowed(w) { return } } else if !s.writeAllowed(w, r.Context()) { return } if request.Kind == "domain_preflight" { preflightRequest, parseErr := parseDomainPreflightRequest(request.Payload) if parseErr != nil || !targetMatchesUUID(request.Target, preflightRequest.UUID) { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain preflight payload"}) return } } if request.Kind == "domain_create_plan" { planRequest, parseErr := parseDomainCreatePlanRequest(request.Payload) if parseErr != nil || !targetMatchesUUID(request.Target, planRequest.Domain.UUID) { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain create plan payload"}) return } } if request.Kind == "domain_create_validate" { validatePayload, parseErr := parseDomainCreateValidatePayload(request.Payload) if parseErr != nil || !targetMatchesUUID(request.Target, validatePayload.Domain.UUID) { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain create validate payload"}) return } } if request.Kind == "domain_power" { powerRequest, parseErr := parseDomainPowerRequest(request.Payload) if parseErr != nil || !targetMatchesUUID(request.Target, powerRequest.UUID) { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain power payload"}) return } } if request.Kind == "domain_delete" { uuid, parseErr := parseDomainDeletePayload(request.Payload) if parseErr != nil || !targetMatchesUUID(request.Target, uuid) { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"}) return } } if request.Kind == "domain_read" { uuid, parseErr := parseDomainReadPayload(request.Payload) if parseErr != nil || !targetMatchesUUID(request.Target, uuid) { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain read payload"}) return } } if request.Kind == "domain_observe" { uuid, parseErr := parseDomainObservePayload(request.Payload) if parseErr != nil || !targetMatchesUUID(request.Target, uuid) { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain observe payload"}) return } } if request.Kind == "seed_prepare" { seedRequest, parseErr := parseSeedPrepareRequest(request.Payload) if parseErr != nil || !targetMatchesUUID(request.Target, seedRequest.ArtifactID) { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid seed prepare payload"}) return } } journalPayload, err := safeJournalPayload(request.Kind, request.Payload) if err != nil { writeJSON(w, http.StatusBadRequest, errorResponse{Error: "operation payload cannot be safely journaled"}) return } record, replay, err := s.store.Begin(operation.BeginRequest{ OperationID: request.OperationID, IdempotencyKey: request.IdempotencyKey, Kind: request.Kind, Target: request.Target, RequestHash: operation.HashPayload(body), // The request hash above covers the complete body for idempotency, but // the durable journal receives only a typed, non-secret intent. XML, // network documents and startup scripts are never copied into a // plaintext operation record merely because a request timed out. Payload: journalPayload, }) if err != nil { status := http.StatusBadRequest if errors.Is(err, operation.ErrConflict) { status = http.StatusConflict } else if errors.Is(err, operation.ErrNotWritable) { status = http.StatusServiceUnavailable } message := "operation request was rejected" if errors.Is(err, operation.ErrConflict) { message = "operation identity conflicts with an existing record" } else if errors.Is(err, operation.ErrNotWritable) { message = "operation journal is not writable" } writeJSON(w, status, errorResponse{Error: message}) return } if replay { writeJSON(w, http.StatusOK, record) return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() if _, err := s.store.Update(record.OperationID, operation.StatusRunning, nil, "", ""); err != nil { writeJSON(w, http.StatusInternalServerError, errorResponse{Error: "operation journal update failed"}) return } if request.Kind == "capability_probe" { s.completeCapabilityProbe(w, record, ctx) return } // domain_preflight owns its own typed nil-backend/unavailable result. Keep // it ahead of the generic backend guard so the durable operation contains // domain_preflight_host_unavailable rather than backend_unavailable. if request.Kind == "domain_preflight" { s.completeDomainPreflight(w, record, request, ctx) return } if request.Kind == "domain_create_plan" { s.completeDomainCreatePlan(w, record, request, ctx) return } if request.Kind == "domain_create_validate" { s.completeDomainCreateValidate(w, record, request, ctx) return } if s.backend == nil { _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "backend_unavailable", "host backend is not configured") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "host backend is not configured"}) return } if request.Kind == "domain_read" { s.completeDomainRead(w, record, request, ctx) return } if request.Kind == "domain_observe" { s.completeDomainObserve(w, record, request, ctx) return } if request.Kind == "domain_power" { s.completeDomainPower(w, record, request, ctx) return } if request.Kind == "domain_delete" { s.completeDomainDelete(w, record, request, ctx) return } if request.Kind == "seed_prepare" { s.completeSeedPrepare(w, record, request, ctx) return } _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "operation_not_implemented", "operation kind is not implemented by the agent") writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "operation kind is not implemented by the agent"}) } func validateOperationHeaders(r *http.Request, request operationRequest, body []byte, required bool) error { requestHash := strings.TrimSpace(r.Header.Get("X-Cloud-Request-Hash")) requestID := strings.TrimSpace(r.Header.Get("X-Cloud-Request-ID")) idempotency := strings.TrimSpace(r.Header.Get("X-Cloud-Idempotency-Key")) if required && requestHash == "" { return fmt.Errorf("request hash header is required") } if requestHash != "" && requestHash != operation.HashPayload(body) { return fmt.Errorf("request hash header does not match body") } if requestID != "" && requestID != request.OperationID { return fmt.Errorf("request ID header does not match operation") } if idempotency != "" && idempotency != request.IdempotencyKey { return fmt.Errorf("idempotency header does not match operation") } return nil } func (s *Server) completeCapabilityProbe(w http.ResponseWriter, record operation.Record, ctx context.Context) { if err := s.caps.Probe(ctx); err != nil { updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, nil, "capability_probe_failed", "capability probe did not complete") if updateErr == nil { writeJSON(w, http.StatusServiceUnavailable, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability probe failed and operation state is uncertain"}) } return } snapshot, err := s.caps.Snapshot(ctx) if err != nil { updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, nil, "capability_snapshot_failed", "capability snapshot did not complete") if updateErr == nil { writeJSON(w, http.StatusServiceUnavailable, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "capability snapshot failed and operation state is uncertain"}) } return } result, _ := json.Marshal(snapshot) if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "capability_readback_confirmed"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, result, "operation_journal_update_failed", "capability probe was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } type domainReadPayload struct { UUID string `json:"uuid"` } type domainPowerPayload struct { UUID string `json:"uuid"` Action string `json:"action"` } // domainDeletePayload is intentionally only an exact UUID. Disk cleanup, // network teardown, and panel billing state are separate contracts and are // not accepted by the host Agent's narrow undefine operation. type domainDeletePayload struct { UUID string `json:"uuid"` } type seedPreparePayload struct { ArtifactID string `json:"artifact_id"` InstanceID string `json:"instance_id"` Hostname string `json:"hostname"` NetworkConfig string `json:"network_config,omitempty"` StartupScript string `json:"startup_script,omitempty"` } // These journal payloads are intentionally smaller than the request payloads. // The operation hash still covers the complete request, but durable plaintext // state keeps only identity and content digests for fields that can contain // credentials, host paths, or tenant-provided scripts. type seedPrepareJournalPayload struct { ArtifactID string `json:"artifact_id"` InstanceID string `json:"instance_id"` Hostname string `json:"hostname"` NetworkDigest string `json:"network_digest"` ScriptDigest string `json:"script_digest"` ScriptPresent bool `json:"script_present"` } type capabilityProbeJournalPayload struct{} type domainReadJournalPayload struct { UUID string `json:"uuid"` } type domainObserveResult struct { UUID string `json:"uuid"` XMLHash string `json:"xml_hash"` State hypervisor.DomainPowerState `json:"state"` Active bool `json:"active"` ObservedAt time.Time `json:"observed_at"` } type domainDeleteJournalPayload struct { UUID string `json:"uuid"` } type domainPowerJournalPayload struct { UUID string `json:"uuid"` Action string `json:"action"` } // domainCreateValidateResult combines the immutable seed/XML validation // boundary with the fresh host capability observation. The embedded result // is already redacted by domaincreatevalidate; capabilities are a typed, // read-only summary. No source XML, script, network document, or artifact // path is returned. type domainCreateValidateResult struct { domaincreatevalidate.Result Capabilities domainpreflight.CapabilitySummary `json:"capabilities"` } // domainXMLResult is the durable, non-sensitive form of a domain read-back. // The full XML can contain host paths, auth metadata, and storage details; the // operation journal only needs a content identity for reconciliation. Callers // that need the live XML must use a separately authorized read path rather than // recovering it from a plaintext restart journal. type domainXMLResult struct { UUID string `json:"uuid"` Revision uint64 `json:"revision,omitempty"` XMLHash string `json:"xml_hash"` } func safeJournalPayload(kind string, raw json.RawMessage) (json.RawMessage, error) { switch kind { case "capability_probe": var payload struct{} if err := decodeJSON(raw, &payload); err != nil { return nil, err } return json.Marshal(capabilityProbeJournalPayload{}) case "domain_read": uuid, err := parseDomainReadPayload(raw) if err != nil { return nil, err } return json.Marshal(domainReadJournalPayload{UUID: uuid}) case "domain_observe": uuid, err := parseDomainObservePayload(raw) if err != nil { return nil, err } return json.Marshal(domainReadJournalPayload{UUID: uuid}) case "domain_power": power, err := parseDomainPowerRequest(raw) if err != nil { return nil, err } return json.Marshal(domainPowerJournalPayload{UUID: power.UUID, Action: string(power.Action)}) case "domain_delete": uuid, err := parseDomainDeletePayload(raw) if err != nil { return nil, err } return json.Marshal(domainDeleteJournalPayload{UUID: uuid}) case "seed_prepare": payload, err := parseSeedPrepareRequest(raw) if err != nil { return nil, err } return json.Marshal(seedPrepareJournalPayload{ ArtifactID: payload.ArtifactID, InstanceID: payload.InstanceID, Hostname: payload.Hostname, NetworkDigest: operation.HashPayload([]byte(payload.NetworkConfig)), ScriptDigest: operation.HashPayload([]byte(payload.StartupScript)), ScriptPresent: payload.StartupScript != "", }) case "domain_preflight": payload, err := parseDomainPreflightRequest(raw) if err != nil { return nil, err } // The request type contains only scalar policy fields; unlike domain // define/seed payloads it needs no digest-only projection. return json.Marshal(payload) case "domain_create_plan": payload, err := parseDomainCreatePlanRequest(raw) if err != nil { return nil, err } journal, err := domaincreateplan.PrepareJournal(payload) if err != nil { return nil, err } return json.Marshal(journal) case "domain_create_validate": payload, err := parseDomainCreateValidatePayload(raw) if err != nil { return nil, err } // The request is deliberately the already-redacted create-plan journal // shape. Re-validate it before persisting so a caller cannot smuggle raw // seed data or an inconsistent plan identity into the operation record. return json.Marshal(payload) default: // Unknown operation kinds remain behind the capability gate. Store's // generic sanitizer is a second defense for any future typed payload, // but unknown requests must not be treated as an approved operation. return append(json.RawMessage(nil), raw...), nil } } func parseDomainPreflightRequest(raw json.RawMessage) (domainpreflight.Request, error) { var payload domainpreflight.Request if err := decodeJSON(raw, &payload); err != nil { return domainpreflight.Request{}, err } return domainpreflight.ValidateRequest(payload) } func parseDomainCreatePlanRequest(raw json.RawMessage) (domaincreateplan.Request, error) { var payload domaincreateplan.Request if err := decodeJSON(raw, &payload); err != nil { return domaincreateplan.Request{}, err } return domaincreateplan.ValidateRequest(payload) } // parseDomainCreateValidatePayload accepts only the redacted journal emitted // by domain_create_plan. A validate operation must refer to a published, // immutable seed artifact; accepting the original create request here would // make the endpoint look as if it could publish or repair the artifact. func parseDomainCreateValidatePayload(raw json.RawMessage) (domaincreateplan.JournalPayload, error) { var payload domaincreateplan.JournalPayload if err := decodeJSON(raw, &payload); err != nil { return domaincreateplan.JournalPayload{}, err } if err := domaincreateplan.ValidateJournal(payload); err != nil { return domaincreateplan.JournalPayload{}, err } return payload, nil } func parseSeedPrepareRequest(raw json.RawMessage) (seedPreparePayload, error) { var payload seedPreparePayload if err := decodeJSON(raw, &payload); err != nil { return seedPreparePayload{}, err } artifactID, err := hypervisor.CanonicalUUID(payload.ArtifactID) if err != nil { return seedPreparePayload{}, err } payload.ArtifactID = artifactID if err := startup.Validate(startup.SeedRequest{ Operation: startup.OperationCreate, InstanceID: payload.InstanceID, Hostname: payload.Hostname, NetworkConfig: payload.NetworkConfig, StartupScript: payload.StartupScript, }); err != nil { return seedPreparePayload{}, err } return payload, nil } func parseDomainPowerRequest(raw json.RawMessage) (hypervisor.PowerRequest, error) { var payload domainPowerPayload if err := decodeJSON(raw, &payload); err != nil { return hypervisor.PowerRequest{}, err } canonical, err := hypervisor.CanonicalUUID(payload.UUID) if err != nil { return hypervisor.PowerRequest{}, err } action, err := hypervisor.ParsePowerAction(payload.Action) if err != nil { return hypervisor.PowerRequest{}, err } return hypervisor.PowerRequest{UUID: canonical, Action: action}, nil } func parseDomainDeletePayload(raw json.RawMessage) (string, error) { var payload domainDeletePayload if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" { return "", fmt.Errorf("invalid domain delete payload") } return hypervisor.CanonicalUUID(payload.UUID) } func parseDomainReadPayload(raw json.RawMessage) (string, error) { var payload domainReadPayload if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" { return "", fmt.Errorf("invalid domain read payload") } canonical, err := hypervisor.CanonicalUUID(payload.UUID) if err != nil { return "", err } return canonical, nil } func parseDomainObservePayload(raw json.RawMessage) (string, error) { return parseDomainReadPayload(raw) } func parseSeedJournalPayload(raw json.RawMessage) (seedPreparePayload, error) { var journal seedPrepareJournalPayload if err := decodeJSON(raw, &journal); err == nil && journal.ArtifactID != "" && journal.NetworkDigest != "" && journal.ScriptDigest != "" { artifactID, canonicalErr := hypervisor.CanonicalUUID(journal.ArtifactID) if canonicalErr != nil { return seedPreparePayload{}, canonicalErr } if !validSHA256(journal.NetworkDigest) || !validSHA256(journal.ScriptDigest) { return seedPreparePayload{}, fmt.Errorf("seed journal content hash is invalid") } if err := startup.Validate(startup.SeedRequest{Operation: startup.OperationCreate, InstanceID: journal.InstanceID, Hostname: journal.Hostname}); err != nil { return seedPreparePayload{}, err } return seedPreparePayload{ ArtifactID: artifactID, InstanceID: journal.InstanceID, Hostname: journal.Hostname, StartupScript: "[redacted]", }, nil } return parseSeedPrepareRequest(raw) } func validSHA256(value string) bool { if len(value) != sha256.Size*2 { return false } _, err := hex.DecodeString(value) return err == nil && strings.ToLower(value) == value } func seedManifestFor(files startup.SeedFiles) startup.SeedManifest { payloads := map[string][]byte{ startup.UserDataFilename: files.UserData, startup.MetaDataFilename: files.MetaData, startup.NetworkConfigFilename: files.NetworkConfig, } manifest := startup.SeedManifest{ Version: startup.SeedManifestVersion, Files: make(map[string]startup.SeedFileDigest, len(payloads)), } for name, data := range payloads { digest := sha256.Sum256(data) manifest.Files[name] = startup.SeedFileDigest{ Bytes: len(data), SHA256: hex.EncodeToString(digest[:]), } } return manifest } type seedReconcileResult struct { ArtifactID string `json:"artifact_id"` Manifest startup.SeedManifest `json:"manifest"` RequestContentVerified bool `json:"request_content_verified"` } func marshalSeedReconcileResult(artifactID string, manifest startup.SeedManifest, requestContentVerified bool) ([]byte, error) { return json.Marshal(seedReconcileResult{ ArtifactID: artifactID, Manifest: manifest, RequestContentVerified: requestContentVerified, }) } func persistedSeedManifest(raw json.RawMessage, artifactID string) (startup.SeedManifest, bool) { if len(bytes.TrimSpace(raw)) == 0 { return startup.SeedManifest{}, false } var result seedReconcileResult if err := decodeJSON(raw, &result); err != nil || result.ArtifactID != artifactID || !result.RequestContentVerified { return startup.SeedManifest{}, false } if result.Manifest.Version != startup.SeedManifestVersion || len(result.Manifest.Files) != 3 { return startup.SeedManifest{}, false } return result.Manifest, true } func seedManifestsEqual(left, right startup.SeedManifest) bool { if left.Version != right.Version || len(left.Files) != len(right.Files) { return false } for name, digest := range right.Files { if observed, ok := left.Files[name]; !ok || observed != digest { return false } } return true } func targetMatchesUUID(target, uuid string) bool { target = strings.TrimSpace(target) if target == "" { return true } canonical, err := hypervisor.CanonicalUUID(target) return err == nil && canonical == uuid } func marshalDomainXMLResult(definition hypervisor.Definition) ([]byte, error) { return json.Marshal(domainXMLResult{ UUID: definition.UUID, Revision: definition.Revision, XMLHash: operation.HashPayload(definition.XML), }) } func (s *Server) completeDomainRead(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { var payload domainReadPayload if err := decodeJSON(request.Payload, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" { _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_request", "domain read payload is invalid") writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain read payload"}) return } canonical, err := hypervisor.CanonicalUUID(payload.UUID) if err != nil || !targetMatchesUUID(request.Target, canonical) { _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_request", "domain read target does not match payload") writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain read payload"}) return } payload.UUID = canonical definition, err := hypervisor.ReadbackDomain(ctx, s.backend, payload.UUID) if err != nil { status := operation.StatusUncertain errorCode := "domain_read_failed" if hypervisor.IsDefinitive(err) { status = operation.StatusFailed errorCode = "domain_read_rejected" } updated, updateErr := s.store.Update(record.OperationID, status, nil, errorCode, "domain read-back did not complete") if updateErr == nil { if status == operation.StatusFailed { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, updated) } } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain read-back failed and operation state is uncertain"}) } return } if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "readback_confirmed"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "operation_journal_update_failed", "domain read-back was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } result, _ := marshalDomainXMLResult(definition) updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } // observeDomain performs a read-only XML and power-state observation for one // explicit UUID. The XML itself never crosses the API or durable journal; // callers receive only its content hash plus the independently read-back // libvirt state. Keeping this helper shared by the request and reconcile paths // prevents the latter from accidentally replaying a lifecycle operation. func observeDomain(ctx context.Context, backend hypervisor.Backend, uuid string) (domainObserveResult, error) { definition, err := hypervisor.ReadbackDomain(ctx, backend, uuid) if err != nil { return domainObserveResult{}, err } state, err := backend.ReadDomainPowerState(ctx, definition.UUID) if err != nil { return domainObserveResult{}, err } return domainObserveResult{ UUID: definition.UUID, XMLHash: operation.HashPayload(definition.XML), State: state, Active: state.IsActive(), ObservedAt: time.Now().UTC(), }, nil } func (s *Server) completeDomainObserve(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { uuid, err := parseDomainObservePayload(request.Payload) if err != nil || !targetMatchesUUID(request.Target, uuid) { _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_observe_request", "domain observe payload is invalid") writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain observe payload"}) return } result, err := observeDomain(ctx, s.backend, uuid) if err != nil { status := operation.StatusUncertain errorCode := "domain_observe_uncertain" if hypervisor.IsDefinitive(err) { status = operation.StatusFailed errorCode = "domain_observe_rejected" } updated, updateErr := s.store.Update(record.OperationID, status, nil, errorCode, "domain observation did not complete") if updateErr == nil { if status == operation.StatusFailed { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, updated) } } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain observation failed and operation state is uncertain"}) } return } resultJSON, marshalErr := json.Marshal(result) if marshalErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_observe_result_failed", "domain observation result could not be encoded") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain observation result could not be encoded"}) return } if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_observe_confirmed"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain observation was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } func (s *Server) completeDomainPower(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { powerRequest, err := parseDomainPowerRequest(request.Payload) if err != nil { _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_power_request", "domain power payload is invalid") writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain power payload"}) return } result, err := s.backend.PowerDomain(ctx, powerRequest.UUID, powerRequest.Action) if err != nil { status := operation.StatusUncertain errorCode := "domain_power_uncertain" if hypervisor.IsDefinitive(err) { status = operation.StatusFailed errorCode = "domain_power_rejected" } var resultJSON json.RawMessage if result.State != "" { resultJSON, _ = json.Marshal(result) } updated, updateErr := s.store.Update(record.OperationID, status, resultJSON, errorCode, "domain power action did not reach a confirmed desired state") if updateErr == nil { if status == operation.StatusFailed { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, updated) } } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power action failed and operation state is uncertain"}) } return } if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "power_readback_confirmed"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "operation_journal_update_failed", "domain power was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } resultJSON, _ := json.Marshal(result) updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } // completeDomainDelete performs only the typed, inactive-domain undefine // boundary. The hypervisor implementation must prove absence with a targeted // read-back before this method can persist success. Disk images, seed // artifacts, firewall rules, and host networking are deliberately untouched; // the panel's full VM cleanup contract is not represented here. func (s *Server) completeDomainDelete(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { uuid, err := parseDomainDeletePayload(request.Payload) if err != nil || !targetMatchesUUID(request.Target, uuid) { _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_delete_request", "domain delete payload is invalid") writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"}) return } result, err := s.backend.DeleteDomain(ctx, uuid) var resultJSON json.RawMessage if result.UUID != "" { resultJSON, _ = json.Marshal(result) } if err != nil { status := operation.StatusUncertain httpStatus := http.StatusServiceUnavailable errorCode := "domain_delete_uncertain" message := "domain undefine or absence read-back did not complete" if errors.Is(err, hypervisor.ErrInvalidUUID) || errors.Is(err, hypervisor.ErrDomainActive) { status = operation.StatusFailed httpStatus = http.StatusUnprocessableEntity if errors.Is(err, hypervisor.ErrDomainActive) { errorCode = "domain_delete_active" message = "domain must be inactive before undefine" } else { errorCode = "invalid_domain_delete_request" message = "domain delete UUID is invalid" } } updated, updateErr := s.store.Update(record.OperationID, status, resultJSON, errorCode, message) if updateErr == nil { writeJSON(w, httpStatus, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete failed and operation state is uncertain"}) } return } if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_delete_readback_confirmed"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain absence was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"}) return } updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } func (s *Server) completeSeedPrepare(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { payload, err := parseSeedPrepareRequest(request.Payload) if err != nil { _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_seed_request", "seed payload is invalid") writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid seed prepare payload"}) return } if err := ctx.Err(); err != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "seed_prepare_canceled", "seed preparation was canceled before publication") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed preparation was canceled"}) return } files, err := startup.BuildCreateSeed(payload.InstanceID, payload.Hostname, payload.NetworkConfig, payload.StartupScript) if err != nil { _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_seed_request", "seed payload is invalid") writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid seed prepare payload"}) return } target := filepath.Join(s.cfg.ArtifactRoot, payload.ArtifactID) artifact, err := startup.WriteSeedDirectory(target, files) if err != nil { status := operation.StatusUncertain errorCode := "seed_prepare_uncertain" if errors.Is(err, startup.ErrInvalidSeed) || errors.Is(err, startup.ErrInvalidScript) || errors.Is(err, startup.ErrScriptCreateOnly) || errors.Is(err, startup.ErrArtifactPath) { status = operation.StatusFailed errorCode = "seed_prepare_rejected" } updated, updateErr := s.store.Update(record.OperationID, status, nil, errorCode, "seed artifact publication was not confirmed") if updateErr == nil { if status == operation.StatusFailed { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, updated) } } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed publication failed and operation state is uncertain"}) } return } result, _ := marshalSeedReconcileResult(payload.ArtifactID, artifact.Manifest, true) if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "seed_published"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, result, "operation_journal_update_failed", "seed artifact was published but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed operation result could not be committed"}) return } updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "seed operation result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } // completeDomainPreflight performs a typed, read-only host capability check. // It intentionally never calls DefineDomain: a successful preflight is only a // capability/normalization decision, not evidence that storage or a domain was // created. func (s *Server) completeDomainPreflight(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { payload, err := parseDomainPreflightRequest(request.Payload) if err != nil || !targetMatchesUUID(request.Target, payload.UUID) { code := domainpreflight.Code(err) if err == nil { code = "domain_preflight_target_mismatch" } updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, "domain preflight request was rejected") if updateErr == nil { writeJSON(w, http.StatusBadRequest, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight request could not be journaled"}) } return } result, err := domainpreflight.Run(ctx, s.backend, payload) if err != nil { status := operation.StatusUncertain httpStatus := http.StatusServiceUnavailable code := domainpreflight.Code(err) message := "domain preflight capability observation did not complete" if domainpreflight.IsDefinitive(err) { status = operation.StatusFailed httpStatus = http.StatusUnprocessableEntity message = "domain preflight request was rejected" } updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message) if updateErr == nil { writeJSON(w, httpStatus, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight failed and operation state is uncertain"}) } return } resultJSON, err := json.Marshal(result) if err != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_preflight_result_failed", "domain preflight result could not be encoded") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight result could not be committed"}) return } if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_preflight_confirmed"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain preflight was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight result could not be committed"}) return } updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } // completeDomainCreatePlan performs a pure creation-plan observation. It // validates the typed domain/seed/image intent, observes current host // capabilities, and persists only redacted plan metadata. No seed directory, // storage volume, domain definition, power state, or network object is // touched here. func (s *Server) completeDomainCreatePlan(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { payload, err := parseDomainCreatePlanRequest(request.Payload) if err != nil || !targetMatchesUUID(request.Target, payload.Domain.UUID) { code := domaincreateplan.Code(err) if err == nil { code = "domain_create_plan_target_mismatch" } updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, "domain create plan request was rejected") if updateErr == nil { writeJSON(w, http.StatusBadRequest, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan request could not be journaled"}) } return } result, err := domaincreateplan.Build(ctx, s.backend, payload) if err != nil { status := operation.StatusUncertain httpStatus := http.StatusServiceUnavailable code := domaincreateplan.Code(err) message := "domain create plan host observation did not complete" if domaincreateplan.IsDefinitive(err) { status = operation.StatusFailed httpStatus = http.StatusUnprocessableEntity message = "domain create plan request was rejected" } updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message) if updateErr == nil { writeJSON(w, httpStatus, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan failed and operation state is uncertain"}) } return } resultJSON, err := json.Marshal(result) if err != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_create_plan_result_failed", "domain create plan result could not be encoded") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan result could not be committed"}) return } if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_create_plan_confirmed"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain create plan was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan result could not be committed"}) return } updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } // completeDomainCreateValidate verifies an already-published immutable seed, // then performs one read-only host capability observation. It deliberately // stops before storage allocation, seed publication, domain definition, // lifecycle, or network operations. The seed check comes first so a missing // or tampered artifact is a definitive rejection even when libvirt is also // unavailable. func (s *Server) completeDomainCreateValidate(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { payload, err := parseDomainCreateValidatePayload(request.Payload) if err != nil || !targetMatchesUUID(request.Target, payload.Domain.UUID) { code := domainCreateValidateCode(err) if err == nil { code = "domain_create_validate_target_mismatch" } updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, "domain create validation request was rejected") if updateErr == nil { writeJSON(w, http.StatusBadRequest, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation request could not be journaled"}) } return } validated, err := domaincreatevalidate.Validate(ctx, s.cfg.ArtifactRoot, payload) if err != nil { status := operation.StatusUncertain httpStatus := http.StatusServiceUnavailable code := domainCreateValidateCode(err) message := "published seed validation did not complete" if domaincreatevalidate.IsDefinitive(err) { status = operation.StatusFailed httpStatus = http.StatusUnprocessableEntity message = "published seed or domain validation was rejected" } updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message) if updateErr == nil { writeJSON(w, httpStatus, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation failed and operation state is uncertain"}) } return } // Keep the nil-backend case typed and uncertain rather than falling through // to the generic mutating-backend error. The artifact has been verified, but // the host capability boundary is not yet proven. if s.backend == nil { s.reconcileUncertain(w, record, nil, "domain_create_validate_host_unavailable", "host capability observation is unavailable", http.StatusServiceUnavailable) return } observation, err := domainpreflight.Run(ctx, s.backend, payload.Domain) if err != nil { status := operation.StatusUncertain httpStatus := http.StatusServiceUnavailable code := domainCreateValidateCode(err) message := "host capability observation did not complete after seed validation" if domainpreflight.IsDefinitive(err) { status = operation.StatusFailed httpStatus = http.StatusUnprocessableEntity message = "host firmware policy rejected the validated domain" } updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message) if updateErr == nil { writeJSON(w, httpStatus, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation failed and operation state is uncertain"}) } return } // Reuse the plan identity check so a fresh capability observation cannot be // paired with a tampered journal or a different deterministic XML policy. plan, err := domaincreateplan.BuildFromJournal(observation, payload) if err != nil { status := operation.StatusUncertain httpStatus := http.StatusServiceUnavailable code := domainCreateValidateCode(err) message := "validated domain identity no longer matches host observation" if domaincreateplan.IsDefinitive(err) { status = operation.StatusFailed httpStatus = http.StatusConflict message = "validated domain identity was rejected" } updated, updateErr := s.store.Update(record.OperationID, status, nil, code, message) if updateErr == nil { writeJSON(w, httpStatus, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation failed and operation state is uncertain"}) } return } resultJSON, err := json.Marshal(domainCreateValidateResult{ Result: validated, Capabilities: plan.Capabilities, }) if err != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, nil, "domain_create_validate_result_failed", "domain create validation result could not be encoded") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation result could not be committed"}) return } if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_create_validate_confirmed"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain create validation was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation result could not be committed"}) return } updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } // domainCreateValidateCode keeps errors from the plan/preflight packages in // this operation's namespace. Stable codes are persisted; host error strings // never cross the API boundary. func domainCreateValidateCode(err error) string { if err == nil { return "domain_create_validate_failed" } code := domaincreatevalidate.Code(err) if strings.HasPrefix(code, "domain_create_validate_") && code != "domain_create_validate_failed" { return code } preflightCode := domainpreflight.Code(err) if strings.HasPrefix(preflightCode, "domain_preflight_") && preflightCode != "domain_preflight_failed" { return "domain_create_validate_" + strings.TrimPrefix(preflightCode, "domain_preflight_") } planCode := domaincreateplan.Code(err) if strings.HasPrefix(planCode, "domain_create_plan_") && planCode != "domain_create_plan_failed" { return "domain_create_validate_" + strings.TrimPrefix(planCode, "domain_create_plan_") } return "domain_create_validate_failed" } func (s *Server) getOperation(w http.ResponseWriter, r *http.Request) { record, err := s.storeRecord(r.PathValue("id")) if err != nil { writeJSON(w, http.StatusNotFound, errorResponse{Error: "operation not found"}) return } writeJSON(w, http.StatusOK, record) } func (s *Server) reconcileOperation(w http.ResponseWriter, r *http.Request) { record, err := s.storeRecord(r.PathValue("id")) if err != nil { writeJSON(w, http.StatusNotFound, errorResponse{Error: "operation not found"}) return } // Every reconcile handler below is a read-back operation. It may update // the local journal, but it must never replay the original host mutation. // Therefore it uses the control-plane gate only and deliberately does not // require a fresh Probe()/hostReady bit. A failed/stale probe is exactly // the state in which an operator needs to read back an already ambiguous // operation. Lifecycle requests created through POST /operations remain // behind writeAllowed and still fail closed while the host is degraded. if !s.readbackAllowed(w) { return } if record.Status != operation.StatusUncertain { writeJSON(w, http.StatusOK, record) return } switch record.Kind { case "capability_probe": s.reconcileCapabilityProbe(w, record, r) case "domain_read": s.reconcileDomainRead(w, record, r) case "domain_observe": s.reconcileDomainObserve(w, record, r) case "domain_power": s.reconcileDomainPower(w, record, r) case "domain_delete": s.reconcileDomainDelete(w, record, r) case "seed_prepare": s.reconcileSeedPrepare(w, record, r) case "domain_preflight": s.reconcileDomainPreflight(w, record, r) case "domain_create_plan": s.reconcileDomainCreatePlan(w, record, r) case "domain_create_validate": s.reconcileDomainCreateValidate(w, record, r) default: // No destructive replay is performed. An operation kind that has no // typed read-back contract remains explicitly unresolved. writeJSON(w, http.StatusConflict, errorResponse{Error: "operation requires backend-specific reconciliation"}) } } // reconcileDomainPreflight repeats only the typed capability observation. It // never defines a domain or allocates storage/network resources, so a retry is // safe even when the original HTTP response was lost. func (s *Server) reconcileDomainPreflight(w http.ResponseWriter, record operation.Record, r *http.Request) { payload, err := parseDomainPreflightRequest(record.Payload) if err != nil || !targetMatchesUUID(record.Target, payload.UUID) { s.reconcileInvalidJournal(w, record, domainpreflight.Code(err), "domain preflight journal payload is invalid") return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() result, err := domainpreflight.Run(ctx, s.backend, payload) if err != nil { if domainpreflight.IsDefinitive(err) { updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainpreflight.Code(err), "domain preflight request was rejected") if updateErr == nil { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain preflight reconciliation failed"}) } return } s.reconcileUncertain(w, record, record.Result, domainpreflight.Code(err), "domain preflight capability observation did not complete", http.StatusServiceUnavailable) return } resultJSON, marshalErr := json.Marshal(result) if marshalErr != nil { s.reconcileUncertain(w, record, record.Result, "domain_preflight_result_failed", "domain preflight result could not be encoded", http.StatusServiceUnavailable) return } s.reconcileSuccess(w, record, "domain_preflight_confirmed", resultJSON) } // reconcileDomainCreatePlan repeats only the read-only capability observation // and rebuilds the plan identity from the redacted journal. It never renders // or republishes a seed and never calls a lifecycle/define operation. func (s *Server) reconcileDomainCreatePlan(w http.ResponseWriter, record operation.Record, r *http.Request) { var payload domaincreateplan.JournalPayload if err := decodeJSON(record.Payload, &payload); err != nil || !targetMatchesUUID(record.Target, payload.Domain.UUID) { s.reconcileInvalidJournal(w, record, "domain_create_plan_journal_invalid", "domain create plan journal payload is invalid") return } if err := domaincreateplan.ValidateJournal(payload); err != nil { s.reconcileInvalidJournal(w, record, domaincreateplan.Code(err), "domain create plan journal payload is invalid") return } if s.backend == nil { s.reconcileUncertain(w, record, record.Result, "domain_create_plan_host_unavailable", "host backend is not configured for plan reconciliation", http.StatusServiceUnavailable) return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() observation, err := domainpreflight.Run(ctx, s.backend, payload.Domain) if err != nil { if domainpreflight.IsDefinitive(err) { updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domaincreateplan.Code(err), "domain create plan host policy was rejected") if updateErr == nil { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan reconciliation failed"}) } return } s.reconcileUncertain(w, record, record.Result, domaincreateplan.Code(err), "domain create plan host observation did not complete", http.StatusServiceUnavailable) return } result, err := domaincreateplan.BuildFromJournal(observation, payload) if err != nil { if domaincreateplan.IsDefinitive(err) { updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domaincreateplan.Code(err), "domain create plan journal no longer proves the same intent") if updateErr == nil { writeJSON(w, http.StatusConflict, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create plan reconciliation failed"}) } return } s.reconcileUncertain(w, record, record.Result, domaincreateplan.Code(err), "domain create plan reconciliation remains uncertain", http.StatusServiceUnavailable) return } resultJSON, marshalErr := json.Marshal(result) if marshalErr != nil { s.reconcileUncertain(w, record, record.Result, "domain_create_plan_result_failed", "domain create plan result could not be encoded", http.StatusServiceUnavailable) return } s.reconcileSuccess(w, record, "domain_create_plan_confirmed", resultJSON) } // reconcileDomainCreateValidate repeats the immutable seed read-back before // obtaining a fresh capability observation. It never republishes a seed, // defines a domain, changes power, or configures networking. func (s *Server) reconcileDomainCreateValidate(w http.ResponseWriter, record operation.Record, r *http.Request) { var payload domaincreateplan.JournalPayload if err := decodeJSON(record.Payload, &payload); err != nil || !targetMatchesUUID(record.Target, payload.Domain.UUID) { s.reconcileInvalidJournal(w, record, "domain_create_validate_journal_invalid", "domain create validation journal payload is invalid") return } if err := domaincreateplan.ValidateJournal(payload); err != nil { s.reconcileInvalidJournal(w, record, domainCreateValidateCode(err), "domain create validation journal payload is invalid") return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() validated, err := domaincreatevalidate.Validate(ctx, s.cfg.ArtifactRoot, payload) if err != nil { if domaincreatevalidate.IsDefinitive(err) { updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainCreateValidateCode(err), "published seed or domain validation was rejected") if updateErr == nil { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation reconciliation failed"}) } return } s.reconcileUncertain(w, record, record.Result, domainCreateValidateCode(err), "published seed validation did not complete", http.StatusServiceUnavailable) return } if s.backend == nil { s.reconcileUncertain(w, record, record.Result, "domain_create_validate_host_unavailable", "host capability observation is unavailable", http.StatusServiceUnavailable) return } observation, err := domainpreflight.Run(ctx, s.backend, payload.Domain) if err != nil { if domainpreflight.IsDefinitive(err) { updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainCreateValidateCode(err), "host firmware policy rejected the validated domain") if updateErr == nil { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation reconciliation failed"}) } return } s.reconcileUncertain(w, record, record.Result, domainCreateValidateCode(err), "host capability observation did not complete after seed validation", http.StatusServiceUnavailable) return } plan, err := domaincreateplan.BuildFromJournal(observation, payload) if err != nil { if domaincreateplan.IsDefinitive(err) { updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, domainCreateValidateCode(err), "validated domain identity no longer matches host observation") if updateErr == nil { writeJSON(w, http.StatusConflict, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain create validation reconciliation failed"}) } return } s.reconcileUncertain(w, record, record.Result, domainCreateValidateCode(err), "domain create validation remains uncertain", http.StatusServiceUnavailable) return } resultJSON, marshalErr := json.Marshal(domainCreateValidateResult{ Result: validated, Capabilities: plan.Capabilities, }) if marshalErr != nil { s.reconcileUncertain(w, record, record.Result, "domain_create_validate_result_failed", "domain create validation result could not be encoded", http.StatusServiceUnavailable) return } s.reconcileSuccess(w, record, "domain_create_validate_confirmed", resultJSON) } // reconcileCapabilityProbe performs only the capability snapshot read-back. // Probe is intentionally not called here: even though the current providers // implement Probe as read-only, reconciliation must depend on the narrowest // contract and never replay the original operation. func (s *Server) reconcileCapabilityProbe(w http.ResponseWriter, record operation.Record, r *http.Request) { if s.caps == nil { s.reconcileUncertain(w, record, record.Result, "capability_reconcile_uncertain", "capability snapshot provider is unavailable", http.StatusServiceUnavailable) return } if payload := bytes.TrimSpace(record.Payload); len(payload) != 0 { var empty struct{} if err := decodeJSON(payload, &empty); err != nil { s.reconcileInvalidJournal(w, record, "capability_probe_payload_invalid", "capability probe journal payload is invalid") return } } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() snapshot, err := s.caps.Snapshot(ctx) if err != nil { s.reconcileUncertain(w, record, record.Result, "capability_reconcile_uncertain", "capability snapshot read-back did not complete", http.StatusServiceUnavailable) return } if target := strings.TrimSpace(record.Target); target != "" && snapshot.AgentID != target { result, _ := json.Marshal(snapshot) s.reconcileUncertain(w, record, result, "capability_target_mismatch", "capability snapshot belongs to a different agent", http.StatusConflict) return } result, _ := json.Marshal(snapshot) s.reconcileSuccess(w, record, "capability_readback_confirmed", result) } // reconcileDomainRead performs a read-only dumpxml and never invokes a // define/start/lifecycle method. A missing or policy-mismatched definition is // left uncertain: the agent must not infer a destructive conclusion from one // failed observation. func (s *Server) reconcileDomainRead(w http.ResponseWriter, record operation.Record, r *http.Request) { if s.backend == nil { s.reconcileUncertain(w, record, record.Result, "domain_read_reconcile_uncertain", "host backend is not configured for domain read-back", http.StatusServiceUnavailable) return } payload, err := parseDomainReadPayload(record.Payload) if err != nil || !targetMatchesUUID(record.Target, payload) { s.reconcileInvalidJournal(w, record, "invalid_domain_request", "domain read journal payload is invalid") return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() definition, err := hypervisor.ReadbackDomain(ctx, s.backend, payload) if err != nil { s.reconcileUncertain(w, record, record.Result, "domain_read_reconcile_uncertain", "domain read-back did not complete", http.StatusServiceUnavailable) return } result, _ := marshalDomainXMLResult(definition) s.reconcileSuccess(w, record, "domain_readback_confirmed", result) } // reconcileDomainObserve repeats only the typed XML/state read-back. It never // invokes a lifecycle or define command, so an uncertain response cannot // cause a host mutation when the panel retries reconciliation. func (s *Server) reconcileDomainObserve(w http.ResponseWriter, record operation.Record, r *http.Request) { if s.backend == nil { s.reconcileUncertain(w, record, record.Result, "domain_observe_reconcile_uncertain", "host backend is not configured for domain observation", http.StatusServiceUnavailable) return } uuid, err := parseDomainObservePayload(record.Payload) if err != nil || !targetMatchesUUID(record.Target, uuid) { s.reconcileInvalidJournal(w, record, "invalid_domain_observe_request", "domain observe journal payload is invalid") return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() result, err := observeDomain(ctx, s.backend, uuid) if err != nil { code := "domain_observe_reconcile_uncertain" if hypervisor.IsDefinitive(err) { updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, "domain_observe_rejected", "domain observation was rejected during reconciliation") if updateErr == nil { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain observation reconciliation failed"}) } return } s.reconcileUncertain(w, record, record.Result, code, "domain observation read-back did not complete", http.StatusServiceUnavailable) return } resultJSON, marshalErr := json.Marshal(result) if marshalErr != nil { s.reconcileUncertain(w, record, record.Result, "domain_observe_result_failed", "domain observation result could not be encoded", http.StatusServiceUnavailable) return } s.reconcileSuccess(w, record, "domain_observe_confirmed", resultJSON) } // reconcileSeedPrepare verifies the already-published immutable seed and // compares its manifest with a deterministic rendering of the journaled // request. It never calls WriteSeedDirectory or otherwise republishes files. func (s *Server) reconcileSeedPrepare(w http.ResponseWriter, record operation.Record, r *http.Request) { payload, err := parseSeedJournalPayload(record.Payload) if err != nil || !targetMatchesUUID(record.Target, payload.ArtifactID) { s.reconcileInvalidJournal(w, record, "invalid_seed_request", "seed prepare journal payload is invalid") return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() if err := ctx.Err(); err != nil { s.reconcileUncertain(w, record, nil, "seed_reconcile_uncertain", "seed artifact read-back was canceled", http.StatusServiceUnavailable) return } observed, err := startup.VerifySeedDirectory(filepath.Join(s.cfg.ArtifactRoot, payload.ArtifactID)) if err != nil { s.reconcileUncertain(w, record, record.Result, "seed_reconcile_uncertain", "seed artifact read-back did not complete", http.StatusServiceUnavailable) return } requestContentVerified := false if expected, ok := persistedSeedManifest(record.Result, payload.ArtifactID); ok { if !seedManifestsEqual(observed, expected) { result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, false) s.reconcileUncertain(w, record, result, "seed_reconcile_mismatch", "seed artifact does not match the persisted request manifest", http.StatusConflict) return } requestContentVerified = true } else if payload.StartupScript != "[redacted]" { expectedFiles, renderErr := startup.BuildCreateSeed(payload.InstanceID, payload.Hostname, payload.NetworkConfig, payload.StartupScript) if renderErr != nil { s.reconcileUncertain(w, record, nil, "seed_reconcile_mismatch", "seed request could not be rendered for comparison", http.StatusConflict) return } expected := seedManifestFor(expectedFiles) if !seedManifestsEqual(observed, expected) { result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, false) s.reconcileUncertain(w, record, result, "seed_reconcile_mismatch", "seed artifact does not match the requested content", http.StatusConflict) return } requestContentVerified = true } else if payload.StartupScript == "[redacted]" { // A safe journal intentionally omits the request's network document and // startup script. Once the response/result manifest is also missing, // an artifact read-back proves only that *some* immutable seed exists; // it cannot prove that it is the artifact requested by this operation. // Never turn that integrity-only observation into a succeeded operation. // Keep the observed manifest (with an explicit false flag) so an // operator can inspect it while the operation remains reconcilable. result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, false) code := "seed_reconcile_content_unproven" message := "seed artifact integrity was observed, but request content equivalence could not be proven" if record.ErrorCode == "seed_reconcile_mismatch" { // Preserve the stronger diagnostic for a previously observed // mismatch, while retaining the non-successful uncertain state. code = "seed_reconcile_mismatch" message = "seed request content remains unavailable for comparison" } s.reconcileUncertain(w, record, result, code, message, http.StatusConflict) return } // At this point the journal carried the original request fields (for // example, a test/recovery record created before the digest-only projection // was enabled), so the deterministic render above proved equivalence. result, _ := marshalSeedReconcileResult(payload.ArtifactID, observed, requestContentVerified) s.reconcileSuccess(w, record, "seed_readback_confirmed", result) } func (s *Server) reconcileSuccess(w http.ResponseWriter, record operation.Record, boundary string, result []byte) { if _, err := s.store.SetCommitBoundary(record.OperationID, boundary); err != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, result, "operation_journal_update_failed", "read-back was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, result, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } func (s *Server) reconcileUncertain(w http.ResponseWriter, record operation.Record, result []byte, code, message string, status int) { updated, err := s.store.Update(record.OperationID, operation.StatusUncertain, result, code, message) if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation reconciliation remains uncertain"}) return } writeJSON(w, status, updated) } func (s *Server) reconcileInvalidJournal(w http.ResponseWriter, record operation.Record, code, message string) { updated, err := s.store.Update(record.OperationID, operation.StatusFailed, nil, code, message) if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation journal payload is invalid"}) return } writeJSON(w, http.StatusUnprocessableEntity, updated) } // reconcileDomainPower performs only a typed, read-only state probe. It never // calls PowerDomain: an ambiguous lifecycle response must not be replayed just // because a caller asked to reconcile it. func (s *Server) reconcileDomainPower(w http.ResponseWriter, record operation.Record, r *http.Request) { if s.backend == nil { updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, record.Result, "domain_power_reconcile_uncertain", "host backend is not configured for state reconciliation") if updateErr == nil { writeJSON(w, http.StatusServiceUnavailable, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power reconciliation is unavailable"}) } return } powerRequest, err := parseDomainPowerRequest(record.Payload) if err != nil || !targetMatchesUUID(record.Target, powerRequest.UUID) { updated, updateErr := s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_power_request", "domain power journal payload is invalid") if updateErr == nil { writeJSON(w, http.StatusUnprocessableEntity, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power journal payload is invalid"}) } return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() state, err := s.backend.ReadDomainPowerState(ctx, powerRequest.UUID) if err != nil { // A failed read cannot prove that the requested outcome did not happen; // keep the operation uncertain and require a later probe. updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, record.Result, "domain_power_reconcile_uncertain", "domain power state read-back did not complete") if updateErr == nil { writeJSON(w, http.StatusServiceUnavailable, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power state reconciliation is uncertain"}) } return } result := hypervisor.PowerResult{ UUID: powerRequest.UUID, Action: powerRequest.Action, State: state, Active: state.IsActive(), ObservedAt: time.Now().UTC(), } resultJSON, _ := json.Marshal(result) if !hypervisor.PowerActionReconcileSatisfied(powerRequest.Action, state) { updated, updateErr := s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "domain_power_not_reconciled", "observed domain power state does not satisfy the requested action") if updateErr == nil { writeJSON(w, http.StatusConflict, updated) } else { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain power state remains uncertain"}) } return } if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "power_readback_confirmed"); boundaryErr != nil { _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain power was confirmed but operation state could not be committed") writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "") if err != nil { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "operation result could not be committed"}) return } writeJSON(w, http.StatusOK, updated) } // reconcileDomainDelete is strictly read-only. It observes whether the exact // definition UUID is absent and never retries undefine after an ambiguous // response. A present definition, an active guest, or a failed observation // keeps the journal uncertain for a later operator-approved read-back. func (s *Server) reconcileDomainDelete(w http.ResponseWriter, record operation.Record, r *http.Request) { if s.backend == nil { s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "host backend is not configured for domain deletion read-back", http.StatusServiceUnavailable) return } uuid, err := parseDomainDeletePayload(record.Payload) if err != nil || !targetMatchesUUID(record.Target, uuid) { s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal payload is invalid") return } ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) defer cancel() observation, err := s.backend.InspectDomainForDeletion(ctx, uuid) if err != nil { if errors.Is(err, hypervisor.ErrInvalidUUID) { s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal UUID is invalid") return } s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "domain deletion absence read-back did not complete", http.StatusServiceUnavailable) return } if observation.Present { result, _ := json.Marshal(observation) s.reconcileUncertain(w, record, result, "domain_delete_not_reconciled", "domain definition is still present; no undefine was replayed", http.StatusConflict) return } result, marshalErr := json.Marshal(hypervisor.DomainDeleteResult{ UUID: uuid, AlreadyAbsent: true, Removed: true, ObservedAt: observation.ObservedAt, }) if marshalErr != nil { s.reconcileUncertain(w, record, record.Result, "domain_delete_result_failed", "domain deletion result could not be encoded", http.StatusServiceUnavailable) return } s.reconcileSuccess(w, record, "domain_delete_readback_confirmed", result) } func (s *Server) storeRecord(id string) (operation.Record, error) { if s.store == nil { return operation.Record{}, operation.ErrNotFound } return s.store.Get(id) } func isPlanningKind(kind string) bool { return kind == "domain_preflight" || kind == "domain_create_plan" || kind == "domain_create_validate" } func (s *Server) writeAllowed(w http.ResponseWriter, ctx context.Context) bool { if s.refreshHostReadiness(ctx) && s.readiness() == config.ReadinessReady { return true } writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent is degraded and mutating operations are disabled"}) return false } // readbackAllowed is the control-plane gate for read-only planning, // validation, and reconciliation operations. These operations may obtain a // fresh capability/artifact/domain observation and commit that observation to // the local journal, but they must not require a successful host Probe(). A // failed/stale probe is precisely when an ambiguous operation needs a later // read-back. Transport authentication is enforced by Handler; this method // covers the remaining local control-plane invariants and keeps the durable // journal mandatory. func (s *Server) readbackAllowed(w http.ResponseWriter) bool { return s.controlPlaneReadOnlyAllowed(w, "read-back") } func (s *Server) controlPlaneReadOnlyAllowed(w http.ResponseWriter, operation string) bool { if s == nil || s.store == nil || !s.store.Writable() || s.cfg.AgentID == "" { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent control plane is degraded and " + operation + " is disabled"}) return false } s.stateMu.RLock() tlsReady := s.tlsReady s.stateMu.RUnlock() if s.cfg.RequireMTLS && (!s.cfg.TLSComplete() || !tlsReady) { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent TLS is not ready for " + operation}) return false } if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent client certificate policy is not ready for " + operation}) return false } if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) { writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent request signing is not ready for " + operation}) return false } return true } // preflightAllowed is kept as a narrow compatibility name for the create // planning path. Both planning and reconciliation are read-only control-plane // operations and intentionally share the same host-readiness-independent // checks, while retaining the preflight-specific error wording. func (s *Server) preflightAllowed(w http.ResponseWriter) bool { return s.controlPlaneReadOnlyAllowed(w, "preflight") } func (s *Server) authorized(r *http.Request) bool { if !s.cfg.RequireMTLS && s.cfg.IsLoopbackListen() { return true } if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 { return false } if !s.cfg.RequireMTLS { return false } leaf := r.TLS.PeerCertificates[0] digest := sha256.Sum256(leaf.Raw) observed := hex.EncodeToString(digest[:]) for _, expected := range s.cfg.ClientCertFingerprints { if strings.EqualFold(strings.TrimSpace(expected), observed) { return true } } return false } // TLSConfig constructs a mutual-TLS configuration only when all configured // files can be read. The caller may keep serving health in degraded mode when // this returns an error. func (s *Server) TLSConfig() (*tls.Config, error) { if !s.cfg.TLSComplete() { return nil, fmt.Errorf("TLS certificate and key are not configured") } cert, err := tls.LoadX509KeyPair(s.cfg.TLSCertFile, s.cfg.TLSKeyFile) if err != nil { return nil, fmt.Errorf("load agent TLS certificate: %w", err) } caData, err := os.ReadFile(s.cfg.TLSCAFile) if err != nil { return nil, fmt.Errorf("read agent TLS CA: %w", err) } pool := x509.NewCertPool() if !pool.AppendCertsFromPEM(caData) { return nil, fmt.Errorf("parse agent TLS CA") } return &tls.Config{ MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{cert}, ClientCAs: pool, ClientAuth: tls.RequireAndVerifyClientCert, NextProtos: []string{"h2", "http/1.1"}, }, nil } func readBody(r *http.Request, max int64) ([]byte, error) { if max <= 0 { return nil, fmt.Errorf("request body limit is not configured") } data, err := io.ReadAll(io.LimitReader(r.Body, max+1)) if err != nil { return nil, fmt.Errorf("read request body: %w", err) } if int64(len(data)) > max { return nil, fmt.Errorf("request body exceeds limit") } return data, nil } func decodeJSON(data []byte, target any) error { decoder := json.NewDecoder(bytes.NewReader(data)) decoder.DisallowUnknownFields() if err := decoder.Decode(target); err != nil { return err } var extra any if err := decoder.Decode(&extra); err != io.EOF { return fmt.Errorf("multiple JSON values") } return nil } func writeJSON(w http.ResponseWriter, status int, value any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) _ = json.NewEncoder(w).Encode(value) } // NewHTTPServer provides conservative timeout defaults for a caller that // wants to run the agent directly from main or an integration test. func (s *Server) NewHTTPServer() *http.Server { return &http.Server{ Addr: s.cfg.ListenAddr, Handler: s.Handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: s.cfg.RequestTimeout, WriteTimeout: s.cfg.RequestTimeout, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 32 << 10, } } func remoteIP(r *http.Request) net.IP { host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { return nil } return net.ParseIP(strings.Trim(host, "[]")) }