You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/internal/api/server_test.go

497 lines
21 KiB
Go

package api
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"path/filepath"
"testing"
"g.luoli.live/root/virt/agent/internal/auth"
"g.luoli.live/root/virt/agent/internal/capability"
"g.luoli.live/root/virt/agent/internal/config"
"g.luoli.live/root/virt/agent/internal/domainpreflight"
"g.luoli.live/root/virt/agent/internal/hypervisor"
"g.luoli.live/root/virt/agent/internal/operation"
"g.luoli.live/root/virt/agent/internal/startup"
)
func testServer(t *testing.T, agentID string, store *operation.Store) *Server {
t.Helper()
c := config.Defaults()
c.AgentID = agentID
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.StaticProvider{AgentID: agentID, ProtocolVersion: c.ProtocolVersion},
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
return s
}
func testBackendServer(t *testing.T, agentID string, store *operation.Store) *Server {
t.Helper()
c := config.Defaults()
c.AgentID = agentID
backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: agentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
return s
}
func TestHealthRemainsAvailableWhenIdentityIsMissing(t *testing.T) {
s := testServer(t, "", operation.NewMemory())
req := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, req)
if recorder.Code != http.StatusOK {
t.Fatalf("health status = %d", recorder.Code)
}
var response healthResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
if response.Readiness != config.ReadinessDegraded {
t.Fatalf("readiness = %q", response.Readiness)
}
}
func TestHealthRemainsLiveWhenJournalAndSigningAreUnavailable(t *testing.T) {
// This models the systemd startup boundary: a bad journal path or missing
// HMAC file must not make liveness disappear. The same degraded state must
// reject a protected write instead of silently falling back to plaintext.
c := config.Defaults()
c.AgentID = "host-103"
c.RequireSigned = true
verifier, err := auth.New(nil, true)
if err != nil {
t.Fatal(err)
}
s, err := New(Options{
Config: c,
Store: nil,
Capabilities: capability.StaticProvider{AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Auth: verifier,
})
if err != nil {
t.Fatal(err)
}
health := httptest.NewRecorder()
s.Handler().ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/v1/health", nil))
if health.Code != http.StatusOK {
t.Fatalf("health status = %d, body=%s", health.Code, health.Body.String())
}
var response healthResponse
if err := json.Unmarshal(health.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
if response.Status != "degraded" || response.Readiness != config.ReadinessDegraded {
t.Fatalf("health = %+v, want degraded liveness", response)
}
write := httptest.NewRecorder()
s.Handler().ServeHTTP(write, httptest.NewRequest(http.MethodPost, "/v1/agent/test", bytes.NewReader([]byte(`{}`))))
if write.Code != http.StatusServiceUnavailable {
t.Fatalf("protected write status = %d, body=%s; unavailable signing must fail closed", write.Code, write.Body.String())
}
}
func TestLiveHostProbeDowngradesReadinessAndBlocksWrites(t *testing.T) {
store := operation.NewMemory()
backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
c := config.Defaults()
c.AgentID = "host-103"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
backend.SetProbeError(context.DeadlineExceeded)
health := httptest.NewRecorder()
s.Handler().ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/v1/health", nil))
if health.Code != http.StatusOK {
t.Fatalf("health status = %d, body=%s", health.Code, health.Body.String())
}
var response healthResponse
if err := json.Unmarshal(health.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
if response.Status != "degraded" || response.Readiness != config.ReadinessDegraded {
t.Fatalf("health = %+v, want degraded after live host probe failure", response)
}
write := httptest.NewRecorder()
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000099","idempotency_key":"idem-live-probe-0001","kind":"capability_probe","target":"host-103","payload":{}}`)
s.Handler().ServeHTTP(write, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)))
if write.Code != http.StatusServiceUnavailable {
t.Fatalf("write status = %d, body=%s; live probe failure must fail closed", write.Code, write.Body.String())
}
}
func TestCapabilityProbeOperationIsIdempotent(t *testing.T) {
s := testServer(t, "host-103", operation.NewMemory())
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000011","idempotency_key":"idem-probe-0001","kind":"capability_probe","target":"host-103","payload":{}}`)
first := httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body))
firstRecorder := httptest.NewRecorder()
s.Handler().ServeHTTP(firstRecorder, first)
if firstRecorder.Code != http.StatusOK {
t.Fatalf("first operation status = %d, body=%s", firstRecorder.Code, firstRecorder.Body.String())
}
second := httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body))
secondRecorder := httptest.NewRecorder()
s.Handler().ServeHTTP(secondRecorder, second)
if secondRecorder.Code != http.StatusOK {
t.Fatalf("replay operation status = %d, body=%s", secondRecorder.Code, secondRecorder.Body.String())
}
}
func TestUnknownOperationKindDoesNotEnterJournal(t *testing.T) {
store := operation.NewMemory()
s := testServer(t, "host-103", store)
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000012","idempotency_key":"idem-unknown-0001","kind":"delete_vm"}`)
req := httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body))
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, req)
if recorder.Code != http.StatusNotImplemented {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
if _, err := store.Get("00000000-0000-4000-8000-000000000012"); err == nil {
t.Fatal("unsupported operation must not be journaled")
}
}
func TestRemoteProtectedRouteRequiresClientCertificate(t *testing.T) {
c := config.Defaults()
c.AgentID = "host-103"
c.ListenAddr = "0.0.0.0:8787"
c.RequireMTLS = true
c.RequireSigned = true
s, err := New(Options{Config: c, Store: operation.NewMemory(), Capabilities: capability.StaticProvider{AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}})
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodGet, "/v1/capabilities", nil)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, req)
if recorder.Code != http.StatusUnauthorized {
t.Fatalf("status = %d", recorder.Code)
}
}
func TestRawDomainDefineIsDisabledAndDomainReadRemainsReadOnly(t *testing.T) {
store := operation.NewMemory()
s := testBackendServer(t, "host-103", store)
defineBody := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000013","idempotency_key":"idem-domain-0001","kind":"domain_define","target":"00000000-0000-4000-8000-000000000001","payload":{"uuid":"00000000-0000-4000-8000-000000000001","xml":"<domain><uuid>00000000-0000-4000-8000-000000000001</uuid><os><type>hvm</type></os></domain>"}}`)
record := httptest.NewRecorder()
s.Handler().ServeHTTP(record, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(defineBody)))
if record.Code != http.StatusNotImplemented {
t.Fatalf("define status=%d body=%s; raw XML define must stay disabled", record.Code, record.Body.String())
}
if _, err := store.Get("00000000-0000-4000-8000-000000000013"); !errors.Is(err, operation.ErrNotFound) {
t.Fatalf("disabled domain define must not enter the journal: err=%v", err)
}
if _, err := s.backend.DefineDomain(context.Background(), hypervisor.DefineRequest{
UUID: "00000000-0000-4000-8000-000000000001",
XML: []byte(`<domain><uuid>00000000-0000-4000-8000-000000000001</uuid><os><type>hvm</type></os></domain>`),
}); err != nil {
t.Fatalf("seed read-only test domain: %v", err)
}
readBody := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000014","idempotency_key":"idem-read-0001","kind":"domain_read","target":"00000000-0000-4000-8000-000000000001","payload":{"uuid":"00000000-0000-4000-8000-000000000001"}}`)
readRecord := httptest.NewRecorder()
s.Handler().ServeHTTP(readRecord, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(readBody)))
if readRecord.Code != http.StatusOK {
t.Fatalf("read status=%d body=%s", readRecord.Code, readRecord.Body.String())
}
var readResult operation.Record
if err := json.Unmarshal(readRecord.Body.Bytes(), &readResult); err != nil {
t.Fatal(err)
}
var readPayload domainXMLResult
if err := json.Unmarshal(readResult.Result, &readPayload); err != nil {
t.Fatal(err)
}
if readPayload.UUID != "00000000-0000-4000-8000-000000000001" || !validSHA256(readPayload.XMLHash) {
t.Fatalf("readback did not include a safe XML identity summary: %s", readRecord.Body.String())
}
if bytes.Contains(readResult.Result, []byte(`"xml"`)) {
t.Fatalf("full domain XML must not be persisted in the operation result: %s", readRecord.Body.String())
}
}
func TestDomainPreflightIsTypedReadOnlyAndReturnsPolicyHash(t *testing.T) {
store := operation.NewMemory()
s := testBackendServer(t, "host-103", store)
uuid := "00000000-0000-4000-8000-000000000041"
payload := `{"uuid":"` + uuid + `","firmware":"efi","architecture":"x86_64","machine":"q35","os_type":"hvm","loader":{"type":"pflash","readonly":true,"secure_boot":false}}`
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000042","idempotency_key":"idem-domain-preflight-0042","kind":"domain_preflight","target":"` + uuid + `","payload":` + payload + `}`)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)))
if recorder.Code != http.StatusOK {
t.Fatalf("preflight status=%d body=%s", recorder.Code, recorder.Body.String())
}
var record operation.Record
if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusSucceeded || record.CommitBoundary != "domain_preflight_confirmed" {
t.Fatalf("preflight record=%+v", record)
}
if bytes.Contains(record.Payload, []byte(`"xml"`)) || bytes.Contains(record.Payload, []byte(`"disk"`)) || bytes.Contains(record.Payload, []byte(`"network"`)) {
t.Fatalf("preflight journal contains untyped host input: %s", record.Payload)
}
var result domainpreflight.Result
if err := json.Unmarshal(record.Result, &result); err != nil {
t.Fatal(err)
}
if result.XMLHash == "" || !result.LoaderPending || result.NormalizedXML != "" {
t.Fatalf("preflight result=%+v", result)
}
// The normalized policy is an internal input to a later host-bound stage;
// the operation/API projection carries only its digest. This prevents the
// plaintext journal from becoming a raw XML store while retaining the
// deterministic identity needed for reconciliation.
for _, forbidden := range []string{"\"normalized_xml\"", "<domain", "<disk", "<interface", "<devices", "<loader"} {
if bytes.Contains(record.Result, []byte(forbidden)) {
t.Fatalf("preflight result contains forbidden raw XML field %q: %s", forbidden, record.Result)
}
}
if _, err := s.backend.ReadDomainXML(context.Background(), uuid); !errors.Is(err, hypervisor.ErrDomainNotFound) {
t.Fatalf("preflight unexpectedly defined domain: err=%v", err)
}
}
// preflightProbeBackend lets the API tests distinguish the initial host
// readiness probe from the capability observation that preflight itself must
// perform. The fake backend's normal Probe and Capabilities methods share an
// error knob, so this wrapper models a stale/unavailable Probe independently.
type preflightProbeBackend struct {
*hypervisor.FakeBackend
probeErr error
capabilityErr error
probeCalls int
capabilityCalls int
}
func (b *preflightProbeBackend) Probe(ctx context.Context) (hypervisor.ProbeResult, error) {
b.probeCalls++
if b.probeErr != nil {
return hypervisor.ProbeResult{}, b.probeErr
}
return b.FakeBackend.Probe(ctx)
}
func (b *preflightProbeBackend) Capabilities(ctx context.Context) (hypervisor.CapabilitySnapshot, error) {
b.capabilityCalls++
if b.capabilityErr != nil {
return hypervisor.CapabilitySnapshot{}, b.capabilityErr
}
return b.FakeBackend.Capabilities(ctx)
}
func preflightOperationBody(operationID, idempotencyKey, uuid string) []byte {
return []byte(`{"operation_id":"` + operationID + `","idempotency_key":"` + idempotencyKey + `","kind":"domain_preflight","target":"` + uuid + `","payload":{"uuid":"` + uuid + `","firmware":"efi","architecture":"x86_64","machine":"q35","os_type":"hvm","loader":{"type":"pflash","readonly":true,"secure_boot":false}}}`)
}
func TestDomainPreflightBypassesStaleHostReadinessGate(t *testing.T) {
store := operation.NewMemory()
backend := &preflightProbeBackend{
FakeBackend: hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()),
probeErr: context.DeadlineExceeded,
}
c := config.Defaults()
c.AgentID = "host-preflight-stale"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: false,
})
if err != nil {
t.Fatal(err)
}
uuid := "00000000-0000-4000-8000-000000000045"
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(preflightOperationBody(
"00000000-0000-4000-8000-000000000046", "idem-preflight-stale-0046", uuid,
))))
if recorder.Code != http.StatusOK {
t.Fatalf("preflight status=%d body=%s", recorder.Code, recorder.Body.String())
}
var record operation.Record
if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusSucceeded || record.ErrorCode != "" {
t.Fatalf("preflight record=%+v", record)
}
if backend.probeCalls != 0 || backend.capabilityCalls != 1 {
t.Fatalf("backend calls probe=%d capabilities=%d; preflight must skip stale readiness probe", backend.probeCalls, backend.capabilityCalls)
}
}
func TestDomainPreflightHostUnavailableIsDurableAndReconciles(t *testing.T) {
store := operation.NewMemory()
backend := &preflightProbeBackend{
FakeBackend: hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()),
capabilityErr: errors.New("libvirt capability transport unavailable"),
}
c := config.Defaults()
c.AgentID = "host-preflight-unavailable"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: false,
})
if err != nil {
t.Fatal(err)
}
uuid := "00000000-0000-4000-8000-000000000047"
id := "00000000-0000-4000-8000-000000000048"
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(preflightOperationBody(id, "idem-preflight-unavailable-0048", uuid))))
if recorder.Code != http.StatusServiceUnavailable {
t.Fatalf("preflight status=%d body=%s", recorder.Code, recorder.Body.String())
}
record, err := store.Get(id)
if err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusUncertain || record.ErrorCode != "domain_preflight_host_unavailable" {
t.Fatalf("unavailable preflight record=%+v", record)
}
reconcile := httptest.NewRecorder()
s.Handler().ServeHTTP(reconcile, httptest.NewRequest(http.MethodPost, "/v1/operations/"+id+"/reconcile", nil))
if reconcile.Code != http.StatusServiceUnavailable {
t.Fatalf("reconcile status=%d body=%s", reconcile.Code, reconcile.Body.String())
}
reconciled, err := store.Get(id)
if err != nil {
t.Fatal(err)
}
if reconciled.Status != operation.StatusUncertain || reconciled.ErrorCode != "domain_preflight_host_unavailable" {
t.Fatalf("reconciled unavailable preflight record=%+v", reconciled)
}
if backend.capabilityCalls != 2 {
t.Fatalf("capability calls=%d, want initial + reconcile", backend.capabilityCalls)
}
}
func TestDomainPreflightHostPolicyRejectionIsDurable(t *testing.T) {
store := operation.NewMemory()
badCaps := hypervisor.DefaultCapabilitySnapshot()
badCaps.SecureBootDisabled = false
backend := &preflightProbeBackend{FakeBackend: hypervisor.NewFakeBackend(badCaps)}
c := config.Defaults()
c.AgentID = "host-preflight-policy"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: false,
})
if err != nil {
t.Fatal(err)
}
uuid := "00000000-0000-4000-8000-000000000049"
id := "00000000-0000-4000-8000-000000000050"
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(preflightOperationBody(id, "idem-preflight-policy-0050", uuid))))
if recorder.Code != http.StatusUnprocessableEntity {
t.Fatalf("preflight status=%d body=%s", recorder.Code, recorder.Body.String())
}
record, err := store.Get(id)
if err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusFailed || record.ErrorCode != "domain_preflight_host_policy_rejected" {
t.Fatalf("policy preflight record=%+v", record)
}
}
func TestDomainPreflightRejectsXMLAndDoesNotEnterJournal(t *testing.T) {
store := operation.NewMemory()
s := testBackendServer(t, "host-103", store)
uuid := "00000000-0000-4000-8000-000000000043"
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000044","idempotency_key":"idem-domain-preflight-0044","kind":"domain_preflight","target":"` + uuid + `","payload":{"uuid":"` + uuid + `","firmware":"efi","architecture":"x86_64","machine":"q35","os_type":"hvm","loader":{"type":"pflash","readonly":true,"secure_boot":false},"xml":"<domain/>"}}`)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)))
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
if _, err := store.Get("00000000-0000-4000-8000-000000000044"); !errors.Is(err, operation.ErrNotFound) {
t.Fatalf("invalid preflight request entered journal: err=%v", err)
}
}
func TestSeedPreparePublishesGreenfieldStartupArtifact(t *testing.T) {
store := operation.NewMemory()
backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
c := config.Defaults()
c.AgentID = "host-103"
c.ArtifactRoot = t.TempDir()
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
artifactID := "00000000-0000-4000-8000-000000000031"
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000032","idempotency_key":"idem-seed-prepare-01","kind":"seed_prepare","target":"` + artifactID + `","payload":{"artifact_id":"` + artifactID + `","instance_id":"instance-031","hostname":"vm-031","network_config":"{\"version\":2,\"ethernets\":{}}","startup_script":"#!/bin/sh\necho first-boot\n"}}`)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)))
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
var record operation.Record
if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusSucceeded || record.CommitBoundary != "seed_published" {
t.Fatalf("record = %+v", record)
}
if bytes.Contains(record.Payload, []byte(`"network_config"`)) || bytes.Contains(record.Payload, []byte(`"startup_script"`)) {
t.Fatalf("seed journal payload contains raw content fields: %s", record.Payload)
}
if !bytes.Contains(record.Payload, []byte(`"network_digest"`)) || !bytes.Contains(record.Payload, []byte(`"script_digest"`)) {
t.Fatalf("seed journal payload lost content identity hashes: %s", record.Payload)
}
manifest, err := startup.VerifySeedDirectory(filepath.Join(c.ArtifactRoot, artifactID))
if err != nil || manifest.Version != startup.SeedManifestVersion {
t.Fatalf("published seed verification = %+v, err=%v", manifest, err)
}
if bytes.Contains(record.Payload, []byte("first-boot")) {
t.Fatal("startup script was persisted in the plaintext operation journal")
}
}