package api import ( "bytes" "context" "encoding/json" "errors" "net/http" "net/http/httptest" "path/filepath" "testing" "g.luoli.live/root/virt/agent/internal/auth" "g.luoli.live/root/virt/agent/internal/capability" "g.luoli.live/root/virt/agent/internal/config" "g.luoli.live/root/virt/agent/internal/domainpreflight" "g.luoli.live/root/virt/agent/internal/hypervisor" "g.luoli.live/root/virt/agent/internal/operation" "g.luoli.live/root/virt/agent/internal/startup" ) func testServer(t *testing.T, agentID string, store *operation.Store) *Server { t.Helper() c := config.Defaults() c.AgentID = agentID s, err := New(Options{ Config: c, Store: store, Capabilities: capability.StaticProvider{AgentID: agentID, ProtocolVersion: c.ProtocolVersion}, HostBackendReady: true, }) if err != nil { t.Fatal(err) } return s } func testBackendServer(t *testing.T, agentID string, store *operation.Store) *Server { t.Helper() c := config.Defaults() c.AgentID = agentID backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) s, err := New(Options{ Config: c, Store: store, Capabilities: capability.BackendProvider{Backend: backend, AgentID: agentID, ProtocolVersion: c.ProtocolVersion}, Backend: backend, HostBackendReady: true, }) if err != nil { t.Fatal(err) } return s } func TestHealthRemainsAvailableWhenIdentityIsMissing(t *testing.T) { s := testServer(t, "", operation.NewMemory()) req := httptest.NewRequest(http.MethodGet, "/v1/health", nil) recorder := httptest.NewRecorder() s.Handler().ServeHTTP(recorder, req) if recorder.Code != http.StatusOK { t.Fatalf("health status = %d", recorder.Code) } var response healthResponse if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { t.Fatal(err) } if response.Readiness != config.ReadinessDegraded { t.Fatalf("readiness = %q", response.Readiness) } } func TestHealthRemainsLiveWhenJournalAndSigningAreUnavailable(t *testing.T) { // This models the systemd startup boundary: a bad journal path or missing // HMAC file must not make liveness disappear. The same degraded state must // reject a protected write instead of silently falling back to plaintext. c := config.Defaults() c.AgentID = "host-103" c.RequireSigned = true verifier, err := auth.New(nil, true) if err != nil { t.Fatal(err) } s, err := New(Options{ Config: c, Store: nil, Capabilities: capability.StaticProvider{AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}, Auth: verifier, }) if err != nil { t.Fatal(err) } health := httptest.NewRecorder() s.Handler().ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/v1/health", nil)) if health.Code != http.StatusOK { t.Fatalf("health status = %d, body=%s", health.Code, health.Body.String()) } var response healthResponse if err := json.Unmarshal(health.Body.Bytes(), &response); err != nil { t.Fatal(err) } if response.Status != "degraded" || response.Readiness != config.ReadinessDegraded { t.Fatalf("health = %+v, want degraded liveness", response) } write := httptest.NewRecorder() s.Handler().ServeHTTP(write, httptest.NewRequest(http.MethodPost, "/v1/agent/test", bytes.NewReader([]byte(`{}`)))) if write.Code != http.StatusServiceUnavailable { t.Fatalf("protected write status = %d, body=%s; unavailable signing must fail closed", write.Code, write.Body.String()) } } func TestLiveHostProbeDowngradesReadinessAndBlocksWrites(t *testing.T) { store := operation.NewMemory() backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) c := config.Defaults() c.AgentID = "host-103" s, err := New(Options{ Config: c, Store: store, Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}, Backend: backend, HostBackendReady: true, }) if err != nil { t.Fatal(err) } backend.SetProbeError(context.DeadlineExceeded) health := httptest.NewRecorder() s.Handler().ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/v1/health", nil)) if health.Code != http.StatusOK { t.Fatalf("health status = %d, body=%s", health.Code, health.Body.String()) } var response healthResponse if err := json.Unmarshal(health.Body.Bytes(), &response); err != nil { t.Fatal(err) } if response.Status != "degraded" || response.Readiness != config.ReadinessDegraded { t.Fatalf("health = %+v, want degraded after live host probe failure", response) } write := httptest.NewRecorder() body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000099","idempotency_key":"idem-live-probe-0001","kind":"capability_probe","target":"host-103","payload":{}}`) s.Handler().ServeHTTP(write, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body))) if write.Code != http.StatusServiceUnavailable { t.Fatalf("write status = %d, body=%s; live probe failure must fail closed", write.Code, write.Body.String()) } } func TestCapabilityProbeOperationIsIdempotent(t *testing.T) { s := testServer(t, "host-103", operation.NewMemory()) body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000011","idempotency_key":"idem-probe-0001","kind":"capability_probe","target":"host-103","payload":{}}`) first := httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)) firstRecorder := httptest.NewRecorder() s.Handler().ServeHTTP(firstRecorder, first) if firstRecorder.Code != http.StatusOK { t.Fatalf("first operation status = %d, body=%s", firstRecorder.Code, firstRecorder.Body.String()) } second := httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)) secondRecorder := httptest.NewRecorder() s.Handler().ServeHTTP(secondRecorder, second) if secondRecorder.Code != http.StatusOK { t.Fatalf("replay operation status = %d, body=%s", secondRecorder.Code, secondRecorder.Body.String()) } } func TestUnknownOperationKindDoesNotEnterJournal(t *testing.T) { store := operation.NewMemory() s := testServer(t, "host-103", store) body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000012","idempotency_key":"idem-unknown-0001","kind":"delete_vm"}`) req := httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)) recorder := httptest.NewRecorder() s.Handler().ServeHTTP(recorder, req) if recorder.Code != http.StatusNotImplemented { t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String()) } if _, err := store.Get("00000000-0000-4000-8000-000000000012"); err == nil { t.Fatal("unsupported operation must not be journaled") } } func TestRemoteProtectedRouteRequiresClientCertificate(t *testing.T) { c := config.Defaults() c.AgentID = "host-103" c.ListenAddr = "0.0.0.0:8787" c.RequireMTLS = true c.RequireSigned = true s, err := New(Options{Config: c, Store: operation.NewMemory(), Capabilities: capability.StaticProvider{AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}}) if err != nil { t.Fatal(err) } req := httptest.NewRequest(http.MethodGet, "/v1/capabilities", nil) recorder := httptest.NewRecorder() s.Handler().ServeHTTP(recorder, req) if recorder.Code != http.StatusUnauthorized { t.Fatalf("status = %d", recorder.Code) } } func TestRawDomainDefineIsDisabledAndDomainReadRemainsReadOnly(t *testing.T) { store := operation.NewMemory() s := testBackendServer(t, "host-103", store) defineBody := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000013","idempotency_key":"idem-domain-0001","kind":"domain_define","target":"00000000-0000-4000-8000-000000000001","payload":{"uuid":"00000000-0000-4000-8000-000000000001","xml":"00000000-0000-4000-8000-000000000001hvm"}}`) record := httptest.NewRecorder() s.Handler().ServeHTTP(record, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(defineBody))) if record.Code != http.StatusNotImplemented { t.Fatalf("define status=%d body=%s; raw XML define must stay disabled", record.Code, record.Body.String()) } if _, err := store.Get("00000000-0000-4000-8000-000000000013"); !errors.Is(err, operation.ErrNotFound) { t.Fatalf("disabled domain define must not enter the journal: err=%v", err) } if _, err := s.backend.DefineDomain(context.Background(), hypervisor.DefineRequest{ UUID: "00000000-0000-4000-8000-000000000001", XML: []byte(`00000000-0000-4000-8000-000000000001hvm`), }); err != nil { t.Fatalf("seed read-only test domain: %v", err) } readBody := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000014","idempotency_key":"idem-read-0001","kind":"domain_read","target":"00000000-0000-4000-8000-000000000001","payload":{"uuid":"00000000-0000-4000-8000-000000000001"}}`) readRecord := httptest.NewRecorder() s.Handler().ServeHTTP(readRecord, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(readBody))) if readRecord.Code != http.StatusOK { t.Fatalf("read status=%d body=%s", readRecord.Code, readRecord.Body.String()) } var readResult operation.Record if err := json.Unmarshal(readRecord.Body.Bytes(), &readResult); err != nil { t.Fatal(err) } var readPayload domainXMLResult if err := json.Unmarshal(readResult.Result, &readPayload); err != nil { t.Fatal(err) } if readPayload.UUID != "00000000-0000-4000-8000-000000000001" || !validSHA256(readPayload.XMLHash) { t.Fatalf("readback did not include a safe XML identity summary: %s", readRecord.Body.String()) } if bytes.Contains(readResult.Result, []byte(`"xml"`)) { t.Fatalf("full domain XML must not be persisted in the operation result: %s", readRecord.Body.String()) } } func TestDomainPreflightIsTypedReadOnlyAndReturnsPolicyHash(t *testing.T) { store := operation.NewMemory() s := testBackendServer(t, "host-103", store) uuid := "00000000-0000-4000-8000-000000000041" payload := `{"uuid":"` + uuid + `","firmware":"efi","architecture":"x86_64","machine":"q35","os_type":"hvm","loader":{"type":"pflash","readonly":true,"secure_boot":false}}` body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000042","idempotency_key":"idem-domain-preflight-0042","kind":"domain_preflight","target":"` + uuid + `","payload":` + payload + `}`) recorder := httptest.NewRecorder() s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body))) if recorder.Code != http.StatusOK { t.Fatalf("preflight status=%d body=%s", recorder.Code, recorder.Body.String()) } var record operation.Record if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil { t.Fatal(err) } if record.Status != operation.StatusSucceeded || record.CommitBoundary != "domain_preflight_confirmed" { t.Fatalf("preflight record=%+v", record) } if bytes.Contains(record.Payload, []byte(`"xml"`)) || bytes.Contains(record.Payload, []byte(`"disk"`)) || bytes.Contains(record.Payload, []byte(`"network"`)) { t.Fatalf("preflight journal contains untyped host input: %s", record.Payload) } var result domainpreflight.Result if err := json.Unmarshal(record.Result, &result); err != nil { t.Fatal(err) } if result.XMLHash == "" || !result.LoaderPending || result.NormalizedXML != "" { t.Fatalf("preflight result=%+v", result) } // The normalized policy is an internal input to a later host-bound stage; // the operation/API projection carries only its digest. This prevents the // plaintext journal from becoming a raw XML store while retaining the // deterministic identity needed for reconciliation. for _, forbidden := range []string{"\"normalized_xml\"", ""}}`) recorder := httptest.NewRecorder() s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body))) if recorder.Code != http.StatusBadRequest { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } if _, err := store.Get("00000000-0000-4000-8000-000000000044"); !errors.Is(err, operation.ErrNotFound) { t.Fatalf("invalid preflight request entered journal: err=%v", err) } } func TestSeedPreparePublishesGreenfieldStartupArtifact(t *testing.T) { store := operation.NewMemory() backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) c := config.Defaults() c.AgentID = "host-103" c.ArtifactRoot = t.TempDir() s, err := New(Options{ Config: c, Store: store, Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}, Backend: backend, HostBackendReady: true, }) if err != nil { t.Fatal(err) } artifactID := "00000000-0000-4000-8000-000000000031" body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000032","idempotency_key":"idem-seed-prepare-01","kind":"seed_prepare","target":"` + artifactID + `","payload":{"artifact_id":"` + artifactID + `","instance_id":"instance-031","hostname":"vm-031","network_config":"{\"version\":2,\"ethernets\":{}}","startup_script":"#!/bin/sh\necho first-boot\n"}}`) recorder := httptest.NewRecorder() s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body))) if recorder.Code != http.StatusOK { t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String()) } var record operation.Record if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil { t.Fatal(err) } if record.Status != operation.StatusSucceeded || record.CommitBoundary != "seed_published" { t.Fatalf("record = %+v", record) } if bytes.Contains(record.Payload, []byte(`"network_config"`)) || bytes.Contains(record.Payload, []byte(`"startup_script"`)) { t.Fatalf("seed journal payload contains raw content fields: %s", record.Payload) } if !bytes.Contains(record.Payload, []byte(`"network_digest"`)) || !bytes.Contains(record.Payload, []byte(`"script_digest"`)) { t.Fatalf("seed journal payload lost content identity hashes: %s", record.Payload) } manifest, err := startup.VerifySeedDirectory(filepath.Join(c.ArtifactRoot, artifactID)) if err != nil || manifest.Version != startup.SeedManifestVersion { t.Fatalf("published seed verification = %+v, err=%v", manifest, err) } if bytes.Contains(record.Payload, []byte("first-boot")) { t.Fatal("startup script was persisted in the plaintext operation journal") } }