You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/internal/api/server_power_test.go

380 lines
16 KiB
Go

package api
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"g.luoli.live/root/virt/agent/internal/capability"
"g.luoli.live/root/virt/agent/internal/config"
"g.luoli.live/root/virt/agent/internal/hypervisor"
"g.luoli.live/root/virt/agent/internal/operation"
)
func newPowerTestServer(t *testing.T) (*Server, *operation.Store, *hypervisor.FakeBackend) {
t.Helper()
store := operation.NewMemory()
backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
if _, err := backend.DefineDomain(context.Background(), hypervisor.DefineRequest{
UUID: testPowerUUID,
XML: []byte(` <domain><uuid>` + testPowerUUID + `</uuid><os><type>hvm</type></os></domain>`),
}); err != nil {
t.Fatal(err)
}
c := config.Defaults()
c.AgentID = "host-103"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
return s, store, backend
}
const testPowerUUID = "00000000-0000-4000-8000-000000000021"
func TestDomainPowerRejectsUnknownActionBeforeJournal(t *testing.T) {
s, store, _ := newPowerTestServer(t)
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000022","idempotency_key":"idem-power-invalid-01","kind":"domain_power","target":"` + testPowerUUID + `","payload":{"uuid":"` + testPowerUUID + `","action":"poweroff"}}`)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)))
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, body=%s; want 400", recorder.Code, recorder.Body.String())
}
if _, err := store.Get("00000000-0000-4000-8000-000000000022"); !errors.Is(err, operation.ErrNotFound) {
t.Fatalf("invalid power action entered journal: %v", err)
}
}
func TestDomainPowerReturnsTypedReadbackResult(t *testing.T) {
s, store, _ := newPowerTestServer(t)
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000023","idempotency_key":"idem-power-start-01","kind":"domain_power","target":"` + testPowerUUID + `","payload":{"uuid":"` + testPowerUUID + `","action":"start"}}`)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)))
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
var record operation.Record
if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusSucceeded || record.CommitBoundary != "power_readback_confirmed" {
t.Fatalf("record = %+v, want succeeded/readback boundary", record)
}
var result hypervisor.PowerResult
if err := json.Unmarshal(record.Result, &result); err != nil {
t.Fatalf("decode power result: %v", err)
}
if result.UUID != testPowerUUID || result.Action != hypervisor.PowerActionStart || result.State != hypervisor.StateRunning || !result.Active {
t.Fatalf("power result = %+v", result)
}
if _, err := store.Get(record.OperationID); err != nil {
t.Fatal(err)
}
}
func TestDomainPowerRejectsUnknownPayloadFields(t *testing.T) {
s, store, _ := newPowerTestServer(t)
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000024","idempotency_key":"idem-power-extra-01","kind":"domain_power","payload":{"uuid":"` + testPowerUUID + `","action":"start","force":true}}`)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)))
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, body=%s; want 400", recorder.Code, recorder.Body.String())
}
if _, err := store.Get("00000000-0000-4000-8000-000000000024"); !errors.Is(err, operation.ErrNotFound) {
t.Fatalf("invalid payload entered journal: %v", err)
}
}
func TestDomainPowerRejectsTargetPayloadMismatchBeforeJournal(t *testing.T) {
s, store, _ := newPowerTestServer(t)
body := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000025","idempotency_key":"idem-power-target-01","kind":"domain_power","target":"00000000-0000-4000-8000-000000000099","payload":{"uuid":"` + testPowerUUID + `","action":"start"}}`)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)))
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, body=%s; want 400", recorder.Code, recorder.Body.String())
}
if _, err := store.Get("00000000-0000-4000-8000-000000000025"); !errors.Is(err, operation.ErrNotFound) {
t.Fatalf("mismatched target entered journal: %v", err)
}
}
// reconcilePowerBackend counts calls so the test can prove that a reconcile
// only reads host state and never replays the lifecycle action.
type reconcilePowerBackend struct {
*hypervisor.FakeBackend
powerCalls int
readCalls int
probeCalls int
readErr error
}
func (b *reconcilePowerBackend) Probe(ctx context.Context) (hypervisor.ProbeResult, error) {
b.probeCalls++
return b.FakeBackend.Probe(ctx)
}
func (b *reconcilePowerBackend) PowerDomain(ctx context.Context, uuid string, action hypervisor.PowerAction) (hypervisor.PowerResult, error) {
b.powerCalls++
return b.FakeBackend.PowerDomain(ctx, uuid, action)
}
func (b *reconcilePowerBackend) ReadDomainPowerState(ctx context.Context, uuid string) (hypervisor.DomainPowerState, error) {
b.readCalls++
if b.readErr != nil {
return "", b.readErr
}
return b.FakeBackend.ReadDomainPowerState(ctx, uuid)
}
func seedUncertainPowerOperation(t *testing.T, store *operation.Store, id, uuid string, action hypervisor.PowerAction) {
t.Helper()
payload := []byte(`{"uuid":"` + uuid + `","action":"` + string(action) + `"}`)
record, replay, err := store.Begin(operation.BeginRequest{
OperationID: id,
IdempotencyKey: "idem-reconcile-" + id[len(id)-4:],
Kind: "domain_power",
Target: uuid,
RequestHash: operation.HashPayload(payload),
Payload: payload,
})
if err != nil || replay {
t.Fatalf("seed Begin() = %+v, replay=%v, err=%v", record, replay, err)
}
if _, err := store.Update(id, operation.StatusUncertain, nil, "domain_power_uncertain", "host result requires read-back"); err != nil {
t.Fatal(err)
}
}
func TestDomainPowerReconcileReadsStateWithoutReplayingAction(t *testing.T) {
store := operation.NewMemory()
fake := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
if _, err := fake.DefineDomain(context.Background(), hypervisor.DefineRequest{
UUID: testPowerUUID,
XML: []byte(` <domain><uuid>` + testPowerUUID + `</uuid><os><type>hvm</type></os></domain>`),
}); err != nil {
t.Fatal(err)
}
backend := &reconcilePowerBackend{FakeBackend: fake}
c := config.Defaults()
c.AgentID = "host-103"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
const operationID = "00000000-0000-4000-8000-000000000026"
seedUncertainPowerOperation(t, store, operationID, testPowerUUID, hypervisor.PowerActionStart)
// The current state is still shut off, so reconcile must preserve uncertain
// and return a safe operation envelope rather than claiming success.
first := httptest.NewRecorder()
s.Handler().ServeHTTP(first, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil))
if first.Code != http.StatusConflict {
t.Fatalf("unsatisfied reconcile status = %d, body=%s; want 409", first.Code, first.Body.String())
}
var uncertain operation.Record
if err := json.Unmarshal(first.Body.Bytes(), &uncertain); err != nil {
t.Fatalf("decode uncertain record: %v", err)
}
if uncertain.Status != operation.StatusUncertain {
t.Fatalf("unsatisfied status = %q, want uncertain", uncertain.Status)
}
if backend.readCalls != 1 || backend.powerCalls != 0 {
t.Fatalf("calls after unsatisfied reconcile: read=%d power=%d; want read=1 power=0", backend.readCalls, backend.powerCalls)
}
// Simulate the host having completed the original action before the
// response was lost. Reconcile should observe it and commit success without
// invoking PowerDomain again.
if _, err := fake.PowerDomain(context.Background(), testPowerUUID, hypervisor.PowerActionStart); err != nil {
t.Fatal(err)
}
backend.powerCalls = 0
second := httptest.NewRecorder()
s.Handler().ServeHTTP(second, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil))
if second.Code != http.StatusOK {
t.Fatalf("satisfied reconcile status = %d, body=%s; want 200", second.Code, second.Body.String())
}
var succeeded operation.Record
if err := json.Unmarshal(second.Body.Bytes(), &succeeded); err != nil {
t.Fatalf("decode succeeded record: %v", err)
}
if succeeded.Status != operation.StatusSucceeded || succeeded.CommitBoundary != "power_readback_confirmed" {
t.Fatalf("reconciled record = %+v, want succeeded/readback boundary", succeeded)
}
var result hypervisor.PowerResult
if err := json.Unmarshal(succeeded.Result, &result); err != nil {
t.Fatalf("decode reconciled result: %v", err)
}
if result.Action != hypervisor.PowerActionStart || result.State != hypervisor.StateRunning || !result.Active {
t.Fatalf("reconciled result = %+v", result)
}
if backend.readCalls != 2 || backend.powerCalls != 0 {
t.Fatalf("calls after satisfied reconcile: read=%d power=%d; want read=2 power=0", backend.readCalls, backend.powerCalls)
}
}
func TestDomainPowerReconcileReadsWhenHostReadinessIsDegraded(t *testing.T) {
store := operation.NewMemory()
fake := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
if _, err := fake.DefineDomain(context.Background(), hypervisor.DefineRequest{
UUID: testPowerUUID,
XML: []byte(` <domain><uuid>` + testPowerUUID + `</uuid><os><type>hvm</type></os></domain>`),
}); err != nil {
t.Fatal(err)
}
// Keep the host readiness bit degraded and make a fresh Probe fail. The
// domain state read itself remains available, which is the boundary a
// reconciliation request must support.
fake.SetProbeError(context.DeadlineExceeded)
backend := &reconcilePowerBackend{FakeBackend: fake}
c := config.Defaults()
c.AgentID = "host-103"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: false,
})
if err != nil {
t.Fatal(err)
}
if s.readiness() != config.ReadinessDegraded {
t.Fatalf("initial readiness = %q, want degraded", s.readiness())
}
const operationID = "00000000-0000-4000-8000-000000000028"
// A shut-off domain satisfies a shutdown read-back. If reconcile were to
// replay the lifecycle request, powerCalls would become non-zero; the
// read-only contract requires only ReadDomainPowerState.
seedUncertainPowerOperation(t, store, operationID, testPowerUUID, hypervisor.PowerActionShutdown)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("degraded reconcile status = %d, body=%s; read-back must remain available", recorder.Code, recorder.Body.String())
}
record, err := store.Get(operationID)
if err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusSucceeded || record.CommitBoundary != "power_readback_confirmed" {
t.Fatalf("degraded reconcile record = %+v", record)
}
if backend.readCalls != 1 || backend.powerCalls != 0 || backend.probeCalls != 0 {
t.Fatalf("degraded reconcile calls: read=%d power=%d probe=%d; want read=1 power=0 probe=0", backend.readCalls, backend.powerCalls, backend.probeCalls)
}
// The relaxed gate is limited to reconciliation. A new lifecycle request
// must still fail closed while the host readiness probe is degraded.
writeBody := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000029","idempotency_key":"idem-degraded-write-0029","kind":"domain_power","target":"` + testPowerUUID + `","payload":{"uuid":"` + testPowerUUID + `","action":"start"}}`)
write := httptest.NewRecorder()
s.Handler().ServeHTTP(write, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(writeBody)))
if write.Code != http.StatusServiceUnavailable {
t.Fatalf("degraded lifecycle status = %d, body=%s; new mutation must remain blocked", write.Code, write.Body.String())
}
if backend.powerCalls != 0 {
t.Fatalf("degraded lifecycle unexpectedly replayed power action: power=%d", backend.powerCalls)
}
}
func TestDomainPowerReconcileReadFailureKeepsUncertain(t *testing.T) {
store := operation.NewMemory()
fake := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
if _, err := fake.DefineDomain(context.Background(), hypervisor.DefineRequest{
UUID: testPowerUUID,
XML: []byte(` <domain><uuid>` + testPowerUUID + `</uuid><os><type>hvm</type></os></domain>`),
}); err != nil {
t.Fatal(err)
}
backend := &reconcilePowerBackend{FakeBackend: fake, readErr: errors.New("read transport failed")}
c := config.Defaults()
c.AgentID = "host-103"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
const operationID = "00000000-0000-4000-8000-000000000027"
seedUncertainPowerOperation(t, store, operationID, testPowerUUID, hypervisor.PowerActionStart)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil))
if recorder.Code != http.StatusServiceUnavailable {
t.Fatalf("read failure status = %d, body=%s; want 503", recorder.Code, recorder.Body.String())
}
record, err := store.Get(operationID)
if err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusUncertain {
t.Fatalf("read failure status = %q, want uncertain", record.Status)
}
if backend.powerCalls != 0 || backend.readCalls != 1 {
t.Fatalf("calls after read failure: read=%d power=%d", backend.readCalls, backend.powerCalls)
}
}
func TestDomainPowerReconcileNeverConfirmsRebootFromRunningState(t *testing.T) {
store := operation.NewMemory()
fake := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
if _, err := fake.DefineDomain(context.Background(), hypervisor.DefineRequest{
UUID: testPowerUUID,
XML: []byte(` <domain><uuid>` + testPowerUUID + `</uuid><os><type>hvm</type></os></domain>`),
}); err != nil {
t.Fatal(err)
}
if _, err := fake.PowerDomain(context.Background(), testPowerUUID, hypervisor.PowerActionStart); err != nil {
t.Fatal(err)
}
backend := &reconcilePowerBackend{FakeBackend: fake}
c := config.Defaults()
c.AgentID = "host-103"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
const operationID = "00000000-0000-4000-8000-000000000028"
seedUncertainPowerOperation(t, store, operationID, testPowerUUID, hypervisor.PowerActionReboot)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil))
if recorder.Code != http.StatusConflict {
t.Fatalf("reboot reconcile status = %d, body=%s; want 409", recorder.Code, recorder.Body.String())
}
record, err := store.Get(operationID)
if err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusUncertain {
t.Fatalf("reboot reconcile status = %q, want uncertain", record.Status)
}
if backend.powerCalls != 0 || backend.readCalls != 1 {
t.Fatalf("reboot reconcile calls: read=%d power=%d; want read=1 power=0", backend.readCalls, backend.powerCalls)
}
}