You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/api/server_test.go

1089 lines
44 KiB
Go

package api
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"git.luoli.live/root/virt/agent/auth"
consolebroker "git.luoli.live/root/virt/agent/console"
"git.luoli.live/root/virt/agent/hypervisor"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/worker"
)
type testHypervisor struct {
hypervisor.Unsupported
mu sync.Mutex
inspections int
configureCalls int
inspectionResult *hypervisor.OperationInspection
inspectionErr error
}
func (h *testHypervisor) Ping(context.Context) error { return nil }
func (h *testHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) {
return []hypervisor.VMMetadata{}, nil
}
func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageConfigurationRequest) (hypervisor.StorageConfigurationResult, error) {
h.mu.Lock()
h.configureCalls++
h.mu.Unlock()
return hypervisor.StorageConfigurationResult{Configured: true}, nil
}
func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) {
h.mu.Lock()
defer h.mu.Unlock()
h.inspections++
if h.inspectionErr != nil {
return hypervisor.OperationInspection{}, h.inspectionErr
}
if h.inspectionResult != nil {
inspection := *h.inspectionResult
inspection.EffectProjection = append(json.RawMessage(nil), inspection.EffectProjection...)
return inspection, nil
}
return hypervisor.OperationInspection{
OperationID: record.OperationID,
OperationStatus: record.State,
CheckedAt: time.Now().Unix(),
Outcome: hypervisor.InspectionOutcomeFailed,
SummaryCode: hypervisor.InspectionSummaryFailed,
ErrorCode: "result_unconfirmed",
EffectResolution: hypervisor.InspectionEffectInconclusive,
FenceResolutionRequired: record.State == operation.StateFailed && record.EffectMayHaveApplied,
}, nil
}
type fakeMutationExecutor struct {
mu sync.Mutex
calls int
requests []worker.Request
result json.RawMessage
err error
}
type blockingMutationExecutor struct {
started chan worker.Request
release chan struct{}
calls atomic.Int32
}
func (e *blockingMutationExecutor) Execute(ctx context.Context, request worker.Request) (json.RawMessage, error) {
e.calls.Add(1)
select {
case e.started <- request:
case <-ctx.Done():
return nil, ctx.Err()
}
select {
case <-e.release:
return json.RawMessage(`{"configured":true}`), nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (e *fakeMutationExecutor) Execute(_ context.Context, request worker.Request) (json.RawMessage, error) {
e.mu.Lock()
defer e.mu.Unlock()
e.calls++
e.requests = append(e.requests, request)
return append(json.RawMessage(nil), e.result...), e.err
}
type testAPI struct {
handler *Server
hv *testHypervisor
journal *operation.Journal
secret []byte
now time.Time
nonce int
}
func newTestAPI(t *testing.T) *testAPI {
t.Helper()
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
secret := []byte("0123456789abcdef0123456789abcdef")
now := time.Unix(2_000_000_000, 0)
journal.SetClockForTesting(func() time.Time { return now })
authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal)
if err != nil {
t.Fatal(err)
}
authenticator.SetClockForTesting(func() time.Time { return now })
broker, err := consolebroker.NewBroker(time.Minute)
if err != nil {
t.Fatal(err)
}
hv := &testHypervisor{}
handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = handler.Shutdown(ctx)
_ = journal.Wait(ctx)
_ = journal.Close()
})
return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now}
}
func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder {
t.Helper()
a.nonce++
request := httptest.NewRequest(method, target, bytes.NewReader(body))
nonce := fmt.Sprintf("nonce%011d", a.nonce)
if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
a.handler.ServeHTTP(response, request)
return response
}
func requireOperationResponseIdentity(t *testing.T, response *httptest.ResponseRecorder,
operationID, action, resourceID, requestFingerprint string) operationResponse {
t.Helper()
var wire operationResponse
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
t.Fatalf("decode operation response %q: %v body=%s", operationID, err, response.Body.String())
}
if wire.OperationID != operationID || wire.Action != action || wire.ResourceID != resourceID ||
wire.RequestFingerprint != requestFingerprint {
t.Fatalf("operation identity = %#v, want operation=%q action=%q resource=%q fingerprint=%q",
wire, operationID, action, resourceID, requestFingerprint)
}
var raw map[string]json.RawMessage
if err := json.Unmarshal(response.Body.Bytes(), &raw); err != nil {
t.Fatal(err)
}
if _, legacy := raw["fingerprint"]; legacy {
t.Fatalf("legacy fingerprint key leaked into operation wire: %s", response.Body.String())
}
return wire
}
func TestExitRescueRequiresDataLossAcknowledgement(t *testing.T) {
api := newTestAPI(t)
operationID := "exit-rescue-without-ack"
actor := auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: operationID,
}
body := []byte(`{"operation_id":"exit-rescue-without-ack","network_suspended":false}`)
response := api.request(t, http.MethodPost,
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/rescue/exit", body, actor)
if response.Code != http.StatusUnprocessableEntity ||
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"data_loss_ack_required"`)) {
t.Fatalf("exit rescue without acknowledgement = %d %s",
response.Code, response.Body.String())
}
}
func TestOperationRecoveryMutationRoutesAreRemoved(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-terminal-inspect"
spec := operation.Spec{
OperationID: operationID,
ResourceID: "vm-inspect",
Action: "delete-vm",
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-inspect", []byte(`{}`)),
Destructive: true,
}
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return nil, &worker.Error{
Code: "result_unconfirmed", Message: "result was not confirmed",
EffectMayHaveApplied: true,
}
}); err == nil {
t.Fatal("seed failure unexpectedly succeeded")
}
actor := auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: operationID,
}
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID, nil, actor); response.Code != http.StatusOK {
t.Fatalf("operation GET = %d %s", response.Code, response.Body.String())
}
inspectionResponse := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
if inspectionResponse.Code != http.StatusOK {
t.Fatalf("read-only inspect = %d %s", inspectionResponse.Code, inspectionResponse.Body.String())
}
var inspection hypervisor.OperationInspection
if err := json.Unmarshal(inspectionResponse.Body.Bytes(), &inspection); err != nil {
t.Fatal(err)
}
if inspection.Action != spec.Action || inspection.ResourceID != spec.ResourceID ||
inspection.RequestFingerprint != spec.Fingerprint {
t.Fatalf("inspection identity = %#v, want action=%q resource=%q fingerprint=%q",
inspection, spec.Action, spec.ResourceID, spec.Fingerprint)
}
for _, suffix := range []string{"reconcile", "confirm-inspection", "abort-job", "cleanup"} {
response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/"+suffix, []byte(`{}`), actor)
if response.Code != http.StatusNotFound {
t.Fatalf("removed %s route = %d %s", suffix, response.Code, response.Body.String())
}
}
api.hv.mu.Lock()
inspections := api.hv.inspections
api.hv.mu.Unlock()
if inspections != 1 {
t.Fatalf("removed mutation routes reached hypervisor; inspections=%d", inspections)
}
}
func TestOperationInspectionReportsCompletionDeliveryQuarantine(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-delivery-quarantine"
spec := operation.Spec{
OperationID: operationID,
ResourceID: "vm-delivery-quarantine",
Action: "delete-vm",
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-delivery-quarantine", []byte(`{}`)),
Destructive: true,
}
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"nested":{"root_password":"must-not-be-delivered"}}`), nil
}); err != nil {
t.Fatal(err)
}
if claimed, err := api.journal.ClaimPendingCompletions(1); err != nil || len(claimed) != 0 {
t.Fatalf("sensitive completion was not quarantined: %#v err=%v", claimed, err)
}
actor := auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: operationID,
}
response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
if response.Code != http.StatusOK {
t.Fatalf("quarantined inspection = %d %s", response.Code, response.Body.String())
}
var inspection hypervisor.OperationInspection
if err := json.Unmarshal(response.Body.Bytes(), &inspection); err != nil {
t.Fatal(err)
}
if inspection.OperationStatus != operation.StateSucceeded ||
inspection.ErrorCode != "" ||
inspection.Outcome != hypervisor.InspectionOutcomeUncertain ||
inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete ||
inspection.EffectResolution != hypervisor.InspectionEffectInconclusive ||
inspection.CompletionDeliveryState != operation.CompletionDeliveryStateQuarantined ||
inspection.CompletionDeliveryErrorCode != operation.CompletionDeliveryErrorReceiptInvalid ||
len(inspection.EffectProjection) != 0 || inspection.AbortJob.Available || inspection.Cleanup.Available {
t.Fatalf("quarantined completion inspection = %#v", inspection)
}
api.hv.mu.Lock()
inspections := api.hv.inspections
api.hv.mu.Unlock()
if inspections != 0 {
t.Fatalf("delivery quarantine reporting depended on %d live host inspections", inspections)
}
stored, err := api.journal.Get(operationID)
if err != nil || stored.State != operation.StateSucceeded || stored.Error != nil {
t.Fatalf("delivery quarantine changed operation status: %#v err=%v", stored, err)
}
}
func TestSuperAdminFenceResolutionIsJournalOnlyAndReadBackIsStable(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-fence-resolution"
spec := operation.Spec{
OperationID: operationID,
ResourceID: "vm-fence-resolution",
Action: "delete-vm",
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-resolution", []byte(`{}`)),
Destructive: true,
}
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return nil, &worker.Error{
Code: "result_unconfirmed", Message: "result was not confirmed",
EffectMayHaveApplied: true, ResubmitAllowed: false,
}
}); err == nil {
t.Fatal("seed effectful failure unexpectedly succeeded")
}
actor := auth.Actor{
UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: operationID,
}
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound ||
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) {
t.Fatalf("missing fence resolution = %d %s", response.Code, response.Body.String())
}
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK {
t.Fatalf("fresh inspection = %d %s", response.Code, response.Body.String())
}
request := operation.FenceResolutionRequest{
ResolutionID: "0123456789abcdef0123456789abcdef",
Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(),
ReasonSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint,
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
missingIdentity := request
missingIdentity.RequestFingerprint = ""
missingIdentityBody, err := json.Marshal(missingIdentity)
if err != nil {
t.Fatal(err)
}
invalidIdentity := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", missingIdentityBody, actor)
if invalidIdentity.Code != http.StatusBadRequest ||
!bytes.Contains(invalidIdentity.Body.Bytes(), []byte(`"code":"invalid_fence_resolution"`)) {
t.Fatalf("missing fence identity = %d %s", invalidIdentity.Code, invalidIdentity.Body.String())
}
mismatch := request
mismatch.ResourceID = "vm-other-fence-resolution"
mismatchBody, err := json.Marshal(mismatch)
if err != nil {
t.Fatal(err)
}
identityConflict := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", mismatchBody, actor)
if identityConflict.Code != http.StatusConflict ||
!bytes.Contains(identityConflict.Body.Bytes(), []byte(`"code":"operation_identity_conflict"`)) {
t.Fatalf("fence identity conflict = %d %s", identityConflict.Code, identityConflict.Body.String())
}
resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"replayed":false`)) {
t.Fatalf("resolve fence = %d %s", resolved.Code, resolved.Body.String())
}
readBack := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor)
if readBack.Code != http.StatusOK ||
!bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution_id":"0123456789abcdef0123456789abcdef"`)) ||
!bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution":"not_applied"`)) {
t.Fatalf("read fence resolution = %d %s", readBack.Code, readBack.Body.String())
}
replayed := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
if replayed.Code != http.StatusOK || !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) {
t.Fatalf("idempotent fence resolution = %d %s", replayed.Code, replayed.Body.String())
}
api.hv.mu.Lock()
inspections := api.hv.inspections
configureCalls := api.hv.configureCalls
api.hv.mu.Unlock()
if inspections != 1 || configureCalls != 0 {
t.Fatalf("fence resolution invoked host work: inspections=%d mutations=%d", inspections, configureCalls)
}
}
func TestFenceResolutionRequiresExactActorOperationIdentity(t *testing.T) {
api := newTestAPI(t)
request := operation.FenceResolutionRequest{
ResolutionID: "fedcba9876543210fedcba9876543210",
Resolution: operation.FenceResolutionApplied, InspectionCheckedAt: api.now.Unix(),
ReasonSHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
Action: "delete-vm", ResourceID: "vm-fence-identity",
RequestFingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-identity", []byte(`{}`)),
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
response := api.request(t, http.MethodPost, "/v1/operations/operation-fence-identity/resolve-fence", body, auth.Actor{
UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: "operation-other-identity",
})
if response.Code != http.StatusBadRequest ||
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"operation_id_mismatch"`)) {
t.Fatalf("identity mismatch = %d %s", response.Code, response.Body.String())
}
}
func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-system-recovery"
spec := operation.Spec{
OperationID: operationID,
ResourceID: "vm-system-recovery",
Action: "update-settings",
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms/vm-system-recovery/settings", []byte(`{}`)),
Destructive: true,
}
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return nil, &worker.Error{
Code: "result_unconfirmed", Message: "result was not confirmed",
EffectMayHaveApplied: true,
}
}); err == nil {
t.Fatal("seed effectful failure unexpectedly succeeded")
}
actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID}
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK ||
!bytes.Contains(response.Body.Bytes(), []byte(`"fence_resolution_required":true`)) ||
!bytes.Contains(response.Body.Bytes(), []byte(`"fence_state":"active"`)) {
t.Fatalf("system inspect = %d %s", response.Code, response.Body.String())
}
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound ||
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) {
t.Fatalf("system unresolved fence read = %d %s", response.Code, response.Body.String())
}
request := operation.FenceResolutionRequest{
ResolutionID: "98769876987698769876987698769876",
Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(),
ReasonSHA256: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint,
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) {
t.Fatalf("system fence resolution = %d %s", resolved.Code, resolved.Body.String())
}
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusOK {
t.Fatalf("system resolved fence read = %d %s", response.Code, response.Body.String())
}
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK ||
!bytes.Contains(response.Body.Bytes(), []byte(`"fence_resolution_required":false`)) ||
!bytes.Contains(response.Body.Bytes(), []byte(`"fence_state":"resolved"`)) {
t.Fatalf("resolved system inspect = %d %s", response.Code, response.Body.String())
}
for _, denied := range []struct {
name, operationID, resourceID, action string
}{
{"unknown", "operation-system-unknown-denied", "vm-system-unknown-denied", "future-unknown"},
} {
t.Run("reject "+denied.name+" operation", func(t *testing.T) {
deniedSpec := operation.Spec{
OperationID: denied.operationID,
ResourceID: denied.resourceID,
Action: denied.action,
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/test/"+denied.resourceID, []byte(`{}`)),
Destructive: true,
}
if _, _, err := api.journal.Execute(context.Background(), deniedSpec, func(context.Context) (json.RawMessage, error) {
return nil, &worker.Error{Code: "result_unconfirmed", Message: "effect may have applied", EffectMayHaveApplied: true}
}); err == nil {
t.Fatal("seed disallowed recovery failure unexpectedly succeeded")
}
deniedActor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: denied.operationID}
for _, route := range []struct {
method, suffix string
body []byte
}{
{http.MethodGet, "/inspect", nil},
{http.MethodGet, "/fence-resolution", nil},
{http.MethodPost, "/resolve-fence", body},
} {
response := api.request(t, route.method, "/v1/operations/"+denied.operationID+route.suffix, route.body, deniedActor)
if response.Code != http.StatusForbidden {
t.Fatalf("system recovery %s %s = %d %s", route.method, route.suffix, response.Code, response.Body.String())
}
}
var successorCalls atomic.Int32
successor := operation.Spec{
OperationID: denied.operationID + "-successor",
ResourceID: denied.resourceID,
Action: "delete-vm",
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/"+denied.resourceID, []byte(`{}`)),
Destructive: true,
}
if _, _, err := api.journal.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
successorCalls.Add(1)
return json.RawMessage(`{"deleted":true}`), nil
}); !errors.Is(err, operation.ErrResourceBlocked) || successorCalls.Load() != 0 {
t.Fatalf("denied system recovery released blocker: calls=%d err=%v", successorCalls.Load(), err)
}
})
}
reviewedActions := []string{
"power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy",
"reinstall-vm",
"enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended",
"change-network", "network-suspend", "network-resume",
"apply-firewall", "delete-vm", "force-delete-vm",
"service-suspend",
"create-snapshot:0123456789abcdef0123456789abcdef:filesystem",
"create-snapshot:0123456789abcdef0123456789abcdef:crash",
"delete-snapshot:0123456789abcdef0123456789abcdef",
"restore-snapshot:0123456789abcdef0123456789abcdef",
"import-template", "update-template", "delete-template",
"import-rescue-template", "delete-rescue-template",
"import-user-iso", "delete-user-iso",
}
for index, action := range reviewedActions {
t.Run("reviewed "+action, func(t *testing.T) {
allowedOperationID := fmt.Sprintf("operation-system-reviewed-%02d", index)
allowedResourceID := fmt.Sprintf("resource-system-reviewed-%02d", index)
allowedSpec := operation.Spec{
OperationID: allowedOperationID,
ResourceID: allowedResourceID,
Action: action,
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/test/"+allowedResourceID, []byte(`{}`)),
Destructive: true,
}
if _, _, err := api.journal.Execute(context.Background(), allowedSpec, func(context.Context) (json.RawMessage, error) {
return nil, &worker.Error{Code: "result_unconfirmed", Message: "effect may have applied", EffectMayHaveApplied: true}
}); err == nil {
t.Fatal("seed reviewed recovery failure unexpectedly succeeded")
}
allowedActor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: allowedOperationID}
if response := api.request(t, http.MethodGet, "/v1/operations/"+allowedOperationID+"/inspect", nil, allowedActor); response.Code != http.StatusOK {
t.Fatalf("reviewed system inspect = %d %s", response.Code, response.Body.String())
}
allowedRequest := operation.FenceResolutionRequest{
ResolutionID: fmt.Sprintf("%032x", index+1),
Resolution: operation.FenceResolutionNotApplied,
InspectionCheckedAt: api.now.Unix(),
ReasonSHA256: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
Action: allowedSpec.Action,
ResourceID: allowedSpec.ResourceID,
RequestFingerprint: allowedSpec.Fingerprint,
}
allowedBody, err := json.Marshal(allowedRequest)
if err != nil {
t.Fatal(err)
}
if response := api.request(t, http.MethodPost, "/v1/operations/"+allowedOperationID+"/resolve-fence", allowedBody, allowedActor); response.Code != http.StatusOK {
t.Fatalf("reviewed system resolution = %d %s", response.Code, response.Body.String())
}
if response := api.request(t, http.MethodGet, "/v1/operations/"+allowedOperationID+"/fence-resolution", nil, allowedActor); response.Code != http.StatusOK {
t.Fatalf("reviewed system fence read = %d %s", response.Code, response.Body.String())
}
})
}
for _, test := range []struct {
name, method, target string
body []byte
}{
{"health", http.MethodGet, "/v1/health", nil},
{"raw operation", http.MethodGet, "/v1/operations/" + operationID, nil},
{"other operation", http.MethodGet, "/v1/operations/operation-other-recovery/inspect", nil},
{"host mutation", http.MethodPost, "/v1/storage/configuration", []byte(`{"operation_id":"operation-system-recovery"}`)},
{"create mutation", http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-recovery"}`)},
} {
t.Run(test.name, func(t *testing.T) {
response := api.request(t, test.method, test.target, test.body, actor)
if response.Code != http.StatusForbidden {
t.Fatalf("least-privilege route = %d %s", response.Code, response.Body.String())
}
})
}
manualZero := auth.Actor{
UserID: 0, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID,
}
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, manualZero); response.Code != http.StatusForbidden {
t.Fatalf("zero-id human inspection = %d %s", response.Code, response.Body.String())
}
api.hv.mu.Lock()
inspections := api.hv.inspections
configureCalls := api.hv.configureCalls
api.hv.mu.Unlock()
if inspections != 2+len(reviewedActions) || configureCalls != 0 {
t.Fatalf("system recovery escaped read-only scope: inspections=%d mutations=%d", inspections, configureCalls)
}
}
func TestSystemCreateRecoveryRequiresFreshCompleteAbsenceEvidence(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-system-create-recovery"
resourceID := "123e4567-e89b-42d3-a456-426614174000"
requestBody := []byte(`{"operation_id":"operation-system-create-recovery"}`)
spec := operation.Spec{
OperationID: operationID,
ResourceID: resourceID,
Action: "create-vm",
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", requestBody),
Destructive: true,
}
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return nil, &worker.Error{
Code: "result_unconfirmed", Message: "create result was not confirmed",
EffectMayHaveApplied: true,
}
}); err == nil {
t.Fatal("seed create failure unexpectedly succeeded")
}
actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID}
request := operation.FenceResolutionRequest{
ResolutionID: "40404040404040404040404040404040",
Resolution: operation.FenceResolutionApplied,
InspectionCheckedAt: api.now.Unix(),
ReasonSHA256: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd",
Action: spec.Action,
ResourceID: spec.ResourceID,
RequestFingerprint: spec.Fingerprint,
}
encodeRequest := func() []byte {
t.Helper()
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
return body
}
assertActive := func() {
t.Helper()
state, err := api.journal.FenceState(operationID)
if err != nil || state != operation.FenceStateActive {
t.Fatalf("create fence state=%q err=%v", state, err)
}
}
applied := api.request(
t,
http.MethodPost,
"/v1/operations/"+operationID+"/resolve-fence",
encodeRequest(),
actor,
)
if applied.Code != http.StatusConflict {
t.Fatalf("system create applied resolution = %d %s", applied.Code, applied.Body.String())
}
assertActive()
request.Resolution = operation.FenceResolutionNotApplied
inconclusive := api.request(
t,
http.MethodPost,
"/v1/operations/"+operationID+"/resolve-fence",
encodeRequest(),
actor,
)
if inconclusive.Code != http.StatusConflict ||
!bytes.Contains(inconclusive.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
t.Fatalf("inconclusive create resolution = %d %s", inconclusive.Code, inconclusive.Body.String())
}
assertActive()
exactInspection := func(checkedAt int64) hypervisor.OperationInspection {
return hypervisor.OperationInspection{
OperationID: operationID,
Action: spec.Action,
ResourceID: resourceID,
RequestFingerprint: spec.Fingerprint,
OperationStatus: operation.StateFailed,
CheckedAt: checkedAt,
Outcome: hypervisor.InspectionOutcomeFailed,
SummaryCode: hypervisor.InspectionSummaryFailed,
ErrorCode: "creation_absent",
EffectResolution: hypervisor.InspectionEffectNotApplied,
EffectProjection: json.RawMessage(
`{"creation_absent":true,"managed_artifacts_absent":true}`,
),
}
}
stale := exactInspection(api.now.Add(-time.Second).Unix())
api.hv.mu.Lock()
api.hv.inspectionResult = &stale
api.hv.mu.Unlock()
staleResponse := api.request(
t,
http.MethodPost,
"/v1/operations/"+operationID+"/resolve-fence",
encodeRequest(),
actor,
)
if staleResponse.Code != http.StatusConflict ||
!bytes.Contains(staleResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
t.Fatalf("stale create resolution = %d %s", staleResponse.Code, staleResponse.Body.String())
}
assertActive()
incomplete := exactInspection(api.now.Unix())
incomplete.EffectProjection = json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":false}`)
api.hv.mu.Lock()
api.hv.inspectionResult = &incomplete
api.hv.mu.Unlock()
incompleteResponse := api.request(
t,
http.MethodPost,
"/v1/operations/"+operationID+"/resolve-fence",
encodeRequest(),
actor,
)
if incompleteResponse.Code != http.StatusConflict ||
!bytes.Contains(incompleteResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
t.Fatalf("incomplete create absence = %d %s", incompleteResponse.Code, incompleteResponse.Body.String())
}
assertActive()
exact := exactInspection(api.now.Unix())
api.hv.mu.Lock()
api.hv.inspectionResult = &exact
api.hv.mu.Unlock()
inspection := api.request(
t,
http.MethodGet,
"/v1/operations/"+operationID+"/inspect",
nil,
actor,
)
if inspection.Code != http.StatusOK ||
!bytes.Contains(inspection.Body.Bytes(), []byte(`"error_code":"creation_absent"`)) ||
!bytes.Contains(inspection.Body.Bytes(), []byte(`"effect_resolution":"not_applied"`)) {
t.Fatalf("system create inspection = %d %s", inspection.Code, inspection.Body.String())
}
resolved := api.request(
t,
http.MethodPost,
"/v1/operations/"+operationID+"/resolve-fence",
encodeRequest(),
actor,
)
if resolved.Code != http.StatusOK ||
!bytes.Contains(resolved.Body.Bytes(), []byte(`"resolution":"not_applied"`)) ||
!bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) {
t.Fatalf("system create not-applied resolution = %d %s", resolved.Code, resolved.Body.String())
}
replayed := api.request(
t,
http.MethodPost,
"/v1/operations/"+operationID+"/resolve-fence",
encodeRequest(),
actor,
)
if replayed.Code != http.StatusOK ||
!bytes.Contains(replayed.Body.Bytes(), []byte(`"resolution":"not_applied"`)) ||
!bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) {
t.Fatalf("system create resolution replay = %d %s", replayed.Code, replayed.Body.String())
}
api.hv.mu.Lock()
inspections := api.hv.inspections
configureCalls := api.hv.configureCalls
api.hv.mu.Unlock()
if inspections != 5 || configureCalls != 0 {
t.Fatalf("system create recovery inspections=%d mutations=%d", inspections, configureCalls)
}
}
func TestSystemCreateRecoveryRejectsExecutionFencedAbsence(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-system-create-unreaped"
resourceID := "123e4567-e89b-42d3-a456-426614174002"
spec := operation.Spec{
OperationID: operationID,
ResourceID: resourceID,
Action: "create-vm",
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-create-unreaped"}`)),
Destructive: true,
}
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return nil, &worker.Error{
Code: "worker_reap_unconfirmed", Message: "worker may still be running",
EffectMayHaveApplied: true, ExecutionFenced: true,
}
}); err == nil {
t.Fatal("seed execution-fenced create unexpectedly succeeded")
}
exactAbsence := hypervisor.OperationInspection{
OperationID: operationID,
Action: spec.Action,
ResourceID: resourceID,
RequestFingerprint: spec.Fingerprint,
OperationStatus: operation.StateFailed,
CheckedAt: api.now.Unix(),
Outcome: hypervisor.InspectionOutcomeFailed,
SummaryCode: hypervisor.InspectionSummaryFailed,
ErrorCode: "creation_absent",
EffectResolution: hypervisor.InspectionEffectNotApplied,
EffectProjection: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`),
}
api.hv.mu.Lock()
api.hv.inspectionResult = &exactAbsence
api.hv.mu.Unlock()
actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID}
inspected := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
if inspected.Code != http.StatusOK {
t.Fatalf("execution-fenced create inspection = %d %s", inspected.Code, inspected.Body.String())
}
var inspection hypervisor.OperationInspection
if err := json.Unmarshal(inspected.Body.Bytes(), &inspection); err != nil {
t.Fatal(err)
}
if inspection.Outcome != hypervisor.InspectionOutcomeUncertain ||
inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete ||
inspection.EffectResolution != hypervisor.InspectionEffectInconclusive ||
len(inspection.EffectProjection) != 0 {
t.Fatalf("execution-fenced create absence was treated as final: %#v", inspection)
}
request := operation.FenceResolutionRequest{
ResolutionID: "50505050505050505050505050505050",
Resolution: operation.FenceResolutionNotApplied,
InspectionCheckedAt: api.now.Unix(),
ReasonSHA256: "dededededededededededededededededededededededededededededededede",
Action: spec.Action,
ResourceID: spec.ResourceID,
RequestFingerprint: spec.Fingerprint,
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
if resolved.Code != http.StatusConflict ||
!bytes.Contains(resolved.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
t.Fatalf("execution-fenced create resolution = %d %s", resolved.Code, resolved.Body.String())
}
if state, err := api.journal.FenceState(operationID); err != nil || state != operation.FenceStateActive {
t.Fatalf("execution-fenced create was released state=%q err=%v", state, err)
}
api.hv.mu.Lock()
inspections := api.hv.inspections
api.hv.mu.Unlock()
if inspections != 1 {
t.Fatalf("execution-fenced create resolve repeated host inspection: %d", inspections)
}
}
func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) {
api := newTestAPI(t)
executor := &fakeMutationExecutor{result: json.RawMessage(`{"configured":true}`)}
if err := api.handler.SetMutationExecutor(executor); err != nil {
t.Fatal(err)
}
request := hypervisor.StorageConfigurationRequest{
OperationID: "operation-isolated-storage",
ManagedDirectory: "/srv/virt/managed",
TemplateDirectory: "/srv/virt/templates",
SeedDirectory: "/srv/virt/seeds",
UserISODirectory: "/srv/virt/user-isos",
RescueDirectory: "/srv/virt/rescue-isos",
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
actor := auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: request.OperationID,
}
first := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
if first.Code != http.StatusAccepted {
t.Fatalf("first isolated response = %d %s", first.Code, first.Body.String())
}
fingerprint := operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", body)
accepted := requireOperationResponseIdentity(t, first, request.OperationID, "configure-storage", "host-storage", fingerprint)
if accepted.Status != operation.StateDispatching || accepted.Replayed {
t.Fatalf("accepted operation response = %#v", accepted)
}
terminal := waitForTerminal(t, api.journal, request.OperationID)
if terminal.State != operation.StateSucceeded || string(terminal.Result) != `{"configured":true}` {
t.Fatalf("terminal operation = %#v", terminal)
}
replay := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
if replay.Code != http.StatusOK {
t.Fatalf("terminal replay = %d %s", replay.Code, replay.Body.String())
}
replayed := requireOperationResponseIdentity(t, replay, request.OperationID, "configure-storage", "host-storage", fingerprint)
if replayed.Status != operation.StateSucceeded || !replayed.Replayed {
t.Fatalf("replayed operation response = %#v", replayed)
}
executor.mu.Lock()
defer executor.mu.Unlock()
if executor.calls != 1 || len(executor.requests) != 1 {
t.Fatalf("executor calls=%d requests=%d", executor.calls, len(executor.requests))
}
workerRequest := executor.requests[0]
if workerRequest.OperationID != request.OperationID || workerRequest.Actor.OperationID != request.OperationID ||
workerRequest.Method != http.MethodPost || workerRequest.RequestTarget != "/v1/storage/configuration" {
t.Fatalf("worker envelope = %#v", workerRequest)
}
}
func TestIsolatedMutationDeadlineCapsOnlyStorageConfiguration(t *testing.T) {
server := &Server{OperationTimeout: 840 * time.Second}
if got := server.isolatedMutationDeadline("configure-storage"); got != 120*time.Second {
t.Fatalf("storage helper deadline = %s, want 120s", got)
}
for _, action := range []string{"create-vm", "delete-vm", "change-network"} {
if got := server.isolatedMutationDeadline(action); got != 840*time.Second {
t.Fatalf("%s helper deadline = %s, want global 840s", action, got)
}
}
server.OperationTimeout = 30 * time.Second
if got := server.isolatedMutationDeadline("configure-storage"); got != 30*time.Second {
t.Fatalf("lower global deadline was widened to %s", got)
}
}
func TestIsolatedMutationAdmissionIsShortAndDoesNotStartSecondWorker(t *testing.T) {
api := newTestAPI(t)
// Give the first request enough time to persist its independent Bolt
// journal under a loaded test host. The short timeout is the behavior under
// test only after that first worker demonstrably owns admission.
api.handler.AdmissionTimeout = time.Second
executor := &blockingMutationExecutor{
started: make(chan worker.Request, 1), release: make(chan struct{}),
}
if err := api.handler.SetMutationExecutor(executor); err != nil {
t.Fatal(err)
}
requestBody := func(operationID string) []byte {
body, err := json.Marshal(hypervisor.StorageConfigurationRequest{
OperationID: operationID, ManagedDirectory: "/srv/virt/managed",
TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds",
UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos",
})
if err != nil {
t.Fatal(err)
}
return body
}
firstID := "operation-admission-first"
firstBody := requestBody(firstID)
first := api.request(t, http.MethodPost, "/v1/storage/configuration", firstBody, auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: firstID,
})
if first.Code != http.StatusAccepted {
t.Fatalf("first response = %d %s", first.Code, first.Body.String())
}
firstWire := requireOperationResponseIdentity(t, first, firstID, "configure-storage", "host-storage",
operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", firstBody))
if firstWire.Status != operation.StateDispatching || firstWire.Replayed {
t.Fatalf("first accepted operation response = %#v", firstWire)
}
select {
case <-executor.started:
case <-time.After(time.Second):
t.Fatal("first worker did not start")
}
api.handler.AdmissionTimeout = 30 * time.Millisecond
secondID := "operation-admission-second"
secondBody := requestBody(secondID)
startedAt := time.Now()
second := api.request(t, http.MethodPost, "/v1/storage/configuration", secondBody, auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: secondID,
})
if elapsed := time.Since(startedAt); elapsed > 500*time.Millisecond {
t.Fatalf("saturated admission waited %s", elapsed)
}
if second.Code != http.StatusOK ||
!bytes.Contains(second.Body.Bytes(), []byte(`"status":"failed"`)) ||
!bytes.Contains(second.Body.Bytes(), []byte(`"code":"operation_not_dispatched"`)) {
t.Fatalf("saturated admission response = %d %s", second.Code, second.Body.String())
}
failedWire := requireOperationResponseIdentity(t, second, secondID, "configure-storage", "host-storage",
operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", secondBody))
if failedWire.Status != operation.StateFailed || failedWire.Replayed {
t.Fatalf("failed operation response = %#v", failedWire)
}
if executor.calls.Load() != 1 {
t.Fatalf("saturated request started a second worker; calls=%d", executor.calls.Load())
}
close(executor.release)
if terminal := waitForTerminal(t, api.journal, firstID); terminal.State != operation.StateSucceeded {
t.Fatalf("first terminal = %#v", terminal)
}
}
func TestParentMutationNeverFallsBackWhenExecutorIsMissing(t *testing.T) {
api := newTestAPI(t)
request := hypervisor.StorageConfigurationRequest{
OperationID: "operation-worker-missing",
ManagedDirectory: "/srv/virt/managed",
TemplateDirectory: "/srv/virt/templates",
SeedDirectory: "/srv/virt/seeds",
UserISODirectory: "/srv/virt/user-isos",
RescueDirectory: "/srv/virt/rescue-isos",
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
response := api.request(t, http.MethodPost, "/v1/storage/configuration", body, auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: request.OperationID,
})
if response.Code != http.StatusServiceUnavailable ||
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"mutation_worker_unavailable"`)) ||
!bytes.Contains(response.Body.Bytes(), []byte(`"dispatch_state":"not_dispatched"`)) {
t.Fatalf("missing worker response = %d %s", response.Code, response.Body.String())
}
if _, err := api.journal.Get(request.OperationID); !errors.Is(err, operation.ErrNotFound) {
t.Fatalf("missing worker created journal state: %v", err)
}
api.hv.mu.Lock()
configureCalls := api.hv.configureCalls
api.hv.mu.Unlock()
if configureCalls != 0 {
t.Fatalf("parent invoked the in-process hypervisor %d times", configureCalls)
}
}
func TestTrustedWorkerExecutesLocallyWithoutRecursiveSpawn(t *testing.T) {
api := newTestAPI(t)
request := hypervisor.StorageConfigurationRequest{
OperationID: "operation-trusted-worker",
ManagedDirectory: "/srv/virt/managed",
TemplateDirectory: "/srv/virt/templates",
SeedDirectory: "/srv/virt/seeds",
UserISODirectory: "/srv/virt/user-isos",
RescueDirectory: "/srv/virt/rescue-isos",
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
httpRequest := httptest.NewRequest(http.MethodPost, "/v1/storage/configuration", bytes.NewReader(body))
response := httptest.NewRecorder()
api.handler.ServeTrustedWorkerRequest(response, httpRequest, body, auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: request.OperationID,
})
if response.Code != http.StatusOK {
t.Fatalf("trusted worker response = %d %s", response.Code, response.Body.String())
}
succeeded := requireOperationResponseIdentity(t, response, request.OperationID, "configure-storage", "host-storage",
operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", body))
if succeeded.Status != operation.StateSucceeded || succeeded.Replayed {
t.Fatalf("successful operation response = %#v", succeeded)
}
api.hv.mu.Lock()
configureCalls := api.hv.configureCalls
api.hv.mu.Unlock()
if configureCalls != 1 {
t.Fatalf("trusted worker local mutation calls=%d", configureCalls)
}
}
func TestListEnvelopeRemainsNonNull(t *testing.T) {
api := newTestAPI(t)
response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1})
if response.Code != http.StatusOK || response.Body.String() != "{\"vms\":[]}\n" {
t.Fatalf("VM list = %d %s", response.Code, response.Body.String())
}
}
func waitForTerminal(t *testing.T, journal *operation.Journal, operationID string) operation.Record {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
record, err := journal.Get(operationID)
if err == nil && (record.State == operation.StateSucceeded || record.State == operation.StateFailed) {
return record
}
time.Sleep(time.Millisecond)
}
t.Fatalf("operation %s did not become terminal", operationID)
return operation.Record{}
}