package api import ( "bytes" "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "path/filepath" "sync" "sync/atomic" "testing" "time" "git.luoli.live/root/virt/agent/auth" consolebroker "git.luoli.live/root/virt/agent/console" "git.luoli.live/root/virt/agent/hypervisor" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/worker" ) type testHypervisor struct { hypervisor.Unsupported mu sync.Mutex inspections int configureCalls int inspectionResult *hypervisor.OperationInspection inspectionErr error } func (h *testHypervisor) Ping(context.Context) error { return nil } func (h *testHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) { return []hypervisor.VMMetadata{}, nil } func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageConfigurationRequest) (hypervisor.StorageConfigurationResult, error) { h.mu.Lock() h.configureCalls++ h.mu.Unlock() return hypervisor.StorageConfigurationResult{Configured: true}, nil } func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) { h.mu.Lock() defer h.mu.Unlock() h.inspections++ if h.inspectionErr != nil { return hypervisor.OperationInspection{}, h.inspectionErr } if h.inspectionResult != nil { inspection := *h.inspectionResult inspection.EffectProjection = append(json.RawMessage(nil), inspection.EffectProjection...) return inspection, nil } return hypervisor.OperationInspection{ OperationID: record.OperationID, OperationStatus: record.State, CheckedAt: time.Now().Unix(), Outcome: hypervisor.InspectionOutcomeFailed, SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "result_unconfirmed", EffectResolution: hypervisor.InspectionEffectInconclusive, FenceResolutionRequired: record.State == operation.StateFailed && record.EffectMayHaveApplied, }, nil } type fakeMutationExecutor struct { mu sync.Mutex calls int requests []worker.Request result json.RawMessage err error } type blockingMutationExecutor struct { started chan worker.Request release chan struct{} calls atomic.Int32 } func (e *blockingMutationExecutor) Execute(ctx context.Context, request worker.Request) (json.RawMessage, error) { e.calls.Add(1) select { case e.started <- request: case <-ctx.Done(): return nil, ctx.Err() } select { case <-e.release: return json.RawMessage(`{"configured":true}`), nil case <-ctx.Done(): return nil, ctx.Err() } } func (e *fakeMutationExecutor) Execute(_ context.Context, request worker.Request) (json.RawMessage, error) { e.mu.Lock() defer e.mu.Unlock() e.calls++ e.requests = append(e.requests, request) return append(json.RawMessage(nil), e.result...), e.err } type testAPI struct { handler *Server hv *testHypervisor journal *operation.Journal secret []byte now time.Time nonce int } func newTestAPI(t *testing.T) *testAPI { t.Helper() journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db")) if err != nil { t.Fatal(err) } secret := []byte("0123456789abcdef0123456789abcdef") now := time.Unix(2_000_000_000, 0) journal.SetClockForTesting(func() time.Time { return now }) authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal) if err != nil { t.Fatal(err) } authenticator.SetClockForTesting(func() time.Time { return now }) broker, err := consolebroker.NewBroker(time.Minute) if err != nil { t.Fatal(err) } hv := &testHypervisor{} handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second) if err != nil { t.Fatal(err) } t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() _ = handler.Shutdown(ctx) _ = journal.Wait(ctx) _ = journal.Close() }) return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now} } func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder { t.Helper() a.nonce++ request := httptest.NewRequest(method, target, bytes.NewReader(body)) nonce := fmt.Sprintf("nonce%011d", a.nonce) if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil { t.Fatal(err) } response := httptest.NewRecorder() a.handler.ServeHTTP(response, request) return response } func requireOperationResponseIdentity(t *testing.T, response *httptest.ResponseRecorder, operationID, action, resourceID, requestFingerprint string) operationResponse { t.Helper() var wire operationResponse if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil { t.Fatalf("decode operation response %q: %v body=%s", operationID, err, response.Body.String()) } if wire.OperationID != operationID || wire.Action != action || wire.ResourceID != resourceID || wire.RequestFingerprint != requestFingerprint { t.Fatalf("operation identity = %#v, want operation=%q action=%q resource=%q fingerprint=%q", wire, operationID, action, resourceID, requestFingerprint) } var raw map[string]json.RawMessage if err := json.Unmarshal(response.Body.Bytes(), &raw); err != nil { t.Fatal(err) } if _, legacy := raw["fingerprint"]; legacy { t.Fatalf("legacy fingerprint key leaked into operation wire: %s", response.Body.String()) } return wire } func TestOperationRecoveryMutationRoutesAreRemoved(t *testing.T) { api := newTestAPI(t) operationID := "operation-terminal-inspect" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-inspect", Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-inspect", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "result was not confirmed", EffectMayHaveApplied: true, } }); err == nil { t.Fatal("seed failure unexpectedly succeeded") } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID, nil, actor); response.Code != http.StatusOK { t.Fatalf("operation GET = %d %s", response.Code, response.Body.String()) } inspectionResponse := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor) if inspectionResponse.Code != http.StatusOK { t.Fatalf("read-only inspect = %d %s", inspectionResponse.Code, inspectionResponse.Body.String()) } var inspection hypervisor.OperationInspection if err := json.Unmarshal(inspectionResponse.Body.Bytes(), &inspection); err != nil { t.Fatal(err) } if inspection.Action != spec.Action || inspection.ResourceID != spec.ResourceID || inspection.RequestFingerprint != spec.Fingerprint { t.Fatalf("inspection identity = %#v, want action=%q resource=%q fingerprint=%q", inspection, spec.Action, spec.ResourceID, spec.Fingerprint) } for _, suffix := range []string{"reconcile", "confirm-inspection", "abort-job", "cleanup"} { response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/"+suffix, []byte(`{}`), actor) if response.Code != http.StatusNotFound { t.Fatalf("removed %s route = %d %s", suffix, response.Code, response.Body.String()) } } api.hv.mu.Lock() inspections := api.hv.inspections api.hv.mu.Unlock() if inspections != 1 { t.Fatalf("removed mutation routes reached hypervisor; inspections=%d", inspections) } } func TestOperationInspectionReportsCompletionDeliveryQuarantine(t *testing.T) { api := newTestAPI(t) operationID := "operation-delivery-quarantine" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-delivery-quarantine", Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-delivery-quarantine", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return json.RawMessage(`{"nested":{"root_password":"must-not-be-delivered"}}`), nil }); err != nil { t.Fatal(err) } if claimed, err := api.journal.ClaimPendingCompletions(1); err != nil || len(claimed) != 0 { t.Fatalf("sensitive completion was not quarantined: %#v err=%v", claimed, err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor) if response.Code != http.StatusOK { t.Fatalf("quarantined inspection = %d %s", response.Code, response.Body.String()) } var inspection hypervisor.OperationInspection if err := json.Unmarshal(response.Body.Bytes(), &inspection); err != nil { t.Fatal(err) } if inspection.OperationStatus != operation.StateSucceeded || inspection.ErrorCode != "" || inspection.Outcome != hypervisor.InspectionOutcomeUncertain || inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete || inspection.EffectResolution != hypervisor.InspectionEffectInconclusive || inspection.CompletionDeliveryState != operation.CompletionDeliveryStateQuarantined || inspection.CompletionDeliveryErrorCode != operation.CompletionDeliveryErrorReceiptInvalid || len(inspection.EffectProjection) != 0 || inspection.AbortJob.Available || inspection.Cleanup.Available { t.Fatalf("quarantined completion inspection = %#v", inspection) } api.hv.mu.Lock() inspections := api.hv.inspections api.hv.mu.Unlock() if inspections != 0 { t.Fatalf("delivery quarantine reporting depended on %d live host inspections", inspections) } stored, err := api.journal.Get(operationID) if err != nil || stored.State != operation.StateSucceeded || stored.Error != nil { t.Fatalf("delivery quarantine changed operation status: %#v err=%v", stored, err) } } func TestSuperAdminFenceResolutionIsJournalOnlyAndReadBackIsStable(t *testing.T) { api := newTestAPI(t) operationID := "operation-fence-resolution" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-fence-resolution", Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-resolution", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "result was not confirmed", EffectMayHaveApplied: true, ResubmitAllowed: false, } }); err == nil { t.Fatal("seed effectful failure unexpectedly succeeded") } actor := auth.Actor{ UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) { t.Fatalf("missing fence resolution = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK { t.Fatalf("fresh inspection = %d %s", response.Code, response.Body.String()) } request := operation.FenceResolutionRequest{ ResolutionID: "0123456789abcdef0123456789abcdef", Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } missingIdentity := request missingIdentity.RequestFingerprint = "" missingIdentityBody, err := json.Marshal(missingIdentity) if err != nil { t.Fatal(err) } invalidIdentity := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", missingIdentityBody, actor) if invalidIdentity.Code != http.StatusBadRequest || !bytes.Contains(invalidIdentity.Body.Bytes(), []byte(`"code":"invalid_fence_resolution"`)) { t.Fatalf("missing fence identity = %d %s", invalidIdentity.Code, invalidIdentity.Body.String()) } mismatch := request mismatch.ResourceID = "vm-other-fence-resolution" mismatchBody, err := json.Marshal(mismatch) if err != nil { t.Fatal(err) } identityConflict := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", mismatchBody, actor) if identityConflict.Code != http.StatusConflict || !bytes.Contains(identityConflict.Body.Bytes(), []byte(`"code":"operation_identity_conflict"`)) { t.Fatalf("fence identity conflict = %d %s", identityConflict.Code, identityConflict.Body.String()) } resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"replayed":false`)) { t.Fatalf("resolve fence = %d %s", resolved.Code, resolved.Body.String()) } readBack := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor) if readBack.Code != http.StatusOK || !bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution_id":"0123456789abcdef0123456789abcdef"`)) || !bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution":"not_applied"`)) { t.Fatalf("read fence resolution = %d %s", readBack.Code, readBack.Body.String()) } replayed := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if replayed.Code != http.StatusOK || !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) { t.Fatalf("idempotent fence resolution = %d %s", replayed.Code, replayed.Body.String()) } api.hv.mu.Lock() inspections := api.hv.inspections configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if inspections != 1 || configureCalls != 0 { t.Fatalf("fence resolution invoked host work: inspections=%d mutations=%d", inspections, configureCalls) } } func TestFenceResolutionRequiresExactActorOperationIdentity(t *testing.T) { api := newTestAPI(t) request := operation.FenceResolutionRequest{ ResolutionID: "fedcba9876543210fedcba9876543210", Resolution: operation.FenceResolutionApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Action: "delete-vm", ResourceID: "vm-fence-identity", RequestFingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-identity", []byte(`{}`)), } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } response := api.request(t, http.MethodPost, "/v1/operations/operation-fence-identity/resolve-fence", body, auth.Actor{ UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: "operation-other-identity", }) if response.Code != http.StatusBadRequest || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"operation_id_mismatch"`)) { t.Fatalf("identity mismatch = %d %s", response.Code, response.Body.String()) } } func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing.T) { api := newTestAPI(t) operationID := "operation-system-recovery" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-system-recovery", Action: "update-settings", Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms/vm-system-recovery/settings", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "result was not confirmed", EffectMayHaveApplied: true, } }); err == nil { t.Fatal("seed effectful failure unexpectedly succeeded") } actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte(`"fence_resolution_required":true`)) || !bytes.Contains(response.Body.Bytes(), []byte(`"fence_state":"active"`)) { t.Fatalf("system inspect = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) { t.Fatalf("system unresolved fence read = %d %s", response.Code, response.Body.String()) } request := operation.FenceResolutionRequest{ ResolutionID: "98769876987698769876987698769876", Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) { t.Fatalf("system fence resolution = %d %s", resolved.Code, resolved.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusOK { t.Fatalf("system resolved fence read = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte(`"fence_resolution_required":false`)) || !bytes.Contains(response.Body.Bytes(), []byte(`"fence_state":"resolved"`)) { t.Fatalf("resolved system inspect = %d %s", response.Code, response.Body.String()) } for _, denied := range []struct { name, operationID, resourceID, action string }{ {"unknown", "operation-system-unknown-denied", "vm-system-unknown-denied", "future-unknown"}, } { t.Run("reject "+denied.name+" operation", func(t *testing.T) { deniedSpec := operation.Spec{ OperationID: denied.operationID, ResourceID: denied.resourceID, Action: denied.action, Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/test/"+denied.resourceID, []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), deniedSpec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{Code: "result_unconfirmed", Message: "effect may have applied", EffectMayHaveApplied: true} }); err == nil { t.Fatal("seed disallowed recovery failure unexpectedly succeeded") } deniedActor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: denied.operationID} for _, route := range []struct { method, suffix string body []byte }{ {http.MethodGet, "/inspect", nil}, {http.MethodGet, "/fence-resolution", nil}, {http.MethodPost, "/resolve-fence", body}, } { response := api.request(t, route.method, "/v1/operations/"+denied.operationID+route.suffix, route.body, deniedActor) if response.Code != http.StatusForbidden { t.Fatalf("system recovery %s %s = %d %s", route.method, route.suffix, response.Code, response.Body.String()) } } var successorCalls atomic.Int32 successor := operation.Spec{ OperationID: denied.operationID + "-successor", ResourceID: denied.resourceID, Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/"+denied.resourceID, []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) { successorCalls.Add(1) return json.RawMessage(`{"deleted":true}`), nil }); !errors.Is(err, operation.ErrResourceBlocked) || successorCalls.Load() != 0 { t.Fatalf("denied system recovery released blocker: calls=%d err=%v", successorCalls.Load(), err) } }) } reviewedActions := []string{ "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy", "reinstall-vm", "enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended", "change-network", "network-suspend", "network-resume", "apply-firewall", "delete-vm", "force-delete-vm", "service-suspend", "create-snapshot:0123456789abcdef0123456789abcdef:filesystem", "create-snapshot:0123456789abcdef0123456789abcdef:crash", "delete-snapshot:0123456789abcdef0123456789abcdef", "restore-snapshot:0123456789abcdef0123456789abcdef", "import-template", "update-template", "delete-template", "import-rescue-template", "delete-rescue-template", "import-user-iso", "delete-user-iso", } for index, action := range reviewedActions { t.Run("reviewed "+action, func(t *testing.T) { allowedOperationID := fmt.Sprintf("operation-system-reviewed-%02d", index) allowedResourceID := fmt.Sprintf("resource-system-reviewed-%02d", index) allowedSpec := operation.Spec{ OperationID: allowedOperationID, ResourceID: allowedResourceID, Action: action, Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/test/"+allowedResourceID, []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), allowedSpec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{Code: "result_unconfirmed", Message: "effect may have applied", EffectMayHaveApplied: true} }); err == nil { t.Fatal("seed reviewed recovery failure unexpectedly succeeded") } allowedActor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: allowedOperationID} if response := api.request(t, http.MethodGet, "/v1/operations/"+allowedOperationID+"/inspect", nil, allowedActor); response.Code != http.StatusOK { t.Fatalf("reviewed system inspect = %d %s", response.Code, response.Body.String()) } allowedRequest := operation.FenceResolutionRequest{ ResolutionID: fmt.Sprintf("%032x", index+1), Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", Action: allowedSpec.Action, ResourceID: allowedSpec.ResourceID, RequestFingerprint: allowedSpec.Fingerprint, } allowedBody, err := json.Marshal(allowedRequest) if err != nil { t.Fatal(err) } if response := api.request(t, http.MethodPost, "/v1/operations/"+allowedOperationID+"/resolve-fence", allowedBody, allowedActor); response.Code != http.StatusOK { t.Fatalf("reviewed system resolution = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+allowedOperationID+"/fence-resolution", nil, allowedActor); response.Code != http.StatusOK { t.Fatalf("reviewed system fence read = %d %s", response.Code, response.Body.String()) } }) } for _, test := range []struct { name, method, target string body []byte }{ {"health", http.MethodGet, "/v1/health", nil}, {"raw operation", http.MethodGet, "/v1/operations/" + operationID, nil}, {"other operation", http.MethodGet, "/v1/operations/operation-other-recovery/inspect", nil}, {"host mutation", http.MethodPost, "/v1/storage/configuration", []byte(`{"operation_id":"operation-system-recovery"}`)}, {"create mutation", http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-recovery"}`)}, } { t.Run(test.name, func(t *testing.T) { response := api.request(t, test.method, test.target, test.body, actor) if response.Code != http.StatusForbidden { t.Fatalf("least-privilege route = %d %s", response.Code, response.Body.String()) } }) } manualZero := auth.Actor{ UserID: 0, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, manualZero); response.Code != http.StatusForbidden { t.Fatalf("zero-id human inspection = %d %s", response.Code, response.Body.String()) } api.hv.mu.Lock() inspections := api.hv.inspections configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if inspections != 2+len(reviewedActions) || configureCalls != 0 { t.Fatalf("system recovery escaped read-only scope: inspections=%d mutations=%d", inspections, configureCalls) } } func TestSystemCreateRecoveryRequiresFreshCompleteAbsenceEvidence(t *testing.T) { api := newTestAPI(t) operationID := "operation-system-create-recovery" resourceID := "123e4567-e89b-42d3-a456-426614174000" requestBody := []byte(`{"operation_id":"operation-system-create-recovery"}`) spec := operation.Spec{ OperationID: operationID, ResourceID: resourceID, Action: "create-vm", Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", requestBody), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "create result was not confirmed", EffectMayHaveApplied: true, } }); err == nil { t.Fatal("seed create failure unexpectedly succeeded") } actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} request := operation.FenceResolutionRequest{ ResolutionID: "40404040404040404040404040404040", Resolution: operation.FenceResolutionApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint, } encodeRequest := func() []byte { t.Helper() body, err := json.Marshal(request) if err != nil { t.Fatal(err) } return body } assertActive := func() { t.Helper() state, err := api.journal.FenceState(operationID) if err != nil || state != operation.FenceStateActive { t.Fatalf("create fence state=%q err=%v", state, err) } } applied := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if applied.Code != http.StatusConflict { t.Fatalf("system create applied resolution = %d %s", applied.Code, applied.Body.String()) } assertActive() request.Resolution = operation.FenceResolutionNotApplied inconclusive := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if inconclusive.Code != http.StatusConflict || !bytes.Contains(inconclusive.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { t.Fatalf("inconclusive create resolution = %d %s", inconclusive.Code, inconclusive.Body.String()) } assertActive() exactInspection := func(checkedAt int64) hypervisor.OperationInspection { return hypervisor.OperationInspection{ OperationID: operationID, Action: spec.Action, ResourceID: resourceID, RequestFingerprint: spec.Fingerprint, OperationStatus: operation.StateFailed, CheckedAt: checkedAt, Outcome: hypervisor.InspectionOutcomeFailed, SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "creation_absent", EffectResolution: hypervisor.InspectionEffectNotApplied, EffectProjection: json.RawMessage( `{"creation_absent":true,"managed_artifacts_absent":true}`, ), } } stale := exactInspection(api.now.Add(-time.Second).Unix()) api.hv.mu.Lock() api.hv.inspectionResult = &stale api.hv.mu.Unlock() staleResponse := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if staleResponse.Code != http.StatusConflict || !bytes.Contains(staleResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { t.Fatalf("stale create resolution = %d %s", staleResponse.Code, staleResponse.Body.String()) } assertActive() incomplete := exactInspection(api.now.Unix()) incomplete.EffectProjection = json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":false}`) api.hv.mu.Lock() api.hv.inspectionResult = &incomplete api.hv.mu.Unlock() incompleteResponse := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if incompleteResponse.Code != http.StatusConflict || !bytes.Contains(incompleteResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { t.Fatalf("incomplete create absence = %d %s", incompleteResponse.Code, incompleteResponse.Body.String()) } assertActive() exact := exactInspection(api.now.Unix()) api.hv.mu.Lock() api.hv.inspectionResult = &exact api.hv.mu.Unlock() inspection := api.request( t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor, ) if inspection.Code != http.StatusOK || !bytes.Contains(inspection.Body.Bytes(), []byte(`"error_code":"creation_absent"`)) || !bytes.Contains(inspection.Body.Bytes(), []byte(`"effect_resolution":"not_applied"`)) { t.Fatalf("system create inspection = %d %s", inspection.Code, inspection.Body.String()) } resolved := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolution":"not_applied"`)) || !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) { t.Fatalf("system create not-applied resolution = %d %s", resolved.Code, resolved.Body.String()) } replayed := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if replayed.Code != http.StatusOK || !bytes.Contains(replayed.Body.Bytes(), []byte(`"resolution":"not_applied"`)) || !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) { t.Fatalf("system create resolution replay = %d %s", replayed.Code, replayed.Body.String()) } api.hv.mu.Lock() inspections := api.hv.inspections configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if inspections != 5 || configureCalls != 0 { t.Fatalf("system create recovery inspections=%d mutations=%d", inspections, configureCalls) } } func TestSystemCreateRecoveryRejectsExecutionFencedAbsence(t *testing.T) { api := newTestAPI(t) operationID := "operation-system-create-unreaped" resourceID := "123e4567-e89b-42d3-a456-426614174002" spec := operation.Spec{ OperationID: operationID, ResourceID: resourceID, Action: "create-vm", Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-create-unreaped"}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "worker_reap_unconfirmed", Message: "worker may still be running", EffectMayHaveApplied: true, ExecutionFenced: true, } }); err == nil { t.Fatal("seed execution-fenced create unexpectedly succeeded") } exactAbsence := hypervisor.OperationInspection{ OperationID: operationID, Action: spec.Action, ResourceID: resourceID, RequestFingerprint: spec.Fingerprint, OperationStatus: operation.StateFailed, CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeFailed, SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "creation_absent", EffectResolution: hypervisor.InspectionEffectNotApplied, EffectProjection: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`), } api.hv.mu.Lock() api.hv.inspectionResult = &exactAbsence api.hv.mu.Unlock() actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} inspected := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor) if inspected.Code != http.StatusOK { t.Fatalf("execution-fenced create inspection = %d %s", inspected.Code, inspected.Body.String()) } var inspection hypervisor.OperationInspection if err := json.Unmarshal(inspected.Body.Bytes(), &inspection); err != nil { t.Fatal(err) } if inspection.Outcome != hypervisor.InspectionOutcomeUncertain || inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete || inspection.EffectResolution != hypervisor.InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 { t.Fatalf("execution-fenced create absence was treated as final: %#v", inspection) } request := operation.FenceResolutionRequest{ ResolutionID: "50505050505050505050505050505050", Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "dededededededededededededededededededededededededededededededede", Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if resolved.Code != http.StatusConflict || !bytes.Contains(resolved.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { t.Fatalf("execution-fenced create resolution = %d %s", resolved.Code, resolved.Body.String()) } if state, err := api.journal.FenceState(operationID); err != nil || state != operation.FenceStateActive { t.Fatalf("execution-fenced create was released state=%q err=%v", state, err) } api.hv.mu.Lock() inspections := api.hv.inspections api.hv.mu.Unlock() if inspections != 1 { t.Fatalf("execution-fenced create resolve repeated host inspection: %d", inspections) } } func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) { api := newTestAPI(t) executor := &fakeMutationExecutor{result: json.RawMessage(`{"configured":true}`)} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } request := hypervisor.StorageConfigurationRequest{ OperationID: "operation-isolated-storage", ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, } first := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor) if first.Code != http.StatusAccepted { t.Fatalf("first isolated response = %d %s", first.Code, first.Body.String()) } fingerprint := operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", body) accepted := requireOperationResponseIdentity(t, first, request.OperationID, "configure-storage", "host-storage", fingerprint) if accepted.Status != operation.StateDispatching || accepted.Replayed { t.Fatalf("accepted operation response = %#v", accepted) } terminal := waitForTerminal(t, api.journal, request.OperationID) if terminal.State != operation.StateSucceeded || string(terminal.Result) != `{"configured":true}` { t.Fatalf("terminal operation = %#v", terminal) } replay := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor) if replay.Code != http.StatusOK { t.Fatalf("terminal replay = %d %s", replay.Code, replay.Body.String()) } replayed := requireOperationResponseIdentity(t, replay, request.OperationID, "configure-storage", "host-storage", fingerprint) if replayed.Status != operation.StateSucceeded || !replayed.Replayed { t.Fatalf("replayed operation response = %#v", replayed) } executor.mu.Lock() defer executor.mu.Unlock() if executor.calls != 1 || len(executor.requests) != 1 { t.Fatalf("executor calls=%d requests=%d", executor.calls, len(executor.requests)) } workerRequest := executor.requests[0] if workerRequest.OperationID != request.OperationID || workerRequest.Actor.OperationID != request.OperationID || workerRequest.Method != http.MethodPost || workerRequest.RequestTarget != "/v1/storage/configuration" { t.Fatalf("worker envelope = %#v", workerRequest) } } func TestIsolatedMutationDeadlineCapsOnlyStorageConfiguration(t *testing.T) { server := &Server{OperationTimeout: 840 * time.Second} if got := server.isolatedMutationDeadline("configure-storage"); got != 120*time.Second { t.Fatalf("storage helper deadline = %s, want 120s", got) } for _, action := range []string{"create-vm", "delete-vm", "change-network"} { if got := server.isolatedMutationDeadline(action); got != 840*time.Second { t.Fatalf("%s helper deadline = %s, want global 840s", action, got) } } server.OperationTimeout = 30 * time.Second if got := server.isolatedMutationDeadline("configure-storage"); got != 30*time.Second { t.Fatalf("lower global deadline was widened to %s", got) } } func TestIsolatedMutationAdmissionIsShortAndDoesNotStartSecondWorker(t *testing.T) { api := newTestAPI(t) // Give the first request enough time to persist its independent Bolt // journal under a loaded test host. The short timeout is the behavior under // test only after that first worker demonstrably owns admission. api.handler.AdmissionTimeout = time.Second executor := &blockingMutationExecutor{ started: make(chan worker.Request, 1), release: make(chan struct{}), } if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } requestBody := func(operationID string) []byte { body, err := json.Marshal(hypervisor.StorageConfigurationRequest{ OperationID: operationID, ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", }) if err != nil { t.Fatal(err) } return body } firstID := "operation-admission-first" firstBody := requestBody(firstID) first := api.request(t, http.MethodPost, "/v1/storage/configuration", firstBody, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: firstID, }) if first.Code != http.StatusAccepted { t.Fatalf("first response = %d %s", first.Code, first.Body.String()) } firstWire := requireOperationResponseIdentity(t, first, firstID, "configure-storage", "host-storage", operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", firstBody)) if firstWire.Status != operation.StateDispatching || firstWire.Replayed { t.Fatalf("first accepted operation response = %#v", firstWire) } select { case <-executor.started: case <-time.After(time.Second): t.Fatal("first worker did not start") } api.handler.AdmissionTimeout = 30 * time.Millisecond secondID := "operation-admission-second" secondBody := requestBody(secondID) startedAt := time.Now() second := api.request(t, http.MethodPost, "/v1/storage/configuration", secondBody, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: secondID, }) if elapsed := time.Since(startedAt); elapsed > 500*time.Millisecond { t.Fatalf("saturated admission waited %s", elapsed) } if second.Code != http.StatusOK || !bytes.Contains(second.Body.Bytes(), []byte(`"status":"failed"`)) || !bytes.Contains(second.Body.Bytes(), []byte(`"code":"operation_not_dispatched"`)) { t.Fatalf("saturated admission response = %d %s", second.Code, second.Body.String()) } failedWire := requireOperationResponseIdentity(t, second, secondID, "configure-storage", "host-storage", operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", secondBody)) if failedWire.Status != operation.StateFailed || failedWire.Replayed { t.Fatalf("failed operation response = %#v", failedWire) } if executor.calls.Load() != 1 { t.Fatalf("saturated request started a second worker; calls=%d", executor.calls.Load()) } close(executor.release) if terminal := waitForTerminal(t, api.journal, firstID); terminal.State != operation.StateSucceeded { t.Fatalf("first terminal = %#v", terminal) } } func TestParentMutationNeverFallsBackWhenExecutorIsMissing(t *testing.T) { api := newTestAPI(t) request := hypervisor.StorageConfigurationRequest{ OperationID: "operation-worker-missing", ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } response := api.request(t, http.MethodPost, "/v1/storage/configuration", body, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, }) if response.Code != http.StatusServiceUnavailable || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"mutation_worker_unavailable"`)) || !bytes.Contains(response.Body.Bytes(), []byte(`"dispatch_state":"not_dispatched"`)) { t.Fatalf("missing worker response = %d %s", response.Code, response.Body.String()) } if _, err := api.journal.Get(request.OperationID); !errors.Is(err, operation.ErrNotFound) { t.Fatalf("missing worker created journal state: %v", err) } api.hv.mu.Lock() configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if configureCalls != 0 { t.Fatalf("parent invoked the in-process hypervisor %d times", configureCalls) } } func TestTrustedWorkerExecutesLocallyWithoutRecursiveSpawn(t *testing.T) { api := newTestAPI(t) request := hypervisor.StorageConfigurationRequest{ OperationID: "operation-trusted-worker", ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } httpRequest := httptest.NewRequest(http.MethodPost, "/v1/storage/configuration", bytes.NewReader(body)) response := httptest.NewRecorder() api.handler.ServeTrustedWorkerRequest(response, httpRequest, body, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, }) if response.Code != http.StatusOK { t.Fatalf("trusted worker response = %d %s", response.Code, response.Body.String()) } succeeded := requireOperationResponseIdentity(t, response, request.OperationID, "configure-storage", "host-storage", operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", body)) if succeeded.Status != operation.StateSucceeded || succeeded.Replayed { t.Fatalf("successful operation response = %#v", succeeded) } api.hv.mu.Lock() configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if configureCalls != 1 { t.Fatalf("trusted worker local mutation calls=%d", configureCalls) } } func TestListEnvelopeRemainsNonNull(t *testing.T) { api := newTestAPI(t) response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1}) if response.Code != http.StatusOK || response.Body.String() != "{\"vms\":[]}\n" { t.Fatalf("VM list = %d %s", response.Code, response.Body.String()) } } func waitForTerminal(t *testing.T, journal *operation.Journal, operationID string) operation.Record { t.Helper() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { record, err := journal.Get(operationID) if err == nil && (record.State == operation.StateSucceeded || record.State == operation.StateFailed) { return record } time.Sleep(time.Millisecond) } t.Fatalf("operation %s did not become terminal", operationID) return operation.Record{} }