package api import ( "bytes" "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "path/filepath" "sync" "sync/atomic" "testing" "time" "git.luoli.live/root/virt/agent/auth" consolebroker "git.luoli.live/root/virt/agent/console" "git.luoli.live/root/virt/agent/hypervisor" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/worker" ) type testHypervisor struct { hypervisor.Unsupported mu sync.Mutex inspections int configureCalls int } func (h *testHypervisor) Ping(context.Context) error { return nil } func (h *testHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) { return []hypervisor.VMMetadata{}, nil } func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageConfigurationRequest) (hypervisor.StorageConfigurationResult, error) { h.mu.Lock() h.configureCalls++ h.mu.Unlock() return hypervisor.StorageConfigurationResult{Configured: true}, nil } func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) { h.mu.Lock() h.inspections++ h.mu.Unlock() return hypervisor.OperationInspection{ OperationID: record.OperationID, CheckedAt: time.Now().Unix(), Outcome: hypervisor.InspectionOutcomeFailed, SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "result_unconfirmed", }, nil } type fakeMutationExecutor struct { mu sync.Mutex calls int requests []worker.Request result json.RawMessage err error } type blockingMutationExecutor struct { started chan worker.Request release chan struct{} calls atomic.Int32 } func (e *blockingMutationExecutor) Execute(ctx context.Context, request worker.Request) (json.RawMessage, error) { e.calls.Add(1) select { case e.started <- request: case <-ctx.Done(): return nil, ctx.Err() } select { case <-e.release: return json.RawMessage(`{"configured":true}`), nil case <-ctx.Done(): return nil, ctx.Err() } } func (e *fakeMutationExecutor) Execute(_ context.Context, request worker.Request) (json.RawMessage, error) { e.mu.Lock() defer e.mu.Unlock() e.calls++ e.requests = append(e.requests, request) return append(json.RawMessage(nil), e.result...), e.err } type testAPI struct { handler *Server hv *testHypervisor journal *operation.Journal secret []byte now time.Time nonce int } func newTestAPI(t *testing.T) *testAPI { t.Helper() journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db")) if err != nil { t.Fatal(err) } secret := []byte("0123456789abcdef0123456789abcdef") now := time.Unix(2_000_000_000, 0) journal.SetClockForTesting(func() time.Time { return now }) authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal) if err != nil { t.Fatal(err) } authenticator.SetClockForTesting(func() time.Time { return now }) broker, err := consolebroker.NewBroker(time.Minute) if err != nil { t.Fatal(err) } hv := &testHypervisor{} handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second) if err != nil { t.Fatal(err) } t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() _ = handler.Shutdown(ctx) _ = journal.Wait(ctx) _ = journal.Close() }) return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now} } func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder { t.Helper() a.nonce++ request := httptest.NewRequest(method, target, bytes.NewReader(body)) nonce := fmt.Sprintf("nonce%011d", a.nonce) if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil { t.Fatal(err) } response := httptest.NewRecorder() a.handler.ServeHTTP(response, request) return response } func TestOperationRecoveryMutationRoutesAreRemoved(t *testing.T) { api := newTestAPI(t) operationID := "operation-terminal-inspect" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-inspect", Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-inspect", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "result was not confirmed", EffectMayHaveApplied: true, } }); err == nil { t.Fatal("seed failure unexpectedly succeeded") } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID, nil, actor); response.Code != http.StatusOK { t.Fatalf("operation GET = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK { t.Fatalf("read-only inspect = %d %s", response.Code, response.Body.String()) } for _, suffix := range []string{"reconcile", "confirm-inspection", "abort-job", "cleanup"} { response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/"+suffix, []byte(`{}`), actor) if response.Code != http.StatusNotFound { t.Fatalf("removed %s route = %d %s", suffix, response.Code, response.Body.String()) } } api.hv.mu.Lock() inspections := api.hv.inspections api.hv.mu.Unlock() if inspections != 1 { t.Fatalf("removed mutation routes reached hypervisor; inspections=%d", inspections) } } func TestSuperAdminFenceResolutionIsJournalOnlyAndReadBackIsStable(t *testing.T) { api := newTestAPI(t) operationID := "operation-fence-resolution" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-fence-resolution", Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-resolution", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "result was not confirmed", EffectMayHaveApplied: true, ResubmitAllowed: false, } }); err == nil { t.Fatal("seed effectful failure unexpectedly succeeded") } actor := auth.Actor{ UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) { t.Fatalf("missing fence resolution = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK { t.Fatalf("fresh inspection = %d %s", response.Code, response.Body.String()) } request := operation.FenceResolutionRequest{ ResolutionID: "0123456789abcdef0123456789abcdef", Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"replayed":false`)) { t.Fatalf("resolve fence = %d %s", resolved.Code, resolved.Body.String()) } readBack := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor) if readBack.Code != http.StatusOK || !bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution_id":"0123456789abcdef0123456789abcdef"`)) || !bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution":"not_applied"`)) { t.Fatalf("read fence resolution = %d %s", readBack.Code, readBack.Body.String()) } replayed := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if replayed.Code != http.StatusOK || !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) { t.Fatalf("idempotent fence resolution = %d %s", replayed.Code, replayed.Body.String()) } api.hv.mu.Lock() inspections := api.hv.inspections configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if inspections != 1 || configureCalls != 0 { t.Fatalf("fence resolution invoked host work: inspections=%d mutations=%d", inspections, configureCalls) } } func TestFenceResolutionRequiresExactActorOperationIdentity(t *testing.T) { api := newTestAPI(t) request := operation.FenceResolutionRequest{ ResolutionID: "fedcba9876543210fedcba9876543210", Resolution: operation.FenceResolutionApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } response := api.request(t, http.MethodPost, "/v1/operations/operation-fence-identity/resolve-fence", body, auth.Actor{ UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: "operation-other-identity", }) if response.Code != http.StatusBadRequest || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"operation_id_mismatch"`)) { t.Fatalf("identity mismatch = %d %s", response.Code, response.Body.String()) } } func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) { api := newTestAPI(t) executor := &fakeMutationExecutor{result: json.RawMessage(`{"configured":true}`)} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } request := hypervisor.StorageConfigurationRequest{ OperationID: "operation-isolated-storage", ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, } first := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor) if first.Code != http.StatusAccepted { t.Fatalf("first isolated response = %d %s", first.Code, first.Body.String()) } terminal := waitForTerminal(t, api.journal, request.OperationID) if terminal.State != operation.StateSucceeded || string(terminal.Result) != `{"configured":true}` { t.Fatalf("terminal operation = %#v", terminal) } replay := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor) if replay.Code != http.StatusOK { t.Fatalf("terminal replay = %d %s", replay.Code, replay.Body.String()) } executor.mu.Lock() defer executor.mu.Unlock() if executor.calls != 1 || len(executor.requests) != 1 { t.Fatalf("executor calls=%d requests=%d", executor.calls, len(executor.requests)) } workerRequest := executor.requests[0] if workerRequest.OperationID != request.OperationID || workerRequest.Actor.OperationID != request.OperationID || workerRequest.Method != http.MethodPost || workerRequest.RequestTarget != "/v1/storage/configuration" { t.Fatalf("worker envelope = %#v", workerRequest) } } func TestIsolatedMutationDeadlineCapsOnlyStorageConfiguration(t *testing.T) { server := &Server{OperationTimeout: 840 * time.Second} if got := server.isolatedMutationDeadline("configure-storage"); got != 120*time.Second { t.Fatalf("storage helper deadline = %s, want 120s", got) } for _, action := range []string{"create-vm", "delete-vm", "change-network"} { if got := server.isolatedMutationDeadline(action); got != 840*time.Second { t.Fatalf("%s helper deadline = %s, want global 840s", action, got) } } server.OperationTimeout = 30 * time.Second if got := server.isolatedMutationDeadline("configure-storage"); got != 30*time.Second { t.Fatalf("lower global deadline was widened to %s", got) } } func TestIsolatedMutationAdmissionIsShortAndDoesNotStartSecondWorker(t *testing.T) { api := newTestAPI(t) api.handler.AdmissionTimeout = 30 * time.Millisecond executor := &blockingMutationExecutor{ started: make(chan worker.Request, 1), release: make(chan struct{}), } if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } requestBody := func(operationID string) []byte { body, err := json.Marshal(hypervisor.StorageConfigurationRequest{ OperationID: operationID, ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", }) if err != nil { t.Fatal(err) } return body } firstID := "operation-admission-first" first := api.request(t, http.MethodPost, "/v1/storage/configuration", requestBody(firstID), auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: firstID, }) if first.Code != http.StatusAccepted { t.Fatalf("first response = %d %s", first.Code, first.Body.String()) } select { case <-executor.started: case <-time.After(time.Second): t.Fatal("first worker did not start") } secondID := "operation-admission-second" startedAt := time.Now() second := api.request(t, http.MethodPost, "/v1/storage/configuration", requestBody(secondID), auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: secondID, }) if elapsed := time.Since(startedAt); elapsed > 500*time.Millisecond { t.Fatalf("saturated admission waited %s", elapsed) } if second.Code != http.StatusOK || !bytes.Contains(second.Body.Bytes(), []byte(`"status":"failed"`)) || !bytes.Contains(second.Body.Bytes(), []byte(`"code":"operation_not_dispatched"`)) { t.Fatalf("saturated admission response = %d %s", second.Code, second.Body.String()) } if executor.calls.Load() != 1 { t.Fatalf("saturated request started a second worker; calls=%d", executor.calls.Load()) } close(executor.release) if terminal := waitForTerminal(t, api.journal, firstID); terminal.State != operation.StateSucceeded { t.Fatalf("first terminal = %#v", terminal) } } func TestParentMutationNeverFallsBackWhenExecutorIsMissing(t *testing.T) { api := newTestAPI(t) request := hypervisor.StorageConfigurationRequest{ OperationID: "operation-worker-missing", ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } response := api.request(t, http.MethodPost, "/v1/storage/configuration", body, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, }) if response.Code != http.StatusServiceUnavailable || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"mutation_worker_unavailable"`)) || !bytes.Contains(response.Body.Bytes(), []byte(`"dispatch_state":"not_dispatched"`)) { t.Fatalf("missing worker response = %d %s", response.Code, response.Body.String()) } if _, err := api.journal.Get(request.OperationID); !errors.Is(err, operation.ErrNotFound) { t.Fatalf("missing worker created journal state: %v", err) } api.hv.mu.Lock() configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if configureCalls != 0 { t.Fatalf("parent invoked the in-process hypervisor %d times", configureCalls) } } func TestTrustedWorkerExecutesLocallyWithoutRecursiveSpawn(t *testing.T) { api := newTestAPI(t) request := hypervisor.StorageConfigurationRequest{ OperationID: "operation-trusted-worker", ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } httpRequest := httptest.NewRequest(http.MethodPost, "/v1/storage/configuration", bytes.NewReader(body)) response := httptest.NewRecorder() api.handler.ServeTrustedWorkerRequest(response, httpRequest, body, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, }) if response.Code != http.StatusOK { t.Fatalf("trusted worker response = %d %s", response.Code, response.Body.String()) } api.hv.mu.Lock() configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if configureCalls != 1 { t.Fatalf("trusted worker local mutation calls=%d", configureCalls) } } func TestListEnvelopeRemainsNonNull(t *testing.T) { api := newTestAPI(t) response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1}) if response.Code != http.StatusOK || response.Body.String() != "{\"vms\":[]}\n" { t.Fatalf("VM list = %d %s", response.Code, response.Body.String()) } } func waitForTerminal(t *testing.T, journal *operation.Journal, operationID string) operation.Record { t.Helper() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { record, err := journal.Get(operationID) if err == nil && (record.State == operation.StateSucceeded || record.State == operation.StateFailed) { return record } time.Sleep(time.Millisecond) } t.Fatalf("operation %s did not become terminal", operationID) return operation.Record{} }