package api import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "path/filepath" "strings" "sync" "testing" "time" "git.luoli.live/root/virt/agent/auth" consolebroker "git.luoli.live/root/virt/agent/console" "git.luoli.live/root/virt/agent/hypervisor" "git.luoli.live/root/virt/agent/operation" ) type fakeHypervisor struct { hypervisor.Unsupported mu sync.Mutex creates int reconciles int lastReconcile operation.Record reconcileTo operation.Resolution reconcileErr error templateCalls chan hypervisor.ImportTemplateRequest templateWait <-chan struct{} rescueCalls chan hypervisor.ImportRescueTemplateRequest userISOCalls chan hypervisor.ImportUserISORequest forceCalls chan hypervisor.ForceRescueStartRequest powerCalls chan hypervisor.PowerRequest } type failingNonceStore struct{} func (failingNonceStore) UseNonce(string, time.Time, time.Time) (bool, error) { return false, errors.New("nonce store failed") } func (f *fakeHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) { return []hypervisor.VMMetadata{}, nil } func (f *fakeHypervisor) Ping(context.Context) error { return nil } func (f *fakeHypervisor) Capabilities(context.Context) (hypervisor.Capabilities, error) { return hypervisor.Capabilities{ Backend: "fake", Features: hypervisor.FeatureSet{CreateVM: true, Console: true, UEFI: true}, PowerActions: []hypervisor.PowerAction{hypervisor.PowerStart}, Firmware: []string{"uefi"}, NetworkSelectors: []string{"public"}, }, nil } func (f *fakeHypervisor) CreateVM(_ context.Context, request hypervisor.CreateVMRequest) (hypervisor.VMMetadata, error) { f.mu.Lock() f.creates++ f.mu.Unlock() return hypervisor.VMMetadata{ UUID: request.UUID, Name: request.Name, VCPUs: request.VCPUs, MemoryMiB: request.MemoryMiB, Machine: "q35", Firmware: "uefi", PublicIPCIDR: request.PublicIPCIDR, PublicPoolKey: request.PublicPoolKey, }, nil } func (f *fakeHypervisor) Reconcile(_ context.Context, record operation.Record) (operation.Resolution, error) { f.mu.Lock() defer f.mu.Unlock() f.reconciles++ f.lastReconcile = record return f.reconcileTo, f.reconcileErr } func (f *fakeHypervisor) GetConsole(context.Context, string) (hypervisor.ConsoleSession, error) { return hypervisor.ConsoleSession{Protocol: "vnc", Address: "127.0.0.1:5900"}, nil } func (f *fakeHypervisor) ResolveNetworkSelector(_ context.Context, selector string) (hypervisor.LocalNetworkSource, error) { if selector != "public" { return hypervisor.LocalNetworkSource{}, hypervisor.NewError(hypervisor.ErrInvalid, "network_not_found", "network selector is not configured", nil) } return hypervisor.LocalNetworkSource{Type: "bridge", Source: "br0"}, nil } func (f *fakeHypervisor) ImportTemplate(_ context.Context, request hypervisor.ImportTemplateRequest) (hypervisor.ImageTemplate, error) { f.templateCalls <- request if f.templateWait != nil { <-f.templateWait } return hypervisor.ImageTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("a", 64)}, nil } func (f *fakeHypervisor) ListRescueTemplates(context.Context) ([]hypervisor.RescueTemplate, error) { return []hypervisor.RescueTemplate{}, nil } func (f *fakeHypervisor) ImportRescueTemplate(_ context.Context, request hypervisor.ImportRescueTemplateRequest) (hypervisor.RescueTemplate, error) { f.rescueCalls <- request return hypervisor.RescueTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("b", 64)}, nil } func (f *fakeHypervisor) ImportUserISO(_ context.Context, request hypervisor.ImportUserISORequest) (hypervisor.UserISO, error) { f.userISOCalls <- request return hypervisor.UserISO{ID: request.ID, OwnerID: request.OwnerID, Name: request.Name, SHA256: strings.Repeat("c", 64)}, nil } func (f *fakeHypervisor) DeleteTemplate(context.Context, string) error { return nil } func (f *fakeHypervisor) DeleteRescueTemplate(context.Context, string) error { return nil } func (f *fakeHypervisor) DeleteUserISO(context.Context, string, int64) error { return nil } func (f *fakeHypervisor) ForceStartRescue(_ context.Context, _ string, request hypervisor.ForceRescueStartRequest) (hypervisor.RescueState, error) { f.forceCalls <- request return hypervisor.RescueState{Active: true, TemplateID: "0123456789abcdef0123456789abcdef"}, nil } func (f *fakeHypervisor) Power(_ context.Context, _ string, request hypervisor.PowerRequest) (hypervisor.VMState, error) { f.powerCalls <- request return hypervisor.VMState{State: hypervisor.VMStateShutoff}, nil } type testAPI struct { handler *Server hv *fakeHypervisor journal *operation.Journal secret []byte now time.Time nonce int } func newTestAPI(t *testing.T) *testAPI { t.Helper() journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = journal.Close() }) secret := []byte("0123456789abcdef0123456789abcdef") now := time.Unix(2_000_000_000, 0) authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal) if err != nil { t.Fatal(err) } authenticator.SetClockForTesting(func() time.Time { return now }) broker, err := consolebroker.NewBroker(time.Minute) if err != nil { t.Fatal(err) } hv := &fakeHypervisor{ templateCalls: make(chan hypervisor.ImportTemplateRequest, 1), rescueCalls: make(chan hypervisor.ImportRescueTemplateRequest, 1), userISOCalls: make(chan hypervisor.ImportUserISORequest, 1), forceCalls: make(chan hypervisor.ForceRescueStartRequest, 1), powerCalls: make(chan hypervisor.PowerRequest, 1), } handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second) if err != nil { t.Fatal(err) } t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() _ = handler.Shutdown(ctx) _ = journal.Wait(ctx) }) return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now} } func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder { t.Helper() a.nonce++ request := httptest.NewRequest(method, target, bytes.NewReader(body)) nonce := fmt.Sprintf("nonce%011d", a.nonce) if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil { t.Fatal(err) } response := httptest.NewRecorder() a.handler.ServeHTTP(response, request) return response } func validCreateBody(operationID, name string) []byte { rules := []hypervisor.PlatformNetworkPolicyRule{{ ID: "ssh", Label: "SSH", Category: "default", Protocols: []string{"tcp"}, PortFrom: 22, PortTo: 22, Directions: []string{"ingress"}, Reason: "default SSH policy", }} digestDocument := struct { SchemaVersion uint64 `json:"schema_version"` BasePolicyVersion uint64 `json:"base_policy_version"` CatalogRevision uint64 `json:"catalog_revision"` Rules []hypervisor.PlatformNetworkPolicyRule `json:"rules"` }{SchemaVersion: 1, BasePolicyVersion: 1, CatalogRevision: 0, Rules: rules} digestJSON, _ := json.Marshal(digestDocument) digest := sha256.Sum256(digestJSON) value := hypervisor.CreateVMRequest{ OperationID: operationID, UUID: "123e4567-e89b-42d3-a456-426614174000", Source: "cloud", Name: name, VCPUs: 2, MemoryMiB: 2048, DiskGiB: 40, BandwidthMbps: 100, NetworkSelector: "public", TemplateID: "debian-12", TemplateRevision: 1, TemplateGenerationToken: "0123456789abcdef0123456789abcdef", RootPassword: "StrongPassword!123", PublicPoolKey: "0123456789abcdef0123456789abcdef", PublicPoolCIDR: "8.8.8.0/24", PublicIPCIDR: "8.8.8.10/24", Gateway: "8.8.8.1", DNSServers: []string{"1.1.1.1"}, PlatformNetworkPolicy: hypervisor.PlatformNetworkPolicySnapshot{ SchemaVersion: 1, BasePolicyVersion: 1, Rules: rules, Digest: hex.EncodeToString(digest[:]), }, } encoded, _ := json.Marshal(value) return encoded } func TestListResponsesUseNamedNonNullEnvelope(t *testing.T) { api := newTestAPI(t) response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1}) if response.Code != http.StatusOK || strings.TrimSpace(response.Body.String()) != `{"vms":[]}` { t.Fatalf("VM list response = %d %s", response.Code, response.Body.String()) } } func TestCapabilitiesExposeStableNetworkSelectorKeys(t *testing.T) { api := newTestAPI(t) response := api.request(t, http.MethodGet, "/v1/capabilities", nil, auth.Actor{UserID: 1}) if response.Code != http.StatusOK { t.Fatalf("capabilities response = %d %s", response.Code, response.Body.String()) } var payload struct { NetworkSelectors []string `json:"network_selectors"` } if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil { t.Fatal(err) } if len(payload.NetworkSelectors) != 1 || payload.NetworkSelectors[0] != "public" { t.Fatalf("network selectors = %#v", payload.NetworkSelectors) } } func TestReinstallRejectsStartupScript(t *testing.T) { api := newTestAPI(t) actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-reinstall"} body := []byte(`{"operation_id":"operation-reinstall","template_id":"debian-12","template_revision":1,"template_generation_token":"0123456789abcdef0123456789abcdef","startup_script":"echo forbidden"}`) response := api.request(t, http.MethodPost, "/v1/vms/123e4567-e89b-42d3-a456-426614174000/reinstall", body, actor) if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), `"code":"invalid_json"`) || !strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) { t.Fatalf("reinstall startup script response = %d %s", response.Code, response.Body.String()) } } func TestPowerDestroyWireActionIsAccepted(t *testing.T) { api := newTestAPI(t) actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-poweroff"} body := []byte(`{"action":"destroy"}`) response := api.request(t, http.MethodPost, "/v1/vms/123e4567-e89b-42d3-a456-426614174000/actions", body, actor) if response.Code != http.StatusOK { t.Fatalf("destroy action response = %d %s", response.Code, response.Body.String()) } select { case request := <-api.hv.powerCalls: if request.Action != hypervisor.PowerDestroy { t.Fatalf("power action = %q, want destroy", request.Action) } default: t.Fatal("destroy action did not reach the hypervisor") } } func TestEveryHTTPRouteRequiresHMACIncludingHealth(t *testing.T) { api := newTestAPI(t) request := httptest.NewRequest(http.MethodGet, "/v1/health", nil) response := httptest.NewRecorder() api.handler.ServeHTTP(response, request) if response.Code != http.StatusUnauthorized || !strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) { t.Fatalf("health without HMAC status = %d, body=%s", response.Code, response.Body.String()) } response = api.request(t, http.MethodGet, "/v1/health", nil, auth.Actor{UserID: 1}) if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"service":"virt-host-agent"`) { t.Fatalf("signed health status = %d, body=%s", response.Code, response.Body.String()) } } func TestReplayAuthenticationFailureDoesNotClaimNotDispatched(t *testing.T) { api := newTestAPI(t) request := httptest.NewRequest(http.MethodGet, "/v1/health", nil) if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, api.secret, api.now.Unix(), "fixed-replay-nonce-1234"); err != nil { t.Fatal(err) } first := httptest.NewRecorder() api.handler.ServeHTTP(first, request) if first.Code != http.StatusOK { t.Fatalf("first request status = %d body=%s", first.Code, first.Body.String()) } replay := httptest.NewRequest(http.MethodGet, "/v1/health", nil) if err := auth.Sign(replay, nil, auth.Actor{UserID: 1}, api.secret, api.now.Unix(), "fixed-replay-nonce-1234"); err != nil { t.Fatal(err) } response := httptest.NewRecorder() api.handler.ServeHTTP(response, replay) if response.Code != http.StatusUnauthorized || !strings.Contains(response.Body.String(), `"code":"replayed_request"`) || strings.Contains(response.Body.String(), `"dispatch_state"`) { t.Fatalf("replay response = %d %s", response.Code, response.Body.String()) } } func TestTerminalPreJournalContractIsExplicit(t *testing.T) { api := newTestAPI(t) tests := []struct { name string method string target string body []byte actor auth.Actor status int code string }{ { name: "forbidden mutation", method: http.MethodPost, target: "/v1/vms", body: []byte(`{}`), actor: auth.Actor{UserID: 1}, status: http.StatusForbidden, code: "forbidden", }, { name: "route missing", method: http.MethodPost, target: "/v1/missing", body: []byte(`{}`), actor: auth.Actor{UserID: 1, CanManage: true}, status: http.StatusNotFound, code: "not_found", }, { name: "method mismatch", method: http.MethodDelete, target: "/v1/health", actor: auth.Actor{UserID: 1, CanManage: true}, status: http.StatusMethodNotAllowed, code: "method_not_allowed", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { response := api.request(t, test.method, test.target, test.body, test.actor) var payload errorEnvelope if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil { t.Fatal(err) } if response.Code != test.status || payload.Error.Code != test.code || payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" || payload.Error.Operation != nil { t.Fatalf("response = %d %#v", response.Code, payload) } }) } } func TestRetryablePreJournalContractIsExplicit(t *testing.T) { t.Run("shutting down", func(t *testing.T) { api := newTestAPI(t) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() if err := api.handler.Shutdown(ctx); err != nil { t.Fatal(err) } response := api.request(t, http.MethodGet, "/v1/health", nil, auth.Actor{UserID: 1}) assertRetryableNotDispatched(t, response, "agent_shutting_down") }) t.Run("nonce store unavailable", func(t *testing.T) { journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db")) if err != nil { t.Fatal(err) } defer journal.Close() secret := []byte("0123456789abcdef0123456789abcdef") now := time.Unix(2_000_000_000, 0) authenticator, err := auth.NewWithNonceStore( secret, 5*time.Minute, 10*time.Minute, failingNonceStore{}, ) if err != nil { t.Fatal(err) } authenticator.SetClockForTesting(func() time.Time { return now }) broker, err := consolebroker.NewBroker(time.Minute) if err != nil { t.Fatal(err) } server, err := NewServer(authenticator, journal, &fakeHypervisor{}, broker, 64<<10, time.Second) if err != nil { t.Fatal(err) } defer func() { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() _ = server.Shutdown(ctx) }() request := httptest.NewRequest(http.MethodGet, "/v1/health", nil) if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, secret, now.Unix(), "nonce-store-failure-1234"); err != nil { t.Fatal(err) } response := httptest.NewRecorder() server.ServeHTTP(response, request) assertRetryableNotDispatched(t, response, "nonce_store_unavailable") }) } func assertRetryableNotDispatched(t *testing.T, response *httptest.ResponseRecorder, code string) { t.Helper() var payload errorEnvelope if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil { t.Fatal(err) } if response.Code != http.StatusServiceUnavailable || payload.Error.Code != code || !payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" || payload.Error.Operation != nil { t.Fatalf("response = %d %#v", response.Code, payload) } } func TestCreateVMOperationIsDurablyIdempotentAndConflictsOnPayloadChange(t *testing.T) { api := newTestAPI(t) actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-create-one"} body := validCreateBody(actor.OperationID, "vm-one") first := api.request(t, http.MethodPost, "/v1/vms", body, actor) if first.Code != http.StatusCreated { t.Fatalf("first create status = %d body=%s", first.Code, first.Body.String()) } if !strings.Contains(first.Body.String(), `"public_pool_key":"0123456789abcdef0123456789abcdef"`) || !strings.Contains(first.Body.String(), `"public_ip_cidr":"8.8.8.10/24"`) { t.Fatalf("create response omitted confirmed network allocation: %s", first.Body.String()) } second := api.request(t, http.MethodPost, "/v1/vms", body, actor) if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), `"replayed":true`) { t.Fatalf("replay status = %d body=%s", second.Code, second.Body.String()) } api.hv.mu.Lock() creates := api.hv.creates api.hv.mu.Unlock() if creates != 1 { t.Fatalf("backend create calls = %d, want 1", creates) } conflict := api.request(t, http.MethodPost, "/v1/vms", validCreateBody(actor.OperationID, "vm-different"), actor) if conflict.Code != http.StatusConflict || !strings.Contains(conflict.Body.String(), "operation_id_conflict") || strings.Contains(conflict.Body.String(), `"dispatch_state"`) { t.Fatalf("conflict status = %d body=%s", conflict.Code, conflict.Body.String()) } } func TestFailedCreateRouteUsesReadOnlyReconciliationWithoutReplay(t *testing.T) { api := newTestAPI(t) operationID := "operation-create-api-failed" uuid := "123e4567-e89b-42d3-a456-426614174000" spec := operation.Spec{ OperationID: operationID, ResourceID: uuid, Action: "create-vm", Fingerprint: operation.Fingerprint("POST", "/v1/vms", []byte(`{}`)), Destructive: true, } if record, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, hypervisor.NewError(hypervisor.ErrInvalid, "network_selector_invalid", "network selector is not configured", nil) }); err == nil || record.State != operation.StateFailed { t.Fatalf("initial failed create = %#v err=%v", record, err) } api.hv.mu.Lock() api.hv.reconcileTo = operation.Resolution{ Resolved: true, Succeeded: false, ErrorCode: "creation_absent", Result: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`), } api.hv.mu.Unlock() actor := auth.Actor{UserID: 1, CanManage: true, OperationID: operationID} response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor) if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"status":"failed"`) || !strings.Contains(response.Body.String(), `"cause_code":"network_selector_invalid"`) || !strings.Contains(response.Body.String(), `"managed_artifacts_absent":true`) { t.Fatalf("failed create reconcile response = %d %s", response.Code, response.Body.String()) } api.hv.mu.Lock() reconciles := api.hv.reconciles last := api.hv.lastReconcile creates := api.hv.creates api.hv.mu.Unlock() if reconciles != 1 || creates != 0 || last.OperationID != operationID || last.ResourceID != uuid || last.Action != "create-vm" { t.Fatalf("reconcile calls=%d creates=%d record=%#v", reconciles, creates, last) } second := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor) api.hv.mu.Lock() reconciles = api.hv.reconciles api.hv.mu.Unlock() if second.Code != http.StatusOK || reconciles != 1 { t.Fatalf("confirmed absence was reinspected: status=%d calls=%d body=%s", second.Code, reconciles, second.Body.String()) } } func TestUnsupportedTypedMutationUsesStable501Contract(t *testing.T) { api := newTestAPI(t) actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-network-one"} body := []byte(`{"operation_id":"operation-network-one","reason":"quota"}`) response := api.request(t, http.MethodPost, "/v1/vms/123e4567-e89b-42d3-a456-426614174000/network/suspend", body, actor) if response.Code != http.StatusNotImplemented || !strings.Contains(response.Body.String(), `"code":"unsupported"`) { t.Fatalf("unsupported status = %d body=%s", response.Code, response.Body.String()) } } func TestReconciliableMutationRoutesPersistOnlySafeExpectedState(t *testing.T) { api := newTestAPI(t) uuid := "123e4567-e89b-42d3-a456-426614174000" tests := []struct { operationID string method string path string body string kind string }{ {"operation-settings-route", http.MethodPatch, "/v1/vms/" + uuid + "/settings", `{"operation_id":"operation-settings-route","root_password":"NeverPersistThis!123","confirm_restart":true}`, operation.ExpectedUpdateSettings}, {"operation-change-network", http.MethodPost, "/v1/vms/" + uuid + "/network", `{"operation_id":"operation-change-network","network":"public","public_pool_key":"0123456789abcdef0123456789abcdef","public_pool_cidr":"8.8.8.0/24","public_ip_cidr":"8.8.8.10/24","gateway":"8.8.8.1","dns_servers":["1.1.1.1"],"confirm_restart":true}`, operation.ExpectedChangeNetwork}, {"operation-network-route", http.MethodPost, "/v1/vms/" + uuid + "/network/resume", `{"operation_id":"operation-network-route"}`, operation.ExpectedNetworkState}, {"operation-service-route", http.MethodPost, "/v1/vms/" + uuid + "/service/resume", `{"operation_id":"operation-service-route","start_if_suspended":true}`, operation.ExpectedServiceState}, {"operation-firewall-route", http.MethodPut, "/v1/vms/" + uuid + "/firewall", `{"operation_id":"operation-firewall-route","revision":1,"default_in":"drop","default_out":"accept","rules":[]}`, operation.ExpectedFirewall}, } for _, test := range tests { actor := auth.Actor{UserID: 1, CanManage: true, OperationID: test.operationID} response := api.request(t, test.method, test.path, []byte(test.body), actor) if response.Code != http.StatusNotImplemented { t.Fatalf("%s status=%d body=%s", test.kind, response.Code, response.Body.String()) } record, err := api.journal.Get(test.operationID) if err != nil || record.ExpectedState == nil || record.ExpectedState.Kind != test.kind || record.ExpectedState.TransitionID != test.operationID { t.Fatalf("%s record=%#v err=%v", test.kind, record, err) } if test.kind == operation.ExpectedChangeNetwork && (record.ExpectedState.NetworkSourceType != "bridge" || record.ExpectedState.NetworkSourceName != "br0") { t.Fatalf("network source was not durably resolved: %#v", record.ExpectedState) } encoded, err := json.Marshal(record.ExpectedState) if err != nil { t.Fatal(err) } if bytes.Contains(encoded, []byte("NeverPersistThis!123")) || bytes.Contains(encoded, []byte("ssh-")) { t.Fatalf("%s expected state contains a credential: %s", test.kind, encoded) } } } func TestConsoleTicketIsHMACProtectedAndDoesNotExposeTarget(t *testing.T) { api := newTestAPI(t) actor := auth.Actor{UserID: 1, CanManage: true} response := api.request(t, http.MethodPost, "/v1/vms/123e4567-e89b-42d3-a456-426614174000/console-ticket", nil, actor) if response.Code != http.StatusCreated { t.Fatalf("console ticket status = %d body=%s", response.Code, response.Body.String()) } if strings.Contains(response.Body.String(), "127.0.0.1") || strings.Contains(response.Body.String(), "/v1/console") || strings.Contains(response.Body.String(), "protocol") || !strings.Contains(response.Body.String(), `"ticket"`) { t.Fatalf("unsafe console ticket response: %s", response.Body.String()) } } func TestBodyLimitIsEnforcedBeforeAuthentication(t *testing.T) { api := newTestAPI(t) api.handler.MaxBodyBytes = 1024 request := httptest.NewRequest(http.MethodPost, "/v1/vms", bytes.NewReader(make([]byte, 1025))) response := httptest.NewRecorder() api.handler.ServeHTTP(response, request) if response.Code != http.StatusRequestEntityTooLarge { t.Fatalf("oversized body status = %d body=%s", response.Code, response.Body.String()) } } func TestCatalogImportsAreDurableAsyncAndAcceptPrivateSourceToken(t *testing.T) { api := newTestAPI(t) token := strings.Repeat("a", 64) tests := []struct { name string target string actor auth.Actor body string check func(*testing.T) }{ { name: "template", target: "/v1/templates", actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-import"}, body: `{"operation_id":"operation-template-import","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","source_token":"` + token + `","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`, check: func(t *testing.T) { select { case request := <-api.hv.templateCalls: if request.SHA256 != "" || request.SourceToken != token { t.Fatalf("template request = %#v", request) } case <-time.After(time.Second): t.Fatal("template import was not dispatched") } }, }, { name: "rescue", target: "/v1/rescue-templates", actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-rescue-import"}, body: `{"operation_id":"operation-rescue-import","name":"Debian Rescue","source_url":"https://images.example.com/rescue.iso","source_token":"` + token + `"}`, check: func(t *testing.T) { select { case request := <-api.hv.rescueCalls: if !catalogIDPattern.MatchString(request.ID) || request.SourceToken != token { t.Fatalf("rescue request = %#v", request) } case <-time.After(time.Second): t.Fatal("rescue import was not dispatched") } }, }, { name: "user ISO", target: "/v1/user-isos", actor: auth.Actor{UserID: 42, CanManage: true, OperationID: "operation-useriso-import"}, body: `{"operation_id":"operation-useriso-import","owner_id":42,"name":"Installer","source_url":"https://images.example.com/user.iso","source_token":"` + token + `"}`, check: func(t *testing.T) { select { case request := <-api.hv.userISOCalls: if !catalogIDPattern.MatchString(request.ID) || request.OwnerID != 42 || request.SourceToken != token { t.Fatalf("user ISO request = %#v", request) } case <-time.After(time.Second): t.Fatal("user ISO import was not dispatched") } }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { response := api.request(t, http.MethodPost, test.target, []byte(test.body), test.actor) if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"status":"dispatching"`) { t.Fatalf("async import response = %d %s", response.Code, response.Body.String()) } test.check(t) }) } } func TestAsyncImportNeverReturnsAcceptedWhileAnotherCallerOwnsStartup(t *testing.T) { api := newTestAPI(t) const operationID = "operation-import-start-race" if !api.handler.claimAsync(operationID) { t.Fatal("could not reserve async startup claim") } defer api.handler.releaseAsync(operationID) body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`) response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{ UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID, }) if response.Code != http.StatusServiceUnavailable || !strings.Contains(response.Body.String(), `"code":"operation_starting"`) || strings.Contains(response.Body.String(), `"status":"dispatching"`) { t.Fatalf("startup loser response = %d %s", response.Code, response.Body.String()) } record, err := api.journal.Get(operationID) if err != nil || record.State != operation.StateReserved { t.Fatalf("startup loser journal = %#v err=%v", record, err) } } func TestImportSourceTokenRejectsShortOrNonURLSafeValues(t *testing.T) { for _, token := range []string{"too-short", strings.Repeat("a", 31), strings.Repeat("a", 32) + "+"} { if err := validateImportSource("https://images.example.com/image.iso", token, ""); err == nil { t.Fatalf("token %q unexpectedly accepted", token) } } if err := validateImportSource("https://images.example.com/image.iso", strings.Repeat("a", 32), ""); err != nil { t.Fatalf("valid token rejected: %v", err) } } func TestCatalogDeleteResultsContainExactManagedCleanupEvidence(t *testing.T) { api := newTestAPI(t) id := "0123456789abcdef0123456789abcdef" tests := []struct { name string target string operationID string resourceType string resourceID string ownerID int64 actor auth.Actor }{ { name: "template", target: "/v1/templates/debian-12", operationID: "operation-delete-template", resourceType: "template", resourceID: "debian-12", actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true}, }, { name: "rescue template", target: "/v1/rescue-templates/" + id, operationID: "operation-delete-rescue", resourceType: "rescue_template", resourceID: id, actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true}, }, { name: "user ISO", target: "/v1/user-isos/" + id, operationID: "operation-delete-useriso", resourceType: "user_iso", resourceID: id, ownerID: 42, actor: auth.Actor{UserID: 42, CanManage: true}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { test.actor.OperationID = test.operationID body := []byte(`{"operation_id":"` + test.operationID + `"}`) response := api.request(t, http.MethodDelete, test.target, body, test.actor) if response.Code != http.StatusOK { t.Fatalf("delete response = %d %s", response.Code, response.Body.String()) } var wire struct { Status operation.State `json:"status"` Result hypervisor.CatalogDeleteResult `json:"result"` } if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil { t.Fatal(err) } if wire.Status != operation.StateSucceeded || wire.Result.ResourceType != test.resourceType || wire.Result.ID != test.resourceID || wire.Result.OwnerID != test.ownerID || !wire.Result.Deleted || !wire.Result.ManagedImageAbsent || !wire.Result.ManagedMetadataAbsent || !wire.Result.ManagedArtifactsAbsent { t.Fatalf("delete result = %#v", wire) } }) } } func TestCatalogDeleteNotFoundCodesAreTerminalDurableFailures(t *testing.T) { for _, code := range []string{"template_not_found", "rescue_template_not_found", "user_iso_not_found"} { t.Run(code, func(t *testing.T) { response := httptest.NewRecorder() writeOperationFailure(response, operation.Record{ OperationID: "operation-delete-missing", State: operation.StateFailed, Error: &operation.PublicError{Code: code, Message: "managed catalog resource was not found"}, }, false) if response.Code != http.StatusNotFound { t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) } var wire operationResponse if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil { t.Fatal(err) } if wire.Status != operation.StateFailed || wire.Error == nil || wire.Error.Code != code || wire.Retryable || len(wire.Result) != 0 { t.Fatalf("failure = %#v", wire) } }) } } func TestForceStartRescueRequiresExplicitDataLossAcknowledgement(t *testing.T) { api := newTestAPI(t) uuid := "123e4567-e89b-42d3-a456-426614174000" target := "/v1/vms/" + uuid + "/rescue/force-start" rejectedActor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rejected"} rejected := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rejected","ack_data_loss":false}`), rejectedActor) if rejected.Code != http.StatusUnprocessableEntity || !strings.Contains(rejected.Body.String(), `"code":"data_loss_ack_required"`) { t.Fatalf("force-start without acknowledgement = %d %s", rejected.Code, rejected.Body.String()) } select { case call := <-api.hv.forceCalls: t.Fatalf("rejected force-start reached backend: %#v", call) default: } actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rescue"} accepted := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rescue","ack_data_loss":true}`), actor) if accepted.Code != http.StatusAccepted || !strings.Contains(accepted.Body.String(), `"active":true`) { t.Fatalf("acknowledged force-start = %d %s", accepted.Code, accepted.Body.String()) } select { case call := <-api.hv.forceCalls: if !call.AckDataLoss || call.OperationID != actor.OperationID { t.Fatalf("force-start request = %#v", call) } default: t.Fatal("acknowledged force-start did not reach backend") } } func TestAsyncImportReplayDoesNotDispatchTwice(t *testing.T) { api := newTestAPI(t) release := make(chan struct{}) api.hv.templateWait = release actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-replay"} body := []byte(`{"operation_id":"operation-template-replay","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`) first := api.request(t, http.MethodPost, "/v1/templates", body, actor) if first.Code != http.StatusAccepted { close(release) t.Fatalf("first import response = %d %s", first.Code, first.Body.String()) } select { case <-api.hv.templateCalls: case <-time.After(time.Second): close(release) t.Fatal("first template import was not dispatched") } second := api.request(t, http.MethodPost, "/v1/templates", body, actor) if second.Code != http.StatusAccepted || !strings.Contains(second.Body.String(), `"replayed":true`) { close(release) t.Fatalf("replayed import response = %d %s", second.Code, second.Body.String()) } select { case duplicate := <-api.hv.templateCalls: close(release) t.Fatalf("template import dispatched twice: %#v", duplicate) case <-time.After(100 * time.Millisecond): } close(release) deadline := time.Now().Add(time.Second) for { record, err := api.handler.Journal.Get(actor.OperationID) if err == nil && record.State == operation.StateSucceeded { break } if time.Now().After(deadline) { t.Fatalf("async import did not finish after release: %#v err=%v", record, err) } time.Sleep(10 * time.Millisecond) } } func TestExplicitReplayResumesReservedImportAfterRestartBoundary(t *testing.T) { api := newTestAPI(t) actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-resume"} body := []byte(`{"operation_id":"operation-template-resume","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`) _, _, err := api.handler.Journal.Reserve(operation.Spec{ OperationID: actor.OperationID, ResourceID: "template:debian-12", Action: "import-template", Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/templates", body), }) if err != nil { t.Fatal(err) } response := api.request(t, http.MethodPost, "/v1/templates", body, actor) if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"replayed":true`) { t.Fatalf("reserved replay response = %d %s", response.Code, response.Body.String()) } select { case <-api.hv.templateCalls: case <-time.After(time.Second): t.Fatal("reserved import was not resumed by explicit authenticated replay") } }