You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/api/server_test.go

858 lines
35 KiB
Go

package api
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.luoli.live/root/virt/agent/auth"
consolebroker "git.luoli.live/root/virt/agent/console"
"git.luoli.live/root/virt/agent/hypervisor"
"git.luoli.live/root/virt/agent/operation"
)
type fakeHypervisor struct {
hypervisor.Unsupported
mu sync.Mutex
creates int
reconciles int
lastReconcile operation.Record
reconcileTo operation.Resolution
reconcileErr error
templateCalls chan hypervisor.ImportTemplateRequest
templateWait <-chan struct{}
rescueCalls chan hypervisor.ImportRescueTemplateRequest
userISOCalls chan hypervisor.ImportUserISORequest
forceCalls chan hypervisor.ForceRescueStartRequest
powerCalls chan hypervisor.PowerRequest
}
type failingNonceStore struct{}
func (failingNonceStore) UseNonce(string, time.Time, time.Time) (bool, error) {
return false, errors.New("nonce store failed")
}
func (f *fakeHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) {
return []hypervisor.VMMetadata{}, nil
}
func (f *fakeHypervisor) Ping(context.Context) error { return nil }
func (f *fakeHypervisor) Capabilities(context.Context) (hypervisor.Capabilities, error) {
return hypervisor.Capabilities{
Backend: "fake",
Features: hypervisor.FeatureSet{CreateVM: true, Console: true, UEFI: true},
PowerActions: []hypervisor.PowerAction{hypervisor.PowerStart},
Firmware: []string{"uefi"},
NetworkSelectors: []string{"public"},
}, nil
}
func (f *fakeHypervisor) CreateVM(_ context.Context, request hypervisor.CreateVMRequest) (hypervisor.VMMetadata, error) {
f.mu.Lock()
f.creates++
f.mu.Unlock()
return hypervisor.VMMetadata{
UUID: request.UUID, Name: request.Name, VCPUs: request.VCPUs,
MemoryMiB: request.MemoryMiB, Machine: "q35", Firmware: "uefi",
PublicIPCIDR: request.PublicIPCIDR, PublicPoolKey: request.PublicPoolKey,
}, nil
}
func (f *fakeHypervisor) Reconcile(_ context.Context, record operation.Record) (operation.Resolution, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.reconciles++
f.lastReconcile = record
return f.reconcileTo, f.reconcileErr
}
func (f *fakeHypervisor) GetConsole(context.Context, string) (hypervisor.ConsoleSession, error) {
return hypervisor.ConsoleSession{Protocol: "vnc", Address: "127.0.0.1:5900"}, nil
}
func (f *fakeHypervisor) ResolveNetworkSelector(_ context.Context, selector string) (hypervisor.LocalNetworkSource, error) {
if selector != "public" {
return hypervisor.LocalNetworkSource{}, hypervisor.NewError(hypervisor.ErrInvalid, "network_not_found", "network selector is not configured", nil)
}
return hypervisor.LocalNetworkSource{Type: "bridge", Source: "br0"}, nil
}
func (f *fakeHypervisor) ImportTemplate(_ context.Context, request hypervisor.ImportTemplateRequest) (hypervisor.ImageTemplate, error) {
f.templateCalls <- request
if f.templateWait != nil {
<-f.templateWait
}
return hypervisor.ImageTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("a", 64)}, nil
}
func (f *fakeHypervisor) ListRescueTemplates(context.Context) ([]hypervisor.RescueTemplate, error) {
return []hypervisor.RescueTemplate{}, nil
}
func (f *fakeHypervisor) ImportRescueTemplate(_ context.Context, request hypervisor.ImportRescueTemplateRequest) (hypervisor.RescueTemplate, error) {
f.rescueCalls <- request
return hypervisor.RescueTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("b", 64)}, nil
}
func (f *fakeHypervisor) ImportUserISO(_ context.Context, request hypervisor.ImportUserISORequest) (hypervisor.UserISO, error) {
f.userISOCalls <- request
return hypervisor.UserISO{ID: request.ID, OwnerID: request.OwnerID, Name: request.Name, SHA256: strings.Repeat("c", 64)}, nil
}
func (f *fakeHypervisor) DeleteTemplate(context.Context, string) error { return nil }
func (f *fakeHypervisor) DeleteRescueTemplate(context.Context, string) error { return nil }
func (f *fakeHypervisor) DeleteUserISO(context.Context, string, int64) error { return nil }
func (f *fakeHypervisor) ForceStartRescue(_ context.Context, _ string, request hypervisor.ForceRescueStartRequest) (hypervisor.RescueState, error) {
f.forceCalls <- request
return hypervisor.RescueState{Active: true, TemplateID: "0123456789abcdef0123456789abcdef"}, nil
}
func (f *fakeHypervisor) Power(_ context.Context, _ string, request hypervisor.PowerRequest) (hypervisor.VMState, error) {
f.powerCalls <- request
return hypervisor.VMState{State: hypervisor.VMStateShutoff}, nil
}
type testAPI struct {
handler *Server
hv *fakeHypervisor
journal *operation.Journal
secret []byte
now time.Time
nonce int
}
func newTestAPI(t *testing.T) *testAPI {
t.Helper()
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = journal.Close() })
secret := []byte("0123456789abcdef0123456789abcdef")
now := time.Unix(2_000_000_000, 0)
authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal)
if err != nil {
t.Fatal(err)
}
authenticator.SetClockForTesting(func() time.Time { return now })
broker, err := consolebroker.NewBroker(time.Minute)
if err != nil {
t.Fatal(err)
}
hv := &fakeHypervisor{
templateCalls: make(chan hypervisor.ImportTemplateRequest, 1),
rescueCalls: make(chan hypervisor.ImportRescueTemplateRequest, 1),
userISOCalls: make(chan hypervisor.ImportUserISORequest, 1),
forceCalls: make(chan hypervisor.ForceRescueStartRequest, 1),
powerCalls: make(chan hypervisor.PowerRequest, 1),
}
handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = handler.Shutdown(ctx)
_ = journal.Wait(ctx)
})
return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now}
}
func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder {
t.Helper()
a.nonce++
request := httptest.NewRequest(method, target, bytes.NewReader(body))
nonce := fmt.Sprintf("nonce%011d", a.nonce)
if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
a.handler.ServeHTTP(response, request)
return response
}
func validCreateBody(operationID, name string) []byte {
rules := []hypervisor.PlatformNetworkPolicyRule{{
ID: "ssh", Label: "SSH", Category: "default", Protocols: []string{"tcp"},
PortFrom: 22, PortTo: 22, Directions: []string{"ingress"}, Reason: "default SSH policy",
}}
digestDocument := struct {
SchemaVersion uint64 `json:"schema_version"`
BasePolicyVersion uint64 `json:"base_policy_version"`
CatalogRevision uint64 `json:"catalog_revision"`
Rules []hypervisor.PlatformNetworkPolicyRule `json:"rules"`
}{SchemaVersion: 1, BasePolicyVersion: 1, CatalogRevision: 0, Rules: rules}
digestJSON, _ := json.Marshal(digestDocument)
digest := sha256.Sum256(digestJSON)
value := hypervisor.CreateVMRequest{
OperationID: operationID,
UUID: "123e4567-e89b-42d3-a456-426614174000",
Source: "cloud", Name: name, VCPUs: 2, MemoryMiB: 2048,
DiskGiB: 40, BandwidthMbps: 100, NetworkSelector: "public",
TemplateID: "debian-12", TemplateRevision: 1,
TemplateGenerationToken: "0123456789abcdef0123456789abcdef",
RootPassword: "StrongPassword!123", PublicPoolKey: "0123456789abcdef0123456789abcdef",
PublicPoolCIDR: "8.8.8.0/24", PublicIPCIDR: "8.8.8.10/24", Gateway: "8.8.8.1",
DNSServers: []string{"1.1.1.1"},
PlatformNetworkPolicy: hypervisor.PlatformNetworkPolicySnapshot{
SchemaVersion: 1, BasePolicyVersion: 1, Rules: rules,
Digest: hex.EncodeToString(digest[:]),
},
}
encoded, _ := json.Marshal(value)
return encoded
}
func TestListResponsesUseNamedNonNullEnvelope(t *testing.T) {
api := newTestAPI(t)
response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1})
if response.Code != http.StatusOK || strings.TrimSpace(response.Body.String()) != `{"vms":[]}` {
t.Fatalf("VM list response = %d %s", response.Code, response.Body.String())
}
}
func TestCapabilitiesExposeStableNetworkSelectorKeys(t *testing.T) {
api := newTestAPI(t)
response := api.request(t, http.MethodGet, "/v1/capabilities", nil,
auth.Actor{UserID: 1})
if response.Code != http.StatusOK {
t.Fatalf("capabilities response = %d %s",
response.Code, response.Body.String())
}
var payload struct {
NetworkSelectors []string `json:"network_selectors"`
}
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if len(payload.NetworkSelectors) != 1 ||
payload.NetworkSelectors[0] != "public" {
t.Fatalf("network selectors = %#v", payload.NetworkSelectors)
}
}
func TestReinstallRejectsStartupScript(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-reinstall"}
body := []byte(`{"operation_id":"operation-reinstall","template_id":"debian-12","template_revision":1,"template_generation_token":"0123456789abcdef0123456789abcdef","startup_script":"echo forbidden"}`)
response := api.request(t, http.MethodPost,
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/reinstall", body, actor)
if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), `"code":"invalid_json"`) ||
!strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) {
t.Fatalf("reinstall startup script response = %d %s", response.Code, response.Body.String())
}
}
func TestPowerDestroyWireActionIsAccepted(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-poweroff"}
body := []byte(`{"action":"destroy"}`)
response := api.request(t, http.MethodPost,
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/actions", body, actor)
if response.Code != http.StatusOK {
t.Fatalf("destroy action response = %d %s", response.Code, response.Body.String())
}
select {
case request := <-api.hv.powerCalls:
if request.Action != hypervisor.PowerDestroy {
t.Fatalf("power action = %q, want destroy", request.Action)
}
default:
t.Fatal("destroy action did not reach the hypervisor")
}
}
func TestEveryHTTPRouteRequiresHMACIncludingHealth(t *testing.T) {
api := newTestAPI(t)
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
response := httptest.NewRecorder()
api.handler.ServeHTTP(response, request)
if response.Code != http.StatusUnauthorized ||
!strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) {
t.Fatalf("health without HMAC status = %d, body=%s", response.Code, response.Body.String())
}
response = api.request(t, http.MethodGet, "/v1/health", nil, auth.Actor{UserID: 1})
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"service":"virt-host-agent"`) {
t.Fatalf("signed health status = %d, body=%s", response.Code, response.Body.String())
}
}
func TestReplayAuthenticationFailureDoesNotClaimNotDispatched(t *testing.T) {
api := newTestAPI(t)
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, api.secret,
api.now.Unix(), "fixed-replay-nonce-1234"); err != nil {
t.Fatal(err)
}
first := httptest.NewRecorder()
api.handler.ServeHTTP(first, request)
if first.Code != http.StatusOK {
t.Fatalf("first request status = %d body=%s", first.Code, first.Body.String())
}
replay := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
if err := auth.Sign(replay, nil, auth.Actor{UserID: 1}, api.secret,
api.now.Unix(), "fixed-replay-nonce-1234"); err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
api.handler.ServeHTTP(response, replay)
if response.Code != http.StatusUnauthorized ||
!strings.Contains(response.Body.String(), `"code":"replayed_request"`) ||
strings.Contains(response.Body.String(), `"dispatch_state"`) {
t.Fatalf("replay response = %d %s", response.Code, response.Body.String())
}
}
func TestTerminalPreJournalContractIsExplicit(t *testing.T) {
api := newTestAPI(t)
tests := []struct {
name string
method string
target string
body []byte
actor auth.Actor
status int
code string
}{
{
name: "forbidden mutation", method: http.MethodPost, target: "/v1/vms",
body: []byte(`{}`), actor: auth.Actor{UserID: 1},
status: http.StatusForbidden, code: "forbidden",
},
{
name: "route missing", method: http.MethodPost, target: "/v1/missing",
body: []byte(`{}`), actor: auth.Actor{UserID: 1, CanManage: true},
status: http.StatusNotFound, code: "not_found",
},
{
name: "method mismatch", method: http.MethodDelete, target: "/v1/health",
actor: auth.Actor{UserID: 1, CanManage: true},
status: http.StatusMethodNotAllowed, code: "method_not_allowed",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
response := api.request(t, test.method, test.target, test.body, test.actor)
var payload errorEnvelope
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if response.Code != test.status || payload.Error.Code != test.code ||
payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" ||
payload.Error.Operation != nil {
t.Fatalf("response = %d %#v", response.Code, payload)
}
})
}
}
func TestRetryablePreJournalContractIsExplicit(t *testing.T) {
t.Run("shutting down", func(t *testing.T) {
api := newTestAPI(t)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := api.handler.Shutdown(ctx); err != nil {
t.Fatal(err)
}
response := api.request(t, http.MethodGet, "/v1/health", nil,
auth.Actor{UserID: 1})
assertRetryableNotDispatched(t, response, "agent_shutting_down")
})
t.Run("nonce store unavailable", func(t *testing.T) {
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
defer journal.Close()
secret := []byte("0123456789abcdef0123456789abcdef")
now := time.Unix(2_000_000_000, 0)
authenticator, err := auth.NewWithNonceStore(
secret, 5*time.Minute, 10*time.Minute, failingNonceStore{},
)
if err != nil {
t.Fatal(err)
}
authenticator.SetClockForTesting(func() time.Time { return now })
broker, err := consolebroker.NewBroker(time.Minute)
if err != nil {
t.Fatal(err)
}
server, err := NewServer(authenticator, journal, &fakeHypervisor{},
broker, 64<<10, time.Second)
if err != nil {
t.Fatal(err)
}
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = server.Shutdown(ctx)
}()
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, secret,
now.Unix(), "nonce-store-failure-1234"); err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
server.ServeHTTP(response, request)
assertRetryableNotDispatched(t, response, "nonce_store_unavailable")
})
}
func assertRetryableNotDispatched(t *testing.T, response *httptest.ResponseRecorder, code string) {
t.Helper()
var payload errorEnvelope
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if response.Code != http.StatusServiceUnavailable ||
payload.Error.Code != code || !payload.Error.Retryable ||
payload.Error.DispatchState != "not_dispatched" ||
payload.Error.Operation != nil {
t.Fatalf("response = %d %#v", response.Code, payload)
}
}
func TestCreateVMOperationIsDurablyIdempotentAndConflictsOnPayloadChange(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-create-one"}
body := validCreateBody(actor.OperationID, "vm-one")
first := api.request(t, http.MethodPost, "/v1/vms", body, actor)
if first.Code != http.StatusCreated {
t.Fatalf("first create status = %d body=%s", first.Code, first.Body.String())
}
if !strings.Contains(first.Body.String(), `"public_pool_key":"0123456789abcdef0123456789abcdef"`) ||
!strings.Contains(first.Body.String(), `"public_ip_cidr":"8.8.8.10/24"`) {
t.Fatalf("create response omitted confirmed network allocation: %s", first.Body.String())
}
second := api.request(t, http.MethodPost, "/v1/vms", body, actor)
if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), `"replayed":true`) {
t.Fatalf("replay status = %d body=%s", second.Code, second.Body.String())
}
api.hv.mu.Lock()
creates := api.hv.creates
api.hv.mu.Unlock()
if creates != 1 {
t.Fatalf("backend create calls = %d, want 1", creates)
}
conflict := api.request(t, http.MethodPost, "/v1/vms", validCreateBody(actor.OperationID, "vm-different"), actor)
if conflict.Code != http.StatusConflict ||
!strings.Contains(conflict.Body.String(), "operation_id_conflict") ||
strings.Contains(conflict.Body.String(), `"dispatch_state"`) {
t.Fatalf("conflict status = %d body=%s", conflict.Code, conflict.Body.String())
}
}
func TestFailedCreateRouteUsesReadOnlyReconciliationWithoutReplay(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-create-api-failed"
uuid := "123e4567-e89b-42d3-a456-426614174000"
spec := operation.Spec{
OperationID: operationID, ResourceID: uuid, Action: "create-vm",
Fingerprint: operation.Fingerprint("POST", "/v1/vms", []byte(`{}`)),
Destructive: true,
}
if record, _, err := api.journal.Execute(context.Background(), spec,
func(context.Context) (json.RawMessage, error) {
return nil, hypervisor.NewError(hypervisor.ErrInvalid, "network_selector_invalid", "network selector is not configured", nil)
}); err == nil || record.State != operation.StateFailed {
t.Fatalf("initial failed create = %#v err=%v", record, err)
}
api.hv.mu.Lock()
api.hv.reconcileTo = operation.Resolution{
Resolved: true, Succeeded: false, ErrorCode: "creation_absent",
Result: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`),
}
api.hv.mu.Unlock()
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: operationID}
response := api.request(t, http.MethodPost,
"/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
if response.Code != http.StatusOK ||
!strings.Contains(response.Body.String(), `"status":"failed"`) ||
!strings.Contains(response.Body.String(), `"cause_code":"network_selector_invalid"`) ||
!strings.Contains(response.Body.String(), `"managed_artifacts_absent":true`) {
t.Fatalf("failed create reconcile response = %d %s", response.Code, response.Body.String())
}
api.hv.mu.Lock()
reconciles := api.hv.reconciles
last := api.hv.lastReconcile
creates := api.hv.creates
api.hv.mu.Unlock()
if reconciles != 1 || creates != 0 || last.OperationID != operationID || last.ResourceID != uuid || last.Action != "create-vm" {
t.Fatalf("reconcile calls=%d creates=%d record=%#v", reconciles, creates, last)
}
second := api.request(t, http.MethodPost,
"/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
api.hv.mu.Lock()
reconciles = api.hv.reconciles
api.hv.mu.Unlock()
if second.Code != http.StatusOK || reconciles != 1 {
t.Fatalf("confirmed absence was reinspected: status=%d calls=%d body=%s",
second.Code, reconciles, second.Body.String())
}
}
func TestUnsupportedTypedMutationUsesStable501Contract(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-network-one"}
body := []byte(`{"operation_id":"operation-network-one","reason":"quota"}`)
response := api.request(t, http.MethodPost,
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/network/suspend", body, actor)
if response.Code != http.StatusNotImplemented || !strings.Contains(response.Body.String(), `"code":"unsupported"`) {
t.Fatalf("unsupported status = %d body=%s", response.Code, response.Body.String())
}
}
func TestReconciliableMutationRoutesPersistOnlySafeExpectedState(t *testing.T) {
api := newTestAPI(t)
uuid := "123e4567-e89b-42d3-a456-426614174000"
tests := []struct {
operationID string
method string
path string
body string
kind string
}{
{"operation-settings-route", http.MethodPatch, "/v1/vms/" + uuid + "/settings", `{"operation_id":"operation-settings-route","root_password":"NeverPersistThis!123","confirm_restart":true}`, operation.ExpectedUpdateSettings},
{"operation-change-network", http.MethodPost, "/v1/vms/" + uuid + "/network", `{"operation_id":"operation-change-network","network":"public","public_pool_key":"0123456789abcdef0123456789abcdef","public_pool_cidr":"8.8.8.0/24","public_ip_cidr":"8.8.8.10/24","gateway":"8.8.8.1","dns_servers":["1.1.1.1"],"confirm_restart":true}`, operation.ExpectedChangeNetwork},
{"operation-network-route", http.MethodPost, "/v1/vms/" + uuid + "/network/resume", `{"operation_id":"operation-network-route"}`, operation.ExpectedNetworkState},
{"operation-service-route", http.MethodPost, "/v1/vms/" + uuid + "/service/resume", `{"operation_id":"operation-service-route","start_if_suspended":true}`, operation.ExpectedServiceState},
{"operation-firewall-route", http.MethodPut, "/v1/vms/" + uuid + "/firewall", `{"operation_id":"operation-firewall-route","revision":1,"default_in":"drop","default_out":"accept","rules":[]}`, operation.ExpectedFirewall},
}
for _, test := range tests {
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: test.operationID}
response := api.request(t, test.method, test.path, []byte(test.body), actor)
if response.Code != http.StatusNotImplemented {
t.Fatalf("%s status=%d body=%s", test.kind, response.Code, response.Body.String())
}
record, err := api.journal.Get(test.operationID)
if err != nil || record.ExpectedState == nil || record.ExpectedState.Kind != test.kind || record.ExpectedState.TransitionID != test.operationID {
t.Fatalf("%s record=%#v err=%v", test.kind, record, err)
}
if test.kind == operation.ExpectedChangeNetwork && (record.ExpectedState.NetworkSourceType != "bridge" || record.ExpectedState.NetworkSourceName != "br0") {
t.Fatalf("network source was not durably resolved: %#v", record.ExpectedState)
}
encoded, err := json.Marshal(record.ExpectedState)
if err != nil {
t.Fatal(err)
}
if bytes.Contains(encoded, []byte("NeverPersistThis!123")) || bytes.Contains(encoded, []byte("ssh-")) {
t.Fatalf("%s expected state contains a credential: %s", test.kind, encoded)
}
}
}
func TestConsoleTicketIsHMACProtectedAndDoesNotExposeTarget(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true}
response := api.request(t, http.MethodPost,
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/console-ticket", nil, actor)
if response.Code != http.StatusCreated {
t.Fatalf("console ticket status = %d body=%s", response.Code, response.Body.String())
}
if strings.Contains(response.Body.String(), "127.0.0.1") ||
strings.Contains(response.Body.String(), "/v1/console") ||
strings.Contains(response.Body.String(), "protocol") ||
!strings.Contains(response.Body.String(), `"ticket"`) {
t.Fatalf("unsafe console ticket response: %s", response.Body.String())
}
}
func TestBodyLimitIsEnforcedBeforeAuthentication(t *testing.T) {
api := newTestAPI(t)
api.handler.MaxBodyBytes = 1024
request := httptest.NewRequest(http.MethodPost, "/v1/vms", bytes.NewReader(make([]byte, 1025)))
response := httptest.NewRecorder()
api.handler.ServeHTTP(response, request)
if response.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("oversized body status = %d body=%s", response.Code, response.Body.String())
}
}
func TestCatalogImportsAreDurableAsyncAndAcceptPrivateSourceToken(t *testing.T) {
api := newTestAPI(t)
token := strings.Repeat("a", 64)
tests := []struct {
name string
target string
actor auth.Actor
body string
check func(*testing.T)
}{
{
name: "template", target: "/v1/templates",
actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-import"},
body: `{"operation_id":"operation-template-import","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","source_token":"` + token + `","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`,
check: func(t *testing.T) {
select {
case request := <-api.hv.templateCalls:
if request.SHA256 != "" || request.SourceToken != token {
t.Fatalf("template request = %#v", request)
}
case <-time.After(time.Second):
t.Fatal("template import was not dispatched")
}
},
},
{
name: "rescue", target: "/v1/rescue-templates",
actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-rescue-import"},
body: `{"operation_id":"operation-rescue-import","name":"Debian Rescue","source_url":"https://images.example.com/rescue.iso","source_token":"` + token + `"}`,
check: func(t *testing.T) {
select {
case request := <-api.hv.rescueCalls:
if !catalogIDPattern.MatchString(request.ID) || request.SourceToken != token {
t.Fatalf("rescue request = %#v", request)
}
case <-time.After(time.Second):
t.Fatal("rescue import was not dispatched")
}
},
},
{
name: "user ISO", target: "/v1/user-isos",
actor: auth.Actor{UserID: 42, CanManage: true, OperationID: "operation-useriso-import"},
body: `{"operation_id":"operation-useriso-import","owner_id":42,"name":"Installer","source_url":"https://images.example.com/user.iso","source_token":"` + token + `"}`,
check: func(t *testing.T) {
select {
case request := <-api.hv.userISOCalls:
if !catalogIDPattern.MatchString(request.ID) || request.OwnerID != 42 || request.SourceToken != token {
t.Fatalf("user ISO request = %#v", request)
}
case <-time.After(time.Second):
t.Fatal("user ISO import was not dispatched")
}
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
response := api.request(t, http.MethodPost, test.target, []byte(test.body), test.actor)
if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"status":"dispatching"`) {
t.Fatalf("async import response = %d %s", response.Code, response.Body.String())
}
test.check(t)
})
}
}
func TestAsyncImportNeverReturnsAcceptedWhileAnotherCallerOwnsStartup(t *testing.T) {
api := newTestAPI(t)
const operationID = "operation-import-start-race"
if !api.handler.claimAsync(operationID) {
t.Fatal("could not reserve async startup claim")
}
defer api.handler.releaseAsync(operationID)
body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
})
if response.Code != http.StatusServiceUnavailable ||
!strings.Contains(response.Body.String(), `"code":"operation_starting"`) ||
strings.Contains(response.Body.String(), `"status":"dispatching"`) {
t.Fatalf("startup loser response = %d %s", response.Code, response.Body.String())
}
record, err := api.journal.Get(operationID)
if err != nil || record.State != operation.StateReserved {
t.Fatalf("startup loser journal = %#v err=%v", record, err)
}
}
func TestImportSourceTokenRejectsShortOrNonURLSafeValues(t *testing.T) {
for _, token := range []string{"too-short", strings.Repeat("a", 31), strings.Repeat("a", 32) + "+"} {
if err := validateImportSource("https://images.example.com/image.iso", token, ""); err == nil {
t.Fatalf("token %q unexpectedly accepted", token)
}
}
if err := validateImportSource("https://images.example.com/image.iso", strings.Repeat("a", 32), ""); err != nil {
t.Fatalf("valid token rejected: %v", err)
}
}
func TestCatalogDeleteResultsContainExactManagedCleanupEvidence(t *testing.T) {
api := newTestAPI(t)
id := "0123456789abcdef0123456789abcdef"
tests := []struct {
name string
target string
operationID string
resourceType string
resourceID string
ownerID int64
actor auth.Actor
}{
{
name: "template", target: "/v1/templates/debian-12", operationID: "operation-delete-template",
resourceType: "template", resourceID: "debian-12", actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true},
},
{
name: "rescue template", target: "/v1/rescue-templates/" + id, operationID: "operation-delete-rescue",
resourceType: "rescue_template", resourceID: id, actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true},
},
{
name: "user ISO", target: "/v1/user-isos/" + id, operationID: "operation-delete-useriso",
resourceType: "user_iso", resourceID: id, ownerID: 42, actor: auth.Actor{UserID: 42, CanManage: true},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
test.actor.OperationID = test.operationID
body := []byte(`{"operation_id":"` + test.operationID + `"}`)
response := api.request(t, http.MethodDelete, test.target, body, test.actor)
if response.Code != http.StatusOK {
t.Fatalf("delete response = %d %s", response.Code, response.Body.String())
}
var wire struct {
Status operation.State `json:"status"`
Result hypervisor.CatalogDeleteResult `json:"result"`
}
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
t.Fatal(err)
}
if wire.Status != operation.StateSucceeded || wire.Result.ResourceType != test.resourceType || wire.Result.ID != test.resourceID ||
wire.Result.OwnerID != test.ownerID || !wire.Result.Deleted || !wire.Result.ManagedImageAbsent ||
!wire.Result.ManagedMetadataAbsent || !wire.Result.ManagedArtifactsAbsent {
t.Fatalf("delete result = %#v", wire)
}
})
}
}
func TestCatalogDeleteNotFoundCodesAreTerminalDurableFailures(t *testing.T) {
for _, code := range []string{"template_not_found", "rescue_template_not_found", "user_iso_not_found"} {
t.Run(code, func(t *testing.T) {
response := httptest.NewRecorder()
writeOperationFailure(response, operation.Record{
OperationID: "operation-delete-missing", State: operation.StateFailed,
Error: &operation.PublicError{Code: code, Message: "managed catalog resource was not found"},
}, false)
if response.Code != http.StatusNotFound {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
var wire operationResponse
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
t.Fatal(err)
}
if wire.Status != operation.StateFailed || wire.Error == nil || wire.Error.Code != code || wire.Retryable || len(wire.Result) != 0 {
t.Fatalf("failure = %#v", wire)
}
})
}
}
func TestForceStartRescueRequiresExplicitDataLossAcknowledgement(t *testing.T) {
api := newTestAPI(t)
uuid := "123e4567-e89b-42d3-a456-426614174000"
target := "/v1/vms/" + uuid + "/rescue/force-start"
rejectedActor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rejected"}
rejected := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rejected","ack_data_loss":false}`), rejectedActor)
if rejected.Code != http.StatusUnprocessableEntity || !strings.Contains(rejected.Body.String(), `"code":"data_loss_ack_required"`) {
t.Fatalf("force-start without acknowledgement = %d %s", rejected.Code, rejected.Body.String())
}
select {
case call := <-api.hv.forceCalls:
t.Fatalf("rejected force-start reached backend: %#v", call)
default:
}
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rescue"}
accepted := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rescue","ack_data_loss":true}`), actor)
if accepted.Code != http.StatusAccepted || !strings.Contains(accepted.Body.String(), `"active":true`) {
t.Fatalf("acknowledged force-start = %d %s", accepted.Code, accepted.Body.String())
}
select {
case call := <-api.hv.forceCalls:
if !call.AckDataLoss || call.OperationID != actor.OperationID {
t.Fatalf("force-start request = %#v", call)
}
default:
t.Fatal("acknowledged force-start did not reach backend")
}
}
func TestAsyncImportReplayDoesNotDispatchTwice(t *testing.T) {
api := newTestAPI(t)
release := make(chan struct{})
api.hv.templateWait = release
actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-replay"}
body := []byte(`{"operation_id":"operation-template-replay","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
first := api.request(t, http.MethodPost, "/v1/templates", body, actor)
if first.Code != http.StatusAccepted {
close(release)
t.Fatalf("first import response = %d %s", first.Code, first.Body.String())
}
select {
case <-api.hv.templateCalls:
case <-time.After(time.Second):
close(release)
t.Fatal("first template import was not dispatched")
}
second := api.request(t, http.MethodPost, "/v1/templates", body, actor)
if second.Code != http.StatusAccepted || !strings.Contains(second.Body.String(), `"replayed":true`) {
close(release)
t.Fatalf("replayed import response = %d %s", second.Code, second.Body.String())
}
select {
case duplicate := <-api.hv.templateCalls:
close(release)
t.Fatalf("template import dispatched twice: %#v", duplicate)
case <-time.After(100 * time.Millisecond):
}
close(release)
deadline := time.Now().Add(time.Second)
for {
record, err := api.handler.Journal.Get(actor.OperationID)
if err == nil && record.State == operation.StateSucceeded {
break
}
if time.Now().After(deadline) {
t.Fatalf("async import did not finish after release: %#v err=%v", record, err)
}
time.Sleep(10 * time.Millisecond)
}
}
func TestExplicitReplayResumesReservedImportAfterRestartBoundary(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-resume"}
body := []byte(`{"operation_id":"operation-template-resume","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
_, _, err := api.handler.Journal.Reserve(operation.Spec{
OperationID: actor.OperationID,
ResourceID: "template:debian-12",
Action: "import-template",
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/templates", body),
})
if err != nil {
t.Fatal(err)
}
response := api.request(t, http.MethodPost, "/v1/templates", body, actor)
if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"replayed":true`) {
t.Fatalf("reserved replay response = %d %s", response.Code, response.Body.String())
}
select {
case <-api.hv.templateCalls:
case <-time.After(time.Second):
t.Fatal("reserved import was not resumed by explicit authenticated replay")
}
}