You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/api/server_test.go

1690 lines
75 KiB
Go

package api
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"time"
"git.luoli.live/root/virt/agent/auth"
consolebroker "git.luoli.live/root/virt/agent/console"
"git.luoli.live/root/virt/agent/hypervisor"
"git.luoli.live/root/virt/agent/operation"
)
type fakeHypervisor struct {
hypervisor.Unsupported
mu sync.Mutex
creates int
lastCreate hypervisor.CreateVMRequest
deleteCalls chan hypervisor.DeleteVMRequest
deleteErr error
forceDeleteCalls chan hypervisor.ForceDeleteVMRequest
reconciles int
lastReconcile operation.Record
reconcileTo operation.Resolution
reconcileErr error
inspections int
lastInspection operation.Record
inspectionTo hypervisor.OperationInspection
inspectionErr error
abortCalls int
lastAbortToken string
abortTo hypervisor.OperationInspection
abortErr error
cleanupCalls int
lastCleanupPlan string
cleanupTo operation.Resolution
cleanupErr error
templateCalls chan hypervisor.ImportTemplateRequest
templateWait <-chan struct{}
rescueCalls chan hypervisor.ImportRescueTemplateRequest
userISOCalls chan hypervisor.ImportUserISORequest
forceCalls chan hypervisor.ForceRescueStartRequest
powerCalls chan hypervisor.PowerRequest
storageCalls int
lastStorage hypervisor.StorageConfigurationRequest
storageExpected operation.ExpectedState
}
type failingNonceStore struct{}
func (failingNonceStore) UseNonce(string, time.Time, time.Time) (bool, error) {
return false, errors.New("nonce store failed")
}
func (f *fakeHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) {
return []hypervisor.VMMetadata{}, nil
}
func (f *fakeHypervisor) Ping(context.Context) error { return nil }
func (f *fakeHypervisor) Capabilities(context.Context) (hypervisor.Capabilities, error) {
return hypervisor.Capabilities{
Backend: "fake",
Features: hypervisor.FeatureSet{CreateVM: true, Console: true, UEFI: true},
PowerActions: []hypervisor.PowerAction{hypervisor.PowerStart},
Firmware: []string{"uefi"},
NetworkSelectors: []string{"bridge:br0"},
}, nil
}
func (f *fakeHypervisor) ConfigureStorage(ctx context.Context, request hypervisor.StorageConfigurationRequest) (hypervisor.StorageConfigurationResult, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.storageCalls++
f.lastStorage = request
f.storageExpected, _ = operation.ExpectedStateFromContext(ctx)
return hypervisor.StorageConfigurationResult{Configured: true}, nil
}
func (f *fakeHypervisor) CreateVM(_ context.Context, request hypervisor.CreateVMRequest) (hypervisor.VMMetadata, error) {
f.mu.Lock()
f.creates++
f.lastCreate = request
f.mu.Unlock()
return hypervisor.VMMetadata{
UUID: request.UUID, Name: request.Name, VCPUs: request.VCPUs,
MemoryMiB: request.MemoryMiB, Machine: "q35", Firmware: "uefi",
PublicIPCIDR: request.PublicIPCIDR, PublicPoolKey: request.PublicPoolKey,
}, nil
}
func (f *fakeHypervisor) DeleteVM(_ context.Context, _ string, request hypervisor.DeleteVMRequest) (hypervisor.DeleteVMResult, error) {
f.deleteCalls <- request
if f.deleteErr != nil {
return hypervisor.DeleteVMResult{}, f.deleteErr
}
return hypervisor.DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true}, nil
}
func (f *fakeHypervisor) ForceDeleteVM(_ context.Context, _ string, request hypervisor.ForceDeleteVMRequest) (hypervisor.DeleteVMResult, error) {
f.forceDeleteCalls <- request
return hypervisor.DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true, Forced: true}, nil
}
func (f *fakeHypervisor) Reconcile(_ context.Context, record operation.Record) (operation.Resolution, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.reconciles++
f.lastReconcile = record
return f.reconcileTo, f.reconcileErr
}
func (f *fakeHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.inspections++
f.lastInspection = record
return f.inspectionTo, f.inspectionErr
}
func (f *fakeHypervisor) AbortOperationJob(_ context.Context, _ operation.Record, token string) (hypervisor.OperationInspection, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.abortCalls++
f.lastAbortToken = token
return f.abortTo, f.abortErr
}
func (f *fakeHypervisor) CleanupOperation(_ context.Context, _ operation.Record, planID string) (operation.Resolution, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.cleanupCalls++
f.lastCleanupPlan = planID
return f.cleanupTo, f.cleanupErr
}
func (f *fakeHypervisor) GetConsole(context.Context, string) (hypervisor.ConsoleSession, error) {
return hypervisor.ConsoleSession{Protocol: "vnc", Address: "127.0.0.1:5900"}, nil
}
func (f *fakeHypervisor) ResolveNetworkSelector(_ context.Context, selector string) (hypervisor.LocalNetworkSource, error) {
return hypervisor.ParseNetworkSelector(selector)
}
func (f *fakeHypervisor) ImportTemplate(_ context.Context, request hypervisor.ImportTemplateRequest) (hypervisor.ImageTemplate, error) {
f.templateCalls <- request
if f.templateWait != nil {
<-f.templateWait
}
return hypervisor.ImageTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("a", 64)}, nil
}
func (f *fakeHypervisor) ListRescueTemplates(context.Context) ([]hypervisor.RescueTemplate, error) {
return []hypervisor.RescueTemplate{}, nil
}
func (f *fakeHypervisor) ImportRescueTemplate(_ context.Context, request hypervisor.ImportRescueTemplateRequest) (hypervisor.RescueTemplate, error) {
f.rescueCalls <- request
return hypervisor.RescueTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("b", 64)}, nil
}
func (f *fakeHypervisor) ImportUserISO(_ context.Context, request hypervisor.ImportUserISORequest) (hypervisor.UserISO, error) {
f.userISOCalls <- request
return hypervisor.UserISO{ID: request.ID, OwnerID: request.OwnerID, Name: request.Name, SHA256: strings.Repeat("c", 64)}, nil
}
func (f *fakeHypervisor) DeleteTemplate(context.Context, string) error { return nil }
func (f *fakeHypervisor) DeleteRescueTemplate(context.Context, string) error { return nil }
func (f *fakeHypervisor) DeleteUserISO(context.Context, string, int64) error { return nil }
func (f *fakeHypervisor) ForceStartRescue(_ context.Context, _ string, request hypervisor.ForceRescueStartRequest) (hypervisor.RescueState, error) {
f.forceCalls <- request
return hypervisor.RescueState{Active: true, TemplateID: "0123456789abcdef0123456789abcdef"}, nil
}
func (f *fakeHypervisor) Power(_ context.Context, _ string, request hypervisor.PowerRequest) (hypervisor.VMState, error) {
f.powerCalls <- request
return hypervisor.VMState{State: hypervisor.VMStateShutoff}, nil
}
type testAPI struct {
handler *Server
hv *fakeHypervisor
journal *operation.Journal
secret []byte
now time.Time
nonce int
}
func newTestAPI(t *testing.T) *testAPI {
t.Helper()
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = journal.Close() })
secret := []byte("0123456789abcdef0123456789abcdef")
now := time.Unix(2_000_000_000, 0)
journal.SetClockForTesting(func() time.Time { return now })
authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal)
if err != nil {
t.Fatal(err)
}
authenticator.SetClockForTesting(func() time.Time { return now })
broker, err := consolebroker.NewBroker(time.Minute)
if err != nil {
t.Fatal(err)
}
hv := &fakeHypervisor{
deleteCalls: make(chan hypervisor.DeleteVMRequest, 1),
forceDeleteCalls: make(chan hypervisor.ForceDeleteVMRequest, 1),
templateCalls: make(chan hypervisor.ImportTemplateRequest, 1),
rescueCalls: make(chan hypervisor.ImportRescueTemplateRequest, 1),
userISOCalls: make(chan hypervisor.ImportUserISORequest, 1),
forceCalls: make(chan hypervisor.ForceRescueStartRequest, 1),
powerCalls: make(chan hypervisor.PowerRequest, 1),
}
handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = handler.Shutdown(ctx)
_ = journal.Wait(ctx)
})
return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now}
}
func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder {
t.Helper()
a.nonce++
request := httptest.NewRequest(method, target, bytes.NewReader(body))
nonce := fmt.Sprintf("nonce%011d", a.nonce)
if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
a.handler.ServeHTTP(response, request)
return response
}
func validCreateBody(operationID, name string) []byte {
rules := []hypervisor.PlatformNetworkPolicyRule{{
ID: "ssh", Label: "SSH", Category: "default", Protocols: []string{"tcp"},
PortFrom: 22, PortTo: 22, Directions: []string{"ingress"}, Reason: "default SSH policy",
}}
digestDocument := struct {
SchemaVersion uint64 `json:"schema_version"`
BasePolicyVersion uint64 `json:"base_policy_version"`
CatalogRevision uint64 `json:"catalog_revision"`
Rules []hypervisor.PlatformNetworkPolicyRule `json:"rules"`
}{SchemaVersion: 1, BasePolicyVersion: 1, CatalogRevision: 0, Rules: rules}
digestJSON, _ := json.Marshal(digestDocument)
digest := sha256.Sum256(digestJSON)
value := hypervisor.CreateVMRequest{
OperationID: operationID,
UUID: "123e4567-e89b-42d3-a456-426614174000",
Source: "cloud", Name: name, VCPUs: 2, MemoryMiB: 2048,
DiskGiB: 40, BandwidthMbps: 100, NetworkSelector: "bridge:br0",
TemplateID: "debian-12", TemplateRevision: 1,
TemplateGenerationToken: "0123456789abcdef0123456789abcdef",
RootPassword: "StrongPassword!123", PublicPoolKey: "0123456789abcdef0123456789abcdef",
PublicPoolCIDR: "8.8.8.0/24", PublicIPCIDR: "8.8.8.10/24", Gateway: "8.8.8.1",
DNSServers: []string{"1.1.1.1"},
PlatformNetworkPolicy: hypervisor.PlatformNetworkPolicySnapshot{
SchemaVersion: 1, BasePolicyVersion: 1, Rules: rules,
Digest: hex.EncodeToString(digest[:]),
},
}
encoded, _ := json.Marshal(value)
return encoded
}
func TestListResponsesUseNamedNonNullEnvelope(t *testing.T) {
api := newTestAPI(t)
response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1})
if response.Code != http.StatusOK || strings.TrimSpace(response.Body.String()) != `{"vms":[]}` {
t.Fatalf("VM list response = %d %s", response.Code, response.Body.String())
}
}
func TestCapabilitiesExposeCanonicalNetworkSelectors(t *testing.T) {
api := newTestAPI(t)
response := api.request(t, http.MethodGet, "/v1/capabilities", nil,
auth.Actor{UserID: 1})
if response.Code != http.StatusOK {
t.Fatalf("capabilities response = %d %s",
response.Code, response.Body.String())
}
var payload struct {
NetworkSelectors []string `json:"network_selectors"`
}
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if len(payload.NetworkSelectors) != 1 ||
payload.NetworkSelectors[0] != "bridge:br0" {
t.Fatalf("network selectors = %#v", payload.NetworkSelectors)
}
}
func TestStorageConfigurationRequiresSuperAdminManageAndIsDurable(t *testing.T) {
api := newTestAPI(t)
getDenied := api.request(t, http.MethodGet, "/v1/storage/configuration", nil,
auth.Actor{UserID: 1, IsSuperAdmin: true, CanManage: true})
if getDenied.Code != http.StatusMethodNotAllowed {
t.Fatalf("storage GET = %d %s", getDenied.Code, getDenied.Body.String())
}
adminDenied := api.request(t, http.MethodPost, "/v1/storage/configuration", []byte(`{}`),
auth.Actor{UserID: 1, IsAdmin: true, CanManage: true})
if adminDenied.Code != http.StatusForbidden {
t.Fatalf("non-superadmin storage POST = %d %s", adminDenied.Code, adminDenied.Body.String())
}
request := hypervisor.StorageConfigurationRequest{
OperationID: "operation-storage-one",
ManagedDirectory: "/srv/virt/vms", TemplateDirectory: "/srv/virt/templates",
SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos",
RescueDirectory: "/srv/virt/rescue-isos",
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID}
first := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
if first.Code != http.StatusOK || !strings.Contains(first.Body.String(), `"configured":true`) {
t.Fatalf("storage POST = %d %s", first.Code, first.Body.String())
}
second := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), `"replayed":true`) {
t.Fatalf("storage replay = %d %s", second.Code, second.Body.String())
}
api.hv.mu.Lock()
calls := api.hv.storageCalls
expected := api.hv.storageExpected
api.hv.mu.Unlock()
if calls != 1 {
t.Fatalf("storage calls = %d, want 1", calls)
}
if expected.Kind != operation.ExpectedStorageConfiguration || expected.ManagedPathDigest == "" ||
strings.Contains(expected.ManagedPathDigest, request.ManagedDirectory) {
t.Fatalf("storage expected state = %#v", expected)
}
operationDenied := api.request(t, http.MethodGet, "/v1/operations/"+request.OperationID, nil,
auth.Actor{UserID: 2})
if operationDenied.Code != http.StatusForbidden {
t.Fatalf("storage operation leaked to non-superadmin = %d %s", operationDenied.Code, operationDenied.Body.String())
}
operationAdminDenied := api.request(t, http.MethodGet, "/v1/operations/"+request.OperationID, nil,
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true})
if operationAdminDenied.Code != http.StatusForbidden {
t.Fatalf("storage operation leaked to ordinary admin = %d %s", operationAdminDenied.Code, operationAdminDenied.Body.String())
}
operationGet := api.request(t, http.MethodGet, "/v1/operations/"+request.OperationID, nil, actor)
if operationGet.Code != http.StatusOK {
t.Fatalf("superadmin storage operation GET = %d %s", operationGet.Code, operationGet.Body.String())
}
reconcileAdminDenied := api.request(t, http.MethodPost, "/v1/operations/"+request.OperationID+"/reconcile", []byte(`{}`),
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true})
if reconcileAdminDenied.Code != http.StatusForbidden {
t.Fatalf("ordinary admin storage reconcile = %d %s", reconcileAdminDenied.Code, reconcileAdminDenied.Body.String())
}
}
func TestReinstallRejectsStartupScript(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-reinstall"}
body := []byte(`{"operation_id":"operation-reinstall","template_id":"debian-12","template_revision":1,"template_generation_token":"0123456789abcdef0123456789abcdef","startup_script":"echo forbidden"}`)
response := api.request(t, http.MethodPost,
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/reinstall", body, actor)
if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), `"code":"invalid_json"`) ||
!strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) {
t.Fatalf("reinstall startup script response = %d %s", response.Code, response.Body.String())
}
}
func TestPowerDestroyWireActionIsAccepted(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-poweroff"}
body := []byte(`{"action":"destroy"}`)
response := api.request(t, http.MethodPost,
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/actions", body, actor)
if response.Code != http.StatusOK {
t.Fatalf("destroy action response = %d %s", response.Code, response.Body.String())
}
select {
case request := <-api.hv.powerCalls:
if request.Action != hypervisor.PowerDestroy {
t.Fatalf("power action = %q, want destroy", request.Action)
}
default:
t.Fatal("destroy action did not reach the hypervisor")
}
}
func TestEveryHTTPRouteRequiresHMACIncludingHealth(t *testing.T) {
api := newTestAPI(t)
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
response := httptest.NewRecorder()
api.handler.ServeHTTP(response, request)
if response.Code != http.StatusUnauthorized ||
!strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) {
t.Fatalf("health without HMAC status = %d, body=%s", response.Code, response.Body.String())
}
response = api.request(t, http.MethodGet, "/v1/health", nil, auth.Actor{UserID: 1})
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"service":"virt-host-agent"`) {
t.Fatalf("signed health status = %d, body=%s", response.Code, response.Body.String())
}
}
func TestNetworkSelectorsRequireAuthenticationAndCanonicalInput(t *testing.T) {
for _, target := range []string{"/v1/vms", "/v1/vms/123e4567-e89b-42d3-a456-426614174000/network"} {
t.Run(target, func(t *testing.T) {
api := newTestAPI(t)
body := validCreateBody("invalid-network-operation", "vm-test")
if strings.HasSuffix(target, "/network") {
body = []byte(`{"operation_id":"invalid-network-operation","network":"bridge:br0","confirm_restart":true}`)
}
body = bytes.ReplaceAll(body, []byte(`"bridge:br0"`), []byte(`"public"`))
unsigned := httptest.NewRecorder()
api.handler.ServeHTTP(unsigned, httptest.NewRequest(http.MethodPost, target, bytes.NewReader(body)))
if unsigned.Code != http.StatusUnauthorized {
t.Fatalf("unsigned invalid selector = %d %s", unsigned.Code, unsigned.Body.String())
}
response := api.request(t, http.MethodPost, target, body,
auth.Actor{UserID: 3, CanManage: true, OperationID: "invalid-network-operation"})
var payload errorEnvelope
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if response.Code != http.StatusUnprocessableEntity || payload.Error.Code != "invalid_network" ||
payload.Error.DispatchState != "not_dispatched" || api.hv.creates != 0 {
t.Fatalf("signed invalid selector = %d %#v", response.Code, payload)
}
if _, err := api.journal.Get("invalid-network-operation"); !errors.Is(err, operation.ErrNotFound) {
t.Fatalf("invalid selector reached the operation journal: %v", err)
}
})
}
}
func TestReplayAuthenticationFailureDoesNotClaimNotDispatched(t *testing.T) {
api := newTestAPI(t)
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, api.secret,
api.now.Unix(), "fixed-replay-nonce-1234"); err != nil {
t.Fatal(err)
}
first := httptest.NewRecorder()
api.handler.ServeHTTP(first, request)
if first.Code != http.StatusOK {
t.Fatalf("first request status = %d body=%s", first.Code, first.Body.String())
}
replay := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
if err := auth.Sign(replay, nil, auth.Actor{UserID: 1}, api.secret,
api.now.Unix(), "fixed-replay-nonce-1234"); err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
api.handler.ServeHTTP(response, replay)
if response.Code != http.StatusUnauthorized ||
!strings.Contains(response.Body.String(), `"code":"replayed_request"`) ||
strings.Contains(response.Body.String(), `"dispatch_state"`) {
t.Fatalf("replay response = %d %s", response.Code, response.Body.String())
}
}
func TestTerminalPreJournalContractIsExplicit(t *testing.T) {
api := newTestAPI(t)
tests := []struct {
name string
method string
target string
body []byte
actor auth.Actor
status int
code string
}{
{
name: "forbidden mutation", method: http.MethodPost, target: "/v1/vms",
body: []byte(`{}`), actor: auth.Actor{UserID: 1},
status: http.StatusForbidden, code: "forbidden",
},
{
name: "route missing", method: http.MethodPost, target: "/v1/missing",
body: []byte(`{}`), actor: auth.Actor{UserID: 1, CanManage: true},
status: http.StatusNotFound, code: "not_found",
},
{
name: "method mismatch", method: http.MethodDelete, target: "/v1/health",
actor: auth.Actor{UserID: 1, CanManage: true},
status: http.StatusMethodNotAllowed, code: "method_not_allowed",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
response := api.request(t, test.method, test.target, test.body, test.actor)
var payload errorEnvelope
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if response.Code != test.status || payload.Error.Code != test.code ||
payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" ||
payload.Error.Operation != nil {
t.Fatalf("response = %d %#v", response.Code, payload)
}
})
}
}
func TestRetryablePreJournalContractIsExplicit(t *testing.T) {
t.Run("shutting down", func(t *testing.T) {
api := newTestAPI(t)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := api.handler.Shutdown(ctx); err != nil {
t.Fatal(err)
}
response := api.request(t, http.MethodGet, "/v1/health", nil,
auth.Actor{UserID: 1})
assertRetryableNotDispatched(t, response, "agent_shutting_down")
})
t.Run("nonce store unavailable", func(t *testing.T) {
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
defer journal.Close()
secret := []byte("0123456789abcdef0123456789abcdef")
now := time.Unix(2_000_000_000, 0)
authenticator, err := auth.NewWithNonceStore(
secret, 5*time.Minute, 10*time.Minute, failingNonceStore{},
)
if err != nil {
t.Fatal(err)
}
authenticator.SetClockForTesting(func() time.Time { return now })
broker, err := consolebroker.NewBroker(time.Minute)
if err != nil {
t.Fatal(err)
}
server, err := NewServer(authenticator, journal, &fakeHypervisor{},
broker, 64<<10, time.Second)
if err != nil {
t.Fatal(err)
}
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = server.Shutdown(ctx)
}()
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, secret,
now.Unix(), "nonce-store-failure-1234"); err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
server.ServeHTTP(response, request)
assertRetryableNotDispatched(t, response, "nonce_store_unavailable")
})
}
func assertRetryableNotDispatched(t *testing.T, response *httptest.ResponseRecorder, code string) {
t.Helper()
var payload errorEnvelope
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if response.Code != http.StatusServiceUnavailable ||
payload.Error.Code != code || !payload.Error.Retryable ||
payload.Error.DispatchState != "not_dispatched" ||
payload.Error.Operation != nil {
t.Fatalf("response = %d %#v", response.Code, payload)
}
}
func TestCreateVMOperationIsDurablyIdempotentAndConflictsOnPayloadChange(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 3, CanManage: true, OperationID: "operation-create-one"}
body := validCreateBody(actor.OperationID, "vm-one")
first := api.request(t, http.MethodPost, "/v1/vms", body, actor)
if first.Code != http.StatusCreated {
t.Fatalf("first create status = %d body=%s", first.Code, first.Body.String())
}
if !strings.Contains(first.Body.String(), `"public_pool_key":"0123456789abcdef0123456789abcdef"`) ||
!strings.Contains(first.Body.String(), `"public_ip_cidr":"8.8.8.10/24"`) {
t.Fatalf("create response omitted confirmed network allocation: %s", first.Body.String())
}
second := api.request(t, http.MethodPost, "/v1/vms", body, actor)
if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), `"replayed":true`) {
t.Fatalf("replay status = %d body=%s", second.Code, second.Body.String())
}
api.hv.mu.Lock()
creates := api.hv.creates
lastCreate := api.hv.lastCreate
api.hv.mu.Unlock()
if creates != 1 {
t.Fatalf("backend create calls = %d, want 1", creates)
}
if lastCreate.DomainName != "3:vm-123e4567e89b" || lastCreate.Name != "vm-one" {
t.Fatalf("create names = internal %q display %q", lastCreate.DomainName, lastCreate.Name)
}
conflict := api.request(t, http.MethodPost, "/v1/vms", validCreateBody(actor.OperationID, "vm-different"), actor)
if conflict.Code != http.StatusConflict ||
!strings.Contains(conflict.Body.String(), "operation_id_conflict") ||
strings.Contains(conflict.Body.String(), `"dispatch_state"`) {
t.Fatalf("conflict status = %d body=%s", conflict.Code, conflict.Body.String())
}
}
func TestCreateVMRejectsClientSuppliedInternalDomainName(t *testing.T) {
api := newTestAPI(t)
body := validCreateBody("operation-create-injected-name", "vm-one")
body = bytes.Replace(body, []byte(`"source":"cloud"`), []byte(`"source":"cloud","domain_name":"9:vm-123e4567e89b"`), 1)
response := api.request(t, http.MethodPost, "/v1/vms", body, auth.Actor{
UserID: 3, CanManage: true, OperationID: "operation-create-injected-name",
})
if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), "invalid_json") {
t.Fatalf("injected domain name response = %d %s", response.Code, response.Body.String())
}
}
func TestFailedCreateRouteUsesReadOnlyReconciliationWithoutReplay(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-create-api-failed"
uuid := "123e4567-e89b-42d3-a456-426614174000"
spec := operation.Spec{
OperationID: operationID, ResourceID: uuid, Action: "create-vm",
Fingerprint: operation.Fingerprint("POST", "/v1/vms", []byte(`{}`)),
Destructive: true,
}
if record, _, err := api.journal.Execute(context.Background(), spec,
func(context.Context) (json.RawMessage, error) {
return nil, hypervisor.NewError(hypervisor.ErrInvalid, "network_selector_invalid", "network selector is not configured", nil)
}); err == nil || record.State != operation.StateFailed {
t.Fatalf("initial failed create = %#v err=%v", record, err)
}
api.hv.mu.Lock()
api.hv.reconcileTo = operation.Resolution{
Resolved: true, Succeeded: false, ErrorCode: "creation_absent",
Result: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`),
}
api.hv.mu.Unlock()
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: operationID}
response := api.request(t, http.MethodPost,
"/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
if response.Code != http.StatusOK ||
!strings.Contains(response.Body.String(), `"status":"failed"`) ||
!strings.Contains(response.Body.String(), `"cause_code":"network_selector_invalid"`) ||
!strings.Contains(response.Body.String(), `"managed_artifacts_absent":true`) {
t.Fatalf("failed create reconcile response = %d %s", response.Code, response.Body.String())
}
api.hv.mu.Lock()
reconciles := api.hv.reconciles
last := api.hv.lastReconcile
creates := api.hv.creates
api.hv.mu.Unlock()
if reconciles != 1 || creates != 0 || last.OperationID != operationID || last.ResourceID != uuid || last.Action != "create-vm" {
t.Fatalf("reconcile calls=%d creates=%d record=%#v", reconciles, creates, last)
}
second := api.request(t, http.MethodPost,
"/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
api.hv.mu.Lock()
reconciles = api.hv.reconciles
api.hv.mu.Unlock()
if second.Code != http.StatusOK || reconciles != 1 {
t.Fatalf("confirmed absence was reinspected: status=%d calls=%d body=%s",
second.Code, reconciles, second.Body.String())
}
}
func TestOperationInspectUsesReadOnlyContract(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-inspect-api"
uuid := "123e4567-e89b-42d3-a456-426614174000"
persistUncertainOperation(t, api, operation.Spec{
OperationID: operationID, ResourceID: uuid, Action: "create-snapshot:0123456789abcdef0123456789abcdef:crash",
Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+uuid+"/snapshots", []byte(`{}`)),
Destructive: true, LockKeys: []string{"domain:" + uuid},
})
token := strings.Repeat("a", 64)
api.hv.inspectionTo = hypervisor.OperationInspection{
OperationID: operationID, OperationStatus: operation.StateUncertain,
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeUncertain,
SummaryCode: hypervisor.InspectionSummaryAbortableJob,
AbortJob: hypervisor.OperationAbortInspection{Available: true, Kind: "backup", InspectionToken: token},
}
denied := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil,
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true, OperationID: operationID})
if denied.Code != http.StatusForbidden {
t.Fatalf("ordinary manager inspect = %d %s", denied.Code, denied.Body.String())
}
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
if response.Code != http.StatusOK ||
!strings.Contains(response.Body.String(), `"summary_code":"abortable_job_active"`) ||
!strings.Contains(response.Body.String(), `"inspection_token":"`+token+`"`) ||
strings.Contains(response.Body.String(), `"result"`) {
t.Fatalf("inspect response = %d %s", response.Code, response.Body.String())
}
api.hv.mu.Lock()
defer api.hv.mu.Unlock()
if api.hv.inspections != 1 || api.hv.reconciles != 0 || api.hv.lastInspection.OperationID != operationID {
t.Fatalf("inspection calls=%d reconcile calls=%d record=%#v",
api.hv.inspections, api.hv.reconciles, api.hv.lastInspection)
}
}
func TestOperationAbortJobRequiresInspectionTokenAndReturnsInspection(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-abort-api"
uuid := "123e4567-e89b-42d3-a456-426614174000"
persistUncertainOperation(t, api, operation.Spec{
OperationID: operationID, ResourceID: uuid, Action: "create-snapshot:0123456789abcdef0123456789abcdef:crash",
Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+uuid+"/snapshots", []byte(`{}`)),
Destructive: true, LockKeys: []string{"domain:" + uuid},
})
token := strings.Repeat("b", 64)
api.hv.abortTo = hypervisor.OperationInspection{
OperationID: operationID, OperationStatus: operation.StateUncertain,
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeUncertain,
SummaryCode: hypervisor.InspectionSummaryIncomplete,
}
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
invalid := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/abort-job",
[]byte(`{"inspection_token":"`+strings.Repeat("A", 64)+`"}`), actor)
if invalid.Code != http.StatusUnprocessableEntity || !strings.Contains(invalid.Body.String(), "invalid_inspection_token") {
t.Fatalf("invalid abort token response = %d %s", invalid.Code, invalid.Body.String())
}
response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/abort-job",
[]byte(`{"inspection_token":"`+token+`"}`), actor)
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"summary_code":"evidence_incomplete"`) {
t.Fatalf("abort response = %d %s", response.Code, response.Body.String())
}
api.hv.mu.Lock()
defer api.hv.mu.Unlock()
if api.hv.abortCalls != 1 || api.hv.lastAbortToken != token {
t.Fatalf("abort calls=%d token=%q", api.hv.abortCalls, api.hv.lastAbortToken)
}
}
func TestOperationCleanupUsesJournalAdmissionAndExactPlan(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-cleanup-api"
uuid := "123e4567-e89b-42d3-a456-426614174000"
persistUncertainOperation(t, api, operation.Spec{
OperationID: operationID, ResourceID: uuid, Action: "delete-vm",
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{}`)),
Destructive: true, LockKeys: []string{"domain:" + uuid},
})
planID := strings.Repeat("c", 64)
api.hv.cleanupTo = operation.Resolution{Resolved: true, Succeeded: true,
Result: json.RawMessage(`{"absent":true,"managed_disks_absent":true}`)}
api.hv.inspectionTo = hypervisor.OperationInspection{
OperationID: operationID, OperationStatus: operation.StateSucceeded,
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeSucceeded,
SummaryCode: hypervisor.InspectionSummarySucceeded,
}
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/cleanup",
[]byte(`{"plan_id":"`+planID+`"}`), actor)
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"outcome":"succeeded"`) {
t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String())
}
stored, err := api.journal.Get(operationID)
if err != nil || stored.State != operation.StateSucceeded {
t.Fatalf("cleanup journal record = %#v err=%v", stored, err)
}
api.hv.mu.Lock()
defer api.hv.mu.Unlock()
if api.hv.cleanupCalls != 1 || api.hv.lastCleanupPlan != planID || api.hv.inspections != 1 {
t.Fatalf("cleanup calls=%d plan=%q inspections=%d",
api.hv.cleanupCalls, api.hv.lastCleanupPlan, api.hv.inspections)
}
}
func TestConfirmInspectionRequiresFreshExactReadOnlyEvidenceAndReleasesBlocker(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-confirm-inspection"
uuid := "123e4567-e89b-42d3-a456-426614174000"
lockKey := "domain:" + uuid
persistUncertainOperation(t, api, operation.Spec{
OperationID: operationID, ResourceID: uuid, Action: "delete-vm",
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{}`)),
Destructive: true, LockKeys: []string{lockKey},
})
escalateOperationNeedsAttention(t, api.journal)
before, err := api.journal.Get(operationID)
if err != nil || before.State != operation.StateNeedsAttention {
t.Fatalf("needs-attention seed = %#v err=%v", before, err)
}
token := strings.Repeat("d", 64)
resolution := operation.Resolution{Resolved: true, Succeeded: true,
Result: json.RawMessage(`{"deleted":true,"managed_artifacts_absent":true}`)}
api.hv.inspectionTo = hypervisor.OperationInspection{
OperationID: operationID, OperationStatus: operation.StateNeedsAttention,
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeSucceeded,
SummaryCode: hypervisor.InspectionSummarySucceeded,
Confirmation: hypervisor.OperationConfirmation{Available: true, InspectionToken: token},
ReadOnlyResolution: &resolution,
}
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
target := "/v1/operations/" + operationID + "/confirm-inspection"
generic := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
if generic.Code != http.StatusConflict || !strings.Contains(generic.Body.String(), `"code":"operation_needs_attention"`) {
t.Fatalf("generic needs-attention reconcile = %d %s", generic.Code, generic.Body.String())
}
if api.hv.reconciles != 0 {
t.Fatalf("generic needs-attention reconcile reached backend: calls=%d", api.hv.reconciles)
}
stale := api.request(t, http.MethodPost, target,
[]byte(`{"inspection_token":"`+strings.Repeat("e", 64)+`"}`), actor)
if stale.Code != http.StatusConflict || !strings.Contains(stale.Body.String(), `"code":"inspection_inconclusive"`) {
t.Fatalf("stale confirmation = %d %s", stale.Code, stale.Body.String())
}
afterStale, err := api.journal.Get(operationID)
if err != nil || !reflect.DeepEqual(before, afterStale) {
t.Fatalf("stale confirmation mutated journal: before=%#v after=%#v err=%v", before, afterStale, err)
}
api.hv.inspectionTo.Confirmation = hypervisor.OperationConfirmation{}
api.hv.inspectionTo.ReadOnlyResolution = nil
inconclusive := api.request(t, http.MethodPost, target,
[]byte(`{"inspection_token":"`+token+`"}`), actor)
if inconclusive.Code != http.StatusConflict || !strings.Contains(inconclusive.Body.String(), `"code":"inspection_inconclusive"`) {
t.Fatalf("inconclusive confirmation = %d %s", inconclusive.Code, inconclusive.Body.String())
}
afterInconclusive, err := api.journal.Get(operationID)
if err != nil || !reflect.DeepEqual(before, afterInconclusive) {
t.Fatalf("inconclusive confirmation mutated journal: before=%#v after=%#v err=%v", before, afterInconclusive, err)
}
api.hv.inspectionTo.Confirmation = hypervisor.OperationConfirmation{Available: true, InspectionToken: token}
api.hv.inspectionTo.ReadOnlyResolution = &resolution
confirmed := api.request(t, http.MethodPost, target,
[]byte(`{"inspection_token":"`+token+`"}`), actor)
if confirmed.Code != http.StatusOK || !strings.Contains(confirmed.Body.String(), `"status":"succeeded"`) ||
!strings.Contains(confirmed.Body.String(), `"managed_artifacts_absent":true`) {
t.Fatalf("confirmed inspection = %d %s", confirmed.Code, confirmed.Body.String())
}
stored, err := api.journal.Get(operationID)
if err != nil || stored.State != operation.StateSucceeded {
t.Fatalf("confirmed operation = %#v err=%v", stored, err)
}
called := false
_, _, err = api.journal.Execute(context.Background(), operation.Spec{
OperationID: "operation-after-confirm", ResourceID: uuid, Action: "delete-vm",
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{"after":true}`)),
Destructive: true, LockKeys: []string{lockKey},
}, func(context.Context) (json.RawMessage, error) {
called = true
return json.RawMessage(`{"deleted":true}`), nil
})
if err != nil || !called {
t.Fatalf("confirmed operation did not release durable blocker: called=%v err=%v", called, err)
}
api.hv.mu.Lock()
defer api.hv.mu.Unlock()
if api.hv.inspections != 3 || api.hv.reconciles != 0 || api.hv.cleanupCalls != 0 || api.hv.abortCalls != 0 {
t.Fatalf("confirmation host calls inspections=%d reconcile=%d cleanup=%d abort=%d",
api.hv.inspections, api.hv.reconciles, api.hv.cleanupCalls, api.hv.abortCalls)
}
}
func TestConfirmInspectionAcceptsAlreadyFailedForceRequiredRecord(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-force-required-confirm"
uuid := "123e4567-e89b-42d3-a456-426614174000"
persistForceRequiredOperation(t, api, operationID, uuid, "delete-vm")
token := strings.Repeat("f", 64)
resolution := operation.Resolution{
Resolved: true, ErrorCode: "force_required",
ErrorMessage: "guest remains active and requires an explicit force delete",
}
api.hv.inspectionTo = hypervisor.OperationInspection{
OperationID: operationID, OperationStatus: operation.StateFailed,
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeFailed,
SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "force_required",
Confirmation: hypervisor.OperationConfirmation{Available: true, InspectionToken: token},
ReadOnlyResolution: &resolution,
}
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/confirm-inspection",
[]byte(`{"inspection_token":"`+token+`"}`), actor)
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"status":"failed"`) ||
!strings.Contains(response.Body.String(), `"error_code":"force_required"`) {
t.Fatalf("force-required confirmation = %d %s", response.Code, response.Body.String())
}
}
func TestFailedForceRequiredInspectAlwaysReadsFreshHostState(t *testing.T) {
for _, action := range []string{"delete-vm", "service-suspend"} {
t.Run(action, func(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-inspect-" + strings.ReplaceAll(action, "-", "")
uuid := "123e4567-e89b-42d3-a456-426614174000"
persistForceRequiredOperation(t, api, operationID, uuid, action)
api.hv.inspectionTo = hypervisor.OperationInspection{
OperationID: operationID, OperationStatus: operation.StateFailed,
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeFailed,
SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "force_required",
}
generic := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/reconcile", []byte(`{}`),
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true, OperationID: operationID})
if generic.Code != http.StatusConflict || !strings.Contains(generic.Body.String(), `"code":"force_required"`) {
t.Fatalf("generic force-required reconcile = %d %s", generic.Code, generic.Body.String())
}
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"error_code":"force_required"`) {
t.Fatalf("fresh force-required inspect = %d %s", response.Code, response.Body.String())
}
api.hv.mu.Lock()
defer api.hv.mu.Unlock()
if api.hv.reconciles != 0 || api.hv.inspections != 1 || api.hv.lastInspection.OperationID != operationID ||
api.hv.lastInspection.Action != action {
t.Fatalf("fresh force-required inspect reconcile=%d inspect=%d record=%#v",
api.hv.reconciles, api.hv.inspections, api.hv.lastInspection)
}
})
}
}
func escalateOperationNeedsAttention(t *testing.T, journal *operation.Journal) {
t.Helper()
coordinator, err := operation.NewRecoveryCoordinatorWithLimits(journal,
func(context.Context, operation.Record) (operation.Resolution, error) {
return operation.Resolution{}, nil
},
nil, operation.RecoveryLimits{MaxAttempts: 1, MaxAge: time.Hour, MinInterval: time.Nanosecond})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := coordinator.Start(ctx); err != nil {
t.Fatal(err)
}
select {
case <-coordinator.Ready():
case <-time.After(time.Second):
t.Fatal("recovery coordinator did not reach needs_attention")
}
coordinator.Stop()
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
defer waitCancel()
if err := coordinator.Wait(waitCtx); err != nil {
t.Fatal(err)
}
}
func TestOperationRecoveryMutationsRejectOrdinaryManager(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-recovery-auth"
uuid := "123e4567-e89b-42d3-a456-426614174000"
persistUncertainOperation(t, api, operation.Spec{
OperationID: operationID, ResourceID: uuid, Action: "delete-vm",
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{}`)),
Destructive: true, LockKeys: []string{"domain:" + uuid},
})
actor := auth.Actor{UserID: 2, IsAdmin: true, CanManage: true, OperationID: operationID}
for _, test := range []struct {
name string
method string
path string
body []byte
}{
{name: "inspect", method: http.MethodGet, path: "/v1/operations/" + operationID + "/inspect"},
{name: "confirm", method: http.MethodPost, path: "/v1/operations/" + operationID + "/confirm-inspection", body: []byte(`{"inspection_token":"` + strings.Repeat("c", 64) + `"}`)},
{name: "abort", method: http.MethodPost, path: "/v1/operations/" + operationID + "/abort-job", body: []byte(`{"inspection_token":"` + strings.Repeat("a", 64) + `"}`)},
{name: "cleanup", method: http.MethodPost, path: "/v1/operations/" + operationID + "/cleanup", body: []byte(`{"plan_id":"` + strings.Repeat("b", 64) + `"}`)},
} {
t.Run(test.name, func(t *testing.T) {
response := api.request(t, test.method, test.path, test.body, actor)
if response.Code != http.StatusForbidden {
t.Fatalf("ordinary manager response = %d %s", response.Code, response.Body.String())
}
})
}
api.hv.mu.Lock()
defer api.hv.mu.Unlock()
if api.hv.inspections != 0 || api.hv.abortCalls != 0 || api.hv.cleanupCalls != 0 {
t.Fatalf("unauthorized calls: inspect=%d abort=%d cleanup=%d",
api.hv.inspections, api.hv.abortCalls, api.hv.cleanupCalls)
}
}
func TestOperationRecoveryErrorsNeverExposeJournalOrRawCause(t *testing.T) {
api := newTestAPI(t)
operationID := "operation-recovery-redaction"
uuid := "123e4567-e89b-42d3-a456-426614174000"
persistUncertainOperation(t, api, operation.Spec{
OperationID: operationID, ResourceID: uuid, Action: "delete-vm",
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{"private":"request"}`)),
Destructive: true, LockKeys: []string{"domain:" + uuid},
})
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
rawSentinel := "raw-libvirt-path-/host/private-and-secret"
stableErr := hypervisor.NewError(hypervisor.ErrUnavailable, "inspection_unavailable", "operation evidence is unavailable", errors.New(rawSentinel))
api.hv.inspectionErr = stableErr
inspect := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
assertRecoveryErrorRedacted(t, inspect, rawSentinel, uuid)
api.hv.inspectionErr = nil
api.hv.abortErr = stableErr
abort := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/abort-job",
[]byte(`{"inspection_token":"`+strings.Repeat("a", 64)+`"}`), actor)
assertRecoveryErrorRedacted(t, abort, rawSentinel, uuid)
api.hv.abortErr = nil
api.hv.cleanupErr = operation.MarkUncertain(stableErr)
cleanup := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/cleanup",
[]byte(`{"plan_id":"`+strings.Repeat("b", 64)+`"}`), actor)
assertRecoveryErrorRedacted(t, cleanup, rawSentinel, uuid)
}
func assertRecoveryErrorRedacted(t *testing.T, response *httptest.ResponseRecorder, forbidden ...string) {
t.Helper()
if response.Code < 400 {
t.Fatalf("recovery error unexpectedly succeeded: %d %s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, field := range []string{`"operation":`, `"result":`, `"cause_code":`, `"fingerprint":`, `"resource_id":`} {
if strings.Contains(body, field) {
t.Fatalf("recovery error exposed journal field %s: %s", field, body)
}
}
for _, value := range forbidden {
if strings.Contains(body, value) {
t.Fatalf("recovery error exposed private value %q: %s", value, body)
}
}
}
func persistUncertainOperation(t *testing.T, api *testAPI, spec operation.Spec) {
t.Helper()
record, _, err := api.journal.Execute(context.Background(), spec,
func(context.Context) (json.RawMessage, error) {
return nil, operation.MarkUncertain(errors.New("injected ambiguous result"))
})
if err == nil || record.State != operation.StateUncertain {
t.Fatalf("persist uncertain operation = %#v err=%v", record, err)
}
}
func persistForceRequiredOperation(t *testing.T, api *testAPI, operationID, uuid, action string) {
t.Helper()
record, _, err := api.journal.Execute(context.Background(), operation.Spec{
OperationID: operationID, ResourceID: uuid, Action: action,
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/"+uuid, []byte(`{"remove_managed_disks":true}`)),
Destructive: true, LockKeys: hypervisor.DeleteVMAdmission(uuid).LockKeys,
}, func(context.Context) (json.RawMessage, error) {
return nil, hypervisor.NewError(hypervisor.ErrConflict, "force_required",
"guest remains active and requires an explicit force delete", nil)
})
if err == nil || record.State != operation.StateFailed || record.Error == nil || record.Error.Code != "force_required" {
t.Fatalf("persist force-required delete = %#v err=%v", record, err)
}
}
func TestForceDeleteRequiresSuperAdminAckAndLinkedGracefulOperation(t *testing.T) {
api := newTestAPI(t)
uuid := "123e4567-e89b-42d3-a456-426614174000"
priorID := "operation-delete-graceful-parent"
persistForceRequiredOperation(t, api, priorID, uuid, "delete-vm")
target := "/v1/vms/" + uuid + "/force-delete"
ordinaryID := "operation-force-ordinary"
ordinaryBody := []byte(`{"operation_id":"` + ordinaryID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":true}`)
ordinary := api.request(t, http.MethodPost, target, ordinaryBody,
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true, OperationID: ordinaryID})
if ordinary.Code != http.StatusForbidden {
t.Fatalf("ordinary manager force delete = %d %s", ordinary.Code, ordinary.Body.String())
}
missingAckID := "operation-force-no-ack"
missingAckBody := []byte(`{"operation_id":"` + missingAckID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":false}`)
missingAck := api.request(t, http.MethodPost, target, missingAckBody,
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: missingAckID})
if missingAck.Code != http.StatusUnprocessableEntity || !strings.Contains(missingAck.Body.String(), `"code":"data_loss_ack_required"`) {
t.Fatalf("force delete without acknowledgement = %d %s", missingAck.Code, missingAck.Body.String())
}
reusedBody := []byte(`{"operation_id":"` + priorID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":true}`)
reused := api.request(t, http.MethodPost, target, reusedBody,
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: priorID})
if reused.Code != http.StatusUnprocessableEntity || !strings.Contains(reused.Body.String(), `"code":"force_operation_id_reused"`) {
t.Fatalf("reused force operation id = %d %s", reused.Code, reused.Body.String())
}
forceID := "operation-force-delete-successor"
body := []byte(`{"operation_id":"` + forceID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":true}`)
response := api.request(t, http.MethodPost, target, body,
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: forceID})
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"forced":true`) ||
!strings.Contains(response.Body.String(), `"managed_artifacts_absent":true`) {
t.Fatalf("authorized force delete = %d %s", response.Code, response.Body.String())
}
select {
case request := <-api.hv.forceDeleteCalls:
if request.OperationID != forceID || request.PriorOperationID != priorID || !request.AckDataLoss || !request.RemoveManagedDisks {
t.Fatalf("force delete request = %#v", request)
}
default:
t.Fatal("authorized force delete did not reach backend")
}
stored, err := api.journal.Get(forceID)
if err != nil || stored.Action != "force-delete-vm" || stored.SupersedesOperationID != priorID || stored.State != operation.StateSucceeded {
t.Fatalf("force delete record = %#v err=%v", stored, err)
}
prior, err := api.journal.Get(priorID)
if err != nil || prior.State != operation.StateFailed || prior.Error == nil || prior.Error.Code != "force_delete_completed" {
t.Fatalf("closed graceful predecessor = %#v err=%v", prior, err)
}
replay := api.request(t, http.MethodPost, target, body,
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: forceID})
if replay.Code != http.StatusOK || !strings.Contains(replay.Body.String(), `"forced":true`) {
t.Fatalf("force delete replay = %d %s", replay.Code, replay.Body.String())
}
select {
case request := <-api.hv.forceDeleteCalls:
t.Fatalf("force delete replay reached backend: %#v", request)
default:
}
reordered := []byte(`{"prior_operation_id":"` + priorID + `","operation_id":"` + forceID + `","remove_managed_disks":true,"ack_data_loss":true}`)
conflict := api.request(t, http.MethodPost, target, reordered,
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: forceID})
if conflict.Code != http.StatusConflict || !strings.Contains(conflict.Body.String(), `"code":"operation_id_conflict"`) {
t.Fatalf("changed force replay = %d %s", conflict.Code, conflict.Body.String())
}
}
func TestGracefulDeleteForceRequiredIsStableAndDoesNotDispatchForce(t *testing.T) {
api := newTestAPI(t)
api.hv.deleteErr = hypervisor.NewError(hypervisor.ErrConflict, "force_required",
"guest remains active and requires an explicit force delete", nil)
uuid := "123e4567-e89b-42d3-a456-426614174000"
operationID := "operation-delete-needs-force"
body := []byte(`{"operation_id":"` + operationID + `","remove_managed_disks":true}`)
response := api.request(t, http.MethodDelete, "/v1/vms/"+uuid, body,
auth.Actor{UserID: 2, CanManage: true, OperationID: operationID})
if response.Code != http.StatusConflict || !strings.Contains(response.Body.String(), `"code":"force_required"`) {
t.Fatalf("graceful delete force-required = %d %s", response.Code, response.Body.String())
}
stored, err := api.journal.Get(operationID)
if err != nil || stored.State != operation.StateFailed || stored.Error == nil || stored.Error.Code != "force_required" {
t.Fatalf("graceful delete record = %#v err=%v", stored, err)
}
select {
case request := <-api.hv.deleteCalls:
if !request.RemoveManagedDisks || request.OperationID != operationID {
t.Fatalf("graceful delete request = %#v", request)
}
default:
t.Fatal("graceful delete did not reach its backend")
}
select {
case request := <-api.hv.forceDeleteCalls:
t.Fatalf("graceful delete dispatched force backend: %#v", request)
default:
}
}
func TestForceDeleteAcceptsExpiryServiceSuspendForceRequiredPredecessor(t *testing.T) {
api := newTestAPI(t)
uuid := "123e4567-e89b-42d3-a456-426614174000"
priorID := "operation-expiry-service-suspend"
persistForceRequiredOperation(t, api, priorID, uuid, "service-suspend")
forceID := "operation-expiry-force-delete"
body := []byte(`{"operation_id":"` + forceID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":true}`)
response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/force-delete", body,
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: forceID})
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"forced":true`) {
t.Fatalf("expiry force delete = %d %s", response.Code, response.Body.String())
}
stored, err := api.journal.Get(forceID)
if err != nil || stored.SupersedesOperationID != priorID || stored.State != operation.StateSucceeded {
t.Fatalf("expiry force delete record = %#v err=%v", stored, err)
}
}
func TestUnsupportedTypedMutationUsesStable501Contract(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-network-one"}
body := []byte(`{"operation_id":"operation-network-one","reason":"quota"}`)
response := api.request(t, http.MethodPost,
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/network/suspend", body, actor)
if response.Code != http.StatusNotImplemented || !strings.Contains(response.Body.String(), `"code":"unsupported"`) {
t.Fatalf("unsupported status = %d body=%s", response.Code, response.Body.String())
}
}
func TestReconciliableMutationRoutesPersistOnlySafeExpectedState(t *testing.T) {
api := newTestAPI(t)
uuid := "123e4567-e89b-42d3-a456-426614174000"
tests := []struct {
operationID string
method string
path string
body string
kind string
}{
{"operation-settings-route", http.MethodPatch, "/v1/vms/" + uuid + "/settings", `{"operation_id":"operation-settings-route","root_password":"NeverPersistThis!123","confirm_restart":true}`, operation.ExpectedUpdateSettings},
{"operation-change-network", http.MethodPost, "/v1/vms/" + uuid + "/network", `{"operation_id":"operation-change-network","network":"bridge:br0","public_pool_key":"0123456789abcdef0123456789abcdef","public_pool_cidr":"8.8.8.0/24","public_ip_cidr":"8.8.8.10/24","gateway":"8.8.8.1","dns_servers":["1.1.1.1"],"confirm_restart":true}`, operation.ExpectedChangeNetwork},
{"operation-network-route", http.MethodPost, "/v1/vms/" + uuid + "/network/resume", `{"operation_id":"operation-network-route"}`, operation.ExpectedNetworkState},
{"operation-service-route", http.MethodPost, "/v1/vms/" + uuid + "/service/resume", `{"operation_id":"operation-service-route","start_if_suspended":true}`, operation.ExpectedServiceState},
{"operation-firewall-route", http.MethodPut, "/v1/vms/" + uuid + "/firewall", `{"operation_id":"operation-firewall-route","revision":1,"default_in":"drop","default_out":"accept","rules":[]}`, operation.ExpectedFirewall},
}
for _, test := range tests {
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: test.operationID}
response := api.request(t, test.method, test.path, []byte(test.body), actor)
if response.Code != http.StatusNotImplemented {
t.Fatalf("%s status=%d body=%s", test.kind, response.Code, response.Body.String())
}
record, err := api.journal.Get(test.operationID)
if err != nil || record.ExpectedState == nil || record.ExpectedState.Kind != test.kind || record.ExpectedState.TransitionID != test.operationID {
t.Fatalf("%s record=%#v err=%v", test.kind, record, err)
}
if test.kind == operation.ExpectedChangeNetwork && (record.ExpectedState.NetworkSourceType != "bridge" || record.ExpectedState.NetworkSourceName != "br0") {
t.Fatalf("network source was not durably resolved: %#v", record.ExpectedState)
}
encoded, err := json.Marshal(record.ExpectedState)
if err != nil {
t.Fatal(err)
}
if bytes.Contains(encoded, []byte("NeverPersistThis!123")) || bytes.Contains(encoded, []byte("ssh-")) {
t.Fatalf("%s expected state contains a credential: %s", test.kind, encoded)
}
}
}
func TestConsoleTicketIsHMACProtectedAndDoesNotExposeTarget(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true}
response := api.request(t, http.MethodPost,
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/console-ticket", nil, actor)
if response.Code != http.StatusCreated {
t.Fatalf("console ticket status = %d body=%s", response.Code, response.Body.String())
}
if strings.Contains(response.Body.String(), "127.0.0.1") ||
strings.Contains(response.Body.String(), "/v1/console") ||
strings.Contains(response.Body.String(), "protocol") ||
!strings.Contains(response.Body.String(), `"ticket"`) {
t.Fatalf("unsafe console ticket response: %s", response.Body.String())
}
}
func TestBodyLimitIsEnforcedBeforeAuthentication(t *testing.T) {
api := newTestAPI(t)
api.handler.MaxBodyBytes = 1024
request := httptest.NewRequest(http.MethodPost, "/v1/vms", bytes.NewReader(make([]byte, 1025)))
response := httptest.NewRecorder()
api.handler.ServeHTTP(response, request)
if response.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("oversized body status = %d body=%s", response.Code, response.Body.String())
}
}
func TestCatalogImportsAreDurableAsyncAndAcceptPrivateSourceToken(t *testing.T) {
api := newTestAPI(t)
token := strings.Repeat("a", 64)
tests := []struct {
name string
target string
actor auth.Actor
body string
check func(*testing.T)
}{
{
name: "template", target: "/v1/templates",
actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-import"},
body: `{"operation_id":"operation-template-import","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","source_token":"` + token + `","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`,
check: func(t *testing.T) {
select {
case request := <-api.hv.templateCalls:
if request.SHA256 != "" || request.SourceToken != token {
t.Fatalf("template request = %#v", request)
}
case <-time.After(time.Second):
t.Fatal("template import was not dispatched")
}
},
},
{
name: "rescue", target: "/v1/rescue-templates",
actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-rescue-import"},
body: `{"operation_id":"operation-rescue-import","name":"Debian Rescue","source_url":"https://images.example.com/rescue.iso","source_token":"` + token + `"}`,
check: func(t *testing.T) {
select {
case request := <-api.hv.rescueCalls:
if !catalogIDPattern.MatchString(request.ID) || request.SourceToken != token {
t.Fatalf("rescue request = %#v", request)
}
case <-time.After(time.Second):
t.Fatal("rescue import was not dispatched")
}
},
},
{
name: "user ISO", target: "/v1/user-isos",
actor: auth.Actor{UserID: 42, CanManage: true, OperationID: "operation-useriso-import"},
body: `{"operation_id":"operation-useriso-import","owner_id":42,"name":"Installer","source_url":"https://images.example.com/user.iso","source_token":"` + token + `"}`,
check: func(t *testing.T) {
select {
case request := <-api.hv.userISOCalls:
if !catalogIDPattern.MatchString(request.ID) || request.OwnerID != 42 || request.SourceToken != token {
t.Fatalf("user ISO request = %#v", request)
}
case <-time.After(time.Second):
t.Fatal("user ISO import was not dispatched")
}
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
response := api.request(t, http.MethodPost, test.target, []byte(test.body), test.actor)
if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"status":"dispatching"`) {
t.Fatalf("async import response = %d %s", response.Code, response.Body.String())
}
test.check(t)
})
}
}
func TestAsyncImportNeverReturnsAcceptedWhileAnotherCallerOwnsStartup(t *testing.T) {
api := newTestAPI(t)
const operationID = "operation-import-start-race"
if !api.handler.claimAsync(operationID) {
t.Fatal("could not reserve async startup claim")
}
defer api.handler.releaseAsync(operationID)
body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
})
if response.Code != http.StatusServiceUnavailable ||
!strings.Contains(response.Body.String(), `"code":"operation_starting"`) ||
!strings.Contains(response.Body.String(), `"status":"reserved"`) ||
strings.Contains(response.Body.String(), `"dispatch_state"`) {
t.Fatalf("startup loser response = %d %s", response.Code, response.Body.String())
}
record, err := api.journal.Get(operationID)
if err != nil || record.State != operation.StateReserved {
t.Fatalf("startup loser journal = %#v err=%v", record, err)
}
}
func TestAsyncImportAdmissionTimeoutIsExplicitlyNotDispatched(t *testing.T) {
api := newTestAPI(t)
api.handler.OperationTimeout = 20 * time.Millisecond
holderStarted := make(chan struct{})
releaseHolder := make(chan struct{})
holderDone := make(chan struct{})
go func() {
defer close(holderDone)
_, _, _ = api.journal.Execute(context.Background(), operation.Spec{
OperationID: "operation-template-lock-holder",
ResourceID: "template:debian-12",
Action: "update-template",
Fingerprint: operation.Fingerprint(http.MethodPatch, "/v1/templates/debian-12", []byte(`{}`)),
LockKeys: []string{"catalog:template/debian-12"},
}, func(context.Context) (json.RawMessage, error) {
close(holderStarted)
<-releaseHolder
return json.RawMessage(`{}`), nil
})
}()
<-holderStarted
const operationID = "operation-template-lock-waiter"
body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
})
close(releaseHolder)
<-holderDone
var payload errorEnvelope
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if response.Code != http.StatusServiceUnavailable || payload.Error.Code != "backend_timeout" ||
!payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" ||
payload.Error.Operation != nil {
t.Fatalf("async admission timeout = %d %#v", response.Code, payload)
}
persisted, err := api.journal.Get(operationID)
if err != nil || persisted.State != operation.StateFailed || persisted.Error == nil ||
persisted.Error.Code != "operation_not_dispatched" {
t.Fatalf("async admission journal = %#v err=%v", persisted, err)
}
select {
case request := <-api.hv.templateCalls:
t.Fatalf("timed-out import reached backend: %#v", request)
default:
}
}
func TestAsyncImportCapacityFailureIsExplicitlyNotDispatched(t *testing.T) {
api := newTestAPI(t)
for index := 0; index < cap(api.handler.asyncSlots); index++ {
api.handler.asyncSlots <- struct{}{}
}
const operationID = "operation-template-no-capacity"
body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
})
var payload errorEnvelope
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if response.Code != http.StatusServiceUnavailable || payload.Error.Code != "agent_busy" ||
!payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" ||
payload.Error.Operation != nil {
t.Fatalf("async capacity response = %d %#v", response.Code, payload)
}
persisted, err := api.journal.Get(operationID)
if err != nil || persisted.State != operation.StateFailed || persisted.Error == nil ||
persisted.Error.Code != "operation_not_dispatched" {
t.Fatalf("async capacity journal = %#v err=%v", persisted, err)
}
select {
case request := <-api.hv.templateCalls:
t.Fatalf("capacity-rejected import reached backend: %#v", request)
default:
}
if !api.handler.claimAsync(operationID) {
t.Fatal("could not hold async restart claim")
}
duplicate := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
})
api.handler.releaseAsync(operationID)
var duplicatePayload errorEnvelope
if err := json.Unmarshal(duplicate.Body.Bytes(), &duplicatePayload); err != nil {
t.Fatal(err)
}
if duplicate.Code != http.StatusServiceUnavailable || duplicatePayload.Error.Code != "operation_starting" ||
!duplicatePayload.Error.Retryable || duplicatePayload.Error.DispatchState != "" ||
duplicatePayload.Error.Operation == nil ||
duplicatePayload.Error.Operation.State != operation.StateReserved ||
duplicatePayload.Error.Operation.Error != nil || duplicatePayload.Error.Operation.Result != nil {
t.Fatalf("async failed-not-dispatched startup duplicate = %d %#v", duplicate.Code, duplicatePayload)
}
persisted, err = api.journal.Get(operationID)
if err != nil || !operation.CanRetryNotDispatched(persisted) {
t.Fatalf("async startup duplicate changed journal = %#v err=%v", persisted, err)
}
for index := 0; index < cap(api.handler.asyncSlots); index++ {
<-api.handler.asyncSlots
}
retry := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
})
if retry.Code != http.StatusAccepted || !strings.Contains(retry.Body.String(), `"status":"dispatching"`) {
t.Fatalf("async capacity retry = %d %s", retry.Code, retry.Body.String())
}
select {
case <-api.hv.templateCalls:
case <-time.After(time.Second):
t.Fatal("same-ID capacity retry was not dispatched")
}
}
func TestNotDispatchedTombstoneResponseIsTerminal(t *testing.T) {
response := httptest.NewRecorder()
writeOperationFailure(response, operation.Record{
OperationID: "operation-expired-not-dispatched",
Action: "create-vm",
State: operation.StateFailed,
Tombstone: true,
Error: &operation.PublicError{
Code: "operation_not_dispatched", Message: "operation details expired; operation_id remains permanently reserved",
},
}, true)
var payload operationResponse
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if response.Code != http.StatusServiceUnavailable || payload.Retryable ||
payload.Status != operation.StateFailed || payload.Error == nil ||
payload.Error.Code != "operation_not_dispatched" || strings.Contains(response.Body.String(), "dispatch_state") {
t.Fatalf("not-dispatched tombstone response = %d %#v body=%s", response.Code, payload, response.Body.String())
}
}
func TestImportSourceTokenRejectsShortOrNonURLSafeValues(t *testing.T) {
for _, token := range []string{"too-short", strings.Repeat("a", 31), strings.Repeat("a", 32) + "+"} {
if err := validateImportSource("https://images.example.com/image.iso", token, ""); err == nil {
t.Fatalf("token %q unexpectedly accepted", token)
}
}
if err := validateImportSource("https://images.example.com/image.iso", strings.Repeat("a", 32), ""); err != nil {
t.Fatalf("valid token rejected: %v", err)
}
}
func TestCatalogDeleteResultsContainExactManagedCleanupEvidence(t *testing.T) {
api := newTestAPI(t)
id := "0123456789abcdef0123456789abcdef"
tests := []struct {
name string
target string
operationID string
resourceType string
resourceID string
ownerID int64
actor auth.Actor
}{
{
name: "template", target: "/v1/templates/debian-12", operationID: "operation-delete-template",
resourceType: "template", resourceID: "debian-12", actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true},
},
{
name: "rescue template", target: "/v1/rescue-templates/" + id, operationID: "operation-delete-rescue",
resourceType: "rescue_template", resourceID: id, actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true},
},
{
name: "user ISO", target: "/v1/user-isos/" + id, operationID: "operation-delete-useriso",
resourceType: "user_iso", resourceID: id, ownerID: 42, actor: auth.Actor{UserID: 42, CanManage: true},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
test.actor.OperationID = test.operationID
body := []byte(`{"operation_id":"` + test.operationID + `"}`)
response := api.request(t, http.MethodDelete, test.target, body, test.actor)
if response.Code != http.StatusOK {
t.Fatalf("delete response = %d %s", response.Code, response.Body.String())
}
var wire struct {
Status operation.State `json:"status"`
Result hypervisor.CatalogDeleteResult `json:"result"`
}
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
t.Fatal(err)
}
if wire.Status != operation.StateSucceeded || wire.Result.ResourceType != test.resourceType || wire.Result.ID != test.resourceID ||
wire.Result.OwnerID != test.ownerID || !wire.Result.Deleted || !wire.Result.ManagedImageAbsent ||
!wire.Result.ManagedMetadataAbsent || !wire.Result.ManagedArtifactsAbsent {
t.Fatalf("delete result = %#v", wire)
}
})
}
}
func TestCatalogDeleteNotFoundCodesAreTerminalDurableFailures(t *testing.T) {
for _, code := range []string{"template_not_found", "rescue_template_not_found", "user_iso_not_found"} {
t.Run(code, func(t *testing.T) {
response := httptest.NewRecorder()
writeOperationFailure(response, operation.Record{
OperationID: "operation-delete-missing", State: operation.StateFailed,
Error: &operation.PublicError{Code: code, Message: "managed catalog resource was not found"},
}, false)
if response.Code != http.StatusNotFound {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
var wire operationResponse
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
t.Fatal(err)
}
if wire.Status != operation.StateFailed || wire.Error == nil || wire.Error.Code != code || wire.Retryable || len(wire.Result) != 0 {
t.Fatalf("failure = %#v", wire)
}
})
}
}
func TestForceStartRescueRequiresExplicitDataLossAcknowledgement(t *testing.T) {
api := newTestAPI(t)
uuid := "123e4567-e89b-42d3-a456-426614174000"
target := "/v1/vms/" + uuid + "/rescue/force-start"
rejectedActor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rejected"}
rejected := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rejected","ack_data_loss":false}`), rejectedActor)
if rejected.Code != http.StatusUnprocessableEntity || !strings.Contains(rejected.Body.String(), `"code":"data_loss_ack_required"`) {
t.Fatalf("force-start without acknowledgement = %d %s", rejected.Code, rejected.Body.String())
}
select {
case call := <-api.hv.forceCalls:
t.Fatalf("rejected force-start reached backend: %#v", call)
default:
}
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rescue"}
accepted := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rescue","ack_data_loss":true}`), actor)
if accepted.Code != http.StatusAccepted || !strings.Contains(accepted.Body.String(), `"active":true`) {
t.Fatalf("acknowledged force-start = %d %s", accepted.Code, accepted.Body.String())
}
select {
case call := <-api.hv.forceCalls:
if !call.AckDataLoss || call.OperationID != actor.OperationID {
t.Fatalf("force-start request = %#v", call)
}
default:
t.Fatal("acknowledged force-start did not reach backend")
}
}
func TestAsyncImportReplayDoesNotDispatchTwice(t *testing.T) {
api := newTestAPI(t)
release := make(chan struct{})
api.hv.templateWait = release
actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-replay"}
body := []byte(`{"operation_id":"operation-template-replay","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
first := api.request(t, http.MethodPost, "/v1/templates", body, actor)
if first.Code != http.StatusAccepted {
close(release)
t.Fatalf("first import response = %d %s", first.Code, first.Body.String())
}
select {
case <-api.hv.templateCalls:
case <-time.After(time.Second):
close(release)
t.Fatal("first template import was not dispatched")
}
second := api.request(t, http.MethodPost, "/v1/templates", body, actor)
if second.Code != http.StatusAccepted || !strings.Contains(second.Body.String(), `"replayed":true`) {
close(release)
t.Fatalf("replayed import response = %d %s", second.Code, second.Body.String())
}
select {
case duplicate := <-api.hv.templateCalls:
close(release)
t.Fatalf("template import dispatched twice: %#v", duplicate)
case <-time.After(100 * time.Millisecond):
}
close(release)
deadline := time.Now().Add(time.Second)
for {
record, err := api.handler.Journal.Get(actor.OperationID)
if err == nil && record.State == operation.StateSucceeded {
break
}
if time.Now().After(deadline) {
t.Fatalf("async import did not finish after release: %#v err=%v", record, err)
}
time.Sleep(10 * time.Millisecond)
}
}
func TestExplicitReplayResumesReservedImportAfterRestartBoundary(t *testing.T) {
api := newTestAPI(t)
actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-resume"}
body := []byte(`{"operation_id":"operation-template-resume","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
_, _, err := api.handler.Journal.Reserve(operation.Spec{
OperationID: actor.OperationID,
ResourceID: "template:debian-12",
Action: "import-template",
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/templates", body),
})
if err != nil {
t.Fatal(err)
}
response := api.request(t, http.MethodPost, "/v1/templates", body, actor)
if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"replayed":true`) {
t.Fatalf("reserved replay response = %d %s", response.Code, response.Body.String())
}
select {
case <-api.hv.templateCalls:
case <-time.After(time.Second):
t.Fatal("reserved import was not resumed by explicit authenticated replay")
}
}