You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
876 lines
36 KiB
Go
876 lines
36 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
consolebroker "git.luoli.live/root/virt/agent/console"
|
|
"git.luoli.live/root/virt/agent/hypervisor"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
)
|
|
|
|
type fakeHypervisor struct {
|
|
hypervisor.Unsupported
|
|
mu sync.Mutex
|
|
creates int
|
|
lastCreate hypervisor.CreateVMRequest
|
|
reconciles int
|
|
lastReconcile operation.Record
|
|
reconcileTo operation.Resolution
|
|
reconcileErr error
|
|
templateCalls chan hypervisor.ImportTemplateRequest
|
|
templateWait <-chan struct{}
|
|
rescueCalls chan hypervisor.ImportRescueTemplateRequest
|
|
userISOCalls chan hypervisor.ImportUserISORequest
|
|
forceCalls chan hypervisor.ForceRescueStartRequest
|
|
powerCalls chan hypervisor.PowerRequest
|
|
}
|
|
|
|
type failingNonceStore struct{}
|
|
|
|
func (failingNonceStore) UseNonce(string, time.Time, time.Time) (bool, error) {
|
|
return false, errors.New("nonce store failed")
|
|
}
|
|
|
|
func (f *fakeHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) {
|
|
return []hypervisor.VMMetadata{}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) Ping(context.Context) error { return nil }
|
|
|
|
func (f *fakeHypervisor) Capabilities(context.Context) (hypervisor.Capabilities, error) {
|
|
return hypervisor.Capabilities{
|
|
Backend: "fake",
|
|
Features: hypervisor.FeatureSet{CreateVM: true, Console: true, UEFI: true},
|
|
PowerActions: []hypervisor.PowerAction{hypervisor.PowerStart},
|
|
Firmware: []string{"uefi"},
|
|
NetworkSelectors: []string{"public"},
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) CreateVM(_ context.Context, request hypervisor.CreateVMRequest) (hypervisor.VMMetadata, error) {
|
|
f.mu.Lock()
|
|
f.creates++
|
|
f.lastCreate = request
|
|
f.mu.Unlock()
|
|
return hypervisor.VMMetadata{
|
|
UUID: request.UUID, Name: request.Name, VCPUs: request.VCPUs,
|
|
MemoryMiB: request.MemoryMiB, Machine: "q35", Firmware: "uefi",
|
|
PublicIPCIDR: request.PublicIPCIDR, PublicPoolKey: request.PublicPoolKey,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) Reconcile(_ context.Context, record operation.Record) (operation.Resolution, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.reconciles++
|
|
f.lastReconcile = record
|
|
return f.reconcileTo, f.reconcileErr
|
|
}
|
|
|
|
func (f *fakeHypervisor) GetConsole(context.Context, string) (hypervisor.ConsoleSession, error) {
|
|
return hypervisor.ConsoleSession{Protocol: "vnc", Address: "127.0.0.1:5900"}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ResolveNetworkSelector(_ context.Context, selector string) (hypervisor.LocalNetworkSource, error) {
|
|
if selector != "public" {
|
|
return hypervisor.LocalNetworkSource{}, hypervisor.NewError(hypervisor.ErrInvalid, "network_not_found", "network selector is not configured", nil)
|
|
}
|
|
return hypervisor.LocalNetworkSource{Type: "bridge", Source: "br0"}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ImportTemplate(_ context.Context, request hypervisor.ImportTemplateRequest) (hypervisor.ImageTemplate, error) {
|
|
f.templateCalls <- request
|
|
if f.templateWait != nil {
|
|
<-f.templateWait
|
|
}
|
|
return hypervisor.ImageTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("a", 64)}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ListRescueTemplates(context.Context) ([]hypervisor.RescueTemplate, error) {
|
|
return []hypervisor.RescueTemplate{}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ImportRescueTemplate(_ context.Context, request hypervisor.ImportRescueTemplateRequest) (hypervisor.RescueTemplate, error) {
|
|
f.rescueCalls <- request
|
|
return hypervisor.RescueTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("b", 64)}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ImportUserISO(_ context.Context, request hypervisor.ImportUserISORequest) (hypervisor.UserISO, error) {
|
|
f.userISOCalls <- request
|
|
return hypervisor.UserISO{ID: request.ID, OwnerID: request.OwnerID, Name: request.Name, SHA256: strings.Repeat("c", 64)}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) DeleteTemplate(context.Context, string) error { return nil }
|
|
|
|
func (f *fakeHypervisor) DeleteRescueTemplate(context.Context, string) error { return nil }
|
|
|
|
func (f *fakeHypervisor) DeleteUserISO(context.Context, string, int64) error { return nil }
|
|
|
|
func (f *fakeHypervisor) ForceStartRescue(_ context.Context, _ string, request hypervisor.ForceRescueStartRequest) (hypervisor.RescueState, error) {
|
|
f.forceCalls <- request
|
|
return hypervisor.RescueState{Active: true, TemplateID: "0123456789abcdef0123456789abcdef"}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) Power(_ context.Context, _ string, request hypervisor.PowerRequest) (hypervisor.VMState, error) {
|
|
f.powerCalls <- request
|
|
return hypervisor.VMState{State: hypervisor.VMStateShutoff}, nil
|
|
}
|
|
|
|
type testAPI struct {
|
|
handler *Server
|
|
hv *fakeHypervisor
|
|
journal *operation.Journal
|
|
secret []byte
|
|
now time.Time
|
|
nonce int
|
|
}
|
|
|
|
func newTestAPI(t *testing.T) *testAPI {
|
|
t.Helper()
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = journal.Close() })
|
|
secret := []byte("0123456789abcdef0123456789abcdef")
|
|
now := time.Unix(2_000_000_000, 0)
|
|
authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
authenticator.SetClockForTesting(func() time.Time { return now })
|
|
broker, err := consolebroker.NewBroker(time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hv := &fakeHypervisor{
|
|
templateCalls: make(chan hypervisor.ImportTemplateRequest, 1),
|
|
rescueCalls: make(chan hypervisor.ImportRescueTemplateRequest, 1),
|
|
userISOCalls: make(chan hypervisor.ImportUserISORequest, 1),
|
|
forceCalls: make(chan hypervisor.ForceRescueStartRequest, 1),
|
|
powerCalls: make(chan hypervisor.PowerRequest, 1),
|
|
}
|
|
handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
_ = handler.Shutdown(ctx)
|
|
_ = journal.Wait(ctx)
|
|
})
|
|
return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now}
|
|
}
|
|
|
|
func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder {
|
|
t.Helper()
|
|
a.nonce++
|
|
request := httptest.NewRequest(method, target, bytes.NewReader(body))
|
|
nonce := fmt.Sprintf("nonce%011d", a.nonce)
|
|
if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := httptest.NewRecorder()
|
|
a.handler.ServeHTTP(response, request)
|
|
return response
|
|
}
|
|
|
|
func validCreateBody(operationID, name string) []byte {
|
|
rules := []hypervisor.PlatformNetworkPolicyRule{{
|
|
ID: "ssh", Label: "SSH", Category: "default", Protocols: []string{"tcp"},
|
|
PortFrom: 22, PortTo: 22, Directions: []string{"ingress"}, Reason: "default SSH policy",
|
|
}}
|
|
digestDocument := struct {
|
|
SchemaVersion uint64 `json:"schema_version"`
|
|
BasePolicyVersion uint64 `json:"base_policy_version"`
|
|
CatalogRevision uint64 `json:"catalog_revision"`
|
|
Rules []hypervisor.PlatformNetworkPolicyRule `json:"rules"`
|
|
}{SchemaVersion: 1, BasePolicyVersion: 1, CatalogRevision: 0, Rules: rules}
|
|
digestJSON, _ := json.Marshal(digestDocument)
|
|
digest := sha256.Sum256(digestJSON)
|
|
value := hypervisor.CreateVMRequest{
|
|
OperationID: operationID,
|
|
UUID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Source: "cloud", Name: name, VCPUs: 2, MemoryMiB: 2048,
|
|
DiskGiB: 40, BandwidthMbps: 100, NetworkSelector: "public",
|
|
TemplateID: "debian-12", TemplateRevision: 1,
|
|
TemplateGenerationToken: "0123456789abcdef0123456789abcdef",
|
|
RootPassword: "StrongPassword!123", PublicPoolKey: "0123456789abcdef0123456789abcdef",
|
|
PublicPoolCIDR: "8.8.8.0/24", PublicIPCIDR: "8.8.8.10/24", Gateway: "8.8.8.1",
|
|
DNSServers: []string{"1.1.1.1"},
|
|
PlatformNetworkPolicy: hypervisor.PlatformNetworkPolicySnapshot{
|
|
SchemaVersion: 1, BasePolicyVersion: 1, Rules: rules,
|
|
Digest: hex.EncodeToString(digest[:]),
|
|
},
|
|
}
|
|
encoded, _ := json.Marshal(value)
|
|
return encoded
|
|
}
|
|
|
|
func TestListResponsesUseNamedNonNullEnvelope(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1})
|
|
if response.Code != http.StatusOK || strings.TrimSpace(response.Body.String()) != `{"vms":[]}` {
|
|
t.Fatalf("VM list response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestCapabilitiesExposeStableNetworkSelectorKeys(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
response := api.request(t, http.MethodGet, "/v1/capabilities", nil,
|
|
auth.Actor{UserID: 1})
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("capabilities response = %d %s",
|
|
response.Code, response.Body.String())
|
|
}
|
|
var payload struct {
|
|
NetworkSelectors []string `json:"network_selectors"`
|
|
}
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(payload.NetworkSelectors) != 1 ||
|
|
payload.NetworkSelectors[0] != "public" {
|
|
t.Fatalf("network selectors = %#v", payload.NetworkSelectors)
|
|
}
|
|
}
|
|
|
|
func TestReinstallRejectsStartupScript(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-reinstall"}
|
|
body := []byte(`{"operation_id":"operation-reinstall","template_id":"debian-12","template_revision":1,"template_generation_token":"0123456789abcdef0123456789abcdef","startup_script":"echo forbidden"}`)
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/reinstall", body, actor)
|
|
if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), `"code":"invalid_json"`) ||
|
|
!strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) {
|
|
t.Fatalf("reinstall startup script response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestPowerDestroyWireActionIsAccepted(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-poweroff"}
|
|
body := []byte(`{"action":"destroy"}`)
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/actions", body, actor)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("destroy action response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
select {
|
|
case request := <-api.hv.powerCalls:
|
|
if request.Action != hypervisor.PowerDestroy {
|
|
t.Fatalf("power action = %q, want destroy", request.Action)
|
|
}
|
|
default:
|
|
t.Fatal("destroy action did not reach the hypervisor")
|
|
}
|
|
}
|
|
|
|
func TestEveryHTTPRouteRequiresHMACIncludingHealth(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
|
|
response := httptest.NewRecorder()
|
|
api.handler.ServeHTTP(response, request)
|
|
if response.Code != http.StatusUnauthorized ||
|
|
!strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) {
|
|
t.Fatalf("health without HMAC status = %d, body=%s", response.Code, response.Body.String())
|
|
}
|
|
|
|
response = api.request(t, http.MethodGet, "/v1/health", nil, auth.Actor{UserID: 1})
|
|
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"service":"virt-host-agent"`) {
|
|
t.Fatalf("signed health status = %d, body=%s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestReplayAuthenticationFailureDoesNotClaimNotDispatched(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
|
|
if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, api.secret,
|
|
api.now.Unix(), "fixed-replay-nonce-1234"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first := httptest.NewRecorder()
|
|
api.handler.ServeHTTP(first, request)
|
|
if first.Code != http.StatusOK {
|
|
t.Fatalf("first request status = %d body=%s", first.Code, first.Body.String())
|
|
}
|
|
|
|
replay := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
|
|
if err := auth.Sign(replay, nil, auth.Actor{UserID: 1}, api.secret,
|
|
api.now.Unix(), "fixed-replay-nonce-1234"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := httptest.NewRecorder()
|
|
api.handler.ServeHTTP(response, replay)
|
|
if response.Code != http.StatusUnauthorized ||
|
|
!strings.Contains(response.Body.String(), `"code":"replayed_request"`) ||
|
|
strings.Contains(response.Body.String(), `"dispatch_state"`) {
|
|
t.Fatalf("replay response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestTerminalPreJournalContractIsExplicit(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
tests := []struct {
|
|
name string
|
|
method string
|
|
target string
|
|
body []byte
|
|
actor auth.Actor
|
|
status int
|
|
code string
|
|
}{
|
|
{
|
|
name: "forbidden mutation", method: http.MethodPost, target: "/v1/vms",
|
|
body: []byte(`{}`), actor: auth.Actor{UserID: 1},
|
|
status: http.StatusForbidden, code: "forbidden",
|
|
},
|
|
{
|
|
name: "route missing", method: http.MethodPost, target: "/v1/missing",
|
|
body: []byte(`{}`), actor: auth.Actor{UserID: 1, CanManage: true},
|
|
status: http.StatusNotFound, code: "not_found",
|
|
},
|
|
{
|
|
name: "method mismatch", method: http.MethodDelete, target: "/v1/health",
|
|
actor: auth.Actor{UserID: 1, CanManage: true},
|
|
status: http.StatusMethodNotAllowed, code: "method_not_allowed",
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
response := api.request(t, test.method, test.target, test.body, test.actor)
|
|
var payload errorEnvelope
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.Code != test.status || payload.Error.Code != test.code ||
|
|
payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" ||
|
|
payload.Error.Operation != nil {
|
|
t.Fatalf("response = %d %#v", response.Code, payload)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRetryablePreJournalContractIsExplicit(t *testing.T) {
|
|
t.Run("shutting down", func(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
if err := api.handler.Shutdown(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := api.request(t, http.MethodGet, "/v1/health", nil,
|
|
auth.Actor{UserID: 1})
|
|
assertRetryableNotDispatched(t, response, "agent_shutting_down")
|
|
})
|
|
|
|
t.Run("nonce store unavailable", func(t *testing.T) {
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer journal.Close()
|
|
secret := []byte("0123456789abcdef0123456789abcdef")
|
|
now := time.Unix(2_000_000_000, 0)
|
|
authenticator, err := auth.NewWithNonceStore(
|
|
secret, 5*time.Minute, 10*time.Minute, failingNonceStore{},
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
authenticator.SetClockForTesting(func() time.Time { return now })
|
|
broker, err := consolebroker.NewBroker(time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
server, err := NewServer(authenticator, journal, &fakeHypervisor{},
|
|
broker, 64<<10, time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
_ = server.Shutdown(ctx)
|
|
}()
|
|
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
|
|
if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, secret,
|
|
now.Unix(), "nonce-store-failure-1234"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := httptest.NewRecorder()
|
|
server.ServeHTTP(response, request)
|
|
assertRetryableNotDispatched(t, response, "nonce_store_unavailable")
|
|
})
|
|
}
|
|
|
|
func assertRetryableNotDispatched(t *testing.T, response *httptest.ResponseRecorder, code string) {
|
|
t.Helper()
|
|
var payload errorEnvelope
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.Code != http.StatusServiceUnavailable ||
|
|
payload.Error.Code != code || !payload.Error.Retryable ||
|
|
payload.Error.DispatchState != "not_dispatched" ||
|
|
payload.Error.Operation != nil {
|
|
t.Fatalf("response = %d %#v", response.Code, payload)
|
|
}
|
|
}
|
|
|
|
func TestCreateVMOperationIsDurablyIdempotentAndConflictsOnPayloadChange(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 3, CanManage: true, OperationID: "operation-create-one"}
|
|
body := validCreateBody(actor.OperationID, "vm-one")
|
|
first := api.request(t, http.MethodPost, "/v1/vms", body, actor)
|
|
if first.Code != http.StatusCreated {
|
|
t.Fatalf("first create status = %d body=%s", first.Code, first.Body.String())
|
|
}
|
|
if !strings.Contains(first.Body.String(), `"public_pool_key":"0123456789abcdef0123456789abcdef"`) ||
|
|
!strings.Contains(first.Body.String(), `"public_ip_cidr":"8.8.8.10/24"`) {
|
|
t.Fatalf("create response omitted confirmed network allocation: %s", first.Body.String())
|
|
}
|
|
second := api.request(t, http.MethodPost, "/v1/vms", body, actor)
|
|
if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), `"replayed":true`) {
|
|
t.Fatalf("replay status = %d body=%s", second.Code, second.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
creates := api.hv.creates
|
|
lastCreate := api.hv.lastCreate
|
|
api.hv.mu.Unlock()
|
|
if creates != 1 {
|
|
t.Fatalf("backend create calls = %d, want 1", creates)
|
|
}
|
|
if lastCreate.DomainName != "3:vm-123e4567e89b" || lastCreate.Name != "vm-one" {
|
|
t.Fatalf("create names = internal %q display %q", lastCreate.DomainName, lastCreate.Name)
|
|
}
|
|
|
|
conflict := api.request(t, http.MethodPost, "/v1/vms", validCreateBody(actor.OperationID, "vm-different"), actor)
|
|
if conflict.Code != http.StatusConflict ||
|
|
!strings.Contains(conflict.Body.String(), "operation_id_conflict") ||
|
|
strings.Contains(conflict.Body.String(), `"dispatch_state"`) {
|
|
t.Fatalf("conflict status = %d body=%s", conflict.Code, conflict.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestCreateVMRejectsClientSuppliedInternalDomainName(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
body := validCreateBody("operation-create-injected-name", "vm-one")
|
|
body = bytes.Replace(body, []byte(`"source":"cloud"`), []byte(`"source":"cloud","domain_name":"9:vm-123e4567e89b"`), 1)
|
|
response := api.request(t, http.MethodPost, "/v1/vms", body, auth.Actor{
|
|
UserID: 3, CanManage: true, OperationID: "operation-create-injected-name",
|
|
})
|
|
if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), "invalid_json") {
|
|
t.Fatalf("injected domain name response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestFailedCreateRouteUsesReadOnlyReconciliationWithoutReplay(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-create-api-failed"
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
spec := operation.Spec{
|
|
OperationID: operationID, ResourceID: uuid, Action: "create-vm",
|
|
Fingerprint: operation.Fingerprint("POST", "/v1/vms", []byte(`{}`)),
|
|
Destructive: true,
|
|
}
|
|
if record, _, err := api.journal.Execute(context.Background(), spec,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
return nil, hypervisor.NewError(hypervisor.ErrInvalid, "network_selector_invalid", "network selector is not configured", nil)
|
|
}); err == nil || record.State != operation.StateFailed {
|
|
t.Fatalf("initial failed create = %#v err=%v", record, err)
|
|
}
|
|
api.hv.mu.Lock()
|
|
api.hv.reconcileTo = operation.Resolution{
|
|
Resolved: true, Succeeded: false, ErrorCode: "creation_absent",
|
|
Result: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`),
|
|
}
|
|
api.hv.mu.Unlock()
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: operationID}
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
|
|
if response.Code != http.StatusOK ||
|
|
!strings.Contains(response.Body.String(), `"status":"failed"`) ||
|
|
!strings.Contains(response.Body.String(), `"cause_code":"network_selector_invalid"`) ||
|
|
!strings.Contains(response.Body.String(), `"managed_artifacts_absent":true`) {
|
|
t.Fatalf("failed create reconcile response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
reconciles := api.hv.reconciles
|
|
last := api.hv.lastReconcile
|
|
creates := api.hv.creates
|
|
api.hv.mu.Unlock()
|
|
if reconciles != 1 || creates != 0 || last.OperationID != operationID || last.ResourceID != uuid || last.Action != "create-vm" {
|
|
t.Fatalf("reconcile calls=%d creates=%d record=%#v", reconciles, creates, last)
|
|
}
|
|
second := api.request(t, http.MethodPost,
|
|
"/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
|
|
api.hv.mu.Lock()
|
|
reconciles = api.hv.reconciles
|
|
api.hv.mu.Unlock()
|
|
if second.Code != http.StatusOK || reconciles != 1 {
|
|
t.Fatalf("confirmed absence was reinspected: status=%d calls=%d body=%s",
|
|
second.Code, reconciles, second.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestUnsupportedTypedMutationUsesStable501Contract(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-network-one"}
|
|
body := []byte(`{"operation_id":"operation-network-one","reason":"quota"}`)
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/network/suspend", body, actor)
|
|
if response.Code != http.StatusNotImplemented || !strings.Contains(response.Body.String(), `"code":"unsupported"`) {
|
|
t.Fatalf("unsupported status = %d body=%s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestReconciliableMutationRoutesPersistOnlySafeExpectedState(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
tests := []struct {
|
|
operationID string
|
|
method string
|
|
path string
|
|
body string
|
|
kind string
|
|
}{
|
|
{"operation-settings-route", http.MethodPatch, "/v1/vms/" + uuid + "/settings", `{"operation_id":"operation-settings-route","root_password":"NeverPersistThis!123","confirm_restart":true}`, operation.ExpectedUpdateSettings},
|
|
{"operation-change-network", http.MethodPost, "/v1/vms/" + uuid + "/network", `{"operation_id":"operation-change-network","network":"public","public_pool_key":"0123456789abcdef0123456789abcdef","public_pool_cidr":"8.8.8.0/24","public_ip_cidr":"8.8.8.10/24","gateway":"8.8.8.1","dns_servers":["1.1.1.1"],"confirm_restart":true}`, operation.ExpectedChangeNetwork},
|
|
{"operation-network-route", http.MethodPost, "/v1/vms/" + uuid + "/network/resume", `{"operation_id":"operation-network-route"}`, operation.ExpectedNetworkState},
|
|
{"operation-service-route", http.MethodPost, "/v1/vms/" + uuid + "/service/resume", `{"operation_id":"operation-service-route","start_if_suspended":true}`, operation.ExpectedServiceState},
|
|
{"operation-firewall-route", http.MethodPut, "/v1/vms/" + uuid + "/firewall", `{"operation_id":"operation-firewall-route","revision":1,"default_in":"drop","default_out":"accept","rules":[]}`, operation.ExpectedFirewall},
|
|
}
|
|
for _, test := range tests {
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: test.operationID}
|
|
response := api.request(t, test.method, test.path, []byte(test.body), actor)
|
|
if response.Code != http.StatusNotImplemented {
|
|
t.Fatalf("%s status=%d body=%s", test.kind, response.Code, response.Body.String())
|
|
}
|
|
record, err := api.journal.Get(test.operationID)
|
|
if err != nil || record.ExpectedState == nil || record.ExpectedState.Kind != test.kind || record.ExpectedState.TransitionID != test.operationID {
|
|
t.Fatalf("%s record=%#v err=%v", test.kind, record, err)
|
|
}
|
|
if test.kind == operation.ExpectedChangeNetwork && (record.ExpectedState.NetworkSourceType != "bridge" || record.ExpectedState.NetworkSourceName != "br0") {
|
|
t.Fatalf("network source was not durably resolved: %#v", record.ExpectedState)
|
|
}
|
|
encoded, err := json.Marshal(record.ExpectedState)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if bytes.Contains(encoded, []byte("NeverPersistThis!123")) || bytes.Contains(encoded, []byte("ssh-")) {
|
|
t.Fatalf("%s expected state contains a credential: %s", test.kind, encoded)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestConsoleTicketIsHMACProtectedAndDoesNotExposeTarget(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true}
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/console-ticket", nil, actor)
|
|
if response.Code != http.StatusCreated {
|
|
t.Fatalf("console ticket status = %d body=%s", response.Code, response.Body.String())
|
|
}
|
|
if strings.Contains(response.Body.String(), "127.0.0.1") ||
|
|
strings.Contains(response.Body.String(), "/v1/console") ||
|
|
strings.Contains(response.Body.String(), "protocol") ||
|
|
!strings.Contains(response.Body.String(), `"ticket"`) {
|
|
t.Fatalf("unsafe console ticket response: %s", response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestBodyLimitIsEnforcedBeforeAuthentication(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
api.handler.MaxBodyBytes = 1024
|
|
request := httptest.NewRequest(http.MethodPost, "/v1/vms", bytes.NewReader(make([]byte, 1025)))
|
|
response := httptest.NewRecorder()
|
|
api.handler.ServeHTTP(response, request)
|
|
if response.Code != http.StatusRequestEntityTooLarge {
|
|
t.Fatalf("oversized body status = %d body=%s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestCatalogImportsAreDurableAsyncAndAcceptPrivateSourceToken(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
token := strings.Repeat("a", 64)
|
|
tests := []struct {
|
|
name string
|
|
target string
|
|
actor auth.Actor
|
|
body string
|
|
check func(*testing.T)
|
|
}{
|
|
{
|
|
name: "template", target: "/v1/templates",
|
|
actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-import"},
|
|
body: `{"operation_id":"operation-template-import","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","source_token":"` + token + `","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`,
|
|
check: func(t *testing.T) {
|
|
select {
|
|
case request := <-api.hv.templateCalls:
|
|
if request.SHA256 != "" || request.SourceToken != token {
|
|
t.Fatalf("template request = %#v", request)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("template import was not dispatched")
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "rescue", target: "/v1/rescue-templates",
|
|
actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-rescue-import"},
|
|
body: `{"operation_id":"operation-rescue-import","name":"Debian Rescue","source_url":"https://images.example.com/rescue.iso","source_token":"` + token + `"}`,
|
|
check: func(t *testing.T) {
|
|
select {
|
|
case request := <-api.hv.rescueCalls:
|
|
if !catalogIDPattern.MatchString(request.ID) || request.SourceToken != token {
|
|
t.Fatalf("rescue request = %#v", request)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("rescue import was not dispatched")
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "user ISO", target: "/v1/user-isos",
|
|
actor: auth.Actor{UserID: 42, CanManage: true, OperationID: "operation-useriso-import"},
|
|
body: `{"operation_id":"operation-useriso-import","owner_id":42,"name":"Installer","source_url":"https://images.example.com/user.iso","source_token":"` + token + `"}`,
|
|
check: func(t *testing.T) {
|
|
select {
|
|
case request := <-api.hv.userISOCalls:
|
|
if !catalogIDPattern.MatchString(request.ID) || request.OwnerID != 42 || request.SourceToken != token {
|
|
t.Fatalf("user ISO request = %#v", request)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("user ISO import was not dispatched")
|
|
}
|
|
},
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
response := api.request(t, http.MethodPost, test.target, []byte(test.body), test.actor)
|
|
if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"status":"dispatching"`) {
|
|
t.Fatalf("async import response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
test.check(t)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAsyncImportNeverReturnsAcceptedWhileAnotherCallerOwnsStartup(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
const operationID = "operation-import-start-race"
|
|
if !api.handler.claimAsync(operationID) {
|
|
t.Fatal("could not reserve async startup claim")
|
|
}
|
|
defer api.handler.releaseAsync(operationID)
|
|
body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
|
|
response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
|
|
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
|
|
})
|
|
if response.Code != http.StatusServiceUnavailable ||
|
|
!strings.Contains(response.Body.String(), `"code":"operation_starting"`) ||
|
|
strings.Contains(response.Body.String(), `"status":"dispatching"`) {
|
|
t.Fatalf("startup loser response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
record, err := api.journal.Get(operationID)
|
|
if err != nil || record.State != operation.StateReserved {
|
|
t.Fatalf("startup loser journal = %#v err=%v", record, err)
|
|
}
|
|
}
|
|
|
|
func TestImportSourceTokenRejectsShortOrNonURLSafeValues(t *testing.T) {
|
|
for _, token := range []string{"too-short", strings.Repeat("a", 31), strings.Repeat("a", 32) + "+"} {
|
|
if err := validateImportSource("https://images.example.com/image.iso", token, ""); err == nil {
|
|
t.Fatalf("token %q unexpectedly accepted", token)
|
|
}
|
|
}
|
|
if err := validateImportSource("https://images.example.com/image.iso", strings.Repeat("a", 32), ""); err != nil {
|
|
t.Fatalf("valid token rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCatalogDeleteResultsContainExactManagedCleanupEvidence(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
id := "0123456789abcdef0123456789abcdef"
|
|
tests := []struct {
|
|
name string
|
|
target string
|
|
operationID string
|
|
resourceType string
|
|
resourceID string
|
|
ownerID int64
|
|
actor auth.Actor
|
|
}{
|
|
{
|
|
name: "template", target: "/v1/templates/debian-12", operationID: "operation-delete-template",
|
|
resourceType: "template", resourceID: "debian-12", actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true},
|
|
},
|
|
{
|
|
name: "rescue template", target: "/v1/rescue-templates/" + id, operationID: "operation-delete-rescue",
|
|
resourceType: "rescue_template", resourceID: id, actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true},
|
|
},
|
|
{
|
|
name: "user ISO", target: "/v1/user-isos/" + id, operationID: "operation-delete-useriso",
|
|
resourceType: "user_iso", resourceID: id, ownerID: 42, actor: auth.Actor{UserID: 42, CanManage: true},
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
test.actor.OperationID = test.operationID
|
|
body := []byte(`{"operation_id":"` + test.operationID + `"}`)
|
|
response := api.request(t, http.MethodDelete, test.target, body, test.actor)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("delete response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
var wire struct {
|
|
Status operation.State `json:"status"`
|
|
Result hypervisor.CatalogDeleteResult `json:"result"`
|
|
}
|
|
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if wire.Status != operation.StateSucceeded || wire.Result.ResourceType != test.resourceType || wire.Result.ID != test.resourceID ||
|
|
wire.Result.OwnerID != test.ownerID || !wire.Result.Deleted || !wire.Result.ManagedImageAbsent ||
|
|
!wire.Result.ManagedMetadataAbsent || !wire.Result.ManagedArtifactsAbsent {
|
|
t.Fatalf("delete result = %#v", wire)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCatalogDeleteNotFoundCodesAreTerminalDurableFailures(t *testing.T) {
|
|
for _, code := range []string{"template_not_found", "rescue_template_not_found", "user_iso_not_found"} {
|
|
t.Run(code, func(t *testing.T) {
|
|
response := httptest.NewRecorder()
|
|
writeOperationFailure(response, operation.Record{
|
|
OperationID: "operation-delete-missing", State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: code, Message: "managed catalog resource was not found"},
|
|
}, false)
|
|
if response.Code != http.StatusNotFound {
|
|
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
|
}
|
|
var wire operationResponse
|
|
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if wire.Status != operation.StateFailed || wire.Error == nil || wire.Error.Code != code || wire.Retryable || len(wire.Result) != 0 {
|
|
t.Fatalf("failure = %#v", wire)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestForceStartRescueRequiresExplicitDataLossAcknowledgement(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
target := "/v1/vms/" + uuid + "/rescue/force-start"
|
|
rejectedActor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rejected"}
|
|
rejected := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rejected","ack_data_loss":false}`), rejectedActor)
|
|
if rejected.Code != http.StatusUnprocessableEntity || !strings.Contains(rejected.Body.String(), `"code":"data_loss_ack_required"`) {
|
|
t.Fatalf("force-start without acknowledgement = %d %s", rejected.Code, rejected.Body.String())
|
|
}
|
|
select {
|
|
case call := <-api.hv.forceCalls:
|
|
t.Fatalf("rejected force-start reached backend: %#v", call)
|
|
default:
|
|
}
|
|
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rescue"}
|
|
accepted := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rescue","ack_data_loss":true}`), actor)
|
|
if accepted.Code != http.StatusAccepted || !strings.Contains(accepted.Body.String(), `"active":true`) {
|
|
t.Fatalf("acknowledged force-start = %d %s", accepted.Code, accepted.Body.String())
|
|
}
|
|
select {
|
|
case call := <-api.hv.forceCalls:
|
|
if !call.AckDataLoss || call.OperationID != actor.OperationID {
|
|
t.Fatalf("force-start request = %#v", call)
|
|
}
|
|
default:
|
|
t.Fatal("acknowledged force-start did not reach backend")
|
|
}
|
|
}
|
|
|
|
func TestAsyncImportReplayDoesNotDispatchTwice(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
release := make(chan struct{})
|
|
api.hv.templateWait = release
|
|
actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-replay"}
|
|
body := []byte(`{"operation_id":"operation-template-replay","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
|
|
first := api.request(t, http.MethodPost, "/v1/templates", body, actor)
|
|
if first.Code != http.StatusAccepted {
|
|
close(release)
|
|
t.Fatalf("first import response = %d %s", first.Code, first.Body.String())
|
|
}
|
|
select {
|
|
case <-api.hv.templateCalls:
|
|
case <-time.After(time.Second):
|
|
close(release)
|
|
t.Fatal("first template import was not dispatched")
|
|
}
|
|
second := api.request(t, http.MethodPost, "/v1/templates", body, actor)
|
|
if second.Code != http.StatusAccepted || !strings.Contains(second.Body.String(), `"replayed":true`) {
|
|
close(release)
|
|
t.Fatalf("replayed import response = %d %s", second.Code, second.Body.String())
|
|
}
|
|
select {
|
|
case duplicate := <-api.hv.templateCalls:
|
|
close(release)
|
|
t.Fatalf("template import dispatched twice: %#v", duplicate)
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
close(release)
|
|
deadline := time.Now().Add(time.Second)
|
|
for {
|
|
record, err := api.handler.Journal.Get(actor.OperationID)
|
|
if err == nil && record.State == operation.StateSucceeded {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("async import did not finish after release: %#v err=%v", record, err)
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestExplicitReplayResumesReservedImportAfterRestartBoundary(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-resume"}
|
|
body := []byte(`{"operation_id":"operation-template-resume","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
|
|
_, _, err := api.handler.Journal.Reserve(operation.Spec{
|
|
OperationID: actor.OperationID,
|
|
ResourceID: "template:debian-12",
|
|
Action: "import-template",
|
|
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/templates", body),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := api.request(t, http.MethodPost, "/v1/templates", body, actor)
|
|
if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"replayed":true`) {
|
|
t.Fatalf("reserved replay response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
select {
|
|
case <-api.hv.templateCalls:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("reserved import was not resumed by explicit authenticated replay")
|
|
}
|
|
}
|