You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1690 lines
75 KiB
Go
1690 lines
75 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
consolebroker "git.luoli.live/root/virt/agent/console"
|
|
"git.luoli.live/root/virt/agent/hypervisor"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
)
|
|
|
|
type fakeHypervisor struct {
|
|
hypervisor.Unsupported
|
|
mu sync.Mutex
|
|
creates int
|
|
lastCreate hypervisor.CreateVMRequest
|
|
deleteCalls chan hypervisor.DeleteVMRequest
|
|
deleteErr error
|
|
forceDeleteCalls chan hypervisor.ForceDeleteVMRequest
|
|
reconciles int
|
|
lastReconcile operation.Record
|
|
reconcileTo operation.Resolution
|
|
reconcileErr error
|
|
inspections int
|
|
lastInspection operation.Record
|
|
inspectionTo hypervisor.OperationInspection
|
|
inspectionErr error
|
|
abortCalls int
|
|
lastAbortToken string
|
|
abortTo hypervisor.OperationInspection
|
|
abortErr error
|
|
cleanupCalls int
|
|
lastCleanupPlan string
|
|
cleanupTo operation.Resolution
|
|
cleanupErr error
|
|
templateCalls chan hypervisor.ImportTemplateRequest
|
|
templateWait <-chan struct{}
|
|
rescueCalls chan hypervisor.ImportRescueTemplateRequest
|
|
userISOCalls chan hypervisor.ImportUserISORequest
|
|
forceCalls chan hypervisor.ForceRescueStartRequest
|
|
powerCalls chan hypervisor.PowerRequest
|
|
storageCalls int
|
|
lastStorage hypervisor.StorageConfigurationRequest
|
|
storageExpected operation.ExpectedState
|
|
}
|
|
|
|
type failingNonceStore struct{}
|
|
|
|
func (failingNonceStore) UseNonce(string, time.Time, time.Time) (bool, error) {
|
|
return false, errors.New("nonce store failed")
|
|
}
|
|
|
|
func (f *fakeHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) {
|
|
return []hypervisor.VMMetadata{}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) Ping(context.Context) error { return nil }
|
|
|
|
func (f *fakeHypervisor) Capabilities(context.Context) (hypervisor.Capabilities, error) {
|
|
return hypervisor.Capabilities{
|
|
Backend: "fake",
|
|
Features: hypervisor.FeatureSet{CreateVM: true, Console: true, UEFI: true},
|
|
PowerActions: []hypervisor.PowerAction{hypervisor.PowerStart},
|
|
Firmware: []string{"uefi"},
|
|
NetworkSelectors: []string{"bridge:br0"},
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ConfigureStorage(ctx context.Context, request hypervisor.StorageConfigurationRequest) (hypervisor.StorageConfigurationResult, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.storageCalls++
|
|
f.lastStorage = request
|
|
f.storageExpected, _ = operation.ExpectedStateFromContext(ctx)
|
|
return hypervisor.StorageConfigurationResult{Configured: true}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) CreateVM(_ context.Context, request hypervisor.CreateVMRequest) (hypervisor.VMMetadata, error) {
|
|
f.mu.Lock()
|
|
f.creates++
|
|
f.lastCreate = request
|
|
f.mu.Unlock()
|
|
return hypervisor.VMMetadata{
|
|
UUID: request.UUID, Name: request.Name, VCPUs: request.VCPUs,
|
|
MemoryMiB: request.MemoryMiB, Machine: "q35", Firmware: "uefi",
|
|
PublicIPCIDR: request.PublicIPCIDR, PublicPoolKey: request.PublicPoolKey,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) DeleteVM(_ context.Context, _ string, request hypervisor.DeleteVMRequest) (hypervisor.DeleteVMResult, error) {
|
|
f.deleteCalls <- request
|
|
if f.deleteErr != nil {
|
|
return hypervisor.DeleteVMResult{}, f.deleteErr
|
|
}
|
|
return hypervisor.DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ForceDeleteVM(_ context.Context, _ string, request hypervisor.ForceDeleteVMRequest) (hypervisor.DeleteVMResult, error) {
|
|
f.forceDeleteCalls <- request
|
|
return hypervisor.DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true, Forced: true}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) Reconcile(_ context.Context, record operation.Record) (operation.Resolution, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.reconciles++
|
|
f.lastReconcile = record
|
|
return f.reconcileTo, f.reconcileErr
|
|
}
|
|
|
|
func (f *fakeHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.inspections++
|
|
f.lastInspection = record
|
|
return f.inspectionTo, f.inspectionErr
|
|
}
|
|
|
|
func (f *fakeHypervisor) AbortOperationJob(_ context.Context, _ operation.Record, token string) (hypervisor.OperationInspection, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.abortCalls++
|
|
f.lastAbortToken = token
|
|
return f.abortTo, f.abortErr
|
|
}
|
|
|
|
func (f *fakeHypervisor) CleanupOperation(_ context.Context, _ operation.Record, planID string) (operation.Resolution, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.cleanupCalls++
|
|
f.lastCleanupPlan = planID
|
|
return f.cleanupTo, f.cleanupErr
|
|
}
|
|
|
|
func (f *fakeHypervisor) GetConsole(context.Context, string) (hypervisor.ConsoleSession, error) {
|
|
return hypervisor.ConsoleSession{Protocol: "vnc", Address: "127.0.0.1:5900"}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ResolveNetworkSelector(_ context.Context, selector string) (hypervisor.LocalNetworkSource, error) {
|
|
return hypervisor.ParseNetworkSelector(selector)
|
|
}
|
|
|
|
func (f *fakeHypervisor) ImportTemplate(_ context.Context, request hypervisor.ImportTemplateRequest) (hypervisor.ImageTemplate, error) {
|
|
f.templateCalls <- request
|
|
if f.templateWait != nil {
|
|
<-f.templateWait
|
|
}
|
|
return hypervisor.ImageTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("a", 64)}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ListRescueTemplates(context.Context) ([]hypervisor.RescueTemplate, error) {
|
|
return []hypervisor.RescueTemplate{}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ImportRescueTemplate(_ context.Context, request hypervisor.ImportRescueTemplateRequest) (hypervisor.RescueTemplate, error) {
|
|
f.rescueCalls <- request
|
|
return hypervisor.RescueTemplate{ID: request.ID, Name: request.Name, SHA256: strings.Repeat("b", 64)}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) ImportUserISO(_ context.Context, request hypervisor.ImportUserISORequest) (hypervisor.UserISO, error) {
|
|
f.userISOCalls <- request
|
|
return hypervisor.UserISO{ID: request.ID, OwnerID: request.OwnerID, Name: request.Name, SHA256: strings.Repeat("c", 64)}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) DeleteTemplate(context.Context, string) error { return nil }
|
|
|
|
func (f *fakeHypervisor) DeleteRescueTemplate(context.Context, string) error { return nil }
|
|
|
|
func (f *fakeHypervisor) DeleteUserISO(context.Context, string, int64) error { return nil }
|
|
|
|
func (f *fakeHypervisor) ForceStartRescue(_ context.Context, _ string, request hypervisor.ForceRescueStartRequest) (hypervisor.RescueState, error) {
|
|
f.forceCalls <- request
|
|
return hypervisor.RescueState{Active: true, TemplateID: "0123456789abcdef0123456789abcdef"}, nil
|
|
}
|
|
|
|
func (f *fakeHypervisor) Power(_ context.Context, _ string, request hypervisor.PowerRequest) (hypervisor.VMState, error) {
|
|
f.powerCalls <- request
|
|
return hypervisor.VMState{State: hypervisor.VMStateShutoff}, nil
|
|
}
|
|
|
|
type testAPI struct {
|
|
handler *Server
|
|
hv *fakeHypervisor
|
|
journal *operation.Journal
|
|
secret []byte
|
|
now time.Time
|
|
nonce int
|
|
}
|
|
|
|
func newTestAPI(t *testing.T) *testAPI {
|
|
t.Helper()
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = journal.Close() })
|
|
secret := []byte("0123456789abcdef0123456789abcdef")
|
|
now := time.Unix(2_000_000_000, 0)
|
|
journal.SetClockForTesting(func() time.Time { return now })
|
|
authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
authenticator.SetClockForTesting(func() time.Time { return now })
|
|
broker, err := consolebroker.NewBroker(time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hv := &fakeHypervisor{
|
|
deleteCalls: make(chan hypervisor.DeleteVMRequest, 1),
|
|
forceDeleteCalls: make(chan hypervisor.ForceDeleteVMRequest, 1),
|
|
templateCalls: make(chan hypervisor.ImportTemplateRequest, 1),
|
|
rescueCalls: make(chan hypervisor.ImportRescueTemplateRequest, 1),
|
|
userISOCalls: make(chan hypervisor.ImportUserISORequest, 1),
|
|
forceCalls: make(chan hypervisor.ForceRescueStartRequest, 1),
|
|
powerCalls: make(chan hypervisor.PowerRequest, 1),
|
|
}
|
|
handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
_ = handler.Shutdown(ctx)
|
|
_ = journal.Wait(ctx)
|
|
})
|
|
return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now}
|
|
}
|
|
|
|
func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder {
|
|
t.Helper()
|
|
a.nonce++
|
|
request := httptest.NewRequest(method, target, bytes.NewReader(body))
|
|
nonce := fmt.Sprintf("nonce%011d", a.nonce)
|
|
if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := httptest.NewRecorder()
|
|
a.handler.ServeHTTP(response, request)
|
|
return response
|
|
}
|
|
|
|
func validCreateBody(operationID, name string) []byte {
|
|
rules := []hypervisor.PlatformNetworkPolicyRule{{
|
|
ID: "ssh", Label: "SSH", Category: "default", Protocols: []string{"tcp"},
|
|
PortFrom: 22, PortTo: 22, Directions: []string{"ingress"}, Reason: "default SSH policy",
|
|
}}
|
|
digestDocument := struct {
|
|
SchemaVersion uint64 `json:"schema_version"`
|
|
BasePolicyVersion uint64 `json:"base_policy_version"`
|
|
CatalogRevision uint64 `json:"catalog_revision"`
|
|
Rules []hypervisor.PlatformNetworkPolicyRule `json:"rules"`
|
|
}{SchemaVersion: 1, BasePolicyVersion: 1, CatalogRevision: 0, Rules: rules}
|
|
digestJSON, _ := json.Marshal(digestDocument)
|
|
digest := sha256.Sum256(digestJSON)
|
|
value := hypervisor.CreateVMRequest{
|
|
OperationID: operationID,
|
|
UUID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Source: "cloud", Name: name, VCPUs: 2, MemoryMiB: 2048,
|
|
DiskGiB: 40, BandwidthMbps: 100, NetworkSelector: "bridge:br0",
|
|
TemplateID: "debian-12", TemplateRevision: 1,
|
|
TemplateGenerationToken: "0123456789abcdef0123456789abcdef",
|
|
RootPassword: "StrongPassword!123", PublicPoolKey: "0123456789abcdef0123456789abcdef",
|
|
PublicPoolCIDR: "8.8.8.0/24", PublicIPCIDR: "8.8.8.10/24", Gateway: "8.8.8.1",
|
|
DNSServers: []string{"1.1.1.1"},
|
|
PlatformNetworkPolicy: hypervisor.PlatformNetworkPolicySnapshot{
|
|
SchemaVersion: 1, BasePolicyVersion: 1, Rules: rules,
|
|
Digest: hex.EncodeToString(digest[:]),
|
|
},
|
|
}
|
|
encoded, _ := json.Marshal(value)
|
|
return encoded
|
|
}
|
|
|
|
func TestListResponsesUseNamedNonNullEnvelope(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1})
|
|
if response.Code != http.StatusOK || strings.TrimSpace(response.Body.String()) != `{"vms":[]}` {
|
|
t.Fatalf("VM list response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestCapabilitiesExposeCanonicalNetworkSelectors(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
response := api.request(t, http.MethodGet, "/v1/capabilities", nil,
|
|
auth.Actor{UserID: 1})
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("capabilities response = %d %s",
|
|
response.Code, response.Body.String())
|
|
}
|
|
var payload struct {
|
|
NetworkSelectors []string `json:"network_selectors"`
|
|
}
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(payload.NetworkSelectors) != 1 ||
|
|
payload.NetworkSelectors[0] != "bridge:br0" {
|
|
t.Fatalf("network selectors = %#v", payload.NetworkSelectors)
|
|
}
|
|
}
|
|
|
|
func TestStorageConfigurationRequiresSuperAdminManageAndIsDurable(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
getDenied := api.request(t, http.MethodGet, "/v1/storage/configuration", nil,
|
|
auth.Actor{UserID: 1, IsSuperAdmin: true, CanManage: true})
|
|
if getDenied.Code != http.StatusMethodNotAllowed {
|
|
t.Fatalf("storage GET = %d %s", getDenied.Code, getDenied.Body.String())
|
|
}
|
|
adminDenied := api.request(t, http.MethodPost, "/v1/storage/configuration", []byte(`{}`),
|
|
auth.Actor{UserID: 1, IsAdmin: true, CanManage: true})
|
|
if adminDenied.Code != http.StatusForbidden {
|
|
t.Fatalf("non-superadmin storage POST = %d %s", adminDenied.Code, adminDenied.Body.String())
|
|
}
|
|
|
|
request := hypervisor.StorageConfigurationRequest{
|
|
OperationID: "operation-storage-one",
|
|
ManagedDirectory: "/srv/virt/vms", TemplateDirectory: "/srv/virt/templates",
|
|
SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos",
|
|
RescueDirectory: "/srv/virt/rescue-isos",
|
|
}
|
|
body, err := json.Marshal(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID}
|
|
first := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
|
|
if first.Code != http.StatusOK || !strings.Contains(first.Body.String(), `"configured":true`) {
|
|
t.Fatalf("storage POST = %d %s", first.Code, first.Body.String())
|
|
}
|
|
second := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
|
|
if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), `"replayed":true`) {
|
|
t.Fatalf("storage replay = %d %s", second.Code, second.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
calls := api.hv.storageCalls
|
|
expected := api.hv.storageExpected
|
|
api.hv.mu.Unlock()
|
|
if calls != 1 {
|
|
t.Fatalf("storage calls = %d, want 1", calls)
|
|
}
|
|
if expected.Kind != operation.ExpectedStorageConfiguration || expected.ManagedPathDigest == "" ||
|
|
strings.Contains(expected.ManagedPathDigest, request.ManagedDirectory) {
|
|
t.Fatalf("storage expected state = %#v", expected)
|
|
}
|
|
operationDenied := api.request(t, http.MethodGet, "/v1/operations/"+request.OperationID, nil,
|
|
auth.Actor{UserID: 2})
|
|
if operationDenied.Code != http.StatusForbidden {
|
|
t.Fatalf("storage operation leaked to non-superadmin = %d %s", operationDenied.Code, operationDenied.Body.String())
|
|
}
|
|
operationAdminDenied := api.request(t, http.MethodGet, "/v1/operations/"+request.OperationID, nil,
|
|
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true})
|
|
if operationAdminDenied.Code != http.StatusForbidden {
|
|
t.Fatalf("storage operation leaked to ordinary admin = %d %s", operationAdminDenied.Code, operationAdminDenied.Body.String())
|
|
}
|
|
operationGet := api.request(t, http.MethodGet, "/v1/operations/"+request.OperationID, nil, actor)
|
|
if operationGet.Code != http.StatusOK {
|
|
t.Fatalf("superadmin storage operation GET = %d %s", operationGet.Code, operationGet.Body.String())
|
|
}
|
|
reconcileAdminDenied := api.request(t, http.MethodPost, "/v1/operations/"+request.OperationID+"/reconcile", []byte(`{}`),
|
|
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true})
|
|
if reconcileAdminDenied.Code != http.StatusForbidden {
|
|
t.Fatalf("ordinary admin storage reconcile = %d %s", reconcileAdminDenied.Code, reconcileAdminDenied.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestReinstallRejectsStartupScript(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-reinstall"}
|
|
body := []byte(`{"operation_id":"operation-reinstall","template_id":"debian-12","template_revision":1,"template_generation_token":"0123456789abcdef0123456789abcdef","startup_script":"echo forbidden"}`)
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/reinstall", body, actor)
|
|
if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), `"code":"invalid_json"`) ||
|
|
!strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) {
|
|
t.Fatalf("reinstall startup script response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestPowerDestroyWireActionIsAccepted(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-poweroff"}
|
|
body := []byte(`{"action":"destroy"}`)
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/actions", body, actor)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("destroy action response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
select {
|
|
case request := <-api.hv.powerCalls:
|
|
if request.Action != hypervisor.PowerDestroy {
|
|
t.Fatalf("power action = %q, want destroy", request.Action)
|
|
}
|
|
default:
|
|
t.Fatal("destroy action did not reach the hypervisor")
|
|
}
|
|
}
|
|
|
|
func TestEveryHTTPRouteRequiresHMACIncludingHealth(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
|
|
response := httptest.NewRecorder()
|
|
api.handler.ServeHTTP(response, request)
|
|
if response.Code != http.StatusUnauthorized ||
|
|
!strings.Contains(response.Body.String(), `"dispatch_state":"not_dispatched"`) {
|
|
t.Fatalf("health without HMAC status = %d, body=%s", response.Code, response.Body.String())
|
|
}
|
|
|
|
response = api.request(t, http.MethodGet, "/v1/health", nil, auth.Actor{UserID: 1})
|
|
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"service":"virt-host-agent"`) {
|
|
t.Fatalf("signed health status = %d, body=%s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestNetworkSelectorsRequireAuthenticationAndCanonicalInput(t *testing.T) {
|
|
for _, target := range []string{"/v1/vms", "/v1/vms/123e4567-e89b-42d3-a456-426614174000/network"} {
|
|
t.Run(target, func(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
body := validCreateBody("invalid-network-operation", "vm-test")
|
|
if strings.HasSuffix(target, "/network") {
|
|
body = []byte(`{"operation_id":"invalid-network-operation","network":"bridge:br0","confirm_restart":true}`)
|
|
}
|
|
body = bytes.ReplaceAll(body, []byte(`"bridge:br0"`), []byte(`"public"`))
|
|
unsigned := httptest.NewRecorder()
|
|
api.handler.ServeHTTP(unsigned, httptest.NewRequest(http.MethodPost, target, bytes.NewReader(body)))
|
|
if unsigned.Code != http.StatusUnauthorized {
|
|
t.Fatalf("unsigned invalid selector = %d %s", unsigned.Code, unsigned.Body.String())
|
|
}
|
|
response := api.request(t, http.MethodPost, target, body,
|
|
auth.Actor{UserID: 3, CanManage: true, OperationID: "invalid-network-operation"})
|
|
var payload errorEnvelope
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.Code != http.StatusUnprocessableEntity || payload.Error.Code != "invalid_network" ||
|
|
payload.Error.DispatchState != "not_dispatched" || api.hv.creates != 0 {
|
|
t.Fatalf("signed invalid selector = %d %#v", response.Code, payload)
|
|
}
|
|
if _, err := api.journal.Get("invalid-network-operation"); !errors.Is(err, operation.ErrNotFound) {
|
|
t.Fatalf("invalid selector reached the operation journal: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReplayAuthenticationFailureDoesNotClaimNotDispatched(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
|
|
if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, api.secret,
|
|
api.now.Unix(), "fixed-replay-nonce-1234"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first := httptest.NewRecorder()
|
|
api.handler.ServeHTTP(first, request)
|
|
if first.Code != http.StatusOK {
|
|
t.Fatalf("first request status = %d body=%s", first.Code, first.Body.String())
|
|
}
|
|
|
|
replay := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
|
|
if err := auth.Sign(replay, nil, auth.Actor{UserID: 1}, api.secret,
|
|
api.now.Unix(), "fixed-replay-nonce-1234"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := httptest.NewRecorder()
|
|
api.handler.ServeHTTP(response, replay)
|
|
if response.Code != http.StatusUnauthorized ||
|
|
!strings.Contains(response.Body.String(), `"code":"replayed_request"`) ||
|
|
strings.Contains(response.Body.String(), `"dispatch_state"`) {
|
|
t.Fatalf("replay response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestTerminalPreJournalContractIsExplicit(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
tests := []struct {
|
|
name string
|
|
method string
|
|
target string
|
|
body []byte
|
|
actor auth.Actor
|
|
status int
|
|
code string
|
|
}{
|
|
{
|
|
name: "forbidden mutation", method: http.MethodPost, target: "/v1/vms",
|
|
body: []byte(`{}`), actor: auth.Actor{UserID: 1},
|
|
status: http.StatusForbidden, code: "forbidden",
|
|
},
|
|
{
|
|
name: "route missing", method: http.MethodPost, target: "/v1/missing",
|
|
body: []byte(`{}`), actor: auth.Actor{UserID: 1, CanManage: true},
|
|
status: http.StatusNotFound, code: "not_found",
|
|
},
|
|
{
|
|
name: "method mismatch", method: http.MethodDelete, target: "/v1/health",
|
|
actor: auth.Actor{UserID: 1, CanManage: true},
|
|
status: http.StatusMethodNotAllowed, code: "method_not_allowed",
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
response := api.request(t, test.method, test.target, test.body, test.actor)
|
|
var payload errorEnvelope
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.Code != test.status || payload.Error.Code != test.code ||
|
|
payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" ||
|
|
payload.Error.Operation != nil {
|
|
t.Fatalf("response = %d %#v", response.Code, payload)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRetryablePreJournalContractIsExplicit(t *testing.T) {
|
|
t.Run("shutting down", func(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
if err := api.handler.Shutdown(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := api.request(t, http.MethodGet, "/v1/health", nil,
|
|
auth.Actor{UserID: 1})
|
|
assertRetryableNotDispatched(t, response, "agent_shutting_down")
|
|
})
|
|
|
|
t.Run("nonce store unavailable", func(t *testing.T) {
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer journal.Close()
|
|
secret := []byte("0123456789abcdef0123456789abcdef")
|
|
now := time.Unix(2_000_000_000, 0)
|
|
authenticator, err := auth.NewWithNonceStore(
|
|
secret, 5*time.Minute, 10*time.Minute, failingNonceStore{},
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
authenticator.SetClockForTesting(func() time.Time { return now })
|
|
broker, err := consolebroker.NewBroker(time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
server, err := NewServer(authenticator, journal, &fakeHypervisor{},
|
|
broker, 64<<10, time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
_ = server.Shutdown(ctx)
|
|
}()
|
|
request := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
|
|
if err := auth.Sign(request, nil, auth.Actor{UserID: 1}, secret,
|
|
now.Unix(), "nonce-store-failure-1234"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := httptest.NewRecorder()
|
|
server.ServeHTTP(response, request)
|
|
assertRetryableNotDispatched(t, response, "nonce_store_unavailable")
|
|
})
|
|
}
|
|
|
|
func assertRetryableNotDispatched(t *testing.T, response *httptest.ResponseRecorder, code string) {
|
|
t.Helper()
|
|
var payload errorEnvelope
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.Code != http.StatusServiceUnavailable ||
|
|
payload.Error.Code != code || !payload.Error.Retryable ||
|
|
payload.Error.DispatchState != "not_dispatched" ||
|
|
payload.Error.Operation != nil {
|
|
t.Fatalf("response = %d %#v", response.Code, payload)
|
|
}
|
|
}
|
|
|
|
func TestCreateVMOperationIsDurablyIdempotentAndConflictsOnPayloadChange(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 3, CanManage: true, OperationID: "operation-create-one"}
|
|
body := validCreateBody(actor.OperationID, "vm-one")
|
|
first := api.request(t, http.MethodPost, "/v1/vms", body, actor)
|
|
if first.Code != http.StatusCreated {
|
|
t.Fatalf("first create status = %d body=%s", first.Code, first.Body.String())
|
|
}
|
|
if !strings.Contains(first.Body.String(), `"public_pool_key":"0123456789abcdef0123456789abcdef"`) ||
|
|
!strings.Contains(first.Body.String(), `"public_ip_cidr":"8.8.8.10/24"`) {
|
|
t.Fatalf("create response omitted confirmed network allocation: %s", first.Body.String())
|
|
}
|
|
second := api.request(t, http.MethodPost, "/v1/vms", body, actor)
|
|
if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), `"replayed":true`) {
|
|
t.Fatalf("replay status = %d body=%s", second.Code, second.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
creates := api.hv.creates
|
|
lastCreate := api.hv.lastCreate
|
|
api.hv.mu.Unlock()
|
|
if creates != 1 {
|
|
t.Fatalf("backend create calls = %d, want 1", creates)
|
|
}
|
|
if lastCreate.DomainName != "3:vm-123e4567e89b" || lastCreate.Name != "vm-one" {
|
|
t.Fatalf("create names = internal %q display %q", lastCreate.DomainName, lastCreate.Name)
|
|
}
|
|
|
|
conflict := api.request(t, http.MethodPost, "/v1/vms", validCreateBody(actor.OperationID, "vm-different"), actor)
|
|
if conflict.Code != http.StatusConflict ||
|
|
!strings.Contains(conflict.Body.String(), "operation_id_conflict") ||
|
|
strings.Contains(conflict.Body.String(), `"dispatch_state"`) {
|
|
t.Fatalf("conflict status = %d body=%s", conflict.Code, conflict.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestCreateVMRejectsClientSuppliedInternalDomainName(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
body := validCreateBody("operation-create-injected-name", "vm-one")
|
|
body = bytes.Replace(body, []byte(`"source":"cloud"`), []byte(`"source":"cloud","domain_name":"9:vm-123e4567e89b"`), 1)
|
|
response := api.request(t, http.MethodPost, "/v1/vms", body, auth.Actor{
|
|
UserID: 3, CanManage: true, OperationID: "operation-create-injected-name",
|
|
})
|
|
if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), "invalid_json") {
|
|
t.Fatalf("injected domain name response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestFailedCreateRouteUsesReadOnlyReconciliationWithoutReplay(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-create-api-failed"
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
spec := operation.Spec{
|
|
OperationID: operationID, ResourceID: uuid, Action: "create-vm",
|
|
Fingerprint: operation.Fingerprint("POST", "/v1/vms", []byte(`{}`)),
|
|
Destructive: true,
|
|
}
|
|
if record, _, err := api.journal.Execute(context.Background(), spec,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
return nil, hypervisor.NewError(hypervisor.ErrInvalid, "network_selector_invalid", "network selector is not configured", nil)
|
|
}); err == nil || record.State != operation.StateFailed {
|
|
t.Fatalf("initial failed create = %#v err=%v", record, err)
|
|
}
|
|
api.hv.mu.Lock()
|
|
api.hv.reconcileTo = operation.Resolution{
|
|
Resolved: true, Succeeded: false, ErrorCode: "creation_absent",
|
|
Result: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`),
|
|
}
|
|
api.hv.mu.Unlock()
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: operationID}
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
|
|
if response.Code != http.StatusOK ||
|
|
!strings.Contains(response.Body.String(), `"status":"failed"`) ||
|
|
!strings.Contains(response.Body.String(), `"cause_code":"network_selector_invalid"`) ||
|
|
!strings.Contains(response.Body.String(), `"managed_artifacts_absent":true`) {
|
|
t.Fatalf("failed create reconcile response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
reconciles := api.hv.reconciles
|
|
last := api.hv.lastReconcile
|
|
creates := api.hv.creates
|
|
api.hv.mu.Unlock()
|
|
if reconciles != 1 || creates != 0 || last.OperationID != operationID || last.ResourceID != uuid || last.Action != "create-vm" {
|
|
t.Fatalf("reconcile calls=%d creates=%d record=%#v", reconciles, creates, last)
|
|
}
|
|
second := api.request(t, http.MethodPost,
|
|
"/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
|
|
api.hv.mu.Lock()
|
|
reconciles = api.hv.reconciles
|
|
api.hv.mu.Unlock()
|
|
if second.Code != http.StatusOK || reconciles != 1 {
|
|
t.Fatalf("confirmed absence was reinspected: status=%d calls=%d body=%s",
|
|
second.Code, reconciles, second.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestOperationInspectUsesReadOnlyContract(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-inspect-api"
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
persistUncertainOperation(t, api, operation.Spec{
|
|
OperationID: operationID, ResourceID: uuid, Action: "create-snapshot:0123456789abcdef0123456789abcdef:crash",
|
|
Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+uuid+"/snapshots", []byte(`{}`)),
|
|
Destructive: true, LockKeys: []string{"domain:" + uuid},
|
|
})
|
|
token := strings.Repeat("a", 64)
|
|
api.hv.inspectionTo = hypervisor.OperationInspection{
|
|
OperationID: operationID, OperationStatus: operation.StateUncertain,
|
|
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeUncertain,
|
|
SummaryCode: hypervisor.InspectionSummaryAbortableJob,
|
|
AbortJob: hypervisor.OperationAbortInspection{Available: true, Kind: "backup", InspectionToken: token},
|
|
}
|
|
denied := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil,
|
|
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true, OperationID: operationID})
|
|
if denied.Code != http.StatusForbidden {
|
|
t.Fatalf("ordinary manager inspect = %d %s", denied.Code, denied.Body.String())
|
|
}
|
|
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
|
|
response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
|
|
if response.Code != http.StatusOK ||
|
|
!strings.Contains(response.Body.String(), `"summary_code":"abortable_job_active"`) ||
|
|
!strings.Contains(response.Body.String(), `"inspection_token":"`+token+`"`) ||
|
|
strings.Contains(response.Body.String(), `"result"`) {
|
|
t.Fatalf("inspect response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
defer api.hv.mu.Unlock()
|
|
if api.hv.inspections != 1 || api.hv.reconciles != 0 || api.hv.lastInspection.OperationID != operationID {
|
|
t.Fatalf("inspection calls=%d reconcile calls=%d record=%#v",
|
|
api.hv.inspections, api.hv.reconciles, api.hv.lastInspection)
|
|
}
|
|
}
|
|
|
|
func TestOperationAbortJobRequiresInspectionTokenAndReturnsInspection(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-abort-api"
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
persistUncertainOperation(t, api, operation.Spec{
|
|
OperationID: operationID, ResourceID: uuid, Action: "create-snapshot:0123456789abcdef0123456789abcdef:crash",
|
|
Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+uuid+"/snapshots", []byte(`{}`)),
|
|
Destructive: true, LockKeys: []string{"domain:" + uuid},
|
|
})
|
|
token := strings.Repeat("b", 64)
|
|
api.hv.abortTo = hypervisor.OperationInspection{
|
|
OperationID: operationID, OperationStatus: operation.StateUncertain,
|
|
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeUncertain,
|
|
SummaryCode: hypervisor.InspectionSummaryIncomplete,
|
|
}
|
|
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
|
|
invalid := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/abort-job",
|
|
[]byte(`{"inspection_token":"`+strings.Repeat("A", 64)+`"}`), actor)
|
|
if invalid.Code != http.StatusUnprocessableEntity || !strings.Contains(invalid.Body.String(), "invalid_inspection_token") {
|
|
t.Fatalf("invalid abort token response = %d %s", invalid.Code, invalid.Body.String())
|
|
}
|
|
response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/abort-job",
|
|
[]byte(`{"inspection_token":"`+token+`"}`), actor)
|
|
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"summary_code":"evidence_incomplete"`) {
|
|
t.Fatalf("abort response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
defer api.hv.mu.Unlock()
|
|
if api.hv.abortCalls != 1 || api.hv.lastAbortToken != token {
|
|
t.Fatalf("abort calls=%d token=%q", api.hv.abortCalls, api.hv.lastAbortToken)
|
|
}
|
|
}
|
|
|
|
func TestOperationCleanupUsesJournalAdmissionAndExactPlan(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-cleanup-api"
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
persistUncertainOperation(t, api, operation.Spec{
|
|
OperationID: operationID, ResourceID: uuid, Action: "delete-vm",
|
|
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{}`)),
|
|
Destructive: true, LockKeys: []string{"domain:" + uuid},
|
|
})
|
|
planID := strings.Repeat("c", 64)
|
|
api.hv.cleanupTo = operation.Resolution{Resolved: true, Succeeded: true,
|
|
Result: json.RawMessage(`{"absent":true,"managed_disks_absent":true}`)}
|
|
api.hv.inspectionTo = hypervisor.OperationInspection{
|
|
OperationID: operationID, OperationStatus: operation.StateSucceeded,
|
|
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeSucceeded,
|
|
SummaryCode: hypervisor.InspectionSummarySucceeded,
|
|
}
|
|
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
|
|
response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/cleanup",
|
|
[]byte(`{"plan_id":"`+planID+`"}`), actor)
|
|
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"outcome":"succeeded"`) {
|
|
t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
stored, err := api.journal.Get(operationID)
|
|
if err != nil || stored.State != operation.StateSucceeded {
|
|
t.Fatalf("cleanup journal record = %#v err=%v", stored, err)
|
|
}
|
|
api.hv.mu.Lock()
|
|
defer api.hv.mu.Unlock()
|
|
if api.hv.cleanupCalls != 1 || api.hv.lastCleanupPlan != planID || api.hv.inspections != 1 {
|
|
t.Fatalf("cleanup calls=%d plan=%q inspections=%d",
|
|
api.hv.cleanupCalls, api.hv.lastCleanupPlan, api.hv.inspections)
|
|
}
|
|
}
|
|
|
|
func TestConfirmInspectionRequiresFreshExactReadOnlyEvidenceAndReleasesBlocker(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-confirm-inspection"
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
lockKey := "domain:" + uuid
|
|
persistUncertainOperation(t, api, operation.Spec{
|
|
OperationID: operationID, ResourceID: uuid, Action: "delete-vm",
|
|
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{}`)),
|
|
Destructive: true, LockKeys: []string{lockKey},
|
|
})
|
|
escalateOperationNeedsAttention(t, api.journal)
|
|
before, err := api.journal.Get(operationID)
|
|
if err != nil || before.State != operation.StateNeedsAttention {
|
|
t.Fatalf("needs-attention seed = %#v err=%v", before, err)
|
|
}
|
|
token := strings.Repeat("d", 64)
|
|
resolution := operation.Resolution{Resolved: true, Succeeded: true,
|
|
Result: json.RawMessage(`{"deleted":true,"managed_artifacts_absent":true}`)}
|
|
api.hv.inspectionTo = hypervisor.OperationInspection{
|
|
OperationID: operationID, OperationStatus: operation.StateNeedsAttention,
|
|
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeSucceeded,
|
|
SummaryCode: hypervisor.InspectionSummarySucceeded,
|
|
Confirmation: hypervisor.OperationConfirmation{Available: true, InspectionToken: token},
|
|
ReadOnlyResolution: &resolution,
|
|
}
|
|
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
|
|
target := "/v1/operations/" + operationID + "/confirm-inspection"
|
|
generic := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/reconcile", []byte(`{}`), actor)
|
|
if generic.Code != http.StatusConflict || !strings.Contains(generic.Body.String(), `"code":"operation_needs_attention"`) {
|
|
t.Fatalf("generic needs-attention reconcile = %d %s", generic.Code, generic.Body.String())
|
|
}
|
|
if api.hv.reconciles != 0 {
|
|
t.Fatalf("generic needs-attention reconcile reached backend: calls=%d", api.hv.reconciles)
|
|
}
|
|
|
|
stale := api.request(t, http.MethodPost, target,
|
|
[]byte(`{"inspection_token":"`+strings.Repeat("e", 64)+`"}`), actor)
|
|
if stale.Code != http.StatusConflict || !strings.Contains(stale.Body.String(), `"code":"inspection_inconclusive"`) {
|
|
t.Fatalf("stale confirmation = %d %s", stale.Code, stale.Body.String())
|
|
}
|
|
afterStale, err := api.journal.Get(operationID)
|
|
if err != nil || !reflect.DeepEqual(before, afterStale) {
|
|
t.Fatalf("stale confirmation mutated journal: before=%#v after=%#v err=%v", before, afterStale, err)
|
|
}
|
|
|
|
api.hv.inspectionTo.Confirmation = hypervisor.OperationConfirmation{}
|
|
api.hv.inspectionTo.ReadOnlyResolution = nil
|
|
inconclusive := api.request(t, http.MethodPost, target,
|
|
[]byte(`{"inspection_token":"`+token+`"}`), actor)
|
|
if inconclusive.Code != http.StatusConflict || !strings.Contains(inconclusive.Body.String(), `"code":"inspection_inconclusive"`) {
|
|
t.Fatalf("inconclusive confirmation = %d %s", inconclusive.Code, inconclusive.Body.String())
|
|
}
|
|
afterInconclusive, err := api.journal.Get(operationID)
|
|
if err != nil || !reflect.DeepEqual(before, afterInconclusive) {
|
|
t.Fatalf("inconclusive confirmation mutated journal: before=%#v after=%#v err=%v", before, afterInconclusive, err)
|
|
}
|
|
|
|
api.hv.inspectionTo.Confirmation = hypervisor.OperationConfirmation{Available: true, InspectionToken: token}
|
|
api.hv.inspectionTo.ReadOnlyResolution = &resolution
|
|
confirmed := api.request(t, http.MethodPost, target,
|
|
[]byte(`{"inspection_token":"`+token+`"}`), actor)
|
|
if confirmed.Code != http.StatusOK || !strings.Contains(confirmed.Body.String(), `"status":"succeeded"`) ||
|
|
!strings.Contains(confirmed.Body.String(), `"managed_artifacts_absent":true`) {
|
|
t.Fatalf("confirmed inspection = %d %s", confirmed.Code, confirmed.Body.String())
|
|
}
|
|
stored, err := api.journal.Get(operationID)
|
|
if err != nil || stored.State != operation.StateSucceeded {
|
|
t.Fatalf("confirmed operation = %#v err=%v", stored, err)
|
|
}
|
|
called := false
|
|
_, _, err = api.journal.Execute(context.Background(), operation.Spec{
|
|
OperationID: "operation-after-confirm", ResourceID: uuid, Action: "delete-vm",
|
|
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{"after":true}`)),
|
|
Destructive: true, LockKeys: []string{lockKey},
|
|
}, func(context.Context) (json.RawMessage, error) {
|
|
called = true
|
|
return json.RawMessage(`{"deleted":true}`), nil
|
|
})
|
|
if err != nil || !called {
|
|
t.Fatalf("confirmed operation did not release durable blocker: called=%v err=%v", called, err)
|
|
}
|
|
api.hv.mu.Lock()
|
|
defer api.hv.mu.Unlock()
|
|
if api.hv.inspections != 3 || api.hv.reconciles != 0 || api.hv.cleanupCalls != 0 || api.hv.abortCalls != 0 {
|
|
t.Fatalf("confirmation host calls inspections=%d reconcile=%d cleanup=%d abort=%d",
|
|
api.hv.inspections, api.hv.reconciles, api.hv.cleanupCalls, api.hv.abortCalls)
|
|
}
|
|
}
|
|
|
|
func TestConfirmInspectionAcceptsAlreadyFailedForceRequiredRecord(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-force-required-confirm"
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
persistForceRequiredOperation(t, api, operationID, uuid, "delete-vm")
|
|
token := strings.Repeat("f", 64)
|
|
resolution := operation.Resolution{
|
|
Resolved: true, ErrorCode: "force_required",
|
|
ErrorMessage: "guest remains active and requires an explicit force delete",
|
|
}
|
|
api.hv.inspectionTo = hypervisor.OperationInspection{
|
|
OperationID: operationID, OperationStatus: operation.StateFailed,
|
|
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeFailed,
|
|
SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "force_required",
|
|
Confirmation: hypervisor.OperationConfirmation{Available: true, InspectionToken: token},
|
|
ReadOnlyResolution: &resolution,
|
|
}
|
|
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
|
|
response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/confirm-inspection",
|
|
[]byte(`{"inspection_token":"`+token+`"}`), actor)
|
|
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"status":"failed"`) ||
|
|
!strings.Contains(response.Body.String(), `"error_code":"force_required"`) {
|
|
t.Fatalf("force-required confirmation = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestFailedForceRequiredInspectAlwaysReadsFreshHostState(t *testing.T) {
|
|
for _, action := range []string{"delete-vm", "service-suspend"} {
|
|
t.Run(action, func(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-inspect-" + strings.ReplaceAll(action, "-", "")
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
persistForceRequiredOperation(t, api, operationID, uuid, action)
|
|
api.hv.inspectionTo = hypervisor.OperationInspection{
|
|
OperationID: operationID, OperationStatus: operation.StateFailed,
|
|
CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeFailed,
|
|
SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "force_required",
|
|
}
|
|
generic := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/reconcile", []byte(`{}`),
|
|
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true, OperationID: operationID})
|
|
if generic.Code != http.StatusConflict || !strings.Contains(generic.Body.String(), `"code":"force_required"`) {
|
|
t.Fatalf("generic force-required reconcile = %d %s", generic.Code, generic.Body.String())
|
|
}
|
|
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
|
|
response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
|
|
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"error_code":"force_required"`) {
|
|
t.Fatalf("fresh force-required inspect = %d %s", response.Code, response.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
defer api.hv.mu.Unlock()
|
|
if api.hv.reconciles != 0 || api.hv.inspections != 1 || api.hv.lastInspection.OperationID != operationID ||
|
|
api.hv.lastInspection.Action != action {
|
|
t.Fatalf("fresh force-required inspect reconcile=%d inspect=%d record=%#v",
|
|
api.hv.reconciles, api.hv.inspections, api.hv.lastInspection)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func escalateOperationNeedsAttention(t *testing.T, journal *operation.Journal) {
|
|
t.Helper()
|
|
coordinator, err := operation.NewRecoveryCoordinatorWithLimits(journal,
|
|
func(context.Context, operation.Record) (operation.Resolution, error) {
|
|
return operation.Resolution{}, nil
|
|
},
|
|
nil, operation.RecoveryLimits{MaxAttempts: 1, MaxAge: time.Hour, MinInterval: time.Nanosecond})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
if err := coordinator.Start(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-coordinator.Ready():
|
|
case <-time.After(time.Second):
|
|
t.Fatal("recovery coordinator did not reach needs_attention")
|
|
}
|
|
coordinator.Stop()
|
|
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer waitCancel()
|
|
if err := coordinator.Wait(waitCtx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestOperationRecoveryMutationsRejectOrdinaryManager(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-recovery-auth"
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
persistUncertainOperation(t, api, operation.Spec{
|
|
OperationID: operationID, ResourceID: uuid, Action: "delete-vm",
|
|
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{}`)),
|
|
Destructive: true, LockKeys: []string{"domain:" + uuid},
|
|
})
|
|
actor := auth.Actor{UserID: 2, IsAdmin: true, CanManage: true, OperationID: operationID}
|
|
for _, test := range []struct {
|
|
name string
|
|
method string
|
|
path string
|
|
body []byte
|
|
}{
|
|
{name: "inspect", method: http.MethodGet, path: "/v1/operations/" + operationID + "/inspect"},
|
|
{name: "confirm", method: http.MethodPost, path: "/v1/operations/" + operationID + "/confirm-inspection", body: []byte(`{"inspection_token":"` + strings.Repeat("c", 64) + `"}`)},
|
|
{name: "abort", method: http.MethodPost, path: "/v1/operations/" + operationID + "/abort-job", body: []byte(`{"inspection_token":"` + strings.Repeat("a", 64) + `"}`)},
|
|
{name: "cleanup", method: http.MethodPost, path: "/v1/operations/" + operationID + "/cleanup", body: []byte(`{"plan_id":"` + strings.Repeat("b", 64) + `"}`)},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
response := api.request(t, test.method, test.path, test.body, actor)
|
|
if response.Code != http.StatusForbidden {
|
|
t.Fatalf("ordinary manager response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
})
|
|
}
|
|
api.hv.mu.Lock()
|
|
defer api.hv.mu.Unlock()
|
|
if api.hv.inspections != 0 || api.hv.abortCalls != 0 || api.hv.cleanupCalls != 0 {
|
|
t.Fatalf("unauthorized calls: inspect=%d abort=%d cleanup=%d",
|
|
api.hv.inspections, api.hv.abortCalls, api.hv.cleanupCalls)
|
|
}
|
|
}
|
|
|
|
func TestOperationRecoveryErrorsNeverExposeJournalOrRawCause(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-recovery-redaction"
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
persistUncertainOperation(t, api, operation.Spec{
|
|
OperationID: operationID, ResourceID: uuid, Action: "delete-vm",
|
|
Fingerprint: operation.Fingerprint("DELETE", "/v1/vms/"+uuid, []byte(`{"private":"request"}`)),
|
|
Destructive: true, LockKeys: []string{"domain:" + uuid},
|
|
})
|
|
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID}
|
|
rawSentinel := "raw-libvirt-path-/host/private-and-secret"
|
|
stableErr := hypervisor.NewError(hypervisor.ErrUnavailable, "inspection_unavailable", "operation evidence is unavailable", errors.New(rawSentinel))
|
|
|
|
api.hv.inspectionErr = stableErr
|
|
inspect := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
|
|
assertRecoveryErrorRedacted(t, inspect, rawSentinel, uuid)
|
|
api.hv.inspectionErr = nil
|
|
|
|
api.hv.abortErr = stableErr
|
|
abort := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/abort-job",
|
|
[]byte(`{"inspection_token":"`+strings.Repeat("a", 64)+`"}`), actor)
|
|
assertRecoveryErrorRedacted(t, abort, rawSentinel, uuid)
|
|
api.hv.abortErr = nil
|
|
|
|
api.hv.cleanupErr = operation.MarkUncertain(stableErr)
|
|
cleanup := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/cleanup",
|
|
[]byte(`{"plan_id":"`+strings.Repeat("b", 64)+`"}`), actor)
|
|
assertRecoveryErrorRedacted(t, cleanup, rawSentinel, uuid)
|
|
}
|
|
|
|
func assertRecoveryErrorRedacted(t *testing.T, response *httptest.ResponseRecorder, forbidden ...string) {
|
|
t.Helper()
|
|
if response.Code < 400 {
|
|
t.Fatalf("recovery error unexpectedly succeeded: %d %s", response.Code, response.Body.String())
|
|
}
|
|
body := response.Body.String()
|
|
for _, field := range []string{`"operation":`, `"result":`, `"cause_code":`, `"fingerprint":`, `"resource_id":`} {
|
|
if strings.Contains(body, field) {
|
|
t.Fatalf("recovery error exposed journal field %s: %s", field, body)
|
|
}
|
|
}
|
|
for _, value := range forbidden {
|
|
if strings.Contains(body, value) {
|
|
t.Fatalf("recovery error exposed private value %q: %s", value, body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func persistUncertainOperation(t *testing.T, api *testAPI, spec operation.Spec) {
|
|
t.Helper()
|
|
record, _, err := api.journal.Execute(context.Background(), spec,
|
|
func(context.Context) (json.RawMessage, error) {
|
|
return nil, operation.MarkUncertain(errors.New("injected ambiguous result"))
|
|
})
|
|
if err == nil || record.State != operation.StateUncertain {
|
|
t.Fatalf("persist uncertain operation = %#v err=%v", record, err)
|
|
}
|
|
}
|
|
|
|
func persistForceRequiredOperation(t *testing.T, api *testAPI, operationID, uuid, action string) {
|
|
t.Helper()
|
|
record, _, err := api.journal.Execute(context.Background(), operation.Spec{
|
|
OperationID: operationID, ResourceID: uuid, Action: action,
|
|
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/"+uuid, []byte(`{"remove_managed_disks":true}`)),
|
|
Destructive: true, LockKeys: hypervisor.DeleteVMAdmission(uuid).LockKeys,
|
|
}, func(context.Context) (json.RawMessage, error) {
|
|
return nil, hypervisor.NewError(hypervisor.ErrConflict, "force_required",
|
|
"guest remains active and requires an explicit force delete", nil)
|
|
})
|
|
if err == nil || record.State != operation.StateFailed || record.Error == nil || record.Error.Code != "force_required" {
|
|
t.Fatalf("persist force-required delete = %#v err=%v", record, err)
|
|
}
|
|
}
|
|
|
|
func TestForceDeleteRequiresSuperAdminAckAndLinkedGracefulOperation(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
priorID := "operation-delete-graceful-parent"
|
|
persistForceRequiredOperation(t, api, priorID, uuid, "delete-vm")
|
|
target := "/v1/vms/" + uuid + "/force-delete"
|
|
|
|
ordinaryID := "operation-force-ordinary"
|
|
ordinaryBody := []byte(`{"operation_id":"` + ordinaryID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":true}`)
|
|
ordinary := api.request(t, http.MethodPost, target, ordinaryBody,
|
|
auth.Actor{UserID: 2, IsAdmin: true, CanManage: true, OperationID: ordinaryID})
|
|
if ordinary.Code != http.StatusForbidden {
|
|
t.Fatalf("ordinary manager force delete = %d %s", ordinary.Code, ordinary.Body.String())
|
|
}
|
|
|
|
missingAckID := "operation-force-no-ack"
|
|
missingAckBody := []byte(`{"operation_id":"` + missingAckID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":false}`)
|
|
missingAck := api.request(t, http.MethodPost, target, missingAckBody,
|
|
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: missingAckID})
|
|
if missingAck.Code != http.StatusUnprocessableEntity || !strings.Contains(missingAck.Body.String(), `"code":"data_loss_ack_required"`) {
|
|
t.Fatalf("force delete without acknowledgement = %d %s", missingAck.Code, missingAck.Body.String())
|
|
}
|
|
|
|
reusedBody := []byte(`{"operation_id":"` + priorID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":true}`)
|
|
reused := api.request(t, http.MethodPost, target, reusedBody,
|
|
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: priorID})
|
|
if reused.Code != http.StatusUnprocessableEntity || !strings.Contains(reused.Body.String(), `"code":"force_operation_id_reused"`) {
|
|
t.Fatalf("reused force operation id = %d %s", reused.Code, reused.Body.String())
|
|
}
|
|
|
|
forceID := "operation-force-delete-successor"
|
|
body := []byte(`{"operation_id":"` + forceID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":true}`)
|
|
response := api.request(t, http.MethodPost, target, body,
|
|
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: forceID})
|
|
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"forced":true`) ||
|
|
!strings.Contains(response.Body.String(), `"managed_artifacts_absent":true`) {
|
|
t.Fatalf("authorized force delete = %d %s", response.Code, response.Body.String())
|
|
}
|
|
select {
|
|
case request := <-api.hv.forceDeleteCalls:
|
|
if request.OperationID != forceID || request.PriorOperationID != priorID || !request.AckDataLoss || !request.RemoveManagedDisks {
|
|
t.Fatalf("force delete request = %#v", request)
|
|
}
|
|
default:
|
|
t.Fatal("authorized force delete did not reach backend")
|
|
}
|
|
stored, err := api.journal.Get(forceID)
|
|
if err != nil || stored.Action != "force-delete-vm" || stored.SupersedesOperationID != priorID || stored.State != operation.StateSucceeded {
|
|
t.Fatalf("force delete record = %#v err=%v", stored, err)
|
|
}
|
|
prior, err := api.journal.Get(priorID)
|
|
if err != nil || prior.State != operation.StateFailed || prior.Error == nil || prior.Error.Code != "force_delete_completed" {
|
|
t.Fatalf("closed graceful predecessor = %#v err=%v", prior, err)
|
|
}
|
|
replay := api.request(t, http.MethodPost, target, body,
|
|
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: forceID})
|
|
if replay.Code != http.StatusOK || !strings.Contains(replay.Body.String(), `"forced":true`) {
|
|
t.Fatalf("force delete replay = %d %s", replay.Code, replay.Body.String())
|
|
}
|
|
select {
|
|
case request := <-api.hv.forceDeleteCalls:
|
|
t.Fatalf("force delete replay reached backend: %#v", request)
|
|
default:
|
|
}
|
|
reordered := []byte(`{"prior_operation_id":"` + priorID + `","operation_id":"` + forceID + `","remove_managed_disks":true,"ack_data_loss":true}`)
|
|
conflict := api.request(t, http.MethodPost, target, reordered,
|
|
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: forceID})
|
|
if conflict.Code != http.StatusConflict || !strings.Contains(conflict.Body.String(), `"code":"operation_id_conflict"`) {
|
|
t.Fatalf("changed force replay = %d %s", conflict.Code, conflict.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestGracefulDeleteForceRequiredIsStableAndDoesNotDispatchForce(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
api.hv.deleteErr = hypervisor.NewError(hypervisor.ErrConflict, "force_required",
|
|
"guest remains active and requires an explicit force delete", nil)
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
operationID := "operation-delete-needs-force"
|
|
body := []byte(`{"operation_id":"` + operationID + `","remove_managed_disks":true}`)
|
|
response := api.request(t, http.MethodDelete, "/v1/vms/"+uuid, body,
|
|
auth.Actor{UserID: 2, CanManage: true, OperationID: operationID})
|
|
if response.Code != http.StatusConflict || !strings.Contains(response.Body.String(), `"code":"force_required"`) {
|
|
t.Fatalf("graceful delete force-required = %d %s", response.Code, response.Body.String())
|
|
}
|
|
stored, err := api.journal.Get(operationID)
|
|
if err != nil || stored.State != operation.StateFailed || stored.Error == nil || stored.Error.Code != "force_required" {
|
|
t.Fatalf("graceful delete record = %#v err=%v", stored, err)
|
|
}
|
|
select {
|
|
case request := <-api.hv.deleteCalls:
|
|
if !request.RemoveManagedDisks || request.OperationID != operationID {
|
|
t.Fatalf("graceful delete request = %#v", request)
|
|
}
|
|
default:
|
|
t.Fatal("graceful delete did not reach its backend")
|
|
}
|
|
select {
|
|
case request := <-api.hv.forceDeleteCalls:
|
|
t.Fatalf("graceful delete dispatched force backend: %#v", request)
|
|
default:
|
|
}
|
|
}
|
|
|
|
func TestForceDeleteAcceptsExpiryServiceSuspendForceRequiredPredecessor(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
priorID := "operation-expiry-service-suspend"
|
|
persistForceRequiredOperation(t, api, priorID, uuid, "service-suspend")
|
|
forceID := "operation-expiry-force-delete"
|
|
body := []byte(`{"operation_id":"` + forceID + `","prior_operation_id":"` + priorID + `","remove_managed_disks":true,"ack_data_loss":true}`)
|
|
response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/force-delete", body,
|
|
auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: forceID})
|
|
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"forced":true`) {
|
|
t.Fatalf("expiry force delete = %d %s", response.Code, response.Body.String())
|
|
}
|
|
stored, err := api.journal.Get(forceID)
|
|
if err != nil || stored.SupersedesOperationID != priorID || stored.State != operation.StateSucceeded {
|
|
t.Fatalf("expiry force delete record = %#v err=%v", stored, err)
|
|
}
|
|
}
|
|
|
|
func TestUnsupportedTypedMutationUsesStable501Contract(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-network-one"}
|
|
body := []byte(`{"operation_id":"operation-network-one","reason":"quota"}`)
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/network/suspend", body, actor)
|
|
if response.Code != http.StatusNotImplemented || !strings.Contains(response.Body.String(), `"code":"unsupported"`) {
|
|
t.Fatalf("unsupported status = %d body=%s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestReconciliableMutationRoutesPersistOnlySafeExpectedState(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
tests := []struct {
|
|
operationID string
|
|
method string
|
|
path string
|
|
body string
|
|
kind string
|
|
}{
|
|
{"operation-settings-route", http.MethodPatch, "/v1/vms/" + uuid + "/settings", `{"operation_id":"operation-settings-route","root_password":"NeverPersistThis!123","confirm_restart":true}`, operation.ExpectedUpdateSettings},
|
|
{"operation-change-network", http.MethodPost, "/v1/vms/" + uuid + "/network", `{"operation_id":"operation-change-network","network":"bridge:br0","public_pool_key":"0123456789abcdef0123456789abcdef","public_pool_cidr":"8.8.8.0/24","public_ip_cidr":"8.8.8.10/24","gateway":"8.8.8.1","dns_servers":["1.1.1.1"],"confirm_restart":true}`, operation.ExpectedChangeNetwork},
|
|
{"operation-network-route", http.MethodPost, "/v1/vms/" + uuid + "/network/resume", `{"operation_id":"operation-network-route"}`, operation.ExpectedNetworkState},
|
|
{"operation-service-route", http.MethodPost, "/v1/vms/" + uuid + "/service/resume", `{"operation_id":"operation-service-route","start_if_suspended":true}`, operation.ExpectedServiceState},
|
|
{"operation-firewall-route", http.MethodPut, "/v1/vms/" + uuid + "/firewall", `{"operation_id":"operation-firewall-route","revision":1,"default_in":"drop","default_out":"accept","rules":[]}`, operation.ExpectedFirewall},
|
|
}
|
|
for _, test := range tests {
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: test.operationID}
|
|
response := api.request(t, test.method, test.path, []byte(test.body), actor)
|
|
if response.Code != http.StatusNotImplemented {
|
|
t.Fatalf("%s status=%d body=%s", test.kind, response.Code, response.Body.String())
|
|
}
|
|
record, err := api.journal.Get(test.operationID)
|
|
if err != nil || record.ExpectedState == nil || record.ExpectedState.Kind != test.kind || record.ExpectedState.TransitionID != test.operationID {
|
|
t.Fatalf("%s record=%#v err=%v", test.kind, record, err)
|
|
}
|
|
if test.kind == operation.ExpectedChangeNetwork && (record.ExpectedState.NetworkSourceType != "bridge" || record.ExpectedState.NetworkSourceName != "br0") {
|
|
t.Fatalf("network source was not durably resolved: %#v", record.ExpectedState)
|
|
}
|
|
encoded, err := json.Marshal(record.ExpectedState)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if bytes.Contains(encoded, []byte("NeverPersistThis!123")) || bytes.Contains(encoded, []byte("ssh-")) {
|
|
t.Fatalf("%s expected state contains a credential: %s", test.kind, encoded)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestConsoleTicketIsHMACProtectedAndDoesNotExposeTarget(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true}
|
|
response := api.request(t, http.MethodPost,
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/console-ticket", nil, actor)
|
|
if response.Code != http.StatusCreated {
|
|
t.Fatalf("console ticket status = %d body=%s", response.Code, response.Body.String())
|
|
}
|
|
if strings.Contains(response.Body.String(), "127.0.0.1") ||
|
|
strings.Contains(response.Body.String(), "/v1/console") ||
|
|
strings.Contains(response.Body.String(), "protocol") ||
|
|
!strings.Contains(response.Body.String(), `"ticket"`) {
|
|
t.Fatalf("unsafe console ticket response: %s", response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestBodyLimitIsEnforcedBeforeAuthentication(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
api.handler.MaxBodyBytes = 1024
|
|
request := httptest.NewRequest(http.MethodPost, "/v1/vms", bytes.NewReader(make([]byte, 1025)))
|
|
response := httptest.NewRecorder()
|
|
api.handler.ServeHTTP(response, request)
|
|
if response.Code != http.StatusRequestEntityTooLarge {
|
|
t.Fatalf("oversized body status = %d body=%s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestCatalogImportsAreDurableAsyncAndAcceptPrivateSourceToken(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
token := strings.Repeat("a", 64)
|
|
tests := []struct {
|
|
name string
|
|
target string
|
|
actor auth.Actor
|
|
body string
|
|
check func(*testing.T)
|
|
}{
|
|
{
|
|
name: "template", target: "/v1/templates",
|
|
actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-import"},
|
|
body: `{"operation_id":"operation-template-import","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","source_token":"` + token + `","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`,
|
|
check: func(t *testing.T) {
|
|
select {
|
|
case request := <-api.hv.templateCalls:
|
|
if request.SHA256 != "" || request.SourceToken != token {
|
|
t.Fatalf("template request = %#v", request)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("template import was not dispatched")
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "rescue", target: "/v1/rescue-templates",
|
|
actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-rescue-import"},
|
|
body: `{"operation_id":"operation-rescue-import","name":"Debian Rescue","source_url":"https://images.example.com/rescue.iso","source_token":"` + token + `"}`,
|
|
check: func(t *testing.T) {
|
|
select {
|
|
case request := <-api.hv.rescueCalls:
|
|
if !catalogIDPattern.MatchString(request.ID) || request.SourceToken != token {
|
|
t.Fatalf("rescue request = %#v", request)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("rescue import was not dispatched")
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "user ISO", target: "/v1/user-isos",
|
|
actor: auth.Actor{UserID: 42, CanManage: true, OperationID: "operation-useriso-import"},
|
|
body: `{"operation_id":"operation-useriso-import","owner_id":42,"name":"Installer","source_url":"https://images.example.com/user.iso","source_token":"` + token + `"}`,
|
|
check: func(t *testing.T) {
|
|
select {
|
|
case request := <-api.hv.userISOCalls:
|
|
if !catalogIDPattern.MatchString(request.ID) || request.OwnerID != 42 || request.SourceToken != token {
|
|
t.Fatalf("user ISO request = %#v", request)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("user ISO import was not dispatched")
|
|
}
|
|
},
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
response := api.request(t, http.MethodPost, test.target, []byte(test.body), test.actor)
|
|
if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"status":"dispatching"`) {
|
|
t.Fatalf("async import response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
test.check(t)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAsyncImportNeverReturnsAcceptedWhileAnotherCallerOwnsStartup(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
const operationID = "operation-import-start-race"
|
|
if !api.handler.claimAsync(operationID) {
|
|
t.Fatal("could not reserve async startup claim")
|
|
}
|
|
defer api.handler.releaseAsync(operationID)
|
|
body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
|
|
response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
|
|
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
|
|
})
|
|
if response.Code != http.StatusServiceUnavailable ||
|
|
!strings.Contains(response.Body.String(), `"code":"operation_starting"`) ||
|
|
!strings.Contains(response.Body.String(), `"status":"reserved"`) ||
|
|
strings.Contains(response.Body.String(), `"dispatch_state"`) {
|
|
t.Fatalf("startup loser response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
record, err := api.journal.Get(operationID)
|
|
if err != nil || record.State != operation.StateReserved {
|
|
t.Fatalf("startup loser journal = %#v err=%v", record, err)
|
|
}
|
|
}
|
|
|
|
func TestAsyncImportAdmissionTimeoutIsExplicitlyNotDispatched(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
api.handler.OperationTimeout = 20 * time.Millisecond
|
|
holderStarted := make(chan struct{})
|
|
releaseHolder := make(chan struct{})
|
|
holderDone := make(chan struct{})
|
|
go func() {
|
|
defer close(holderDone)
|
|
_, _, _ = api.journal.Execute(context.Background(), operation.Spec{
|
|
OperationID: "operation-template-lock-holder",
|
|
ResourceID: "template:debian-12",
|
|
Action: "update-template",
|
|
Fingerprint: operation.Fingerprint(http.MethodPatch, "/v1/templates/debian-12", []byte(`{}`)),
|
|
LockKeys: []string{"catalog:template/debian-12"},
|
|
}, func(context.Context) (json.RawMessage, error) {
|
|
close(holderStarted)
|
|
<-releaseHolder
|
|
return json.RawMessage(`{}`), nil
|
|
})
|
|
}()
|
|
<-holderStarted
|
|
|
|
const operationID = "operation-template-lock-waiter"
|
|
body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
|
|
response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
|
|
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
|
|
})
|
|
close(releaseHolder)
|
|
<-holderDone
|
|
|
|
var payload errorEnvelope
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.Code != http.StatusServiceUnavailable || payload.Error.Code != "backend_timeout" ||
|
|
!payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" ||
|
|
payload.Error.Operation != nil {
|
|
t.Fatalf("async admission timeout = %d %#v", response.Code, payload)
|
|
}
|
|
persisted, err := api.journal.Get(operationID)
|
|
if err != nil || persisted.State != operation.StateFailed || persisted.Error == nil ||
|
|
persisted.Error.Code != "operation_not_dispatched" {
|
|
t.Fatalf("async admission journal = %#v err=%v", persisted, err)
|
|
}
|
|
select {
|
|
case request := <-api.hv.templateCalls:
|
|
t.Fatalf("timed-out import reached backend: %#v", request)
|
|
default:
|
|
}
|
|
}
|
|
|
|
func TestAsyncImportCapacityFailureIsExplicitlyNotDispatched(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
for index := 0; index < cap(api.handler.asyncSlots); index++ {
|
|
api.handler.asyncSlots <- struct{}{}
|
|
}
|
|
|
|
const operationID = "operation-template-no-capacity"
|
|
body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
|
|
response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
|
|
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
|
|
})
|
|
var payload errorEnvelope
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.Code != http.StatusServiceUnavailable || payload.Error.Code != "agent_busy" ||
|
|
!payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" ||
|
|
payload.Error.Operation != nil {
|
|
t.Fatalf("async capacity response = %d %#v", response.Code, payload)
|
|
}
|
|
persisted, err := api.journal.Get(operationID)
|
|
if err != nil || persisted.State != operation.StateFailed || persisted.Error == nil ||
|
|
persisted.Error.Code != "operation_not_dispatched" {
|
|
t.Fatalf("async capacity journal = %#v err=%v", persisted, err)
|
|
}
|
|
select {
|
|
case request := <-api.hv.templateCalls:
|
|
t.Fatalf("capacity-rejected import reached backend: %#v", request)
|
|
default:
|
|
}
|
|
if !api.handler.claimAsync(operationID) {
|
|
t.Fatal("could not hold async restart claim")
|
|
}
|
|
duplicate := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
|
|
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
|
|
})
|
|
api.handler.releaseAsync(operationID)
|
|
var duplicatePayload errorEnvelope
|
|
if err := json.Unmarshal(duplicate.Body.Bytes(), &duplicatePayload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if duplicate.Code != http.StatusServiceUnavailable || duplicatePayload.Error.Code != "operation_starting" ||
|
|
!duplicatePayload.Error.Retryable || duplicatePayload.Error.DispatchState != "" ||
|
|
duplicatePayload.Error.Operation == nil ||
|
|
duplicatePayload.Error.Operation.State != operation.StateReserved ||
|
|
duplicatePayload.Error.Operation.Error != nil || duplicatePayload.Error.Operation.Result != nil {
|
|
t.Fatalf("async failed-not-dispatched startup duplicate = %d %#v", duplicate.Code, duplicatePayload)
|
|
}
|
|
persisted, err = api.journal.Get(operationID)
|
|
if err != nil || !operation.CanRetryNotDispatched(persisted) {
|
|
t.Fatalf("async startup duplicate changed journal = %#v err=%v", persisted, err)
|
|
}
|
|
for index := 0; index < cap(api.handler.asyncSlots); index++ {
|
|
<-api.handler.asyncSlots
|
|
}
|
|
retry := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{
|
|
UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID,
|
|
})
|
|
if retry.Code != http.StatusAccepted || !strings.Contains(retry.Body.String(), `"status":"dispatching"`) {
|
|
t.Fatalf("async capacity retry = %d %s", retry.Code, retry.Body.String())
|
|
}
|
|
select {
|
|
case <-api.hv.templateCalls:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("same-ID capacity retry was not dispatched")
|
|
}
|
|
}
|
|
|
|
func TestNotDispatchedTombstoneResponseIsTerminal(t *testing.T) {
|
|
response := httptest.NewRecorder()
|
|
writeOperationFailure(response, operation.Record{
|
|
OperationID: "operation-expired-not-dispatched",
|
|
Action: "create-vm",
|
|
State: operation.StateFailed,
|
|
Tombstone: true,
|
|
Error: &operation.PublicError{
|
|
Code: "operation_not_dispatched", Message: "operation details expired; operation_id remains permanently reserved",
|
|
},
|
|
}, true)
|
|
var payload operationResponse
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.Code != http.StatusServiceUnavailable || payload.Retryable ||
|
|
payload.Status != operation.StateFailed || payload.Error == nil ||
|
|
payload.Error.Code != "operation_not_dispatched" || strings.Contains(response.Body.String(), "dispatch_state") {
|
|
t.Fatalf("not-dispatched tombstone response = %d %#v body=%s", response.Code, payload, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestImportSourceTokenRejectsShortOrNonURLSafeValues(t *testing.T) {
|
|
for _, token := range []string{"too-short", strings.Repeat("a", 31), strings.Repeat("a", 32) + "+"} {
|
|
if err := validateImportSource("https://images.example.com/image.iso", token, ""); err == nil {
|
|
t.Fatalf("token %q unexpectedly accepted", token)
|
|
}
|
|
}
|
|
if err := validateImportSource("https://images.example.com/image.iso", strings.Repeat("a", 32), ""); err != nil {
|
|
t.Fatalf("valid token rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCatalogDeleteResultsContainExactManagedCleanupEvidence(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
id := "0123456789abcdef0123456789abcdef"
|
|
tests := []struct {
|
|
name string
|
|
target string
|
|
operationID string
|
|
resourceType string
|
|
resourceID string
|
|
ownerID int64
|
|
actor auth.Actor
|
|
}{
|
|
{
|
|
name: "template", target: "/v1/templates/debian-12", operationID: "operation-delete-template",
|
|
resourceType: "template", resourceID: "debian-12", actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true},
|
|
},
|
|
{
|
|
name: "rescue template", target: "/v1/rescue-templates/" + id, operationID: "operation-delete-rescue",
|
|
resourceType: "rescue_template", resourceID: id, actor: auth.Actor{UserID: 1, CanManage: true, IsAdmin: true},
|
|
},
|
|
{
|
|
name: "user ISO", target: "/v1/user-isos/" + id, operationID: "operation-delete-useriso",
|
|
resourceType: "user_iso", resourceID: id, ownerID: 42, actor: auth.Actor{UserID: 42, CanManage: true},
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
test.actor.OperationID = test.operationID
|
|
body := []byte(`{"operation_id":"` + test.operationID + `"}`)
|
|
response := api.request(t, http.MethodDelete, test.target, body, test.actor)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("delete response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
var wire struct {
|
|
Status operation.State `json:"status"`
|
|
Result hypervisor.CatalogDeleteResult `json:"result"`
|
|
}
|
|
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if wire.Status != operation.StateSucceeded || wire.Result.ResourceType != test.resourceType || wire.Result.ID != test.resourceID ||
|
|
wire.Result.OwnerID != test.ownerID || !wire.Result.Deleted || !wire.Result.ManagedImageAbsent ||
|
|
!wire.Result.ManagedMetadataAbsent || !wire.Result.ManagedArtifactsAbsent {
|
|
t.Fatalf("delete result = %#v", wire)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCatalogDeleteNotFoundCodesAreTerminalDurableFailures(t *testing.T) {
|
|
for _, code := range []string{"template_not_found", "rescue_template_not_found", "user_iso_not_found"} {
|
|
t.Run(code, func(t *testing.T) {
|
|
response := httptest.NewRecorder()
|
|
writeOperationFailure(response, operation.Record{
|
|
OperationID: "operation-delete-missing", State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: code, Message: "managed catalog resource was not found"},
|
|
}, false)
|
|
if response.Code != http.StatusNotFound {
|
|
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
|
}
|
|
var wire operationResponse
|
|
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if wire.Status != operation.StateFailed || wire.Error == nil || wire.Error.Code != code || wire.Retryable || len(wire.Result) != 0 {
|
|
t.Fatalf("failure = %#v", wire)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestForceStartRescueRequiresExplicitDataLossAcknowledgement(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
uuid := "123e4567-e89b-42d3-a456-426614174000"
|
|
target := "/v1/vms/" + uuid + "/rescue/force-start"
|
|
rejectedActor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rejected"}
|
|
rejected := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rejected","ack_data_loss":false}`), rejectedActor)
|
|
if rejected.Code != http.StatusUnprocessableEntity || !strings.Contains(rejected.Body.String(), `"code":"data_loss_ack_required"`) {
|
|
t.Fatalf("force-start without acknowledgement = %d %s", rejected.Code, rejected.Body.String())
|
|
}
|
|
select {
|
|
case call := <-api.hv.forceCalls:
|
|
t.Fatalf("rejected force-start reached backend: %#v", call)
|
|
default:
|
|
}
|
|
|
|
actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-force-rescue"}
|
|
accepted := api.request(t, http.MethodPost, target, []byte(`{"operation_id":"operation-force-rescue","ack_data_loss":true}`), actor)
|
|
if accepted.Code != http.StatusAccepted || !strings.Contains(accepted.Body.String(), `"active":true`) {
|
|
t.Fatalf("acknowledged force-start = %d %s", accepted.Code, accepted.Body.String())
|
|
}
|
|
select {
|
|
case call := <-api.hv.forceCalls:
|
|
if !call.AckDataLoss || call.OperationID != actor.OperationID {
|
|
t.Fatalf("force-start request = %#v", call)
|
|
}
|
|
default:
|
|
t.Fatal("acknowledged force-start did not reach backend")
|
|
}
|
|
}
|
|
|
|
func TestAsyncImportReplayDoesNotDispatchTwice(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
release := make(chan struct{})
|
|
api.hv.templateWait = release
|
|
actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-replay"}
|
|
body := []byte(`{"operation_id":"operation-template-replay","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
|
|
first := api.request(t, http.MethodPost, "/v1/templates", body, actor)
|
|
if first.Code != http.StatusAccepted {
|
|
close(release)
|
|
t.Fatalf("first import response = %d %s", first.Code, first.Body.String())
|
|
}
|
|
select {
|
|
case <-api.hv.templateCalls:
|
|
case <-time.After(time.Second):
|
|
close(release)
|
|
t.Fatal("first template import was not dispatched")
|
|
}
|
|
second := api.request(t, http.MethodPost, "/v1/templates", body, actor)
|
|
if second.Code != http.StatusAccepted || !strings.Contains(second.Body.String(), `"replayed":true`) {
|
|
close(release)
|
|
t.Fatalf("replayed import response = %d %s", second.Code, second.Body.String())
|
|
}
|
|
select {
|
|
case duplicate := <-api.hv.templateCalls:
|
|
close(release)
|
|
t.Fatalf("template import dispatched twice: %#v", duplicate)
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
close(release)
|
|
deadline := time.Now().Add(time.Second)
|
|
for {
|
|
record, err := api.handler.Journal.Get(actor.OperationID)
|
|
if err == nil && record.State == operation.StateSucceeded {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("async import did not finish after release: %#v err=%v", record, err)
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestExplicitReplayResumesReservedImportAfterRestartBoundary(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
actor := auth.Actor{UserID: 1, CanManage: true, IsAdmin: true, OperationID: "operation-template-resume"}
|
|
body := []byte(`{"operation_id":"operation-template-resume","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`)
|
|
_, _, err := api.handler.Journal.Reserve(operation.Spec{
|
|
OperationID: actor.OperationID,
|
|
ResourceID: "template:debian-12",
|
|
Action: "import-template",
|
|
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/templates", body),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := api.request(t, http.MethodPost, "/v1/templates", body, actor)
|
|
if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"replayed":true`) {
|
|
t.Fatalf("reserved replay response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
select {
|
|
case <-api.hv.templateCalls:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("reserved import was not resumed by explicit authenticated replay")
|
|
}
|
|
}
|