You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
460 lines
17 KiB
Go
460 lines
17 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
consolebroker "git.luoli.live/root/virt/agent/console"
|
|
"git.luoli.live/root/virt/agent/hypervisor"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"git.luoli.live/root/virt/agent/worker"
|
|
)
|
|
|
|
type testHypervisor struct {
|
|
hypervisor.Unsupported
|
|
mu sync.Mutex
|
|
inspections int
|
|
configureCalls int
|
|
}
|
|
|
|
func (h *testHypervisor) Ping(context.Context) error { return nil }
|
|
|
|
func (h *testHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) {
|
|
return []hypervisor.VMMetadata{}, nil
|
|
}
|
|
|
|
func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageConfigurationRequest) (hypervisor.StorageConfigurationResult, error) {
|
|
h.mu.Lock()
|
|
h.configureCalls++
|
|
h.mu.Unlock()
|
|
return hypervisor.StorageConfigurationResult{Configured: true}, nil
|
|
}
|
|
|
|
func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) {
|
|
h.mu.Lock()
|
|
h.inspections++
|
|
h.mu.Unlock()
|
|
return hypervisor.OperationInspection{
|
|
OperationID: record.OperationID,
|
|
CheckedAt: time.Now().Unix(),
|
|
Outcome: hypervisor.InspectionOutcomeFailed,
|
|
SummaryCode: hypervisor.InspectionSummaryFailed,
|
|
ErrorCode: "result_unconfirmed",
|
|
}, nil
|
|
}
|
|
|
|
type fakeMutationExecutor struct {
|
|
mu sync.Mutex
|
|
calls int
|
|
requests []worker.Request
|
|
result json.RawMessage
|
|
err error
|
|
}
|
|
|
|
type blockingMutationExecutor struct {
|
|
started chan worker.Request
|
|
release chan struct{}
|
|
calls atomic.Int32
|
|
}
|
|
|
|
func (e *blockingMutationExecutor) Execute(ctx context.Context, request worker.Request) (json.RawMessage, error) {
|
|
e.calls.Add(1)
|
|
select {
|
|
case e.started <- request:
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
select {
|
|
case <-e.release:
|
|
return json.RawMessage(`{"configured":true}`), nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (e *fakeMutationExecutor) Execute(_ context.Context, request worker.Request) (json.RawMessage, error) {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
e.calls++
|
|
e.requests = append(e.requests, request)
|
|
return append(json.RawMessage(nil), e.result...), e.err
|
|
}
|
|
|
|
type testAPI struct {
|
|
handler *Server
|
|
hv *testHypervisor
|
|
journal *operation.Journal
|
|
secret []byte
|
|
now time.Time
|
|
nonce int
|
|
}
|
|
|
|
func newTestAPI(t *testing.T) *testAPI {
|
|
t.Helper()
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secret := []byte("0123456789abcdef0123456789abcdef")
|
|
now := time.Unix(2_000_000_000, 0)
|
|
journal.SetClockForTesting(func() time.Time { return now })
|
|
authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
authenticator.SetClockForTesting(func() time.Time { return now })
|
|
broker, err := consolebroker.NewBroker(time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hv := &testHypervisor{}
|
|
handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
_ = handler.Shutdown(ctx)
|
|
_ = journal.Wait(ctx)
|
|
_ = journal.Close()
|
|
})
|
|
return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now}
|
|
}
|
|
|
|
func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder {
|
|
t.Helper()
|
|
a.nonce++
|
|
request := httptest.NewRequest(method, target, bytes.NewReader(body))
|
|
nonce := fmt.Sprintf("nonce%011d", a.nonce)
|
|
if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := httptest.NewRecorder()
|
|
a.handler.ServeHTTP(response, request)
|
|
return response
|
|
}
|
|
|
|
func TestOperationRecoveryMutationRoutesAreRemoved(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-terminal-inspect"
|
|
spec := operation.Spec{
|
|
OperationID: operationID,
|
|
ResourceID: "vm-inspect",
|
|
Action: "delete-vm",
|
|
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-inspect", []byte(`{}`)),
|
|
Destructive: true,
|
|
}
|
|
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return nil, &worker.Error{
|
|
Code: "result_unconfirmed", Message: "result was not confirmed",
|
|
EffectMayHaveApplied: true,
|
|
}
|
|
}); err == nil {
|
|
t.Fatal("seed failure unexpectedly succeeded")
|
|
}
|
|
actor := auth.Actor{
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
OperationID: operationID,
|
|
}
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID, nil, actor); response.Code != http.StatusOK {
|
|
t.Fatalf("operation GET = %d %s", response.Code, response.Body.String())
|
|
}
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK {
|
|
t.Fatalf("read-only inspect = %d %s", response.Code, response.Body.String())
|
|
}
|
|
for _, suffix := range []string{"reconcile", "confirm-inspection", "abort-job", "cleanup"} {
|
|
response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/"+suffix, []byte(`{}`), actor)
|
|
if response.Code != http.StatusNotFound {
|
|
t.Fatalf("removed %s route = %d %s", suffix, response.Code, response.Body.String())
|
|
}
|
|
}
|
|
api.hv.mu.Lock()
|
|
inspections := api.hv.inspections
|
|
api.hv.mu.Unlock()
|
|
if inspections != 1 {
|
|
t.Fatalf("removed mutation routes reached hypervisor; inspections=%d", inspections)
|
|
}
|
|
}
|
|
|
|
func TestSuperAdminFenceResolutionIsJournalOnlyAndReadBackIsStable(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
operationID := "operation-fence-resolution"
|
|
spec := operation.Spec{
|
|
OperationID: operationID,
|
|
ResourceID: "vm-fence-resolution",
|
|
Action: "delete-vm",
|
|
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-resolution", []byte(`{}`)),
|
|
Destructive: true,
|
|
}
|
|
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return nil, &worker.Error{
|
|
Code: "result_unconfirmed", Message: "result was not confirmed",
|
|
EffectMayHaveApplied: true, ResubmitAllowed: false,
|
|
}
|
|
}); err == nil {
|
|
t.Fatal("seed effectful failure unexpectedly succeeded")
|
|
}
|
|
actor := auth.Actor{
|
|
UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
OperationID: operationID,
|
|
}
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound ||
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) {
|
|
t.Fatalf("missing fence resolution = %d %s", response.Code, response.Body.String())
|
|
}
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK {
|
|
t.Fatalf("fresh inspection = %d %s", response.Code, response.Body.String())
|
|
}
|
|
request := operation.FenceResolutionRequest{
|
|
ResolutionID: "0123456789abcdef0123456789abcdef",
|
|
Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(),
|
|
ReasonSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
}
|
|
body, err := json.Marshal(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
|
|
if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"replayed":false`)) {
|
|
t.Fatalf("resolve fence = %d %s", resolved.Code, resolved.Body.String())
|
|
}
|
|
readBack := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor)
|
|
if readBack.Code != http.StatusOK ||
|
|
!bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution_id":"0123456789abcdef0123456789abcdef"`)) ||
|
|
!bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution":"not_applied"`)) {
|
|
t.Fatalf("read fence resolution = %d %s", readBack.Code, readBack.Body.String())
|
|
}
|
|
replayed := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
|
|
if replayed.Code != http.StatusOK || !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) {
|
|
t.Fatalf("idempotent fence resolution = %d %s", replayed.Code, replayed.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
inspections := api.hv.inspections
|
|
configureCalls := api.hv.configureCalls
|
|
api.hv.mu.Unlock()
|
|
if inspections != 1 || configureCalls != 0 {
|
|
t.Fatalf("fence resolution invoked host work: inspections=%d mutations=%d", inspections, configureCalls)
|
|
}
|
|
}
|
|
|
|
func TestFenceResolutionRequiresExactActorOperationIdentity(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
request := operation.FenceResolutionRequest{
|
|
ResolutionID: "fedcba9876543210fedcba9876543210",
|
|
Resolution: operation.FenceResolutionApplied, InspectionCheckedAt: api.now.Unix(),
|
|
ReasonSHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
}
|
|
body, err := json.Marshal(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := api.request(t, http.MethodPost, "/v1/operations/operation-fence-identity/resolve-fence", body, auth.Actor{
|
|
UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
OperationID: "operation-other-identity",
|
|
})
|
|
if response.Code != http.StatusBadRequest ||
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"operation_id_mismatch"`)) {
|
|
t.Fatalf("identity mismatch = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
executor := &fakeMutationExecutor{result: json.RawMessage(`{"configured":true}`)}
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request := hypervisor.StorageConfigurationRequest{
|
|
OperationID: "operation-isolated-storage",
|
|
ManagedDirectory: "/srv/virt/managed",
|
|
TemplateDirectory: "/srv/virt/templates",
|
|
SeedDirectory: "/srv/virt/seeds",
|
|
UserISODirectory: "/srv/virt/user-isos",
|
|
RescueDirectory: "/srv/virt/rescue-isos",
|
|
}
|
|
body, err := json.Marshal(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
actor := auth.Actor{
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
OperationID: request.OperationID,
|
|
}
|
|
first := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
|
|
if first.Code != http.StatusAccepted {
|
|
t.Fatalf("first isolated response = %d %s", first.Code, first.Body.String())
|
|
}
|
|
terminal := waitForTerminal(t, api.journal, request.OperationID)
|
|
if terminal.State != operation.StateSucceeded || string(terminal.Result) != `{"configured":true}` {
|
|
t.Fatalf("terminal operation = %#v", terminal)
|
|
}
|
|
replay := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
|
|
if replay.Code != http.StatusOK {
|
|
t.Fatalf("terminal replay = %d %s", replay.Code, replay.Body.String())
|
|
}
|
|
executor.mu.Lock()
|
|
defer executor.mu.Unlock()
|
|
if executor.calls != 1 || len(executor.requests) != 1 {
|
|
t.Fatalf("executor calls=%d requests=%d", executor.calls, len(executor.requests))
|
|
}
|
|
workerRequest := executor.requests[0]
|
|
if workerRequest.OperationID != request.OperationID || workerRequest.Actor.OperationID != request.OperationID ||
|
|
workerRequest.Method != http.MethodPost || workerRequest.RequestTarget != "/v1/storage/configuration" {
|
|
t.Fatalf("worker envelope = %#v", workerRequest)
|
|
}
|
|
}
|
|
|
|
func TestIsolatedMutationAdmissionIsShortAndDoesNotStartSecondWorker(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
api.handler.AdmissionTimeout = 30 * time.Millisecond
|
|
executor := &blockingMutationExecutor{
|
|
started: make(chan worker.Request, 1), release: make(chan struct{}),
|
|
}
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
requestBody := func(operationID string) []byte {
|
|
body, err := json.Marshal(hypervisor.StorageConfigurationRequest{
|
|
OperationID: operationID, ManagedDirectory: "/srv/virt/managed",
|
|
TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds",
|
|
UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return body
|
|
}
|
|
firstID := "operation-admission-first"
|
|
first := api.request(t, http.MethodPost, "/v1/storage/configuration", requestBody(firstID), auth.Actor{
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: firstID,
|
|
})
|
|
if first.Code != http.StatusAccepted {
|
|
t.Fatalf("first response = %d %s", first.Code, first.Body.String())
|
|
}
|
|
select {
|
|
case <-executor.started:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("first worker did not start")
|
|
}
|
|
|
|
secondID := "operation-admission-second"
|
|
startedAt := time.Now()
|
|
second := api.request(t, http.MethodPost, "/v1/storage/configuration", requestBody(secondID), auth.Actor{
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: secondID,
|
|
})
|
|
if elapsed := time.Since(startedAt); elapsed > 500*time.Millisecond {
|
|
t.Fatalf("saturated admission waited %s", elapsed)
|
|
}
|
|
if second.Code != http.StatusOK ||
|
|
!bytes.Contains(second.Body.Bytes(), []byte(`"status":"failed"`)) ||
|
|
!bytes.Contains(second.Body.Bytes(), []byte(`"code":"operation_not_dispatched"`)) {
|
|
t.Fatalf("saturated admission response = %d %s", second.Code, second.Body.String())
|
|
}
|
|
if executor.calls.Load() != 1 {
|
|
t.Fatalf("saturated request started a second worker; calls=%d", executor.calls.Load())
|
|
}
|
|
close(executor.release)
|
|
if terminal := waitForTerminal(t, api.journal, firstID); terminal.State != operation.StateSucceeded {
|
|
t.Fatalf("first terminal = %#v", terminal)
|
|
}
|
|
}
|
|
|
|
func TestParentMutationNeverFallsBackWhenExecutorIsMissing(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
request := hypervisor.StorageConfigurationRequest{
|
|
OperationID: "operation-worker-missing",
|
|
ManagedDirectory: "/srv/virt/managed",
|
|
TemplateDirectory: "/srv/virt/templates",
|
|
SeedDirectory: "/srv/virt/seeds",
|
|
UserISODirectory: "/srv/virt/user-isos",
|
|
RescueDirectory: "/srv/virt/rescue-isos",
|
|
}
|
|
body, err := json.Marshal(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
response := api.request(t, http.MethodPost, "/v1/storage/configuration", body, auth.Actor{
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
OperationID: request.OperationID,
|
|
})
|
|
if response.Code != http.StatusServiceUnavailable ||
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"mutation_worker_unavailable"`)) ||
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"dispatch_state":"not_dispatched"`)) {
|
|
t.Fatalf("missing worker response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
if _, err := api.journal.Get(request.OperationID); !errors.Is(err, operation.ErrNotFound) {
|
|
t.Fatalf("missing worker created journal state: %v", err)
|
|
}
|
|
api.hv.mu.Lock()
|
|
configureCalls := api.hv.configureCalls
|
|
api.hv.mu.Unlock()
|
|
if configureCalls != 0 {
|
|
t.Fatalf("parent invoked the in-process hypervisor %d times", configureCalls)
|
|
}
|
|
}
|
|
|
|
func TestTrustedWorkerExecutesLocallyWithoutRecursiveSpawn(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
request := hypervisor.StorageConfigurationRequest{
|
|
OperationID: "operation-trusted-worker",
|
|
ManagedDirectory: "/srv/virt/managed",
|
|
TemplateDirectory: "/srv/virt/templates",
|
|
SeedDirectory: "/srv/virt/seeds",
|
|
UserISODirectory: "/srv/virt/user-isos",
|
|
RescueDirectory: "/srv/virt/rescue-isos",
|
|
}
|
|
body, err := json.Marshal(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
httpRequest := httptest.NewRequest(http.MethodPost, "/v1/storage/configuration", bytes.NewReader(body))
|
|
response := httptest.NewRecorder()
|
|
api.handler.ServeTrustedWorkerRequest(response, httpRequest, body, auth.Actor{
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
OperationID: request.OperationID,
|
|
})
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("trusted worker response = %d %s", response.Code, response.Body.String())
|
|
}
|
|
api.hv.mu.Lock()
|
|
configureCalls := api.hv.configureCalls
|
|
api.hv.mu.Unlock()
|
|
if configureCalls != 1 {
|
|
t.Fatalf("trusted worker local mutation calls=%d", configureCalls)
|
|
}
|
|
}
|
|
|
|
func TestListEnvelopeRemainsNonNull(t *testing.T) {
|
|
api := newTestAPI(t)
|
|
response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1})
|
|
if response.Code != http.StatusOK || response.Body.String() != "{\"vms\":[]}\n" {
|
|
t.Fatalf("VM list = %d %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func waitForTerminal(t *testing.T, journal *operation.Journal, operationID string) operation.Record {
|
|
t.Helper()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
record, err := journal.Get(operationID)
|
|
if err == nil && (record.State == operation.StateSucceeded || record.State == operation.StateFailed) {
|
|
return record
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
t.Fatalf("operation %s did not become terminal", operationID)
|
|
return operation.Record{}
|
|
}
|