|
|
|
|
package api
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"bytes"
|
|
|
|
|
"context"
|
|
|
|
|
"crypto/sha256"
|
|
|
|
|
"encoding/hex"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"errors"
|
|
|
|
|
"fmt"
|
|
|
|
|
"net/http"
|
|
|
|
|
"net/http/httptest"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"reflect"
|
|
|
|
|
"slices"
|
|
|
|
|
"strings"
|
|
|
|
|
"sync"
|
|
|
|
|
"sync/atomic"
|
|
|
|
|
"testing"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
|
|
|
consolebroker "git.luoli.live/root/virt/agent/console"
|
|
|
|
|
"git.luoli.live/root/virt/agent/hypervisor"
|
|
|
|
|
"git.luoli.live/root/virt/agent/objecttransfer"
|
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
|
|
|
"git.luoli.live/root/virt/agent/publicnetwork"
|
|
|
|
|
"git.luoli.live/root/virt/agent/vpcrouting"
|
|
|
|
|
"git.luoli.live/root/virt/agent/worker"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
type testHypervisor struct {
|
|
|
|
|
hypervisor.Unsupported
|
|
|
|
|
mu sync.Mutex
|
|
|
|
|
inspections int
|
|
|
|
|
configureCalls int
|
|
|
|
|
cleanupPeerTargetCalls int
|
|
|
|
|
cleanupPeerTargetLast hypervisor.CleanupPeerMigrationTargetRequest
|
|
|
|
|
cleanupBackupCalls int
|
|
|
|
|
cleanupBackupLast hypervisor.CleanupBackupWorkspaceRequest
|
|
|
|
|
inspectionResult *hypervisor.OperationInspection
|
|
|
|
|
inspectionErr error
|
|
|
|
|
backupReadiness hypervisor.BackupReadiness
|
|
|
|
|
backupReadinessErr error
|
|
|
|
|
metrics hypervisor.Metrics
|
|
|
|
|
metricsErr error
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type apiFailure struct {
|
|
|
|
|
code string
|
|
|
|
|
effect bool
|
|
|
|
|
safe bool
|
|
|
|
|
fence bool
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e apiFailure) Error() string { return e.code }
|
|
|
|
|
func (e apiFailure) PublicCode() string { return e.code }
|
|
|
|
|
func (e apiFailure) PublicMessage() string { return e.code }
|
|
|
|
|
func (e apiFailure) FailureSemantics() (bool, bool) {
|
|
|
|
|
return e.effect, e.safe
|
|
|
|
|
}
|
|
|
|
|
func (e apiFailure) RequiresExecutionFence() bool { return e.fence }
|
|
|
|
|
|
|
|
|
|
func (h *testHypervisor) Ping(context.Context) error { return nil }
|
|
|
|
|
|
|
|
|
|
func (h *testHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) {
|
|
|
|
|
return []hypervisor.VMMetadata{}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (h *testHypervisor) GetBackupReadiness(context.Context, string) (hypervisor.BackupReadiness, error) {
|
|
|
|
|
return h.backupReadiness, h.backupReadinessErr
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (h *testHypervisor) GetMetrics(context.Context, string) (hypervisor.Metrics, error) {
|
|
|
|
|
return h.metrics, h.metricsErr
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageConfigurationRequest) (hypervisor.StorageConfigurationResult, error) {
|
|
|
|
|
h.mu.Lock()
|
|
|
|
|
h.configureCalls++
|
|
|
|
|
h.mu.Unlock()
|
|
|
|
|
return hypervisor.StorageConfigurationResult{Configured: true}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (h *testHypervisor) CleanupPeerMigrationTarget(_ context.Context, _ string,
|
|
|
|
|
request hypervisor.CleanupPeerMigrationTargetRequest) (hypervisor.DeleteVMResult, error) {
|
|
|
|
|
h.mu.Lock()
|
|
|
|
|
h.cleanupPeerTargetCalls++
|
|
|
|
|
h.cleanupPeerTargetLast = request
|
|
|
|
|
h.mu.Unlock()
|
|
|
|
|
return hypervisor.DeleteVMResult{
|
|
|
|
|
Deleted: true, ManagedArtifactsAbsent: true, Forced: false,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (h *testHypervisor) CleanupBackupWorkspace(_ context.Context, _ string,
|
|
|
|
|
request hypervisor.CleanupBackupWorkspaceRequest) (hypervisor.CleanupBackupWorkspaceResult, error) {
|
|
|
|
|
h.mu.Lock()
|
|
|
|
|
h.cleanupBackupCalls++
|
|
|
|
|
h.cleanupBackupLast = request
|
|
|
|
|
h.mu.Unlock()
|
|
|
|
|
parts := strings.Split(request.PriorOperation.Action, ":")
|
|
|
|
|
key := ""
|
|
|
|
|
if len(parts) == 3 {
|
|
|
|
|
key = parts[1]
|
|
|
|
|
}
|
|
|
|
|
return hypervisor.CleanupBackupWorkspaceResult{
|
|
|
|
|
BackupKey: key, Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) {
|
|
|
|
|
h.mu.Lock()
|
|
|
|
|
defer h.mu.Unlock()
|
|
|
|
|
h.inspections++
|
|
|
|
|
if h.inspectionErr != nil {
|
|
|
|
|
return hypervisor.OperationInspection{}, h.inspectionErr
|
|
|
|
|
}
|
|
|
|
|
if h.inspectionResult != nil {
|
|
|
|
|
inspection := *h.inspectionResult
|
|
|
|
|
inspection.EffectProjection = append(json.RawMessage(nil), inspection.EffectProjection...)
|
|
|
|
|
return inspection, nil
|
|
|
|
|
}
|
|
|
|
|
return hypervisor.OperationInspection{
|
|
|
|
|
OperationID: record.OperationID,
|
|
|
|
|
OperationStatus: record.State,
|
|
|
|
|
CheckedAt: time.Now().Unix(),
|
|
|
|
|
Outcome: hypervisor.InspectionOutcomeFailed,
|
|
|
|
|
SummaryCode: hypervisor.InspectionSummaryFailed,
|
|
|
|
|
ErrorCode: "result_unconfirmed",
|
|
|
|
|
EffectResolution: hypervisor.InspectionEffectInconclusive,
|
|
|
|
|
FenceResolutionRequired: record.State == operation.StateFailed && record.EffectMayHaveApplied,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type fakeMutationExecutor struct {
|
|
|
|
|
mu sync.Mutex
|
|
|
|
|
calls int
|
|
|
|
|
requests []worker.Request
|
|
|
|
|
result json.RawMessage
|
|
|
|
|
err error
|
|
|
|
|
deadline time.Time
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type blockingMutationExecutor struct {
|
|
|
|
|
started chan worker.Request
|
|
|
|
|
release chan struct{}
|
|
|
|
|
calls atomic.Int32
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e *blockingMutationExecutor) Execute(ctx context.Context, request worker.Request) (json.RawMessage, error) {
|
|
|
|
|
e.calls.Add(1)
|
|
|
|
|
select {
|
|
|
|
|
case e.started <- request:
|
|
|
|
|
case <-ctx.Done():
|
|
|
|
|
return nil, ctx.Err()
|
|
|
|
|
}
|
|
|
|
|
select {
|
|
|
|
|
case <-e.release:
|
|
|
|
|
return json.RawMessage(`{"configured":true}`), nil
|
|
|
|
|
case <-ctx.Done():
|
|
|
|
|
return nil, ctx.Err()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e *fakeMutationExecutor) Execute(ctx context.Context, request worker.Request) (json.RawMessage, error) {
|
|
|
|
|
e.mu.Lock()
|
|
|
|
|
defer e.mu.Unlock()
|
|
|
|
|
e.calls++
|
|
|
|
|
e.requests = append(e.requests, request)
|
|
|
|
|
if deadline, ok := ctx.Deadline(); ok {
|
|
|
|
|
e.deadline = deadline
|
|
|
|
|
}
|
|
|
|
|
return append(json.RawMessage(nil), e.result...), e.err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type testAPI struct {
|
|
|
|
|
handler *Server
|
|
|
|
|
hv *testHypervisor
|
|
|
|
|
journal *operation.Journal
|
|
|
|
|
secret []byte
|
|
|
|
|
now time.Time
|
|
|
|
|
nonce int
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func newTestAPI(t *testing.T) *testAPI {
|
|
|
|
|
t.Helper()
|
|
|
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
secret := []byte("0123456789abcdef0123456789abcdef")
|
|
|
|
|
now := time.Unix(2_000_000_000, 0)
|
|
|
|
|
journal.SetClockForTesting(func() time.Time { return now })
|
|
|
|
|
authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
authenticator.SetClockForTesting(func() time.Time { return now })
|
|
|
|
|
broker, err := consolebroker.NewBroker(time.Minute)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
hv := &testHypervisor{}
|
|
|
|
|
handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
t.Cleanup(func() {
|
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
|
|
|
defer cancel()
|
|
|
|
|
_ = handler.Shutdown(ctx)
|
|
|
|
|
_ = journal.Wait(ctx)
|
|
|
|
|
_ = journal.Close()
|
|
|
|
|
})
|
|
|
|
|
return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder {
|
|
|
|
|
t.Helper()
|
|
|
|
|
a.nonce++
|
|
|
|
|
request := httptest.NewRequest(method, target, bytes.NewReader(body))
|
|
|
|
|
nonce := fmt.Sprintf("nonce%011d", a.nonce)
|
|
|
|
|
if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
response := httptest.NewRecorder()
|
|
|
|
|
a.handler.ServeHTTP(response, request)
|
|
|
|
|
return response
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestLegacyDailyTrafficRouteIsRemoved(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
response := api.request(t, http.MethodGet,
|
|
|
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/traffic/daily?days=30",
|
|
|
|
|
nil, auth.Actor{UserID: 1, CanManage: true})
|
|
|
|
|
if response.Code != http.StatusNotFound {
|
|
|
|
|
t.Fatalf("legacy daily traffic status=%d body=%s",
|
|
|
|
|
response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestMetricsRoutePublishesPublicInterfaceHealthCounters(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
rxPackets := uint64(101)
|
|
|
|
|
txPackets := uint64(202)
|
|
|
|
|
rxErrors := uint64(3)
|
|
|
|
|
txErrors := uint64(4)
|
|
|
|
|
rxDrops := uint64(5)
|
|
|
|
|
txDrops := uint64(6)
|
|
|
|
|
api.hv.metrics = hypervisor.Metrics{
|
|
|
|
|
UUID: "123e4567-e89b-42d3-a456-426614174000",
|
|
|
|
|
RXBytes: 1000, TXBytes: 2000,
|
|
|
|
|
RXPackets: &rxPackets, TXPackets: &txPackets,
|
|
|
|
|
RXErrors: &rxErrors, TXErrors: &txErrors,
|
|
|
|
|
RXDrops: &rxDrops, TXDrops: &txDrops,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodGet,
|
|
|
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/metrics",
|
|
|
|
|
nil, auth.Actor{UserID: 1, CanManage: true})
|
|
|
|
|
if response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
var payload map[string]any
|
|
|
|
|
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
for field, expected := range map[string]float64{
|
|
|
|
|
"rx_packets": 101, "tx_packets": 202,
|
|
|
|
|
"rx_errors": 3, "tx_errors": 4,
|
|
|
|
|
"rx_drops": 5, "tx_drops": 6,
|
|
|
|
|
} {
|
|
|
|
|
if payload[field] != expected {
|
|
|
|
|
t.Fatalf("%s = %#v", field, payload[field])
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if payload["rx_bytes"] != float64(1000) || payload["tx_bytes"] != float64(2000) {
|
|
|
|
|
t.Fatalf("durable byte counters changed shape: %#v", payload)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestBackupReadinessIsBoundedReadOnlyEvidence(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
api.hv.backupReadiness = hypervisor.BackupReadiness{
|
|
|
|
|
Eligible: false, Retryable: true, Reason: "backup_rescue_active",
|
|
|
|
|
}
|
|
|
|
|
const uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
response := api.request(t, http.MethodGet,
|
|
|
|
|
"/v1/vms/"+uuid+"/backup-readiness", nil,
|
|
|
|
|
auth.Actor{UserID: 1, CanManage: true})
|
|
|
|
|
if response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("backup readiness status=%d body=%s",
|
|
|
|
|
response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
var readiness hypervisor.BackupReadiness
|
|
|
|
|
if err := json.Unmarshal(response.Body.Bytes(), &readiness); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if readiness != api.hv.backupReadiness {
|
|
|
|
|
t.Fatalf("backup readiness=%#v want=%#v",
|
|
|
|
|
readiness, api.hv.backupReadiness)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestBackupReadinessRequiresManageCapability(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
response := api.request(t, http.MethodGet,
|
|
|
|
|
"/v1/vms/"+uuid+"/backup-readiness", nil,
|
|
|
|
|
auth.Actor{UserID: 1})
|
|
|
|
|
if response.Code != http.StatusForbidden {
|
|
|
|
|
t.Fatalf("backup readiness without manage=%d body=%s",
|
|
|
|
|
response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func requireOperationResponseIdentity(t *testing.T, response *httptest.ResponseRecorder,
|
|
|
|
|
operationID, action, resourceID, requestFingerprint string) operationResponse {
|
|
|
|
|
t.Helper()
|
|
|
|
|
var wire operationResponse
|
|
|
|
|
if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil {
|
|
|
|
|
t.Fatalf("decode operation response %q: %v body=%s", operationID, err, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
if wire.OperationID != operationID || wire.Action != action || wire.ResourceID != resourceID ||
|
|
|
|
|
wire.RequestFingerprint != requestFingerprint {
|
|
|
|
|
t.Fatalf("operation identity = %#v, want operation=%q action=%q resource=%q fingerprint=%q",
|
|
|
|
|
wire, operationID, action, resourceID, requestFingerprint)
|
|
|
|
|
}
|
|
|
|
|
var raw map[string]json.RawMessage
|
|
|
|
|
if err := json.Unmarshal(response.Body.Bytes(), &raw); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if _, legacy := raw["fingerprint"]; legacy {
|
|
|
|
|
t.Fatalf("legacy fingerprint key leaked into operation wire: %s", response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
return wire
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestVPCNetworkMutationUsesCanonicalDigestAndDurableExpectedState(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
vpcKey = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
operationID = "operation-vpc-network-apply"
|
|
|
|
|
)
|
|
|
|
|
network := vpcrouting.Network{
|
|
|
|
|
VPCKey: vpcKey, DesiredRevision: 4,
|
|
|
|
|
CIDR: "10.88.0.0/24", Gateway: "10.88.0.1",
|
|
|
|
|
Transport: vpcrouting.Transport{
|
|
|
|
|
AddressCIDR: "172.16.50.11/24", Interface: "br-private", MTU: 1500,
|
|
|
|
|
},
|
|
|
|
|
LocalPorts: []vpcrouting.LocalPort{{IP: "10.88.0.2", MAC: "02:00:00:00:00:02"}},
|
|
|
|
|
PeerRoutes: []vpcrouting.PeerRoute{{IP: "10.88.0.3", Via: "172.16.50.12"}},
|
|
|
|
|
}
|
|
|
|
|
digest, err := vpcrouting.Digest(network)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
request := vpcrouting.ApplyRequest{OperationID: operationID, DesiredDigest: digest, Network: network}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
bridge, _ := vpcrouting.BridgeName(vpcKey)
|
|
|
|
|
result, _ := json.Marshal(vpcrouting.Result{
|
|
|
|
|
VPCKey: vpcKey, DesiredRevision: 4, DesiredDigest: digest,
|
|
|
|
|
Bridge: bridge, Applied: true,
|
|
|
|
|
})
|
|
|
|
|
executor := &fakeMutationExecutor{result: result}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost, "/v1/vpc/networks/"+vpcKey+"/apply", body, actor)
|
|
|
|
|
if response.Code != http.StatusAccepted && response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("VPC apply status=%d body=%s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
record := waitForTerminal(t, api.journal, operationID)
|
|
|
|
|
if record.State != operation.StateSucceeded || record.Action != "apply-vpc-network:"+vpcKey ||
|
|
|
|
|
record.ResourceID != "vpc:"+vpcKey || record.ExpectedState == nil ||
|
|
|
|
|
record.ExpectedState.Kind != operation.ExpectedVPCNetwork ||
|
|
|
|
|
record.ExpectedState.VPCKey != vpcKey || record.ExpectedState.VPCRevision != 4 ||
|
|
|
|
|
record.ExpectedState.VPCDigest != digest || !record.ExpectedState.VPCPresent {
|
|
|
|
|
t.Fatalf("VPC operation record = %#v", record)
|
|
|
|
|
}
|
|
|
|
|
if len(record.LockKeys) != 2 || !slices.Contains(record.LockKeys, "catalog:vpc-host/state") ||
|
|
|
|
|
!slices.Contains(record.LockKeys, "catalog:vpc/"+vpcKey) {
|
|
|
|
|
t.Fatalf("VPC operation locks = %#v", record.LockKeys)
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Lock()
|
|
|
|
|
defer executor.mu.Unlock()
|
|
|
|
|
if executor.calls != 1 || executor.requests[0].RequestTarget != "/v1/vpc/networks/"+vpcKey+"/apply" {
|
|
|
|
|
t.Fatalf("VPC worker requests = %#v", executor.requests)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestVPCNetworkRepairCarriesExactSupersededGeneration(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
vpcKey = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
priorID = "operation-vpc-repair-prior"
|
|
|
|
|
repairID = "operation-vpc-repair-next"
|
|
|
|
|
)
|
|
|
|
|
network := vpcrouting.Network{
|
|
|
|
|
VPCKey: vpcKey, DesiredRevision: 9,
|
|
|
|
|
CIDR: "10.89.0.0/24", Gateway: "10.89.0.1",
|
|
|
|
|
Transport: vpcrouting.Transport{
|
|
|
|
|
AddressCIDR: "172.16.50.11/24", Interface: "br-private", MTU: 1500,
|
|
|
|
|
},
|
|
|
|
|
LocalPorts: []vpcrouting.LocalPort{{IP: "10.89.0.2", MAC: "02:00:00:00:00:12"}},
|
|
|
|
|
}
|
|
|
|
|
digest, err := vpcrouting.Digest(network)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
prior := operation.Spec{
|
|
|
|
|
OperationID: priorID, ResourceID: "vpc:" + vpcKey,
|
|
|
|
|
Action: "apply-vpc-network:" + vpcKey,
|
|
|
|
|
Fingerprint: operation.Fingerprint(
|
|
|
|
|
http.MethodPost, "/v1/vpc/networks/"+vpcKey+"/apply", []byte(`{}`),
|
|
|
|
|
),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
ExpectedState: &operation.ExpectedState{
|
|
|
|
|
Kind: operation.ExpectedVPCNetwork, TransitionID: priorID,
|
|
|
|
|
VPCKey: vpcKey, VPCRevision: 9,
|
|
|
|
|
VPCDigest: digest, VPCPresent: true,
|
|
|
|
|
},
|
|
|
|
|
LockKeys: hypervisor.VPCNetworkAdmission(vpcKey).LockKeys,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(
|
|
|
|
|
context.Background(), prior,
|
|
|
|
|
func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, apiFailure{
|
|
|
|
|
code: "vpc_reconcile_unconfirmed", effect: true,
|
|
|
|
|
safe: false, fence: true,
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
); err == nil {
|
|
|
|
|
t.Fatal("VPC repair predecessor unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
request := vpcrouting.ApplyRequest{
|
|
|
|
|
OperationID: repairID, DesiredDigest: digest, Network: network,
|
|
|
|
|
VPCRepair: true, SupersedesOperationID: priorID,
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
bridge, _ := vpcrouting.BridgeName(vpcKey)
|
|
|
|
|
result, _ := json.Marshal(vpcrouting.Result{
|
|
|
|
|
VPCKey: vpcKey, DesiredRevision: 9, DesiredDigest: digest,
|
|
|
|
|
Bridge: bridge, Applied: true,
|
|
|
|
|
})
|
|
|
|
|
executor := &fakeMutationExecutor{result: result}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost,
|
|
|
|
|
"/v1/vpc/networks/"+vpcKey+"/apply/repair", body,
|
|
|
|
|
auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: repairID,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
if response.Code != http.StatusAccepted && response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("VPC repair status=%d body=%s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
record := waitForTerminal(t, api.journal, repairID)
|
|
|
|
|
if record.State != operation.StateSucceeded ||
|
|
|
|
|
record.SupersedesOperationID != priorID {
|
|
|
|
|
t.Fatalf("VPC repair record = %#v", record)
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Lock()
|
|
|
|
|
defer executor.mu.Unlock()
|
|
|
|
|
if executor.calls != 1 || len(executor.requests) != 1 ||
|
|
|
|
|
executor.requests[0].SupersededOperation == nil ||
|
|
|
|
|
executor.requests[0].SupersededOperation.OperationID != priorID ||
|
|
|
|
|
executor.requests[0].SupersededExpectedState == nil ||
|
|
|
|
|
executor.requests[0].SupersededExpectedState.VPCDigest != digest {
|
|
|
|
|
t.Fatalf("VPC repair worker envelope = %#v", executor.requests)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestVPCPortMutationBindsGuestAndHostProjectionIdentity(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
vpcKey = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
portKey = "1123456789abcdef0123456789abcdef"
|
|
|
|
|
operationID = "operation-vpc-port-apply"
|
|
|
|
|
digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
|
|
|
|
)
|
|
|
|
|
request := hypervisor.VPCPortRequest{
|
|
|
|
|
OperationID: operationID, PortKey: portKey, VPCKey: vpcKey,
|
|
|
|
|
PrivateIP: "10.60.0.9", CIDR: "10.60.0.0/24", Gateway: "10.60.0.1",
|
|
|
|
|
MAC: "02:00:00:00:00:09", HostRevision: 7, HostDigest: digest,
|
|
|
|
|
ConfirmRestart: true,
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
bridge, err := vpcrouting.BridgeName(vpcKey)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
result, _ := json.Marshal(hypervisor.VPCPortResult{
|
|
|
|
|
UUID: uuid, PortKey: portKey, VPCKey: vpcKey,
|
|
|
|
|
Status: hypervisor.VMStateRunning, Attached: true,
|
|
|
|
|
})
|
|
|
|
|
executor := &fakeMutationExecutor{result: result}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/vpc-port", body, actor)
|
|
|
|
|
if response.Code != http.StatusAccepted && response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("VPC port apply status=%d body=%s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
record := waitForTerminal(t, api.journal, operationID)
|
|
|
|
|
if record.State != operation.StateSucceeded || record.Action != "apply-vpc-port" ||
|
|
|
|
|
record.ResourceID != uuid || record.ExpectedState == nil ||
|
|
|
|
|
record.ExpectedState.Kind != operation.ExpectedVPCPort ||
|
|
|
|
|
record.ExpectedState.VPCPortKey != portKey || record.ExpectedState.VPCKey != vpcKey ||
|
|
|
|
|
record.ExpectedState.VPCPrivateIP != request.PrivateIP ||
|
|
|
|
|
record.ExpectedState.VPCCIDR != request.CIDR ||
|
|
|
|
|
record.ExpectedState.VPCGateway != request.Gateway ||
|
|
|
|
|
record.ExpectedState.VPCMAC != request.MAC ||
|
|
|
|
|
record.ExpectedState.VPCBridge != bridge ||
|
|
|
|
|
record.ExpectedState.VPCRevision != request.HostRevision ||
|
|
|
|
|
record.ExpectedState.VPCDigest != digest || !record.ExpectedState.VPCPresent {
|
|
|
|
|
t.Fatalf("VPC port operation record = %#v", record)
|
|
|
|
|
}
|
|
|
|
|
for _, required := range []string{
|
|
|
|
|
"catalog:vpc-port/" + portKey, "domain:" + uuid,
|
|
|
|
|
} {
|
|
|
|
|
if !slices.Contains(record.LockKeys, required) {
|
|
|
|
|
t.Fatalf("VPC port locks missing %q: %#v", required, record.LockKeys)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
for _, global := range []string{
|
|
|
|
|
"catalog:vpc-host/state", "catalog:vpc/" + vpcKey,
|
|
|
|
|
} {
|
|
|
|
|
if slices.Contains(record.LockKeys, global) {
|
|
|
|
|
t.Fatalf("VPC port operation retained global lock %q: %#v",
|
|
|
|
|
global, record.LockKeys)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestExitRescueRequiresDataLossAcknowledgement(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
operationID := "exit-rescue-without-ack"
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
}
|
|
|
|
|
body := []byte(`{"operation_id":"exit-rescue-without-ack","network_suspended":false}`)
|
|
|
|
|
response := api.request(t, http.MethodPost,
|
|
|
|
|
"/v1/vms/123e4567-e89b-42d3-a456-426614174000/rescue/exit", body, actor)
|
|
|
|
|
if response.Code != http.StatusUnprocessableEntity ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"data_loss_ack_required"`)) {
|
|
|
|
|
t.Fatalf("exit rescue without acknowledgement = %d %s",
|
|
|
|
|
response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestOperationRecoveryMutationRoutesAreRemoved(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
operationID := "operation-terminal-inspect"
|
|
|
|
|
spec := operation.Spec{
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
ResourceID: "vm-inspect",
|
|
|
|
|
Action: "delete-vm",
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-inspect", []byte(`{}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, &worker.Error{
|
|
|
|
|
Code: "result_unconfirmed", Message: "result was not confirmed",
|
|
|
|
|
EffectMayHaveApplied: true,
|
|
|
|
|
}
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("seed failure unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID, nil, actor); response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("operation GET = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
inspectionResponse := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
|
|
|
|
|
if inspectionResponse.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("read-only inspect = %d %s", inspectionResponse.Code, inspectionResponse.Body.String())
|
|
|
|
|
}
|
|
|
|
|
var inspection hypervisor.OperationInspection
|
|
|
|
|
if err := json.Unmarshal(inspectionResponse.Body.Bytes(), &inspection); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if inspection.Action != spec.Action || inspection.ResourceID != spec.ResourceID ||
|
|
|
|
|
inspection.RequestFingerprint != spec.Fingerprint {
|
|
|
|
|
t.Fatalf("inspection identity = %#v, want action=%q resource=%q fingerprint=%q",
|
|
|
|
|
inspection, spec.Action, spec.ResourceID, spec.Fingerprint)
|
|
|
|
|
}
|
|
|
|
|
for _, suffix := range []string{"reconcile", "confirm-inspection", "abort-job", "cleanup"} {
|
|
|
|
|
response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/"+suffix, []byte(`{}`), actor)
|
|
|
|
|
if response.Code != http.StatusNotFound {
|
|
|
|
|
t.Fatalf("removed %s route = %d %s", suffix, response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
inspections := api.hv.inspections
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
if inspections != 1 {
|
|
|
|
|
t.Fatalf("removed mutation routes reached hypervisor; inspections=%d", inspections)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestOperationInspectionReportsCompletionDeliveryQuarantine(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
operationID := "operation-delivery-quarantine"
|
|
|
|
|
spec := operation.Spec{
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
ResourceID: "vm-delivery-quarantine",
|
|
|
|
|
Action: "delete-vm",
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-delivery-quarantine", []byte(`{}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return json.RawMessage(`{"nested":{"root_password":"must-not-be-delivered"}}`), nil
|
|
|
|
|
}); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if claimed, err := api.journal.ClaimPendingCompletions(1); err != nil || len(claimed) != 0 {
|
|
|
|
|
t.Fatalf("sensitive completion was not quarantined: %#v err=%v", claimed, err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
|
|
|
|
|
if response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("quarantined inspection = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
var inspection hypervisor.OperationInspection
|
|
|
|
|
if err := json.Unmarshal(response.Body.Bytes(), &inspection); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if inspection.OperationStatus != operation.StateSucceeded ||
|
|
|
|
|
inspection.ErrorCode != "" ||
|
|
|
|
|
inspection.Outcome != hypervisor.InspectionOutcomeUncertain ||
|
|
|
|
|
inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete ||
|
|
|
|
|
inspection.EffectResolution != hypervisor.InspectionEffectInconclusive ||
|
|
|
|
|
inspection.CompletionDeliveryState != operation.CompletionDeliveryStateQuarantined ||
|
|
|
|
|
inspection.CompletionDeliveryErrorCode != operation.CompletionDeliveryErrorReceiptInvalid ||
|
|
|
|
|
len(inspection.EffectProjection) != 0 || inspection.AbortJob.Available || inspection.Cleanup.Available {
|
|
|
|
|
t.Fatalf("quarantined completion inspection = %#v", inspection)
|
|
|
|
|
}
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
inspections := api.hv.inspections
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
if inspections != 0 {
|
|
|
|
|
t.Fatalf("delivery quarantine reporting depended on %d live host inspections", inspections)
|
|
|
|
|
}
|
|
|
|
|
stored, err := api.journal.Get(operationID)
|
|
|
|
|
if err != nil || stored.State != operation.StateSucceeded || stored.Error != nil {
|
|
|
|
|
t.Fatalf("delivery quarantine changed operation status: %#v err=%v", stored, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestSuperAdminFenceResolutionIsJournalOnlyAndReadBackIsStable(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
operationID := "operation-fence-resolution"
|
|
|
|
|
spec := operation.Spec{
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
ResourceID: "vm-fence-resolution",
|
|
|
|
|
Action: "delete-vm",
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-resolution", []byte(`{}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, &worker.Error{
|
|
|
|
|
Code: "result_unconfirmed", Message: "result was not confirmed",
|
|
|
|
|
EffectMayHaveApplied: true, ResubmitAllowed: false,
|
|
|
|
|
}
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("seed effectful failure unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) {
|
|
|
|
|
t.Fatalf("missing fence resolution = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("fresh inspection = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
request := operation.FenceResolutionRequest{
|
|
|
|
|
ResolutionID: "0123456789abcdef0123456789abcdef",
|
|
|
|
|
Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(),
|
|
|
|
|
ReasonSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
|
|
|
Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint,
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
missingIdentity := request
|
|
|
|
|
missingIdentity.RequestFingerprint = ""
|
|
|
|
|
missingIdentityBody, err := json.Marshal(missingIdentity)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
invalidIdentity := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", missingIdentityBody, actor)
|
|
|
|
|
if invalidIdentity.Code != http.StatusBadRequest ||
|
|
|
|
|
!bytes.Contains(invalidIdentity.Body.Bytes(), []byte(`"code":"invalid_fence_resolution"`)) {
|
|
|
|
|
t.Fatalf("missing fence identity = %d %s", invalidIdentity.Code, invalidIdentity.Body.String())
|
|
|
|
|
}
|
|
|
|
|
mismatch := request
|
|
|
|
|
mismatch.ResourceID = "vm-other-fence-resolution"
|
|
|
|
|
mismatchBody, err := json.Marshal(mismatch)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
identityConflict := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", mismatchBody, actor)
|
|
|
|
|
if identityConflict.Code != http.StatusConflict ||
|
|
|
|
|
!bytes.Contains(identityConflict.Body.Bytes(), []byte(`"code":"operation_identity_conflict"`)) {
|
|
|
|
|
t.Fatalf("fence identity conflict = %d %s", identityConflict.Code, identityConflict.Body.String())
|
|
|
|
|
}
|
|
|
|
|
resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
|
|
|
|
|
if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"replayed":false`)) {
|
|
|
|
|
t.Fatalf("resolve fence = %d %s", resolved.Code, resolved.Body.String())
|
|
|
|
|
}
|
|
|
|
|
readBack := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor)
|
|
|
|
|
if readBack.Code != http.StatusOK ||
|
|
|
|
|
!bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution_id":"0123456789abcdef0123456789abcdef"`)) ||
|
|
|
|
|
!bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution":"not_applied"`)) {
|
|
|
|
|
t.Fatalf("read fence resolution = %d %s", readBack.Code, readBack.Body.String())
|
|
|
|
|
}
|
|
|
|
|
replayed := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
|
|
|
|
|
if replayed.Code != http.StatusOK || !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) {
|
|
|
|
|
t.Fatalf("idempotent fence resolution = %d %s", replayed.Code, replayed.Body.String())
|
|
|
|
|
}
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
inspections := api.hv.inspections
|
|
|
|
|
configureCalls := api.hv.configureCalls
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
if inspections != 1 || configureCalls != 0 {
|
|
|
|
|
t.Fatalf("fence resolution invoked host work: inspections=%d mutations=%d", inspections, configureCalls)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestFenceResolutionRequiresExactActorOperationIdentity(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
request := operation.FenceResolutionRequest{
|
|
|
|
|
ResolutionID: "fedcba9876543210fedcba9876543210",
|
|
|
|
|
Resolution: operation.FenceResolutionApplied, InspectionCheckedAt: api.now.Unix(),
|
|
|
|
|
ReasonSHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
|
|
|
Action: "delete-vm", ResourceID: "vm-fence-identity",
|
|
|
|
|
RequestFingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-identity", []byte(`{}`)),
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost, "/v1/operations/operation-fence-identity/resolve-fence", body, auth.Actor{
|
|
|
|
|
UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: "operation-other-identity",
|
|
|
|
|
})
|
|
|
|
|
if response.Code != http.StatusBadRequest ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"operation_id_mismatch"`)) {
|
|
|
|
|
t.Fatalf("identity mismatch = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
operationID := "operation-system-recovery"
|
|
|
|
|
spec := operation.Spec{
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
ResourceID: "vm-system-recovery",
|
|
|
|
|
Action: "update-settings",
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms/vm-system-recovery/settings", []byte(`{}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, &worker.Error{
|
|
|
|
|
Code: "result_unconfirmed", Message: "result was not confirmed",
|
|
|
|
|
EffectMayHaveApplied: true,
|
|
|
|
|
}
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("seed effectful failure unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"fence_resolution_required":true`)) ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"fence_state":"active"`)) {
|
|
|
|
|
t.Fatalf("system inspect = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) {
|
|
|
|
|
t.Fatalf("system unresolved fence read = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
request := operation.FenceResolutionRequest{
|
|
|
|
|
ResolutionID: "98769876987698769876987698769876",
|
|
|
|
|
Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(),
|
|
|
|
|
ReasonSHA256: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
|
|
|
|
|
Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint,
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
|
|
|
|
|
if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) {
|
|
|
|
|
t.Fatalf("system fence resolution = %d %s", resolved.Code, resolved.Body.String())
|
|
|
|
|
}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("system resolved fence read = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"fence_resolution_required":false`)) ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"fence_state":"resolved"`)) {
|
|
|
|
|
t.Fatalf("resolved system inspect = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, denied := range []struct {
|
|
|
|
|
name, operationID, resourceID, action string
|
|
|
|
|
}{
|
|
|
|
|
{"unknown", "operation-system-unknown-denied", "vm-system-unknown-denied", "future-unknown"},
|
|
|
|
|
} {
|
|
|
|
|
t.Run("reject "+denied.name+" operation", func(t *testing.T) {
|
|
|
|
|
deniedSpec := operation.Spec{
|
|
|
|
|
OperationID: denied.operationID,
|
|
|
|
|
ResourceID: denied.resourceID,
|
|
|
|
|
Action: denied.action,
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/test/"+denied.resourceID, []byte(`{}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), deniedSpec, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, &worker.Error{Code: "result_unconfirmed", Message: "effect may have applied", EffectMayHaveApplied: true}
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("seed disallowed recovery failure unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
deniedActor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: denied.operationID}
|
|
|
|
|
for _, route := range []struct {
|
|
|
|
|
method, suffix string
|
|
|
|
|
body []byte
|
|
|
|
|
}{
|
|
|
|
|
{http.MethodGet, "/inspect", nil},
|
|
|
|
|
{http.MethodGet, "/fence-resolution", nil},
|
|
|
|
|
{http.MethodPost, "/resolve-fence", body},
|
|
|
|
|
} {
|
|
|
|
|
response := api.request(t, route.method, "/v1/operations/"+denied.operationID+route.suffix, route.body, deniedActor)
|
|
|
|
|
if response.Code != http.StatusForbidden {
|
|
|
|
|
t.Fatalf("system recovery %s %s = %d %s", route.method, route.suffix, response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
var successorCalls atomic.Int32
|
|
|
|
|
successor := operation.Spec{
|
|
|
|
|
OperationID: denied.operationID + "-successor",
|
|
|
|
|
ResourceID: denied.resourceID,
|
|
|
|
|
Action: "delete-vm",
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/"+denied.resourceID, []byte(`{}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
successorCalls.Add(1)
|
|
|
|
|
return json.RawMessage(`{"deleted":true}`), nil
|
|
|
|
|
}); !errors.Is(err, operation.ErrResourceBlocked) || successorCalls.Load() != 0 {
|
|
|
|
|
t.Fatalf("denied system recovery released blocker: calls=%d err=%v", successorCalls.Load(), err)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
reviewedActions := []string{
|
|
|
|
|
"power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy",
|
|
|
|
|
"reinstall-vm",
|
|
|
|
|
"enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended",
|
|
|
|
|
"change-network", "network-suspend", "network-resume",
|
|
|
|
|
"apply-firewall", "delete-vm", "force-delete-vm",
|
|
|
|
|
"service-suspend",
|
|
|
|
|
"cleanup-peer-migration-target:0123456789abcdef0123456789abcdef",
|
|
|
|
|
"create-snapshot:0123456789abcdef0123456789abcdef:filesystem",
|
|
|
|
|
"create-snapshot:0123456789abcdef0123456789abcdef:crash",
|
|
|
|
|
"delete-snapshot:0123456789abcdef0123456789abcdef",
|
|
|
|
|
"restore-snapshot:0123456789abcdef0123456789abcdef",
|
|
|
|
|
"import-template", "update-template", "delete-template",
|
|
|
|
|
"import-rescue-template", "delete-rescue-template",
|
|
|
|
|
"import-user-iso", "delete-user-iso",
|
|
|
|
|
}
|
|
|
|
|
for index, action := range reviewedActions {
|
|
|
|
|
t.Run("reviewed "+action, func(t *testing.T) {
|
|
|
|
|
allowedOperationID := fmt.Sprintf("operation-system-reviewed-%02d", index)
|
|
|
|
|
allowedResourceID := fmt.Sprintf("resource-system-reviewed-%02d", index)
|
|
|
|
|
allowedSpec := operation.Spec{
|
|
|
|
|
OperationID: allowedOperationID,
|
|
|
|
|
ResourceID: allowedResourceID,
|
|
|
|
|
Action: action,
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/test/"+allowedResourceID, []byte(`{}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), allowedSpec, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, &worker.Error{Code: "result_unconfirmed", Message: "effect may have applied", EffectMayHaveApplied: true}
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("seed reviewed recovery failure unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
allowedActor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: allowedOperationID}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+allowedOperationID+"/inspect", nil, allowedActor); response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("reviewed system inspect = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
allowedRequest := operation.FenceResolutionRequest{
|
|
|
|
|
ResolutionID: fmt.Sprintf("%032x", index+1),
|
|
|
|
|
Resolution: operation.FenceResolutionNotApplied,
|
|
|
|
|
InspectionCheckedAt: api.now.Unix(),
|
|
|
|
|
ReasonSHA256: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
|
|
|
|
|
Action: allowedSpec.Action,
|
|
|
|
|
ResourceID: allowedSpec.ResourceID,
|
|
|
|
|
RequestFingerprint: allowedSpec.Fingerprint,
|
|
|
|
|
}
|
|
|
|
|
allowedBody, err := json.Marshal(allowedRequest)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if response := api.request(t, http.MethodPost, "/v1/operations/"+allowedOperationID+"/resolve-fence", allowedBody, allowedActor); response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("reviewed system resolution = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+allowedOperationID+"/fence-resolution", nil, allowedActor); response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("reviewed system fence read = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, test := range []struct {
|
|
|
|
|
name, method, target string
|
|
|
|
|
body []byte
|
|
|
|
|
}{
|
|
|
|
|
{"health", http.MethodGet, "/v1/health", nil},
|
|
|
|
|
{"raw operation", http.MethodGet, "/v1/operations/" + operationID, nil},
|
|
|
|
|
{"other operation", http.MethodGet, "/v1/operations/operation-other-recovery/inspect", nil},
|
|
|
|
|
{"host mutation", http.MethodPost, "/v1/storage/configuration", []byte(`{"operation_id":"operation-system-recovery"}`)},
|
|
|
|
|
{"create mutation", http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-recovery"}`)},
|
|
|
|
|
} {
|
|
|
|
|
t.Run(test.name, func(t *testing.T) {
|
|
|
|
|
response := api.request(t, test.method, test.target, test.body, actor)
|
|
|
|
|
if response.Code != http.StatusForbidden {
|
|
|
|
|
t.Fatalf("least-privilege route = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
manualZero := auth.Actor{
|
|
|
|
|
UserID: 0, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID,
|
|
|
|
|
}
|
|
|
|
|
if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, manualZero); response.Code != http.StatusForbidden {
|
|
|
|
|
t.Fatalf("zero-id human inspection = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
inspections := api.hv.inspections
|
|
|
|
|
configureCalls := api.hv.configureCalls
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
if inspections != 2+len(reviewedActions) || configureCalls != 0 {
|
|
|
|
|
t.Fatalf("system recovery escaped read-only scope: inspections=%d mutations=%d", inspections, configureCalls)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestSystemCreateRecoveryRequiresFreshCompleteAbsenceEvidence(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
operationID := "operation-system-create-recovery"
|
|
|
|
|
resourceID := "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
requestBody := []byte(`{"operation_id":"operation-system-create-recovery"}`)
|
|
|
|
|
spec := operation.Spec{
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
ResourceID: resourceID,
|
|
|
|
|
Action: "create-vm",
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", requestBody),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, &worker.Error{
|
|
|
|
|
Code: "result_unconfirmed", Message: "create result was not confirmed",
|
|
|
|
|
EffectMayHaveApplied: true,
|
|
|
|
|
}
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("seed create failure unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID}
|
|
|
|
|
request := operation.FenceResolutionRequest{
|
|
|
|
|
ResolutionID: "40404040404040404040404040404040",
|
|
|
|
|
Resolution: operation.FenceResolutionApplied,
|
|
|
|
|
InspectionCheckedAt: api.now.Unix(),
|
|
|
|
|
ReasonSHA256: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd",
|
|
|
|
|
Action: spec.Action,
|
|
|
|
|
ResourceID: spec.ResourceID,
|
|
|
|
|
RequestFingerprint: spec.Fingerprint,
|
|
|
|
|
}
|
|
|
|
|
encodeRequest := func() []byte {
|
|
|
|
|
t.Helper()
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return body
|
|
|
|
|
}
|
|
|
|
|
assertActive := func() {
|
|
|
|
|
t.Helper()
|
|
|
|
|
state, err := api.journal.FenceState(operationID)
|
|
|
|
|
if err != nil || state != operation.FenceStateActive {
|
|
|
|
|
t.Fatalf("create fence state=%q err=%v", state, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
applied := api.request(
|
|
|
|
|
t,
|
|
|
|
|
http.MethodPost,
|
|
|
|
|
"/v1/operations/"+operationID+"/resolve-fence",
|
|
|
|
|
encodeRequest(),
|
|
|
|
|
actor,
|
|
|
|
|
)
|
|
|
|
|
if applied.Code != http.StatusConflict {
|
|
|
|
|
t.Fatalf("system create applied resolution = %d %s", applied.Code, applied.Body.String())
|
|
|
|
|
}
|
|
|
|
|
assertActive()
|
|
|
|
|
|
|
|
|
|
request.Resolution = operation.FenceResolutionNotApplied
|
|
|
|
|
inconclusive := api.request(
|
|
|
|
|
t,
|
|
|
|
|
http.MethodPost,
|
|
|
|
|
"/v1/operations/"+operationID+"/resolve-fence",
|
|
|
|
|
encodeRequest(),
|
|
|
|
|
actor,
|
|
|
|
|
)
|
|
|
|
|
if inconclusive.Code != http.StatusConflict ||
|
|
|
|
|
!bytes.Contains(inconclusive.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
|
|
|
|
|
t.Fatalf("inconclusive create resolution = %d %s", inconclusive.Code, inconclusive.Body.String())
|
|
|
|
|
}
|
|
|
|
|
assertActive()
|
|
|
|
|
|
|
|
|
|
exactInspection := func(checkedAt int64) hypervisor.OperationInspection {
|
|
|
|
|
return hypervisor.OperationInspection{
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
Action: spec.Action,
|
|
|
|
|
ResourceID: resourceID,
|
|
|
|
|
RequestFingerprint: spec.Fingerprint,
|
|
|
|
|
OperationStatus: operation.StateFailed,
|
|
|
|
|
CheckedAt: checkedAt,
|
|
|
|
|
Outcome: hypervisor.InspectionOutcomeFailed,
|
|
|
|
|
SummaryCode: hypervisor.InspectionSummaryFailed,
|
|
|
|
|
ErrorCode: "creation_absent",
|
|
|
|
|
EffectResolution: hypervisor.InspectionEffectNotApplied,
|
|
|
|
|
EffectProjection: json.RawMessage(
|
|
|
|
|
`{"creation_absent":true,"managed_artifacts_absent":true}`,
|
|
|
|
|
),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
stale := exactInspection(api.now.Add(-time.Second).Unix())
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
api.hv.inspectionResult = &stale
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
staleResponse := api.request(
|
|
|
|
|
t,
|
|
|
|
|
http.MethodPost,
|
|
|
|
|
"/v1/operations/"+operationID+"/resolve-fence",
|
|
|
|
|
encodeRequest(),
|
|
|
|
|
actor,
|
|
|
|
|
)
|
|
|
|
|
if staleResponse.Code != http.StatusConflict ||
|
|
|
|
|
!bytes.Contains(staleResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
|
|
|
|
|
t.Fatalf("stale create resolution = %d %s", staleResponse.Code, staleResponse.Body.String())
|
|
|
|
|
}
|
|
|
|
|
assertActive()
|
|
|
|
|
|
|
|
|
|
incomplete := exactInspection(api.now.Unix())
|
|
|
|
|
incomplete.EffectProjection = json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":false}`)
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
api.hv.inspectionResult = &incomplete
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
incompleteResponse := api.request(
|
|
|
|
|
t,
|
|
|
|
|
http.MethodPost,
|
|
|
|
|
"/v1/operations/"+operationID+"/resolve-fence",
|
|
|
|
|
encodeRequest(),
|
|
|
|
|
actor,
|
|
|
|
|
)
|
|
|
|
|
if incompleteResponse.Code != http.StatusConflict ||
|
|
|
|
|
!bytes.Contains(incompleteResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
|
|
|
|
|
t.Fatalf("incomplete create absence = %d %s", incompleteResponse.Code, incompleteResponse.Body.String())
|
|
|
|
|
}
|
|
|
|
|
assertActive()
|
|
|
|
|
|
|
|
|
|
exact := exactInspection(api.now.Unix())
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
api.hv.inspectionResult = &exact
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
inspection := api.request(
|
|
|
|
|
t,
|
|
|
|
|
http.MethodGet,
|
|
|
|
|
"/v1/operations/"+operationID+"/inspect",
|
|
|
|
|
nil,
|
|
|
|
|
actor,
|
|
|
|
|
)
|
|
|
|
|
if inspection.Code != http.StatusOK ||
|
|
|
|
|
!bytes.Contains(inspection.Body.Bytes(), []byte(`"error_code":"creation_absent"`)) ||
|
|
|
|
|
!bytes.Contains(inspection.Body.Bytes(), []byte(`"effect_resolution":"not_applied"`)) {
|
|
|
|
|
t.Fatalf("system create inspection = %d %s", inspection.Code, inspection.Body.String())
|
|
|
|
|
}
|
|
|
|
|
resolved := api.request(
|
|
|
|
|
t,
|
|
|
|
|
http.MethodPost,
|
|
|
|
|
"/v1/operations/"+operationID+"/resolve-fence",
|
|
|
|
|
encodeRequest(),
|
|
|
|
|
actor,
|
|
|
|
|
)
|
|
|
|
|
if resolved.Code != http.StatusOK ||
|
|
|
|
|
!bytes.Contains(resolved.Body.Bytes(), []byte(`"resolution":"not_applied"`)) ||
|
|
|
|
|
!bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) {
|
|
|
|
|
t.Fatalf("system create not-applied resolution = %d %s", resolved.Code, resolved.Body.String())
|
|
|
|
|
}
|
|
|
|
|
replayed := api.request(
|
|
|
|
|
t,
|
|
|
|
|
http.MethodPost,
|
|
|
|
|
"/v1/operations/"+operationID+"/resolve-fence",
|
|
|
|
|
encodeRequest(),
|
|
|
|
|
actor,
|
|
|
|
|
)
|
|
|
|
|
if replayed.Code != http.StatusOK ||
|
|
|
|
|
!bytes.Contains(replayed.Body.Bytes(), []byte(`"resolution":"not_applied"`)) ||
|
|
|
|
|
!bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) {
|
|
|
|
|
t.Fatalf("system create resolution replay = %d %s", replayed.Code, replayed.Body.String())
|
|
|
|
|
}
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
inspections := api.hv.inspections
|
|
|
|
|
configureCalls := api.hv.configureCalls
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
if inspections != 5 || configureCalls != 0 {
|
|
|
|
|
t.Fatalf("system create recovery inspections=%d mutations=%d", inspections, configureCalls)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestSystemCreateRecoveryRejectsExecutionFencedAbsence(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
operationID := "operation-system-create-unreaped"
|
|
|
|
|
resourceID := "123e4567-e89b-42d3-a456-426614174002"
|
|
|
|
|
spec := operation.Spec{
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
ResourceID: resourceID,
|
|
|
|
|
Action: "create-vm",
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-create-unreaped"}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, &worker.Error{
|
|
|
|
|
Code: "worker_reap_unconfirmed", Message: "worker may still be running",
|
|
|
|
|
EffectMayHaveApplied: true, ExecutionFenced: true,
|
|
|
|
|
}
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("seed execution-fenced create unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
exactAbsence := hypervisor.OperationInspection{
|
|
|
|
|
OperationID: operationID,
|
|
|
|
|
Action: spec.Action,
|
|
|
|
|
ResourceID: resourceID,
|
|
|
|
|
RequestFingerprint: spec.Fingerprint,
|
|
|
|
|
OperationStatus: operation.StateFailed,
|
|
|
|
|
CheckedAt: api.now.Unix(),
|
|
|
|
|
Outcome: hypervisor.InspectionOutcomeFailed,
|
|
|
|
|
SummaryCode: hypervisor.InspectionSummaryFailed,
|
|
|
|
|
ErrorCode: "creation_absent",
|
|
|
|
|
EffectResolution: hypervisor.InspectionEffectNotApplied,
|
|
|
|
|
EffectProjection: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`),
|
|
|
|
|
}
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
api.hv.inspectionResult = &exactAbsence
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID}
|
|
|
|
|
|
|
|
|
|
inspected := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
|
|
|
|
|
if inspected.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("execution-fenced create inspection = %d %s", inspected.Code, inspected.Body.String())
|
|
|
|
|
}
|
|
|
|
|
var inspection hypervisor.OperationInspection
|
|
|
|
|
if err := json.Unmarshal(inspected.Body.Bytes(), &inspection); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if inspection.Outcome != hypervisor.InspectionOutcomeUncertain ||
|
|
|
|
|
inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete ||
|
|
|
|
|
inspection.EffectResolution != hypervisor.InspectionEffectInconclusive ||
|
|
|
|
|
len(inspection.EffectProjection) != 0 {
|
|
|
|
|
t.Fatalf("execution-fenced create absence was treated as final: %#v", inspection)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
request := operation.FenceResolutionRequest{
|
|
|
|
|
ResolutionID: "50505050505050505050505050505050",
|
|
|
|
|
Resolution: operation.FenceResolutionNotApplied,
|
|
|
|
|
InspectionCheckedAt: api.now.Unix(),
|
|
|
|
|
ReasonSHA256: "dededededededededededededededededededededededededededededededede",
|
|
|
|
|
Action: spec.Action,
|
|
|
|
|
ResourceID: spec.ResourceID,
|
|
|
|
|
RequestFingerprint: spec.Fingerprint,
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
|
|
|
|
|
if resolved.Code != http.StatusConflict ||
|
|
|
|
|
!bytes.Contains(resolved.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
|
|
|
|
|
t.Fatalf("execution-fenced create resolution = %d %s", resolved.Code, resolved.Body.String())
|
|
|
|
|
}
|
|
|
|
|
if state, err := api.journal.FenceState(operationID); err != nil || state != operation.FenceStateActive {
|
|
|
|
|
t.Fatalf("execution-fenced create was released state=%q err=%v", state, err)
|
|
|
|
|
}
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
inspections := api.hv.inspections
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
if inspections != 1 {
|
|
|
|
|
t.Fatalf("execution-fenced create resolve repeated host inspection: %d", inspections)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
executor := &fakeMutationExecutor{result: json.RawMessage(`{"configured":true}`)}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
request := hypervisor.StorageConfigurationRequest{
|
|
|
|
|
OperationID: "operation-isolated-storage",
|
|
|
|
|
ManagedDirectory: "/srv/virt/managed",
|
|
|
|
|
TemplateDirectory: "/srv/virt/templates",
|
|
|
|
|
SeedDirectory: "/srv/virt/seeds",
|
|
|
|
|
UserISODirectory: "/srv/virt/user-isos",
|
|
|
|
|
RescueDirectory: "/srv/virt/rescue-isos",
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: request.OperationID,
|
|
|
|
|
}
|
|
|
|
|
first := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
|
|
|
|
|
if first.Code != http.StatusAccepted {
|
|
|
|
|
t.Fatalf("first isolated response = %d %s", first.Code, first.Body.String())
|
|
|
|
|
}
|
|
|
|
|
fingerprint := operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", body)
|
|
|
|
|
accepted := requireOperationResponseIdentity(t, first, request.OperationID, "configure-storage", "host-storage", fingerprint)
|
|
|
|
|
if accepted.Status != operation.StateDispatching || accepted.Replayed {
|
|
|
|
|
t.Fatalf("accepted operation response = %#v", accepted)
|
|
|
|
|
}
|
|
|
|
|
terminal := waitForTerminal(t, api.journal, request.OperationID)
|
|
|
|
|
if terminal.State != operation.StateSucceeded || string(terminal.Result) != `{"configured":true}` {
|
|
|
|
|
t.Fatalf("terminal operation = %#v", terminal)
|
|
|
|
|
}
|
|
|
|
|
replay := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor)
|
|
|
|
|
if replay.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("terminal replay = %d %s", replay.Code, replay.Body.String())
|
|
|
|
|
}
|
|
|
|
|
replayed := requireOperationResponseIdentity(t, replay, request.OperationID, "configure-storage", "host-storage", fingerprint)
|
|
|
|
|
if replayed.Status != operation.StateSucceeded || !replayed.Replayed {
|
|
|
|
|
t.Fatalf("replayed operation response = %#v", replayed)
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Lock()
|
|
|
|
|
defer executor.mu.Unlock()
|
|
|
|
|
if executor.calls != 1 || len(executor.requests) != 1 {
|
|
|
|
|
t.Fatalf("executor calls=%d requests=%d", executor.calls, len(executor.requests))
|
|
|
|
|
}
|
|
|
|
|
workerRequest := executor.requests[0]
|
|
|
|
|
if workerRequest.OperationID != request.OperationID || workerRequest.Actor.OperationID != request.OperationID ||
|
|
|
|
|
workerRequest.Method != http.MethodPost || workerRequest.RequestTarget != "/v1/storage/configuration" {
|
|
|
|
|
t.Fatalf("worker envelope = %#v", workerRequest)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestPeerMigrationTargetCleanupRequiresFreshExactPlanAndRetainsReceiveFence(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
migrationID = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
priorID = "operation-peer-receive-residue"
|
|
|
|
|
cleanupID = "operation-peer-target-cleanup"
|
|
|
|
|
)
|
|
|
|
|
prior := operation.Spec{
|
|
|
|
|
OperationID: priorID, ResourceID: uuid,
|
|
|
|
|
Action: "receive-peer-migration:" + migrationID,
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodPost,
|
|
|
|
|
"/v1/vms/"+uuid+"/peer-migration/receive", []byte(`{}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
LockKeys: hypervisor.ReceivePeerMigrationAdmission(hypervisor.ReceivePeerMigrationRequest{
|
|
|
|
|
CreateVMRequest: hypervisor.CreateVMRequest{UUID: uuid}, MigrationID: migrationID,
|
|
|
|
|
}).LockKeys,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), prior, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, &worker.Error{Code: "result_unconfirmed", Message: "receive result unknown", EffectMayHaveApplied: true}
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("seed receive failure unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
stored, err := api.journal.Get(priorID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
planID := strings.Repeat("a", 64)
|
|
|
|
|
api.hv.inspectionResult = &hypervisor.OperationInspection{
|
|
|
|
|
OperationID: stored.OperationID, Action: stored.Action,
|
|
|
|
|
ResourceID: stored.ResourceID, RequestFingerprint: stored.Fingerprint,
|
|
|
|
|
OperationStatus: stored.State, CheckedAt: api.now.Unix(),
|
|
|
|
|
Outcome: hypervisor.InspectionOutcomeUncertain,
|
|
|
|
|
SummaryCode: hypervisor.InspectionSummaryCleanupRequired,
|
|
|
|
|
EffectResolution: hypervisor.InspectionEffectPartial,
|
|
|
|
|
Cleanup: hypervisor.OperationCleanupPlan{Available: true, PlanID: planID,
|
|
|
|
|
Steps: []string{hypervisor.CleanupStepDeleteManagedVolumes}},
|
|
|
|
|
}
|
|
|
|
|
executor := &fakeMutationExecutor{result: json.RawMessage(`{"deleted":true,"managed_artifacts_absent":true,"forced":false}`)}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
request := hypervisor.CleanupPeerMigrationTargetRequest{
|
|
|
|
|
OperationID: cleanupID, MigrationID: migrationID,
|
|
|
|
|
PriorOperationID: priorID, CleanupPlanID: planID,
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: cleanupID}
|
|
|
|
|
response := api.request(t, http.MethodPost,
|
|
|
|
|
"/v1/vms/"+uuid+"/peer-migration/cleanup-target", body, actor)
|
|
|
|
|
if response.Code != http.StatusAccepted {
|
|
|
|
|
t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
terminal := waitForTerminal(t, api.journal, cleanupID)
|
|
|
|
|
if terminal.State != operation.StateSucceeded || terminal.SupersedesOperationID != priorID {
|
|
|
|
|
t.Fatalf("cleanup terminal = %#v", terminal)
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Lock()
|
|
|
|
|
if len(executor.requests) != 1 || executor.requests[0].SupersededOperation == nil ||
|
|
|
|
|
executor.requests[0].SupersededOperation.OperationID != priorID {
|
|
|
|
|
executor.mu.Unlock()
|
|
|
|
|
t.Fatalf("cleanup worker predecessor = %#v", executor.requests)
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Unlock()
|
|
|
|
|
if state, err := api.journal.FenceState(priorID); err != nil || state != operation.FenceStateActive {
|
|
|
|
|
t.Fatalf("receive fence state=%q err=%v", state, err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
stale := request
|
|
|
|
|
stale.OperationID = "operation-peer-target-cleanup-stale"
|
|
|
|
|
stale.CleanupPlanID = strings.Repeat("b", 64)
|
|
|
|
|
staleBody, _ := json.Marshal(stale)
|
|
|
|
|
staleActor := actor
|
|
|
|
|
staleActor.OperationID = stale.OperationID
|
|
|
|
|
denied := api.request(t, http.MethodPost,
|
|
|
|
|
"/v1/vms/"+uuid+"/peer-migration/cleanup-target", staleBody, staleActor)
|
|
|
|
|
if denied.Code != http.StatusConflict ||
|
|
|
|
|
!bytes.Contains(denied.Body.Bytes(), []byte(`"code":"peer_migration_cleanup_plan_changed"`)) {
|
|
|
|
|
t.Fatalf("stale cleanup = %d %s", denied.Code, denied.Body.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestTrustedWorkerPeerMigrationTargetCleanupUsesParentPredecessor(t *testing.T) {
|
|
|
|
|
parent := newTestAPI(t)
|
|
|
|
|
child := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
migrationID = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
priorID = "operation-peer-receive-worker-residue"
|
|
|
|
|
cleanupID = "operation-peer-target-worker-cleanup"
|
|
|
|
|
)
|
|
|
|
|
priorSpec := operation.Spec{
|
|
|
|
|
OperationID: priorID, ResourceID: uuid,
|
|
|
|
|
Action: "receive-peer-migration:" + migrationID,
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodPost,
|
|
|
|
|
"/v1/vms/"+uuid+"/peer-migration/receive", []byte(`{}`)),
|
|
|
|
|
Destructive: true,
|
|
|
|
|
LockKeys: hypervisor.ReceivePeerMigrationAdmission(hypervisor.ReceivePeerMigrationRequest{
|
|
|
|
|
CreateVMRequest: hypervisor.CreateVMRequest{UUID: uuid}, MigrationID: migrationID,
|
|
|
|
|
}).LockKeys,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := parent.journal.Execute(context.Background(), priorSpec, func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return nil, &worker.Error{Code: "result_unconfirmed", Message: "receive result unknown", EffectMayHaveApplied: true}
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("seed receive failure unexpectedly succeeded")
|
|
|
|
|
}
|
|
|
|
|
prior, err := parent.journal.Get(priorID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
planID := strings.Repeat("c", 64)
|
|
|
|
|
child.hv.inspectionResult = &hypervisor.OperationInspection{
|
|
|
|
|
OperationID: prior.OperationID, Action: prior.Action,
|
|
|
|
|
ResourceID: prior.ResourceID, RequestFingerprint: prior.Fingerprint,
|
|
|
|
|
OperationStatus: prior.State, CheckedAt: child.now.Unix(),
|
|
|
|
|
Outcome: hypervisor.InspectionOutcomeUncertain,
|
|
|
|
|
SummaryCode: hypervisor.InspectionSummaryCleanupRequired,
|
|
|
|
|
EffectResolution: hypervisor.InspectionEffectPartial,
|
|
|
|
|
Cleanup: hypervisor.OperationCleanupPlan{Available: true, PlanID: planID,
|
|
|
|
|
Steps: []string{hypervisor.CleanupStepDeleteManagedVolumes}},
|
|
|
|
|
}
|
|
|
|
|
requestBody, err := json.Marshal(hypervisor.CleanupPeerMigrationTargetRequest{
|
|
|
|
|
OperationID: cleanupID, MigrationID: migrationID,
|
|
|
|
|
PriorOperationID: priorID, CleanupPlanID: planID,
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: cleanupID,
|
|
|
|
|
}
|
|
|
|
|
request := httptest.NewRequest(http.MethodPost,
|
|
|
|
|
"/v1/vms/"+uuid+"/peer-migration/cleanup-target", bytes.NewReader(requestBody))
|
|
|
|
|
response := httptest.NewRecorder()
|
|
|
|
|
child.handler.ServeTrustedWorkerRequest(response, request, requestBody, actor, prior)
|
|
|
|
|
if response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("trusted cleanup response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
terminal, err := child.journal.Get(cleanupID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if terminal.State != operation.StateSucceeded || terminal.SupersedesOperationID != "" {
|
|
|
|
|
t.Fatalf("trusted cleanup terminal = %#v", terminal)
|
|
|
|
|
}
|
|
|
|
|
child.hv.mu.Lock()
|
|
|
|
|
defer child.hv.mu.Unlock()
|
|
|
|
|
if child.hv.cleanupPeerTargetCalls != 1 ||
|
|
|
|
|
child.hv.cleanupPeerTargetLast.PriorOperation.OperationID != priorID {
|
|
|
|
|
t.Fatalf("trusted cleanup calls=%d request=%#v",
|
|
|
|
|
child.hv.cleanupPeerTargetCalls, child.hv.cleanupPeerTargetLast)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestCreateBackupUsesDedicatedAsyncActionAndHeavyAdmission(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
backupKey = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
)
|
|
|
|
|
result := hypervisor.BackupResult{
|
|
|
|
|
BackupKey: backupKey, SizeBytes: 1024, VirtualSizeBytes: 40 << 30,
|
|
|
|
|
SHA256: strings.Repeat("a", 64), Consistency: "crash",
|
|
|
|
|
SourceContract: &hypervisor.PortableRestoreContract{
|
|
|
|
|
Contract: 1, Architecture: "x86_64", Machine: "q35",
|
|
|
|
|
Firmware: "uefi", SecureBoot: false, DiskFormat: "qcow2",
|
|
|
|
|
SystemDiskOnly: true, PublicNetworkContract: 1, VPCInherited: false,
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
encodedResult, err := json.Marshal(result)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
executor := &fakeMutationExecutor{result: encodedResult}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
request := hypervisor.BackupRequest{
|
|
|
|
|
OperationID: "operation-create-backup", BackupKey: backupKey,
|
|
|
|
|
Name: "portable backup", ConsistencyMode: "filesystem",
|
|
|
|
|
ObjectUpload: objecttransfer.MultipartPlan{
|
|
|
|
|
PartSizeBytes: 5 << 20,
|
|
|
|
|
PartURLs: []string{
|
|
|
|
|
"https://objects.example.test/object?partNumber=1&uploadId=secret",
|
|
|
|
|
},
|
|
|
|
|
CompleteURL: "https://objects.example.test/object?uploadId=secret",
|
|
|
|
|
AbortURL: "https://objects.example.test/object?uploadId=secret",
|
|
|
|
|
VerifyURL: "https://objects.example.test/object?signature=secret",
|
|
|
|
|
MaximumSizeBytes: 5 << 20,
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: request.OperationID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/backups", body, actor)
|
|
|
|
|
if response.Code != http.StatusAccepted {
|
|
|
|
|
t.Fatalf("backup response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
record := waitForTerminal(t, api.journal, request.OperationID)
|
|
|
|
|
if record.State != operation.StateSucceeded || record.Action != "create-backup:"+backupKey+":filesystem" ||
|
|
|
|
|
record.ResourceID != uuid || record.Destructive || !record.HeavyIO {
|
|
|
|
|
t.Fatalf("backup operation record = %#v", record)
|
|
|
|
|
}
|
|
|
|
|
for _, want := range []string{
|
|
|
|
|
"domain:" + uuid,
|
|
|
|
|
"volume:virt-cloud-managed/vm-" + uuid,
|
|
|
|
|
"volume:virt-cloud-managed/vbackup-123e4567e89b42d3a456426614174000-" + backupKey + ".qcow2",
|
|
|
|
|
"catalog:backup/" + backupKey,
|
|
|
|
|
} {
|
|
|
|
|
if !containsTestString(record.LockKeys, want) {
|
|
|
|
|
t.Fatalf("backup operation locks missing %q: %v", want, record.LockKeys)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
receipts, err := api.journal.ClaimPendingCompletions(1)
|
|
|
|
|
if err != nil || len(receipts) != 1 || receipts[0].OperationID != request.OperationID ||
|
|
|
|
|
receipts[0].Action != record.Action || receipts[0].Status != string(operation.StateSucceeded) {
|
|
|
|
|
t.Fatalf("backup completion outbox = %#v err=%v", receipts, err)
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Lock()
|
|
|
|
|
deadline := executor.deadline
|
|
|
|
|
executor.mu.Unlock()
|
|
|
|
|
remaining := time.Until(deadline)
|
|
|
|
|
if remaining < 23*time.Hour+59*time.Minute || remaining > 24*time.Hour {
|
|
|
|
|
t.Fatalf("backup helper deadline remaining = %s, want dedicated 24h bound", remaining)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestBackupWorkspaceCleanupRequiresSuccessfulPriorAndFreshExactPlan(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
backupKey = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
priorID = "operation-backup-cleanup-pending"
|
|
|
|
|
cleanupID = "operation-backup-workspace-cleanup"
|
|
|
|
|
)
|
|
|
|
|
priorSpec := operation.Spec{
|
|
|
|
|
OperationID: priorID, ResourceID: uuid,
|
|
|
|
|
Action: "create-backup:" + backupKey + ":filesystem",
|
|
|
|
|
Fingerprint: operation.Fingerprint(http.MethodPost,
|
|
|
|
|
"/v1/vms/"+uuid+"/backups", []byte(`{}`)),
|
|
|
|
|
LockKeys: hypervisor.CreateBackupAdmission(uuid, hypervisor.BackupRequest{
|
|
|
|
|
BackupKey: backupKey,
|
|
|
|
|
}).LockKeys,
|
|
|
|
|
HeavyIO: true,
|
|
|
|
|
}
|
|
|
|
|
if _, _, err := api.journal.Execute(context.Background(), priorSpec,
|
|
|
|
|
func(context.Context) (json.RawMessage, error) {
|
|
|
|
|
return json.RawMessage(`{"backup_key":"` + backupKey + `","size_bytes":1024,"virtual_size_bytes":4096,"sha256":"` + strings.Repeat("a", 64) + `","consistency":"crash","cleanup_pending":true}`), nil
|
|
|
|
|
}); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
prior, err := api.journal.Get(priorID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
planID := strings.Repeat("b", 64)
|
|
|
|
|
api.hv.inspectionResult = &hypervisor.OperationInspection{
|
|
|
|
|
OperationID: prior.OperationID, Action: prior.Action,
|
|
|
|
|
ResourceID: prior.ResourceID, RequestFingerprint: prior.Fingerprint,
|
|
|
|
|
OperationStatus: prior.State, CheckedAt: api.now.Unix(),
|
|
|
|
|
Outcome: hypervisor.InspectionOutcomeSucceeded,
|
|
|
|
|
SummaryCode: hypervisor.InspectionSummaryCleanupRequired,
|
|
|
|
|
EffectResolution: hypervisor.InspectionEffectApplied,
|
|
|
|
|
EffectProjection: append(json.RawMessage(nil), prior.Result...),
|
|
|
|
|
Cleanup: hypervisor.OperationCleanupPlan{
|
|
|
|
|
Available: true, PlanID: planID,
|
|
|
|
|
Steps: []string{hypervisor.CleanupStepDeleteSnapshotVolume},
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
executor := &fakeMutationExecutor{result: json.RawMessage(`{"backup_key":"` + backupKey + `","cleaned":true,"workspace_absent":true,"metadata_absent":true}`)}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
request := hypervisor.CleanupBackupWorkspaceRequest{
|
|
|
|
|
OperationID: cleanupID, PriorOperationID: priorID, CleanupPlanID: planID,
|
|
|
|
|
}
|
|
|
|
|
body, _ := json.Marshal(request)
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: cleanupID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost,
|
|
|
|
|
"/v1/operations/"+cleanupID+"/cleanup-backup-workspace", body, actor)
|
|
|
|
|
if response.Code != http.StatusAccepted {
|
|
|
|
|
t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
terminal := waitForTerminal(t, api.journal, cleanupID)
|
|
|
|
|
if terminal.State != operation.StateSucceeded ||
|
|
|
|
|
terminal.SupersedesOperationID != priorID ||
|
|
|
|
|
terminal.Action != "cleanup-backup-workspace:"+backupKey ||
|
|
|
|
|
terminal.ResourceID != uuid {
|
|
|
|
|
t.Fatalf("cleanup terminal = %#v", terminal)
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Lock()
|
|
|
|
|
if len(executor.requests) != 1 || executor.requests[0].SupersededOperation == nil ||
|
|
|
|
|
executor.requests[0].SupersededOperation.OperationID != priorID {
|
|
|
|
|
executor.mu.Unlock()
|
|
|
|
|
t.Fatalf("cleanup worker predecessor = %#v", executor.requests)
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Unlock()
|
|
|
|
|
|
|
|
|
|
stale := request
|
|
|
|
|
stale.OperationID = "operation-backup-workspace-stale"
|
|
|
|
|
stale.CleanupPlanID = strings.Repeat("c", 64)
|
|
|
|
|
staleBody, _ := json.Marshal(stale)
|
|
|
|
|
staleActor := actor
|
|
|
|
|
staleActor.OperationID = stale.OperationID
|
|
|
|
|
denied := api.request(t, http.MethodPost,
|
|
|
|
|
"/v1/operations/"+stale.OperationID+"/cleanup-backup-workspace",
|
|
|
|
|
staleBody, staleActor)
|
|
|
|
|
if denied.Code != http.StatusConflict ||
|
|
|
|
|
!bytes.Contains(denied.Body.Bytes(), []byte(`"code":"backup_workspace_cleanup_plan_changed"`)) {
|
|
|
|
|
t.Fatalf("stale cleanup = %d %s", denied.Code, denied.Body.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRecoverBackupUsesDestructiveDiskTransitionAdmission(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
backupKey = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
)
|
|
|
|
|
backupSHA := strings.Repeat("b", 64)
|
|
|
|
|
result := hypervisor.RecoverBackupResult{
|
|
|
|
|
BackupKey: backupKey, SHA256: backupSHA,
|
|
|
|
|
Restored: true, PowerState: hypervisor.VMStateRunning,
|
|
|
|
|
}
|
|
|
|
|
encodedResult, err := json.Marshal(result)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
executor := &fakeMutationExecutor{result: encodedResult}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
request := hypervisor.RecoverBackupRequest{
|
|
|
|
|
OperationID: "operation-recover-backup", BackupKey: backupKey,
|
|
|
|
|
SourceURL: "https://objects.example.test/object?signature=secret",
|
|
|
|
|
SourceSizeBytes: 4 << 20, SourceSHA256: backupSHA,
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: request.OperationID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost,
|
|
|
|
|
"/v1/vms/"+uuid+"/recover-backup", body, actor)
|
|
|
|
|
if response.Code != http.StatusAccepted {
|
|
|
|
|
t.Fatalf("backup recovery response = %d %s", response.Code,
|
|
|
|
|
response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
record := waitForTerminal(t, api.journal, request.OperationID)
|
|
|
|
|
if record.State != operation.StateSucceeded ||
|
|
|
|
|
record.Action != "recover-backup:"+backupKey+":"+backupSHA ||
|
|
|
|
|
record.ResourceID != uuid || !record.Destructive || !record.HeavyIO {
|
|
|
|
|
t.Fatalf("backup recovery operation record = %#v", record)
|
|
|
|
|
}
|
|
|
|
|
for _, want := range []string{
|
|
|
|
|
"domain:" + uuid,
|
|
|
|
|
"volume:virt-cloud-managed/vm-" + uuid,
|
|
|
|
|
"catalog:backup/" + backupKey,
|
|
|
|
|
} {
|
|
|
|
|
if !containsTestString(record.LockKeys, want) {
|
|
|
|
|
t.Fatalf("backup recovery locks missing %q: %v", want,
|
|
|
|
|
record.LockKeys)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRestoreBackupDerivesDomainAndUsesCreateLikeAction(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
backupKey = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
)
|
|
|
|
|
result := hypervisor.RestoreBackupResult{
|
|
|
|
|
VMMetadata: hypervisor.VMMetadata{UUID: uuid, Name: "restored-vm"},
|
|
|
|
|
BackupKey: backupKey, SHA256: strings.Repeat("b", 64),
|
|
|
|
|
}
|
|
|
|
|
encodedResult, err := json.Marshal(result)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
executor := &fakeMutationExecutor{result: encodedResult}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
request := hypervisor.RestoreBackupRequest{
|
|
|
|
|
CreateVMRequest: hypervisor.CreateVMRequest{
|
|
|
|
|
OperationID: "operation-restore-backup", UUID: uuid,
|
|
|
|
|
Name: "restored-vm", VCPUs: 2, MemoryMiB: 2048, DiskGiB: 40,
|
|
|
|
|
BandwidthMbps: 100,
|
|
|
|
|
PublicNetwork: publicnetwork.Intent{
|
|
|
|
|
Contract: 1, NetworkSelector: "bridge:br0",
|
|
|
|
|
IPv4: publicnetwork.AddressIntent{PoolKey: backupKey, PoolCIDR: "8.8.8.0/24", AddressCIDR: "8.8.8.10/24", Gateway: "8.8.8.1"},
|
|
|
|
|
IPv6: publicnetwork.AddressIntent{PoolKey: "abcdef0123456789abcdef0123456789", PoolCIDR: "2404:c140:156:3a::/64", AddressCIDR: "2404:c140:156:3a::1000/64", Gateway: "2404:c140:156::1", GatewayOnLink: true},
|
|
|
|
|
DNSServers: []string{"1.1.1.1", "2606:4700:4700::1111"},
|
|
|
|
|
},
|
|
|
|
|
PlatformNetworkPolicy: apiTestPlatformPolicy(t),
|
|
|
|
|
},
|
|
|
|
|
RequestedDomainName: "1:vm-123e4567e89b",
|
|
|
|
|
BackupKey: backupKey, SourceURL: "https://objects.example.test/object?signature=secret",
|
|
|
|
|
SourceSizeBytes: 4 << 20, SourceSHA256: strings.Repeat("b", 64),
|
|
|
|
|
}
|
|
|
|
|
debian12NetworkFix := true
|
|
|
|
|
request.Debian12NetworkFix = &debian12NetworkFix
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: request.OperationID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/restore-backup", body, actor)
|
|
|
|
|
if response.Code != http.StatusAccepted {
|
|
|
|
|
t.Fatalf("restore response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
record := waitForTerminal(t, api.journal, request.OperationID)
|
|
|
|
|
if record.State != operation.StateSucceeded || record.Action != "restore-backup:"+backupKey ||
|
|
|
|
|
record.ResourceID != uuid || !record.Destructive || !record.HeavyIO ||
|
|
|
|
|
!operation.IsCreateLikeAction(record.Action) || record.ExpectedState == nil ||
|
|
|
|
|
record.ExpectedState.Kind != operation.ExpectedCreateLikeNetwork ||
|
|
|
|
|
record.ExpectedState.TransitionID != request.OperationID ||
|
|
|
|
|
record.ExpectedState.NetworkSourceType != "bridge" ||
|
|
|
|
|
record.ExpectedState.NetworkSourceName != "br0" ||
|
|
|
|
|
!reflect.DeepEqual(record.ExpectedState.PublicNetwork, request.PublicNetwork) ||
|
|
|
|
|
record.ExpectedState.VPCPresent || record.ExpectedState.VPCKey != "" ||
|
|
|
|
|
record.ExpectedState.VPCPortKey != "" {
|
|
|
|
|
t.Fatalf("restore operation record = %#v", record)
|
|
|
|
|
}
|
|
|
|
|
receipts, err := api.journal.ClaimPendingCompletions(1)
|
|
|
|
|
if err != nil || len(receipts) != 1 || receipts[0].OperationID != request.OperationID ||
|
|
|
|
|
receipts[0].Action != record.Action || receipts[0].Status != string(operation.StateSucceeded) {
|
|
|
|
|
t.Fatalf("restore completion outbox = %#v err=%v", receipts, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestPeerMigrationReceivePersistsExactPublicAndVPCExpectedState(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const (
|
|
|
|
|
uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
migrationID = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
vpcKey = "1123456789abcdef0123456789abcdef"
|
|
|
|
|
portKey = "2123456789abcdef0123456789abcdef"
|
|
|
|
|
hostDigest = "3123456789abcdef0123456789abcdef3123456789abcdef0123456789abcdef"
|
|
|
|
|
)
|
|
|
|
|
bridge, err := vpcrouting.BridgeName(vpcKey)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
public := publicnetwork.Intent{
|
|
|
|
|
Contract: 1, NetworkSelector: "network:public-dual-stack",
|
|
|
|
|
IPv4: publicnetwork.AddressIntent{
|
|
|
|
|
PoolKey: migrationID, PoolCIDR: "8.8.8.0/24",
|
|
|
|
|
AddressCIDR: "8.8.8.10/24", Gateway: "8.8.8.1",
|
|
|
|
|
},
|
|
|
|
|
IPv6: publicnetwork.AddressIntent{
|
|
|
|
|
PoolKey: "abcdef0123456789abcdef0123456789",
|
|
|
|
|
PoolCIDR: "2404:c140:156:3a::/64", AddressCIDR: "2404:c140:156:3a::1000/64",
|
|
|
|
|
Gateway: "2404:c140:156::1", GatewayOnLink: true,
|
|
|
|
|
},
|
|
|
|
|
DNSServers: []string{"1.1.1.1", "2606:4700:4700::1111"},
|
|
|
|
|
}
|
|
|
|
|
port := hypervisor.VPCPortMetadata{
|
|
|
|
|
PortKey: portKey, VPCKey: vpcKey, PrivateIP: "10.60.0.9",
|
|
|
|
|
CIDR: "10.60.0.0/24", Gateway: "10.60.0.1", MAC: "02:00:00:00:00:09",
|
|
|
|
|
Bridge: bridge, HostRevision: 7, HostDigest: hostDigest,
|
|
|
|
|
}
|
|
|
|
|
request := hypervisor.ReceivePeerMigrationRequest{
|
|
|
|
|
CreateVMRequest: hypervisor.CreateVMRequest{
|
|
|
|
|
OperationID: "operation-peer-expected-state", UUID: uuid,
|
|
|
|
|
Name: "migrated-vm", VCPUs: 2, MemoryMiB: 2048, DiskGiB: 40,
|
|
|
|
|
BandwidthMbps: 100, PublicNetwork: public,
|
|
|
|
|
PlatformNetworkPolicy: apiTestPlatformPolicy(t),
|
|
|
|
|
},
|
|
|
|
|
MigrationID: migrationID, Ticket: strings.Repeat("2", 64),
|
|
|
|
|
SourceEndpoints: []string{"https://192.0.2.10:9443"},
|
|
|
|
|
SourceIdentitySHA256: strings.Repeat("3", 64),
|
|
|
|
|
SourceUUID: "123e4567-e89b-42d3-a456-426614174001",
|
|
|
|
|
ManifestDigest: strings.Repeat("4", 64), Debian12NetworkFix: true,
|
|
|
|
|
VPCPort: &port,
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
executor := &fakeMutationExecutor{result: json.RawMessage(`{}`)}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: request.OperationID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/peer-migration/receive", body, actor)
|
|
|
|
|
if response.Code != http.StatusAccepted {
|
|
|
|
|
t.Fatalf("peer migration receive response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
record := waitForTerminal(t, api.journal, request.OperationID)
|
|
|
|
|
expected := record.ExpectedState
|
|
|
|
|
if record.State != operation.StateSucceeded ||
|
|
|
|
|
record.Action != "receive-peer-migration:"+migrationID ||
|
|
|
|
|
record.ResourceID != uuid || expected == nil ||
|
|
|
|
|
expected.Kind != operation.ExpectedCreateLikeNetwork ||
|
|
|
|
|
expected.TransitionID != request.OperationID ||
|
|
|
|
|
expected.NetworkSourceType != "network" ||
|
|
|
|
|
expected.NetworkSourceName != "public-dual-stack" ||
|
|
|
|
|
!reflect.DeepEqual(expected.PublicNetwork, public) || !expected.VPCPresent ||
|
|
|
|
|
expected.VPCKey != port.VPCKey || expected.VPCPortKey != port.PortKey ||
|
|
|
|
|
expected.VPCRevision != port.HostRevision || expected.VPCDigest != port.HostDigest ||
|
|
|
|
|
expected.VPCCIDR != port.CIDR || expected.VPCGateway != port.Gateway ||
|
|
|
|
|
expected.VPCPrivateIP != port.PrivateIP || expected.VPCMAC != port.MAC ||
|
|
|
|
|
expected.VPCBridge != port.Bridge {
|
|
|
|
|
t.Fatalf("peer migration expected state = %#v record=%#v", expected, record)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestDiskAuthoritativeRestoreWireContractsRejectHostnameBeforeDispatch(t *testing.T) {
|
|
|
|
|
const (
|
|
|
|
|
uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
backupKey = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
)
|
|
|
|
|
public := publicnetwork.Intent{
|
|
|
|
|
Contract: 1, NetworkSelector: "bridge:br0",
|
|
|
|
|
IPv4: publicnetwork.AddressIntent{
|
|
|
|
|
PoolKey: backupKey, PoolCIDR: "8.8.8.0/24",
|
|
|
|
|
AddressCIDR: "8.8.8.10/24", Gateway: "8.8.8.1",
|
|
|
|
|
},
|
|
|
|
|
IPv6: publicnetwork.AddressIntent{
|
|
|
|
|
PoolKey: "abcdef0123456789abcdef0123456789", PoolCIDR: "2404:c140:156:3a::/64",
|
|
|
|
|
AddressCIDR: "2404:c140:156:3a::1000/64", Gateway: "2404:c140:156::1", GatewayOnLink: true,
|
|
|
|
|
},
|
|
|
|
|
DNSServers: []string{"1.1.1.1", "2606:4700:4700::1111"},
|
|
|
|
|
}
|
|
|
|
|
create := hypervisor.CreateVMRequest{
|
|
|
|
|
UUID: uuid, Name: "restored-vm", VCPUs: 2, MemoryMiB: 2048,
|
|
|
|
|
DiskGiB: 40, BandwidthMbps: 100, Hostname: "replacement.example.com",
|
|
|
|
|
PublicNetwork: public, PlatformNetworkPolicy: apiTestPlatformPolicy(t),
|
|
|
|
|
}
|
|
|
|
|
debian12NetworkFix := true
|
|
|
|
|
restoreCreate := create
|
|
|
|
|
restoreCreate.OperationID = "operation-restore-hostname-contract"
|
|
|
|
|
peerCreate := create
|
|
|
|
|
peerCreate.OperationID = "operation-peer-hostname-contract"
|
|
|
|
|
tests := []struct {
|
|
|
|
|
name string
|
|
|
|
|
path string
|
|
|
|
|
operationID string
|
|
|
|
|
errorCode string
|
|
|
|
|
request any
|
|
|
|
|
}{
|
|
|
|
|
{
|
|
|
|
|
name: "backup restore", path: "/v1/vms/" + uuid + "/restore-backup",
|
|
|
|
|
operationID: restoreCreate.OperationID, errorCode: "invalid_backup_restore",
|
|
|
|
|
request: hypervisor.RestoreBackupRequest{
|
|
|
|
|
CreateVMRequest: restoreCreate,
|
|
|
|
|
RequestedDomainName: "1:vm-123e4567e89b",
|
|
|
|
|
BackupKey: backupKey, SourceURL: "https://objects.example.test/object?signature=secret",
|
|
|
|
|
SourceSizeBytes: 4 << 20, SourceSHA256: strings.Repeat("b", 64),
|
|
|
|
|
Debian12NetworkFix: &debian12NetworkFix,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
name: "peer migration receive", path: "/v1/vms/" + uuid + "/peer-migration/receive",
|
|
|
|
|
operationID: peerCreate.OperationID, errorCode: "invalid_peer_migration_receive",
|
|
|
|
|
request: hypervisor.ReceivePeerMigrationRequest{
|
|
|
|
|
CreateVMRequest: peerCreate,
|
|
|
|
|
MigrationID: strings.Repeat("1", 32), Ticket: strings.Repeat("2", 64),
|
|
|
|
|
SourceEndpoints: []string{"https://192.0.2.10:9443"},
|
|
|
|
|
SourceIdentitySHA256: strings.Repeat("3", 64),
|
|
|
|
|
SourceUUID: "123e4567-e89b-42d3-a456-426614174001",
|
|
|
|
|
ManifestDigest: strings.Repeat("4", 64), Debian12NetworkFix: true,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
for _, testCase := range tests {
|
|
|
|
|
t.Run(testCase.name, func(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
executor := &fakeMutationExecutor{result: json.RawMessage(`{"unexpected":true}`)}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(testCase.request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: testCase.operationID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost, testCase.path, body, actor)
|
|
|
|
|
if response.Code != http.StatusUnprocessableEntity ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"`+testCase.errorCode+`"`)) {
|
|
|
|
|
t.Fatalf("hostname replacement response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Lock()
|
|
|
|
|
calls := executor.calls
|
|
|
|
|
executor.mu.Unlock()
|
|
|
|
|
if calls != 0 {
|
|
|
|
|
t.Fatalf("hostname replacement dispatched %d worker calls", calls)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func apiTestPlatformPolicy(t *testing.T) hypervisor.PlatformNetworkPolicySnapshot {
|
|
|
|
|
t.Helper()
|
|
|
|
|
rules := []hypervisor.PlatformNetworkPolicyRule{{
|
|
|
|
|
ID: "ssh", Label: "SSH", Category: "default", Protocols: []string{"tcp"},
|
|
|
|
|
PortFrom: 22, PortTo: 22, Directions: []string{"ingress"}, Reason: "default SSH policy",
|
|
|
|
|
}}
|
|
|
|
|
document := struct {
|
|
|
|
|
SchemaVersion uint64 `json:"schema_version"`
|
|
|
|
|
BasePolicyVersion uint64 `json:"base_policy_version"`
|
|
|
|
|
CatalogRevision uint64 `json:"catalog_revision"`
|
|
|
|
|
Rules []hypervisor.PlatformNetworkPolicyRule `json:"rules"`
|
|
|
|
|
}{SchemaVersion: 1, BasePolicyVersion: 1, CatalogRevision: 1, Rules: rules}
|
|
|
|
|
encoded, err := json.Marshal(document)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
digest := sha256.Sum256(encoded)
|
|
|
|
|
return hypervisor.PlatformNetworkPolicySnapshot{
|
|
|
|
|
SchemaVersion: 1, BasePolicyVersion: 1, CatalogRevision: 1,
|
|
|
|
|
Rules: rules, Digest: hex.EncodeToString(digest[:]),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func containsTestString(values []string, want string) bool {
|
|
|
|
|
for _, value := range values {
|
|
|
|
|
if value == want {
|
|
|
|
|
return true
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestResizeResourcesPersistsExactSourceTargetAndAdmission(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
const uuid = "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
|
request := hypervisor.ResizeResourcesRequest{
|
|
|
|
|
OperationID: "operation-resize-resources",
|
|
|
|
|
FromVCPUs: 2, TargetVCPUs: 4,
|
|
|
|
|
FromMemoryMiB: 2048, TargetMemoryMiB: 4096,
|
|
|
|
|
FromDiskGiB: 40, TargetDiskGiB: 80,
|
|
|
|
|
FromBandwidthMbps: 100, TargetBandwidthMbps: 200,
|
|
|
|
|
ConfirmShutdown: true,
|
|
|
|
|
}
|
|
|
|
|
result := hypervisor.ResizeResourcesResult{
|
|
|
|
|
UUID: uuid,
|
|
|
|
|
FromVCPUs: 2, TargetVCPUs: 4, AppliedVCPUs: 4,
|
|
|
|
|
FromMemoryMiB: 2048, TargetMemoryMiB: 4096, AppliedMemoryMiB: 4096,
|
|
|
|
|
FromDiskGiB: 40, TargetDiskGiB: 80, AppliedDiskGiB: 80,
|
|
|
|
|
FromBandwidthMbps: 100, TargetBandwidthMbps: 200, AppliedBandwidthMbps: 200,
|
|
|
|
|
Status: hypervisor.VMStateShutoff, OriginallyRunning: false,
|
|
|
|
|
PowerStateRestored: true, ResourcesResized: true,
|
|
|
|
|
}
|
|
|
|
|
encodedResult, err := json.Marshal(result)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
executor := &fakeMutationExecutor{result: encodedResult}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor := auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: request.OperationID,
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/resources", body, actor)
|
|
|
|
|
if response.Code != http.StatusAccepted {
|
|
|
|
|
t.Fatalf("resize response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
record := waitForTerminal(t, api.journal, request.OperationID)
|
|
|
|
|
if record.State != operation.StateSucceeded || record.Action != "resize-vm" || record.ResourceID != uuid || !record.Destructive ||
|
|
|
|
|
record.ExpectedState == nil || record.ExpectedState.Kind != operation.ExpectedResizeResources ||
|
|
|
|
|
record.ExpectedState.TransitionID != request.OperationID ||
|
|
|
|
|
record.ExpectedState.FromVCPUs != request.FromVCPUs || record.ExpectedState.TargetVCPUs != request.TargetVCPUs ||
|
|
|
|
|
record.ExpectedState.FromMemoryMiB != request.FromMemoryMiB || record.ExpectedState.TargetMemoryMiB != request.TargetMemoryMiB ||
|
|
|
|
|
record.ExpectedState.FromDiskGiB != request.FromDiskGiB || record.ExpectedState.TargetDiskGiB != request.TargetDiskGiB ||
|
|
|
|
|
record.ExpectedState.FromBandwidthMbps != request.FromBandwidthMbps || record.ExpectedState.TargetBandwidthMbps != request.TargetBandwidthMbps {
|
|
|
|
|
t.Fatalf("resize operation record = %#v", record)
|
|
|
|
|
}
|
|
|
|
|
wantLocks := []string{"domain:" + uuid, "volume:virt-cloud-managed/vm-" + uuid}
|
|
|
|
|
if !record.HeavyIO || !reflect.DeepEqual(record.LockKeys, wantLocks) {
|
|
|
|
|
t.Fatalf("resize admission = heavy=%v locks=%v, want heavy locks=%v", record.HeavyIO, record.LockKeys, wantLocks)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
request.OperationID = "operation-resize-unconfirmed"
|
|
|
|
|
request.ConfirmShutdown = false
|
|
|
|
|
body, err = json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actor.OperationID = request.OperationID
|
|
|
|
|
rejected := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/resources", body, actor)
|
|
|
|
|
if rejected.Code != http.StatusUnprocessableEntity ||
|
|
|
|
|
!bytes.Contains(rejected.Body.Bytes(), []byte(`"code":"shutdown_confirmation_required"`)) {
|
|
|
|
|
t.Fatalf("missing shutdown confirmation = %d %s", rejected.Code, rejected.Body.String())
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Lock()
|
|
|
|
|
calls := executor.calls
|
|
|
|
|
executor.mu.Unlock()
|
|
|
|
|
if calls != 1 {
|
|
|
|
|
t.Fatalf("invalid resize crossed worker boundary: calls=%d", calls)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
methodRejected := api.request(t, http.MethodPatch, "/v1/vms/"+uuid+"/resources", body, actor)
|
|
|
|
|
if methodRejected.Code != http.StatusMethodNotAllowed || methodRejected.Header().Get("Allow") != http.MethodPost ||
|
|
|
|
|
!bytes.Contains(methodRejected.Body.Bytes(), []byte(`"code":"method_not_allowed"`)) {
|
|
|
|
|
t.Fatalf("PATCH resize response = %d allow=%q %s", methodRejected.Code, methodRejected.Header().Get("Allow"), methodRejected.Body.String())
|
|
|
|
|
}
|
|
|
|
|
executor.mu.Lock()
|
|
|
|
|
calls = executor.calls
|
|
|
|
|
executor.mu.Unlock()
|
|
|
|
|
if calls != 1 {
|
|
|
|
|
t.Fatalf("PATCH resize crossed worker boundary: calls=%d", calls)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestIsolatedMutationDeadlineCapsOnlyStorageConfiguration(t *testing.T) {
|
|
|
|
|
server := &Server{OperationTimeout: 840 * time.Second}
|
|
|
|
|
if got := server.isolatedMutationDeadline("configure-storage"); got != 120*time.Second {
|
|
|
|
|
t.Fatalf("storage helper deadline = %s, want 120s", got)
|
|
|
|
|
}
|
|
|
|
|
for _, action := range []string{"create-vm", "delete-vm", "change-network"} {
|
|
|
|
|
if got := server.isolatedMutationDeadline(action); got != 840*time.Second {
|
|
|
|
|
t.Fatalf("%s helper deadline = %s, want global 840s", action, got)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
server.OperationTimeout = 30 * time.Second
|
|
|
|
|
if got := server.isolatedMutationDeadline("configure-storage"); got != 30*time.Second {
|
|
|
|
|
t.Fatalf("lower global deadline was widened to %s", got)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestIsolatedMutationAdmissionIsShortAndDoesNotStartSecondWorker(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
// Give the first request enough time to persist its independent Bolt
|
|
|
|
|
// journal under a loaded test host. The short timeout is the behavior under
|
|
|
|
|
// test only after that first worker demonstrably owns admission.
|
|
|
|
|
api.handler.AdmissionTimeout = time.Second
|
|
|
|
|
executor := &blockingMutationExecutor{
|
|
|
|
|
started: make(chan worker.Request, 1), release: make(chan struct{}),
|
|
|
|
|
}
|
|
|
|
|
if err := api.handler.SetMutationExecutor(executor); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
requestBody := func(operationID string) []byte {
|
|
|
|
|
body, err := json.Marshal(hypervisor.StorageConfigurationRequest{
|
|
|
|
|
OperationID: operationID, ManagedDirectory: "/srv/virt/managed",
|
|
|
|
|
TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds",
|
|
|
|
|
UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos",
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return body
|
|
|
|
|
}
|
|
|
|
|
firstID := "operation-admission-first"
|
|
|
|
|
firstBody := requestBody(firstID)
|
|
|
|
|
first := api.request(t, http.MethodPost, "/v1/storage/configuration", firstBody, auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: firstID,
|
|
|
|
|
})
|
|
|
|
|
if first.Code != http.StatusAccepted {
|
|
|
|
|
t.Fatalf("first response = %d %s", first.Code, first.Body.String())
|
|
|
|
|
}
|
|
|
|
|
firstWire := requireOperationResponseIdentity(t, first, firstID, "configure-storage", "host-storage",
|
|
|
|
|
operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", firstBody))
|
|
|
|
|
if firstWire.Status != operation.StateDispatching || firstWire.Replayed {
|
|
|
|
|
t.Fatalf("first accepted operation response = %#v", firstWire)
|
|
|
|
|
}
|
|
|
|
|
select {
|
|
|
|
|
case <-executor.started:
|
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
|
t.Fatal("first worker did not start")
|
|
|
|
|
}
|
|
|
|
|
api.handler.AdmissionTimeout = 30 * time.Millisecond
|
|
|
|
|
|
|
|
|
|
secondID := "operation-admission-second"
|
|
|
|
|
secondBody := requestBody(secondID)
|
|
|
|
|
startedAt := time.Now()
|
|
|
|
|
second := api.request(t, http.MethodPost, "/v1/storage/configuration", secondBody, auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: secondID,
|
|
|
|
|
})
|
|
|
|
|
if elapsed := time.Since(startedAt); elapsed > 500*time.Millisecond {
|
|
|
|
|
t.Fatalf("saturated admission waited %s", elapsed)
|
|
|
|
|
}
|
|
|
|
|
if second.Code != http.StatusOK ||
|
|
|
|
|
!bytes.Contains(second.Body.Bytes(), []byte(`"status":"failed"`)) ||
|
|
|
|
|
!bytes.Contains(second.Body.Bytes(), []byte(`"code":"operation_not_dispatched"`)) {
|
|
|
|
|
t.Fatalf("saturated admission response = %d %s", second.Code, second.Body.String())
|
|
|
|
|
}
|
|
|
|
|
failedWire := requireOperationResponseIdentity(t, second, secondID, "configure-storage", "host-storage",
|
|
|
|
|
operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", secondBody))
|
|
|
|
|
if failedWire.Status != operation.StateFailed || failedWire.Replayed {
|
|
|
|
|
t.Fatalf("failed operation response = %#v", failedWire)
|
|
|
|
|
}
|
|
|
|
|
if executor.calls.Load() != 1 {
|
|
|
|
|
t.Fatalf("saturated request started a second worker; calls=%d", executor.calls.Load())
|
|
|
|
|
}
|
|
|
|
|
close(executor.release)
|
|
|
|
|
if terminal := waitForTerminal(t, api.journal, firstID); terminal.State != operation.StateSucceeded {
|
|
|
|
|
t.Fatalf("first terminal = %#v", terminal)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestParentMutationNeverFallsBackWhenExecutorIsMissing(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
request := hypervisor.StorageConfigurationRequest{
|
|
|
|
|
OperationID: "operation-worker-missing",
|
|
|
|
|
ManagedDirectory: "/srv/virt/managed",
|
|
|
|
|
TemplateDirectory: "/srv/virt/templates",
|
|
|
|
|
SeedDirectory: "/srv/virt/seeds",
|
|
|
|
|
UserISODirectory: "/srv/virt/user-isos",
|
|
|
|
|
RescueDirectory: "/srv/virt/rescue-isos",
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
response := api.request(t, http.MethodPost, "/v1/storage/configuration", body, auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: request.OperationID,
|
|
|
|
|
})
|
|
|
|
|
if response.Code != http.StatusServiceUnavailable ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"code":"mutation_worker_unavailable"`)) ||
|
|
|
|
|
!bytes.Contains(response.Body.Bytes(), []byte(`"dispatch_state":"not_dispatched"`)) {
|
|
|
|
|
t.Fatalf("missing worker response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
if _, err := api.journal.Get(request.OperationID); !errors.Is(err, operation.ErrNotFound) {
|
|
|
|
|
t.Fatalf("missing worker created journal state: %v", err)
|
|
|
|
|
}
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
configureCalls := api.hv.configureCalls
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
if configureCalls != 0 {
|
|
|
|
|
t.Fatalf("parent invoked the in-process hypervisor %d times", configureCalls)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestTrustedWorkerExecutesLocallyWithoutRecursiveSpawn(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
request := hypervisor.StorageConfigurationRequest{
|
|
|
|
|
OperationID: "operation-trusted-worker",
|
|
|
|
|
ManagedDirectory: "/srv/virt/managed",
|
|
|
|
|
TemplateDirectory: "/srv/virt/templates",
|
|
|
|
|
SeedDirectory: "/srv/virt/seeds",
|
|
|
|
|
UserISODirectory: "/srv/virt/user-isos",
|
|
|
|
|
RescueDirectory: "/srv/virt/rescue-isos",
|
|
|
|
|
}
|
|
|
|
|
body, err := json.Marshal(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
httpRequest := httptest.NewRequest(http.MethodPost, "/v1/storage/configuration", bytes.NewReader(body))
|
|
|
|
|
response := httptest.NewRecorder()
|
|
|
|
|
api.handler.ServeTrustedWorkerRequest(response, httpRequest, body, auth.Actor{
|
|
|
|
|
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
|
|
|
|
|
OperationID: request.OperationID,
|
|
|
|
|
})
|
|
|
|
|
if response.Code != http.StatusOK {
|
|
|
|
|
t.Fatalf("trusted worker response = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
succeeded := requireOperationResponseIdentity(t, response, request.OperationID, "configure-storage", "host-storage",
|
|
|
|
|
operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", body))
|
|
|
|
|
if succeeded.Status != operation.StateSucceeded || succeeded.Replayed {
|
|
|
|
|
t.Fatalf("successful operation response = %#v", succeeded)
|
|
|
|
|
}
|
|
|
|
|
api.hv.mu.Lock()
|
|
|
|
|
configureCalls := api.hv.configureCalls
|
|
|
|
|
api.hv.mu.Unlock()
|
|
|
|
|
if configureCalls != 1 {
|
|
|
|
|
t.Fatalf("trusted worker local mutation calls=%d", configureCalls)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestListEnvelopeRemainsNonNull(t *testing.T) {
|
|
|
|
|
api := newTestAPI(t)
|
|
|
|
|
response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1})
|
|
|
|
|
if response.Code != http.StatusOK || response.Body.String() != "{\"vms\":[]}\n" {
|
|
|
|
|
t.Fatalf("VM list = %d %s", response.Code, response.Body.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func waitForTerminal(t *testing.T, journal *operation.Journal, operationID string) operation.Record {
|
|
|
|
|
t.Helper()
|
|
|
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
|
|
|
for time.Now().Before(deadline) {
|
|
|
|
|
record, err := journal.Get(operationID)
|
|
|
|
|
if err == nil && (record.State == operation.StateSucceeded || record.State == operation.StateFailed) {
|
|
|
|
|
return record
|
|
|
|
|
}
|
|
|
|
|
time.Sleep(time.Millisecond)
|
|
|
|
|
}
|
|
|
|
|
t.Fatalf("operation %s did not become terminal", operationID)
|
|
|
|
|
return operation.Record{}
|
|
|
|
|
}
|