package api import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "path/filepath" "reflect" "slices" "strings" "sync" "sync/atomic" "testing" "time" "git.luoli.live/root/virt/agent/auth" consolebroker "git.luoli.live/root/virt/agent/console" "git.luoli.live/root/virt/agent/hypervisor" "git.luoli.live/root/virt/agent/objecttransfer" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/publicnetwork" "git.luoli.live/root/virt/agent/vpcrouting" "git.luoli.live/root/virt/agent/worker" ) type testHypervisor struct { hypervisor.Unsupported mu sync.Mutex inspections int configureCalls int cleanupPeerTargetCalls int cleanupPeerTargetLast hypervisor.CleanupPeerMigrationTargetRequest cleanupBackupCalls int cleanupBackupLast hypervisor.CleanupBackupWorkspaceRequest inspectionResult *hypervisor.OperationInspection inspectionErr error backupReadiness hypervisor.BackupReadiness backupReadinessErr error metrics hypervisor.Metrics metricsErr error } type apiFailure struct { code string effect bool safe bool fence bool } func (e apiFailure) Error() string { return e.code } func (e apiFailure) PublicCode() string { return e.code } func (e apiFailure) PublicMessage() string { return e.code } func (e apiFailure) FailureSemantics() (bool, bool) { return e.effect, e.safe } func (e apiFailure) RequiresExecutionFence() bool { return e.fence } func (h *testHypervisor) Ping(context.Context) error { return nil } func (h *testHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) { return []hypervisor.VMMetadata{}, nil } func (h *testHypervisor) GetBackupReadiness(context.Context, string) (hypervisor.BackupReadiness, error) { return h.backupReadiness, h.backupReadinessErr } func (h *testHypervisor) GetMetrics(context.Context, string) (hypervisor.Metrics, error) { return h.metrics, h.metricsErr } func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageConfigurationRequest) (hypervisor.StorageConfigurationResult, error) { h.mu.Lock() h.configureCalls++ h.mu.Unlock() return hypervisor.StorageConfigurationResult{Configured: true}, nil } func (h *testHypervisor) CleanupPeerMigrationTarget(_ context.Context, _ string, request hypervisor.CleanupPeerMigrationTargetRequest) (hypervisor.DeleteVMResult, error) { h.mu.Lock() h.cleanupPeerTargetCalls++ h.cleanupPeerTargetLast = request h.mu.Unlock() return hypervisor.DeleteVMResult{ Deleted: true, ManagedArtifactsAbsent: true, Forced: false, }, nil } func (h *testHypervisor) CleanupBackupWorkspace(_ context.Context, _ string, request hypervisor.CleanupBackupWorkspaceRequest) (hypervisor.CleanupBackupWorkspaceResult, error) { h.mu.Lock() h.cleanupBackupCalls++ h.cleanupBackupLast = request h.mu.Unlock() parts := strings.Split(request.PriorOperation.Action, ":") key := "" if len(parts) == 3 { key = parts[1] } return hypervisor.CleanupBackupWorkspaceResult{ BackupKey: key, Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true, }, nil } func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) { h.mu.Lock() defer h.mu.Unlock() h.inspections++ if h.inspectionErr != nil { return hypervisor.OperationInspection{}, h.inspectionErr } if h.inspectionResult != nil { inspection := *h.inspectionResult inspection.EffectProjection = append(json.RawMessage(nil), inspection.EffectProjection...) return inspection, nil } return hypervisor.OperationInspection{ OperationID: record.OperationID, OperationStatus: record.State, CheckedAt: time.Now().Unix(), Outcome: hypervisor.InspectionOutcomeFailed, SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "result_unconfirmed", EffectResolution: hypervisor.InspectionEffectInconclusive, FenceResolutionRequired: record.State == operation.StateFailed && record.EffectMayHaveApplied, }, nil } type fakeMutationExecutor struct { mu sync.Mutex calls int requests []worker.Request result json.RawMessage err error deadline time.Time } type blockingMutationExecutor struct { started chan worker.Request release chan struct{} calls atomic.Int32 } func (e *blockingMutationExecutor) Execute(ctx context.Context, request worker.Request) (json.RawMessage, error) { e.calls.Add(1) select { case e.started <- request: case <-ctx.Done(): return nil, ctx.Err() } select { case <-e.release: return json.RawMessage(`{"configured":true}`), nil case <-ctx.Done(): return nil, ctx.Err() } } func (e *fakeMutationExecutor) Execute(ctx context.Context, request worker.Request) (json.RawMessage, error) { e.mu.Lock() defer e.mu.Unlock() e.calls++ e.requests = append(e.requests, request) if deadline, ok := ctx.Deadline(); ok { e.deadline = deadline } return append(json.RawMessage(nil), e.result...), e.err } type testAPI struct { handler *Server hv *testHypervisor journal *operation.Journal secret []byte now time.Time nonce int } func newTestAPI(t *testing.T) *testAPI { t.Helper() journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db")) if err != nil { t.Fatal(err) } secret := []byte("0123456789abcdef0123456789abcdef") now := time.Unix(2_000_000_000, 0) journal.SetClockForTesting(func() time.Time { return now }) authenticator, err := auth.NewWithNonceStore(secret, 5*time.Minute, 10*time.Minute, journal) if err != nil { t.Fatal(err) } authenticator.SetClockForTesting(func() time.Time { return now }) broker, err := consolebroker.NewBroker(time.Minute) if err != nil { t.Fatal(err) } hv := &testHypervisor{} handler, err := NewServer(authenticator, journal, hv, broker, 64<<10, time.Second) if err != nil { t.Fatal(err) } t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() _ = handler.Shutdown(ctx) _ = journal.Wait(ctx) _ = journal.Close() }) return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now} } func (a *testAPI) request(t *testing.T, method, target string, body []byte, actor auth.Actor) *httptest.ResponseRecorder { t.Helper() a.nonce++ request := httptest.NewRequest(method, target, bytes.NewReader(body)) nonce := fmt.Sprintf("nonce%011d", a.nonce) if err := auth.Sign(request, body, actor, a.secret, a.now.Unix(), nonce); err != nil { t.Fatal(err) } response := httptest.NewRecorder() a.handler.ServeHTTP(response, request) return response } func TestLegacyDailyTrafficRouteIsRemoved(t *testing.T) { api := newTestAPI(t) response := api.request(t, http.MethodGet, "/v1/vms/123e4567-e89b-42d3-a456-426614174000/traffic/daily?days=30", nil, auth.Actor{UserID: 1, CanManage: true}) if response.Code != http.StatusNotFound { t.Fatalf("legacy daily traffic status=%d body=%s", response.Code, response.Body.String()) } } func TestMetricsRoutePublishesPublicInterfaceHealthCounters(t *testing.T) { api := newTestAPI(t) rxPackets := uint64(101) txPackets := uint64(202) rxErrors := uint64(3) txErrors := uint64(4) rxDrops := uint64(5) txDrops := uint64(6) api.hv.metrics = hypervisor.Metrics{ UUID: "123e4567-e89b-42d3-a456-426614174000", RXBytes: 1000, TXBytes: 2000, RXPackets: &rxPackets, TXPackets: &txPackets, RXErrors: &rxErrors, TXErrors: &txErrors, RXDrops: &rxDrops, TXDrops: &txDrops, } response := api.request(t, http.MethodGet, "/v1/vms/123e4567-e89b-42d3-a456-426614174000/metrics", nil, auth.Actor{UserID: 1, CanManage: true}) if response.Code != http.StatusOK { t.Fatalf("status = %d, body = %s", response.Code, response.Body.String()) } var payload map[string]any if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil { t.Fatal(err) } for field, expected := range map[string]float64{ "rx_packets": 101, "tx_packets": 202, "rx_errors": 3, "tx_errors": 4, "rx_drops": 5, "tx_drops": 6, } { if payload[field] != expected { t.Fatalf("%s = %#v", field, payload[field]) } } if payload["rx_bytes"] != float64(1000) || payload["tx_bytes"] != float64(2000) { t.Fatalf("durable byte counters changed shape: %#v", payload) } } func TestBackupReadinessIsBoundedReadOnlyEvidence(t *testing.T) { api := newTestAPI(t) api.hv.backupReadiness = hypervisor.BackupReadiness{ Eligible: false, Retryable: true, Reason: "backup_rescue_active", } const uuid = "123e4567-e89b-42d3-a456-426614174000" response := api.request(t, http.MethodGet, "/v1/vms/"+uuid+"/backup-readiness", nil, auth.Actor{UserID: 1, CanManage: true}) if response.Code != http.StatusOK { t.Fatalf("backup readiness status=%d body=%s", response.Code, response.Body.String()) } var readiness hypervisor.BackupReadiness if err := json.Unmarshal(response.Body.Bytes(), &readiness); err != nil { t.Fatal(err) } if readiness != api.hv.backupReadiness { t.Fatalf("backup readiness=%#v want=%#v", readiness, api.hv.backupReadiness) } } func TestBackupReadinessRequiresManageCapability(t *testing.T) { api := newTestAPI(t) const uuid = "123e4567-e89b-42d3-a456-426614174000" response := api.request(t, http.MethodGet, "/v1/vms/"+uuid+"/backup-readiness", nil, auth.Actor{UserID: 1}) if response.Code != http.StatusForbidden { t.Fatalf("backup readiness without manage=%d body=%s", response.Code, response.Body.String()) } } func requireOperationResponseIdentity(t *testing.T, response *httptest.ResponseRecorder, operationID, action, resourceID, requestFingerprint string) operationResponse { t.Helper() var wire operationResponse if err := json.Unmarshal(response.Body.Bytes(), &wire); err != nil { t.Fatalf("decode operation response %q: %v body=%s", operationID, err, response.Body.String()) } if wire.OperationID != operationID || wire.Action != action || wire.ResourceID != resourceID || wire.RequestFingerprint != requestFingerprint { t.Fatalf("operation identity = %#v, want operation=%q action=%q resource=%q fingerprint=%q", wire, operationID, action, resourceID, requestFingerprint) } var raw map[string]json.RawMessage if err := json.Unmarshal(response.Body.Bytes(), &raw); err != nil { t.Fatal(err) } if _, legacy := raw["fingerprint"]; legacy { t.Fatalf("legacy fingerprint key leaked into operation wire: %s", response.Body.String()) } return wire } func TestVPCNetworkMutationUsesCanonicalDigestAndDurableExpectedState(t *testing.T) { api := newTestAPI(t) const ( vpcKey = "0123456789abcdef0123456789abcdef" operationID = "operation-vpc-network-apply" ) network := vpcrouting.Network{ VPCKey: vpcKey, DesiredRevision: 4, CIDR: "10.88.0.0/24", Gateway: "10.88.0.1", Transport: vpcrouting.Transport{ AddressCIDR: "172.16.50.11/24", Interface: "br-private", MTU: 1500, }, LocalPorts: []vpcrouting.LocalPort{{IP: "10.88.0.2", MAC: "02:00:00:00:00:02"}}, PeerRoutes: []vpcrouting.PeerRoute{{IP: "10.88.0.3", Via: "172.16.50.12"}}, } digest, err := vpcrouting.Digest(network) if err != nil { t.Fatal(err) } request := vpcrouting.ApplyRequest{OperationID: operationID, DesiredDigest: digest, Network: network} body, err := json.Marshal(request) if err != nil { t.Fatal(err) } bridge, _ := vpcrouting.BridgeName(vpcKey) result, _ := json.Marshal(vpcrouting.Result{ VPCKey: vpcKey, DesiredRevision: 4, DesiredDigest: digest, Bridge: bridge, Applied: true, }) executor := &fakeMutationExecutor{result: result} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } response := api.request(t, http.MethodPost, "/v1/vpc/networks/"+vpcKey+"/apply", body, actor) if response.Code != http.StatusAccepted && response.Code != http.StatusOK { t.Fatalf("VPC apply status=%d body=%s", response.Code, response.Body.String()) } record := waitForTerminal(t, api.journal, operationID) if record.State != operation.StateSucceeded || record.Action != "apply-vpc-network:"+vpcKey || record.ResourceID != "vpc:"+vpcKey || record.ExpectedState == nil || record.ExpectedState.Kind != operation.ExpectedVPCNetwork || record.ExpectedState.VPCKey != vpcKey || record.ExpectedState.VPCRevision != 4 || record.ExpectedState.VPCDigest != digest || !record.ExpectedState.VPCPresent { t.Fatalf("VPC operation record = %#v", record) } if len(record.LockKeys) != 2 || !slices.Contains(record.LockKeys, "catalog:vpc-host/state") || !slices.Contains(record.LockKeys, "catalog:vpc/"+vpcKey) { t.Fatalf("VPC operation locks = %#v", record.LockKeys) } executor.mu.Lock() defer executor.mu.Unlock() if executor.calls != 1 || executor.requests[0].RequestTarget != "/v1/vpc/networks/"+vpcKey+"/apply" { t.Fatalf("VPC worker requests = %#v", executor.requests) } } func TestVPCNetworkRepairCarriesExactSupersededGeneration(t *testing.T) { api := newTestAPI(t) const ( vpcKey = "0123456789abcdef0123456789abcdef" priorID = "operation-vpc-repair-prior" repairID = "operation-vpc-repair-next" ) network := vpcrouting.Network{ VPCKey: vpcKey, DesiredRevision: 9, CIDR: "10.89.0.0/24", Gateway: "10.89.0.1", Transport: vpcrouting.Transport{ AddressCIDR: "172.16.50.11/24", Interface: "br-private", MTU: 1500, }, LocalPorts: []vpcrouting.LocalPort{{IP: "10.89.0.2", MAC: "02:00:00:00:00:12"}}, } digest, err := vpcrouting.Digest(network) if err != nil { t.Fatal(err) } prior := operation.Spec{ OperationID: priorID, ResourceID: "vpc:" + vpcKey, Action: "apply-vpc-network:" + vpcKey, Fingerprint: operation.Fingerprint( http.MethodPost, "/v1/vpc/networks/"+vpcKey+"/apply", []byte(`{}`), ), Destructive: true, ExpectedState: &operation.ExpectedState{ Kind: operation.ExpectedVPCNetwork, TransitionID: priorID, VPCKey: vpcKey, VPCRevision: 9, VPCDigest: digest, VPCPresent: true, }, LockKeys: hypervisor.VPCNetworkAdmission(vpcKey).LockKeys, } if _, _, err := api.journal.Execute( context.Background(), prior, func(context.Context) (json.RawMessage, error) { return nil, apiFailure{ code: "vpc_reconcile_unconfirmed", effect: true, safe: false, fence: true, } }, ); err == nil { t.Fatal("VPC repair predecessor unexpectedly succeeded") } request := vpcrouting.ApplyRequest{ OperationID: repairID, DesiredDigest: digest, Network: network, VPCRepair: true, SupersedesOperationID: priorID, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } bridge, _ := vpcrouting.BridgeName(vpcKey) result, _ := json.Marshal(vpcrouting.Result{ VPCKey: vpcKey, DesiredRevision: 9, DesiredDigest: digest, Bridge: bridge, Applied: true, }) executor := &fakeMutationExecutor{result: result} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } response := api.request(t, http.MethodPost, "/v1/vpc/networks/"+vpcKey+"/apply/repair", body, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: repairID, }, ) if response.Code != http.StatusAccepted && response.Code != http.StatusOK { t.Fatalf("VPC repair status=%d body=%s", response.Code, response.Body.String()) } record := waitForTerminal(t, api.journal, repairID) if record.State != operation.StateSucceeded || record.SupersedesOperationID != priorID { t.Fatalf("VPC repair record = %#v", record) } executor.mu.Lock() defer executor.mu.Unlock() if executor.calls != 1 || len(executor.requests) != 1 || executor.requests[0].SupersededOperation == nil || executor.requests[0].SupersededOperation.OperationID != priorID || executor.requests[0].SupersededExpectedState == nil || executor.requests[0].SupersededExpectedState.VPCDigest != digest { t.Fatalf("VPC repair worker envelope = %#v", executor.requests) } } func TestVPCPortMutationBindsGuestAndHostProjectionIdentity(t *testing.T) { api := newTestAPI(t) const ( uuid = "123e4567-e89b-42d3-a456-426614174000" vpcKey = "0123456789abcdef0123456789abcdef" portKey = "1123456789abcdef0123456789abcdef" operationID = "operation-vpc-port-apply" digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" ) request := hypervisor.VPCPortRequest{ OperationID: operationID, PortKey: portKey, VPCKey: vpcKey, PrivateIP: "10.60.0.9", CIDR: "10.60.0.0/24", Gateway: "10.60.0.1", MAC: "02:00:00:00:00:09", HostRevision: 7, HostDigest: digest, ConfirmRestart: true, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } bridge, err := vpcrouting.BridgeName(vpcKey) if err != nil { t.Fatal(err) } result, _ := json.Marshal(hypervisor.VPCPortResult{ UUID: uuid, PortKey: portKey, VPCKey: vpcKey, Status: hypervisor.VMStateRunning, Attached: true, }) executor := &fakeMutationExecutor{result: result} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/vpc-port", body, actor) if response.Code != http.StatusAccepted && response.Code != http.StatusOK { t.Fatalf("VPC port apply status=%d body=%s", response.Code, response.Body.String()) } record := waitForTerminal(t, api.journal, operationID) if record.State != operation.StateSucceeded || record.Action != "apply-vpc-port" || record.ResourceID != uuid || record.ExpectedState == nil || record.ExpectedState.Kind != operation.ExpectedVPCPort || record.ExpectedState.VPCPortKey != portKey || record.ExpectedState.VPCKey != vpcKey || record.ExpectedState.VPCPrivateIP != request.PrivateIP || record.ExpectedState.VPCCIDR != request.CIDR || record.ExpectedState.VPCGateway != request.Gateway || record.ExpectedState.VPCMAC != request.MAC || record.ExpectedState.VPCBridge != bridge || record.ExpectedState.VPCRevision != request.HostRevision || record.ExpectedState.VPCDigest != digest || !record.ExpectedState.VPCPresent { t.Fatalf("VPC port operation record = %#v", record) } for _, required := range []string{ "catalog:vpc-port/" + portKey, "domain:" + uuid, } { if !slices.Contains(record.LockKeys, required) { t.Fatalf("VPC port locks missing %q: %#v", required, record.LockKeys) } } for _, global := range []string{ "catalog:vpc-host/state", "catalog:vpc/" + vpcKey, } { if slices.Contains(record.LockKeys, global) { t.Fatalf("VPC port operation retained global lock %q: %#v", global, record.LockKeys) } } } func TestExitRescueRequiresDataLossAcknowledgement(t *testing.T) { api := newTestAPI(t) operationID := "exit-rescue-without-ack" actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } body := []byte(`{"operation_id":"exit-rescue-without-ack","network_suspended":false}`) response := api.request(t, http.MethodPost, "/v1/vms/123e4567-e89b-42d3-a456-426614174000/rescue/exit", body, actor) if response.Code != http.StatusUnprocessableEntity || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"data_loss_ack_required"`)) { t.Fatalf("exit rescue without acknowledgement = %d %s", response.Code, response.Body.String()) } } func TestOperationRecoveryMutationRoutesAreRemoved(t *testing.T) { api := newTestAPI(t) operationID := "operation-terminal-inspect" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-inspect", Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-inspect", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "result was not confirmed", EffectMayHaveApplied: true, } }); err == nil { t.Fatal("seed failure unexpectedly succeeded") } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID, nil, actor); response.Code != http.StatusOK { t.Fatalf("operation GET = %d %s", response.Code, response.Body.String()) } inspectionResponse := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor) if inspectionResponse.Code != http.StatusOK { t.Fatalf("read-only inspect = %d %s", inspectionResponse.Code, inspectionResponse.Body.String()) } var inspection hypervisor.OperationInspection if err := json.Unmarshal(inspectionResponse.Body.Bytes(), &inspection); err != nil { t.Fatal(err) } if inspection.Action != spec.Action || inspection.ResourceID != spec.ResourceID || inspection.RequestFingerprint != spec.Fingerprint { t.Fatalf("inspection identity = %#v, want action=%q resource=%q fingerprint=%q", inspection, spec.Action, spec.ResourceID, spec.Fingerprint) } for _, suffix := range []string{"reconcile", "confirm-inspection", "abort-job", "cleanup"} { response := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/"+suffix, []byte(`{}`), actor) if response.Code != http.StatusNotFound { t.Fatalf("removed %s route = %d %s", suffix, response.Code, response.Body.String()) } } api.hv.mu.Lock() inspections := api.hv.inspections api.hv.mu.Unlock() if inspections != 1 { t.Fatalf("removed mutation routes reached hypervisor; inspections=%d", inspections) } } func TestOperationInspectionReportsCompletionDeliveryQuarantine(t *testing.T) { api := newTestAPI(t) operationID := "operation-delivery-quarantine" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-delivery-quarantine", Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-delivery-quarantine", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return json.RawMessage(`{"nested":{"root_password":"must-not-be-delivered"}}`), nil }); err != nil { t.Fatal(err) } if claimed, err := api.journal.ClaimPendingCompletions(1); err != nil || len(claimed) != 0 { t.Fatalf("sensitive completion was not quarantined: %#v err=%v", claimed, err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor) if response.Code != http.StatusOK { t.Fatalf("quarantined inspection = %d %s", response.Code, response.Body.String()) } var inspection hypervisor.OperationInspection if err := json.Unmarshal(response.Body.Bytes(), &inspection); err != nil { t.Fatal(err) } if inspection.OperationStatus != operation.StateSucceeded || inspection.ErrorCode != "" || inspection.Outcome != hypervisor.InspectionOutcomeUncertain || inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete || inspection.EffectResolution != hypervisor.InspectionEffectInconclusive || inspection.CompletionDeliveryState != operation.CompletionDeliveryStateQuarantined || inspection.CompletionDeliveryErrorCode != operation.CompletionDeliveryErrorReceiptInvalid || len(inspection.EffectProjection) != 0 || inspection.AbortJob.Available || inspection.Cleanup.Available { t.Fatalf("quarantined completion inspection = %#v", inspection) } api.hv.mu.Lock() inspections := api.hv.inspections api.hv.mu.Unlock() if inspections != 0 { t.Fatalf("delivery quarantine reporting depended on %d live host inspections", inspections) } stored, err := api.journal.Get(operationID) if err != nil || stored.State != operation.StateSucceeded || stored.Error != nil { t.Fatalf("delivery quarantine changed operation status: %#v err=%v", stored, err) } } func TestSuperAdminFenceResolutionIsJournalOnlyAndReadBackIsStable(t *testing.T) { api := newTestAPI(t) operationID := "operation-fence-resolution" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-fence-resolution", Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-resolution", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "result was not confirmed", EffectMayHaveApplied: true, ResubmitAllowed: false, } }); err == nil { t.Fatal("seed effectful failure unexpectedly succeeded") } actor := auth.Actor{ UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) { t.Fatalf("missing fence resolution = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK { t.Fatalf("fresh inspection = %d %s", response.Code, response.Body.String()) } request := operation.FenceResolutionRequest{ ResolutionID: "0123456789abcdef0123456789abcdef", Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } missingIdentity := request missingIdentity.RequestFingerprint = "" missingIdentityBody, err := json.Marshal(missingIdentity) if err != nil { t.Fatal(err) } invalidIdentity := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", missingIdentityBody, actor) if invalidIdentity.Code != http.StatusBadRequest || !bytes.Contains(invalidIdentity.Body.Bytes(), []byte(`"code":"invalid_fence_resolution"`)) { t.Fatalf("missing fence identity = %d %s", invalidIdentity.Code, invalidIdentity.Body.String()) } mismatch := request mismatch.ResourceID = "vm-other-fence-resolution" mismatchBody, err := json.Marshal(mismatch) if err != nil { t.Fatal(err) } identityConflict := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", mismatchBody, actor) if identityConflict.Code != http.StatusConflict || !bytes.Contains(identityConflict.Body.Bytes(), []byte(`"code":"operation_identity_conflict"`)) { t.Fatalf("fence identity conflict = %d %s", identityConflict.Code, identityConflict.Body.String()) } resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"replayed":false`)) { t.Fatalf("resolve fence = %d %s", resolved.Code, resolved.Body.String()) } readBack := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor) if readBack.Code != http.StatusOK || !bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution_id":"0123456789abcdef0123456789abcdef"`)) || !bytes.Contains(readBack.Body.Bytes(), []byte(`"resolution":"not_applied"`)) { t.Fatalf("read fence resolution = %d %s", readBack.Code, readBack.Body.String()) } replayed := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if replayed.Code != http.StatusOK || !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) { t.Fatalf("idempotent fence resolution = %d %s", replayed.Code, replayed.Body.String()) } api.hv.mu.Lock() inspections := api.hv.inspections configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if inspections != 1 || configureCalls != 0 { t.Fatalf("fence resolution invoked host work: inspections=%d mutations=%d", inspections, configureCalls) } } func TestFenceResolutionRequiresExactActorOperationIdentity(t *testing.T) { api := newTestAPI(t) request := operation.FenceResolutionRequest{ ResolutionID: "fedcba9876543210fedcba9876543210", Resolution: operation.FenceResolutionApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Action: "delete-vm", ResourceID: "vm-fence-identity", RequestFingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/vm-fence-identity", []byte(`{}`)), } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } response := api.request(t, http.MethodPost, "/v1/operations/operation-fence-identity/resolve-fence", body, auth.Actor{ UserID: 9, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: "operation-other-identity", }) if response.Code != http.StatusBadRequest || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"operation_id_mismatch"`)) { t.Fatalf("identity mismatch = %d %s", response.Code, response.Body.String()) } } func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing.T) { api := newTestAPI(t) operationID := "operation-system-recovery" spec := operation.Spec{ OperationID: operationID, ResourceID: "vm-system-recovery", Action: "update-settings", Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms/vm-system-recovery/settings", []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "result was not confirmed", EffectMayHaveApplied: true, } }); err == nil { t.Fatal("seed effectful failure unexpectedly succeeded") } actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte(`"fence_resolution_required":true`)) || !bytes.Contains(response.Body.Bytes(), []byte(`"fence_state":"active"`)) { t.Fatalf("system inspect = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusNotFound || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"fence_resolution_not_found"`)) { t.Fatalf("system unresolved fence read = %d %s", response.Code, response.Body.String()) } request := operation.FenceResolutionRequest{ ResolutionID: "98769876987698769876987698769876", Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) { t.Fatalf("system fence resolution = %d %s", resolved.Code, resolved.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/fence-resolution", nil, actor); response.Code != http.StatusOK { t.Fatalf("system resolved fence read = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor); response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte(`"fence_resolution_required":false`)) || !bytes.Contains(response.Body.Bytes(), []byte(`"fence_state":"resolved"`)) { t.Fatalf("resolved system inspect = %d %s", response.Code, response.Body.String()) } for _, denied := range []struct { name, operationID, resourceID, action string }{ {"unknown", "operation-system-unknown-denied", "vm-system-unknown-denied", "future-unknown"}, } { t.Run("reject "+denied.name+" operation", func(t *testing.T) { deniedSpec := operation.Spec{ OperationID: denied.operationID, ResourceID: denied.resourceID, Action: denied.action, Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/test/"+denied.resourceID, []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), deniedSpec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{Code: "result_unconfirmed", Message: "effect may have applied", EffectMayHaveApplied: true} }); err == nil { t.Fatal("seed disallowed recovery failure unexpectedly succeeded") } deniedActor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: denied.operationID} for _, route := range []struct { method, suffix string body []byte }{ {http.MethodGet, "/inspect", nil}, {http.MethodGet, "/fence-resolution", nil}, {http.MethodPost, "/resolve-fence", body}, } { response := api.request(t, route.method, "/v1/operations/"+denied.operationID+route.suffix, route.body, deniedActor) if response.Code != http.StatusForbidden { t.Fatalf("system recovery %s %s = %d %s", route.method, route.suffix, response.Code, response.Body.String()) } } var successorCalls atomic.Int32 successor := operation.Spec{ OperationID: denied.operationID + "-successor", ResourceID: denied.resourceID, Action: "delete-vm", Fingerprint: operation.Fingerprint(http.MethodDelete, "/v1/vms/"+denied.resourceID, []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) { successorCalls.Add(1) return json.RawMessage(`{"deleted":true}`), nil }); !errors.Is(err, operation.ErrResourceBlocked) || successorCalls.Load() != 0 { t.Fatalf("denied system recovery released blocker: calls=%d err=%v", successorCalls.Load(), err) } }) } reviewedActions := []string{ "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy", "reinstall-vm", "enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended", "change-network", "network-suspend", "network-resume", "apply-firewall", "delete-vm", "force-delete-vm", "service-suspend", "cleanup-peer-migration-target:0123456789abcdef0123456789abcdef", "create-snapshot:0123456789abcdef0123456789abcdef:filesystem", "create-snapshot:0123456789abcdef0123456789abcdef:crash", "delete-snapshot:0123456789abcdef0123456789abcdef", "restore-snapshot:0123456789abcdef0123456789abcdef", "import-template", "update-template", "delete-template", "import-rescue-template", "delete-rescue-template", "import-user-iso", "delete-user-iso", } for index, action := range reviewedActions { t.Run("reviewed "+action, func(t *testing.T) { allowedOperationID := fmt.Sprintf("operation-system-reviewed-%02d", index) allowedResourceID := fmt.Sprintf("resource-system-reviewed-%02d", index) allowedSpec := operation.Spec{ OperationID: allowedOperationID, ResourceID: allowedResourceID, Action: action, Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/test/"+allowedResourceID, []byte(`{}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), allowedSpec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{Code: "result_unconfirmed", Message: "effect may have applied", EffectMayHaveApplied: true} }); err == nil { t.Fatal("seed reviewed recovery failure unexpectedly succeeded") } allowedActor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: allowedOperationID} if response := api.request(t, http.MethodGet, "/v1/operations/"+allowedOperationID+"/inspect", nil, allowedActor); response.Code != http.StatusOK { t.Fatalf("reviewed system inspect = %d %s", response.Code, response.Body.String()) } allowedRequest := operation.FenceResolutionRequest{ ResolutionID: fmt.Sprintf("%032x", index+1), Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", Action: allowedSpec.Action, ResourceID: allowedSpec.ResourceID, RequestFingerprint: allowedSpec.Fingerprint, } allowedBody, err := json.Marshal(allowedRequest) if err != nil { t.Fatal(err) } if response := api.request(t, http.MethodPost, "/v1/operations/"+allowedOperationID+"/resolve-fence", allowedBody, allowedActor); response.Code != http.StatusOK { t.Fatalf("reviewed system resolution = %d %s", response.Code, response.Body.String()) } if response := api.request(t, http.MethodGet, "/v1/operations/"+allowedOperationID+"/fence-resolution", nil, allowedActor); response.Code != http.StatusOK { t.Fatalf("reviewed system fence read = %d %s", response.Code, response.Body.String()) } }) } for _, test := range []struct { name, method, target string body []byte }{ {"health", http.MethodGet, "/v1/health", nil}, {"raw operation", http.MethodGet, "/v1/operations/" + operationID, nil}, {"other operation", http.MethodGet, "/v1/operations/operation-other-recovery/inspect", nil}, {"host mutation", http.MethodPost, "/v1/storage/configuration", []byte(`{"operation_id":"operation-system-recovery"}`)}, {"create mutation", http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-recovery"}`)}, } { t.Run(test.name, func(t *testing.T) { response := api.request(t, test.method, test.target, test.body, actor) if response.Code != http.StatusForbidden { t.Fatalf("least-privilege route = %d %s", response.Code, response.Body.String()) } }) } manualZero := auth.Actor{ UserID: 0, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: operationID, } if response := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, manualZero); response.Code != http.StatusForbidden { t.Fatalf("zero-id human inspection = %d %s", response.Code, response.Body.String()) } api.hv.mu.Lock() inspections := api.hv.inspections configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if inspections != 2+len(reviewedActions) || configureCalls != 0 { t.Fatalf("system recovery escaped read-only scope: inspections=%d mutations=%d", inspections, configureCalls) } } func TestSystemCreateRecoveryRequiresFreshCompleteAbsenceEvidence(t *testing.T) { api := newTestAPI(t) operationID := "operation-system-create-recovery" resourceID := "123e4567-e89b-42d3-a456-426614174000" requestBody := []byte(`{"operation_id":"operation-system-create-recovery"}`) spec := operation.Spec{ OperationID: operationID, ResourceID: resourceID, Action: "create-vm", Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", requestBody), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "result_unconfirmed", Message: "create result was not confirmed", EffectMayHaveApplied: true, } }); err == nil { t.Fatal("seed create failure unexpectedly succeeded") } actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} request := operation.FenceResolutionRequest{ ResolutionID: "40404040404040404040404040404040", Resolution: operation.FenceResolutionApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint, } encodeRequest := func() []byte { t.Helper() body, err := json.Marshal(request) if err != nil { t.Fatal(err) } return body } assertActive := func() { t.Helper() state, err := api.journal.FenceState(operationID) if err != nil || state != operation.FenceStateActive { t.Fatalf("create fence state=%q err=%v", state, err) } } applied := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if applied.Code != http.StatusConflict { t.Fatalf("system create applied resolution = %d %s", applied.Code, applied.Body.String()) } assertActive() request.Resolution = operation.FenceResolutionNotApplied inconclusive := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if inconclusive.Code != http.StatusConflict || !bytes.Contains(inconclusive.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { t.Fatalf("inconclusive create resolution = %d %s", inconclusive.Code, inconclusive.Body.String()) } assertActive() exactInspection := func(checkedAt int64) hypervisor.OperationInspection { return hypervisor.OperationInspection{ OperationID: operationID, Action: spec.Action, ResourceID: resourceID, RequestFingerprint: spec.Fingerprint, OperationStatus: operation.StateFailed, CheckedAt: checkedAt, Outcome: hypervisor.InspectionOutcomeFailed, SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "creation_absent", EffectResolution: hypervisor.InspectionEffectNotApplied, EffectProjection: json.RawMessage( `{"creation_absent":true,"managed_artifacts_absent":true}`, ), } } stale := exactInspection(api.now.Add(-time.Second).Unix()) api.hv.mu.Lock() api.hv.inspectionResult = &stale api.hv.mu.Unlock() staleResponse := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if staleResponse.Code != http.StatusConflict || !bytes.Contains(staleResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { t.Fatalf("stale create resolution = %d %s", staleResponse.Code, staleResponse.Body.String()) } assertActive() incomplete := exactInspection(api.now.Unix()) incomplete.EffectProjection = json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":false}`) api.hv.mu.Lock() api.hv.inspectionResult = &incomplete api.hv.mu.Unlock() incompleteResponse := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if incompleteResponse.Code != http.StatusConflict || !bytes.Contains(incompleteResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { t.Fatalf("incomplete create absence = %d %s", incompleteResponse.Code, incompleteResponse.Body.String()) } assertActive() exact := exactInspection(api.now.Unix()) api.hv.mu.Lock() api.hv.inspectionResult = &exact api.hv.mu.Unlock() inspection := api.request( t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor, ) if inspection.Code != http.StatusOK || !bytes.Contains(inspection.Body.Bytes(), []byte(`"error_code":"creation_absent"`)) || !bytes.Contains(inspection.Body.Bytes(), []byte(`"effect_resolution":"not_applied"`)) { t.Fatalf("system create inspection = %d %s", inspection.Code, inspection.Body.String()) } resolved := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if resolved.Code != http.StatusOK || !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolution":"not_applied"`)) || !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) { t.Fatalf("system create not-applied resolution = %d %s", resolved.Code, resolved.Body.String()) } replayed := api.request( t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", encodeRequest(), actor, ) if replayed.Code != http.StatusOK || !bytes.Contains(replayed.Body.Bytes(), []byte(`"resolution":"not_applied"`)) || !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) { t.Fatalf("system create resolution replay = %d %s", replayed.Code, replayed.Body.String()) } api.hv.mu.Lock() inspections := api.hv.inspections configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if inspections != 5 || configureCalls != 0 { t.Fatalf("system create recovery inspections=%d mutations=%d", inspections, configureCalls) } } func TestSystemCreateRecoveryRejectsExecutionFencedAbsence(t *testing.T) { api := newTestAPI(t) operationID := "operation-system-create-unreaped" resourceID := "123e4567-e89b-42d3-a456-426614174002" spec := operation.Spec{ OperationID: operationID, ResourceID: resourceID, Action: "create-vm", Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-create-unreaped"}`)), Destructive: true, } if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{ Code: "worker_reap_unconfirmed", Message: "worker may still be running", EffectMayHaveApplied: true, ExecutionFenced: true, } }); err == nil { t.Fatal("seed execution-fenced create unexpectedly succeeded") } exactAbsence := hypervisor.OperationInspection{ OperationID: operationID, Action: spec.Action, ResourceID: resourceID, RequestFingerprint: spec.Fingerprint, OperationStatus: operation.StateFailed, CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeFailed, SummaryCode: hypervisor.InspectionSummaryFailed, ErrorCode: "creation_absent", EffectResolution: hypervisor.InspectionEffectNotApplied, EffectProjection: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`), } api.hv.mu.Lock() api.hv.inspectionResult = &exactAbsence api.hv.mu.Unlock() actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} inspected := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor) if inspected.Code != http.StatusOK { t.Fatalf("execution-fenced create inspection = %d %s", inspected.Code, inspected.Body.String()) } var inspection hypervisor.OperationInspection if err := json.Unmarshal(inspected.Body.Bytes(), &inspection); err != nil { t.Fatal(err) } if inspection.Outcome != hypervisor.InspectionOutcomeUncertain || inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete || inspection.EffectResolution != hypervisor.InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 { t.Fatalf("execution-fenced create absence was treated as final: %#v", inspection) } request := operation.FenceResolutionRequest{ ResolutionID: "50505050505050505050505050505050", Resolution: operation.FenceResolutionNotApplied, InspectionCheckedAt: api.now.Unix(), ReasonSHA256: "dededededededededededededededededededededededededededededededede", Action: spec.Action, ResourceID: spec.ResourceID, RequestFingerprint: spec.Fingerprint, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) if resolved.Code != http.StatusConflict || !bytes.Contains(resolved.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { t.Fatalf("execution-fenced create resolution = %d %s", resolved.Code, resolved.Body.String()) } if state, err := api.journal.FenceState(operationID); err != nil || state != operation.FenceStateActive { t.Fatalf("execution-fenced create was released state=%q err=%v", state, err) } api.hv.mu.Lock() inspections := api.hv.inspections api.hv.mu.Unlock() if inspections != 1 { t.Fatalf("execution-fenced create resolve repeated host inspection: %d", inspections) } } func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) { api := newTestAPI(t) executor := &fakeMutationExecutor{result: json.RawMessage(`{"configured":true}`)} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } request := hypervisor.StorageConfigurationRequest{ OperationID: "operation-isolated-storage", ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, } first := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor) if first.Code != http.StatusAccepted { t.Fatalf("first isolated response = %d %s", first.Code, first.Body.String()) } fingerprint := operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", body) accepted := requireOperationResponseIdentity(t, first, request.OperationID, "configure-storage", "host-storage", fingerprint) if accepted.Status != operation.StateDispatching || accepted.Replayed { t.Fatalf("accepted operation response = %#v", accepted) } terminal := waitForTerminal(t, api.journal, request.OperationID) if terminal.State != operation.StateSucceeded || string(terminal.Result) != `{"configured":true}` { t.Fatalf("terminal operation = %#v", terminal) } replay := api.request(t, http.MethodPost, "/v1/storage/configuration", body, actor) if replay.Code != http.StatusOK { t.Fatalf("terminal replay = %d %s", replay.Code, replay.Body.String()) } replayed := requireOperationResponseIdentity(t, replay, request.OperationID, "configure-storage", "host-storage", fingerprint) if replayed.Status != operation.StateSucceeded || !replayed.Replayed { t.Fatalf("replayed operation response = %#v", replayed) } executor.mu.Lock() defer executor.mu.Unlock() if executor.calls != 1 || len(executor.requests) != 1 { t.Fatalf("executor calls=%d requests=%d", executor.calls, len(executor.requests)) } workerRequest := executor.requests[0] if workerRequest.OperationID != request.OperationID || workerRequest.Actor.OperationID != request.OperationID || workerRequest.Method != http.MethodPost || workerRequest.RequestTarget != "/v1/storage/configuration" { t.Fatalf("worker envelope = %#v", workerRequest) } } func TestPeerMigrationTargetCleanupRequiresFreshExactPlanAndRetainsReceiveFence(t *testing.T) { api := newTestAPI(t) const ( uuid = "123e4567-e89b-42d3-a456-426614174000" migrationID = "0123456789abcdef0123456789abcdef" priorID = "operation-peer-receive-residue" cleanupID = "operation-peer-target-cleanup" ) prior := operation.Spec{ OperationID: priorID, ResourceID: uuid, Action: "receive-peer-migration:" + migrationID, Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms/"+uuid+"/peer-migration/receive", []byte(`{}`)), Destructive: true, LockKeys: hypervisor.ReceivePeerMigrationAdmission(hypervisor.ReceivePeerMigrationRequest{ CreateVMRequest: hypervisor.CreateVMRequest{UUID: uuid}, MigrationID: migrationID, }).LockKeys, } if _, _, err := api.journal.Execute(context.Background(), prior, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{Code: "result_unconfirmed", Message: "receive result unknown", EffectMayHaveApplied: true} }); err == nil { t.Fatal("seed receive failure unexpectedly succeeded") } stored, err := api.journal.Get(priorID) if err != nil { t.Fatal(err) } planID := strings.Repeat("a", 64) api.hv.inspectionResult = &hypervisor.OperationInspection{ OperationID: stored.OperationID, Action: stored.Action, ResourceID: stored.ResourceID, RequestFingerprint: stored.Fingerprint, OperationStatus: stored.State, CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeUncertain, SummaryCode: hypervisor.InspectionSummaryCleanupRequired, EffectResolution: hypervisor.InspectionEffectPartial, Cleanup: hypervisor.OperationCleanupPlan{Available: true, PlanID: planID, Steps: []string{hypervisor.CleanupStepDeleteManagedVolumes}}, } executor := &fakeMutationExecutor{result: json.RawMessage(`{"deleted":true,"managed_artifacts_absent":true,"forced":false}`)} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } request := hypervisor.CleanupPeerMigrationTargetRequest{ OperationID: cleanupID, MigrationID: migrationID, PriorOperationID: priorID, CleanupPlanID: planID, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: cleanupID} response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/peer-migration/cleanup-target", body, actor) if response.Code != http.StatusAccepted { t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String()) } terminal := waitForTerminal(t, api.journal, cleanupID) if terminal.State != operation.StateSucceeded || terminal.SupersedesOperationID != priorID { t.Fatalf("cleanup terminal = %#v", terminal) } executor.mu.Lock() if len(executor.requests) != 1 || executor.requests[0].SupersededOperation == nil || executor.requests[0].SupersededOperation.OperationID != priorID { executor.mu.Unlock() t.Fatalf("cleanup worker predecessor = %#v", executor.requests) } executor.mu.Unlock() if state, err := api.journal.FenceState(priorID); err != nil || state != operation.FenceStateActive { t.Fatalf("receive fence state=%q err=%v", state, err) } stale := request stale.OperationID = "operation-peer-target-cleanup-stale" stale.CleanupPlanID = strings.Repeat("b", 64) staleBody, _ := json.Marshal(stale) staleActor := actor staleActor.OperationID = stale.OperationID denied := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/peer-migration/cleanup-target", staleBody, staleActor) if denied.Code != http.StatusConflict || !bytes.Contains(denied.Body.Bytes(), []byte(`"code":"peer_migration_cleanup_plan_changed"`)) { t.Fatalf("stale cleanup = %d %s", denied.Code, denied.Body.String()) } } func TestTrustedWorkerPeerMigrationTargetCleanupUsesParentPredecessor(t *testing.T) { parent := newTestAPI(t) child := newTestAPI(t) const ( uuid = "123e4567-e89b-42d3-a456-426614174000" migrationID = "0123456789abcdef0123456789abcdef" priorID = "operation-peer-receive-worker-residue" cleanupID = "operation-peer-target-worker-cleanup" ) priorSpec := operation.Spec{ OperationID: priorID, ResourceID: uuid, Action: "receive-peer-migration:" + migrationID, Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms/"+uuid+"/peer-migration/receive", []byte(`{}`)), Destructive: true, LockKeys: hypervisor.ReceivePeerMigrationAdmission(hypervisor.ReceivePeerMigrationRequest{ CreateVMRequest: hypervisor.CreateVMRequest{UUID: uuid}, MigrationID: migrationID, }).LockKeys, } if _, _, err := parent.journal.Execute(context.Background(), priorSpec, func(context.Context) (json.RawMessage, error) { return nil, &worker.Error{Code: "result_unconfirmed", Message: "receive result unknown", EffectMayHaveApplied: true} }); err == nil { t.Fatal("seed receive failure unexpectedly succeeded") } prior, err := parent.journal.Get(priorID) if err != nil { t.Fatal(err) } planID := strings.Repeat("c", 64) child.hv.inspectionResult = &hypervisor.OperationInspection{ OperationID: prior.OperationID, Action: prior.Action, ResourceID: prior.ResourceID, RequestFingerprint: prior.Fingerprint, OperationStatus: prior.State, CheckedAt: child.now.Unix(), Outcome: hypervisor.InspectionOutcomeUncertain, SummaryCode: hypervisor.InspectionSummaryCleanupRequired, EffectResolution: hypervisor.InspectionEffectPartial, Cleanup: hypervisor.OperationCleanupPlan{Available: true, PlanID: planID, Steps: []string{hypervisor.CleanupStepDeleteManagedVolumes}}, } requestBody, err := json.Marshal(hypervisor.CleanupPeerMigrationTargetRequest{ OperationID: cleanupID, MigrationID: migrationID, PriorOperationID: priorID, CleanupPlanID: planID, }) if err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: cleanupID, } request := httptest.NewRequest(http.MethodPost, "/v1/vms/"+uuid+"/peer-migration/cleanup-target", bytes.NewReader(requestBody)) response := httptest.NewRecorder() child.handler.ServeTrustedWorkerRequest(response, request, requestBody, actor, prior) if response.Code != http.StatusOK { t.Fatalf("trusted cleanup response = %d %s", response.Code, response.Body.String()) } terminal, err := child.journal.Get(cleanupID) if err != nil { t.Fatal(err) } if terminal.State != operation.StateSucceeded || terminal.SupersedesOperationID != "" { t.Fatalf("trusted cleanup terminal = %#v", terminal) } child.hv.mu.Lock() defer child.hv.mu.Unlock() if child.hv.cleanupPeerTargetCalls != 1 || child.hv.cleanupPeerTargetLast.PriorOperation.OperationID != priorID { t.Fatalf("trusted cleanup calls=%d request=%#v", child.hv.cleanupPeerTargetCalls, child.hv.cleanupPeerTargetLast) } } func TestCreateBackupUsesDedicatedAsyncActionAndHeavyAdmission(t *testing.T) { api := newTestAPI(t) const ( uuid = "123e4567-e89b-42d3-a456-426614174000" backupKey = "0123456789abcdef0123456789abcdef" ) result := hypervisor.BackupResult{ BackupKey: backupKey, SizeBytes: 1024, VirtualSizeBytes: 40 << 30, SHA256: strings.Repeat("a", 64), Consistency: "crash", SourceContract: &hypervisor.PortableRestoreContract{ Contract: 1, Architecture: "x86_64", Machine: "q35", Firmware: "uefi", SecureBoot: false, DiskFormat: "qcow2", SystemDiskOnly: true, PublicNetworkContract: 1, VPCInherited: false, }, } encodedResult, err := json.Marshal(result) if err != nil { t.Fatal(err) } executor := &fakeMutationExecutor{result: encodedResult} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } request := hypervisor.BackupRequest{ OperationID: "operation-create-backup", BackupKey: backupKey, Name: "portable backup", ConsistencyMode: "filesystem", ObjectUpload: objecttransfer.MultipartPlan{ PartSizeBytes: 5 << 20, PartURLs: []string{ "https://objects.example.test/object?partNumber=1&uploadId=secret", }, CompleteURL: "https://objects.example.test/object?uploadId=secret", AbortURL: "https://objects.example.test/object?uploadId=secret", VerifyURL: "https://objects.example.test/object?signature=secret", MaximumSizeBytes: 5 << 20, }, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, } response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/backups", body, actor) if response.Code != http.StatusAccepted { t.Fatalf("backup response = %d %s", response.Code, response.Body.String()) } record := waitForTerminal(t, api.journal, request.OperationID) if record.State != operation.StateSucceeded || record.Action != "create-backup:"+backupKey+":filesystem" || record.ResourceID != uuid || record.Destructive || !record.HeavyIO { t.Fatalf("backup operation record = %#v", record) } for _, want := range []string{ "domain:" + uuid, "volume:virt-cloud-managed/vm-" + uuid, "volume:virt-cloud-managed/vbackup-123e4567e89b42d3a456426614174000-" + backupKey + ".qcow2", "catalog:backup/" + backupKey, } { if !containsTestString(record.LockKeys, want) { t.Fatalf("backup operation locks missing %q: %v", want, record.LockKeys) } } receipts, err := api.journal.ClaimPendingCompletions(1) if err != nil || len(receipts) != 1 || receipts[0].OperationID != request.OperationID || receipts[0].Action != record.Action || receipts[0].Status != string(operation.StateSucceeded) { t.Fatalf("backup completion outbox = %#v err=%v", receipts, err) } executor.mu.Lock() deadline := executor.deadline executor.mu.Unlock() remaining := time.Until(deadline) if remaining < 23*time.Hour+59*time.Minute || remaining > 24*time.Hour { t.Fatalf("backup helper deadline remaining = %s, want dedicated 24h bound", remaining) } } func TestBackupWorkspaceCleanupRequiresSuccessfulPriorAndFreshExactPlan(t *testing.T) { api := newTestAPI(t) const ( uuid = "123e4567-e89b-42d3-a456-426614174000" backupKey = "0123456789abcdef0123456789abcdef" priorID = "operation-backup-cleanup-pending" cleanupID = "operation-backup-workspace-cleanup" ) priorSpec := operation.Spec{ OperationID: priorID, ResourceID: uuid, Action: "create-backup:" + backupKey + ":filesystem", Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms/"+uuid+"/backups", []byte(`{}`)), LockKeys: hypervisor.CreateBackupAdmission(uuid, hypervisor.BackupRequest{ BackupKey: backupKey, }).LockKeys, HeavyIO: true, } if _, _, err := api.journal.Execute(context.Background(), priorSpec, func(context.Context) (json.RawMessage, error) { return json.RawMessage(`{"backup_key":"` + backupKey + `","size_bytes":1024,"virtual_size_bytes":4096,"sha256":"` + strings.Repeat("a", 64) + `","consistency":"crash","cleanup_pending":true}`), nil }); err != nil { t.Fatal(err) } prior, err := api.journal.Get(priorID) if err != nil { t.Fatal(err) } planID := strings.Repeat("b", 64) api.hv.inspectionResult = &hypervisor.OperationInspection{ OperationID: prior.OperationID, Action: prior.Action, ResourceID: prior.ResourceID, RequestFingerprint: prior.Fingerprint, OperationStatus: prior.State, CheckedAt: api.now.Unix(), Outcome: hypervisor.InspectionOutcomeSucceeded, SummaryCode: hypervisor.InspectionSummaryCleanupRequired, EffectResolution: hypervisor.InspectionEffectApplied, EffectProjection: append(json.RawMessage(nil), prior.Result...), Cleanup: hypervisor.OperationCleanupPlan{ Available: true, PlanID: planID, Steps: []string{hypervisor.CleanupStepDeleteSnapshotVolume}, }, } executor := &fakeMutationExecutor{result: json.RawMessage(`{"backup_key":"` + backupKey + `","cleaned":true,"workspace_absent":true,"metadata_absent":true}`)} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } request := hypervisor.CleanupBackupWorkspaceRequest{ OperationID: cleanupID, PriorOperationID: priorID, CleanupPlanID: planID, } body, _ := json.Marshal(request) actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: cleanupID, } response := api.request(t, http.MethodPost, "/v1/operations/"+cleanupID+"/cleanup-backup-workspace", body, actor) if response.Code != http.StatusAccepted { t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String()) } terminal := waitForTerminal(t, api.journal, cleanupID) if terminal.State != operation.StateSucceeded || terminal.SupersedesOperationID != priorID || terminal.Action != "cleanup-backup-workspace:"+backupKey || terminal.ResourceID != uuid { t.Fatalf("cleanup terminal = %#v", terminal) } executor.mu.Lock() if len(executor.requests) != 1 || executor.requests[0].SupersededOperation == nil || executor.requests[0].SupersededOperation.OperationID != priorID { executor.mu.Unlock() t.Fatalf("cleanup worker predecessor = %#v", executor.requests) } executor.mu.Unlock() stale := request stale.OperationID = "operation-backup-workspace-stale" stale.CleanupPlanID = strings.Repeat("c", 64) staleBody, _ := json.Marshal(stale) staleActor := actor staleActor.OperationID = stale.OperationID denied := api.request(t, http.MethodPost, "/v1/operations/"+stale.OperationID+"/cleanup-backup-workspace", staleBody, staleActor) if denied.Code != http.StatusConflict || !bytes.Contains(denied.Body.Bytes(), []byte(`"code":"backup_workspace_cleanup_plan_changed"`)) { t.Fatalf("stale cleanup = %d %s", denied.Code, denied.Body.String()) } } func TestRecoverBackupUsesDestructiveDiskTransitionAdmission(t *testing.T) { api := newTestAPI(t) const ( uuid = "123e4567-e89b-42d3-a456-426614174000" backupKey = "0123456789abcdef0123456789abcdef" ) backupSHA := strings.Repeat("b", 64) result := hypervisor.RecoverBackupResult{ BackupKey: backupKey, SHA256: backupSHA, Restored: true, PowerState: hypervisor.VMStateRunning, } encodedResult, err := json.Marshal(result) if err != nil { t.Fatal(err) } executor := &fakeMutationExecutor{result: encodedResult} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } request := hypervisor.RecoverBackupRequest{ OperationID: "operation-recover-backup", BackupKey: backupKey, SourceURL: "https://objects.example.test/object?signature=secret", SourceSizeBytes: 4 << 20, SourceSHA256: backupSHA, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, } response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/recover-backup", body, actor) if response.Code != http.StatusAccepted { t.Fatalf("backup recovery response = %d %s", response.Code, response.Body.String()) } record := waitForTerminal(t, api.journal, request.OperationID) if record.State != operation.StateSucceeded || record.Action != "recover-backup:"+backupKey+":"+backupSHA || record.ResourceID != uuid || !record.Destructive || !record.HeavyIO { t.Fatalf("backup recovery operation record = %#v", record) } for _, want := range []string{ "domain:" + uuid, "volume:virt-cloud-managed/vm-" + uuid, "catalog:backup/" + backupKey, } { if !containsTestString(record.LockKeys, want) { t.Fatalf("backup recovery locks missing %q: %v", want, record.LockKeys) } } } func TestRestoreBackupDerivesDomainAndUsesCreateLikeAction(t *testing.T) { api := newTestAPI(t) const ( uuid = "123e4567-e89b-42d3-a456-426614174000" backupKey = "0123456789abcdef0123456789abcdef" ) result := hypervisor.RestoreBackupResult{ VMMetadata: hypervisor.VMMetadata{UUID: uuid, Name: "restored-vm"}, BackupKey: backupKey, SHA256: strings.Repeat("b", 64), } encodedResult, err := json.Marshal(result) if err != nil { t.Fatal(err) } executor := &fakeMutationExecutor{result: encodedResult} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } request := hypervisor.RestoreBackupRequest{ CreateVMRequest: hypervisor.CreateVMRequest{ OperationID: "operation-restore-backup", UUID: uuid, Name: "restored-vm", VCPUs: 2, MemoryMiB: 2048, DiskGiB: 40, BandwidthMbps: 100, PublicNetwork: publicnetwork.Intent{ Contract: 1, NetworkSelector: "bridge:br0", IPv4: publicnetwork.AddressIntent{PoolKey: backupKey, PoolCIDR: "8.8.8.0/24", AddressCIDR: "8.8.8.10/24", Gateway: "8.8.8.1"}, IPv6: publicnetwork.AddressIntent{PoolKey: "abcdef0123456789abcdef0123456789", PoolCIDR: "2404:c140:156:3a::/64", AddressCIDR: "2404:c140:156:3a::1000/64", Gateway: "2404:c140:156::1", GatewayOnLink: true}, DNSServers: []string{"1.1.1.1", "2606:4700:4700::1111"}, }, PlatformNetworkPolicy: apiTestPlatformPolicy(t), }, RequestedDomainName: "1:vm-123e4567e89b", BackupKey: backupKey, SourceURL: "https://objects.example.test/object?signature=secret", SourceSizeBytes: 4 << 20, SourceSHA256: strings.Repeat("b", 64), } debian12NetworkFix := true request.Debian12NetworkFix = &debian12NetworkFix body, err := json.Marshal(request) if err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, } response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/restore-backup", body, actor) if response.Code != http.StatusAccepted { t.Fatalf("restore response = %d %s", response.Code, response.Body.String()) } record := waitForTerminal(t, api.journal, request.OperationID) if record.State != operation.StateSucceeded || record.Action != "restore-backup:"+backupKey || record.ResourceID != uuid || !record.Destructive || !record.HeavyIO || !operation.IsCreateLikeAction(record.Action) || record.ExpectedState == nil || record.ExpectedState.Kind != operation.ExpectedCreateLikeNetwork || record.ExpectedState.TransitionID != request.OperationID || record.ExpectedState.NetworkSourceType != "bridge" || record.ExpectedState.NetworkSourceName != "br0" || !reflect.DeepEqual(record.ExpectedState.PublicNetwork, request.PublicNetwork) || record.ExpectedState.VPCPresent || record.ExpectedState.VPCKey != "" || record.ExpectedState.VPCPortKey != "" { t.Fatalf("restore operation record = %#v", record) } receipts, err := api.journal.ClaimPendingCompletions(1) if err != nil || len(receipts) != 1 || receipts[0].OperationID != request.OperationID || receipts[0].Action != record.Action || receipts[0].Status != string(operation.StateSucceeded) { t.Fatalf("restore completion outbox = %#v err=%v", receipts, err) } } func TestPeerMigrationReceivePersistsExactPublicAndVPCExpectedState(t *testing.T) { api := newTestAPI(t) const ( uuid = "123e4567-e89b-42d3-a456-426614174000" migrationID = "0123456789abcdef0123456789abcdef" vpcKey = "1123456789abcdef0123456789abcdef" portKey = "2123456789abcdef0123456789abcdef" hostDigest = "3123456789abcdef0123456789abcdef3123456789abcdef0123456789abcdef" ) bridge, err := vpcrouting.BridgeName(vpcKey) if err != nil { t.Fatal(err) } public := publicnetwork.Intent{ Contract: 1, NetworkSelector: "network:public-dual-stack", IPv4: publicnetwork.AddressIntent{ PoolKey: migrationID, PoolCIDR: "8.8.8.0/24", AddressCIDR: "8.8.8.10/24", Gateway: "8.8.8.1", }, IPv6: publicnetwork.AddressIntent{ PoolKey: "abcdef0123456789abcdef0123456789", PoolCIDR: "2404:c140:156:3a::/64", AddressCIDR: "2404:c140:156:3a::1000/64", Gateway: "2404:c140:156::1", GatewayOnLink: true, }, DNSServers: []string{"1.1.1.1", "2606:4700:4700::1111"}, } port := hypervisor.VPCPortMetadata{ PortKey: portKey, VPCKey: vpcKey, PrivateIP: "10.60.0.9", CIDR: "10.60.0.0/24", Gateway: "10.60.0.1", MAC: "02:00:00:00:00:09", Bridge: bridge, HostRevision: 7, HostDigest: hostDigest, } request := hypervisor.ReceivePeerMigrationRequest{ CreateVMRequest: hypervisor.CreateVMRequest{ OperationID: "operation-peer-expected-state", UUID: uuid, Name: "migrated-vm", VCPUs: 2, MemoryMiB: 2048, DiskGiB: 40, BandwidthMbps: 100, PublicNetwork: public, PlatformNetworkPolicy: apiTestPlatformPolicy(t), }, MigrationID: migrationID, Ticket: strings.Repeat("2", 64), SourceEndpoints: []string{"https://192.0.2.10:9443"}, SourceIdentitySHA256: strings.Repeat("3", 64), SourceUUID: "123e4567-e89b-42d3-a456-426614174001", ManifestDigest: strings.Repeat("4", 64), Debian12NetworkFix: true, VPCPort: &port, } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } executor := &fakeMutationExecutor{result: json.RawMessage(`{}`)} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, } response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/peer-migration/receive", body, actor) if response.Code != http.StatusAccepted { t.Fatalf("peer migration receive response = %d %s", response.Code, response.Body.String()) } record := waitForTerminal(t, api.journal, request.OperationID) expected := record.ExpectedState if record.State != operation.StateSucceeded || record.Action != "receive-peer-migration:"+migrationID || record.ResourceID != uuid || expected == nil || expected.Kind != operation.ExpectedCreateLikeNetwork || expected.TransitionID != request.OperationID || expected.NetworkSourceType != "network" || expected.NetworkSourceName != "public-dual-stack" || !reflect.DeepEqual(expected.PublicNetwork, public) || !expected.VPCPresent || expected.VPCKey != port.VPCKey || expected.VPCPortKey != port.PortKey || expected.VPCRevision != port.HostRevision || expected.VPCDigest != port.HostDigest || expected.VPCCIDR != port.CIDR || expected.VPCGateway != port.Gateway || expected.VPCPrivateIP != port.PrivateIP || expected.VPCMAC != port.MAC || expected.VPCBridge != port.Bridge { t.Fatalf("peer migration expected state = %#v record=%#v", expected, record) } } func TestDiskAuthoritativeRestoreWireContractsRejectHostnameBeforeDispatch(t *testing.T) { const ( uuid = "123e4567-e89b-42d3-a456-426614174000" backupKey = "0123456789abcdef0123456789abcdef" ) public := publicnetwork.Intent{ Contract: 1, NetworkSelector: "bridge:br0", IPv4: publicnetwork.AddressIntent{ PoolKey: backupKey, PoolCIDR: "8.8.8.0/24", AddressCIDR: "8.8.8.10/24", Gateway: "8.8.8.1", }, IPv6: publicnetwork.AddressIntent{ PoolKey: "abcdef0123456789abcdef0123456789", PoolCIDR: "2404:c140:156:3a::/64", AddressCIDR: "2404:c140:156:3a::1000/64", Gateway: "2404:c140:156::1", GatewayOnLink: true, }, DNSServers: []string{"1.1.1.1", "2606:4700:4700::1111"}, } create := hypervisor.CreateVMRequest{ UUID: uuid, Name: "restored-vm", VCPUs: 2, MemoryMiB: 2048, DiskGiB: 40, BandwidthMbps: 100, Hostname: "replacement.example.com", PublicNetwork: public, PlatformNetworkPolicy: apiTestPlatformPolicy(t), } debian12NetworkFix := true restoreCreate := create restoreCreate.OperationID = "operation-restore-hostname-contract" peerCreate := create peerCreate.OperationID = "operation-peer-hostname-contract" tests := []struct { name string path string operationID string errorCode string request any }{ { name: "backup restore", path: "/v1/vms/" + uuid + "/restore-backup", operationID: restoreCreate.OperationID, errorCode: "invalid_backup_restore", request: hypervisor.RestoreBackupRequest{ CreateVMRequest: restoreCreate, RequestedDomainName: "1:vm-123e4567e89b", BackupKey: backupKey, SourceURL: "https://objects.example.test/object?signature=secret", SourceSizeBytes: 4 << 20, SourceSHA256: strings.Repeat("b", 64), Debian12NetworkFix: &debian12NetworkFix, }, }, { name: "peer migration receive", path: "/v1/vms/" + uuid + "/peer-migration/receive", operationID: peerCreate.OperationID, errorCode: "invalid_peer_migration_receive", request: hypervisor.ReceivePeerMigrationRequest{ CreateVMRequest: peerCreate, MigrationID: strings.Repeat("1", 32), Ticket: strings.Repeat("2", 64), SourceEndpoints: []string{"https://192.0.2.10:9443"}, SourceIdentitySHA256: strings.Repeat("3", 64), SourceUUID: "123e4567-e89b-42d3-a456-426614174001", ManifestDigest: strings.Repeat("4", 64), Debian12NetworkFix: true, }, }, } for _, testCase := range tests { t.Run(testCase.name, func(t *testing.T) { api := newTestAPI(t) executor := &fakeMutationExecutor{result: json.RawMessage(`{"unexpected":true}`)} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } body, err := json.Marshal(testCase.request) if err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: testCase.operationID, } response := api.request(t, http.MethodPost, testCase.path, body, actor) if response.Code != http.StatusUnprocessableEntity || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"`+testCase.errorCode+`"`)) { t.Fatalf("hostname replacement response = %d %s", response.Code, response.Body.String()) } executor.mu.Lock() calls := executor.calls executor.mu.Unlock() if calls != 0 { t.Fatalf("hostname replacement dispatched %d worker calls", calls) } }) } } func apiTestPlatformPolicy(t *testing.T) hypervisor.PlatformNetworkPolicySnapshot { t.Helper() rules := []hypervisor.PlatformNetworkPolicyRule{{ ID: "ssh", Label: "SSH", Category: "default", Protocols: []string{"tcp"}, PortFrom: 22, PortTo: 22, Directions: []string{"ingress"}, Reason: "default SSH policy", }} document := struct { SchemaVersion uint64 `json:"schema_version"` BasePolicyVersion uint64 `json:"base_policy_version"` CatalogRevision uint64 `json:"catalog_revision"` Rules []hypervisor.PlatformNetworkPolicyRule `json:"rules"` }{SchemaVersion: 1, BasePolicyVersion: 1, CatalogRevision: 1, Rules: rules} encoded, err := json.Marshal(document) if err != nil { t.Fatal(err) } digest := sha256.Sum256(encoded) return hypervisor.PlatformNetworkPolicySnapshot{ SchemaVersion: 1, BasePolicyVersion: 1, CatalogRevision: 1, Rules: rules, Digest: hex.EncodeToString(digest[:]), } } func containsTestString(values []string, want string) bool { for _, value := range values { if value == want { return true } } return false } func TestResizeResourcesPersistsExactSourceTargetAndAdmission(t *testing.T) { api := newTestAPI(t) const uuid = "123e4567-e89b-42d3-a456-426614174000" request := hypervisor.ResizeResourcesRequest{ OperationID: "operation-resize-resources", FromVCPUs: 2, TargetVCPUs: 4, FromMemoryMiB: 2048, TargetMemoryMiB: 4096, FromDiskGiB: 40, TargetDiskGiB: 80, FromBandwidthMbps: 100, TargetBandwidthMbps: 200, ConfirmShutdown: true, } result := hypervisor.ResizeResourcesResult{ UUID: uuid, FromVCPUs: 2, TargetVCPUs: 4, AppliedVCPUs: 4, FromMemoryMiB: 2048, TargetMemoryMiB: 4096, AppliedMemoryMiB: 4096, FromDiskGiB: 40, TargetDiskGiB: 80, AppliedDiskGiB: 80, FromBandwidthMbps: 100, TargetBandwidthMbps: 200, AppliedBandwidthMbps: 200, Status: hypervisor.VMStateShutoff, OriginallyRunning: false, PowerStateRestored: true, ResourcesResized: true, } encodedResult, err := json.Marshal(result) if err != nil { t.Fatal(err) } executor := &fakeMutationExecutor{result: encodedResult} if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } actor := auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, } response := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/resources", body, actor) if response.Code != http.StatusAccepted { t.Fatalf("resize response = %d %s", response.Code, response.Body.String()) } record := waitForTerminal(t, api.journal, request.OperationID) if record.State != operation.StateSucceeded || record.Action != "resize-vm" || record.ResourceID != uuid || !record.Destructive || record.ExpectedState == nil || record.ExpectedState.Kind != operation.ExpectedResizeResources || record.ExpectedState.TransitionID != request.OperationID || record.ExpectedState.FromVCPUs != request.FromVCPUs || record.ExpectedState.TargetVCPUs != request.TargetVCPUs || record.ExpectedState.FromMemoryMiB != request.FromMemoryMiB || record.ExpectedState.TargetMemoryMiB != request.TargetMemoryMiB || record.ExpectedState.FromDiskGiB != request.FromDiskGiB || record.ExpectedState.TargetDiskGiB != request.TargetDiskGiB || record.ExpectedState.FromBandwidthMbps != request.FromBandwidthMbps || record.ExpectedState.TargetBandwidthMbps != request.TargetBandwidthMbps { t.Fatalf("resize operation record = %#v", record) } wantLocks := []string{"domain:" + uuid, "volume:virt-cloud-managed/vm-" + uuid} if !record.HeavyIO || !reflect.DeepEqual(record.LockKeys, wantLocks) { t.Fatalf("resize admission = heavy=%v locks=%v, want heavy locks=%v", record.HeavyIO, record.LockKeys, wantLocks) } request.OperationID = "operation-resize-unconfirmed" request.ConfirmShutdown = false body, err = json.Marshal(request) if err != nil { t.Fatal(err) } actor.OperationID = request.OperationID rejected := api.request(t, http.MethodPost, "/v1/vms/"+uuid+"/resources", body, actor) if rejected.Code != http.StatusUnprocessableEntity || !bytes.Contains(rejected.Body.Bytes(), []byte(`"code":"shutdown_confirmation_required"`)) { t.Fatalf("missing shutdown confirmation = %d %s", rejected.Code, rejected.Body.String()) } executor.mu.Lock() calls := executor.calls executor.mu.Unlock() if calls != 1 { t.Fatalf("invalid resize crossed worker boundary: calls=%d", calls) } methodRejected := api.request(t, http.MethodPatch, "/v1/vms/"+uuid+"/resources", body, actor) if methodRejected.Code != http.StatusMethodNotAllowed || methodRejected.Header().Get("Allow") != http.MethodPost || !bytes.Contains(methodRejected.Body.Bytes(), []byte(`"code":"method_not_allowed"`)) { t.Fatalf("PATCH resize response = %d allow=%q %s", methodRejected.Code, methodRejected.Header().Get("Allow"), methodRejected.Body.String()) } executor.mu.Lock() calls = executor.calls executor.mu.Unlock() if calls != 1 { t.Fatalf("PATCH resize crossed worker boundary: calls=%d", calls) } } func TestIsolatedMutationDeadlineCapsOnlyStorageConfiguration(t *testing.T) { server := &Server{OperationTimeout: 840 * time.Second} if got := server.isolatedMutationDeadline("configure-storage"); got != 120*time.Second { t.Fatalf("storage helper deadline = %s, want 120s", got) } for _, action := range []string{"create-vm", "delete-vm", "change-network"} { if got := server.isolatedMutationDeadline(action); got != 840*time.Second { t.Fatalf("%s helper deadline = %s, want global 840s", action, got) } } server.OperationTimeout = 30 * time.Second if got := server.isolatedMutationDeadline("configure-storage"); got != 30*time.Second { t.Fatalf("lower global deadline was widened to %s", got) } } func TestIsolatedMutationAdmissionIsShortAndDoesNotStartSecondWorker(t *testing.T) { api := newTestAPI(t) // Give the first request enough time to persist its independent Bolt // journal under a loaded test host. The short timeout is the behavior under // test only after that first worker demonstrably owns admission. api.handler.AdmissionTimeout = time.Second executor := &blockingMutationExecutor{ started: make(chan worker.Request, 1), release: make(chan struct{}), } if err := api.handler.SetMutationExecutor(executor); err != nil { t.Fatal(err) } requestBody := func(operationID string) []byte { body, err := json.Marshal(hypervisor.StorageConfigurationRequest{ OperationID: operationID, ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", }) if err != nil { t.Fatal(err) } return body } firstID := "operation-admission-first" firstBody := requestBody(firstID) first := api.request(t, http.MethodPost, "/v1/storage/configuration", firstBody, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: firstID, }) if first.Code != http.StatusAccepted { t.Fatalf("first response = %d %s", first.Code, first.Body.String()) } firstWire := requireOperationResponseIdentity(t, first, firstID, "configure-storage", "host-storage", operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", firstBody)) if firstWire.Status != operation.StateDispatching || firstWire.Replayed { t.Fatalf("first accepted operation response = %#v", firstWire) } select { case <-executor.started: case <-time.After(time.Second): t.Fatal("first worker did not start") } api.handler.AdmissionTimeout = 30 * time.Millisecond secondID := "operation-admission-second" secondBody := requestBody(secondID) startedAt := time.Now() second := api.request(t, http.MethodPost, "/v1/storage/configuration", secondBody, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: secondID, }) if elapsed := time.Since(startedAt); elapsed > 500*time.Millisecond { t.Fatalf("saturated admission waited %s", elapsed) } if second.Code != http.StatusOK || !bytes.Contains(second.Body.Bytes(), []byte(`"status":"failed"`)) || !bytes.Contains(second.Body.Bytes(), []byte(`"code":"operation_not_dispatched"`)) { t.Fatalf("saturated admission response = %d %s", second.Code, second.Body.String()) } failedWire := requireOperationResponseIdentity(t, second, secondID, "configure-storage", "host-storage", operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", secondBody)) if failedWire.Status != operation.StateFailed || failedWire.Replayed { t.Fatalf("failed operation response = %#v", failedWire) } if executor.calls.Load() != 1 { t.Fatalf("saturated request started a second worker; calls=%d", executor.calls.Load()) } close(executor.release) if terminal := waitForTerminal(t, api.journal, firstID); terminal.State != operation.StateSucceeded { t.Fatalf("first terminal = %#v", terminal) } } func TestParentMutationNeverFallsBackWhenExecutorIsMissing(t *testing.T) { api := newTestAPI(t) request := hypervisor.StorageConfigurationRequest{ OperationID: "operation-worker-missing", ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } response := api.request(t, http.MethodPost, "/v1/storage/configuration", body, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, }) if response.Code != http.StatusServiceUnavailable || !bytes.Contains(response.Body.Bytes(), []byte(`"code":"mutation_worker_unavailable"`)) || !bytes.Contains(response.Body.Bytes(), []byte(`"dispatch_state":"not_dispatched"`)) { t.Fatalf("missing worker response = %d %s", response.Code, response.Body.String()) } if _, err := api.journal.Get(request.OperationID); !errors.Is(err, operation.ErrNotFound) { t.Fatalf("missing worker created journal state: %v", err) } api.hv.mu.Lock() configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if configureCalls != 0 { t.Fatalf("parent invoked the in-process hypervisor %d times", configureCalls) } } func TestTrustedWorkerExecutesLocallyWithoutRecursiveSpawn(t *testing.T) { api := newTestAPI(t) request := hypervisor.StorageConfigurationRequest{ OperationID: "operation-trusted-worker", ManagedDirectory: "/srv/virt/managed", TemplateDirectory: "/srv/virt/templates", SeedDirectory: "/srv/virt/seeds", UserISODirectory: "/srv/virt/user-isos", RescueDirectory: "/srv/virt/rescue-isos", } body, err := json.Marshal(request) if err != nil { t.Fatal(err) } httpRequest := httptest.NewRequest(http.MethodPost, "/v1/storage/configuration", bytes.NewReader(body)) response := httptest.NewRecorder() api.handler.ServeTrustedWorkerRequest(response, httpRequest, body, auth.Actor{ UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: request.OperationID, }) if response.Code != http.StatusOK { t.Fatalf("trusted worker response = %d %s", response.Code, response.Body.String()) } succeeded := requireOperationResponseIdentity(t, response, request.OperationID, "configure-storage", "host-storage", operation.Fingerprint(http.MethodPost, "/v1/storage/configuration", body)) if succeeded.Status != operation.StateSucceeded || succeeded.Replayed { t.Fatalf("successful operation response = %#v", succeeded) } api.hv.mu.Lock() configureCalls := api.hv.configureCalls api.hv.mu.Unlock() if configureCalls != 1 { t.Fatalf("trusted worker local mutation calls=%d", configureCalls) } } func TestListEnvelopeRemainsNonNull(t *testing.T) { api := newTestAPI(t) response := api.request(t, http.MethodGet, "/v1/vms", nil, auth.Actor{UserID: 1}) if response.Code != http.StatusOK || response.Body.String() != "{\"vms\":[]}\n" { t.Fatalf("VM list = %d %s", response.Code, response.Body.String()) } } func waitForTerminal(t *testing.T, journal *operation.Journal, operationID string) operation.Record { t.Helper() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { record, err := journal.Get(operationID) if err == nil && (record.State == operation.StateSucceeded || record.State == operation.StateFailed) { return record } time.Sleep(time.Millisecond) } t.Fatalf("operation %s did not become terminal", operationID) return operation.Record{} }