diff --git a/agent/internal/api/server.go b/agent/internal/api/server.go index 1df48a41..e8f3cf18 100644 --- a/agent/internal/api/server.go +++ b/agent/internal/api/server.go @@ -381,7 +381,7 @@ func (s *Server) createOperation(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "domain define is disabled until the typed create executor is available"}) return } - if request.Kind != "capability_probe" && request.Kind != "domain_read" && request.Kind != "domain_observe" && request.Kind != "domain_power" && request.Kind != "seed_prepare" && request.Kind != "domain_preflight" && request.Kind != "domain_create_plan" && request.Kind != "domain_create_validate" { + if request.Kind != "capability_probe" && request.Kind != "domain_read" && request.Kind != "domain_observe" && request.Kind != "domain_power" && request.Kind != "domain_delete" && request.Kind != "seed_prepare" && request.Kind != "domain_preflight" && request.Kind != "domain_create_plan" && request.Kind != "domain_create_validate" { writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "operation kind is not implemented by the agent"}) return } @@ -426,6 +426,13 @@ func (s *Server) createOperation(w http.ResponseWriter, r *http.Request) { return } } + if request.Kind == "domain_delete" { + uuid, parseErr := parseDomainDeletePayload(request.Payload) + if parseErr != nil || !targetMatchesUUID(request.Target, uuid) { + writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"}) + return + } + } if request.Kind == "domain_read" { uuid, parseErr := parseDomainReadPayload(request.Payload) if parseErr != nil || !targetMatchesUUID(request.Target, uuid) { @@ -526,6 +533,10 @@ func (s *Server) createOperation(w http.ResponseWriter, r *http.Request) { s.completeDomainPower(w, record, request, ctx) return } + if request.Kind == "domain_delete" { + s.completeDomainDelete(w, record, request, ctx) + return + } if request.Kind == "seed_prepare" { s.completeSeedPrepare(w, record, request, ctx) return @@ -596,6 +607,13 @@ type domainPowerPayload struct { Action string `json:"action"` } +// domainDeletePayload is intentionally only an exact UUID. Disk cleanup, +// network teardown, and panel billing state are separate contracts and are +// not accepted by the host Agent's narrow undefine operation. +type domainDeletePayload struct { + UUID string `json:"uuid"` +} + type seedPreparePayload struct { ArtifactID string `json:"artifact_id"` InstanceID string `json:"instance_id"` @@ -631,6 +649,10 @@ type domainObserveResult struct { ObservedAt time.Time `json:"observed_at"` } +type domainDeleteJournalPayload struct { + UUID string `json:"uuid"` +} + type domainPowerJournalPayload struct { UUID string `json:"uuid"` Action string `json:"action"` @@ -683,6 +705,12 @@ func safeJournalPayload(kind string, raw json.RawMessage) (json.RawMessage, erro return nil, err } return json.Marshal(domainPowerJournalPayload{UUID: power.UUID, Action: string(power.Action)}) + case "domain_delete": + uuid, err := parseDomainDeletePayload(raw) + if err != nil { + return nil, err + } + return json.Marshal(domainDeleteJournalPayload{UUID: uuid}) case "seed_prepare": payload, err := parseSeedPrepareRequest(raw) if err != nil { @@ -800,6 +828,14 @@ func parseDomainPowerRequest(raw json.RawMessage) (hypervisor.PowerRequest, erro return hypervisor.PowerRequest{UUID: canonical, Action: action}, nil } +func parseDomainDeletePayload(raw json.RawMessage) (string, error) { + var payload domainDeletePayload + if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" { + return "", fmt.Errorf("invalid domain delete payload") + } + return hypervisor.CanonicalUUID(payload.UUID) +} + func parseDomainReadPayload(raw json.RawMessage) (string, error) { var payload domainReadPayload if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" { @@ -1086,6 +1122,60 @@ func (s *Server) completeDomainPower(w http.ResponseWriter, record operation.Rec writeJSON(w, http.StatusOK, updated) } +// completeDomainDelete performs only the typed, inactive-domain undefine +// boundary. The hypervisor implementation must prove absence with a targeted +// read-back before this method can persist success. Disk images, seed +// artifacts, firewall rules, and host networking are deliberately untouched; +// the panel's full VM cleanup contract is not represented here. +func (s *Server) completeDomainDelete(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { + uuid, err := parseDomainDeletePayload(request.Payload) + if err != nil || !targetMatchesUUID(request.Target, uuid) { + _, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_delete_request", "domain delete payload is invalid") + writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"}) + return + } + result, err := s.backend.DeleteDomain(ctx, uuid) + var resultJSON json.RawMessage + if result.UUID != "" { + resultJSON, _ = json.Marshal(result) + } + if err != nil { + status := operation.StatusUncertain + httpStatus := http.StatusServiceUnavailable + errorCode := "domain_delete_uncertain" + message := "domain undefine or absence read-back did not complete" + if errors.Is(err, hypervisor.ErrInvalidUUID) || errors.Is(err, hypervisor.ErrDomainActive) { + status = operation.StatusFailed + httpStatus = http.StatusUnprocessableEntity + if errors.Is(err, hypervisor.ErrDomainActive) { + errorCode = "domain_delete_active" + message = "domain must be inactive before undefine" + } else { + errorCode = "invalid_domain_delete_request" + message = "domain delete UUID is invalid" + } + } + updated, updateErr := s.store.Update(record.OperationID, status, resultJSON, errorCode, message) + if updateErr == nil { + writeJSON(w, httpStatus, updated) + } else { + writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete failed and operation state is uncertain"}) + } + return + } + if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_delete_readback_confirmed"); boundaryErr != nil { + _, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain absence was confirmed but operation state could not be committed") + writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"}) + return + } + updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "") + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"}) + return + } + writeJSON(w, http.StatusOK, updated) +} + func (s *Server) completeSeedPrepare(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) { payload, err := parseSeedPrepareRequest(request.Payload) if err != nil { @@ -1429,6 +1519,8 @@ func (s *Server) reconcileOperation(w http.ResponseWriter, r *http.Request) { s.reconcileDomainObserve(w, record, r) case "domain_power": s.reconcileDomainPower(w, record, r) + case "domain_delete": + s.reconcileDomainDelete(w, record, r) case "seed_prepare": s.reconcileSeedPrepare(w, record, r) case "domain_preflight": @@ -1866,6 +1958,49 @@ func (s *Server) reconcileDomainPower(w http.ResponseWriter, record operation.Re writeJSON(w, http.StatusOK, updated) } +// reconcileDomainDelete is strictly read-only. It observes whether the exact +// definition UUID is absent and never retries undefine after an ambiguous +// response. A present definition, an active guest, or a failed observation +// keeps the journal uncertain for a later operator-approved read-back. +func (s *Server) reconcileDomainDelete(w http.ResponseWriter, record operation.Record, r *http.Request) { + if s.backend == nil { + s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "host backend is not configured for domain deletion read-back", http.StatusServiceUnavailable) + return + } + uuid, err := parseDomainDeletePayload(record.Payload) + if err != nil || !targetMatchesUUID(record.Target, uuid) { + s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal payload is invalid") + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout) + defer cancel() + observation, err := s.backend.InspectDomainForDeletion(ctx, uuid) + if err != nil { + if errors.Is(err, hypervisor.ErrInvalidUUID) { + s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal UUID is invalid") + return + } + s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "domain deletion absence read-back did not complete", http.StatusServiceUnavailable) + return + } + if observation.Present { + result, _ := json.Marshal(observation) + s.reconcileUncertain(w, record, result, "domain_delete_not_reconciled", "domain definition is still present; no undefine was replayed", http.StatusConflict) + return + } + result, marshalErr := json.Marshal(hypervisor.DomainDeleteResult{ + UUID: uuid, + AlreadyAbsent: true, + Removed: true, + ObservedAt: observation.ObservedAt, + }) + if marshalErr != nil { + s.reconcileUncertain(w, record, record.Result, "domain_delete_result_failed", "domain deletion result could not be encoded", http.StatusServiceUnavailable) + return + } + s.reconcileSuccess(w, record, "domain_delete_readback_confirmed", result) +} + func (s *Server) storeRecord(id string) (operation.Record, error) { if s.store == nil { return operation.Record{}, operation.ErrNotFound diff --git a/agent/internal/api/server_delete_test.go b/agent/internal/api/server_delete_test.go new file mode 100644 index 00000000..c50b1da0 --- /dev/null +++ b/agent/internal/api/server_delete_test.go @@ -0,0 +1,204 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "g.luoli.live/root/virt/agent/internal/capability" + "g.luoli.live/root/virt/agent/internal/config" + "g.luoli.live/root/virt/agent/internal/hypervisor" + "g.luoli.live/root/virt/agent/internal/operation" +) + +const deleteTestUUID = "00000000-0000-4000-8000-000000000401" + +func newDeleteTestServer(t *testing.T, backend hypervisor.Backend) (*Server, *operation.Store) { + t.Helper() + store := operation.NewMemory() + c := config.Defaults() + c.AgentID = "host-delete-test" + s, err := New(Options{ + Config: c, + Store: store, + Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}, + Backend: backend, + HostBackendReady: true, + }) + if err != nil { + t.Fatal(err) + } + return s, store +} + +func defineDeleteTestDomain(t *testing.T, backend *hypervisor.FakeBackend) { + t.Helper() + if _, err := backend.DefineDomain(context.Background(), hypervisor.DefineRequest{ + UUID: deleteTestUUID, + XML: []byte(`` + deleteTestUUID + `hvm`), + }); err != nil { + t.Fatal(err) + } +} + +func deleteOperationBody(id, key string) []byte { + return []byte(`{"operation_id":"` + id + `","idempotency_key":"` + key + `","kind":"domain_delete","target":"` + deleteTestUUID + `","payload":{"uuid":"` + deleteTestUUID + `"}}`) +} + +func TestDomainDeleteUndefinesOnlyInactiveDefinitionAndJournalsReadback(t *testing.T) { + backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) + defineDeleteTestDomain(t, backend) + s, store := newDeleteTestServer(t, backend) + operationID := "00000000-0000-4000-8000-000000000402" + recorder := httptest.NewRecorder() + s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader( + deleteOperationBody(operationID, "delete-operation-0402"), + ))) + if recorder.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + var record operation.Record + if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil { + t.Fatal(err) + } + if record.Status != operation.StatusSucceeded || record.CommitBoundary != "domain_delete_readback_confirmed" { + t.Fatalf("record=%+v", record) + } + var result hypervisor.DomainDeleteResult + if err := json.Unmarshal(record.Result, &result); err != nil { + t.Fatalf("decode result: %v raw=%s", err, record.Result) + } + if result.UUID != deleteTestUUID || !result.Existed || !result.Removed || result.AlreadyAbsent || + result.StorageRemoved || result.NetworkChanged || result.XMLHash == "" { + t.Fatalf("result=%+v", result) + } + observation, err := backend.InspectDomainForDeletion(context.Background(), deleteTestUUID) + if err != nil { + t.Fatal(err) + } + if observation.Present { + t.Fatalf("definition remains after successful delete: %+v", observation) + } + if _, err := store.Get(operationID); err != nil { + t.Fatal(err) + } +} + +func TestDomainDeleteRejectsActiveDomainBeforeUndefine(t *testing.T) { + backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) + defineDeleteTestDomain(t, backend) + if _, err := backend.PowerDomain(context.Background(), deleteTestUUID, hypervisor.PowerActionStart); err != nil { + t.Fatal(err) + } + s, store := newDeleteTestServer(t, backend) + operationID := "00000000-0000-4000-8000-000000000403" + recorder := httptest.NewRecorder() + s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader( + deleteOperationBody(operationID, "delete-operation-0403"), + ))) + if recorder.Code != http.StatusUnprocessableEntity { + t.Fatalf("status=%d body=%s; want 422", recorder.Code, recorder.Body.String()) + } + record, err := store.Get(operationID) + if err != nil { + t.Fatal(err) + } + if record.Status != operation.StatusFailed || record.ErrorCode != "domain_delete_active" { + t.Fatalf("record=%+v", record) + } + observation, err := backend.InspectDomainForDeletion(context.Background(), deleteTestUUID) + if err != nil || !observation.Present || !observation.Active { + t.Fatalf("domain changed after active rejection: %+v err=%v", observation, err) + } +} + +type deleteCountingBackend struct { + *hypervisor.FakeBackend + deleteCalls int + inspectCalls int +} + +func (b *deleteCountingBackend) DeleteDomain(ctx context.Context, uuid string) (hypervisor.DomainDeleteResult, error) { + b.deleteCalls++ + return b.FakeBackend.DeleteDomain(ctx, uuid) +} + +func (b *deleteCountingBackend) InspectDomainForDeletion(ctx context.Context, uuid string) (hypervisor.DomainDeletionObservation, error) { + b.inspectCalls++ + return b.FakeBackend.InspectDomainForDeletion(ctx, uuid) +} + +func TestDomainDeleteReconcileReadsAbsenceWithoutReplayingUndefine(t *testing.T) { + base := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) + defineDeleteTestDomain(t, base) + backend := &deleteCountingBackend{FakeBackend: base} + store := operation.NewMemory() + const operationID = "00000000-0000-4000-8000-000000000404" + payload := []byte(`{"uuid":"` + deleteTestUUID + `"}`) + journalUncertain(t, store, operationID, "delete-reconcile-0404", "domain_delete", deleteTestUUID, payload) + s, _ := newDeleteTestServerWithStore(t, backend, store) + + first := httptest.NewRecorder() + s.Handler().ServeHTTP(first, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil)) + if first.Code != http.StatusConflict { + t.Fatalf("first reconcile status=%d body=%s; want 409", first.Code, first.Body.String()) + } + if backend.deleteCalls != 0 || backend.inspectCalls != 1 { + t.Fatalf("first reconcile calls delete=%d inspect=%d; undefine must not replay", backend.deleteCalls, backend.inspectCalls) + } + + if _, err := base.DeleteDomain(context.Background(), deleteTestUUID); err != nil { + t.Fatal(err) + } + second := httptest.NewRecorder() + s.Handler().ServeHTTP(second, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil)) + if second.Code != http.StatusOK { + t.Fatalf("second reconcile status=%d body=%s", second.Code, second.Body.String()) + } + record, err := store.Get(operationID) + if err != nil { + t.Fatal(err) + } + if record.Status != operation.StatusSucceeded || record.CommitBoundary != "domain_delete_readback_confirmed" { + t.Fatalf("record=%+v", record) + } + if backend.deleteCalls != 0 || backend.inspectCalls != 2 { + t.Fatalf("second reconcile calls delete=%d inspect=%d; want read-only reconcile", backend.deleteCalls, backend.inspectCalls) + } +} + +func newDeleteTestServerWithStore(t *testing.T, backend hypervisor.Backend, store *operation.Store) (*Server, *operation.Store) { + t.Helper() + c := config.Defaults() + c.AgentID = "host-delete-reconcile" + s, err := New(Options{ + Config: c, + Store: store, + Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}, + Backend: backend, + HostBackendReady: true, + }) + if err != nil { + t.Fatal(err) + } + return s, store +} + +func TestDomainDeleteRejectsExtraPayloadBeforeJournal(t *testing.T) { + backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) + s, store := newDeleteTestServer(t, backend) + operationID := "00000000-0000-4000-8000-000000000405" + body := []byte(`{"operation_id":"` + operationID + `","idempotency_key":"delete-operation-0405","kind":"domain_delete","target":"` + deleteTestUUID + `","payload":{"uuid":"` + deleteTestUUID + `","delete_disks":true}}`) + recorder := httptest.NewRecorder() + s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body))) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status=%d body=%s; want 400", recorder.Code, recorder.Body.String()) + } + if _, err := store.Get(operationID); !errors.Is(err, operation.ErrNotFound) { + t.Fatalf("invalid delete entered journal: %v", err) + } +} diff --git a/agent/internal/hypervisor/backend.go b/agent/internal/hypervisor/backend.go index c0b3d4aa..ea773548 100644 --- a/agent/internal/hypervisor/backend.go +++ b/agent/internal/hypervisor/backend.go @@ -9,6 +9,8 @@ package hypervisor import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/xml" "errors" "fmt" @@ -32,6 +34,16 @@ var ( ErrDomainNotFound = errors.New("domain definition not found") // ErrDomainConflict means an existing UUID was defined with different XML. ErrDomainConflict = errors.New("domain definition conflicts with existing UUID") + // ErrDomainActive means an undefine request was rejected because the + // targeted domain is still active. The Agent never force-destroys a guest + // as part of deletion; callers must perform an explicit, separately + // journaled power operation first. + ErrDomainActive = errors.New("domain is active") + // ErrDeleteReadbackMismatch means an undefine command returned but the + // subsequent targeted read-back could not prove that the definition is + // absent. It is intentionally not a definitive error: the operation may + // have committed and must remain uncertain until a later observation. + ErrDeleteReadbackMismatch = errors.New("domain deletion read-back mismatch") // ErrReadbackMismatch means the backend returned XML that cannot prove the // requested UUID and firmware policy were retained. ErrReadbackMismatch = errors.New("domain definition read-back mismatch") @@ -57,6 +69,7 @@ func IsDefinitive(err error) bool { return errors.Is(err, ErrInvalidUUID) || errors.Is(err, ErrUUIDMismatch) || errors.Is(err, ErrDomainConflict) || + errors.Is(err, ErrDomainActive) || errors.Is(err, ErrInvalidPowerAction) || errors.Is(err, ErrPowerActionRejected) || errors.Is(err, ErrCapabilityPolicy) || @@ -162,6 +175,35 @@ type Definition struct { DefinedAt time.Time `json:"defined_at"` } +// DomainDeletionObservation is a read-only, identity-bound snapshot used by +// delete preflight and uncertain-operation reconciliation. It deliberately +// contains only a digest of the inactive XML; host paths, disk names and +// other definition details never cross the Agent API boundary. +type DomainDeletionObservation struct { + UUID string `json:"uuid"` + Present bool `json:"present"` + State DomainPowerState `json:"state,omitempty"` + Active bool `json:"active"` + XMLHash string `json:"xml_hash,omitempty"` + ObservedAt time.Time `json:"observed_at"` +} + +// DomainDeleteResult records the narrow mutation performed by DeleteDomain. +// Removed means only the libvirt domain definition was undefined. Storage +// and network objects are explicitly reported untouched so a caller cannot +// mistake this operation for the panel's full VM cleanup workflow. +type DomainDeleteResult struct { + UUID string `json:"uuid"` + Existed bool `json:"existed"` + AlreadyAbsent bool `json:"already_absent"` + Removed bool `json:"removed"` + StateBefore DomainPowerState `json:"state_before,omitempty"` + XMLHash string `json:"xml_hash,omitempty"` + StorageRemoved bool `json:"storage_removed"` + NetworkChanged bool `json:"network_changed"` + ObservedAt time.Time `json:"observed_at"` +} + // Backend is the intentionally small host boundary. Methods are context // aware because a panel request timeout must cancel an in-flight RPC without // leaving an unbounded libvirt call behind. @@ -178,6 +220,14 @@ type Backend interface { // ReadDomainPowerState is a read-only host probe used by reconciliation. // It must never invoke a lifecycle command or otherwise mutate the domain. ReadDomainPowerState(context.Context, string) (DomainPowerState, error) + // InspectDomainForDeletion is a read-only, exact-UUID preflight. It must + // never undefine, power off, remove storage, or touch host networking. + InspectDomainForDeletion(context.Context, string) (DomainDeletionObservation, error) + // DeleteDomain only undefines one inactive domain definition. Implementations + // must perform a targeted absence read-back before returning success; any + // lost/ambiguous read-back must return an error that the API journals as + // uncertain. Disk and network cleanup are outside this seam. + DeleteDomain(context.Context, string) (DomainDeleteResult, error) } // LoaderPathProvider is an optional host-bound capability seam. Implementations @@ -227,6 +277,9 @@ type FakeBackend struct { domains map[string]Definition states map[string]DomainPowerState nextRev uint64 + // mutationMu serializes define, lifecycle and undefine sequences. Reads do + // not take this lock and remain safe through mu's snapshot semantics. + mutationMu sync.Mutex } // NewFakeBackend creates a backend with the supplied capability snapshot. @@ -311,6 +364,13 @@ func (f *FakeBackend) PreferredNonSecureLoaderPath(ctx context.Context) (string, // identical UUID/XML pair is idempotent; redefining the UUID with different // XML is rejected rather than overwriting an unknown guest. func (f *FakeBackend) DefineDomain(ctx context.Context, request DefineRequest) (Definition, error) { + if err := contextErr(ctx); err != nil { + return Definition{}, err + } + f.mutationMu.Lock() + defer f.mutationMu.Unlock() + // A request can wait behind another host mutation. Re-check cancellation + // before beginning a new definition sequence. if err := contextErr(ctx); err != nil { return Definition{}, err } @@ -401,6 +461,136 @@ func (f *FakeBackend) ReadDefinition(ctx context.Context, uuid string) (Definiti return cloneDefinition(definition), nil } +// InspectDomainForDeletion performs the exact-UUID, read-only preflight used +// by DeleteDomain and its reconcile path. An absent definition is a valid +// observation rather than an error: delete is idempotent at the definition +// boundary, while transport/read failures remain errors and therefore +// uncertain to the operation layer. +func (f *FakeBackend) InspectDomainForDeletion(ctx context.Context, uuid string) (DomainDeletionObservation, error) { + if err := contextErr(ctx); err != nil { + return DomainDeletionObservation{}, err + } + canonical, err := CanonicalUUID(uuid) + if err != nil { + return DomainDeletionObservation{}, err + } + f.mu.RLock() + definition, present := f.domains[canonical] + state := f.states[canonical] + f.mu.RUnlock() + if !present { + return DomainDeletionObservation{ + UUID: canonical, + Present: false, + ObservedAt: time.Now().UTC(), + }, nil + } + if err := contextErr(ctx); err != nil { + return DomainDeletionObservation{}, err + } + if state == "" { + state = StateShutoff + } + observed, err := domainUUID(definition.XML) + if err != nil || observed != canonical { + return DomainDeletionObservation{}, fmt.Errorf("%w: requested %s", ErrReadbackMismatch, canonical) + } + return DomainDeletionObservation{ + UUID: canonical, + Present: true, + State: state, + Active: state.IsActive(), + XMLHash: hashDomainXML(definition.XML), + ObservedAt: time.Now().UTC(), + }, nil +} + +// DeleteDomain removes only one inactive definition. The operation is fenced +// by the fake's mutation lock and proves absence from a fresh read-back before +// returning success. No disk or network map exists in this backend, so those +// result fields remain explicit false values. +func (f *FakeBackend) DeleteDomain(ctx context.Context, uuid string) (DomainDeleteResult, error) { + if err := contextErr(ctx); err != nil { + return DomainDeleteResult{}, err + } + canonical, err := CanonicalUUID(uuid) + if err != nil { + return DomainDeleteResult{}, err + } + f.mutationMu.Lock() + defer f.mutationMu.Unlock() + if err := contextErr(ctx); err != nil { + return DomainDeleteResult{}, err + } + observation, err := f.InspectDomainForDeletion(ctx, canonical) + if err != nil { + return DomainDeleteResult{UUID: canonical}, err + } + if !observation.Present { + return DomainDeleteResult{ + UUID: canonical, + AlreadyAbsent: true, + Removed: true, + ObservedAt: time.Now().UTC(), + }, nil + } + if observation.Active { + return DomainDeleteResult{ + UUID: canonical, + Existed: true, + StateBefore: observation.State, + XMLHash: observation.XMLHash, + ObservedAt: time.Now().UTC(), + }, fmt.Errorf("%w: %s", ErrDomainActive, canonical) + } + + // Remove the definition only after the read-only preflight proved the + // canonical UUID and inactive state. Keep the state map in lockstep; no + // storage or network cleanup is attempted. + f.mu.Lock() + delete(f.domains, canonical) + delete(f.states, canonical) + f.mu.Unlock() + if err := contextErr(ctx); err != nil { + return DomainDeleteResult{ + UUID: canonical, + Existed: true, + StateBefore: observation.State, + XMLHash: observation.XMLHash, + ObservedAt: time.Now().UTC(), + }, err + } + after, err := f.InspectDomainForDeletion(ctx, canonical) + if err != nil { + return DomainDeleteResult{ + UUID: canonical, + Existed: true, + StateBefore: observation.State, + XMLHash: observation.XMLHash, + ObservedAt: time.Now().UTC(), + }, err + } + if after.Present { + return DomainDeleteResult{ + UUID: canonical, + Existed: true, + StateBefore: observation.State, + XMLHash: observation.XMLHash, + ObservedAt: time.Now().UTC(), + }, fmt.Errorf("%w: %s", ErrDeleteReadbackMismatch, canonical) + } + return DomainDeleteResult{ + UUID: canonical, + Existed: true, + Removed: true, + StateBefore: observation.State, + XMLHash: observation.XMLHash, + StorageRemoved: false, + NetworkChanged: false, + ObservedAt: after.ObservedAt, + }, nil +} + func contextErr(ctx context.Context) error { if ctx == nil { return nil @@ -449,6 +639,11 @@ func isHex(value rune) bool { return value >= '0' && value <= '9' || value >= 'a' && value <= 'f' } +func hashDomainXML(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} + // domainUUID extracts only the direct child. Nested metadata fields // are not identity and must not satisfy this check. func domainUUID(data []byte) (string, error) { diff --git a/agent/internal/hypervisor/backend_test.go b/agent/internal/hypervisor/backend_test.go index ef5be793..bc1c5094 100644 --- a/agent/internal/hypervisor/backend_test.go +++ b/agent/internal/hypervisor/backend_test.go @@ -7,6 +7,7 @@ import ( "strings" "sync" "testing" + "time" ) const testUUID = "00000000-0000-4000-8000-000000000001" @@ -243,6 +244,32 @@ func (s *staticXMLBackend) ReadDomainPowerState(context.Context, string) (Domain return StateShutoff, nil } +func (s *staticXMLBackend) InspectDomainForDeletion(ctx context.Context, uuid string) (DomainDeletionObservation, error) { + data, err := s.ReadDomainXML(ctx, uuid) + if err != nil { + return DomainDeletionObservation{}, err + } + canonical, err := CanonicalUUID(uuid) + if err != nil { + return DomainDeletionObservation{}, err + } + observed, err := domainUUID(data) + if err != nil || observed != canonical { + return DomainDeletionObservation{}, ErrReadbackMismatch + } + return DomainDeletionObservation{ + UUID: canonical, + Present: true, + State: StateShutoff, + XMLHash: hashDomainXML(data), + ObservedAt: time.Now().UTC(), + }, nil +} + +func (s *staticXMLBackend) DeleteDomain(context.Context, string) (DomainDeleteResult, error) { + return DomainDeleteResult{}, errors.New("not implemented") +} + func domainPolicyFromDefinition(data []byte) error { // This helper intentionally uses the same exported policy validator that a // real binding must call after libvirt XMLDesc read-back. diff --git a/agent/internal/hypervisor/delete_test.go b/agent/internal/hypervisor/delete_test.go new file mode 100644 index 00000000..64f2474b --- /dev/null +++ b/agent/internal/hypervisor/delete_test.go @@ -0,0 +1,170 @@ +package hypervisor + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" +) + +func TestFakeDeleteDomainRequiresInactiveAndProvesAbsence(t *testing.T) { + backend := NewFakeBackend(DefaultCapabilitySnapshot()) + if _, err := backend.DefineDomain(context.Background(), DefineRequest{ + UUID: testUUID, + XML: testDomainXML(testUUID), + }); err != nil { + t.Fatal(err) + } + preflight, err := backend.InspectDomainForDeletion(context.Background(), testUUID) + if err != nil { + t.Fatal(err) + } + if !preflight.Present || preflight.Active || preflight.State != StateShutoff || preflight.XMLHash == "" { + t.Fatalf("preflight = %+v", preflight) + } + + result, err := backend.DeleteDomain(context.Background(), strings.ToUpper(testUUID)) + if err != nil { + t.Fatalf("DeleteDomain() error = %v", err) + } + if result.UUID != testUUID || !result.Existed || !result.Removed || result.AlreadyAbsent || + result.StorageRemoved || result.NetworkChanged || result.XMLHash != preflight.XMLHash { + t.Fatalf("delete result = %+v", result) + } + after, err := backend.InspectDomainForDeletion(context.Background(), testUUID) + if err != nil { + t.Fatal(err) + } + if after.Present || after.UUID != testUUID { + t.Fatalf("post-delete observation = %+v", after) + } + if _, err := backend.ReadDomainXML(context.Background(), testUUID); !errors.Is(err, ErrDomainNotFound) { + t.Fatalf("ReadDomainXML() after delete = %v, want ErrDomainNotFound", err) + } + + // A repeated request is an idempotent, already-absent observation; it does + // not resurrect or touch any unrelated host resource. + repeated, err := backend.DeleteDomain(context.Background(), testUUID) + if err != nil { + t.Fatalf("repeated DeleteDomain() error = %v", err) + } + if !repeated.AlreadyAbsent || !repeated.Removed || repeated.Existed { + t.Fatalf("repeated delete result = %+v", repeated) + } +} + +func TestFakeDeleteDomainRejectsActiveWithoutRemovingDefinition(t *testing.T) { + backend := NewFakeBackend(DefaultCapabilitySnapshot()) + if _, err := backend.DefineDomain(context.Background(), DefineRequest{UUID: testUUID, XML: testDomainXML(testUUID)}); err != nil { + t.Fatal(err) + } + if _, err := backend.PowerDomain(context.Background(), testUUID, PowerActionStart); err != nil { + t.Fatal(err) + } + result, err := backend.DeleteDomain(context.Background(), testUUID) + if !errors.Is(err, ErrDomainActive) { + t.Fatalf("DeleteDomain() error = %v, want ErrDomainActive", err) + } + if result.Removed || !result.Existed || result.StateBefore != StateRunning { + t.Fatalf("active delete result = %+v", result) + } + observation, err := backend.InspectDomainForDeletion(context.Background(), testUUID) + if err != nil || !observation.Present || !observation.Active { + t.Fatalf("active definition after rejected delete = %+v, err=%v", observation, err) + } +} + +type deleteRecordingRunner struct { + mu sync.Mutex + commands [][]string + present bool + state []byte + readbackErr error + undefineErr error + returnNotFound bool +} + +func (r *deleteRecordingRunner) Run(_ context.Context, args ...string) ([]byte, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.commands = append(r.commands, append([]string(nil), args...)) + if len(args) < 3 { + return nil, fmt.Errorf("missing virsh command") + } + switch args[2] { + case "dumpxml": + if !r.present || r.returnNotFound { + if !r.present && r.readbackErr != nil { + return nil, r.readbackErr + } + return nil, ErrDomainNotFound + } + return []byte(`` + testUUID + `hvm`), nil + case "domstate": + if !r.present { + return nil, ErrDomainNotFound + } + return append([]byte(nil), r.state...), nil + case "undefine": + if r.undefineErr != nil { + return nil, r.undefineErr + } + r.present = false + return []byte("undefined"), nil + default: + return nil, fmt.Errorf("unexpected command %q", args[2]) + } +} + +func TestVirshDeleteDomainUsesCanonicalUndefineAndAbsenceReadback(t *testing.T) { + runner := &deleteRecordingRunner{present: true, state: []byte("shut off\n")} + backend, err := NewVirshBackendWithRunner("qemu:///system", runner) + if err != nil { + t.Fatal(err) + } + result, err := backend.DeleteDomain(context.Background(), strings.ToUpper(testUUID)) + if err != nil { + t.Fatalf("DeleteDomain() error = %v", err) + } + if !result.Existed || !result.Removed || result.StorageRemoved || result.NetworkChanged { + t.Fatalf("result = %+v", result) + } + runner.mu.Lock() + commands := append([][]string(nil), runner.commands...) + runner.mu.Unlock() + want := [][]string{ + {"-c", "qemu:///system", "dumpxml", "--inactive", testUUID}, + {"-c", "qemu:///system", "domstate", "--domain", testUUID}, + {"-c", "qemu:///system", "undefine", "--domain", testUUID}, + {"-c", "qemu:///system", "dumpxml", "--inactive", testUUID}, + } + if len(commands) != len(want) { + t.Fatalf("commands = %+v, want %+v", commands, want) + } + for i := range want { + if strings.Join(commands[i], "\x00") != strings.Join(want[i], "\x00") { + t.Fatalf("command[%d] = %+v, want %+v", i, commands[i], want[i]) + } + } +} + +func TestVirshDeleteDomainLeavesUncertainWhenAbsenceReadbackFails(t *testing.T) { + runner := &deleteRecordingRunner{ + present: true, + state: []byte("shut off\n"), + readbackErr: errors.New("transport lost"), + } + backend, err := NewVirshBackendWithRunner("qemu:///system", runner) + if err != nil { + t.Fatal(err) + } + result, err := backend.DeleteDomain(context.Background(), testUUID) + if !errors.Is(err, ErrDeleteReadbackMismatch) { + t.Fatalf("DeleteDomain() error = %v, want ErrDeleteReadbackMismatch", err) + } + if result.Removed { + t.Fatalf("ambiguous delete was reported removed: %+v", result) + } +} diff --git a/agent/internal/hypervisor/power.go b/agent/internal/hypervisor/power.go index 2ef20928..f7c0156c 100644 --- a/agent/internal/hypervisor/power.go +++ b/agent/internal/hypervisor/power.go @@ -248,6 +248,11 @@ func powerResult(uuid string, action PowerAction, state DomainPowerState) PowerR // returns the state that was immediately observed. It mirrors the real // adapter's read-back contract while keeping tests deterministic. func (f *FakeBackend) PowerDomain(ctx context.Context, uuid string, action PowerAction) (PowerResult, error) { + if err := contextErr(ctx); err != nil { + return PowerResult{}, err + } + f.mutationMu.Lock() + defer f.mutationMu.Unlock() if err := contextErr(ctx); err != nil { return PowerResult{}, err } diff --git a/agent/internal/hypervisor/virsh.go b/agent/internal/hypervisor/virsh.go index 3b352422..14698a98 100644 --- a/agent/internal/hypervisor/virsh.go +++ b/agent/internal/hypervisor/virsh.go @@ -11,7 +11,6 @@ import ( "context" "errors" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -97,17 +96,37 @@ func (r ExecRunner) Run(ctx context.Context, args ...string) ([]byte, error) { command := exec.CommandContext(ctx, binary, args...) command.Env = fixedVirshEnv() stdout := boundedBuffer{limit: maxVirshOutput} + stderr := boundedBuffer{limit: maxVirshOutput} command.Stdout = &stdout - command.Stderr = io.Discard + // Capture stderr only inside the adapter so a fixed, locale-pinned + // "domain not found" result can be distinguished from a transport failure. + // The bytes are never returned or included in an error message; virsh may + // otherwise disclose host paths, storage names, or authentication details. + command.Stderr = &stderr if err := command.Run(); err != nil { if errors.Is(ctx.Err(), context.Canceled) || errors.Is(ctx.Err(), context.DeadlineExceeded) { return nil, ctx.Err() } + if virshReportsMissingDomain(stderr.Bytes()) { + return nil, ErrDomainNotFound + } return nil, fmt.Errorf("%w: virsh command failed", ErrVirshUnavailable) } return append([]byte(nil), stdout.Bytes()...), nil } +// virshReportsMissingDomain is intentionally narrow and only consumes +// diagnostics from the adapter's own LC_ALL=C subprocess. It provides a +// typed absence read-back for delete/idempotent probes without exposing the +// diagnostic text to the panel. Any wording not listed here remains an +// opaque transport failure and therefore uncertain. +func virshReportsMissingDomain(data []byte) bool { + message := strings.ToLower(string(data)) + return strings.Contains(message, "domain not found") || + strings.Contains(message, "no domain with matching uuid") || + strings.Contains(message, "no domain with matching name") +} + // VirshBackend is a host-local implementation of Backend. It never changes // host networking; domain definitions and the explicitly allowlisted power // actions are the only mutating operations, and every one performs a typed @@ -448,6 +467,120 @@ func (b *VirshBackend) ReadDomainXML(ctx context.Context, uuid string) ([]byte, return cloneBytes(data), nil } +// InspectDomainForDeletion performs a targeted, read-only preflight. It does +// not enumerate domains and never invokes undefine, lifecycle, storage, or +// networking commands. A typed ErrDomainNotFound from the fixed virsh adapter +// is normalized into an absent observation; all other command/read failures +// stay opaque and are left for the operation layer to mark uncertain. +func (b *VirshBackend) InspectDomainForDeletion(ctx context.Context, uuid string) (DomainDeletionObservation, error) { + if err := contextErr(ctx); err != nil { + return DomainDeletionObservation{}, err + } + canonical, err := CanonicalUUID(uuid) + if err != nil { + return DomainDeletionObservation{}, err + } + xmlData, err := b.ReadDomainXML(ctx, canonical) + if errors.Is(err, ErrDomainNotFound) { + return DomainDeletionObservation{ + UUID: canonical, + Present: false, + ObservedAt: time.Now().UTC(), + }, nil + } + if err != nil { + return DomainDeletionObservation{}, err + } + observed, err := domainUUID(xmlData) + if err != nil || observed != canonical { + return DomainDeletionObservation{}, fmt.Errorf("%w: requested %s", ErrReadbackMismatch, canonical) + } + state, err := b.readDomainState(ctx, canonical) + if err != nil { + // The definition was observed but its active/inactive state was not. Do + // not guess that it is safe to undefine; the caller must reconcile. + return DomainDeletionObservation{}, fmt.Errorf("%w: domain state read-back did not complete", ErrDeleteReadbackMismatch) + } + return DomainDeletionObservation{ + UUID: canonical, + Present: true, + State: state, + Active: state.IsActive(), + XMLHash: hashDomainXML(xmlData), + ObservedAt: time.Now().UTC(), + }, nil +} + +// DeleteDomain undefines one inactive domain and nothing else. The fixed +// virsh argv contains only the canonical UUID; no shell, XML, disk path, or +// network operation is accepted. Success requires a second targeted +// read-back proving that the definition is absent. If the command or that +// read-back is ambiguous, the returned error is deliberately non-definitive +// so the API can retain an uncertain journal record instead of replaying. +func (b *VirshBackend) DeleteDomain(ctx context.Context, uuid string) (DomainDeleteResult, error) { + if err := contextErr(ctx); err != nil { + return DomainDeleteResult{}, err + } + b.mutationMu.Lock() + defer b.mutationMu.Unlock() + if err := contextErr(ctx); err != nil { + return DomainDeleteResult{}, err + } + canonical, err := CanonicalUUID(uuid) + if err != nil { + return DomainDeleteResult{}, err + } + before, err := b.InspectDomainForDeletion(ctx, canonical) + if err != nil { + return DomainDeleteResult{UUID: canonical}, err + } + if !before.Present { + return DomainDeleteResult{ + UUID: canonical, + AlreadyAbsent: true, + Removed: true, + ObservedAt: time.Now().UTC(), + }, nil + } + base := DomainDeleteResult{ + UUID: canonical, + Existed: true, + StateBefore: before.State, + XMLHash: before.XMLHash, + StorageRemoved: false, + NetworkChanged: false, + } + if before.Active { + base.ObservedAt = time.Now().UTC() + return base, fmt.Errorf("%w: %s", ErrDomainActive, canonical) + } + _, commandErr := b.runner.Run(ctx, b.args("undefine", "--domain", canonical)...) + if err := contextErr(ctx); err != nil { + base.ObservedAt = time.Now().UTC() + return base, err + } + after, readErr := b.InspectDomainForDeletion(ctx, canonical) + if readErr != nil { + base.ObservedAt = time.Now().UTC() + return base, fmt.Errorf("%w: post-undefine observation failed", ErrDeleteReadbackMismatch) + } + if after.Present { + base.ObservedAt = after.ObservedAt + return base, fmt.Errorf("%w: %s", ErrDeleteReadbackMismatch, canonical) + } + // Absence proves the definition boundary even when virsh returned a + // non-zero status (for example, an idempotent race with another actor). + base.Removed = true + base.ObservedAt = after.ObservedAt + if commandErr != nil { + // Deliberately accept only the authoritative absence read-back here; no + // command error text is surfaced. This keeps retries idempotent without + // turning a transport failure into a blind replay. + return base, nil + } + return base, nil +} + // PowerDomain executes one fixed virsh lifecycle subcommand and then reads // domstate back through the same local connection. A command error is not // blindly replayed: for idempotent actions, an authoritative desired state is diff --git a/docs/GO_AGENT_DOMAIN_DELETE.md b/docs/GO_AGENT_DOMAIN_DELETE.md new file mode 100644 index 00000000..38733966 --- /dev/null +++ b/docs/GO_AGENT_DOMAIN_DELETE.md @@ -0,0 +1,33 @@ +# Go Agent `domain_delete` contract + +This slice implements a narrow, identity-bound **libvirt definition undefine** +operation. It is not the panel's full VM deletion workflow. + +## Host boundary + +- The request payload is exactly `{ "uuid": "" }`. +- `InspectDomainForDeletion` performs a targeted read-only `dumpxml --inactive` + and `domstate` observation. It does not enumerate domains or touch storage, + networking, firewall state, or seed artifacts. +- `DeleteDomain` refuses an active domain. It invokes only + `virsh undefine --domain ` and then requires a targeted + absence read-back before returning success. +- Fake and virsh backends report only a SHA-256 XML digest; raw XML and host + paths never enter the operation result or journal. +- `storage_removed=false` and `network_changed=false` are explicit result + facts. The operation never deletes disks or changes a network. + +If the undefine command or absence read-back is ambiguous, the operation is +`uncertain`. Reconciliation calls only `InspectDomainForDeletion`; it never +replays `undefine`. A present definition, failed transport, or missing state +read-back therefore cannot become a false success. + +## PHP boundary + +The Go API accepts `domain_delete`, but the PHP `CloudAgentVmGateway` keeps the +legacy panel `delete` mapping disabled (`null`). The existing panel operation +also owns disk/seed cleanup, billing state, firewall history, and irreversible +instance tombstoning. Mapping it to definition-only undefine would report a +successful VM deletion while leaving those resources behind. A future mapping +requires a separate typed storage/seed/network cleanup contract, panel CAS +updates, and real-host read-back evidence.