feat: add typed Go Agent domain delete boundary

go-agent
admin 1 month ago
parent 41ee38b9f2
commit 0599909d28

@ -381,7 +381,7 @@ func (s *Server) createOperation(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "domain define is disabled until the typed create executor is available"})
return
}
if request.Kind != "capability_probe" && request.Kind != "domain_read" && request.Kind != "domain_observe" && request.Kind != "domain_power" && request.Kind != "seed_prepare" && request.Kind != "domain_preflight" && request.Kind != "domain_create_plan" && request.Kind != "domain_create_validate" {
if request.Kind != "capability_probe" && request.Kind != "domain_read" && request.Kind != "domain_observe" && request.Kind != "domain_power" && request.Kind != "domain_delete" && request.Kind != "seed_prepare" && request.Kind != "domain_preflight" && request.Kind != "domain_create_plan" && request.Kind != "domain_create_validate" {
writeJSON(w, http.StatusNotImplemented, errorResponse{Error: "operation kind is not implemented by the agent"})
return
}
@ -426,6 +426,13 @@ func (s *Server) createOperation(w http.ResponseWriter, r *http.Request) {
return
}
}
if request.Kind == "domain_delete" {
uuid, parseErr := parseDomainDeletePayload(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, uuid) {
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"})
return
}
}
if request.Kind == "domain_read" {
uuid, parseErr := parseDomainReadPayload(request.Payload)
if parseErr != nil || !targetMatchesUUID(request.Target, uuid) {
@ -526,6 +533,10 @@ func (s *Server) createOperation(w http.ResponseWriter, r *http.Request) {
s.completeDomainPower(w, record, request, ctx)
return
}
if request.Kind == "domain_delete" {
s.completeDomainDelete(w, record, request, ctx)
return
}
if request.Kind == "seed_prepare" {
s.completeSeedPrepare(w, record, request, ctx)
return
@ -596,6 +607,13 @@ type domainPowerPayload struct {
Action string `json:"action"`
}
// domainDeletePayload is intentionally only an exact UUID. Disk cleanup,
// network teardown, and panel billing state are separate contracts and are
// not accepted by the host Agent's narrow undefine operation.
type domainDeletePayload struct {
UUID string `json:"uuid"`
}
type seedPreparePayload struct {
ArtifactID string `json:"artifact_id"`
InstanceID string `json:"instance_id"`
@ -631,6 +649,10 @@ type domainObserveResult struct {
ObservedAt time.Time `json:"observed_at"`
}
type domainDeleteJournalPayload struct {
UUID string `json:"uuid"`
}
type domainPowerJournalPayload struct {
UUID string `json:"uuid"`
Action string `json:"action"`
@ -683,6 +705,12 @@ func safeJournalPayload(kind string, raw json.RawMessage) (json.RawMessage, erro
return nil, err
}
return json.Marshal(domainPowerJournalPayload{UUID: power.UUID, Action: string(power.Action)})
case "domain_delete":
uuid, err := parseDomainDeletePayload(raw)
if err != nil {
return nil, err
}
return json.Marshal(domainDeleteJournalPayload{UUID: uuid})
case "seed_prepare":
payload, err := parseSeedPrepareRequest(raw)
if err != nil {
@ -800,6 +828,14 @@ func parseDomainPowerRequest(raw json.RawMessage) (hypervisor.PowerRequest, erro
return hypervisor.PowerRequest{UUID: canonical, Action: action}, nil
}
func parseDomainDeletePayload(raw json.RawMessage) (string, error) {
var payload domainDeletePayload
if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" {
return "", fmt.Errorf("invalid domain delete payload")
}
return hypervisor.CanonicalUUID(payload.UUID)
}
func parseDomainReadPayload(raw json.RawMessage) (string, error) {
var payload domainReadPayload
if err := decodeJSON(raw, &payload); err != nil || strings.TrimSpace(payload.UUID) == "" {
@ -1086,6 +1122,60 @@ func (s *Server) completeDomainPower(w http.ResponseWriter, record operation.Rec
writeJSON(w, http.StatusOK, updated)
}
// completeDomainDelete performs only the typed, inactive-domain undefine
// boundary. The hypervisor implementation must prove absence with a targeted
// read-back before this method can persist success. Disk images, seed
// artifacts, firewall rules, and host networking are deliberately untouched;
// the panel's full VM cleanup contract is not represented here.
func (s *Server) completeDomainDelete(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
uuid, err := parseDomainDeletePayload(request.Payload)
if err != nil || !targetMatchesUUID(request.Target, uuid) {
_, _ = s.store.Update(record.OperationID, operation.StatusFailed, nil, "invalid_domain_delete_request", "domain delete payload is invalid")
writeJSON(w, http.StatusBadRequest, errorResponse{Error: "invalid domain delete payload"})
return
}
result, err := s.backend.DeleteDomain(ctx, uuid)
var resultJSON json.RawMessage
if result.UUID != "" {
resultJSON, _ = json.Marshal(result)
}
if err != nil {
status := operation.StatusUncertain
httpStatus := http.StatusServiceUnavailable
errorCode := "domain_delete_uncertain"
message := "domain undefine or absence read-back did not complete"
if errors.Is(err, hypervisor.ErrInvalidUUID) || errors.Is(err, hypervisor.ErrDomainActive) {
status = operation.StatusFailed
httpStatus = http.StatusUnprocessableEntity
if errors.Is(err, hypervisor.ErrDomainActive) {
errorCode = "domain_delete_active"
message = "domain must be inactive before undefine"
} else {
errorCode = "invalid_domain_delete_request"
message = "domain delete UUID is invalid"
}
}
updated, updateErr := s.store.Update(record.OperationID, status, resultJSON, errorCode, message)
if updateErr == nil {
writeJSON(w, httpStatus, updated)
} else {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete failed and operation state is uncertain"})
}
return
}
if _, boundaryErr := s.store.SetCommitBoundary(record.OperationID, "domain_delete_readback_confirmed"); boundaryErr != nil {
_, _ = s.store.Update(record.OperationID, operation.StatusUncertain, resultJSON, "operation_journal_update_failed", "domain absence was confirmed but operation state could not be committed")
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"})
return
}
updated, err := s.store.Update(record.OperationID, operation.StatusSucceeded, resultJSON, "", "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "domain delete result could not be committed"})
return
}
writeJSON(w, http.StatusOK, updated)
}
func (s *Server) completeSeedPrepare(w http.ResponseWriter, record operation.Record, request operationRequest, ctx context.Context) {
payload, err := parseSeedPrepareRequest(request.Payload)
if err != nil {
@ -1429,6 +1519,8 @@ func (s *Server) reconcileOperation(w http.ResponseWriter, r *http.Request) {
s.reconcileDomainObserve(w, record, r)
case "domain_power":
s.reconcileDomainPower(w, record, r)
case "domain_delete":
s.reconcileDomainDelete(w, record, r)
case "seed_prepare":
s.reconcileSeedPrepare(w, record, r)
case "domain_preflight":
@ -1866,6 +1958,49 @@ func (s *Server) reconcileDomainPower(w http.ResponseWriter, record operation.Re
writeJSON(w, http.StatusOK, updated)
}
// reconcileDomainDelete is strictly read-only. It observes whether the exact
// definition UUID is absent and never retries undefine after an ambiguous
// response. A present definition, an active guest, or a failed observation
// keeps the journal uncertain for a later operator-approved read-back.
func (s *Server) reconcileDomainDelete(w http.ResponseWriter, record operation.Record, r *http.Request) {
if s.backend == nil {
s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "host backend is not configured for domain deletion read-back", http.StatusServiceUnavailable)
return
}
uuid, err := parseDomainDeletePayload(record.Payload)
if err != nil || !targetMatchesUUID(record.Target, uuid) {
s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal payload is invalid")
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
defer cancel()
observation, err := s.backend.InspectDomainForDeletion(ctx, uuid)
if err != nil {
if errors.Is(err, hypervisor.ErrInvalidUUID) {
s.reconcileInvalidJournal(w, record, "invalid_domain_delete_request", "domain delete journal UUID is invalid")
return
}
s.reconcileUncertain(w, record, record.Result, "domain_delete_reconcile_uncertain", "domain deletion absence read-back did not complete", http.StatusServiceUnavailable)
return
}
if observation.Present {
result, _ := json.Marshal(observation)
s.reconcileUncertain(w, record, result, "domain_delete_not_reconciled", "domain definition is still present; no undefine was replayed", http.StatusConflict)
return
}
result, marshalErr := json.Marshal(hypervisor.DomainDeleteResult{
UUID: uuid,
AlreadyAbsent: true,
Removed: true,
ObservedAt: observation.ObservedAt,
})
if marshalErr != nil {
s.reconcileUncertain(w, record, record.Result, "domain_delete_result_failed", "domain deletion result could not be encoded", http.StatusServiceUnavailable)
return
}
s.reconcileSuccess(w, record, "domain_delete_readback_confirmed", result)
}
func (s *Server) storeRecord(id string) (operation.Record, error) {
if s.store == nil {
return operation.Record{}, operation.ErrNotFound

@ -0,0 +1,204 @@
package api
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"g.luoli.live/root/virt/agent/internal/capability"
"g.luoli.live/root/virt/agent/internal/config"
"g.luoli.live/root/virt/agent/internal/hypervisor"
"g.luoli.live/root/virt/agent/internal/operation"
)
const deleteTestUUID = "00000000-0000-4000-8000-000000000401"
func newDeleteTestServer(t *testing.T, backend hypervisor.Backend) (*Server, *operation.Store) {
t.Helper()
store := operation.NewMemory()
c := config.Defaults()
c.AgentID = "host-delete-test"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
return s, store
}
func defineDeleteTestDomain(t *testing.T, backend *hypervisor.FakeBackend) {
t.Helper()
if _, err := backend.DefineDomain(context.Background(), hypervisor.DefineRequest{
UUID: deleteTestUUID,
XML: []byte(`<domain><uuid>` + deleteTestUUID + `</uuid><os><type>hvm</type></os></domain>`),
}); err != nil {
t.Fatal(err)
}
}
func deleteOperationBody(id, key string) []byte {
return []byte(`{"operation_id":"` + id + `","idempotency_key":"` + key + `","kind":"domain_delete","target":"` + deleteTestUUID + `","payload":{"uuid":"` + deleteTestUUID + `"}}`)
}
func TestDomainDeleteUndefinesOnlyInactiveDefinitionAndJournalsReadback(t *testing.T) {
backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
defineDeleteTestDomain(t, backend)
s, store := newDeleteTestServer(t, backend)
operationID := "00000000-0000-4000-8000-000000000402"
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(
deleteOperationBody(operationID, "delete-operation-0402"),
)))
if recorder.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
var record operation.Record
if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusSucceeded || record.CommitBoundary != "domain_delete_readback_confirmed" {
t.Fatalf("record=%+v", record)
}
var result hypervisor.DomainDeleteResult
if err := json.Unmarshal(record.Result, &result); err != nil {
t.Fatalf("decode result: %v raw=%s", err, record.Result)
}
if result.UUID != deleteTestUUID || !result.Existed || !result.Removed || result.AlreadyAbsent ||
result.StorageRemoved || result.NetworkChanged || result.XMLHash == "" {
t.Fatalf("result=%+v", result)
}
observation, err := backend.InspectDomainForDeletion(context.Background(), deleteTestUUID)
if err != nil {
t.Fatal(err)
}
if observation.Present {
t.Fatalf("definition remains after successful delete: %+v", observation)
}
if _, err := store.Get(operationID); err != nil {
t.Fatal(err)
}
}
func TestDomainDeleteRejectsActiveDomainBeforeUndefine(t *testing.T) {
backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
defineDeleteTestDomain(t, backend)
if _, err := backend.PowerDomain(context.Background(), deleteTestUUID, hypervisor.PowerActionStart); err != nil {
t.Fatal(err)
}
s, store := newDeleteTestServer(t, backend)
operationID := "00000000-0000-4000-8000-000000000403"
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(
deleteOperationBody(operationID, "delete-operation-0403"),
)))
if recorder.Code != http.StatusUnprocessableEntity {
t.Fatalf("status=%d body=%s; want 422", recorder.Code, recorder.Body.String())
}
record, err := store.Get(operationID)
if err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusFailed || record.ErrorCode != "domain_delete_active" {
t.Fatalf("record=%+v", record)
}
observation, err := backend.InspectDomainForDeletion(context.Background(), deleteTestUUID)
if err != nil || !observation.Present || !observation.Active {
t.Fatalf("domain changed after active rejection: %+v err=%v", observation, err)
}
}
type deleteCountingBackend struct {
*hypervisor.FakeBackend
deleteCalls int
inspectCalls int
}
func (b *deleteCountingBackend) DeleteDomain(ctx context.Context, uuid string) (hypervisor.DomainDeleteResult, error) {
b.deleteCalls++
return b.FakeBackend.DeleteDomain(ctx, uuid)
}
func (b *deleteCountingBackend) InspectDomainForDeletion(ctx context.Context, uuid string) (hypervisor.DomainDeletionObservation, error) {
b.inspectCalls++
return b.FakeBackend.InspectDomainForDeletion(ctx, uuid)
}
func TestDomainDeleteReconcileReadsAbsenceWithoutReplayingUndefine(t *testing.T) {
base := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
defineDeleteTestDomain(t, base)
backend := &deleteCountingBackend{FakeBackend: base}
store := operation.NewMemory()
const operationID = "00000000-0000-4000-8000-000000000404"
payload := []byte(`{"uuid":"` + deleteTestUUID + `"}`)
journalUncertain(t, store, operationID, "delete-reconcile-0404", "domain_delete", deleteTestUUID, payload)
s, _ := newDeleteTestServerWithStore(t, backend, store)
first := httptest.NewRecorder()
s.Handler().ServeHTTP(first, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil))
if first.Code != http.StatusConflict {
t.Fatalf("first reconcile status=%d body=%s; want 409", first.Code, first.Body.String())
}
if backend.deleteCalls != 0 || backend.inspectCalls != 1 {
t.Fatalf("first reconcile calls delete=%d inspect=%d; undefine must not replay", backend.deleteCalls, backend.inspectCalls)
}
if _, err := base.DeleteDomain(context.Background(), deleteTestUUID); err != nil {
t.Fatal(err)
}
second := httptest.NewRecorder()
s.Handler().ServeHTTP(second, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil))
if second.Code != http.StatusOK {
t.Fatalf("second reconcile status=%d body=%s", second.Code, second.Body.String())
}
record, err := store.Get(operationID)
if err != nil {
t.Fatal(err)
}
if record.Status != operation.StatusSucceeded || record.CommitBoundary != "domain_delete_readback_confirmed" {
t.Fatalf("record=%+v", record)
}
if backend.deleteCalls != 0 || backend.inspectCalls != 2 {
t.Fatalf("second reconcile calls delete=%d inspect=%d; want read-only reconcile", backend.deleteCalls, backend.inspectCalls)
}
}
func newDeleteTestServerWithStore(t *testing.T, backend hypervisor.Backend, store *operation.Store) (*Server, *operation.Store) {
t.Helper()
c := config.Defaults()
c.AgentID = "host-delete-reconcile"
s, err := New(Options{
Config: c,
Store: store,
Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion},
Backend: backend,
HostBackendReady: true,
})
if err != nil {
t.Fatal(err)
}
return s, store
}
func TestDomainDeleteRejectsExtraPayloadBeforeJournal(t *testing.T) {
backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot())
s, store := newDeleteTestServer(t, backend)
operationID := "00000000-0000-4000-8000-000000000405"
body := []byte(`{"operation_id":"` + operationID + `","idempotency_key":"delete-operation-0405","kind":"domain_delete","target":"` + deleteTestUUID + `","payload":{"uuid":"` + deleteTestUUID + `","delete_disks":true}}`)
recorder := httptest.NewRecorder()
s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body)))
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status=%d body=%s; want 400", recorder.Code, recorder.Body.String())
}
if _, err := store.Get(operationID); !errors.Is(err, operation.ErrNotFound) {
t.Fatalf("invalid delete entered journal: %v", err)
}
}

@ -9,6 +9,8 @@ package hypervisor
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"errors"
"fmt"
@ -32,6 +34,16 @@ var (
ErrDomainNotFound = errors.New("domain definition not found")
// ErrDomainConflict means an existing UUID was defined with different XML.
ErrDomainConflict = errors.New("domain definition conflicts with existing UUID")
// ErrDomainActive means an undefine request was rejected because the
// targeted domain is still active. The Agent never force-destroys a guest
// as part of deletion; callers must perform an explicit, separately
// journaled power operation first.
ErrDomainActive = errors.New("domain is active")
// ErrDeleteReadbackMismatch means an undefine command returned but the
// subsequent targeted read-back could not prove that the definition is
// absent. It is intentionally not a definitive error: the operation may
// have committed and must remain uncertain until a later observation.
ErrDeleteReadbackMismatch = errors.New("domain deletion read-back mismatch")
// ErrReadbackMismatch means the backend returned XML that cannot prove the
// requested UUID and firmware policy were retained.
ErrReadbackMismatch = errors.New("domain definition read-back mismatch")
@ -57,6 +69,7 @@ func IsDefinitive(err error) bool {
return errors.Is(err, ErrInvalidUUID) ||
errors.Is(err, ErrUUIDMismatch) ||
errors.Is(err, ErrDomainConflict) ||
errors.Is(err, ErrDomainActive) ||
errors.Is(err, ErrInvalidPowerAction) ||
errors.Is(err, ErrPowerActionRejected) ||
errors.Is(err, ErrCapabilityPolicy) ||
@ -162,6 +175,35 @@ type Definition struct {
DefinedAt time.Time `json:"defined_at"`
}
// DomainDeletionObservation is a read-only, identity-bound snapshot used by
// delete preflight and uncertain-operation reconciliation. It deliberately
// contains only a digest of the inactive XML; host paths, disk names and
// other definition details never cross the Agent API boundary.
type DomainDeletionObservation struct {
UUID string `json:"uuid"`
Present bool `json:"present"`
State DomainPowerState `json:"state,omitempty"`
Active bool `json:"active"`
XMLHash string `json:"xml_hash,omitempty"`
ObservedAt time.Time `json:"observed_at"`
}
// DomainDeleteResult records the narrow mutation performed by DeleteDomain.
// Removed means only the libvirt domain definition was undefined. Storage
// and network objects are explicitly reported untouched so a caller cannot
// mistake this operation for the panel's full VM cleanup workflow.
type DomainDeleteResult struct {
UUID string `json:"uuid"`
Existed bool `json:"existed"`
AlreadyAbsent bool `json:"already_absent"`
Removed bool `json:"removed"`
StateBefore DomainPowerState `json:"state_before,omitempty"`
XMLHash string `json:"xml_hash,omitempty"`
StorageRemoved bool `json:"storage_removed"`
NetworkChanged bool `json:"network_changed"`
ObservedAt time.Time `json:"observed_at"`
}
// Backend is the intentionally small host boundary. Methods are context
// aware because a panel request timeout must cancel an in-flight RPC without
// leaving an unbounded libvirt call behind.
@ -178,6 +220,14 @@ type Backend interface {
// ReadDomainPowerState is a read-only host probe used by reconciliation.
// It must never invoke a lifecycle command or otherwise mutate the domain.
ReadDomainPowerState(context.Context, string) (DomainPowerState, error)
// InspectDomainForDeletion is a read-only, exact-UUID preflight. It must
// never undefine, power off, remove storage, or touch host networking.
InspectDomainForDeletion(context.Context, string) (DomainDeletionObservation, error)
// DeleteDomain only undefines one inactive domain definition. Implementations
// must perform a targeted absence read-back before returning success; any
// lost/ambiguous read-back must return an error that the API journals as
// uncertain. Disk and network cleanup are outside this seam.
DeleteDomain(context.Context, string) (DomainDeleteResult, error)
}
// LoaderPathProvider is an optional host-bound capability seam. Implementations
@ -227,6 +277,9 @@ type FakeBackend struct {
domains map[string]Definition
states map[string]DomainPowerState
nextRev uint64
// mutationMu serializes define, lifecycle and undefine sequences. Reads do
// not take this lock and remain safe through mu's snapshot semantics.
mutationMu sync.Mutex
}
// NewFakeBackend creates a backend with the supplied capability snapshot.
@ -311,6 +364,13 @@ func (f *FakeBackend) PreferredNonSecureLoaderPath(ctx context.Context) (string,
// identical UUID/XML pair is idempotent; redefining the UUID with different
// XML is rejected rather than overwriting an unknown guest.
func (f *FakeBackend) DefineDomain(ctx context.Context, request DefineRequest) (Definition, error) {
if err := contextErr(ctx); err != nil {
return Definition{}, err
}
f.mutationMu.Lock()
defer f.mutationMu.Unlock()
// A request can wait behind another host mutation. Re-check cancellation
// before beginning a new definition sequence.
if err := contextErr(ctx); err != nil {
return Definition{}, err
}
@ -401,6 +461,136 @@ func (f *FakeBackend) ReadDefinition(ctx context.Context, uuid string) (Definiti
return cloneDefinition(definition), nil
}
// InspectDomainForDeletion performs the exact-UUID, read-only preflight used
// by DeleteDomain and its reconcile path. An absent definition is a valid
// observation rather than an error: delete is idempotent at the definition
// boundary, while transport/read failures remain errors and therefore
// uncertain to the operation layer.
func (f *FakeBackend) InspectDomainForDeletion(ctx context.Context, uuid string) (DomainDeletionObservation, error) {
if err := contextErr(ctx); err != nil {
return DomainDeletionObservation{}, err
}
canonical, err := CanonicalUUID(uuid)
if err != nil {
return DomainDeletionObservation{}, err
}
f.mu.RLock()
definition, present := f.domains[canonical]
state := f.states[canonical]
f.mu.RUnlock()
if !present {
return DomainDeletionObservation{
UUID: canonical,
Present: false,
ObservedAt: time.Now().UTC(),
}, nil
}
if err := contextErr(ctx); err != nil {
return DomainDeletionObservation{}, err
}
if state == "" {
state = StateShutoff
}
observed, err := domainUUID(definition.XML)
if err != nil || observed != canonical {
return DomainDeletionObservation{}, fmt.Errorf("%w: requested %s", ErrReadbackMismatch, canonical)
}
return DomainDeletionObservation{
UUID: canonical,
Present: true,
State: state,
Active: state.IsActive(),
XMLHash: hashDomainXML(definition.XML),
ObservedAt: time.Now().UTC(),
}, nil
}
// DeleteDomain removes only one inactive definition. The operation is fenced
// by the fake's mutation lock and proves absence from a fresh read-back before
// returning success. No disk or network map exists in this backend, so those
// result fields remain explicit false values.
func (f *FakeBackend) DeleteDomain(ctx context.Context, uuid string) (DomainDeleteResult, error) {
if err := contextErr(ctx); err != nil {
return DomainDeleteResult{}, err
}
canonical, err := CanonicalUUID(uuid)
if err != nil {
return DomainDeleteResult{}, err
}
f.mutationMu.Lock()
defer f.mutationMu.Unlock()
if err := contextErr(ctx); err != nil {
return DomainDeleteResult{}, err
}
observation, err := f.InspectDomainForDeletion(ctx, canonical)
if err != nil {
return DomainDeleteResult{UUID: canonical}, err
}
if !observation.Present {
return DomainDeleteResult{
UUID: canonical,
AlreadyAbsent: true,
Removed: true,
ObservedAt: time.Now().UTC(),
}, nil
}
if observation.Active {
return DomainDeleteResult{
UUID: canonical,
Existed: true,
StateBefore: observation.State,
XMLHash: observation.XMLHash,
ObservedAt: time.Now().UTC(),
}, fmt.Errorf("%w: %s", ErrDomainActive, canonical)
}
// Remove the definition only after the read-only preflight proved the
// canonical UUID and inactive state. Keep the state map in lockstep; no
// storage or network cleanup is attempted.
f.mu.Lock()
delete(f.domains, canonical)
delete(f.states, canonical)
f.mu.Unlock()
if err := contextErr(ctx); err != nil {
return DomainDeleteResult{
UUID: canonical,
Existed: true,
StateBefore: observation.State,
XMLHash: observation.XMLHash,
ObservedAt: time.Now().UTC(),
}, err
}
after, err := f.InspectDomainForDeletion(ctx, canonical)
if err != nil {
return DomainDeleteResult{
UUID: canonical,
Existed: true,
StateBefore: observation.State,
XMLHash: observation.XMLHash,
ObservedAt: time.Now().UTC(),
}, err
}
if after.Present {
return DomainDeleteResult{
UUID: canonical,
Existed: true,
StateBefore: observation.State,
XMLHash: observation.XMLHash,
ObservedAt: time.Now().UTC(),
}, fmt.Errorf("%w: %s", ErrDeleteReadbackMismatch, canonical)
}
return DomainDeleteResult{
UUID: canonical,
Existed: true,
Removed: true,
StateBefore: observation.State,
XMLHash: observation.XMLHash,
StorageRemoved: false,
NetworkChanged: false,
ObservedAt: after.ObservedAt,
}, nil
}
func contextErr(ctx context.Context) error {
if ctx == nil {
return nil
@ -449,6 +639,11 @@ func isHex(value rune) bool {
return value >= '0' && value <= '9' || value >= 'a' && value <= 'f'
}
func hashDomainXML(data []byte) string {
digest := sha256.Sum256(data)
return hex.EncodeToString(digest[:])
}
// domainUUID extracts only the direct <uuid> child. Nested metadata fields
// are not identity and must not satisfy this check.
func domainUUID(data []byte) (string, error) {

@ -7,6 +7,7 @@ import (
"strings"
"sync"
"testing"
"time"
)
const testUUID = "00000000-0000-4000-8000-000000000001"
@ -243,6 +244,32 @@ func (s *staticXMLBackend) ReadDomainPowerState(context.Context, string) (Domain
return StateShutoff, nil
}
func (s *staticXMLBackend) InspectDomainForDeletion(ctx context.Context, uuid string) (DomainDeletionObservation, error) {
data, err := s.ReadDomainXML(ctx, uuid)
if err != nil {
return DomainDeletionObservation{}, err
}
canonical, err := CanonicalUUID(uuid)
if err != nil {
return DomainDeletionObservation{}, err
}
observed, err := domainUUID(data)
if err != nil || observed != canonical {
return DomainDeletionObservation{}, ErrReadbackMismatch
}
return DomainDeletionObservation{
UUID: canonical,
Present: true,
State: StateShutoff,
XMLHash: hashDomainXML(data),
ObservedAt: time.Now().UTC(),
}, nil
}
func (s *staticXMLBackend) DeleteDomain(context.Context, string) (DomainDeleteResult, error) {
return DomainDeleteResult{}, errors.New("not implemented")
}
func domainPolicyFromDefinition(data []byte) error {
// This helper intentionally uses the same exported policy validator that a
// real binding must call after libvirt XMLDesc read-back.

@ -0,0 +1,170 @@
package hypervisor
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"testing"
)
func TestFakeDeleteDomainRequiresInactiveAndProvesAbsence(t *testing.T) {
backend := NewFakeBackend(DefaultCapabilitySnapshot())
if _, err := backend.DefineDomain(context.Background(), DefineRequest{
UUID: testUUID,
XML: testDomainXML(testUUID),
}); err != nil {
t.Fatal(err)
}
preflight, err := backend.InspectDomainForDeletion(context.Background(), testUUID)
if err != nil {
t.Fatal(err)
}
if !preflight.Present || preflight.Active || preflight.State != StateShutoff || preflight.XMLHash == "" {
t.Fatalf("preflight = %+v", preflight)
}
result, err := backend.DeleteDomain(context.Background(), strings.ToUpper(testUUID))
if err != nil {
t.Fatalf("DeleteDomain() error = %v", err)
}
if result.UUID != testUUID || !result.Existed || !result.Removed || result.AlreadyAbsent ||
result.StorageRemoved || result.NetworkChanged || result.XMLHash != preflight.XMLHash {
t.Fatalf("delete result = %+v", result)
}
after, err := backend.InspectDomainForDeletion(context.Background(), testUUID)
if err != nil {
t.Fatal(err)
}
if after.Present || after.UUID != testUUID {
t.Fatalf("post-delete observation = %+v", after)
}
if _, err := backend.ReadDomainXML(context.Background(), testUUID); !errors.Is(err, ErrDomainNotFound) {
t.Fatalf("ReadDomainXML() after delete = %v, want ErrDomainNotFound", err)
}
// A repeated request is an idempotent, already-absent observation; it does
// not resurrect or touch any unrelated host resource.
repeated, err := backend.DeleteDomain(context.Background(), testUUID)
if err != nil {
t.Fatalf("repeated DeleteDomain() error = %v", err)
}
if !repeated.AlreadyAbsent || !repeated.Removed || repeated.Existed {
t.Fatalf("repeated delete result = %+v", repeated)
}
}
func TestFakeDeleteDomainRejectsActiveWithoutRemovingDefinition(t *testing.T) {
backend := NewFakeBackend(DefaultCapabilitySnapshot())
if _, err := backend.DefineDomain(context.Background(), DefineRequest{UUID: testUUID, XML: testDomainXML(testUUID)}); err != nil {
t.Fatal(err)
}
if _, err := backend.PowerDomain(context.Background(), testUUID, PowerActionStart); err != nil {
t.Fatal(err)
}
result, err := backend.DeleteDomain(context.Background(), testUUID)
if !errors.Is(err, ErrDomainActive) {
t.Fatalf("DeleteDomain() error = %v, want ErrDomainActive", err)
}
if result.Removed || !result.Existed || result.StateBefore != StateRunning {
t.Fatalf("active delete result = %+v", result)
}
observation, err := backend.InspectDomainForDeletion(context.Background(), testUUID)
if err != nil || !observation.Present || !observation.Active {
t.Fatalf("active definition after rejected delete = %+v, err=%v", observation, err)
}
}
type deleteRecordingRunner struct {
mu sync.Mutex
commands [][]string
present bool
state []byte
readbackErr error
undefineErr error
returnNotFound bool
}
func (r *deleteRecordingRunner) Run(_ context.Context, args ...string) ([]byte, error) {
r.mu.Lock()
defer r.mu.Unlock()
r.commands = append(r.commands, append([]string(nil), args...))
if len(args) < 3 {
return nil, fmt.Errorf("missing virsh command")
}
switch args[2] {
case "dumpxml":
if !r.present || r.returnNotFound {
if !r.present && r.readbackErr != nil {
return nil, r.readbackErr
}
return nil, ErrDomainNotFound
}
return []byte(`<domain><uuid>` + testUUID + `</uuid><os><type>hvm</type></os></domain>`), nil
case "domstate":
if !r.present {
return nil, ErrDomainNotFound
}
return append([]byte(nil), r.state...), nil
case "undefine":
if r.undefineErr != nil {
return nil, r.undefineErr
}
r.present = false
return []byte("undefined"), nil
default:
return nil, fmt.Errorf("unexpected command %q", args[2])
}
}
func TestVirshDeleteDomainUsesCanonicalUndefineAndAbsenceReadback(t *testing.T) {
runner := &deleteRecordingRunner{present: true, state: []byte("shut off\n")}
backend, err := NewVirshBackendWithRunner("qemu:///system", runner)
if err != nil {
t.Fatal(err)
}
result, err := backend.DeleteDomain(context.Background(), strings.ToUpper(testUUID))
if err != nil {
t.Fatalf("DeleteDomain() error = %v", err)
}
if !result.Existed || !result.Removed || result.StorageRemoved || result.NetworkChanged {
t.Fatalf("result = %+v", result)
}
runner.mu.Lock()
commands := append([][]string(nil), runner.commands...)
runner.mu.Unlock()
want := [][]string{
{"-c", "qemu:///system", "dumpxml", "--inactive", testUUID},
{"-c", "qemu:///system", "domstate", "--domain", testUUID},
{"-c", "qemu:///system", "undefine", "--domain", testUUID},
{"-c", "qemu:///system", "dumpxml", "--inactive", testUUID},
}
if len(commands) != len(want) {
t.Fatalf("commands = %+v, want %+v", commands, want)
}
for i := range want {
if strings.Join(commands[i], "\x00") != strings.Join(want[i], "\x00") {
t.Fatalf("command[%d] = %+v, want %+v", i, commands[i], want[i])
}
}
}
func TestVirshDeleteDomainLeavesUncertainWhenAbsenceReadbackFails(t *testing.T) {
runner := &deleteRecordingRunner{
present: true,
state: []byte("shut off\n"),
readbackErr: errors.New("transport lost"),
}
backend, err := NewVirshBackendWithRunner("qemu:///system", runner)
if err != nil {
t.Fatal(err)
}
result, err := backend.DeleteDomain(context.Background(), testUUID)
if !errors.Is(err, ErrDeleteReadbackMismatch) {
t.Fatalf("DeleteDomain() error = %v, want ErrDeleteReadbackMismatch", err)
}
if result.Removed {
t.Fatalf("ambiguous delete was reported removed: %+v", result)
}
}

@ -248,6 +248,11 @@ func powerResult(uuid string, action PowerAction, state DomainPowerState) PowerR
// returns the state that was immediately observed. It mirrors the real
// adapter's read-back contract while keeping tests deterministic.
func (f *FakeBackend) PowerDomain(ctx context.Context, uuid string, action PowerAction) (PowerResult, error) {
if err := contextErr(ctx); err != nil {
return PowerResult{}, err
}
f.mutationMu.Lock()
defer f.mutationMu.Unlock()
if err := contextErr(ctx); err != nil {
return PowerResult{}, err
}

@ -11,7 +11,6 @@ import (
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
@ -97,17 +96,37 @@ func (r ExecRunner) Run(ctx context.Context, args ...string) ([]byte, error) {
command := exec.CommandContext(ctx, binary, args...)
command.Env = fixedVirshEnv()
stdout := boundedBuffer{limit: maxVirshOutput}
stderr := boundedBuffer{limit: maxVirshOutput}
command.Stdout = &stdout
command.Stderr = io.Discard
// Capture stderr only inside the adapter so a fixed, locale-pinned
// "domain not found" result can be distinguished from a transport failure.
// The bytes are never returned or included in an error message; virsh may
// otherwise disclose host paths, storage names, or authentication details.
command.Stderr = &stderr
if err := command.Run(); err != nil {
if errors.Is(ctx.Err(), context.Canceled) || errors.Is(ctx.Err(), context.DeadlineExceeded) {
return nil, ctx.Err()
}
if virshReportsMissingDomain(stderr.Bytes()) {
return nil, ErrDomainNotFound
}
return nil, fmt.Errorf("%w: virsh command failed", ErrVirshUnavailable)
}
return append([]byte(nil), stdout.Bytes()...), nil
}
// virshReportsMissingDomain is intentionally narrow and only consumes
// diagnostics from the adapter's own LC_ALL=C subprocess. It provides a
// typed absence read-back for delete/idempotent probes without exposing the
// diagnostic text to the panel. Any wording not listed here remains an
// opaque transport failure and therefore uncertain.
func virshReportsMissingDomain(data []byte) bool {
message := strings.ToLower(string(data))
return strings.Contains(message, "domain not found") ||
strings.Contains(message, "no domain with matching uuid") ||
strings.Contains(message, "no domain with matching name")
}
// VirshBackend is a host-local implementation of Backend. It never changes
// host networking; domain definitions and the explicitly allowlisted power
// actions are the only mutating operations, and every one performs a typed
@ -448,6 +467,120 @@ func (b *VirshBackend) ReadDomainXML(ctx context.Context, uuid string) ([]byte,
return cloneBytes(data), nil
}
// InspectDomainForDeletion performs a targeted, read-only preflight. It does
// not enumerate domains and never invokes undefine, lifecycle, storage, or
// networking commands. A typed ErrDomainNotFound from the fixed virsh adapter
// is normalized into an absent observation; all other command/read failures
// stay opaque and are left for the operation layer to mark uncertain.
func (b *VirshBackend) InspectDomainForDeletion(ctx context.Context, uuid string) (DomainDeletionObservation, error) {
if err := contextErr(ctx); err != nil {
return DomainDeletionObservation{}, err
}
canonical, err := CanonicalUUID(uuid)
if err != nil {
return DomainDeletionObservation{}, err
}
xmlData, err := b.ReadDomainXML(ctx, canonical)
if errors.Is(err, ErrDomainNotFound) {
return DomainDeletionObservation{
UUID: canonical,
Present: false,
ObservedAt: time.Now().UTC(),
}, nil
}
if err != nil {
return DomainDeletionObservation{}, err
}
observed, err := domainUUID(xmlData)
if err != nil || observed != canonical {
return DomainDeletionObservation{}, fmt.Errorf("%w: requested %s", ErrReadbackMismatch, canonical)
}
state, err := b.readDomainState(ctx, canonical)
if err != nil {
// The definition was observed but its active/inactive state was not. Do
// not guess that it is safe to undefine; the caller must reconcile.
return DomainDeletionObservation{}, fmt.Errorf("%w: domain state read-back did not complete", ErrDeleteReadbackMismatch)
}
return DomainDeletionObservation{
UUID: canonical,
Present: true,
State: state,
Active: state.IsActive(),
XMLHash: hashDomainXML(xmlData),
ObservedAt: time.Now().UTC(),
}, nil
}
// DeleteDomain undefines one inactive domain and nothing else. The fixed
// virsh argv contains only the canonical UUID; no shell, XML, disk path, or
// network operation is accepted. Success requires a second targeted
// read-back proving that the definition is absent. If the command or that
// read-back is ambiguous, the returned error is deliberately non-definitive
// so the API can retain an uncertain journal record instead of replaying.
func (b *VirshBackend) DeleteDomain(ctx context.Context, uuid string) (DomainDeleteResult, error) {
if err := contextErr(ctx); err != nil {
return DomainDeleteResult{}, err
}
b.mutationMu.Lock()
defer b.mutationMu.Unlock()
if err := contextErr(ctx); err != nil {
return DomainDeleteResult{}, err
}
canonical, err := CanonicalUUID(uuid)
if err != nil {
return DomainDeleteResult{}, err
}
before, err := b.InspectDomainForDeletion(ctx, canonical)
if err != nil {
return DomainDeleteResult{UUID: canonical}, err
}
if !before.Present {
return DomainDeleteResult{
UUID: canonical,
AlreadyAbsent: true,
Removed: true,
ObservedAt: time.Now().UTC(),
}, nil
}
base := DomainDeleteResult{
UUID: canonical,
Existed: true,
StateBefore: before.State,
XMLHash: before.XMLHash,
StorageRemoved: false,
NetworkChanged: false,
}
if before.Active {
base.ObservedAt = time.Now().UTC()
return base, fmt.Errorf("%w: %s", ErrDomainActive, canonical)
}
_, commandErr := b.runner.Run(ctx, b.args("undefine", "--domain", canonical)...)
if err := contextErr(ctx); err != nil {
base.ObservedAt = time.Now().UTC()
return base, err
}
after, readErr := b.InspectDomainForDeletion(ctx, canonical)
if readErr != nil {
base.ObservedAt = time.Now().UTC()
return base, fmt.Errorf("%w: post-undefine observation failed", ErrDeleteReadbackMismatch)
}
if after.Present {
base.ObservedAt = after.ObservedAt
return base, fmt.Errorf("%w: %s", ErrDeleteReadbackMismatch, canonical)
}
// Absence proves the definition boundary even when virsh returned a
// non-zero status (for example, an idempotent race with another actor).
base.Removed = true
base.ObservedAt = after.ObservedAt
if commandErr != nil {
// Deliberately accept only the authoritative absence read-back here; no
// command error text is surfaced. This keeps retries idempotent without
// turning a transport failure into a blind replay.
return base, nil
}
return base, nil
}
// PowerDomain executes one fixed virsh lifecycle subcommand and then reads
// domstate back through the same local connection. A command error is not
// blindly replayed: for idempotent actions, an authoritative desired state is

@ -0,0 +1,33 @@
# Go Agent `domain_delete` contract
This slice implements a narrow, identity-bound **libvirt definition undefine**
operation. It is not the panel's full VM deletion workflow.
## Host boundary
- The request payload is exactly `{ "uuid": "<canonical UUID>" }`.
- `InspectDomainForDeletion` performs a targeted read-only `dumpxml --inactive`
and `domstate` observation. It does not enumerate domains or touch storage,
networking, firewall state, or seed artifacts.
- `DeleteDomain` refuses an active domain. It invokes only
`virsh undefine --domain <canonical UUID>` and then requires a targeted
absence read-back before returning success.
- Fake and virsh backends report only a SHA-256 XML digest; raw XML and host
paths never enter the operation result or journal.
- `storage_removed=false` and `network_changed=false` are explicit result
facts. The operation never deletes disks or changes a network.
If the undefine command or absence read-back is ambiguous, the operation is
`uncertain`. Reconciliation calls only `InspectDomainForDeletion`; it never
replays `undefine`. A present definition, failed transport, or missing state
read-back therefore cannot become a false success.
## PHP boundary
The Go API accepts `domain_delete`, but the PHP `CloudAgentVmGateway` keeps the
legacy panel `delete` mapping disabled (`null`). The existing panel operation
also owns disk/seed cleanup, billing state, firewall history, and irreversible
instance tombstoning. Mapping it to definition-only undefine would report a
successful VM deletion while leaving those resources behind. A future mapping
requires a separate typed storage/seed/network cleanup contract, panel CAS
updates, and real-host read-back evidence.
Loading…
Cancel
Save