diff --git a/agent/cmd/virt-cloud-agent/main.go b/agent/cmd/virt-cloud-agent/main.go index d13d984b..8badaff8 100644 --- a/agent/cmd/virt-cloud-agent/main.go +++ b/agent/cmd/virt-cloud-agent/main.go @@ -64,8 +64,10 @@ func main() { logger.Warn("request signing key unavailable; starting degraded", "error", authErr.Error()) // Preserve the same agent binding and durable nonce consumer while the // verifier is degraded. This keeps a repaired key from changing the - // request envelope or replay boundary after startup. - requestAuth, _ = auth.NewWithNonceConsumerForAgent(nil, true, store, cfg.AgentID) + // request envelope or replay boundary after startup. If signing was + // explicitly optional, keep it optional; an unreadable optional key + // must not silently upgrade the control plane to required signing. + requestAuth, _ = auth.NewWithNonceConsumerForAgent(nil, cfg.RequireSigned, store, cfg.AgentID) } provider, backendWarning := buildProvider(cfg, os.Getenv, runtime.GOOS) if backendWarning != "" { diff --git a/agent/internal/api/server.go b/agent/internal/api/server.go index 8faaaa3d..0c78d264 100644 --- a/agent/internal/api/server.go +++ b/agent/internal/api/server.go @@ -59,10 +59,21 @@ type Server struct { stateMu sync.RWMutex hostReady bool tlsReady bool - auth *auth.Verifier - logger *slog.Logger - mux *http.ServeMux -} + // readinessProbeMu serializes health-triggered host probes. Health is an + // unauthenticated liveness endpoint, so without a small single-flight + // boundary every remote poll could start a fresh virsh/libvirt probe. + readinessProbeMu sync.Mutex + readinessProbeAt time.Time + readinessProbeDone chan struct{} + auth *auth.Verifier + logger *slog.Logger + mux *http.ServeMux +} + +// healthReadinessProbeTTL bounds how often the unauthenticated health route +// starts a new host capability probe. Mutating request gates continue to use +// refreshHostReadiness directly so they retain a fresh, fail-closed probe. +const healthReadinessProbeTTL = time.Second type healthResponse struct { Status string `json:"status"` @@ -93,7 +104,19 @@ func New(opts Options) (*Server, error) { return nil, fmt.Errorf("capability provider is required") } if opts.Auth == nil { - opts.Auth = auth.NewDisabled() + if opts.Config.RequireSigned { + // A required verifier with no configured key must remain unavailable. + // Falling back to NewDisabled here would silently turn a required + // authenticated control plane into an unsigned one whenever the + // caller forgot to inject Auth. + verifier, err := auth.NewWithNonceConsumerForAgent(nil, true, opts.Store, opts.Config.AgentID) + if err != nil { + return nil, fmt.Errorf("configure request signing verifier: %w", err) + } + opts.Auth = verifier + } else { + opts.Auth = auth.NewDisabled() + } } if opts.Logger == nil { opts.Logger = slog.New(slog.NewTextHandler(io.Discard, nil)) @@ -156,7 +179,7 @@ func (s *Server) health(w http.ResponseWriter, r *http.Request) { // daemon or socket can disappear after the process starts; refresh it on // health and write boundaries so monitoring does not report a stale green // state and mutating requests fail closed. - s.refreshHostReadiness(r.Context()) + s.refreshHostReadinessCached(r.Context()) readiness := s.readiness() status := "ok" if readiness != config.ReadinessReady { @@ -225,6 +248,56 @@ func (s *Server) SetTLSReady(ready bool) { // intentionally independent from operation execution: a failed probe never // invokes a lifecycle or define command and only downgrades readiness. func (s *Server) refreshHostReadiness(parent context.Context) bool { + if s == nil || s.backend == nil { + return s.hostBackendReady() + } + return s.probeHostReadiness(parent) +} + +// refreshHostReadinessCached is used only by health. It coalesces concurrent +// requests into one bounded probe and briefly reuses its result, including a +// failed result. The latter matters for fail-closed behavior: an attacker +// must not be able to turn a failed host into an unbounded subprocess storm by +// polling the public health endpoint. +func (s *Server) refreshHostReadinessCached(parent context.Context) bool { + if s == nil || s.backend == nil { + return s.hostBackendReady() + } + if parent == nil { + parent = context.Background() + } + + s.readinessProbeMu.Lock() + now := time.Now() + if !s.readinessProbeAt.IsZero() && now.Sub(s.readinessProbeAt) < healthReadinessProbeTTL { + s.readinessProbeMu.Unlock() + return s.hostBackendReady() + } + if done := s.readinessProbeDone; done != nil { + s.readinessProbeMu.Unlock() + select { + case <-done: + return s.hostBackendReady() + case <-parent.Done(): + return s.hostBackendReady() + } + } + done := make(chan struct{}) + s.readinessProbeDone = done + s.readinessProbeMu.Unlock() + + ready := s.probeHostReadiness(parent) + s.readinessProbeMu.Lock() + s.readinessProbeAt = time.Now() + s.readinessProbeDone = nil + close(done) + s.readinessProbeMu.Unlock() + return ready +} + +// probeHostReadiness performs the actual bounded, read-only backend probe. +// Callers choose whether to serialize/cache it based on the request boundary. +func (s *Server) probeHostReadiness(parent context.Context) bool { if s == nil || s.backend == nil { return s.hostBackendReady() } diff --git a/agent/internal/api/server_security_test.go b/agent/internal/api/server_security_test.go new file mode 100644 index 00000000..6a3e4798 --- /dev/null +++ b/agent/internal/api/server_security_test.go @@ -0,0 +1,148 @@ +package api + +import ( + "bytes" + "context" + "errors" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "g.luoli.live/root/virt/agent/internal/capability" + "g.luoli.live/root/virt/agent/internal/config" + "g.luoli.live/root/virt/agent/internal/hypervisor" + "g.luoli.live/root/virt/agent/internal/operation" +) + +func TestNewRequiredSigningWithoutAuthFailsClosed(t *testing.T) { + c := config.Defaults() + c.AgentID = "host-required-auth" + c.RequireSigned = true + store := operation.NewMemory() + backend := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) + s, err := New(Options{ + Config: c, + Store: store, + Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}, + Backend: backend, + HostBackendReady: true, + }) + if err != nil { + t.Fatal(err) + } + if s.auth == nil || s.auth.Ready() { + t.Fatal("required signing without injected Auth must construct an unavailable verifier") + } + + operationID := "00000000-0000-4000-8000-000000000091" + body := []byte(`{"operation_id":"` + operationID + `","idempotency_key":"idem-required-auth-0091","kind":"capability_probe","target":"` + c.AgentID + `","payload":{}}`) + recorder := httptest.NewRecorder() + s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(body))) + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, body = %s; required signing must fail closed", recorder.Code, recorder.Body.String()) + } + if _, err := store.Get(operationID); !errors.Is(err, operation.ErrNotFound) { + t.Fatalf("unsigned request entered operation journal: err=%v", err) + } +} + +// probeCountingBackend blocks the first host probe so concurrent health calls +// have a deterministic in-flight window in which to exercise single-flight. +type probeCountingBackend struct { + *hypervisor.FakeBackend + calls atomic.Int32 + startedOnce sync.Once + started chan struct{} + release chan struct{} +} + +func (b *probeCountingBackend) Probe(ctx context.Context) (hypervisor.ProbeResult, error) { + b.calls.Add(1) + b.startedOnce.Do(func() { close(b.started) }) + select { + case <-b.release: + return b.FakeBackend.Probe(ctx) + case <-ctx.Done(): + return hypervisor.ProbeResult{}, ctx.Err() + } +} + +func TestHealthReadinessProbeSingleFlightAndThrottle(t *testing.T) { + c := config.Defaults() + c.AgentID = "host-health-probe" + base := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) + backend := &probeCountingBackend{ + FakeBackend: base, + started: make(chan struct{}), + release: make(chan struct{}), + } + s, err := New(Options{ + Config: c, + Store: operation.NewMemory(), + Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}, + Backend: backend, + HostBackendReady: true, + }) + if err != nil { + t.Fatal(err) + } + + firstDone := make(chan struct{}) + go func() { + s.refreshHostReadinessCached(context.Background()) + close(firstDone) + }() + select { + case <-backend.started: + case <-time.After(time.Second): + t.Fatal("timed out waiting for first host probe") + } + + const concurrent = 16 + var wg sync.WaitGroup + wg.Add(concurrent) + for i := 0; i < concurrent; i++ { + go func() { + defer wg.Done() + s.refreshHostReadinessCached(context.Background()) + }() + } + // Give the waiters a scheduling window while the first probe is blocked. + time.Sleep(25 * time.Millisecond) + if got := backend.calls.Load(); got != 1 { + t.Fatalf("concurrent health probes = %d, want one in-flight probe", got) + } + close(backend.release) + select { + case <-firstDone: + case <-time.After(time.Second): + t.Fatal("first host probe did not finish") + } + wg.Wait() + if got := backend.calls.Load(); got != 1 { + t.Fatalf("coalesced health probes = %d, want one", got) + } + + // The health handler must use the same short cache rather than bypassing it. + recorder := httptest.NewRecorder() + s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/v1/health", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("health status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + if got := backend.calls.Load(); got != 1 { + t.Fatalf("health request inside throttle window started %d probes, want one", got) + } + + time.Sleep(healthReadinessProbeTTL + 20*time.Millisecond) + recorder = httptest.NewRecorder() + s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/v1/health", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("second health status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + if got := backend.calls.Load(); got != 2 { + t.Fatalf("health probe after throttle window = %d, want two total probes", got) + } +} diff --git a/agent/internal/auth/auth.go b/agent/internal/auth/auth.go index 3cd1cb1e..f8cf9fc9 100644 --- a/agent/internal/auth/auth.go +++ b/agent/internal/auth/auth.go @@ -13,7 +13,9 @@ import ( "io" "net/http" "os" + "path/filepath" "regexp" + "runtime" "strconv" "strings" "sync" @@ -169,12 +171,23 @@ func FromFileWithNonceConsumerForAgent(path string, required bool, consumer Nonc // narrows the lstat/open replacement window without introducing a platform- // specific syscall into the cross-built agent. func readKeyFile(path string) ([]byte, error) { + path, err := normalizeSecretPath(path, "request signing key") + if err != nil { + return nil, fmt.Errorf("read request signing key: %w", err) + } + parentInfo, err := checkSecretParent(filepath.Dir(path)) + if err != nil { + return nil, fmt.Errorf("read request signing key: %w", err) + } info, err := os.Lstat(path) if err != nil { return nil, fmt.Errorf("read request signing key: %w", err) } - if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { - return nil, fmt.Errorf("request signing key must be a regular file, not a symlink") + if info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("request signing key must not be a symlink") + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("request signing key must be a regular file") } if !secureKeyMode(info.Mode()) { return nil, fmt.Errorf("request signing key permissions must be owner-read-only or owner-read-write") @@ -188,9 +201,26 @@ func readKeyFile(path string) ([]byte, error) { if err != nil { return nil, fmt.Errorf("stat request signing key: %w", err) } - if !openedInfo.Mode().IsRegular() || !secureKeyMode(openedInfo.Mode()) { + if !secureSecretFile(openedInfo) || !os.SameFile(info, openedInfo) { return nil, fmt.Errorf("request signing key changed to an unsafe file") } + currentParent, err := checkSecretParent(filepath.Dir(path)) + if err != nil || !os.SameFile(parentInfo, currentParent) { + if err != nil { + return nil, fmt.Errorf("recheck request signing key parent: %w", err) + } + return nil, fmt.Errorf("request signing key parent changed while opening") + } + // Lstat the name after opening as well. This catches a replacement with a + // symlink (including a link back to the same inode) during the lstat/open + // window while the descriptor keeps the bytes we read stable. + currentInfo, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("recheck request signing key: %w", err) + } + if !secureSecretFile(currentInfo) || !os.SameFile(info, currentInfo) { + return nil, fmt.Errorf("request signing key changed while opening") + } data, err := io.ReadAll(io.LimitReader(file, 4097)) if err != nil { return nil, fmt.Errorf("read request signing key: %w", err) @@ -198,12 +228,142 @@ func readKeyFile(path string) ([]byte, error) { if len(data) > 4096 { return nil, fmt.Errorf("request signing key exceeds 4096 bytes") } + currentInfo, err = os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("recheck request signing key after read: %w", err) + } + if !secureSecretFile(currentInfo) || !os.SameFile(info, currentInfo) { + return nil, fmt.Errorf("request signing key changed while reading") + } return data, nil } func secureKeyMode(mode os.FileMode) bool { permissions := mode.Perm() - return permissions&0o077 == 0 && permissions&0o400 != 0 + return (permissions == 0o400 || permissions == 0o600) && + mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) == 0 +} + +// normalizeSecretPath makes the path used for every subsequent check and +// open unambiguous. Relative paths remain supported for local/test callers, +// but traversal components are rejected so cleaning cannot skip a symlinked +// component that was present in the caller's spelling. +func normalizeSecretPath(path, label string) (string, error) { + if path == "" || strings.TrimSpace(path) != path || strings.IndexByte(path, 0) >= 0 { + return "", fmt.Errorf("%s path is empty or malformed", label) + } + for _, component := range strings.FieldsFunc(path, func(r rune) bool { + return r == '/' || r == '\\' + }) { + if component == ".." { + return "", fmt.Errorf("%s path must not contain traversal components", label) + } + } + abs, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("resolve %s path: %w", label, err) + } + return filepath.Clean(abs), nil +} + +// checkSecretParent checks every directory component rather than only the +// immediate parent. Lstat on the complete path follows an intermediate +// symlink, so each component is inspected independently. Parent directories +// may be readable by other users, but must never be writable by group/other +// unless the OS sticky bit protects the standard shared temporary boundary +// (for example /tmp). The final key file remains owner-only. +func checkSecretParent(path string) (os.FileInfo, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("resolve secret parent: %w", err) + } + abs = filepath.Clean(abs) + root := filepath.VolumeName(abs) + string(filepath.Separator) + if filepath.VolumeName(abs) == "" { + root = string(filepath.Separator) + } + rest := strings.TrimPrefix(abs, root) + current := root + if rest == "" { + return checkSecretDirectory(current, path) + } + var last os.FileInfo + for _, component := range strings.Split(rest, string(filepath.Separator)) { + if component == "" || component == "." { + continue + } + current = filepath.Join(current, component) + info, err := checkSecretDirectory(current, path) + if err != nil { + return nil, err + } + last = info + } + return last, nil +} + +func checkSecretDirectory(path, original string) (os.FileInfo, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("inspect request signing key parent %q: %w", original, err) + } + if info.Mode()&os.ModeSymlink != 0 { + if !trustedDarwinAlias(path) { + return nil, fmt.Errorf("request signing key parent %q must contain only real directories", original) + } + resolved, resolveErr := filepath.EvalSymlinks(path) + if resolveErr != nil { + return nil, fmt.Errorf("resolve request signing key parent %q: %w", original, resolveErr) + } + resolvedInfo, statErr := os.Lstat(resolved) + if statErr != nil || resolvedInfo.Mode()&os.ModeSymlink != 0 || !resolvedInfo.IsDir() { + if statErr != nil { + return nil, fmt.Errorf("inspect request signing key parent %q: %w", original, statErr) + } + return nil, fmt.Errorf("request signing key parent %q must contain only real directories", original) + } + info = resolvedInfo + } + if !info.IsDir() { + return nil, fmt.Errorf("request signing key parent %q must contain only real directories", original) + } + if !safeSecretDirectoryMode(info.Mode()) { + return nil, fmt.Errorf("request signing key parent %q must not be group/world writable", original) + } + return info, nil +} + +func safeSecretDirectoryMode(mode os.FileMode) bool { + if mode&(os.ModeSetuid|os.ModeSetgid) != 0 { + return false + } + if mode.Perm()&0o022 == 0 { + return true + } + // A sticky directory prevents another user from replacing entries owned by + // the agent. This is the standard protection on /tmp and lets test/runtime + // paths use a private child directory without weakening the key-file mode. + return mode&os.ModeSticky != 0 +} + +// macOS exposes /var as a fixed root-level compatibility alias. It is unlike a +// caller-controlled symlink nested below the configured state directory. Keep +// this one narrow exception so t.TempDir paths remain usable on macOS while a +// custom symlinked ancestor is still rejected. +func trustedDarwinAlias(path string) bool { + if runtime.GOOS != "darwin" { + return false + } + switch filepath.Clean(path) { + case "/var": + return true + default: + return false + } +} + +func secureSecretFile(info os.FileInfo) bool { + return info != nil && info.Mode()&os.ModeSymlink == 0 && info.Mode().IsRegular() && secureKeyMode(info.Mode()) } // decodeFileKey enforces the on-disk protocol shared with PHP CloudSecrets: diff --git a/agent/internal/auth/auth_test.go b/agent/internal/auth/auth_test.go index e2b7159c..71fcef7e 100644 --- a/agent/internal/auth/auth_test.go +++ b/agent/internal/auth/auth_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -222,3 +223,83 @@ func TestFromFileRejectsSymlinkAndGroupReadableKey(t *testing.T) { t.Fatalf("symlink key was accepted: verifier=%v err=%v", verifier, err) } } + +func TestFromFileRejectsUnsafeAncestorPaths(t *testing.T) { + t.Parallel() + base := t.TempDir() + keyData := []byte(protocolEncodedKey + "\n") + + goodParent := filepath.Join(base, "good") + if err := os.Mkdir(goodParent, 0o755); err != nil { + t.Fatal(err) + } + goodKey := filepath.Join(goodParent, "request.hmac") + if err := os.WriteFile(goodKey, keyData, 0o600); err != nil { + t.Fatal(err) + } + if verifier, err := FromFile(goodKey, true); err != nil || verifier == nil || !verifier.Ready() { + t.Fatalf("secure ancestor path was rejected: verifier=%v err=%v", verifier, err) + } + + realParent := filepath.Join(base, "real") + if err := os.Mkdir(realParent, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(realParent, "request.hmac"), keyData, 0o600); err != nil { + t.Fatal(err) + } + linkParent := filepath.Join(base, "linked-parent") + if err := os.Symlink(realParent, linkParent); err != nil { + t.Fatal(err) + } + if verifier, err := FromFile(filepath.Join(linkParent, "request.hmac"), true); err == nil || verifier == nil || verifier.Ready() { + t.Fatalf("symlinked ancestor was accepted: verifier=%v err=%v", verifier, err) + } + + nonDir := filepath.Join(base, "not-a-directory") + if err := os.WriteFile(nonDir, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if verifier, err := FromFile(filepath.Join(nonDir, "request.hmac"), true); err == nil || verifier == nil || verifier.Ready() { + t.Fatalf("non-directory ancestor was accepted: verifier=%v err=%v", verifier, err) + } + + writableParent := filepath.Join(base, "writable") + if err := os.Mkdir(writableParent, 0o770); err != nil { + t.Fatal(err) + } + if err := os.Chmod(writableParent, 0o770); err != nil { + t.Fatal(err) + } + writableKey := filepath.Join(writableParent, "request.hmac") + if err := os.WriteFile(writableKey, keyData, 0o600); err != nil { + t.Fatal(err) + } + if verifier, err := FromFile(writableKey, true); err == nil || verifier == nil || verifier.Ready() { + t.Fatalf("group-writable ancestor was accepted: verifier=%v err=%v", verifier, err) + } +} + +func TestFromFileRejectsUnsafeRegularFileModes(t *testing.T) { + t.Parallel() + for _, mode := range []os.FileMode{0o644, 0o700, 0o600 | os.ModeSticky} { + mode := mode + t.Run(modeString(mode), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "request.hmac") + if err := os.WriteFile(path, []byte(protocolEncodedKey+"\n"), mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, mode); err != nil { + t.Fatal(err) + } + verifier, err := FromFile(path, true) + if err == nil || verifier == nil || verifier.Ready() { + t.Fatalf("mode %o was accepted: verifier=%v err=%v", mode, verifier, err) + } + }) + } +} + +func modeString(mode os.FileMode) string { + return "mode-" + strconv.FormatUint(uint64(mode.Perm()), 8) +} diff --git a/agent/internal/domaincreateattach/attach.go b/agent/internal/domaincreateattach/attach.go new file mode 100644 index 00000000..edcda95b --- /dev/null +++ b/agent/internal/domaincreateattach/attach.go @@ -0,0 +1,635 @@ +// Package domaincreateattach defines the read-only host-bound boundary that +// precedes a future VM create operation. +// +// The package turns a typed, pre-host domain policy into a host-bound base XML +// (by applying a loader selected from a fresh capability observation), verifies +// an immutable NoCloud artifact, and validates already-resolved disk paths +// against the agent storage allowlist. It does not create files, allocate +// volumes, attach devices, define a domain, change power, or configure a +// network. The returned Plan is an executor input; its safe Result projection +// deliberately excludes host paths and raw XML. +package domaincreateattach + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "g.luoli.live/root/virt/agent/internal/domaincreateplan" + "g.luoli.live/root/virt/agent/internal/domainpreflight" + "g.luoli.live/root/virt/agent/internal/domainxml" + "g.luoli.live/root/virt/agent/internal/hypervisor" + "g.luoli.live/root/virt/agent/internal/startup" + "g.luoli.live/root/virt/agent/internal/storagepolicy" +) + +const ( + codePrefix = "domain_create_attach_" + // Kind is the reserved greenfield operation kind for a future API adapter. + Kind = "domain_create_attach" + // CommitBoundaryChecked means all immutable host-bound inputs passed the + // read-only gate; it does not mean a disk, seed, or domain was attached. + CommitBoundaryChecked = "domain_create_attach_checked" + maxDiskAttachments = 32 +) + +var ( + // ErrInvalidRequest identifies a malformed typed request. It is definitive + // because no host mutation is needed to reject it. + ErrInvalidRequest = errors.New("invalid domain create attach request") + // ErrSeedRejected identifies a missing, malformed, or mismatched immutable + // seed. A caller must publish a new artifact instead of retrying the same + // identity after this error. + ErrSeedRejected = errors.New("domain create seed rejected") + // ErrStorageRejected identifies a disk intent that is outside the explicit + // host storage policy or otherwise cannot be accepted as an existing image. + ErrStorageRejected = errors.New("domain create storage rejected") + // ErrHostUnavailable identifies a host filesystem or loader observation that + // did not complete. Reconciliation may retry after the operator repairs the + // host; this package never retries a mutation. + ErrHostUnavailable = errors.New("domain create host attachment inputs unavailable") + // ErrCanceled identifies a caller cancellation. The operation remains + // uncertain because this boundary intentionally does not claim a mutation. + ErrCanceled = errors.New("domain create attach validation canceled") + + volumeIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$`) +) + +// Error carries a stable operation code while retaining a local cause for +// tests and diagnostics. Message is generic and must not contain a host path, +// seed content, or another tenant-controlled secret. +type Error struct { + CodeValue string `json:"code"` + Message string `json:"message"` + Definitive bool `json:"definitive"` + Cause error `json:"-"` +} + +func (e *Error) Error() string { + if e == nil { + return codePrefix + "failed" + } + if e.Message == "" { + return e.CodeValue + } + return e.CodeValue + ": " + e.Message +} + +func (e *Error) Unwrap() error { + if e == nil { + return nil + } + return e.Cause +} + +func newError(code, message string, definitive bool, cause error) error { + return &Error{CodeValue: code, Message: message, Definitive: definitive, Cause: cause} +} + +// Code returns the stable operation code intended for operation.ErrorCode. +func Code(err error) string { + var typed *Error + if errors.As(err, &typed) && typed != nil && typed.CodeValue != "" { + return typed.CodeValue + } + return codePrefix + "failed" +} + +// IsDefinitive reports whether the request was rejected without an ambiguous +// host-side mutation. Host filesystem/loader availability and cancellation are +// intentionally non-definitive. +func IsDefinitive(err error) bool { + var typed *Error + return errors.As(err, &typed) && typed != nil && typed.Definitive +} + +// DiskAttachment is a host-side resolution of one opaque volume identity. +// Path must come from an agent-owned storage resolver, not directly from a +// panel payload; it is still checked against StorageRoots as defense in depth. +// Slot is translated to a device name by a later executor, which keeps device +// XML and bus choices out of this boundary. +type DiskAttachment struct { + VolumeID string `json:"volume_id"` + Slot int `json:"slot"` + Readonly bool `json:"readonly"` + Path string `json:"-"` +} + +// Request is the minimal host-bound input. Domain remains typed and contains +// no XML, disk, interface, or network fields. LoaderPath must be selected by a +// fresh host capability observation; ArtifactRoot and disk Path values are +// agent-side configuration/resolver outputs and are never accepted from the +// HTTP payload directly. +type Request struct { + Domain domainpreflight.Request `json:"domain"` + LoaderPath string `json:"-"` + ArtifactRoot string `json:"-"` + StorageRoots []string `json:"-"` + SeedArtifactID string `json:"seed_artifact_id"` + SeedManifest domaincreateplan.SeedManifest `json:"seed_manifest"` + StartupScriptPresent bool `json:"first_boot_present"` + StartupScriptSHA256 string `json:"first_boot_digest"` + Disks []DiskAttachment `json:"disks,omitempty"` +} + +// Result is the safe journal/API projection. It contains identities and +// counts, but no host path, raw domain XML, disk source, or seed bytes. +type Result struct { + CommitBoundary string `json:"commit_boundary"` + DomainUUID string `json:"domain_uuid"` + DomainXMLHash string `json:"domain_xml_hash"` + SeedArtifactID string `json:"seed_artifact_id"` + SeedManifest domaincreateplan.SeedManifest `json:"seed_manifest"` + StartupScriptPresent bool `json:"first_boot_present"` + StartupScriptSHA256 string `json:"first_boot_digest"` + DiskCount int `json:"disk_count"` + DiskPlanHash string `json:"disk_plan_hash"` + PlanHash string `json:"plan_hash"` + LoaderPending bool `json:"loader_pending"` +} + +// Plan is the executor-only result. The unexported fields cannot be serialized +// accidentally with encoding/json. Accessors return defensive copies; an +// eventual mutating executor must perform its own open/read-back/TOCTOU checks +// immediately before define or attach. +type Plan struct { + Result + hostXML []byte + seedDirectory string + disks []DiskAttachment +} + +// String deliberately renders only the safe Result projection. The executor +// plan carries host paths and raw XML in private fields; default fmt output +// must not turn an accidental diagnostic log into a host-path disclosure. +func (p Plan) String() string { + encoded, err := json.Marshal(p.Result) + if err != nil { + return "{\"plan\":\"unavailable\"}" + } + return string(encoded) +} + +// GoString protects %#v diagnostics for the same reason as String. +func (p Plan) GoString() string { return p.String() } + +// HostXML returns the host-bound base XML with the capability-selected loader. +// It intentionally contains no disk or network elements; a future executor +// must construct those from the typed DiskAttachments contract. +func (p Plan) HostXML() []byte { return append([]byte(nil), p.hostXML...) } + +// SeedDirectory returns the immutable artifact directory selected by the +// configured root and canonical artifact UUID. It is for the host executor, +// not for a durable API result. +func (p Plan) SeedDirectory() string { return p.seedDirectory } + +// DiskAttachments returns defensive copies of the already policy-checked +// resolver outputs. Path remains excluded from JSON, but the executor must +// revalidate it with an openat/O_NOFOLLOW-style boundary before attach. +func (p Plan) DiskAttachments() []DiskAttachment { + return cloneDisks(p.disks) +} + +// Build validates all host-bound inputs without performing a host mutation. +// It is safe to call before durable commit; a caller should persist only Result +// and retain the operation ID, not the returned host paths/XML, in its journal. +func Build(ctx context.Context, request Request) (Plan, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return Plan{}, newError(codePrefix+"canceled", "attach validation was canceled", false, fmt.Errorf("%w: %v", ErrCanceled, err)) + } + + canonicalDomain, err := domainpreflight.ValidateRequest(request.Domain) + if err != nil { + return Plan{}, newError(codePrefix+"invalid_domain", "domain policy is invalid", true, fmt.Errorf("%w: %v", ErrInvalidRequest, err)) + } + request.Domain = canonicalDomain + if err := validateArtifactRoot(request.ArtifactRoot); err != nil { + return Plan{}, err + } + artifactID, err := canonicalArtifactID(request.SeedArtifactID) + if err != nil { + return Plan{}, newError(codePrefix+"invalid_seed_artifact", "seed artifact identity is invalid", true, fmt.Errorf("%w: %v", ErrInvalidRequest, err)) + } + request.SeedArtifactID = artifactID + if err := validateExpectedManifest(request.SeedManifest); err != nil { + return Plan{}, newError(codePrefix+"invalid_seed_manifest", "seed manifest identity is invalid", true, fmt.Errorf("%w: %v", ErrInvalidRequest, err)) + } + if err := validateStartupExpectation(request.StartupScriptPresent, request.StartupScriptSHA256); err != nil { + return Plan{}, err + } + for _, root := range request.StorageRoots { + if strings.TrimSpace(root) != root || filepath.Clean(root) != root { + return Plan{}, newError(codePrefix+"storage_rejected", "storage root is not canonical", true, fmt.Errorf("%w: non-canonical storage root", ErrStorageRejected)) + } + } + if err := validateDisks(request.Disks); err != nil { + return Plan{}, err + } + + // Render and pin the loader before touching the artifact or disk paths. The + // raw caller cannot supply XML or a loader flag that bypasses this policy. + if request.LoaderPath == "" || strings.TrimSpace(request.LoaderPath) != request.LoaderPath || filepath.Clean(request.LoaderPath) != request.LoaderPath { + return Plan{}, newError(codePrefix+"loader_rejected", "host firmware loader path is not canonical", true, fmt.Errorf("%w: non-canonical loader path", ErrInvalidRequest)) + } + preHostXML, err := domainpreflight.RenderRequest(request.Domain) + if err != nil { + return Plan{}, newError(codePrefix+"invalid_domain", "domain policy could not be rendered", true, fmt.Errorf("%w: %v", ErrInvalidRequest, err)) + } + hostXML, err := domainxml.NormalizeWithLoader(preHostXML, request.LoaderPath) + if err != nil { + return Plan{}, newError(codePrefix+"loader_rejected", "host firmware loader does not satisfy policy", true, fmt.Errorf("%w: %v", ErrInvalidRequest, err)) + } + if err := verifyLoaderFile(request.LoaderPath); err != nil { + return Plan{}, err + } + if err := domainxml.Validate(hostXML); err != nil { + return Plan{}, newError(codePrefix+"loader_rejected", "host-bound firmware XML failed policy validation", true, fmt.Errorf("%w: %v", ErrInvalidRequest, err)) + } + + if err := checkContext(ctx); err != nil { + return Plan{}, err + } + seedDirectory := filepath.Join(request.ArtifactRoot, artifactID) + if filepath.Dir(seedDirectory) != request.ArtifactRoot || filepath.Base(seedDirectory) != artifactID { + return Plan{}, newError(codePrefix+"invalid_seed_artifact", "seed artifact identity is not safely scoped", true, fmt.Errorf("%w: %v", ErrInvalidRequest, hypervisor.ErrInvalidUUID)) + } + verified, err := startup.VerifySeedDirectoryDetails(seedDirectory) + if err != nil { + return Plan{}, classifySeedError(err) + } + if err := checkContext(ctx); err != nil { + return Plan{}, err + } + if !manifestEqual(verified.Manifest, request.SeedManifest) || + verified.StartupScriptIncluded != request.StartupScriptPresent || + verified.StartupScriptSHA256 != request.StartupScriptSHA256 { + return Plan{}, newError(codePrefix+"seed_artifact_rejected", "seed artifact does not match the requested identity", true, ErrSeedRejected) + } + + paths := make([]string, 0, len(request.Disks)) + for _, disk := range request.Disks { + paths = append(paths, disk.Path) + } + policy, err := storagepolicy.New(request.StorageRoots) + if err != nil { + return Plan{}, newError(codePrefix+"storage_rejected", "storage policy is invalid", true, fmt.Errorf("%w: %v", ErrStorageRejected, err)) + } + if err := policy.ValidatePaths(paths); err != nil { + return Plan{}, classifyStorageError(err) + } + if duplicate := duplicateDiskPath(request.Disks); duplicate { + return Plan{}, newError(codePrefix+"storage_rejected", "disk paths must resolve to distinct volumes", true, fmt.Errorf("%w: duplicate resolved path", ErrStorageRejected)) + } + if err := checkContext(ctx); err != nil { + return Plan{}, err + } + + xmlHash := digest(hostXML) + diskHash := digestDiskPlan(request.Disks) + planHash := digestPlan(planIdentity{ + DomainUUID: request.Domain.UUID, + DomainXMLHash: xmlHash, + SeedArtifactID: request.SeedArtifactID, + SeedManifest: request.SeedManifest, + StartupScriptPresent: request.StartupScriptPresent, + StartupScriptSHA256: request.StartupScriptSHA256, + DiskPlanHash: diskHash, + }) + result := Result{ + CommitBoundary: CommitBoundaryChecked, + DomainUUID: request.Domain.UUID, + DomainXMLHash: xmlHash, + SeedArtifactID: request.SeedArtifactID, + SeedManifest: request.SeedManifest, + StartupScriptPresent: request.StartupScriptPresent, + StartupScriptSHA256: request.StartupScriptSHA256, + DiskCount: len(request.Disks), + DiskPlanHash: diskHash, + PlanHash: planHash, + LoaderPending: false, + } + return Plan{ + Result: result, + hostXML: append([]byte(nil), hostXML...), + seedDirectory: seedDirectory, + disks: cloneDisks(request.Disks), + }, nil +} + +// Validate is an alias that makes the no-side-effect behavior explicit at +// call sites that do not yet need to retain the executor plan. +func Validate(ctx context.Context, request Request) (Plan, error) { + return Build(ctx, request) +} + +// BuildWithLoaderProvider obtains the firmware loader from a fresh host +// capability observation before building the host-bound plan. The caller +// cannot choose or reuse a loader path through this entry point; a provider +// failure is recoverable host unavailability, while the typed request and +// filesystem policy remain handled by Build. +func BuildWithLoaderProvider(ctx context.Context, provider hypervisor.LoaderPathProvider, request Request) (Plan, error) { + if ctx == nil { + ctx = context.Background() + } + if err := checkContext(ctx); err != nil { + return Plan{}, err + } + if provider == nil { + return Plan{}, newError(codePrefix+"loader_unavailable", "host firmware loader selection is unavailable", false, ErrHostUnavailable) + } + canonicalDomain, err := domainpreflight.ValidateRequest(request.Domain) + if err != nil { + return Plan{}, newError(codePrefix+"invalid_domain", "domain policy is invalid", true, fmt.Errorf("%w: %v", ErrInvalidRequest, err)) + } + request.Domain = canonicalDomain + loaderPath, err := provider.PreferredNonSecureLoaderPath(ctx) + if err != nil { + if contextErr := ctx.Err(); contextErr != nil { + return Plan{}, newError(codePrefix+"canceled", "attach validation was canceled", false, fmt.Errorf("%w: %v", ErrCanceled, contextErr)) + } + // A fresh capability observation can prove that the host has no + // acceptable non-Secure-Boot loader. That is a definitive host policy + // rejection, not a transport outcome that should be retried forever. + // Keep the hypervisor sentinel wrapped so callers can distinguish this + // from a missing/unreadable capability response. + if hypervisor.IsDefinitive(err) { + return Plan{}, newError(codePrefix+"loader_rejected", "host firmware policy does not provide a non-secure loader", true, fmt.Errorf("%w: %v", hypervisor.ErrCapabilityPolicy, err)) + } + return Plan{}, newError(codePrefix+"loader_unavailable", "host firmware loader selection is unavailable", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) + } + request.LoaderPath = loaderPath + return Build(ctx, request) +} + +type planIdentity struct { + DomainUUID string `json:"domain_uuid"` + DomainXMLHash string `json:"domain_xml_hash"` + SeedArtifactID string `json:"seed_artifact_id"` + SeedManifest domaincreateplan.SeedManifest `json:"seed_manifest"` + StartupScriptPresent bool `json:"first_boot_present"` + StartupScriptSHA256 string `json:"first_boot_digest"` + DiskPlanHash string `json:"disk_plan_hash"` +} + +type diskIdentity struct { + VolumeID string `json:"volume_id"` + Slot int `json:"slot"` + Readonly bool `json:"readonly"` +} + +func validateDisks(disks []DiskAttachment) error { + if len(disks) > maxDiskAttachments { + return newError(codePrefix+"storage_rejected", "too many disk attachments", true, fmt.Errorf("%w: disk count exceeds %d", ErrStorageRejected, maxDiskAttachments)) + } + seenIDs := make(map[string]struct{}, len(disks)) + seenSlots := make(map[int]struct{}, len(disks)) + for _, disk := range disks { + if !volumeIDPattern.MatchString(disk.VolumeID) { + return newError(codePrefix+"storage_rejected", "disk volume identity is invalid", true, fmt.Errorf("%w: invalid volume identity", ErrStorageRejected)) + } + if _, exists := seenIDs[disk.VolumeID]; exists { + return newError(codePrefix+"storage_rejected", "disk volume identities must be unique", true, fmt.Errorf("%w: duplicate volume identity", ErrStorageRejected)) + } + seenIDs[disk.VolumeID] = struct{}{} + if disk.Slot < 0 || disk.Slot >= maxDiskAttachments { + return newError(codePrefix+"storage_rejected", "disk slot is invalid", true, fmt.Errorf("%w: slot out of range", ErrStorageRejected)) + } + if _, exists := seenSlots[disk.Slot]; exists { + return newError(codePrefix+"storage_rejected", "disk slots must be unique", true, fmt.Errorf("%w: duplicate disk slot", ErrStorageRejected)) + } + seenSlots[disk.Slot] = struct{}{} + if disk.Path == "" { + return newError(codePrefix+"storage_rejected", "disk path was not resolved by the agent", true, fmt.Errorf("%w: missing resolved disk path", ErrStorageRejected)) + } + if strings.TrimSpace(disk.Path) != disk.Path || filepath.Clean(disk.Path) != disk.Path { + return newError(codePrefix+"storage_rejected", "disk path is not canonical", true, fmt.Errorf("%w: non-canonical disk path", ErrStorageRejected)) + } + } + return nil +} + +func validateExpectedManifest(manifest domaincreateplan.SeedManifest) error { + if manifest.Version != startup.SeedManifestVersion { + return fmt.Errorf("%w: unsupported seed manifest version", ErrInvalidRequest) + } + for _, digest := range []domaincreateplan.FileDigest{manifest.User, manifest.Meta, manifest.Network} { + if digest.Bytes < 0 || digest.Bytes > startup.MaxFileBytes || len(digest.SHA256) != sha256.Size*2 || strings.ToLower(digest.SHA256) != digest.SHA256 { + return fmt.Errorf("%w: seed manifest digest is invalid", ErrInvalidRequest) + } + if _, err := hex.DecodeString(digest.SHA256); err != nil { + return fmt.Errorf("%w: seed manifest digest is invalid", ErrInvalidRequest) + } + } + return nil +} + +func validateStartupExpectation(present bool, digestValue string) error { + if len(digestValue) != sha256.Size*2 || strings.ToLower(digestValue) != digestValue { + return newError(codePrefix+"invalid_seed_manifest", "startup-script digest identity is invalid", true, fmt.Errorf("%w: invalid startup digest", ErrInvalidRequest)) + } + if _, err := hex.DecodeString(digestValue); err != nil { + return newError(codePrefix+"invalid_seed_manifest", "startup-script digest identity is invalid", true, fmt.Errorf("%w: invalid startup digest", ErrInvalidRequest)) + } + emptyDigest := digest(nil) + if present == (digestValue == emptyDigest) { + return newError(codePrefix+"invalid_seed_manifest", "startup-script presence and digest disagree", true, fmt.Errorf("%w: inconsistent startup identity", ErrInvalidRequest)) + } + return nil +} + +func validateArtifactRoot(root string) error { + if root == "" || strings.TrimSpace(root) != root || strings.IndexByte(root, 0) >= 0 { + return newError(codePrefix+"artifact_unavailable", "seed artifact storage is unavailable", false, fmt.Errorf("%w: %v", ErrHostUnavailable, startup.ErrArtifactPath)) + } + clean := filepath.Clean(root) + if !filepath.IsAbs(root) || clean != root || clean == string(filepath.Separator) { + return newError(codePrefix+"invalid_artifact_root", "seed artifact root is invalid", true, fmt.Errorf("%w: %v", ErrInvalidRequest, startup.ErrArtifactPath)) + } + return nil +} + +func canonicalArtifactID(value string) (string, error) { + canonical, err := hypervisor.CanonicalUUID(value) + if err != nil || canonical != value { + if err == nil { + err = fmt.Errorf("%w: UUID must already be canonical", hypervisor.ErrInvalidUUID) + } + return "", err + } + return canonical, nil +} + +func verifyLoaderFile(path string) error { + if err := verifyLoaderAncestors(path); err != nil { + return err + } + info, err := os.Lstat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrPermission) || os.IsPermission(err) { + return newError(codePrefix+"loader_unavailable", "host firmware loader is unavailable", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) + } + return newError(codePrefix+"loader_unavailable", "host firmware loader could not be inspected", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() || info.Mode().Perm()&0o022 != 0 || info.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) != 0 { + return newError(codePrefix+"loader_rejected", "host firmware loader is not a private regular file", true, fmt.Errorf("%w: %v", ErrInvalidRequest, domainxml.ErrPolicyViolation)) + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return newError(codePrefix+"loader_unavailable", "host firmware loader could not be resolved", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) + } + if resolved != path { + return newError(codePrefix+"loader_rejected", "host firmware loader path is an alias", true, fmt.Errorf("%w: %v", ErrInvalidRequest, domainxml.ErrPolicyViolation)) + } + return nil +} + +// verifyLoaderAncestors rejects caller-controlled aliases and writable +// directory components before a capability-selected loader is embedded in +// host XML. The eventual executor still has to repeat this check immediately +// before define; this is a preflight boundary, not an atomic open guarantee. +func verifyLoaderAncestors(path string) error { + clean := filepath.Clean(path) + if clean == "." || !filepath.IsAbs(clean) { + return newError(codePrefix+"loader_rejected", "host firmware loader path is not canonical", true, fmt.Errorf("%w: %v", ErrInvalidRequest, domainxml.ErrPolicyViolation)) + } + root := filepath.VolumeName(clean) + string(filepath.Separator) + if filepath.VolumeName(clean) == "" { + root = string(filepath.Separator) + } + rest := strings.TrimPrefix(clean, root) + current := root + for _, component := range strings.Split(rest, string(filepath.Separator)) { + if component == "" || component == "." { + continue + } + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if err != nil { + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrPermission) || os.IsPermission(err) { + return newError(codePrefix+"loader_unavailable", "host firmware loader is unavailable", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) + } + return newError(codePrefix+"loader_unavailable", "host firmware loader could not be inspected", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) + } + if info.Mode()&os.ModeSymlink != 0 { + return newError(codePrefix+"loader_rejected", "host firmware loader path is an alias", true, fmt.Errorf("%w: %v", ErrInvalidRequest, domainxml.ErrPolicyViolation)) + } + if current != clean && !info.IsDir() { + return newError(codePrefix+"loader_rejected", "host firmware loader parent is not a directory", true, fmt.Errorf("%w: %v", ErrInvalidRequest, domainxml.ErrPolicyViolation)) + } + if current != clean && !safeLoaderDirectoryMode(info.Mode()) { + return newError(codePrefix+"loader_rejected", "host firmware loader parent is writable by group or other users", true, fmt.Errorf("%w: %v", ErrInvalidRequest, domainxml.ErrPolicyViolation)) + } + } + return nil +} + +func safeLoaderDirectoryMode(mode os.FileMode) bool { + if mode&(os.ModeSetuid|os.ModeSetgid) != 0 { + return false + } + if mode.Perm()&0o022 == 0 { + return true + } + // Standard shared temporary roots are sticky-protected; the actual loader + // file is still required to be a private regular file below this boundary. + return mode&os.ModeSticky != 0 +} + +func classifySeedError(err error) error { + if errors.Is(err, fs.ErrPermission) || os.IsPermission(err) || errors.Is(err, startup.ErrArtifactPath) { + return newError(codePrefix+"artifact_unavailable", "seed artifact storage is unavailable", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) + } + if errors.Is(err, startup.ErrInvalidArtifact) { + return newError(codePrefix+"seed_artifact_rejected", "seed artifact is missing or invalid", true, fmt.Errorf("%w: %v", ErrSeedRejected, err)) + } + return newError(codePrefix+"artifact_unavailable", "seed artifact could not be read", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) +} + +func classifyStorageError(err error) error { + if errors.Is(err, storagepolicy.ErrRootUnavailable) || errors.Is(err, fs.ErrPermission) || os.IsPermission(err) { + return newError(codePrefix+"storage_unavailable", "configured storage is unavailable", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) + } + if storagepolicy.IsDefinitive(err) { + return newError(codePrefix+"storage_rejected", "disk path does not satisfy the storage policy", true, fmt.Errorf("%w: %v", ErrStorageRejected, err)) + } + return newError(codePrefix+"storage_unavailable", "disk path could not be checked", false, fmt.Errorf("%w: %v", ErrHostUnavailable, err)) +} + +func manifestEqual(observed startup.SeedManifest, expected domaincreateplan.SeedManifest) bool { + if observed.Version != expected.Version || len(observed.Files) != 3 { + return false + } + wanted := map[string]domaincreateplan.FileDigest{ + startup.UserDataFilename: expected.User, + startup.MetaDataFilename: expected.Meta, + startup.NetworkConfigFilename: expected.Network, + } + for name, want := range wanted { + got, ok := observed.Files[name] + if !ok || got.Bytes != want.Bytes || got.SHA256 != want.SHA256 { + return false + } + } + return true +} + +func digestDiskPlan(disks []DiskAttachment) string { + identities := make([]diskIdentity, 0, len(disks)) + for _, disk := range disks { + identities = append(identities, diskIdentity{VolumeID: disk.VolumeID, Slot: disk.Slot, Readonly: disk.Readonly}) + } + sort.Slice(identities, func(left, right int) bool { + if identities[left].Slot != identities[right].Slot { + return identities[left].Slot < identities[right].Slot + } + return identities[left].VolumeID < identities[right].VolumeID + }) + data, _ := json.Marshal(identities) + return digest(data) +} + +func digestPlan(identity planIdentity) string { + data, _ := json.Marshal(identity) + return digest(data) +} + +func duplicateDiskPath(disks []DiskAttachment) bool { + seen := make(map[string]struct{}, len(disks)) + for _, disk := range disks { + if _, exists := seen[disk.Path]; exists { + return true + } + seen[disk.Path] = struct{}{} + } + return false +} + +func digest(value []byte) string { + sum := sha256.Sum256(value) + return hex.EncodeToString(sum[:]) +} + +func checkContext(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return newError(codePrefix+"canceled", "attach validation was canceled", false, fmt.Errorf("%w: %v", ErrCanceled, err)) + } + return nil +} + +func cloneDisks(disks []DiskAttachment) []DiskAttachment { + result := make([]DiskAttachment, len(disks)) + copy(result, disks) + return result +} diff --git a/agent/internal/domaincreateattach/attach_test.go b/agent/internal/domaincreateattach/attach_test.go new file mode 100644 index 00000000..56015ad9 --- /dev/null +++ b/agent/internal/domaincreateattach/attach_test.go @@ -0,0 +1,421 @@ +package domaincreateattach + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "g.luoli.live/root/virt/agent/internal/domaincreateplan" + "g.luoli.live/root/virt/agent/internal/domainpreflight" + "g.luoli.live/root/virt/agent/internal/hypervisor" + "g.luoli.live/root/virt/agent/internal/startup" +) + +const ( + attachDomainUUID = "00000000-0000-4000-8000-000000001101" + attachSeedUUID = "00000000-0000-4000-8000-000000001102" +) + +func attachDomainRequest() domainpreflight.Request { + readonly := true + secureBoot := false + return domainpreflight.Request{ + UUID: attachDomainUUID, + Firmware: domainpreflight.FirmwareEFI, + Architecture: domainpreflight.ArchitectureX86, + Machine: domainpreflight.MachineQ35, + OSType: domainpreflight.OSTypeHVM, + Loader: domainpreflight.LoaderPolicy{ + Type: domainpreflight.LoaderPFlash, + Readonly: &readonly, + SecureBoot: &secureBoot, + }, + } +} + +type attachFixture struct { + request Request + artifact string + loader string + diskRoot string + diskPath string + seedFiles startup.SeedFiles +} + +type fixtureLoaderProvider struct { + path string + err error +} + +func (p fixtureLoaderProvider) PreferredNonSecureLoaderPath(context.Context) (string, error) { + return p.path, p.err +} + +func newAttachFixture(t *testing.T) attachFixture { + t.Helper() + artifactRoot := canonicalAttachDir(t) + diskRoot := canonicalAttachDir(t) + loader := filepath.Join(canonicalAttachDir(t), "OVMF_CODE_4M.fd") + if err := os.WriteFile(loader, []byte("firmware"), 0o600); err != nil { + t.Fatal(err) + } + diskPath := filepath.Join(diskRoot, "volume-1.qcow2") + if err := os.WriteFile(diskPath, []byte("disk contents"), 0o600); err != nil { + t.Fatal(err) + } + seedFiles, err := startup.BuildCreateSeed( + "instance-attach-1", + "vm-attach-1", + `{"version":2,"ethernets":{}}`, + "#!/bin/sh\necho first boot\n", + ) + if err != nil { + t.Fatal(err) + } + artifactPath := filepath.Join(artifactRoot, attachSeedUUID) + artifact, err := startup.WriteSeedDirectory(artifactPath, seedFiles) + if err != nil { + t.Fatal(err) + } + manifest := domaincreateplan.SeedManifest{ + Version: artifact.Manifest.Version, + User: domaincreateplan.FileDigest{ + Bytes: artifact.Manifest.Files[startup.UserDataFilename].Bytes, + SHA256: artifact.Manifest.Files[startup.UserDataFilename].SHA256, + }, + Meta: domaincreateplan.FileDigest{ + Bytes: artifact.Manifest.Files[startup.MetaDataFilename].Bytes, + SHA256: artifact.Manifest.Files[startup.MetaDataFilename].SHA256, + }, + Network: domaincreateplan.FileDigest{ + Bytes: artifact.Manifest.Files[startup.NetworkConfigFilename].Bytes, + SHA256: artifact.Manifest.Files[startup.NetworkConfigFilename].SHA256, + }, + } + return attachFixture{ + request: Request{ + Domain: attachDomainRequest(), + LoaderPath: loader, + ArtifactRoot: artifactRoot, + StorageRoots: []string{diskRoot}, + SeedArtifactID: attachSeedUUID, + SeedManifest: manifest, + StartupScriptPresent: true, + StartupScriptSHA256: attachScriptDigest(), + Disks: []DiskAttachment{{ + VolumeID: "volume-attach-1", + Slot: 0, + Path: diskPath, + }}, + }, + artifact: artifactPath, + loader: loader, + diskRoot: diskRoot, + diskPath: diskPath, + seedFiles: seedFiles, + } +} + +func attachScriptDigest() string { + sum := sha256.Sum256([]byte("#!/bin/sh\necho first boot\n")) + return hex.EncodeToString(sum[:]) +} + +func canonicalAttachDir(t *testing.T) string { + t.Helper() + path := t.TempDir() + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + t.Fatal(err) + } + return resolved +} + +func TestBuildChecksSeedAndStorageWithoutMutation(t *testing.T) { + fixture := newAttachFixture(t) + plan, err := Build(context.Background(), fixture.request) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + if plan.CommitBoundary != CommitBoundaryChecked || plan.LoaderPending { + t.Fatalf("result boundary/loader state = %q/%v", plan.CommitBoundary, plan.LoaderPending) + } + if plan.DomainUUID != attachDomainUUID || plan.SeedArtifactID != attachSeedUUID || plan.DiskCount != 1 { + t.Fatalf("result identity = %+v", plan.Result) + } + if len(plan.DomainXMLHash) != 64 || len(plan.DiskPlanHash) != 64 || len(plan.PlanHash) != 64 { + t.Fatalf("result hashes = %+v", plan.Result) + } + xml := string(plan.HostXML()) + if !strings.Contains(xml, `loader`) || !strings.Contains(xml, fixture.loader) { + t.Fatalf("host XML did not pin selected loader: %s", xml) + } + for _, forbidden := range []string{" is not allowed", ErrPolicyViolation, node.name.Local) + } + for _, attr := range node.attrs { + if attr.Name.Space != "" { + return fmt.Errorf("%w: namespaced attribute on <%s> is not allowed", ErrPolicyViolation, node.name.Local) + } + } + name := strings.ToLower(node.name.Local) + switch name { + case "interface": + return fmt.Errorf("%w: network interfaces require a typed network operation", ErrPolicyViolation) + case "filesystem": + return fmt.Errorf("%w: host filesystems are not allowed", ErrPolicyViolation) + case "nvram", "emulator": + return fmt.Errorf("%w: host firmware/emulator paths require an agent-owned resolver", ErrPolicyViolation) + case "serial", "parallel", "console", "channel", "shmem", "rng", "tpm", "redirdev", "smartcard": + if elementContainsHostSource(node) { + return fmt.Errorf("%w: host-backed device <%s> is not allowed", ErrPolicyViolation, name) + } + } + if name == "disk" { + if err := validateFileDiskSource(node); err != nil { + return err + } + } + for _, child := range node.children { + if child.kind != contentElement { + continue + } + if strings.EqualFold(child.elem.name.Local, "source") && name != "disk" && elementContainsHostSource(child.elem) { + return fmt.Errorf("%w: host source under <%s> is not allowed", ErrPolicyViolation, name) + } + if err := validateHostPathElements(child.elem); err != nil { + return err + } + } + return nil +} + +func validateFileDiskSource(node *element) error { + diskType, ok := attributeValue(node, "type") + if ok && diskType != "file" { + return fmt.Errorf("%w: disk type %q is not an allowlisted file-backed disk", ErrPolicyViolation, diskType) + } + // XML element names are case-sensitive to libvirt, but rejecting a + // case-variant explicitly is safer than letting it fall through the + // case-sensitive childrenNamed helper and then missing a host path. + sources := make([]*element, 0, 1) + for _, child := range node.children { + if child.kind != contentElement || !strings.EqualFold(child.elem.name.Local, "source") { + continue + } + if child.elem.name.Local != "source" { + return fmt.Errorf("%w: disk source element name must be exactly ", ErrPolicyViolation) + } + sources = append(sources, child.elem) + } + if len(sources) > 1 { + return fmt.Errorf("%w: disk contains multiple source elements", ErrPolicyViolation) + } + if len(sources) == 0 { + return nil + } + source := sources[0] + filePath := "" + for _, attr := range source.attrs { + if attr.Name.Space != "" { + return fmt.Errorf("%w: namespaced disk source attributes are not allowed", ErrPolicyViolation) + } + switch attr.Name.Local { + case "file": + if filePath != "" { + return fmt.Errorf("%w: disk source contains duplicate file attributes", ErrPolicyViolation) + } + filePath = strings.TrimSpace(attr.Value) + default: + return fmt.Errorf("%w: disk source attribute %q is not an allowlisted local file", ErrPolicyViolation, attr.Name.Local) + } + } + if filePath == "" { + return fmt.Errorf("%w: disk source must name a local file", ErrPolicyViolation) + } + if source.hasElementChildren() || strings.TrimSpace(source.text()) != "" { + return fmt.Errorf("%w: disk source must contain only a file attribute", ErrPolicyViolation) + } + return nil +} + +func elementContainsHostSource(node *element) bool { + if node == nil { + return false + } + if strings.EqualFold(node.name.Local, "source") { + return true + } + for _, attr := range node.attrs { + switch strings.ToLower(attr.Name.Local) { + case "file", "path", "dev", "dir", "protocol", "socket", "volume", "pool": + return true + } + } + if strings.TrimSpace(node.text()) != "" { + return true + } + for _, child := range node.children { + if child.kind == contentElement && elementContainsHostSource(child.elem) { + return true + } + } + return false +} + // FileBackedDiskPaths returns the host paths used by file-backed disk // sources. It is intentionally a parser-level operation rather than a // regexp: XML escaping, element ordering and namespace-qualified elements @@ -673,11 +822,11 @@ func parse(input []byte) (*document, error) { return doc, nil } -// validateUnsafeElements rejects host-escape constructs that should never be -// accepted from a panel payload. Ordinary file-backed disks and libvirt -// network interfaces remain valid inputs, but raw block devices, host-device -// interfaces/filesystems and qemu command injection are reserved for a future, -// separately reviewed capability with explicit admin policy. +// validateUnsafeElements rejects parser-level escape constructs before the +// firmware and host-path policies run. The latter deliberately applies a +// stricter greenfield rule: ordinary libvirt network interfaces and all other +// host-backed devices remain reserved for typed Agent operations with explicit +// ownership and read-back contracts. func validateUnsafeElements(root *element) error { if root == nil { return nil diff --git a/agent/internal/domainxml/domainxml_test.go b/agent/internal/domainxml/domainxml_test.go index 75ff9022..191fbdf7 100644 --- a/agent/internal/domainxml/domainxml_test.go +++ b/agent/internal/domainxml/domainxml_test.go @@ -190,6 +190,82 @@ func TestDomainXMLRejectsDirectivesAndHostEscapeNodes(t *testing.T) { } } +func TestValidateHostPathPolicyRejectsUntypedHostCarriers(t *testing.T) { + base := `hvm/usr/share/OVMF/OVMF_CODE.fd` + tests := []struct { + name string + extra string + }{ + { + name: "network interface", + extra: ``, + }, + { + name: "remote disk", + extra: ``, + }, + { + name: "block disk", + extra: ``, + }, + { + name: "firmware nvram", + extra: `/var/lib/libvirt/nvram/vm.fd`, + }, + { + name: "emulator path", + extra: `/usr/bin/qemu-system-x86_64`, + }, + { + name: "memory backing file", + extra: ``, + }, + { + name: "host serial socket", + extra: ``, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input := strings.Replace(base, ``, tt.extra+``, 1) + if err := Validate([]byte(input)); !errors.Is(err, ErrPolicyViolation) { + t.Fatalf("Validate() error = %v, want ErrPolicyViolation", err) + } + if err := ValidateHostPathPolicy([]byte(input)); !errors.Is(err, ErrPolicyViolation) { + t.Fatalf("ValidateHostPathPolicy() error = %v, want ErrPolicyViolation", err) + } + }) + } +} + +func TestValidateHostPathPolicyAcceptsAllowlistedShapeOnly(t *testing.T) { + input := `hvm/usr/share/OVMF/OVMF_CODE.fd` + if err := ValidateHostPathPolicy([]byte(input)); err != nil { + t.Fatalf("ValidateHostPathPolicy() rejected file-backed disk/pty shape: %v", err) + } + if err := Validate([]byte(input)); err != nil { + t.Fatalf("Validate() rejected file-backed disk/pty shape: %v", err) + } +} + +func TestValidateHostPathPolicyRejectsAmbiguousDiskSource(t *testing.T) { + tests := []string{ + ``, + ``, + ``, + ``, + ``, + ``, + } + for index, input := range tests { + t.Run(fmt.Sprintf("case-%d", index), func(t *testing.T) { + if err := ValidateHostPathPolicy([]byte(input)); !errors.Is(err, ErrPolicyViolation) { + t.Fatalf("ValidateHostPathPolicy() error = %v, want ErrPolicyViolation", err) + } + }) + } +} + func TestDomainXMLRejectsOversizedInput(t *testing.T) { input := append([]byte(`hvm`), bytes.Repeat([]byte{'x'}, maxDomainXMLBytes)...) input = append(input, []byte(``)...) diff --git a/agent/internal/hypervisor/backend.go b/agent/internal/hypervisor/backend.go index a30636e2..c0b3d4aa 100644 --- a/agent/internal/hypervisor/backend.go +++ b/agent/internal/hypervisor/backend.go @@ -180,6 +180,16 @@ type Backend interface { ReadDomainPowerState(context.Context, string) (DomainPowerState, error) } +// LoaderPathProvider is an optional host-bound capability seam. Implementations +// must select the path from a fresh, read-only domain-capabilities observation; +// callers must not persist or accept a loader path supplied by the panel. +// Keeping this separate from Backend lets older test doubles and future native +// bindings adopt the create pipeline incrementally without weakening the +// existing lifecycle interface. +type LoaderPathProvider interface { + PreferredNonSecureLoaderPath(context.Context) (string, error) +} + // ReadbackDomain obtains and validates one explicit UUID from a Backend. It // is the common post-define assertion used by future libvirt implementations. func ReadbackDomain(ctx context.Context, backend Backend, uuid string) (Definition, error) { @@ -287,6 +297,16 @@ func (f *FakeBackend) Capabilities(ctx context.Context) (CapabilitySnapshot, err return caps, nil } +// PreferredNonSecureLoaderPath mirrors the real host seam with a deterministic +// descriptor. The fake still probes first so tests exercise the same fresh +// capability/error boundary as a libvirt adapter. +func (f *FakeBackend) PreferredNonSecureLoaderPath(ctx context.Context) (string, error) { + if _, err := f.Probe(ctx); err != nil { + return "", err + } + return fakeLoaderPath, nil +} + // DefineDomain normalizes and stores one explicit definition. Re-defining an // identical UUID/XML pair is idempotent; redefining the UUID with different // XML is rejected rather than overwriting an unknown guest. diff --git a/agent/internal/hypervisor/backend_test.go b/agent/internal/hypervisor/backend_test.go index 1da7dcaf..ef5be793 100644 --- a/agent/internal/hypervisor/backend_test.go +++ b/agent/internal/hypervisor/backend_test.go @@ -65,6 +65,22 @@ func TestCheckEFIQ35NoSecureBoot(t *testing.T) { } } +func TestFakeBackendSelectsFreshNonSecureLoader(t *testing.T) { + backend := NewFakeBackend(DefaultCapabilitySnapshot()) + path, err := backend.PreferredNonSecureLoaderPath(context.Background()) + if err != nil { + t.Fatalf("PreferredNonSecureLoaderPath() error = %v", err) + } + if path != fakeLoaderPath { + t.Fatalf("loader path = %q, want %q", path, fakeLoaderPath) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := backend.PreferredNonSecureLoaderPath(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled loader selection error = %v, want context.Canceled", err) + } +} + func TestFakeBackendDefineAndReadback(t *testing.T) { backend := NewFakeBackend(DefaultCapabilitySnapshot()) defined, err := backend.DefineDomain(context.Background(), DefineRequest{ diff --git a/agent/internal/hypervisor/virsh.go b/agent/internal/hypervisor/virsh.go index 19f68a82..3b352422 100644 --- a/agent/internal/hypervisor/virsh.go +++ b/agent/internal/hypervisor/virsh.go @@ -305,6 +305,18 @@ func (b *VirshBackend) Capabilities(ctx context.Context) (CapabilitySnapshot, er return result.Capabilities, nil } +// PreferredNonSecureLoaderPath selects a loader from a fresh host capability +// observation. It is intentionally separate from Capabilities so a future +// create executor can retain the observation boundary together with the path +// it uses, instead of trusting a stale or panel-supplied string. +func (b *VirshBackend) PreferredNonSecureLoaderPath(ctx context.Context) (string, error) { + domainCaps, _, err := b.probeDetails(ctx) + if err != nil { + return "", err + } + return domainCaps.PreferredNonSecureLoaderPath() +} + func (b *VirshBackend) DefineDomain(ctx context.Context, request DefineRequest) (Definition, error) { if err := contextErr(ctx); err != nil { return Definition{}, err diff --git a/agent/internal/hypervisor/virsh_test.go b/agent/internal/hypervisor/virsh_test.go index 436fccbc..770aa66c 100644 --- a/agent/internal/hypervisor/virsh_test.go +++ b/agent/internal/hypervisor/virsh_test.go @@ -90,6 +90,13 @@ func TestVirshBackendUsesFixedArgumentsAndReadback(t *testing.T) { if _, err := backend.Probe(context.Background()); err != nil { t.Fatal(err) } + loaderPath, err := backend.PreferredNonSecureLoaderPath(context.Background()) + if err != nil { + t.Fatalf("PreferredNonSecureLoaderPath() error = %v", err) + } + if loaderPath != "/usr/share/OVMF/OVMF_CODE_4M.fd" { + t.Fatalf("loader path = %q", loaderPath) + } definition, err := backend.DefineDomain(context.Background(), DefineRequest{ UUID: testUUID, XML: testDomainXML(testUUID), diff --git a/agent/internal/operation/store.go b/agent/internal/operation/store.go index 642d970c..59e48f4b 100644 --- a/agent/internal/operation/store.go +++ b/agent/internal/operation/store.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" "sync" "time" @@ -97,8 +98,13 @@ type journalFile struct { } type Store struct { - mu sync.RWMutex - path string + mu sync.RWMutex + path string + // fileInfo identifies the journal inode that was opened or most recently + // published. It lets later atomic writes fail closed if another local + // process replaces the path between our checks instead of silently + // overwriting an unrelated file. + fileInfo os.FileInfo records map[string]Record byIdempotency map[string]string nonces map[string]time.Time @@ -123,6 +129,13 @@ func Open(path string) (*Store, error) { if strings.TrimSpace(path) == "" { return nil, fmt.Errorf("journal path is empty") } + if path != ":memory:" { + var err error + path, err = normalizeJournalPath(path) + if err != nil { + return nil, err + } + } s := &Store{ path: path, records: make(map[string]Record), @@ -133,31 +146,21 @@ func Open(path string) (*Store, error) { if path == ":memory:" { return s, nil } + if _, err := ensureJournalParent(filepath.Dir(path), true); err != nil { + return nil, err + } if info, err := os.Lstat(path); err == nil { - if info.Mode()&os.ModeSymlink != 0 { - return nil, fmt.Errorf("journal path must not be a symlink") - } - if !info.Mode().IsRegular() { - return nil, fmt.Errorf("operation journal path is not a regular file") - } - if info.Mode().Perm()&0o077 != 0 || info.Mode().Perm()&0o600 != 0o600 { - return nil, fmt.Errorf("operation journal permissions must be owner read/write only") + if err := validateJournalFile(info); err != nil { + return nil, err } if info.Size() > maxJournalBytes { return nil, fmt.Errorf("operation journal exceeds %d bytes", maxJournalBytes) } - fileHandle, openErr := os.Open(path) - if openErr != nil { - return nil, fmt.Errorf("read operation journal: %w", openErr) - } - data, readErr := io.ReadAll(io.LimitReader(fileHandle, maxJournalBytes+1)) - closeErr := fileHandle.Close() + data, openedInfo, readErr := readJournalFile(path, info) if readErr != nil { - return nil, fmt.Errorf("read operation journal: %w", readErr) - } - if closeErr != nil { - return nil, fmt.Errorf("close operation journal: %w", closeErr) + return nil, readErr } + s.fileInfo = openedInfo if len(data) > maxJournalBytes { return nil, fmt.Errorf("operation journal exceeds %d bytes", maxJournalBytes) } @@ -197,7 +200,12 @@ func Open(path string) (*Store, error) { } else if !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("inspect operation journal: %w", err) } - if err := probeWritable(path); err != nil { + if info, err := probeWritableWithInfo(path, s.fileInfo); err != nil { + return nil, err + } else if info != nil { + s.fileInfo = info + } + if _, err := ensureJournalParent(filepath.Dir(path), false); err != nil { return nil, err } return s, nil @@ -237,41 +245,362 @@ func (s *Store) ConsumeNonce(nonce string, expiresAt, now time.Time) bool { return true } +// probeWritable retains the small internal probe used by older package tests; +// production Open uses probeWritableWithInfo so it can retain the checked +// inode identity across the lstat/open boundary. func probeWritable(path string) error { - if info, err := os.Stat(path); err == nil { - file, openErr := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0) + // Preserve the historical helper behavior for callers/tests that probe a + // brand-new journal directly: create missing ancestors, but validate every + // existing component before doing so. Open uses ensureJournalParent itself + // and therefore passes the stricter non-creating path to the detailed probe. + if _, err := ensureJournalParent(filepath.Dir(path), true); err != nil { + return err + } + _, err := probeWritableWithInfo(path, nil) + return err +} + +func probeWritableWithInfo(path string, expected os.FileInfo) (os.FileInfo, error) { + if info, err := os.Lstat(path); err == nil { + if err := validateJournalFile(info); err != nil { + return nil, err + } + parentInfo, parentErr := ensureJournalParent(filepath.Dir(path), false) + if parentErr != nil { + return nil, parentErr + } + if expected != nil && !os.SameFile(expected, info) { + return nil, fmt.Errorf("operation journal changed while probing") + } + file, openErr := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o600) if openErr != nil { - return fmt.Errorf("operation journal is not writable: %w", openErr) + return nil, fmt.Errorf("operation journal is not writable: %w", openErr) } - if closeErr := file.Close(); closeErr != nil { - return fmt.Errorf("close operation journal probe: %w", closeErr) + openedInfo, statErr := file.Stat() + if statErr != nil { + _ = file.Close() + return nil, fmt.Errorf("stat operation journal probe: %w", statErr) } - if info.IsDir() { - return fmt.Errorf("operation journal path is a directory") + if !secureJournalFile(openedInfo) || !os.SameFile(info, openedInfo) { + _ = file.Close() + return nil, fmt.Errorf("operation journal changed while probing") } - return nil + currentInfo, currentErr := os.Lstat(path) + if currentErr != nil { + _ = file.Close() + return nil, fmt.Errorf("recheck operation journal probe: %w", currentErr) + } + if !secureJournalFile(currentInfo) || !os.SameFile(info, currentInfo) { + _ = file.Close() + return nil, fmt.Errorf("operation journal changed while probing") + } + currentParent, parentErr := ensureJournalParent(filepath.Dir(path), false) + if parentErr != nil || !os.SameFile(parentInfo, currentParent) { + _ = file.Close() + if parentErr != nil { + return nil, fmt.Errorf("recheck operation journal parent: %w", parentErr) + } + return nil, fmt.Errorf("operation journal parent changed while probing") + } + if closeErr := file.Close(); closeErr != nil { + return nil, fmt.Errorf("close operation journal probe: %w", closeErr) + } + return currentInfo, nil } else if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("probe operation journal: %w", err) + return nil, fmt.Errorf("probe operation journal: %w", err) } - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o750); err != nil { - return fmt.Errorf("create journal directory: %w", err) + parentInfo, err := ensureJournalParent(filepath.Dir(path), false) + if err != nil { + return nil, err + } + if parentInfo == nil { + return nil, fmt.Errorf("operation journal parent is unavailable") } - tmp, err := os.CreateTemp(dir, ".operations-probe-*.tmp") + tmp, err := os.CreateTemp(filepath.Dir(path), ".operations-probe-*.tmp") if err != nil { - return fmt.Errorf("operation journal is not writable: %w", err) + return nil, fmt.Errorf("operation journal is not writable: %w", err) } name := tmp.Name() + removed := false + defer func() { + if !removed { + _ = os.Remove(name) + } + }() + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return nil, fmt.Errorf("set operation journal probe mode: %w", err) + } + if info, statErr := tmp.Stat(); statErr != nil || !secureJournalFile(info) { + _ = tmp.Close() + if statErr != nil { + return nil, fmt.Errorf("stat operation journal probe: %w", statErr) + } + return nil, fmt.Errorf("operation journal probe is not a secure regular file") + } if closeErr := tmp.Close(); closeErr != nil { - _ = os.Remove(name) - return fmt.Errorf("close operation journal probe: %w", closeErr) + return nil, fmt.Errorf("close operation journal probe: %w", closeErr) + } + currentParent, err := ensureJournalParent(filepath.Dir(path), false) + if err != nil { + return nil, err + } + if !os.SameFile(parentInfo, currentParent) { + return nil, fmt.Errorf("operation journal parent changed during probe") + } + if _, err := os.Lstat(path); err == nil { + return nil, fmt.Errorf("operation journal appeared during probe") + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("recheck operation journal probe: %w", err) } if err := os.Remove(name); err != nil { - return fmt.Errorf("remove operation journal probe: %w", err) + return nil, fmt.Errorf("remove operation journal probe: %w", err) + } + removed = true + return nil, nil +} + +func normalizeJournalPath(path string) (string, error) { + if path == "" || strings.TrimSpace(path) != path || strings.IndexByte(path, 0) >= 0 { + return "", fmt.Errorf("journal path is empty or malformed") + } + for _, component := range strings.FieldsFunc(path, func(r rune) bool { + return r == '/' || r == '\\' + }) { + if component == ".." { + return "", fmt.Errorf("journal path must not contain traversal components") + } + } + abs, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("resolve journal path: %w", err) + } + return filepath.Clean(abs), nil +} + +// ensureJournalParent verifies every component of a journal's parent. When +// create is true, missing components are created one at a time with a +// non-writable-by-group/other mode and then verified again. Checking one +// complete path with Lstat would follow an intermediate symlink. +func ensureJournalParent(path string, create bool) (os.FileInfo, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("resolve journal parent: %w", err) + } + abs = filepath.Clean(abs) + if info, err := checkJournalDirectoryTree(abs); err == nil { + return info, nil + } else if !create || !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + root := filepath.VolumeName(abs) + string(filepath.Separator) + if filepath.VolumeName(abs) == "" { + root = string(filepath.Separator) + } + rest := strings.TrimPrefix(abs, root) + current := root + missing := make([]string, 0) + for _, component := range strings.Split(rest, string(filepath.Separator)) { + if component == "" || component == "." { + continue + } + current = filepath.Join(current, component) + info, statErr := os.Lstat(current) + if statErr == nil { + if _, checkErr := validateJournalDirectory(info, current); checkErr != nil { + return nil, checkErr + } + continue + } + if !errors.Is(statErr, os.ErrNotExist) { + return nil, fmt.Errorf("inspect journal parent %q: %w", current, statErr) + } + missing = append(missing, current) + } + for index := 0; index < len(missing); index++ { + candidate := missing[index] + if err := os.Mkdir(candidate, 0o750); err != nil && !errors.Is(err, os.ErrExist) { + return nil, fmt.Errorf("create journal directory %q: %w", candidate, err) + } + info, statErr := os.Lstat(candidate) + if statErr != nil { + return nil, fmt.Errorf("inspect created journal directory %q: %w", candidate, statErr) + } + if _, checkErr := validateJournalDirectory(info, candidate); checkErr != nil { + return nil, checkErr + } + } + return checkJournalDirectoryTree(abs) +} + +func checkJournalDirectoryTree(path string) (os.FileInfo, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("resolve journal parent: %w", err) + } + abs = filepath.Clean(abs) + root := filepath.VolumeName(abs) + string(filepath.Separator) + if filepath.VolumeName(abs) == "" { + root = string(filepath.Separator) + } + rest := strings.TrimPrefix(abs, root) + current := root + if rest == "" { + return validateJournalDirectoryAt(current) + } + var last os.FileInfo + for _, component := range strings.Split(rest, string(filepath.Separator)) { + if component == "" || component == "." { + continue + } + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if err != nil { + return nil, fmt.Errorf("inspect journal parent %q: %w", current, err) + } + validated, err := validateJournalDirectory(info, current) + if err != nil { + return nil, err + } + last = validated + } + if last == nil { + return validateJournalDirectoryAt(root) + } + return last, nil +} + +func validateJournalDirectoryAt(path string) (os.FileInfo, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("inspect journal parent %q: %w", path, err) + } + return validateJournalDirectory(info, path) +} + +func validateJournalDirectory(info os.FileInfo, path string) (os.FileInfo, error) { + if info == nil { + return nil, fmt.Errorf("journal parent %q must contain only real directories", path) + } + if info.Mode()&os.ModeSymlink != 0 { + if !trustedDarwinAlias(path) { + return nil, fmt.Errorf("journal parent %q must contain only real directories", path) + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return nil, fmt.Errorf("resolve journal parent %q: %w", path, err) + } + resolvedInfo, err := os.Lstat(resolved) + if err != nil { + return nil, fmt.Errorf("inspect journal parent %q: %w", path, err) + } + if resolvedInfo.Mode()&os.ModeSymlink != 0 || !resolvedInfo.IsDir() { + return nil, fmt.Errorf("journal parent %q must contain only real directories", path) + } + info = resolvedInfo + } + if !info.IsDir() { + return nil, fmt.Errorf("journal parent %q must contain only real directories", path) + } + if !safeJournalDirectoryMode(info.Mode()) { + return nil, fmt.Errorf("journal parent %q must not be group/world writable", path) + } + return info, nil +} + +func safeJournalDirectoryMode(mode os.FileMode) bool { + if mode&(os.ModeSetuid|os.ModeSetgid) != 0 { + return false + } + if mode.Perm()&0o022 == 0 { + return true + } + // Permit the kernel-protected sticky temporary directory used by standard + // test/runtime environments; all agent-created child directories remain + // owner-only (0750) and the journal file remains 0600. + return mode&os.ModeSticky != 0 +} + +func trustedDarwinAlias(path string) bool { + // macOS maps /var to /private/var. Keep the same narrow compatibility + // exception used by auth while rejecting caller-controlled links. + if runtime.GOOS != "darwin" { + return false + } + switch filepath.Clean(path) { + case "/var": + return true + default: + return false + } +} + +func secureJournalFile(info os.FileInfo) bool { + return info != nil && info.Mode()&os.ModeSymlink == 0 && info.Mode().IsRegular() && + info.Mode().Perm() == 0o600 && info.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) == 0 +} + +func validateJournalFile(info os.FileInfo) error { + if info == nil { + return fmt.Errorf("operation journal path is unavailable") + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("journal path must not be a symlink") + } + if !info.Mode().IsRegular() { + return fmt.Errorf("operation journal path is not a regular file") + } + if !secureJournalFile(info) { + return fmt.Errorf("operation journal permissions must be owner read/write only (0600)") } return nil } +func readJournalFile(path string, expected os.FileInfo) ([]byte, os.FileInfo, error) { + fileHandle, openErr := os.Open(path) + if openErr != nil { + return nil, nil, fmt.Errorf("read operation journal: %w", openErr) + } + closed := false + defer func() { + if !closed { + _ = fileHandle.Close() + } + }() + openedInfo, err := fileHandle.Stat() + if err != nil { + return nil, nil, fmt.Errorf("stat operation journal: %w", err) + } + if !secureJournalFile(openedInfo) || expected == nil || !os.SameFile(expected, openedInfo) { + return nil, nil, fmt.Errorf("operation journal changed while opening") + } + currentInfo, err := os.Lstat(path) + if err != nil { + return nil, nil, fmt.Errorf("recheck operation journal: %w", err) + } + if !secureJournalFile(currentInfo) || !os.SameFile(expected, currentInfo) { + return nil, nil, fmt.Errorf("operation journal changed while opening") + } + data, readErr := io.ReadAll(io.LimitReader(fileHandle, maxJournalBytes+1)) + if readErr != nil { + return nil, nil, fmt.Errorf("read operation journal: %w", readErr) + } + currentInfo, err = os.Lstat(path) + if err != nil { + return nil, nil, fmt.Errorf("recheck operation journal: %w", err) + } + if !secureJournalFile(currentInfo) || !os.SameFile(expected, currentInfo) { + return nil, nil, fmt.Errorf("operation journal changed while reading") + } + if err := fileHandle.Close(); err != nil { + closed = true + return nil, nil, fmt.Errorf("close operation journal: %w", err) + } + closed = true + return data, currentInfo, nil +} + func (s *Store) addLoaded(record Record) error { canonicalID, idOK := canonicalOperationID(record.OperationID) canonicalKey, keyOK := canonicalIdempotencyKey(record.IdempotencyKey) @@ -536,12 +865,26 @@ func (s *Store) persistLocked() error { if len(s.records) > maxJournalRows || len(s.nonces) > maxNonceRows { return fmt.Errorf("operation journal row limit exceeded") } - if info, err := os.Lstat(s.path); err == nil && info.Mode()&os.ModeSymlink != 0 { - return fmt.Errorf("journal path became a symlink") - } dir := filepath.Dir(s.path) - if err := os.MkdirAll(dir, 0o750); err != nil { - return fmt.Errorf("create journal directory: %w", err) + parentInfo, err := ensureJournalParent(dir, false) + if err != nil { + return err + } + if parentInfo == nil { + return fmt.Errorf("operation journal parent is unavailable") + } + existingInfo, err := os.Lstat(s.path) + if err == nil { + if !secureJournalFile(existingInfo) { + return fmt.Errorf("operation journal path is no longer a secure regular file") + } + if s.fileInfo == nil || !os.SameFile(s.fileInfo, existingInfo) { + return fmt.Errorf("operation journal path changed before commit") + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect operation journal before commit: %w", err) + } else if s.fileInfo != nil { + return fmt.Errorf("operation journal disappeared before commit") } records := make([]Record, 0, len(s.records)) for _, record := range s.records { @@ -569,6 +912,15 @@ func (s *Store) persistLocked() error { _ = tmp.Close() return fmt.Errorf("set journal mode: %w", err) } + tmpInfo, err := tmp.Stat() + if err != nil { + _ = tmp.Close() + return fmt.Errorf("stat journal temporary file: %w", err) + } + if !secureJournalFile(tmpInfo) { + _ = tmp.Close() + return fmt.Errorf("journal temporary file is not a secure regular file") + } if _, err := tmp.Write(data); err != nil { _ = tmp.Close() return fmt.Errorf("write operation journal: %w", err) @@ -580,15 +932,60 @@ func (s *Store) persistLocked() error { if err := tmp.Close(); err != nil { return fmt.Errorf("close operation journal: %w", err) } + // Recheck both the directory identity and the destination immediately + // before rename. This narrows the lstat/create/rename window while keeping + // the implementation portable across macOS and Linux (no O_NOFOLLOW or + // openat-specific syscall is required). + currentParent, err := ensureJournalParent(dir, false) + if err != nil { + return err + } + if !os.SameFile(parentInfo, currentParent) { + return fmt.Errorf("operation journal parent changed before commit") + } + currentInfo, err := os.Lstat(s.path) + if err == nil { + if !secureJournalFile(currentInfo) || s.fileInfo == nil || !os.SameFile(s.fileInfo, currentInfo) { + return fmt.Errorf("operation journal path changed before commit") + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("recheck operation journal before commit: %w", err) + } else if s.fileInfo != nil { + return fmt.Errorf("operation journal disappeared before commit") + } if err := os.Rename(tmpName, s.path); err != nil { return fmt.Errorf("commit operation journal: %w", err) } + committedInfo, err := os.Lstat(s.path) + if err != nil { + return fmt.Errorf("recheck committed operation journal: %w", err) + } + if !secureJournalFile(committedInfo) { + return fmt.Errorf("committed operation journal is not a secure regular file") + } + s.fileInfo = committedInfo + currentParent, err = ensureJournalParent(dir, false) + if err != nil { + return err + } + if !os.SameFile(parentInfo, currentParent) { + return fmt.Errorf("operation journal parent changed after commit") + } // fsync the parent directory so a successful rename is durable across a // host crash, not merely visible in the page cache. directory, err := os.Open(dir) if err != nil { return fmt.Errorf("open journal directory for sync: %w", err) } + directoryInfo, err := directory.Stat() + if err != nil { + _ = directory.Close() + return fmt.Errorf("stat journal directory for sync: %w", err) + } + if !os.SameFile(parentInfo, directoryInfo) { + _ = directory.Close() + return fmt.Errorf("journal directory changed before sync") + } if err := directory.Sync(); err != nil { _ = directory.Close() return fmt.Errorf("sync journal directory: %w", err) diff --git a/agent/internal/operation/store_test.go b/agent/internal/operation/store_test.go index f50c611a..dcb7cc33 100644 --- a/agent/internal/operation/store_test.go +++ b/agent/internal/operation/store_test.go @@ -105,3 +105,110 @@ func TestNonceConsumptionSurvivesJournalReload(t *testing.T) { t.Fatal("an expired journal nonce should be reusable with a fresh expiry") } } + +func TestOpenCreatesAndChecksNestedJournalAncestors(t *testing.T) { + t.Parallel() + base := t.TempDir() + path := filepath.Join(base, "one", "two", "operations.json") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + if _, _, err := store.Begin(beginRequest()); err != nil { + t.Fatal(err) + } + for _, directory := range []string{filepath.Dir(path), filepath.Dir(filepath.Dir(path))} { + info, err := os.Lstat(directory) + if err != nil { + t.Fatal(err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() || info.Mode().Perm()&0o022 != 0 { + t.Fatalf("created journal ancestor %q is unsafe: mode=%v", directory, info.Mode()) + } + } + if info, err := os.Lstat(path); err != nil || !secureJournalFile(info) { + t.Fatalf("created journal file is unsafe: info=%v err=%v", info, err) + } +} + +func TestOpenRejectsUnsafeJournalAncestors(t *testing.T) { + t.Parallel() + base := t.TempDir() + content := []byte(`{"version":1,"records":[]}`) + + realParent := filepath.Join(base, "real") + if err := os.Mkdir(realParent, 0o700); err != nil { + t.Fatal(err) + } + linkedParent := filepath.Join(base, "linked") + if err := os.Symlink(realParent, linkedParent); err != nil { + t.Fatal(err) + } + if _, err := Open(filepath.Join(linkedParent, "operations.json")); err == nil { + t.Fatal("journal under symlinked ancestor was accepted") + } + + nonDir := filepath.Join(base, "not-a-directory") + if err := os.WriteFile(nonDir, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Open(filepath.Join(nonDir, "operations.json")); err == nil { + t.Fatal("journal under non-directory ancestor was accepted") + } + + writableParent := filepath.Join(base, "writable") + if err := os.Mkdir(writableParent, 0o770); err != nil { + t.Fatal(err) + } + if err := os.Chmod(writableParent, 0o770); err != nil { + t.Fatal(err) + } + if _, err := Open(filepath.Join(writableParent, "operations.json")); err == nil { + t.Fatal("journal under group-writable ancestor was accepted") + } + + unsafeFile := filepath.Join(base, "unsafe.json") + if err := os.WriteFile(unsafeFile, content, 0o640); err != nil { + t.Fatal(err) + } + if _, err := Open(unsafeFile); err == nil { + t.Fatal("group-readable journal was accepted") + } + + link := filepath.Join(base, "link.json") + goodFile := filepath.Join(base, "good.json") + if err := os.WriteFile(goodFile, content, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(goodFile, link); err != nil { + t.Fatal(err) + } + if _, err := Open(link); err == nil { + t.Fatal("symlink journal was accepted") + } +} + +func TestJournalCommitRejectsPathReplacement(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "operations.json") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + if _, _, err := store.Begin(beginRequest()); err != nil { + t.Fatal(err) + } + replacement := filepath.Join(filepath.Dir(path), "replacement.json") + if err := os.WriteFile(replacement, []byte(`{"version":1,"records":[]}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Rename(replacement, path); err != nil { + t.Fatal(err) + } + if _, err := store.Update(beginRequest().OperationID, StatusRunning, nil, "", ""); err == nil { + t.Fatal("journal update succeeded after path replacement") + } + if store.Writable() { + t.Fatal("store remained writable after path replacement") + } +} diff --git a/agent/internal/startup/artifact.go b/agent/internal/startup/artifact.go index 75697665..9df1e14b 100644 --- a/agent/internal/startup/artifact.go +++ b/agent/internal/startup/artifact.go @@ -8,7 +8,6 @@ package startup // command arguments to come from a request. import ( - "bytes" "crypto/sha256" "encoding/hex" "encoding/json" @@ -80,13 +79,14 @@ type SeedArtifact struct { } // SeedVerificationDetails is the safe read-only result of verifying a -// published NoCloud seed. It intentionally contains only the digest -// manifest and a boolean describing whether the fixed first-boot runner was -// included. It never exposes seed bytes, startup-script text, or the host -// filesystem path that was inspected. +// published NoCloud seed. It intentionally contains only the digest manifest, +// a boolean describing whether the fixed first-boot runner was included, and +// the SHA-256 of the normalized startup script. It never exposes seed bytes, +// startup-script text, or the host filesystem path that was inspected. type SeedVerificationDetails struct { Manifest SeedManifest `json:"manifest"` StartupScriptIncluded bool `json:"first_boot_present"` + StartupScriptSHA256 string `json:"first_boot_digest"` } // WriteSeedDirectory validates files, writes them to a private sibling @@ -259,18 +259,15 @@ func VerifySeedDirectoryDetails(target string) (SeedVerificationDetails, error) return SeedVerificationDetails{}, fmt.Errorf("%w: manifest is missing %q", ErrInvalidArtifact, name) } } - // Preserve the historical validation behavior: detecting either fixed - // runner path opts into SeedFiles.Validate, which then requires both paths. - // Once validation succeeds, the returned bit is the stricter AND result so - // callers never infer script inclusion from a partial/malformed document. - hasScriptPath := bytes.Contains(payloads[UserDataFilename], []byte(ScriptPath)) - hasRunnerPath := bytes.Contains(payloads[UserDataFilename], []byte(RunnerPath)) - startupScriptIncluded := hasScriptPath || hasRunnerPath + startupContract, contractErr := inspectStartupContract(payloads[UserDataFilename]) + if contractErr != nil { + return SeedVerificationDetails{}, fmt.Errorf("%w: payload validation: %v", ErrInvalidArtifact, contractErr) + } seedFiles := SeedFiles{ UserData: payloads[UserDataFilename], MetaData: payloads[MetaDataFilename], NetworkConfig: payloads[NetworkConfigFilename], - StartupScriptIncluded: startupScriptIncluded, + StartupScriptIncluded: startupContract.Included, } if err := seedFiles.Validate(); err != nil { return SeedVerificationDetails{}, fmt.Errorf("%w: payload validation: %v", ErrInvalidArtifact, err) @@ -280,7 +277,8 @@ func VerifySeedDirectoryDetails(target string) (SeedVerificationDetails, error) } return SeedVerificationDetails{ Manifest: manifest, - StartupScriptIncluded: hasScriptPath && hasRunnerPath, + StartupScriptIncluded: startupContract.Included, + StartupScriptSHA256: startupContract.ScriptSHA256, }, nil } diff --git a/agent/internal/startup/startup.go b/agent/internal/startup/startup.go index 5ab6bed9..8d57fe60 100644 --- a/agent/internal/startup/startup.go +++ b/agent/internal/startup/startup.go @@ -9,9 +9,12 @@ package startup import ( "bytes" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" + "io" "strings" "unicode" "unicode/utf8" @@ -263,11 +266,12 @@ func (files SeedFiles) Validate() error { if !validJSONObjectDocument(files.MetaData) || !validJSONObjectDocument(files.NetworkConfig) { return fmt.Errorf("%w: metadata and network-config must be JSON", ErrInvalidSeed) } - if files.StartupScriptIncluded { - if !bytes.Contains(files.UserData, []byte(ScriptPath)) || - !bytes.Contains(files.UserData, []byte(RunnerPath)) { - return fmt.Errorf("%w: startup runner files are missing from user-data", ErrInvalidSeed) - } + startupContract, err := inspectStartupContract(files.UserData) + if err != nil { + return err + } + if startupContract.Included != files.StartupScriptIncluded { + return fmt.Errorf("%w: startup runner inclusion flag does not match user-data", ErrInvalidSeed) } return nil } @@ -303,6 +307,118 @@ type writeFile struct { Permissions string `json:"permissions"` } +type startupContractDetails struct { + Included bool + ScriptSHA256 string +} + +// inspectStartupContract parses the generated cloud-config instead of using +// byte-string presence checks. A tenant-provided note, script body, or other +// arbitrary JSON value may mention the fixed guest paths, but that must never +// make first_boot_present true. The only accepted startup shape is exactly one +// private script file, exactly one fixed runner file, and one exact /bin/sh +// command invoking that runner. +func inspectStartupContract(userData []byte) (startupContractDetails, error) { + if !bytes.HasPrefix(userData, []byte("#cloud-config\n")) { + return startupContractDetails{}, fmt.Errorf("%w: user-data must begin with #cloud-config", ErrInvalidSeed) + } + var document map[string]json.RawMessage + if err := decodeStrictJSON(userData[len("#cloud-config\n"):], &document); err != nil { + return startupContractDetails{}, fmt.Errorf("%w: user-data JSON could not be inspected", ErrInvalidSeed) + } + + writeFiles, _, err := decodeOptionalStrict[[]writeFile](document, "write_files") + if err != nil { + return startupContractDetails{}, fmt.Errorf("%w: write_files has invalid shape", ErrInvalidSeed) + } + runCommands, _, err := decodeOptionalStrict[[][]string](document, "runcmd") + if err != nil { + return startupContractDetails{}, fmt.Errorf("%w: runcmd has invalid shape", ErrInvalidSeed) + } + + scriptIndex := -1 + runnerIndex := -1 + for index, file := range writeFiles { + switch file.Path { + case ScriptPath: + if scriptIndex >= 0 || file.Owner != "root:root" || file.Permissions != "0600" || file.Content == "" { + return startupContractDetails{}, fmt.Errorf("%w: startup script write_file is not canonical", ErrInvalidSeed) + } + normalized, normalizeErr := NormalizeScript(file.Content) + if normalizeErr != nil || normalized != file.Content { + return startupContractDetails{}, fmt.Errorf("%w: startup script write_file is not normalized", ErrInvalidSeed) + } + scriptIndex = index + case RunnerPath: + if runnerIndex >= 0 || file.Owner != "root:root" || file.Permissions != "0700" || file.Content != RunnerScript { + return startupContractDetails{}, fmt.Errorf("%w: startup runner write_file is not canonical", ErrInvalidSeed) + } + runnerIndex = index + } + } + + runnerCommandCount := 0 + for _, command := range runCommands { + if len(command) == 0 { + return startupContractDetails{}, fmt.Errorf("%w: runcmd contains an empty command", ErrInvalidSeed) + } + mentionsRunner := false + for _, argument := range command { + if argument == RunnerPath { + mentionsRunner = true + } + } + if mentionsRunner { + if len(command) != 2 || command[0] != "/bin/sh" || command[1] != RunnerPath { + return startupContractDetails{}, fmt.Errorf("%w: startup runner command is not canonical", ErrInvalidSeed) + } + runnerCommandCount++ + } + } + + if scriptIndex < 0 && runnerIndex < 0 && runnerCommandCount == 0 { + return startupContractDetails{ScriptSHA256: digestScript(nil)}, nil + } + if scriptIndex < 0 || runnerIndex < 0 || runnerCommandCount != 1 { + return startupContractDetails{}, fmt.Errorf("%w: startup runner files or command are incomplete", ErrInvalidSeed) + } + return startupContractDetails{Included: true, ScriptSHA256: digestScript([]byte(writeFiles[scriptIndex].Content))}, nil +} + +func digestScript(content []byte) string { + sum := sha256.Sum256(content) + return hex.EncodeToString(sum[:]) +} + +func decodeStrictJSON(data []byte, destination any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(destination); err != nil { + return err + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("multiple JSON values") + } + return err + } + return nil +} + +func decodeOptionalStrict[T any](document map[string]json.RawMessage, key string) (T, bool, error) { + var zero T + raw, ok := document[key] + if !ok { + return zero, false, nil + } + var value T + if err := decodeStrictJSON(raw, &value); err != nil { + return zero, true, err + } + return value, true, nil +} + type metadataDocument struct { InstanceID string `json:"instance-id"` LocalHostname string `json:"local-hostname"` diff --git a/agent/internal/startup/startup_test.go b/agent/internal/startup/startup_test.go index 8bab7bc8..239e9bf3 100644 --- a/agent/internal/startup/startup_test.go +++ b/agent/internal/startup/startup_test.go @@ -272,6 +272,34 @@ func TestSeedFilesValidate(t *testing.T) { } } +func TestSeedFilesDoesNotInferStartupFromArbitraryPathText(t *testing.T) { + files, err := BuildCreateSeed("id", "host", "", "") + if err != nil { + t.Fatal(err) + } + files.UserData = []byte("#cloud-config\n{\"note\":\"paths " + ScriptPath + " " + RunnerPath + "\"}\n") + files.StartupScriptIncluded = false + if err := files.Validate(); err != nil { + t.Fatalf("path text without startup contract was rejected: %v", err) + } + files.StartupScriptIncluded = true + if err := files.Validate(); !errors.Is(err, ErrInvalidSeed) { + t.Fatalf("spoofed startup flag was accepted: %v", err) + } +} + +func TestSeedFilesRejectsPartialStartupContract(t *testing.T) { + files, err := BuildCreateSeed("id", "host", "", "") + if err != nil { + t.Fatal(err) + } + files.UserData = []byte("#cloud-config\n{\"write_files\":[{\"path\":\"" + RunnerPath + "\",\"content\":\"not-the-runner\",\"owner\":\"root:root\",\"permissions\":\"0700\"}]}\n") + files.StartupScriptIncluded = false + if err := files.Validate(); !errors.Is(err, ErrInvalidSeed) { + t.Fatalf("partial startup contract was accepted: %v", err) + } +} + func TestShouldRunNeverReplaysAfterMarker(t *testing.T) { tests := []struct { name string @@ -385,6 +413,9 @@ func TestVerifySeedDirectoryDetailsReturnsOnlySafeStartupMetadata(t *testing.T) if details.StartupScriptIncluded != tt.wantIncluded { t.Fatalf("StartupScriptIncluded = %v, want %v", details.StartupScriptIncluded, tt.wantIncluded) } + if details.StartupScriptSHA256 == "" { + t.Fatal("StartupScriptSHA256 is empty") + } legacy, err := VerifySeedDirectory(target) if err != nil { t.Fatal(err) diff --git a/agent/internal/storagepolicy/storagepolicy.go b/agent/internal/storagepolicy/storagepolicy.go index bdd28248..03b73ef2 100644 --- a/agent/internal/storagepolicy/storagepolicy.go +++ b/agent/internal/storagepolicy/storagepolicy.go @@ -27,6 +27,10 @@ var ( // ErrPathUnavailable means that a referenced disk path cannot be proven to // be an existing regular file. ErrPathUnavailable = errors.New("file-backed disk path is unavailable") + // ErrPathPermission means the host refused a path inspection. The caller + // must keep this uncertain: an operator may repair ownership or mounts and + // reconcile the same operation later. + ErrPathPermission = errors.New("file-backed disk path permission is unavailable") // ErrPathSymlink means that the root, a root ancestor, a disk path, or a // disk path ancestor is a symbolic link. Rejecting aliases prevents a // later replacement from redirecting libvirt outside the allowlist. @@ -43,6 +47,9 @@ var ( // changed. ErrRootUnavailable is intentionally excluded because an operator // may still mount/provision the configured root and then reconcile safely. func IsDefinitive(err error) bool { + if errors.Is(err, ErrPathPermission) { + return false + } return errors.Is(err, ErrPolicyViolation) || errors.Is(err, ErrNotConfigured) || errors.Is(err, ErrPathUnavailable) || @@ -128,6 +135,9 @@ func (p Policy) ValidatePaths(paths []string) error { if errors.Is(err, os.ErrNotExist) { return fmt.Errorf("%w: %q", ErrPathUnavailable, path) } + if errors.Is(err, os.ErrPermission) || os.IsPermission(err) { + return fmt.Errorf("%w: %w: inspect %q", ErrPathUnavailable, ErrPathPermission, path) + } return fmt.Errorf("%w: inspect %q: %v", ErrPathUnavailable, path, err) } if info.Mode()&os.ModeSymlink != 0 { @@ -141,6 +151,9 @@ func (p Policy) ValidatePaths(paths []string) error { } realPath, err := filepath.EvalSymlinks(path) if err != nil { + if errors.Is(err, os.ErrPermission) || os.IsPermission(err) { + return fmt.Errorf("%w: %w: resolve %q", ErrPathUnavailable, ErrPathPermission, path) + } return fmt.Errorf("%w: resolve %q: %v", ErrPathUnavailable, path, err) } if realPath != path { diff --git a/agent/internal/storagepolicy/storagepolicy_test.go b/agent/internal/storagepolicy/storagepolicy_test.go index f71793d2..f1d5cdc0 100644 --- a/agent/internal/storagepolicy/storagepolicy_test.go +++ b/agent/internal/storagepolicy/storagepolicy_test.go @@ -60,6 +60,9 @@ func TestIsDefinitiveDistinguishesPolicyAndUnavailableRoot(t *testing.T) { if IsDefinitive(fmt.Errorf("wrapped: %w", ErrRootUnavailable)) { t.Fatal("ErrRootUnavailable should remain retryable after host provisioning") } + if IsDefinitive(fmt.Errorf("wrapped: %w: %w", ErrPathUnavailable, ErrPathPermission)) { + t.Fatal("permission failures should remain retryable for reconciliation") + } } func TestValidatePathsAcceptsExistingRegularFileInsideRoot(t *testing.T) { diff --git a/docs/GO_AGENT_API_V1.md b/docs/GO_AGENT_API_V1.md index 5d563a6b..9e234201 100644 --- a/docs/GO_AGENT_API_V1.md +++ b/docs/GO_AGENT_API_V1.md @@ -15,6 +15,8 @@ canonical standard base64 文本,解码后必须恰好 32 个 raw bytes;文件末尾可有一个换行。 Go `FromFile` 会拒绝 symlink、非 regular file、group/other 权限、原始字节、嵌入空白、 非 canonical padding 和其他长度;权限错误只让 Agent 保持 degraded,不会放行写请求。 + 密钥路径的祖先目录也会逐级检查;仅内核 sticky 保护的标准共享临时根(例如 + `/tmp`)可作为测试边界,生产 secret/journal 应放在单 Agent 独占的私有目录。 PHP `CloudSecrets::key('engine_hmac_key')` 返回解码后的 raw bytes。`CloudAgentClient` 只接受 该 raw key,不能把 44 字符的 base64 文本直接拿来做 HMAC。 - `GET /v1/health` 是 liveness;配置 mode/owner 宽松时仍返回 200,但 @@ -73,6 +75,11 @@ readback、清理清单和 commit boundary 完成前,Agent 会返回“未实 误报为完全相同。面板若要求精确 desired XML 比对,应等待后续 XML intent digest 契约完成。 +当前 host-bound XML 还会拒绝通用 ``、远程/块设备磁盘、NVRAM、emulator、 +memory-backed file 和文件/套接字型外设。它们必须先进入有 owner、allowlist、TOCTOU +和读回证据的 typed Agent operation;因此这一版不能把任意网络或宿主路径拼进 domain +XML,网络/完整挂载 executor 仍属于后续阶段。 + `domain_define` 对 `` 还会在 host-bound 阶段执行存储路径 allowlist:每个路径必须是现有 regular file、不能是 symlink 或 路径别名,并且必须位于 Agent 配置的 `VIRT_CLOUD_AGENT_STORAGE_ROOTS` 之一。 diff --git a/docs/GO_AGENT_ATTACH_CONTRACT.md b/docs/GO_AGENT_ATTACH_CONTRACT.md new file mode 100644 index 00000000..1ff465e1 --- /dev/null +++ b/docs/GO_AGENT_ATTACH_CONTRACT.md @@ -0,0 +1,43 @@ +# Go Agent host-bound attach contract + +`agent/internal/domaincreateattach` is the next boundary after +`domain_create_plan`/`domain_create_validate`. It is a greenfield contract and +does not read or translate the Python engine or an old database. + +## What it proves + +`Build` accepts a typed EFI/x86_64/q35/no-Secure-Boot domain policy, a loader +path and disk intents that a future Agent resolver is expected to supply, the +expected immutable NoCloud manifest, and a script digest. `BuildWithLoaderProvider` +can obtain the loader through the fresh-observation seam, but this package does +not itself prove volume ownership or resolver generation. It then, without +changing the host: + +1. renders and validates host-bound base XML with the selected loader; +2. verifies the fixed NoCloud artifact and first-boot inclusion bit; +3. validates every resolved disk path against the configured storage-root + allowlist; and +4. returns a deterministic, safe identity (`domain_xml_hash`, disk-plan hash, + seed manifest and `plan_hash`). + +The base XML intentionally contains no ``, ``, or network +configuration. Disk attachments are typed `{volume_id, slot, readonly}` values +with an agent-side path that is excluded from JSON. The `Plan` accessors expose +the host-only values only to a future executor; its `Result` is the journal/API +projection. + +## What it does not prove + +This package does not write or attach a CIDATA image, allocate or create a +volume, call libvirt, define/start a domain, or modify network state. A future +executor must re-open the loader, seed directory, and disk paths using a +no-follow/openat-style boundary immediately before mutation, then perform +explicit define/read-back and cleanup reconciliation. The current filesystem +checks are a preflight gate, not a TOCTOU proof. + +Missing or unreadable host inputs are classified as `uncertain`; malformed +typed policy, seed mismatch, loader policy violations, and storage allowlist +violations are definitive failures. No raw script, network document, host path, +or domain XML is included in the safe result. A successful result is only an +input-validation checkpoint, not an attach/create success or a proof that a +guest will execute the startup script. diff --git a/docs/GO_AGENT_DEPLOYMENT.md b/docs/GO_AGENT_DEPLOYMENT.md index 06780b0a..8c7ff998 100644 --- a/docs/GO_AGENT_DEPLOYMENT.md +++ b/docs/GO_AGENT_DEPLOYMENT.md @@ -97,6 +97,10 @@ symlink。Agent 每次 define 前都会重新检查根、文件和 realpath; 解码,PHP `CloudSecrets::key('engine_hmac_key')` 和 `CloudAgentClient` 使用同一解码结果。 不要把原始二进制或 44 字符 base64 文本直接当作 HMAC key 传给客户端。实际证书、私钥、CA 和 HMAC key 不应由 Git、面板数据库或日志分发。远程部署 还应先完成面板 CA 签发、客户端证书指纹登记和时间同步,再开放管理地址端口。 +密钥、journal 和 loader 路径的祖先目录默认必须不可由 group/other 写入;为支持 +只读冒烟和临时运行目录,内核 sticky 保护的标准共享临时根(例如 `/tmp` 的 +`01777`)是唯一例外,Agent 仍会要求其下的实际文件/子目录使用私有权限。生产 +journal 应由单个 Agent service 独占,不要让两个进程共用同一 journal 路径。 安装 unit 并先以前台/单次方式验收: diff --git a/docs/GO_AGENT_OPERATION_MATRIX.md b/docs/GO_AGENT_OPERATION_MATRIX.md index e02a013f..dd1aeae0 100644 --- a/docs/GO_AGENT_OPERATION_MATRIX.md +++ b/docs/GO_AGENT_OPERATION_MATRIX.md @@ -9,6 +9,7 @@ | 域创建预检(`domain_preflight`) | 已实现只读首版 | 严格 UUID/EFI/x86_64/q35/loader policy;只读 capability、规范 XML/hash;不分配存储、不改网络、不 define;不依赖先前 host readiness probe,但仍要求认证/TLS/HMAC、Agent 身份和 durable journal;loader 路径仍待 host-bound 阶段选择 | | 域创建规划(`domain_create_plan`) | 已实现只读首版 | 在 preflight 基础上校验 immutable seed/image 摘要并生成 plan hash;journal 只保留 seed manifest/digest,不发布 seed、不分配存储、不 define、不改电源/网络;未知响应只重做 capability 观察并核对同一计划身份 | | 已发布 seed 创建核验(`domain_create_validate`) | 已实现只读首版 | 按 UUID 验证 immutable NoCloud artifact、manifest/hash 和 pre-host XML,再只读回读宿主能力;seed 缺失/篡改 definitive,能力/文件传输不可用 uncertain;不写 seed、不分配存储、不 define、不改电源/网络,reconcile 先 seed 后 capability | +| Host-bound 创建/挂载输入核验(`domaincreateattach` 内部契约) | 已实现只读内部边界,尚未开放 HTTP operation | 消费未来 Agent resolver 提供的 loader/卷路径前置条件,并验证 immutable seed、路径规范和 storage allowlist;当前不证明 capability freshness、卷所有权/generation 或 openat/TOCTOU;不制作 CIDATA、不分配卷、不 attach/define、不改网络;待 resolver、fd 门控、libvirt read-back 和清理 executor 完成后再开放 API | | 域 XML 读取/定义 | 已实现 | UUID、loader、机器类型和 inactive XML 读回一致;未知 define 结果当前只做同 UUID/固件策略的只读核对,不重放 XML | | 电源 start/shutdown/destroy/reboot/suspend/resume | 已实现首版 | 固定 argv、动作后状态回读;未知结果只读 reconcile,reboot 不以 running 单态确认 | | NoCloud seed 与云镜像首次启动脚本 | 已实现 artifact 首版 | 目录 manifest/hash、CIDATA 消费链、guest 一次性 marker 验收 |