You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/internal/startup/artifact.go

546 lines
20 KiB
Go

package startup
// This file contains the host-side NoCloud seed artifact boundary. It writes
// only the three fixed datasource files and a digest manifest; it deliberately
// does not invoke mkisofs, genisoimage, shell commands, or any other external
// program. A later libvirt layer may attach this directory through a trusted
// storage implementation or turn it into an ISO without allowing filenames or
// command arguments to come from a request.
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
)
const (
// ManifestFilename is the only metadata file added to a seed directory.
// Its entries cover the three NoCloud payloads, not the manifest itself.
ManifestFilename = "manifest.json"
// SeedManifestVersion is incremented when the on-disk manifest contract
// changes incompatibly.
SeedManifestVersion = 1
seedDirectoryMode fs.FileMode = 0o700
seedFileMode fs.FileMode = 0o600
)
var (
// ErrInvalidArtifact means a published seed is malformed, incomplete, or
// no longer matches its digest manifest.
ErrInvalidArtifact = errors.New("invalid seed artifact")
// ErrArtifactPath means the requested publication path is not an
// agent-owned absolute directory location.
ErrArtifactPath = errors.New("invalid seed artifact path")
// ErrArtifactConflict means an artifact ID was already published with a
// different immutable seed. Artifact IDs are content identities in the
// greenfield protocol; callers must allocate a new ID instead of replacing
// a seed that may already be attached to a domain.
ErrArtifactConflict = errors.New("seed artifact identity conflict")
)
// A single process may receive duplicate retries for one artifact while the
// panel is reconciling an uncertain operation. Serialize publication so the
// target-existence check and final rename cannot race another writer. The
// artifact itself is immutable after publication, so this lock is only a
// process-local coordination aid; VerifySeedDirectory remains the durable
// cross-process integrity check.
var publishMu sync.Mutex
// SeedFileDigest is the integrity record for one fixed NoCloud file.
type SeedFileDigest struct {
Bytes int `json:"bytes"`
SHA256 string `json:"sha256"`
}
// SeedManifest is written as manifest.json after all payload files have been
// fsync'd. A map is used for lookup by readers; encoding/json emits map keys
// in deterministic lexical order, so identical seeds have identical manifests.
type SeedManifest struct {
Version int `json:"version"`
Files map[string]SeedFileDigest `json:"files"`
}
// SeedArtifact describes one successfully published seed directory.
// Manifest is a copy of the bytes written to disk, while Directory is the
// cleaned absolute path supplied to WriteSeedDirectory.
type SeedArtifact struct {
Directory string `json:"directory"`
Manifest SeedManifest `json:"manifest"`
}
// SeedVerificationDetails is the safe read-only result of verifying a
// published NoCloud seed. It intentionally contains only the digest manifest,
// a boolean describing whether the fixed first-boot runner was included, and
// the SHA-256 of the normalized startup script. It never exposes seed bytes,
// startup-script text, or the host filesystem path that was inspected.
type SeedVerificationDetails struct {
Manifest SeedManifest `json:"manifest"`
StartupScriptIncluded bool `json:"first_boot_present"`
StartupScriptSHA256 string `json:"first_boot_digest"`
}
// WriteSeedDirectory validates files, writes them to a private sibling
// directory, writes a SHA-256 manifest, fsyncs the complete tree, and then
// publishes the directory at target. The target is immutable after a
// successful publish, so readers never observe a partially populated seed
// directory and retries cannot silently replace an attached seed. No
// caller-controlled filename is accepted; all names come from the constants in
// this package.
//
// target must be an absolute path whose immediate parent already exists. The
// parent is not created implicitly, which avoids accidentally creating a
// writable tree through a symlink or a mistyped path. The target itself may be
// absent or an identical directory previously published by this function;
// a different existing artifact is an immutable identity conflict.
func WriteSeedDirectory(target string, files SeedFiles) (SeedArtifact, error) {
parent, cleanTarget, err := validateArtifactTarget(target)
if err != nil {
return SeedArtifact{}, err
}
if err := files.Validate(); err != nil {
return SeedArtifact{}, err
}
if err := validateArtifactParent(parent); err != nil {
return SeedArtifact{}, err
}
manifest, payloads := manifestFor(files)
publishMu.Lock()
defer publishMu.Unlock()
if exists, inspectErr := inspectExistingArtifact(cleanTarget); inspectErr != nil {
return SeedArtifact{}, inspectErr
} else if exists {
observed, verifyErr := VerifySeedDirectory(cleanTarget)
if verifyErr != nil {
return SeedArtifact{}, fmt.Errorf("%w: existing seed is not a verified agent artifact: %v", ErrArtifactPath, verifyErr)
}
if !manifestsEqual(observed, manifest) {
return SeedArtifact{}, fmt.Errorf("%w: %s", ErrArtifactConflict, cleanTarget)
}
return SeedArtifact{Directory: cleanTarget, Manifest: observed}, nil
}
temporary, err := os.MkdirTemp(parent, ".virt-cloud-seed-*")
if err != nil {
return SeedArtifact{}, fmt.Errorf("create seed staging directory: %w", err)
}
keep := false
defer func() {
if !keep {
_ = os.RemoveAll(temporary)
}
}()
if err := os.Chmod(temporary, seedDirectoryMode); err != nil {
return SeedArtifact{}, fmt.Errorf("set seed staging directory mode: %w", err)
}
for _, name := range []string{UserDataFilename, MetaDataFilename, NetworkConfigFilename} {
if err := writeSeedFile(temporary, name, payloads[name]); err != nil {
return SeedArtifact{}, err
}
}
manifestBytes, err := marshalManifest(manifest)
if err != nil {
return SeedArtifact{}, err
}
if err := writeSeedFile(temporary, ManifestFilename, manifestBytes); err != nil {
return SeedArtifact{}, err
}
if err := syncDirectory(temporary); err != nil {
return SeedArtifact{}, fmt.Errorf("sync seed staging directory: %w", err)
}
if err := publishSeedDirectory(temporary, cleanTarget, parent); err != nil {
return SeedArtifact{}, err
}
keep = true
// Read back the published tree. This catches a concurrent replacement or
// an unexpected mode/symlink before the caller records the artifact as
// usable. It also makes the returned manifest an observed value, not merely
// an in-memory calculation.
observed, err := VerifySeedDirectory(cleanTarget)
if err != nil {
return SeedArtifact{}, err
}
return SeedArtifact{Directory: cleanTarget, Manifest: observed}, nil
}
// VerifySeedDirectory validates the manifest, modes, fixed file set, sizes,
// and SHA-256 digests of a published seed. It is read-only and can be used by
// a later attach/read-back operation before exposing the artifact to libvirt.
//
// The manifest-only return shape is retained for callers that already depend
// on this API. Call VerifySeedDirectoryDetails when the safe first-boot
// inclusion bit is also needed.
func VerifySeedDirectory(target string) (SeedManifest, error) {
details, err := VerifySeedDirectoryDetails(target)
if err != nil {
return SeedManifest{}, err
}
return details.Manifest, nil
}
// VerifySeedDirectoryDetails validates a published seed and returns its
// manifest together with a safe first-boot inclusion bit. The validation
// rules and error behavior are shared with VerifySeedDirectory; in
// particular, a user-data document containing only one of the two fixed
// runner paths is still rejected as an invalid artifact.
func VerifySeedDirectoryDetails(target string) (SeedVerificationDetails, error) {
parent, cleanTarget, err := validateArtifactTarget(target)
if err != nil {
return SeedVerificationDetails{}, err
}
if err := validateArtifactParent(parent); err != nil {
return SeedVerificationDetails{}, err
}
info, err := os.Lstat(cleanTarget)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return SeedVerificationDetails{}, fmt.Errorf("%w: seed directory does not exist", ErrInvalidArtifact)
}
return SeedVerificationDetails{}, fmt.Errorf("stat seed directory: %w", err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return SeedVerificationDetails{}, fmt.Errorf("%w: seed target must be a directory", ErrInvalidArtifact)
}
if !exactMode(info.Mode(), seedDirectoryMode) {
return SeedVerificationDetails{}, fmt.Errorf("%w: seed directory mode is %o, want %o", ErrInvalidArtifact, info.Mode().Perm(), seedDirectoryMode.Perm())
}
manifestBytes, err := readRegularFile(cleanTarget, ManifestFilename, MaxFileBytes)
if err != nil {
return SeedVerificationDetails{}, err
}
manifest, err := parseManifest(manifestBytes)
if err != nil {
return SeedVerificationDetails{}, err
}
wanted := map[string]struct{}{
UserDataFilename: {},
MetaDataFilename: {},
NetworkConfigFilename: {},
ManifestFilename: {},
}
entries := map[string]struct{}{}
payloads := make(map[string][]byte, 3)
for name, digest := range manifest.Files {
if _, ok := wanted[name]; !ok || name == ManifestFilename {
return SeedVerificationDetails{}, fmt.Errorf("%w: manifest contains unexpected file %q", ErrInvalidArtifact, name)
}
if !validDigest(digest) {
return SeedVerificationDetails{}, fmt.Errorf("%w: invalid digest entry for %q", ErrInvalidArtifact, name)
}
entries[name] = struct{}{}
data, err := readRegularFile(cleanTarget, name, MaxFileBytes)
if err != nil {
return SeedVerificationDetails{}, err
}
if len(data) != digest.Bytes {
return SeedVerificationDetails{}, fmt.Errorf("%w: %s size does not match manifest", ErrInvalidArtifact, name)
}
sum := sha256.Sum256(data)
if hex.EncodeToString(sum[:]) != digest.SHA256 {
return SeedVerificationDetails{}, fmt.Errorf("%w: %s digest does not match manifest", ErrInvalidArtifact, name)
}
payloads[name] = data
}
for _, name := range []string{UserDataFilename, MetaDataFilename, NetworkConfigFilename} {
if _, ok := entries[name]; !ok {
return SeedVerificationDetails{}, fmt.Errorf("%w: manifest is missing %q", ErrInvalidArtifact, name)
}
}
startupContract, contractErr := inspectStartupContract(payloads[UserDataFilename])
if contractErr != nil {
return SeedVerificationDetails{}, fmt.Errorf("%w: payload validation: %v", ErrInvalidArtifact, contractErr)
}
seedFiles := SeedFiles{
UserData: payloads[UserDataFilename],
MetaData: payloads[MetaDataFilename],
NetworkConfig: payloads[NetworkConfigFilename],
StartupScriptIncluded: startupContract.Included,
}
if err := seedFiles.Validate(); err != nil {
return SeedVerificationDetails{}, fmt.Errorf("%w: payload validation: %v", ErrInvalidArtifact, err)
}
if err := validateNoExtraSeedFiles(cleanTarget, wanted); err != nil {
return SeedVerificationDetails{}, err
}
return SeedVerificationDetails{
Manifest: manifest,
StartupScriptIncluded: startupContract.Included,
StartupScriptSHA256: startupContract.ScriptSHA256,
}, nil
}
func manifestFor(files SeedFiles) (SeedManifest, map[string][]byte) {
payloads := map[string][]byte{
UserDataFilename: append([]byte(nil), files.UserData...),
MetaDataFilename: append([]byte(nil), files.MetaData...),
NetworkConfigFilename: append([]byte(nil), files.NetworkConfig...),
}
manifest := SeedManifest{
Version: SeedManifestVersion,
Files: make(map[string]SeedFileDigest, len(payloads)),
}
for name, data := range payloads {
sum := sha256.Sum256(data)
manifest.Files[name] = SeedFileDigest{
Bytes: len(data),
SHA256: hex.EncodeToString(sum[:]),
}
}
return manifest, payloads
}
func marshalManifest(manifest SeedManifest) ([]byte, error) {
if manifest.Version != SeedManifestVersion || len(manifest.Files) != 3 {
return nil, fmt.Errorf("%w: manifest shape is invalid", ErrInvalidArtifact)
}
encoded, err := json.Marshal(manifest)
if err != nil {
return nil, fmt.Errorf("marshal seed manifest: %w", err)
}
return append(encoded, '\n'), nil
}
func parseManifest(data []byte) (SeedManifest, error) {
if len(data) == 0 || len(data) > MaxFileBytes || data[len(data)-1] != '\n' {
return SeedManifest{}, fmt.Errorf("%w: manifest framing is invalid", ErrInvalidArtifact)
}
var manifest SeedManifest
decoder := json.NewDecoder(strings.NewReader(string(data)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&manifest); err != nil {
return SeedManifest{}, fmt.Errorf("%w: decode manifest: %v", ErrInvalidArtifact, err)
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
if err == nil {
return SeedManifest{}, fmt.Errorf("%w: manifest has trailing JSON", ErrInvalidArtifact)
}
return SeedManifest{}, fmt.Errorf("%w: manifest trailing data: %v", ErrInvalidArtifact, err)
}
if manifest.Version != SeedManifestVersion || len(manifest.Files) != 3 {
return SeedManifest{}, fmt.Errorf("%w: unsupported manifest version or file count", ErrInvalidArtifact)
}
return manifest, nil
}
func validDigest(digest SeedFileDigest) bool {
if digest.Bytes < 0 || digest.Bytes > MaxFileBytes || len(digest.SHA256) != sha256.Size*2 {
return false
}
_, err := hex.DecodeString(digest.SHA256)
return err == nil && strings.ToLower(digest.SHA256) == digest.SHA256
}
func writeSeedFile(directory, name string, data []byte) error {
if !isFixedSeedName(name) {
return fmt.Errorf("%w: unsupported seed filename %q", ErrInvalidArtifact, name)
}
if len(data) > MaxFileBytes {
return fmt.Errorf("%w: %s exceeds %d bytes", ErrInvalidArtifact, name, MaxFileBytes)
}
path := filepath.Join(directory, name)
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, seedFileMode)
if err != nil {
return fmt.Errorf("create seed file %s: %w", name, err)
}
closeFile := func() error {
if closeErr := file.Close(); closeErr != nil {
return fmt.Errorf("close seed file %s: %w", name, closeErr)
}
return nil
}
if err := file.Chmod(seedFileMode); err != nil {
_ = file.Close()
return fmt.Errorf("set seed file %s mode: %w", name, err)
}
for offset := 0; offset < len(data); {
written, writeErr := file.Write(data[offset:])
if writeErr != nil {
_ = file.Close()
return fmt.Errorf("write seed file %s: %w", name, writeErr)
}
if written <= 0 {
_ = file.Close()
return fmt.Errorf("write seed file %s: %w", name, io.ErrShortWrite)
}
offset += written
}
if err := file.Sync(); err != nil {
_ = file.Close()
return fmt.Errorf("sync seed file %s: %w", name, err)
}
return closeFile()
}
func readRegularFile(directory, name string, maxBytes int) ([]byte, error) {
if !isFixedSeedName(name) {
return nil, fmt.Errorf("%w: unsupported seed filename %q", ErrInvalidArtifact, name)
}
path := filepath.Join(directory, name)
info, err := os.Lstat(path)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, fmt.Errorf("%w: missing seed file %q", ErrInvalidArtifact, name)
}
return nil, fmt.Errorf("stat seed file %s: %w", name, err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return nil, fmt.Errorf("%w: seed file %s is not a regular file", ErrInvalidArtifact, name)
}
if !exactMode(info.Mode(), seedFileMode) {
return nil, fmt.Errorf("%w: seed file %s mode is %o, want %o", ErrInvalidArtifact, name, info.Mode().Perm(), seedFileMode.Perm())
}
if info.Size() < 0 || info.Size() > int64(maxBytes) {
return nil, fmt.Errorf("%w: seed file %s is too large", ErrInvalidArtifact, name)
}
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("open seed file %s: %w", name, err)
}
defer file.Close()
data, err := io.ReadAll(io.LimitReader(file, int64(maxBytes)+1))
if err != nil {
return nil, fmt.Errorf("read seed file %s: %w", name, err)
}
if len(data) > maxBytes {
return nil, fmt.Errorf("%w: seed file %s is too large", ErrInvalidArtifact, name)
}
return data, nil
}
func validateNoExtraSeedFiles(directory string, allowed map[string]struct{}) error {
entries, err := os.ReadDir(directory)
if err != nil {
return fmt.Errorf("read seed directory: %w", err)
}
for _, entry := range entries {
if _, ok := allowed[entry.Name()]; !ok {
return fmt.Errorf("%w: unexpected seed directory entry %q", ErrInvalidArtifact, entry.Name())
}
}
return nil
}
func isFixedSeedName(name string) bool {
switch name {
case UserDataFilename, MetaDataFilename, NetworkConfigFilename, ManifestFilename:
return true
default:
return false
}
}
func exactMode(mode, want fs.FileMode) bool {
if mode.Perm() != want.Perm() {
return false
}
return mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) == 0
}
func validateArtifactTarget(target string) (parent, cleanTarget string, err error) {
if target == "" || strings.TrimSpace(target) != target || strings.IndexByte(target, 0) >= 0 {
return "", "", fmt.Errorf("%w: target must be a non-empty absolute path", ErrArtifactPath)
}
cleanTarget = filepath.Clean(target)
if !filepath.IsAbs(target) || cleanTarget != target || cleanTarget == string(filepath.Separator) {
return "", "", fmt.Errorf("%w: target must be a clean absolute directory path", ErrArtifactPath)
}
base := filepath.Base(cleanTarget)
if base == "." || base == ".." || base == string(filepath.Separator) {
return "", "", fmt.Errorf("%w: target basename is invalid", ErrArtifactPath)
}
parent = filepath.Dir(cleanTarget)
return parent, cleanTarget, nil
}
func validateArtifactParent(parent string) error {
info, err := os.Lstat(parent)
if err != nil {
return fmt.Errorf("%w: artifact parent is unavailable: %v", ErrArtifactPath, err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return fmt.Errorf("%w: artifact parent must be a directory", ErrArtifactPath)
}
// Publication uses a sibling temporary directory followed by rename. A
// group/world-writable or special-bit parent would let another local user
// race that boundary, replace the staging entry, or observe tenant seed
// material before the immutable target is verified.
if info.Mode().Perm()&0o022 != 0 || info.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) != 0 {
return fmt.Errorf("%w: artifact parent must not be group/world writable or carry special mode bits", ErrArtifactPath)
}
return nil
}
func publishSeedDirectory(temporary, target, parent string) error {
// WriteSeedDirectory has already checked that the target is absent while
// holding publishMu. A single rename now publishes a complete tree with no
// target->backup gap. If another process wins the race, fail closed; the
// caller can verify/reconcile the existing immutable artifact by ID.
if info, err := os.Lstat(target); err == nil {
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return fmt.Errorf("%w: existing seed target is not a directory", ErrArtifactPath)
}
return fmt.Errorf("%w: target already exists", ErrArtifactConflict)
} else if !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("inspect existing seed directory: %w", err)
}
if err := os.Rename(temporary, target); err != nil {
return fmt.Errorf("publish seed directory: %w", err)
}
if err := syncDirectory(parent); err != nil {
return fmt.Errorf("sync seed parent directory: %w", err)
}
return nil
}
func inspectExistingArtifact(target string) (bool, error) {
info, err := os.Lstat(target)
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("inspect existing seed directory: %w", err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return true, fmt.Errorf("%w: existing seed target is not a directory", ErrArtifactPath)
}
return true, nil
}
func manifestsEqual(left, right SeedManifest) bool {
if left.Version != right.Version || len(left.Files) != len(right.Files) {
return false
}
for name, digest := range right.Files {
if digest != left.Files[name] {
return false
}
}
return true
}
func syncDirectory(path string) error {
directory, err := os.Open(path)
if err != nil {
return err
}
if err := directory.Sync(); err != nil {
_ = directory.Close()
return err
}
return directory.Close()
}