You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/worker/executor.go

581 lines
17 KiB
Go

package worker
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"git.luoli.live/root/virt/agent/operation"
)
const (
workerReceiptName = "completion.json"
workerJournalName = "journal.db"
workerIdentityName = "request-identity.json"
terminationGrace = 500 * time.Millisecond
killWait = 2 * time.Second
maxIdentityBytes = 4 << 10
)
var (
errInvalidWorkerIdentity = errors.New("invalid worker recovery identity")
errInvalidWorkerReceipt = errors.New("invalid worker receipt")
)
// TerminalEvidence is a committed child-Journal result discovered before an
// immutable completion.json was published. trustedFilesystem is deliberately
// private: only the scanner that verifies file type, mode, and ownership can
// authorize conversion into a main-Journal receipt.
type TerminalEvidence struct {
Record operation.Record
Identity *RequestIdentity
Legacy bool
trustedFilesystem bool
}
// InterruptedReceiptFor accepts child evidence only when every identity field
// available to both Journals matches exactly. New workers additionally require
// the durable request identity (actor owner, target, body digest). A legacy
// Journal can be adopted only from a trusted pre-upgrade worker directory.
func (e TerminalEvidence) InterruptedReceiptFor(main operation.Record) (operation.InterruptedReceipt, bool) {
if !e.trustedFilesystem || main.State != operation.StateDispatching ||
main.OperationID != e.Record.OperationID || main.Action != e.Record.Action ||
main.ResourceID != e.Record.ResourceID || main.Fingerprint != e.Record.Fingerprint ||
(e.Record.State != operation.StateSucceeded && e.Record.State != operation.StateFailed) {
return operation.InterruptedReceipt{}, false
}
if e.Identity == nil {
if !e.Legacy {
return operation.InterruptedReceipt{}, false
}
} else if e.Legacy || e.Identity.Validate() != nil ||
e.Identity.OperationID != e.Record.OperationID || e.Identity.Fingerprint != e.Record.Fingerprint {
return operation.InterruptedReceipt{}, false
}
result := append(json.RawMessage(nil), e.Record.Result...)
if len(result) > MaxWorkerResponse/2 {
result = nil
}
response := Response{
Version: ProtocolVersion, OperationID: e.Record.OperationID, Fingerprint: e.Record.Fingerprint,
Status: string(e.Record.State), Result: result,
EffectMayHaveApplied: e.Record.EffectMayHaveApplied,
SafeToResubmit: e.Record.SafeToResubmit,
}
if e.Record.Error != nil {
response.ErrorCode = e.Record.Error.Code
response.ErrorMessage = e.Record.Error.Message
if len(response.ErrorMessage) > 1024 {
response.ErrorMessage = response.ErrorMessage[:1024]
}
}
if response.Validate() != nil {
return operation.InterruptedReceipt{}, false
}
return operation.InterruptedReceipt{
OperationID: response.OperationID, Fingerprint: response.Fingerprint,
Status: response.Status, Result: response.Result,
ErrorCode: response.ErrorCode, ErrorMessage: response.ErrorMessage,
EffectMayHaveApplied: response.EffectMayHaveApplied,
SafeToResubmit: response.SafeToResubmit,
}, true
}
type SubprocessExecutor struct {
executable string
configPath string
stateDir string
}
func NewSubprocessExecutor(executable, configPath, stateDir string) (*SubprocessExecutor, error) {
for name, value := range map[string]string{"executable": executable, "config path": configPath, "state directory": stateDir} {
if strings.TrimSpace(value) == "" || !filepath.IsAbs(value) || filepath.Clean(value) != value {
return nil, fmt.Errorf("worker %s must be a clean absolute path", name)
}
}
if err := os.MkdirAll(stateDir, 0o700); err != nil {
return nil, fmt.Errorf("create worker state directory: %w", err)
}
if err := os.Chmod(stateDir, 0o700); err != nil {
return nil, fmt.Errorf("secure worker state directory: %w", err)
}
return &SubprocessExecutor{executable: executable, configPath: configPath, stateDir: stateDir}, nil
}
func (e *SubprocessExecutor) Execute(ctx context.Context, request Request) (json.RawMessage, error) {
if err := request.Validate(); err != nil {
return nil, err
}
operationDir := filepath.Join(e.stateDir, request.OperationID)
receiptPath := filepath.Join(operationDir, workerReceiptName)
journalPath := filepath.Join(operationDir, workerJournalName)
if err := os.Mkdir(operationDir, 0o700); err != nil {
if !errors.Is(err, os.ErrExist) {
return nil, fmt.Errorf("create operation worker directory: %w", err)
}
response, readErr := ReadReceipt(receiptPath)
if readErr != nil || response.Fingerprint != request.Fingerprint {
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
}
return Result(response)
}
payload, err := json.Marshal(request)
if err != nil || len(payload) > MaxEnvelopeBytes {
_ = os.RemoveAll(operationDir)
return nil, errors.New("encode bounded worker request")
}
identity, err := requestIdentityFromRequest(request)
if err != nil {
_ = os.RemoveAll(operationDir)
return nil, err
}
if err := writeRequestIdentity(filepath.Join(operationDir, workerIdentityName), identity); err != nil {
_ = os.RemoveAll(operationDir)
return nil, fmt.Errorf("persist worker recovery identity: %w", err)
}
command := exec.Command(e.executable,
"--config", e.configPath,
"--operation-worker",
"--worker-journal", journalPath,
"--worker-receipt", receiptPath,
)
configureProcess(command)
command.Stdin = bytes.NewReader(payload)
stdout := &limitedBuffer{limit: MaxWorkerResponse}
stderr := &limitedBuffer{limit: 32 << 10}
command.Stdout = stdout
command.Stderr = stderr
if err := command.Start(); err != nil {
_ = os.RemoveAll(operationDir)
return nil, &Error{
Code: "worker_start_failed", Message: "isolated mutation worker could not be started",
EffectMayHaveApplied: false, ResubmitAllowed: true,
}
}
wait := make(chan error, 1)
go func() { wait <- command.Wait() }()
select {
case waitErr := <-wait:
response, readErr := ReadReceipt(receiptPath)
if readErr != nil || response.OperationID != request.OperationID || response.Fingerprint != request.Fingerprint {
if waitErr != nil {
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
}
return nil, errors.New("isolated mutation worker returned no valid receipt")
}
return Result(response)
case <-ctx.Done():
reaped := false
terminateProcessGroup(command)
select {
case <-wait:
reaped = true
case <-time.After(terminationGrace):
killProcessGroup(command)
select {
case <-wait:
reaped = true
case <-time.After(killWait):
return nil, &Error{
Code: "worker_reap_unconfirmed",
Message: "isolated mutation worker did not exit after forced termination",
EffectMayHaveApplied: true, ResubmitAllowed: false, ExecutionFenced: true,
}
}
}
if reaped {
if response, readErr := ReadReceipt(receiptPath); readErr == nil &&
response.OperationID == request.OperationID && response.Fingerprint == request.Fingerprint {
return Result(response)
}
}
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
}
}
func (e *SubprocessExecutor) PendingReceipts() ([]Response, error) {
entries, err := os.ReadDir(e.stateDir)
if err != nil {
return nil, err
}
results := make([]Response, 0, len(entries))
for _, entry := range entries {
if !entry.IsDir() || !operationIDPattern.MatchString(entry.Name()) {
continue
}
response, found, readErr := readTrustedReceipt(
filepath.Join(e.stateDir, entry.Name(), workerReceiptName), entry.Name(),
)
if readErr != nil {
return nil, readErr
}
if found {
results = append(results, response)
}
}
return results, nil
}
// PendingJournalTerminals finds the crash window where a worker committed its
// terminal Journal transaction but did not yet publish completion.json. An
// invalid/mismatched artifact is skipped and retained for manual inspection;
// transient filesystem or database-lock errors are returned so startup can
// stop before irreversibly terminalizing the main Journal as unconfirmed.
func (e *SubprocessExecutor) PendingJournalTerminals() ([]TerminalEvidence, error) {
entries, err := os.ReadDir(e.stateDir)
if err != nil {
return nil, err
}
results := make([]TerminalEvidence, 0, len(entries))
for _, entry := range entries {
if !entry.IsDir() || !operationIDPattern.MatchString(entry.Name()) {
continue
}
operationDir := filepath.Join(e.stateDir, entry.Name())
trusted, trustErr := trustedWorkerPath(operationDir, true)
if trustErr != nil {
return nil, trustErr
}
if !trusted {
continue
}
_, receiptFound, receiptErr := readTrustedReceipt(
filepath.Join(operationDir, workerReceiptName), entry.Name(),
)
if receiptErr != nil {
return nil, receiptErr
}
if receiptFound {
continue
}
journalPath := filepath.Join(operationDir, workerJournalName)
trusted, trustErr = trustedWorkerPath(journalPath, false)
if trustErr != nil {
if errors.Is(trustErr, os.ErrNotExist) {
continue
}
return nil, trustErr
}
if !trusted {
continue
}
var identity *RequestIdentity
identityPath := filepath.Join(operationDir, workerIdentityName)
trustedIdentity, identityTrustErr := trustedWorkerPath(identityPath, false)
switch {
case identityTrustErr == nil && trustedIdentity:
read, readErr := readRequestIdentity(identityPath)
if readErr != nil {
if errors.Is(readErr, errInvalidWorkerIdentity) {
continue
}
return nil, readErr
}
if read.OperationID != entry.Name() {
continue
}
identity = &read
case identityTrustErr == nil:
continue
case errors.Is(identityTrustErr, os.ErrNotExist):
// Pre-upgrade workers did not persist request-identity.json. Their
// exact operation/action/resource/fingerprint still has to match the
// main Journal before it can be adopted.
case identityTrustErr != nil:
return nil, identityTrustErr
}
record, readErr := operation.ReadTerminalRecord(journalPath, entry.Name())
if readErr != nil {
if errors.Is(readErr, operation.ErrTerminalEvidenceInvalid) ||
errors.Is(readErr, operation.ErrTerminalEvidencePending) || errors.Is(readErr, os.ErrNotExist) {
continue
}
return nil, readErr
}
results = append(results, TerminalEvidence{
Record: record, Identity: identity, Legacy: identity == nil, trustedFilesystem: true,
})
}
return results, nil
}
func (e *SubprocessExecutor) PendingOperationIDs() ([]string, error) {
entries, err := os.ReadDir(e.stateDir)
if err != nil {
return nil, err
}
operationIDs := make([]string, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() && operationIDPattern.MatchString(entry.Name()) {
operationIDs = append(operationIDs, entry.Name())
}
}
return operationIDs, nil
}
func (e *SubprocessExecutor) Cleanup(operationID string) error {
if !operationIDPattern.MatchString(operationID) {
return errors.New("invalid operation ID")
}
return os.RemoveAll(filepath.Join(e.stateDir, operationID))
}
// CleanupCompleted removes helper state only when its immutable receipt still
// matches the main Journal request fingerprint. This prevents completion
// delivery of a synthesized result_unconfirmed record from deleting corrupt or
// mismatched child evidence that was never adopted.
func (e *SubprocessExecutor) CleanupCompleted(operationID, fingerprint string) error {
if !operationIDPattern.MatchString(operationID) || !fingerprintPattern.MatchString(fingerprint) {
return errors.New("invalid completed worker identity")
}
response, err := ReadReceipt(filepath.Join(e.stateDir, operationID, workerReceiptName))
if err != nil || response.OperationID != operationID || response.Fingerprint != fingerprint {
return nil
}
return e.Cleanup(operationID)
}
func WriteReceipt(path string, response Response) error {
if err := response.Validate(); err != nil {
return err
}
encoded, err := json.Marshal(response)
if err != nil || len(encoded) > MaxWorkerResponse {
return errors.New("encode bounded worker receipt")
}
directory := filepath.Dir(path)
if err := os.MkdirAll(directory, 0o700); err != nil {
return err
}
temporary, err := os.CreateTemp(directory, ".completion-*.tmp")
if err != nil {
return err
}
temporaryName := temporary.Name()
remove := true
defer func() {
if remove {
_ = os.Remove(temporaryName)
}
}()
if err := temporary.Chmod(0o600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(encoded); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
// Link is an atomic no-replace publication on the same filesystem. A
// receipt that already exists is immutable, even if a buggy caller invokes
// the writer twice.
if err := os.Link(temporaryName, path); err != nil {
return err
}
if err := os.Remove(temporaryName); err != nil {
return err
}
remove = false
dir, err := os.Open(directory)
if err != nil {
return err
}
err = dir.Sync()
_ = dir.Close()
return err
}
func writeRequestIdentity(path string, identity RequestIdentity) error {
if err := identity.Validate(); err != nil {
return err
}
encoded, err := json.Marshal(identity)
if err != nil || len(encoded) > maxIdentityBytes {
return errInvalidWorkerIdentity
}
directory := filepath.Dir(path)
temporary, err := os.CreateTemp(directory, ".identity-*.tmp")
if err != nil {
return err
}
temporaryName := temporary.Name()
remove := true
defer func() {
if remove {
_ = os.Remove(temporaryName)
}
}()
if err := temporary.Chmod(0o600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(encoded); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
if err := os.Link(temporaryName, path); err != nil {
return err
}
if err := os.Remove(temporaryName); err != nil {
return err
}
remove = false
dir, err := os.Open(directory)
if err != nil {
return err
}
err = dir.Sync()
_ = dir.Close()
return err
}
func readRequestIdentity(path string) (RequestIdentity, error) {
file, err := os.Open(path)
if err != nil {
return RequestIdentity{}, err
}
defer file.Close()
encoded, err := io.ReadAll(io.LimitReader(file, maxIdentityBytes+1))
if err != nil {
return RequestIdentity{}, err
}
if len(encoded) > maxIdentityBytes {
return RequestIdentity{}, errInvalidWorkerIdentity
}
decoder := json.NewDecoder(bytes.NewReader(encoded))
decoder.DisallowUnknownFields()
var identity RequestIdentity
if err := decoder.Decode(&identity); err != nil {
return RequestIdentity{}, fmt.Errorf("%w: %v", errInvalidWorkerIdentity, err)
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return RequestIdentity{}, errInvalidWorkerIdentity
}
if err := identity.Validate(); err != nil {
return RequestIdentity{}, fmt.Errorf("%w: %v", errInvalidWorkerIdentity, err)
}
return identity, nil
}
func trustedWorkerPath(path string, directory bool) (bool, error) {
info, err := os.Lstat(path)
if err != nil {
return false, err
}
if info.Mode()&os.ModeSymlink != 0 || directory != info.IsDir() ||
(!directory && !info.Mode().IsRegular()) || info.Mode().Perm()&0o022 != 0 {
return false, nil
}
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || int(stat.Uid) != os.Geteuid() {
return false, nil
}
return true, nil
}
func readTrustedReceipt(path, operationID string) (Response, bool, error) {
trusted, err := trustedWorkerPath(path, false)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return Response{}, false, nil
}
return Response{}, false, err
}
if !trusted {
return Response{}, false, nil
}
response, err := ReadReceipt(path)
if err != nil {
if errors.Is(err, errInvalidWorkerReceipt) {
return Response{}, false, nil
}
return Response{}, false, err
}
if response.OperationID != operationID {
return Response{}, false, nil
}
return response, true, nil
}
func ReadReceipt(path string) (Response, error) {
file, err := os.Open(path)
if err != nil {
return Response{}, err
}
defer file.Close()
encoded, err := io.ReadAll(io.LimitReader(file, MaxWorkerResponse+1))
if err != nil {
return Response{}, err
}
if len(encoded) > MaxWorkerResponse {
return Response{}, errInvalidWorkerReceipt
}
decoder := json.NewDecoder(bytes.NewReader(encoded))
decoder.DisallowUnknownFields()
var response Response
if err := decoder.Decode(&response); err != nil {
return Response{}, fmt.Errorf("%w: %v", errInvalidWorkerReceipt, err)
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return Response{}, errInvalidWorkerReceipt
}
if err := response.Validate(); err != nil {
return Response{}, fmt.Errorf("%w: %v", errInvalidWorkerReceipt, err)
}
return response, nil
}
type limitedBuffer struct {
mu sync.Mutex
buffer bytes.Buffer
limit int
overflow bool
}
func (b *limitedBuffer) Write(value []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
remaining := b.limit - b.buffer.Len()
if remaining <= 0 {
b.overflow = true
return len(value), nil
}
if len(value) > remaining {
_, _ = b.buffer.Write(value[:remaining])
b.overflow = true
return len(value), nil
}
_, _ = b.buffer.Write(value)
return len(value), nil
}