You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/worker/executor.go

270 lines
7.6 KiB
Go

package worker
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
)
const (
workerReceiptName = "completion.json"
workerJournalName = "journal.db"
terminationGrace = 500 * time.Millisecond
killWait = 2 * time.Second
)
type SubprocessExecutor struct {
executable string
configPath string
stateDir string
}
func NewSubprocessExecutor(executable, configPath, stateDir string) (*SubprocessExecutor, error) {
for name, value := range map[string]string{"executable": executable, "config path": configPath, "state directory": stateDir} {
if strings.TrimSpace(value) == "" || !filepath.IsAbs(value) || filepath.Clean(value) != value {
return nil, fmt.Errorf("worker %s must be a clean absolute path", name)
}
}
if err := os.MkdirAll(stateDir, 0o700); err != nil {
return nil, fmt.Errorf("create worker state directory: %w", err)
}
if err := os.Chmod(stateDir, 0o700); err != nil {
return nil, fmt.Errorf("secure worker state directory: %w", err)
}
return &SubprocessExecutor{executable: executable, configPath: configPath, stateDir: stateDir}, nil
}
func (e *SubprocessExecutor) Execute(ctx context.Context, request Request) (json.RawMessage, error) {
if err := request.Validate(); err != nil {
return nil, err
}
operationDir := filepath.Join(e.stateDir, request.OperationID)
receiptPath := filepath.Join(operationDir, workerReceiptName)
journalPath := filepath.Join(operationDir, workerJournalName)
if err := os.Mkdir(operationDir, 0o700); err != nil {
if !errors.Is(err, os.ErrExist) {
return nil, fmt.Errorf("create operation worker directory: %w", err)
}
response, readErr := ReadReceipt(receiptPath)
if readErr != nil || response.Fingerprint != request.Fingerprint {
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
}
return Result(response)
}
payload, err := json.Marshal(request)
if err != nil || len(payload) > MaxEnvelopeBytes {
_ = os.RemoveAll(operationDir)
return nil, errors.New("encode bounded worker request")
}
command := exec.Command(e.executable,
"--config", e.configPath,
"--operation-worker",
"--worker-journal", journalPath,
"--worker-receipt", receiptPath,
)
configureProcess(command)
command.Stdin = bytes.NewReader(payload)
stdout := &limitedBuffer{limit: MaxWorkerResponse}
stderr := &limitedBuffer{limit: 32 << 10}
command.Stdout = stdout
command.Stderr = stderr
if err := command.Start(); err != nil {
_ = os.RemoveAll(operationDir)
return nil, &Error{
Code: "worker_start_failed", Message: "isolated mutation worker could not be started",
EffectMayHaveApplied: false, ResubmitAllowed: true,
}
}
wait := make(chan error, 1)
go func() { wait <- command.Wait() }()
select {
case waitErr := <-wait:
response, readErr := ReadReceipt(receiptPath)
if readErr != nil || response.OperationID != request.OperationID || response.Fingerprint != request.Fingerprint {
if waitErr != nil {
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
}
return nil, errors.New("isolated mutation worker returned no valid receipt")
}
return Result(response)
case <-ctx.Done():
reaped := false
terminateProcessGroup(command)
select {
case <-wait:
reaped = true
case <-time.After(terminationGrace):
killProcessGroup(command)
select {
case <-wait:
reaped = true
case <-time.After(killWait):
return nil, &Error{
Code: "worker_reap_unconfirmed",
Message: "isolated mutation worker did not exit after forced termination",
EffectMayHaveApplied: true, ResubmitAllowed: false, ExecutionFenced: true,
}
}
}
if reaped {
if response, readErr := ReadReceipt(receiptPath); readErr == nil &&
response.OperationID == request.OperationID && response.Fingerprint == request.Fingerprint {
return Result(response)
}
}
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
}
}
func (e *SubprocessExecutor) PendingReceipts() ([]Response, error) {
entries, err := os.ReadDir(e.stateDir)
if err != nil {
return nil, err
}
results := make([]Response, 0, len(entries))
for _, entry := range entries {
if !entry.IsDir() || !operationIDPattern.MatchString(entry.Name()) {
continue
}
response, readErr := ReadReceipt(filepath.Join(e.stateDir, entry.Name(), workerReceiptName))
if readErr == nil && response.OperationID == entry.Name() {
results = append(results, response)
}
}
return results, nil
}
func (e *SubprocessExecutor) PendingOperationIDs() ([]string, error) {
entries, err := os.ReadDir(e.stateDir)
if err != nil {
return nil, err
}
operationIDs := make([]string, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() && operationIDPattern.MatchString(entry.Name()) {
operationIDs = append(operationIDs, entry.Name())
}
}
return operationIDs, nil
}
func (e *SubprocessExecutor) Cleanup(operationID string) error {
if !operationIDPattern.MatchString(operationID) {
return errors.New("invalid operation ID")
}
return os.RemoveAll(filepath.Join(e.stateDir, operationID))
}
func WriteReceipt(path string, response Response) error {
if err := response.Validate(); err != nil {
return err
}
encoded, err := json.Marshal(response)
if err != nil || len(encoded) > MaxWorkerResponse {
return errors.New("encode bounded worker receipt")
}
directory := filepath.Dir(path)
if err := os.MkdirAll(directory, 0o700); err != nil {
return err
}
temporary, err := os.CreateTemp(directory, ".completion-*.tmp")
if err != nil {
return err
}
temporaryName := temporary.Name()
remove := true
defer func() {
if remove {
_ = os.Remove(temporaryName)
}
}()
if err := temporary.Chmod(0o600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(encoded); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
// Link is an atomic no-replace publication on the same filesystem. A
// receipt that already exists is immutable, even if a buggy caller invokes
// the writer twice.
if err := os.Link(temporaryName, path); err != nil {
return err
}
if err := os.Remove(temporaryName); err != nil {
return err
}
remove = false
dir, err := os.Open(directory)
if err != nil {
return err
}
err = dir.Sync()
_ = dir.Close()
return err
}
func ReadReceipt(path string) (Response, error) {
file, err := os.Open(path)
if err != nil {
return Response{}, err
}
defer file.Close()
encoded, err := io.ReadAll(io.LimitReader(file, MaxWorkerResponse+1))
if err != nil || len(encoded) > MaxWorkerResponse {
return Response{}, errors.New("read bounded worker receipt")
}
decoder := json.NewDecoder(bytes.NewReader(encoded))
decoder.DisallowUnknownFields()
var response Response
if err := decoder.Decode(&response); err != nil {
return Response{}, err
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return Response{}, errors.New("worker receipt contains trailing data")
}
return response, response.Validate()
}
type limitedBuffer struct {
mu sync.Mutex
buffer bytes.Buffer
limit int
overflow bool
}
func (b *limitedBuffer) Write(value []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
remaining := b.limit - b.buffer.Len()
if remaining <= 0 {
b.overflow = true
return len(value), nil
}
if len(value) > remaining {
_, _ = b.buffer.Write(value[:remaining])
b.overflow = true
return len(value), nil
}
_, _ = b.buffer.Write(value)
return len(value), nil
}