You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
270 lines
7.6 KiB
Go
270 lines
7.6 KiB
Go
package worker
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
workerReceiptName = "completion.json"
|
|
workerJournalName = "journal.db"
|
|
terminationGrace = 500 * time.Millisecond
|
|
killWait = 2 * time.Second
|
|
)
|
|
|
|
type SubprocessExecutor struct {
|
|
executable string
|
|
configPath string
|
|
stateDir string
|
|
}
|
|
|
|
func NewSubprocessExecutor(executable, configPath, stateDir string) (*SubprocessExecutor, error) {
|
|
for name, value := range map[string]string{"executable": executable, "config path": configPath, "state directory": stateDir} {
|
|
if strings.TrimSpace(value) == "" || !filepath.IsAbs(value) || filepath.Clean(value) != value {
|
|
return nil, fmt.Errorf("worker %s must be a clean absolute path", name)
|
|
}
|
|
}
|
|
if err := os.MkdirAll(stateDir, 0o700); err != nil {
|
|
return nil, fmt.Errorf("create worker state directory: %w", err)
|
|
}
|
|
if err := os.Chmod(stateDir, 0o700); err != nil {
|
|
return nil, fmt.Errorf("secure worker state directory: %w", err)
|
|
}
|
|
return &SubprocessExecutor{executable: executable, configPath: configPath, stateDir: stateDir}, nil
|
|
}
|
|
|
|
func (e *SubprocessExecutor) Execute(ctx context.Context, request Request) (json.RawMessage, error) {
|
|
if err := request.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
operationDir := filepath.Join(e.stateDir, request.OperationID)
|
|
receiptPath := filepath.Join(operationDir, workerReceiptName)
|
|
journalPath := filepath.Join(operationDir, workerJournalName)
|
|
if err := os.Mkdir(operationDir, 0o700); err != nil {
|
|
if !errors.Is(err, os.ErrExist) {
|
|
return nil, fmt.Errorf("create operation worker directory: %w", err)
|
|
}
|
|
response, readErr := ReadReceipt(receiptPath)
|
|
if readErr != nil || response.Fingerprint != request.Fingerprint {
|
|
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
|
|
}
|
|
return Result(response)
|
|
}
|
|
payload, err := json.Marshal(request)
|
|
if err != nil || len(payload) > MaxEnvelopeBytes {
|
|
_ = os.RemoveAll(operationDir)
|
|
return nil, errors.New("encode bounded worker request")
|
|
}
|
|
command := exec.Command(e.executable,
|
|
"--config", e.configPath,
|
|
"--operation-worker",
|
|
"--worker-journal", journalPath,
|
|
"--worker-receipt", receiptPath,
|
|
)
|
|
configureProcess(command)
|
|
command.Stdin = bytes.NewReader(payload)
|
|
stdout := &limitedBuffer{limit: MaxWorkerResponse}
|
|
stderr := &limitedBuffer{limit: 32 << 10}
|
|
command.Stdout = stdout
|
|
command.Stderr = stderr
|
|
if err := command.Start(); err != nil {
|
|
_ = os.RemoveAll(operationDir)
|
|
return nil, &Error{
|
|
Code: "worker_start_failed", Message: "isolated mutation worker could not be started",
|
|
EffectMayHaveApplied: false, ResubmitAllowed: true,
|
|
}
|
|
}
|
|
wait := make(chan error, 1)
|
|
go func() { wait <- command.Wait() }()
|
|
select {
|
|
case waitErr := <-wait:
|
|
response, readErr := ReadReceipt(receiptPath)
|
|
if readErr != nil || response.OperationID != request.OperationID || response.Fingerprint != request.Fingerprint {
|
|
if waitErr != nil {
|
|
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
|
|
}
|
|
return nil, errors.New("isolated mutation worker returned no valid receipt")
|
|
}
|
|
return Result(response)
|
|
case <-ctx.Done():
|
|
reaped := false
|
|
terminateProcessGroup(command)
|
|
select {
|
|
case <-wait:
|
|
reaped = true
|
|
case <-time.After(terminationGrace):
|
|
killProcessGroup(command)
|
|
select {
|
|
case <-wait:
|
|
reaped = true
|
|
case <-time.After(killWait):
|
|
return nil, &Error{
|
|
Code: "worker_reap_unconfirmed",
|
|
Message: "isolated mutation worker did not exit after forced termination",
|
|
EffectMayHaveApplied: true, ResubmitAllowed: false, ExecutionFenced: true,
|
|
}
|
|
}
|
|
}
|
|
if reaped {
|
|
if response, readErr := ReadReceipt(receiptPath); readErr == nil &&
|
|
response.OperationID == request.OperationID && response.Fingerprint == request.Fingerprint {
|
|
return Result(response)
|
|
}
|
|
}
|
|
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
|
|
}
|
|
}
|
|
|
|
func (e *SubprocessExecutor) PendingReceipts() ([]Response, error) {
|
|
entries, err := os.ReadDir(e.stateDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
results := make([]Response, 0, len(entries))
|
|
for _, entry := range entries {
|
|
if !entry.IsDir() || !operationIDPattern.MatchString(entry.Name()) {
|
|
continue
|
|
}
|
|
response, readErr := ReadReceipt(filepath.Join(e.stateDir, entry.Name(), workerReceiptName))
|
|
if readErr == nil && response.OperationID == entry.Name() {
|
|
results = append(results, response)
|
|
}
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (e *SubprocessExecutor) PendingOperationIDs() ([]string, error) {
|
|
entries, err := os.ReadDir(e.stateDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
operationIDs := make([]string, 0, len(entries))
|
|
for _, entry := range entries {
|
|
if entry.IsDir() && operationIDPattern.MatchString(entry.Name()) {
|
|
operationIDs = append(operationIDs, entry.Name())
|
|
}
|
|
}
|
|
return operationIDs, nil
|
|
}
|
|
|
|
func (e *SubprocessExecutor) Cleanup(operationID string) error {
|
|
if !operationIDPattern.MatchString(operationID) {
|
|
return errors.New("invalid operation ID")
|
|
}
|
|
return os.RemoveAll(filepath.Join(e.stateDir, operationID))
|
|
}
|
|
|
|
func WriteReceipt(path string, response Response) error {
|
|
if err := response.Validate(); err != nil {
|
|
return err
|
|
}
|
|
encoded, err := json.Marshal(response)
|
|
if err != nil || len(encoded) > MaxWorkerResponse {
|
|
return errors.New("encode bounded worker receipt")
|
|
}
|
|
directory := filepath.Dir(path)
|
|
if err := os.MkdirAll(directory, 0o700); err != nil {
|
|
return err
|
|
}
|
|
temporary, err := os.CreateTemp(directory, ".completion-*.tmp")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
temporaryName := temporary.Name()
|
|
remove := true
|
|
defer func() {
|
|
if remove {
|
|
_ = os.Remove(temporaryName)
|
|
}
|
|
}()
|
|
if err := temporary.Chmod(0o600); err != nil {
|
|
_ = temporary.Close()
|
|
return err
|
|
}
|
|
if _, err := temporary.Write(encoded); err != nil {
|
|
_ = temporary.Close()
|
|
return err
|
|
}
|
|
if err := temporary.Sync(); err != nil {
|
|
_ = temporary.Close()
|
|
return err
|
|
}
|
|
if err := temporary.Close(); err != nil {
|
|
return err
|
|
}
|
|
// Link is an atomic no-replace publication on the same filesystem. A
|
|
// receipt that already exists is immutable, even if a buggy caller invokes
|
|
// the writer twice.
|
|
if err := os.Link(temporaryName, path); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Remove(temporaryName); err != nil {
|
|
return err
|
|
}
|
|
remove = false
|
|
dir, err := os.Open(directory)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = dir.Sync()
|
|
_ = dir.Close()
|
|
return err
|
|
}
|
|
|
|
func ReadReceipt(path string) (Response, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return Response{}, err
|
|
}
|
|
defer file.Close()
|
|
encoded, err := io.ReadAll(io.LimitReader(file, MaxWorkerResponse+1))
|
|
if err != nil || len(encoded) > MaxWorkerResponse {
|
|
return Response{}, errors.New("read bounded worker receipt")
|
|
}
|
|
decoder := json.NewDecoder(bytes.NewReader(encoded))
|
|
decoder.DisallowUnknownFields()
|
|
var response Response
|
|
if err := decoder.Decode(&response); err != nil {
|
|
return Response{}, err
|
|
}
|
|
var trailing any
|
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
|
return Response{}, errors.New("worker receipt contains trailing data")
|
|
}
|
|
return response, response.Validate()
|
|
}
|
|
|
|
type limitedBuffer struct {
|
|
mu sync.Mutex
|
|
buffer bytes.Buffer
|
|
limit int
|
|
overflow bool
|
|
}
|
|
|
|
func (b *limitedBuffer) Write(value []byte) (int, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
remaining := b.limit - b.buffer.Len()
|
|
if remaining <= 0 {
|
|
b.overflow = true
|
|
return len(value), nil
|
|
}
|
|
if len(value) > remaining {
|
|
_, _ = b.buffer.Write(value[:remaining])
|
|
b.overflow = true
|
|
return len(value), nil
|
|
}
|
|
_, _ = b.buffer.Write(value)
|
|
return len(value), nil
|
|
}
|