package worker import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "strings" "sync" "time" ) const ( workerReceiptName = "completion.json" workerJournalName = "journal.db" terminationGrace = 500 * time.Millisecond killWait = 2 * time.Second ) type SubprocessExecutor struct { executable string configPath string stateDir string } func NewSubprocessExecutor(executable, configPath, stateDir string) (*SubprocessExecutor, error) { for name, value := range map[string]string{"executable": executable, "config path": configPath, "state directory": stateDir} { if strings.TrimSpace(value) == "" || !filepath.IsAbs(value) || filepath.Clean(value) != value { return nil, fmt.Errorf("worker %s must be a clean absolute path", name) } } if err := os.MkdirAll(stateDir, 0o700); err != nil { return nil, fmt.Errorf("create worker state directory: %w", err) } if err := os.Chmod(stateDir, 0o700); err != nil { return nil, fmt.Errorf("secure worker state directory: %w", err) } return &SubprocessExecutor{executable: executable, configPath: configPath, stateDir: stateDir}, nil } func (e *SubprocessExecutor) Execute(ctx context.Context, request Request) (json.RawMessage, error) { if err := request.Validate(); err != nil { return nil, err } operationDir := filepath.Join(e.stateDir, request.OperationID) receiptPath := filepath.Join(operationDir, workerReceiptName) journalPath := filepath.Join(operationDir, workerJournalName) if err := os.Mkdir(operationDir, 0o700); err != nil { if !errors.Is(err, os.ErrExist) { return nil, fmt.Errorf("create operation worker directory: %w", err) } response, readErr := ReadReceipt(receiptPath) if readErr != nil || response.Fingerprint != request.Fingerprint { return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint)) } return Result(response) } payload, err := json.Marshal(request) if err != nil || len(payload) > MaxEnvelopeBytes { _ = os.RemoveAll(operationDir) return nil, errors.New("encode bounded worker request") } command := exec.Command(e.executable, "--config", e.configPath, "--operation-worker", "--worker-journal", journalPath, "--worker-receipt", receiptPath, ) configureProcess(command) command.Stdin = bytes.NewReader(payload) stdout := &limitedBuffer{limit: MaxWorkerResponse} stderr := &limitedBuffer{limit: 32 << 10} command.Stdout = stdout command.Stderr = stderr if err := command.Start(); err != nil { _ = os.RemoveAll(operationDir) return nil, &Error{ Code: "worker_start_failed", Message: "isolated mutation worker could not be started", EffectMayHaveApplied: false, ResubmitAllowed: true, } } wait := make(chan error, 1) go func() { wait <- command.Wait() }() select { case waitErr := <-wait: response, readErr := ReadReceipt(receiptPath) if readErr != nil || response.OperationID != request.OperationID || response.Fingerprint != request.Fingerprint { if waitErr != nil { return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint)) } return nil, errors.New("isolated mutation worker returned no valid receipt") } return Result(response) case <-ctx.Done(): reaped := false terminateProcessGroup(command) select { case <-wait: reaped = true case <-time.After(terminationGrace): killProcessGroup(command) select { case <-wait: reaped = true case <-time.After(killWait): return nil, &Error{ Code: "worker_reap_unconfirmed", Message: "isolated mutation worker did not exit after forced termination", EffectMayHaveApplied: true, ResubmitAllowed: false, ExecutionFenced: true, } } } if reaped { if response, readErr := ReadReceipt(receiptPath); readErr == nil && response.OperationID == request.OperationID && response.Fingerprint == request.Fingerprint { return Result(response) } } return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint)) } } func (e *SubprocessExecutor) PendingReceipts() ([]Response, error) { entries, err := os.ReadDir(e.stateDir) if err != nil { return nil, err } results := make([]Response, 0, len(entries)) for _, entry := range entries { if !entry.IsDir() || !operationIDPattern.MatchString(entry.Name()) { continue } response, readErr := ReadReceipt(filepath.Join(e.stateDir, entry.Name(), workerReceiptName)) if readErr == nil && response.OperationID == entry.Name() { results = append(results, response) } } return results, nil } func (e *SubprocessExecutor) PendingOperationIDs() ([]string, error) { entries, err := os.ReadDir(e.stateDir) if err != nil { return nil, err } operationIDs := make([]string, 0, len(entries)) for _, entry := range entries { if entry.IsDir() && operationIDPattern.MatchString(entry.Name()) { operationIDs = append(operationIDs, entry.Name()) } } return operationIDs, nil } func (e *SubprocessExecutor) Cleanup(operationID string) error { if !operationIDPattern.MatchString(operationID) { return errors.New("invalid operation ID") } return os.RemoveAll(filepath.Join(e.stateDir, operationID)) } func WriteReceipt(path string, response Response) error { if err := response.Validate(); err != nil { return err } encoded, err := json.Marshal(response) if err != nil || len(encoded) > MaxWorkerResponse { return errors.New("encode bounded worker receipt") } directory := filepath.Dir(path) if err := os.MkdirAll(directory, 0o700); err != nil { return err } temporary, err := os.CreateTemp(directory, ".completion-*.tmp") if err != nil { return err } temporaryName := temporary.Name() remove := true defer func() { if remove { _ = os.Remove(temporaryName) } }() if err := temporary.Chmod(0o600); err != nil { _ = temporary.Close() return err } if _, err := temporary.Write(encoded); err != nil { _ = temporary.Close() return err } if err := temporary.Sync(); err != nil { _ = temporary.Close() return err } if err := temporary.Close(); err != nil { return err } // Link is an atomic no-replace publication on the same filesystem. A // receipt that already exists is immutable, even if a buggy caller invokes // the writer twice. if err := os.Link(temporaryName, path); err != nil { return err } if err := os.Remove(temporaryName); err != nil { return err } remove = false dir, err := os.Open(directory) if err != nil { return err } err = dir.Sync() _ = dir.Close() return err } func ReadReceipt(path string) (Response, error) { file, err := os.Open(path) if err != nil { return Response{}, err } defer file.Close() encoded, err := io.ReadAll(io.LimitReader(file, MaxWorkerResponse+1)) if err != nil || len(encoded) > MaxWorkerResponse { return Response{}, errors.New("read bounded worker receipt") } decoder := json.NewDecoder(bytes.NewReader(encoded)) decoder.DisallowUnknownFields() var response Response if err := decoder.Decode(&response); err != nil { return Response{}, err } var trailing any if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { return Response{}, errors.New("worker receipt contains trailing data") } return response, response.Validate() } type limitedBuffer struct { mu sync.Mutex buffer bytes.Buffer limit int overflow bool } func (b *limitedBuffer) Write(value []byte) (int, error) { b.mu.Lock() defer b.mu.Unlock() remaining := b.limit - b.buffer.Len() if remaining <= 0 { b.overflow = true return len(value), nil } if len(value) > remaining { _, _ = b.buffer.Write(value[:remaining]) b.overflow = true return len(value), nil } _, _ = b.buffer.Write(value) return len(value), nil }