package worker import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "strings" "sync" "syscall" "time" "git.luoli.live/root/virt/agent/operation" ) const ( workerReceiptName = "completion.json" workerJournalName = "journal.db" workerIdentityName = "request-identity.json" terminationGrace = 500 * time.Millisecond killWait = 2 * time.Second maxIdentityBytes = 4 << 10 ) var ( errInvalidWorkerIdentity = errors.New("invalid worker recovery identity") errInvalidWorkerReceipt = errors.New("invalid worker receipt") ) // TerminalEvidence is a committed child-Journal result discovered before an // immutable completion.json was published. trustedFilesystem is deliberately // private: only the scanner that verifies file type, mode, and ownership can // authorize conversion into a main-Journal receipt. type TerminalEvidence struct { Record operation.Record Identity *RequestIdentity Legacy bool trustedFilesystem bool } // InterruptedReceiptFor accepts child evidence only when every identity field // available to both Journals matches exactly. New workers additionally require // the durable request identity (actor owner, target, body digest). A legacy // Journal can be adopted only from a trusted pre-upgrade worker directory. func (e TerminalEvidence) InterruptedReceiptFor(main operation.Record) (operation.InterruptedReceipt, bool) { if !e.trustedFilesystem || main.State != operation.StateDispatching || main.OperationID != e.Record.OperationID || main.Action != e.Record.Action || main.ResourceID != e.Record.ResourceID || main.Fingerprint != e.Record.Fingerprint || (e.Record.State != operation.StateSucceeded && e.Record.State != operation.StateFailed) { return operation.InterruptedReceipt{}, false } if e.Identity == nil { if !e.Legacy { return operation.InterruptedReceipt{}, false } } else if e.Legacy || e.Identity.Validate() != nil || e.Identity.OperationID != e.Record.OperationID || e.Identity.Fingerprint != e.Record.Fingerprint { return operation.InterruptedReceipt{}, false } result := append(json.RawMessage(nil), e.Record.Result...) if len(result) > MaxWorkerResponse/2 { result = nil } response := Response{ Version: ProtocolVersion, OperationID: e.Record.OperationID, Fingerprint: e.Record.Fingerprint, Status: string(e.Record.State), Result: result, EffectMayHaveApplied: e.Record.EffectMayHaveApplied, SafeToResubmit: e.Record.SafeToResubmit, } if e.Record.Error != nil { response.ErrorCode = e.Record.Error.Code response.ErrorMessage = e.Record.Error.Message if len(response.ErrorMessage) > 1024 { response.ErrorMessage = response.ErrorMessage[:1024] } } if response.Validate() != nil { return operation.InterruptedReceipt{}, false } return operation.InterruptedReceipt{ OperationID: response.OperationID, Fingerprint: response.Fingerprint, Status: response.Status, Result: response.Result, ErrorCode: response.ErrorCode, ErrorMessage: response.ErrorMessage, EffectMayHaveApplied: response.EffectMayHaveApplied, SafeToResubmit: response.SafeToResubmit, }, true } type SubprocessExecutor struct { executable string configPath string stateDir string } func NewSubprocessExecutor(executable, configPath, stateDir string) (*SubprocessExecutor, error) { for name, value := range map[string]string{"executable": executable, "config path": configPath, "state directory": stateDir} { if strings.TrimSpace(value) == "" || !filepath.IsAbs(value) || filepath.Clean(value) != value { return nil, fmt.Errorf("worker %s must be a clean absolute path", name) } } if err := os.MkdirAll(stateDir, 0o700); err != nil { return nil, fmt.Errorf("create worker state directory: %w", err) } if err := os.Chmod(stateDir, 0o700); err != nil { return nil, fmt.Errorf("secure worker state directory: %w", err) } return &SubprocessExecutor{executable: executable, configPath: configPath, stateDir: stateDir}, nil } func (e *SubprocessExecutor) Execute(ctx context.Context, request Request) (json.RawMessage, error) { if err := request.Validate(); err != nil { return nil, err } operationDir := filepath.Join(e.stateDir, request.OperationID) receiptPath := filepath.Join(operationDir, workerReceiptName) journalPath := filepath.Join(operationDir, workerJournalName) if err := os.Mkdir(operationDir, 0o700); err != nil { if !errors.Is(err, os.ErrExist) { return nil, fmt.Errorf("create operation worker directory: %w", err) } response, readErr := ReadReceipt(receiptPath) if readErr != nil || response.Fingerprint != request.Fingerprint { return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint)) } return Result(response) } payload, err := json.Marshal(request) if err != nil || len(payload) > MaxEnvelopeBytes { _ = os.RemoveAll(operationDir) return nil, errors.New("encode bounded worker request") } identity, err := requestIdentityFromRequest(request) if err != nil { _ = os.RemoveAll(operationDir) return nil, err } if err := writeRequestIdentity(filepath.Join(operationDir, workerIdentityName), identity); err != nil { _ = os.RemoveAll(operationDir) return nil, fmt.Errorf("persist worker recovery identity: %w", err) } command := exec.Command(e.executable, "--config", e.configPath, "--operation-worker", "--worker-journal", journalPath, "--worker-receipt", receiptPath, ) configureProcess(command) command.Stdin = bytes.NewReader(payload) stdout := &limitedBuffer{limit: MaxWorkerResponse} stderr := &limitedBuffer{limit: 32 << 10} command.Stdout = stdout command.Stderr = stderr if err := command.Start(); err != nil { _ = os.RemoveAll(operationDir) return nil, &Error{ Code: "worker_start_failed", Message: "isolated mutation worker could not be started", EffectMayHaveApplied: false, ResubmitAllowed: true, } } wait := make(chan error, 1) go func() { wait <- command.Wait() }() select { case waitErr := <-wait: response, readErr := ReadReceipt(receiptPath) if readErr != nil || response.OperationID != request.OperationID || response.Fingerprint != request.Fingerprint { if waitErr != nil { return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint)) } return nil, errors.New("isolated mutation worker returned no valid receipt") } return Result(response) case <-ctx.Done(): reaped := false terminateProcessGroup(command) select { case <-wait: reaped = true case <-time.After(terminationGrace): killProcessGroup(command) select { case <-wait: reaped = true case <-time.After(killWait): return nil, &Error{ Code: "worker_reap_unconfirmed", Message: "isolated mutation worker did not exit after forced termination", EffectMayHaveApplied: true, ResubmitAllowed: false, ExecutionFenced: true, } } } if reaped { if response, readErr := ReadReceipt(receiptPath); readErr == nil && response.OperationID == request.OperationID && response.Fingerprint == request.Fingerprint { return Result(response) } } return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint)) } } func (e *SubprocessExecutor) PendingReceipts() ([]Response, error) { entries, err := os.ReadDir(e.stateDir) if err != nil { return nil, err } results := make([]Response, 0, len(entries)) for _, entry := range entries { if !entry.IsDir() || !operationIDPattern.MatchString(entry.Name()) { continue } response, found, readErr := readTrustedReceipt( filepath.Join(e.stateDir, entry.Name(), workerReceiptName), entry.Name(), ) if readErr != nil { return nil, readErr } if found { results = append(results, response) } } return results, nil } // PendingJournalTerminals finds the crash window where a worker committed its // terminal Journal transaction but did not yet publish completion.json. An // invalid/mismatched artifact is skipped and retained for manual inspection; // transient filesystem or database-lock errors are returned so startup can // stop before irreversibly terminalizing the main Journal as unconfirmed. func (e *SubprocessExecutor) PendingJournalTerminals() ([]TerminalEvidence, error) { entries, err := os.ReadDir(e.stateDir) if err != nil { return nil, err } results := make([]TerminalEvidence, 0, len(entries)) for _, entry := range entries { if !entry.IsDir() || !operationIDPattern.MatchString(entry.Name()) { continue } operationDir := filepath.Join(e.stateDir, entry.Name()) trusted, trustErr := trustedWorkerPath(operationDir, true) if trustErr != nil { return nil, trustErr } if !trusted { continue } _, receiptFound, receiptErr := readTrustedReceipt( filepath.Join(operationDir, workerReceiptName), entry.Name(), ) if receiptErr != nil { return nil, receiptErr } if receiptFound { continue } journalPath := filepath.Join(operationDir, workerJournalName) trusted, trustErr = trustedWorkerPath(journalPath, false) if trustErr != nil { if errors.Is(trustErr, os.ErrNotExist) { continue } return nil, trustErr } if !trusted { continue } var identity *RequestIdentity identityPath := filepath.Join(operationDir, workerIdentityName) trustedIdentity, identityTrustErr := trustedWorkerPath(identityPath, false) switch { case identityTrustErr == nil && trustedIdentity: read, readErr := readRequestIdentity(identityPath) if readErr != nil { if errors.Is(readErr, errInvalidWorkerIdentity) { continue } return nil, readErr } if read.OperationID != entry.Name() { continue } identity = &read case identityTrustErr == nil: continue case errors.Is(identityTrustErr, os.ErrNotExist): // Pre-upgrade workers did not persist request-identity.json. Their // exact operation/action/resource/fingerprint still has to match the // main Journal before it can be adopted. case identityTrustErr != nil: return nil, identityTrustErr } record, readErr := operation.ReadTerminalRecord(journalPath, entry.Name()) if readErr != nil { if errors.Is(readErr, operation.ErrTerminalEvidenceInvalid) || errors.Is(readErr, operation.ErrTerminalEvidencePending) || errors.Is(readErr, os.ErrNotExist) { continue } return nil, readErr } results = append(results, TerminalEvidence{ Record: record, Identity: identity, Legacy: identity == nil, trustedFilesystem: true, }) } return results, nil } func (e *SubprocessExecutor) PendingOperationIDs() ([]string, error) { entries, err := os.ReadDir(e.stateDir) if err != nil { return nil, err } operationIDs := make([]string, 0, len(entries)) for _, entry := range entries { if entry.IsDir() && operationIDPattern.MatchString(entry.Name()) { operationIDs = append(operationIDs, entry.Name()) } } return operationIDs, nil } func (e *SubprocessExecutor) Cleanup(operationID string) error { if !operationIDPattern.MatchString(operationID) { return errors.New("invalid operation ID") } return os.RemoveAll(filepath.Join(e.stateDir, operationID)) } // CleanupCompleted removes helper state only when its immutable receipt still // matches the main Journal request fingerprint. This prevents completion // delivery of a synthesized result_unconfirmed record from deleting corrupt or // mismatched child evidence that was never adopted. func (e *SubprocessExecutor) CleanupCompleted(operationID, fingerprint string) error { if !operationIDPattern.MatchString(operationID) || !fingerprintPattern.MatchString(fingerprint) { return errors.New("invalid completed worker identity") } response, err := ReadReceipt(filepath.Join(e.stateDir, operationID, workerReceiptName)) if err != nil || response.OperationID != operationID || response.Fingerprint != fingerprint { return nil } return e.Cleanup(operationID) } func WriteReceipt(path string, response Response) error { if err := response.Validate(); err != nil { return err } encoded, err := json.Marshal(response) if err != nil || len(encoded) > MaxWorkerResponse { return errors.New("encode bounded worker receipt") } directory := filepath.Dir(path) if err := os.MkdirAll(directory, 0o700); err != nil { return err } temporary, err := os.CreateTemp(directory, ".completion-*.tmp") if err != nil { return err } temporaryName := temporary.Name() remove := true defer func() { if remove { _ = os.Remove(temporaryName) } }() if err := temporary.Chmod(0o600); err != nil { _ = temporary.Close() return err } if _, err := temporary.Write(encoded); err != nil { _ = temporary.Close() return err } if err := temporary.Sync(); err != nil { _ = temporary.Close() return err } if err := temporary.Close(); err != nil { return err } // Link is an atomic no-replace publication on the same filesystem. A // receipt that already exists is immutable, even if a buggy caller invokes // the writer twice. if err := os.Link(temporaryName, path); err != nil { return err } if err := os.Remove(temporaryName); err != nil { return err } remove = false dir, err := os.Open(directory) if err != nil { return err } err = dir.Sync() _ = dir.Close() return err } func writeRequestIdentity(path string, identity RequestIdentity) error { if err := identity.Validate(); err != nil { return err } encoded, err := json.Marshal(identity) if err != nil || len(encoded) > maxIdentityBytes { return errInvalidWorkerIdentity } directory := filepath.Dir(path) temporary, err := os.CreateTemp(directory, ".identity-*.tmp") if err != nil { return err } temporaryName := temporary.Name() remove := true defer func() { if remove { _ = os.Remove(temporaryName) } }() if err := temporary.Chmod(0o600); err != nil { _ = temporary.Close() return err } if _, err := temporary.Write(encoded); err != nil { _ = temporary.Close() return err } if err := temporary.Sync(); err != nil { _ = temporary.Close() return err } if err := temporary.Close(); err != nil { return err } if err := os.Link(temporaryName, path); err != nil { return err } if err := os.Remove(temporaryName); err != nil { return err } remove = false dir, err := os.Open(directory) if err != nil { return err } err = dir.Sync() _ = dir.Close() return err } func readRequestIdentity(path string) (RequestIdentity, error) { file, err := os.Open(path) if err != nil { return RequestIdentity{}, err } defer file.Close() encoded, err := io.ReadAll(io.LimitReader(file, maxIdentityBytes+1)) if err != nil { return RequestIdentity{}, err } if len(encoded) > maxIdentityBytes { return RequestIdentity{}, errInvalidWorkerIdentity } decoder := json.NewDecoder(bytes.NewReader(encoded)) decoder.DisallowUnknownFields() var identity RequestIdentity if err := decoder.Decode(&identity); err != nil { return RequestIdentity{}, fmt.Errorf("%w: %v", errInvalidWorkerIdentity, err) } var trailing any if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { return RequestIdentity{}, errInvalidWorkerIdentity } if err := identity.Validate(); err != nil { return RequestIdentity{}, fmt.Errorf("%w: %v", errInvalidWorkerIdentity, err) } return identity, nil } func trustedWorkerPath(path string, directory bool) (bool, error) { info, err := os.Lstat(path) if err != nil { return false, err } if info.Mode()&os.ModeSymlink != 0 || directory != info.IsDir() || (!directory && !info.Mode().IsRegular()) || info.Mode().Perm()&0o022 != 0 { return false, nil } stat, ok := info.Sys().(*syscall.Stat_t) if !ok || int(stat.Uid) != os.Geteuid() { return false, nil } return true, nil } func readTrustedReceipt(path, operationID string) (Response, bool, error) { trusted, err := trustedWorkerPath(path, false) if err != nil { if errors.Is(err, os.ErrNotExist) { return Response{}, false, nil } return Response{}, false, err } if !trusted { return Response{}, false, nil } response, err := ReadReceipt(path) if err != nil { if errors.Is(err, errInvalidWorkerReceipt) { return Response{}, false, nil } return Response{}, false, err } if response.OperationID != operationID { return Response{}, false, nil } return response, true, nil } func ReadReceipt(path string) (Response, error) { file, err := os.Open(path) if err != nil { return Response{}, err } defer file.Close() encoded, err := io.ReadAll(io.LimitReader(file, MaxWorkerResponse+1)) if err != nil { return Response{}, err } if len(encoded) > MaxWorkerResponse { return Response{}, errInvalidWorkerReceipt } decoder := json.NewDecoder(bytes.NewReader(encoded)) decoder.DisallowUnknownFields() var response Response if err := decoder.Decode(&response); err != nil { return Response{}, fmt.Errorf("%w: %v", errInvalidWorkerReceipt, err) } var trailing any if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { return Response{}, errInvalidWorkerReceipt } if err := response.Validate(); err != nil { return Response{}, fmt.Errorf("%w: %v", errInvalidWorkerReceipt, err) } return response, nil } type limitedBuffer struct { mu sync.Mutex buffer bytes.Buffer limit int overflow bool } func (b *limitedBuffer) Write(value []byte) (int, error) { b.mu.Lock() defer b.mu.Unlock() remaining := b.limit - b.buffer.Len() if remaining <= 0 { b.overflow = true return len(value), nil } if len(value) > remaining { _, _ = b.buffer.Write(value[:remaining]) b.overflow = true return len(value), nil } _, _ = b.buffer.Write(value) return len(value), nil }