|
|
|
|
package worker
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"bytes"
|
|
|
|
|
"context"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"errors"
|
|
|
|
|
"fmt"
|
|
|
|
|
"io"
|
|
|
|
|
"os"
|
|
|
|
|
"os/exec"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"strings"
|
|
|
|
|
"sync"
|
|
|
|
|
"syscall"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const (
|
|
|
|
|
workerReceiptName = "completion.json"
|
|
|
|
|
workerJournalName = "journal.db"
|
|
|
|
|
workerIdentityName = "request-identity.json"
|
|
|
|
|
terminationGrace = 500 * time.Millisecond
|
|
|
|
|
killWait = 2 * time.Second
|
|
|
|
|
maxIdentityBytes = 4 << 10
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
var (
|
|
|
|
|
errInvalidWorkerIdentity = errors.New("invalid worker recovery identity")
|
|
|
|
|
errInvalidWorkerReceipt = errors.New("invalid worker receipt")
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// TerminalEvidence is a committed child-Journal result discovered before an
|
|
|
|
|
// immutable completion.json was published. trustedFilesystem is deliberately
|
|
|
|
|
// private: only the scanner that verifies file type, mode, and ownership can
|
|
|
|
|
// authorize conversion into a main-Journal receipt.
|
|
|
|
|
type TerminalEvidence struct {
|
|
|
|
|
Record operation.Record
|
|
|
|
|
Identity *RequestIdentity
|
|
|
|
|
Legacy bool
|
|
|
|
|
|
|
|
|
|
trustedFilesystem bool
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// InterruptedReceiptFor accepts child evidence only when every identity field
|
|
|
|
|
// available to both Journals matches exactly. New workers additionally require
|
|
|
|
|
// the durable request identity (actor owner, target, body digest). A legacy
|
|
|
|
|
// Journal can be adopted only from a trusted pre-upgrade worker directory.
|
|
|
|
|
func (e TerminalEvidence) InterruptedReceiptFor(main operation.Record) (operation.InterruptedReceipt, bool) {
|
|
|
|
|
if !e.trustedFilesystem || main.State != operation.StateDispatching ||
|
|
|
|
|
main.OperationID != e.Record.OperationID || main.Action != e.Record.Action ||
|
|
|
|
|
main.ResourceID != e.Record.ResourceID || main.Fingerprint != e.Record.Fingerprint ||
|
|
|
|
|
(e.Record.State != operation.StateSucceeded && e.Record.State != operation.StateFailed) {
|
|
|
|
|
return operation.InterruptedReceipt{}, false
|
|
|
|
|
}
|
|
|
|
|
if e.Identity == nil {
|
|
|
|
|
if !e.Legacy {
|
|
|
|
|
return operation.InterruptedReceipt{}, false
|
|
|
|
|
}
|
|
|
|
|
} else if e.Legacy || e.Identity.Validate() != nil ||
|
|
|
|
|
e.Identity.OperationID != e.Record.OperationID || e.Identity.Fingerprint != e.Record.Fingerprint {
|
|
|
|
|
return operation.InterruptedReceipt{}, false
|
|
|
|
|
}
|
|
|
|
|
result := append(json.RawMessage(nil), e.Record.Result...)
|
|
|
|
|
if len(result) > MaxWorkerResponse/2 {
|
|
|
|
|
result = nil
|
|
|
|
|
}
|
|
|
|
|
response := Response{
|
|
|
|
|
Version: ProtocolVersion, OperationID: e.Record.OperationID, Fingerprint: e.Record.Fingerprint,
|
|
|
|
|
Status: string(e.Record.State), Result: result,
|
|
|
|
|
EffectMayHaveApplied: e.Record.EffectMayHaveApplied,
|
|
|
|
|
SafeToResubmit: e.Record.SafeToResubmit,
|
|
|
|
|
}
|
|
|
|
|
if e.Record.Error != nil {
|
|
|
|
|
response.ErrorCode = e.Record.Error.Code
|
|
|
|
|
response.ErrorMessage = e.Record.Error.Message
|
|
|
|
|
if len(response.ErrorMessage) > 1024 {
|
|
|
|
|
response.ErrorMessage = response.ErrorMessage[:1024]
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if response.Validate() != nil {
|
|
|
|
|
return operation.InterruptedReceipt{}, false
|
|
|
|
|
}
|
|
|
|
|
return operation.InterruptedReceipt{
|
|
|
|
|
OperationID: response.OperationID, Fingerprint: response.Fingerprint,
|
|
|
|
|
Status: response.Status, Result: response.Result,
|
|
|
|
|
ErrorCode: response.ErrorCode, ErrorMessage: response.ErrorMessage,
|
|
|
|
|
EffectMayHaveApplied: response.EffectMayHaveApplied,
|
|
|
|
|
SafeToResubmit: response.SafeToResubmit,
|
|
|
|
|
}, true
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type SubprocessExecutor struct {
|
|
|
|
|
executable string
|
|
|
|
|
configPath string
|
|
|
|
|
stateDir string
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func NewSubprocessExecutor(executable, configPath, stateDir string) (*SubprocessExecutor, error) {
|
|
|
|
|
for name, value := range map[string]string{"executable": executable, "config path": configPath, "state directory": stateDir} {
|
|
|
|
|
if strings.TrimSpace(value) == "" || !filepath.IsAbs(value) || filepath.Clean(value) != value {
|
|
|
|
|
return nil, fmt.Errorf("worker %s must be a clean absolute path", name)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if err := os.MkdirAll(stateDir, 0o700); err != nil {
|
|
|
|
|
return nil, fmt.Errorf("create worker state directory: %w", err)
|
|
|
|
|
}
|
|
|
|
|
if err := os.Chmod(stateDir, 0o700); err != nil {
|
|
|
|
|
return nil, fmt.Errorf("secure worker state directory: %w", err)
|
|
|
|
|
}
|
|
|
|
|
return &SubprocessExecutor{executable: executable, configPath: configPath, stateDir: stateDir}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e *SubprocessExecutor) Execute(ctx context.Context, request Request) (json.RawMessage, error) {
|
|
|
|
|
if err := request.Validate(); err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
operationDir := filepath.Join(e.stateDir, request.OperationID)
|
|
|
|
|
receiptPath := filepath.Join(operationDir, workerReceiptName)
|
|
|
|
|
journalPath := filepath.Join(operationDir, workerJournalName)
|
|
|
|
|
if err := os.Mkdir(operationDir, 0o700); err != nil {
|
|
|
|
|
if !errors.Is(err, os.ErrExist) {
|
|
|
|
|
return nil, fmt.Errorf("create operation worker directory: %w", err)
|
|
|
|
|
}
|
|
|
|
|
response, readErr := ReadReceipt(receiptPath)
|
|
|
|
|
if readErr != nil || response.Fingerprint != request.Fingerprint {
|
|
|
|
|
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
|
|
|
|
|
}
|
|
|
|
|
return Result(response)
|
|
|
|
|
}
|
|
|
|
|
payload, err := json.Marshal(request)
|
|
|
|
|
if err != nil || len(payload) > MaxEnvelopeBytes {
|
|
|
|
|
_ = os.RemoveAll(operationDir)
|
|
|
|
|
return nil, errors.New("encode bounded worker request")
|
|
|
|
|
}
|
|
|
|
|
identity, err := requestIdentityFromRequest(request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
_ = os.RemoveAll(operationDir)
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
if err := writeRequestIdentity(filepath.Join(operationDir, workerIdentityName), identity); err != nil {
|
|
|
|
|
_ = os.RemoveAll(operationDir)
|
|
|
|
|
return nil, fmt.Errorf("persist worker recovery identity: %w", err)
|
|
|
|
|
}
|
|
|
|
|
command := exec.Command(e.executable,
|
|
|
|
|
"--config", e.configPath,
|
|
|
|
|
"--operation-worker",
|
|
|
|
|
"--worker-journal", journalPath,
|
|
|
|
|
"--worker-receipt", receiptPath,
|
|
|
|
|
)
|
|
|
|
|
configureProcess(command)
|
|
|
|
|
command.Stdin = bytes.NewReader(payload)
|
|
|
|
|
stdout := &limitedBuffer{limit: MaxWorkerResponse}
|
|
|
|
|
stderr := &limitedBuffer{limit: 32 << 10}
|
|
|
|
|
command.Stdout = stdout
|
|
|
|
|
command.Stderr = stderr
|
|
|
|
|
if err := command.Start(); err != nil {
|
|
|
|
|
_ = os.RemoveAll(operationDir)
|
|
|
|
|
return nil, &Error{
|
|
|
|
|
Code: "worker_start_failed", Message: "isolated mutation worker could not be started",
|
|
|
|
|
EffectMayHaveApplied: false, ResubmitAllowed: true,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
wait := make(chan error, 1)
|
|
|
|
|
go func() { wait <- command.Wait() }()
|
|
|
|
|
select {
|
|
|
|
|
case waitErr := <-wait:
|
|
|
|
|
response, readErr := ReadReceipt(receiptPath)
|
|
|
|
|
if readErr != nil || response.OperationID != request.OperationID || response.Fingerprint != request.Fingerprint {
|
|
|
|
|
if waitErr != nil {
|
|
|
|
|
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
|
|
|
|
|
}
|
|
|
|
|
return nil, errors.New("isolated mutation worker returned no valid receipt")
|
|
|
|
|
}
|
|
|
|
|
return Result(response)
|
|
|
|
|
case <-ctx.Done():
|
|
|
|
|
reaped := false
|
|
|
|
|
terminateProcessGroup(command)
|
|
|
|
|
select {
|
|
|
|
|
case <-wait:
|
|
|
|
|
reaped = true
|
|
|
|
|
case <-time.After(terminationGrace):
|
|
|
|
|
killProcessGroup(command)
|
|
|
|
|
select {
|
|
|
|
|
case <-wait:
|
|
|
|
|
reaped = true
|
|
|
|
|
case <-time.After(killWait):
|
|
|
|
|
return nil, &Error{
|
|
|
|
|
Code: "worker_reap_unconfirmed",
|
|
|
|
|
Message: "isolated mutation worker did not exit after forced termination",
|
|
|
|
|
EffectMayHaveApplied: true, ResubmitAllowed: false, ExecutionFenced: true,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if reaped {
|
|
|
|
|
if response, readErr := ReadReceipt(receiptPath); readErr == nil &&
|
|
|
|
|
response.OperationID == request.OperationID && response.Fingerprint == request.Fingerprint {
|
|
|
|
|
return Result(response)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return Result(UnconfirmedResponse(request.OperationID, request.Fingerprint))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e *SubprocessExecutor) PendingReceipts() ([]Response, error) {
|
|
|
|
|
entries, err := os.ReadDir(e.stateDir)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
results := make([]Response, 0, len(entries))
|
|
|
|
|
for _, entry := range entries {
|
|
|
|
|
if !entry.IsDir() || !operationIDPattern.MatchString(entry.Name()) {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
response, found, readErr := readTrustedReceipt(
|
|
|
|
|
filepath.Join(e.stateDir, entry.Name(), workerReceiptName), entry.Name(),
|
|
|
|
|
)
|
|
|
|
|
if readErr != nil {
|
|
|
|
|
return nil, readErr
|
|
|
|
|
}
|
|
|
|
|
if found {
|
|
|
|
|
results = append(results, response)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return results, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// PendingJournalTerminals finds the crash window where a worker committed its
|
|
|
|
|
// terminal Journal transaction but did not yet publish completion.json. An
|
|
|
|
|
// invalid/mismatched artifact is skipped and retained for manual inspection;
|
|
|
|
|
// transient filesystem or database-lock errors are returned so startup can
|
|
|
|
|
// stop before irreversibly terminalizing the main Journal as unconfirmed.
|
|
|
|
|
func (e *SubprocessExecutor) PendingJournalTerminals() ([]TerminalEvidence, error) {
|
|
|
|
|
entries, err := os.ReadDir(e.stateDir)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
results := make([]TerminalEvidence, 0, len(entries))
|
|
|
|
|
for _, entry := range entries {
|
|
|
|
|
if !entry.IsDir() || !operationIDPattern.MatchString(entry.Name()) {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
operationDir := filepath.Join(e.stateDir, entry.Name())
|
|
|
|
|
trusted, trustErr := trustedWorkerPath(operationDir, true)
|
|
|
|
|
if trustErr != nil {
|
|
|
|
|
return nil, trustErr
|
|
|
|
|
}
|
|
|
|
|
if !trusted {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
_, receiptFound, receiptErr := readTrustedReceipt(
|
|
|
|
|
filepath.Join(operationDir, workerReceiptName), entry.Name(),
|
|
|
|
|
)
|
|
|
|
|
if receiptErr != nil {
|
|
|
|
|
return nil, receiptErr
|
|
|
|
|
}
|
|
|
|
|
if receiptFound {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
journalPath := filepath.Join(operationDir, workerJournalName)
|
|
|
|
|
trusted, trustErr = trustedWorkerPath(journalPath, false)
|
|
|
|
|
if trustErr != nil {
|
|
|
|
|
if errors.Is(trustErr, os.ErrNotExist) {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
return nil, trustErr
|
|
|
|
|
}
|
|
|
|
|
if !trusted {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var identity *RequestIdentity
|
|
|
|
|
identityPath := filepath.Join(operationDir, workerIdentityName)
|
|
|
|
|
trustedIdentity, identityTrustErr := trustedWorkerPath(identityPath, false)
|
|
|
|
|
switch {
|
|
|
|
|
case identityTrustErr == nil && trustedIdentity:
|
|
|
|
|
read, readErr := readRequestIdentity(identityPath)
|
|
|
|
|
if readErr != nil {
|
|
|
|
|
if errors.Is(readErr, errInvalidWorkerIdentity) {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
return nil, readErr
|
|
|
|
|
}
|
|
|
|
|
if read.OperationID != entry.Name() {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
identity = &read
|
|
|
|
|
case identityTrustErr == nil:
|
|
|
|
|
continue
|
|
|
|
|
case errors.Is(identityTrustErr, os.ErrNotExist):
|
|
|
|
|
// Pre-upgrade workers did not persist request-identity.json. Their
|
|
|
|
|
// exact operation/action/resource/fingerprint still has to match the
|
|
|
|
|
// main Journal before it can be adopted.
|
|
|
|
|
case identityTrustErr != nil:
|
|
|
|
|
return nil, identityTrustErr
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
record, readErr := operation.ReadTerminalRecord(journalPath, entry.Name())
|
|
|
|
|
if readErr != nil {
|
|
|
|
|
if errors.Is(readErr, operation.ErrTerminalEvidenceInvalid) ||
|
|
|
|
|
errors.Is(readErr, operation.ErrTerminalEvidencePending) || errors.Is(readErr, os.ErrNotExist) {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
return nil, readErr
|
|
|
|
|
}
|
|
|
|
|
results = append(results, TerminalEvidence{
|
|
|
|
|
Record: record, Identity: identity, Legacy: identity == nil, trustedFilesystem: true,
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
return results, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e *SubprocessExecutor) PendingOperationIDs() ([]string, error) {
|
|
|
|
|
entries, err := os.ReadDir(e.stateDir)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
operationIDs := make([]string, 0, len(entries))
|
|
|
|
|
for _, entry := range entries {
|
|
|
|
|
if entry.IsDir() && operationIDPattern.MatchString(entry.Name()) {
|
|
|
|
|
operationIDs = append(operationIDs, entry.Name())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return operationIDs, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e *SubprocessExecutor) Cleanup(operationID string) error {
|
|
|
|
|
if !operationIDPattern.MatchString(operationID) {
|
|
|
|
|
return errors.New("invalid operation ID")
|
|
|
|
|
}
|
|
|
|
|
return os.RemoveAll(filepath.Join(e.stateDir, operationID))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// CleanupCompleted removes helper state only when its immutable receipt still
|
|
|
|
|
// matches the main Journal request fingerprint. This prevents completion
|
|
|
|
|
// delivery of a synthesized result_unconfirmed record from deleting corrupt or
|
|
|
|
|
// mismatched child evidence that was never adopted.
|
|
|
|
|
func (e *SubprocessExecutor) CleanupCompleted(operationID, fingerprint string) error {
|
|
|
|
|
if !operationIDPattern.MatchString(operationID) || !fingerprintPattern.MatchString(fingerprint) {
|
|
|
|
|
return errors.New("invalid completed worker identity")
|
|
|
|
|
}
|
|
|
|
|
response, err := ReadReceipt(filepath.Join(e.stateDir, operationID, workerReceiptName))
|
|
|
|
|
if err != nil || response.OperationID != operationID || response.Fingerprint != fingerprint {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
return e.Cleanup(operationID)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func WriteReceipt(path string, response Response) error {
|
|
|
|
|
if err := response.Validate(); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
encoded, err := json.Marshal(response)
|
|
|
|
|
if err != nil || len(encoded) > MaxWorkerResponse {
|
|
|
|
|
return errors.New("encode bounded worker receipt")
|
|
|
|
|
}
|
|
|
|
|
directory := filepath.Dir(path)
|
|
|
|
|
if err := os.MkdirAll(directory, 0o700); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
temporary, err := os.CreateTemp(directory, ".completion-*.tmp")
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
temporaryName := temporary.Name()
|
|
|
|
|
remove := true
|
|
|
|
|
defer func() {
|
|
|
|
|
if remove {
|
|
|
|
|
_ = os.Remove(temporaryName)
|
|
|
|
|
}
|
|
|
|
|
}()
|
|
|
|
|
if err := temporary.Chmod(0o600); err != nil {
|
|
|
|
|
_ = temporary.Close()
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if _, err := temporary.Write(encoded); err != nil {
|
|
|
|
|
_ = temporary.Close()
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if err := temporary.Sync(); err != nil {
|
|
|
|
|
_ = temporary.Close()
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if err := temporary.Close(); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
// Link is an atomic no-replace publication on the same filesystem. A
|
|
|
|
|
// receipt that already exists is immutable, even if a buggy caller invokes
|
|
|
|
|
// the writer twice.
|
|
|
|
|
if err := os.Link(temporaryName, path); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if err := os.Remove(temporaryName); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
remove = false
|
|
|
|
|
dir, err := os.Open(directory)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
err = dir.Sync()
|
|
|
|
|
_ = dir.Close()
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func writeRequestIdentity(path string, identity RequestIdentity) error {
|
|
|
|
|
if err := identity.Validate(); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
encoded, err := json.Marshal(identity)
|
|
|
|
|
if err != nil || len(encoded) > maxIdentityBytes {
|
|
|
|
|
return errInvalidWorkerIdentity
|
|
|
|
|
}
|
|
|
|
|
directory := filepath.Dir(path)
|
|
|
|
|
temporary, err := os.CreateTemp(directory, ".identity-*.tmp")
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
temporaryName := temporary.Name()
|
|
|
|
|
remove := true
|
|
|
|
|
defer func() {
|
|
|
|
|
if remove {
|
|
|
|
|
_ = os.Remove(temporaryName)
|
|
|
|
|
}
|
|
|
|
|
}()
|
|
|
|
|
if err := temporary.Chmod(0o600); err != nil {
|
|
|
|
|
_ = temporary.Close()
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if _, err := temporary.Write(encoded); err != nil {
|
|
|
|
|
_ = temporary.Close()
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if err := temporary.Sync(); err != nil {
|
|
|
|
|
_ = temporary.Close()
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if err := temporary.Close(); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if err := os.Link(temporaryName, path); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if err := os.Remove(temporaryName); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
remove = false
|
|
|
|
|
dir, err := os.Open(directory)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
err = dir.Sync()
|
|
|
|
|
_ = dir.Close()
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func readRequestIdentity(path string) (RequestIdentity, error) {
|
|
|
|
|
file, err := os.Open(path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return RequestIdentity{}, err
|
|
|
|
|
}
|
|
|
|
|
defer file.Close()
|
|
|
|
|
encoded, err := io.ReadAll(io.LimitReader(file, maxIdentityBytes+1))
|
|
|
|
|
if err != nil {
|
|
|
|
|
return RequestIdentity{}, err
|
|
|
|
|
}
|
|
|
|
|
if len(encoded) > maxIdentityBytes {
|
|
|
|
|
return RequestIdentity{}, errInvalidWorkerIdentity
|
|
|
|
|
}
|
|
|
|
|
decoder := json.NewDecoder(bytes.NewReader(encoded))
|
|
|
|
|
decoder.DisallowUnknownFields()
|
|
|
|
|
var identity RequestIdentity
|
|
|
|
|
if err := decoder.Decode(&identity); err != nil {
|
|
|
|
|
return RequestIdentity{}, fmt.Errorf("%w: %v", errInvalidWorkerIdentity, err)
|
|
|
|
|
}
|
|
|
|
|
var trailing any
|
|
|
|
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
|
|
|
|
return RequestIdentity{}, errInvalidWorkerIdentity
|
|
|
|
|
}
|
|
|
|
|
if err := identity.Validate(); err != nil {
|
|
|
|
|
return RequestIdentity{}, fmt.Errorf("%w: %v", errInvalidWorkerIdentity, err)
|
|
|
|
|
}
|
|
|
|
|
return identity, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func trustedWorkerPath(path string, directory bool) (bool, error) {
|
|
|
|
|
info, err := os.Lstat(path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return false, err
|
|
|
|
|
}
|
|
|
|
|
if info.Mode()&os.ModeSymlink != 0 || directory != info.IsDir() ||
|
|
|
|
|
(!directory && !info.Mode().IsRegular()) || info.Mode().Perm()&0o022 != 0 {
|
|
|
|
|
return false, nil
|
|
|
|
|
}
|
|
|
|
|
stat, ok := info.Sys().(*syscall.Stat_t)
|
|
|
|
|
if !ok || int(stat.Uid) != os.Geteuid() {
|
|
|
|
|
return false, nil
|
|
|
|
|
}
|
|
|
|
|
return true, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func readTrustedReceipt(path, operationID string) (Response, bool, error) {
|
|
|
|
|
trusted, err := trustedWorkerPath(path, false)
|
|
|
|
|
if err != nil {
|
|
|
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
|
|
|
return Response{}, false, nil
|
|
|
|
|
}
|
|
|
|
|
return Response{}, false, err
|
|
|
|
|
}
|
|
|
|
|
if !trusted {
|
|
|
|
|
return Response{}, false, nil
|
|
|
|
|
}
|
|
|
|
|
response, err := ReadReceipt(path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
if errors.Is(err, errInvalidWorkerReceipt) {
|
|
|
|
|
return Response{}, false, nil
|
|
|
|
|
}
|
|
|
|
|
return Response{}, false, err
|
|
|
|
|
}
|
|
|
|
|
if response.OperationID != operationID {
|
|
|
|
|
return Response{}, false, nil
|
|
|
|
|
}
|
|
|
|
|
return response, true, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func ReadReceipt(path string) (Response, error) {
|
|
|
|
|
file, err := os.Open(path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return Response{}, err
|
|
|
|
|
}
|
|
|
|
|
defer file.Close()
|
|
|
|
|
encoded, err := io.ReadAll(io.LimitReader(file, MaxWorkerResponse+1))
|
|
|
|
|
if err != nil {
|
|
|
|
|
return Response{}, err
|
|
|
|
|
}
|
|
|
|
|
if len(encoded) > MaxWorkerResponse {
|
|
|
|
|
return Response{}, errInvalidWorkerReceipt
|
|
|
|
|
}
|
|
|
|
|
decoder := json.NewDecoder(bytes.NewReader(encoded))
|
|
|
|
|
decoder.DisallowUnknownFields()
|
|
|
|
|
var response Response
|
|
|
|
|
if err := decoder.Decode(&response); err != nil {
|
|
|
|
|
return Response{}, fmt.Errorf("%w: %v", errInvalidWorkerReceipt, err)
|
|
|
|
|
}
|
|
|
|
|
var trailing any
|
|
|
|
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
|
|
|
|
return Response{}, errInvalidWorkerReceipt
|
|
|
|
|
}
|
|
|
|
|
if err := response.Validate(); err != nil {
|
|
|
|
|
return Response{}, fmt.Errorf("%w: %v", errInvalidWorkerReceipt, err)
|
|
|
|
|
}
|
|
|
|
|
return response, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type limitedBuffer struct {
|
|
|
|
|
mu sync.Mutex
|
|
|
|
|
buffer bytes.Buffer
|
|
|
|
|
limit int
|
|
|
|
|
overflow bool
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (b *limitedBuffer) Write(value []byte) (int, error) {
|
|
|
|
|
b.mu.Lock()
|
|
|
|
|
defer b.mu.Unlock()
|
|
|
|
|
remaining := b.limit - b.buffer.Len()
|
|
|
|
|
if remaining <= 0 {
|
|
|
|
|
b.overflow = true
|
|
|
|
|
return len(value), nil
|
|
|
|
|
}
|
|
|
|
|
if len(value) > remaining {
|
|
|
|
|
_, _ = b.buffer.Write(value[:remaining])
|
|
|
|
|
b.overflow = true
|
|
|
|
|
return len(value), nil
|
|
|
|
|
}
|
|
|
|
|
_, _ = b.buffer.Write(value)
|
|
|
|
|
return len(value), nil
|
|
|
|
|
}
|