You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
616 lines
22 KiB
Go
616 lines
22 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/api"
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
"git.luoli.live/root/virt/agent/config"
|
|
consolebroker "git.luoli.live/root/virt/agent/console"
|
|
"git.luoli.live/root/virt/agent/control"
|
|
"git.luoli.live/root/virt/agent/hypervisor"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"git.luoli.live/root/virt/agent/peermigration"
|
|
"git.luoli.live/root/virt/agent/worker"
|
|
)
|
|
|
|
func main() {
|
|
configPath := flag.String("config", "/etc/virt-agent/config.json", "path to agent JSON configuration")
|
|
operationWorker := flag.Bool("operation-worker", false, "run one internal isolated mutation worker")
|
|
workerJournal := flag.String("worker-journal", "", "internal isolated worker journal")
|
|
workerReceipt := flag.String("worker-receipt", "", "internal isolated worker receipt")
|
|
publicGuardPreflight := flag.Bool("public-guard-preflight", false, "restore the cached public guard before libvirt starts")
|
|
flag.Parse()
|
|
if err := validateRuntimeMode(*publicGuardPreflight, *operationWorker, *workerJournal, *workerReceipt); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
if *publicGuardPreflight {
|
|
if err := runPublicGuardPreflight(*configPath); err != nil {
|
|
log.Printf("public guard preflight failed: %v", err)
|
|
os.Exit(1)
|
|
}
|
|
return
|
|
}
|
|
if *operationWorker {
|
|
if err := runOperationWorker(*configPath, *workerJournal, *workerReceipt); err != nil {
|
|
log.Printf("isolated mutation worker failed: %v", err)
|
|
os.Exit(1)
|
|
}
|
|
return
|
|
}
|
|
cfg, err := config.Load(*configPath)
|
|
if err != nil {
|
|
log.Fatalf("load configuration: %v", err)
|
|
}
|
|
journal, err := operation.OpenWithHeavyIOConcurrency(cfg.JournalPath, cfg.HeavyIOConcurrency)
|
|
if err != nil {
|
|
log.Fatalf("open durable operation journal: %v", err)
|
|
}
|
|
executable, err := os.Executable()
|
|
if err != nil {
|
|
log.Fatalf("resolve Agent executable: %v", err)
|
|
}
|
|
executable, err = filepath.Abs(executable)
|
|
if err != nil {
|
|
log.Fatalf("resolve absolute Agent executable: %v", err)
|
|
}
|
|
absoluteConfig, err := filepath.Abs(*configPath)
|
|
if err != nil {
|
|
log.Fatalf("resolve absolute configuration path: %v", err)
|
|
}
|
|
executor, err := worker.NewSubprocessExecutor(executable, filepath.Clean(absoluteConfig), cfg.JournalPath+".workers")
|
|
if err != nil {
|
|
log.Fatalf("initialize isolated mutation executor: %v", err)
|
|
}
|
|
if err := recoverInterruptedWorkerState(journal, executor); err != nil {
|
|
log.Fatalf("recover interrupted isolated mutations: %v", err)
|
|
}
|
|
authenticator, err := auth.NewWithNonceStore([]byte(cfg.AgentSecret), cfg.ClockSkew(), cfg.NonceTTL(), journal)
|
|
if err != nil {
|
|
log.Fatalf("initialize request authentication: %v", err)
|
|
}
|
|
peerRuntime, peerErr := startPeerMigration(cfg)
|
|
if peerErr != nil {
|
|
log.Printf("PeerMigration V3 unavailable; ordinary Agent operations remain enabled: %v", peerErr)
|
|
}
|
|
local := localConfig(cfg)
|
|
local.RestoreVPCState = true
|
|
if peerRuntime != nil {
|
|
local.PeerMigrationEndpoints = append([]string(nil), peerRuntime.endpoints...)
|
|
local.PeerMigrationIdentitySHA256 = peerRuntime.identitySHA256
|
|
}
|
|
hv, err := hypervisor.OpenLibvirt(cfg.LibvirtURI, local)
|
|
if err != nil {
|
|
log.Fatalf("connect to libvirt: %v", err)
|
|
}
|
|
observationRuntime, err := newNetworkObservationRuntime(hv)
|
|
if err != nil {
|
|
log.Fatalf("initialize network abuse observation: %v", err)
|
|
}
|
|
console, err := consolebroker.NewBroker(60 * time.Second)
|
|
if err != nil {
|
|
log.Fatalf("initialize console broker: %v", err)
|
|
}
|
|
handler, err := api.NewServer(authenticator, journal, hv, console, cfg.MaxBodyBytes, cfg.OperationTimeout())
|
|
if err != nil {
|
|
log.Fatalf("initialize Agent request dispatcher: %v", err)
|
|
}
|
|
if err := handler.SetMutationExecutor(executor); err != nil {
|
|
log.Fatalf("enable isolated mutation execution: %v", err)
|
|
}
|
|
if err := handler.SetNetworkObserver(observationRuntime.collector); err != nil {
|
|
log.Fatalf("publish network abuse observation: %v", err)
|
|
}
|
|
var peerListener *peermigration.Listener
|
|
if peerRuntime != nil {
|
|
peerListener = peerRuntime.listener
|
|
log.Printf("PeerMigration V3 listening on %s with %d automatically discovered endpoint(s)",
|
|
cfg.PeerMigrationListenAddress(), len(peerRuntime.endpoints))
|
|
}
|
|
client, err := control.NewClient(
|
|
cfg.PanelURL, cfg.AgentID, []byte(cfg.AgentSecret), handler, console, cfg.MaxBodyBytes, cfg.OperationTimeout(),
|
|
&workerCompletionOutbox{journal: journal, cleaner: executor},
|
|
)
|
|
if err != nil {
|
|
log.Fatalf("initialize outbound control channel: %v", err)
|
|
}
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
if err := observationRuntime.Start(ctx); err != nil {
|
|
log.Fatalf("start network abuse observation: %v", err)
|
|
}
|
|
var runErr error
|
|
if ctx.Err() == nil {
|
|
log.Printf("virt host agent %s connecting to panel as Agent %s", api.Version, cfg.AgentID)
|
|
var peerFailures <-chan error
|
|
if peerListener != nil {
|
|
peerFailures = peerListener.Errors()
|
|
}
|
|
runErr = runControlUntilStopped(
|
|
ctx, journal.BackgroundFailures(), client.Run, peerFailures,
|
|
)
|
|
}
|
|
stop()
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
if err := peerListener.Shutdown(shutdownCtx); err != nil {
|
|
log.Printf("PeerMigration listener did not stop safely: %v", err)
|
|
os.Exit(1)
|
|
}
|
|
if err := handler.Shutdown(shutdownCtx); err != nil {
|
|
log.Printf("Agent handlers did not drain safely: %v", err)
|
|
os.Exit(1)
|
|
}
|
|
if err := observationRuntime.Shutdown(shutdownCtx); err != nil {
|
|
log.Printf("network abuse observation did not stop safely: %v", err)
|
|
os.Exit(1)
|
|
}
|
|
if err := journal.Wait(shutdownCtx); err != nil {
|
|
log.Printf("Agent operation workers did not drain safely: %v", err)
|
|
os.Exit(1)
|
|
}
|
|
if err := hv.Close(); err != nil {
|
|
log.Printf("close libvirt connection: %v", err)
|
|
}
|
|
if err := journal.Close(); err != nil {
|
|
log.Printf("close operation journal: %v", err)
|
|
}
|
|
if runErr != nil {
|
|
log.Fatalf("run outbound control channel: %v", runErr)
|
|
}
|
|
}
|
|
|
|
type peerMigrationRuntime struct {
|
|
listener *peermigration.Listener
|
|
endpoints []string
|
|
identitySHA256 string
|
|
}
|
|
|
|
func startPeerMigration(cfg config.Config) (*peerMigrationRuntime, error) {
|
|
store, err := peermigration.NewStore(
|
|
cfg.PeerMigrationStatePath(), cfg.StoragePathRoots,
|
|
cfg.PeerMigrationStagingPath(),
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("initialize automatic peer migration store: %w", err)
|
|
}
|
|
identity, err := peermigration.LoadOrCreateIdentity(cfg.PeerMigrationStatePath(), cfg.AgentID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("initialize automatic peer migration identity: %w", err)
|
|
}
|
|
endpoints, err := peermigration.DiscoverEndpoints(cfg.PeerMigrationPort)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("discover peer migration endpoints: %w", err)
|
|
}
|
|
handler, err := peermigration.NewHandler(store)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("initialize peer migration handler: %w", err)
|
|
}
|
|
tlsConfig, err := identity.ServerTLSConfig()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("initialize automatic peer migration TLS: %w", err)
|
|
}
|
|
listener, err := peermigration.StartListener(cfg.PeerMigrationListenAddress(), handler, tlsConfig)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("start peer migration listener: %w", err)
|
|
}
|
|
return &peerMigrationRuntime{
|
|
listener: listener, endpoints: endpoints, identitySHA256: identity.Fingerprint(),
|
|
}, nil
|
|
}
|
|
|
|
func localConfig(cfg config.Config) hypervisor.LocalConfig {
|
|
return hypervisor.LocalConfig{
|
|
StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...),
|
|
StorageConfigurationLockPath: cfg.JournalPath + ".storage-config.lock",
|
|
FirmwareLoader: cfg.FirmwareLoader,
|
|
FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate,
|
|
NVRAMDirectory: cfg.NVRAMDirectory,
|
|
TrafficStatePath: cfg.JournalPath + ".traffic.json",
|
|
GuestAgentTimeoutSeconds: int(cfg.GuestAgentTimeoutSec),
|
|
PeerMigrationStatePath: cfg.PeerMigrationStatePath(),
|
|
PeerMigrationStagingPath: cfg.PeerMigrationStagingPath(),
|
|
VPCStatePath: cfg.JournalPath + ".vpc-l3-v1.json",
|
|
PublicGuardLockPath: cfg.PublicGuardLockPath(),
|
|
PublicGuardStatePath: cfg.PublicGuardStatePath(),
|
|
}
|
|
}
|
|
|
|
func runControlUntilStopped(ctx context.Context, backgroundFailures <-chan error,
|
|
run func(context.Context) error, optionalFailures ...<-chan error) error {
|
|
var peerFailures <-chan error
|
|
if len(optionalFailures) > 0 {
|
|
peerFailures = optionalFailures[0]
|
|
}
|
|
runCtx, cancelRun := context.WithCancel(ctx)
|
|
defer cancelRun()
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- run(runCtx)
|
|
}()
|
|
select {
|
|
case err := <-done:
|
|
return err
|
|
case persistErr := <-backgroundFailures:
|
|
log.Printf("durable background operation finalization failed; stopping Agent: %v", persistErr)
|
|
cancelRun()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(5 * time.Second):
|
|
log.Printf("outbound control channel did not stop within the finalization failure grace period")
|
|
}
|
|
return fmt.Errorf("durable background operation finalization failed: %w", persistErr)
|
|
case peerErr, open := <-peerFailures:
|
|
if !open || peerErr == nil {
|
|
peerErr = errors.New("listener stopped unexpectedly")
|
|
}
|
|
cancelRun()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(5 * time.Second):
|
|
log.Printf("outbound control channel did not stop within the PeerMigration listener failure grace period")
|
|
}
|
|
return fmt.Errorf("PeerMigration listener failed: %w", peerErr)
|
|
}
|
|
}
|
|
|
|
type workerStateCleaner interface {
|
|
Cleanup(string) error
|
|
}
|
|
|
|
type completedWorkerStateCleaner interface {
|
|
CleanupCompleted(string, string) error
|
|
}
|
|
|
|
type recoveredWorkerState interface {
|
|
workerStateCleaner
|
|
PendingReceipts() ([]worker.Response, error)
|
|
PendingJournalTerminals() ([]worker.TerminalEvidence, error)
|
|
}
|
|
|
|
func recoverInterruptedWorkerState(journal *operation.Journal, state recoveredWorkerState) error {
|
|
// Inspect child Journals first. A still-running orphan worker keeps its Bolt
|
|
// file locked, which stops startup before main-Journal terminalization. If a
|
|
// receipt already exists the scanner skips that Journal and the receipt scan
|
|
// immediately below imports it, closing the publication race.
|
|
terminalEvidence, err := state.PendingJournalTerminals()
|
|
if err != nil {
|
|
// Do not irreversibly turn the main dispatching row into
|
|
// result_unconfirmed while terminal child evidence is temporarily
|
|
// unreadable (for example, a surviving worker still holds its DB).
|
|
return fmt.Errorf("inspect isolated mutation journals: %w", err)
|
|
}
|
|
pendingReceipts, err := state.PendingReceipts()
|
|
if err != nil {
|
|
return fmt.Errorf("read isolated mutation receipts: %w", err)
|
|
}
|
|
|
|
interrupted := make([]operation.InterruptedReceipt, 0, len(pendingReceipts)+len(terminalEvidence))
|
|
cleanup := make(map[string]struct{}, len(pendingReceipts)+len(terminalEvidence))
|
|
for _, receipt := range pendingReceipts {
|
|
record, getErr := journal.Get(receipt.OperationID)
|
|
if errors.Is(getErr, operation.ErrNotFound) || getErr == nil &&
|
|
(record.State != operation.StateDispatching || record.Fingerprint != receipt.Fingerprint) {
|
|
continue
|
|
}
|
|
if getErr != nil {
|
|
return fmt.Errorf("read main operation %s: %w", receipt.OperationID, getErr)
|
|
}
|
|
interrupted = append(interrupted, operation.InterruptedReceipt{
|
|
OperationID: receipt.OperationID, Fingerprint: receipt.Fingerprint,
|
|
Status: receipt.Status, Result: receipt.Result,
|
|
ErrorCode: receipt.ErrorCode, ErrorMessage: receipt.ErrorMessage,
|
|
EffectMayHaveApplied: receipt.EffectMayHaveApplied,
|
|
SafeToResubmit: receipt.SafeToResubmit,
|
|
})
|
|
cleanup[receipt.OperationID] = struct{}{}
|
|
}
|
|
for _, evidence := range terminalEvidence {
|
|
record, getErr := journal.Get(evidence.Record.OperationID)
|
|
if errors.Is(getErr, operation.ErrNotFound) {
|
|
continue
|
|
}
|
|
if getErr != nil {
|
|
return fmt.Errorf("read main operation %s: %w", evidence.Record.OperationID, getErr)
|
|
}
|
|
receipt, ok := evidence.InterruptedReceiptFor(record)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if _, duplicate := cleanup[receipt.OperationID]; duplicate {
|
|
continue
|
|
}
|
|
interrupted = append(interrupted, receipt)
|
|
cleanup[receipt.OperationID] = struct{}{}
|
|
}
|
|
if _, err := journal.FinalizeInterruptedOperations(interrupted); err != nil {
|
|
return fmt.Errorf("finalize interrupted isolated mutations: %w", err)
|
|
}
|
|
return cleanupRecoveredWorkerState(state, cleanup)
|
|
}
|
|
|
|
// cleanupRecoveredWorkerState removes only helper directories whose terminal
|
|
// evidence was successfully matched and imported. Corrupt, mismatched, or
|
|
// otherwise unadopted evidence remains available for inspection.
|
|
func cleanupRecoveredWorkerState(state workerStateCleaner, operationIDs map[string]struct{}) error {
|
|
for operationID := range operationIDs {
|
|
if err := state.Cleanup(operationID); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return fmt.Errorf("clean isolated mutation state %s: %w", operationID, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// workerCompletionOutbox keeps completion delivery in the main Journal until
|
|
// the panel ACKs it. Disposable helper state is cleaned independently after
|
|
// the durable receipt is claimed, after its ACK, or during the next startup.
|
|
type workerCompletionOutbox struct {
|
|
journal *operation.Journal
|
|
cleaner completedWorkerStateCleaner
|
|
}
|
|
|
|
func (o *workerCompletionOutbox) ClaimPendingCompletions(limit int, excluded []string) ([]operation.CompletionReceipt, error) {
|
|
claimed, _, err := o.journal.ClaimPendingCompletionsWithExpiredExcluding(limit, excluded)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, receipt := range claimed {
|
|
if cleanupErr := o.cleaner.CleanupCompleted(receipt.OperationID, receipt.RequestFingerprint); cleanupErr != nil && !errors.Is(cleanupErr, os.ErrNotExist) {
|
|
// The main Journal receipt is already durable. Disposable helper
|
|
// cleanup must neither suppress its delivery nor remove it from the
|
|
// ACK-driven outbox; startup will retry any failed cleanup.
|
|
log.Printf("clean delivered worker state operation_id=%s: %v", receipt.OperationID, cleanupErr)
|
|
}
|
|
}
|
|
return claimed, nil
|
|
}
|
|
|
|
func (o *workerCompletionOutbox) AcknowledgeCompletion(eventID string) error {
|
|
operationID, err := o.journal.CompletionOperationID(eventID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if operationID != "" {
|
|
record, getErr := o.journal.Get(operationID)
|
|
if getErr != nil {
|
|
log.Printf("read acknowledged worker identity operation_id=%s: %v", operationID, getErr)
|
|
} else if cleanupErr := o.cleaner.CleanupCompleted(operationID, record.Fingerprint); cleanupErr != nil && !errors.Is(cleanupErr, os.ErrNotExist) {
|
|
// An ACK is authoritative delivery state. A disposable helper
|
|
// cleanup failure must not keep the receipt in the delivery loop.
|
|
log.Printf("clean acknowledged worker state operation_id=%s: %v", operationID, cleanupErr)
|
|
}
|
|
}
|
|
return o.journal.AcknowledgeCompletion(eventID)
|
|
}
|
|
|
|
const (
|
|
isolatedWorkerMaximumLifetime = 2*time.Hour + 30*time.Second
|
|
isolatedBackupWorkerMaximumLifetime = 24*time.Hour + 30*time.Second
|
|
)
|
|
|
|
func isolatedWorkerLifetime(method, path string) time.Duration {
|
|
if method != http.MethodPost {
|
|
return isolatedWorkerMaximumLifetime
|
|
}
|
|
segments := strings.Split(strings.Trim(path, "/"), "/")
|
|
if len(segments) >= 4 && segments[0] == "v1" && segments[1] == "vms" &&
|
|
((len(segments) == 4 && (segments[3] == "backups" || segments[3] == "restore-backup" ||
|
|
segments[3] == "recover-backup")) ||
|
|
(len(segments) == 5 && segments[3] == "peer-migration" &&
|
|
(segments[4] == "prepare" || segments[4] == "receive"))) {
|
|
return isolatedBackupWorkerMaximumLifetime
|
|
}
|
|
return isolatedWorkerMaximumLifetime
|
|
}
|
|
|
|
func runOperationWorker(configPath, journalPath, receiptPath string) error {
|
|
if !cleanAbsolutePath(journalPath) || !cleanAbsolutePath(receiptPath) ||
|
|
filepath.Base(journalPath) != "journal.db" || filepath.Base(receiptPath) != "completion.json" ||
|
|
filepath.Dir(journalPath) != filepath.Dir(receiptPath) {
|
|
return errors.New("invalid isolated worker paths")
|
|
}
|
|
raw, err := io.ReadAll(io.LimitReader(os.Stdin, worker.MaxEnvelopeBytes+1))
|
|
if err != nil || len(raw) > worker.MaxEnvelopeBytes {
|
|
return errors.New("read bounded isolated worker request")
|
|
}
|
|
decoder := json.NewDecoder(bytes.NewReader(raw))
|
|
decoder.DisallowUnknownFields()
|
|
var request worker.Request
|
|
if err := decoder.Decode(&request); err != nil {
|
|
return errors.New("decode isolated worker request")
|
|
}
|
|
var trailing any
|
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
|
return errors.New("isolated worker request contains trailing data")
|
|
}
|
|
if err := request.Validate(); err != nil {
|
|
return err
|
|
}
|
|
if filepath.Base(filepath.Dir(journalPath)) != request.OperationID {
|
|
return errors.New("isolated worker directory does not match operation")
|
|
}
|
|
cfg, err := config.Load(configPath)
|
|
if err != nil {
|
|
return fmt.Errorf("load worker configuration: %w", err)
|
|
}
|
|
journal, err := operation.OpenWithHeavyIOConcurrency(journalPath, cfg.HeavyIOConcurrency)
|
|
if err != nil {
|
|
return fmt.Errorf("open worker journal: %w", err)
|
|
}
|
|
authenticator, err := auth.NewWithNonceStore([]byte(cfg.AgentSecret), cfg.ClockSkew(), cfg.NonceTTL(), journal)
|
|
if err != nil {
|
|
return fmt.Errorf("initialize worker authentication: %w", err)
|
|
}
|
|
local := localConfig(cfg)
|
|
hv, err := hypervisor.OpenLibvirt(cfg.LibvirtURI, local)
|
|
if err != nil {
|
|
return fmt.Errorf("connect worker to libvirt: %w", err)
|
|
}
|
|
console, err := consolebroker.NewBroker(60 * time.Second)
|
|
if err != nil {
|
|
return fmt.Errorf("initialize worker console broker: %w", err)
|
|
}
|
|
handler, err := api.NewServer(authenticator, journal, hv, console, cfg.MaxBodyBytes, cfg.OperationTimeout())
|
|
if err != nil {
|
|
return fmt.Errorf("initialize worker dispatcher: %w", err)
|
|
}
|
|
parsed, err := url.ParseRequestURI(request.RequestTarget)
|
|
if err != nil {
|
|
return errors.New("parse isolated worker request target")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(),
|
|
isolatedWorkerLifetime(request.Method, parsed.Path))
|
|
defer cancel()
|
|
httpRequest := (&http.Request{
|
|
Method: request.Method, URL: parsed, RequestURI: request.RequestTarget,
|
|
Header: make(http.Header), Body: io.NopCloser(strings.NewReader(request.Body)),
|
|
ContentLength: int64(len(request.Body)), Host: "agent.worker",
|
|
}).WithContext(ctx)
|
|
responseCapture := newBoundedWorkerResponse(worker.MaxWorkerResponse)
|
|
if request.SupersededOperation != nil {
|
|
prior := *request.SupersededOperation
|
|
prior.ExpectedState = request.SupersededExpectedState
|
|
handler.ServeTrustedWorkerRequest(
|
|
responseCapture, httpRequest, []byte(request.Body), request.Actor,
|
|
prior,
|
|
)
|
|
} else {
|
|
handler.ServeTrustedWorkerRequest(
|
|
responseCapture, httpRequest, []byte(request.Body), request.Actor,
|
|
)
|
|
}
|
|
|
|
response := waitForWorkerTerminal(ctx, journal, request, responseCapture)
|
|
if err := worker.WriteReceipt(receiptPath, response); err != nil {
|
|
return fmt.Errorf("persist isolated worker receipt: %w", err)
|
|
}
|
|
encoded, err := json.Marshal(response)
|
|
if err != nil {
|
|
return errors.New("encode isolated worker receipt")
|
|
}
|
|
if _, err := os.Stdout.Write(append(encoded, '\n')); err != nil {
|
|
return errors.New("write isolated worker response")
|
|
}
|
|
// Avoid any potentially blocking libvirt cleanup after the immutable
|
|
// receipt is durable. The operating system closes descriptors on exit.
|
|
os.Exit(0)
|
|
return nil
|
|
}
|
|
|
|
func cleanAbsolutePath(value string) bool {
|
|
return value != "" && filepath.IsAbs(value) && filepath.Clean(value) == value && !strings.ContainsRune(value, 0)
|
|
}
|
|
|
|
func waitForWorkerTerminal(ctx context.Context, journal *operation.Journal, request worker.Request,
|
|
capture *boundedWorkerResponse) worker.Response {
|
|
ticker := time.NewTicker(25 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
for {
|
|
record, err := journal.Get(request.OperationID)
|
|
if err == nil && (record.State == operation.StateSucceeded || record.State == operation.StateFailed) {
|
|
response := worker.Response{
|
|
Version: worker.ProtocolVersion, OperationID: request.OperationID, Fingerprint: request.Fingerprint,
|
|
Status: string(record.State), Result: boundedWorkerResult(record.Result),
|
|
EffectMayHaveApplied: record.EffectMayHaveApplied, SafeToResubmit: record.SafeToResubmit,
|
|
}
|
|
if record.Error != nil {
|
|
response.ErrorCode = record.Error.Code
|
|
response.ErrorMessage = boundedWorkerText(record.Error.Message, 1024)
|
|
}
|
|
if response.Validate() == nil {
|
|
return response
|
|
}
|
|
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
|
|
}
|
|
if err != nil && !errors.Is(err, operation.ErrNotFound) {
|
|
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
|
|
}
|
|
if errors.Is(err, operation.ErrNotFound) && capture.status >= http.StatusBadRequest {
|
|
var envelope struct {
|
|
Error struct {
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
} `json:"error"`
|
|
}
|
|
if json.Unmarshal(capture.body.Bytes(), &envelope) == nil &&
|
|
envelope.Error.Code != "" && len(envelope.Error.Code) <= 64 {
|
|
response := worker.Response{
|
|
Version: worker.ProtocolVersion, OperationID: request.OperationID, Fingerprint: request.Fingerprint,
|
|
Status: "failed", ErrorCode: envelope.Error.Code,
|
|
ErrorMessage: boundedWorkerText(envelope.Error.Message, 1024),
|
|
EffectMayHaveApplied: false, SafeToResubmit: true,
|
|
}
|
|
if response.Validate() == nil {
|
|
return response
|
|
}
|
|
}
|
|
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func boundedWorkerResult(value json.RawMessage) json.RawMessage {
|
|
if len(value) > worker.MaxWorkerResponse/2 {
|
|
return nil
|
|
}
|
|
return append(json.RawMessage(nil), value...)
|
|
}
|
|
|
|
func boundedWorkerText(value string, limit int) string {
|
|
if len(value) <= limit {
|
|
return value
|
|
}
|
|
return value[:limit]
|
|
}
|
|
|
|
type boundedWorkerResponse struct {
|
|
header http.Header
|
|
status int
|
|
body bytes.Buffer
|
|
limit int
|
|
overflow bool
|
|
}
|
|
|
|
func newBoundedWorkerResponse(limit int) *boundedWorkerResponse {
|
|
return &boundedWorkerResponse{header: make(http.Header), limit: limit}
|
|
}
|
|
|
|
func (w *boundedWorkerResponse) Header() http.Header { return w.header }
|
|
|
|
func (w *boundedWorkerResponse) WriteHeader(status int) {
|
|
if w.status == 0 {
|
|
w.status = status
|
|
}
|
|
}
|
|
|
|
func (w *boundedWorkerResponse) Write(value []byte) (int, error) {
|
|
if w.status == 0 {
|
|
w.status = http.StatusOK
|
|
}
|
|
if w.body.Len()+len(value) > w.limit {
|
|
w.overflow = true
|
|
return len(value), nil
|
|
}
|
|
return w.body.Write(value)
|
|
}
|