You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/cmd/virt-agent/main.go

422 lines
15 KiB
Go

package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"git.luoli.live/root/virt/agent/api"
"git.luoli.live/root/virt/agent/auth"
"git.luoli.live/root/virt/agent/config"
consolebroker "git.luoli.live/root/virt/agent/console"
"git.luoli.live/root/virt/agent/control"
"git.luoli.live/root/virt/agent/hypervisor"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/worker"
)
func main() {
configPath := flag.String("config", "/etc/virt-agent/config.json", "path to agent JSON configuration")
operationWorker := flag.Bool("operation-worker", false, "run one internal isolated mutation worker")
workerJournal := flag.String("worker-journal", "", "internal isolated worker journal")
workerReceipt := flag.String("worker-receipt", "", "internal isolated worker receipt")
flag.Parse()
if *operationWorker {
if err := runOperationWorker(*configPath, *workerJournal, *workerReceipt); err != nil {
log.Printf("isolated mutation worker failed: %v", err)
os.Exit(1)
}
return
}
if *workerJournal != "" || *workerReceipt != "" {
log.Fatal("worker paths require --operation-worker")
}
cfg, err := config.Load(*configPath)
if err != nil {
log.Fatalf("load configuration: %v", err)
}
journal, err := operation.OpenWithHeavyIOConcurrency(cfg.JournalPath, cfg.HeavyIOConcurrency)
if err != nil {
log.Fatalf("open durable operation journal: %v", err)
}
executable, err := os.Executable()
if err != nil {
log.Fatalf("resolve Agent executable: %v", err)
}
executable, err = filepath.Abs(executable)
if err != nil {
log.Fatalf("resolve absolute Agent executable: %v", err)
}
absoluteConfig, err := filepath.Abs(*configPath)
if err != nil {
log.Fatalf("resolve absolute configuration path: %v", err)
}
executor, err := worker.NewSubprocessExecutor(executable, filepath.Clean(absoluteConfig), cfg.JournalPath+".workers")
if err != nil {
log.Fatalf("initialize isolated mutation executor: %v", err)
}
pendingReceipts, err := executor.PendingReceipts()
if err != nil {
log.Fatalf("read isolated mutation receipts: %v", err)
}
interrupted := make([]operation.InterruptedReceipt, 0, len(pendingReceipts))
for _, receipt := range pendingReceipts {
interrupted = append(interrupted, operation.InterruptedReceipt{
OperationID: receipt.OperationID, Fingerprint: receipt.Fingerprint,
Status: receipt.Status, Result: receipt.Result,
ErrorCode: receipt.ErrorCode, ErrorMessage: receipt.ErrorMessage,
EffectMayHaveApplied: receipt.EffectMayHaveApplied,
SafeToResubmit: receipt.SafeToResubmit,
})
}
if _, err := journal.FinalizeInterruptedOperations(interrupted); err != nil {
log.Fatalf("finalize interrupted isolated mutations: %v", err)
}
pendingCompletions, err := journal.PendingCompletionOperationIDs()
if err != nil {
log.Fatalf("read pending completion deliveries: %v", err)
}
pendingOperationIDs, err := executor.PendingOperationIDs()
if err != nil {
log.Fatalf("list isolated mutation state: %v", err)
}
for _, operationID := range pendingOperationIDs {
if _, retain := pendingCompletions[operationID]; retain {
continue
}
if err := executor.Cleanup(operationID); err != nil {
log.Fatalf("clean isolated mutation state %s: %v", operationID, err)
}
}
authenticator, err := auth.NewWithNonceStore([]byte(cfg.AgentSecret), cfg.ClockSkew(), cfg.NonceTTL(), journal)
if err != nil {
log.Fatalf("initialize request authentication: %v", err)
}
local := hypervisor.LocalConfig{
StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...),
StorageConfigurationLockPath: cfg.JournalPath + ".storage-config.lock",
FirmwareLoader: cfg.FirmwareLoader,
FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate, NVRAMDirectory: cfg.NVRAMDirectory,
TrafficStatePath: cfg.JournalPath + ".traffic.json",
GuestAgentTimeoutSeconds: int(cfg.GuestAgentTimeoutSec),
}
hv, err := hypervisor.OpenLibvirt(cfg.LibvirtURI, local)
if err != nil {
log.Fatalf("connect to libvirt: %v", err)
}
console, err := consolebroker.NewBroker(60 * time.Second)
if err != nil {
log.Fatalf("initialize console broker: %v", err)
}
handler, err := api.NewServer(authenticator, journal, hv, console, cfg.MaxBodyBytes, cfg.OperationTimeout())
if err != nil {
log.Fatalf("initialize Agent request dispatcher: %v", err)
}
if err := handler.SetMutationExecutor(executor); err != nil {
log.Fatalf("enable isolated mutation execution: %v", err)
}
client, err := control.NewClient(
cfg.PanelURL, cfg.AgentID, []byte(cfg.AgentSecret), handler, console, cfg.MaxBodyBytes, cfg.OperationTimeout(),
&workerCompletionOutbox{journal: journal, cleaner: executor},
)
if err != nil {
log.Fatalf("initialize outbound control channel: %v", err)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
var runErr error
if ctx.Err() == nil {
log.Printf("virt host agent %s connecting to panel as Agent %s", api.Version, cfg.AgentID)
runErr = runControlUntilStopped(ctx, journal.BackgroundFailures(), client.Run)
}
stop()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := handler.Shutdown(shutdownCtx); err != nil {
log.Printf("Agent handlers did not drain safely: %v", err)
os.Exit(1)
}
if err := journal.Wait(shutdownCtx); err != nil {
log.Printf("Agent operation workers did not drain safely: %v", err)
os.Exit(1)
}
if err := hv.Close(); err != nil {
log.Printf("close libvirt connection: %v", err)
}
if err := journal.Close(); err != nil {
log.Printf("close operation journal: %v", err)
}
if runErr != nil {
log.Fatalf("run outbound control channel: %v", runErr)
}
}
func runControlUntilStopped(ctx context.Context, backgroundFailures <-chan error,
run func(context.Context) error) error {
runCtx, cancelRun := context.WithCancel(ctx)
defer cancelRun()
done := make(chan error, 1)
go func() {
done <- run(runCtx)
}()
select {
case err := <-done:
return err
case persistErr := <-backgroundFailures:
log.Printf("durable background operation finalization failed; stopping Agent: %v", persistErr)
cancelRun()
select {
case <-done:
case <-time.After(5 * time.Second):
log.Printf("outbound control channel did not stop within the finalization failure grace period")
}
return fmt.Errorf("durable background operation finalization failed: %w", persistErr)
}
}
type workerStateCleaner interface {
Cleanup(string) error
}
// workerCompletionOutbox couples receipt delivery lifecycle to disposable
// helper state. The main Journal remains the permanent tombstone; a helper's
// journal and receipt survive restarts until the panel ACKs the immutable
// completion, or until the bounded delivery window expires.
type workerCompletionOutbox struct {
journal *operation.Journal
cleaner workerStateCleaner
}
func (o *workerCompletionOutbox) ClaimPendingCompletions(limit int, excluded []string) ([]operation.CompletionReceipt, error) {
claimed, expired, err := o.journal.ClaimPendingCompletionsWithExpiredExcluding(limit, excluded)
if err != nil {
return nil, err
}
for _, operationID := range expired {
log.Printf("completion outbox exhausted operation_id=%s", operationID)
if err := o.cleaner.Cleanup(operationID); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("clean expired worker state %s: %w", operationID, err)
}
}
return claimed, nil
}
func (o *workerCompletionOutbox) AcknowledgeCompletion(eventID string) error {
operationID, err := o.journal.CompletionOperationID(eventID)
if err != nil {
return err
}
if operationID != "" {
if err := o.cleaner.Cleanup(operationID); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("clean acknowledged worker state %s: %w", operationID, err)
}
}
return o.journal.AcknowledgeCompletion(eventID)
}
const isolatedWorkerMaximumLifetime = 2*time.Hour + 30*time.Second
func runOperationWorker(configPath, journalPath, receiptPath string) error {
if !cleanAbsolutePath(journalPath) || !cleanAbsolutePath(receiptPath) ||
filepath.Base(journalPath) != "journal.db" || filepath.Base(receiptPath) != "completion.json" ||
filepath.Dir(journalPath) != filepath.Dir(receiptPath) {
return errors.New("invalid isolated worker paths")
}
raw, err := io.ReadAll(io.LimitReader(os.Stdin, worker.MaxEnvelopeBytes+1))
if err != nil || len(raw) > worker.MaxEnvelopeBytes {
return errors.New("read bounded isolated worker request")
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.DisallowUnknownFields()
var request worker.Request
if err := decoder.Decode(&request); err != nil {
return errors.New("decode isolated worker request")
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return errors.New("isolated worker request contains trailing data")
}
if err := request.Validate(); err != nil {
return err
}
if filepath.Base(filepath.Dir(journalPath)) != request.OperationID {
return errors.New("isolated worker directory does not match operation")
}
cfg, err := config.Load(configPath)
if err != nil {
return fmt.Errorf("load worker configuration: %w", err)
}
journal, err := operation.OpenWithHeavyIOConcurrency(journalPath, cfg.HeavyIOConcurrency)
if err != nil {
return fmt.Errorf("open worker journal: %w", err)
}
authenticator, err := auth.NewWithNonceStore([]byte(cfg.AgentSecret), cfg.ClockSkew(), cfg.NonceTTL(), journal)
if err != nil {
return fmt.Errorf("initialize worker authentication: %w", err)
}
local := hypervisor.LocalConfig{
StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...),
StorageConfigurationLockPath: cfg.JournalPath + ".storage-config.lock",
FirmwareLoader: cfg.FirmwareLoader, FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate,
NVRAMDirectory: cfg.NVRAMDirectory, TrafficStatePath: cfg.JournalPath + ".traffic.json",
GuestAgentTimeoutSeconds: int(cfg.GuestAgentTimeoutSec),
}
hv, err := hypervisor.OpenLibvirt(cfg.LibvirtURI, local)
if err != nil {
return fmt.Errorf("connect worker to libvirt: %w", err)
}
console, err := consolebroker.NewBroker(60 * time.Second)
if err != nil {
return fmt.Errorf("initialize worker console broker: %w", err)
}
handler, err := api.NewServer(authenticator, journal, hv, console, cfg.MaxBodyBytes, cfg.OperationTimeout())
if err != nil {
return fmt.Errorf("initialize worker dispatcher: %w", err)
}
parsed, err := url.ParseRequestURI(request.RequestTarget)
if err != nil {
return errors.New("parse isolated worker request target")
}
ctx, cancel := context.WithTimeout(context.Background(), isolatedWorkerMaximumLifetime)
defer cancel()
httpRequest := (&http.Request{
Method: request.Method, URL: parsed, RequestURI: request.RequestTarget,
Header: make(http.Header), Body: io.NopCloser(strings.NewReader(request.Body)),
ContentLength: int64(len(request.Body)), Host: "agent.worker",
}).WithContext(ctx)
responseCapture := newBoundedWorkerResponse(worker.MaxWorkerResponse)
handler.ServeTrustedWorkerRequest(responseCapture, httpRequest, []byte(request.Body), request.Actor)
response := waitForWorkerTerminal(ctx, journal, request, responseCapture)
if err := worker.WriteReceipt(receiptPath, response); err != nil {
return fmt.Errorf("persist isolated worker receipt: %w", err)
}
encoded, err := json.Marshal(response)
if err != nil {
return errors.New("encode isolated worker receipt")
}
if _, err := os.Stdout.Write(append(encoded, '\n')); err != nil {
return errors.New("write isolated worker response")
}
// Avoid any potentially blocking libvirt cleanup after the immutable
// receipt is durable. The operating system closes descriptors on exit.
os.Exit(0)
return nil
}
func cleanAbsolutePath(value string) bool {
return value != "" && filepath.IsAbs(value) && filepath.Clean(value) == value && !strings.ContainsRune(value, 0)
}
func waitForWorkerTerminal(ctx context.Context, journal *operation.Journal, request worker.Request,
capture *boundedWorkerResponse) worker.Response {
ticker := time.NewTicker(25 * time.Millisecond)
defer ticker.Stop()
for {
record, err := journal.Get(request.OperationID)
if err == nil && (record.State == operation.StateSucceeded || record.State == operation.StateFailed) {
response := worker.Response{
Version: worker.ProtocolVersion, OperationID: request.OperationID, Fingerprint: request.Fingerprint,
Status: string(record.State), Result: boundedWorkerResult(record.Result),
EffectMayHaveApplied: record.EffectMayHaveApplied, SafeToResubmit: record.SafeToResubmit,
}
if record.Error != nil {
response.ErrorCode = record.Error.Code
response.ErrorMessage = boundedWorkerText(record.Error.Message, 1024)
}
if response.Validate() == nil {
return response
}
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
}
if err != nil && !errors.Is(err, operation.ErrNotFound) {
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
}
if errors.Is(err, operation.ErrNotFound) && capture.status >= http.StatusBadRequest {
var envelope struct {
Error struct {
Code string `json:"code"`
Message string `json:"message"`
} `json:"error"`
}
if json.Unmarshal(capture.body.Bytes(), &envelope) == nil &&
envelope.Error.Code != "" && len(envelope.Error.Code) <= 64 {
response := worker.Response{
Version: worker.ProtocolVersion, OperationID: request.OperationID, Fingerprint: request.Fingerprint,
Status: "failed", ErrorCode: envelope.Error.Code,
ErrorMessage: boundedWorkerText(envelope.Error.Message, 1024),
EffectMayHaveApplied: false, SafeToResubmit: true,
}
if response.Validate() == nil {
return response
}
}
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
}
select {
case <-ctx.Done():
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
case <-ticker.C:
}
}
}
func boundedWorkerResult(value json.RawMessage) json.RawMessage {
if len(value) > worker.MaxWorkerResponse/2 {
return nil
}
return append(json.RawMessage(nil), value...)
}
func boundedWorkerText(value string, limit int) string {
if len(value) <= limit {
return value
}
return value[:limit]
}
type boundedWorkerResponse struct {
header http.Header
status int
body bytes.Buffer
limit int
overflow bool
}
func newBoundedWorkerResponse(limit int) *boundedWorkerResponse {
return &boundedWorkerResponse{header: make(http.Header), limit: limit}
}
func (w *boundedWorkerResponse) Header() http.Header { return w.header }
func (w *boundedWorkerResponse) WriteHeader(status int) {
if w.status == 0 {
w.status = status
}
}
func (w *boundedWorkerResponse) Write(value []byte) (int, error) {
if w.status == 0 {
w.status = http.StatusOK
}
if w.body.Len()+len(value) > w.limit {
w.overflow = true
return len(value), nil
}
return w.body.Write(value)
}