You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

616 lines
22 KiB
Go

package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"git.luoli.live/root/virt/agent/api"
"git.luoli.live/root/virt/agent/auth"
"git.luoli.live/root/virt/agent/config"
consolebroker "git.luoli.live/root/virt/agent/console"
"git.luoli.live/root/virt/agent/control"
"git.luoli.live/root/virt/agent/hypervisor"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/peermigration"
"git.luoli.live/root/virt/agent/worker"
)
func main() {
configPath := flag.String("config", "/etc/virt-agent/config.json", "path to agent JSON configuration")
operationWorker := flag.Bool("operation-worker", false, "run one internal isolated mutation worker")
workerJournal := flag.String("worker-journal", "", "internal isolated worker journal")
workerReceipt := flag.String("worker-receipt", "", "internal isolated worker receipt")
publicGuardPreflight := flag.Bool("public-guard-preflight", false, "restore the cached public guard before libvirt starts")
flag.Parse()
if err := validateRuntimeMode(*publicGuardPreflight, *operationWorker, *workerJournal, *workerReceipt); err != nil {
log.Fatal(err)
}
if *publicGuardPreflight {
if err := runPublicGuardPreflight(*configPath); err != nil {
log.Printf("public guard preflight failed: %v", err)
os.Exit(1)
}
return
}
if *operationWorker {
if err := runOperationWorker(*configPath, *workerJournal, *workerReceipt); err != nil {
log.Printf("isolated mutation worker failed: %v", err)
os.Exit(1)
}
return
}
cfg, err := config.Load(*configPath)
if err != nil {
log.Fatalf("load configuration: %v", err)
}
journal, err := operation.OpenWithHeavyIOConcurrency(cfg.JournalPath, cfg.HeavyIOConcurrency)
if err != nil {
log.Fatalf("open durable operation journal: %v", err)
}
executable, err := os.Executable()
if err != nil {
log.Fatalf("resolve Agent executable: %v", err)
}
executable, err = filepath.Abs(executable)
if err != nil {
log.Fatalf("resolve absolute Agent executable: %v", err)
}
absoluteConfig, err := filepath.Abs(*configPath)
if err != nil {
log.Fatalf("resolve absolute configuration path: %v", err)
}
executor, err := worker.NewSubprocessExecutor(executable, filepath.Clean(absoluteConfig), cfg.JournalPath+".workers")
if err != nil {
log.Fatalf("initialize isolated mutation executor: %v", err)
}
if err := recoverInterruptedWorkerState(journal, executor); err != nil {
log.Fatalf("recover interrupted isolated mutations: %v", err)
}
authenticator, err := auth.NewWithNonceStore([]byte(cfg.AgentSecret), cfg.ClockSkew(), cfg.NonceTTL(), journal)
if err != nil {
log.Fatalf("initialize request authentication: %v", err)
}
peerRuntime, peerErr := startPeerMigration(cfg)
if peerErr != nil {
log.Printf("PeerMigration V3 unavailable; ordinary Agent operations remain enabled: %v", peerErr)
}
local := localConfig(cfg)
local.RestoreVPCState = true
if peerRuntime != nil {
local.PeerMigrationEndpoints = append([]string(nil), peerRuntime.endpoints...)
local.PeerMigrationIdentitySHA256 = peerRuntime.identitySHA256
}
hv, err := hypervisor.OpenLibvirt(cfg.LibvirtURI, local)
if err != nil {
log.Fatalf("connect to libvirt: %v", err)
}
observationRuntime, err := newNetworkObservationRuntime(hv)
if err != nil {
log.Fatalf("initialize network abuse observation: %v", err)
}
console, err := consolebroker.NewBroker(60 * time.Second)
if err != nil {
log.Fatalf("initialize console broker: %v", err)
}
handler, err := api.NewServer(authenticator, journal, hv, console, cfg.MaxBodyBytes, cfg.OperationTimeout())
if err != nil {
log.Fatalf("initialize Agent request dispatcher: %v", err)
}
if err := handler.SetMutationExecutor(executor); err != nil {
log.Fatalf("enable isolated mutation execution: %v", err)
}
if err := handler.SetNetworkObserver(observationRuntime.collector); err != nil {
log.Fatalf("publish network abuse observation: %v", err)
}
var peerListener *peermigration.Listener
if peerRuntime != nil {
peerListener = peerRuntime.listener
log.Printf("PeerMigration V3 listening on %s with %d automatically discovered endpoint(s)",
cfg.PeerMigrationListenAddress(), len(peerRuntime.endpoints))
}
client, err := control.NewClient(
cfg.PanelURL, cfg.AgentID, []byte(cfg.AgentSecret), handler, console, cfg.MaxBodyBytes, cfg.OperationTimeout(),
&workerCompletionOutbox{journal: journal, cleaner: executor},
)
if err != nil {
log.Fatalf("initialize outbound control channel: %v", err)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := observationRuntime.Start(ctx); err != nil {
log.Fatalf("start network abuse observation: %v", err)
}
var runErr error
if ctx.Err() == nil {
log.Printf("virt host agent %s connecting to panel as Agent %s", api.Version, cfg.AgentID)
var peerFailures <-chan error
if peerListener != nil {
peerFailures = peerListener.Errors()
}
runErr = runControlUntilStopped(
ctx, journal.BackgroundFailures(), client.Run, peerFailures,
)
}
stop()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := peerListener.Shutdown(shutdownCtx); err != nil {
log.Printf("PeerMigration listener did not stop safely: %v", err)
os.Exit(1)
}
if err := handler.Shutdown(shutdownCtx); err != nil {
log.Printf("Agent handlers did not drain safely: %v", err)
os.Exit(1)
}
if err := observationRuntime.Shutdown(shutdownCtx); err != nil {
log.Printf("network abuse observation did not stop safely: %v", err)
os.Exit(1)
}
if err := journal.Wait(shutdownCtx); err != nil {
log.Printf("Agent operation workers did not drain safely: %v", err)
os.Exit(1)
}
if err := hv.Close(); err != nil {
log.Printf("close libvirt connection: %v", err)
}
if err := journal.Close(); err != nil {
log.Printf("close operation journal: %v", err)
}
if runErr != nil {
log.Fatalf("run outbound control channel: %v", runErr)
}
}
type peerMigrationRuntime struct {
listener *peermigration.Listener
endpoints []string
identitySHA256 string
}
func startPeerMigration(cfg config.Config) (*peerMigrationRuntime, error) {
store, err := peermigration.NewStore(
cfg.PeerMigrationStatePath(), cfg.StoragePathRoots,
cfg.PeerMigrationStagingPath(),
)
if err != nil {
return nil, fmt.Errorf("initialize automatic peer migration store: %w", err)
}
identity, err := peermigration.LoadOrCreateIdentity(cfg.PeerMigrationStatePath(), cfg.AgentID)
if err != nil {
return nil, fmt.Errorf("initialize automatic peer migration identity: %w", err)
}
endpoints, err := peermigration.DiscoverEndpoints(cfg.PeerMigrationPort)
if err != nil {
return nil, fmt.Errorf("discover peer migration endpoints: %w", err)
}
handler, err := peermigration.NewHandler(store)
if err != nil {
return nil, fmt.Errorf("initialize peer migration handler: %w", err)
}
tlsConfig, err := identity.ServerTLSConfig()
if err != nil {
return nil, fmt.Errorf("initialize automatic peer migration TLS: %w", err)
}
listener, err := peermigration.StartListener(cfg.PeerMigrationListenAddress(), handler, tlsConfig)
if err != nil {
return nil, fmt.Errorf("start peer migration listener: %w", err)
}
return &peerMigrationRuntime{
listener: listener, endpoints: endpoints, identitySHA256: identity.Fingerprint(),
}, nil
}
func localConfig(cfg config.Config) hypervisor.LocalConfig {
return hypervisor.LocalConfig{
StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...),
StorageConfigurationLockPath: cfg.JournalPath + ".storage-config.lock",
FirmwareLoader: cfg.FirmwareLoader,
FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate,
NVRAMDirectory: cfg.NVRAMDirectory,
TrafficStatePath: cfg.JournalPath + ".traffic.json",
GuestAgentTimeoutSeconds: int(cfg.GuestAgentTimeoutSec),
PeerMigrationStatePath: cfg.PeerMigrationStatePath(),
PeerMigrationStagingPath: cfg.PeerMigrationStagingPath(),
VPCStatePath: cfg.JournalPath + ".vpc-l3-v1.json",
PublicGuardLockPath: cfg.PublicGuardLockPath(),
PublicGuardStatePath: cfg.PublicGuardStatePath(),
}
}
func runControlUntilStopped(ctx context.Context, backgroundFailures <-chan error,
run func(context.Context) error, optionalFailures ...<-chan error) error {
var peerFailures <-chan error
if len(optionalFailures) > 0 {
peerFailures = optionalFailures[0]
}
runCtx, cancelRun := context.WithCancel(ctx)
defer cancelRun()
done := make(chan error, 1)
go func() {
done <- run(runCtx)
}()
select {
case err := <-done:
return err
case persistErr := <-backgroundFailures:
log.Printf("durable background operation finalization failed; stopping Agent: %v", persistErr)
cancelRun()
select {
case <-done:
case <-time.After(5 * time.Second):
log.Printf("outbound control channel did not stop within the finalization failure grace period")
}
return fmt.Errorf("durable background operation finalization failed: %w", persistErr)
case peerErr, open := <-peerFailures:
if !open || peerErr == nil {
peerErr = errors.New("listener stopped unexpectedly")
}
cancelRun()
select {
case <-done:
case <-time.After(5 * time.Second):
log.Printf("outbound control channel did not stop within the PeerMigration listener failure grace period")
}
return fmt.Errorf("PeerMigration listener failed: %w", peerErr)
}
}
type workerStateCleaner interface {
Cleanup(string) error
}
type completedWorkerStateCleaner interface {
CleanupCompleted(string, string) error
}
type recoveredWorkerState interface {
workerStateCleaner
PendingReceipts() ([]worker.Response, error)
PendingJournalTerminals() ([]worker.TerminalEvidence, error)
}
func recoverInterruptedWorkerState(journal *operation.Journal, state recoveredWorkerState) error {
// Inspect child Journals first. A still-running orphan worker keeps its Bolt
// file locked, which stops startup before main-Journal terminalization. If a
// receipt already exists the scanner skips that Journal and the receipt scan
// immediately below imports it, closing the publication race.
terminalEvidence, err := state.PendingJournalTerminals()
if err != nil {
// Do not irreversibly turn the main dispatching row into
// result_unconfirmed while terminal child evidence is temporarily
// unreadable (for example, a surviving worker still holds its DB).
return fmt.Errorf("inspect isolated mutation journals: %w", err)
}
pendingReceipts, err := state.PendingReceipts()
if err != nil {
return fmt.Errorf("read isolated mutation receipts: %w", err)
}
interrupted := make([]operation.InterruptedReceipt, 0, len(pendingReceipts)+len(terminalEvidence))
cleanup := make(map[string]struct{}, len(pendingReceipts)+len(terminalEvidence))
for _, receipt := range pendingReceipts {
record, getErr := journal.Get(receipt.OperationID)
if errors.Is(getErr, operation.ErrNotFound) || getErr == nil &&
(record.State != operation.StateDispatching || record.Fingerprint != receipt.Fingerprint) {
continue
}
if getErr != nil {
return fmt.Errorf("read main operation %s: %w", receipt.OperationID, getErr)
}
interrupted = append(interrupted, operation.InterruptedReceipt{
OperationID: receipt.OperationID, Fingerprint: receipt.Fingerprint,
Status: receipt.Status, Result: receipt.Result,
ErrorCode: receipt.ErrorCode, ErrorMessage: receipt.ErrorMessage,
EffectMayHaveApplied: receipt.EffectMayHaveApplied,
SafeToResubmit: receipt.SafeToResubmit,
})
cleanup[receipt.OperationID] = struct{}{}
}
for _, evidence := range terminalEvidence {
record, getErr := journal.Get(evidence.Record.OperationID)
if errors.Is(getErr, operation.ErrNotFound) {
continue
}
if getErr != nil {
return fmt.Errorf("read main operation %s: %w", evidence.Record.OperationID, getErr)
}
receipt, ok := evidence.InterruptedReceiptFor(record)
if !ok {
continue
}
if _, duplicate := cleanup[receipt.OperationID]; duplicate {
continue
}
interrupted = append(interrupted, receipt)
cleanup[receipt.OperationID] = struct{}{}
}
if _, err := journal.FinalizeInterruptedOperations(interrupted); err != nil {
return fmt.Errorf("finalize interrupted isolated mutations: %w", err)
}
return cleanupRecoveredWorkerState(state, cleanup)
}
// cleanupRecoveredWorkerState removes only helper directories whose terminal
// evidence was successfully matched and imported. Corrupt, mismatched, or
// otherwise unadopted evidence remains available for inspection.
func cleanupRecoveredWorkerState(state workerStateCleaner, operationIDs map[string]struct{}) error {
for operationID := range operationIDs {
if err := state.Cleanup(operationID); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("clean isolated mutation state %s: %w", operationID, err)
}
}
return nil
}
// workerCompletionOutbox keeps completion delivery in the main Journal until
// the panel ACKs it. Disposable helper state is cleaned independently after
// the durable receipt is claimed, after its ACK, or during the next startup.
type workerCompletionOutbox struct {
journal *operation.Journal
cleaner completedWorkerStateCleaner
}
func (o *workerCompletionOutbox) ClaimPendingCompletions(limit int, excluded []string) ([]operation.CompletionReceipt, error) {
claimed, _, err := o.journal.ClaimPendingCompletionsWithExpiredExcluding(limit, excluded)
if err != nil {
return nil, err
}
for _, receipt := range claimed {
if cleanupErr := o.cleaner.CleanupCompleted(receipt.OperationID, receipt.RequestFingerprint); cleanupErr != nil && !errors.Is(cleanupErr, os.ErrNotExist) {
// The main Journal receipt is already durable. Disposable helper
// cleanup must neither suppress its delivery nor remove it from the
// ACK-driven outbox; startup will retry any failed cleanup.
log.Printf("clean delivered worker state operation_id=%s: %v", receipt.OperationID, cleanupErr)
}
}
return claimed, nil
}
func (o *workerCompletionOutbox) AcknowledgeCompletion(eventID string) error {
operationID, err := o.journal.CompletionOperationID(eventID)
if err != nil {
return err
}
if operationID != "" {
record, getErr := o.journal.Get(operationID)
if getErr != nil {
log.Printf("read acknowledged worker identity operation_id=%s: %v", operationID, getErr)
} else if cleanupErr := o.cleaner.CleanupCompleted(operationID, record.Fingerprint); cleanupErr != nil && !errors.Is(cleanupErr, os.ErrNotExist) {
// An ACK is authoritative delivery state. A disposable helper
// cleanup failure must not keep the receipt in the delivery loop.
log.Printf("clean acknowledged worker state operation_id=%s: %v", operationID, cleanupErr)
}
}
return o.journal.AcknowledgeCompletion(eventID)
}
const (
isolatedWorkerMaximumLifetime = 2*time.Hour + 30*time.Second
isolatedBackupWorkerMaximumLifetime = 24*time.Hour + 30*time.Second
)
func isolatedWorkerLifetime(method, path string) time.Duration {
if method != http.MethodPost {
return isolatedWorkerMaximumLifetime
}
segments := strings.Split(strings.Trim(path, "/"), "/")
if len(segments) >= 4 && segments[0] == "v1" && segments[1] == "vms" &&
((len(segments) == 4 && (segments[3] == "backups" || segments[3] == "restore-backup" ||
segments[3] == "recover-backup")) ||
(len(segments) == 5 && segments[3] == "peer-migration" &&
(segments[4] == "prepare" || segments[4] == "receive"))) {
return isolatedBackupWorkerMaximumLifetime
}
return isolatedWorkerMaximumLifetime
}
func runOperationWorker(configPath, journalPath, receiptPath string) error {
if !cleanAbsolutePath(journalPath) || !cleanAbsolutePath(receiptPath) ||
filepath.Base(journalPath) != "journal.db" || filepath.Base(receiptPath) != "completion.json" ||
filepath.Dir(journalPath) != filepath.Dir(receiptPath) {
return errors.New("invalid isolated worker paths")
}
raw, err := io.ReadAll(io.LimitReader(os.Stdin, worker.MaxEnvelopeBytes+1))
if err != nil || len(raw) > worker.MaxEnvelopeBytes {
return errors.New("read bounded isolated worker request")
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.DisallowUnknownFields()
var request worker.Request
if err := decoder.Decode(&request); err != nil {
return errors.New("decode isolated worker request")
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return errors.New("isolated worker request contains trailing data")
}
if err := request.Validate(); err != nil {
return err
}
if filepath.Base(filepath.Dir(journalPath)) != request.OperationID {
return errors.New("isolated worker directory does not match operation")
}
cfg, err := config.Load(configPath)
if err != nil {
return fmt.Errorf("load worker configuration: %w", err)
}
journal, err := operation.OpenWithHeavyIOConcurrency(journalPath, cfg.HeavyIOConcurrency)
if err != nil {
return fmt.Errorf("open worker journal: %w", err)
}
authenticator, err := auth.NewWithNonceStore([]byte(cfg.AgentSecret), cfg.ClockSkew(), cfg.NonceTTL(), journal)
if err != nil {
return fmt.Errorf("initialize worker authentication: %w", err)
}
local := localConfig(cfg)
hv, err := hypervisor.OpenLibvirt(cfg.LibvirtURI, local)
if err != nil {
return fmt.Errorf("connect worker to libvirt: %w", err)
}
console, err := consolebroker.NewBroker(60 * time.Second)
if err != nil {
return fmt.Errorf("initialize worker console broker: %w", err)
}
handler, err := api.NewServer(authenticator, journal, hv, console, cfg.MaxBodyBytes, cfg.OperationTimeout())
if err != nil {
return fmt.Errorf("initialize worker dispatcher: %w", err)
}
parsed, err := url.ParseRequestURI(request.RequestTarget)
if err != nil {
return errors.New("parse isolated worker request target")
}
ctx, cancel := context.WithTimeout(context.Background(),
isolatedWorkerLifetime(request.Method, parsed.Path))
defer cancel()
httpRequest := (&http.Request{
Method: request.Method, URL: parsed, RequestURI: request.RequestTarget,
Header: make(http.Header), Body: io.NopCloser(strings.NewReader(request.Body)),
ContentLength: int64(len(request.Body)), Host: "agent.worker",
}).WithContext(ctx)
responseCapture := newBoundedWorkerResponse(worker.MaxWorkerResponse)
if request.SupersededOperation != nil {
prior := *request.SupersededOperation
prior.ExpectedState = request.SupersededExpectedState
handler.ServeTrustedWorkerRequest(
responseCapture, httpRequest, []byte(request.Body), request.Actor,
prior,
)
} else {
handler.ServeTrustedWorkerRequest(
responseCapture, httpRequest, []byte(request.Body), request.Actor,
)
}
response := waitForWorkerTerminal(ctx, journal, request, responseCapture)
if err := worker.WriteReceipt(receiptPath, response); err != nil {
return fmt.Errorf("persist isolated worker receipt: %w", err)
}
encoded, err := json.Marshal(response)
if err != nil {
return errors.New("encode isolated worker receipt")
}
if _, err := os.Stdout.Write(append(encoded, '\n')); err != nil {
return errors.New("write isolated worker response")
}
// Avoid any potentially blocking libvirt cleanup after the immutable
// receipt is durable. The operating system closes descriptors on exit.
os.Exit(0)
return nil
}
func cleanAbsolutePath(value string) bool {
return value != "" && filepath.IsAbs(value) && filepath.Clean(value) == value && !strings.ContainsRune(value, 0)
}
func waitForWorkerTerminal(ctx context.Context, journal *operation.Journal, request worker.Request,
capture *boundedWorkerResponse) worker.Response {
ticker := time.NewTicker(25 * time.Millisecond)
defer ticker.Stop()
for {
record, err := journal.Get(request.OperationID)
if err == nil && (record.State == operation.StateSucceeded || record.State == operation.StateFailed) {
response := worker.Response{
Version: worker.ProtocolVersion, OperationID: request.OperationID, Fingerprint: request.Fingerprint,
Status: string(record.State), Result: boundedWorkerResult(record.Result),
EffectMayHaveApplied: record.EffectMayHaveApplied, SafeToResubmit: record.SafeToResubmit,
}
if record.Error != nil {
response.ErrorCode = record.Error.Code
response.ErrorMessage = boundedWorkerText(record.Error.Message, 1024)
}
if response.Validate() == nil {
return response
}
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
}
if err != nil && !errors.Is(err, operation.ErrNotFound) {
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
}
if errors.Is(err, operation.ErrNotFound) && capture.status >= http.StatusBadRequest {
var envelope struct {
Error struct {
Code string `json:"code"`
Message string `json:"message"`
} `json:"error"`
}
if json.Unmarshal(capture.body.Bytes(), &envelope) == nil &&
envelope.Error.Code != "" && len(envelope.Error.Code) <= 64 {
response := worker.Response{
Version: worker.ProtocolVersion, OperationID: request.OperationID, Fingerprint: request.Fingerprint,
Status: "failed", ErrorCode: envelope.Error.Code,
ErrorMessage: boundedWorkerText(envelope.Error.Message, 1024),
EffectMayHaveApplied: false, SafeToResubmit: true,
}
if response.Validate() == nil {
return response
}
}
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
}
select {
case <-ctx.Done():
return worker.UnconfirmedResponse(request.OperationID, request.Fingerprint)
case <-ticker.C:
}
}
}
func boundedWorkerResult(value json.RawMessage) json.RawMessage {
if len(value) > worker.MaxWorkerResponse/2 {
return nil
}
return append(json.RawMessage(nil), value...)
}
func boundedWorkerText(value string, limit int) string {
if len(value) <= limit {
return value
}
return value[:limit]
}
type boundedWorkerResponse struct {
header http.Header
status int
body bytes.Buffer
limit int
overflow bool
}
func newBoundedWorkerResponse(limit int) *boundedWorkerResponse {
return &boundedWorkerResponse{header: make(http.Header), limit: limit}
}
func (w *boundedWorkerResponse) Header() http.Header { return w.header }
func (w *boundedWorkerResponse) WriteHeader(status int) {
if w.status == 0 {
w.status = status
}
}
func (w *boundedWorkerResponse) Write(value []byte) (int, error) {
if w.status == 0 {
w.status = http.StatusOK
}
if w.body.Len()+len(value) > w.limit {
w.overflow = true
return len(value), nil
}
return w.body.Write(value)
}