You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/control/client_test.go

705 lines
23 KiB
Go

package control
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/coder/websocket"
"git.luoli.live/root/virt/agent/auth"
consolebroker "git.luoli.live/root/virt/agent/console"
"git.luoli.live/root/virt/agent/operation"
)
const controlTestSecret = "0123456789abcdef0123456789abcdef"
func TestCompletionDeliveryWindowReleasesOnlyOnDisposition(t *testing.T) {
window := newCompletionDeliveryWindow()
for index := 0; index < completionWindowSize; index++ {
if !window.reserve(context.Background()) {
t.Fatal("completion window reservation failed")
}
window.track(fmt.Sprintf("completion_window_%02d", index), fmt.Sprintf("operation_window_%02d", index))
}
blockedCtx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
if window.reserve(blockedCtx) {
t.Fatal("completion window exceeded its bounded capacity")
}
if operationID, released := window.release("completion_window_00"); !released || operationID != "operation_window_00" {
t.Fatal("ACK/retry disposition did not release the tracked event")
}
if !window.reserve(context.Background()) {
t.Fatal("released completion slot was not reusable")
}
window.abandonReservation()
if _, released := window.release("completion_window_00"); released {
t.Fatal("duplicate disposition released a second slot")
}
}
func TestCompletionMessageCarriesStrictImmutableOperationIdentity(t *testing.T) {
receipt := operation.CompletionReceipt{
EventID: "completion_identity_123", OperationID: "operation_identity_123",
Action: "delete-vm", ResourceID: "vm-completion-identity",
RequestFingerprint: strings.Repeat("a", 64),
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
}
message, err := completionMessageFromReceipt(receipt)
if err != nil {
t.Fatal(err)
}
if message.Action != receipt.Action || message.ResourceID != receipt.ResourceID ||
message.RequestFingerprint != receipt.RequestFingerprint {
t.Fatalf("completion message identity = %#v", message)
}
tests := []struct {
name string
mutate func(*operation.CompletionReceipt)
}{
{name: "missing action", mutate: func(value *operation.CompletionReceipt) { value.Action = "" }},
{name: "uppercase action", mutate: func(value *operation.CompletionReceipt) { value.Action = "Delete-VM" }},
{name: "resource path", mutate: func(value *operation.CompletionReceipt) { value.ResourceID = "vm/unsafe" }},
{name: "missing resource", mutate: func(value *operation.CompletionReceipt) { value.ResourceID = "" }},
{name: "uppercase fingerprint", mutate: func(value *operation.CompletionReceipt) { value.RequestFingerprint = strings.Repeat("A", 64) }},
{name: "short fingerprint", mutate: func(value *operation.CompletionReceipt) { value.RequestFingerprint = strings.Repeat("a", 63) }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
candidate := receipt
test.mutate(&candidate)
if _, err := completionMessageFromReceipt(candidate); err == nil {
t.Fatal("invalid immutable operation identity was accepted")
}
})
}
}
func TestControlSubprotocolUsesCleanBreakV3(t *testing.T) {
if ControlSubprotocol != "virt-agent-control.v3" {
t.Fatalf("control subprotocol = %q, want clean-break v3", ControlSubprotocol)
}
}
func TestControlSessionRejectsLegacyV2GatewayDuringHandshake(t *testing.T) {
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.Header.Get("Sec-WebSocket-Protocol") != "virt-agent-control.v2" {
http.Error(writer, "legacy v2 control subprotocol required", http.StatusBadRequest)
return
}
connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{
Subprotocols: []string{"virt-agent-control.v2"}, CompressionMode: websocket.CompressionDisabled,
})
if err != nil {
return
}
defer connection.CloseNow()
_, _, _ = connection.Read(context.Background())
}))
defer server.Close()
broker, err := consolebroker.NewBroker(time.Minute)
if err != nil {
t.Fatal(err)
}
client, err := NewClient(
server.URL, "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, time.Minute,
)
if err != nil {
t.Fatal(err)
}
client.httpClient = server.Client()
err = client.runSession(context.Background())
if err == nil || !strings.Contains(err.Error(), "dial panel control channel") {
t.Fatalf("legacy v2 gateway handshake error = %v", err)
}
}
func TestCompletionDeliveryLogContainsOnlyStableMetadata(t *testing.T) {
logger := log.Default()
previousWriter, previousFlags, previousPrefix := logger.Writer(), logger.Flags(), logger.Prefix()
var output bytes.Buffer
logger.SetOutput(&output)
logger.SetFlags(0)
logger.SetPrefix("")
t.Cleanup(func() {
logger.SetOutput(previousWriter)
logger.SetFlags(previousFlags)
logger.SetPrefix(previousPrefix)
})
logCompletionDelivery(completionPhaseAgentRetryReceived, "operation_log_123", "completion_log_123")
got := strings.TrimSpace(output.String())
want := "completion_delivery phase=agent_retry_received operation_id=operation_log_123 event_id=completion_log_123"
if got != want {
t.Fatalf("completion log = %q, want %q", got, want)
}
}
func TestControlDeadlineWrapsAPIOperationDeadline(t *testing.T) {
broker, err := consolebroker.NewBroker(time.Minute)
if err != nil {
t.Fatal(err)
}
client, err := NewClient(
"https://panel.example", "42", []byte(controlTestSecret),
http.NotFoundHandler(), broker, 64<<10, maximumAPITimeout,
)
if err != nil {
t.Fatal(err)
}
if client.requestTimeout != 855*time.Second {
t.Fatalf("control request timeout = %s, want 855s", client.requestTimeout)
}
if _, err := NewClient(
"https://panel.example", "42", []byte(controlTestSecret),
http.NotFoundHandler(), broker, 64<<10, maximumAPITimeout+time.Second,
); err == nil {
t.Fatal("control client accepted an API timeout that crosses the Gateway margin")
}
}
type controlPingerFunc func(context.Context) error
func (function controlPingerFunc) Ping(ctx context.Context) error {
return function(ctx)
}
func TestControlHeartbeatPingsUntilSessionCloses(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
pinged := make(chan struct{}, 1)
finished := make(chan error, 1)
go func() {
finished <- maintainControlHeartbeat(ctx, controlPingerFunc(
func(context.Context) error {
select {
case pinged <- struct{}{}:
default:
}
return nil
},
), time.Millisecond, 50*time.Millisecond)
}()
select {
case <-pinged:
cancel()
case <-time.After(time.Second):
cancel()
t.Fatal("control heartbeat did not send a Ping")
}
select {
case err := <-finished:
if err != nil {
t.Fatalf("heartbeat returned an error after session cancellation: %v", err)
}
case <-time.After(time.Second):
t.Fatal("control heartbeat did not stop with its session")
}
}
func TestControlHeartbeatRejectsMissingPong(t *testing.T) {
finished := make(chan error, 1)
go func() {
finished <- maintainControlHeartbeat(context.Background(), controlPingerFunc(
func(ctx context.Context) error {
<-ctx.Done()
return ctx.Err()
},
), time.Millisecond, 5*time.Millisecond)
}()
select {
case err := <-finished:
if err == nil || !strings.Contains(err.Error(), "control ping failed") ||
!errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("missing Pong error = %v", err)
}
case <-time.After(time.Second):
t.Fatal("missing Pong did not fail the control heartbeat")
}
}
func TestControlSessionAuthenticatesAndDispatchesInnerSignedRequest(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
authenticator, err := auth.New([]byte(controlTestSecret), time.Minute, 2*time.Minute)
if err != nil {
t.Fatal(err)
}
authenticator.SetClockForTesting(func() time.Time { return now })
inner := http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
body, readErr := io.ReadAll(request.Body)
if readErr != nil {
t.Errorf("read reconstructed request: %v", readErr)
response.WriteHeader(http.StatusBadRequest)
return
}
actor, authErr := authenticator.Authenticate(request, body)
if authErr != nil || actor.UserID != 42 || request.RequestURI != "/v1/health?full=1" || string(body) != `{"probe":true}` {
t.Errorf("reconstructed request actor=%#v uri=%q body=%q err=%v", actor, request.RequestURI, body, authErr)
response.WriteHeader(http.StatusUnauthorized)
return
}
response.Header().Set("Content-Type", "application/json")
response.WriteHeader(http.StatusCreated)
_, _ = response.Write([]byte(`{"status":"ok"}`))
})
result := make(chan response, 1)
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Path != controlPath || request.Header.Get("X-Virt-Agent-ID") != "42" ||
request.Header.Get("Authorization") != "Bearer "+controlTestSecret {
http.Error(writer, "bad Agent handshake", http.StatusUnauthorized)
return
}
connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
})
if acceptErr != nil {
return
}
defer connection.CloseNow()
body := []byte(`{"probe":true}`)
signed, _ := http.NewRequest(http.MethodPost, "https://agent.invalid/v1/health?full=1", bytes.NewReader(body))
_ = auth.Sign(signed, body, auth.Actor{UserID: 42}, []byte(controlTestSecret), now.Unix(), "0123456789abcdef0123456789abcdef")
headers := make(map[string]string)
for _, name := range []string{auth.HeaderTimestamp, auth.HeaderNonce, auth.HeaderBodyDigest, auth.HeaderActor, auth.HeaderSignature} {
headers[name] = signed.Header.Get(name)
}
payload, _ := json.Marshal(message{
Type: "request", ID: "request_12345678", Method: http.MethodPost,
RequestTarget: "/v1/health?full=1", Body: string(body), Headers: headers,
})
if writeErr := connection.Write(request.Context(), websocket.MessageText, payload); writeErr != nil {
t.Errorf("send request: %v", writeErr)
return
}
messageType, raw, readErr := connection.Read(request.Context())
if readErr != nil || messageType != websocket.MessageText {
t.Errorf("read response type=%v err=%v", messageType, readErr)
return
}
var got response
if err := json.Unmarshal(raw, &got); err != nil {
t.Errorf("decode response: %v", err)
return
}
result <- got
_ = connection.Close(websocket.StatusNormalClosure, "done")
}))
defer server.Close()
broker, _ := consolebroker.NewBroker(time.Minute)
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute)
if err != nil {
t.Fatal(err)
}
client.httpClient = server.Client()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- client.runSession(ctx) }()
select {
case got := <-result:
if got.Type != "response" || got.ID != "request_12345678" || got.Status != http.StatusCreated || got.Body != `{"status":"ok"}` {
t.Fatalf("control response = %#v", got)
}
case <-ctx.Done():
t.Fatal("timed out waiting for control response")
}
select {
case <-done:
case <-ctx.Done():
t.Fatal("control session did not close")
}
}
func TestConsoleOpenCreatesReverseWSSAndRelaysBinary(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
var operationError *net.OpError
if errors.As(err, &operationError) {
t.Skipf("sandbox does not permit loopback listeners: %v", err)
}
t.Fatal(err)
}
defer listener.Close()
consoleRelease := make(chan struct{})
var releaseOnce sync.Once
releaseConsole := func() { releaseOnce.Do(func() { close(consoleRelease) }) }
defer releaseConsole()
backendDone := make(chan struct{})
go func() {
defer close(backendDone)
connection, acceptErr := listener.Accept()
if acceptErr != nil {
return
}
defer connection.Close()
buffer := make([]byte, 128)
count, readErr := connection.Read(buffer)
if readErr == nil {
_, _ = connection.Write(buffer[:count])
}
<-consoleRelease
}()
broker, _ := consolebroker.NewBroker(time.Minute)
ticket, err := broker.Issue(listener.Addr().String())
if err != nil {
t.Fatal(err)
}
const sessionID = "session_0123456789abcdef01234567"
const sessionToken = "token_0123456789abcdef0123456789abcdef"
controlResult := make(chan response, 1)
consoleResult := make(chan string, 1)
relayDone := make(chan struct{})
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
switch request.URL.Path {
case controlPath:
connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
})
if acceptErr != nil {
return
}
defer connection.CloseNow()
payload, _ := json.Marshal(message{
Type: "console_open", ID: "console_12345678", Ticket: ticket.Ticket,
SessionID: sessionID, SessionToken: sessionToken,
})
if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil {
return
}
messageType, raw, err := connection.Read(request.Context())
if err != nil || messageType != websocket.MessageText {
return
}
var got response
if json.Unmarshal(raw, &got) == nil {
controlResult <- got
}
select {
case <-relayDone:
case <-request.Context().Done():
}
_ = connection.Close(websocket.StatusNormalClosure, "done")
case consolePath:
expectedProtocol := ConsoleProtocol + sessionToken
if request.Header.Get("X-Virt-Agent-ID") != "42" ||
request.Header.Get("Authorization") != "" ||
request.Header.Get("X-Virt-Session-ID") != sessionID {
http.Error(writer, "bad console handshake", http.StatusUnauthorized)
return
}
connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{
Subprotocols: []string{expectedProtocol}, CompressionMode: websocket.CompressionDisabled,
})
if acceptErr != nil {
return
}
defer connection.CloseNow()
if err := connection.Write(request.Context(), websocket.MessageBinary, []byte("RFB-test")); err != nil {
return
}
messageType, raw, err := connection.Read(request.Context())
if err == nil && messageType == websocket.MessageBinary {
consoleResult <- string(raw)
}
<-consoleRelease
close(relayDone)
default:
http.NotFound(writer, request)
}
}))
defer server.Close()
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, time.Minute)
if err != nil {
t.Fatal(err)
}
client.httpClient = server.Client()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- client.runSession(ctx) }()
select {
case got := <-controlResult:
if got.Status != http.StatusOK || got.ID != "console_12345678" || !strings.Contains(got.Body, `"status":"connected"`) {
t.Fatalf("console-open response = %#v", got)
}
case <-ctx.Done():
t.Fatal("timed out waiting for console-open response")
}
select {
case got := <-consoleResult:
if got != "RFB-test" {
t.Fatalf("reverse console body = %q", got)
}
if len(client.activeConsoles) != 1 {
t.Fatalf("active console slot was not held for relay lifetime: %d", len(client.activeConsoles))
}
releaseConsole()
case <-ctx.Done():
t.Fatal("timed out waiting for reverse console relay")
}
select {
case <-backendDone:
case <-ctx.Done():
t.Fatal("local console connection did not close")
}
select {
case <-done:
case <-ctx.Done():
t.Fatal("control session did not close")
}
deadline := time.Now().Add(time.Second)
for len(client.activeConsoles) != 0 && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
if got := len(client.activeConsoles); got != 0 {
t.Fatalf("active console slot was not released: %d", got)
}
}
func TestClientBoundsActiveConsoles(t *testing.T) {
client := &Client{activeConsoles: make(chan struct{}, maxActiveConsoles)}
for index := 0; index < maxActiveConsoles; index++ {
if !client.acquireConsole() {
t.Fatalf("console slot %d was rejected", index)
}
}
if client.acquireConsole() {
t.Fatal("console capacity exceeded configured limit")
}
client.releaseConsole()
if !client.acquireConsole() {
t.Fatal("released console capacity was not reusable")
}
}
func TestParsePanelOriginRejectsInvalidPorts(t *testing.T) {
for _, value := range []string{
"https://panel.example.com:",
"https://panel.example.com:0",
"https://panel.example.com:65536",
} {
if _, err := parsePanelOrigin(value); err == nil {
t.Fatalf("invalid panel origin %q was accepted", value)
}
}
if _, err := parsePanelOrigin("https://panel.example.com:443"); err != nil {
t.Fatalf("valid explicit TLS port was rejected: %v", err)
}
}
func TestControlRequestBodyAndResponseLimitsAreExact(t *testing.T) {
client := &Client{maxBodyBytes: 1024}
headers := map[string]string{
auth.HeaderTimestamp: "1",
auth.HeaderNonce: "0123456789abcdef0123456789abcdef",
auth.HeaderBodyDigest: "digest",
auth.HeaderActor: "actor",
auth.HeaderSignature: "signature",
}
request := message{
Type: "request", ID: "request_boundary", Method: http.MethodPost,
RequestTarget: "/v1/health", Body: strings.Repeat("x", 1024), Headers: headers,
}
if err := client.validateRequest(request); err != nil {
t.Fatalf("exact maximum request body was rejected: %v", err)
}
request.Body += "x"
if err := client.validateRequest(request); err == nil {
t.Fatal("oversized request body was accepted")
}
capture := newResponseCapture(8)
if count, err := capture.Write([]byte("12345678")); err != nil || count != 8 {
t.Fatalf("exact maximum response write count=%d err=%v", count, err)
}
status, body := capture.result()
if status != http.StatusOK || body != "12345678" {
t.Fatalf("exact maximum response = %d %q", status, body)
}
overflow := newResponseCapture(8)
if _, err := overflow.Write([]byte("123456789")); err == nil {
t.Fatal("oversized response body was accepted")
}
status, body = overflow.result()
if status != http.StatusBadGateway || !strings.Contains(body, `"code":"agent_response_too_large"`) {
t.Fatalf("oversized response = %d %q", status, body)
}
}
func TestControlSessionBoundsInFlightRequests(t *testing.T) {
release := make(chan struct{})
inner := http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
<-release
response.WriteHeader(http.StatusNoContent)
})
result := make(chan response, 1)
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
})
if err != nil {
return
}
defer connection.CloseNow()
headers := map[string]string{
auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef",
auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature",
}
for index := 0; index <= maxInFlight; index++ {
payload, _ := json.Marshal(message{
Type: "request", ID: fmt.Sprintf("request_%08d", index), Method: http.MethodGet,
RequestTarget: "/v1/health", Headers: headers,
})
if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil {
return
}
}
messageType, raw, err := connection.Read(request.Context())
if err != nil || messageType != websocket.MessageText {
return
}
var got response
if json.Unmarshal(raw, &got) == nil {
result <- got
}
_ = connection.Close(websocket.StatusNormalClosure, "done")
}))
defer server.Close()
broker, _ := consolebroker.NewBroker(time.Minute)
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute)
if err != nil {
t.Fatal(err)
}
client.httpClient = server.Client()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- client.runSession(ctx) }()
select {
case got := <-result:
if got.Status != http.StatusServiceUnavailable ||
!strings.Contains(got.Body, `"code":"agent_busy"`) ||
!strings.Contains(got.Body, `"dispatch_state":"not_dispatched"`) {
t.Fatalf("saturation response = %#v", got)
}
case <-ctx.Done():
t.Fatal("timed out waiting for saturation response")
}
close(release)
select {
case <-done:
case <-ctx.Done():
t.Fatal("control session did not close")
}
}
func TestControlRequestDeadlineReturnsGatewayTimeout(t *testing.T) {
release := make(chan struct{}, 1)
defer func() {
select {
case release <- struct{}{}:
default:
}
}()
inner := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
<-release
})
result := make(chan response, 1)
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
})
if err != nil {
return
}
defer connection.CloseNow()
payload, _ := json.Marshal(message{
Type: "request", ID: "request_deadline", Method: http.MethodGet, RequestTarget: "/v1/health",
Headers: map[string]string{
auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef",
auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature",
},
})
if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil {
return
}
messageType, raw, err := connection.Read(request.Context())
if err != nil || messageType != websocket.MessageText {
return
}
var got response
if json.Unmarshal(raw, &got) == nil {
result <- got
}
_ = connection.Close(websocket.StatusNormalClosure, "done")
}))
defer server.Close()
broker, _ := consolebroker.NewBroker(time.Minute)
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute)
if err != nil {
t.Fatal(err)
}
client.requestTimeout = 25 * time.Millisecond
client.httpClient = server.Client()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- client.runSession(ctx) }()
select {
case got := <-result:
if got.Status != http.StatusGatewayTimeout || !strings.Contains(got.Body, `"code":"agent_request_timeout"`) {
t.Fatalf("deadline response = %#v", got)
}
deadline := time.Now().Add(time.Second)
for len(client.inFlight) != 0 && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
if len(client.inFlight) != 0 {
t.Fatalf("timed-out handler retained its WSS admission slot: %d", len(client.inFlight))
}
release <- struct{}{}
case <-ctx.Done():
t.Fatal("timed out waiting for request deadline response")
}
select {
case <-done:
case <-ctx.Done():
t.Fatal("control session did not close")
}
}
func newTLSServer(t *testing.T, handler http.Handler) *httptest.Server {
t.Helper()
var server *httptest.Server
var panicValue any
func() {
defer func() { panicValue = recover() }()
server = httptest.NewTLSServer(handler)
}()
if panicValue != nil {
t.Skipf("sandbox does not permit a TLS loopback server: %v", panicValue)
}
return server
}