You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
708 lines
23 KiB
Go
708 lines
23 KiB
Go
package control
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/coder/websocket"
|
|
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
consolebroker "git.luoli.live/root/virt/agent/console"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
)
|
|
|
|
const controlTestSecret = "0123456789abcdef0123456789abcdef"
|
|
|
|
func TestCompletionDeliveryWindowReleasesOnlyOnDisposition(t *testing.T) {
|
|
window := newCompletionDeliveryWindow()
|
|
for index := 0; index < completionWindowSize; index++ {
|
|
if !window.reserve(context.Background()) {
|
|
t.Fatal("completion window reservation failed")
|
|
}
|
|
window.track(fmt.Sprintf("completion_window_%02d", index), fmt.Sprintf("operation_window_%02d", index))
|
|
}
|
|
blockedCtx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
|
defer cancel()
|
|
if window.reserve(blockedCtx) {
|
|
t.Fatal("completion window exceeded its bounded capacity")
|
|
}
|
|
if operationID, released := window.release("completion_window_00"); !released || operationID != "operation_window_00" {
|
|
t.Fatal("ACK/retry disposition did not release the tracked event")
|
|
}
|
|
if !window.reserve(context.Background()) {
|
|
t.Fatal("released completion slot was not reusable")
|
|
}
|
|
window.abandonReservation()
|
|
if _, released := window.release("completion_window_00"); released {
|
|
t.Fatal("duplicate disposition released a second slot")
|
|
}
|
|
}
|
|
|
|
func TestCompletionMessageCarriesStrictImmutableOperationIdentity(t *testing.T) {
|
|
receipt := operation.CompletionReceipt{
|
|
EventID: "completion_identity_123", OperationID: "operation_identity_123",
|
|
Action: "delete-vm", ResourceID: "vm-completion-identity",
|
|
RequestFingerprint: strings.Repeat("a", 64),
|
|
Status: "failed", Result: json.RawMessage("null"),
|
|
ErrorCode: "operation_failed",
|
|
CauseCode: "backup_disk_transition_active",
|
|
}
|
|
message, err := completionMessageFromReceipt(receipt)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if message.Action != receipt.Action || message.ResourceID != receipt.ResourceID ||
|
|
message.RequestFingerprint != receipt.RequestFingerprint ||
|
|
message.CauseCode != receipt.CauseCode {
|
|
t.Fatalf("completion message identity = %#v", message)
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*operation.CompletionReceipt)
|
|
}{
|
|
{name: "missing action", mutate: func(value *operation.CompletionReceipt) { value.Action = "" }},
|
|
{name: "uppercase action", mutate: func(value *operation.CompletionReceipt) { value.Action = "Delete-VM" }},
|
|
{name: "resource path", mutate: func(value *operation.CompletionReceipt) { value.ResourceID = "vm/unsafe" }},
|
|
{name: "missing resource", mutate: func(value *operation.CompletionReceipt) { value.ResourceID = "" }},
|
|
{name: "uppercase fingerprint", mutate: func(value *operation.CompletionReceipt) { value.RequestFingerprint = strings.Repeat("A", 64) }},
|
|
{name: "short fingerprint", mutate: func(value *operation.CompletionReceipt) { value.RequestFingerprint = strings.Repeat("a", 63) }},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
candidate := receipt
|
|
test.mutate(&candidate)
|
|
if _, err := completionMessageFromReceipt(candidate); err == nil {
|
|
t.Fatal("invalid immutable operation identity was accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestControlSubprotocolUsesCleanBreakV3(t *testing.T) {
|
|
if ControlSubprotocol != "virt-agent-control.v3" {
|
|
t.Fatalf("control subprotocol = %q, want clean-break v3", ControlSubprotocol)
|
|
}
|
|
}
|
|
|
|
func TestControlSessionRejectsLegacyV2GatewayDuringHandshake(t *testing.T) {
|
|
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
|
if request.Header.Get("Sec-WebSocket-Protocol") != "virt-agent-control.v2" {
|
|
http.Error(writer, "legacy v2 control subprotocol required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
|
Subprotocols: []string{"virt-agent-control.v2"}, CompressionMode: websocket.CompressionDisabled,
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer connection.CloseNow()
|
|
_, _, _ = connection.Read(context.Background())
|
|
}))
|
|
defer server.Close()
|
|
|
|
broker, err := consolebroker.NewBroker(time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client, err := NewClient(
|
|
server.URL, "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, time.Minute,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client.httpClient = server.Client()
|
|
err = client.runSession(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "dial panel control channel") {
|
|
t.Fatalf("legacy v2 gateway handshake error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCompletionDeliveryLogContainsOnlyStableMetadata(t *testing.T) {
|
|
logger := log.Default()
|
|
previousWriter, previousFlags, previousPrefix := logger.Writer(), logger.Flags(), logger.Prefix()
|
|
var output bytes.Buffer
|
|
logger.SetOutput(&output)
|
|
logger.SetFlags(0)
|
|
logger.SetPrefix("")
|
|
t.Cleanup(func() {
|
|
logger.SetOutput(previousWriter)
|
|
logger.SetFlags(previousFlags)
|
|
logger.SetPrefix(previousPrefix)
|
|
})
|
|
|
|
logCompletionDelivery(completionPhaseAgentRetryReceived, "operation_log_123", "completion_log_123")
|
|
got := strings.TrimSpace(output.String())
|
|
want := "completion_delivery phase=agent_retry_received operation_id=operation_log_123 event_id=completion_log_123"
|
|
if got != want {
|
|
t.Fatalf("completion log = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestControlDeadlineWrapsAPIOperationDeadline(t *testing.T) {
|
|
broker, err := consolebroker.NewBroker(time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client, err := NewClient(
|
|
"https://panel.example", "42", []byte(controlTestSecret),
|
|
http.NotFoundHandler(), broker, 64<<10, maximumAPITimeout,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if client.requestTimeout != 855*time.Second {
|
|
t.Fatalf("control request timeout = %s, want 855s", client.requestTimeout)
|
|
}
|
|
if _, err := NewClient(
|
|
"https://panel.example", "42", []byte(controlTestSecret),
|
|
http.NotFoundHandler(), broker, 64<<10, maximumAPITimeout+time.Second,
|
|
); err == nil {
|
|
t.Fatal("control client accepted an API timeout that crosses the Gateway margin")
|
|
}
|
|
}
|
|
|
|
type controlPingerFunc func(context.Context) error
|
|
|
|
func (function controlPingerFunc) Ping(ctx context.Context) error {
|
|
return function(ctx)
|
|
}
|
|
|
|
func TestControlHeartbeatPingsUntilSessionCloses(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
pinged := make(chan struct{}, 1)
|
|
finished := make(chan error, 1)
|
|
go func() {
|
|
finished <- maintainControlHeartbeat(ctx, controlPingerFunc(
|
|
func(context.Context) error {
|
|
select {
|
|
case pinged <- struct{}{}:
|
|
default:
|
|
}
|
|
return nil
|
|
},
|
|
), time.Millisecond, 50*time.Millisecond)
|
|
}()
|
|
|
|
select {
|
|
case <-pinged:
|
|
cancel()
|
|
case <-time.After(time.Second):
|
|
cancel()
|
|
t.Fatal("control heartbeat did not send a Ping")
|
|
}
|
|
select {
|
|
case err := <-finished:
|
|
if err != nil {
|
|
t.Fatalf("heartbeat returned an error after session cancellation: %v", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("control heartbeat did not stop with its session")
|
|
}
|
|
}
|
|
|
|
func TestControlHeartbeatRejectsMissingPong(t *testing.T) {
|
|
finished := make(chan error, 1)
|
|
go func() {
|
|
finished <- maintainControlHeartbeat(context.Background(), controlPingerFunc(
|
|
func(ctx context.Context) error {
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
},
|
|
), time.Millisecond, 5*time.Millisecond)
|
|
}()
|
|
|
|
select {
|
|
case err := <-finished:
|
|
if err == nil || !strings.Contains(err.Error(), "control ping failed") ||
|
|
!errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("missing Pong error = %v", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("missing Pong did not fail the control heartbeat")
|
|
}
|
|
}
|
|
|
|
func TestControlSessionAuthenticatesAndDispatchesInnerSignedRequest(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
authenticator, err := auth.New([]byte(controlTestSecret), time.Minute, 2*time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
authenticator.SetClockForTesting(func() time.Time { return now })
|
|
inner := http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
body, readErr := io.ReadAll(request.Body)
|
|
if readErr != nil {
|
|
t.Errorf("read reconstructed request: %v", readErr)
|
|
response.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
actor, authErr := authenticator.Authenticate(request, body)
|
|
if authErr != nil || actor.UserID != 42 || request.RequestURI != "/v1/health?full=1" || string(body) != `{"probe":true}` {
|
|
t.Errorf("reconstructed request actor=%#v uri=%q body=%q err=%v", actor, request.RequestURI, body, authErr)
|
|
response.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
response.Header().Set("Content-Type", "application/json")
|
|
response.WriteHeader(http.StatusCreated)
|
|
_, _ = response.Write([]byte(`{"status":"ok"}`))
|
|
})
|
|
|
|
result := make(chan response, 1)
|
|
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
|
if request.URL.Path != controlPath || request.Header.Get("X-Virt-Agent-ID") != "42" ||
|
|
request.Header.Get("Authorization") != "Bearer "+controlTestSecret {
|
|
http.Error(writer, "bad Agent handshake", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
|
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
|
|
})
|
|
if acceptErr != nil {
|
|
return
|
|
}
|
|
defer connection.CloseNow()
|
|
body := []byte(`{"probe":true}`)
|
|
signed, _ := http.NewRequest(http.MethodPost, "https://agent.invalid/v1/health?full=1", bytes.NewReader(body))
|
|
_ = auth.Sign(signed, body, auth.Actor{UserID: 42}, []byte(controlTestSecret), now.Unix(), "0123456789abcdef0123456789abcdef")
|
|
headers := make(map[string]string)
|
|
for _, name := range []string{auth.HeaderTimestamp, auth.HeaderNonce, auth.HeaderBodyDigest, auth.HeaderActor, auth.HeaderSignature} {
|
|
headers[name] = signed.Header.Get(name)
|
|
}
|
|
payload, _ := json.Marshal(message{
|
|
Type: "request", ID: "request_12345678", Method: http.MethodPost,
|
|
RequestTarget: "/v1/health?full=1", Body: string(body), Headers: headers,
|
|
})
|
|
if writeErr := connection.Write(request.Context(), websocket.MessageText, payload); writeErr != nil {
|
|
t.Errorf("send request: %v", writeErr)
|
|
return
|
|
}
|
|
messageType, raw, readErr := connection.Read(request.Context())
|
|
if readErr != nil || messageType != websocket.MessageText {
|
|
t.Errorf("read response type=%v err=%v", messageType, readErr)
|
|
return
|
|
}
|
|
var got response
|
|
if err := json.Unmarshal(raw, &got); err != nil {
|
|
t.Errorf("decode response: %v", err)
|
|
return
|
|
}
|
|
result <- got
|
|
_ = connection.Close(websocket.StatusNormalClosure, "done")
|
|
}))
|
|
defer server.Close()
|
|
|
|
broker, _ := consolebroker.NewBroker(time.Minute)
|
|
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client.httpClient = server.Client()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() { done <- client.runSession(ctx) }()
|
|
select {
|
|
case got := <-result:
|
|
if got.Type != "response" || got.ID != "request_12345678" || got.Status != http.StatusCreated || got.Body != `{"status":"ok"}` {
|
|
t.Fatalf("control response = %#v", got)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("timed out waiting for control response")
|
|
}
|
|
select {
|
|
case <-done:
|
|
case <-ctx.Done():
|
|
t.Fatal("control session did not close")
|
|
}
|
|
}
|
|
|
|
func TestConsoleOpenCreatesReverseWSSAndRelaysBinary(t *testing.T) {
|
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
var operationError *net.OpError
|
|
if errors.As(err, &operationError) {
|
|
t.Skipf("sandbox does not permit loopback listeners: %v", err)
|
|
}
|
|
t.Fatal(err)
|
|
}
|
|
defer listener.Close()
|
|
consoleRelease := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
releaseConsole := func() { releaseOnce.Do(func() { close(consoleRelease) }) }
|
|
defer releaseConsole()
|
|
backendDone := make(chan struct{})
|
|
go func() {
|
|
defer close(backendDone)
|
|
connection, acceptErr := listener.Accept()
|
|
if acceptErr != nil {
|
|
return
|
|
}
|
|
defer connection.Close()
|
|
buffer := make([]byte, 128)
|
|
count, readErr := connection.Read(buffer)
|
|
if readErr == nil {
|
|
_, _ = connection.Write(buffer[:count])
|
|
}
|
|
<-consoleRelease
|
|
}()
|
|
|
|
broker, _ := consolebroker.NewBroker(time.Minute)
|
|
ticket, err := broker.Issue(listener.Addr().String())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const sessionID = "session_0123456789abcdef01234567"
|
|
const sessionToken = "token_0123456789abcdef0123456789abcdef"
|
|
controlResult := make(chan response, 1)
|
|
consoleResult := make(chan string, 1)
|
|
relayDone := make(chan struct{})
|
|
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
|
switch request.URL.Path {
|
|
case controlPath:
|
|
connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
|
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
|
|
})
|
|
if acceptErr != nil {
|
|
return
|
|
}
|
|
defer connection.CloseNow()
|
|
payload, _ := json.Marshal(message{
|
|
Type: "console_open", ID: "console_12345678", Ticket: ticket.Ticket,
|
|
SessionID: sessionID, SessionToken: sessionToken,
|
|
})
|
|
if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil {
|
|
return
|
|
}
|
|
messageType, raw, err := connection.Read(request.Context())
|
|
if err != nil || messageType != websocket.MessageText {
|
|
return
|
|
}
|
|
var got response
|
|
if json.Unmarshal(raw, &got) == nil {
|
|
controlResult <- got
|
|
}
|
|
select {
|
|
case <-relayDone:
|
|
case <-request.Context().Done():
|
|
}
|
|
_ = connection.Close(websocket.StatusNormalClosure, "done")
|
|
case consolePath:
|
|
expectedProtocol := ConsoleProtocol + sessionToken
|
|
if request.Header.Get("X-Virt-Agent-ID") != "42" ||
|
|
request.Header.Get("Authorization") != "" ||
|
|
request.Header.Get("X-Virt-Session-ID") != sessionID {
|
|
http.Error(writer, "bad console handshake", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
|
Subprotocols: []string{expectedProtocol}, CompressionMode: websocket.CompressionDisabled,
|
|
})
|
|
if acceptErr != nil {
|
|
return
|
|
}
|
|
defer connection.CloseNow()
|
|
if err := connection.Write(request.Context(), websocket.MessageBinary, []byte("RFB-test")); err != nil {
|
|
return
|
|
}
|
|
messageType, raw, err := connection.Read(request.Context())
|
|
if err == nil && messageType == websocket.MessageBinary {
|
|
consoleResult <- string(raw)
|
|
}
|
|
<-consoleRelease
|
|
close(relayDone)
|
|
default:
|
|
http.NotFound(writer, request)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client.httpClient = server.Client()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() { done <- client.runSession(ctx) }()
|
|
select {
|
|
case got := <-controlResult:
|
|
if got.Status != http.StatusOK || got.ID != "console_12345678" || !strings.Contains(got.Body, `"status":"connected"`) {
|
|
t.Fatalf("console-open response = %#v", got)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("timed out waiting for console-open response")
|
|
}
|
|
select {
|
|
case got := <-consoleResult:
|
|
if got != "RFB-test" {
|
|
t.Fatalf("reverse console body = %q", got)
|
|
}
|
|
if len(client.activeConsoles) != 1 {
|
|
t.Fatalf("active console slot was not held for relay lifetime: %d", len(client.activeConsoles))
|
|
}
|
|
releaseConsole()
|
|
case <-ctx.Done():
|
|
t.Fatal("timed out waiting for reverse console relay")
|
|
}
|
|
select {
|
|
case <-backendDone:
|
|
case <-ctx.Done():
|
|
t.Fatal("local console connection did not close")
|
|
}
|
|
select {
|
|
case <-done:
|
|
case <-ctx.Done():
|
|
t.Fatal("control session did not close")
|
|
}
|
|
deadline := time.Now().Add(time.Second)
|
|
for len(client.activeConsoles) != 0 && time.Now().Before(deadline) {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if got := len(client.activeConsoles); got != 0 {
|
|
t.Fatalf("active console slot was not released: %d", got)
|
|
}
|
|
}
|
|
|
|
func TestClientBoundsActiveConsoles(t *testing.T) {
|
|
client := &Client{activeConsoles: make(chan struct{}, maxActiveConsoles)}
|
|
for index := 0; index < maxActiveConsoles; index++ {
|
|
if !client.acquireConsole() {
|
|
t.Fatalf("console slot %d was rejected", index)
|
|
}
|
|
}
|
|
if client.acquireConsole() {
|
|
t.Fatal("console capacity exceeded configured limit")
|
|
}
|
|
client.releaseConsole()
|
|
if !client.acquireConsole() {
|
|
t.Fatal("released console capacity was not reusable")
|
|
}
|
|
}
|
|
|
|
func TestParsePanelOriginRejectsInvalidPorts(t *testing.T) {
|
|
for _, value := range []string{
|
|
"https://panel.example.com:",
|
|
"https://panel.example.com:0",
|
|
"https://panel.example.com:65536",
|
|
} {
|
|
if _, err := parsePanelOrigin(value); err == nil {
|
|
t.Fatalf("invalid panel origin %q was accepted", value)
|
|
}
|
|
}
|
|
if _, err := parsePanelOrigin("https://panel.example.com:443"); err != nil {
|
|
t.Fatalf("valid explicit TLS port was rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestControlRequestBodyAndResponseLimitsAreExact(t *testing.T) {
|
|
client := &Client{maxBodyBytes: 1024}
|
|
headers := map[string]string{
|
|
auth.HeaderTimestamp: "1",
|
|
auth.HeaderNonce: "0123456789abcdef0123456789abcdef",
|
|
auth.HeaderBodyDigest: "digest",
|
|
auth.HeaderActor: "actor",
|
|
auth.HeaderSignature: "signature",
|
|
}
|
|
request := message{
|
|
Type: "request", ID: "request_boundary", Method: http.MethodPost,
|
|
RequestTarget: "/v1/health", Body: strings.Repeat("x", 1024), Headers: headers,
|
|
}
|
|
if err := client.validateRequest(request); err != nil {
|
|
t.Fatalf("exact maximum request body was rejected: %v", err)
|
|
}
|
|
request.Body += "x"
|
|
if err := client.validateRequest(request); err == nil {
|
|
t.Fatal("oversized request body was accepted")
|
|
}
|
|
|
|
capture := newResponseCapture(8)
|
|
if count, err := capture.Write([]byte("12345678")); err != nil || count != 8 {
|
|
t.Fatalf("exact maximum response write count=%d err=%v", count, err)
|
|
}
|
|
status, body := capture.result()
|
|
if status != http.StatusOK || body != "12345678" {
|
|
t.Fatalf("exact maximum response = %d %q", status, body)
|
|
}
|
|
overflow := newResponseCapture(8)
|
|
if _, err := overflow.Write([]byte("123456789")); err == nil {
|
|
t.Fatal("oversized response body was accepted")
|
|
}
|
|
status, body = overflow.result()
|
|
if status != http.StatusBadGateway || !strings.Contains(body, `"code":"agent_response_too_large"`) {
|
|
t.Fatalf("oversized response = %d %q", status, body)
|
|
}
|
|
}
|
|
|
|
func TestControlSessionBoundsInFlightRequests(t *testing.T) {
|
|
release := make(chan struct{})
|
|
inner := http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
|
|
<-release
|
|
response.WriteHeader(http.StatusNoContent)
|
|
})
|
|
result := make(chan response, 1)
|
|
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
|
connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
|
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer connection.CloseNow()
|
|
headers := map[string]string{
|
|
auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef",
|
|
auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature",
|
|
}
|
|
for index := 0; index <= maxInFlight; index++ {
|
|
payload, _ := json.Marshal(message{
|
|
Type: "request", ID: fmt.Sprintf("request_%08d", index), Method: http.MethodGet,
|
|
RequestTarget: "/v1/health", Headers: headers,
|
|
})
|
|
if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil {
|
|
return
|
|
}
|
|
}
|
|
messageType, raw, err := connection.Read(request.Context())
|
|
if err != nil || messageType != websocket.MessageText {
|
|
return
|
|
}
|
|
var got response
|
|
if json.Unmarshal(raw, &got) == nil {
|
|
result <- got
|
|
}
|
|
_ = connection.Close(websocket.StatusNormalClosure, "done")
|
|
}))
|
|
defer server.Close()
|
|
|
|
broker, _ := consolebroker.NewBroker(time.Minute)
|
|
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client.httpClient = server.Client()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() { done <- client.runSession(ctx) }()
|
|
select {
|
|
case got := <-result:
|
|
if got.Status != http.StatusServiceUnavailable ||
|
|
!strings.Contains(got.Body, `"code":"agent_busy"`) ||
|
|
!strings.Contains(got.Body, `"dispatch_state":"not_dispatched"`) {
|
|
t.Fatalf("saturation response = %#v", got)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("timed out waiting for saturation response")
|
|
}
|
|
close(release)
|
|
select {
|
|
case <-done:
|
|
case <-ctx.Done():
|
|
t.Fatal("control session did not close")
|
|
}
|
|
}
|
|
|
|
func TestControlRequestDeadlineReturnsGatewayTimeout(t *testing.T) {
|
|
release := make(chan struct{}, 1)
|
|
defer func() {
|
|
select {
|
|
case release <- struct{}{}:
|
|
default:
|
|
}
|
|
}()
|
|
inner := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
|
|
<-release
|
|
})
|
|
result := make(chan response, 1)
|
|
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
|
connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
|
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer connection.CloseNow()
|
|
payload, _ := json.Marshal(message{
|
|
Type: "request", ID: "request_deadline", Method: http.MethodGet, RequestTarget: "/v1/health",
|
|
Headers: map[string]string{
|
|
auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef",
|
|
auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature",
|
|
},
|
|
})
|
|
if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil {
|
|
return
|
|
}
|
|
messageType, raw, err := connection.Read(request.Context())
|
|
if err != nil || messageType != websocket.MessageText {
|
|
return
|
|
}
|
|
var got response
|
|
if json.Unmarshal(raw, &got) == nil {
|
|
result <- got
|
|
}
|
|
_ = connection.Close(websocket.StatusNormalClosure, "done")
|
|
}))
|
|
defer server.Close()
|
|
|
|
broker, _ := consolebroker.NewBroker(time.Minute)
|
|
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client.requestTimeout = 25 * time.Millisecond
|
|
client.httpClient = server.Client()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() { done <- client.runSession(ctx) }()
|
|
select {
|
|
case got := <-result:
|
|
if got.Status != http.StatusGatewayTimeout || !strings.Contains(got.Body, `"code":"agent_request_timeout"`) {
|
|
t.Fatalf("deadline response = %#v", got)
|
|
}
|
|
deadline := time.Now().Add(time.Second)
|
|
for len(client.inFlight) != 0 && time.Now().Before(deadline) {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if len(client.inFlight) != 0 {
|
|
t.Fatalf("timed-out handler retained its WSS admission slot: %d", len(client.inFlight))
|
|
}
|
|
release <- struct{}{}
|
|
case <-ctx.Done():
|
|
t.Fatal("timed out waiting for request deadline response")
|
|
}
|
|
select {
|
|
case <-done:
|
|
case <-ctx.Done():
|
|
t.Fatal("control session did not close")
|
|
}
|
|
}
|
|
|
|
func newTLSServer(t *testing.T, handler http.Handler) *httptest.Server {
|
|
t.Helper()
|
|
var server *httptest.Server
|
|
var panicValue any
|
|
func() {
|
|
defer func() { panicValue = recover() }()
|
|
server = httptest.NewTLSServer(handler)
|
|
}()
|
|
if panicValue != nil {
|
|
t.Skipf("sandbox does not permit a TLS loopback server: %v", panicValue)
|
|
}
|
|
return server
|
|
}
|