package control import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "log" "net" "net/http" "net/http/httptest" "strings" "sync" "testing" "time" "github.com/coder/websocket" "git.luoli.live/root/virt/agent/auth" consolebroker "git.luoli.live/root/virt/agent/console" "git.luoli.live/root/virt/agent/operation" ) const controlTestSecret = "0123456789abcdef0123456789abcdef" func TestCompletionDeliveryWindowReleasesOnlyOnDisposition(t *testing.T) { window := newCompletionDeliveryWindow() for index := 0; index < completionWindowSize; index++ { if !window.reserve(context.Background()) { t.Fatal("completion window reservation failed") } window.track(fmt.Sprintf("completion_window_%02d", index), fmt.Sprintf("operation_window_%02d", index)) } blockedCtx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) defer cancel() if window.reserve(blockedCtx) { t.Fatal("completion window exceeded its bounded capacity") } if operationID, released := window.release("completion_window_00"); !released || operationID != "operation_window_00" { t.Fatal("ACK/retry disposition did not release the tracked event") } if !window.reserve(context.Background()) { t.Fatal("released completion slot was not reusable") } window.abandonReservation() if _, released := window.release("completion_window_00"); released { t.Fatal("duplicate disposition released a second slot") } } func TestCompletionMessageCarriesStrictImmutableOperationIdentity(t *testing.T) { receipt := operation.CompletionReceipt{ EventID: "completion_identity_123", OperationID: "operation_identity_123", Action: "delete-vm", ResourceID: "vm-completion-identity", RequestFingerprint: strings.Repeat("a", 64), Status: "failed", Result: json.RawMessage("null"), ErrorCode: "operation_failed", CauseCode: "backup_disk_transition_active", } message, err := completionMessageFromReceipt(receipt) if err != nil { t.Fatal(err) } if message.Action != receipt.Action || message.ResourceID != receipt.ResourceID || message.RequestFingerprint != receipt.RequestFingerprint || message.CauseCode != receipt.CauseCode { t.Fatalf("completion message identity = %#v", message) } tests := []struct { name string mutate func(*operation.CompletionReceipt) }{ {name: "missing action", mutate: func(value *operation.CompletionReceipt) { value.Action = "" }}, {name: "uppercase action", mutate: func(value *operation.CompletionReceipt) { value.Action = "Delete-VM" }}, {name: "resource path", mutate: func(value *operation.CompletionReceipt) { value.ResourceID = "vm/unsafe" }}, {name: "missing resource", mutate: func(value *operation.CompletionReceipt) { value.ResourceID = "" }}, {name: "uppercase fingerprint", mutate: func(value *operation.CompletionReceipt) { value.RequestFingerprint = strings.Repeat("A", 64) }}, {name: "short fingerprint", mutate: func(value *operation.CompletionReceipt) { value.RequestFingerprint = strings.Repeat("a", 63) }}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { candidate := receipt test.mutate(&candidate) if _, err := completionMessageFromReceipt(candidate); err == nil { t.Fatal("invalid immutable operation identity was accepted") } }) } } func TestControlSubprotocolUsesCleanBreakV3(t *testing.T) { if ControlSubprotocol != "virt-agent-control.v3" { t.Fatalf("control subprotocol = %q, want clean-break v3", ControlSubprotocol) } } func TestControlSessionRejectsLegacyV2GatewayDuringHandshake(t *testing.T) { server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { if request.Header.Get("Sec-WebSocket-Protocol") != "virt-agent-control.v2" { http.Error(writer, "legacy v2 control subprotocol required", http.StatusBadRequest) return } connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{ Subprotocols: []string{"virt-agent-control.v2"}, CompressionMode: websocket.CompressionDisabled, }) if err != nil { return } defer connection.CloseNow() _, _, _ = connection.Read(context.Background()) })) defer server.Close() broker, err := consolebroker.NewBroker(time.Minute) if err != nil { t.Fatal(err) } client, err := NewClient( server.URL, "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, time.Minute, ) if err != nil { t.Fatal(err) } client.httpClient = server.Client() err = client.runSession(context.Background()) if err == nil || !strings.Contains(err.Error(), "dial panel control channel") { t.Fatalf("legacy v2 gateway handshake error = %v", err) } } func TestCompletionDeliveryLogContainsOnlyStableMetadata(t *testing.T) { logger := log.Default() previousWriter, previousFlags, previousPrefix := logger.Writer(), logger.Flags(), logger.Prefix() var output bytes.Buffer logger.SetOutput(&output) logger.SetFlags(0) logger.SetPrefix("") t.Cleanup(func() { logger.SetOutput(previousWriter) logger.SetFlags(previousFlags) logger.SetPrefix(previousPrefix) }) logCompletionDelivery(completionPhaseAgentRetryReceived, "operation_log_123", "completion_log_123") got := strings.TrimSpace(output.String()) want := "completion_delivery phase=agent_retry_received operation_id=operation_log_123 event_id=completion_log_123" if got != want { t.Fatalf("completion log = %q, want %q", got, want) } } func TestControlDeadlineWrapsAPIOperationDeadline(t *testing.T) { broker, err := consolebroker.NewBroker(time.Minute) if err != nil { t.Fatal(err) } client, err := NewClient( "https://panel.example", "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, maximumAPITimeout, ) if err != nil { t.Fatal(err) } if client.requestTimeout != 855*time.Second { t.Fatalf("control request timeout = %s, want 855s", client.requestTimeout) } if _, err := NewClient( "https://panel.example", "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, maximumAPITimeout+time.Second, ); err == nil { t.Fatal("control client accepted an API timeout that crosses the Gateway margin") } } type controlPingerFunc func(context.Context) error func (function controlPingerFunc) Ping(ctx context.Context) error { return function(ctx) } func TestControlHeartbeatPingsUntilSessionCloses(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) pinged := make(chan struct{}, 1) finished := make(chan error, 1) go func() { finished <- maintainControlHeartbeat(ctx, controlPingerFunc( func(context.Context) error { select { case pinged <- struct{}{}: default: } return nil }, ), time.Millisecond, 50*time.Millisecond) }() select { case <-pinged: cancel() case <-time.After(time.Second): cancel() t.Fatal("control heartbeat did not send a Ping") } select { case err := <-finished: if err != nil { t.Fatalf("heartbeat returned an error after session cancellation: %v", err) } case <-time.After(time.Second): t.Fatal("control heartbeat did not stop with its session") } } func TestControlHeartbeatRejectsMissingPong(t *testing.T) { finished := make(chan error, 1) go func() { finished <- maintainControlHeartbeat(context.Background(), controlPingerFunc( func(ctx context.Context) error { <-ctx.Done() return ctx.Err() }, ), time.Millisecond, 5*time.Millisecond) }() select { case err := <-finished: if err == nil || !strings.Contains(err.Error(), "control ping failed") || !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("missing Pong error = %v", err) } case <-time.After(time.Second): t.Fatal("missing Pong did not fail the control heartbeat") } } func TestControlSessionAuthenticatesAndDispatchesInnerSignedRequest(t *testing.T) { now := time.Now().UTC().Truncate(time.Second) authenticator, err := auth.New([]byte(controlTestSecret), time.Minute, 2*time.Minute) if err != nil { t.Fatal(err) } authenticator.SetClockForTesting(func() time.Time { return now }) inner := http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { body, readErr := io.ReadAll(request.Body) if readErr != nil { t.Errorf("read reconstructed request: %v", readErr) response.WriteHeader(http.StatusBadRequest) return } actor, authErr := authenticator.Authenticate(request, body) if authErr != nil || actor.UserID != 42 || request.RequestURI != "/v1/health?full=1" || string(body) != `{"probe":true}` { t.Errorf("reconstructed request actor=%#v uri=%q body=%q err=%v", actor, request.RequestURI, body, authErr) response.WriteHeader(http.StatusUnauthorized) return } response.Header().Set("Content-Type", "application/json") response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"status":"ok"}`)) }) result := make(chan response, 1) server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { if request.URL.Path != controlPath || request.Header.Get("X-Virt-Agent-ID") != "42" || request.Header.Get("Authorization") != "Bearer "+controlTestSecret { http.Error(writer, "bad Agent handshake", http.StatusUnauthorized) return } connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{ Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled, }) if acceptErr != nil { return } defer connection.CloseNow() body := []byte(`{"probe":true}`) signed, _ := http.NewRequest(http.MethodPost, "https://agent.invalid/v1/health?full=1", bytes.NewReader(body)) _ = auth.Sign(signed, body, auth.Actor{UserID: 42}, []byte(controlTestSecret), now.Unix(), "0123456789abcdef0123456789abcdef") headers := make(map[string]string) for _, name := range []string{auth.HeaderTimestamp, auth.HeaderNonce, auth.HeaderBodyDigest, auth.HeaderActor, auth.HeaderSignature} { headers[name] = signed.Header.Get(name) } payload, _ := json.Marshal(message{ Type: "request", ID: "request_12345678", Method: http.MethodPost, RequestTarget: "/v1/health?full=1", Body: string(body), Headers: headers, }) if writeErr := connection.Write(request.Context(), websocket.MessageText, payload); writeErr != nil { t.Errorf("send request: %v", writeErr) return } messageType, raw, readErr := connection.Read(request.Context()) if readErr != nil || messageType != websocket.MessageText { t.Errorf("read response type=%v err=%v", messageType, readErr) return } var got response if err := json.Unmarshal(raw, &got); err != nil { t.Errorf("decode response: %v", err) return } result <- got _ = connection.Close(websocket.StatusNormalClosure, "done") })) defer server.Close() broker, _ := consolebroker.NewBroker(time.Minute) client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute) if err != nil { t.Fatal(err) } client.httpClient = server.Client() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() done := make(chan error, 1) go func() { done <- client.runSession(ctx) }() select { case got := <-result: if got.Type != "response" || got.ID != "request_12345678" || got.Status != http.StatusCreated || got.Body != `{"status":"ok"}` { t.Fatalf("control response = %#v", got) } case <-ctx.Done(): t.Fatal("timed out waiting for control response") } select { case <-done: case <-ctx.Done(): t.Fatal("control session did not close") } } func TestConsoleOpenCreatesReverseWSSAndRelaysBinary(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { var operationError *net.OpError if errors.As(err, &operationError) { t.Skipf("sandbox does not permit loopback listeners: %v", err) } t.Fatal(err) } defer listener.Close() consoleRelease := make(chan struct{}) var releaseOnce sync.Once releaseConsole := func() { releaseOnce.Do(func() { close(consoleRelease) }) } defer releaseConsole() backendDone := make(chan struct{}) go func() { defer close(backendDone) connection, acceptErr := listener.Accept() if acceptErr != nil { return } defer connection.Close() buffer := make([]byte, 128) count, readErr := connection.Read(buffer) if readErr == nil { _, _ = connection.Write(buffer[:count]) } <-consoleRelease }() broker, _ := consolebroker.NewBroker(time.Minute) ticket, err := broker.Issue(listener.Addr().String()) if err != nil { t.Fatal(err) } const sessionID = "session_0123456789abcdef01234567" const sessionToken = "token_0123456789abcdef0123456789abcdef" controlResult := make(chan response, 1) consoleResult := make(chan string, 1) relayDone := make(chan struct{}) server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { switch request.URL.Path { case controlPath: connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{ Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled, }) if acceptErr != nil { return } defer connection.CloseNow() payload, _ := json.Marshal(message{ Type: "console_open", ID: "console_12345678", Ticket: ticket.Ticket, SessionID: sessionID, SessionToken: sessionToken, }) if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil { return } messageType, raw, err := connection.Read(request.Context()) if err != nil || messageType != websocket.MessageText { return } var got response if json.Unmarshal(raw, &got) == nil { controlResult <- got } select { case <-relayDone: case <-request.Context().Done(): } _ = connection.Close(websocket.StatusNormalClosure, "done") case consolePath: expectedProtocol := ConsoleProtocol + sessionToken if request.Header.Get("X-Virt-Agent-ID") != "42" || request.Header.Get("Authorization") != "" || request.Header.Get("X-Virt-Session-ID") != sessionID { http.Error(writer, "bad console handshake", http.StatusUnauthorized) return } connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{ Subprotocols: []string{expectedProtocol}, CompressionMode: websocket.CompressionDisabled, }) if acceptErr != nil { return } defer connection.CloseNow() if err := connection.Write(request.Context(), websocket.MessageBinary, []byte("RFB-test")); err != nil { return } messageType, raw, err := connection.Read(request.Context()) if err == nil && messageType == websocket.MessageBinary { consoleResult <- string(raw) } <-consoleRelease close(relayDone) default: http.NotFound(writer, request) } })) defer server.Close() client, err := NewClient(server.URL, "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, time.Minute) if err != nil { t.Fatal(err) } client.httpClient = server.Client() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() done := make(chan error, 1) go func() { done <- client.runSession(ctx) }() select { case got := <-controlResult: if got.Status != http.StatusOK || got.ID != "console_12345678" || !strings.Contains(got.Body, `"status":"connected"`) { t.Fatalf("console-open response = %#v", got) } case <-ctx.Done(): t.Fatal("timed out waiting for console-open response") } select { case got := <-consoleResult: if got != "RFB-test" { t.Fatalf("reverse console body = %q", got) } if len(client.activeConsoles) != 1 { t.Fatalf("active console slot was not held for relay lifetime: %d", len(client.activeConsoles)) } releaseConsole() case <-ctx.Done(): t.Fatal("timed out waiting for reverse console relay") } select { case <-backendDone: case <-ctx.Done(): t.Fatal("local console connection did not close") } select { case <-done: case <-ctx.Done(): t.Fatal("control session did not close") } deadline := time.Now().Add(time.Second) for len(client.activeConsoles) != 0 && time.Now().Before(deadline) { time.Sleep(time.Millisecond) } if got := len(client.activeConsoles); got != 0 { t.Fatalf("active console slot was not released: %d", got) } } func TestClientBoundsActiveConsoles(t *testing.T) { client := &Client{activeConsoles: make(chan struct{}, maxActiveConsoles)} for index := 0; index < maxActiveConsoles; index++ { if !client.acquireConsole() { t.Fatalf("console slot %d was rejected", index) } } if client.acquireConsole() { t.Fatal("console capacity exceeded configured limit") } client.releaseConsole() if !client.acquireConsole() { t.Fatal("released console capacity was not reusable") } } func TestParsePanelOriginRejectsInvalidPorts(t *testing.T) { for _, value := range []string{ "https://panel.example.com:", "https://panel.example.com:0", "https://panel.example.com:65536", } { if _, err := parsePanelOrigin(value); err == nil { t.Fatalf("invalid panel origin %q was accepted", value) } } if _, err := parsePanelOrigin("https://panel.example.com:443"); err != nil { t.Fatalf("valid explicit TLS port was rejected: %v", err) } } func TestControlRequestBodyAndResponseLimitsAreExact(t *testing.T) { client := &Client{maxBodyBytes: 1024} headers := map[string]string{ auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef", auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature", } request := message{ Type: "request", ID: "request_boundary", Method: http.MethodPost, RequestTarget: "/v1/health", Body: strings.Repeat("x", 1024), Headers: headers, } if err := client.validateRequest(request); err != nil { t.Fatalf("exact maximum request body was rejected: %v", err) } request.Body += "x" if err := client.validateRequest(request); err == nil { t.Fatal("oversized request body was accepted") } capture := newResponseCapture(8) if count, err := capture.Write([]byte("12345678")); err != nil || count != 8 { t.Fatalf("exact maximum response write count=%d err=%v", count, err) } status, body := capture.result() if status != http.StatusOK || body != "12345678" { t.Fatalf("exact maximum response = %d %q", status, body) } overflow := newResponseCapture(8) if _, err := overflow.Write([]byte("123456789")); err == nil { t.Fatal("oversized response body was accepted") } status, body = overflow.result() if status != http.StatusBadGateway || !strings.Contains(body, `"code":"agent_response_too_large"`) { t.Fatalf("oversized response = %d %q", status, body) } } func TestControlSessionBoundsInFlightRequests(t *testing.T) { release := make(chan struct{}) inner := http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { <-release response.WriteHeader(http.StatusNoContent) }) result := make(chan response, 1) server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{ Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled, }) if err != nil { return } defer connection.CloseNow() headers := map[string]string{ auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef", auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature", } for index := 0; index <= maxInFlight; index++ { payload, _ := json.Marshal(message{ Type: "request", ID: fmt.Sprintf("request_%08d", index), Method: http.MethodGet, RequestTarget: "/v1/health", Headers: headers, }) if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil { return } } messageType, raw, err := connection.Read(request.Context()) if err != nil || messageType != websocket.MessageText { return } var got response if json.Unmarshal(raw, &got) == nil { result <- got } _ = connection.Close(websocket.StatusNormalClosure, "done") })) defer server.Close() broker, _ := consolebroker.NewBroker(time.Minute) client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute) if err != nil { t.Fatal(err) } client.httpClient = server.Client() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() done := make(chan error, 1) go func() { done <- client.runSession(ctx) }() select { case got := <-result: if got.Status != http.StatusServiceUnavailable || !strings.Contains(got.Body, `"code":"agent_busy"`) || !strings.Contains(got.Body, `"dispatch_state":"not_dispatched"`) { t.Fatalf("saturation response = %#v", got) } case <-ctx.Done(): t.Fatal("timed out waiting for saturation response") } close(release) select { case <-done: case <-ctx.Done(): t.Fatal("control session did not close") } } func TestControlRequestDeadlineReturnsGatewayTimeout(t *testing.T) { release := make(chan struct{}, 1) defer func() { select { case release <- struct{}{}: default: } }() inner := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { <-release }) result := make(chan response, 1) server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{ Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled, }) if err != nil { return } defer connection.CloseNow() payload, _ := json.Marshal(message{ Type: "request", ID: "request_deadline", Method: http.MethodGet, RequestTarget: "/v1/health", Headers: map[string]string{ auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef", auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature", }, }) if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil { return } messageType, raw, err := connection.Read(request.Context()) if err != nil || messageType != websocket.MessageText { return } var got response if json.Unmarshal(raw, &got) == nil { result <- got } _ = connection.Close(websocket.StatusNormalClosure, "done") })) defer server.Close() broker, _ := consolebroker.NewBroker(time.Minute) client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute) if err != nil { t.Fatal(err) } client.requestTimeout = 25 * time.Millisecond client.httpClient = server.Client() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() done := make(chan error, 1) go func() { done <- client.runSession(ctx) }() select { case got := <-result: if got.Status != http.StatusGatewayTimeout || !strings.Contains(got.Body, `"code":"agent_request_timeout"`) { t.Fatalf("deadline response = %#v", got) } deadline := time.Now().Add(time.Second) for len(client.inFlight) != 0 && time.Now().Before(deadline) { time.Sleep(time.Millisecond) } if len(client.inFlight) != 0 { t.Fatalf("timed-out handler retained its WSS admission slot: %d", len(client.inFlight)) } release <- struct{}{} case <-ctx.Done(): t.Fatal("timed out waiting for request deadline response") } select { case <-done: case <-ctx.Done(): t.Fatal("control session did not close") } } func newTLSServer(t *testing.T, handler http.Handler) *httptest.Server { t.Helper() var server *httptest.Server var panicValue any func() { defer func() { panicValue = recover() }() server = httptest.NewTLSServer(handler) }() if panicValue != nil { t.Skipf("sandbox does not permit a TLS loopback server: %v", panicValue) } return server }