You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
811 lines
26 KiB
Go
811 lines
26 KiB
Go
// Package control implements the Agent-initiated control and reverse-console
|
|
// channels. The panel still signs every inner API request; this transport only
|
|
// replaces the retired public HTTPS listener.
|
|
package control
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/tls"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/coder/websocket"
|
|
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
consolebroker "git.luoli.live/root/virt/agent/console"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
)
|
|
|
|
const (
|
|
ControlSubprotocol = "virt-agent-control.v2"
|
|
ConsoleProtocol = "virt-agent-console."
|
|
controlPath = "/agent/connect"
|
|
consolePath = "/agent/console"
|
|
maxControlFrame = 4 << 20
|
|
maxResponseBytes = 2 << 20
|
|
maxRequestBody = 1 << 20
|
|
maxInFlight = 32
|
|
maxActiveConsoles = 16
|
|
reconnectDelay = 5 * time.Second
|
|
controlPingInterval = 30 * time.Second
|
|
controlPingTimeout = 10 * time.Second
|
|
controlResultMargin = 15 * time.Second
|
|
maximumAPITimeout = 840 * time.Second
|
|
completionPollInterval = 5 * time.Second
|
|
completionWindowSize = 8
|
|
)
|
|
|
|
const (
|
|
completionPhaseAgentSent = "agent_sent"
|
|
completionPhaseAgentSendFailed = "agent_send_failed"
|
|
completionPhaseAgentACKReceived = "agent_ack_received"
|
|
completionPhaseAgentRetryReceived = "agent_retry_received"
|
|
)
|
|
|
|
var (
|
|
transportIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,128}$`)
|
|
agentIDPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`)
|
|
opaqueTokenPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,512}$`)
|
|
)
|
|
|
|
type message struct {
|
|
Type string `json:"type"`
|
|
ID string `json:"id"`
|
|
Method string `json:"method,omitempty"`
|
|
RequestTarget string `json:"request_target,omitempty"`
|
|
Body string `json:"body,omitempty"`
|
|
Headers map[string]string `json:"headers,omitempty"`
|
|
Ticket string `json:"ticket,omitempty"`
|
|
SessionID string `json:"session_id,omitempty"`
|
|
SessionToken string `json:"session_token,omitempty"`
|
|
}
|
|
|
|
type response struct {
|
|
Type string `json:"type"`
|
|
ID string `json:"id"`
|
|
Status int `json:"status"`
|
|
Body string `json:"body"`
|
|
}
|
|
|
|
type completionMessage struct {
|
|
Type string `json:"type"`
|
|
ID string `json:"id"`
|
|
OperationID string `json:"operation_id"`
|
|
Status string `json:"status"`
|
|
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
|
|
SafeToResubmit bool `json:"safe_to_resubmit"`
|
|
ErrorCode string `json:"error_code,omitempty"`
|
|
Result json.RawMessage `json:"result"`
|
|
}
|
|
|
|
type CompletionOutbox interface {
|
|
ClaimPendingCompletions(int, []string) ([]operation.CompletionReceipt, error)
|
|
AcknowledgeCompletion(string) error
|
|
}
|
|
|
|
type completionDeliveryWindow struct {
|
|
slots chan struct{}
|
|
wake chan struct{}
|
|
mu sync.Mutex
|
|
pending map[string]string
|
|
}
|
|
|
|
func newCompletionDeliveryWindow() *completionDeliveryWindow {
|
|
return &completionDeliveryWindow{
|
|
slots: make(chan struct{}, completionWindowSize), wake: make(chan struct{}, 1),
|
|
pending: make(map[string]string),
|
|
}
|
|
}
|
|
|
|
func (w *completionDeliveryWindow) reserve(ctx context.Context) bool {
|
|
select {
|
|
case w.slots <- struct{}{}:
|
|
return true
|
|
case <-ctx.Done():
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (w *completionDeliveryWindow) abandonReservation() {
|
|
<-w.slots
|
|
}
|
|
|
|
func (w *completionDeliveryWindow) track(eventID, operationID string) {
|
|
w.mu.Lock()
|
|
w.pending[eventID] = operationID
|
|
w.mu.Unlock()
|
|
}
|
|
|
|
func (w *completionDeliveryWindow) release(eventID string) (string, bool) {
|
|
w.mu.Lock()
|
|
operationID, ok := w.pending[eventID]
|
|
if !ok {
|
|
w.mu.Unlock()
|
|
return "", false
|
|
}
|
|
delete(w.pending, eventID)
|
|
w.mu.Unlock()
|
|
<-w.slots
|
|
select {
|
|
case w.wake <- struct{}{}:
|
|
default:
|
|
}
|
|
return operationID, true
|
|
}
|
|
|
|
func (w *completionDeliveryWindow) pendingIDs() []string {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
result := make([]string, 0, len(w.pending))
|
|
for eventID := range w.pending {
|
|
result = append(result, eventID)
|
|
}
|
|
return result
|
|
}
|
|
|
|
type Client struct {
|
|
panelOrigin *url.URL
|
|
agentID string
|
|
agentSecret string
|
|
handler http.Handler
|
|
console *consolebroker.Broker
|
|
maxBodyBytes int64
|
|
requestTimeout time.Duration
|
|
inFlight chan struct{}
|
|
activeConsoles chan struct{}
|
|
httpClient *http.Client
|
|
completionOutbox CompletionOutbox
|
|
}
|
|
|
|
func NewClient(
|
|
panelURL string,
|
|
agentID string,
|
|
agentSecret []byte,
|
|
handler http.Handler,
|
|
console *consolebroker.Broker,
|
|
maxBodyBytes int64,
|
|
operationTimeout time.Duration,
|
|
completionOutboxes ...CompletionOutbox,
|
|
) (*Client, error) {
|
|
origin, err := parsePanelOrigin(panelURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !agentIDPattern.MatchString(agentID) {
|
|
return nil, errors.New("agent ID must be a positive decimal host ID")
|
|
}
|
|
if len(agentSecret) < 32 || len(agentSecret) > 256 ||
|
|
strings.IndexFunc(string(agentSecret), func(value rune) bool { return value < 0x21 || value > 0x7e }) >= 0 {
|
|
return nil, errors.New("agent secret must contain 32-256 non-space printable ASCII bytes")
|
|
}
|
|
if handler == nil || console == nil {
|
|
return nil, errors.New("API handler and console broker are required")
|
|
}
|
|
if maxBodyBytes < 1024 || maxBodyBytes > maxRequestBody {
|
|
return nil, errors.New("maximum body size is invalid")
|
|
}
|
|
if operationTimeout < 5*time.Second || operationTimeout > maximumAPITimeout {
|
|
return nil, errors.New("operation timeout must be between 5 and 840 seconds")
|
|
}
|
|
dialer := &net.Dialer{Timeout: 15 * time.Second, KeepAlive: 30 * time.Second}
|
|
transport := &http.Transport{
|
|
Proxy: nil,
|
|
DialContext: dialer.DialContext,
|
|
ForceAttemptHTTP2: false,
|
|
TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12},
|
|
TLSHandshakeTimeout: 15 * time.Second,
|
|
ResponseHeaderTimeout: 15 * time.Second,
|
|
}
|
|
var completionOutbox CompletionOutbox
|
|
if len(completionOutboxes) > 1 {
|
|
return nil, errors.New("only one completion outbox may be configured")
|
|
}
|
|
if len(completionOutboxes) == 1 {
|
|
if completionOutboxes[0] == nil {
|
|
return nil, errors.New("completion outbox is nil")
|
|
}
|
|
completionOutbox = completionOutboxes[0]
|
|
}
|
|
return &Client{
|
|
panelOrigin: origin,
|
|
agentID: agentID,
|
|
agentSecret: string(append([]byte(nil), agentSecret...)),
|
|
handler: handler,
|
|
console: console,
|
|
maxBodyBytes: maxBodyBytes,
|
|
// The API owns the durable mutation deadline. Keep the WSS response
|
|
// envelope alive long enough to carry its structured result, while the
|
|
// Gateway still retains another 15 seconds before its 870s deadline.
|
|
requestTimeout: operationTimeout + controlResultMargin,
|
|
inFlight: make(chan struct{}, maxInFlight),
|
|
activeConsoles: make(chan struct{}, maxActiveConsoles),
|
|
httpClient: &http.Client{
|
|
Transport: transport,
|
|
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
|
|
return http.ErrUseLastResponse
|
|
},
|
|
},
|
|
completionOutbox: completionOutbox,
|
|
}, nil
|
|
}
|
|
|
|
// Run maintains one outbound control session. Bounded transport reconnects are
|
|
// independent from durable mutation execution; reconnecting never replays a
|
|
// completed or already-dispatched mutation.
|
|
func (c *Client) Run(ctx context.Context) error {
|
|
for {
|
|
err := c.runSession(ctx)
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
log.Printf("Agent control connection closed: %v; retrying in %s", err, reconnectDelay)
|
|
timer := time.NewTimer(reconnectDelay)
|
|
select {
|
|
case <-ctx.Done():
|
|
if !timer.Stop() {
|
|
<-timer.C
|
|
}
|
|
return nil
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *Client) runSession(parent context.Context) error {
|
|
ctx, cancel := context.WithCancel(parent)
|
|
defer cancel()
|
|
connection, response, err := websocket.Dial(ctx, c.websocketURL(controlPath), &websocket.DialOptions{
|
|
HTTPClient: c.httpClient,
|
|
HTTPHeader: c.controlHeaders(),
|
|
Subprotocols: []string{ControlSubprotocol},
|
|
CompressionMode: websocket.CompressionDisabled,
|
|
})
|
|
if response != nil && response.Body != nil {
|
|
_ = response.Body.Close()
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("dial panel control channel: %w", err)
|
|
}
|
|
defer connection.CloseNow()
|
|
if connection.Subprotocol() != ControlSubprotocol {
|
|
_ = connection.Close(websocket.StatusPolicyViolation, "control subprotocol required")
|
|
return errors.New("panel selected an invalid control subprotocol")
|
|
}
|
|
connection.SetReadLimit(maxControlFrame)
|
|
writer := &connectionWriter{connection: connection}
|
|
completionWindow := newCompletionDeliveryWindow()
|
|
heartbeatErrors := make(chan error, 1)
|
|
go func() {
|
|
if heartbeatErr := maintainControlHeartbeat(
|
|
ctx, connection, controlPingInterval, controlPingTimeout,
|
|
); heartbeatErr != nil {
|
|
heartbeatErrors <- heartbeatErr
|
|
cancel()
|
|
}
|
|
}()
|
|
completionErrors := make(chan error, 1)
|
|
if c.completionOutbox != nil {
|
|
go func() {
|
|
if completionErr := c.pumpCompletions(ctx, writer, completionWindow); completionErr != nil {
|
|
completionErrors <- completionErr
|
|
cancel()
|
|
}
|
|
}()
|
|
}
|
|
for {
|
|
messageType, payload, err := connection.Read(ctx)
|
|
if err != nil {
|
|
select {
|
|
case heartbeatErr := <-heartbeatErrors:
|
|
return fmt.Errorf("maintain panel control heartbeat: %w", heartbeatErr)
|
|
case completionErr := <-completionErrors:
|
|
return fmt.Errorf("deliver operation completion: %w", completionErr)
|
|
default:
|
|
}
|
|
return fmt.Errorf("read panel control message: %w", err)
|
|
}
|
|
if messageType != websocket.MessageText {
|
|
_ = connection.Close(websocket.StatusUnsupportedData, "text control messages required")
|
|
return errors.New("panel sent a non-text control message")
|
|
}
|
|
input, err := decodeMessage(payload)
|
|
if err != nil {
|
|
_ = connection.Close(websocket.StatusInvalidFramePayloadData, "invalid control message")
|
|
return err
|
|
}
|
|
switch input.Type {
|
|
case "request":
|
|
c.dispatch(ctx, writer, input, c.handleRequest)
|
|
case "console_open":
|
|
c.dispatch(ctx, writer, input, c.handleConsoleOpen)
|
|
case "completion_ack":
|
|
if c.completionOutbox == nil || !validCompletionDisposition(input) {
|
|
_ = connection.Close(websocket.StatusPolicyViolation, "invalid completion acknowledgement")
|
|
return errors.New("panel sent an invalid completion acknowledgement")
|
|
}
|
|
if err := c.completionOutbox.AcknowledgeCompletion(input.ID); err != nil {
|
|
return fmt.Errorf("acknowledge operation completion: %w", err)
|
|
}
|
|
if operationID, released := completionWindow.release(input.ID); released {
|
|
logCompletionDelivery(completionPhaseAgentACKReceived, operationID, input.ID)
|
|
}
|
|
case "completion_retry":
|
|
if c.completionOutbox == nil || !validCompletionDisposition(input) {
|
|
_ = connection.Close(websocket.StatusPolicyViolation, "invalid completion retry disposition")
|
|
return errors.New("panel sent an invalid completion retry disposition")
|
|
}
|
|
if operationID, released := completionWindow.release(input.ID); released {
|
|
logCompletionDelivery(completionPhaseAgentRetryReceived, operationID, input.ID)
|
|
}
|
|
default:
|
|
_ = connection.Close(websocket.StatusPolicyViolation, "unsupported control message")
|
|
return errors.New("panel sent an unsupported control message")
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *Client) pumpCompletions(ctx context.Context, writer *connectionWriter,
|
|
window *completionDeliveryWindow) error {
|
|
ticker := time.NewTicker(completionPollInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
if !window.reserve(ctx) {
|
|
return nil
|
|
}
|
|
completions, err := c.completionOutbox.ClaimPendingCompletions(1, window.pendingIDs())
|
|
if err != nil {
|
|
window.abandonReservation()
|
|
return err
|
|
}
|
|
if len(completions) == 1 {
|
|
completion := completions[0]
|
|
window.track(completion.EventID, completion.OperationID)
|
|
message := completionMessage{
|
|
Type: "completion", ID: completion.EventID, OperationID: completion.OperationID,
|
|
Status: completion.Status, Result: completion.Result,
|
|
ErrorCode: completion.ErrorCode, EffectMayHaveApplied: completion.EffectMayHaveApplied,
|
|
SafeToResubmit: completion.SafeToResubmit,
|
|
}
|
|
if err := writer.write(ctx, message); err != nil {
|
|
window.release(completion.EventID)
|
|
logCompletionDelivery(completionPhaseAgentSendFailed, completion.OperationID, completion.EventID)
|
|
return err
|
|
}
|
|
logCompletionDelivery(completionPhaseAgentSent, completion.OperationID, completion.EventID)
|
|
continue
|
|
}
|
|
window.abandonReservation()
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case <-ticker.C:
|
|
case <-window.wake:
|
|
}
|
|
}
|
|
}
|
|
|
|
func logCompletionDelivery(phase, operationID, eventID string) {
|
|
log.Printf("completion_delivery phase=%s operation_id=%s event_id=%s", phase, operationID, eventID)
|
|
}
|
|
|
|
func validCompletionDisposition(input message) bool {
|
|
return transportIDPattern.MatchString(input.ID) && input.Method == "" && input.RequestTarget == "" &&
|
|
input.Body == "" && len(input.Headers) == 0 && input.Ticket == "" &&
|
|
input.SessionID == "" && input.SessionToken == ""
|
|
}
|
|
|
|
type controlPinger interface {
|
|
Ping(context.Context) error
|
|
}
|
|
|
|
// maintainControlHeartbeat creates traffic in both directions often enough to
|
|
// keep an otherwise idle WSS session alive through reverse proxies and CDNs.
|
|
// Ping waits for the peer's Pong, so an unresponsive gateway also causes the
|
|
// session to close and reconnect instead of remaining half-open indefinitely.
|
|
func maintainControlHeartbeat(
|
|
ctx context.Context,
|
|
connection controlPinger,
|
|
interval time.Duration,
|
|
timeout time.Duration,
|
|
) error {
|
|
if interval <= 0 || timeout <= 0 {
|
|
return errors.New("control heartbeat timing is invalid")
|
|
}
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case <-ticker.C:
|
|
pingCtx, cancel := context.WithTimeout(ctx, timeout)
|
|
err := connection.Ping(pingCtx)
|
|
cancel()
|
|
if err != nil {
|
|
return fmt.Errorf("control ping failed: %w", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *Client) dispatch(
|
|
ctx context.Context,
|
|
writer *connectionWriter,
|
|
input message,
|
|
handle func(context.Context, context.Context, *connectionWriter, message),
|
|
) {
|
|
select {
|
|
case c.inFlight <- struct{}{}:
|
|
go func() {
|
|
defer func() { <-c.inFlight }()
|
|
requestCtx, cancel := context.WithTimeout(ctx, c.requestTimeout)
|
|
defer cancel()
|
|
handle(ctx, requestCtx, writer, input)
|
|
}()
|
|
default:
|
|
_ = writer.respond(ctx, notDispatchedErrorResponse(input.ID,
|
|
http.StatusServiceUnavailable, "agent_busy"))
|
|
}
|
|
}
|
|
|
|
func (c *Client) handleRequest(sessionCtx, requestCtx context.Context, writer *connectionWriter, input message) {
|
|
if err := c.validateRequest(input); err != nil {
|
|
_ = writer.respond(sessionCtx, notDispatchedErrorResponse(
|
|
input.ID, http.StatusBadRequest, "invalid_control_request",
|
|
))
|
|
return
|
|
}
|
|
requestURL, _ := url.ParseRequestURI(input.RequestTarget)
|
|
request := &http.Request{
|
|
Method: input.Method,
|
|
URL: requestURL,
|
|
RequestURI: input.RequestTarget,
|
|
Header: make(http.Header),
|
|
Body: io.NopCloser(strings.NewReader(input.Body)),
|
|
ContentLength: int64(len(input.Body)),
|
|
Host: "agent.internal",
|
|
}
|
|
request = request.WithContext(requestCtx)
|
|
for name, value := range input.Headers {
|
|
request.Header.Set(http.CanonicalHeaderKey(name), value)
|
|
}
|
|
type handlerResult struct {
|
|
status int
|
|
body string
|
|
}
|
|
done := make(chan handlerResult, 1)
|
|
go func() {
|
|
capture := newResponseCapture(maxResponseBytes)
|
|
defer func() {
|
|
if recover() != nil {
|
|
capture = newResponseCapture(maxResponseBytes)
|
|
capture.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = capture.Write([]byte(`{"error":{"code":"agent_internal_error","message":"Agent request failed","retryable":false}}`))
|
|
}
|
|
status, body := capture.result()
|
|
done <- handlerResult{status: status, body: body}
|
|
}()
|
|
c.handler.ServeHTTP(capture, request)
|
|
}()
|
|
select {
|
|
case result := <-done:
|
|
if errors.Is(requestCtx.Err(), context.DeadlineExceeded) {
|
|
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusGatewayTimeout, "agent_request_timeout"))
|
|
} else {
|
|
_ = writer.respond(sessionCtx, response{Type: "response", ID: input.ID, Status: result.status, Body: result.body})
|
|
}
|
|
case <-requestCtx.Done():
|
|
if errors.Is(requestCtx.Err(), context.DeadlineExceeded) {
|
|
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusGatewayTimeout, "agent_request_timeout"))
|
|
}
|
|
// Mutation handlers return after durable dispatch and run only in a
|
|
// supervised child process. A stuck read-only handler must not retain a
|
|
// WSS admission slot indefinitely; its buffered result is discarded.
|
|
return
|
|
}
|
|
}
|
|
|
|
func (c *Client) validateRequest(input message) error {
|
|
if !transportIDPattern.MatchString(input.ID) || input.Ticket != "" || input.SessionID != "" || input.SessionToken != "" {
|
|
return errors.New("invalid request envelope")
|
|
}
|
|
switch input.Method {
|
|
case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete:
|
|
default:
|
|
return errors.New("unsupported request method")
|
|
}
|
|
if len(input.RequestTarget) == 0 || len(input.RequestTarget) > 2048 ||
|
|
strings.ContainsAny(input.RequestTarget, "\x00\r\n\\# ") ||
|
|
(input.RequestTarget != "/v1" && !strings.HasPrefix(input.RequestTarget, "/v1/")) {
|
|
return errors.New("invalid request target")
|
|
}
|
|
parsed, err := url.ParseRequestURI(input.RequestTarget)
|
|
if err != nil || parsed.IsAbs() || parsed.Host != "" {
|
|
return errors.New("invalid request target")
|
|
}
|
|
probe := &http.Request{URL: parsed}
|
|
if auth.RequestTarget(probe) != input.RequestTarget {
|
|
return errors.New("request target is not canonical")
|
|
}
|
|
if int64(len(input.Body)) > c.maxBodyBytes {
|
|
return errors.New("request body exceeds configured limit")
|
|
}
|
|
required := map[string]bool{
|
|
http.CanonicalHeaderKey(auth.HeaderTimestamp): false,
|
|
http.CanonicalHeaderKey(auth.HeaderNonce): false,
|
|
http.CanonicalHeaderKey(auth.HeaderBodyDigest): false,
|
|
http.CanonicalHeaderKey(auth.HeaderActor): false,
|
|
http.CanonicalHeaderKey(auth.HeaderSignature): false,
|
|
}
|
|
for name, value := range input.Headers {
|
|
canonical := http.CanonicalHeaderKey(name)
|
|
if _, ok := required[canonical]; ok {
|
|
if value == "" || strings.ContainsAny(value, "\r\n") || required[canonical] {
|
|
return errors.New("invalid security header")
|
|
}
|
|
required[canonical] = true
|
|
continue
|
|
}
|
|
if (canonical == "Accept" || canonical == "Content-Type") && value == "application/json" {
|
|
continue
|
|
}
|
|
return errors.New("unsupported request header")
|
|
}
|
|
for _, present := range required {
|
|
if !present {
|
|
return errors.New("missing security header")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Client) handleConsoleOpen(sessionCtx, requestCtx context.Context, writer *connectionWriter, input message) {
|
|
if !transportIDPattern.MatchString(input.ID) || input.Method != "" || input.RequestTarget != "" ||
|
|
input.Body != "" || len(input.Headers) != 0 || !validTicket(input.Ticket) ||
|
|
!opaqueTokenPattern.MatchString(input.SessionID) || !opaqueTokenPattern.MatchString(input.SessionToken) {
|
|
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusBadRequest, "invalid_console_request"))
|
|
return
|
|
}
|
|
if !c.acquireConsole() {
|
|
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusServiceUnavailable, "console_capacity_exhausted"))
|
|
return
|
|
}
|
|
releaseConsole := true
|
|
defer func() {
|
|
if releaseConsole {
|
|
c.releaseConsole()
|
|
}
|
|
}()
|
|
backend, err := c.console.Open(requestCtx, input.Ticket)
|
|
if err != nil {
|
|
status := http.StatusBadGateway
|
|
code := "console_unavailable"
|
|
if errors.Is(err, consolebroker.ErrInvalidTicket) {
|
|
status = http.StatusUnauthorized
|
|
code = "invalid_console_ticket"
|
|
}
|
|
_ = writer.respond(sessionCtx, errorResponse(input.ID, status, code))
|
|
return
|
|
}
|
|
consoleHeaders := c.consoleHeaders(input.SessionID)
|
|
protocol := ConsoleProtocol + input.SessionToken
|
|
connection, handshake, err := websocket.Dial(requestCtx, c.websocketURL(consolePath), &websocket.DialOptions{
|
|
HTTPClient: c.httpClient,
|
|
HTTPHeader: consoleHeaders,
|
|
Subprotocols: []string{protocol},
|
|
CompressionMode: websocket.CompressionDisabled,
|
|
})
|
|
if handshake != nil && handshake.Body != nil {
|
|
_ = handshake.Body.Close()
|
|
}
|
|
if err != nil || connection == nil || connection.Subprotocol() != protocol {
|
|
_ = backend.Close()
|
|
if connection != nil {
|
|
_ = connection.CloseNow()
|
|
}
|
|
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusBadGateway, "console_gateway_unavailable"))
|
|
return
|
|
}
|
|
connection.SetReadLimit(8 << 20)
|
|
body, _ := json.Marshal(map[string]string{"session_id": input.SessionID, "status": "connected"})
|
|
if err := writer.respond(sessionCtx, response{Type: "response", ID: input.ID, Status: http.StatusOK, Body: string(body)}); err != nil {
|
|
_ = backend.Close()
|
|
_ = connection.CloseNow()
|
|
return
|
|
}
|
|
releaseConsole = false
|
|
go func() {
|
|
defer c.releaseConsole()
|
|
defer backend.Close()
|
|
defer connection.CloseNow()
|
|
c.console.Relay(sessionCtx, connection, backend)
|
|
}()
|
|
}
|
|
|
|
func (c *Client) acquireConsole() bool {
|
|
select {
|
|
case c.activeConsoles <- struct{}{}:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (c *Client) releaseConsole() {
|
|
<-c.activeConsoles
|
|
}
|
|
|
|
func (c *Client) controlHeaders() http.Header {
|
|
headers := make(http.Header)
|
|
headers.Set("X-Virt-Agent-ID", c.agentID)
|
|
headers.Set("Authorization", "Bearer "+c.agentSecret)
|
|
return headers
|
|
}
|
|
|
|
func (c *Client) consoleHeaders(sessionID string) http.Header {
|
|
headers := make(http.Header)
|
|
headers.Set("X-Virt-Agent-ID", c.agentID)
|
|
headers.Set("X-Virt-Session-ID", sessionID)
|
|
return headers
|
|
}
|
|
|
|
func (c *Client) websocketURL(path string) string {
|
|
value := *c.panelOrigin
|
|
value.Scheme = "wss"
|
|
value.Path = path
|
|
return value.String()
|
|
}
|
|
|
|
func parsePanelOrigin(value string) (*url.URL, error) {
|
|
parsed, err := url.Parse(value)
|
|
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.Hostname() == "" ||
|
|
parsed.User != nil || parsed.Opaque != "" || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" ||
|
|
parsed.RawPath != "" || (parsed.Path != "" && parsed.Path != "/") || strings.HasSuffix(parsed.Host, ":") {
|
|
return nil, errors.New("panel URL must be an HTTPS origin")
|
|
}
|
|
if parsed.Port() != "" {
|
|
port, portErr := strconv.Atoi(parsed.Port())
|
|
if portErr != nil || port < 1 || port > 65535 {
|
|
return nil, errors.New("panel URL contains an invalid port")
|
|
}
|
|
}
|
|
parsed.Path = ""
|
|
return parsed, nil
|
|
}
|
|
|
|
func decodeMessage(payload []byte) (message, error) {
|
|
decoder := json.NewDecoder(bytes.NewReader(payload))
|
|
decoder.DisallowUnknownFields()
|
|
var input message
|
|
if err := decoder.Decode(&input); err != nil {
|
|
return message{}, errors.New("decode control message")
|
|
}
|
|
var extra any
|
|
if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {
|
|
return message{}, errors.New("control message contains trailing data")
|
|
}
|
|
return input, nil
|
|
}
|
|
|
|
func validTicket(value string) bool {
|
|
decoded, err := base64.RawURLEncoding.DecodeString(value)
|
|
return err == nil && len(decoded) == 32
|
|
}
|
|
|
|
func errorResponse(id string, status int, code string) response {
|
|
body, _ := json.Marshal(map[string]any{
|
|
"error": map[string]any{"code": code, "message": "Agent control request failed", "retryable": false},
|
|
})
|
|
return response{Type: "response", ID: id, Status: status, Body: string(body)}
|
|
}
|
|
|
|
// notDispatchedErrorResponse is reserved for failures that occur before the
|
|
// signed request reaches the API handler (and therefore before the durable
|
|
// operation journal). The marker classifies host-effect/refund safety only;
|
|
// it never authorizes either side to requeue or automatically redispatch a
|
|
// mutation. A later explicit request must use a new operation identifier.
|
|
func notDispatchedErrorResponse(id string, status int, code string) response {
|
|
body, _ := json.Marshal(map[string]any{
|
|
"error": map[string]any{
|
|
"code": code, "message": "Agent control request was not dispatched",
|
|
"retryable": false, "dispatch_state": "not_dispatched",
|
|
},
|
|
})
|
|
return response{Type: "response", ID: id, Status: status, Body: string(body)}
|
|
}
|
|
|
|
type connectionWriter struct {
|
|
connection *websocket.Conn
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func (w *connectionWriter) respond(ctx context.Context, value response) error {
|
|
payload, err := json.Marshal(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(payload) > maxControlFrame {
|
|
value = errorResponse(value.ID, http.StatusBadGateway, "agent_response_too_large")
|
|
payload, err = json.Marshal(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return w.writePayload(ctx, payload)
|
|
}
|
|
|
|
func (w *connectionWriter) write(ctx context.Context, value any) error {
|
|
payload, err := json.Marshal(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(payload) > maxControlFrame {
|
|
return errors.New("control message exceeds frame limit")
|
|
}
|
|
return w.writePayload(ctx, payload)
|
|
}
|
|
|
|
func (w *connectionWriter) writePayload(ctx context.Context, payload []byte) error {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
writeContext, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
defer cancel()
|
|
return w.connection.Write(writeContext, websocket.MessageText, payload)
|
|
}
|
|
|
|
type responseCapture struct {
|
|
header http.Header
|
|
status int
|
|
body bytes.Buffer
|
|
limit int64
|
|
overflow bool
|
|
}
|
|
|
|
func newResponseCapture(limit int64) *responseCapture {
|
|
return &responseCapture{header: make(http.Header), limit: limit}
|
|
}
|
|
|
|
func (r *responseCapture) Header() http.Header { return r.header }
|
|
|
|
func (r *responseCapture) WriteHeader(status int) {
|
|
if r.status == 0 {
|
|
r.status = status
|
|
}
|
|
}
|
|
|
|
func (r *responseCapture) Write(value []byte) (int, error) {
|
|
if r.status == 0 {
|
|
r.status = http.StatusOK
|
|
}
|
|
remaining := r.limit - int64(r.body.Len())
|
|
if remaining <= 0 || int64(len(value)) > remaining {
|
|
r.overflow = true
|
|
return 0, errors.New("Agent response exceeds transport limit")
|
|
}
|
|
return r.body.Write(value)
|
|
}
|
|
|
|
func (r *responseCapture) result() (int, string) {
|
|
if r.overflow {
|
|
value := errorResponse("unused", http.StatusBadGateway, "agent_response_too_large")
|
|
return value.Status, value.Body
|
|
}
|
|
if r.status == 0 {
|
|
r.status = http.StatusOK
|
|
}
|
|
return r.status, r.body.String()
|
|
}
|