feat: adopt outbound agent gateway
parent
97a17edb92
commit
177fa24dd5
@ -0,0 +1,556 @@
|
||||
// Package control implements the Agent-initiated control and reverse-console
|
||||
// channels. The panel still signs every inner API request; this transport only
|
||||
// replaces the retired public HTTPS listener.
|
||||
package control
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
|
||||
"git.luoli.live/root/virt/agent/auth"
|
||||
consolebroker "git.luoli.live/root/virt/agent/console"
|
||||
)
|
||||
|
||||
const (
|
||||
ControlSubprotocol = "virt-agent-control.v1"
|
||||
ConsoleProtocol = "virt-agent-console."
|
||||
controlPath = "/agent/connect"
|
||||
consolePath = "/agent/console"
|
||||
maxControlFrame = 4 << 20
|
||||
maxResponseBytes = 2 << 20
|
||||
maxRequestBody = 1 << 20
|
||||
maxInFlight = 32
|
||||
maxActiveConsoles = 16
|
||||
reconnectDelay = 5 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
transportIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,128}$`)
|
||||
agentIDPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`)
|
||||
opaqueTokenPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,512}$`)
|
||||
)
|
||||
|
||||
type message struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Method string `json:"method,omitempty"`
|
||||
RequestTarget string `json:"request_target,omitempty"`
|
||||
Body string `json:"body,omitempty"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
Ticket string `json:"ticket,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
SessionToken string `json:"session_token,omitempty"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Status int `json:"status"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
panelOrigin *url.URL
|
||||
agentID string
|
||||
agentSecret string
|
||||
handler http.Handler
|
||||
console *consolebroker.Broker
|
||||
maxBodyBytes int64
|
||||
requestTimeout time.Duration
|
||||
inFlight chan struct{}
|
||||
activeConsoles chan struct{}
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewClient(
|
||||
panelURL string,
|
||||
agentID string,
|
||||
agentSecret []byte,
|
||||
handler http.Handler,
|
||||
console *consolebroker.Broker,
|
||||
maxBodyBytes int64,
|
||||
requestTimeout time.Duration,
|
||||
) (*Client, error) {
|
||||
origin, err := parsePanelOrigin(panelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !agentIDPattern.MatchString(agentID) {
|
||||
return nil, errors.New("agent ID must be a positive decimal host ID")
|
||||
}
|
||||
if len(agentSecret) < 32 || len(agentSecret) > 256 ||
|
||||
strings.IndexFunc(string(agentSecret), func(value rune) bool { return value < 0x21 || value > 0x7e }) >= 0 {
|
||||
return nil, errors.New("agent secret must contain 32-256 non-space printable ASCII bytes")
|
||||
}
|
||||
if handler == nil || console == nil {
|
||||
return nil, errors.New("API handler and console broker are required")
|
||||
}
|
||||
if maxBodyBytes < 1024 || maxBodyBytes > maxRequestBody {
|
||||
return nil, errors.New("maximum body size is invalid")
|
||||
}
|
||||
if requestTimeout < 5*time.Second || requestTimeout > time.Hour {
|
||||
return nil, errors.New("request timeout must be between 5 seconds and 1 hour")
|
||||
}
|
||||
dialer := &net.Dialer{Timeout: 15 * time.Second, KeepAlive: 30 * time.Second}
|
||||
transport := &http.Transport{
|
||||
Proxy: nil,
|
||||
DialContext: dialer.DialContext,
|
||||
ForceAttemptHTTP2: false,
|
||||
TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12},
|
||||
TLSHandshakeTimeout: 15 * time.Second,
|
||||
ResponseHeaderTimeout: 15 * time.Second,
|
||||
}
|
||||
return &Client{
|
||||
panelOrigin: origin,
|
||||
agentID: agentID,
|
||||
agentSecret: string(append([]byte(nil), agentSecret...)),
|
||||
handler: handler,
|
||||
console: console,
|
||||
maxBodyBytes: maxBodyBytes,
|
||||
requestTimeout: requestTimeout,
|
||||
inFlight: make(chan struct{}, maxInFlight),
|
||||
activeConsoles: make(chan struct{}, maxActiveConsoles),
|
||||
httpClient: &http.Client{
|
||||
Transport: transport,
|
||||
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run maintains one outbound control session. A fixed retry is intentional in
|
||||
// the first protocol version; durable operation safety does not depend on the
|
||||
// transport reconnect policy.
|
||||
func (c *Client) Run(ctx context.Context) error {
|
||||
for {
|
||||
err := c.runSession(ctx)
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
log.Printf("Agent control connection closed: %v; retrying in %s", err, reconnectDelay)
|
||||
timer := time.NewTimer(reconnectDelay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return nil
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) runSession(parent context.Context) error {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
defer cancel()
|
||||
connection, response, err := websocket.Dial(ctx, c.websocketURL(controlPath), &websocket.DialOptions{
|
||||
HTTPClient: c.httpClient,
|
||||
HTTPHeader: c.controlHeaders(),
|
||||
Subprotocols: []string{ControlSubprotocol},
|
||||
CompressionMode: websocket.CompressionDisabled,
|
||||
})
|
||||
if response != nil && response.Body != nil {
|
||||
_ = response.Body.Close()
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("dial panel control channel: %w", err)
|
||||
}
|
||||
defer connection.CloseNow()
|
||||
if connection.Subprotocol() != ControlSubprotocol {
|
||||
_ = connection.Close(websocket.StatusPolicyViolation, "control subprotocol required")
|
||||
return errors.New("panel selected an invalid control subprotocol")
|
||||
}
|
||||
connection.SetReadLimit(maxControlFrame)
|
||||
writer := &connectionWriter{connection: connection}
|
||||
for {
|
||||
messageType, payload, err := connection.Read(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read panel control message: %w", err)
|
||||
}
|
||||
if messageType != websocket.MessageText {
|
||||
_ = connection.Close(websocket.StatusUnsupportedData, "text control messages required")
|
||||
return errors.New("panel sent a non-text control message")
|
||||
}
|
||||
input, err := decodeMessage(payload)
|
||||
if err != nil {
|
||||
_ = connection.Close(websocket.StatusInvalidFramePayloadData, "invalid control message")
|
||||
return err
|
||||
}
|
||||
switch input.Type {
|
||||
case "request":
|
||||
c.dispatch(ctx, writer, input, c.handleRequest)
|
||||
case "console_open":
|
||||
c.dispatch(ctx, writer, input, c.handleConsoleOpen)
|
||||
default:
|
||||
_ = connection.Close(websocket.StatusPolicyViolation, "unsupported control message")
|
||||
return errors.New("panel sent an unsupported control message")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) dispatch(
|
||||
ctx context.Context,
|
||||
writer *connectionWriter,
|
||||
input message,
|
||||
handle func(context.Context, context.Context, *connectionWriter, message),
|
||||
) {
|
||||
select {
|
||||
case c.inFlight <- struct{}{}:
|
||||
go func() {
|
||||
defer func() { <-c.inFlight }()
|
||||
requestCtx, cancel := context.WithTimeout(ctx, c.requestTimeout)
|
||||
defer cancel()
|
||||
handle(ctx, requestCtx, writer, input)
|
||||
}()
|
||||
default:
|
||||
_ = writer.respond(ctx, notDispatchedErrorResponse(input.ID,
|
||||
http.StatusServiceUnavailable, "agent_busy"))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) handleRequest(sessionCtx, requestCtx context.Context, writer *connectionWriter, input message) {
|
||||
if err := c.validateRequest(input); err != nil {
|
||||
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusBadRequest, "invalid_control_request"))
|
||||
return
|
||||
}
|
||||
requestURL, _ := url.ParseRequestURI(input.RequestTarget)
|
||||
request := &http.Request{
|
||||
Method: input.Method,
|
||||
URL: requestURL,
|
||||
RequestURI: input.RequestTarget,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(input.Body)),
|
||||
ContentLength: int64(len(input.Body)),
|
||||
Host: "agent.internal",
|
||||
}
|
||||
request = request.WithContext(requestCtx)
|
||||
for name, value := range input.Headers {
|
||||
request.Header.Set(http.CanonicalHeaderKey(name), value)
|
||||
}
|
||||
type handlerResult struct {
|
||||
status int
|
||||
body string
|
||||
}
|
||||
done := make(chan handlerResult, 1)
|
||||
go func() {
|
||||
capture := newResponseCapture(maxResponseBytes)
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
capture = newResponseCapture(maxResponseBytes)
|
||||
capture.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = capture.Write([]byte(`{"error":{"code":"agent_internal_error","message":"Agent request failed","retryable":true}}`))
|
||||
}
|
||||
status, body := capture.result()
|
||||
done <- handlerResult{status: status, body: body}
|
||||
}()
|
||||
c.handler.ServeHTTP(capture, request)
|
||||
}()
|
||||
select {
|
||||
case result := <-done:
|
||||
if errors.Is(requestCtx.Err(), context.DeadlineExceeded) {
|
||||
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusGatewayTimeout, "agent_request_timeout"))
|
||||
} else {
|
||||
_ = writer.respond(sessionCtx, response{Type: "response", ID: input.ID, Status: result.status, Body: result.body})
|
||||
}
|
||||
case <-requestCtx.Done():
|
||||
if errors.Is(requestCtx.Err(), context.DeadlineExceeded) {
|
||||
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusGatewayTimeout, "agent_request_timeout"))
|
||||
}
|
||||
// Keep this admission slot occupied until an uncooperative handler
|
||||
// actually returns. This bounds leaked libvirt calls across reconnects.
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) validateRequest(input message) error {
|
||||
if !transportIDPattern.MatchString(input.ID) || input.Ticket != "" || input.SessionID != "" || input.SessionToken != "" {
|
||||
return errors.New("invalid request envelope")
|
||||
}
|
||||
switch input.Method {
|
||||
case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete:
|
||||
default:
|
||||
return errors.New("unsupported request method")
|
||||
}
|
||||
if len(input.RequestTarget) == 0 || len(input.RequestTarget) > 2048 ||
|
||||
strings.ContainsAny(input.RequestTarget, "\x00\r\n\\# ") ||
|
||||
(input.RequestTarget != "/v1" && !strings.HasPrefix(input.RequestTarget, "/v1/")) {
|
||||
return errors.New("invalid request target")
|
||||
}
|
||||
parsed, err := url.ParseRequestURI(input.RequestTarget)
|
||||
if err != nil || parsed.IsAbs() || parsed.Host != "" {
|
||||
return errors.New("invalid request target")
|
||||
}
|
||||
probe := &http.Request{URL: parsed}
|
||||
if auth.RequestTarget(probe) != input.RequestTarget {
|
||||
return errors.New("request target is not canonical")
|
||||
}
|
||||
if int64(len(input.Body)) > c.maxBodyBytes {
|
||||
return errors.New("request body exceeds configured limit")
|
||||
}
|
||||
required := map[string]bool{
|
||||
http.CanonicalHeaderKey(auth.HeaderTimestamp): false,
|
||||
http.CanonicalHeaderKey(auth.HeaderNonce): false,
|
||||
http.CanonicalHeaderKey(auth.HeaderBodyDigest): false,
|
||||
http.CanonicalHeaderKey(auth.HeaderActor): false,
|
||||
http.CanonicalHeaderKey(auth.HeaderSignature): false,
|
||||
}
|
||||
for name, value := range input.Headers {
|
||||
canonical := http.CanonicalHeaderKey(name)
|
||||
if _, ok := required[canonical]; ok {
|
||||
if value == "" || strings.ContainsAny(value, "\r\n") || required[canonical] {
|
||||
return errors.New("invalid security header")
|
||||
}
|
||||
required[canonical] = true
|
||||
continue
|
||||
}
|
||||
if (canonical == "Accept" || canonical == "Content-Type") && value == "application/json" {
|
||||
continue
|
||||
}
|
||||
return errors.New("unsupported request header")
|
||||
}
|
||||
for _, present := range required {
|
||||
if !present {
|
||||
return errors.New("missing security header")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) handleConsoleOpen(sessionCtx, requestCtx context.Context, writer *connectionWriter, input message) {
|
||||
if !transportIDPattern.MatchString(input.ID) || input.Method != "" || input.RequestTarget != "" ||
|
||||
input.Body != "" || len(input.Headers) != 0 || !validTicket(input.Ticket) ||
|
||||
!opaqueTokenPattern.MatchString(input.SessionID) || !opaqueTokenPattern.MatchString(input.SessionToken) {
|
||||
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusBadRequest, "invalid_console_request"))
|
||||
return
|
||||
}
|
||||
if !c.acquireConsole() {
|
||||
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusServiceUnavailable, "console_capacity_exhausted"))
|
||||
return
|
||||
}
|
||||
releaseConsole := true
|
||||
defer func() {
|
||||
if releaseConsole {
|
||||
c.releaseConsole()
|
||||
}
|
||||
}()
|
||||
backend, err := c.console.Open(requestCtx, input.Ticket)
|
||||
if err != nil {
|
||||
status := http.StatusBadGateway
|
||||
code := "console_unavailable"
|
||||
if errors.Is(err, consolebroker.ErrInvalidTicket) {
|
||||
status = http.StatusUnauthorized
|
||||
code = "invalid_console_ticket"
|
||||
}
|
||||
_ = writer.respond(sessionCtx, errorResponse(input.ID, status, code))
|
||||
return
|
||||
}
|
||||
consoleHeaders := c.consoleHeaders(input.SessionID)
|
||||
protocol := ConsoleProtocol + input.SessionToken
|
||||
connection, handshake, err := websocket.Dial(requestCtx, c.websocketURL(consolePath), &websocket.DialOptions{
|
||||
HTTPClient: c.httpClient,
|
||||
HTTPHeader: consoleHeaders,
|
||||
Subprotocols: []string{protocol},
|
||||
CompressionMode: websocket.CompressionDisabled,
|
||||
})
|
||||
if handshake != nil && handshake.Body != nil {
|
||||
_ = handshake.Body.Close()
|
||||
}
|
||||
if err != nil || connection == nil || connection.Subprotocol() != protocol {
|
||||
_ = backend.Close()
|
||||
if connection != nil {
|
||||
_ = connection.CloseNow()
|
||||
}
|
||||
_ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusBadGateway, "console_gateway_unavailable"))
|
||||
return
|
||||
}
|
||||
connection.SetReadLimit(8 << 20)
|
||||
body, _ := json.Marshal(map[string]string{"session_id": input.SessionID, "status": "connected"})
|
||||
if err := writer.respond(sessionCtx, response{Type: "response", ID: input.ID, Status: http.StatusOK, Body: string(body)}); err != nil {
|
||||
_ = backend.Close()
|
||||
_ = connection.CloseNow()
|
||||
return
|
||||
}
|
||||
releaseConsole = false
|
||||
go func() {
|
||||
defer c.releaseConsole()
|
||||
defer backend.Close()
|
||||
defer connection.CloseNow()
|
||||
c.console.Relay(sessionCtx, connection, backend)
|
||||
}()
|
||||
}
|
||||
|
||||
func (c *Client) acquireConsole() bool {
|
||||
select {
|
||||
case c.activeConsoles <- struct{}{}:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) releaseConsole() {
|
||||
<-c.activeConsoles
|
||||
}
|
||||
|
||||
func (c *Client) controlHeaders() http.Header {
|
||||
headers := make(http.Header)
|
||||
headers.Set("X-Virt-Agent-ID", c.agentID)
|
||||
headers.Set("Authorization", "Bearer "+c.agentSecret)
|
||||
return headers
|
||||
}
|
||||
|
||||
func (c *Client) consoleHeaders(sessionID string) http.Header {
|
||||
headers := make(http.Header)
|
||||
headers.Set("X-Virt-Agent-ID", c.agentID)
|
||||
headers.Set("X-Virt-Session-ID", sessionID)
|
||||
return headers
|
||||
}
|
||||
|
||||
func (c *Client) websocketURL(path string) string {
|
||||
value := *c.panelOrigin
|
||||
value.Scheme = "wss"
|
||||
value.Path = path
|
||||
return value.String()
|
||||
}
|
||||
|
||||
func parsePanelOrigin(value string) (*url.URL, error) {
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.Hostname() == "" ||
|
||||
parsed.User != nil || parsed.Opaque != "" || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" ||
|
||||
parsed.RawPath != "" || (parsed.Path != "" && parsed.Path != "/") || strings.HasSuffix(parsed.Host, ":") {
|
||||
return nil, errors.New("panel URL must be an HTTPS origin")
|
||||
}
|
||||
if parsed.Port() != "" {
|
||||
port, portErr := strconv.Atoi(parsed.Port())
|
||||
if portErr != nil || port < 1 || port > 65535 {
|
||||
return nil, errors.New("panel URL contains an invalid port")
|
||||
}
|
||||
}
|
||||
parsed.Path = ""
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func decodeMessage(payload []byte) (message, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
var input message
|
||||
if err := decoder.Decode(&input); err != nil {
|
||||
return message{}, errors.New("decode control message")
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {
|
||||
return message{}, errors.New("control message contains trailing data")
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
|
||||
func validTicket(value string) bool {
|
||||
decoded, err := base64.RawURLEncoding.DecodeString(value)
|
||||
return err == nil && len(decoded) == 32
|
||||
}
|
||||
|
||||
func errorResponse(id string, status int, code string) response {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"error": map[string]any{"code": code, "message": "Agent control request failed", "retryable": status >= 500},
|
||||
})
|
||||
return response{Type: "response", ID: id, Status: status, Body: string(body)}
|
||||
}
|
||||
|
||||
// notDispatchedErrorResponse is reserved for failures that occur before the
|
||||
// signed request reaches the API handler (and therefore before the durable
|
||||
// operation journal). The panel may safely put the same operation back in
|
||||
// its persistent queue only when this exact marker is present.
|
||||
func notDispatchedErrorResponse(id string, status int, code string) response {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": code, "message": "Agent control request was not dispatched",
|
||||
"retryable": true, "dispatch_state": "not_dispatched",
|
||||
},
|
||||
})
|
||||
return response{Type: "response", ID: id, Status: status, Body: string(body)}
|
||||
}
|
||||
|
||||
type connectionWriter struct {
|
||||
connection *websocket.Conn
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (w *connectionWriter) respond(ctx context.Context, value response) error {
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(payload) > maxControlFrame {
|
||||
value = errorResponse(value.ID, http.StatusBadGateway, "agent_response_too_large")
|
||||
payload, err = json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
writeContext, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
return w.connection.Write(writeContext, websocket.MessageText, payload)
|
||||
}
|
||||
|
||||
type responseCapture struct {
|
||||
header http.Header
|
||||
status int
|
||||
body bytes.Buffer
|
||||
limit int64
|
||||
overflow bool
|
||||
}
|
||||
|
||||
func newResponseCapture(limit int64) *responseCapture {
|
||||
return &responseCapture{header: make(http.Header), limit: limit}
|
||||
}
|
||||
|
||||
func (r *responseCapture) Header() http.Header { return r.header }
|
||||
|
||||
func (r *responseCapture) WriteHeader(status int) {
|
||||
if r.status == 0 {
|
||||
r.status = status
|
||||
}
|
||||
}
|
||||
|
||||
func (r *responseCapture) Write(value []byte) (int, error) {
|
||||
if r.status == 0 {
|
||||
r.status = http.StatusOK
|
||||
}
|
||||
remaining := r.limit - int64(r.body.Len())
|
||||
if remaining <= 0 || int64(len(value)) > remaining {
|
||||
r.overflow = true
|
||||
return 0, errors.New("Agent response exceeds transport limit")
|
||||
}
|
||||
return r.body.Write(value)
|
||||
}
|
||||
|
||||
func (r *responseCapture) result() (int, string) {
|
||||
if r.overflow {
|
||||
value := errorResponse("unused", http.StatusBadGateway, "agent_response_too_large")
|
||||
return value.Status, value.Body
|
||||
}
|
||||
if r.status == 0 {
|
||||
r.status = http.StatusOK
|
||||
}
|
||||
return r.status, r.body.String()
|
||||
}
|
||||
@ -0,0 +1,490 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
|
||||
"git.luoli.live/root/virt/agent/auth"
|
||||
consolebroker "git.luoli.live/root/virt/agent/console"
|
||||
)
|
||||
|
||||
const controlTestSecret = "0123456789abcdef0123456789abcdef"
|
||||
|
||||
func TestControlSessionAuthenticatesAndDispatchesInnerSignedRequest(t *testing.T) {
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
authenticator, err := auth.New([]byte(controlTestSecret), time.Minute, 2*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
authenticator.SetClockForTesting(func() time.Time { return now })
|
||||
inner := http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
body, readErr := io.ReadAll(request.Body)
|
||||
if readErr != nil {
|
||||
t.Errorf("read reconstructed request: %v", readErr)
|
||||
response.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
actor, authErr := authenticator.Authenticate(request, body)
|
||||
if authErr != nil || actor.UserID != 42 || request.RequestURI != "/v1/health?full=1" || string(body) != `{"probe":true}` {
|
||||
t.Errorf("reconstructed request actor=%#v uri=%q body=%q err=%v", actor, request.RequestURI, body, authErr)
|
||||
response.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
response.WriteHeader(http.StatusCreated)
|
||||
_, _ = response.Write([]byte(`{"status":"ok"}`))
|
||||
})
|
||||
|
||||
result := make(chan response, 1)
|
||||
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != controlPath || request.Header.Get("X-Virt-Agent-ID") != "42" ||
|
||||
request.Header.Get("Authorization") != "Bearer "+controlTestSecret {
|
||||
http.Error(writer, "bad Agent handshake", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
||||
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
|
||||
})
|
||||
if acceptErr != nil {
|
||||
return
|
||||
}
|
||||
defer connection.CloseNow()
|
||||
body := []byte(`{"probe":true}`)
|
||||
signed, _ := http.NewRequest(http.MethodPost, "https://agent.invalid/v1/health?full=1", bytes.NewReader(body))
|
||||
_ = auth.Sign(signed, body, auth.Actor{UserID: 42}, []byte(controlTestSecret), now.Unix(), "0123456789abcdef0123456789abcdef")
|
||||
headers := make(map[string]string)
|
||||
for _, name := range []string{auth.HeaderTimestamp, auth.HeaderNonce, auth.HeaderBodyDigest, auth.HeaderActor, auth.HeaderSignature} {
|
||||
headers[name] = signed.Header.Get(name)
|
||||
}
|
||||
payload, _ := json.Marshal(message{
|
||||
Type: "request", ID: "request_12345678", Method: http.MethodPost,
|
||||
RequestTarget: "/v1/health?full=1", Body: string(body), Headers: headers,
|
||||
})
|
||||
if writeErr := connection.Write(request.Context(), websocket.MessageText, payload); writeErr != nil {
|
||||
t.Errorf("send request: %v", writeErr)
|
||||
return
|
||||
}
|
||||
messageType, raw, readErr := connection.Read(request.Context())
|
||||
if readErr != nil || messageType != websocket.MessageText {
|
||||
t.Errorf("read response type=%v err=%v", messageType, readErr)
|
||||
return
|
||||
}
|
||||
var got response
|
||||
if err := json.Unmarshal(raw, &got); err != nil {
|
||||
t.Errorf("decode response: %v", err)
|
||||
return
|
||||
}
|
||||
result <- got
|
||||
_ = connection.Close(websocket.StatusNormalClosure, "done")
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
broker, _ := consolebroker.NewBroker(time.Minute)
|
||||
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.httpClient = server.Client()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- client.runSession(ctx) }()
|
||||
select {
|
||||
case got := <-result:
|
||||
if got.Type != "response" || got.ID != "request_12345678" || got.Status != http.StatusCreated || got.Body != `{"status":"ok"}` {
|
||||
t.Fatalf("control response = %#v", got)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("timed out waiting for control response")
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("control session did not close")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsoleOpenCreatesReverseWSSAndRelaysBinary(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
var operationError *net.OpError
|
||||
if errors.As(err, &operationError) {
|
||||
t.Skipf("sandbox does not permit loopback listeners: %v", err)
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
consoleRelease := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
releaseConsole := func() { releaseOnce.Do(func() { close(consoleRelease) }) }
|
||||
defer releaseConsole()
|
||||
backendDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(backendDone)
|
||||
connection, acceptErr := listener.Accept()
|
||||
if acceptErr != nil {
|
||||
return
|
||||
}
|
||||
defer connection.Close()
|
||||
buffer := make([]byte, 128)
|
||||
count, readErr := connection.Read(buffer)
|
||||
if readErr == nil {
|
||||
_, _ = connection.Write(buffer[:count])
|
||||
}
|
||||
<-consoleRelease
|
||||
}()
|
||||
|
||||
broker, _ := consolebroker.NewBroker(time.Minute)
|
||||
ticket, err := broker.Issue(listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const sessionID = "session_0123456789abcdef01234567"
|
||||
const sessionToken = "token_0123456789abcdef0123456789abcdef"
|
||||
controlResult := make(chan response, 1)
|
||||
consoleResult := make(chan string, 1)
|
||||
relayDone := make(chan struct{})
|
||||
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
switch request.URL.Path {
|
||||
case controlPath:
|
||||
connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
||||
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
|
||||
})
|
||||
if acceptErr != nil {
|
||||
return
|
||||
}
|
||||
defer connection.CloseNow()
|
||||
payload, _ := json.Marshal(message{
|
||||
Type: "console_open", ID: "console_12345678", Ticket: ticket.Ticket,
|
||||
SessionID: sessionID, SessionToken: sessionToken,
|
||||
})
|
||||
if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil {
|
||||
return
|
||||
}
|
||||
messageType, raw, err := connection.Read(request.Context())
|
||||
if err != nil || messageType != websocket.MessageText {
|
||||
return
|
||||
}
|
||||
var got response
|
||||
if json.Unmarshal(raw, &got) == nil {
|
||||
controlResult <- got
|
||||
}
|
||||
select {
|
||||
case <-relayDone:
|
||||
case <-request.Context().Done():
|
||||
}
|
||||
_ = connection.Close(websocket.StatusNormalClosure, "done")
|
||||
case consolePath:
|
||||
expectedProtocol := ConsoleProtocol + sessionToken
|
||||
if request.Header.Get("X-Virt-Agent-ID") != "42" ||
|
||||
request.Header.Get("Authorization") != "" ||
|
||||
request.Header.Get("X-Virt-Session-ID") != sessionID {
|
||||
http.Error(writer, "bad console handshake", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
||||
Subprotocols: []string{expectedProtocol}, CompressionMode: websocket.CompressionDisabled,
|
||||
})
|
||||
if acceptErr != nil {
|
||||
return
|
||||
}
|
||||
defer connection.CloseNow()
|
||||
if err := connection.Write(request.Context(), websocket.MessageBinary, []byte("RFB-test")); err != nil {
|
||||
return
|
||||
}
|
||||
messageType, raw, err := connection.Read(request.Context())
|
||||
if err == nil && messageType == websocket.MessageBinary {
|
||||
consoleResult <- string(raw)
|
||||
}
|
||||
<-consoleRelease
|
||||
close(relayDone)
|
||||
default:
|
||||
http.NotFound(writer, request)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.httpClient = server.Client()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- client.runSession(ctx) }()
|
||||
select {
|
||||
case got := <-controlResult:
|
||||
if got.Status != http.StatusOK || got.ID != "console_12345678" || !strings.Contains(got.Body, `"status":"connected"`) {
|
||||
t.Fatalf("console-open response = %#v", got)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("timed out waiting for console-open response")
|
||||
}
|
||||
select {
|
||||
case got := <-consoleResult:
|
||||
if got != "RFB-test" {
|
||||
t.Fatalf("reverse console body = %q", got)
|
||||
}
|
||||
if len(client.activeConsoles) != 1 {
|
||||
t.Fatalf("active console slot was not held for relay lifetime: %d", len(client.activeConsoles))
|
||||
}
|
||||
releaseConsole()
|
||||
case <-ctx.Done():
|
||||
t.Fatal("timed out waiting for reverse console relay")
|
||||
}
|
||||
select {
|
||||
case <-backendDone:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("local console connection did not close")
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("control session did not close")
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for len(client.activeConsoles) != 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if got := len(client.activeConsoles); got != 0 {
|
||||
t.Fatalf("active console slot was not released: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientBoundsActiveConsoles(t *testing.T) {
|
||||
client := &Client{activeConsoles: make(chan struct{}, maxActiveConsoles)}
|
||||
for index := 0; index < maxActiveConsoles; index++ {
|
||||
if !client.acquireConsole() {
|
||||
t.Fatalf("console slot %d was rejected", index)
|
||||
}
|
||||
}
|
||||
if client.acquireConsole() {
|
||||
t.Fatal("console capacity exceeded configured limit")
|
||||
}
|
||||
client.releaseConsole()
|
||||
if !client.acquireConsole() {
|
||||
t.Fatal("released console capacity was not reusable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePanelOriginRejectsInvalidPorts(t *testing.T) {
|
||||
for _, value := range []string{
|
||||
"https://panel.example.com:",
|
||||
"https://panel.example.com:0",
|
||||
"https://panel.example.com:65536",
|
||||
} {
|
||||
if _, err := parsePanelOrigin(value); err == nil {
|
||||
t.Fatalf("invalid panel origin %q was accepted", value)
|
||||
}
|
||||
}
|
||||
if _, err := parsePanelOrigin("https://panel.example.com:443"); err != nil {
|
||||
t.Fatalf("valid explicit TLS port was rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlRequestBodyAndResponseLimitsAreExact(t *testing.T) {
|
||||
client := &Client{maxBodyBytes: 1024}
|
||||
headers := map[string]string{
|
||||
auth.HeaderTimestamp: "1",
|
||||
auth.HeaderNonce: "0123456789abcdef0123456789abcdef",
|
||||
auth.HeaderBodyDigest: "digest",
|
||||
auth.HeaderActor: "actor",
|
||||
auth.HeaderSignature: "signature",
|
||||
}
|
||||
request := message{
|
||||
Type: "request", ID: "request_boundary", Method: http.MethodPost,
|
||||
RequestTarget: "/v1/health", Body: strings.Repeat("x", 1024), Headers: headers,
|
||||
}
|
||||
if err := client.validateRequest(request); err != nil {
|
||||
t.Fatalf("exact maximum request body was rejected: %v", err)
|
||||
}
|
||||
request.Body += "x"
|
||||
if err := client.validateRequest(request); err == nil {
|
||||
t.Fatal("oversized request body was accepted")
|
||||
}
|
||||
|
||||
capture := newResponseCapture(8)
|
||||
if count, err := capture.Write([]byte("12345678")); err != nil || count != 8 {
|
||||
t.Fatalf("exact maximum response write count=%d err=%v", count, err)
|
||||
}
|
||||
status, body := capture.result()
|
||||
if status != http.StatusOK || body != "12345678" {
|
||||
t.Fatalf("exact maximum response = %d %q", status, body)
|
||||
}
|
||||
overflow := newResponseCapture(8)
|
||||
if _, err := overflow.Write([]byte("123456789")); err == nil {
|
||||
t.Fatal("oversized response body was accepted")
|
||||
}
|
||||
status, body = overflow.result()
|
||||
if status != http.StatusBadGateway || !strings.Contains(body, `"code":"agent_response_too_large"`) {
|
||||
t.Fatalf("oversized response = %d %q", status, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlSessionBoundsInFlightRequests(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
inner := http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
|
||||
<-release
|
||||
response.WriteHeader(http.StatusNoContent)
|
||||
})
|
||||
result := make(chan response, 1)
|
||||
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
||||
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer connection.CloseNow()
|
||||
headers := map[string]string{
|
||||
auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef",
|
||||
auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature",
|
||||
}
|
||||
for index := 0; index <= maxInFlight; index++ {
|
||||
payload, _ := json.Marshal(message{
|
||||
Type: "request", ID: fmt.Sprintf("request_%08d", index), Method: http.MethodGet,
|
||||
RequestTarget: "/v1/health", Headers: headers,
|
||||
})
|
||||
if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
messageType, raw, err := connection.Read(request.Context())
|
||||
if err != nil || messageType != websocket.MessageText {
|
||||
return
|
||||
}
|
||||
var got response
|
||||
if json.Unmarshal(raw, &got) == nil {
|
||||
result <- got
|
||||
}
|
||||
_ = connection.Close(websocket.StatusNormalClosure, "done")
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
broker, _ := consolebroker.NewBroker(time.Minute)
|
||||
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.httpClient = server.Client()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- client.runSession(ctx) }()
|
||||
select {
|
||||
case got := <-result:
|
||||
if got.Status != http.StatusServiceUnavailable ||
|
||||
!strings.Contains(got.Body, `"code":"agent_busy"`) ||
|
||||
!strings.Contains(got.Body, `"dispatch_state":"not_dispatched"`) {
|
||||
t.Fatalf("saturation response = %#v", got)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("timed out waiting for saturation response")
|
||||
}
|
||||
close(release)
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("control session did not close")
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlRequestDeadlineReturnsGatewayTimeout(t *testing.T) {
|
||||
release := make(chan struct{}, 1)
|
||||
defer func() {
|
||||
select {
|
||||
case release <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}()
|
||||
inner := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
|
||||
<-release
|
||||
})
|
||||
result := make(chan response, 1)
|
||||
server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{
|
||||
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer connection.CloseNow()
|
||||
payload, _ := json.Marshal(message{
|
||||
Type: "request", ID: "request_deadline", Method: http.MethodGet, RequestTarget: "/v1/health",
|
||||
Headers: map[string]string{
|
||||
auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef",
|
||||
auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature",
|
||||
},
|
||||
})
|
||||
if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil {
|
||||
return
|
||||
}
|
||||
messageType, raw, err := connection.Read(request.Context())
|
||||
if err != nil || messageType != websocket.MessageText {
|
||||
return
|
||||
}
|
||||
var got response
|
||||
if json.Unmarshal(raw, &got) == nil {
|
||||
result <- got
|
||||
}
|
||||
_ = connection.Close(websocket.StatusNormalClosure, "done")
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
broker, _ := consolebroker.NewBroker(time.Minute)
|
||||
client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.requestTimeout = 25 * time.Millisecond
|
||||
client.httpClient = server.Client()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- client.runSession(ctx) }()
|
||||
select {
|
||||
case got := <-result:
|
||||
if got.Status != http.StatusGatewayTimeout || !strings.Contains(got.Body, `"code":"agent_request_timeout"`) {
|
||||
t.Fatalf("deadline response = %#v", got)
|
||||
}
|
||||
if len(client.inFlight) != 1 {
|
||||
t.Fatalf("timed-out handler released admission slot before returning: %d", len(client.inFlight))
|
||||
}
|
||||
release <- struct{}{}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("timed out waiting for request deadline response")
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("control session did not close")
|
||||
}
|
||||
}
|
||||
|
||||
func newTLSServer(t *testing.T, handler http.Handler) *httptest.Server {
|
||||
t.Helper()
|
||||
var server *httptest.Server
|
||||
var panicValue any
|
||||
func() {
|
||||
defer func() { panicValue = recover() }()
|
||||
server = httptest.NewTLSServer(handler)
|
||||
}()
|
||||
if panicValue != nil {
|
||||
t.Skipf("sandbox does not permit a TLS loopback server: %v", panicValue)
|
||||
}
|
||||
return server
|
||||
}
|
||||
@ -1,6 +1,8 @@
|
||||
{
|
||||
"listen_address": "127.0.0.1:8001",
|
||||
"panel_introspection_url": "http://127.0.0.1/api/internal/cloud/session-introspection",
|
||||
"panel_agent_auth_url": "http://127.0.0.1/api/internal/cloud/agent-auth",
|
||||
"panel_shared_secret": "REPLACE_ME",
|
||||
"connect_timeout_seconds": 10
|
||||
"connect_timeout_seconds": 10,
|
||||
"agent_request_timeout_seconds": 900
|
||||
}
|
||||
@ -0,0 +1,121 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.luoli.live/root/virt/gateway/internal/protocol"
|
||||
)
|
||||
|
||||
const maxPanelResponseBytes = 64 << 10
|
||||
|
||||
type ConnectionAuthenticator interface {
|
||||
Authenticate(context.Context, string, string) error
|
||||
}
|
||||
|
||||
type HTTPConnectionAuthenticator struct {
|
||||
endpoint string
|
||||
secret []byte
|
||||
client *http.Client
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewHTTPConnectionAuthenticator(endpoint string, secret []byte, timeout time.Duration) (*HTTPConnectionAuthenticator, error) {
|
||||
if err := validateLoopbackPanelURL(endpoint, "/api/internal/cloud/agent-auth"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(secret) < 32 || timeout <= 0 || timeout > 30*time.Second {
|
||||
return nil, errors.New("invalid Agent authentication client settings")
|
||||
}
|
||||
dialer := &net.Dialer{Timeout: timeout, KeepAlive: 30 * time.Second}
|
||||
transport := &http.Transport{
|
||||
Proxy: nil, DialContext: dialer.DialContext, ForceAttemptHTTP2: false,
|
||||
TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, TLSHandshakeTimeout: timeout,
|
||||
}
|
||||
return &HTTPConnectionAuthenticator{
|
||||
endpoint: endpoint,
|
||||
secret: append([]byte(nil), secret...),
|
||||
client: &http.Client{Transport: transport, Timeout: timeout,
|
||||
CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }},
|
||||
now: time.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *HTTPConnectionAuthenticator) Authenticate(ctx context.Context, agentID, agentSecret string) error {
|
||||
body, err := json.Marshal(struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
AgentSecret string `json:"agent_secret"`
|
||||
}{AgentID: agentID, AgentSecret: agentSecret})
|
||||
if err != nil {
|
||||
return errors.New("encode Agent authentication request")
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, a.endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return errors.New("create Agent authentication request")
|
||||
}
|
||||
request.Header.Set("Accept", "application/json")
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
nonce, err := protocol.RandomNonce()
|
||||
if err != nil {
|
||||
return errors.New("create Agent authentication nonce")
|
||||
}
|
||||
if err := protocol.Sign(request, body, protocol.Actor{Service: "agent-gateway"}, a.secret, a.now(), nonce); err != nil {
|
||||
return errors.New("sign Agent authentication request")
|
||||
}
|
||||
response, err := a.client.Do(request)
|
||||
if err != nil {
|
||||
return errors.New("panel Agent authentication unavailable")
|
||||
}
|
||||
defer response.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(response.Body, maxPanelResponseBytes+1))
|
||||
if err != nil || len(raw) > maxPanelResponseBytes || response.StatusCode != http.StatusOK {
|
||||
return errors.New("panel rejected Agent authentication")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
var result struct {
|
||||
Active bool `json:"active"`
|
||||
AgentID string `json:"agent_id"`
|
||||
}
|
||||
if err := decoder.Decode(&result); err != nil {
|
||||
return errors.New("panel Agent authentication response is invalid")
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) || !result.Active || result.AgentID != agentID {
|
||||
return errors.New("panel rejected Agent authentication")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateLoopbackPanelURL(value, requiredPath string) error {
|
||||
if value == "" || len(value) > 2048 || strings.ContainsAny(value, "\x00\r\n\\") {
|
||||
return errors.New("panel internal URL is invalid")
|
||||
}
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.User != nil ||
|
||||
parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" || parsed.Path != requiredPath || parsed.RawPath != "" ||
|
||||
strings.HasSuffix(parsed.Host, ":") {
|
||||
return errors.New("panel internal URL must use its fixed path")
|
||||
}
|
||||
ip := net.ParseIP(parsed.Hostname())
|
||||
if ip == nil || !ip.IsLoopback() {
|
||||
return errors.New("panel internal URL must use a literal loopback address")
|
||||
}
|
||||
if port := parsed.Port(); port != "" {
|
||||
number, err := strconv.Atoi(port)
|
||||
if err != nil || number < 1 || number > 65535 {
|
||||
return errors.New("panel internal URL has an invalid port")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,91 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.luoli.live/root/virt/gateway/internal/protocol"
|
||||
)
|
||||
|
||||
type authNonceStore struct {
|
||||
mu sync.Mutex
|
||||
m map[string]time.Time
|
||||
}
|
||||
|
||||
func (s *authNonceStore) UseNonce(nonce string, now, expiresAt time.Time) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if expiry := s.m[nonce]; expiry.After(now) {
|
||||
return false, nil
|
||||
}
|
||||
s.m[nonce] = expiresAt
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func TestHTTPConnectionAuthenticatorSignsSecretForFixedPHPRoute(t *testing.T) {
|
||||
secret := []byte(strings.Repeat("g", 32))
|
||||
verifier, _ := protocol.NewAuthenticator(secret, time.Minute, 2*time.Minute,
|
||||
&authNonceStore{m: make(map[string]time.Time)})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
body, _ := io.ReadAll(request.Body)
|
||||
actor, err := verifier.Authenticate(request, body)
|
||||
var input struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
AgentSecret string `json:"agent_secret"`
|
||||
}
|
||||
if err != nil || actor.Service != "agent-gateway" || json.Unmarshal(body, &input) != nil ||
|
||||
input.AgentID != "8" || input.AgentSecret != strings.Repeat("a", 32) {
|
||||
http.Error(response, "forbidden", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
_, _ = response.Write([]byte(`{"active":true,"agent_id":"8"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := NewHTTPConnectionAuthenticator(server.URL+"/api/internal/cloud/agent-auth",
|
||||
secret, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := client.Authenticate(t.Context(), "8", strings.Repeat("a", 32)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPConnectionAuthenticatorRejectsMismatchedPanelIdentity(t *testing.T) {
|
||||
secret := []byte(strings.Repeat("g", 32))
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
_, _ = response.Write([]byte(`{"active":true,"agent_id":"9"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := NewHTTPConnectionAuthenticator(server.URL+"/api/internal/cloud/agent-auth",
|
||||
secret, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := client.Authenticate(t.Context(), "8", strings.Repeat("a", 32)); err == nil {
|
||||
t.Fatal("mismatched panel Agent identity was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPConnectionAuthenticatorRejectsExplicitEmptyPort(t *testing.T) {
|
||||
for _, endpoint := range []string{
|
||||
"http://127.0.0.1:/api/internal/cloud/agent-auth",
|
||||
"http://127.0.0.1:0/api/internal/cloud/agent-auth",
|
||||
"http://127.0.0.1:65536/api/internal/cloud/agent-auth",
|
||||
"http://127.0.0.1/api/internal/cloud/agent-auth?",
|
||||
} {
|
||||
if _, err := NewHTTPConnectionAuthenticator(
|
||||
endpoint, []byte(strings.Repeat("g", 32)), time.Second,
|
||||
); err == nil {
|
||||
t.Fatalf("invalid Agent authentication endpoint accepted: %s", endpoint)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type memoryNonceStore struct {
|
||||
mu sync.Mutex
|
||||
values map[string]time.Time
|
||||
}
|
||||
|
||||
func newMemoryNonceStore() *memoryNonceStore {
|
||||
return &memoryNonceStore{values: make(map[string]time.Time)}
|
||||
}
|
||||
|
||||
func (s *memoryNonceStore) UseNonce(nonce string, now, expiresAt time.Time) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for value, expiry := range s.values {
|
||||
if !expiry.After(now) {
|
||||
delete(s.values, value)
|
||||
}
|
||||
}
|
||||
if expiry, exists := s.values[nonce]; exists && expiry.After(now) {
|
||||
return false, nil
|
||||
}
|
||||
s.values[nonce] = expiresAt
|
||||
return true, nil
|
||||
}
|
||||
@ -0,0 +1,684 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.luoli.live/root/virt/gateway/internal/protocol"
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
const (
|
||||
ControlPath = "/agent/connect"
|
||||
ConsolePath = "/agent/console"
|
||||
InternalRequestPath = "/internal/agent/request"
|
||||
ControlSubprotocol = "virt-agent-control.v1"
|
||||
ConsoleSubprotocolPrefix = "virt-agent-console."
|
||||
maxControlFrame = 4 << 20
|
||||
maxInternalRequest = 4 << 20
|
||||
maxAgentRequestBody = 1 << 20
|
||||
maxConsoleFrame = 8 << 20
|
||||
maxInFlightPerAgent = 32
|
||||
maxPendingConsoles = 64
|
||||
)
|
||||
|
||||
var (
|
||||
ErrAgentBusy = errors.New("Agent request capacity is exhausted")
|
||||
ErrConsoleBusy = errors.New("reverse console capacity is exhausted")
|
||||
ErrGatewayDraining = errors.New("Agent gateway is draining")
|
||||
ErrNotDispatched = errors.New("request was not dispatched to the Agent")
|
||||
)
|
||||
|
||||
var (
|
||||
agentIDPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`)
|
||||
requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`)
|
||||
tokenPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{32,256}$`)
|
||||
)
|
||||
|
||||
type wireRequest struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Method string `json:"method"`
|
||||
RequestTarget string `json:"request_target"`
|
||||
Body string `json:"body"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
}
|
||||
|
||||
type wireResponse struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Status int `json:"status"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
type consoleOpen struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Ticket string `json:"ticket"`
|
||||
SessionID string `json:"session_id"`
|
||||
SessionToken string `json:"session_token"`
|
||||
}
|
||||
|
||||
type dispatchResult struct {
|
||||
status int
|
||||
body []byte
|
||||
}
|
||||
|
||||
type pendingConsole struct {
|
||||
agentID string
|
||||
ready chan *websocket.Conn
|
||||
done <-chan struct{}
|
||||
}
|
||||
|
||||
type session struct {
|
||||
agentID string
|
||||
secret string
|
||||
conn *websocket.Conn
|
||||
writeMu sync.Mutex
|
||||
mu sync.Mutex
|
||||
pending map[string]chan dispatchResult
|
||||
inFlight chan struct{}
|
||||
done chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
type Registry struct {
|
||||
authenticator ConnectionAuthenticator
|
||||
requestAuth *protocol.Authenticator
|
||||
requestTimeout time.Duration
|
||||
consoleTimeout time.Duration
|
||||
mu sync.RWMutex
|
||||
sessions map[string]*session
|
||||
consoles map[string]pendingConsole
|
||||
lifecycleMu sync.Mutex
|
||||
draining bool
|
||||
active sync.WaitGroup
|
||||
}
|
||||
|
||||
func NewRegistry(authenticator ConnectionAuthenticator, sharedSecret []byte, requestTimeout, consoleTimeout time.Duration) (*Registry, error) {
|
||||
if authenticator == nil || len(sharedSecret) < 32 || requestTimeout <= 0 || requestTimeout > 15*time.Minute ||
|
||||
consoleTimeout <= 0 || consoleTimeout > 30*time.Second {
|
||||
return nil, errors.New("invalid Agent registry settings")
|
||||
}
|
||||
requestAuth, err := protocol.NewAuthenticator(sharedSecret, time.Minute, 2*time.Minute, newMemoryNonceStore())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Registry{
|
||||
authenticator: authenticator, requestAuth: requestAuth,
|
||||
requestTimeout: requestTimeout, consoleTimeout: consoleTimeout,
|
||||
sessions: make(map[string]*session), consoles: make(map[string]pendingConsole),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Registry) ServeControl(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Method != http.MethodGet || request.URL.Path != ControlPath || request.URL.RawQuery != "" {
|
||||
http.Error(response, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if !headerExactly(request.Header.Values("Sec-WebSocket-Protocol"), ControlSubprotocol) {
|
||||
http.Error(response, "Agent control subprotocol required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if r.isDraining() {
|
||||
http.Error(response, "Agent gateway is draining", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
agentID, secret, ok := agentCredentials(request)
|
||||
if !ok {
|
||||
http.Error(response, "Agent authentication required", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(request.Context(), r.consoleTimeout)
|
||||
err := r.authenticator.Authenticate(ctx, agentID, secret)
|
||||
cancel()
|
||||
if err != nil {
|
||||
http.Error(response, "Agent authentication failed", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
conn, err := websocket.Accept(response, request, &websocket.AcceptOptions{
|
||||
Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if conn.Subprotocol() != ControlSubprotocol {
|
||||
_ = conn.Close(websocket.StatusPolicyViolation, "control subprotocol required")
|
||||
return
|
||||
}
|
||||
conn.SetReadLimit(maxControlFrame)
|
||||
current := &session{
|
||||
agentID: agentID, secret: secret, conn: conn, pending: make(map[string]chan dispatchResult),
|
||||
inFlight: make(chan struct{}, maxInFlightPerAgent), done: make(chan struct{}),
|
||||
}
|
||||
if !r.register(current) {
|
||||
_ = conn.Close(websocket.StatusPolicyViolation, "Agent is already connected")
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
r.unregister(current)
|
||||
current.close()
|
||||
_ = conn.CloseNow()
|
||||
}()
|
||||
current.readLoop(request.Context())
|
||||
}
|
||||
|
||||
func (r *Registry) ServeInternalRequest(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Method != http.MethodPost || request.URL.Path != InternalRequestPath || request.URL.RawQuery != "" {
|
||||
writeInternalError(response, http.StatusNotFound, "gateway_route_not_found", "Agent gateway route not found")
|
||||
return
|
||||
}
|
||||
if !remoteIsLoopback(request.RemoteAddr) {
|
||||
writeInternalError(response, http.StatusForbidden, "gateway_forbidden", "Agent gateway request is forbidden")
|
||||
return
|
||||
}
|
||||
body, err := readBounded(response, request, maxInternalRequest)
|
||||
if err != nil {
|
||||
writeInternalError(response, http.StatusBadRequest, "gateway_request_invalid", "Agent gateway request is invalid")
|
||||
return
|
||||
}
|
||||
actor, err := r.requestAuth.Authenticate(request, body)
|
||||
if err != nil || !exactServiceActor(actor, "php-panel") {
|
||||
writeInternalError(response, http.StatusForbidden, "gateway_forbidden", "Agent gateway request is forbidden")
|
||||
return
|
||||
}
|
||||
var envelope struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
wireRequest
|
||||
}
|
||||
if err := decodeStrict(body, &envelope); err != nil || !agentIDPattern.MatchString(envelope.AgentID) || validateWireRequest(envelope.wireRequest) != nil {
|
||||
writeInternalError(response, http.StatusBadRequest, "gateway_request_invalid", "Agent gateway request is invalid")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(request.Context(), r.requestTimeout)
|
||||
defer cancel()
|
||||
result, err := r.dispatch(ctx, envelope.AgentID, envelope.wireRequest)
|
||||
if err != nil {
|
||||
status := http.StatusBadGateway
|
||||
code := "agent_disconnected"
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
status, code = http.StatusGatewayTimeout, "agent_timeout"
|
||||
} else if errors.Is(err, ErrAgentBusy) {
|
||||
status, code = http.StatusServiceUnavailable, "agent_busy"
|
||||
} else if errors.Is(err, ErrGatewayDraining) {
|
||||
status, code = http.StatusServiceUnavailable, "gateway_draining"
|
||||
}
|
||||
errorPayload := map[string]any{
|
||||
"code": code, "message": "host Agent is unavailable", "retryable": true,
|
||||
}
|
||||
if errors.Is(err, ErrNotDispatched) {
|
||||
errorPayload["dispatch_state"] = "not_dispatched"
|
||||
}
|
||||
writeJSON(response, status, map[string]any{"error": errorPayload})
|
||||
return
|
||||
}
|
||||
response.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
response.Header().Set("Cache-Control", "no-store")
|
||||
response.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
response.WriteHeader(result.status)
|
||||
_, _ = response.Write(result.body)
|
||||
}
|
||||
|
||||
func (r *Registry) OpenConsole(ctx context.Context, agentID, ticket, sessionID string) (*websocket.Conn, error) {
|
||||
if !agentIDPattern.MatchString(agentID) || !tokenPattern.MatchString(ticket) || !tokenPattern.MatchString(sessionID) {
|
||||
return nil, errors.New("invalid reverse console request")
|
||||
}
|
||||
if !r.beginActivity() {
|
||||
return nil, ErrGatewayDraining
|
||||
}
|
||||
defer r.active.Done()
|
||||
current := r.get(agentID)
|
||||
if current == nil {
|
||||
return nil, errors.New("Agent is not connected")
|
||||
}
|
||||
if err := r.revalidate(ctx, current); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
streamID, err := randomToken(16)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sessionToken, err := randomToken(32)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wait, cancel := context.WithTimeout(ctx, r.consoleTimeout)
|
||||
defer cancel()
|
||||
acknowledgement, releasePending, err := current.reservePending(streamID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer releasePending()
|
||||
ready := make(chan *websocket.Conn)
|
||||
r.mu.Lock()
|
||||
if len(r.consoles) >= maxPendingConsoles {
|
||||
r.mu.Unlock()
|
||||
return nil, ErrConsoleBusy
|
||||
}
|
||||
if _, exists := r.consoles[sessionToken]; exists {
|
||||
r.mu.Unlock()
|
||||
return nil, errors.New("reverse console token collision")
|
||||
}
|
||||
r.consoles[sessionToken] = pendingConsole{agentID: agentID, ready: ready, done: wait.Done()}
|
||||
r.mu.Unlock()
|
||||
defer func() {
|
||||
r.mu.Lock()
|
||||
delete(r.consoles, sessionToken)
|
||||
r.mu.Unlock()
|
||||
}()
|
||||
if err := current.write(wait, consoleOpen{
|
||||
Type: "console_open", ID: streamID, Ticket: ticket,
|
||||
SessionID: sessionID, SessionToken: sessionToken,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var conn *websocket.Conn
|
||||
acknowledged := false
|
||||
for !acknowledged || conn == nil {
|
||||
select {
|
||||
case result := <-acknowledgement:
|
||||
if result.status < http.StatusOK || result.status >= http.StatusMultipleChoices {
|
||||
if conn != nil {
|
||||
_ = conn.CloseNow()
|
||||
}
|
||||
return nil, fmt.Errorf("Agent rejected reverse console with status %d", result.status)
|
||||
}
|
||||
acknowledged = true
|
||||
case candidate := <-ready:
|
||||
if candidate == nil {
|
||||
return nil, errors.New("reverse console connection failed")
|
||||
}
|
||||
conn = candidate
|
||||
case <-current.done:
|
||||
if conn != nil {
|
||||
_ = conn.CloseNow()
|
||||
}
|
||||
return nil, errors.New("Agent disconnected while opening console")
|
||||
case <-wait.Done():
|
||||
if conn != nil {
|
||||
_ = conn.CloseNow()
|
||||
}
|
||||
return nil, wait.Err()
|
||||
}
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func (r *Registry) ServeAgentConsole(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Method != http.MethodGet || request.URL.Path != ConsolePath || request.URL.RawQuery != "" {
|
||||
http.Error(response, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
agentValues := request.Header.Values("X-Virt-Agent-ID")
|
||||
if len(agentValues) != 1 || !agentIDPattern.MatchString(agentValues[0]) {
|
||||
http.Error(response, "invalid Agent identity", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
protocolValue, token, ok := consoleProtocol(request.Header.Values("Sec-WebSocket-Protocol"))
|
||||
if !ok {
|
||||
http.Error(response, "reverse console token required", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
pending, exists := r.consoles[token]
|
||||
if exists && pending.agentID == agentValues[0] {
|
||||
delete(r.consoles, token)
|
||||
} else {
|
||||
exists = false
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if !exists {
|
||||
http.Error(response, "reverse console token is invalid or expired", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
conn, err := websocket.Accept(response, request, &websocket.AcceptOptions{
|
||||
Subprotocols: []string{protocolValue}, CompressionMode: websocket.CompressionDisabled,
|
||||
})
|
||||
if err != nil {
|
||||
select {
|
||||
case pending.ready <- nil:
|
||||
case <-pending.done:
|
||||
case <-request.Context().Done():
|
||||
}
|
||||
return
|
||||
}
|
||||
conn.SetReadLimit(maxConsoleFrame)
|
||||
select {
|
||||
case pending.ready <- conn:
|
||||
// Ownership transfers to OpenConsole and then to the browser relay.
|
||||
// Returning is required: a hijacked request context is cancelled only
|
||||
// after ServeHTTP returns, so waiting on it here would leak forever.
|
||||
return
|
||||
case <-pending.done:
|
||||
_ = conn.CloseNow()
|
||||
case <-request.Context().Done():
|
||||
_ = conn.CloseNow()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) register(value *session) bool {
|
||||
r.lifecycleMu.Lock()
|
||||
defer r.lifecycleMu.Unlock()
|
||||
if r.draining {
|
||||
return false
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, exists := r.sessions[value.agentID]; exists {
|
||||
return false
|
||||
}
|
||||
r.sessions[value.agentID] = value
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *Registry) unregister(value *session) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.sessions[value.agentID] == value {
|
||||
delete(r.sessions, value.agentID)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) get(agentID string) *session {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.sessions[agentID]
|
||||
}
|
||||
|
||||
func (r *Registry) dispatch(ctx context.Context, agentID string, request wireRequest) (dispatchResult, error) {
|
||||
if !r.beginActivity() {
|
||||
return dispatchResult{}, errors.Join(ErrNotDispatched, ErrGatewayDraining)
|
||||
}
|
||||
defer r.active.Done()
|
||||
current := r.get(agentID)
|
||||
if current == nil {
|
||||
return dispatchResult{}, errors.Join(ErrNotDispatched, errors.New("Agent is not connected"))
|
||||
}
|
||||
if err := r.revalidate(ctx, current); err != nil {
|
||||
return dispatchResult{}, errors.Join(ErrNotDispatched, err)
|
||||
}
|
||||
return current.dispatch(ctx, request)
|
||||
}
|
||||
|
||||
func (r *Registry) revalidate(ctx context.Context, current *session) error {
|
||||
if err := r.authenticator.Authenticate(ctx, current.agentID, current.secret); err != nil {
|
||||
r.unregister(current)
|
||||
current.close()
|
||||
_ = current.conn.CloseNow()
|
||||
return errors.New("Agent credentials are no longer valid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *session) dispatch(ctx context.Context, request wireRequest) (dispatchResult, error) {
|
||||
result, release, err := s.reservePending(request.ID)
|
||||
if err != nil {
|
||||
return dispatchResult{}, errors.Join(ErrNotDispatched, err)
|
||||
}
|
||||
defer release()
|
||||
if err := s.write(ctx, request); err != nil {
|
||||
return dispatchResult{}, err
|
||||
}
|
||||
select {
|
||||
case response := <-result:
|
||||
return response, nil
|
||||
case <-s.done:
|
||||
return dispatchResult{}, errors.New("Agent disconnected")
|
||||
case <-ctx.Done():
|
||||
return dispatchResult{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) beginActivity() bool {
|
||||
r.lifecycleMu.Lock()
|
||||
defer r.lifecycleMu.Unlock()
|
||||
if r.draining {
|
||||
return false
|
||||
}
|
||||
r.active.Add(1)
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *Registry) isDraining() bool {
|
||||
r.lifecycleMu.Lock()
|
||||
defer r.lifecycleMu.Unlock()
|
||||
return r.draining
|
||||
}
|
||||
|
||||
// BeginDrain stops new control sessions, requests, and console opens while
|
||||
// allowing work that already crossed the gateway admission boundary to finish.
|
||||
func (r *Registry) BeginDrain() {
|
||||
r.lifecycleMu.Lock()
|
||||
r.draining = true
|
||||
r.lifecycleMu.Unlock()
|
||||
}
|
||||
|
||||
// Wait blocks until admitted dispatches and reverse-console opens have
|
||||
// returned. BeginDrain must be called first so no new WaitGroup additions race
|
||||
// with this wait.
|
||||
func (r *Registry) Wait(ctx context.Context) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
r.active.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Close aborts any remaining hijacked control sockets after the graceful
|
||||
// drain deadline. Active reverse-console data sockets are owned by Relay.
|
||||
func (r *Registry) Close() {
|
||||
r.BeginDrain()
|
||||
r.mu.RLock()
|
||||
sessions := make([]*session, 0, len(r.sessions))
|
||||
for _, current := range r.sessions {
|
||||
sessions = append(sessions, current)
|
||||
}
|
||||
r.mu.RUnlock()
|
||||
for _, current := range sessions {
|
||||
current.close()
|
||||
if current.conn != nil {
|
||||
_ = current.conn.CloseNow()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *session) reservePending(id string) (chan dispatchResult, func(), error) {
|
||||
select {
|
||||
case s.inFlight <- struct{}{}:
|
||||
default:
|
||||
return nil, nil, ErrAgentBusy
|
||||
}
|
||||
result := make(chan dispatchResult, 1)
|
||||
s.mu.Lock()
|
||||
if _, exists := s.pending[id]; exists {
|
||||
s.mu.Unlock()
|
||||
<-s.inFlight
|
||||
return nil, nil, errors.New("request ID is already pending")
|
||||
}
|
||||
s.pending[id] = result
|
||||
s.mu.Unlock()
|
||||
var once sync.Once
|
||||
release := func() {
|
||||
once.Do(func() {
|
||||
s.mu.Lock()
|
||||
delete(s.pending, id)
|
||||
s.mu.Unlock()
|
||||
<-s.inFlight
|
||||
})
|
||||
}
|
||||
return result, release, nil
|
||||
}
|
||||
|
||||
func (s *session) write(ctx context.Context, value any) error {
|
||||
body, err := json.Marshal(value)
|
||||
if err != nil || len(body) > maxControlFrame {
|
||||
return errors.New("encode Agent control message")
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
return s.conn.Write(ctx, websocket.MessageText, body)
|
||||
}
|
||||
|
||||
func (s *session) readLoop(ctx context.Context) {
|
||||
for {
|
||||
messageType, raw, err := s.conn.Read(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if messageType != websocket.MessageText || len(raw) > maxControlFrame {
|
||||
_ = s.conn.Close(websocket.StatusUnsupportedData, "text control messages required")
|
||||
return
|
||||
}
|
||||
var response wireResponse
|
||||
if decodeStrict(raw, &response) != nil || response.Type != "response" ||
|
||||
!requestIDPattern.MatchString(response.ID) || response.Status < 200 || response.Status > 599 ||
|
||||
len(response.Body) == 0 || len(response.Body) > maxControlFrame || !json.Valid([]byte(response.Body)) {
|
||||
_ = s.conn.Close(websocket.StatusPolicyViolation, "invalid control response")
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
pending := s.pending[response.ID]
|
||||
s.mu.Unlock()
|
||||
if pending != nil {
|
||||
select {
|
||||
case pending <- dispatchResult{status: response.Status, body: []byte(response.Body)}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *session) close() {
|
||||
s.once.Do(func() { close(s.done) })
|
||||
}
|
||||
|
||||
func validateWireRequest(request wireRequest) error {
|
||||
if request.Type != "request" || !requestIDPattern.MatchString(request.ID) ||
|
||||
!validMethod(request.Method) || !validRequestTarget(request.RequestTarget) ||
|
||||
len(request.Body) > maxAgentRequestBody || (request.Body != "" && !json.Valid([]byte(request.Body))) ||
|
||||
len(request.Headers) > 32 {
|
||||
return errors.New("invalid request envelope")
|
||||
}
|
||||
for name, value := range request.Headers {
|
||||
if name == "" || len(name) > 128 || len(value) > 4096 || strings.ContainsAny(name+value, "\x00\r\n") {
|
||||
return errors.New("invalid request headers")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validMethod(value string) bool {
|
||||
switch value {
|
||||
case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func validRequestTarget(value string) bool {
|
||||
return len(value) <= 2048 && (value == "/v1" || strings.HasPrefix(value, "/v1/")) &&
|
||||
!strings.ContainsAny(value, "\x00\r\n\\# ")
|
||||
}
|
||||
|
||||
func agentCredentials(request *http.Request) (string, string, bool) {
|
||||
ids := request.Header.Values("X-Virt-Agent-ID")
|
||||
authorizations := request.Header.Values("Authorization")
|
||||
if len(ids) != 1 || len(authorizations) != 1 || !agentIDPattern.MatchString(ids[0]) ||
|
||||
!strings.HasPrefix(authorizations[0], "Bearer ") {
|
||||
return "", "", false
|
||||
}
|
||||
secret := strings.TrimPrefix(authorizations[0], "Bearer ")
|
||||
if len(secret) < 32 || len(secret) > 256 || strings.ContainsAny(secret, "\x00\r\n \t") {
|
||||
return "", "", false
|
||||
}
|
||||
return ids[0], secret, true
|
||||
}
|
||||
|
||||
func headerExactly(values []string, expected string) bool {
|
||||
return len(values) == 1 && strings.TrimSpace(values[0]) == expected
|
||||
}
|
||||
|
||||
func consoleProtocol(values []string) (string, string, bool) {
|
||||
if len(values) != 1 || strings.Contains(values[0], ",") {
|
||||
return "", "", false
|
||||
}
|
||||
value := strings.TrimSpace(values[0])
|
||||
if !strings.HasPrefix(value, ConsoleSubprotocolPrefix) {
|
||||
return "", "", false
|
||||
}
|
||||
token := strings.TrimPrefix(value, ConsoleSubprotocolPrefix)
|
||||
return value, token, tokenPattern.MatchString(token)
|
||||
}
|
||||
|
||||
func randomToken(size int) (string, error) {
|
||||
value := make([]byte, size)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return "", fmt.Errorf("generate token: %w", err)
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(value), nil
|
||||
}
|
||||
|
||||
func remoteIsLoopback(value string) bool {
|
||||
host, _, err := net.SplitHostPort(value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
|
||||
func exactServiceActor(actor protocol.Actor, service string) bool {
|
||||
return actor.Service == service && actor.UserID == 0 && !actor.IsAdmin &&
|
||||
!actor.IsSuperAdmin && !actor.CanManage
|
||||
}
|
||||
|
||||
func readBounded(response http.ResponseWriter, request *http.Request, maximum int64) ([]byte, error) {
|
||||
request.Body = http.MaxBytesReader(response, request.Body, maximum)
|
||||
defer request.Body.Close()
|
||||
return io.ReadAll(request.Body)
|
||||
}
|
||||
|
||||
func decodeStrict(raw []byte, target any) error {
|
||||
decoder := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {
|
||||
return errors.New("trailing JSON data")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeJSON(response http.ResponseWriter, status int, value any) {
|
||||
response.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
response.Header().Set("Cache-Control", "no-store")
|
||||
response.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
response.WriteHeader(status)
|
||||
_ = json.NewEncoder(response).Encode(value)
|
||||
}
|
||||
|
||||
func writeInternalError(response http.ResponseWriter, status int, code, message string) {
|
||||
writeJSON(response, status, map[string]any{"error": map[string]any{
|
||||
"code": code, "message": message, "retryable": false,
|
||||
}})
|
||||
}
|
||||
@ -0,0 +1,532 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.luoli.live/root/virt/gateway/internal/protocol"
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
func TestWireRequestBodyLimitIsExact(t *testing.T) {
|
||||
request := wireRequest{
|
||||
Type: "request", ID: "request_boundary", Method: http.MethodPost,
|
||||
RequestTarget: "/v1/health", Body: `{"payload":"` + strings.Repeat("x", maxAgentRequestBody-14) + `"}`,
|
||||
Headers: map[string]string{},
|
||||
}
|
||||
if len(request.Body) != maxAgentRequestBody {
|
||||
t.Fatalf("boundary fixture length = %d", len(request.Body))
|
||||
}
|
||||
if err := validateWireRequest(request); err != nil {
|
||||
t.Fatalf("exact maximum Agent body was rejected: %v", err)
|
||||
}
|
||||
request.Body += " "
|
||||
if err := validateWireRequest(request); err == nil {
|
||||
t.Fatal("oversized Agent body was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
type allowConnectionAuthenticator struct {
|
||||
mu sync.Mutex
|
||||
agentID string
|
||||
secret string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (a *allowConnectionAuthenticator) Authenticate(_ context.Context, agentID, secret string) error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.agentID, a.secret = agentID, secret
|
||||
a.calls++
|
||||
return a.err
|
||||
}
|
||||
|
||||
func (a *allowConnectionAuthenticator) reject(err error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.err = err
|
||||
}
|
||||
|
||||
func (a *allowConnectionAuthenticator) callCount() int {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.calls
|
||||
}
|
||||
|
||||
func (a *allowConnectionAuthenticator) credentials() (string, string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.agentID, a.secret
|
||||
}
|
||||
|
||||
func newTestRegistry(t *testing.T) (*Registry, *httptest.Server, []byte, <-chan struct{}) {
|
||||
t.Helper()
|
||||
shared := []byte(strings.Repeat("p", 32))
|
||||
registry, err := NewRegistry(&allowConnectionAuthenticator{}, shared, 5*time.Second, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc(ControlPath, registry.ServeControl)
|
||||
consoleReturned := make(chan struct{}, 8)
|
||||
mux.HandleFunc(ConsolePath, func(response http.ResponseWriter, request *http.Request) {
|
||||
registry.ServeAgentConsole(response, request)
|
||||
consoleReturned <- struct{}{}
|
||||
})
|
||||
mux.HandleFunc(InternalRequestPath, registry.ServeInternalRequest)
|
||||
server := httptest.NewServer(mux)
|
||||
t.Cleanup(server.Close)
|
||||
return registry, server, shared, consoleReturned
|
||||
}
|
||||
|
||||
func dialControl(t *testing.T, serverURL, agentID string) *websocket.Conn {
|
||||
t.Helper()
|
||||
headers := http.Header{
|
||||
"X-Virt-Agent-ID": []string{agentID},
|
||||
"Authorization": []string{"Bearer " + strings.Repeat("a", 32)},
|
||||
}
|
||||
conn, response, err := websocket.Dial(t.Context(), "ws"+strings.TrimPrefix(serverURL, "http")+ControlPath,
|
||||
&websocket.DialOptions{HTTPHeader: headers, Subprotocols: []string{ControlSubprotocol},
|
||||
CompressionMode: websocket.CompressionDisabled})
|
||||
if response != nil && response.Body != nil {
|
||||
_ = response.Body.Close()
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("dial Agent control: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.CloseNow() })
|
||||
return conn
|
||||
}
|
||||
|
||||
func TestRegistryDispatchesAuthenticatedRequestAndReturnsExactResponse(t *testing.T) {
|
||||
_, server, shared, _ := newTestRegistry(t)
|
||||
control := dialControl(t, server.URL, "12")
|
||||
|
||||
agentDone := make(chan error, 1)
|
||||
go func() {
|
||||
messageType, raw, err := control.Read(context.Background())
|
||||
if err != nil {
|
||||
agentDone <- err
|
||||
return
|
||||
}
|
||||
var request wireRequest
|
||||
if messageType != websocket.MessageText || json.Unmarshal(raw, &request) != nil ||
|
||||
request.Type != "request" || request.ID != "request_12345678" ||
|
||||
request.Body != `{"probe":true}` || request.Headers["X-Virt-Actor"] != "actor" {
|
||||
agentDone <- io.ErrUnexpectedEOF
|
||||
return
|
||||
}
|
||||
response, _ := json.Marshal(wireResponse{Type: "response", ID: request.ID,
|
||||
Status: http.StatusCreated, Body: `{"result":"exact"}`})
|
||||
agentDone <- control.Write(context.Background(), websocket.MessageText, response)
|
||||
}()
|
||||
|
||||
envelope := map[string]any{
|
||||
"agent_id": "12", "type": "request", "id": "request_12345678",
|
||||
"method": "POST", "request_target": "/v1/health", "body": `{"probe":true}`,
|
||||
"headers": map[string]string{"X-Virt-Actor": "actor"},
|
||||
}
|
||||
body, _ := json.Marshal(envelope)
|
||||
request, _ := http.NewRequest(http.MethodPost, server.URL+InternalRequestPath, bytes.NewReader(body))
|
||||
nonce, _ := protocol.RandomNonce()
|
||||
if err := protocol.Sign(request, body, protocol.Actor{Service: "php-panel"}, shared, time.Now(), nonce); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
raw, _ := io.ReadAll(response.Body)
|
||||
if response.StatusCode != http.StatusCreated || string(raw) != `{"result":"exact"}` {
|
||||
t.Fatalf("status=%d body=%q", response.StatusCode, raw)
|
||||
}
|
||||
if err := <-agentDone; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRejectsInternalRequestFromWrongServiceActor(t *testing.T) {
|
||||
_, server, shared, _ := newTestRegistry(t)
|
||||
body := []byte(`{"agent_id":"12","type":"request","id":"request_12345678","method":"GET","request_target":"/v1/health","body":"","headers":{}}`)
|
||||
request, _ := http.NewRequest(http.MethodPost, server.URL+InternalRequestPath, bytes.NewReader(body))
|
||||
nonce, _ := protocol.RandomNonce()
|
||||
_ = protocol.Sign(request, body, protocol.Actor{Service: "agent-gateway"}, shared, time.Now(), nonce)
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("wrong service actor status=%d", response.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRejectsInternalRequestFromMixedServiceActor(t *testing.T) {
|
||||
_, server, shared, _ := newTestRegistry(t)
|
||||
body := []byte(`{"agent_id":"12","type":"request","id":"request_12345678","method":"GET","request_target":"/v1/health","body":"","headers":{}}`)
|
||||
request, _ := http.NewRequest(http.MethodPost, server.URL+InternalRequestPath, bytes.NewReader(body))
|
||||
nonce, _ := protocol.RandomNonce()
|
||||
_ = protocol.Sign(request, body, protocol.Actor{Service: "php-panel", UserID: 1}, shared, time.Now(), nonce)
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("mixed service actor status=%d", response.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryReturnsTypedJSONForRejectedInternalRequests(t *testing.T) {
|
||||
registry, _, shared, _ := newTestRegistry(t)
|
||||
|
||||
remoteRequest := httptest.NewRequest(http.MethodPost, InternalRequestPath, strings.NewReader("{}"))
|
||||
remoteRequest.RemoteAddr = "203.0.113.10:43123"
|
||||
remoteResponse := httptest.NewRecorder()
|
||||
registry.ServeInternalRequest(remoteResponse, remoteRequest)
|
||||
assertInternalError(t, remoteResponse.Result(), http.StatusForbidden, "gateway_forbidden")
|
||||
|
||||
body := []byte(`{"agent_id":"invalid","type":"request","id":"request_12345678","method":"GET","request_target":"/v1/health","body":"","headers":{}}`)
|
||||
invalidRequest := httptest.NewRequest(http.MethodPost, InternalRequestPath, bytes.NewReader(body))
|
||||
invalidRequest.RemoteAddr = "127.0.0.1:43124"
|
||||
nonce, _ := protocol.RandomNonce()
|
||||
if err := protocol.Sign(invalidRequest, body, protocol.Actor{Service: "php-panel"}, shared, time.Now(), nonce); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
invalidResponse := httptest.NewRecorder()
|
||||
registry.ServeInternalRequest(invalidResponse, invalidRequest)
|
||||
assertInternalError(t, invalidResponse.Result(), http.StatusBadRequest, "gateway_request_invalid")
|
||||
}
|
||||
|
||||
func TestSessionRejectsRequestBeforeDispatchWhenCapacityIsExhausted(t *testing.T) {
|
||||
current := &session{inFlight: make(chan struct{}, maxInFlightPerAgent)}
|
||||
for index := 0; index < maxInFlightPerAgent; index++ {
|
||||
current.inFlight <- struct{}{}
|
||||
}
|
||||
_, err := current.dispatch(t.Context(), wireRequest{ID: "request_12345678"})
|
||||
if !errors.Is(err, ErrAgentBusy) || !errors.Is(err, ErrNotDispatched) {
|
||||
t.Fatalf("capacity error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryMarksCapacityRejectionAsNotDispatched(t *testing.T) {
|
||||
registry, _, shared, _ := newTestRegistry(t)
|
||||
current := &session{
|
||||
agentID: "12", secret: strings.Repeat("a", 32),
|
||||
pending: make(map[string]chan dispatchResult),
|
||||
inFlight: make(chan struct{}, maxInFlightPerAgent), done: make(chan struct{}),
|
||||
}
|
||||
for index := 0; index < maxInFlightPerAgent; index++ {
|
||||
current.inFlight <- struct{}{}
|
||||
}
|
||||
registry.sessions[current.agentID] = current
|
||||
body := []byte(`{"agent_id":"12","type":"request","id":"request_12345678","method":"POST","request_target":"/v1/vms","body":"{}","headers":{}}`)
|
||||
request := httptest.NewRequest(http.MethodPost, InternalRequestPath, bytes.NewReader(body))
|
||||
request.RemoteAddr = "127.0.0.1:43125"
|
||||
nonce, _ := protocol.RandomNonce()
|
||||
if err := protocol.Sign(request, body, protocol.Actor{Service: "php-panel"}, shared, time.Now(), nonce); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response := httptest.NewRecorder()
|
||||
registry.ServeInternalRequest(response, request)
|
||||
var payload struct {
|
||||
Error struct {
|
||||
Code string `json:"code"`
|
||||
Retryable bool `json:"retryable"`
|
||||
DispatchState string `json:"dispatch_state"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Code != http.StatusServiceUnavailable || payload.Error.Code != "agent_busy" ||
|
||||
!payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" {
|
||||
t.Fatalf("capacity response status=%d payload=%#v", response.Code, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryDrainRejectsNewDispatchBeforeAgentWrite(t *testing.T) {
|
||||
registry, _, _, _ := newTestRegistry(t)
|
||||
registry.BeginDrain()
|
||||
_, err := registry.dispatch(t.Context(), "12", wireRequest{ID: "request_12345678"})
|
||||
if !errors.Is(err, ErrGatewayDraining) || !errors.Is(err, ErrNotDispatched) {
|
||||
t.Fatalf("draining dispatch error = %v", err)
|
||||
}
|
||||
if err := registry.Wait(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryDrainWaitsForAdmittedDispatch(t *testing.T) {
|
||||
registry, server, _, _ := newTestRegistry(t)
|
||||
control := dialControl(t, server.URL, "47")
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := registry.dispatch(context.Background(), "47", wireRequest{
|
||||
Type: "request", ID: "request_12345678", Method: http.MethodGet,
|
||||
RequestTarget: "/v1/health", Headers: map[string]string{},
|
||||
})
|
||||
result <- err
|
||||
}()
|
||||
messageType, raw, err := control.Read(t.Context())
|
||||
if err != nil || messageType != websocket.MessageText {
|
||||
t.Fatalf("read admitted dispatch: type=%v err=%v", messageType, err)
|
||||
}
|
||||
var request wireRequest
|
||||
if json.Unmarshal(raw, &request) != nil || request.ID != "request_12345678" {
|
||||
t.Fatalf("admitted request = %#v", request)
|
||||
}
|
||||
registry.BeginDrain()
|
||||
short, cancelShort := context.WithTimeout(t.Context(), 20*time.Millisecond)
|
||||
if err := registry.Wait(short); !errors.Is(err, context.DeadlineExceeded) {
|
||||
cancelShort()
|
||||
t.Fatalf("drain did not wait for active dispatch: %v", err)
|
||||
}
|
||||
cancelShort()
|
||||
encoded, _ := json.Marshal(wireResponse{Type: "response", ID: request.ID,
|
||||
Status: http.StatusOK, Body: `{"status":"ok"}`})
|
||||
if err := control.Write(t.Context(), websocket.MessageText, encoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-result; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registry.Wait(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryTimeoutAfterAgentWriteRemainsAmbiguous(t *testing.T) {
|
||||
registry, server, _, _ := newTestRegistry(t)
|
||||
control := dialControl(t, server.URL, "48")
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := registry.dispatch(ctx, "48", wireRequest{
|
||||
Type: "request", ID: "request_87654321", Method: http.MethodGet,
|
||||
RequestTarget: "/v1/health", Headers: map[string]string{},
|
||||
})
|
||||
result <- err
|
||||
}()
|
||||
if _, _, err := control.Read(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := <-result
|
||||
if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrNotDispatched) {
|
||||
t.Fatalf("post-write timeout certainty = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRevokesConnectedAgentBeforeOrdinaryDispatch(t *testing.T) {
|
||||
registry, server, _, _ := newTestRegistry(t)
|
||||
control := dialControl(t, server.URL, "12")
|
||||
authenticator := registry.authenticator.(*allowConnectionAuthenticator)
|
||||
authenticator.reject(errors.New("Agent secret was rotated"))
|
||||
|
||||
_, err := registry.dispatch(t.Context(), "12", wireRequest{ID: "request_12345678"})
|
||||
if err == nil || registry.get("12") != nil {
|
||||
t.Fatalf("revoked Agent dispatch err=%v session=%v", err, registry.get("12"))
|
||||
}
|
||||
if authenticator.callCount() != 2 {
|
||||
t.Fatalf("authentication calls=%d want=2", authenticator.callCount())
|
||||
}
|
||||
agentID, secret := authenticator.credentials()
|
||||
if agentID != "12" || secret != strings.Repeat("a", 32) {
|
||||
t.Fatalf("revalidation credentials agent_id=%q secret_length=%d", agentID, len(secret))
|
||||
}
|
||||
readContext, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
if _, _, err := control.Read(readContext); err == nil {
|
||||
t.Fatal("revoked Agent control session remained open")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRevokesConnectedAgentBeforeOpeningConsole(t *testing.T) {
|
||||
registry, server, _, _ := newTestRegistry(t)
|
||||
control := dialControl(t, server.URL, "44")
|
||||
authenticator := registry.authenticator.(*allowConnectionAuthenticator)
|
||||
authenticator.reject(errors.New("Agent was disabled"))
|
||||
|
||||
conn, err := registry.OpenConsole(t.Context(), "44", strings.Repeat("b", 32), strings.Repeat("c", 32))
|
||||
if conn != nil {
|
||||
_ = conn.CloseNow()
|
||||
}
|
||||
if err == nil || registry.get("44") != nil {
|
||||
t.Fatalf("revoked Agent console err=%v session=%v", err, registry.get("44"))
|
||||
}
|
||||
if authenticator.callCount() != 2 {
|
||||
t.Fatalf("authentication calls=%d want=2", authenticator.callCount())
|
||||
}
|
||||
readContext, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
if _, _, err := control.Read(readContext); err == nil {
|
||||
t.Fatal("revoked Agent control session remained open")
|
||||
}
|
||||
}
|
||||
|
||||
func assertInternalError(t *testing.T, response *http.Response, status int, code string) {
|
||||
t.Helper()
|
||||
defer response.Body.Close()
|
||||
var payload struct {
|
||||
Error struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Retryable bool `json:"retryable"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
|
||||
t.Fatalf("decode internal error: %v", err)
|
||||
}
|
||||
if response.StatusCode != status || payload.Error.Code != code ||
|
||||
payload.Error.Message == "" || payload.Error.Retryable ||
|
||||
!strings.HasPrefix(response.Header.Get("Content-Type"), "application/json") {
|
||||
t.Fatalf("unexpected internal error status=%d payload=%#v headers=%v", response.StatusCode, payload, response.Header)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryPairsOneUseReverseConsoleWithConnectedAgent(t *testing.T) {
|
||||
registry, server, _, consoleReturned := newTestRegistry(t)
|
||||
control := dialControl(t, server.URL, "44")
|
||||
|
||||
opened := make(chan *websocket.Conn, 1)
|
||||
errors := make(chan error, 1)
|
||||
go func() {
|
||||
conn, err := registry.OpenConsole(context.Background(), "44", strings.Repeat("b", 32), strings.Repeat("c", 32))
|
||||
if err != nil {
|
||||
errors <- err
|
||||
return
|
||||
}
|
||||
opened <- conn
|
||||
}()
|
||||
|
||||
messageType, raw, err := control.Read(t.Context())
|
||||
if err != nil || messageType != websocket.MessageText {
|
||||
t.Fatalf("read console_open: type=%v err=%v", messageType, err)
|
||||
}
|
||||
var command consoleOpen
|
||||
if json.Unmarshal(raw, &command) != nil || command.Type != "console_open" ||
|
||||
command.Ticket != strings.Repeat("b", 32) || command.SessionID != strings.Repeat("c", 32) ||
|
||||
!tokenPattern.MatchString(command.SessionToken) {
|
||||
t.Fatalf("invalid console_open: %#v", command)
|
||||
}
|
||||
protocolValue := ConsoleSubprotocolPrefix + command.SessionToken
|
||||
agentConsole, response, err := websocket.Dial(t.Context(),
|
||||
"ws"+strings.TrimPrefix(server.URL, "http")+ConsolePath,
|
||||
&websocket.DialOptions{HTTPHeader: http.Header{"X-Virt-Agent-ID": []string{"44"}},
|
||||
Subprotocols: []string{protocolValue}, CompressionMode: websocket.CompressionDisabled})
|
||||
if response != nil && response.Body != nil {
|
||||
_ = response.Body.Close()
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer agentConsole.CloseNow()
|
||||
acknowledgement, _ := json.Marshal(wireResponse{Type: "response", ID: command.ID,
|
||||
Status: http.StatusOK, Body: `{"status":"connected"}`})
|
||||
if err := control.Write(t.Context(), websocket.MessageText, acknowledgement); err != nil {
|
||||
t.Fatalf("send console acknowledgement: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-consoleReturned:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("reverse console handler did not return after transferring connection ownership")
|
||||
}
|
||||
duplicate, duplicateResponse, duplicateErr := websocket.Dial(t.Context(),
|
||||
"ws"+strings.TrimPrefix(server.URL, "http")+ConsolePath,
|
||||
&websocket.DialOptions{HTTPHeader: http.Header{"X-Virt-Agent-ID": []string{"44"}},
|
||||
Subprotocols: []string{protocolValue}, CompressionMode: websocket.CompressionDisabled})
|
||||
if duplicate != nil {
|
||||
_ = duplicate.CloseNow()
|
||||
}
|
||||
if duplicateResponse != nil && duplicateResponse.Body != nil {
|
||||
defer duplicateResponse.Body.Close()
|
||||
}
|
||||
if duplicateErr == nil || duplicateResponse == nil || duplicateResponse.StatusCode != http.StatusUnauthorized {
|
||||
t.Fatalf("reused console token err=%v response=%v", duplicateErr, duplicateResponse)
|
||||
}
|
||||
|
||||
var gatewayConsole *websocket.Conn
|
||||
select {
|
||||
case gatewayConsole = <-opened:
|
||||
case err := <-errors:
|
||||
t.Fatal(err)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("reverse console was not paired")
|
||||
}
|
||||
defer gatewayConsole.CloseNow()
|
||||
if err := agentConsole.Write(t.Context(), websocket.MessageBinary, []byte("RFB")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kind, payload, err := gatewayConsole.Read(t.Context())
|
||||
if err != nil || kind != websocket.MessageBinary || string(payload) != "RFB" {
|
||||
t.Fatalf("reverse console frame kind=%v payload=%q err=%v", kind, payload, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryReturnsConsoleRejectionWithoutWaitingForReverseDial(t *testing.T) {
|
||||
registry, server, _, _ := newTestRegistry(t)
|
||||
control := dialControl(t, server.URL, "45")
|
||||
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := registry.OpenConsole(context.Background(), "45", strings.Repeat("b", 32), strings.Repeat("c", 32))
|
||||
result <- err
|
||||
}()
|
||||
|
||||
messageType, raw, err := control.Read(t.Context())
|
||||
if err != nil || messageType != websocket.MessageText {
|
||||
t.Fatalf("read console_open: type=%v err=%v", messageType, err)
|
||||
}
|
||||
var command consoleOpen
|
||||
if err := json.Unmarshal(raw, &command); err != nil {
|
||||
t.Fatalf("decode console_open: %v", err)
|
||||
}
|
||||
rejection, _ := json.Marshal(wireResponse{Type: "response", ID: command.ID,
|
||||
Status: http.StatusUnauthorized, Body: `{"error":{"code":"invalid_console_ticket"}}`})
|
||||
if err := control.Write(t.Context(), websocket.MessageText, rejection); err != nil {
|
||||
t.Fatalf("send console rejection: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
if err == nil || !strings.Contains(err.Error(), "status 401") {
|
||||
t.Fatalf("console rejection error = %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("console rejection waited for the reverse connection timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryBoundsPendingReverseConsoles(t *testing.T) {
|
||||
registry, err := NewRegistry(&allowConnectionAuthenticator{}, []byte(strings.Repeat("p", 32)),
|
||||
5*time.Second, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := &session{
|
||||
agentID: "46", pending: make(map[string]chan dispatchResult),
|
||||
inFlight: make(chan struct{}, maxInFlightPerAgent), done: make(chan struct{}),
|
||||
}
|
||||
registry.sessions[current.agentID] = current
|
||||
for index := 0; index < maxPendingConsoles; index++ {
|
||||
registry.consoles[string(rune(index+1))] = pendingConsole{agentID: current.agentID}
|
||||
}
|
||||
_, err = registry.OpenConsole(t.Context(), current.agentID, strings.Repeat("b", 32), strings.Repeat("c", 32))
|
||||
if !errors.Is(err, ErrConsoleBusy) {
|
||||
t.Fatalf("pending console capacity error = %v", err)
|
||||
}
|
||||
if len(current.inFlight) != 0 || len(current.pending) != 0 {
|
||||
t.Fatalf("rejected console leaked session admission: in_flight=%d pending=%d", len(current.inFlight), len(current.pending))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
agentgateway "git.luoli.live/root/virt/gateway/agent"
|
||||
"git.luoli.live/root/virt/gateway/console"
|
||||
)
|
||||
|
||||
func main() {
|
||||
configPath := flag.String("config", "", "absolute path to the JSON configuration")
|
||||
flag.Parse()
|
||||
if *configPath == "" || !filepath.IsAbs(*configPath) {
|
||||
log.Fatal("--config must be an absolute path")
|
||||
}
|
||||
config, err := console.LoadConfig(*configPath)
|
||||
if err != nil {
|
||||
log.Fatalf("Agent gateway configuration is invalid: %v", err)
|
||||
}
|
||||
introspector, err := console.NewHTTPIntrospector(config.PanelIntrospectionURL,
|
||||
[]byte(config.PanelSharedSecret), config.ConnectTimeout())
|
||||
if err != nil {
|
||||
log.Fatalf("console introspection client cannot start: %v", err)
|
||||
}
|
||||
authenticator, err := agentgateway.NewHTTPConnectionAuthenticator(
|
||||
config.PanelAgentAuthURL, []byte(config.PanelSharedSecret), config.ConnectTimeout(),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("Agent authentication client cannot start: %v", err)
|
||||
}
|
||||
registry, err := agentgateway.NewRegistry(authenticator,
|
||||
[]byte(config.PanelSharedSecret), config.AgentRequestTimeout(),
|
||||
config.ConnectTimeout())
|
||||
if err != nil {
|
||||
log.Fatalf("Agent registry cannot start: %v", err)
|
||||
}
|
||||
relay, err := console.NewRelay(introspector, registry)
|
||||
if err != nil {
|
||||
log.Fatalf("reverse console cannot start: %v", err)
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc(agentgateway.ControlPath, registry.ServeControl)
|
||||
mux.HandleFunc(agentgateway.ConsolePath, registry.ServeAgentConsole)
|
||||
mux.HandleFunc(agentgateway.InternalRequestPath, registry.ServeInternalRequest)
|
||||
mux.Handle("/engine-console", relay)
|
||||
server := &http.Server{
|
||||
Addr: config.ListenAddress, Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 60 * time.Second,
|
||||
MaxHeaderBytes: 32 << 10,
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
serveErrors := make(chan error, 1)
|
||||
go func() {
|
||||
serveErrors <- server.ListenAndServe()
|
||||
}()
|
||||
log.Printf("virt-cloud Agent gateway listening on %s", config.ListenAddress)
|
||||
select {
|
||||
case err := <-serveErrors:
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatalf("Agent gateway stopped unexpectedly: %v", err)
|
||||
}
|
||||
return
|
||||
case <-ctx.Done():
|
||||
}
|
||||
|
||||
// Keep the listener available during the grace period so an already
|
||||
// admitted reverse-console open can complete its Agent callback. All other
|
||||
// new work is rejected by the draining gates below.
|
||||
registry.BeginDrain()
|
||||
relay.BeginDrain()
|
||||
drainContext, cancelDrain := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
var waits sync.WaitGroup
|
||||
waits.Add(2)
|
||||
go func() {
|
||||
defer waits.Done()
|
||||
_ = registry.Wait(drainContext)
|
||||
}()
|
||||
go func() {
|
||||
defer waits.Done()
|
||||
_ = relay.Wait(drainContext)
|
||||
}()
|
||||
waits.Wait()
|
||||
close(drained)
|
||||
}()
|
||||
select {
|
||||
case <-drained:
|
||||
case <-drainContext.Done():
|
||||
}
|
||||
cancelDrain()
|
||||
relay.Close()
|
||||
registry.Close()
|
||||
|
||||
shutdown, cancelShutdown := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
if err := server.Shutdown(shutdown); err != nil {
|
||||
_ = server.Close()
|
||||
log.Printf("Agent gateway HTTP shutdown was forced: %v", err)
|
||||
}
|
||||
cancelShutdown()
|
||||
select {
|
||||
case err := <-serveErrors:
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Printf("Agent gateway stopped unexpectedly: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
_ = server.Close()
|
||||
log.Print("Agent gateway listener did not stop before the shutdown deadline")
|
||||
}
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.luoli.live/root/virt/gateway/console"
|
||||
)
|
||||
|
||||
func main() {
|
||||
configPath := flag.String("config", "", "absolute path to the JSON configuration")
|
||||
flag.Parse()
|
||||
if *configPath == "" || !filepath.IsAbs(*configPath) {
|
||||
log.Fatal("--config must be an absolute path")
|
||||
}
|
||||
config, err := console.LoadConfig(*configPath)
|
||||
if err != nil {
|
||||
log.Fatalf("console relay configuration is invalid: %v", err)
|
||||
}
|
||||
introspector, err := console.NewHTTPIntrospector(config.PanelIntrospectionURL, []byte(config.PanelSharedSecret), config.ConnectTimeout())
|
||||
if err != nil {
|
||||
log.Fatalf("console introspection client cannot start: %v", err)
|
||||
}
|
||||
relay, err := console.NewRelay(introspector, config.ConnectTimeout())
|
||||
if err != nil {
|
||||
log.Fatalf("console relay cannot start: %v", err)
|
||||
}
|
||||
server := &http.Server{Addr: config.ListenAddress, Handler: relay, ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 15 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 32 << 10}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = server.Shutdown(shutdown)
|
||||
}()
|
||||
log.Printf("virt-cloud-console-relay listening on %s", config.ListenAddress)
|
||||
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatalf("console relay stopped unexpectedly: %v", err)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
final class CloudRescueTemplateImport extends Model
|
||||
{
|
||||
protected $connection = 'default';
|
||||
protected $table = 'cloud_rescue_template_imports';
|
||||
protected $primaryKey = 'operation_id';
|
||||
public $incrementing = false;
|
||||
protected $keyType = 'string';
|
||||
protected $casts = [
|
||||
'host_id' => 'int',
|
||||
'created_at' => 'int',
|
||||
'updated_at' => 'int',
|
||||
];
|
||||
}
|
||||
@ -1,74 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
/** Strict fixed-IP Agent endpoints avoid redirects, DNS rebinding and metadata SSRF. */
|
||||
final class CloudAgentEndpoint
|
||||
{
|
||||
public static function normalize($value): string
|
||||
{
|
||||
if (!is_string($value)) {
|
||||
throw new CloudError('Agent API 地址格式无效');
|
||||
}
|
||||
$value = trim($value);
|
||||
if ($value === '' || strlen($value) > 255
|
||||
|| preg_match('/[\x00-\x20\x7f]/', $value) === 1) {
|
||||
throw new CloudError('Agent API 地址需为不超过 255 字符的固定 IP URL');
|
||||
}
|
||||
$parts = parse_url($value);
|
||||
if (!is_array($parts)) {
|
||||
throw new CloudError('Agent API 地址格式无效');
|
||||
}
|
||||
$scheme = strtolower((string) ($parts['scheme'] ?? ''));
|
||||
$hostValue = (string) ($parts['host'] ?? '');
|
||||
$host = trim($hostValue, '[]');
|
||||
if ($scheme !== 'https'
|
||||
|| $host === '' || filter_var($host, FILTER_VALIDATE_IP) === false
|
||||
|| isset($parts['user']) || isset($parts['pass'])
|
||||
|| isset($parts['query']) || isset($parts['fragment'])
|
||||
|| (($parts['path'] ?? '') !== '' && ($parts['path'] ?? '') !== '/')) {
|
||||
throw new CloudError('Agent API 地址必须是无账号、路径、查询参数的固定 IP HTTPS URL');
|
||||
}
|
||||
if (self::blockedAddress($host)) {
|
||||
throw new CloudError('Agent API 地址不能使用回环、链路本地、元数据、组播或未指定地址');
|
||||
}
|
||||
$port = $parts['port'] ?? null;
|
||||
if ($port !== null && ($port < 1 || $port > 65535)) {
|
||||
throw new CloudError('Agent API 端口无效');
|
||||
}
|
||||
$renderedHost = str_contains($host, ':') ? '[' . strtolower($host) . ']' : $host;
|
||||
return $scheme . '://' . $renderedHost . ($port === null ? '' : ':' . $port);
|
||||
}
|
||||
|
||||
private static function blockedAddress(string $host): bool
|
||||
{
|
||||
if (filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false) {
|
||||
$value = self::ipv4($host);
|
||||
return self::ipv4In($value, '0.0.0.0', 8)
|
||||
|| self::ipv4In($value, '127.0.0.0', 8)
|
||||
|| self::ipv4In($value, '169.254.0.0', 16)
|
||||
|| self::ipv4In($value, '224.0.0.0', 4)
|
||||
|| self::ipv4In($value, '240.0.0.0', 4);
|
||||
}
|
||||
$packed = inet_pton($host);
|
||||
if ($packed === false) {
|
||||
return true;
|
||||
}
|
||||
return $packed === str_repeat("\0", 16)
|
||||
|| $packed === str_repeat("\0", 15) . "\1"
|
||||
|| (ord($packed[0]) === 0xfe && (ord($packed[1]) & 0xc0) === 0x80)
|
||||
|| ord($packed[0]) === 0xff
|
||||
|| substr($packed, 0, 12) === str_repeat("\0", 10) . "\xff\xff";
|
||||
}
|
||||
|
||||
private static function ipv4(string $address): int
|
||||
{
|
||||
return (int) sprintf('%u', ip2long($address));
|
||||
}
|
||||
|
||||
private static function ipv4In(int $value, string $network, int $prefix): bool
|
||||
{
|
||||
$mask = $prefix === 0 ? 0 : ((0xffffffff << (32 - $prefix)) & 0xffffffff);
|
||||
return ($value & $mask) === (self::ipv4($network) & $mask);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
/** Strict loopback endpoint for PHP-to-Go Agent gateway dispatch. */
|
||||
final class CloudAgentGatewayEndpoint
|
||||
{
|
||||
public const PATH = '/internal/agent/request';
|
||||
public const URL = 'http://127.0.0.1:8001' . self::PATH;
|
||||
|
||||
public static function normalize($value): string
|
||||
{
|
||||
if (!is_string($value)) {
|
||||
throw new \InvalidArgumentException('Agent gateway URL is invalid');
|
||||
}
|
||||
$value = trim($value);
|
||||
if ($value === '' || strlen($value) > 2048
|
||||
|| preg_match('/[\x00-\x20\x7f\\\\]/', $value) === 1) {
|
||||
throw new \InvalidArgumentException('Agent gateway URL is invalid');
|
||||
}
|
||||
$parts = parse_url($value);
|
||||
if (!is_array($parts)) {
|
||||
throw new \InvalidArgumentException('Agent gateway URL is invalid');
|
||||
}
|
||||
$scheme = strtolower((string) ($parts['scheme'] ?? ''));
|
||||
$host = (string) ($parts['host'] ?? '');
|
||||
if ($scheme !== 'http' || $host !== '127.0.0.1'
|
||||
|| ($parts['path'] ?? '') !== self::PATH
|
||||
|| isset($parts['user']) || isset($parts['pass'])
|
||||
|| isset($parts['query']) || isset($parts['fragment'])
|
||||
|| ($parts['port'] ?? null) !== 8001) {
|
||||
throw new \InvalidArgumentException(
|
||||
'Agent gateway URL must be the fixed 127.0.0.1:8001 dispatch endpoint'
|
||||
);
|
||||
}
|
||||
return self::URL;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,276 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudRescueTemplateImport;
|
||||
|
||||
/**
|
||||
* Persists one host-local rescue-template import before its only dispatch.
|
||||
* Every later request may only reconcile the originally reserved operation.
|
||||
*/
|
||||
final class CloudRescueTemplateImportService
|
||||
{
|
||||
public static function start(
|
||||
int $hostId,
|
||||
string $importType,
|
||||
string $name,
|
||||
string $sourceUrl,
|
||||
string $sourceToken,
|
||||
string $sha256,
|
||||
array $actor
|
||||
): array {
|
||||
self::validateInput($hostId, $importType, $name, $sourceUrl,
|
||||
$sourceToken, $sha256);
|
||||
$operationId = bin2hex(random_bytes(16));
|
||||
$templateId = bin2hex(random_bytes(16));
|
||||
$record = self::reserve(
|
||||
$operationId, $hostId, $templateId, $importType
|
||||
);
|
||||
|
||||
try {
|
||||
$client = CloudAgentSettingsService::clientForHost($hostId, 900.0);
|
||||
} catch (\Throwable $error) {
|
||||
self::mark($record, 'failed', 'agent_client_unavailable');
|
||||
return self::dto(self::reload($record));
|
||||
}
|
||||
|
||||
self::mark($record, 'sending', '');
|
||||
$record = self::reload($record);
|
||||
$requestActor = $actor;
|
||||
$requestActor['operation_id'] = $operationId;
|
||||
$body = [
|
||||
'operation_id' => $operationId,
|
||||
'id' => $templateId,
|
||||
'name' => $name,
|
||||
'source_url' => $sourceUrl,
|
||||
'sha256' => $sha256,
|
||||
];
|
||||
if ($sourceToken !== '') {
|
||||
$body['source_token'] = $sourceToken;
|
||||
}
|
||||
|
||||
try {
|
||||
$operation = $client->request(
|
||||
'POST', '/v1/rescue-templates', $requestActor, $body
|
||||
);
|
||||
self::applyOperation($record, $operation);
|
||||
} catch (CloudAgentError $error) {
|
||||
self::mark(
|
||||
$record,
|
||||
$error->safePreDispatchFailure
|
||||
? 'failed' : ($error->retryable ? 'running' : 'failed'),
|
||||
$error->agentCode
|
||||
);
|
||||
} catch (\Throwable $error) {
|
||||
// The request may already have crossed the dispatch boundary.
|
||||
// Preserve the operation ID and require explicit reconciliation.
|
||||
self::mark($record, 'uncertain', 'import_dispatch_unconfirmed');
|
||||
}
|
||||
return self::dto(self::reload($record));
|
||||
}
|
||||
|
||||
public static function job(int $hostId, string $operationId,
|
||||
array $actor): array
|
||||
{
|
||||
self::validateOperationId($operationId);
|
||||
$record = CloudRescueTemplateImport::query()
|
||||
->where('host_id', $hostId)
|
||||
->where('operation_id', $operationId)->first();
|
||||
if ($record === null) {
|
||||
throw new CloudError('救援镜像导入任务不存在', 404);
|
||||
}
|
||||
return self::pollIfAllowed($record, $actor);
|
||||
}
|
||||
|
||||
private static function reserve(string $operationId, int $hostId,
|
||||
string $templateId, string $importType): CloudRescueTemplateImport
|
||||
{
|
||||
$record = new CloudRescueTemplateImport();
|
||||
$record->operation_id = $operationId;
|
||||
$record->host_id = $hostId;
|
||||
$record->template_id = $templateId;
|
||||
$record->import_type = $importType;
|
||||
$record->state = 'dispatching';
|
||||
$record->error_code = '';
|
||||
$record->created_at = $record->updated_at = time();
|
||||
$record->save();
|
||||
return $record;
|
||||
}
|
||||
|
||||
private static function pollIfAllowed(CloudRescueTemplateImport $record,
|
||||
array $actor): array
|
||||
{
|
||||
$state = (string) $record->state;
|
||||
if (in_array($state, ['dispatching', 'sending'], true)) {
|
||||
if ((int) $record->updated_at >= time() - 90) {
|
||||
return self::dto($record);
|
||||
}
|
||||
self::mark($record, 'uncertain', 'import_dispatch_unconfirmed');
|
||||
return self::dto(self::reload($record));
|
||||
}
|
||||
if ($state !== 'running') {
|
||||
return self::dto($record);
|
||||
}
|
||||
|
||||
try {
|
||||
$operation = CloudAgentSettingsService::clientForHost(
|
||||
(int) $record->host_id
|
||||
)->request('GET', '/v1/operations/'
|
||||
. rawurlencode((string) $record->operation_id), $actor);
|
||||
self::applyOperation($record, $operation);
|
||||
} catch (CloudAgentError $error) {
|
||||
if ($error->agentHttpStatus === 404) {
|
||||
self::mark($record, 'uncertain', 'operation_not_found');
|
||||
} else {
|
||||
// An outage never authorizes a second import POST. Keep the
|
||||
// same operation available for later reconciliation.
|
||||
self::mark($record, 'running', 'agent_unavailable');
|
||||
}
|
||||
} catch (\Throwable $error) {
|
||||
self::mark($record, 'running', 'agent_unavailable');
|
||||
}
|
||||
return self::dto(self::reload($record));
|
||||
}
|
||||
|
||||
private static function applyOperation(CloudRescueTemplateImport $record,
|
||||
array $operation): void
|
||||
{
|
||||
if (($operation['operation_id'] ?? null)
|
||||
!== (string) $record->operation_id) {
|
||||
self::mark($record, 'uncertain', 'operation_id_mismatch');
|
||||
return;
|
||||
}
|
||||
$status = strtolower(trim((string) ($operation['status'] ?? '')));
|
||||
if ($status === 'succeeded') {
|
||||
$result = $operation['result'] ?? null;
|
||||
if (!is_array($result) || array_is_list($result)
|
||||
|| ($result['id'] ?? null) !== (string) $record->template_id) {
|
||||
self::mark($record, 'uncertain', 'agent_result_mismatch');
|
||||
return;
|
||||
}
|
||||
self::mark($record, 'succeeded', '');
|
||||
return;
|
||||
}
|
||||
if ($status === 'failed') {
|
||||
self::mark($record, 'failed', self::operationErrorCode(
|
||||
$operation, 'agent_import_failed'
|
||||
));
|
||||
return;
|
||||
}
|
||||
if ($status === 'uncertain') {
|
||||
self::mark($record, 'uncertain', self::operationErrorCode(
|
||||
$operation, 'agent_import_uncertain'
|
||||
));
|
||||
return;
|
||||
}
|
||||
if (in_array($status,
|
||||
['reserved', 'pending', 'running', 'dispatching'], true)) {
|
||||
self::mark($record, 'running', '');
|
||||
return;
|
||||
}
|
||||
self::mark($record, 'uncertain', 'agent_status_invalid');
|
||||
}
|
||||
|
||||
private static function mark(CloudRescueTemplateImport $record,
|
||||
string $state, string $errorCode): void
|
||||
{
|
||||
$current = (string) $record->state;
|
||||
$allowed = [
|
||||
'dispatching' => ['sending', 'failed', 'uncertain'],
|
||||
'sending' => ['running', 'succeeded', 'failed', 'uncertain'],
|
||||
'running' => ['running', 'succeeded', 'failed', 'uncertain'],
|
||||
];
|
||||
if (!isset($allowed[$current])
|
||||
|| !in_array($state, $allowed[$current], true)) {
|
||||
return;
|
||||
}
|
||||
CloudRescueTemplateImport::query()
|
||||
->where('operation_id', (string) $record->operation_id)
|
||||
->where('host_id', (int) $record->host_id)
|
||||
->where('template_id', (string) $record->template_id)
|
||||
->where('state', $current)
|
||||
->update([
|
||||
'state' => $state,
|
||||
'error_code' => substr($errorCode, 0, 64),
|
||||
'updated_at' => time(),
|
||||
]);
|
||||
}
|
||||
|
||||
private static function reload(
|
||||
CloudRescueTemplateImport $record
|
||||
): CloudRescueTemplateImport {
|
||||
$fresh = CloudRescueTemplateImport::query()
|
||||
->where('operation_id', (string) $record->operation_id)
|
||||
->where('host_id', (int) $record->host_id)
|
||||
->where('template_id', (string) $record->template_id)->first();
|
||||
if ($fresh === null) {
|
||||
throw new CloudError('救援镜像导入任务状态丢失', 503);
|
||||
}
|
||||
return $fresh;
|
||||
}
|
||||
|
||||
private static function dto(CloudRescueTemplateImport $record): array
|
||||
{
|
||||
$state = (string) $record->state;
|
||||
$status = in_array($state, ['failed', 'uncertain', 'succeeded'], true)
|
||||
? $state : 'running';
|
||||
$result = [
|
||||
'job_id' => (string) $record->operation_id,
|
||||
'operation_id' => (string) $record->operation_id,
|
||||
'host_id' => (int) $record->host_id,
|
||||
'template_id' => (string) $record->template_id,
|
||||
'import_type' => (string) $record->import_type,
|
||||
'status' => $status,
|
||||
'phase' => $status === 'succeeded'
|
||||
? '救援镜像导入已完成'
|
||||
: ($status === 'running' ? '正在核对 Agent 导入结果'
|
||||
: '救援镜像导入未得到安全确认,禁止自动重试'),
|
||||
'status_url' => '/api/v1/admin/rescue-templates/import-jobs/'
|
||||
. rawurlencode((string) $record->operation_id)
|
||||
. '?host_id=' . (int) $record->host_id,
|
||||
];
|
||||
if (in_array($status, ['failed', 'uncertain'], true)) {
|
||||
$result['error'] = $status === 'uncertain'
|
||||
? 'Agent 导入结果不确定,请按 operation_id 人工核对'
|
||||
: 'Agent 明确拒绝或无法开始救援镜像导入';
|
||||
$result['error_code'] = (string) $record->error_code;
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
private static function operationErrorCode(array $operation,
|
||||
string $fallback): string
|
||||
{
|
||||
$error = is_array($operation['error'] ?? null)
|
||||
? $operation['error'] : [];
|
||||
$code = $error['code'] ?? ($operation['error_code'] ?? null);
|
||||
return is_string($code)
|
||||
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $code) === 1
|
||||
? $code : $fallback;
|
||||
}
|
||||
|
||||
private static function validateInput(int $hostId, string $importType,
|
||||
string $name, string $sourceUrl, string $sourceToken,
|
||||
string $sha256): void
|
||||
{
|
||||
if ($hostId < 1 || !in_array($importType, ['remote', 'upload'], true)
|
||||
|| $name === '' || mb_strlen($name, 'UTF-8') > 64
|
||||
|| $sourceUrl === '' || strlen($sourceUrl) > 4096
|
||||
|| ($sourceToken !== ''
|
||||
&& preg_match('/^[A-Za-z0-9_-]{32,512}$/D', $sourceToken)
|
||||
!== 1)
|
||||
|| ($sha256 !== ''
|
||||
&& preg_match('/^[a-f0-9]{64}$/D', $sha256) !== 1)) {
|
||||
throw new \InvalidArgumentException(
|
||||
'Invalid rescue-template import reservation'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static function validateOperationId(string $operationId): void
|
||||
{
|
||||
if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) {
|
||||
throw new CloudError('救援镜像导入任务编号无效');
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,423 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Cloud {
|
||||
/** Test-only sealing stub; the state-machine assertions do not inspect ciphertext. */
|
||||
final class CloudSecrets
|
||||
{
|
||||
public static function encrypt(array $payload, string $purpose): string
|
||||
{
|
||||
return 'test-sealed:' . $purpose;
|
||||
}
|
||||
}
|
||||
|
||||
final class CatalogImportFakeAgent
|
||||
{
|
||||
public array $requests = [];
|
||||
/** @var null|callable */
|
||||
public $beforeResponse = null;
|
||||
public ?array $getResponse = null;
|
||||
|
||||
public function request(string $method, string $target, array $actor,
|
||||
?array $payload = null): array
|
||||
{
|
||||
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
||||
if (is_callable($this->beforeResponse)) {
|
||||
($this->beforeResponse)($method, $target, $actor, $payload);
|
||||
}
|
||||
$operationId = (string) ($payload['operation_id']
|
||||
?? $actor['operation_id'] ?? basename($target));
|
||||
if ($method === 'GET') {
|
||||
return ['operation_id' => $operationId]
|
||||
+ ($this->getResponse ?? ['status' => 'running']);
|
||||
}
|
||||
if ($target === '/v1/templates') {
|
||||
return [
|
||||
'operation_id' => $operationId,
|
||||
'status' => 'succeeded',
|
||||
'result' => [
|
||||
'id' => (string) $payload['id'],
|
||||
'revision' => (int) $payload['revision'],
|
||||
'generation_token' =>
|
||||
(string) $payload['generation_token'],
|
||||
'size_bytes' => 4096,
|
||||
'sha256' => str_repeat('a', 64),
|
||||
],
|
||||
];
|
||||
}
|
||||
if ($target === '/v1/user-isos') {
|
||||
return [
|
||||
'operation_id' => $operationId,
|
||||
'status' => 'succeeded',
|
||||
'result' => [
|
||||
'id' => (string) $payload['id'],
|
||||
'owner_id' => (int) $payload['owner_id'],
|
||||
'size_bytes' => 8192,
|
||||
'sha256' => str_repeat('b', 64),
|
||||
],
|
||||
];
|
||||
}
|
||||
throw new \RuntimeException('Unexpected fake Agent request');
|
||||
}
|
||||
}
|
||||
|
||||
final class CloudAgentSettingsService
|
||||
{
|
||||
public static array $hostIds = [101];
|
||||
public static ?CatalogImportFakeAgent $client = null;
|
||||
|
||||
public static function enabledHostIds(): array
|
||||
{
|
||||
return self::$hostIds;
|
||||
}
|
||||
|
||||
public static function clientForHost(int $hostId,
|
||||
float $timeoutSeconds = 20.0): CatalogImportFakeAgent
|
||||
{
|
||||
if (!in_array($hostId, self::$hostIds, true)
|
||||
|| self::$client === null) {
|
||||
throw new \RuntimeException('Missing fake Agent');
|
||||
}
|
||||
return self::$client;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateReplica;
|
||||
use App\Models\CloudUserIso;
|
||||
use App\Models\CloudUserIsoReplica;
|
||||
use App\Services\Cloud\CatalogImportFakeAgent;
|
||||
use App\Services\Cloud\CloudAgentSettingsService;
|
||||
use App\Services\Cloud\CloudTemplateCatalogService;
|
||||
use App\Services\Cloud\CloudUserIsoService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
||||
'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
|
||||
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
||||
$table->string('id')->primary();
|
||||
$table->string('name');
|
||||
$table->string('os_family');
|
||||
$table->string('os_version');
|
||||
$table->string('source_type');
|
||||
$table->text('sealed_source_url');
|
||||
$table->integer('image_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->text('startup_script')->default('');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token')->unique();
|
||||
$table->string('state');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_template_replicas',
|
||||
static function (Blueprint $table): void {
|
||||
$table->integer('host_id');
|
||||
$table->string('template_id');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('image_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->string('operation_id')->default('');
|
||||
$table->string('error_code')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->primary(['host_id', 'template_id']);
|
||||
});
|
||||
$schema->create('cloud_orders', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('status');
|
||||
$table->text('spec_json');
|
||||
});
|
||||
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('kind');
|
||||
$table->string('status');
|
||||
$table->text('sealed_payload');
|
||||
});
|
||||
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
|
||||
$table->string('id')->primary();
|
||||
$table->integer('user_id');
|
||||
$table->string('name');
|
||||
$table->text('sealed_source_url');
|
||||
$table->integer('size_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->string('state');
|
||||
$table->string('import_operation_id');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_user_iso_replicas',
|
||||
static function (Blueprint $table): void {
|
||||
$table->integer('host_id');
|
||||
$table->string('user_iso_id');
|
||||
$table->integer('user_id');
|
||||
$table->string('state');
|
||||
$table->integer('size_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->string('operation_id')->default('');
|
||||
$table->string('error_code')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->primary(['host_id', 'user_iso_id']);
|
||||
});
|
||||
|
||||
$agent = new CatalogImportFakeAgent();
|
||||
CloudAgentSettingsService::$client = $agent;
|
||||
|
||||
// An Agent success followed by a local metadata write failure is uncertain,
|
||||
// never a definite dispatch failure. The transaction leaves the replica
|
||||
// pending long enough for the guarded uncertainty transition to latch.
|
||||
Capsule::statement("CREATE TRIGGER reject_template_digest "
|
||||
. "BEFORE UPDATE OF sha256 ON cloud_templates "
|
||||
. "WHEN NEW.sha256 <> OLD.sha256 BEGIN "
|
||||
. "SELECT RAISE(ABORT, 'template persistence failure'); END");
|
||||
$templateJob = CloudTemplateCatalogService::importRemote([
|
||||
'name' => 'Debian 12 persistence test',
|
||||
'url' => 'https://images.example/debian-12.qcow2',
|
||||
'sha256' => '',
|
||||
], ['user_id' => 1, 'can_manage' => true]);
|
||||
$template = CloudTemplate::query()->where('id',
|
||||
(string) $templateJob['job_id'])->first();
|
||||
$templateReplica = CloudTemplateReplica::query()
|
||||
->where('template_id', (string) $template->id)->first();
|
||||
$assert($templateJob['status'] === 'uncertain'
|
||||
&& (string) $template->state === 'error'
|
||||
&& (string) $templateReplica->state === 'uncertain'
|
||||
&& (string) $templateReplica->error_code === 'panel_persist_failed',
|
||||
'Template Agent success was mislabeled after panel persistence failed.');
|
||||
Capsule::statement('DROP TRIGGER reject_template_digest');
|
||||
|
||||
Capsule::statement("CREATE TRIGGER reject_iso_digest "
|
||||
. "BEFORE UPDATE OF sha256 ON cloud_user_isos "
|
||||
. "WHEN NEW.sha256 <> OLD.sha256 BEGIN "
|
||||
. "SELECT RAISE(ABORT, 'ISO persistence failure'); END");
|
||||
$isoJob = CloudUserIsoService::import([
|
||||
'name' => 'Recovery ISO',
|
||||
'url' => 'https://images.example/recovery.iso',
|
||||
], ['user_id' => 42, 'can_manage' => false]);
|
||||
$iso = CloudUserIso::query()->where('import_operation_id',
|
||||
(string) $isoJob['job_id'])->first();
|
||||
$isoReplica = CloudUserIsoReplica::query()
|
||||
->where('user_iso_id', (string) $iso->id)->first();
|
||||
$assert($isoJob['status'] === 'uncertain'
|
||||
&& (string) $iso->state === 'failed'
|
||||
&& (string) $isoReplica->state === 'uncertain'
|
||||
&& (string) $isoReplica->error_code === 'panel_persist_failed',
|
||||
'ISO Agent success was mislabeled after panel persistence failed.');
|
||||
Capsule::statement('DROP TRIGGER reject_iso_digest');
|
||||
|
||||
// A process interruption after the last replica update must be repaired by
|
||||
// status lookup even when there is no pending operation left to poll.
|
||||
Capsule::table('cloud_templates')->where('id', (string) $template->id)
|
||||
->update(['state' => 'importing']);
|
||||
Capsule::table('cloud_template_replicas')
|
||||
->where('template_id', (string) $template->id)->update([
|
||||
'state' => 'active',
|
||||
'image_bytes' => 4096,
|
||||
'sha256' => str_repeat('a', 64),
|
||||
'error_code' => '',
|
||||
]);
|
||||
$requestsBeforeTemplateRepair = count($agent->requests);
|
||||
$repairedTemplate = CloudTemplateCatalogService::importJob(
|
||||
(string) $template->id, ['user_id' => 1, 'can_manage' => true]
|
||||
);
|
||||
$assert($repairedTemplate['status'] === 'succeeded'
|
||||
&& (string) CloudTemplate::find((string) $template->id)->state
|
||||
=== 'active'
|
||||
&& count($agent->requests) === $requestsBeforeTemplateRepair,
|
||||
'Template aggregate state did not self-heal without pending replicas.');
|
||||
|
||||
Capsule::table('cloud_user_isos')->where('id', (string) $iso->id)
|
||||
->update([
|
||||
'state' => 'importing',
|
||||
'size_bytes' => 8192,
|
||||
'sha256' => str_repeat('b', 64),
|
||||
]);
|
||||
Capsule::table('cloud_user_iso_replicas')
|
||||
->where('user_iso_id', (string) $iso->id)->update([
|
||||
'state' => 'active',
|
||||
'size_bytes' => 8192,
|
||||
'sha256' => str_repeat('b', 64),
|
||||
'error_code' => '',
|
||||
]);
|
||||
$requestsBeforeIsoRepair = count($agent->requests);
|
||||
$repairedIso = CloudUserIsoService::job((string) $iso->import_operation_id,
|
||||
['user_id' => 42, 'can_manage' => false]);
|
||||
$assert($repairedIso['status'] === 'succeeded'
|
||||
&& (string) CloudUserIso::find((string) $iso->id)->state === 'active'
|
||||
&& count($agent->requests) === $requestsBeforeIsoRepair,
|
||||
'ISO aggregate state did not self-heal without pending replicas.');
|
||||
|
||||
// Quota aggregation must use the bytes committed through the locked model,
|
||||
// not the stale outer object that existed before the Agent response.
|
||||
$quotaOwner = 77;
|
||||
$quotaBytes = 10 * 1024 * 1024 * 1024;
|
||||
Capsule::table('cloud_user_isos')->insert([
|
||||
'id' => str_repeat('9', 32),
|
||||
'user_id' => $quotaOwner,
|
||||
'name' => 'Existing quota usage',
|
||||
'sealed_source_url' => 'test-sealed:user-iso-source',
|
||||
'size_bytes' => $quotaBytes - 4096,
|
||||
'sha256' => str_repeat('9', 64),
|
||||
'state' => 'active',
|
||||
'import_operation_id' => str_repeat('8', 32),
|
||||
'created_at' => time(),
|
||||
'updated_at' => time(),
|
||||
]);
|
||||
$quotaJob = CloudUserIsoService::import([
|
||||
'name' => 'Quota overflow ISO',
|
||||
'url' => 'https://images.example/quota-overflow.iso',
|
||||
], ['user_id' => $quotaOwner, 'can_manage' => false]);
|
||||
$quotaIso = CloudUserIso::query()->where('import_operation_id',
|
||||
(string) $quotaJob['job_id'])->first();
|
||||
$assert($quotaJob['status'] === 'failed'
|
||||
&& (int) $quotaIso->size_bytes === 8192
|
||||
&& (string) $quotaIso->state === 'failed',
|
||||
'ISO quota aggregation ignored the persisted Agent result size.');
|
||||
|
||||
// A successful stale template response must not overwrite a concurrently
|
||||
// installed delete operation or move the parent out of deleting.
|
||||
$staleTemplateId = 'debian-12-stale-import';
|
||||
$oldTemplateOperation = str_repeat('c', 32);
|
||||
$newTemplateOperation = str_repeat('d', 32);
|
||||
$generation = str_repeat('e', 32);
|
||||
$now = time();
|
||||
Capsule::table('cloud_templates')->insert([
|
||||
'id' => $staleTemplateId,
|
||||
'name' => 'Debian 12 stale import',
|
||||
'os_family' => 'Debian',
|
||||
'os_version' => '12',
|
||||
'source_type' => 'remote',
|
||||
'sealed_source_url' => 'test-sealed:template-source',
|
||||
'image_bytes' => 0,
|
||||
'sha256' => '',
|
||||
'startup_script' => '',
|
||||
'revision' => 1,
|
||||
'generation_token' => $generation,
|
||||
'state' => 'importing',
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_template_replicas')->insert([
|
||||
'host_id' => 101,
|
||||
'template_id' => $staleTemplateId,
|
||||
'revision' => 1,
|
||||
'generation_token' => $generation,
|
||||
'state' => 'pending',
|
||||
'image_bytes' => 0,
|
||||
'sha256' => '',
|
||||
'operation_id' => $oldTemplateOperation,
|
||||
'error_code' => '',
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
$agent->beforeResponse = static function (string $method) use (
|
||||
$staleTemplateId, $newTemplateOperation
|
||||
): void {
|
||||
if ($method !== 'GET') {
|
||||
return;
|
||||
}
|
||||
Capsule::table('cloud_templates')->where('id', $staleTemplateId)
|
||||
->update(['state' => 'deleting']);
|
||||
Capsule::table('cloud_template_replicas')
|
||||
->where('template_id', $staleTemplateId)->update([
|
||||
'state' => 'delete_succeeded',
|
||||
'operation_id' => $newTemplateOperation,
|
||||
'error_code' => '',
|
||||
]);
|
||||
};
|
||||
$agent->getResponse = [
|
||||
'status' => 'succeeded',
|
||||
'result' => [
|
||||
'id' => $staleTemplateId,
|
||||
'revision' => 1,
|
||||
'generation_token' => $generation,
|
||||
'size_bytes' => 4096,
|
||||
'sha256' => str_repeat('a', 64),
|
||||
],
|
||||
];
|
||||
CloudTemplateCatalogService::importJob($staleTemplateId,
|
||||
['user_id' => 1, 'can_manage' => true]);
|
||||
$staleTemplate = CloudTemplate::find($staleTemplateId);
|
||||
$staleTemplateReplica = CloudTemplateReplica::query()
|
||||
->where('template_id', $staleTemplateId)->first();
|
||||
$assert((string) $staleTemplate->state === 'deleting'
|
||||
&& (string) $staleTemplateReplica->state === 'delete_succeeded'
|
||||
&& (string) $staleTemplateReplica->operation_id
|
||||
=== $newTemplateOperation,
|
||||
'Stale template polling rolled back a concurrent terminal delete.');
|
||||
|
||||
// The state predicate is independently required: even the same operation
|
||||
// ID cannot move a terminal ISO replica back to pending.
|
||||
$staleIsoId = str_repeat('f', 32);
|
||||
$staleIsoJob = str_repeat('1', 32);
|
||||
$staleIsoOperation = str_repeat('2', 32);
|
||||
Capsule::table('cloud_user_isos')->insert([
|
||||
'id' => $staleIsoId,
|
||||
'user_id' => 42,
|
||||
'name' => 'Stale ISO',
|
||||
'sealed_source_url' => 'test-sealed:user-iso-source',
|
||||
'size_bytes' => 0,
|
||||
'sha256' => '',
|
||||
'state' => 'importing',
|
||||
'import_operation_id' => $staleIsoJob,
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_user_iso_replicas')->insert([
|
||||
'host_id' => 101,
|
||||
'user_iso_id' => $staleIsoId,
|
||||
'user_id' => 42,
|
||||
'state' => 'pending',
|
||||
'size_bytes' => 0,
|
||||
'sha256' => '',
|
||||
'operation_id' => $staleIsoOperation,
|
||||
'error_code' => '',
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
$agent->beforeResponse = static function (string $method) use (
|
||||
$staleIsoId
|
||||
): void {
|
||||
if ($method !== 'GET') {
|
||||
return;
|
||||
}
|
||||
Capsule::table('cloud_user_isos')->where('id', $staleIsoId)
|
||||
->update(['state' => 'active']);
|
||||
Capsule::table('cloud_user_iso_replicas')
|
||||
->where('user_iso_id', $staleIsoId)->update([
|
||||
'state' => 'active',
|
||||
'error_code' => '',
|
||||
]);
|
||||
};
|
||||
$agent->getResponse = null;
|
||||
CloudUserIsoService::job($staleIsoJob,
|
||||
['user_id' => 42, 'can_manage' => false]);
|
||||
$staleIso = CloudUserIso::find($staleIsoId);
|
||||
$staleIsoReplica = CloudUserIsoReplica::query()
|
||||
->where('user_iso_id', $staleIsoId)->first();
|
||||
$assert((string) $staleIso->state === 'active'
|
||||
&& (string) $staleIsoReplica->state === 'active'
|
||||
&& (string) $staleIsoReplica->operation_id === $staleIsoOperation,
|
||||
'Stale ISO polling moved a terminal replica back to pending.');
|
||||
|
||||
echo "Catalog import persistence failures and stale-response CAS passed.\n";
|
||||
}
|
||||
@ -0,0 +1,331 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Cloud {
|
||||
/** Deterministic in-process Agent for rescue import reconciliation tests. */
|
||||
final class RescueImportFakeAgent
|
||||
{
|
||||
public array $requests = [];
|
||||
public array $operations = [];
|
||||
public string $mode = 'running';
|
||||
public $onRequest = null;
|
||||
|
||||
public function request(string $method, string $target, array $actor,
|
||||
?array $payload = null): array
|
||||
{
|
||||
$method = strtoupper($method);
|
||||
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
||||
if (is_callable($this->onRequest)) {
|
||||
($this->onRequest)($method, $target, $actor, $payload);
|
||||
}
|
||||
|
||||
if ($method === 'POST' && $target === '/v1/rescue-templates') {
|
||||
$operationId = (string) ($payload['operation_id'] ?? '');
|
||||
$templateId = (string) ($payload['id'] ?? '');
|
||||
if ($this->mode === 'pre_dispatch_busy') {
|
||||
throw new CloudAgentError('not dispatched', 503,
|
||||
'agent_busy', true, 503, false, true);
|
||||
}
|
||||
$this->operations[$operationId] = $templateId;
|
||||
if ($this->mode === 'transport') {
|
||||
throw new CloudAgentError('transport uncertain', 502,
|
||||
'agent_transport_error', true);
|
||||
}
|
||||
} elseif ($method === 'GET'
|
||||
&& preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
|
||||
$target, $match)) {
|
||||
$operationId = rawurldecode($match[1]);
|
||||
$templateId = $this->operations[$operationId] ?? '';
|
||||
if ($this->mode === 'not_found' || $templateId === '') {
|
||||
throw new CloudAgentError('operation missing', 404,
|
||||
'operation_not_found', false, 404);
|
||||
}
|
||||
if ($this->mode === 'transport') {
|
||||
throw new CloudAgentError('transport uncertain', 502,
|
||||
'agent_transport_error', true);
|
||||
}
|
||||
} else {
|
||||
throw new \RuntimeException('Unexpected fake Agent request: '
|
||||
. $method . ' ' . $target);
|
||||
}
|
||||
|
||||
if ($this->mode === 'wrong_operation') {
|
||||
return ['operation_id' => str_repeat('f', 32),
|
||||
'status' => 'running'];
|
||||
}
|
||||
if ($this->mode === 'failed') {
|
||||
return ['operation_id' => $operationId, 'status' => 'failed',
|
||||
'error' => ['code' => 'rescue_template_exists']];
|
||||
}
|
||||
if ($this->mode === 'success') {
|
||||
return [
|
||||
'operation_id' => $operationId,
|
||||
'status' => 'succeeded',
|
||||
'result' => [
|
||||
'id' => $templateId,
|
||||
'name' => 'Rescue',
|
||||
'size_bytes' => 1048576,
|
||||
'sha256' => str_repeat('a', 64),
|
||||
],
|
||||
];
|
||||
}
|
||||
return ['operation_id' => $operationId, 'status' => 'running'];
|
||||
}
|
||||
}
|
||||
|
||||
/** Replaces only Agent lookup in this isolated process. */
|
||||
final class CloudAgentSettingsService
|
||||
{
|
||||
/** @var array<int, RescueImportFakeAgent> */
|
||||
public static array $clients = [];
|
||||
|
||||
public static function clientForHost(int $hostId,
|
||||
float $timeoutSeconds = 20.0): RescueImportFakeAgent
|
||||
{
|
||||
if (!isset(self::$clients[$hostId])) {
|
||||
throw new \RuntimeException('Missing fake Agent for host '
|
||||
. $hostId);
|
||||
}
|
||||
return self::$clients[$hostId];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudRescueTemplateImport;
|
||||
use App\Services\Cloud\CloudAgentSettingsService;
|
||||
use App\Services\Cloud\CloudError;
|
||||
use App\Services\Cloud\CloudRescueTemplateImportService;
|
||||
use App\Services\Cloud\RescueImportFakeAgent;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new \RuntimeException($message);
|
||||
}
|
||||
};
|
||||
$requestCount = static fn (RescueImportFakeAgent $agent,
|
||||
string $method): int => count(array_filter($agent->requests,
|
||||
static fn (array $request): bool => $request['method'] === $method));
|
||||
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
||||
'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
$schema->create('cloud_rescue_template_imports',
|
||||
static function (Blueprint $table): void {
|
||||
$table->string('operation_id')->primary();
|
||||
$table->integer('host_id');
|
||||
$table->string('template_id');
|
||||
$table->string('import_type');
|
||||
$table->string('state');
|
||||
$table->string('error_code')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->unique(['host_id', 'template_id']);
|
||||
});
|
||||
|
||||
$actor = ['user_id' => 1, 'is_admin' => true, 'can_manage' => true];
|
||||
|
||||
// The row must exist with its immutable identity before the first POST.
|
||||
$remoteAgent = new RescueImportFakeAgent();
|
||||
$remoteAgent->onRequest = static function (string $method, string $target,
|
||||
array $requestActor, ?array $payload) use ($assert): void {
|
||||
if ($method !== 'POST') {
|
||||
return;
|
||||
}
|
||||
$row = CloudRescueTemplateImport::query()
|
||||
->where('operation_id', (string) $payload['operation_id'])->first();
|
||||
$assert($row !== null
|
||||
&& (int) $row->host_id === 10
|
||||
&& (string) $row->template_id === (string) $payload['id']
|
||||
&& (string) $row->import_type === 'remote'
|
||||
&& (string) $row->state === 'sending'
|
||||
&& (string) ($requestActor['operation_id'] ?? '')
|
||||
=== (string) $row->operation_id,
|
||||
'Rescue import crossed dispatch before its durable identity existed.');
|
||||
};
|
||||
CloudAgentSettingsService::$clients[10] = $remoteAgent;
|
||||
$remote = CloudRescueTemplateImportService::start(
|
||||
10,
|
||||
'remote',
|
||||
'Rescue',
|
||||
'https://images.example.com/rescue.iso',
|
||||
'',
|
||||
str_repeat('a', 64),
|
||||
$actor
|
||||
);
|
||||
$remoteRow = CloudRescueTemplateImport::query()
|
||||
->where('operation_id', (string) $remote['operation_id'])->first();
|
||||
$assert($remote['status'] === 'running'
|
||||
&& $remote['job_id'] === $remote['operation_id']
|
||||
&& $remote['host_id'] === 10
|
||||
&& $remote['template_id'] === (string) $remoteRow->template_id
|
||||
&& $remote['import_type'] === 'remote'
|
||||
&& $remote['status_url']
|
||||
=== '/api/v1/admin/rescue-templates/import-jobs/'
|
||||
. $remote['operation_id'] . '?host_id=10'
|
||||
&& (string) $remoteRow->state === 'running'
|
||||
&& $requestCount($remoteAgent, 'POST') === 1,
|
||||
'Remote rescue import did not persist its running operation.');
|
||||
|
||||
// Authenticated capacity rejection is known to precede the Agent journal,
|
||||
// so it is a deterministic failure rather than an ambiguous operation.
|
||||
$busyAgent = new RescueImportFakeAgent();
|
||||
$busyAgent->mode = 'pre_dispatch_busy';
|
||||
CloudAgentSettingsService::$clients[11] = $busyAgent;
|
||||
$busy = CloudRescueTemplateImportService::start(
|
||||
11, 'remote', 'Busy rescue',
|
||||
'https://images.example.com/busy.iso', '', '', $actor
|
||||
);
|
||||
$assert($busy['status'] === 'failed'
|
||||
&& $busy['error_code'] === 'agent_busy'
|
||||
&& $requestCount($busyAgent, 'POST') === 1
|
||||
&& $busyAgent->operations === [],
|
||||
'Definitive pre-dispatch rescue import was left for impossible reconciliation.');
|
||||
|
||||
// Reconciliation uses the exact saved operation and terminal rows are
|
||||
// read-only; no later call is allowed to issue another import POST.
|
||||
$remoteAgent->mode = 'success';
|
||||
$completed = CloudRescueTemplateImportService::job(
|
||||
10, (string) $remote['operation_id'], $actor
|
||||
);
|
||||
$requestsAfterSuccess = count($remoteAgent->requests);
|
||||
$completedAgain = CloudRescueTemplateImportService::job(
|
||||
10, (string) $remote['operation_id'], $actor
|
||||
);
|
||||
$lastPoll = $remoteAgent->requests[$requestsAfterSuccess - 1];
|
||||
$assert($completed['status'] === 'succeeded'
|
||||
&& $completedAgain['status'] === 'succeeded'
|
||||
&& $requestCount($remoteAgent, 'POST') === 1
|
||||
&& count($remoteAgent->requests) === $requestsAfterSuccess
|
||||
&& $lastPoll['method'] === 'GET'
|
||||
&& $lastPoll['target'] === '/v1/operations/'
|
||||
. $remote['operation_id'],
|
||||
'Rescue import reconciliation changed or replayed its operation.');
|
||||
|
||||
// A retryable transport failure may have crossed the boundary. Preserve
|
||||
// the upload operation, then lock it uncertain if Agent has no record.
|
||||
$uploadAgent = new RescueImportFakeAgent();
|
||||
$uploadAgent->mode = 'transport';
|
||||
CloudAgentSettingsService::$clients[20] = $uploadAgent;
|
||||
$upload = CloudRescueTemplateImportService::start(
|
||||
20,
|
||||
'upload',
|
||||
'Uploaded rescue',
|
||||
'https://panel.example.com/api/internal/cloud/upload-staging/'
|
||||
. str_repeat('b', 32),
|
||||
str_repeat('c', 64),
|
||||
str_repeat('d', 64),
|
||||
$actor
|
||||
);
|
||||
$uploadRow = CloudRescueTemplateImport::query()
|
||||
->where('operation_id', (string) $upload['operation_id'])->first();
|
||||
$assert($upload['status'] === 'running'
|
||||
&& $upload['import_type'] === 'upload'
|
||||
&& (string) $uploadRow->state === 'running'
|
||||
&& (string) $uploadRow->error_code === 'agent_transport_error'
|
||||
&& $requestCount($uploadAgent, 'POST') === 1,
|
||||
'Ambiguous upload dispatch was not retained for reconciliation.');
|
||||
$uploadAgent->mode = 'not_found';
|
||||
$uncertain = CloudRescueTemplateImportService::job(
|
||||
20, (string) $upload['operation_id'], $actor
|
||||
);
|
||||
$requestsAfterUncertain = count($uploadAgent->requests);
|
||||
$uncertainAgain = CloudRescueTemplateImportService::job(
|
||||
20, (string) $upload['operation_id'], $actor
|
||||
);
|
||||
$assert($uncertain['status'] === 'uncertain'
|
||||
&& $uncertainAgain['status'] === 'uncertain'
|
||||
&& $uncertain['error_code'] === 'operation_not_found'
|
||||
&& $requestCount($uploadAgent, 'POST') === 1
|
||||
&& count($uploadAgent->requests) === $requestsAfterUncertain,
|
||||
'Missing ambiguous operation was replayed instead of locked uncertain.');
|
||||
|
||||
// A mismatched Agent operation is terminally uncertain and host scoping
|
||||
// prevents another host from learning or polling the job.
|
||||
$mismatchAgent = new RescueImportFakeAgent();
|
||||
$mismatchAgent->mode = 'wrong_operation';
|
||||
CloudAgentSettingsService::$clients[30] = $mismatchAgent;
|
||||
$mismatch = CloudRescueTemplateImportService::start(
|
||||
30, 'remote', 'Mismatch', 'https://images.example.com/other.iso',
|
||||
'', '', $actor
|
||||
);
|
||||
$assert($mismatch['status'] === 'uncertain'
|
||||
&& $mismatch['error_code'] === 'operation_id_mismatch'
|
||||
&& $requestCount($mismatchAgent, 'POST') === 1,
|
||||
'Mismatched Agent operation identity was accepted.');
|
||||
try {
|
||||
CloudRescueTemplateImportService::job(
|
||||
31, (string) $mismatch['operation_id'], $actor
|
||||
);
|
||||
throw new \RuntimeException('Cross-host rescue import lookup succeeded.');
|
||||
} catch (CloudError $error) {
|
||||
$assert($error->httpStatus === 404,
|
||||
'Cross-host rescue import lookup returned the wrong status.');
|
||||
}
|
||||
|
||||
// A process crash between durable reservation and completion cannot cause
|
||||
// a later status request to issue the original POST.
|
||||
$staleOperation = str_repeat('e', 32);
|
||||
Capsule::table('cloud_rescue_template_imports')->insert([
|
||||
'operation_id' => $staleOperation,
|
||||
'host_id' => 40,
|
||||
'template_id' => str_repeat('9', 32),
|
||||
'import_type' => 'remote',
|
||||
'state' => 'sending',
|
||||
'error_code' => '',
|
||||
'created_at' => time() - 180,
|
||||
'updated_at' => time() - 180,
|
||||
]);
|
||||
$staleAgent = new RescueImportFakeAgent();
|
||||
CloudAgentSettingsService::$clients[40] = $staleAgent;
|
||||
$stale = CloudRescueTemplateImportService::job(
|
||||
40, $staleOperation, $actor
|
||||
);
|
||||
$assert($stale['status'] === 'uncertain'
|
||||
&& $stale['error_code'] === 'import_dispatch_unconfirmed'
|
||||
&& $staleAgent->requests === [],
|
||||
'Stale dispatch boundary caused an unsafe import replay.');
|
||||
|
||||
// A response produced from a stale concurrent poll must not overwrite a
|
||||
// terminal state committed while that request was in flight.
|
||||
$raceAgent = new RescueImportFakeAgent();
|
||||
CloudAgentSettingsService::$clients[50] = $raceAgent;
|
||||
$race = CloudRescueTemplateImportService::start(
|
||||
50, 'remote', 'Concurrent poll',
|
||||
'https://images.example.com/concurrent.iso', '', '', $actor
|
||||
);
|
||||
$raceAgent->onRequest = static function (string $method, string $target)
|
||||
use ($race): void {
|
||||
if ($method === 'GET') {
|
||||
CloudRescueTemplateImport::query()
|
||||
->where('operation_id', (string) $race['operation_id'])
|
||||
->update([
|
||||
'state' => 'succeeded',
|
||||
'error_code' => '',
|
||||
'updated_at' => time(),
|
||||
]);
|
||||
}
|
||||
};
|
||||
$raced = CloudRescueTemplateImportService::job(
|
||||
50, (string) $race['operation_id'], $actor
|
||||
);
|
||||
$assert($raced['status'] === 'succeeded'
|
||||
&& (string) CloudRescueTemplateImport::find(
|
||||
(string) $race['operation_id']
|
||||
)->state === 'succeeded',
|
||||
'A stale concurrent poll moved a terminal import state backwards.');
|
||||
|
||||
$columns = $schema->getColumnListing('cloud_rescue_template_imports');
|
||||
$assert(!in_array('source_url', $columns, true)
|
||||
&& !in_array('source_token', $columns, true),
|
||||
'Durable rescue import table persisted source credentials.');
|
||||
|
||||
echo "Rescue import persistence, host binding and no-replay reconciliation passed.\n";
|
||||
}
|
||||
Loading…
Reference in New Issue