You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
364 lines
14 KiB
Go
364 lines
14 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/gateway/internal/protocol"
|
|
)
|
|
|
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
|
|
|
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
|
return f(request)
|
|
}
|
|
|
|
type acceptNonceStore struct{}
|
|
|
|
func (acceptNonceStore) UseNonce(string, time.Time, time.Time) (bool, error) { return true, nil }
|
|
|
|
func TestHTTPCompletionSinkUsesExactPanelContractAndGatewayIdentity(t *testing.T) {
|
|
secret := []byte(strings.Repeat("s", 32))
|
|
now := time.Date(2026, 10, 19, 2, 0, 0, 0, time.UTC)
|
|
sink, err := NewHTTPCompletionSink("http://127.0.0.1:8080/api/internal/cloud/agent-auth", secret, time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sink.now = func() time.Time { return now }
|
|
var captured *http.Request
|
|
var capturedBody []byte
|
|
sink.client = &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
|
captured = request.Clone(request.Context())
|
|
capturedBody, err = io.ReadAll(request.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &http.Response{
|
|
StatusCode: http.StatusOK,
|
|
Body: io.NopCloser(strings.NewReader(`{"accepted":true,"disposition":"applied"}`)),
|
|
Header: http.Header{"Content-Type": []string{"application/json; charset=utf-8"}},
|
|
Request: request,
|
|
}, nil
|
|
})}
|
|
completion := Completion{
|
|
Type: "completion", ID: "completion_event_123", OperationID: "operation_terminal_123",
|
|
Action: "delete-vm", ResourceID: "vm-completion-123",
|
|
RequestFingerprint: strings.Repeat("a", 64),
|
|
Status: "failed", ErrorCode: "result_unconfirmed",
|
|
CauseCode: "backup_disk_transition_active", Result: json.RawMessage("null"),
|
|
EffectMayHaveApplied: true,
|
|
}
|
|
if err := sink.Deliver(context.Background(), "42", completion); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if captured == nil || captured.Method != http.MethodPost || captured.URL.Path != completionEndpointPath {
|
|
t.Fatalf("completion request = %#v", captured)
|
|
}
|
|
var payload map[string]json.RawMessage
|
|
if err := json.Unmarshal(capturedBody, &payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantKeys := []string{
|
|
"type", "id", "agent_id", "operation_id", "action", "resource_id", "request_fingerprint",
|
|
"status", "effect_may_have_applied", "safe_to_resubmit", "error_code", "cause_code", "result",
|
|
}
|
|
if len(payload) != len(wantKeys) {
|
|
t.Fatalf("completion keys = %#v body=%s", payload, capturedBody)
|
|
}
|
|
for _, key := range wantKeys {
|
|
if _, ok := payload[key]; !ok {
|
|
t.Fatalf("completion omitted %q: %s", key, capturedBody)
|
|
}
|
|
}
|
|
if string(payload["type"]) != `"completion"` || string(payload["id"]) != `"completion_event_123"` ||
|
|
string(payload["agent_id"]) != `"42"` || string(payload["action"]) != `"delete-vm"` ||
|
|
string(payload["resource_id"]) != `"vm-completion-123"` ||
|
|
string(payload["request_fingerprint"]) != `"`+strings.Repeat("a", 64)+`"` ||
|
|
string(payload["cause_code"]) != `"backup_disk_transition_active"` ||
|
|
string(payload["result"]) != "null" {
|
|
t.Fatalf("completion payload = %s", capturedBody)
|
|
}
|
|
if _, legacy := payload["event_id"]; legacy {
|
|
t.Fatalf("legacy event_id leaked into completion: %s", capturedBody)
|
|
}
|
|
authenticator, err := protocol.NewAuthenticator(secret, time.Minute, 2*time.Minute, acceptNonceStore{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
authenticator.SetClockForTesting(func() time.Time { return now })
|
|
actor, err := authenticator.Authenticate(captured, capturedBody)
|
|
if err != nil || actor.Service != "agent-gateway" || actor.UserID != 0 || actor.IsAdmin || actor.IsSuperAdmin || actor.CanManage {
|
|
t.Fatalf("completion actor = %#v err=%v", actor, err)
|
|
}
|
|
}
|
|
|
|
func TestHTTPCompletionSinkRequiresExactTerminalAcceptance(t *testing.T) {
|
|
completion := Completion{
|
|
Type: "completion", ID: "completion_event_strict", OperationID: "operation_terminal_strict",
|
|
Action: "delete-vm", ResourceID: "vm-completion-strict",
|
|
RequestFingerprint: strings.Repeat("b", 64),
|
|
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
status int
|
|
contentType string
|
|
body string
|
|
wantError bool
|
|
}{
|
|
{name: "applied", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"applied"}`},
|
|
{name: "late", status: 200, contentType: "application/json; charset=UTF-8", body: `{"accepted":true,"disposition":"late"}`},
|
|
{name: "rejected durable receipt", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"rejected"}`},
|
|
{name: "received is not terminal", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"received"}`, wantError: true},
|
|
{name: "accepted false", status: 200, contentType: "application/json", body: `{"accepted":false,"disposition":"rejected"}`, wantError: true},
|
|
{name: "unknown field", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"applied","extra":1}`, wantError: true},
|
|
{name: "trailing JSON", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"applied"}{}`, wantError: true},
|
|
{name: "missing body", status: 204, contentType: "application/json", wantError: true},
|
|
{name: "wrong content type", status: 200, contentType: "text/json", body: `{"accepted":true,"disposition":"applied"}`, wantError: true},
|
|
{name: "unknown content type parameter", status: 200, contentType: "application/json; version=1", body: `{"accepted":true,"disposition":"applied"}`, wantError: true},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
sink, err := NewHTTPCompletionSink("http://127.0.0.1:8080/api/internal/cloud/agent-auth", []byte(strings.Repeat("s", 32)), time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sink.client = &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
|
header := make(http.Header)
|
|
if test.contentType != "" {
|
|
header.Set("Content-Type", test.contentType)
|
|
}
|
|
return &http.Response{StatusCode: test.status, Header: header,
|
|
Body: io.NopCloser(strings.NewReader(test.body)), Request: request}, nil
|
|
})}
|
|
err = sink.Deliver(context.Background(), "42", completion)
|
|
if (err != nil) != test.wantError {
|
|
t.Fatalf("Deliver() error = %v, wantError=%v", err, test.wantError)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type captureCompletionSink struct {
|
|
mu sync.Mutex
|
|
agentID string
|
|
completion Completion
|
|
called chan struct{}
|
|
deliveryErr error
|
|
}
|
|
|
|
func (s *captureCompletionSink) Deliver(_ context.Context, agentID string, completion Completion) error {
|
|
s.mu.Lock()
|
|
s.agentID = agentID
|
|
s.completion = completion
|
|
s.mu.Unlock()
|
|
close(s.called)
|
|
return s.deliveryErr
|
|
}
|
|
|
|
func TestCompletionDeliveryACKsOnlyTerminalPanelAcceptance(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
sinkError error
|
|
wantType string
|
|
}{
|
|
{name: "terminal acceptance", wantType: "completion_ack"},
|
|
{name: "retryable delivery", sinkError: errors.New("injected delivery failure"), wantType: "completion_retry"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
sink := &captureCompletionSink{called: make(chan struct{}), deliveryErr: test.sinkError}
|
|
written := make(chan any, 1)
|
|
registry := &Registry{completionSink: sink}
|
|
current := &session{
|
|
agentID: "84", completionSlots: make(chan struct{}, maxCompletionDeliveries), done: make(chan struct{}),
|
|
writeHook: func(_ context.Context, value any) error { written <- value; return nil },
|
|
}
|
|
completion := Completion{
|
|
Type: "completion", ID: "completion_disposition_123", OperationID: "operation_disposition_123",
|
|
Action: "delete-vm", ResourceID: "vm-completion-disposition",
|
|
RequestFingerprint: strings.Repeat("c", 64),
|
|
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
|
|
}
|
|
registry.deliverCompletion(context.Background(), current, completion)
|
|
select {
|
|
case value := <-written:
|
|
gotType := ""
|
|
switch message := value.(type) {
|
|
case completionAck:
|
|
gotType = message.Type
|
|
case completionRetry:
|
|
gotType = message.Type
|
|
}
|
|
if gotType != test.wantType {
|
|
t.Fatalf("completion disposition type = %q, want %q", gotType, test.wantType)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("completion disposition was not written")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRegistryBindsCompletionToAuthenticatedSessionAgent(t *testing.T) {
|
|
sink := &captureCompletionSink{called: make(chan struct{})}
|
|
registry := &Registry{completionSink: sink}
|
|
current := &session{
|
|
agentID: "84", completionSlots: make(chan struct{}, maxCompletionDeliveries), done: make(chan struct{}),
|
|
writeHook: func(context.Context, any) error { return nil },
|
|
}
|
|
completion := Completion{
|
|
Type: "completion", ID: "completion_event_456", OperationID: "operation_terminal_456",
|
|
Action: "delete-vm", ResourceID: "vm-completion-456",
|
|
RequestFingerprint: strings.Repeat("d", 64),
|
|
Status: "succeeded", Result: json.RawMessage(`{"accepted":true}`),
|
|
}
|
|
registry.deliverCompletion(context.Background(), current, completion)
|
|
select {
|
|
case <-sink.called:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("completion was not delivered")
|
|
}
|
|
sink.mu.Lock()
|
|
agentID := sink.agentID
|
|
delivered := sink.completion
|
|
sink.mu.Unlock()
|
|
if agentID != current.agentID || delivered.ID != completion.ID {
|
|
t.Fatalf("delivery agent=%q completion=%#v", agentID, delivered)
|
|
}
|
|
deadline := time.Now().Add(time.Second)
|
|
for len(current.completionSlots) != 0 && time.Now().Before(deadline) {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if len(current.completionSlots) != 0 {
|
|
t.Fatal("completion delivery slot was not released after failure")
|
|
}
|
|
}
|
|
|
|
func TestCompletionDeliveryAppliesBackpressureInsteadOfDropping(t *testing.T) {
|
|
sink := &captureCompletionSink{called: make(chan struct{})}
|
|
registry := &Registry{completionSink: sink}
|
|
current := &session{
|
|
agentID: "84", completionSlots: make(chan struct{}, maxCompletionDeliveries), done: make(chan struct{}),
|
|
writeHook: func(context.Context, any) error { return nil },
|
|
}
|
|
for range maxCompletionDeliveries {
|
|
current.completionSlots <- struct{}{}
|
|
}
|
|
completion := Completion{
|
|
Type: "completion", ID: "completion_backpressure_123", OperationID: "operation_backpressure_123",
|
|
Action: "delete-vm", ResourceID: "vm-completion-backpressure",
|
|
RequestFingerprint: strings.Repeat("e", 64),
|
|
Status: "succeeded", Result: json.RawMessage(`{"accepted":true}`),
|
|
}
|
|
returned := make(chan struct{})
|
|
go func() {
|
|
registry.deliverCompletion(context.Background(), current, completion)
|
|
close(returned)
|
|
}()
|
|
select {
|
|
case <-returned:
|
|
t.Fatal("full completion window silently dropped the delivery")
|
|
case <-time.After(30 * time.Millisecond):
|
|
}
|
|
<-current.completionSlots
|
|
select {
|
|
case <-returned:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("completion did not enter the delivery queue after capacity was released")
|
|
}
|
|
select {
|
|
case <-sink.called:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("queued completion was not delivered")
|
|
}
|
|
}
|
|
|
|
func TestCompletionValidationRequiresExplicitNullResult(t *testing.T) {
|
|
valid := Completion{
|
|
Type: "completion", ID: "completion_event_789", OperationID: "operation_terminal_789",
|
|
Action: "delete-vm", ResourceID: "vm-completion-789",
|
|
RequestFingerprint: strings.Repeat("f", 64),
|
|
Status: "failed", ErrorCode: "operation_failed", Result: json.RawMessage("null"),
|
|
}
|
|
if err := validateCompletion(valid); err != nil {
|
|
t.Fatalf("explicit null result rejected: %v", err)
|
|
}
|
|
withCause := valid
|
|
withCause.CauseCode = "backup_disk_transition_active"
|
|
if err := validateCompletion(withCause); err != nil {
|
|
t.Fatalf("stable cause code rejected: %v", err)
|
|
}
|
|
invalidCause := valid
|
|
invalidCause.CauseCode = "Backup Disk Transition Active"
|
|
if err := validateCompletion(invalidCause); err == nil {
|
|
t.Fatal("invalid cause code was accepted")
|
|
}
|
|
missing := valid
|
|
missing.Result = nil
|
|
if err := validateCompletion(missing); err == nil {
|
|
t.Fatal("missing result key was accepted")
|
|
}
|
|
}
|
|
|
|
func TestCompletionValidationRequiresStrictImmutableOperationIdentity(t *testing.T) {
|
|
valid := Completion{
|
|
Type: "completion", ID: "completion_identity_123", OperationID: "operation_identity_123",
|
|
Action: "create-snapshot:0123456789abcdef0123456789abcdef:filesystem",
|
|
ResourceID: "template:debian-13", RequestFingerprint: strings.Repeat("a", 64),
|
|
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
|
|
}
|
|
if err := validateCompletion(valid); err != nil {
|
|
t.Fatalf("valid immutable identity rejected: %v", err)
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*Completion)
|
|
}{
|
|
{name: "missing action", mutate: func(value *Completion) { value.Action = "" }},
|
|
{name: "uppercase action", mutate: func(value *Completion) { value.Action = "Delete-VM" }},
|
|
{name: "resource path", mutate: func(value *Completion) { value.ResourceID = "vm/unsafe" }},
|
|
{name: "missing resource", mutate: func(value *Completion) { value.ResourceID = "" }},
|
|
{name: "uppercase fingerprint", mutate: func(value *Completion) { value.RequestFingerprint = strings.Repeat("A", 64) }},
|
|
{name: "short fingerprint", mutate: func(value *Completion) { value.RequestFingerprint = strings.Repeat("a", 63) }},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
candidate := valid
|
|
test.mutate(&candidate)
|
|
if err := validateCompletion(candidate); err == nil {
|
|
t.Fatal("invalid immutable operation identity was accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGatewayCompletionDeliveryLogContainsOnlyStableMetadata(t *testing.T) {
|
|
logger := log.Default()
|
|
previousWriter, previousFlags, previousPrefix := logger.Writer(), logger.Flags(), logger.Prefix()
|
|
var output strings.Builder
|
|
logger.SetOutput(&output)
|
|
logger.SetFlags(0)
|
|
logger.SetPrefix("")
|
|
t.Cleanup(func() {
|
|
logger.SetOutput(previousWriter)
|
|
logger.SetFlags(previousFlags)
|
|
logger.SetPrefix(previousPrefix)
|
|
})
|
|
|
|
logGatewayCompletionDelivery(completionPhaseGatewayACKSent, "operation_log_456", "completion_log_456")
|
|
got := strings.TrimSpace(output.String())
|
|
want := "completion_delivery phase=gateway_ack_sent operation_id=operation_log_456 event_id=completion_log_456"
|
|
if got != want {
|
|
t.Fatalf("completion log = %q, want %q", got, want)
|
|
}
|
|
}
|