You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/gateway/agent/completion_test.go

364 lines
14 KiB
Go

package agent
import (
"context"
"encoding/json"
"errors"
"io"
"log"
"net/http"
"strings"
"sync"
"testing"
"time"
"git.luoli.live/root/virt/gateway/internal/protocol"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return f(request)
}
type acceptNonceStore struct{}
func (acceptNonceStore) UseNonce(string, time.Time, time.Time) (bool, error) { return true, nil }
func TestHTTPCompletionSinkUsesExactPanelContractAndGatewayIdentity(t *testing.T) {
secret := []byte(strings.Repeat("s", 32))
now := time.Date(2026, 10, 19, 2, 0, 0, 0, time.UTC)
sink, err := NewHTTPCompletionSink("http://127.0.0.1:8080/api/internal/cloud/agent-auth", secret, time.Second)
if err != nil {
t.Fatal(err)
}
sink.now = func() time.Time { return now }
var captured *http.Request
var capturedBody []byte
sink.client = &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
captured = request.Clone(request.Context())
capturedBody, err = io.ReadAll(request.Body)
if err != nil {
return nil, err
}
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(strings.NewReader(`{"accepted":true,"disposition":"applied"}`)),
Header: http.Header{"Content-Type": []string{"application/json; charset=utf-8"}},
Request: request,
}, nil
})}
completion := Completion{
Type: "completion", ID: "completion_event_123", OperationID: "operation_terminal_123",
Action: "delete-vm", ResourceID: "vm-completion-123",
RequestFingerprint: strings.Repeat("a", 64),
Status: "failed", ErrorCode: "result_unconfirmed",
CauseCode: "backup_disk_transition_active", Result: json.RawMessage("null"),
EffectMayHaveApplied: true,
}
if err := sink.Deliver(context.Background(), "42", completion); err != nil {
t.Fatal(err)
}
if captured == nil || captured.Method != http.MethodPost || captured.URL.Path != completionEndpointPath {
t.Fatalf("completion request = %#v", captured)
}
var payload map[string]json.RawMessage
if err := json.Unmarshal(capturedBody, &payload); err != nil {
t.Fatal(err)
}
wantKeys := []string{
"type", "id", "agent_id", "operation_id", "action", "resource_id", "request_fingerprint",
"status", "effect_may_have_applied", "safe_to_resubmit", "error_code", "cause_code", "result",
}
if len(payload) != len(wantKeys) {
t.Fatalf("completion keys = %#v body=%s", payload, capturedBody)
}
for _, key := range wantKeys {
if _, ok := payload[key]; !ok {
t.Fatalf("completion omitted %q: %s", key, capturedBody)
}
}
if string(payload["type"]) != `"completion"` || string(payload["id"]) != `"completion_event_123"` ||
string(payload["agent_id"]) != `"42"` || string(payload["action"]) != `"delete-vm"` ||
string(payload["resource_id"]) != `"vm-completion-123"` ||
string(payload["request_fingerprint"]) != `"`+strings.Repeat("a", 64)+`"` ||
string(payload["cause_code"]) != `"backup_disk_transition_active"` ||
string(payload["result"]) != "null" {
t.Fatalf("completion payload = %s", capturedBody)
}
if _, legacy := payload["event_id"]; legacy {
t.Fatalf("legacy event_id leaked into completion: %s", capturedBody)
}
authenticator, err := protocol.NewAuthenticator(secret, time.Minute, 2*time.Minute, acceptNonceStore{})
if err != nil {
t.Fatal(err)
}
authenticator.SetClockForTesting(func() time.Time { return now })
actor, err := authenticator.Authenticate(captured, capturedBody)
if err != nil || actor.Service != "agent-gateway" || actor.UserID != 0 || actor.IsAdmin || actor.IsSuperAdmin || actor.CanManage {
t.Fatalf("completion actor = %#v err=%v", actor, err)
}
}
func TestHTTPCompletionSinkRequiresExactTerminalAcceptance(t *testing.T) {
completion := Completion{
Type: "completion", ID: "completion_event_strict", OperationID: "operation_terminal_strict",
Action: "delete-vm", ResourceID: "vm-completion-strict",
RequestFingerprint: strings.Repeat("b", 64),
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
}
tests := []struct {
name string
status int
contentType string
body string
wantError bool
}{
{name: "applied", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"applied"}`},
{name: "late", status: 200, contentType: "application/json; charset=UTF-8", body: `{"accepted":true,"disposition":"late"}`},
{name: "rejected durable receipt", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"rejected"}`},
{name: "received is not terminal", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"received"}`, wantError: true},
{name: "accepted false", status: 200, contentType: "application/json", body: `{"accepted":false,"disposition":"rejected"}`, wantError: true},
{name: "unknown field", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"applied","extra":1}`, wantError: true},
{name: "trailing JSON", status: 200, contentType: "application/json", body: `{"accepted":true,"disposition":"applied"}{}`, wantError: true},
{name: "missing body", status: 204, contentType: "application/json", wantError: true},
{name: "wrong content type", status: 200, contentType: "text/json", body: `{"accepted":true,"disposition":"applied"}`, wantError: true},
{name: "unknown content type parameter", status: 200, contentType: "application/json; version=1", body: `{"accepted":true,"disposition":"applied"}`, wantError: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
sink, err := NewHTTPCompletionSink("http://127.0.0.1:8080/api/internal/cloud/agent-auth", []byte(strings.Repeat("s", 32)), time.Second)
if err != nil {
t.Fatal(err)
}
sink.client = &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
header := make(http.Header)
if test.contentType != "" {
header.Set("Content-Type", test.contentType)
}
return &http.Response{StatusCode: test.status, Header: header,
Body: io.NopCloser(strings.NewReader(test.body)), Request: request}, nil
})}
err = sink.Deliver(context.Background(), "42", completion)
if (err != nil) != test.wantError {
t.Fatalf("Deliver() error = %v, wantError=%v", err, test.wantError)
}
})
}
}
type captureCompletionSink struct {
mu sync.Mutex
agentID string
completion Completion
called chan struct{}
deliveryErr error
}
func (s *captureCompletionSink) Deliver(_ context.Context, agentID string, completion Completion) error {
s.mu.Lock()
s.agentID = agentID
s.completion = completion
s.mu.Unlock()
close(s.called)
return s.deliveryErr
}
func TestCompletionDeliveryACKsOnlyTerminalPanelAcceptance(t *testing.T) {
for _, test := range []struct {
name string
sinkError error
wantType string
}{
{name: "terminal acceptance", wantType: "completion_ack"},
{name: "retryable delivery", sinkError: errors.New("injected delivery failure"), wantType: "completion_retry"},
} {
t.Run(test.name, func(t *testing.T) {
sink := &captureCompletionSink{called: make(chan struct{}), deliveryErr: test.sinkError}
written := make(chan any, 1)
registry := &Registry{completionSink: sink}
current := &session{
agentID: "84", completionSlots: make(chan struct{}, maxCompletionDeliveries), done: make(chan struct{}),
writeHook: func(_ context.Context, value any) error { written <- value; return nil },
}
completion := Completion{
Type: "completion", ID: "completion_disposition_123", OperationID: "operation_disposition_123",
Action: "delete-vm", ResourceID: "vm-completion-disposition",
RequestFingerprint: strings.Repeat("c", 64),
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
}
registry.deliverCompletion(context.Background(), current, completion)
select {
case value := <-written:
gotType := ""
switch message := value.(type) {
case completionAck:
gotType = message.Type
case completionRetry:
gotType = message.Type
}
if gotType != test.wantType {
t.Fatalf("completion disposition type = %q, want %q", gotType, test.wantType)
}
case <-time.After(time.Second):
t.Fatal("completion disposition was not written")
}
})
}
}
func TestRegistryBindsCompletionToAuthenticatedSessionAgent(t *testing.T) {
sink := &captureCompletionSink{called: make(chan struct{})}
registry := &Registry{completionSink: sink}
current := &session{
agentID: "84", completionSlots: make(chan struct{}, maxCompletionDeliveries), done: make(chan struct{}),
writeHook: func(context.Context, any) error { return nil },
}
completion := Completion{
Type: "completion", ID: "completion_event_456", OperationID: "operation_terminal_456",
Action: "delete-vm", ResourceID: "vm-completion-456",
RequestFingerprint: strings.Repeat("d", 64),
Status: "succeeded", Result: json.RawMessage(`{"accepted":true}`),
}
registry.deliverCompletion(context.Background(), current, completion)
select {
case <-sink.called:
case <-time.After(time.Second):
t.Fatal("completion was not delivered")
}
sink.mu.Lock()
agentID := sink.agentID
delivered := sink.completion
sink.mu.Unlock()
if agentID != current.agentID || delivered.ID != completion.ID {
t.Fatalf("delivery agent=%q completion=%#v", agentID, delivered)
}
deadline := time.Now().Add(time.Second)
for len(current.completionSlots) != 0 && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
if len(current.completionSlots) != 0 {
t.Fatal("completion delivery slot was not released after failure")
}
}
func TestCompletionDeliveryAppliesBackpressureInsteadOfDropping(t *testing.T) {
sink := &captureCompletionSink{called: make(chan struct{})}
registry := &Registry{completionSink: sink}
current := &session{
agentID: "84", completionSlots: make(chan struct{}, maxCompletionDeliveries), done: make(chan struct{}),
writeHook: func(context.Context, any) error { return nil },
}
for range maxCompletionDeliveries {
current.completionSlots <- struct{}{}
}
completion := Completion{
Type: "completion", ID: "completion_backpressure_123", OperationID: "operation_backpressure_123",
Action: "delete-vm", ResourceID: "vm-completion-backpressure",
RequestFingerprint: strings.Repeat("e", 64),
Status: "succeeded", Result: json.RawMessage(`{"accepted":true}`),
}
returned := make(chan struct{})
go func() {
registry.deliverCompletion(context.Background(), current, completion)
close(returned)
}()
select {
case <-returned:
t.Fatal("full completion window silently dropped the delivery")
case <-time.After(30 * time.Millisecond):
}
<-current.completionSlots
select {
case <-returned:
case <-time.After(time.Second):
t.Fatal("completion did not enter the delivery queue after capacity was released")
}
select {
case <-sink.called:
case <-time.After(time.Second):
t.Fatal("queued completion was not delivered")
}
}
func TestCompletionValidationRequiresExplicitNullResult(t *testing.T) {
valid := Completion{
Type: "completion", ID: "completion_event_789", OperationID: "operation_terminal_789",
Action: "delete-vm", ResourceID: "vm-completion-789",
RequestFingerprint: strings.Repeat("f", 64),
Status: "failed", ErrorCode: "operation_failed", Result: json.RawMessage("null"),
}
if err := validateCompletion(valid); err != nil {
t.Fatalf("explicit null result rejected: %v", err)
}
withCause := valid
withCause.CauseCode = "backup_disk_transition_active"
if err := validateCompletion(withCause); err != nil {
t.Fatalf("stable cause code rejected: %v", err)
}
invalidCause := valid
invalidCause.CauseCode = "Backup Disk Transition Active"
if err := validateCompletion(invalidCause); err == nil {
t.Fatal("invalid cause code was accepted")
}
missing := valid
missing.Result = nil
if err := validateCompletion(missing); err == nil {
t.Fatal("missing result key was accepted")
}
}
func TestCompletionValidationRequiresStrictImmutableOperationIdentity(t *testing.T) {
valid := Completion{
Type: "completion", ID: "completion_identity_123", OperationID: "operation_identity_123",
Action: "create-snapshot:0123456789abcdef0123456789abcdef:filesystem",
ResourceID: "template:debian-13", RequestFingerprint: strings.Repeat("a", 64),
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
}
if err := validateCompletion(valid); err != nil {
t.Fatalf("valid immutable identity rejected: %v", err)
}
tests := []struct {
name string
mutate func(*Completion)
}{
{name: "missing action", mutate: func(value *Completion) { value.Action = "" }},
{name: "uppercase action", mutate: func(value *Completion) { value.Action = "Delete-VM" }},
{name: "resource path", mutate: func(value *Completion) { value.ResourceID = "vm/unsafe" }},
{name: "missing resource", mutate: func(value *Completion) { value.ResourceID = "" }},
{name: "uppercase fingerprint", mutate: func(value *Completion) { value.RequestFingerprint = strings.Repeat("A", 64) }},
{name: "short fingerprint", mutate: func(value *Completion) { value.RequestFingerprint = strings.Repeat("a", 63) }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
candidate := valid
test.mutate(&candidate)
if err := validateCompletion(candidate); err == nil {
t.Fatal("invalid immutable operation identity was accepted")
}
})
}
}
func TestGatewayCompletionDeliveryLogContainsOnlyStableMetadata(t *testing.T) {
logger := log.Default()
previousWriter, previousFlags, previousPrefix := logger.Writer(), logger.Flags(), logger.Prefix()
var output strings.Builder
logger.SetOutput(&output)
logger.SetFlags(0)
logger.SetPrefix("")
t.Cleanup(func() {
logger.SetOutput(previousWriter)
logger.SetFlags(previousFlags)
logger.SetPrefix(previousPrefix)
})
logGatewayCompletionDelivery(completionPhaseGatewayACKSent, "operation_log_456", "completion_log_456")
got := strings.TrimSpace(output.String())
want := "completion_delivery phase=gateway_ack_sent operation_id=operation_log_456 event_id=completion_log_456"
if got != want {
t.Fatalf("completion log = %q, want %q", got, want)
}
}