You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/worker/executor_test.go

293 lines
9.9 KiB
Go

package worker
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"syscall"
"testing"
"time"
"git.luoli.live/root/virt/agent/auth"
"git.luoli.live/root/virt/agent/operation"
)
func validRequest(operationID string) Request {
requestTarget := "/v1/vms/123e4567-e89b-42d3-a456-426614174000/power"
body := `{"operation_id":"` + operationID + `","action":"reboot"}`
return Request{
Version: ProtocolVersion,
Method: "POST",
RequestTarget: requestTarget,
Body: body,
Actor: auth.Actor{UserID: 1, CanManage: true, OperationID: operationID},
OperationID: operationID,
Fingerprint: Fingerprint("POST", requestTarget, []byte(body)),
}
}
func TestRequestRequiresExactActorOperationIdentity(t *testing.T) {
request := validRequest("operation-worker-identity")
if err := request.Validate(); err != nil {
t.Fatalf("valid request: %v", err)
}
missing := request
missing.Actor.OperationID = ""
if err := missing.Validate(); err == nil {
t.Fatal("missing actor operation ID was accepted")
}
mismatch := request
mismatch.Actor.OperationID = "operation-worker-other"
if err := mismatch.Validate(); err == nil {
t.Fatal("mismatched actor operation ID was accepted")
}
}
func TestReceiptPublicationIsImmutable(t *testing.T) {
path := filepath.Join(t.TempDir(), "operation-receipt", workerReceiptName)
request := validRequest("operation-immutable-receipt")
first := Response{
Version: ProtocolVersion, OperationID: request.OperationID, Fingerprint: request.Fingerprint,
Status: "succeeded", Result: json.RawMessage(`{"accepted":true}`),
}
if err := WriteReceipt(path, first); err != nil {
t.Fatal(err)
}
second := first
second.Result = json.RawMessage(`{"accepted":false}`)
if err := WriteReceipt(path, second); err == nil {
t.Fatal("second receipt replaced immutable completion")
}
stored, err := ReadReceipt(path)
if err != nil || string(stored.Result) != `{"accepted":true}` {
t.Fatalf("stored receipt = %#v err=%v", stored, err)
}
}
func TestExistingOperationDirectoryNeverSpawnsSecondWorker(t *testing.T) {
temp := t.TempDir()
executable := filepath.Join(temp, "must-not-run")
marker := filepath.Join(temp, "spawned")
script := "#!/bin/sh\n: > " + strconv.Quote(marker) + "\nexit 0\n"
if err := os.WriteFile(executable, []byte(script), 0o700); err != nil {
t.Fatal(err)
}
configPath := filepath.Join(temp, "config.json")
if err := os.WriteFile(configPath, []byte(`{}`), 0o600); err != nil {
t.Fatal(err)
}
stateDir := filepath.Join(temp, "workers")
executor, err := NewSubprocessExecutor(executable, configPath, stateDir)
if err != nil {
t.Fatal(err)
}
request := validRequest("operation-existing-directory")
if err := os.Mkdir(filepath.Join(stateDir, request.OperationID), 0o700); err != nil {
t.Fatal(err)
}
_, err = executor.Execute(context.Background(), request)
var workerErr *Error
if !errors.As(err, &workerErr) || workerErr.Code != ResultUnconfirmed || !workerErr.EffectMayHaveApplied {
t.Fatalf("existing directory result = %#v err=%v", workerErr, err)
}
if _, statErr := os.Stat(marker); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("duplicate worker was spawned: %v", statErr)
}
}
func writeTerminalJournalWithoutReceipt(t *testing.T, operationDir string, request Request) operation.Record {
t.Helper()
j, err := operation.Open(filepath.Join(operationDir, workerJournalName))
if err != nil {
t.Fatal(err)
}
spec := operation.Spec{
OperationID: request.OperationID,
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "power-reboot",
Fingerprint: request.Fingerprint,
Destructive: true,
}
record, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"state":"running"}`), nil
})
if err != nil {
_ = j.Close()
t.Fatal(err)
}
if err := j.Close(); err != nil {
t.Fatal(err)
}
return record
}
func TestPendingJournalTerminalsRecoversDurableTerminalWithoutReceipt(t *testing.T) {
temp := t.TempDir()
executor, err := NewSubprocessExecutor(
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
)
if err != nil {
t.Fatal(err)
}
request := validRequest("operation-terminal-no-receipt")
operationDir := filepath.Join(executor.stateDir, request.OperationID)
if err := os.Mkdir(operationDir, 0o700); err != nil {
t.Fatal(err)
}
identity, err := requestIdentityFromRequest(request)
if err != nil {
t.Fatal(err)
}
if err := writeRequestIdentity(filepath.Join(operationDir, workerIdentityName), identity); err != nil {
t.Fatal(err)
}
want := writeTerminalJournalWithoutReceipt(t, operationDir, request)
evidence, err := executor.PendingJournalTerminals()
if err != nil || len(evidence) != 1 {
t.Fatalf("pending terminal evidence = %#v err=%v", evidence, err)
}
got := evidence[0]
if got.Record.OperationID != want.OperationID || got.Record.Action != want.Action ||
got.Record.ResourceID != want.ResourceID || got.Record.Fingerprint != want.Fingerprint ||
got.Identity == nil || got.Identity.ActorUserID != request.Actor.UserID ||
got.Identity.RequestTarget != request.RequestTarget ||
got.Identity.BodySHA256 != auth.BodyDigest([]byte(request.Body)) || got.Legacy {
t.Fatalf("terminal evidence = %#v", got)
}
mainRecord := want
mainRecord.State = operation.StateDispatching
receipt, ok := got.InterruptedReceiptFor(mainRecord)
if !ok || receipt.Status != string(operation.StateSucceeded) || string(receipt.Result) != `{"state":"running"}` {
t.Fatalf("interrupted receipt = %#v ok=%v", receipt, ok)
}
mismatch := mainRecord
mismatch.Action = "power-reset"
if _, ok := got.InterruptedReceiptFor(mismatch); ok {
t.Fatal("mismatched action manufactured terminal success")
}
}
func TestPendingJournalTerminalsPreservesCorruptIdentityEvidence(t *testing.T) {
temp := t.TempDir()
executor, err := NewSubprocessExecutor(
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
)
if err != nil {
t.Fatal(err)
}
request := validRequest("operation-corrupt-identity")
operationDir := filepath.Join(executor.stateDir, request.OperationID)
if err := os.Mkdir(operationDir, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(operationDir, workerIdentityName), []byte(`{"operation_id":"other"}`), 0o600); err != nil {
t.Fatal(err)
}
writeTerminalJournalWithoutReceipt(t, operationDir, request)
evidence, err := executor.PendingJournalTerminals()
if err != nil || len(evidence) != 0 {
t.Fatalf("corrupt identity evidence = %#v err=%v", evidence, err)
}
if _, err := os.Stat(operationDir); err != nil {
t.Fatalf("unadopted worker evidence was deleted: %v", err)
}
}
func TestPendingJournalTerminalsSupportsTrustedLegacyJournal(t *testing.T) {
temp := t.TempDir()
executor, err := NewSubprocessExecutor(
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
)
if err != nil {
t.Fatal(err)
}
request := validRequest("operation-legacy-terminal")
operationDir := filepath.Join(executor.stateDir, request.OperationID)
if err := os.Mkdir(operationDir, 0o700); err != nil {
t.Fatal(err)
}
want := writeTerminalJournalWithoutReceipt(t, operationDir, request)
evidence, err := executor.PendingJournalTerminals()
if err != nil || len(evidence) != 1 || !evidence[0].Legacy || evidence[0].Identity != nil {
t.Fatalf("legacy terminal evidence = %#v err=%v", evidence, err)
}
mainRecord := want
mainRecord.State = operation.StateDispatching
if _, ok := evidence[0].InterruptedReceiptFor(mainRecord); !ok {
t.Fatal("trusted pre-upgrade terminal Journal was not recoverable")
}
}
func TestRequestValidationBindsFingerprintToExactTargetAndBody(t *testing.T) {
request := validRequest("operation-request-fingerprint")
request.Body = `{"operation_id":"operation-request-fingerprint","action":"reset"}`
if err := request.Validate(); err == nil {
t.Fatal("request body changed without changing fingerprint")
}
request = validRequest("operation-target-fingerprint")
request.RequestTarget += "?unexpected=true"
if err := request.Validate(); err == nil {
t.Fatal("request target changed without changing fingerprint")
}
}
func TestCancellationReapsHelperBeforeExecuteReturns(t *testing.T) {
temp := t.TempDir()
executable := filepath.Join(temp, "blocking-worker")
pidPath := filepath.Join(temp, "worker.pid")
script := fmt.Sprintf("#!/bin/sh\necho $$ > %s\ntrap 'exit 0' TERM INT\nwhile :; do sleep 1; done\n", strconv.Quote(pidPath))
if err := os.WriteFile(executable, []byte(script), 0o700); err != nil {
t.Fatal(err)
}
configPath := filepath.Join(temp, "config.json")
if err := os.WriteFile(configPath, []byte(`{}`), 0o600); err != nil {
t.Fatal(err)
}
executor, err := NewSubprocessExecutor(executable, configPath, filepath.Join(temp, "workers"))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
_, executeErr := executor.Execute(ctx, validRequest("operation-cancel-reap"))
done <- executeErr
}()
deadline := time.Now().Add(10 * time.Second)
var pid int
for {
pidBytes, readErr := os.ReadFile(pidPath)
if readErr == nil {
if _, scanErr := fmt.Sscanf(string(pidBytes), "%d", &pid); scanErr == nil && pid > 0 {
break
}
}
if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {
t.Fatal(readErr)
}
if time.Now().After(deadline) {
t.Fatalf("helper did not publish a valid pid: %v", readErr)
}
time.Sleep(time.Millisecond)
}
cancel()
err = <-done
var workerErr *Error
if !errors.As(err, &workerErr) || workerErr.Code != ResultUnconfirmed {
t.Fatalf("canceled worker result = %#v err=%v", workerErr, err)
}
if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) {
t.Fatalf("Execute returned before worker was reaped; kill(0) err=%v", err)
}
if _, statErr := os.Stat(pidPath); statErr != nil {
t.Fatalf("worker pid evidence disappeared: %v", statErr)
}
}