You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
293 lines
9.9 KiB
Go
293 lines
9.9 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/auth"
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
)
|
|
|
|
func validRequest(operationID string) Request {
|
|
requestTarget := "/v1/vms/123e4567-e89b-42d3-a456-426614174000/power"
|
|
body := `{"operation_id":"` + operationID + `","action":"reboot"}`
|
|
return Request{
|
|
Version: ProtocolVersion,
|
|
Method: "POST",
|
|
RequestTarget: requestTarget,
|
|
Body: body,
|
|
Actor: auth.Actor{UserID: 1, CanManage: true, OperationID: operationID},
|
|
OperationID: operationID,
|
|
Fingerprint: Fingerprint("POST", requestTarget, []byte(body)),
|
|
}
|
|
}
|
|
|
|
func TestRequestRequiresExactActorOperationIdentity(t *testing.T) {
|
|
request := validRequest("operation-worker-identity")
|
|
if err := request.Validate(); err != nil {
|
|
t.Fatalf("valid request: %v", err)
|
|
}
|
|
missing := request
|
|
missing.Actor.OperationID = ""
|
|
if err := missing.Validate(); err == nil {
|
|
t.Fatal("missing actor operation ID was accepted")
|
|
}
|
|
mismatch := request
|
|
mismatch.Actor.OperationID = "operation-worker-other"
|
|
if err := mismatch.Validate(); err == nil {
|
|
t.Fatal("mismatched actor operation ID was accepted")
|
|
}
|
|
}
|
|
|
|
func TestReceiptPublicationIsImmutable(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "operation-receipt", workerReceiptName)
|
|
request := validRequest("operation-immutable-receipt")
|
|
first := Response{
|
|
Version: ProtocolVersion, OperationID: request.OperationID, Fingerprint: request.Fingerprint,
|
|
Status: "succeeded", Result: json.RawMessage(`{"accepted":true}`),
|
|
}
|
|
if err := WriteReceipt(path, first); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second := first
|
|
second.Result = json.RawMessage(`{"accepted":false}`)
|
|
if err := WriteReceipt(path, second); err == nil {
|
|
t.Fatal("second receipt replaced immutable completion")
|
|
}
|
|
stored, err := ReadReceipt(path)
|
|
if err != nil || string(stored.Result) != `{"accepted":true}` {
|
|
t.Fatalf("stored receipt = %#v err=%v", stored, err)
|
|
}
|
|
}
|
|
|
|
func TestExistingOperationDirectoryNeverSpawnsSecondWorker(t *testing.T) {
|
|
temp := t.TempDir()
|
|
executable := filepath.Join(temp, "must-not-run")
|
|
marker := filepath.Join(temp, "spawned")
|
|
script := "#!/bin/sh\n: > " + strconv.Quote(marker) + "\nexit 0\n"
|
|
if err := os.WriteFile(executable, []byte(script), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configPath := filepath.Join(temp, "config.json")
|
|
if err := os.WriteFile(configPath, []byte(`{}`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stateDir := filepath.Join(temp, "workers")
|
|
executor, err := NewSubprocessExecutor(executable, configPath, stateDir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request := validRequest("operation-existing-directory")
|
|
if err := os.Mkdir(filepath.Join(stateDir, request.OperationID), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = executor.Execute(context.Background(), request)
|
|
var workerErr *Error
|
|
if !errors.As(err, &workerErr) || workerErr.Code != ResultUnconfirmed || !workerErr.EffectMayHaveApplied {
|
|
t.Fatalf("existing directory result = %#v err=%v", workerErr, err)
|
|
}
|
|
if _, statErr := os.Stat(marker); !errors.Is(statErr, os.ErrNotExist) {
|
|
t.Fatalf("duplicate worker was spawned: %v", statErr)
|
|
}
|
|
}
|
|
|
|
func writeTerminalJournalWithoutReceipt(t *testing.T, operationDir string, request Request) operation.Record {
|
|
t.Helper()
|
|
j, err := operation.Open(filepath.Join(operationDir, workerJournalName))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := operation.Spec{
|
|
OperationID: request.OperationID,
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "power-reboot",
|
|
Fingerprint: request.Fingerprint,
|
|
Destructive: true,
|
|
}
|
|
record, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return json.RawMessage(`{"state":"running"}`), nil
|
|
})
|
|
if err != nil {
|
|
_ = j.Close()
|
|
t.Fatal(err)
|
|
}
|
|
if err := j.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return record
|
|
}
|
|
|
|
func TestPendingJournalTerminalsRecoversDurableTerminalWithoutReceipt(t *testing.T) {
|
|
temp := t.TempDir()
|
|
executor, err := NewSubprocessExecutor(
|
|
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request := validRequest("operation-terminal-no-receipt")
|
|
operationDir := filepath.Join(executor.stateDir, request.OperationID)
|
|
if err := os.Mkdir(operationDir, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
identity, err := requestIdentityFromRequest(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := writeRequestIdentity(filepath.Join(operationDir, workerIdentityName), identity); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := writeTerminalJournalWithoutReceipt(t, operationDir, request)
|
|
|
|
evidence, err := executor.PendingJournalTerminals()
|
|
if err != nil || len(evidence) != 1 {
|
|
t.Fatalf("pending terminal evidence = %#v err=%v", evidence, err)
|
|
}
|
|
got := evidence[0]
|
|
if got.Record.OperationID != want.OperationID || got.Record.Action != want.Action ||
|
|
got.Record.ResourceID != want.ResourceID || got.Record.Fingerprint != want.Fingerprint ||
|
|
got.Identity == nil || got.Identity.ActorUserID != request.Actor.UserID ||
|
|
got.Identity.RequestTarget != request.RequestTarget ||
|
|
got.Identity.BodySHA256 != auth.BodyDigest([]byte(request.Body)) || got.Legacy {
|
|
t.Fatalf("terminal evidence = %#v", got)
|
|
}
|
|
mainRecord := want
|
|
mainRecord.State = operation.StateDispatching
|
|
receipt, ok := got.InterruptedReceiptFor(mainRecord)
|
|
if !ok || receipt.Status != string(operation.StateSucceeded) || string(receipt.Result) != `{"state":"running"}` {
|
|
t.Fatalf("interrupted receipt = %#v ok=%v", receipt, ok)
|
|
}
|
|
mismatch := mainRecord
|
|
mismatch.Action = "power-reset"
|
|
if _, ok := got.InterruptedReceiptFor(mismatch); ok {
|
|
t.Fatal("mismatched action manufactured terminal success")
|
|
}
|
|
}
|
|
|
|
func TestPendingJournalTerminalsPreservesCorruptIdentityEvidence(t *testing.T) {
|
|
temp := t.TempDir()
|
|
executor, err := NewSubprocessExecutor(
|
|
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request := validRequest("operation-corrupt-identity")
|
|
operationDir := filepath.Join(executor.stateDir, request.OperationID)
|
|
if err := os.Mkdir(operationDir, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(operationDir, workerIdentityName), []byte(`{"operation_id":"other"}`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeTerminalJournalWithoutReceipt(t, operationDir, request)
|
|
|
|
evidence, err := executor.PendingJournalTerminals()
|
|
if err != nil || len(evidence) != 0 {
|
|
t.Fatalf("corrupt identity evidence = %#v err=%v", evidence, err)
|
|
}
|
|
if _, err := os.Stat(operationDir); err != nil {
|
|
t.Fatalf("unadopted worker evidence was deleted: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPendingJournalTerminalsSupportsTrustedLegacyJournal(t *testing.T) {
|
|
temp := t.TempDir()
|
|
executor, err := NewSubprocessExecutor(
|
|
filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"),
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request := validRequest("operation-legacy-terminal")
|
|
operationDir := filepath.Join(executor.stateDir, request.OperationID)
|
|
if err := os.Mkdir(operationDir, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := writeTerminalJournalWithoutReceipt(t, operationDir, request)
|
|
|
|
evidence, err := executor.PendingJournalTerminals()
|
|
if err != nil || len(evidence) != 1 || !evidence[0].Legacy || evidence[0].Identity != nil {
|
|
t.Fatalf("legacy terminal evidence = %#v err=%v", evidence, err)
|
|
}
|
|
mainRecord := want
|
|
mainRecord.State = operation.StateDispatching
|
|
if _, ok := evidence[0].InterruptedReceiptFor(mainRecord); !ok {
|
|
t.Fatal("trusted pre-upgrade terminal Journal was not recoverable")
|
|
}
|
|
}
|
|
|
|
func TestRequestValidationBindsFingerprintToExactTargetAndBody(t *testing.T) {
|
|
request := validRequest("operation-request-fingerprint")
|
|
request.Body = `{"operation_id":"operation-request-fingerprint","action":"reset"}`
|
|
if err := request.Validate(); err == nil {
|
|
t.Fatal("request body changed without changing fingerprint")
|
|
}
|
|
request = validRequest("operation-target-fingerprint")
|
|
request.RequestTarget += "?unexpected=true"
|
|
if err := request.Validate(); err == nil {
|
|
t.Fatal("request target changed without changing fingerprint")
|
|
}
|
|
}
|
|
|
|
func TestCancellationReapsHelperBeforeExecuteReturns(t *testing.T) {
|
|
temp := t.TempDir()
|
|
executable := filepath.Join(temp, "blocking-worker")
|
|
pidPath := filepath.Join(temp, "worker.pid")
|
|
script := fmt.Sprintf("#!/bin/sh\necho $$ > %s\ntrap 'exit 0' TERM INT\nwhile :; do sleep 1; done\n", strconv.Quote(pidPath))
|
|
if err := os.WriteFile(executable, []byte(script), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configPath := filepath.Join(temp, "config.json")
|
|
if err := os.WriteFile(configPath, []byte(`{}`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
executor, err := NewSubprocessExecutor(executable, configPath, filepath.Join(temp, "workers"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, executeErr := executor.Execute(ctx, validRequest("operation-cancel-reap"))
|
|
done <- executeErr
|
|
}()
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
var pid int
|
|
for {
|
|
pidBytes, readErr := os.ReadFile(pidPath)
|
|
if readErr == nil {
|
|
if _, scanErr := fmt.Sscanf(string(pidBytes), "%d", &pid); scanErr == nil && pid > 0 {
|
|
break
|
|
}
|
|
}
|
|
if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {
|
|
t.Fatal(readErr)
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("helper did not publish a valid pid: %v", readErr)
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
cancel()
|
|
err = <-done
|
|
var workerErr *Error
|
|
if !errors.As(err, &workerErr) || workerErr.Code != ResultUnconfirmed {
|
|
t.Fatalf("canceled worker result = %#v err=%v", workerErr, err)
|
|
}
|
|
if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) {
|
|
t.Fatalf("Execute returned before worker was reaped; kill(0) err=%v", err)
|
|
}
|
|
if _, statErr := os.Stat(pidPath); statErr != nil {
|
|
t.Fatalf("worker pid evidence disappeared: %v", statErr)
|
|
}
|
|
}
|