package worker import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "strconv" "syscall" "testing" "time" "git.luoli.live/root/virt/agent/auth" "git.luoli.live/root/virt/agent/operation" ) func validRequest(operationID string) Request { requestTarget := "/v1/vms/123e4567-e89b-42d3-a456-426614174000/power" body := `{"operation_id":"` + operationID + `","action":"reboot"}` return Request{ Version: ProtocolVersion, Method: "POST", RequestTarget: requestTarget, Body: body, Actor: auth.Actor{UserID: 1, CanManage: true, OperationID: operationID}, OperationID: operationID, Fingerprint: Fingerprint("POST", requestTarget, []byte(body)), } } func TestRequestRequiresExactActorOperationIdentity(t *testing.T) { request := validRequest("operation-worker-identity") if err := request.Validate(); err != nil { t.Fatalf("valid request: %v", err) } missing := request missing.Actor.OperationID = "" if err := missing.Validate(); err == nil { t.Fatal("missing actor operation ID was accepted") } mismatch := request mismatch.Actor.OperationID = "operation-worker-other" if err := mismatch.Validate(); err == nil { t.Fatal("mismatched actor operation ID was accepted") } } func TestReceiptPublicationIsImmutable(t *testing.T) { path := filepath.Join(t.TempDir(), "operation-receipt", workerReceiptName) request := validRequest("operation-immutable-receipt") first := Response{ Version: ProtocolVersion, OperationID: request.OperationID, Fingerprint: request.Fingerprint, Status: "succeeded", Result: json.RawMessage(`{"accepted":true}`), } if err := WriteReceipt(path, first); err != nil { t.Fatal(err) } second := first second.Result = json.RawMessage(`{"accepted":false}`) if err := WriteReceipt(path, second); err == nil { t.Fatal("second receipt replaced immutable completion") } stored, err := ReadReceipt(path) if err != nil || string(stored.Result) != `{"accepted":true}` { t.Fatalf("stored receipt = %#v err=%v", stored, err) } } func TestExistingOperationDirectoryNeverSpawnsSecondWorker(t *testing.T) { temp := t.TempDir() executable := filepath.Join(temp, "must-not-run") marker := filepath.Join(temp, "spawned") script := "#!/bin/sh\n: > " + strconv.Quote(marker) + "\nexit 0\n" if err := os.WriteFile(executable, []byte(script), 0o700); err != nil { t.Fatal(err) } configPath := filepath.Join(temp, "config.json") if err := os.WriteFile(configPath, []byte(`{}`), 0o600); err != nil { t.Fatal(err) } stateDir := filepath.Join(temp, "workers") executor, err := NewSubprocessExecutor(executable, configPath, stateDir) if err != nil { t.Fatal(err) } request := validRequest("operation-existing-directory") if err := os.Mkdir(filepath.Join(stateDir, request.OperationID), 0o700); err != nil { t.Fatal(err) } _, err = executor.Execute(context.Background(), request) var workerErr *Error if !errors.As(err, &workerErr) || workerErr.Code != ResultUnconfirmed || !workerErr.EffectMayHaveApplied { t.Fatalf("existing directory result = %#v err=%v", workerErr, err) } if _, statErr := os.Stat(marker); !errors.Is(statErr, os.ErrNotExist) { t.Fatalf("duplicate worker was spawned: %v", statErr) } } func writeTerminalJournalWithoutReceipt(t *testing.T, operationDir string, request Request) operation.Record { t.Helper() j, err := operation.Open(filepath.Join(operationDir, workerJournalName)) if err != nil { t.Fatal(err) } spec := operation.Spec{ OperationID: request.OperationID, ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "power-reboot", Fingerprint: request.Fingerprint, Destructive: true, } record, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return json.RawMessage(`{"state":"running"}`), nil }) if err != nil { _ = j.Close() t.Fatal(err) } if err := j.Close(); err != nil { t.Fatal(err) } return record } func TestPendingJournalTerminalsRecoversDurableTerminalWithoutReceipt(t *testing.T) { temp := t.TempDir() executor, err := NewSubprocessExecutor( filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"), ) if err != nil { t.Fatal(err) } request := validRequest("operation-terminal-no-receipt") operationDir := filepath.Join(executor.stateDir, request.OperationID) if err := os.Mkdir(operationDir, 0o700); err != nil { t.Fatal(err) } identity, err := requestIdentityFromRequest(request) if err != nil { t.Fatal(err) } if err := writeRequestIdentity(filepath.Join(operationDir, workerIdentityName), identity); err != nil { t.Fatal(err) } want := writeTerminalJournalWithoutReceipt(t, operationDir, request) evidence, err := executor.PendingJournalTerminals() if err != nil || len(evidence) != 1 { t.Fatalf("pending terminal evidence = %#v err=%v", evidence, err) } got := evidence[0] if got.Record.OperationID != want.OperationID || got.Record.Action != want.Action || got.Record.ResourceID != want.ResourceID || got.Record.Fingerprint != want.Fingerprint || got.Identity == nil || got.Identity.ActorUserID != request.Actor.UserID || got.Identity.RequestTarget != request.RequestTarget || got.Identity.BodySHA256 != auth.BodyDigest([]byte(request.Body)) || got.Legacy { t.Fatalf("terminal evidence = %#v", got) } mainRecord := want mainRecord.State = operation.StateDispatching receipt, ok := got.InterruptedReceiptFor(mainRecord) if !ok || receipt.Status != string(operation.StateSucceeded) || string(receipt.Result) != `{"state":"running"}` { t.Fatalf("interrupted receipt = %#v ok=%v", receipt, ok) } mismatch := mainRecord mismatch.Action = "power-reset" if _, ok := got.InterruptedReceiptFor(mismatch); ok { t.Fatal("mismatched action manufactured terminal success") } } func TestPendingJournalTerminalsPreservesCorruptIdentityEvidence(t *testing.T) { temp := t.TempDir() executor, err := NewSubprocessExecutor( filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"), ) if err != nil { t.Fatal(err) } request := validRequest("operation-corrupt-identity") operationDir := filepath.Join(executor.stateDir, request.OperationID) if err := os.Mkdir(operationDir, 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(operationDir, workerIdentityName), []byte(`{"operation_id":"other"}`), 0o600); err != nil { t.Fatal(err) } writeTerminalJournalWithoutReceipt(t, operationDir, request) evidence, err := executor.PendingJournalTerminals() if err != nil || len(evidence) != 0 { t.Fatalf("corrupt identity evidence = %#v err=%v", evidence, err) } if _, err := os.Stat(operationDir); err != nil { t.Fatalf("unadopted worker evidence was deleted: %v", err) } } func TestPendingJournalTerminalsSupportsTrustedLegacyJournal(t *testing.T) { temp := t.TempDir() executor, err := NewSubprocessExecutor( filepath.Join(temp, "virt-agent"), filepath.Join(temp, "config.json"), filepath.Join(temp, "workers"), ) if err != nil { t.Fatal(err) } request := validRequest("operation-legacy-terminal") operationDir := filepath.Join(executor.stateDir, request.OperationID) if err := os.Mkdir(operationDir, 0o700); err != nil { t.Fatal(err) } want := writeTerminalJournalWithoutReceipt(t, operationDir, request) evidence, err := executor.PendingJournalTerminals() if err != nil || len(evidence) != 1 || !evidence[0].Legacy || evidence[0].Identity != nil { t.Fatalf("legacy terminal evidence = %#v err=%v", evidence, err) } mainRecord := want mainRecord.State = operation.StateDispatching if _, ok := evidence[0].InterruptedReceiptFor(mainRecord); !ok { t.Fatal("trusted pre-upgrade terminal Journal was not recoverable") } } func TestRequestValidationBindsFingerprintToExactTargetAndBody(t *testing.T) { request := validRequest("operation-request-fingerprint") request.Body = `{"operation_id":"operation-request-fingerprint","action":"reset"}` if err := request.Validate(); err == nil { t.Fatal("request body changed without changing fingerprint") } request = validRequest("operation-target-fingerprint") request.RequestTarget += "?unexpected=true" if err := request.Validate(); err == nil { t.Fatal("request target changed without changing fingerprint") } } func TestCancellationReapsHelperBeforeExecuteReturns(t *testing.T) { temp := t.TempDir() executable := filepath.Join(temp, "blocking-worker") pidPath := filepath.Join(temp, "worker.pid") script := fmt.Sprintf("#!/bin/sh\necho $$ > %s\ntrap 'exit 0' TERM INT\nwhile :; do sleep 1; done\n", strconv.Quote(pidPath)) if err := os.WriteFile(executable, []byte(script), 0o700); err != nil { t.Fatal(err) } configPath := filepath.Join(temp, "config.json") if err := os.WriteFile(configPath, []byte(`{}`), 0o600); err != nil { t.Fatal(err) } executor, err := NewSubprocessExecutor(executable, configPath, filepath.Join(temp, "workers")) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { _, executeErr := executor.Execute(ctx, validRequest("operation-cancel-reap")) done <- executeErr }() deadline := time.Now().Add(10 * time.Second) var pid int for { pidBytes, readErr := os.ReadFile(pidPath) if readErr == nil { if _, scanErr := fmt.Sscanf(string(pidBytes), "%d", &pid); scanErr == nil && pid > 0 { break } } if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { t.Fatal(readErr) } if time.Now().After(deadline) { t.Fatalf("helper did not publish a valid pid: %v", readErr) } time.Sleep(time.Millisecond) } cancel() err = <-done var workerErr *Error if !errors.As(err, &workerErr) || workerErr.Code != ResultUnconfirmed { t.Fatalf("canceled worker result = %#v err=%v", workerErr, err) } if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) { t.Fatalf("Execute returned before worker was reaped; kill(0) err=%v", err) } if _, statErr := os.Stat(pidPath); statErr != nil { t.Fatalf("worker pid evidence disappeared: %v", statErr) } }