diff --git a/README.md b/README.md index fe94109e..cc265229 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,10 @@ Virt Cloud 是 PHP 8.5 面板与宿主机 Go Agent 组成的虚拟机销售平 ## 架构与功能 -- PHP 负责账户、授权、余额、支付、订单、账期、资源账本、Agent 注册和操作协调。管理员在 `/admin/agents` 为每台宿主机配置固定 IP HTTPS Agent endpoint 与独立 HMAC 密钥。 +- PHP 负责账户、授权、余额、支付、订单、账期、资源账本、Agent 注册和持久任务协调。管理员在 `/admin/agents` 为每台宿主机配置独立 Agent 密钥;宿主机不再向面板暴露管理端口。 - PHP 使用 `cloud_data_key` 的用途隔离密钥密封 Agent 密钥、任务载荷和凭据;只接受当前版本密文,不导入或尝试解密旧格式。 - 每台虚拟化宿主机运行 `virt-agent`,只连接本机 `qemu:///system`,执行 libvirt、存储、网络、快照、救援、指标和控制台后端操作。面板不保存 SSH/libvirt 登录凭据,也不直接执行宿主机命令。 -- 面板与 Agent 使用 TLS、请求体 SHA-256、时间戳、随机 nonce、操作者上下文和 HMAC-SHA256 签名;每个请求都必须认证,不能把 Agent 端口暴露为公共管理 API。 +- Agent 以 `panel_url + agent_id + agent_secret` 主动建立 WSS 长连接。PHP 先持久化任务,再通过面板本机 Go Agent Gateway 转发带 SHA-256、时间戳、nonce、操作者上下文和 HMAC-SHA256 签名的请求。断线后只按原 `operation_id` 对账,不自动重放。 - 所有虚拟机使用宿主机能力支持的 Q35 UEFI,并明确关闭 Secure Boot。不要硬编码某个发行版的 OVMF 路径。 - 每个云镜像可配置仅首次安装执行的启动脚本。创建任务会快照脚本版本;脚本成功、失败或中断后都不会自动重放,租户不能读取或覆盖脚本正文。 - 用户购买套餐开通一台 VM;每次购买或续费生成独立预付费账期及流量额度。额度用尽只断网、不关机;下一已付费账期恢复网络。最终到期只断网;未支付续费账单进入三天宽限期,之后永久删除 VM 定义和面板管理的磁盘。 @@ -20,14 +20,14 @@ Virt Cloud 是 PHP 8.5 面板与宿主机 Go Agent 组成的虚拟机销售平 virt-cloud/ agent/ 宿主机 Go Agent 与测试 web/ PHP 面板、页面、全新 SQL schema 与测试 - deploy/ Agent、PHP workers、控制台 relay 和更新 helper 的部署样例 + deploy/ Agent、PHP workers、Agent Gateway 和更新 helper 的部署样例 runtime/ 安装时生成的私有状态与密钥,不入 Git docs/ 架构、部署、运维和验证说明 ``` -HTTP 文档根目录必须是 `web/public/`。普通页面和 API 全部由 PHP 提供。浏览器控制台需要面板侧 `virt-cloud-console-relay`;它只接收 PHP 签发的一次性票据,不持有宿主机长期密钥。未部署并验收 relay 时必须将控制台功能标记为不可用,不能回退到旧引擎。 +HTTP 文档根目录必须是 `web/public/`。普通页面和 API 全部由 PHP 提供。面板侧 `virt-cloud-agent-gateway` 同时承载 Agent 主动控制通道和反向控制台:浏览器只提交 PHP 签发的一次性票据;Gateway 仅在已连接会话的内存中暂存该 Agent 握手密钥,用于每次普通请求或控制台打开前向 PHP 复核,绝不持久化。未部署并验收 Gateway 时必须将 Agent 操作和控制台标记为不可用,不能回退到旧引擎。 -Agent、更新 helper 和 console relay 只校验配置文件是否可读、JSON 与字段内容是否有效;它们不审计 owner、group 或 mode,不会因为可读文件是 `0600`、`0640` 或 `0644` 而拒绝启动。最小权限仍是运维加固建议,不是应用启动的权限位合规检查。 +Agent、更新 helper 和 Agent Gateway 只校验配置文件是否可读、JSON 与字段内容是否有效;它们不审计 owner、group 或 mode,不会因为可读文件是 `0600`、`0640` 或 `0644` 而拒绝启动。最小权限仍是运维加固建议,不是应用启动的权限位合规检查。 云镜像只接受经过校验的独立 QCOW2 内容;远程导入只允许可公开解析的 HTTPS 直链,并对初始地址及全部跳转执行 SSRF、大小与摘要校验,自制镜像通过管理员上传入口导入。拒绝 raw/ISO、加密镜像、外部 data file、backing file 和超过限制的文件。镜像来源的发行方可信度由管理员负责确认。 @@ -35,4 +35,4 @@ Agent、更新 helper 和 console relay 只校验配置文件是否可读、JSON 这是全新数据库版本。只能向空的业务库和日志库安装,旧表、旧密文、旧宿主机凭据及旧任务不能迁移或复用。完整步骤见 [部署说明](docs/DEPLOYMENT.md),职责与协议见 [集成架构](docs/INTEGRATION_PLAN.md),验证门槛见 [验证说明](docs/VALIDATION.md)。 -开放销售前必须在非生产宿主机完成 MariaDB 并发、Agent TLS/HMAC、UEFI、cloud-init 启动脚本、VM 生命周期、控制台、指标、流量、镜像/救援、付款和故障核对验收。单元测试通过不代表生产环境已验收。 +开放销售前必须在非生产宿主机完成 MariaDB 并发、Agent WSS/HMAC、UEFI、cloud-init 启动脚本、VM 生命周期、反向控制台、指标、流量、HTTPS 镜像/救援、付款和故障核对验收。单元测试通过不代表生产环境已验收。 diff --git a/agent/api/server.go b/agent/api/server.go index 96534a5d..bba2e859 100644 --- a/agent/api/server.go +++ b/agent/api/server.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/sha256" - "crypto/tls" "encoding/json" "errors" "fmt" @@ -35,7 +34,10 @@ var ( sourceTokenPattern = regexp.MustCompile(`^[A-Za-z0-9._~-]{32,512}$`) ) -const catalogImportTimeout = 2 * time.Hour +const ( + catalogImportTimeout = 2 * time.Hour + maxAsyncImports = 2 +) type Server struct { Authenticator *auth.Authenticator @@ -46,6 +48,13 @@ type Server struct { OperationTimeout time.Duration asyncMu sync.Mutex asyncRunning map[string]struct{} + asyncSlots chan struct{} + asyncWG sync.WaitGroup + lifecycleMu sync.Mutex + closing bool + requestWG sync.WaitGroup + backgroundCtx context.Context + backgroundCancel context.CancelFunc } type errorBody struct { @@ -85,33 +94,22 @@ func NewServer( if operationTimeout <= 0 { return nil, errors.New("operation timeout must be positive") } + backgroundCtx, backgroundCancel := context.WithCancel(context.Background()) return &Server{ Authenticator: authenticator, Journal: journal, Hypervisor: hv, Console: console, MaxBodyBytes: maxBodyBytes, OperationTimeout: operationTimeout, - asyncRunning: make(map[string]struct{}), + asyncRunning: make(map[string]struct{}), asyncSlots: make(chan struct{}, maxAsyncImports), + backgroundCtx: backgroundCtx, backgroundCancel: backgroundCancel, }, nil } -func BoundedHTTPServer(address string, handler http.Handler) *http.Server { - return &http.Server{ - Addr: address, Handler: handler, - ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 15 * time.Second, - WriteTimeout: 3 * time.Minute, - IdleTimeout: 60 * time.Second, - MaxHeaderBytes: 32 << 10, - TLSConfig: &tls.Config{ - MinVersion: tls.VersionTLS12, - }, - } -} - func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { setResponseHeaders(w) - if r.URL.Path == "/v1/console" { - s.Console.ServeHTTP(w, r) + if !s.beginRequest() { + writeError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down", nil) return } + defer s.requestWG.Done() if !strings.HasPrefix(r.URL.Path, "/v1/") || strings.HasSuffix(r.URL.Path, "/") { writeError(w, http.StatusNotFound, "not_found", "endpoint not found", nil) return @@ -983,23 +981,98 @@ func (s *Server) runAsyncMutation( writeMappedError(w, err, nil) return } - if record.State == operation.StateReserved && s.claimAsync(operationID) { - go func() { - defer s.releaseAsync(operationID) - ctx, cancel := context.WithTimeout(context.Background(), catalogImportTimeout) - defer cancel() - _, _, _ = s.Journal.Execute(ctx, spec, func(ctx context.Context) (json.RawMessage, error) { + if record.State == operation.StateReserved && !s.claimAsync(operationID) { + writeError(w, http.StatusServiceUnavailable, "operation_starting", "operation dispatch is already starting", &record) + return + } + if record.State == operation.StateReserved { + select { + case s.asyncSlots <- struct{}{}: + default: + failed, failErr := s.Journal.FailNotDispatched(operationID) + s.releaseAsync(operationID) + if failErr != nil { + writeMappedError(w, failErr, &record) + return + } + writeOperationFailure(w, failed, replayed) + return + } + workerCtx, cancel := context.WithTimeout(s.backgroundCtx, catalogImportTimeout) + started, _, done, startErr := s.Journal.ExecuteBackground( + r.Context(), workerCtx, spec, + func(ctx context.Context) (json.RawMessage, error) { result, callErr := call(ctx) if callErr != nil { return nil, callErr } return json.Marshal(result) - }) - }() + }, + ) + if startErr != nil || done == nil { + cancel() + <-s.asyncSlots + s.releaseAsync(operationID) + if startErr != nil { + writeMappedError(w, startErr, &started) + return + } + record = started + } else { + record = started + s.asyncWG.Add(1) + go func() { + defer s.asyncWG.Done() + defer cancel() + defer func() { <-s.asyncSlots }() + defer s.releaseAsync(operationID) + <-done + }() + } } writeJSON(w, http.StatusAccepted, operationWire(record, replayed)) } +func (s *Server) beginRequest() bool { + s.lifecycleMu.Lock() + defer s.lifecycleMu.Unlock() + if s.closing { + return false + } + s.requestWG.Add(1) + return true +} + +// Shutdown stops background imports and waits for every admitted API handler +// and import watcher. The journal's own workers are drained separately before +// libvirt or BoltDB are closed. +func (s *Server) Shutdown(ctx context.Context) error { + s.lifecycleMu.Lock() + if !s.closing { + s.closing = true + s.backgroundCancel() + } + s.lifecycleMu.Unlock() + if err := waitGroup(ctx, &s.requestWG); err != nil { + return err + } + return waitGroup(ctx, &s.asyncWG) +} + +func waitGroup(ctx context.Context, group *sync.WaitGroup) error { + done := make(chan struct{}) + go func() { + group.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + func (s *Server) claimAsync(operationID string) bool { s.asyncMu.Lock() defer s.asyncMu.Unlock() @@ -1181,6 +1254,9 @@ func classifyError(err error) (int, string, string) { if errors.Is(err, operation.ErrNotFound) { return http.StatusNotFound, "operation_not_found", "operation was not found" } + if errors.Is(err, operation.ErrBusy) { + return http.StatusServiceUnavailable, "agent_busy", "Agent mutation capacity is exhausted" + } var backend *hypervisor.Error if errors.As(err, &backend) { status := statusForKind(backend.Kind) diff --git a/agent/api/server_test.go b/agent/api/server_test.go index 49b6a6ea..56307148 100644 --- a/agent/api/server_test.go +++ b/agent/api/server_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/sha256" - "crypto/tls" "encoding/hex" "encoding/json" "fmt" @@ -140,6 +139,12 @@ func newTestAPI(t *testing.T) *testAPI { if err != nil { t.Fatal(err) } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = handler.Shutdown(ctx) + _ = journal.Wait(ctx) + }) return &testAPI{handler: handler, hv: hv, journal: journal, secret: secret, now: now} } @@ -196,13 +201,6 @@ func TestListResponsesUseNamedNonNullEnvelope(t *testing.T) { } } -func TestBoundedHTTPServerSetsTLSFloor(t *testing.T) { - server := BoundedHTTPServer("127.0.0.1:9443", http.NotFoundHandler()) - if server.TLSConfig == nil || server.TLSConfig.MinVersion != tls.VersionTLS12 { - t.Fatalf("TLS minimum version = %#v", server.TLSConfig) - } -} - func TestReinstallRejectsStartupScript(t *testing.T) { api := newTestAPI(t) actor := auth.Actor{UserID: 1, CanManage: true, OperationID: "operation-reinstall"} @@ -316,7 +314,10 @@ func TestConsoleTicketIsHMACProtectedAndDoesNotExposeTarget(t *testing.T) { if response.Code != http.StatusCreated { t.Fatalf("console ticket status = %d body=%s", response.Code, response.Body.String()) } - if strings.Contains(response.Body.String(), "127.0.0.1") || !strings.Contains(response.Body.String(), "cloud-agent-ticket.") { + if strings.Contains(response.Body.String(), "127.0.0.1") || + strings.Contains(response.Body.String(), "/v1/console") || + strings.Contains(response.Body.String(), "protocol") || + !strings.Contains(response.Body.String(), `"ticket"`) { t.Fatalf("unsafe console ticket response: %s", response.Body.String()) } } @@ -391,7 +392,7 @@ func TestCatalogImportsAreDurableAsyncAndAcceptPrivateSourceToken(t *testing.T) for _, test := range tests { t.Run(test.name, func(t *testing.T) { response := api.request(t, http.MethodPost, test.target, []byte(test.body), test.actor) - if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"status":"reserved"`) { + if response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"status":"dispatching"`) { t.Fatalf("async import response = %d %s", response.Code, response.Body.String()) } test.check(t) @@ -399,6 +400,28 @@ func TestCatalogImportsAreDurableAsyncAndAcceptPrivateSourceToken(t *testing.T) } } +func TestAsyncImportNeverReturnsAcceptedWhileAnotherCallerOwnsStartup(t *testing.T) { + api := newTestAPI(t) + const operationID = "operation-import-start-race" + if !api.handler.claimAsync(operationID) { + t.Fatal("could not reserve async startup claim") + } + defer api.handler.releaseAsync(operationID) + body := []byte(`{"operation_id":"` + operationID + `","id":"debian-12","name":"Debian 12","source_url":"https://images.example.com/debian.qcow2","os_family":"debian","version":"12","architecture":"x86_64","revision":1,"generation_token":"0123456789abcdef0123456789abcdef"}`) + response := api.request(t, http.MethodPost, "/v1/templates", body, auth.Actor{ + UserID: 1, CanManage: true, IsAdmin: true, OperationID: operationID, + }) + if response.Code != http.StatusServiceUnavailable || + !strings.Contains(response.Body.String(), `"code":"operation_starting"`) || + strings.Contains(response.Body.String(), `"status":"dispatching"`) { + t.Fatalf("startup loser response = %d %s", response.Code, response.Body.String()) + } + record, err := api.journal.Get(operationID) + if err != nil || record.State != operation.StateReserved { + t.Fatalf("startup loser journal = %#v err=%v", record, err) + } +} + func TestImportSourceTokenRejectsShortOrNonURLSafeValues(t *testing.T) { for _, token := range []string{"too-short", strings.Repeat("a", 31), strings.Repeat("a", 32) + "+"} { if err := validateImportSource("https://images.example.com/image.iso", token, ""); err == nil { diff --git a/agent/cmd/virt-agent/main.go b/agent/cmd/virt-agent/main.go index 8833f5f5..07ed2423 100644 --- a/agent/cmd/virt-agent/main.go +++ b/agent/cmd/virt-agent/main.go @@ -2,10 +2,8 @@ package main import ( "context" - "errors" "flag" "log" - "net/http" "os" "os/signal" "syscall" @@ -15,6 +13,7 @@ import ( "git.luoli.live/root/virt/agent/auth" "git.luoli.live/root/virt/agent/config" consolebroker "git.luoli.live/root/virt/agent/console" + "git.luoli.live/root/virt/agent/control" "git.luoli.live/root/virt/agent/hypervisor" "git.luoli.live/root/virt/agent/operation" ) @@ -31,8 +30,7 @@ func main() { if err != nil { log.Fatalf("open durable operation journal: %v", err) } - defer journal.Close() - authenticator, err := auth.NewWithNonceStore([]byte(cfg.SharedSecret), cfg.ClockSkew(), cfg.NonceTTL(), journal) + authenticator, err := auth.NewWithNonceStore([]byte(cfg.AgentSecret), cfg.ClockSkew(), cfg.NonceTTL(), journal) if err != nil { log.Fatalf("initialize request authentication: %v", err) } @@ -50,31 +48,41 @@ func main() { if err != nil { log.Fatalf("connect to libvirt: %v", err) } - defer hv.Close() console, err := consolebroker.NewBroker(60 * time.Second) if err != nil { log.Fatalf("initialize console broker: %v", err) } handler, err := api.NewServer(authenticator, journal, hv, console, cfg.MaxBodyBytes, cfg.OperationTimeout()) if err != nil { - log.Fatalf("initialize HTTP API: %v", err) + log.Fatalf("initialize Agent request dispatcher: %v", err) } - httpServer := api.BoundedHTTPServer(cfg.ListenAddress, handler) - - shutdownSignals := make(chan os.Signal, 1) - signal.Notify(shutdownSignals, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-shutdownSignals - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - if err := httpServer.Shutdown(ctx); err != nil { - log.Printf("HTTP shutdown: %v", err) - } - }() - - log.Printf("virt host agent %s listening with TLS on %s", api.Version, cfg.ListenAddress) - err = httpServer.ListenAndServeTLS(cfg.TLSCertFile, cfg.TLSKeyFile) - if err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Fatalf("serve agent API: %v", err) + client, err := control.NewClient( + cfg.PanelURL, cfg.AgentID, []byte(cfg.AgentSecret), handler, console, cfg.MaxBodyBytes, cfg.OperationTimeout(), + ) + if err != nil { + log.Fatalf("initialize outbound control channel: %v", err) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + log.Printf("virt host agent %s connecting to panel as Agent %s", api.Version, cfg.AgentID) + runErr := client.Run(ctx) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := handler.Shutdown(shutdownCtx); err != nil { + log.Printf("Agent handlers did not drain safely: %v", err) + os.Exit(1) + } + if err := journal.Wait(shutdownCtx); err != nil { + log.Printf("Agent operation workers did not drain safely: %v", err) + os.Exit(1) + } + if err := hv.Close(); err != nil { + log.Printf("close libvirt connection: %v", err) + } + if err := journal.Close(); err != nil { + log.Printf("close operation journal: %v", err) + } + if runErr != nil { + log.Fatalf("run outbound control channel: %v", runErr) } } diff --git a/agent/config/config.go b/agent/config/config.go index b5c064a9..96581817 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -6,10 +6,11 @@ import ( "errors" "fmt" "io" - "net" + "net/url" "os" "path/filepath" "regexp" + "strconv" "strings" "time" ) @@ -20,12 +21,11 @@ const maxConfigBytes = 1 << 20 // deliberately not part of validation: a readable configuration is accepted // regardless of how it was provisioned. type Config struct { - ListenAddress string `json:"listen_address"` - SharedSecret string `json:"shared_secret"` + PanelURL string `json:"panel_url"` + AgentID string `json:"agent_id"` + AgentSecret string `json:"agent_secret"` JournalPath string `json:"journal_path"` LibvirtURI string `json:"libvirt_uri,omitempty"` - TLSCertFile string `json:"tls_cert_file"` - TLSKeyFile string `json:"tls_key_file"` MaxBodyBytes int64 `json:"max_body_bytes,omitempty"` ClockSkewSeconds int64 `json:"clock_skew_seconds,omitempty"` NonceTTLSeconds int64 `json:"nonce_ttl_seconds,omitempty"` @@ -45,6 +45,7 @@ type NetworkSource struct { } var localNamePattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,128}$`) +var agentIDPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`) func (c *Config) applyDefaults() { if c.LibvirtURI == "" { @@ -66,13 +67,15 @@ func (c *Config) applyDefaults() { func (c Config) Validate() error { var problems []string - if strings.TrimSpace(c.ListenAddress) == "" { - problems = append(problems, "listen_address is required") - } else if _, _, err := net.SplitHostPort(c.ListenAddress); err != nil { - problems = append(problems, "listen_address must be host:port") + if err := validatePanelOrigin(c.PanelURL); err != nil { + problems = append(problems, err.Error()) } - if len(c.SharedSecret) < 32 { - problems = append(problems, "shared_secret must contain at least 32 bytes") + if !agentIDPattern.MatchString(c.AgentID) { + problems = append(problems, "agent_id must be a positive decimal host ID") + } + if len(c.AgentSecret) < 32 || len(c.AgentSecret) > 256 || + strings.IndexFunc(c.AgentSecret, func(value rune) bool { return value < 0x21 || value > 0x7e }) >= 0 { + problems = append(problems, "agent_secret must contain 32-256 non-space printable ASCII bytes") } if strings.TrimSpace(c.JournalPath) == "" { problems = append(problems, "journal_path is required") @@ -80,20 +83,8 @@ func (c Config) Validate() error { if c.LibvirtURI != "qemu:///system" { problems = append(problems, "libvirt_uri must be qemu:///system") } - if strings.TrimSpace(c.TLSCertFile) == "" || strings.TrimSpace(c.TLSKeyFile) == "" { - problems = append(problems, "tls_cert_file and tls_key_file are required") - } else { - for label, value := range map[string]string{ - "tls_cert_file": c.TLSCertFile, - "tls_key_file": c.TLSKeyFile, - } { - if !filepath.IsAbs(value) || filepath.Clean(value) != value || strings.ContainsRune(value, 0) { - problems = append(problems, label+" must be a clean absolute path") - } - } - } - if c.MaxBodyBytes < 1024 || c.MaxBodyBytes > 64<<20 { - problems = append(problems, "max_body_bytes must be between 1024 and 67108864") + if c.MaxBodyBytes < 1024 || c.MaxBodyBytes > 1<<20 { + problems = append(problems, "max_body_bytes must be between 1024 and 1048576") } if c.ClockSkewSeconds < 30 || c.ClockSkewSeconds > 3600 { problems = append(problems, "clock_skew_seconds must be between 30 and 3600") @@ -140,6 +131,25 @@ func (c Config) Validate() error { return nil } +func validatePanelOrigin(value string) error { + if value == "" || len(value) > 2048 || strings.TrimSpace(value) != value || strings.ContainsAny(value, "\x00\r\n\\") { + return errors.New("panel_url must be an HTTPS origin") + } + parsed, err := url.Parse(value) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.Hostname() == "" || + parsed.User != nil || parsed.Opaque != "" || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" || + parsed.RawPath != "" || (parsed.Path != "" && parsed.Path != "/") || strings.HasSuffix(parsed.Host, ":") { + return errors.New("panel_url must be an HTTPS origin without credentials, path, query, or fragment") + } + if parsed.Port() != "" { + port, err := strconv.Atoi(parsed.Port()) + if err != nil || port < 1 || port > 65535 { + return errors.New("panel_url contains an invalid port") + } + } + return nil +} + func (c Config) ClockSkew() time.Duration { return time.Duration(c.ClockSkewSeconds) * time.Second } diff --git a/agent/config/config_test.go b/agent/config/config_test.go index a4f056f2..659493a5 100644 --- a/agent/config/config_test.go +++ b/agent/config/config_test.go @@ -11,9 +11,8 @@ import ( const testSecret = "0123456789abcdef0123456789abcdef" func validConfigJSON(extra string) string { - return `{"listen_address":"127.0.0.1:9443","shared_secret":"` + testSecret + + return `{"panel_url":"https://panel.example.com","agent_id":"42","agent_secret":"` + testSecret + `","journal_path":"/tmp/virt-agent-operations.db",` + - `"tls_cert_file":"/etc/virt-agent/tls/agent.crt","tls_key_file":"/etc/virt-agent/tls/agent.key",` + `"managed_storage_pool":"virt-vms","template_storage_pool":"virt-templates","seed_storage_pool":"virt-seeds",` + `"firmware_loader":"/usr/share/OVMF/OVMF_CODE.fd","firmware_nvram_template":"/usr/share/OVMF/OVMF_VARS.fd",` + `"nvram_directory":"/var/lib/libvirt/qemu/nvram",` + @@ -45,14 +44,14 @@ func TestLoadAcceptsReadableConfigRegardlessOfMode(t *testing.T) { func TestLoadValidatesContentNotPermissions(t *testing.T) { path := filepath.Join(t.TempDir(), "agent.json") - if err := os.WriteFile(path, []byte(`{"listen_address":"bad","shared_secret":"short","journal_path":""}`), 0o600); err != nil { + if err := os.WriteFile(path, []byte(`{"panel_url":"http://panel.example.com/path","agent_id":"bad id","agent_secret":"short","journal_path":""}`), 0o600); err != nil { t.Fatal(err) } _, err := Load(path) if err == nil { t.Fatal("invalid content was accepted") } - for _, want := range []string{"listen_address", "shared_secret", "journal_path"} { + for _, want := range []string{"panel_url", "agent_id", "agent_secret", "journal_path"} { if !strings.Contains(err.Error(), want) { t.Fatalf("error %q does not mention %q", err, want) } @@ -61,8 +60,10 @@ func TestLoadValidatesContentNotPermissions(t *testing.T) { func TestLoadRejectsUnknownAndTrailingJSON(t *testing.T) { for name, body := range map[string]string{ - "unknown": validConfigJSON(`,"permission_mode":"0600"`), - "trailing": validConfigJSON("") + ` {}`, + "unknown": validConfigJSON(`,"permission_mode":"0600"`), + "retired inbound listener": validConfigJSON(`,"listen_address":"0.0.0.0:9443"`), + "retired TLS key": validConfigJSON(`,"tls_key_file":"/etc/virt-agent/agent.key"`), + "trailing": validConfigJSON("") + ` {}`, } { t.Run(name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "agent.json") @@ -76,29 +77,51 @@ func TestLoadRejectsUnknownAndTrailingJSON(t *testing.T) { } } -func TestLoadRequiresTLSCertificateAndKey(t *testing.T) { +func TestLoadRequiresHTTPSPanelOriginAndAgentIdentity(t *testing.T) { for name, body := range map[string]string{ - "both missing": strings.ReplaceAll(validConfigJSON(""), - `"tls_cert_file":"/etc/virt-agent/tls/agent.crt","tls_key_file":"/etc/virt-agent/tls/agent.key",`, ""), - "certificate missing": strings.ReplaceAll(validConfigJSON(""), - `"tls_cert_file":"/etc/virt-agent/tls/agent.crt",`, ""), - "key missing": strings.ReplaceAll(validConfigJSON(""), - `"tls_key_file":"/etc/virt-agent/tls/agent.key",`, ""), - "relative certificate": strings.ReplaceAll(validConfigJSON(""), - `/etc/virt-agent/tls/agent.crt`, `agent.crt`), + "plain HTTP": strings.ReplaceAll(validConfigJSON(""), + `https://panel.example.com`, `http://panel.example.com`), + "path": strings.ReplaceAll(validConfigJSON(""), + `https://panel.example.com`, `https://panel.example.com/cloud`), + "credentials": strings.ReplaceAll(validConfigJSON(""), + `https://panel.example.com`, `https://user@panel.example.com`), + "zero port": strings.ReplaceAll(validConfigJSON(""), + `https://panel.example.com`, `https://panel.example.com:0`), + "empty port": strings.ReplaceAll(validConfigJSON(""), + `https://panel.example.com`, `https://panel.example.com:`), + "oversized port": strings.ReplaceAll(validConfigJSON(""), + `https://panel.example.com`, `https://panel.example.com:99999`), + "non-decimal ID": strings.ReplaceAll(validConfigJSON(""), `"agent_id":"42"`, `"agent_id":"host-hk-01"`), + "zero ID": strings.ReplaceAll(validConfigJSON(""), `"agent_id":"42"`, `"agent_id":"0"`), + "leading zero ID": strings.ReplaceAll(validConfigJSON(""), `"agent_id":"42"`, `"agent_id":"042"`), + "20 digit ID": strings.ReplaceAll(validConfigJSON(""), `"agent_id":"42"`, + `"agent_id":"12345678901234567890"`), + "short secret": strings.ReplaceAll(validConfigJSON(""), testSecret, `short`), + "oversized body": validConfigJSON(`,"max_body_bytes":1048577`), } { t.Run(name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "agent.json") if err := os.WriteFile(path, []byte(body), 0o600); err != nil { t.Fatal(err) } - if _, err := Load(path); err == nil || !strings.Contains(err.Error(), "tls_") { - t.Fatalf("invalid TLS configuration error = %v", err) + if _, err := Load(path); err == nil { + t.Fatal("invalid outbound Agent configuration was accepted") } }) } } +func TestLoadAcceptsMaximumLengthDecimalAgentID(t *testing.T) { + path := filepath.Join(t.TempDir(), "agent.json") + body := strings.ReplaceAll(validConfigJSON(""), `"agent_id":"42"`, `"agent_id":"1234567890123456789"`) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err != nil { + t.Fatalf("valid 19 digit Agent ID was rejected: %v", err) + } +} + func TestLoadRequiresNonceTTLToCoverFullClockSkewWindow(t *testing.T) { path := filepath.Join(t.TempDir(), "agent.json") if err := os.WriteFile(path, []byte(validConfigJSON(`,"clock_skew_seconds":300,"nonce_ttl_seconds":599`)), 0o600); err != nil { diff --git a/agent/console/broker.go b/agent/console/broker.go index 1bf5bd0d..b6753994 100644 --- a/agent/console/broker.go +++ b/agent/console/broker.go @@ -8,8 +8,6 @@ import ( "fmt" "io" "net" - "net/http" - "strings" "sync" "time" @@ -17,14 +15,17 @@ import ( ) const ( - SubprotocolPrefix = "cloud-agent-ticket." defaultMaxFrame = 8 << 20 + maxPendingTickets = 256 +) + +var ( + ErrInvalidTicket = errors.New("console ticket is invalid, expired, or already used") + ErrTicketCapacity = errors.New("console ticket capacity is exhausted") ) type Ticket struct { Ticket string `json:"ticket"` - URL string `json:"url"` - Protocol string `json:"protocol"` ExpiresAt time.Time `json:"expires_at"` } @@ -71,50 +72,33 @@ func (b *Broker) Issue(address string) (Ticket, error) { defer b.mu.Unlock() now := b.now().UTC() b.purgeLocked(now) + if len(b.tickets) >= maxPendingTickets { + return Ticket{}, ErrTicketCapacity + } expires := now.Add(b.ttl) b.tickets[value] = entry{address: address, expiresAt: expires} return Ticket{ - Ticket: value, URL: "/v1/console", Protocol: SubprotocolPrefix + value, - ExpiresAt: expires, + Ticket: value, ExpiresAt: expires, }, nil } -func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "websocket upgrade requires GET", http.StatusMethodNotAllowed) - return - } - protocol, ticket, ok := requestedTicket(r.Header.Values("Sec-WebSocket-Protocol")) - if !ok { - http.Error(w, "valid console ticket subprotocol required", http.StatusUnauthorized) - return - } +// Open atomically consumes one short-lived ticket before dialing the local +// loopback VNC target. A failed dial never makes the ticket reusable. +func (b *Broker) Open(ctx context.Context, ticket string) (net.Conn, error) { address, ok := b.consume(ticket) if !ok { - http.Error(w, "console ticket is invalid, expired, or already used", http.StatusUnauthorized) - return + return nil, ErrInvalidTicket } - - backend, err := b.dialer.DialContext(r.Context(), "tcp", address) + backend, err := b.dialer.DialContext(ctx, "tcp", address) if err != nil { - http.Error(w, "local console is unavailable", http.StatusBadGateway) - return + return nil, fmt.Errorf("dial local console: %w", err) } - defer backend.Close() - - conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{ - Subprotocols: []string{protocol}, - CompressionMode: websocket.CompressionDisabled, - }) - if err != nil { - return - } - defer conn.Close(websocket.StatusNormalClosure, "") - conn.SetReadLimit(b.maxFrame) - b.relay(r.Context(), conn, backend) + return backend, nil } -func (b *Broker) relay(parent context.Context, conn *websocket.Conn, backend net.Conn) { +// Relay copies only binary WebSocket messages between the authenticated +// reverse console channel and the local VNC TCP socket. +func (b *Broker) Relay(parent context.Context, conn *websocket.Conn, backend net.Conn) { ctx, cancel := context.WithCancel(parent) defer cancel() errorsChannel := make(chan error, 2) @@ -182,26 +166,6 @@ func (b *Broker) purgeLocked(now time.Time) { } } -func requestedTicket(values []string) (string, string, bool) { - var protocols []string - for _, value := range values { - for _, part := range strings.Split(value, ",") { - if protocol := strings.TrimSpace(part); protocol != "" { - protocols = append(protocols, protocol) - } - } - } - if len(protocols) != 1 || !strings.HasPrefix(protocols[0], SubprotocolPrefix) { - return "", "", false - } - ticket := strings.TrimPrefix(protocols[0], SubprotocolPrefix) - decoded, err := base64.RawURLEncoding.DecodeString(ticket) - if err != nil || len(decoded) != 32 { - return "", "", false - } - return protocols[0], ticket, true -} - func validateLoopbackTarget(address string) error { host, port, err := net.SplitHostPort(address) if err != nil { diff --git a/agent/console/broker_test.go b/agent/console/broker_test.go index 8e3452eb..c1a6e97a 100644 --- a/agent/console/broker_test.go +++ b/agent/console/broker_test.go @@ -4,12 +4,8 @@ import ( "context" "errors" "net" - "net/http/httptest" - "strings" "testing" "time" - - "github.com/coder/websocket" ) func TestTicketIsOneTimeAndExpires(t *testing.T) { @@ -23,7 +19,7 @@ func TestTicketIsOneTimeAndExpires(t *testing.T) { if err != nil { t.Fatal(err) } - if !strings.HasPrefix(ticket.Protocol, SubprotocolPrefix) || ticket.Ticket == "" { + if ticket.Ticket == "" { t.Fatalf("bad ticket: %#v", ticket) } address, ok := b.consume(ticket.Ticket) @@ -50,7 +46,28 @@ func TestIssueRejectsNonLoopbackTargets(t *testing.T) { } } -func TestWebSocketRelaysBinaryAndRejectsTicketReplay(t *testing.T) { +func TestIssueBoundsPendingTicketsAndRecoversAfterExpiry(t *testing.T) { + b, err := NewBroker(time.Minute) + if err != nil { + t.Fatal(err) + } + now := time.Unix(2_000_000_000, 0) + b.SetClockForTesting(func() time.Time { return now }) + for index := 0; index < maxPendingTickets; index++ { + if _, err := b.Issue("127.0.0.1:5900"); err != nil { + t.Fatalf("issue ticket %d: %v", index, err) + } + } + if _, err := b.Issue("127.0.0.1:5900"); !errors.Is(err, ErrTicketCapacity) { + t.Fatalf("capacity error = %v", err) + } + now = now.Add(61 * time.Second) + if _, err := b.Issue("127.0.0.1:5900"); err != nil { + t.Fatalf("expired tickets did not release capacity: %v", err) + } +} + +func TestOpenDialsLoopbackAndRejectsTicketReplay(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { var operationError *net.OpError @@ -78,33 +95,40 @@ func TestWebSocketRelaysBinaryAndRejectsTicketReplay(t *testing.T) { if err != nil { t.Fatal(err) } - server := httptest.NewServer(b) - defer server.Close() - url := "ws" + strings.TrimPrefix(server.URL, "http") ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - conn, response, err := websocket.Dial(ctx, url, &websocket.DialOptions{Subprotocols: []string{ticket.Protocol}}) + conn, err := b.Open(ctx, ticket.Ticket) if err != nil { - t.Fatalf("dial relay: %v (response %#v)", err, response) + t.Fatalf("open console: %v", err) } - if conn.Subprotocol() != ticket.Protocol { - t.Fatalf("selected protocol = %q", conn.Subprotocol()) - } - if err := conn.Write(ctx, websocket.MessageBinary, []byte("RFB-test")); err != nil { + defer conn.Close() + if _, err := conn.Write([]byte("RFB-test")); err != nil { t.Fatal(err) } - messageType, payload, err := conn.Read(ctx) - if err != nil || messageType != websocket.MessageBinary || string(payload) != "RFB-test" { - t.Fatalf("relay response type=%v payload=%q err=%v", messageType, payload, err) + payload := make([]byte, len("RFB-test")) + if _, err := conn.Read(payload); err != nil || string(payload) != "RFB-test" { + t.Fatalf("console response payload=%q err=%v", payload, err) + } + if replay, err := b.Open(ctx, ticket.Ticket); !errors.Is(err, ErrInvalidTicket) { + if replay != nil { + _ = replay.Close() + } + t.Fatalf("ticket replay error = %v", err) } - _ = conn.Close(websocket.StatusNormalClosure, "") - conn, response, err = websocket.Dial(ctx, url, &websocket.DialOptions{Subprotocols: []string{ticket.Protocol}}) - if err == nil { - _ = conn.CloseNow() - t.Fatal("one-time ticket was accepted twice") + closedTicket, err := b.Issue(listener.Addr().String()) + if err != nil { + t.Fatal(err) } - if response == nil || response.StatusCode != 401 { - t.Fatalf("ticket replay status = %#v, err=%v", response, err) + _ = listener.Close() + if failed, err := b.Open(ctx, closedTicket.Ticket); err == nil { + _ = failed.Close() + t.Fatal("closed local console unexpectedly connected") + } + if replay, err := b.Open(ctx, closedTicket.Ticket); !errors.Is(err, ErrInvalidTicket) { + if replay != nil { + _ = replay.Close() + } + t.Fatalf("failed dial made ticket reusable: %v", err) } } diff --git a/agent/control/client.go b/agent/control/client.go new file mode 100644 index 00000000..4d2e5bf8 --- /dev/null +++ b/agent/control/client.go @@ -0,0 +1,556 @@ +// Package control implements the Agent-initiated control and reverse-console +// channels. The panel still signs every inner API request; this transport only +// replaces the retired public HTTPS listener. +package control + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/coder/websocket" + + "git.luoli.live/root/virt/agent/auth" + consolebroker "git.luoli.live/root/virt/agent/console" +) + +const ( + ControlSubprotocol = "virt-agent-control.v1" + ConsoleProtocol = "virt-agent-console." + controlPath = "/agent/connect" + consolePath = "/agent/console" + maxControlFrame = 4 << 20 + maxResponseBytes = 2 << 20 + maxRequestBody = 1 << 20 + maxInFlight = 32 + maxActiveConsoles = 16 + reconnectDelay = 5 * time.Second +) + +var ( + transportIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,128}$`) + agentIDPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`) + opaqueTokenPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,512}$`) +) + +type message struct { + Type string `json:"type"` + ID string `json:"id"` + Method string `json:"method,omitempty"` + RequestTarget string `json:"request_target,omitempty"` + Body string `json:"body,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Ticket string `json:"ticket,omitempty"` + SessionID string `json:"session_id,omitempty"` + SessionToken string `json:"session_token,omitempty"` +} + +type response struct { + Type string `json:"type"` + ID string `json:"id"` + Status int `json:"status"` + Body string `json:"body"` +} + +type Client struct { + panelOrigin *url.URL + agentID string + agentSecret string + handler http.Handler + console *consolebroker.Broker + maxBodyBytes int64 + requestTimeout time.Duration + inFlight chan struct{} + activeConsoles chan struct{} + httpClient *http.Client +} + +func NewClient( + panelURL string, + agentID string, + agentSecret []byte, + handler http.Handler, + console *consolebroker.Broker, + maxBodyBytes int64, + requestTimeout time.Duration, +) (*Client, error) { + origin, err := parsePanelOrigin(panelURL) + if err != nil { + return nil, err + } + if !agentIDPattern.MatchString(agentID) { + return nil, errors.New("agent ID must be a positive decimal host ID") + } + if len(agentSecret) < 32 || len(agentSecret) > 256 || + strings.IndexFunc(string(agentSecret), func(value rune) bool { return value < 0x21 || value > 0x7e }) >= 0 { + return nil, errors.New("agent secret must contain 32-256 non-space printable ASCII bytes") + } + if handler == nil || console == nil { + return nil, errors.New("API handler and console broker are required") + } + if maxBodyBytes < 1024 || maxBodyBytes > maxRequestBody { + return nil, errors.New("maximum body size is invalid") + } + if requestTimeout < 5*time.Second || requestTimeout > time.Hour { + return nil, errors.New("request timeout must be between 5 seconds and 1 hour") + } + dialer := &net.Dialer{Timeout: 15 * time.Second, KeepAlive: 30 * time.Second} + transport := &http.Transport{ + Proxy: nil, + DialContext: dialer.DialContext, + ForceAttemptHTTP2: false, + TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, + TLSHandshakeTimeout: 15 * time.Second, + ResponseHeaderTimeout: 15 * time.Second, + } + return &Client{ + panelOrigin: origin, + agentID: agentID, + agentSecret: string(append([]byte(nil), agentSecret...)), + handler: handler, + console: console, + maxBodyBytes: maxBodyBytes, + requestTimeout: requestTimeout, + inFlight: make(chan struct{}, maxInFlight), + activeConsoles: make(chan struct{}, maxActiveConsoles), + httpClient: &http.Client{ + Transport: transport, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +// Run maintains one outbound control session. A fixed retry is intentional in +// the first protocol version; durable operation safety does not depend on the +// transport reconnect policy. +func (c *Client) Run(ctx context.Context) error { + for { + err := c.runSession(ctx) + if ctx.Err() != nil { + return nil + } + log.Printf("Agent control connection closed: %v; retrying in %s", err, reconnectDelay) + timer := time.NewTimer(reconnectDelay) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return nil + case <-timer.C: + } + } +} + +func (c *Client) runSession(parent context.Context) error { + ctx, cancel := context.WithCancel(parent) + defer cancel() + connection, response, err := websocket.Dial(ctx, c.websocketURL(controlPath), &websocket.DialOptions{ + HTTPClient: c.httpClient, + HTTPHeader: c.controlHeaders(), + Subprotocols: []string{ControlSubprotocol}, + CompressionMode: websocket.CompressionDisabled, + }) + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + if err != nil { + return fmt.Errorf("dial panel control channel: %w", err) + } + defer connection.CloseNow() + if connection.Subprotocol() != ControlSubprotocol { + _ = connection.Close(websocket.StatusPolicyViolation, "control subprotocol required") + return errors.New("panel selected an invalid control subprotocol") + } + connection.SetReadLimit(maxControlFrame) + writer := &connectionWriter{connection: connection} + for { + messageType, payload, err := connection.Read(ctx) + if err != nil { + return fmt.Errorf("read panel control message: %w", err) + } + if messageType != websocket.MessageText { + _ = connection.Close(websocket.StatusUnsupportedData, "text control messages required") + return errors.New("panel sent a non-text control message") + } + input, err := decodeMessage(payload) + if err != nil { + _ = connection.Close(websocket.StatusInvalidFramePayloadData, "invalid control message") + return err + } + switch input.Type { + case "request": + c.dispatch(ctx, writer, input, c.handleRequest) + case "console_open": + c.dispatch(ctx, writer, input, c.handleConsoleOpen) + default: + _ = connection.Close(websocket.StatusPolicyViolation, "unsupported control message") + return errors.New("panel sent an unsupported control message") + } + } +} + +func (c *Client) dispatch( + ctx context.Context, + writer *connectionWriter, + input message, + handle func(context.Context, context.Context, *connectionWriter, message), +) { + select { + case c.inFlight <- struct{}{}: + go func() { + defer func() { <-c.inFlight }() + requestCtx, cancel := context.WithTimeout(ctx, c.requestTimeout) + defer cancel() + handle(ctx, requestCtx, writer, input) + }() + default: + _ = writer.respond(ctx, notDispatchedErrorResponse(input.ID, + http.StatusServiceUnavailable, "agent_busy")) + } +} + +func (c *Client) handleRequest(sessionCtx, requestCtx context.Context, writer *connectionWriter, input message) { + if err := c.validateRequest(input); err != nil { + _ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusBadRequest, "invalid_control_request")) + return + } + requestURL, _ := url.ParseRequestURI(input.RequestTarget) + request := &http.Request{ + Method: input.Method, + URL: requestURL, + RequestURI: input.RequestTarget, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(input.Body)), + ContentLength: int64(len(input.Body)), + Host: "agent.internal", + } + request = request.WithContext(requestCtx) + for name, value := range input.Headers { + request.Header.Set(http.CanonicalHeaderKey(name), value) + } + type handlerResult struct { + status int + body string + } + done := make(chan handlerResult, 1) + go func() { + capture := newResponseCapture(maxResponseBytes) + defer func() { + if recover() != nil { + capture = newResponseCapture(maxResponseBytes) + capture.WriteHeader(http.StatusInternalServerError) + _, _ = capture.Write([]byte(`{"error":{"code":"agent_internal_error","message":"Agent request failed","retryable":true}}`)) + } + status, body := capture.result() + done <- handlerResult{status: status, body: body} + }() + c.handler.ServeHTTP(capture, request) + }() + select { + case result := <-done: + if errors.Is(requestCtx.Err(), context.DeadlineExceeded) { + _ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusGatewayTimeout, "agent_request_timeout")) + } else { + _ = writer.respond(sessionCtx, response{Type: "response", ID: input.ID, Status: result.status, Body: result.body}) + } + case <-requestCtx.Done(): + if errors.Is(requestCtx.Err(), context.DeadlineExceeded) { + _ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusGatewayTimeout, "agent_request_timeout")) + } + // Keep this admission slot occupied until an uncooperative handler + // actually returns. This bounds leaked libvirt calls across reconnects. + <-done + } +} + +func (c *Client) validateRequest(input message) error { + if !transportIDPattern.MatchString(input.ID) || input.Ticket != "" || input.SessionID != "" || input.SessionToken != "" { + return errors.New("invalid request envelope") + } + switch input.Method { + case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete: + default: + return errors.New("unsupported request method") + } + if len(input.RequestTarget) == 0 || len(input.RequestTarget) > 2048 || + strings.ContainsAny(input.RequestTarget, "\x00\r\n\\# ") || + (input.RequestTarget != "/v1" && !strings.HasPrefix(input.RequestTarget, "/v1/")) { + return errors.New("invalid request target") + } + parsed, err := url.ParseRequestURI(input.RequestTarget) + if err != nil || parsed.IsAbs() || parsed.Host != "" { + return errors.New("invalid request target") + } + probe := &http.Request{URL: parsed} + if auth.RequestTarget(probe) != input.RequestTarget { + return errors.New("request target is not canonical") + } + if int64(len(input.Body)) > c.maxBodyBytes { + return errors.New("request body exceeds configured limit") + } + required := map[string]bool{ + http.CanonicalHeaderKey(auth.HeaderTimestamp): false, + http.CanonicalHeaderKey(auth.HeaderNonce): false, + http.CanonicalHeaderKey(auth.HeaderBodyDigest): false, + http.CanonicalHeaderKey(auth.HeaderActor): false, + http.CanonicalHeaderKey(auth.HeaderSignature): false, + } + for name, value := range input.Headers { + canonical := http.CanonicalHeaderKey(name) + if _, ok := required[canonical]; ok { + if value == "" || strings.ContainsAny(value, "\r\n") || required[canonical] { + return errors.New("invalid security header") + } + required[canonical] = true + continue + } + if (canonical == "Accept" || canonical == "Content-Type") && value == "application/json" { + continue + } + return errors.New("unsupported request header") + } + for _, present := range required { + if !present { + return errors.New("missing security header") + } + } + return nil +} + +func (c *Client) handleConsoleOpen(sessionCtx, requestCtx context.Context, writer *connectionWriter, input message) { + if !transportIDPattern.MatchString(input.ID) || input.Method != "" || input.RequestTarget != "" || + input.Body != "" || len(input.Headers) != 0 || !validTicket(input.Ticket) || + !opaqueTokenPattern.MatchString(input.SessionID) || !opaqueTokenPattern.MatchString(input.SessionToken) { + _ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusBadRequest, "invalid_console_request")) + return + } + if !c.acquireConsole() { + _ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusServiceUnavailable, "console_capacity_exhausted")) + return + } + releaseConsole := true + defer func() { + if releaseConsole { + c.releaseConsole() + } + }() + backend, err := c.console.Open(requestCtx, input.Ticket) + if err != nil { + status := http.StatusBadGateway + code := "console_unavailable" + if errors.Is(err, consolebroker.ErrInvalidTicket) { + status = http.StatusUnauthorized + code = "invalid_console_ticket" + } + _ = writer.respond(sessionCtx, errorResponse(input.ID, status, code)) + return + } + consoleHeaders := c.consoleHeaders(input.SessionID) + protocol := ConsoleProtocol + input.SessionToken + connection, handshake, err := websocket.Dial(requestCtx, c.websocketURL(consolePath), &websocket.DialOptions{ + HTTPClient: c.httpClient, + HTTPHeader: consoleHeaders, + Subprotocols: []string{protocol}, + CompressionMode: websocket.CompressionDisabled, + }) + if handshake != nil && handshake.Body != nil { + _ = handshake.Body.Close() + } + if err != nil || connection == nil || connection.Subprotocol() != protocol { + _ = backend.Close() + if connection != nil { + _ = connection.CloseNow() + } + _ = writer.respond(sessionCtx, errorResponse(input.ID, http.StatusBadGateway, "console_gateway_unavailable")) + return + } + connection.SetReadLimit(8 << 20) + body, _ := json.Marshal(map[string]string{"session_id": input.SessionID, "status": "connected"}) + if err := writer.respond(sessionCtx, response{Type: "response", ID: input.ID, Status: http.StatusOK, Body: string(body)}); err != nil { + _ = backend.Close() + _ = connection.CloseNow() + return + } + releaseConsole = false + go func() { + defer c.releaseConsole() + defer backend.Close() + defer connection.CloseNow() + c.console.Relay(sessionCtx, connection, backend) + }() +} + +func (c *Client) acquireConsole() bool { + select { + case c.activeConsoles <- struct{}{}: + return true + default: + return false + } +} + +func (c *Client) releaseConsole() { + <-c.activeConsoles +} + +func (c *Client) controlHeaders() http.Header { + headers := make(http.Header) + headers.Set("X-Virt-Agent-ID", c.agentID) + headers.Set("Authorization", "Bearer "+c.agentSecret) + return headers +} + +func (c *Client) consoleHeaders(sessionID string) http.Header { + headers := make(http.Header) + headers.Set("X-Virt-Agent-ID", c.agentID) + headers.Set("X-Virt-Session-ID", sessionID) + return headers +} + +func (c *Client) websocketURL(path string) string { + value := *c.panelOrigin + value.Scheme = "wss" + value.Path = path + return value.String() +} + +func parsePanelOrigin(value string) (*url.URL, error) { + parsed, err := url.Parse(value) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.Hostname() == "" || + parsed.User != nil || parsed.Opaque != "" || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" || + parsed.RawPath != "" || (parsed.Path != "" && parsed.Path != "/") || strings.HasSuffix(parsed.Host, ":") { + return nil, errors.New("panel URL must be an HTTPS origin") + } + if parsed.Port() != "" { + port, portErr := strconv.Atoi(parsed.Port()) + if portErr != nil || port < 1 || port > 65535 { + return nil, errors.New("panel URL contains an invalid port") + } + } + parsed.Path = "" + return parsed, nil +} + +func decodeMessage(payload []byte) (message, error) { + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + var input message + if err := decoder.Decode(&input); err != nil { + return message{}, errors.New("decode control message") + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return message{}, errors.New("control message contains trailing data") + } + return input, nil +} + +func validTicket(value string) bool { + decoded, err := base64.RawURLEncoding.DecodeString(value) + return err == nil && len(decoded) == 32 +} + +func errorResponse(id string, status int, code string) response { + body, _ := json.Marshal(map[string]any{ + "error": map[string]any{"code": code, "message": "Agent control request failed", "retryable": status >= 500}, + }) + return response{Type: "response", ID: id, Status: status, Body: string(body)} +} + +// notDispatchedErrorResponse is reserved for failures that occur before the +// signed request reaches the API handler (and therefore before the durable +// operation journal). The panel may safely put the same operation back in +// its persistent queue only when this exact marker is present. +func notDispatchedErrorResponse(id string, status int, code string) response { + body, _ := json.Marshal(map[string]any{ + "error": map[string]any{ + "code": code, "message": "Agent control request was not dispatched", + "retryable": true, "dispatch_state": "not_dispatched", + }, + }) + return response{Type: "response", ID: id, Status: status, Body: string(body)} +} + +type connectionWriter struct { + connection *websocket.Conn + mu sync.Mutex +} + +func (w *connectionWriter) respond(ctx context.Context, value response) error { + payload, err := json.Marshal(value) + if err != nil { + return err + } + if len(payload) > maxControlFrame { + value = errorResponse(value.ID, http.StatusBadGateway, "agent_response_too_large") + payload, err = json.Marshal(value) + if err != nil { + return err + } + } + w.mu.Lock() + defer w.mu.Unlock() + writeContext, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + return w.connection.Write(writeContext, websocket.MessageText, payload) +} + +type responseCapture struct { + header http.Header + status int + body bytes.Buffer + limit int64 + overflow bool +} + +func newResponseCapture(limit int64) *responseCapture { + return &responseCapture{header: make(http.Header), limit: limit} +} + +func (r *responseCapture) Header() http.Header { return r.header } + +func (r *responseCapture) WriteHeader(status int) { + if r.status == 0 { + r.status = status + } +} + +func (r *responseCapture) Write(value []byte) (int, error) { + if r.status == 0 { + r.status = http.StatusOK + } + remaining := r.limit - int64(r.body.Len()) + if remaining <= 0 || int64(len(value)) > remaining { + r.overflow = true + return 0, errors.New("Agent response exceeds transport limit") + } + return r.body.Write(value) +} + +func (r *responseCapture) result() (int, string) { + if r.overflow { + value := errorResponse("unused", http.StatusBadGateway, "agent_response_too_large") + return value.Status, value.Body + } + if r.status == 0 { + r.status = http.StatusOK + } + return r.status, r.body.String() +} diff --git a/agent/control/client_test.go b/agent/control/client_test.go new file mode 100644 index 00000000..aff89fb0 --- /dev/null +++ b/agent/control/client_test.go @@ -0,0 +1,490 @@ +package control + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/coder/websocket" + + "git.luoli.live/root/virt/agent/auth" + consolebroker "git.luoli.live/root/virt/agent/console" +) + +const controlTestSecret = "0123456789abcdef0123456789abcdef" + +func TestControlSessionAuthenticatesAndDispatchesInnerSignedRequest(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + authenticator, err := auth.New([]byte(controlTestSecret), time.Minute, 2*time.Minute) + if err != nil { + t.Fatal(err) + } + authenticator.SetClockForTesting(func() time.Time { return now }) + inner := http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + body, readErr := io.ReadAll(request.Body) + if readErr != nil { + t.Errorf("read reconstructed request: %v", readErr) + response.WriteHeader(http.StatusBadRequest) + return + } + actor, authErr := authenticator.Authenticate(request, body) + if authErr != nil || actor.UserID != 42 || request.RequestURI != "/v1/health?full=1" || string(body) != `{"probe":true}` { + t.Errorf("reconstructed request actor=%#v uri=%q body=%q err=%v", actor, request.RequestURI, body, authErr) + response.WriteHeader(http.StatusUnauthorized) + return + } + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"status":"ok"}`)) + }) + + result := make(chan response, 1) + server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != controlPath || request.Header.Get("X-Virt-Agent-ID") != "42" || + request.Header.Get("Authorization") != "Bearer "+controlTestSecret { + http.Error(writer, "bad Agent handshake", http.StatusUnauthorized) + return + } + connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{ + Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled, + }) + if acceptErr != nil { + return + } + defer connection.CloseNow() + body := []byte(`{"probe":true}`) + signed, _ := http.NewRequest(http.MethodPost, "https://agent.invalid/v1/health?full=1", bytes.NewReader(body)) + _ = auth.Sign(signed, body, auth.Actor{UserID: 42}, []byte(controlTestSecret), now.Unix(), "0123456789abcdef0123456789abcdef") + headers := make(map[string]string) + for _, name := range []string{auth.HeaderTimestamp, auth.HeaderNonce, auth.HeaderBodyDigest, auth.HeaderActor, auth.HeaderSignature} { + headers[name] = signed.Header.Get(name) + } + payload, _ := json.Marshal(message{ + Type: "request", ID: "request_12345678", Method: http.MethodPost, + RequestTarget: "/v1/health?full=1", Body: string(body), Headers: headers, + }) + if writeErr := connection.Write(request.Context(), websocket.MessageText, payload); writeErr != nil { + t.Errorf("send request: %v", writeErr) + return + } + messageType, raw, readErr := connection.Read(request.Context()) + if readErr != nil || messageType != websocket.MessageText { + t.Errorf("read response type=%v err=%v", messageType, readErr) + return + } + var got response + if err := json.Unmarshal(raw, &got); err != nil { + t.Errorf("decode response: %v", err) + return + } + result <- got + _ = connection.Close(websocket.StatusNormalClosure, "done") + })) + defer server.Close() + + broker, _ := consolebroker.NewBroker(time.Minute) + client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute) + if err != nil { + t.Fatal(err) + } + client.httpClient = server.Client() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- client.runSession(ctx) }() + select { + case got := <-result: + if got.Type != "response" || got.ID != "request_12345678" || got.Status != http.StatusCreated || got.Body != `{"status":"ok"}` { + t.Fatalf("control response = %#v", got) + } + case <-ctx.Done(): + t.Fatal("timed out waiting for control response") + } + select { + case <-done: + case <-ctx.Done(): + t.Fatal("control session did not close") + } +} + +func TestConsoleOpenCreatesReverseWSSAndRelaysBinary(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + var operationError *net.OpError + if errors.As(err, &operationError) { + t.Skipf("sandbox does not permit loopback listeners: %v", err) + } + t.Fatal(err) + } + defer listener.Close() + consoleRelease := make(chan struct{}) + var releaseOnce sync.Once + releaseConsole := func() { releaseOnce.Do(func() { close(consoleRelease) }) } + defer releaseConsole() + backendDone := make(chan struct{}) + go func() { + defer close(backendDone) + connection, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + defer connection.Close() + buffer := make([]byte, 128) + count, readErr := connection.Read(buffer) + if readErr == nil { + _, _ = connection.Write(buffer[:count]) + } + <-consoleRelease + }() + + broker, _ := consolebroker.NewBroker(time.Minute) + ticket, err := broker.Issue(listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + const sessionID = "session_0123456789abcdef01234567" + const sessionToken = "token_0123456789abcdef0123456789abcdef" + controlResult := make(chan response, 1) + consoleResult := make(chan string, 1) + relayDone := make(chan struct{}) + server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case controlPath: + connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{ + Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled, + }) + if acceptErr != nil { + return + } + defer connection.CloseNow() + payload, _ := json.Marshal(message{ + Type: "console_open", ID: "console_12345678", Ticket: ticket.Ticket, + SessionID: sessionID, SessionToken: sessionToken, + }) + if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil { + return + } + messageType, raw, err := connection.Read(request.Context()) + if err != nil || messageType != websocket.MessageText { + return + } + var got response + if json.Unmarshal(raw, &got) == nil { + controlResult <- got + } + select { + case <-relayDone: + case <-request.Context().Done(): + } + _ = connection.Close(websocket.StatusNormalClosure, "done") + case consolePath: + expectedProtocol := ConsoleProtocol + sessionToken + if request.Header.Get("X-Virt-Agent-ID") != "42" || + request.Header.Get("Authorization") != "" || + request.Header.Get("X-Virt-Session-ID") != sessionID { + http.Error(writer, "bad console handshake", http.StatusUnauthorized) + return + } + connection, acceptErr := websocket.Accept(writer, request, &websocket.AcceptOptions{ + Subprotocols: []string{expectedProtocol}, CompressionMode: websocket.CompressionDisabled, + }) + if acceptErr != nil { + return + } + defer connection.CloseNow() + if err := connection.Write(request.Context(), websocket.MessageBinary, []byte("RFB-test")); err != nil { + return + } + messageType, raw, err := connection.Read(request.Context()) + if err == nil && messageType == websocket.MessageBinary { + consoleResult <- string(raw) + } + <-consoleRelease + close(relayDone) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + + client, err := NewClient(server.URL, "42", []byte(controlTestSecret), http.NotFoundHandler(), broker, 64<<10, time.Minute) + if err != nil { + t.Fatal(err) + } + client.httpClient = server.Client() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- client.runSession(ctx) }() + select { + case got := <-controlResult: + if got.Status != http.StatusOK || got.ID != "console_12345678" || !strings.Contains(got.Body, `"status":"connected"`) { + t.Fatalf("console-open response = %#v", got) + } + case <-ctx.Done(): + t.Fatal("timed out waiting for console-open response") + } + select { + case got := <-consoleResult: + if got != "RFB-test" { + t.Fatalf("reverse console body = %q", got) + } + if len(client.activeConsoles) != 1 { + t.Fatalf("active console slot was not held for relay lifetime: %d", len(client.activeConsoles)) + } + releaseConsole() + case <-ctx.Done(): + t.Fatal("timed out waiting for reverse console relay") + } + select { + case <-backendDone: + case <-ctx.Done(): + t.Fatal("local console connection did not close") + } + select { + case <-done: + case <-ctx.Done(): + t.Fatal("control session did not close") + } + deadline := time.Now().Add(time.Second) + for len(client.activeConsoles) != 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := len(client.activeConsoles); got != 0 { + t.Fatalf("active console slot was not released: %d", got) + } +} + +func TestClientBoundsActiveConsoles(t *testing.T) { + client := &Client{activeConsoles: make(chan struct{}, maxActiveConsoles)} + for index := 0; index < maxActiveConsoles; index++ { + if !client.acquireConsole() { + t.Fatalf("console slot %d was rejected", index) + } + } + if client.acquireConsole() { + t.Fatal("console capacity exceeded configured limit") + } + client.releaseConsole() + if !client.acquireConsole() { + t.Fatal("released console capacity was not reusable") + } +} + +func TestParsePanelOriginRejectsInvalidPorts(t *testing.T) { + for _, value := range []string{ + "https://panel.example.com:", + "https://panel.example.com:0", + "https://panel.example.com:65536", + } { + if _, err := parsePanelOrigin(value); err == nil { + t.Fatalf("invalid panel origin %q was accepted", value) + } + } + if _, err := parsePanelOrigin("https://panel.example.com:443"); err != nil { + t.Fatalf("valid explicit TLS port was rejected: %v", err) + } +} + +func TestControlRequestBodyAndResponseLimitsAreExact(t *testing.T) { + client := &Client{maxBodyBytes: 1024} + headers := map[string]string{ + auth.HeaderTimestamp: "1", + auth.HeaderNonce: "0123456789abcdef0123456789abcdef", + auth.HeaderBodyDigest: "digest", + auth.HeaderActor: "actor", + auth.HeaderSignature: "signature", + } + request := message{ + Type: "request", ID: "request_boundary", Method: http.MethodPost, + RequestTarget: "/v1/health", Body: strings.Repeat("x", 1024), Headers: headers, + } + if err := client.validateRequest(request); err != nil { + t.Fatalf("exact maximum request body was rejected: %v", err) + } + request.Body += "x" + if err := client.validateRequest(request); err == nil { + t.Fatal("oversized request body was accepted") + } + + capture := newResponseCapture(8) + if count, err := capture.Write([]byte("12345678")); err != nil || count != 8 { + t.Fatalf("exact maximum response write count=%d err=%v", count, err) + } + status, body := capture.result() + if status != http.StatusOK || body != "12345678" { + t.Fatalf("exact maximum response = %d %q", status, body) + } + overflow := newResponseCapture(8) + if _, err := overflow.Write([]byte("123456789")); err == nil { + t.Fatal("oversized response body was accepted") + } + status, body = overflow.result() + if status != http.StatusBadGateway || !strings.Contains(body, `"code":"agent_response_too_large"`) { + t.Fatalf("oversized response = %d %q", status, body) + } +} + +func TestControlSessionBoundsInFlightRequests(t *testing.T) { + release := make(chan struct{}) + inner := http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + <-release + response.WriteHeader(http.StatusNoContent) + }) + result := make(chan response, 1) + server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{ + Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled, + }) + if err != nil { + return + } + defer connection.CloseNow() + headers := map[string]string{ + auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef", + auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature", + } + for index := 0; index <= maxInFlight; index++ { + payload, _ := json.Marshal(message{ + Type: "request", ID: fmt.Sprintf("request_%08d", index), Method: http.MethodGet, + RequestTarget: "/v1/health", Headers: headers, + }) + if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil { + return + } + } + messageType, raw, err := connection.Read(request.Context()) + if err != nil || messageType != websocket.MessageText { + return + } + var got response + if json.Unmarshal(raw, &got) == nil { + result <- got + } + _ = connection.Close(websocket.StatusNormalClosure, "done") + })) + defer server.Close() + + broker, _ := consolebroker.NewBroker(time.Minute) + client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute) + if err != nil { + t.Fatal(err) + } + client.httpClient = server.Client() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- client.runSession(ctx) }() + select { + case got := <-result: + if got.Status != http.StatusServiceUnavailable || + !strings.Contains(got.Body, `"code":"agent_busy"`) || + !strings.Contains(got.Body, `"dispatch_state":"not_dispatched"`) { + t.Fatalf("saturation response = %#v", got) + } + case <-ctx.Done(): + t.Fatal("timed out waiting for saturation response") + } + close(release) + select { + case <-done: + case <-ctx.Done(): + t.Fatal("control session did not close") + } +} + +func TestControlRequestDeadlineReturnsGatewayTimeout(t *testing.T) { + release := make(chan struct{}, 1) + defer func() { + select { + case release <- struct{}{}: + default: + } + }() + inner := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + <-release + }) + result := make(chan response, 1) + server := newTLSServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + connection, err := websocket.Accept(writer, request, &websocket.AcceptOptions{ + Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled, + }) + if err != nil { + return + } + defer connection.CloseNow() + payload, _ := json.Marshal(message{ + Type: "request", ID: "request_deadline", Method: http.MethodGet, RequestTarget: "/v1/health", + Headers: map[string]string{ + auth.HeaderTimestamp: "1", auth.HeaderNonce: "0123456789abcdef0123456789abcdef", + auth.HeaderBodyDigest: "digest", auth.HeaderActor: "actor", auth.HeaderSignature: "signature", + }, + }) + if err := connection.Write(request.Context(), websocket.MessageText, payload); err != nil { + return + } + messageType, raw, err := connection.Read(request.Context()) + if err != nil || messageType != websocket.MessageText { + return + } + var got response + if json.Unmarshal(raw, &got) == nil { + result <- got + } + _ = connection.Close(websocket.StatusNormalClosure, "done") + })) + defer server.Close() + + broker, _ := consolebroker.NewBroker(time.Minute) + client, err := NewClient(server.URL, "42", []byte(controlTestSecret), inner, broker, 64<<10, time.Minute) + if err != nil { + t.Fatal(err) + } + client.requestTimeout = 25 * time.Millisecond + client.httpClient = server.Client() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- client.runSession(ctx) }() + select { + case got := <-result: + if got.Status != http.StatusGatewayTimeout || !strings.Contains(got.Body, `"code":"agent_request_timeout"`) { + t.Fatalf("deadline response = %#v", got) + } + if len(client.inFlight) != 1 { + t.Fatalf("timed-out handler released admission slot before returning: %d", len(client.inFlight)) + } + release <- struct{}{} + case <-ctx.Done(): + t.Fatal("timed out waiting for request deadline response") + } + select { + case <-done: + case <-ctx.Done(): + t.Fatal("control session did not close") + } +} + +func newTLSServer(t *testing.T, handler http.Handler) *httptest.Server { + t.Helper() + var server *httptest.Server + var panicValue any + func() { + defer func() { panicValue = recover() }() + server = httptest.NewTLSServer(handler) + }() + if panicValue != nil { + t.Skipf("sandbox does not permit a TLS loopback server: %v", panicValue) + } + return server +} diff --git a/agent/hypervisor/feature_helpers.go b/agent/hypervisor/feature_helpers.go index 934b90b5..6b5ae499 100644 --- a/agent/hypervisor/feature_helpers.go +++ b/agent/hypervisor/feature_helpers.go @@ -570,15 +570,20 @@ func validSourceToken(token string) bool { func remoteHTTPClient(allowRedirects bool) *http.Client { dialer := &net.Dialer{Timeout: 15 * time.Second, KeepAlive: 30 * time.Second} transport := &http.Transport{ - Proxy: nil, - DisableCompression: true, - TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, + Proxy: nil, + DisableCompression: true, + TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, + TLSHandshakeTimeout: 15 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, + ExpectContinueTimeout: 1 * time.Second, DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { host, port, err := net.SplitHostPort(address) if err != nil { return nil, err } - resolved, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host) + lookupCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + resolved, err := net.DefaultResolver.LookupNetIP(lookupCtx, "ip", host) + cancel() if err != nil { return nil, err } diff --git a/agent/hypervisor/feature_helpers_test.go b/agent/hypervisor/feature_helpers_test.go index e8b0857a..e3a2c183 100644 --- a/agent/hypervisor/feature_helpers_test.go +++ b/agent/hypervisor/feature_helpers_test.go @@ -8,6 +8,7 @@ import ( "net/netip" "strings" "testing" + "time" "git.luoli.live/root/virt/agent/cloudinit" ) @@ -139,8 +140,9 @@ func TestRemoteImportAddressPolicyBlocksLocalNetworks(t *testing.T) { } client := remoteHTTPClient(true) transport, ok := client.Transport.(*http.Transport) - if !ok || !transport.DisableCompression { - t.Fatal("remote import client did not disable transparent decompression") + if !ok || !transport.DisableCompression || transport.TLSHandshakeTimeout != 15*time.Second || + transport.ResponseHeaderTimeout != 30*time.Second || transport.ExpectContinueTimeout != time.Second { + t.Fatal("remote import client transport bounds are incomplete") } client.CloseIdleConnections() if _, err := validateRemoteSource("http://example.com/image.iso"); err == nil { diff --git a/agent/hypervisor/libvirt_catalog_features.go b/agent/hypervisor/libvirt_catalog_features.go index a8a2fda6..7d278da7 100644 --- a/agent/hypervisor/libvirt_catalog_features.go +++ b/agent/hypervisor/libvirt_catalog_features.go @@ -16,6 +16,7 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "git.luoli.live/root/virt/agent/operation" @@ -24,6 +25,11 @@ import ( const catalogMetadataLimit = 64 << 10 +// Imports are serialized with each other but do not hold the global VM +// mutation lock while an HTTPS body is in flight. Until metadata is published, +// the incomplete volume is not addressable through catalog APIs. +var catalogImportMu sync.Mutex + func templateMetadataVolumeName(id string) string { return "virt-template-" + id + ".json" } func rescueImageVolumeName(id string) string { return "rescue-" + id + ".iso" } func rescueMetadataVolumeName(id string) string { return "virt-rescue-" + id + ".json" } @@ -96,8 +102,8 @@ func (l *Libvirt) ImportTemplate(ctx context.Context, request ImportTemplateRequ if _, err := validateRemoteSource(request.SourceURL); err != nil { return ImageTemplate{}, err } - domainMutationMu.Lock() - defer domainMutationMu.Unlock() + catalogImportMu.Lock() + defer catalogImportMu.Unlock() templates, err := l.ListTemplates(ctx) if err != nil { return ImageTemplate{}, err @@ -290,8 +296,8 @@ func (l *Libvirt) ImportRescueTemplate(ctx context.Context, request ImportRescue return RescueTemplate{}, NewError(ErrUnavailable, "entropy_unavailable", "rescue template identity could not be generated", err) } } - domainMutationMu.Lock() - defer domainMutationMu.Unlock() + catalogImportMu.Lock() + defer catalogImportMu.Unlock() existing, err := l.ListRescueTemplates(ctx) if err != nil { return RescueTemplate{}, err @@ -449,8 +455,8 @@ func (l *Libvirt) ImportUserISO(ctx context.Context, request ImportUserISOReques return UserISO{}, NewError(ErrUnavailable, "entropy_unavailable", "user ISO identity could not be generated", err) } } - domainMutationMu.Lock() - defer domainMutationMu.Unlock() + catalogImportMu.Lock() + defer catalogImportMu.Unlock() items, err := l.ListUserISOs(ctx, request.OwnerID) if err != nil { return UserISO{}, err @@ -773,15 +779,17 @@ func (l *Libvirt) downloadVolume(ctx context.Context, pool *libvirt.StoragePool, } stream, err := l.conn.NewStream(0) if err != nil { - volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) + failure := translateLibvirtError("create image upload stream", err) + failure = l.rollbackImportedVolume(volume, failure, "imported image") volume.Free() - return nil, 0, "", translateLibvirtError("create image upload stream", err) + return nil, 0, "", failure } if err := volume.Upload(stream, 0, uint64(response.ContentLength), 0); err != nil { stream.Free() - volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) + failure := mutationLibvirtError("open imported image upload", err) + failure = l.rollbackImportedVolume(volume, failure, "imported image") volume.Free() - return nil, 0, "", mutationLibvirtError("open imported image upload", err) + return nil, 0, "", failure } hash := sha256.New() written, copyErr := io.Copy(io.MultiWriter(&libvirtStreamWriter{ctx: ctx, stream: stream}, hash), io.LimitReader(source, maximum+1)) diff --git a/agent/operation/journal.go b/agent/operation/journal.go index 310eb09c..25fca27a 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -41,6 +41,7 @@ var ( ErrNotFound = errors.New("operation not found") ErrFingerprint = errors.New("operation_id payload conflict") ErrOperationUncertain = errors.New("operation result is uncertain") + ErrBusy = errors.New("operation execution capacity is exhausted") operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`) hexDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) noncePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`) @@ -169,8 +170,13 @@ type Journal struct { resourceMu sync.Mutex resources map[string]*resourceLock + + executionSlots chan struct{} + workerWG sync.WaitGroup } +const maxConcurrentExecutions = 32 + func Open(path string) (*Journal, error) { if strings.TrimSpace(path) == "" { return nil, errors.New("journal path is required") @@ -182,7 +188,10 @@ func Open(path string) (*Journal, error) { if err != nil { return nil, fmt.Errorf("open operation journal database: %w", err) } - j := &Journal{path: path, db: db, now: time.Now, resources: make(map[string]*resourceLock)} + j := &Journal{ + path: path, db: db, now: time.Now, resources: make(map[string]*resourceLock), + executionSlots: make(chan struct{}, maxConcurrentExecutions), + } if err := j.initialize(); err != nil { _ = db.Close() return nil, err @@ -197,6 +206,23 @@ func (j *Journal) Close() error { return j.db.Close() } +// Wait blocks until every real mutation and late-result finalizer has stopped. +// Callers must stop API admission before invoking it so no new worker can be +// added concurrently with the wait. +func (j *Journal) Wait(ctx context.Context) error { + done := make(chan struct{}) + go func() { + j.workerWG.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + // UseNonce uses two Bolt buckets: direct lookup is O(1), while the ordered // expiry index makes cleanup proportional only to expired entries. It never // rewrites or decodes the operation history. @@ -350,13 +376,24 @@ func (j *Journal) Execute(ctx context.Context, spec Spec, fn func(context.Contex } return failed, existed, err } + releaseExecution, err := j.acquireExecution(ctx) + if err != nil { + release() + failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID) + if failErr != nil { + return failed, existed, failErr + } + return failed, existed, err + } record, err = j.markDispatching(spec.OperationID) if err != nil { + releaseExecution() release() return record, existed, err } if record.State != StateDispatching { + releaseExecution() release() if record.State == StateUncertain { return record, true, ErrOperationUncertain @@ -370,7 +407,9 @@ func (j *Journal) Execute(ctx context.Context, spec Spec, fn func(context.Contex } done := make(chan outcome, 1) mutationContext := context.WithValue(ctx, expectedStateContextKey{}, cloneExpectedState(record.ExpectedState)) + j.workerWG.Add(1) go func() { + defer j.workerWG.Done() result, callErr := fn(mutationContext) done <- outcome{result: result, err: callErr} }() @@ -379,25 +418,101 @@ func (j *Journal) Execute(ctx context.Context, spec Spec, fn func(context.Contex case completed := <-done: terminal, persistErr := j.finish(spec.OperationID, completed.result, completed.err) release() + releaseExecution() if persistErr != nil { return terminal, existed, persistErr } return terminal, existed, completed.err case <-ctx.Done(): uncertain, persistErr := j.markUncertain(spec.OperationID, ctx.Err()) - if persistErr != nil { - release() - return uncertain, existed, persistErr - } + j.workerWG.Add(1) go func() { + defer j.workerWG.Done() completed := <-done _, _ = j.finishIfUncertain(spec.OperationID, completed.result, completed.err) release() + releaseExecution() }() + if persistErr != nil { + return uncertain, existed, persistErr + } return uncertain, existed, MarkUncertain(ctx.Err()) } } +// ExecuteBackground persists the dispatch boundary and starts one bounded, +// tracked worker before returning. A nil done channel means an existing +// non-reserved record was replayed and no new call was started. +func (j *Journal) ExecuteBackground( + dispatchCtx, workerCtx context.Context, + spec Spec, + fn func(context.Context) (json.RawMessage, error), +) (Record, bool, <-chan struct{}, error) { + record, existed, err := j.Reserve(spec) + if err != nil { + return Record{}, existed, nil, err + } + if existed && record.State != StateReserved { + return record, true, nil, nil + } + release, err := j.acquire(dispatchCtx, lockKey(spec)) + if err != nil { + failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID) + if failErr != nil { + return failed, existed, nil, failErr + } + return failed, existed, nil, err + } + releaseExecution, err := j.acquireExecution(dispatchCtx) + if err != nil { + release() + failed, failErr := j.failNotDispatchedIfReserved(spec.OperationID) + if failErr != nil { + return failed, existed, nil, failErr + } + return failed, existed, nil, err + } + record, err = j.markDispatching(spec.OperationID) + if err != nil { + releaseExecution() + release() + return record, existed, nil, err + } + if record.State != StateDispatching { + releaseExecution() + release() + return record, true, nil, nil + } + type outcome struct { + result json.RawMessage + err error + } + callDone := make(chan outcome, 1) + done := make(chan struct{}) + mutationContext := context.WithValue(workerCtx, expectedStateContextKey{}, cloneExpectedState(record.ExpectedState)) + j.workerWG.Add(2) + go func() { + defer j.workerWG.Done() + result, callErr := fn(mutationContext) + callDone <- outcome{result: result, err: callErr} + }() + go func() { + defer j.workerWG.Done() + defer close(done) + defer release() + defer releaseExecution() + select { + case completed := <-callDone: + _, _ = j.finishIfUncertain(spec.OperationID, completed.result, completed.err) + case <-workerCtx.Done(): + _, _ = j.markUncertain(spec.OperationID, workerCtx.Err()) + completed := <-callDone + _, _ = j.finishIfUncertain(spec.OperationID, completed.result, completed.err) + } + }() + return record, existed, done, nil +} + // ExpectedStateFromContext returns the descriptor atomically persisted with // the dispatch record. Mutation implementations use it to write a matching // host-side transition marker before reporting success. @@ -634,6 +749,12 @@ func (j *Journal) failNotDispatchedIfReserved(operationID string) (Record, error }) } +// FailNotDispatched terminally records that a reserved operation never +// crossed the dispatch boundary. It is safe for admission failures only. +func (j *Journal) FailNotDispatched(operationID string) (Record, error) { + return j.failNotDispatchedIfReserved(operationID) +} + func notDispatchedEvidence(action string) json.RawMessage { if action == "create-vm" { return json.RawMessage(`{"not_dispatched":true,"creation_absent":true}`) @@ -724,6 +845,18 @@ func (j *Journal) acquire(ctx context.Context, key string) (func(), error) { } } +func (j *Journal) acquireExecution(_ context.Context) (func(), error) { + select { + case j.executionSlots <- struct{}{}: + var once sync.Once + return func() { + once.Do(func() { <-j.executionSlots }) + }, nil + default: + return nil, ErrBusy + } +} + func (j *Journal) releaseResourceReference(key string, lock *resourceLock) { j.resourceMu.Lock() defer j.resourceMu.Unlock() diff --git a/agent/operation/journal_test.go b/agent/operation/journal_test.go index cdc9983f..5f4a21c3 100644 --- a/agent/operation/journal_test.go +++ b/agent/operation/journal_test.go @@ -313,6 +313,90 @@ func TestTimeoutAfterDispatchIsDurablyUncertainThenRecordsLateResult(t *testing. } } +func TestBackgroundTimeoutMarksUncertainAndRetainsAdmissionUntilLateResult(t *testing.T) { + j := testJournal(t) + spec := testSpec("operation-background-timeout", "vm-background-timeout", `{}`) + workerCtx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + started := make(chan struct{}) + releaseWorker := make(chan struct{}) + record, replayed, done, err := j.ExecuteBackground( + context.Background(), workerCtx, spec, + func(context.Context) (json.RawMessage, error) { + close(started) + <-releaseWorker + return json.RawMessage(`{"deleted":true}`), nil + }, + ) + if err != nil || replayed || done == nil || record.State != StateDispatching { + t.Fatalf("background start = %#v replayed=%v done=%v err=%v", record, replayed, done != nil, err) + } + <-started + + deadline := time.Now().Add(time.Second) + for { + record, err = j.Get(spec.OperationID) + if err == nil && record.State == StateUncertain { + break + } + if time.Now().After(deadline) { + t.Fatalf("background timeout was not persisted: %#v err=%v", record, err) + } + time.Sleep(time.Millisecond) + } + if len(j.executionSlots) != 1 { + t.Fatalf("background execution slots in use = %d, want 1", len(j.executionSlots)) + } + select { + case <-done: + t.Fatal("background completion closed before the real worker returned") + default: + } + + replayedCall := false + replayedRecord, wasReplay, replayErr := j.Execute( + context.Background(), spec, + func(context.Context) (json.RawMessage, error) { + replayedCall = true + return nil, nil + }, + ) + if !wasReplay || !errors.Is(replayErr, ErrOperationUncertain) || replayedCall || replayedRecord.State != StateUncertain { + t.Fatalf("background timeout replay = %#v replayed=%v called=%v err=%v", + replayedRecord, wasReplay, replayedCall, replayErr) + } + + waitCtx, waitCancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer waitCancel() + waiterCalled := false + waiter, _, waitErr := j.Execute( + waitCtx, + testSpec("operation-background-waiter", spec.ResourceID, `{"waiter":true}`), + func(context.Context) (json.RawMessage, error) { + waiterCalled = true + return nil, nil + }, + ) + if !errors.Is(waitErr, context.DeadlineExceeded) || waiterCalled || + waiter.State != StateFailed || waiter.Error == nil || waiter.Error.Code != "operation_not_dispatched" { + t.Fatalf("background resource lock was released early: %#v called=%v err=%v", waiter, waiterCalled, waitErr) + } + + close(releaseWorker) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("background late-result finalizer did not finish") + } + record, err = j.Get(spec.OperationID) + if err != nil || record.State != StateSucceeded || string(record.Result) != `{"deleted":true}` { + t.Fatalf("background late result = %#v err=%v", record, err) + } + if len(j.executionSlots) != 0 { + t.Fatalf("background execution slots retained after completion = %d", len(j.executionSlots)) + } +} + func TestExpectedStateSurvivesCrashWithoutAppearingOnPublicRecordJSON(t *testing.T) { path := filepath.Join(t.TempDir(), "operations.db") j, err := Open(path) diff --git a/deploy/virt-cloud-console-relay.json.example b/deploy/virt-cloud-agent-gateway.json.example similarity index 53% rename from deploy/virt-cloud-console-relay.json.example rename to deploy/virt-cloud-agent-gateway.json.example index 7712cfeb..ba9bba50 100644 --- a/deploy/virt-cloud-console-relay.json.example +++ b/deploy/virt-cloud-agent-gateway.json.example @@ -1,6 +1,8 @@ { "listen_address": "127.0.0.1:8001", "panel_introspection_url": "http://127.0.0.1/api/internal/cloud/session-introspection", + "panel_agent_auth_url": "http://127.0.0.1/api/internal/cloud/agent-auth", "panel_shared_secret": "REPLACE_ME", - "connect_timeout_seconds": 10 + "connect_timeout_seconds": 10, + "agent_request_timeout_seconds": 900 } diff --git a/deploy/virt-cloud-console-relay.service b/deploy/virt-cloud-agent-gateway.service similarity index 59% rename from deploy/virt-cloud-console-relay.service rename to deploy/virt-cloud-agent-gateway.service index 0b424be5..eac24515 100644 --- a/deploy/virt-cloud-console-relay.service +++ b/deploy/virt-cloud-agent-gateway.service @@ -1,5 +1,5 @@ [Unit] -Description=Virt Cloud local console WebSocket relay +Description=Virt Cloud Agent connection and reverse console gateway After=network-online.target Wants=network-online.target @@ -8,11 +8,11 @@ Type=simple User=apache Group=apache WorkingDirectory=/home/virt-cloud -ExecStart=/home/virt-cloud/bin/virt-cloud-console-relay --config /home/virt-cloud/runtime/console-relay.json +ExecStart=/home/virt-cloud/bin/virt-cloud-agent-gateway --config /home/virt-cloud/runtime/agent-gateway.json Restart=on-failure RestartSec=5 UMask=0077 -# UMask only hardens newly created files; the relay does not audit the owner, +# UMask only hardens newly created files; the gateway does not audit the owner, # group, or mode of its readable JSON configuration. NoNewPrivileges=true PrivateTmp=true diff --git a/deploy/virt-cloud-agent.json.example b/deploy/virt-cloud-agent.json.example index d0e1856a..69a1edf9 100644 --- a/deploy/virt-cloud-agent.json.example +++ b/deploy/virt-cloud-agent.json.example @@ -1,10 +1,9 @@ { - "listen_address": "0.0.0.0:9443", - "shared_secret": "REPLACE_ME", + "panel_url": "https://panel.example.com", + "agent_id": "1", + "agent_secret": "REPLACE_ME", "journal_path": "/var/lib/virt-agent/operations.db", "libvirt_uri": "qemu:///system", - "tls_cert_file": "/etc/virt-agent/tls/agent.crt", - "tls_key_file": "/etc/virt-agent/tls/agent.key", "max_body_bytes": 1048576, "clock_skew_seconds": 300, "nonce_ttl_seconds": 600, diff --git a/deploy/virt-cloud-update.json.example b/deploy/virt-cloud-update.json.example index aa7153ba..77a0e8c4 100644 --- a/deploy/virt-cloud-update.json.example +++ b/deploy/virt-cloud-update.json.example @@ -12,6 +12,6 @@ "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", "virt-cloud-policy-worker@2.service", - "virt-cloud-console-relay.service" + "virt-cloud-agent-gateway.service" ] } diff --git a/docs/ADMIN_OPERATIONS.md b/docs/ADMIN_OPERATIONS.md index c4e86060..19a4c454 100644 --- a/docs/ADMIN_OPERATIONS.md +++ b/docs/ADMIN_OPERATIONS.md @@ -25,7 +25,7 @@ POST /api/v1/admin/operations/jobs/{id}/cancel ## 运行状态的含义 - 业务/日志数据库:只读 `SELECT 1` 成功,不代表全部表结构或业务数据正确。MySQL 使用独立、非持久连接,连接及读取超时各为 2 秒;仅在支持 mysqlnd 的 PDO 驱动下探测,临时读取超时配置在检查后恢复,不修改应用共享连接。驱动或运行环境不支持短超时设置时显示未知,不回退到无界连接;SQLite 保持原有只读查询。日志库未启用则明确显示未启用。DNS 解析不受这些连接/读取超时完整约束,因此不能将它们理解为整个健康接口的硬总时限。 -- 宿主机 Agent:TLS、签名健康接口和本地操作日志可用,不代表每台虚拟机都正常。页面分别展示各 Agent 的 endpoint、最后测试时间和受控错误;健康请求必须有短超时。 +- 宿主机 Agent:已认证出站 WSS 会话、内层签名健康接口和本地操作日志可用,不代表每台虚拟机都正常。页面分别展示数字 Agent ID、最后测试时间和受控错误,不存储或展示宿主机 endpoint;健康请求必须有短超时。 - Redis:配置后检查 PING;独立客户端使用 1 秒连接/读写超时,不影响应用共享客户端,也不验证缓存内容。 - 普通 worker、策略 worker、生命周期调度:读取本机最近运行记录;不是 `systemctl` 状态,也不是对进程仍然存活的保证。页面可见时每 30 秒检查一次,不刷新整个任务列表;检查失败时明确标记保留的是旧状态。 @@ -43,7 +43,7 @@ virt-cloud-lifecycle.service virt-cloud-policy-worker@1.service virt-cloud-policy-worker@2.service virt-cloud-update.service -virt-cloud-console-relay.service +virt-cloud-agent-gateway.service ``` 不要仅为获得心跳而中断正在重装/删除的任务。通过既有服务日志确认任务情况;缺少心跳不会改变现有任务处理逻辑。 diff --git a/docs/ADMIN_SYSTEM_LOGS.md b/docs/ADMIN_SYSTEM_LOGS.md index 6c1e021f..60746e36 100644 --- a/docs/ADMIN_SYSTEM_LOGS.md +++ b/docs/ADMIN_SYSTEM_LOGS.md @@ -10,7 +10,7 @@ - `virt-cloud-policy-worker@1.service` - `virt-cloud-policy-worker@2.service` - `virt-cloud-update.service` -- `virt-cloud-console-relay.service` +- `virt-cloud-agent-gateway.service` 接口使用参数数组启动 `/usr/bin/journalctl`(系统没有该路径时使用 `/bin/journalctl`),不会经过 shell,也不接受任意 unit、命令或文件路径。单次最多读取 200 条,使用 `--cursor` 加 `--reverse` 加载更早记录;journalctl 会从游标向更早记录移动,返回结果通常不包含边界,服务端也会防御性移除意外返回的边界记录。这样可以避开部分旧 systemd 版本在 `--after-cursor` 与 unit 筛选组合下漏掉首条记录的问题。带时间下界的游标页会在 PHP 中继续应用下界,因为 systemd 不允许 `--since` 与游标同时出现。消息预览、详情字段、输出、stderr 和子进程运行时间都有上限。每条列表记录保留 journal cursor、服务和 UTC 微秒时间戳,点击后通过 `GET /api/v1/admin/system-logs/detail?cursor=...&unit=...×tamp=...` 在该时间戳窗口内重新读取,再校验返回 cursor,避免不同 systemd 版本的 cursor seek 差异影响详情;日志轮转或游标不存在时返回 404。命令行、可执行文件和 core-dump 字段不会返回,名称明显表示密码、令牌、密钥、凭据或授权信息的结构化字段会隐藏值,消息中的常见凭据形式也会遮罩。日志内容可能包含运行时上下文,因此接口仅对管理员开放,并设置 `private, no-store`。 diff --git a/docs/AGENT_API.md b/docs/AGENT_API.md index e21cbd73..96e25ab3 100644 --- a/docs/AGENT_API.md +++ b/docs/AGENT_API.md @@ -1,15 +1,18 @@ -# Virt Host Agent API v1 +# Virt Host Agent WSS/API v1 -本协议只供 PHP 面板调用宿主机 Go Agent。浏览器、第三方客户端和另一台宿主机不得直接使用;不提供旧引擎 API 兼容。 +本协议只供 PHP 面板通过本机 Go Gateway 调用宿主机 Go Agent。浏览器、第三方客户端和另一台宿主机不得直接使用;不提供旧引擎 API 兼容。 ## 传输与编码 -- 基础路径为 `/v1`。面板只接受固定 IP 的 HTTPS endpoint;私网、CGNAT 和测试部署也不能降级为明文 HTTP。Agent 配置必须提供证书与私钥并只启动 TLS listener;证书必须包含 endpoint IP 的 SAN,且其签发链须受面板系统信任。Agent 不跟随请求重定向;PHP 也不得跟随 Agent 响应的重定向。 +- Agent 配置面板 HTTPS origin、数字 `agent_id` 和独立 `agent_secret`,主动连接 `WSS /agent/connect`,子协议固定为 `virt-agent-control.v1`。请求头携带 `X-Virt-Agent-ID` 与 `Authorization: Bearer `;Gateway 必须通过 PHP 内部接口校验后才接管会话。Agent 不启动入站 HTTP/TLS listener,不需要部署宿主机证书或开放管理端口。 +- 基础 API 语义仍使用 `/v1`。PHP 先用每宿主机密钥对内层 method/target/body 签名,再用 Gateway 本机共享密钥向字面 loopback 的 `POST /internal/agent/request` 提交转发 envelope。Gateway 不改写内层字节,Agent 在进程内将其交给同一 API handler 校验与执行。 - 请求和响应使用 UTF-8 JSON,`Content-Type: application/json`。无正文 GET 的摘要按空字节串计算。 - JSON decoder 拒绝未知字段、多个顶层值、尾随非空内容、错误类型和超过 `max_body_bytes` 的正文。 - ID、UUID、路径、URL、网桥名、MAC、CIDR、脚本和数组数量都必须有显式格式/长度上限;不能把任意 XML、shell 命令或 libvirt flags 作为 API 输入。 - 时间使用 UTC RFC 3339;字节数使用整数,不用浮点;响应禁止包含密钥、宿主机文件内容、VNC 地址或底层凭据。 +控制帧是严格 JSON:Gateway 发送 `{"type":"request","id":"...","method":"GET","request_target":"/v1/...","body":"...","headers":{...}}`,Agent 返回 `{"type":"response","id":"...","status":200,"body":"{...}"}`。`id` 只用于连接内匹配响应;业务幂等和防重放仍以内层 HMAC、nonce 和 `operation_id` 为准。任一方断线或超时只能返回不确定运输结果,不得为此生成新 `operation_id` 或自动重放变更。 + ## HMAC 认证 每台宿主机使用不同的至少 32 字节密钥。每个请求包含: @@ -144,17 +147,17 @@ Agent 在调用 libvirt 前将 `operation_id`、VM UUID/资源 ID、动作、指 ## 控制台票据 -浏览器先从 PHP 获取一次性面板票据,再连接面板本机 `virt-cloud-console-relay`。relay 通过 PHP 内省得到 VM UUID、目标 Agent 和一个 Agent console ticket;浏览器不能提供 Agent endpoint。 +浏览器先从 PHP 获取一次性面板票据,再连接面板 `virt-cloud-agent-gateway` 的 `/engine-console`。Gateway 通过 PHP 内省得到数字 Agent ID 和一个 Agent console ticket;浏览器不能提供 Agent ID、endpoint 或 VNC 目标。 Agent console ticket 的当前边界: - 只能由通过 HMAC 且具有 `can_manage` 的 panel-to-Agent `POST /v1/vms/{uuid}/console-ticket` 签发,TTL 固定 60 秒。 - Agent 仅在内存中将随机票据映射到该 VM 的本地 loopback VNC 目标;进程重启会使未用票据全部失效。 - 票据在尝试建立上游连接前就被原子消费;过期、重复或已消费一律拒绝,连接失败也不退回票据。 -- 面板浏览器票据另行绑定会话、用户和 VM;relay 每 15 秒向 PHP 重新内省,负责在登出、撤销或权限变更后断开会话。 +- 面板浏览器票据另行绑定会话、用户和 VM;Gateway 每 15 秒向 PHP 重新内省,负责在登出、撤销或权限变更后断开会话。 - 只把 VNC 绑定地址留在 Agent 内部,API 和浏览器都不能看到宿主机 VNC 地址/端口。 -relay 使用 Agent 返回的 `protocol`(`cloud-agent-ticket.`)连接固定 `GET /v1/console`。该 WebSocket 路由不使用长期 HMAC,只消费前述短时一次性票据;只转发 binary frame。`features.console` 只有在完整 broker 与本地 VNC 连接可用时才能为 true。 +Gateway 在已认证控制通道发送 `console_open`,其中包含一次性 Agent ticket、会话 ID 和随机反向通道 token。Agent 在尝试 VNC 前原子消费票据,然后主动连接 `WSS /agent/console`,子协议为 `virt-agent-console.`;反向通道只转发 binary frame。`features.console` 只有在完整 broker、本地 VNC 与反向 Gateway 连接可用时才能为 true。 ## UEFI 与 Secure Boot diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 91eb79c3..345d139e 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -2,7 +2,7 @@ 本版本只支持全新安装:不导入旧数据库,不读取旧引擎密文、配置或宿主机凭据,也不提供兼容回退。示例使用 AlmaLinux、项目目录 `/home/virt-cloud`、PHP-FPM 用户 `apache`;请按实际发行版调整包名和服务名。 -面板服务器运行 PHP、数据库客户端、workers、更新 helper 和可选的控制台 relay。每台 KVM 宿主机单独运行 Go `virt-agent`,Agent 只连接该机的 `qemu:///system`,不经 SSH 管理另一台宿主机。 +面板服务器运行 PHP、数据库客户端、workers、更新 helper 和 Go Agent Gateway。每台 KVM 宿主机单独运行 Go `virt-agent`,Agent 只连接该机的 `qemu:///system`,并主动连接面板 WSS;不经 SSH 管理另一台宿主机。 ## 1. 依赖 @@ -11,7 +11,7 @@ - PHP 8.5 CLI/FPM、PDO MySQL、sodium、mbstring、curl、XML/DOM、OpenSSL、session,Composer 2。 - MariaDB:一个业务库和一个日志库;初始化账户需要建表权限。当前 SQL 以 MariaDB 为目标,不承诺兼容 Oracle MySQL。 - Redis 或兼容 Valkey,用于会话、验证码与限流。 -- Go 1.24 或更新版本,用于构建本机更新 helper 与控制台 relay。 +- Go 1.24 或更新版本,用于构建本机更新 helper 与 Agent Gateway。 - Nginx/Apache 与有效 HTTPS 证书。 每台虚拟化宿主机: @@ -34,7 +34,7 @@ composer install --no-dev --prefer-dist --optimize-autoloader ```bash cd /home/virt-cloud/web -cp config/.config.php.example config/config.php +install -o apache -g apache -m 0600 config/.config.php.example config/config.php ``` 至少设置业务库、日志库、Redis、站点 HTTPS、Cookie、SMTP、支付以及 `cloud_data_key_file`。`cloud_data_key` 必须是 base64 编码的 32 字节独立随机密钥,PHP 使用版本化、用途/AAD 隔离的 XChaCha20-Poly1305 格式密封每台 Agent 的 HMAC 密钥、任务载荷和凭据;不能使用网站通用 key 代替,也不接受旧密文格式。 @@ -43,9 +43,8 @@ cp config/.config.php.example config/config.php ```bash install -d -o apache -g apache /home/virt-cloud/runtime/php +install -o apache -g apache -m 0600 /dev/null /home/virt-cloud/runtime/php/data.key openssl rand -base64 32 > /home/virt-cloud/runtime/php/data.key -chown apache:apache /home/virt-cloud/runtime/php/data.key -chmod 0640 /home/virt-cloud/runtime/php/data.key ``` 上述 owner/mode 只是使文件可读并收紧操作系统访问的部署建议;PHP 不审计它们,也不会因为可读密钥文件使用其他 mode 而拒绝启动。 @@ -77,22 +76,23 @@ go mod verify go test ./... go test -tags libvirt ./... go build -tags libvirt -trimpath -o /usr/local/sbin/virt-agent ./cmd/virt-agent -install -d /etc/virt-agent /etc/virt-agent/tls /var/lib/virt-agent -cp /home/virt-cloud/deploy/virt-cloud-agent.json.example /etc/virt-agent/config.json +install -d /etc/virt-agent /var/lib/virt-agent +install -o root -g root -m 0600 \ + /home/virt-cloud/deploy/virt-cloud-agent.json.example /etc/virt-agent/config.json ``` -为每台宿主机生成不同的至少 32 字节随机密钥,并把相同值分别写入该机 Agent 配置和面板 `/admin/agents` 对应记录。示例文件只有占位符,绝不能直接启动或提交真实密钥。保存启用配置后必须立即在该页执行“测试连接”;面板仅在当前 `revision` 已通过健康与必要能力检查时才允许调度、业务 Agent 请求和控制台票据发放。任何 endpoint、密钥或启用状态修改都会清除该就绪标记,要求重新测试。 +为每台宿主机生成不同的至少 32 字节随机密钥,并把相同值分别写入该机 Agent 配置和面板 `/admin/agents` 对应记录。`agent_id` 就是面板宿主机记录的正整数 ID,`panel_url` 是面板对 Agent 可达的 HTTPS origin。示例文件只有占位符,绝不能直接启动或提交真实密钥。保存启用配置后,先启动 Agent 建立出站会话,再在管理页执行“测试连接”。面板仅在当前 `revision` 已通过健康与必要能力检查时才允许调度、业务 Agent 请求和控制台票据发放。密钥或启用状态修改会清除就绪标记。 Agent 配置字段: | 字段 | 说明 | | --- | --- | -| `listen_address` | 默认建议 `0.0.0.0:9443`,防火墙只允许面板服务器访问 | -| `shared_secret` | 该宿主机独立 HMAC 密钥,至少 32 字节 | +| `panel_url` | 面板 HTTPS origin,例如 `https://panel.example.com`;Agent 由此派生 `wss://.../agent/connect` 和 `/agent/console` | +| `agent_id` | 面板对应宿主机的正整数 ID | +| `agent_secret` | 该宿主机独立连接/HMAC 密钥,32~256 个非空格可打印 ASCII 字符 | | `journal_path` | bbolt 事务型持久库,保存 operation_id、结果、expected-state 与 nonce 到期索引;systemd 样例下使用 `/var/lib/virt-agent/operations.db` | | `libvirt_uri` | 必须为 `qemu:///system` | -| `tls_cert_file` / `tls_key_file` | 必填的 TLS 证书与私钥绝对路径;证书必须包含面板中所填固定 IP 的 SAN,签发链须受面板系统信任 | -| `max_body_bytes` | 请求体上限,默认 1 MiB,允许 1 KiB~64 MiB | +| `max_body_bytes` | 请求体上限,默认 1 MiB,允许 1 KiB~1 MiB;云镜像正文不经过控制通道 | | `clock_skew_seconds` | 签名时间窗,默认 300 秒 | | `nonce_ttl_seconds` | nonce 保留时间,默认 600 秒且不得小于 `2 × clock_skew_seconds`,以覆盖未来时间戳请求的完整有效窗 | | `operation_timeout_seconds` | 操作等待上限,默认 120 秒;超时不等于底层调用已取消 | @@ -110,7 +110,7 @@ Agent 的 systemd 单元保持 `ProtectSystem=strict`,进程只获准写 `/var `journal_path` 是 bbolt 数据库,不是可编辑 JSON。nonce 通过独立到期索引清理,不会重写 operation 历史;到期时刻本身仍视为占用,只清理严格早于当前时刻的 nonce。独立 inflight 索引使启动只恢复 `reserved`/`dispatching`,不会扫描永久历史。终态结果详情保留 90 天后压缩,释放的 bbolt 页面会被后续写入复用,但 operation_id、请求指纹、资源、动作和终态会永久保留为防重放 tombstone。禁止以“清理历史”为由删除这些 tombstone。当前为全新 Go Agent 格式,不导入旧 Python 数据或早期 JSON 原型 journal。 -配置必须存在、可读、是单个严格 JSON 对象并通过内容校验。Agent **不会**检查文件 owner、group 或 mode,也不会因 `0600`、`0640`、`0644` 等可读权限差异拒绝启动;文档中的收紧权限只是运维建议。操作系统本身无权读取、JSON 损坏、字段未知、TLS 文件不可读或配置值无效仍会明确启动失败;这些是可用性/内容错误,不是权限位审计。`tls_cert_file` 与 `tls_key_file` 永远必填,任何旧明文传输开关都会作为未知字段被拒绝;开发健康测试应直接使用进程内 HTTP handler/`httptest`,不能给生产二进制留下明文监听开关。 +配置必须存在、可读、是单个严格 JSON 对象并通过内容校验。Agent **不会**检查文件 owner、group 或 mode,也不会因 `0600`、`0640`、`0644` 等可读权限差异拒绝启动;文档中的收紧权限只是运维建议。操作系统本身无权读取、JSON 损坏、字段未知或配置值无效仍会明确启动失败;这些是可用性/内容错误,不是权限位审计。旧的 `listen_address`、`shared_secret`、`tls_cert_file` 和 `tls_key_file` 会作为未知字段被拒绝;Agent 不启动入站监听。 安装并启动 service: @@ -122,11 +122,11 @@ systemctl status virt-cloud-agent.service --no-pager journalctl -u virt-cloud-agent.service -n 100 --no-pager ``` -在面板 `/admin/agents` 填写固定 IP `https://IP:9443`、对应密钥并执行连接测试;域名和任何 `http://` 地址(包括私网/CGNAT)都会被拒绝。先保持宿主机调度状态为 disabled;健康和 capabilities 通过后,再配置资源池、网络和销售状态。如果二进制未带 `libvirt` build tag,Agent 会以 `libvirt_adapter_not_built` 明确失败,不会退化成外部命令或远程连接。 +在面板 `/admin/agents` 为相同宿主机 ID 保存对应密钥,启动 Agent 后执行连接测试。先保持宿主机调度状态为 disabled;健康和 capabilities 通过后,再配置资源池、网络和销售状态。如果二进制未带 `libvirt` build tag,Agent 会以 `libvirt_adapter_not_built` 明确失败,不会退化成外部命令或远程连接。 -## 5. 面板 workers、更新 helper 与控制台 relay +## 5. 面板 workers、更新 helper 与 Agent Gateway -PHP workers 不依赖本机虚拟化服务;它们按任务绑定的宿主机直接调用对应 Agent: +PHP workers 不依赖本机虚拟化服务;它们先持久化任务,再通过本机 Agent Gateway 将请求发给任务绑定的 Agent: ```bash cp /home/virt-cloud/deploy/virt-cloud-worker.service /etc/systemd/system/ @@ -139,7 +139,7 @@ systemctl enable --now virt-cloud-worker.service virt-cloud-lifecycle.service \ 普通 worker 负责购买、创建、重装、邮件和最终删除;至少两个策略 worker 处理到期断网、网络恢复与删除前策略;lifecycle 生成续费账单并推进账期、流量、宽限和删除状态。不要因某个 Agent 离线而停止其他宿主机的任务处理。 -在线更新由本机 root Go helper 提供固定 Unix socket API,不通过 Agent 执行。在面板机器构建并安装两个 helper: +在线更新由本机 root Go helper 提供固定 Unix socket API,不通过 Agent 执行。在面板机器构建并安装更新 helper 和 Agent Gateway: ```bash install -d /home/virt-cloud/bin /home/virt-cloud/runtime/update @@ -148,18 +148,20 @@ go mod download go mod verify go test ./... go build -trimpath -o /home/virt-cloud/bin/virt-cloud-update ./cmd/virt-cloud-update -go build -trimpath -o /home/virt-cloud/bin/virt-cloud-console-relay ./cmd/virt-cloud-console-relay -cp /home/virt-cloud/deploy/virt-cloud-update.json.example /home/virt-cloud/runtime/update.json -cp /home/virt-cloud/deploy/virt-cloud-console-relay.json.example /home/virt-cloud/runtime/console-relay.json +go build -trimpath -o /home/virt-cloud/bin/virt-cloud-agent-gateway ./cmd/virt-cloud-agent-gateway +install -o root -g root -m 0600 \ + /home/virt-cloud/deploy/virt-cloud-update.json.example /home/virt-cloud/runtime/update.json +install -o apache -g apache -m 0600 \ + /home/virt-cloud/deploy/virt-cloud-agent-gateway.json.example /home/virt-cloud/runtime/agent-gateway.json cp /home/virt-cloud/deploy/virt-cloud-update.service /etc/systemd/system/ -cp /home/virt-cloud/deploy/virt-cloud-console-relay.service /etc/systemd/system/ +cp /home/virt-cloud/deploy/virt-cloud-agent-gateway.service /etc/systemd/system/ ``` -启动前先编辑两个 runtime JSON,把所有 `REPLACE_ME` 占位符换成不同的安全随机密钥,并在 PHP `config.php` 中配置完全一致的 `cloud_update_helper_secret` 和 `cloud_console_relay_secret`。占位符短于 32 字节,保留它会使 helper 拒绝启动,避免误用公开样例密钥。然后才启动: +启动前先编辑两个 runtime JSON,把所有 `REPLACE_ME` 占位符换成不同的安全随机密钥,并在 PHP `config.php` 中配置完全一致的 `cloud_update_helper_secret` 和 `cloud_agent_gateway_secret`;`cloud_agent_gateway_url` 必须严格为 `http://127.0.0.1:8001/internal/agent/request`。占位符短于 32 字节,保留它会使服务拒绝启动。然后才启动: ```bash systemctl daemon-reload -systemctl enable --now virt-cloud-update.service virt-cloud-console-relay.service +systemctl enable --now virt-cloud-update.service virt-cloud-agent-gateway.service ``` `virt-cloud-update` 常驻并仅监听 `/run/virt-cloud/update.sock`,PHP 使用固定的 settings/check/apply/jobs API。配置字段为: @@ -178,33 +180,56 @@ systemctl enable --now virt-cloud-update.service virt-cloud-console-relay.servic 仓库访问令牌经 API 保存到 helper 状态,不写配置样例、URL、进程参数或日志。helper 只允许固定远端、固定 `go` 分支和干净工作树 fast-forward;`cloudInit` 结果不确定时保留维护锁,必须 SSH 人工核对。如果未部署某个可选应用 unit,启动 update helper 前必须把它从 `service_units` 样例中删除;不得为绕过预检而填写虚假 unit。 -控制台 relay 只监听 `127.0.0.1:8001`。`runtime/console-relay.json` 的 `panel_shared_secret` 必须与 PHP `cloud_console_relay_secret` 一致,`panel_introspection_url` 必须是字面 loopback IP 上的 `/api/internal/cloud/session-introspection`,`connect_timeout_seconds` 只允许 1–30。relay 不持有每宿主机长期 Agent 密钥。 +面板 Agent Gateway 只监听 `127.0.0.1:8001`。`runtime/agent-gateway.json` 的 `panel_shared_secret` 必须与 PHP `cloud_agent_gateway_secret` 一致;`panel_introspection_url` 和 `panel_agent_auth_url` 必须分别是字面 loopback IP 上的 `/api/internal/cloud/session-introspection` 与 `/api/internal/cloud/agent-auth`。Gateway 在 Agent WSS 握手以及每次普通请求或控制台打开前请 PHP 重新核验当前连接保存的 Agent 凭据;核验失败会撤销该连接。 -relay 用 `X-Virt-*` HMAC 以 actor `{"service":"console-relay"}` 调用 PHP。首次内省必须原子消费面板票据: +Gateway 用 `X-Virt-*` HMAC 以 actor `{"service":"agent-gateway"}` 调用 PHP。首次内省必须原子消费面板票据: ```json {"ticket":"","consume":true} ``` -PHP 只返回下列字段;Agent URL 必须是固定非环回 IP 的 `wss://.../v1/console`,过期时间为 UTC RFC3339: +PHP 只返回下列字段;`agent_id` 对应已连接的数字 Agent 身份,过期时间为 UTC RFC3339: ```json { "active": true, "connection_id": "", - "agent_websocket_url": "wss://198.51.100.2:9443/v1/console", + "agent_id": "42", "agent_ticket": "", "agent_expires_at": "2026-10-15T12:00:00Z" } ``` -relay 每 15 秒发送 `{"ticket":"","connection_id":"","consume":false}`。会话、用户、VM 所有权和服务均仍有效时 PHP 必须严格只返回 `{"active":true}`。任何拒绝、额外/缺失字段、超时或 `active:false` 都必须断开两端。 +Gateway 向对应 Agent 的控制长连接发送 `console_open`,Agent 消费票据并主动建立 `/agent/console` 反向 WSS。Gateway 每 15 秒发送 `{"ticket":"","connection_id":"","consume":false}`。会话、用户、VM 所有权和服务均仍有效时 PHP 必须严格只返回 `{"active":true}`。任何拒绝、超时或 `active:false` 都必须断开两端。 这两个 helper 的配置加载都不把 owner/group/mode 当成启动条件;只要实际 service 用户能读取并且 JSON/字段有效,不会因权限位不符合某个预设值而拒绝启动。建议使用最小读取范围,绝不要 `chmod 777`。 ## 6. Web 与 WebSocket -所有普通页面和 JSON API 交给 PHP。仅将现有控制台 WebSocket 路径反代到本机 Go relay: +所有普通页面和 JSON API 交给 PHP。将控制台以及 Agent 主动连接的 WebSocket 路径反代到本机 Go Gateway: + +Gateway 配置样例使用 `http://127.0.0.1/...` 回调 PHP,因此必须另有一个明确绑定字面 loopback 的 HTTP vhost,把且只把两个固定回调路径交给同一 PHP front controller。该 vhost 不得执行 HTTP 到 HTTPS 跳转、域名规范化或代理到公开站点;否则 Gateway 会因禁止重定向而拒绝回调。下面的 PHP-FPM socket 路径应按发行版调整: + +```nginx +server { + listen 127.0.0.1:80; + server_name 127.0.0.1; + root /home/virt-cloud/web/public; + + location ~ ^/api/internal/cloud/(?:session-introspection|agent-auth)$ { + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root/index.php; + fastcgi_param SCRIPT_NAME /index.php; + fastcgi_pass unix:/run/php-fpm/www.sock; + } + + location / { + return 404; + } +} +``` + +公开 HTTPS vhost 必须为上述两个 PHP 内部路径配置精确 `return 404`,不能让普通 PHP catch-all 暴露它们。启动 Gateway 前,应从面板本机分别向两个 loopback URL 发送未签名 POST,确认请求直达 PHP 并被 HMAC 校验拒绝,而不是得到 `301`、`302` 或 PHP 路由 `404`。 ```nginx map $http_upgrade $cloud_connection_upgrade { @@ -217,20 +242,48 @@ location = /engine-console { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $cloud_connection_upgrade; - proxy_set_header Host $host; + # Preserve a non-default public port because Origin validation compares + # the complete authority, not only the hostname. + proxy_set_header Host $http_host; proxy_set_header Origin $http_origin; proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 3600s; } + +# These PHP callbacks and the Gateway dispatch endpoint are local-only. +location = /api/internal/cloud/session-introspection { return 404; } +location = /api/internal/cloud/agent-auth { return 404; } +location = /internal/agent/request { return 404; } + +location = /agent/connect { + proxy_pass http://127.0.0.1:8001/agent/connect; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $cloud_connection_upgrade; + proxy_set_header Sec-WebSocket-Protocol $http_sec_websocket_protocol; + proxy_set_header X-Virt-Agent-ID $http_x_virt_agent_id; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 3600s; +} + +location = /agent/console { + proxy_pass http://127.0.0.1:8001/agent/console; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $cloud_connection_upgrade; + proxy_set_header Sec-WebSocket-Protocol $http_sec_websocket_protocol; + proxy_set_header X-Virt-Agent-ID $http_x_virt_agent_id; + proxy_read_timeout 3600s; +} ``` -不要公开 PHP 内省接口、helper Unix socket、relay 的本机端口或 libvirt。Agent 9443 只允许面板服务器访问并必须使用有效 TLS/HMAC。 +不要公开 PHP 内省/认证接口、`/internal/agent/request`、helper Unix socket、Gateway 的本机端口或 libvirt。Agent 宿主机无需开放管理端口;出站网络只需能访问面板 HTTPS/WSS 和经校验的 HTTPS 镜像源。 控制台上线前必须验收:一次性票据只能消费一次、过期拒绝、登出/撤销后断开、跨用户/跨 VM 拒绝、Origin 校验、上游 Agent 断线及时关闭、浏览器不能注入 endpoint。Agent 未声明 console capability 时,PHP 必须显示不可用,不能回退旧实现。 ## 7. 管理页面配置顺序 -1. 在 `/admin/agents` 为每台宿主机保存 endpoint 与独立密钥,连接测试通过但保持 disabled。 +1. 在 `/admin/agents` 为每台宿主机保存独立密钥,在 Agent 配置中写入面板 HTTPS 地址、对应数字 ID 和同一密钥;出站连接测试通过但先保持调度 disabled。 2. 创建宿主机业务记录、地区、资源池成员关系、容量和安全余量。面板策略、reservation 与具体 IP 账本是调度事实源。 3. 配置预先存在的 bridge 和一个或多个公网 IPv4 CIDR;显式保留宿主机管理 IP。Agent 不通过扫描推测可售地址。 4. 导入云镜像和救援 ISO。远程下载只接受可公开解析的 HTTPS 直链,Agent 会校验初始地址与全部跳转;来源的发行方可信度由管理员确认。本地上传必须是独立、未加密、无 backing/data file 的 QCOW2。 @@ -243,7 +296,7 @@ location = /engine-console { 在非生产宿主机完成以下测试后才开放销售: -- `/v1/health`、`/v1/capabilities` 的 TLS/HMAC、时间窗、nonce 重放和错误遮罩。 +- 通过 WSS 转发 `/v1/health`、`/v1/capabilities` 的双层 HMAC、时间窗、nonce 重放和错误遮罩。 - 创建、启动、软/硬关机、软/硬重启、删除、重装、救援、快照、网络、防火墙、指标和控制台。 - Q35 UEFI 启动且 Secure Boot 关闭;不同宿主机固件路径通过 capabilities 选择。 - 云镜像启动脚本只执行一次,并覆盖成功、非零退出、重启及 Agent 断线。 @@ -260,10 +313,10 @@ location = /engine-console { journalctl -u virt-cloud-agent.service -n 200 --no-pager journalctl -u virt-cloud-worker.service -u virt-cloud-lifecycle.service \ -u 'virt-cloud-policy-worker@*' -n 200 --no-pager -journalctl -u virt-cloud-update.service -u virt-cloud-console-relay.service \ +journalctl -u virt-cloud-update.service -u virt-cloud-agent-gateway.service \ -n 200 --no-pager ``` -先区分 PHP 任务尚未派发、Agent 认证/网络失败、已派发但响应丢失、libvirt 明确失败和业务库提交失败。权限位与所有者不是应用层合规检查项;遇到启动错误应读取具体的“不可读、JSON 无效、TLS 缺失或字段无效”诊断,而不是盲目放宽权限。 +先区分 PHP 任务尚未派发、Gateway 未运行、Agent 出站 WSS 认证/网络失败、已派发但响应丢失、libvirt 明确失败和业务库提交失败。权限位与所有者不是应用层合规检查项;遇到启动错误应读取具体的“不可读、JSON 无效或字段无效”诊断,而不是盲目放宽权限。 任何破坏性任务在派发后超时,都保留原 `operation_id`、reservation、IP 和账单状态进行核对。不要换新标识重试、直接改库释放、自动退款或再次毁盘。 diff --git a/docs/INTEGRATION_PLAN.md b/docs/INTEGRATION_PLAN.md index d83b3455..f19fb668 100644 --- a/docs/INTEGRATION_PLAN.md +++ b/docs/INTEGRATION_PLAN.md @@ -6,16 +6,18 @@ ```text 浏览器 - ├─ HTTPS 页面/API ──> PHP 面板 ── HMAC + TLS ──> 目标宿主机 virt-agent - │ │ └─ qemu:///system - │ └─ MariaDB/Redis - └─ WSS /engine-console ──> 本机 Go console relay - └─ PHP 一次性票据内省 ──> 目标宿主机 Agent + ├─ HTTPS 页面/API ──> PHP 面板 ──> MariaDB/Redis + └─ WSS /engine-console ──> 本机 Go Agent Gateway + └─ 通过已有出站会话要求 Agent 反向建立控制台通道 -PHP workers ──> 持久 cloud job/operation_id ──> Agent 本地操作日志 ──> 对账 +宿主机 virt-agent ── WSS /agent/connect ──> 面板 Go Agent Gateway + ├─ qemu:///system ├─ 本机 HMAC API ──> PHP + └─ WSS /agent/console <─────────────┘ + +PHP workers ──> 持久 cloud job/operation_id ──> 本机 Gateway ──> Agent 本地操作日志 ──> 对账 ``` -外部 `/engine-console` 只是为现有用户界面保留的 WebSocket 路径名,不指向旧引擎。普通 HTTP 页面、管理 API、用户 API 和授权全部由 PHP 提供。 +外部 `/engine-console` 只是为现有用户界面保留的 WebSocket 路径名,不指向旧引擎。`/agent/connect` 和 `/agent/console` 也由同一个 Gateway 承载。普通 HTTP 页面、管理 API、用户 API 和授权全部由 PHP 提供;`/internal/agent/request` 只在字面 loopback 地址上接受 PHP 签名请求。 ## 责任边界 @@ -23,7 +25,7 @@ PHP workers ──> 持久 cloud job/operation_id ──> Agent 本地操作日 | --- | --- | --- | | 账户、会话、余额、订单、发票、支付、公钥所有权 | PHP | 业务库 | | 地区、套餐、容量策略、reservation、公网 IP 账本 | PHP | 业务库 | -| Agent endpoint、启用状态、配置版本、已测试版本和连接测试状态 | PHP | `cloud_host_agents` | +| Agent 数字身份、启用状态、配置版本、已测试版本和连接测试状态 | PHP | `cloud_host_agents`;不保存宿主机 endpoint | | 每宿主机 Agent HMAC 密钥 | PHP | 使用面板 `cloud_data_key` 的用途隔离密钥密封后保存 | | 云镜像元数据、启动脚本、修订和不可变 generation | PHP | `cloud_templates` | | 任务状态、操作者、`operation_id`、重试/核对决策 | PHP | 业务库 | @@ -35,7 +37,7 @@ PHP 不打开 libvirt 连接,不接收任意宿主机命令,也不把 Agent ## Agent API 与认证 -基础 API 使用 `/v1`;最低健康与能力接口为 `GET /v1/health`、`GET /v1/capabilities`,VM 资源位于 `/v1/vms`。可选功能必须由 capability 声明;PHP 在能力缺失时返回明确的“不支持”,不能猜测、静默降级或调用旧接口。 +基础 API 使用 `/v1`;最低健康与能力接口为 `GET /v1/health`、`GET /v1/capabilities`,VM 资源位于 `/v1/vms`。这些是长连接中的内层 HTTP 语义,Agent 不再对外监听 HTTP/TLS 端口。可选功能必须由 capability 声明;PHP 在能力缺失时返回明确的“不支持”,不能猜测、静默降级或调用旧接口。 每个请求携带: @@ -58,7 +60,7 @@ Unix 时间戳 原始 X-Virt-Actor 值 ``` -最后用该宿主机独立密钥计算十六进制 HMAC-SHA256。Agent 校验时间窗、正文摘要、常量时间签名和 nonce 防重放。面板只接受固定 IP HTTPS endpoint,私网、CGNAT 和开发部署也不允许明文;Agent 生产入口无 HTTP 监听分支。不能关闭 TLS 或认证来排错。 +最后用该宿主机独立密钥计算十六进制 HMAC-SHA256。Agent 校验时间窗、正文摘要、常量时间签名和 nonce 防重放。Agent 配置 `panel_url`、数字 `agent_id` 和独立 `agent_secret`,只连接面板 HTTPS/WSS 入口。Gateway 使用 PHP 内部认证接口校验连接;PHP 到 Gateway 另使用面板本机共享密钥签名,两层认证不可关闭。 ## 操作一致性 @@ -73,11 +75,11 @@ Unix 时间戳 ## 虚拟机、镜像与启动脚本 - 域 XML 使用结构化类型生成并验证输入,所有变更按 UUID 定位。部署每台宿主机时先查询 domain capabilities,把该机支持的 Q35/EFI loader 与 NVRAM template 写入本机 Agent 配置;创建请求不能传入任意固件路径。域 XML 明确关闭 `secure-boot` 和 enrolled keys,不在 PHP 中硬编码单一发行版 OVMF 路径。 -- Agent 配置使用 `revision`/`tested_revision` 就绪门闩。保存永远将 `tested_revision` 清零;只有同一 revision 的 `/v1/health` 与 `/v1/capabilities` 全部通过才记录为就绪。调度、Agent 客户端以及控制台路径均同时要求 `enabled=1`、`last_test_status=healthy` 且 `tested_revision=revision`;失败重测立即失效门闩。 +- Agent 配置使用 `revision`/`tested_revision` 就绪门闩。保存密钥或启用状态会将 `tested_revision` 清零;只有同一 revision 经已认证出站会话完成 `/v1/health` 与 `/v1/capabilities` 才记录为就绪。调度、Agent 客户端以及控制台路径均要求 `enabled=1`、`last_test_status=healthy` 且 `tested_revision=revision`。 - 云镜像由 PHP 保存目录元数据和启动脚本。创建任务快照镜像 generation、revision 和脚本正文后签名发送给目标 Agent;删除后重新导入的同名镜像不能继承旧脚本。 - 启动脚本仅用于新云镜像实例的首次安装,不用于普通重启、网络设置、重装、救援或用户 ISO。脚本限制 32 KiB UTF-8,拒绝 NUL,CRLF 规范化为 LF,空值表示禁用。 - Agent 将脚本以 root 私有文件写入 cloud-init seed,由一次性 runner 原子记录 started 状态。成功、失败和中断都不自动重放;VM 创建成功不等待脚本结束。 -- 镜像下载必须限制 HTTPS 来源、解析后的公网地址、跳转次数、大小、超时和摘要;校验 QCOW2 格式,拒绝 backing file、外部 data file、加密或 raw 内容。 +- 镜像下载始终使用独立 HTTPS 请求,WSS 只传递一次性下载 URL/token/摘要等任务元数据,不传镜像字节。Agent 限制来源、解析后的公网地址、全部跳转、大小、超时和摘要;校验 QCOW2 格式,拒绝 backing file、外部 data file、加密或 raw 内容。 ## 网络与账期 @@ -87,11 +89,11 @@ PHP 的容量策略、reservation 和具体 IP 账本是调度事实源。Agent ## 控制台 -浏览器只向面板侧 `virt-cloud-console-relay` 提交 PHP 签发的短时、一次性票据。relay 通过本机 PHP 内省接口验证当前会话与 VM 所有权,再请求目标 Agent 的短时 console capability 并中继 WebSocket。relay 不保存每宿主机长期密钥,不接受浏览器指定 endpoint,也不能把 VNC 端口直接暴露公网。 +浏览器只向面板侧 `virt-cloud-agent-gateway` 的 `/engine-console` 提交 PHP 签发的短时、一次性票据。Gateway 通过本机 PHP 内省接口验证当前会话与 VM 所有权,再在 Agent 控制长连接上发送 `console_open`。Agent 原子消费本机 console ticket、连接 loopback VNC,然后主动连回 `/agent/console`。Gateway 只在已连接会话内存中暂存该 Agent 握手密钥,用于每次普通请求或控制台打开前向 PHP 复核,不持久化密钥、不接受浏览器指定 Agent 或 endpoint,也不把 VNC 端口暴露公网。 -relay 固定以 `{"service":"console-relay"}` actor 签名 POST `/api/internal/cloud/session-introspection`。首次请求为 `{"ticket":"...","consume":true}`,PHP 原子消费后返回 `active`、`connection_id`、固定 IP 的 `wss://.../v1/console`、Agent 一次性票据及 RFC3339 过期时间。随后每 15 秒的 `consume:false` 请求带原票据与 `connection_id`,成功响应必须严格为 `{"active":true}`。不得把 Agent endpoint、密钥或 VNC 目标交给浏览器决定。 +Gateway 固定以 `{"service":"agent-gateway"}` actor 签名 POST `/api/internal/cloud/session-introspection`。首次请求为 `{"ticket":"...","consume":true}`,PHP 原子消费后返回 `active`、`connection_id`、数字 `agent_id`、Agent 一次性票据及 RFC3339 过期时间;不再返回宿主机 WebSocket URL。随后每 15 秒的 `consume:false` 请求带原票据与 `connection_id`,成功响应必须严格为 `{"active":true}`。 -当前部署只有在 relay 二进制、Agent console capability、一次性消费、会话撤销、Origin、超时和断线关闭测试全部通过后才能启用控制台;否则页面应显示不可用,不能回退到旧引擎。 +当前部署只有在 Gateway 二进制、Agent console capability、一次性消费、会话撤销、Origin、超时和断线关闭测试全部通过后才能启用控制台;否则页面应显示不可用,不能回退到旧引擎。 ## 在线更新 @@ -101,4 +103,4 @@ helper 只允许干净工作树 fast-forward,按持久检查点停止服务、 ## 上线边界 -生产前必须实测 Agent TLS/HMAC、防重放、断线后对账、宿主机重启、UEFI、cloud-init、快照/救援/重装、控制台撤销、流量跨 QEMU 重启、到期断网、宽限删除、镜像安全和并发购买。测试不得对未明确授权的宿主机做创建、写配置、改网络或修改虚拟机操作。 +生产前必须实测 Agent WSS/HMAC、防重放、断线后对账、宿主机重启、UEFI、cloud-init、快照/救援/重装、反向控制台撤销、流量跨 QEMU 重启、到期断网、宽限删除、HTTPS 镜像安全和并发购买。测试不得对未明确授权的宿主机做创建、写配置、改网络或修改虚拟机操作。 diff --git a/docs/MULTI_PUBLIC_IPV4_POOLS.md b/docs/MULTI_PUBLIC_IPV4_POOLS.md index fbc10987..f11577f5 100644 --- a/docs/MULTI_PUBLIC_IPV4_POOLS.md +++ b/docs/MULTI_PUBLIC_IPV4_POOLS.md @@ -30,7 +30,7 @@ ## 配置与幂等顺序 1. 按全新部署文档创建空数据库、运行 `php xcat cloudInit`,并启动 worker 与 lifecycle。 -2. 在 `/admin/agents` 添加宿主机 Agent endpoint 与独立 HMAC 密钥,TLS/HMAC 健康测试通过后设置地区和唯一 `default_network`;桥接网络由管理员预先在宿主机准备好。 +2. 在 `/admin/agents` 为宿主机保存独立 HMAC 密钥,在宿主机 Agent 中配置面板 HTTPS 地址、数字 Agent ID 和同一密钥;WSS/HMAC 健康测试通过后设置地区和唯一 `default_network`,桥接网络由管理员预先在宿主机准备好。 3. 在“资源调度”中加入资源池,填写 CPU、内存、磁盘、安全余量和最大实例数。面板用这些策略减去 reservation/IP 账本计算剩余资源。 4. 在“公网地址池”逐个添加实际 CIDR。创建使用稳定 `request_key`,它确定性映射到池标识;更新使用页面读取的 `expected_revision`,旧表单不能覆盖新版本。 5. 面板在事务中保存配置并生成版本,然后向目标 Agent 提交同版本。签名响应匹配后标记 `applied`,自动物化 CIDR 中未保留的具体 IP;响应超时则标记 `uncertain`。 diff --git a/docs/PRUNING_REPORT.md b/docs/PRUNING_REPORT.md index 1bdaaaf7..e180af8a 100644 --- a/docs/PRUNING_REPORT.md +++ b/docs/PRUNING_REPORT.md @@ -24,14 +24,14 @@ - PHP 是账户、财务、资源账本、Agent 注册、公共 HTTP 授权和操作协调的唯一权威。 - 每台宿主机的 `virt-agent` 只负责本机 `qemu:///system`、受管磁盘、网络、防火墙、快照、救援、指标和控制台后端。 -- 面板到 Agent 的每个请求使用该宿主机独立 HMAC 密钥并通过 TLS。Agent 不连接面板数据库,浏览器不能直接调用 Agent。 -- 面板侧 Go helper 只负责受限在线更新和控制台 WebSocket relay;它们不承担账户或 VM 调度逻辑。 +- 面板到 Agent 的每个请求使用该宿主机独立 HMAC 密钥,并在 Agent 主动建立的 WSS 会话上转发。Agent 不连接面板数据库,浏览器不能直接调用 Agent。 +- 面板侧 Go helper 负责受限在线更新;Agent Gateway 负责 Agent 主动 WSS 会话、PHP 请求转发和反向控制台,不承担账户或 VM 调度逻辑。 ## 数据库边界 `web/sql/ss.sql`、`web/sql/user_log.sql` 和 `php xcat cloudInit` 只用于全新空数据库。安装过程不读取、转换或删除旧生产库;旧库应作为离线只读备份单独保存。 -Agent endpoint、启用状态、版本和测试结果保存在 PHP 业务库;每宿主机 HMAC 密钥使用面板 `cloud_data_key` 的用途隔离密钥密封。云镜像及首次启动脚本同样由 PHP 保存,任务派发时快照 revision 与 generation。Agent 的操作日志只保存本机操作边界和结果,不取代业务库。旧格式密文不受理;只能创建全新数据。 +Agent 数字身份、启用状态、版本和测试结果保存在 PHP 业务库,不保存宿主机 endpoint;每宿主机 HMAC 密钥使用面板 `cloud_data_key` 的用途隔离密钥密封。云镜像及首次启动脚本同样由 PHP 保存,任务派发时快照 revision 与 generation。Agent 的操作日志只保存本机操作边界和结果,不取代业务库。旧格式密文不受理;只能创建全新数据。 配置文件必须可读且内容有效,但应用不得用 owner/group/mode 审核阻止启动。推荐最小权限只是部署加固,不是兼容条件;绝不能因此增加弱密钥、明文传输或公开端口的回退。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 7e1b2763..6ca7c27d 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -24,7 +24,7 @@ go test ./... go test -race ./... ``` -Go 测试至少覆盖:配置内容校验但不检查 owner/mode、TLS 证书/私钥强制项、未知的旧明文传输字段被拒绝、HMAC 正文摘要、完整时间窗(含未来偏差与到期边界)、nonce 防重放、不同请求复用 `operation_id`、按 VM 串行、派发前/后故障注入、仅按 inflight 索引重启恢复、超过 16 MiB 的长期 journal 重启、nonce 到期索引清理、operation tombstone 保留、五类 expected-state/transition marker 只读核对、换网 source 反例、libvirt typed error、UEFI XML 和启动脚本一次性 runner。不带 tag 的 portable 测试只能验证缺失 adapter 时的明确失败;生产交付必须通过 `-tags libvirt` 的编译和测试。PHP 契约测试还必须证明所有固定 IP 网段都只接受 HTTPS,任何 `http://` endpoint 都被拒绝。 +Go 测试至少覆盖:配置内容校验但不检查 owner/mode、HTTPS `panel_url`、数字 Agent ID、旧 listener/TLS 字段被拒绝、WSS 子协议与认证、严格控制帧、HMAC 正文摘要、完整时间窗(含未来偏差与到期边界)、nonce 防重放、不同请求复用 `operation_id`、按 VM 串行、派发前/后故障注入、仅按 inflight 索引重启恢复、超过 16 MiB 的长期 journal 重启、nonce 到期索引清理、operation tombstone 保留、五类 expected-state/transition marker 只读核对、换网 source 反例、libvirt typed error、UEFI XML、启动脚本一次性 runner 和反向控制台。不带 tag 的 portable 测试只能验证缺失 adapter 时的明确失败;生产交付必须通过 `-tags libvirt` 的编译和测试。PHP 契约测试还必须证明 PHP→Gateway 只使用字面 loopback 固定路径,不再存储或请求宿主机 endpoint。 PHP: @@ -34,7 +34,7 @@ find app config public tests -name '*.php' -print0 | xargs -0 -n1 php -l for test_file in tests/*Test.php; do php "$test_file" || exit $?; done ``` -PHP 测试至少覆盖:Agent endpoint 防 SSRF、每宿主机密钥加密、签名与 Go golden vector 一致、管理员权限、普通用户路由隔离、模板脚本读取/写入权限、32 KiB/UTF-8/NUL 边界、revision/generation、任务快照、账单与资源 reservation、`uncertain` 不自动重放/退款。 +PHP 测试至少覆盖:Agent Gateway loopback 约束、Gateway 外层 HMAC、每宿主机内层密钥加密与签名 golden vector、Agent 握手认证、管理员权限、普通用户路由隔离、模板脚本读取/写入权限、32 KiB/UTF-8/NUL 边界、revision/generation、持久任务快照、账单与资源 reservation、`uncertain` 不自动重放/退款。 浏览器脚本与部署文件: @@ -57,8 +57,13 @@ systemd-analyze verify /home/virt-cloud/deploy/*.service cd /home/virt-cloud test -z "$(git ls-files '*.py' '*.pyi' '*requirements*.txt' 'pyproject.toml')" test ! -e deploy/virt-cloud-engine.service +test ! -e deploy/virt-cloud-console-relay.service +test ! -e deploy/virt-cloud-console-relay.json.example +test ! -e gateway/cmd/virt-cloud-console-relay ! rg -n 'virt-cloud-engine\.service|/home/virt-cloud/engine|CLOUD_ENGINE_CONFIG' \ deploy README.md docs --glob '!VALIDATION.md' +! rg -n 'virt-cloud-console-relay|cloud_console_relay_secret|agent_websocket_url' \ + deploy gateway web/app web/config README.md docs --glob '!VALIDATION.md' ``` 上述 `requirements*.txt` 只是对已退役文件类型的残留检查,不是当前部署依赖;产品运行路径中不存在 Python 或 pip 安装步骤。 @@ -74,7 +79,7 @@ test ! -e deploy/virt-cloud-engine.service - 缺少任一签名头、摘要不符、过期/未来时间、nonce 重放、actor 字段未知或签名错误全部拒绝。 - 认证错误不回显密钥、canonical 内容、内部路径或 libvirt 原始敏感信息。 -每个 Agent 的 `/v1/health` 和 `/v1/capabilities` 都要从面板 `/admin/agents` 测试;一个宿主机失败不得污染另一个宿主机的配置或状态。 +每个 Agent 的 `/v1/health` 和 `/v1/capabilities` 都要从面板 `/admin/agents` 经当前已认证 WSS 会话测试;一个宿主机失败不得污染另一个宿主机的配置或状态。 ## 3. 操作与故障注入 @@ -119,7 +124,7 @@ virsh -c qemu:///system pool-list --all 管理员页面保持 venix,用户/登录页保持原虚拟机面板样式。逐项验收: - 注册、登录、密码重置、充值、支付、公告、工单、管理员账户。 -- `/admin/agents` 新增/修改/禁用/连接测试;普通用户不可见且 API 返回拒绝。 +- `/admin/agents` 新增/修改密钥/禁用/出站连接测试;普通用户不可见且 API 返回拒绝。 - 地区、资源池、套餐、库存、多公网池、镜像、救援 ISO、系统日志和异常任务。 - 购买只扣款一次并只创建一台 VM;明确创建失败才退款,未知结果不退款。 - 控制台短票据一次消费、到期、登出撤销、跨用户拒绝和上游断线。 @@ -167,3 +172,21 @@ authorized host mutation test: 未执行;用户明确禁止修改指定宿主 browser/payment/SMTP test: 自动化 PHP/Node 契约测试通过;真实支付、SMTP 和浏览器人工验收未执行 known skips or risks: 生产 Linux 上的 systemd-analyze、MariaDB 全新建库、真实 libvirt 破坏性生命周期与外部服务联调仍须在获授权测试环境验收;macOS 静态 libvirt 链接仅有重复库警告 ``` + +### 2026-10-16 出站 WSS Agent Gateway 提交前验收 + +```text +commit: 本记录所在 go 分支提交(基线 97a17ed) +Go version / libvirt version / PHP version / MariaDB version: Go 1.27.1 / libvirt 12.8.0 / PHP 8.5.11 / MariaDB 未在本机运行 +go vet: Agent 默认、libvirt、libvirt+dlopen 与 Gateway 均通过 +go test: Agent 默认、libvirt、libvirt+dlopen 与 Gateway 均通过;go mod verify 与 tidy -diff 通过;反向控制台目标测试分别连续运行 50 次和 30 次通过 +go test -race: Agent 默认、libvirt、libvirt+dlopen 与 Gateway 均通过 +Linux AMD64 build: linux/amd64 CGo + libvirt_dlopen 交叉编译通过;提交后按实际提交 SHA 重建发布包 +PHP tests: SourceSyntax 检查 391 个 PHP 文件通过,115 个测试文件通过;现有测试依赖仅输出已知弃用/警告信息,退出码为 0 +Node tests: 93 个 JavaScript 文件语法检查通过,4 个行为测试通过;3 个部署 JSON 示例有效 +systemd-analyze: 未执行;验收机为 macOS,不提供 systemd-analyze +read-only host capability check: 本轮未完成;当前 Mac 的 SSH 别名 103 解析为 0.0.0.103,网络不可达 +authorized host mutation test: 未执行;用户明确禁止修改指定宿主机的任何配置 +browser/payment/SMTP test: 自动化 PHP/Node 契约测试通过;真实支付、SMTP 和浏览器人工验收未执行 +known skips or risks: 生产 Linux 上的 systemd-analyze、MariaDB 全新建库、真实 libvirt 生命周期与外部服务联调仍须在获授权测试环境验收;macOS libvirt 链接测试仅有重复库警告 +``` diff --git a/gateway/README.md b/gateway/README.md index 881fd3cd..e782c1a3 100644 --- a/gateway/README.md +++ b/gateway/README.md @@ -1,11 +1,12 @@ # Virt Cloud panel-side Go helpers -`gateway` is an independent Go module containing the two small services that -cannot safely run inside PHP-FPM: +`gateway` is an independent Go module containing the two services that cannot +safely run inside PHP-FPM: - `virt-cloud-update` is the privileged, Unix-socket-only update coordinator. -- `virt-cloud-console-relay` is the loopback-only WebSocket bridge between the - existing browser URL and a host Agent. +- `virt-cloud-agent-gateway` keeps Agent-initiated control WebSockets, accepts + authenticated dispatches from local PHP, and pairs reverse console streams + with the existing browser console URL. Neither service is a compatibility layer for the retired Python engine. @@ -14,7 +15,7 @@ Neither service is a compatibility layer for the retired Python engine. ```bash cd gateway go test ./... -go build ./cmd/virt-cloud-update ./cmd/virt-cloud-console-relay +go build ./cmd/virt-cloud-update ./cmd/virt-cloud-agent-gateway ``` Both commands require an absolute `--config` path. The JSON decoders reject @@ -24,7 +25,7 @@ unreadable, malformed, or unsafe configuration still fails startup. ## Authentication -The update API and relay-to-PHP requests use the shared `X-Virt-*` contract: +The update API and gateway-to-PHP requests use the shared `X-Virt-*` contract: ```text METHOD\n @@ -37,8 +38,9 @@ base64url-json-actor The signature is lowercase HMAC-SHA256. The update helper persists nonce expiry state, accepts a 60-second clock window, and requires a managing super -administrator actor. The relay signs outbound requests with -`{"service":"console-relay"}`. +administrator actor. The Agent gateway signs PHP authentication and console +introspection requests with `{"service":"agent-gateway"}` and accepts local +PHP dispatches only with `{"service":"php-panel"}`. ## Update API @@ -100,11 +102,29 @@ exposed as terminal `blocked`, `checkpoint` as `phase`, the applied commit as `new_commit`, and the pre-update commit as an informational `rollback_ref`. That reference never triggers an automatic rollback. -## Console relay contract +## Agent gateway contract -The relay listens only on `127.0.0.1:8001` at `/engine-console`. It requires a -same-origin `Origin`, the trusted proxy's single `X-Forwarded-Proto`, and -exactly one browser subprotocol: +The gateway listens only on `127.0.0.1:8001`. The public reverse proxy exposes +`GET /agent/connect`, `GET /agent/console`, and `GET /engine-console`; it must +not expose `POST /internal/agent/request`. + +An Agent opens `/agent/connect` with `X-Virt-Agent-ID`, `Authorization: Bearer +`, and the exact `virt-agent-control.v1` subprotocol. The +gateway validates those credentials through the fixed loopback PHP endpoint +`/api/internal/cloud/agent-auth`. It retains the presented secret only in that +live session's memory and revalidates it before every ordinary dispatch or +console open; rejection unregisters and closes the session. PHP dispatches one +signed request envelope to the loopback-only `/internal/agent/request`; the +gateway sends it to the live Agent and returns the Agent's exact HTTP status and +JSON body. Control frames are bounded JSON text frames: + +```json +{"type":"request","id":"...","method":"POST","request_target":"/v1/...","body":"{...}","headers":{"X-Virt-Actor":"..."}} +{"type":"response","id":"...","status":200,"body":"{...}"} +``` + +For a browser console, `/engine-console` requires a same-origin `Origin`, the +trusted proxy's single `X-Forwarded-Proto`, and exactly one browser subprotocol: ```text cloud-ticket. @@ -116,27 +136,33 @@ Before accepting the browser WebSocket it signs this PHP introspection body: {"ticket":"...","consume":true} ``` -PHP must atomically consume that ticket and return exactly: +PHP must atomically consume that ticket and return: ```json { "active": true, "connection_id": "", - "agent_websocket_url": "wss://[:port]/v1/console", + "agent_id": "", "agent_ticket": "", "agent_expires_at": "2026-10-15T12:00:00Z" } ``` -The relay accepts only WSS, a literal non-loopback/non-link-local IP, the exact -`/v1/console` path, verified TLS, no redirect, credentials, query, fragment, or -proxy. It connects using only: +The gateway sends the authenticated control session a one-use `console_open` +message containing PHP's Agent ticket and a gateway-only session token: + +```json +{"type":"console_open","id":"...","ticket":"...","session_id":"...","session_token":"..."} +``` + +The Agent consumes the ticket locally, then connects outward to +`/agent/console` with its ID and the returned one-use subprotocol: ```text -cloud-agent-ticket. +virt-agent-console. ``` -Only binary WebSocket messages are copied. Every 15 seconds the relay signs a +Only binary WebSocket messages are copied. Every 15 seconds the gateway signs a second introspection request: ```json @@ -145,10 +171,17 @@ second introspection request: PHP returns exactly `{"active":true}` while the browser session, user, VM ownership, and service remain valid. A denial, malformed response, timeout, or -`active:false` immediately tears down both sides. The relay never accepts an -Agent endpoint from the browser and never stores a host Agent secret. +`active:false` immediately tears down both sides. The gateway never accepts an +Agent endpoint from the browser and never persists a host Agent secret. PHP implements the browser-ticket and connection lifecycle in -`CloudConsoleTicketService`; the Agent issues and consumes the one-use -`/v1/console` ticket. If any validation or periodic session recheck fails, the -relay closes both WebSocket sides. +`CloudConsoleTicketService`. The Agent never listens for a console connection; +it opens the short-lived reverse stream only after `console_open`. If any +validation or periodic session recheck fails, the gateway closes both +WebSocket sides. + +Cloud-image bytes never cross the control WebSocket. PHP persists a staged +object and issues a short-lived HTTPS URL plus bearer token; the control request +carries only that URL, token, and optional digest, and the Agent downloads the +image over HTTPS while enforcing the response length, type-specific size limit, +and computed SHA-256. diff --git a/gateway/agent/auth.go b/gateway/agent/auth.go new file mode 100644 index 00000000..6c572954 --- /dev/null +++ b/gateway/agent/auth.go @@ -0,0 +1,121 @@ +package agent + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "git.luoli.live/root/virt/gateway/internal/protocol" +) + +const maxPanelResponseBytes = 64 << 10 + +type ConnectionAuthenticator interface { + Authenticate(context.Context, string, string) error +} + +type HTTPConnectionAuthenticator struct { + endpoint string + secret []byte + client *http.Client + now func() time.Time +} + +func NewHTTPConnectionAuthenticator(endpoint string, secret []byte, timeout time.Duration) (*HTTPConnectionAuthenticator, error) { + if err := validateLoopbackPanelURL(endpoint, "/api/internal/cloud/agent-auth"); err != nil { + return nil, err + } + if len(secret) < 32 || timeout <= 0 || timeout > 30*time.Second { + return nil, errors.New("invalid Agent authentication client settings") + } + dialer := &net.Dialer{Timeout: timeout, KeepAlive: 30 * time.Second} + transport := &http.Transport{ + Proxy: nil, DialContext: dialer.DialContext, ForceAttemptHTTP2: false, + TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, TLSHandshakeTimeout: timeout, + } + return &HTTPConnectionAuthenticator{ + endpoint: endpoint, + secret: append([]byte(nil), secret...), + client: &http.Client{Transport: transport, Timeout: timeout, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }}, + now: time.Now, + }, nil +} + +func (a *HTTPConnectionAuthenticator) Authenticate(ctx context.Context, agentID, agentSecret string) error { + body, err := json.Marshal(struct { + AgentID string `json:"agent_id"` + AgentSecret string `json:"agent_secret"` + }{AgentID: agentID, AgentSecret: agentSecret}) + if err != nil { + return errors.New("encode Agent authentication request") + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, a.endpoint, bytes.NewReader(body)) + if err != nil { + return errors.New("create Agent authentication request") + } + request.Header.Set("Accept", "application/json") + request.Header.Set("Content-Type", "application/json") + nonce, err := protocol.RandomNonce() + if err != nil { + return errors.New("create Agent authentication nonce") + } + if err := protocol.Sign(request, body, protocol.Actor{Service: "agent-gateway"}, a.secret, a.now(), nonce); err != nil { + return errors.New("sign Agent authentication request") + } + response, err := a.client.Do(request) + if err != nil { + return errors.New("panel Agent authentication unavailable") + } + defer response.Body.Close() + raw, err := io.ReadAll(io.LimitReader(response.Body, maxPanelResponseBytes+1)) + if err != nil || len(raw) > maxPanelResponseBytes || response.StatusCode != http.StatusOK { + return errors.New("panel rejected Agent authentication") + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var result struct { + Active bool `json:"active"` + AgentID string `json:"agent_id"` + } + if err := decoder.Decode(&result); err != nil { + return errors.New("panel Agent authentication response is invalid") + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) || !result.Active || result.AgentID != agentID { + return errors.New("panel rejected Agent authentication") + } + return nil +} + +func validateLoopbackPanelURL(value, requiredPath string) error { + if value == "" || len(value) > 2048 || strings.ContainsAny(value, "\x00\r\n\\") { + return errors.New("panel internal URL is invalid") + } + parsed, err := url.Parse(value) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.User != nil || + parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" || parsed.Path != requiredPath || parsed.RawPath != "" || + strings.HasSuffix(parsed.Host, ":") { + return errors.New("panel internal URL must use its fixed path") + } + ip := net.ParseIP(parsed.Hostname()) + if ip == nil || !ip.IsLoopback() { + return errors.New("panel internal URL must use a literal loopback address") + } + if port := parsed.Port(); port != "" { + number, err := strconv.Atoi(port) + if err != nil || number < 1 || number > 65535 { + return errors.New("panel internal URL has an invalid port") + } + } + return nil +} diff --git a/gateway/agent/auth_test.go b/gateway/agent/auth_test.go new file mode 100644 index 00000000..536e78f8 --- /dev/null +++ b/gateway/agent/auth_test.go @@ -0,0 +1,91 @@ +package agent + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "git.luoli.live/root/virt/gateway/internal/protocol" +) + +type authNonceStore struct { + mu sync.Mutex + m map[string]time.Time +} + +func (s *authNonceStore) UseNonce(nonce string, now, expiresAt time.Time) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if expiry := s.m[nonce]; expiry.After(now) { + return false, nil + } + s.m[nonce] = expiresAt + return true, nil +} + +func TestHTTPConnectionAuthenticatorSignsSecretForFixedPHPRoute(t *testing.T) { + secret := []byte(strings.Repeat("g", 32)) + verifier, _ := protocol.NewAuthenticator(secret, time.Minute, 2*time.Minute, + &authNonceStore{m: make(map[string]time.Time)}) + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + body, _ := io.ReadAll(request.Body) + actor, err := verifier.Authenticate(request, body) + var input struct { + AgentID string `json:"agent_id"` + AgentSecret string `json:"agent_secret"` + } + if err != nil || actor.Service != "agent-gateway" || json.Unmarshal(body, &input) != nil || + input.AgentID != "8" || input.AgentSecret != strings.Repeat("a", 32) { + http.Error(response, "forbidden", http.StatusForbidden) + return + } + response.Header().Set("Content-Type", "application/json") + _, _ = response.Write([]byte(`{"active":true,"agent_id":"8"}`)) + })) + defer server.Close() + client, err := NewHTTPConnectionAuthenticator(server.URL+"/api/internal/cloud/agent-auth", + secret, time.Second) + if err != nil { + t.Fatal(err) + } + if err := client.Authenticate(t.Context(), "8", strings.Repeat("a", 32)); err != nil { + t.Fatal(err) + } +} + +func TestHTTPConnectionAuthenticatorRejectsMismatchedPanelIdentity(t *testing.T) { + secret := []byte(strings.Repeat("g", 32)) + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.Header().Set("Content-Type", "application/json") + _, _ = response.Write([]byte(`{"active":true,"agent_id":"9"}`)) + })) + defer server.Close() + client, err := NewHTTPConnectionAuthenticator(server.URL+"/api/internal/cloud/agent-auth", + secret, time.Second) + if err != nil { + t.Fatal(err) + } + if err := client.Authenticate(t.Context(), "8", strings.Repeat("a", 32)); err == nil { + t.Fatal("mismatched panel Agent identity was accepted") + } +} + +func TestHTTPConnectionAuthenticatorRejectsExplicitEmptyPort(t *testing.T) { + for _, endpoint := range []string{ + "http://127.0.0.1:/api/internal/cloud/agent-auth", + "http://127.0.0.1:0/api/internal/cloud/agent-auth", + "http://127.0.0.1:65536/api/internal/cloud/agent-auth", + "http://127.0.0.1/api/internal/cloud/agent-auth?", + } { + if _, err := NewHTTPConnectionAuthenticator( + endpoint, []byte(strings.Repeat("g", 32)), time.Second, + ); err == nil { + t.Fatalf("invalid Agent authentication endpoint accepted: %s", endpoint) + } + } +} diff --git a/gateway/agent/nonce.go b/gateway/agent/nonce.go new file mode 100644 index 00000000..0e58a057 --- /dev/null +++ b/gateway/agent/nonce.go @@ -0,0 +1,30 @@ +package agent + +import ( + "sync" + "time" +) + +type memoryNonceStore struct { + mu sync.Mutex + values map[string]time.Time +} + +func newMemoryNonceStore() *memoryNonceStore { + return &memoryNonceStore{values: make(map[string]time.Time)} +} + +func (s *memoryNonceStore) UseNonce(nonce string, now, expiresAt time.Time) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for value, expiry := range s.values { + if !expiry.After(now) { + delete(s.values, value) + } + } + if expiry, exists := s.values[nonce]; exists && expiry.After(now) { + return false, nil + } + s.values[nonce] = expiresAt + return true, nil +} diff --git a/gateway/agent/registry.go b/gateway/agent/registry.go new file mode 100644 index 00000000..51c69918 --- /dev/null +++ b/gateway/agent/registry.go @@ -0,0 +1,684 @@ +package agent + +import ( + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "regexp" + "strings" + "sync" + "time" + + "git.luoli.live/root/virt/gateway/internal/protocol" + "github.com/coder/websocket" +) + +const ( + ControlPath = "/agent/connect" + ConsolePath = "/agent/console" + InternalRequestPath = "/internal/agent/request" + ControlSubprotocol = "virt-agent-control.v1" + ConsoleSubprotocolPrefix = "virt-agent-console." + maxControlFrame = 4 << 20 + maxInternalRequest = 4 << 20 + maxAgentRequestBody = 1 << 20 + maxConsoleFrame = 8 << 20 + maxInFlightPerAgent = 32 + maxPendingConsoles = 64 +) + +var ( + ErrAgentBusy = errors.New("Agent request capacity is exhausted") + ErrConsoleBusy = errors.New("reverse console capacity is exhausted") + ErrGatewayDraining = errors.New("Agent gateway is draining") + ErrNotDispatched = errors.New("request was not dispatched to the Agent") +) + +var ( + agentIDPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`) + requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`) + tokenPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{32,256}$`) +) + +type wireRequest struct { + Type string `json:"type"` + ID string `json:"id"` + Method string `json:"method"` + RequestTarget string `json:"request_target"` + Body string `json:"body"` + Headers map[string]string `json:"headers"` +} + +type wireResponse struct { + Type string `json:"type"` + ID string `json:"id"` + Status int `json:"status"` + Body string `json:"body"` +} + +type consoleOpen struct { + Type string `json:"type"` + ID string `json:"id"` + Ticket string `json:"ticket"` + SessionID string `json:"session_id"` + SessionToken string `json:"session_token"` +} + +type dispatchResult struct { + status int + body []byte +} + +type pendingConsole struct { + agentID string + ready chan *websocket.Conn + done <-chan struct{} +} + +type session struct { + agentID string + secret string + conn *websocket.Conn + writeMu sync.Mutex + mu sync.Mutex + pending map[string]chan dispatchResult + inFlight chan struct{} + done chan struct{} + once sync.Once +} + +type Registry struct { + authenticator ConnectionAuthenticator + requestAuth *protocol.Authenticator + requestTimeout time.Duration + consoleTimeout time.Duration + mu sync.RWMutex + sessions map[string]*session + consoles map[string]pendingConsole + lifecycleMu sync.Mutex + draining bool + active sync.WaitGroup +} + +func NewRegistry(authenticator ConnectionAuthenticator, sharedSecret []byte, requestTimeout, consoleTimeout time.Duration) (*Registry, error) { + if authenticator == nil || len(sharedSecret) < 32 || requestTimeout <= 0 || requestTimeout > 15*time.Minute || + consoleTimeout <= 0 || consoleTimeout > 30*time.Second { + return nil, errors.New("invalid Agent registry settings") + } + requestAuth, err := protocol.NewAuthenticator(sharedSecret, time.Minute, 2*time.Minute, newMemoryNonceStore()) + if err != nil { + return nil, err + } + return &Registry{ + authenticator: authenticator, requestAuth: requestAuth, + requestTimeout: requestTimeout, consoleTimeout: consoleTimeout, + sessions: make(map[string]*session), consoles: make(map[string]pendingConsole), + }, nil +} + +func (r *Registry) ServeControl(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet || request.URL.Path != ControlPath || request.URL.RawQuery != "" { + http.Error(response, "not found", http.StatusNotFound) + return + } + if !headerExactly(request.Header.Values("Sec-WebSocket-Protocol"), ControlSubprotocol) { + http.Error(response, "Agent control subprotocol required", http.StatusBadRequest) + return + } + if r.isDraining() { + http.Error(response, "Agent gateway is draining", http.StatusServiceUnavailable) + return + } + agentID, secret, ok := agentCredentials(request) + if !ok { + http.Error(response, "Agent authentication required", http.StatusUnauthorized) + return + } + ctx, cancel := context.WithTimeout(request.Context(), r.consoleTimeout) + err := r.authenticator.Authenticate(ctx, agentID, secret) + cancel() + if err != nil { + http.Error(response, "Agent authentication failed", http.StatusUnauthorized) + return + } + conn, err := websocket.Accept(response, request, &websocket.AcceptOptions{ + Subprotocols: []string{ControlSubprotocol}, CompressionMode: websocket.CompressionDisabled, + }) + if err != nil { + return + } + if conn.Subprotocol() != ControlSubprotocol { + _ = conn.Close(websocket.StatusPolicyViolation, "control subprotocol required") + return + } + conn.SetReadLimit(maxControlFrame) + current := &session{ + agentID: agentID, secret: secret, conn: conn, pending: make(map[string]chan dispatchResult), + inFlight: make(chan struct{}, maxInFlightPerAgent), done: make(chan struct{}), + } + if !r.register(current) { + _ = conn.Close(websocket.StatusPolicyViolation, "Agent is already connected") + return + } + defer func() { + r.unregister(current) + current.close() + _ = conn.CloseNow() + }() + current.readLoop(request.Context()) +} + +func (r *Registry) ServeInternalRequest(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost || request.URL.Path != InternalRequestPath || request.URL.RawQuery != "" { + writeInternalError(response, http.StatusNotFound, "gateway_route_not_found", "Agent gateway route not found") + return + } + if !remoteIsLoopback(request.RemoteAddr) { + writeInternalError(response, http.StatusForbidden, "gateway_forbidden", "Agent gateway request is forbidden") + return + } + body, err := readBounded(response, request, maxInternalRequest) + if err != nil { + writeInternalError(response, http.StatusBadRequest, "gateway_request_invalid", "Agent gateway request is invalid") + return + } + actor, err := r.requestAuth.Authenticate(request, body) + if err != nil || !exactServiceActor(actor, "php-panel") { + writeInternalError(response, http.StatusForbidden, "gateway_forbidden", "Agent gateway request is forbidden") + return + } + var envelope struct { + AgentID string `json:"agent_id"` + wireRequest + } + if err := decodeStrict(body, &envelope); err != nil || !agentIDPattern.MatchString(envelope.AgentID) || validateWireRequest(envelope.wireRequest) != nil { + writeInternalError(response, http.StatusBadRequest, "gateway_request_invalid", "Agent gateway request is invalid") + return + } + ctx, cancel := context.WithTimeout(request.Context(), r.requestTimeout) + defer cancel() + result, err := r.dispatch(ctx, envelope.AgentID, envelope.wireRequest) + if err != nil { + status := http.StatusBadGateway + code := "agent_disconnected" + if errors.Is(err, context.DeadlineExceeded) { + status, code = http.StatusGatewayTimeout, "agent_timeout" + } else if errors.Is(err, ErrAgentBusy) { + status, code = http.StatusServiceUnavailable, "agent_busy" + } else if errors.Is(err, ErrGatewayDraining) { + status, code = http.StatusServiceUnavailable, "gateway_draining" + } + errorPayload := map[string]any{ + "code": code, "message": "host Agent is unavailable", "retryable": true, + } + if errors.Is(err, ErrNotDispatched) { + errorPayload["dispatch_state"] = "not_dispatched" + } + writeJSON(response, status, map[string]any{"error": errorPayload}) + return + } + response.Header().Set("Content-Type", "application/json; charset=utf-8") + response.Header().Set("Cache-Control", "no-store") + response.Header().Set("X-Content-Type-Options", "nosniff") + response.WriteHeader(result.status) + _, _ = response.Write(result.body) +} + +func (r *Registry) OpenConsole(ctx context.Context, agentID, ticket, sessionID string) (*websocket.Conn, error) { + if !agentIDPattern.MatchString(agentID) || !tokenPattern.MatchString(ticket) || !tokenPattern.MatchString(sessionID) { + return nil, errors.New("invalid reverse console request") + } + if !r.beginActivity() { + return nil, ErrGatewayDraining + } + defer r.active.Done() + current := r.get(agentID) + if current == nil { + return nil, errors.New("Agent is not connected") + } + if err := r.revalidate(ctx, current); err != nil { + return nil, err + } + streamID, err := randomToken(16) + if err != nil { + return nil, err + } + sessionToken, err := randomToken(32) + if err != nil { + return nil, err + } + wait, cancel := context.WithTimeout(ctx, r.consoleTimeout) + defer cancel() + acknowledgement, releasePending, err := current.reservePending(streamID) + if err != nil { + return nil, err + } + defer releasePending() + ready := make(chan *websocket.Conn) + r.mu.Lock() + if len(r.consoles) >= maxPendingConsoles { + r.mu.Unlock() + return nil, ErrConsoleBusy + } + if _, exists := r.consoles[sessionToken]; exists { + r.mu.Unlock() + return nil, errors.New("reverse console token collision") + } + r.consoles[sessionToken] = pendingConsole{agentID: agentID, ready: ready, done: wait.Done()} + r.mu.Unlock() + defer func() { + r.mu.Lock() + delete(r.consoles, sessionToken) + r.mu.Unlock() + }() + if err := current.write(wait, consoleOpen{ + Type: "console_open", ID: streamID, Ticket: ticket, + SessionID: sessionID, SessionToken: sessionToken, + }); err != nil { + return nil, err + } + var conn *websocket.Conn + acknowledged := false + for !acknowledged || conn == nil { + select { + case result := <-acknowledgement: + if result.status < http.StatusOK || result.status >= http.StatusMultipleChoices { + if conn != nil { + _ = conn.CloseNow() + } + return nil, fmt.Errorf("Agent rejected reverse console with status %d", result.status) + } + acknowledged = true + case candidate := <-ready: + if candidate == nil { + return nil, errors.New("reverse console connection failed") + } + conn = candidate + case <-current.done: + if conn != nil { + _ = conn.CloseNow() + } + return nil, errors.New("Agent disconnected while opening console") + case <-wait.Done(): + if conn != nil { + _ = conn.CloseNow() + } + return nil, wait.Err() + } + } + return conn, nil +} + +func (r *Registry) ServeAgentConsole(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet || request.URL.Path != ConsolePath || request.URL.RawQuery != "" { + http.Error(response, "not found", http.StatusNotFound) + return + } + agentValues := request.Header.Values("X-Virt-Agent-ID") + if len(agentValues) != 1 || !agentIDPattern.MatchString(agentValues[0]) { + http.Error(response, "invalid Agent identity", http.StatusUnauthorized) + return + } + protocolValue, token, ok := consoleProtocol(request.Header.Values("Sec-WebSocket-Protocol")) + if !ok { + http.Error(response, "reverse console token required", http.StatusUnauthorized) + return + } + r.mu.Lock() + pending, exists := r.consoles[token] + if exists && pending.agentID == agentValues[0] { + delete(r.consoles, token) + } else { + exists = false + } + r.mu.Unlock() + if !exists { + http.Error(response, "reverse console token is invalid or expired", http.StatusUnauthorized) + return + } + conn, err := websocket.Accept(response, request, &websocket.AcceptOptions{ + Subprotocols: []string{protocolValue}, CompressionMode: websocket.CompressionDisabled, + }) + if err != nil { + select { + case pending.ready <- nil: + case <-pending.done: + case <-request.Context().Done(): + } + return + } + conn.SetReadLimit(maxConsoleFrame) + select { + case pending.ready <- conn: + // Ownership transfers to OpenConsole and then to the browser relay. + // Returning is required: a hijacked request context is cancelled only + // after ServeHTTP returns, so waiting on it here would leak forever. + return + case <-pending.done: + _ = conn.CloseNow() + case <-request.Context().Done(): + _ = conn.CloseNow() + } +} + +func (r *Registry) register(value *session) bool { + r.lifecycleMu.Lock() + defer r.lifecycleMu.Unlock() + if r.draining { + return false + } + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.sessions[value.agentID]; exists { + return false + } + r.sessions[value.agentID] = value + return true +} + +func (r *Registry) unregister(value *session) { + r.mu.Lock() + defer r.mu.Unlock() + if r.sessions[value.agentID] == value { + delete(r.sessions, value.agentID) + } +} + +func (r *Registry) get(agentID string) *session { + r.mu.RLock() + defer r.mu.RUnlock() + return r.sessions[agentID] +} + +func (r *Registry) dispatch(ctx context.Context, agentID string, request wireRequest) (dispatchResult, error) { + if !r.beginActivity() { + return dispatchResult{}, errors.Join(ErrNotDispatched, ErrGatewayDraining) + } + defer r.active.Done() + current := r.get(agentID) + if current == nil { + return dispatchResult{}, errors.Join(ErrNotDispatched, errors.New("Agent is not connected")) + } + if err := r.revalidate(ctx, current); err != nil { + return dispatchResult{}, errors.Join(ErrNotDispatched, err) + } + return current.dispatch(ctx, request) +} + +func (r *Registry) revalidate(ctx context.Context, current *session) error { + if err := r.authenticator.Authenticate(ctx, current.agentID, current.secret); err != nil { + r.unregister(current) + current.close() + _ = current.conn.CloseNow() + return errors.New("Agent credentials are no longer valid") + } + return nil +} + +func (s *session) dispatch(ctx context.Context, request wireRequest) (dispatchResult, error) { + result, release, err := s.reservePending(request.ID) + if err != nil { + return dispatchResult{}, errors.Join(ErrNotDispatched, err) + } + defer release() + if err := s.write(ctx, request); err != nil { + return dispatchResult{}, err + } + select { + case response := <-result: + return response, nil + case <-s.done: + return dispatchResult{}, errors.New("Agent disconnected") + case <-ctx.Done(): + return dispatchResult{}, ctx.Err() + } +} + +func (r *Registry) beginActivity() bool { + r.lifecycleMu.Lock() + defer r.lifecycleMu.Unlock() + if r.draining { + return false + } + r.active.Add(1) + return true +} + +func (r *Registry) isDraining() bool { + r.lifecycleMu.Lock() + defer r.lifecycleMu.Unlock() + return r.draining +} + +// BeginDrain stops new control sessions, requests, and console opens while +// allowing work that already crossed the gateway admission boundary to finish. +func (r *Registry) BeginDrain() { + r.lifecycleMu.Lock() + r.draining = true + r.lifecycleMu.Unlock() +} + +// Wait blocks until admitted dispatches and reverse-console opens have +// returned. BeginDrain must be called first so no new WaitGroup additions race +// with this wait. +func (r *Registry) Wait(ctx context.Context) error { + done := make(chan struct{}) + go func() { + r.active.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Close aborts any remaining hijacked control sockets after the graceful +// drain deadline. Active reverse-console data sockets are owned by Relay. +func (r *Registry) Close() { + r.BeginDrain() + r.mu.RLock() + sessions := make([]*session, 0, len(r.sessions)) + for _, current := range r.sessions { + sessions = append(sessions, current) + } + r.mu.RUnlock() + for _, current := range sessions { + current.close() + if current.conn != nil { + _ = current.conn.CloseNow() + } + } +} + +func (s *session) reservePending(id string) (chan dispatchResult, func(), error) { + select { + case s.inFlight <- struct{}{}: + default: + return nil, nil, ErrAgentBusy + } + result := make(chan dispatchResult, 1) + s.mu.Lock() + if _, exists := s.pending[id]; exists { + s.mu.Unlock() + <-s.inFlight + return nil, nil, errors.New("request ID is already pending") + } + s.pending[id] = result + s.mu.Unlock() + var once sync.Once + release := func() { + once.Do(func() { + s.mu.Lock() + delete(s.pending, id) + s.mu.Unlock() + <-s.inFlight + }) + } + return result, release, nil +} + +func (s *session) write(ctx context.Context, value any) error { + body, err := json.Marshal(value) + if err != nil || len(body) > maxControlFrame { + return errors.New("encode Agent control message") + } + s.writeMu.Lock() + defer s.writeMu.Unlock() + return s.conn.Write(ctx, websocket.MessageText, body) +} + +func (s *session) readLoop(ctx context.Context) { + for { + messageType, raw, err := s.conn.Read(ctx) + if err != nil { + return + } + if messageType != websocket.MessageText || len(raw) > maxControlFrame { + _ = s.conn.Close(websocket.StatusUnsupportedData, "text control messages required") + return + } + var response wireResponse + if decodeStrict(raw, &response) != nil || response.Type != "response" || + !requestIDPattern.MatchString(response.ID) || response.Status < 200 || response.Status > 599 || + len(response.Body) == 0 || len(response.Body) > maxControlFrame || !json.Valid([]byte(response.Body)) { + _ = s.conn.Close(websocket.StatusPolicyViolation, "invalid control response") + return + } + s.mu.Lock() + pending := s.pending[response.ID] + s.mu.Unlock() + if pending != nil { + select { + case pending <- dispatchResult{status: response.Status, body: []byte(response.Body)}: + default: + } + } + } +} + +func (s *session) close() { + s.once.Do(func() { close(s.done) }) +} + +func validateWireRequest(request wireRequest) error { + if request.Type != "request" || !requestIDPattern.MatchString(request.ID) || + !validMethod(request.Method) || !validRequestTarget(request.RequestTarget) || + len(request.Body) > maxAgentRequestBody || (request.Body != "" && !json.Valid([]byte(request.Body))) || + len(request.Headers) > 32 { + return errors.New("invalid request envelope") + } + for name, value := range request.Headers { + if name == "" || len(name) > 128 || len(value) > 4096 || strings.ContainsAny(name+value, "\x00\r\n") { + return errors.New("invalid request headers") + } + } + return nil +} + +func validMethod(value string) bool { + switch value { + case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete: + return true + default: + return false + } +} + +func validRequestTarget(value string) bool { + return len(value) <= 2048 && (value == "/v1" || strings.HasPrefix(value, "/v1/")) && + !strings.ContainsAny(value, "\x00\r\n\\# ") +} + +func agentCredentials(request *http.Request) (string, string, bool) { + ids := request.Header.Values("X-Virt-Agent-ID") + authorizations := request.Header.Values("Authorization") + if len(ids) != 1 || len(authorizations) != 1 || !agentIDPattern.MatchString(ids[0]) || + !strings.HasPrefix(authorizations[0], "Bearer ") { + return "", "", false + } + secret := strings.TrimPrefix(authorizations[0], "Bearer ") + if len(secret) < 32 || len(secret) > 256 || strings.ContainsAny(secret, "\x00\r\n \t") { + return "", "", false + } + return ids[0], secret, true +} + +func headerExactly(values []string, expected string) bool { + return len(values) == 1 && strings.TrimSpace(values[0]) == expected +} + +func consoleProtocol(values []string) (string, string, bool) { + if len(values) != 1 || strings.Contains(values[0], ",") { + return "", "", false + } + value := strings.TrimSpace(values[0]) + if !strings.HasPrefix(value, ConsoleSubprotocolPrefix) { + return "", "", false + } + token := strings.TrimPrefix(value, ConsoleSubprotocolPrefix) + return value, token, tokenPattern.MatchString(token) +} + +func randomToken(size int) (string, error) { + value := make([]byte, size) + if _, err := rand.Read(value); err != nil { + return "", fmt.Errorf("generate token: %w", err) + } + return base64.RawURLEncoding.EncodeToString(value), nil +} + +func remoteIsLoopback(value string) bool { + host, _, err := net.SplitHostPort(value) + if err != nil { + return false + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +func exactServiceActor(actor protocol.Actor, service string) bool { + return actor.Service == service && actor.UserID == 0 && !actor.IsAdmin && + !actor.IsSuperAdmin && !actor.CanManage +} + +func readBounded(response http.ResponseWriter, request *http.Request, maximum int64) ([]byte, error) { + request.Body = http.MaxBytesReader(response, request.Body, maximum) + defer request.Body.Close() + return io.ReadAll(request.Body) +} + +func decodeStrict(raw []byte, target any) error { + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return err + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return errors.New("trailing JSON data") + } + return nil +} + +func writeJSON(response http.ResponseWriter, status int, value any) { + response.Header().Set("Content-Type", "application/json; charset=utf-8") + response.Header().Set("Cache-Control", "no-store") + response.Header().Set("X-Content-Type-Options", "nosniff") + response.WriteHeader(status) + _ = json.NewEncoder(response).Encode(value) +} + +func writeInternalError(response http.ResponseWriter, status int, code, message string) { + writeJSON(response, status, map[string]any{"error": map[string]any{ + "code": code, "message": message, "retryable": false, + }}) +} diff --git a/gateway/agent/registry_test.go b/gateway/agent/registry_test.go new file mode 100644 index 00000000..a7865bd2 --- /dev/null +++ b/gateway/agent/registry_test.go @@ -0,0 +1,532 @@ +package agent + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "git.luoli.live/root/virt/gateway/internal/protocol" + "github.com/coder/websocket" +) + +func TestWireRequestBodyLimitIsExact(t *testing.T) { + request := wireRequest{ + Type: "request", ID: "request_boundary", Method: http.MethodPost, + RequestTarget: "/v1/health", Body: `{"payload":"` + strings.Repeat("x", maxAgentRequestBody-14) + `"}`, + Headers: map[string]string{}, + } + if len(request.Body) != maxAgentRequestBody { + t.Fatalf("boundary fixture length = %d", len(request.Body)) + } + if err := validateWireRequest(request); err != nil { + t.Fatalf("exact maximum Agent body was rejected: %v", err) + } + request.Body += " " + if err := validateWireRequest(request); err == nil { + t.Fatal("oversized Agent body was accepted") + } +} + +type allowConnectionAuthenticator struct { + mu sync.Mutex + agentID string + secret string + err error + calls int +} + +func (a *allowConnectionAuthenticator) Authenticate(_ context.Context, agentID, secret string) error { + a.mu.Lock() + defer a.mu.Unlock() + a.agentID, a.secret = agentID, secret + a.calls++ + return a.err +} + +func (a *allowConnectionAuthenticator) reject(err error) { + a.mu.Lock() + defer a.mu.Unlock() + a.err = err +} + +func (a *allowConnectionAuthenticator) callCount() int { + a.mu.Lock() + defer a.mu.Unlock() + return a.calls +} + +func (a *allowConnectionAuthenticator) credentials() (string, string) { + a.mu.Lock() + defer a.mu.Unlock() + return a.agentID, a.secret +} + +func newTestRegistry(t *testing.T) (*Registry, *httptest.Server, []byte, <-chan struct{}) { + t.Helper() + shared := []byte(strings.Repeat("p", 32)) + registry, err := NewRegistry(&allowConnectionAuthenticator{}, shared, 5*time.Second, 2*time.Second) + if err != nil { + t.Fatal(err) + } + mux := http.NewServeMux() + mux.HandleFunc(ControlPath, registry.ServeControl) + consoleReturned := make(chan struct{}, 8) + mux.HandleFunc(ConsolePath, func(response http.ResponseWriter, request *http.Request) { + registry.ServeAgentConsole(response, request) + consoleReturned <- struct{}{} + }) + mux.HandleFunc(InternalRequestPath, registry.ServeInternalRequest) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + return registry, server, shared, consoleReturned +} + +func dialControl(t *testing.T, serverURL, agentID string) *websocket.Conn { + t.Helper() + headers := http.Header{ + "X-Virt-Agent-ID": []string{agentID}, + "Authorization": []string{"Bearer " + strings.Repeat("a", 32)}, + } + conn, response, err := websocket.Dial(t.Context(), "ws"+strings.TrimPrefix(serverURL, "http")+ControlPath, + &websocket.DialOptions{HTTPHeader: headers, Subprotocols: []string{ControlSubprotocol}, + CompressionMode: websocket.CompressionDisabled}) + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + if err != nil { + t.Fatalf("dial Agent control: %v", err) + } + t.Cleanup(func() { _ = conn.CloseNow() }) + return conn +} + +func TestRegistryDispatchesAuthenticatedRequestAndReturnsExactResponse(t *testing.T) { + _, server, shared, _ := newTestRegistry(t) + control := dialControl(t, server.URL, "12") + + agentDone := make(chan error, 1) + go func() { + messageType, raw, err := control.Read(context.Background()) + if err != nil { + agentDone <- err + return + } + var request wireRequest + if messageType != websocket.MessageText || json.Unmarshal(raw, &request) != nil || + request.Type != "request" || request.ID != "request_12345678" || + request.Body != `{"probe":true}` || request.Headers["X-Virt-Actor"] != "actor" { + agentDone <- io.ErrUnexpectedEOF + return + } + response, _ := json.Marshal(wireResponse{Type: "response", ID: request.ID, + Status: http.StatusCreated, Body: `{"result":"exact"}`}) + agentDone <- control.Write(context.Background(), websocket.MessageText, response) + }() + + envelope := map[string]any{ + "agent_id": "12", "type": "request", "id": "request_12345678", + "method": "POST", "request_target": "/v1/health", "body": `{"probe":true}`, + "headers": map[string]string{"X-Virt-Actor": "actor"}, + } + body, _ := json.Marshal(envelope) + request, _ := http.NewRequest(http.MethodPost, server.URL+InternalRequestPath, bytes.NewReader(body)) + nonce, _ := protocol.RandomNonce() + if err := protocol.Sign(request, body, protocol.Actor{Service: "php-panel"}, shared, time.Now(), nonce); err != nil { + t.Fatal(err) + } + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + raw, _ := io.ReadAll(response.Body) + if response.StatusCode != http.StatusCreated || string(raw) != `{"result":"exact"}` { + t.Fatalf("status=%d body=%q", response.StatusCode, raw) + } + if err := <-agentDone; err != nil { + t.Fatal(err) + } +} + +func TestRegistryRejectsInternalRequestFromWrongServiceActor(t *testing.T) { + _, server, shared, _ := newTestRegistry(t) + body := []byte(`{"agent_id":"12","type":"request","id":"request_12345678","method":"GET","request_target":"/v1/health","body":"","headers":{}}`) + request, _ := http.NewRequest(http.MethodPost, server.URL+InternalRequestPath, bytes.NewReader(body)) + nonce, _ := protocol.RandomNonce() + _ = protocol.Sign(request, body, protocol.Actor{Service: "agent-gateway"}, shared, time.Now(), nonce) + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusForbidden { + t.Fatalf("wrong service actor status=%d", response.StatusCode) + } +} + +func TestRegistryRejectsInternalRequestFromMixedServiceActor(t *testing.T) { + _, server, shared, _ := newTestRegistry(t) + body := []byte(`{"agent_id":"12","type":"request","id":"request_12345678","method":"GET","request_target":"/v1/health","body":"","headers":{}}`) + request, _ := http.NewRequest(http.MethodPost, server.URL+InternalRequestPath, bytes.NewReader(body)) + nonce, _ := protocol.RandomNonce() + _ = protocol.Sign(request, body, protocol.Actor{Service: "php-panel", UserID: 1}, shared, time.Now(), nonce) + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusForbidden { + t.Fatalf("mixed service actor status=%d", response.StatusCode) + } +} + +func TestRegistryReturnsTypedJSONForRejectedInternalRequests(t *testing.T) { + registry, _, shared, _ := newTestRegistry(t) + + remoteRequest := httptest.NewRequest(http.MethodPost, InternalRequestPath, strings.NewReader("{}")) + remoteRequest.RemoteAddr = "203.0.113.10:43123" + remoteResponse := httptest.NewRecorder() + registry.ServeInternalRequest(remoteResponse, remoteRequest) + assertInternalError(t, remoteResponse.Result(), http.StatusForbidden, "gateway_forbidden") + + body := []byte(`{"agent_id":"invalid","type":"request","id":"request_12345678","method":"GET","request_target":"/v1/health","body":"","headers":{}}`) + invalidRequest := httptest.NewRequest(http.MethodPost, InternalRequestPath, bytes.NewReader(body)) + invalidRequest.RemoteAddr = "127.0.0.1:43124" + nonce, _ := protocol.RandomNonce() + if err := protocol.Sign(invalidRequest, body, protocol.Actor{Service: "php-panel"}, shared, time.Now(), nonce); err != nil { + t.Fatal(err) + } + invalidResponse := httptest.NewRecorder() + registry.ServeInternalRequest(invalidResponse, invalidRequest) + assertInternalError(t, invalidResponse.Result(), http.StatusBadRequest, "gateway_request_invalid") +} + +func TestSessionRejectsRequestBeforeDispatchWhenCapacityIsExhausted(t *testing.T) { + current := &session{inFlight: make(chan struct{}, maxInFlightPerAgent)} + for index := 0; index < maxInFlightPerAgent; index++ { + current.inFlight <- struct{}{} + } + _, err := current.dispatch(t.Context(), wireRequest{ID: "request_12345678"}) + if !errors.Is(err, ErrAgentBusy) || !errors.Is(err, ErrNotDispatched) { + t.Fatalf("capacity error = %v", err) + } +} + +func TestRegistryMarksCapacityRejectionAsNotDispatched(t *testing.T) { + registry, _, shared, _ := newTestRegistry(t) + current := &session{ + agentID: "12", secret: strings.Repeat("a", 32), + pending: make(map[string]chan dispatchResult), + inFlight: make(chan struct{}, maxInFlightPerAgent), done: make(chan struct{}), + } + for index := 0; index < maxInFlightPerAgent; index++ { + current.inFlight <- struct{}{} + } + registry.sessions[current.agentID] = current + body := []byte(`{"agent_id":"12","type":"request","id":"request_12345678","method":"POST","request_target":"/v1/vms","body":"{}","headers":{}}`) + request := httptest.NewRequest(http.MethodPost, InternalRequestPath, bytes.NewReader(body)) + request.RemoteAddr = "127.0.0.1:43125" + nonce, _ := protocol.RandomNonce() + if err := protocol.Sign(request, body, protocol.Actor{Service: "php-panel"}, shared, time.Now(), nonce); err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + registry.ServeInternalRequest(response, request) + var payload struct { + Error struct { + Code string `json:"code"` + Retryable bool `json:"retryable"` + DispatchState string `json:"dispatch_state"` + } `json:"error"` + } + if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil { + t.Fatal(err) + } + if response.Code != http.StatusServiceUnavailable || payload.Error.Code != "agent_busy" || + !payload.Error.Retryable || payload.Error.DispatchState != "not_dispatched" { + t.Fatalf("capacity response status=%d payload=%#v", response.Code, payload) + } +} + +func TestRegistryDrainRejectsNewDispatchBeforeAgentWrite(t *testing.T) { + registry, _, _, _ := newTestRegistry(t) + registry.BeginDrain() + _, err := registry.dispatch(t.Context(), "12", wireRequest{ID: "request_12345678"}) + if !errors.Is(err, ErrGatewayDraining) || !errors.Is(err, ErrNotDispatched) { + t.Fatalf("draining dispatch error = %v", err) + } + if err := registry.Wait(t.Context()); err != nil { + t.Fatal(err) + } +} + +func TestRegistryDrainWaitsForAdmittedDispatch(t *testing.T) { + registry, server, _, _ := newTestRegistry(t) + control := dialControl(t, server.URL, "47") + result := make(chan error, 1) + go func() { + _, err := registry.dispatch(context.Background(), "47", wireRequest{ + Type: "request", ID: "request_12345678", Method: http.MethodGet, + RequestTarget: "/v1/health", Headers: map[string]string{}, + }) + result <- err + }() + messageType, raw, err := control.Read(t.Context()) + if err != nil || messageType != websocket.MessageText { + t.Fatalf("read admitted dispatch: type=%v err=%v", messageType, err) + } + var request wireRequest + if json.Unmarshal(raw, &request) != nil || request.ID != "request_12345678" { + t.Fatalf("admitted request = %#v", request) + } + registry.BeginDrain() + short, cancelShort := context.WithTimeout(t.Context(), 20*time.Millisecond) + if err := registry.Wait(short); !errors.Is(err, context.DeadlineExceeded) { + cancelShort() + t.Fatalf("drain did not wait for active dispatch: %v", err) + } + cancelShort() + encoded, _ := json.Marshal(wireResponse{Type: "response", ID: request.ID, + Status: http.StatusOK, Body: `{"status":"ok"}`}) + if err := control.Write(t.Context(), websocket.MessageText, encoded); err != nil { + t.Fatal(err) + } + if err := <-result; err != nil { + t.Fatal(err) + } + if err := registry.Wait(t.Context()); err != nil { + t.Fatal(err) + } +} + +func TestRegistryTimeoutAfterAgentWriteRemainsAmbiguous(t *testing.T) { + registry, server, _, _ := newTestRegistry(t) + control := dialControl(t, server.URL, "48") + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Millisecond) + defer cancel() + result := make(chan error, 1) + go func() { + _, err := registry.dispatch(ctx, "48", wireRequest{ + Type: "request", ID: "request_87654321", Method: http.MethodGet, + RequestTarget: "/v1/health", Headers: map[string]string{}, + }) + result <- err + }() + if _, _, err := control.Read(t.Context()); err != nil { + t.Fatal(err) + } + err := <-result + if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrNotDispatched) { + t.Fatalf("post-write timeout certainty = %v", err) + } +} + +func TestRegistryRevokesConnectedAgentBeforeOrdinaryDispatch(t *testing.T) { + registry, server, _, _ := newTestRegistry(t) + control := dialControl(t, server.URL, "12") + authenticator := registry.authenticator.(*allowConnectionAuthenticator) + authenticator.reject(errors.New("Agent secret was rotated")) + + _, err := registry.dispatch(t.Context(), "12", wireRequest{ID: "request_12345678"}) + if err == nil || registry.get("12") != nil { + t.Fatalf("revoked Agent dispatch err=%v session=%v", err, registry.get("12")) + } + if authenticator.callCount() != 2 { + t.Fatalf("authentication calls=%d want=2", authenticator.callCount()) + } + agentID, secret := authenticator.credentials() + if agentID != "12" || secret != strings.Repeat("a", 32) { + t.Fatalf("revalidation credentials agent_id=%q secret_length=%d", agentID, len(secret)) + } + readContext, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if _, _, err := control.Read(readContext); err == nil { + t.Fatal("revoked Agent control session remained open") + } +} + +func TestRegistryRevokesConnectedAgentBeforeOpeningConsole(t *testing.T) { + registry, server, _, _ := newTestRegistry(t) + control := dialControl(t, server.URL, "44") + authenticator := registry.authenticator.(*allowConnectionAuthenticator) + authenticator.reject(errors.New("Agent was disabled")) + + conn, err := registry.OpenConsole(t.Context(), "44", strings.Repeat("b", 32), strings.Repeat("c", 32)) + if conn != nil { + _ = conn.CloseNow() + } + if err == nil || registry.get("44") != nil { + t.Fatalf("revoked Agent console err=%v session=%v", err, registry.get("44")) + } + if authenticator.callCount() != 2 { + t.Fatalf("authentication calls=%d want=2", authenticator.callCount()) + } + readContext, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if _, _, err := control.Read(readContext); err == nil { + t.Fatal("revoked Agent control session remained open") + } +} + +func assertInternalError(t *testing.T, response *http.Response, status int, code string) { + t.Helper() + defer response.Body.Close() + var payload struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + Retryable bool `json:"retryable"` + } `json:"error"` + } + if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { + t.Fatalf("decode internal error: %v", err) + } + if response.StatusCode != status || payload.Error.Code != code || + payload.Error.Message == "" || payload.Error.Retryable || + !strings.HasPrefix(response.Header.Get("Content-Type"), "application/json") { + t.Fatalf("unexpected internal error status=%d payload=%#v headers=%v", response.StatusCode, payload, response.Header) + } +} + +func TestRegistryPairsOneUseReverseConsoleWithConnectedAgent(t *testing.T) { + registry, server, _, consoleReturned := newTestRegistry(t) + control := dialControl(t, server.URL, "44") + + opened := make(chan *websocket.Conn, 1) + errors := make(chan error, 1) + go func() { + conn, err := registry.OpenConsole(context.Background(), "44", strings.Repeat("b", 32), strings.Repeat("c", 32)) + if err != nil { + errors <- err + return + } + opened <- conn + }() + + messageType, raw, err := control.Read(t.Context()) + if err != nil || messageType != websocket.MessageText { + t.Fatalf("read console_open: type=%v err=%v", messageType, err) + } + var command consoleOpen + if json.Unmarshal(raw, &command) != nil || command.Type != "console_open" || + command.Ticket != strings.Repeat("b", 32) || command.SessionID != strings.Repeat("c", 32) || + !tokenPattern.MatchString(command.SessionToken) { + t.Fatalf("invalid console_open: %#v", command) + } + protocolValue := ConsoleSubprotocolPrefix + command.SessionToken + agentConsole, response, err := websocket.Dial(t.Context(), + "ws"+strings.TrimPrefix(server.URL, "http")+ConsolePath, + &websocket.DialOptions{HTTPHeader: http.Header{"X-Virt-Agent-ID": []string{"44"}}, + Subprotocols: []string{protocolValue}, CompressionMode: websocket.CompressionDisabled}) + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + if err != nil { + t.Fatal(err) + } + defer agentConsole.CloseNow() + acknowledgement, _ := json.Marshal(wireResponse{Type: "response", ID: command.ID, + Status: http.StatusOK, Body: `{"status":"connected"}`}) + if err := control.Write(t.Context(), websocket.MessageText, acknowledgement); err != nil { + t.Fatalf("send console acknowledgement: %v", err) + } + select { + case <-consoleReturned: + case <-time.After(time.Second): + t.Fatal("reverse console handler did not return after transferring connection ownership") + } + duplicate, duplicateResponse, duplicateErr := websocket.Dial(t.Context(), + "ws"+strings.TrimPrefix(server.URL, "http")+ConsolePath, + &websocket.DialOptions{HTTPHeader: http.Header{"X-Virt-Agent-ID": []string{"44"}}, + Subprotocols: []string{protocolValue}, CompressionMode: websocket.CompressionDisabled}) + if duplicate != nil { + _ = duplicate.CloseNow() + } + if duplicateResponse != nil && duplicateResponse.Body != nil { + defer duplicateResponse.Body.Close() + } + if duplicateErr == nil || duplicateResponse == nil || duplicateResponse.StatusCode != http.StatusUnauthorized { + t.Fatalf("reused console token err=%v response=%v", duplicateErr, duplicateResponse) + } + + var gatewayConsole *websocket.Conn + select { + case gatewayConsole = <-opened: + case err := <-errors: + t.Fatal(err) + case <-time.After(2 * time.Second): + t.Fatal("reverse console was not paired") + } + defer gatewayConsole.CloseNow() + if err := agentConsole.Write(t.Context(), websocket.MessageBinary, []byte("RFB")); err != nil { + t.Fatal(err) + } + kind, payload, err := gatewayConsole.Read(t.Context()) + if err != nil || kind != websocket.MessageBinary || string(payload) != "RFB" { + t.Fatalf("reverse console frame kind=%v payload=%q err=%v", kind, payload, err) + } +} + +func TestRegistryReturnsConsoleRejectionWithoutWaitingForReverseDial(t *testing.T) { + registry, server, _, _ := newTestRegistry(t) + control := dialControl(t, server.URL, "45") + + result := make(chan error, 1) + go func() { + _, err := registry.OpenConsole(context.Background(), "45", strings.Repeat("b", 32), strings.Repeat("c", 32)) + result <- err + }() + + messageType, raw, err := control.Read(t.Context()) + if err != nil || messageType != websocket.MessageText { + t.Fatalf("read console_open: type=%v err=%v", messageType, err) + } + var command consoleOpen + if err := json.Unmarshal(raw, &command); err != nil { + t.Fatalf("decode console_open: %v", err) + } + rejection, _ := json.Marshal(wireResponse{Type: "response", ID: command.ID, + Status: http.StatusUnauthorized, Body: `{"error":{"code":"invalid_console_ticket"}}`}) + if err := control.Write(t.Context(), websocket.MessageText, rejection); err != nil { + t.Fatalf("send console rejection: %v", err) + } + select { + case err := <-result: + if err == nil || !strings.Contains(err.Error(), "status 401") { + t.Fatalf("console rejection error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("console rejection waited for the reverse connection timeout") + } +} + +func TestRegistryBoundsPendingReverseConsoles(t *testing.T) { + registry, err := NewRegistry(&allowConnectionAuthenticator{}, []byte(strings.Repeat("p", 32)), + 5*time.Second, 2*time.Second) + if err != nil { + t.Fatal(err) + } + current := &session{ + agentID: "46", pending: make(map[string]chan dispatchResult), + inFlight: make(chan struct{}, maxInFlightPerAgent), done: make(chan struct{}), + } + registry.sessions[current.agentID] = current + for index := 0; index < maxPendingConsoles; index++ { + registry.consoles[string(rune(index+1))] = pendingConsole{agentID: current.agentID} + } + _, err = registry.OpenConsole(t.Context(), current.agentID, strings.Repeat("b", 32), strings.Repeat("c", 32)) + if !errors.Is(err, ErrConsoleBusy) { + t.Fatalf("pending console capacity error = %v", err) + } + if len(current.inFlight) != 0 || len(current.pending) != 0 { + t.Fatalf("rejected console leaked session admission: in_flight=%d pending=%d", len(current.inFlight), len(current.pending)) + } +} diff --git a/gateway/cmd/virt-cloud-agent-gateway/main.go b/gateway/cmd/virt-cloud-agent-gateway/main.go new file mode 100644 index 00000000..236ca576 --- /dev/null +++ b/gateway/cmd/virt-cloud-agent-gateway/main.go @@ -0,0 +1,120 @@ +package main + +import ( + "context" + "errors" + "flag" + "log" + "net/http" + "os/signal" + "path/filepath" + "sync" + "syscall" + "time" + + agentgateway "git.luoli.live/root/virt/gateway/agent" + "git.luoli.live/root/virt/gateway/console" +) + +func main() { + configPath := flag.String("config", "", "absolute path to the JSON configuration") + flag.Parse() + if *configPath == "" || !filepath.IsAbs(*configPath) { + log.Fatal("--config must be an absolute path") + } + config, err := console.LoadConfig(*configPath) + if err != nil { + log.Fatalf("Agent gateway configuration is invalid: %v", err) + } + introspector, err := console.NewHTTPIntrospector(config.PanelIntrospectionURL, + []byte(config.PanelSharedSecret), config.ConnectTimeout()) + if err != nil { + log.Fatalf("console introspection client cannot start: %v", err) + } + authenticator, err := agentgateway.NewHTTPConnectionAuthenticator( + config.PanelAgentAuthURL, []byte(config.PanelSharedSecret), config.ConnectTimeout(), + ) + if err != nil { + log.Fatalf("Agent authentication client cannot start: %v", err) + } + registry, err := agentgateway.NewRegistry(authenticator, + []byte(config.PanelSharedSecret), config.AgentRequestTimeout(), + config.ConnectTimeout()) + if err != nil { + log.Fatalf("Agent registry cannot start: %v", err) + } + relay, err := console.NewRelay(introspector, registry) + if err != nil { + log.Fatalf("reverse console cannot start: %v", err) + } + mux := http.NewServeMux() + mux.HandleFunc(agentgateway.ControlPath, registry.ServeControl) + mux.HandleFunc(agentgateway.ConsolePath, registry.ServeAgentConsole) + mux.HandleFunc(agentgateway.InternalRequestPath, registry.ServeInternalRequest) + mux.Handle("/engine-console", relay) + server := &http.Server{ + Addr: config.ListenAddress, Handler: mux, + ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 32 << 10, + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + serveErrors := make(chan error, 1) + go func() { + serveErrors <- server.ListenAndServe() + }() + log.Printf("virt-cloud Agent gateway listening on %s", config.ListenAddress) + select { + case err := <-serveErrors: + if err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("Agent gateway stopped unexpectedly: %v", err) + } + return + case <-ctx.Done(): + } + + // Keep the listener available during the grace period so an already + // admitted reverse-console open can complete its Agent callback. All other + // new work is rejected by the draining gates below. + registry.BeginDrain() + relay.BeginDrain() + drainContext, cancelDrain := context.WithTimeout(context.Background(), 10*time.Second) + drained := make(chan struct{}) + go func() { + var waits sync.WaitGroup + waits.Add(2) + go func() { + defer waits.Done() + _ = registry.Wait(drainContext) + }() + go func() { + defer waits.Done() + _ = relay.Wait(drainContext) + }() + waits.Wait() + close(drained) + }() + select { + case <-drained: + case <-drainContext.Done(): + } + cancelDrain() + relay.Close() + registry.Close() + + shutdown, cancelShutdown := context.WithTimeout(context.Background(), 2*time.Second) + if err := server.Shutdown(shutdown); err != nil { + _ = server.Close() + log.Printf("Agent gateway HTTP shutdown was forced: %v", err) + } + cancelShutdown() + select { + case err := <-serveErrors: + if err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("Agent gateway stopped unexpectedly: %v", err) + } + case <-time.After(2 * time.Second): + _ = server.Close() + log.Print("Agent gateway listener did not stop before the shutdown deadline") + } +} diff --git a/gateway/cmd/virt-cloud-console-relay/main.go b/gateway/cmd/virt-cloud-console-relay/main.go deleted file mode 100644 index ef605dd9..00000000 --- a/gateway/cmd/virt-cloud-console-relay/main.go +++ /dev/null @@ -1,49 +0,0 @@ -package main - -import ( - "context" - "errors" - "flag" - "log" - "net/http" - "os/signal" - "path/filepath" - "syscall" - "time" - - "git.luoli.live/root/virt/gateway/console" -) - -func main() { - configPath := flag.String("config", "", "absolute path to the JSON configuration") - flag.Parse() - if *configPath == "" || !filepath.IsAbs(*configPath) { - log.Fatal("--config must be an absolute path") - } - config, err := console.LoadConfig(*configPath) - if err != nil { - log.Fatalf("console relay configuration is invalid: %v", err) - } - introspector, err := console.NewHTTPIntrospector(config.PanelIntrospectionURL, []byte(config.PanelSharedSecret), config.ConnectTimeout()) - if err != nil { - log.Fatalf("console introspection client cannot start: %v", err) - } - relay, err := console.NewRelay(introspector, config.ConnectTimeout()) - if err != nil { - log.Fatalf("console relay cannot start: %v", err) - } - server := &http.Server{Addr: config.ListenAddress, Handler: relay, ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 15 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 32 << 10} - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - go func() { - <-ctx.Done() - shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = server.Shutdown(shutdown) - }() - log.Printf("virt-cloud-console-relay listening on %s", config.ListenAddress) - if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Fatalf("console relay stopped unexpectedly: %v", err) - } -} diff --git a/gateway/console/config.go b/gateway/console/config.go index bf320ea8..a82eb5ff 100644 --- a/gateway/console/config.go +++ b/gateway/console/config.go @@ -9,6 +9,7 @@ import ( "net" "net/url" "os" + "strconv" "strings" "time" ) @@ -16,38 +17,41 @@ import ( const ( RequiredListenAddress = "127.0.0.1:8001" RequiredIntrospectionPath = "/api/internal/cloud/session-introspection" + RequiredAgentAuthPath = "/api/internal/cloud/agent-auth" maxConfigBytes = 64 << 10 ) type Config struct { - ListenAddress string `json:"listen_address"` - PanelIntrospectionURL string `json:"panel_introspection_url"` - PanelSharedSecret string `json:"panel_shared_secret"` - ConnectTimeoutSeconds int `json:"connect_timeout_seconds"` + ListenAddress string `json:"listen_address"` + PanelIntrospectionURL string `json:"panel_introspection_url"` + PanelAgentAuthURL string `json:"panel_agent_auth_url"` + PanelSharedSecret string `json:"panel_shared_secret"` + ConnectTimeoutSeconds int `json:"connect_timeout_seconds"` + AgentRequestTimeoutSeconds int `json:"agent_request_timeout_seconds"` } func LoadConfig(path string) (Config, error) { file, err := os.Open(path) if err != nil { - return Config{}, fmt.Errorf("open relay configuration: %w", err) + return Config{}, fmt.Errorf("open Agent gateway configuration: %w", err) } defer file.Close() raw, err := io.ReadAll(io.LimitReader(file, maxConfigBytes+1)) if err != nil { - return Config{}, fmt.Errorf("read relay configuration: %w", err) + return Config{}, fmt.Errorf("read Agent gateway configuration: %w", err) } if len(raw) > maxConfigBytes { - return Config{}, errors.New("relay configuration exceeds 65536 bytes") + return Config{}, errors.New("Agent gateway configuration exceeds 65536 bytes") } decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.DisallowUnknownFields() var config Config if err := decoder.Decode(&config); err != nil { - return Config{}, fmt.Errorf("decode relay configuration: %w", err) + return Config{}, fmt.Errorf("decode Agent gateway configuration: %w", err) } var extra any if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { - return Config{}, errors.New("relay configuration contains trailing data") + return Config{}, errors.New("Agent gateway configuration contains trailing data") } if err := config.Validate(); err != nil { return Config{}, err @@ -63,12 +67,18 @@ func (c Config) Validate() error { if err := validateIntrospectionURL(c.PanelIntrospectionURL); err != nil { problems = append(problems, err.Error()) } + if err := validatePanelURL(c.PanelAgentAuthURL, RequiredAgentAuthPath, "panel_agent_auth_url"); err != nil { + problems = append(problems, err.Error()) + } if len(c.PanelSharedSecret) < 32 || strings.ContainsAny(c.PanelSharedSecret, "\x00\r\n") { problems = append(problems, "panel_shared_secret must contain at least 32 non-line-breaking bytes") } if c.ConnectTimeoutSeconds < 1 || c.ConnectTimeoutSeconds > 30 { problems = append(problems, "connect_timeout_seconds must be between 1 and 30") } + if c.AgentRequestTimeoutSeconds < 1 || c.AgentRequestTimeoutSeconds > 900 { + problems = append(problems, "agent_request_timeout_seconds must be between 1 and 900") + } if len(problems) != 0 { return errors.New(strings.Join(problems, "; ")) } @@ -79,24 +89,34 @@ func (c Config) ConnectTimeout() time.Duration { return time.Duration(c.ConnectTimeoutSeconds) * time.Second } +func (c Config) AgentRequestTimeout() time.Duration { + return time.Duration(c.AgentRequestTimeoutSeconds) * time.Second +} + func validateIntrospectionURL(value string) error { + return validatePanelURL(value, RequiredIntrospectionPath, "panel_introspection_url") +} + +func validatePanelURL(value, requiredPath, field string) error { if value == "" || len(value) > 2048 || strings.ContainsAny(value, "\x00\r\n\\") { - return errors.New("panel_introspection_url is invalid") + return errors.New(field + " is invalid") } parsed, err := url.Parse(value) if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.User != nil || - parsed.RawQuery != "" || parsed.Fragment != "" || parsed.Path != RequiredIntrospectionPath || parsed.RawPath != "" { - return errors.New("panel_introspection_url must be the fixed local introspection endpoint") + parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" || parsed.Path != requiredPath || parsed.RawPath != "" || + strings.HasSuffix(parsed.Host, ":") { + return errors.New(field + " must be its fixed local endpoint") } host := parsed.Hostname() ip := net.ParseIP(host) if ip == nil || !ip.IsLoopback() { - return errors.New("panel_introspection_url must use a literal loopback address") + return errors.New(field + " must use a literal loopback address") } port := parsed.Port() if port != "" { - if _, err := net.LookupPort("tcp", port); err != nil { - return errors.New("panel_introspection_url has an invalid port") + number, err := strconv.Atoi(port) + if err != nil || number < 1 || number > 65535 { + return errors.New(field + " has an invalid port") } } return nil diff --git a/gateway/console/config_test.go b/gateway/console/config_test.go index e13194ff..1d82cefc 100644 --- a/gateway/console/config_test.go +++ b/gateway/console/config_test.go @@ -8,8 +8,8 @@ import ( ) func TestLoadConfigDoesNotAuditOwnershipOrMode(t *testing.T) { - path := filepath.Join(t.TempDir(), "relay.json") - raw := `{"listen_address":"127.0.0.1:8001","panel_introspection_url":"http://127.0.0.1/api/internal/cloud/session-introspection","panel_shared_secret":"0123456789abcdef0123456789abcdef","connect_timeout_seconds":10}` + path := filepath.Join(t.TempDir(), "agent-gateway.json") + raw := `{"listen_address":"127.0.0.1:8001","panel_introspection_url":"http://127.0.0.1/api/internal/cloud/session-introspection","panel_agent_auth_url":"http://127.0.0.1/api/internal/cloud/agent-auth","panel_shared_secret":"0123456789abcdef0123456789abcdef","connect_timeout_seconds":10,"agent_request_timeout_seconds":900}` if err := os.WriteFile(path, []byte(raw), 0o644); err != nil { t.Fatal(err) } @@ -19,25 +19,27 @@ func TestLoadConfigDoesNotAuditOwnershipOrMode(t *testing.T) { } func TestLoadConfigRejectsDataPastSizeLimit(t *testing.T) { - path := filepath.Join(t.TempDir(), "relay.json") + path := filepath.Join(t.TempDir(), "agent-gateway.json") raw := []byte(`{}` + strings.Repeat(" ", maxConfigBytes)) if err := os.WriteFile(path, raw, 0o644); err != nil { t.Fatal(err) } if _, err := LoadConfig(path); err == nil || !strings.Contains(err.Error(), "exceeds") { - t.Fatalf("oversized relay configuration error = %v", err) + t.Fatalf("oversized Agent gateway configuration error = %v", err) } } func TestConfigRejectsRemoteIntrospectionAndUnknownFields(t *testing.T) { config := Config{ListenAddress: RequiredListenAddress, PanelIntrospectionURL: "https://198.51.100.2" + RequiredIntrospectionPath, - PanelSharedSecret: strings.Repeat("s", 32), ConnectTimeoutSeconds: 10} + PanelAgentAuthURL: "http://127.0.0.1" + RequiredAgentAuthPath, + PanelSharedSecret: strings.Repeat("s", 32), ConnectTimeoutSeconds: 10, + AgentRequestTimeoutSeconds: 900} if err := config.Validate(); err == nil { t.Fatal("remote introspection endpoint accepted") } - path := filepath.Join(t.TempDir(), "relay.json") - raw := `{"listen_address":"127.0.0.1:8001","panel_introspection_url":"http://127.0.0.1/api/internal/cloud/session-introspection","panel_shared_secret":"0123456789abcdef0123456789abcdef","connect_timeout_seconds":10,"unexpected":true}` + path := filepath.Join(t.TempDir(), "agent-gateway.json") + raw := `{"listen_address":"127.0.0.1:8001","panel_introspection_url":"http://127.0.0.1/api/internal/cloud/session-introspection","panel_agent_auth_url":"http://127.0.0.1/api/internal/cloud/agent-auth","panel_shared_secret":"0123456789abcdef0123456789abcdef","connect_timeout_seconds":10,"agent_request_timeout_seconds":900,"unexpected":true}` if err := os.WriteFile(path, []byte(raw), 0o600); err != nil { t.Fatal(err) } @@ -45,3 +47,29 @@ func TestConfigRejectsRemoteIntrospectionAndUnknownFields(t *testing.T) { t.Fatal("unknown configuration field accepted") } } + +func TestConfigRejectsExplicitEmptyPanelPorts(t *testing.T) { + config := Config{ListenAddress: RequiredListenAddress, + PanelIntrospectionURL: "http://127.0.0.1:" + RequiredIntrospectionPath, + PanelAgentAuthURL: "http://127.0.0.1" + RequiredAgentAuthPath, + PanelSharedSecret: strings.Repeat("s", 32), ConnectTimeoutSeconds: 10, + AgentRequestTimeoutSeconds: 900} + if err := config.Validate(); err == nil { + t.Fatal("explicit empty introspection port was accepted") + } + config.PanelIntrospectionURL = "http://127.0.0.1" + RequiredIntrospectionPath + config.PanelAgentAuthURL = "http://127.0.0.1:" + RequiredAgentAuthPath + if err := config.Validate(); err == nil { + t.Fatal("explicit empty Agent authentication port was accepted") + } + for _, endpoint := range []string{ + "http://127.0.0.1:0" + RequiredAgentAuthPath, + "http://127.0.0.1:65536" + RequiredAgentAuthPath, + "http://127.0.0.1" + RequiredAgentAuthPath + "?", + } { + config.PanelAgentAuthURL = endpoint + if err := config.Validate(); err == nil { + t.Fatalf("invalid Agent authentication endpoint accepted: %s", endpoint) + } + } +} diff --git a/gateway/console/introspection.go b/gateway/console/introspection.go index 8cf5fd5c..7cc6c21a 100644 --- a/gateway/console/introspection.go +++ b/gateway/console/introspection.go @@ -9,8 +9,6 @@ import ( "io" "net" "net/http" - "net/url" - "strings" "time" "git.luoli.live/root/virt/gateway/internal/protocol" @@ -25,11 +23,11 @@ type IntrospectionRequest struct { } type IntrospectionResult struct { - Active bool `json:"active"` - ConnectionID string `json:"connection_id,omitempty"` - AgentWebSocketURL string `json:"agent_websocket_url,omitempty"` - AgentTicket string `json:"agent_ticket,omitempty"` - AgentExpiresAt string `json:"agent_expires_at,omitempty"` + Active bool `json:"active"` + ConnectionID string `json:"connection_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` + AgentTicket string `json:"agent_ticket,omitempty"` + AgentExpiresAt string `json:"agent_expires_at,omitempty"` } type SessionIntrospector interface { @@ -75,7 +73,7 @@ func (i *HTTPIntrospector) Inspect(ctx context.Context, input IntrospectionReque if err != nil { return IntrospectionResult{}, errors.New("create introspection request nonce") } - if err := protocol.Sign(request, body, protocol.Actor{Service: "console-relay"}, i.secret, i.now(), nonce); err != nil { + if err := protocol.Sign(request, body, protocol.Actor{Service: "agent-gateway"}, i.secret, i.now(), nonce); err != nil { return IntrospectionResult{}, errors.New("sign introspection request") } response, err := i.client.Do(request) @@ -108,34 +106,3 @@ func (i *HTTPIntrospector) Inspect(ctx context.Context, input IntrospectionReque func rejectRedirect(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse } - -func validateAgentWebSocketURL(value string) error { - if value == "" || len(value) > 2048 || strings.ContainsAny(value, "\x00\r\n\\") { - return errors.New("agent WebSocket URL is invalid") - } - parsed, err := url.Parse(value) - if err != nil || parsed.Scheme != "wss" || parsed.User != nil || parsed.Opaque != "" || parsed.RawQuery != "" || - parsed.ForceQuery || parsed.Fragment != "" || parsed.Path != "/v1/console" || parsed.RawPath != "" { - return errors.New("agent WebSocket URL must be a fixed WSS console endpoint") - } - ip := net.ParseIP(parsed.Hostname()) - if ip == nil || blockedAgentIP(ip) { - return errors.New("agent WebSocket URL must use an allowed fixed IP address") - } - if port := parsed.Port(); port != "" { - if _, err := net.LookupPort("tcp", port); err != nil { - return errors.New("agent WebSocket URL has an invalid port") - } - } - return nil -} - -func blockedAgentIP(ip net.IP) bool { - if ip.IsUnspecified() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() { - return true - } - if ipv4 := ip.To4(); ipv4 != nil { - return ipv4[0] == 0 || ipv4[0] >= 224 - } - return false -} diff --git a/gateway/console/introspection_test.go b/gateway/console/introspection_test.go index 304a74e5..4e8f4435 100644 --- a/gateway/console/introspection_test.go +++ b/gateway/console/introspection_test.go @@ -35,7 +35,7 @@ func TestHTTPIntrospectorSignsExactBodyAndTarget(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { body, _ := io.ReadAll(request.Body) actor, err := authenticator.Authenticate(request, body) - if err != nil || actor.Service != "console-relay" { + if err != nil || actor.Service != "agent-gateway" { http.Error(response, "forbidden", http.StatusForbidden) return } @@ -45,8 +45,9 @@ func TestHTTPIntrospectorSignsExactBodyAndTarget(t *testing.T) { return } response.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(response).Encode(IntrospectionResult{Active: true, ConnectionID: strings.Repeat("c", 32), - AgentWebSocketURL: "wss://198.51.100.2:9443/v1/console", AgentTicket: strings.Repeat("a", 32), + _ = json.NewEncoder(response).Encode(IntrospectionResult{Active: true, + ConnectionID: strings.Repeat("c", 32), AgentID: "12", + AgentTicket: strings.Repeat("a", 32), AgentExpiresAt: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)}) })) defer server.Close() @@ -59,21 +60,3 @@ func TestHTTPIntrospectorSignsExactBodyAndTarget(t *testing.T) { t.Fatalf("result=%#v err=%v", result, err) } } - -func TestAgentURLValidationIsTLSAndSSRFSafe(t *testing.T) { - for _, good := range []string{"wss://198.51.100.2:9443/v1/console", "wss://10.0.0.2/v1/console", "wss://[fd00::2]:9443/v1/console"} { - if err := validateAgentWebSocketURL(good); err != nil { - t.Errorf("valid URL %q rejected: %v", good, err) - } - } - for _, bad := range []string{ - "ws://198.51.100.2:9443/v1/console", "wss://agent.example/v1/console", - "wss://127.0.0.1:9443/v1/console", "wss://169.254.169.254/v1/console", - "wss://198.51.100.2/v1/console?target=127.0.0.1", "wss://user@198.51.100.2/v1/console", - "wss://198.51.100.2/v1/other", - } { - if err := validateAgentWebSocketURL(bad); err == nil { - t.Errorf("unsafe URL %q accepted", bad) - } - } -} diff --git a/gateway/console/relay.go b/gateway/console/relay.go index 98c5ebc1..503cbc59 100644 --- a/gateway/console/relay.go +++ b/gateway/console/relay.go @@ -2,14 +2,13 @@ package console import ( "context" - "crypto/tls" "errors" "io" - "net" "net/http" "net/url" "regexp" "strings" + "sync" "time" "github.com/coder/websocket" @@ -17,34 +16,36 @@ import ( const ( BrowserProtocolPrefix = "cloud-ticket." - AgentProtocolPrefix = "cloud-agent-ticket." defaultMaximumFrame = 8 << 20 ) var opaqueTokenPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{32,256}$`) +var agentIDPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`) + +type ReverseConsoleConnector interface { + OpenConsole(context.Context, string, string, string) (*websocket.Conn, error) +} type Relay struct { introspector SessionIntrospector - agentClient *http.Client - now func() time.Time + connector ReverseConsoleConnector recheckEvery time.Duration maxFrame int64 - validateURL func(string) error + now func() time.Time + lifecycleMu sync.Mutex + draining bool + active sync.WaitGroup + next uint64 + cancels map[uint64]context.CancelFunc } -func NewRelay(introspector SessionIntrospector, connectTimeout time.Duration) (*Relay, error) { - if introspector == nil || connectTimeout <= 0 || connectTimeout > 30*time.Second { +func NewRelay(introspector SessionIntrospector, connector ReverseConsoleConnector) (*Relay, error) { + if introspector == nil || connector == nil { return nil, errors.New("invalid console relay settings") } - dialer := &net.Dialer{Timeout: connectTimeout, KeepAlive: 30 * time.Second} - transport := &http.Transport{Proxy: nil, DialContext: dialer.DialContext, ForceAttemptHTTP2: false, - TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, TLSHandshakeTimeout: connectTimeout, - ResponseHeaderTimeout: connectTimeout} - // Do not set http.Client.Timeout here: for an upgraded response it would - // terminate a healthy WebSocket after the handshake timeout elapsed. - return &Relay{introspector: introspector, agentClient: &http.Client{Transport: transport, - CheckRedirect: rejectRedirect}, now: time.Now, recheckEvery: 15 * time.Second, - maxFrame: defaultMaximumFrame, validateURL: validateAgentWebSocketURL}, nil + return &Relay{introspector: introspector, connector: connector, + recheckEvery: 15 * time.Second, maxFrame: defaultMaximumFrame, now: time.Now, + cancels: make(map[uint64]context.CancelFunc)}, nil } func (r *Relay) ServeHTTP(response http.ResponseWriter, request *http.Request) { @@ -65,19 +66,21 @@ func (r *Relay) ServeHTTP(response http.ResponseWriter, request *http.Request) { http.Error(response, "console ticket required", http.StatusUnauthorized) return } + relayContext, finish, ok := r.begin(request.Context()) + if !ok { + http.Error(response, "console gateway is draining", http.StatusServiceUnavailable) + return + } + defer finish() + request = request.WithContext(relayContext) initial, err := r.introspector.Inspect(request.Context(), IntrospectionRequest{Ticket: browserTicket, Consume: true}) - if err != nil || validateInitialResultWith(r.validateURL, initial, r.now().UTC()) != nil { + if err != nil || validateInitialResult(initial, r.now().UTC()) != nil { http.Error(response, "console authorization failed", http.StatusUnauthorized) return } - agentProtocol := AgentProtocolPrefix + initial.AgentTicket - agent, handshake, err := websocket.Dial(request.Context(), initial.AgentWebSocketURL, &websocket.DialOptions{ - HTTPClient: r.agentClient, Subprotocols: []string{agentProtocol}, CompressionMode: websocket.CompressionDisabled, - }) - if handshake != nil && handshake.Body != nil { - _ = handshake.Body.Close() - } - if err != nil || agent == nil || agent.Subprotocol() != agentProtocol { + agent, err := r.connector.OpenConsole(request.Context(), initial.AgentID, + initial.AgentTicket, initial.ConnectionID) + if err != nil || agent == nil { if agent != nil { _ = agent.CloseNow() } @@ -96,12 +99,75 @@ func (r *Relay) ServeHTTP(response http.ResponseWriter, request *http.Request) { r.relay(request.Context(), browser, agent, browserTicket, initial.ConnectionID) } -func validateInitialResultWith(validator func(string) error, result IntrospectionResult, now time.Time) error { - if !result.Active || !opaqueTokenPattern.MatchString(result.ConnectionID) || !opaqueTokenPattern.MatchString(result.AgentTicket) { - return errors.New("inactive or malformed console session") +func (r *Relay) begin(parent context.Context) (context.Context, func(), bool) { + r.lifecycleMu.Lock() + if r.draining { + r.lifecycleMu.Unlock() + return nil, nil, false } - if err := validator(result.AgentWebSocketURL); err != nil { - return err + ctx, cancel := context.WithCancel(parent) + r.next++ + id := r.next + r.cancels[id] = cancel + r.active.Add(1) + r.lifecycleMu.Unlock() + var once sync.Once + finish := func() { + once.Do(func() { + r.lifecycleMu.Lock() + delete(r.cancels, id) + r.lifecycleMu.Unlock() + cancel() + r.active.Done() + }) + } + return ctx, finish, true +} + +// BeginDrain rejects new browser console sessions while allowing already +// accepted relays to finish within the process shutdown grace period. +func (r *Relay) BeginDrain() { + r.lifecycleMu.Lock() + r.draining = true + r.lifecycleMu.Unlock() +} + +// Wait blocks until every accepted console handler has returned. BeginDrain +// must be called before Wait so no new WaitGroup additions can race the wait. +func (r *Relay) Wait(ctx context.Context) error { + done := make(chan struct{}) + go func() { + r.active.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Close cancels remaining hijacked WebSocket relays after the graceful drain +// deadline. Each relay owns and closes its browser and Agent sockets. +func (r *Relay) Close() { + r.BeginDrain() + r.lifecycleMu.Lock() + cancels := make([]context.CancelFunc, 0, len(r.cancels)) + for _, cancel := range r.cancels { + cancels = append(cancels, cancel) + } + r.lifecycleMu.Unlock() + for _, cancel := range cancels { + cancel() + } +} + +func validateInitialResult(result IntrospectionResult, now time.Time) error { + if !result.Active || !opaqueTokenPattern.MatchString(result.ConnectionID) || + !agentIDPattern.MatchString(result.AgentID) || + !opaqueTokenPattern.MatchString(result.AgentTicket) { + return errors.New("inactive or malformed console session") } expiresAt, err := time.Parse(time.RFC3339, result.AgentExpiresAt) if err != nil || !expiresAt.After(now) || expiresAt.After(now.Add(2*time.Minute)) { diff --git a/gateway/console/relay_test.go b/gateway/console/relay_test.go index f9234aa6..2d1b67ca 100644 --- a/gateway/console/relay_test.go +++ b/gateway/console/relay_test.go @@ -2,6 +2,7 @@ package console import ( "context" + "errors" "net/http" "net/http/httptest" "strings" @@ -13,10 +14,8 @@ import ( ) type fakeIntrospector struct { - agentURL string - agentTicket string - revoke bool - calls atomic.Int32 + revoke bool + calls atomic.Int32 } func (f *fakeIntrospector) Inspect(_ context.Context, request IntrospectionRequest) (IntrospectionResult, error) { @@ -26,7 +25,7 @@ func (f *fakeIntrospector) Inspect(_ context.Context, request IntrospectionReque return IntrospectionResult{}, errSessionRevoked } return IntrospectionResult{Active: true, ConnectionID: strings.Repeat("c", 32), - AgentWebSocketURL: f.agentURL, AgentTicket: f.agentTicket, + AgentID: "12", AgentTicket: strings.Repeat("a", 32), AgentExpiresAt: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)}, nil } if request.Consume || request.ConnectionID != strings.Repeat("c", 32) { @@ -35,12 +34,32 @@ func (f *fakeIntrospector) Inspect(_ context.Context, request IntrospectionReque return IntrospectionResult{Active: !f.revoke}, nil } -func newAgentServer(t *testing.T, ticket string) *httptest.Server { +type fakeConnector struct { + url string + client *http.Client + calls atomic.Int32 +} + +func (f *fakeConnector) OpenConsole(ctx context.Context, agentID, ticket, sessionID string) (*websocket.Conn, error) { + f.calls.Add(1) + if agentID != "12" || ticket != strings.Repeat("a", 32) || + sessionID != strings.Repeat("c", 32) { + return nil, errSessionRevoked + } + conn, response, err := websocket.Dial(ctx, f.url, &websocket.DialOptions{ + HTTPClient: f.client, CompressionMode: websocket.CompressionDisabled, + }) + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + return conn, err +} + +func newAgentServer(t *testing.T) *httptest.Server { t.Helper() - expected := AgentProtocolPrefix + ticket return httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { connection, err := websocket.Accept(response, request, &websocket.AcceptOptions{ - Subprotocols: []string{expected}, CompressionMode: websocket.CompressionDisabled, + CompressionMode: websocket.CompressionDisabled, }) if err != nil { return @@ -59,13 +78,11 @@ func newAgentServer(t *testing.T, ticket string) *httptest.Server { } func TestRelayForwardsOnlyBinaryAndConsumesTicketOnce(t *testing.T) { - agentTicket := strings.Repeat("a", 32) - agentServer := newAgentServer(t, agentTicket) + agentServer := newAgentServer(t) defer agentServer.Close() - introspector := &fakeIntrospector{agentURL: "wss" + strings.TrimPrefix(agentServer.URL, "https") + "/v1/console", agentTicket: agentTicket} - relay, _ := NewRelay(introspector, time.Second) - relay.agentClient = agentServer.Client() - relay.validateURL = func(string) error { return nil } + introspector := &fakeIntrospector{} + connector := &fakeConnector{url: "wss" + strings.TrimPrefix(agentServer.URL, "https"), client: agentServer.Client()} + relay, _ := NewRelay(introspector, connector) relay.recheckEvery = time.Hour server := httptest.NewServer(relay) defer server.Close() @@ -92,14 +109,11 @@ func TestRelayForwardsOnlyBinaryAndConsumesTicketOnce(t *testing.T) { } func TestRelayDisconnectsWhenPeriodicIntrospectionRevokes(t *testing.T) { - agentTicket := strings.Repeat("d", 32) - agentServer := newAgentServer(t, agentTicket) + agentServer := newAgentServer(t) defer agentServer.Close() - introspector := &fakeIntrospector{agentURL: "wss" + strings.TrimPrefix(agentServer.URL, "https") + "/v1/console", - agentTicket: agentTicket, revoke: true} - relay, _ := NewRelay(introspector, time.Second) - relay.agentClient = agentServer.Client() - relay.validateURL = func(string) error { return nil } + introspector := &fakeIntrospector{revoke: true} + connector := &fakeConnector{url: "wss" + strings.TrimPrefix(agentServer.URL, "https"), client: agentServer.Client()} + relay, _ := NewRelay(introspector, connector) relay.recheckEvery = 20 * time.Millisecond server := httptest.NewServer(relay) defer server.Close() @@ -116,7 +130,7 @@ func TestRelayDisconnectsWhenPeriodicIntrospectionRevokes(t *testing.T) { func TestRelayRejectsMissingOrCrossSiteOriginBeforeIntrospection(t *testing.T) { introspector := &fakeIntrospector{} - relay, _ := NewRelay(introspector, time.Second) + relay, _ := NewRelay(introspector, &fakeConnector{}) request := httptest.NewRequest(http.MethodGet, "http://panel.example/engine-console", nil) request.Host = "panel.example" request.Header.Set("Origin", "https://evil.example") @@ -131,7 +145,7 @@ func TestRelayRejectsMissingOrCrossSiteOriginBeforeIntrospection(t *testing.T) { func TestRelayRejectsNonWebSocketRequestBeforeConsumingTicket(t *testing.T) { introspector := &fakeIntrospector{} - relay, _ := NewRelay(introspector, time.Second) + relay, _ := NewRelay(introspector, &fakeConnector{}) request := httptest.NewRequest(http.MethodGet, "http://panel.example/engine-console", nil) request.Host = "panel.example" request.Header.Set("Origin", "https://panel.example") @@ -144,13 +158,81 @@ func TestRelayRejectsNonWebSocketRequestBeforeConsumingTicket(t *testing.T) { } } -func TestRelayHandshakeTimeoutDoesNotBecomeConnectionLifetime(t *testing.T) { - relay, err := NewRelay(&fakeIntrospector{}, time.Second) - if err != nil { +func TestRelayDrainRejectsNewConsoleBeforeTicketConsumption(t *testing.T) { + introspector := &fakeIntrospector{} + relay, _ := NewRelay(introspector, &fakeConnector{}) + relay.BeginDrain() + request := httptest.NewRequest(http.MethodGet, + "http://panel.example/engine-console", nil) + request.Host = "panel.example" + request.Header.Set("Origin", "http://panel.example") + request.Header.Set("X-Forwarded-Proto", "http") + request.Header.Set("Connection", "Upgrade") + request.Header.Set("Upgrade", "websocket") + request.Header.Set("Sec-WebSocket-Version", "13") + request.Header.Set("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==") + request.Header.Set("Sec-WebSocket-Protocol", + BrowserProtocolPrefix+strings.Repeat("b", 32)) + response := httptest.NewRecorder() + relay.ServeHTTP(response, request) + if response.Code != http.StatusServiceUnavailable || introspector.calls.Load() != 0 { + t.Fatalf("draining relay status=%d introspection_calls=%d", + response.Code, introspector.calls.Load()) + } + if err := relay.Wait(t.Context()); err != nil { t.Fatal(err) } - if relay.agentClient.Timeout != 0 { - t.Fatalf("agent WebSocket client has a lifetime timeout: %v", relay.agentClient.Timeout) +} + +func TestRelayCloseCancelsActiveHijackedConsole(t *testing.T) { + agentServer := newAgentServer(t) + defer agentServer.Close() + relay, _ := NewRelay(&fakeIntrospector{}, &fakeConnector{ + url: "wss" + strings.TrimPrefix(agentServer.URL, "https"), + client: agentServer.Client(), + }) + relay.recheckEvery = time.Hour + server := httptest.NewServer(relay) + defer server.Close() + connection := dialBrowser(t, server.URL, strings.Repeat("f", 32)) + defer connection.CloseNow() + relay.BeginDrain() + short, cancelShort := context.WithTimeout(t.Context(), 20*time.Millisecond) + if err := relay.Wait(short); !errors.Is(err, context.DeadlineExceeded) { + cancelShort() + t.Fatalf("drain did not wait for active console: %v", err) + } + cancelShort() + relay.Close() + wait, cancelWait := context.WithTimeout(t.Context(), time.Second) + defer cancelWait() + if err := relay.Wait(wait); err != nil { + t.Fatalf("active console did not close: %v", err) + } +} + +func TestRelayRequiresReverseConnector(t *testing.T) { + if _, err := NewRelay(&fakeIntrospector{}, nil); err == nil { + t.Fatal("relay accepted a missing reverse console connector") + } +} + +func TestValidateInitialResultRequiresLiveShortLivedAgentTicket(t *testing.T) { + now := time.Date(2026, time.October, 16, 12, 0, 0, 0, time.UTC) + valid := IntrospectionResult{ + Active: true, ConnectionID: strings.Repeat("c", 32), AgentID: "12", + AgentTicket: strings.Repeat("a", 32), AgentExpiresAt: now.Add(time.Minute).Format(time.RFC3339), + } + if err := validateInitialResult(valid, now); err != nil { + t.Fatalf("valid introspection result rejected: %v", err) + } + valid.AgentExpiresAt = now.Format(time.RFC3339) + if err := validateInitialResult(valid, now); err == nil { + t.Fatal("expired Agent ticket was accepted") + } + valid.AgentExpiresAt = now.Add(2*time.Minute + time.Second).Format(time.RFC3339) + if err := validateInitialResult(valid, now); err == nil { + t.Fatal("overlong Agent ticket was accepted") } } diff --git a/gateway/internal/protocol/hmac.go b/gateway/internal/protocol/hmac.go index 2cdbfc0b..32a183af 100644 --- a/gateway/internal/protocol/hmac.go +++ b/gateway/internal/protocol/hmac.go @@ -35,9 +35,8 @@ var ( ErrNonceStore = errors.New("request nonce store unavailable") ) -// Actor is deliberately small. The update helper only accepts a managing -// super-administrator; the console relay uses a fixed service actor when it -// calls PHP. +// Actor is deliberately small. Public callers never choose service identities; +// each local helper pins the exact service value it accepts. type Actor struct { UserID int64 `json:"user_id,omitempty"` IsAdmin bool `json:"is_admin,omitempty"` @@ -184,7 +183,8 @@ func decodeActor(value string) (Actor, error) { if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { return Actor{}, errors.New("actor contains trailing data") } - if actor.UserID < 0 || (actor.Service != "" && actor.Service != "console-relay") { + if actor.UserID < 0 || (actor.Service != "" && actor.Service != "agent-gateway" && + actor.Service != "php-panel") { return Actor{}, errors.New("invalid actor") } return actor, nil diff --git a/gateway/update/config_test.go b/gateway/update/config_test.go index 0c10bdd4..93fd501a 100644 --- a/gateway/update/config_test.go +++ b/gateway/update/config_test.go @@ -12,7 +12,7 @@ func validTestConfig(directory string) Config { SocketPath: RequiredSocketPath, RepositoryDir: directory, RemoteURL: RequiredRemoteURL, Branch: RequiredBranch, StatePath: filepath.Join(directory, "runtime", "update", "state.json"), ServiceSecret: strings.Repeat("s", 32), PHPCommand: "/usr/bin/php", PHPFPMUnit: "php-fpm.service", - ServiceUnits: []string{"virt-cloud-worker.service", "virt-cloud-console-relay.service"}, + ServiceUnits: []string{"virt-cloud-worker.service", "virt-cloud-agent-gateway.service"}, } } diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index ee47d599..12cb3fd4 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -8,13 +8,13 @@ use App\Services\Config; /** Initializes only the clean-break PHP business and Agent log schemas. */ final class CloudSetup { - private const SCHEMA_VERSION = 'go-v1'; + private const SCHEMA_VERSION = 'go-agent-wss-v1'; private const MAIN_TABLES = [ 'cloud_regions', 'cloud_network_lines', 'cloud_resource_pools', 'cloud_hosts', 'cloud_host_agents', 'cloud_templates', 'cloud_template_replicas', 'cloud_user_isos', 'cloud_user_iso_replicas', 'cloud_rescue_template_deletions', - 'cloud_upload_staging', + 'cloud_rescue_template_imports', 'cloud_upload_staging', 'cloud_host_resource_pool_memberships', 'cloud_host_network_attachments', 'cloud_public_ip_pools', 'cloud_host_capacity_policies', 'cloud_public_ips', 'cloud_offers', diff --git a/web/app/Controllers/CloudAdminController.php b/web/app/Controllers/CloudAdminController.php index bc96d367..c0978079 100644 --- a/web/app/Controllers/CloudAdminController.php +++ b/web/app/Controllers/CloudAdminController.php @@ -9,6 +9,7 @@ use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudInput; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudRescueTemplateDeletionService; +use App\Services\Cloud\CloudRescueTemplateImportService; use App\Services\Cloud\CloudTemplateCatalogService; use App\Services\Cloud\CloudUploadStagingService; use App\Services\Cloud\CloudUserIsoFeature; @@ -228,8 +229,6 @@ final class CloudAdminController throw new CloudError('请选择宿主机'); } $this->host($hostId); - $client = CloudAgentSettingsService::clientForHost((int) $hostId, - $method === 'POST' ? 900.0 : 20.0); if ($method === 'GET' && isset($args['delete_job'])) { return $response->withJson( CloudRescueTemplateDeletionService::job( @@ -238,18 +237,17 @@ final class CloudAdminController )->withHeader('Cache-Control', 'no-store'); } if ($method === 'GET' && isset($args['job_id'])) { - $jobId = (string) $args['job_id']; - if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $jobId) !== 1) { - throw new CloudError('救援镜像任务编号无效'); - } - $job = $client->request('GET', '/v1/operations/' - . rawurlencode($jobId), $actor); - $job['job_id'] = $jobId; - return $response->withJson($job) + return $response->withJson( + CloudRescueTemplateImportService::job( + (int) $hostId, (string) $args['job_id'], $actor + ) + ) ->withHeader('Cache-Control', 'no-store'); } if ($method === 'GET' && !isset($args['id'])) { - $catalog = $client->request( + $catalog = CloudAgentSettingsService::clientForHost( + (int) $hostId + )->request( 'GET', '/v1/rescue-templates', $actor ); return $response->withJson([ @@ -268,23 +266,18 @@ final class CloudAdminController && preg_match('/^[a-f0-9]{64}$/D', $digest) !== 1) { throw new CloudError('SHA-256 格式无效'); } - $operationId = bin2hex(random_bytes(16)); - $requestActor = $actor; - $requestActor['operation_id'] = $operationId; - $operation = $client->request('POST', '/v1/rescue-templates', - $requestActor, [ - 'operation_id' => $operationId, - 'id' => bin2hex(random_bytes(16)), - 'name' => $name, - 'source_url' => $url, - 'sha256' => $digest, - ]); - return $response->withJson([ - 'job_id' => $operationId, - 'status' => $operation['status'] ?? 'running', - 'status_url' => '/api/v1/admin/rescue-templates/import-jobs/' - . rawurlencode($operationId) . '?host_id=' . (int) $hostId, - ], 202)->withHeader('Cache-Control', 'no-store'); + return $response->withJson( + CloudRescueTemplateImportService::start( + (int) $hostId, + 'remote', + $name, + $url, + '', + $digest, + $actor + ), + 202 + )->withHeader('Cache-Control', 'no-store'); } if ($method === 'DELETE' && isset($args['id'])) { $id = (string) $args['id']; @@ -316,35 +309,18 @@ final class CloudAdminController $staging = CloudUploadStagingService::issue( $file, (int) $hostId, 'rescue-template' ); - $operationId = bin2hex(random_bytes(16)); - $requestActor = $actor; - $requestActor['operation_id'] = $operationId; - try { - $operation = CloudAgentSettingsService::clientForHost( - (int) $hostId - )->request('POST', '/v1/rescue-templates', $requestActor, [ - 'operation_id' => $operationId, - 'id' => bin2hex(random_bytes(16)), - 'name' => $name, - 'source_url' => $staging['url'], - 'source_token' => $staging['token'], - 'sha256' => $staging['sha256'], - ]); - $status = (string) ($operation['status'] ?? 'reserved'); - } catch (\App\Services\Cloud\CloudAgentError $error) { - if (!$error->retryable) { - throw $error; - } - // Dispatch may have crossed the network boundary. Poll the - // durable operation ID; never replay the binary import. - $status = 'reserved'; - } - return $response->withJson([ - 'job_id' => $operationId, - 'status' => $status, - 'status_url' => '/api/v1/admin/rescue-templates/import-jobs/' - . rawurlencode($operationId) . '?host_id=' . (int) $hostId, - ], 202)->withHeader('Cache-Control', 'no-store'); + return $response->withJson( + CloudRescueTemplateImportService::start( + (int) $hostId, + 'upload', + $name, + (string) $staging['url'], + (string) $staging['token'], + (string) $staging['sha256'], + $actor + ), + 202 + )->withHeader('Cache-Control', 'no-store'); } finally { CloudUploadStagingService::discard($file); } diff --git a/web/app/Controllers/CloudInternalController.php b/web/app/Controllers/CloudInternalController.php index 10ff555b..49ca60b1 100644 --- a/web/app/Controllers/CloudInternalController.php +++ b/web/app/Controllers/CloudInternalController.php @@ -5,10 +5,12 @@ namespace App\Controllers; use App\Services\Config; use App\Services\RedisClient; use App\Services\Cloud\CloudAgentClient; +use App\Services\Cloud\CloudAgentSecretCipher; use App\Services\Cloud\CloudConsoleTicketService; use App\Services\Cloud\CloudError; +use App\Models\CloudHostAgent; -/** Private HMAC endpoint used only by the local console relay. */ +/** Private HMAC endpoints used only by the local Agent gateway. */ final class CloudInternalController { public static function validSignature( @@ -17,7 +19,8 @@ final class CloudInternalController string $body, array $headers, string $key, - int $now + int $now, + string $expectedService = 'agent-gateway' ): bool { $time = $headers['timestamp'] ?? ''; $nonce = $headers['nonce'] ?? ''; @@ -36,7 +39,7 @@ final class CloudInternalController || !is_string($signature) || preg_match('/^[a-f0-9]{64}$/D', $signature) !== 1 || !is_string($actor) || strlen($actor) > 4096 - || !self::validRelayActor($actor)) { + || !self::validGatewayActor($actor, $expectedService)) { return false; } $expected = hash_hmac('sha256', CloudAgentClient::canonical( @@ -54,7 +57,7 @@ final class CloudInternalController { try { $body = (string) $request->getBody(); - $secret = (string) Config::get('cloud_console_relay_secret', ''); + $secret = (string) Config::get('cloud_agent_gateway_secret', ''); $headers = [ 'timestamp' => $request->getHeaderLine('X-Virt-Timestamp'), 'nonce' => $request->getHeaderLine('X-Virt-Nonce'), @@ -78,14 +81,7 @@ final class CloudInternalController return $response->withJson(['active' => false], 403) ->withHeader('Cache-Control', 'no-store'); } - $redis = (new RedisClient())->getClient(); - if ((string) $redis->set( - 'virt-console-introspection-nonce:' . $headers['nonce'], - '1', - 'EX', - 120, - 'NX' - ) !== 'OK') { + if (!self::consumeNonce($headers['nonce'])) { return $response->withJson(['active' => false], 403) ->withHeader('Cache-Control', 'no-store'); } @@ -113,7 +109,98 @@ final class CloudInternalController } } - private static function validRelayActor(string $encoded): bool + public function agentAuth($request, $response) + { + try { + $body = (string) $request->getBody(); + $secret = (string) Config::get('cloud_agent_gateway_secret', ''); + $headers = [ + 'timestamp' => $request->getHeaderLine('X-Virt-Timestamp'), + 'nonce' => $request->getHeaderLine('X-Virt-Nonce'), + 'digest' => $request->getHeaderLine('X-Virt-Content-SHA256'), + 'actor' => $request->getHeaderLine('X-Virt-Actor'), + 'signature' => $request->getHeaderLine('X-Virt-Signature'), + ]; + if ($request->getMethod() !== 'POST' + || $request->getUri()->getPath() + !== '/api/internal/cloud/agent-auth' + || $request->getUri()->getQuery() !== '' + || strlen($body) > 1024 + || !self::validSignature( + $request->getMethod(), + $request->getRequestTarget(), + $body, + $headers, + $secret, + time() + ) + || !self::consumeNonce($headers['nonce'])) { + return $response->withJson(['active' => false], 403) + ->withHeader('Cache-Control', 'no-store'); + } + try { + $value = json_decode($body, true, 8, JSON_THROW_ON_ERROR); + } catch (\Throwable $error) { + throw new CloudError('Agent 认证请求无效', 400); + } + if (!is_array($value) || array_is_list($value) + || count($value) !== 2 + || !array_key_exists('agent_id', $value) + || !array_key_exists('agent_secret', $value) + || !is_string($value['agent_id']) + || preg_match('/^[1-9][0-9]{0,18}$/D', $value['agent_id']) !== 1 + || !is_string($value['agent_secret']) + || strlen($value['agent_secret']) < 32 + || strlen($value['agent_secret']) > 256 + || preg_match('/^[\x21-\x7e]+$/D', $value['agent_secret']) !== 1) { + throw new CloudError('Agent 认证请求无效', 400); + } + $hostId = (int) $value['agent_id']; + $setting = CloudHostAgent::find($hostId); + if ($setting === null || !(bool) $setting->enabled + || trim((string) $setting->sealed_api_secret) === '') { + return $response->withJson(['active' => false], 403) + ->withHeader('Cache-Control', 'no-store'); + } + $expected = CloudAgentSecretCipher::decrypt( + (string) $setting->sealed_api_secret, + $hostId + ); + if (!hash_equals($expected, $value['agent_secret'])) { + return $response->withJson(['active' => false], 403) + ->withHeader('Cache-Control', 'no-store'); + } + return $response->withJson([ + 'active' => true, + 'agent_id' => (string) $hostId, + ])->withHeader('Cache-Control', 'no-store'); + } catch (CloudError $error) { + return $response->withJson(['active' => false], $error->httpStatus) + ->withHeader('Cache-Control', 'no-store'); + } catch (\Throwable $error) { + error_log('Cloud Agent authentication unavailable: ' + . get_class($error)); + return $response->withJson(['active' => false], 503) + ->withHeader('Cache-Control', 'no-store'); + } + } + + private static function consumeNonce(string $nonce): bool + { + $redis = (new RedisClient())->getClient(); + return (string) $redis->set( + 'virt-agent-gateway-nonce:' . $nonce, + '1', + 'EX', + 120, + 'NX' + ) === 'OK'; + } + + private static function validGatewayActor( + string $encoded, + string $expectedService + ): bool { if (preg_match('/^[A-Za-z0-9_-]+$/D', $encoded) !== 1) { return false; @@ -133,6 +220,6 @@ final class CloudInternalController } return is_array($actor) && !array_is_list($actor) && count($actor) === 1 - && ($actor['service'] ?? null) === 'console-relay'; + && ($actor['service'] ?? null) === $expectedService; } } diff --git a/web/app/Models/CloudRescueTemplateImport.php b/web/app/Models/CloudRescueTemplateImport.php new file mode 100644 index 00000000..28067db4 --- /dev/null +++ b/web/app/Models/CloudRescueTemplateImport.php @@ -0,0 +1,17 @@ + 'int', + 'created_at' => 'int', + 'updated_at' => 'int', + ]; +} diff --git a/web/app/Services/Cloud/CloudAgentClient.php b/web/app/Services/Cloud/CloudAgentClient.php index b95f368b..31fbb2d9 100644 --- a/web/app/Services/Cloud/CloudAgentClient.php +++ b/web/app/Services/Cloud/CloudAgentClient.php @@ -2,25 +2,45 @@ namespace App\Services\Cloud; +use App\Services\Config; use GuzzleHttp\Client; -/** Signed panel-to-host Agent client using only the per-host API secret. */ +/** Signed PHP-to-Agent client transported by the local outbound-connection gateway. */ final class CloudAgentClient { private Client $client; + private string $gatewayEndpoint; + private string $gatewaySecret; public function __construct( - private readonly string $endpoint, + private readonly string $agentId, private readonly string $apiSecret, ?Client $client = null, - private readonly float $timeoutSeconds = 20.0 + private readonly float $timeoutSeconds = 20.0, + ?string $gatewayEndpoint = null, + ?string $gatewaySecret = null ) { - if ($endpoint !== CloudAgentEndpoint::normalize($endpoint)) { - throw new \InvalidArgumentException('Agent endpoint must be normalized'); - } - if ($apiSecret === '' || $timeoutSeconds < 1 || $timeoutSeconds > 900) { + if (preg_match('/^[1-9][0-9]{0,18}$/D', $agentId) !== 1 + || strlen($apiSecret) < 32 || strlen($apiSecret) > 256 + || preg_match('/^[\x21-\x7e]+$/D', $apiSecret) !== 1 + || $timeoutSeconds < 1 || $timeoutSeconds > 900) { throw new \InvalidArgumentException('Invalid Agent client configuration'); } + $gatewayEndpoint ??= (string) Config::get( + 'cloud_agent_gateway_url', + CloudAgentGatewayEndpoint::URL + ); + $gatewaySecret ??= (string) Config::get('cloud_agent_gateway_secret', ''); + $this->gatewayEndpoint = CloudAgentGatewayEndpoint::normalize( + $gatewayEndpoint + ); + if (strlen($gatewaySecret) < 32 + || str_contains($gatewaySecret, "\0") + || str_contains($gatewaySecret, "\r") + || str_contains($gatewaySecret, "\n")) { + throw new \InvalidArgumentException('Invalid Agent gateway secret'); + } + $this->gatewaySecret = $gatewaySecret; $this->client = $client ?? new Client([ 'connect_timeout' => 3, 'timeout' => $timeoutSeconds, @@ -82,6 +102,44 @@ final class CloudAgentClient ]; } + /** Sign a local PHP-to-gateway request without exposing a panel user actor. */ + public static function gatewayHeaders( + string $method, + string $requestTarget, + string $body, + string $secret, + ?int $now = null, + ?string $nonce = null + ): array { + $actor = rtrim(strtr(base64_encode(json_encode( + ['service' => 'php-panel'], + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + )), '+/', '-_'), '='); + $timestamp = (string) ($now ?? time()); + $nonce ??= bin2hex(random_bytes(16)); + if (preg_match('/^[a-f0-9]{32}$/D', $nonce) !== 1 + || strlen($secret) < 32) { + throw new \InvalidArgumentException('Invalid gateway signing parameters'); + } + $digest = hash('sha256', $body); + return [ + 'X-Virt-Timestamp' => $timestamp, + 'X-Virt-Nonce' => $nonce, + 'X-Virt-Content-SHA256' => $digest, + 'X-Virt-Actor' => $actor, + 'X-Virt-Signature' => hash_hmac('sha256', self::canonical( + $method, + $requestTarget, + $digest, + $timestamp, + $nonce, + $actor + ), $secret), + 'Accept' => 'application/json', + 'Content-Type' => 'application/json', + ]; + } + public function request( string $method, string $requestTarget, @@ -93,13 +151,39 @@ final class CloudAgentClient $payload === [] ? (object) [] : $payload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); + $signed = self::headers( + $method, + $requestTarget, + $body, + $actor, + $this->apiSecret + ); + $transportId = bin2hex(random_bytes(16)); + $envelope = json_encode([ + 'agent_id' => $this->agentId, + 'type' => 'request', + 'id' => $transportId, + 'method' => strtoupper($method), + 'request_target' => $requestTarget, + 'body' => $body, + 'headers' => [ + 'X-Virt-Timestamp' => $signed['X-Virt-Timestamp'], + 'X-Virt-Nonce' => $signed['X-Virt-Nonce'], + 'X-Virt-Content-SHA256' => $signed['X-Virt-Content-SHA256'], + 'X-Virt-Actor' => $signed['X-Virt-Actor'], + 'X-Virt-Signature' => $signed['X-Virt-Signature'], + ], + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); try { - $response = $this->client->request(strtoupper($method), - $this->endpoint . $requestTarget, [ - 'headers' => self::headers( - $method, $requestTarget, $body, $actor, $this->apiSecret + $response = $this->client->request('POST', + $this->gatewayEndpoint, [ + 'headers' => self::gatewayHeaders( + 'POST', + CloudAgentGatewayEndpoint::PATH, + $envelope, + $this->gatewaySecret ), - 'body' => $body, + 'body' => $envelope, 'allow_redirects' => false, 'http_errors' => false, 'proxy' => '', @@ -107,14 +191,14 @@ final class CloudAgentClient ]); } catch (\GuzzleHttp\Exception\ConnectException $exception) { throw new CloudAgentError( - '无法连接宿主机 Agent,请检查地址、TLS 和防火墙', + '无法连接本机 Agent 网关,请检查网关服务', 502, - 'agent_unreachable', + 'agent_gateway_unreachable', true ); } catch (\Throwable $exception) { throw new CloudAgentError( - '宿主机 Agent 响应不确定,请核对实际状态后再重试', + 'Agent 网关响应不确定,请核对实际状态后再重试', 502, 'agent_transport_error', true @@ -240,8 +324,22 @@ final class CloudAgentClient && is_bool($error['retryable'] ?? null) && $error['retryable'] === false && !array_key_exists('operation', $error); + $safePreDispatchFailure = in_array($status, [502, 503], true) + && count($payload) === 1 + && array_key_exists('error', $payload) + && count($error) === 4 + && array_diff_key($error, array_flip([ + 'code', 'message', 'retryable', 'dispatch_state', + ])) === [] + && in_array($code, [ + 'agent_busy', 'agent_disconnected', 'gateway_draining', + ], true) + && is_string($error['message'] ?? null) + && ($error['retryable'] ?? null) === true + && ($error['dispatch_state'] ?? null) === 'not_dispatched'; return new CloudAgentError($message, $panelStatus, $code, - $retryable, $status, $safeCreateFailure); + $retryable, $status, $safeCreateFailure, + $safePreDispatchFailure); } private static function agentActor(array $actor): array @@ -267,7 +365,9 @@ final class CloudAgentClient { if (($requestTarget !== '/v1' && !str_starts_with($requestTarget, '/v1/')) || strlen($requestTarget) > 2048 - || preg_match('/[\x00-\x20\x7f\\#]/', $requestTarget) === 1 + || preg_match('/[\x00-\x20\x7f]/', $requestTarget) === 1 + || str_contains($requestTarget, '\\') + || str_contains($requestTarget, '#') || str_contains(strtolower($requestTarget), '%0a') || str_contains(strtolower($requestTarget), '%0d')) { throw new \InvalidArgumentException('Invalid Agent request target'); diff --git a/web/app/Services/Cloud/CloudAgentEndpoint.php b/web/app/Services/Cloud/CloudAgentEndpoint.php deleted file mode 100644 index 0294a09b..00000000 --- a/web/app/Services/Cloud/CloudAgentEndpoint.php +++ /dev/null @@ -1,74 +0,0 @@ - 255 - || preg_match('/[\x00-\x20\x7f]/', $value) === 1) { - throw new CloudError('Agent API 地址需为不超过 255 字符的固定 IP URL'); - } - $parts = parse_url($value); - if (!is_array($parts)) { - throw new CloudError('Agent API 地址格式无效'); - } - $scheme = strtolower((string) ($parts['scheme'] ?? '')); - $hostValue = (string) ($parts['host'] ?? ''); - $host = trim($hostValue, '[]'); - if ($scheme !== 'https' - || $host === '' || filter_var($host, FILTER_VALIDATE_IP) === false - || isset($parts['user']) || isset($parts['pass']) - || isset($parts['query']) || isset($parts['fragment']) - || (($parts['path'] ?? '') !== '' && ($parts['path'] ?? '') !== '/')) { - throw new CloudError('Agent API 地址必须是无账号、路径、查询参数的固定 IP HTTPS URL'); - } - if (self::blockedAddress($host)) { - throw new CloudError('Agent API 地址不能使用回环、链路本地、元数据、组播或未指定地址'); - } - $port = $parts['port'] ?? null; - if ($port !== null && ($port < 1 || $port > 65535)) { - throw new CloudError('Agent API 端口无效'); - } - $renderedHost = str_contains($host, ':') ? '[' . strtolower($host) . ']' : $host; - return $scheme . '://' . $renderedHost . ($port === null ? '' : ':' . $port); - } - - private static function blockedAddress(string $host): bool - { - if (filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false) { - $value = self::ipv4($host); - return self::ipv4In($value, '0.0.0.0', 8) - || self::ipv4In($value, '127.0.0.0', 8) - || self::ipv4In($value, '169.254.0.0', 16) - || self::ipv4In($value, '224.0.0.0', 4) - || self::ipv4In($value, '240.0.0.0', 4); - } - $packed = inet_pton($host); - if ($packed === false) { - return true; - } - return $packed === str_repeat("\0", 16) - || $packed === str_repeat("\0", 15) . "\1" - || (ord($packed[0]) === 0xfe && (ord($packed[1]) & 0xc0) === 0x80) - || ord($packed[0]) === 0xff - || substr($packed, 0, 12) === str_repeat("\0", 10) . "\xff\xff"; - } - - private static function ipv4(string $address): int - { - return (int) sprintf('%u', ip2long($address)); - } - - private static function ipv4In(int $value, string $network, int $prefix): bool - { - $mask = $prefix === 0 ? 0 : ((0xffffffff << (32 - $prefix)) & 0xffffffff); - return ($value & $mask) === (self::ipv4($network) & $mask); - } -} diff --git a/web/app/Services/Cloud/CloudAgentError.php b/web/app/Services/Cloud/CloudAgentError.php index 43971b35..6713bd0d 100644 --- a/web/app/Services/Cloud/CloudAgentError.php +++ b/web/app/Services/Cloud/CloudAgentError.php @@ -11,7 +11,9 @@ final class CloudAgentError extends CloudError public readonly bool $retryable = false, public readonly ?int $agentHttpStatus = null, /** Exact 400/422 error envelope without an Agent operation record. */ - public readonly bool $safeCreateFailure = false + public readonly bool $safeCreateFailure = false, + /** Gateway/Agent proof that the API handler was never entered. */ + public readonly bool $safePreDispatchFailure = false ) { parent::__construct($message, $httpStatus); } diff --git a/web/app/Services/Cloud/CloudAgentGatewayEndpoint.php b/web/app/Services/Cloud/CloudAgentGatewayEndpoint.php new file mode 100644 index 00000000..fa8ede00 --- /dev/null +++ b/web/app/Services/Cloud/CloudAgentGatewayEndpoint.php @@ -0,0 +1,38 @@ + 2048 + || preg_match('/[\x00-\x20\x7f\\\\]/', $value) === 1) { + throw new \InvalidArgumentException('Agent gateway URL is invalid'); + } + $parts = parse_url($value); + if (!is_array($parts)) { + throw new \InvalidArgumentException('Agent gateway URL is invalid'); + } + $scheme = strtolower((string) ($parts['scheme'] ?? '')); + $host = (string) ($parts['host'] ?? ''); + if ($scheme !== 'http' || $host !== '127.0.0.1' + || ($parts['path'] ?? '') !== self::PATH + || isset($parts['user']) || isset($parts['pass']) + || isset($parts['query']) || isset($parts['fragment']) + || ($parts['port'] ?? null) !== 8001) { + throw new \InvalidArgumentException( + 'Agent gateway URL must be the fixed 127.0.0.1:8001 dispatch endpoint' + ); + } + return self::URL; + } +} diff --git a/web/app/Services/Cloud/CloudAgentSettingsService.php b/web/app/Services/Cloud/CloudAgentSettingsService.php index 15d0b073..2e239cc6 100644 --- a/web/app/Services/Cloud/CloudAgentSettingsService.php +++ b/web/app/Services/Cloud/CloudAgentSettingsService.php @@ -55,9 +55,8 @@ final class CloudAgentSettingsService public static function save(int $hostId, array $input): array { CloudInput::only($input, [ - 'endpoint', 'enabled', 'api_secret', 'expected_revision', + 'enabled', 'api_secret', 'expected_revision', ]); - $endpoint = CloudAgentEndpoint::normalize($input['endpoint'] ?? null); $enabled = filter_var($input['enabled'] ?? null, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); $secret = $input['api_secret'] ?? ''; @@ -78,7 +77,7 @@ final class CloudAgentSettingsService $host = self::host($hostId); $connection = (new CloudHostAgent())->getConnection(); return $connection->transaction(static function () use ( - $host, $hostId, $endpoint, $enabled, $secret, $expectedRevision + $host, $hostId, $enabled, $secret, $expectedRevision ): array { $setting = CloudHostAgent::query()->where('host_id', $hostId) ->lockForUpdate()->first(); @@ -96,7 +95,6 @@ final class CloudAgentSettingsService } elseif ($secret === '' && trim((string) $setting->sealed_api_secret) === '') { throw new CloudError('启用 Agent 前必须配置 API 密钥'); } - $setting->endpoint = $endpoint; $setting->enabled = $enabled; if ($secret !== '') { $setting->sealed_api_secret = CloudAgentSecretCipher::encrypt($secret, $hostId); @@ -228,7 +226,7 @@ final class CloudAgentSettingsService ): CloudAgentClient { return new CloudAgentClient( - (string) $setting->endpoint, + (string) $setting->host_id, CloudAgentSecretCipher::decrypt( (string) $setting->sealed_api_secret, (int) $setting->host_id @@ -277,7 +275,7 @@ final class CloudAgentSettingsService 'region' => (string) $host->region, 'country_code' => (string) $host->country_code, 'host_enabled' => (bool) $host->enabled, - 'endpoint' => $setting === null ? '' : (string) $setting->endpoint, + 'agent_id' => (string) $host->id, 'enabled' => $setting !== null && (bool) $setting->enabled, 'secret_configured' => $setting !== null && trim((string) $setting->sealed_api_secret) !== '', diff --git a/web/app/Services/Cloud/CloudConsoleTicketService.php b/web/app/Services/Cloud/CloudConsoleTicketService.php index b2b21ae9..54615ce5 100644 --- a/web/app/Services/Cloud/CloudConsoleTicketService.php +++ b/web/app/Services/Cloud/CloudConsoleTicketService.php @@ -7,7 +7,7 @@ use App\Models\CloudHostAgent; use App\Models\CloudInstance; use App\Models\CloudSession; -/** Panel-owned, one-use browser ticket mapping for the local console relay. */ +/** Panel-owned, one-use browser ticket mapping for the local Agent gateway. */ final class CloudConsoleTicketService { private const BROWSER_TTL = 45; @@ -40,10 +40,6 @@ final class CloudConsoleTicketService 503 ); } - $endpoint = CloudAgentEndpoint::normalize((string) $setting->endpoint); - if (!str_starts_with($endpoint, 'https://')) { - throw new CloudError('控制台连接要求宿主机 Agent 使用 HTTPS', 503); - } $now = time(); $ticketExpiry = min($agentExpiry, $now + self::BROWSER_TTL); if ($ticketExpiry <= $now) { @@ -57,8 +53,6 @@ final class CloudConsoleTicketService $record->instance_id = (int) $instance->id; $record->host_id = (int) $instance->host_id; $record->vm_uuid = strtolower((string) $instance->uuid); - $record->agent_websocket_url = 'wss://' . substr($endpoint, 8) - . '/v1/console'; $record->sealed_agent_ticket = CloudSecrets::encrypt([ 'ticket' => $ticket, 'host_id' => (int) $instance->host_id, @@ -128,6 +122,10 @@ final class CloudConsoleTicketService )) { return ['active' => false]; } + $setting = CloudHostAgent::find((int) $record->host_id); + if ($setting === null || !$setting->isReady()) { + return ['active' => false]; + } if ($consume) { if ($record->consumed_at !== null || (int) $record->ticket_expires_at <= $now @@ -154,7 +152,7 @@ final class CloudConsoleTicketService return [ 'active' => true, 'connection_id' => $newConnectionId, - 'agent_websocket_url' => (string) $record->agent_websocket_url, + 'agent_id' => (string) $record->host_id, 'agent_ticket' => $sealed['ticket'], 'agent_expires_at' => gmdate( 'Y-m-d\\TH:i:s\\Z', @@ -174,14 +172,12 @@ final class CloudConsoleTicketService private static function validateAgentTicket(array $value): array { $ticket = $value['ticket'] ?? null; - $url = $value['url'] ?? null; - $protocol = $value['protocol'] ?? null; $expiresAt = $value['expires_at'] ?? null; - if (!is_string($ticket) + if (array_is_list($value) || count($value) !== 2 + || !array_key_exists('ticket', $value) + || !array_key_exists('expires_at', $value) + || !is_string($ticket) || preg_match('/^[A-Za-z0-9_-]{32,256}$/D', $ticket) !== 1 - || $url !== '/v1/console' - || !is_string($protocol) - || !hash_equals('cloud-agent-ticket.' . $ticket, $protocol) || !is_string($expiresAt)) { throw new CloudError('宿主机 Agent 返回了无效控制台票据', 502); } diff --git a/web/app/Services/Cloud/CloudRescueTemplateDeletionService.php b/web/app/Services/Cloud/CloudRescueTemplateDeletionService.php index b722d936..116a8055 100644 --- a/web/app/Services/Cloud/CloudRescueTemplateDeletionService.php +++ b/web/app/Services/Cloud/CloudRescueTemplateDeletionService.php @@ -32,7 +32,7 @@ final class CloudRescueTemplateDeletionService ]); self::applyOperation($record, $operation); } catch (CloudAgentError $error) { - if ($error->retryable) { + if ($error->retryable && !$error->safePreDispatchFailure) { self::mark($record, 'running', 'agent_unavailable'); } else { self::mark($record, 'failed', $error->agentCode); diff --git a/web/app/Services/Cloud/CloudRescueTemplateImportService.php b/web/app/Services/Cloud/CloudRescueTemplateImportService.php new file mode 100644 index 00000000..6f113adc --- /dev/null +++ b/web/app/Services/Cloud/CloudRescueTemplateImportService.php @@ -0,0 +1,276 @@ + $operationId, + 'id' => $templateId, + 'name' => $name, + 'source_url' => $sourceUrl, + 'sha256' => $sha256, + ]; + if ($sourceToken !== '') { + $body['source_token'] = $sourceToken; + } + + try { + $operation = $client->request( + 'POST', '/v1/rescue-templates', $requestActor, $body + ); + self::applyOperation($record, $operation); + } catch (CloudAgentError $error) { + self::mark( + $record, + $error->safePreDispatchFailure + ? 'failed' : ($error->retryable ? 'running' : 'failed'), + $error->agentCode + ); + } catch (\Throwable $error) { + // The request may already have crossed the dispatch boundary. + // Preserve the operation ID and require explicit reconciliation. + self::mark($record, 'uncertain', 'import_dispatch_unconfirmed'); + } + return self::dto(self::reload($record)); + } + + public static function job(int $hostId, string $operationId, + array $actor): array + { + self::validateOperationId($operationId); + $record = CloudRescueTemplateImport::query() + ->where('host_id', $hostId) + ->where('operation_id', $operationId)->first(); + if ($record === null) { + throw new CloudError('救援镜像导入任务不存在', 404); + } + return self::pollIfAllowed($record, $actor); + } + + private static function reserve(string $operationId, int $hostId, + string $templateId, string $importType): CloudRescueTemplateImport + { + $record = new CloudRescueTemplateImport(); + $record->operation_id = $operationId; + $record->host_id = $hostId; + $record->template_id = $templateId; + $record->import_type = $importType; + $record->state = 'dispatching'; + $record->error_code = ''; + $record->created_at = $record->updated_at = time(); + $record->save(); + return $record; + } + + private static function pollIfAllowed(CloudRescueTemplateImport $record, + array $actor): array + { + $state = (string) $record->state; + if (in_array($state, ['dispatching', 'sending'], true)) { + if ((int) $record->updated_at >= time() - 90) { + return self::dto($record); + } + self::mark($record, 'uncertain', 'import_dispatch_unconfirmed'); + return self::dto(self::reload($record)); + } + if ($state !== 'running') { + return self::dto($record); + } + + try { + $operation = CloudAgentSettingsService::clientForHost( + (int) $record->host_id + )->request('GET', '/v1/operations/' + . rawurlencode((string) $record->operation_id), $actor); + self::applyOperation($record, $operation); + } catch (CloudAgentError $error) { + if ($error->agentHttpStatus === 404) { + self::mark($record, 'uncertain', 'operation_not_found'); + } else { + // An outage never authorizes a second import POST. Keep the + // same operation available for later reconciliation. + self::mark($record, 'running', 'agent_unavailable'); + } + } catch (\Throwable $error) { + self::mark($record, 'running', 'agent_unavailable'); + } + return self::dto(self::reload($record)); + } + + private static function applyOperation(CloudRescueTemplateImport $record, + array $operation): void + { + if (($operation['operation_id'] ?? null) + !== (string) $record->operation_id) { + self::mark($record, 'uncertain', 'operation_id_mismatch'); + return; + } + $status = strtolower(trim((string) ($operation['status'] ?? ''))); + if ($status === 'succeeded') { + $result = $operation['result'] ?? null; + if (!is_array($result) || array_is_list($result) + || ($result['id'] ?? null) !== (string) $record->template_id) { + self::mark($record, 'uncertain', 'agent_result_mismatch'); + return; + } + self::mark($record, 'succeeded', ''); + return; + } + if ($status === 'failed') { + self::mark($record, 'failed', self::operationErrorCode( + $operation, 'agent_import_failed' + )); + return; + } + if ($status === 'uncertain') { + self::mark($record, 'uncertain', self::operationErrorCode( + $operation, 'agent_import_uncertain' + )); + return; + } + if (in_array($status, + ['reserved', 'pending', 'running', 'dispatching'], true)) { + self::mark($record, 'running', ''); + return; + } + self::mark($record, 'uncertain', 'agent_status_invalid'); + } + + private static function mark(CloudRescueTemplateImport $record, + string $state, string $errorCode): void + { + $current = (string) $record->state; + $allowed = [ + 'dispatching' => ['sending', 'failed', 'uncertain'], + 'sending' => ['running', 'succeeded', 'failed', 'uncertain'], + 'running' => ['running', 'succeeded', 'failed', 'uncertain'], + ]; + if (!isset($allowed[$current]) + || !in_array($state, $allowed[$current], true)) { + return; + } + CloudRescueTemplateImport::query() + ->where('operation_id', (string) $record->operation_id) + ->where('host_id', (int) $record->host_id) + ->where('template_id', (string) $record->template_id) + ->where('state', $current) + ->update([ + 'state' => $state, + 'error_code' => substr($errorCode, 0, 64), + 'updated_at' => time(), + ]); + } + + private static function reload( + CloudRescueTemplateImport $record + ): CloudRescueTemplateImport { + $fresh = CloudRescueTemplateImport::query() + ->where('operation_id', (string) $record->operation_id) + ->where('host_id', (int) $record->host_id) + ->where('template_id', (string) $record->template_id)->first(); + if ($fresh === null) { + throw new CloudError('救援镜像导入任务状态丢失', 503); + } + return $fresh; + } + + private static function dto(CloudRescueTemplateImport $record): array + { + $state = (string) $record->state; + $status = in_array($state, ['failed', 'uncertain', 'succeeded'], true) + ? $state : 'running'; + $result = [ + 'job_id' => (string) $record->operation_id, + 'operation_id' => (string) $record->operation_id, + 'host_id' => (int) $record->host_id, + 'template_id' => (string) $record->template_id, + 'import_type' => (string) $record->import_type, + 'status' => $status, + 'phase' => $status === 'succeeded' + ? '救援镜像导入已完成' + : ($status === 'running' ? '正在核对 Agent 导入结果' + : '救援镜像导入未得到安全确认,禁止自动重试'), + 'status_url' => '/api/v1/admin/rescue-templates/import-jobs/' + . rawurlencode((string) $record->operation_id) + . '?host_id=' . (int) $record->host_id, + ]; + if (in_array($status, ['failed', 'uncertain'], true)) { + $result['error'] = $status === 'uncertain' + ? 'Agent 导入结果不确定,请按 operation_id 人工核对' + : 'Agent 明确拒绝或无法开始救援镜像导入'; + $result['error_code'] = (string) $record->error_code; + } + return $result; + } + + private static function operationErrorCode(array $operation, + string $fallback): string + { + $error = is_array($operation['error'] ?? null) + ? $operation['error'] : []; + $code = $error['code'] ?? ($operation['error_code'] ?? null); + return is_string($code) + && preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $code) === 1 + ? $code : $fallback; + } + + private static function validateInput(int $hostId, string $importType, + string $name, string $sourceUrl, string $sourceToken, + string $sha256): void + { + if ($hostId < 1 || !in_array($importType, ['remote', 'upload'], true) + || $name === '' || mb_strlen($name, 'UTF-8') > 64 + || $sourceUrl === '' || strlen($sourceUrl) > 4096 + || ($sourceToken !== '' + && preg_match('/^[A-Za-z0-9_-]{32,512}$/D', $sourceToken) + !== 1) + || ($sha256 !== '' + && preg_match('/^[a-f0-9]{64}$/D', $sha256) !== 1)) { + throw new \InvalidArgumentException( + 'Invalid rescue-template import reservation' + ); + } + } + + private static function validateOperationId(string $operationId): void + { + if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) { + throw new CloudError('救援镜像导入任务编号无效'); + } + } +} diff --git a/web/app/Services/Cloud/CloudSystemJournalService.php b/web/app/Services/Cloud/CloudSystemJournalService.php index 3a6aad56..de43c7ed 100644 --- a/web/app/Services/Cloud/CloudSystemJournalService.php +++ b/web/app/Services/Cloud/CloudSystemJournalService.php @@ -34,7 +34,7 @@ final class CloudSystemJournalService 'virt-cloud-policy-worker@1.service', 'virt-cloud-policy-worker@2.service', 'virt-cloud-update.service', - 'virt-cloud-console-relay.service', + 'virt-cloud-agent-gateway.service', ]; private const PRIORITIES = [ diff --git a/web/app/Services/Cloud/CloudTemplateCatalogService.php b/web/app/Services/Cloud/CloudTemplateCatalogService.php index 2caf39b1..3e02a08c 100644 --- a/web/app/Services/Cloud/CloudTemplateCatalogService.php +++ b/web/app/Services/Cloud/CloudTemplateCatalogService.php @@ -98,37 +98,40 @@ final class CloudTemplateCatalogService public static function importJob(string $id, array $actor): array { $template = self::find($id); - if (in_array((string) $template->state, ['importing', 'error'], true) - && CloudTemplateReplica::query()->where('template_id', + if (in_array((string) $template->state, + ['importing', 'error'], true)) { + if (CloudTemplateReplica::query()->where('template_id', (string) $template->id)->where('state', 'pending')->exists()) { - foreach (CloudTemplateReplica::query() - ->where('template_id', (string) $template->id) - ->where('state', 'pending')->orderBy('host_id')->get() - as $replica) { - $operationId = (string) $replica->operation_id; - if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) - !== 1) { - self::markReplica((int) $replica->host_id, - (string) $template->id, 'uncertain', - 'operation_id_invalid'); - continue; - } - try { - $operation = CloudAgentSettingsService::clientForHost( - (int) $replica->host_id - )->request('GET', '/v1/operations/' - . rawurlencode($operationId), $actor); - self::applyImportOperation($template, - (int) $replica->host_id, $operationId, $operation); - } catch (CloudAgentError $error) { - if ($error->httpStatus === 404) { + foreach (CloudTemplateReplica::query() + ->where('template_id', (string) $template->id) + ->where('state', 'pending')->orderBy('host_id')->get() + as $replica) { + $operationId = (string) $replica->operation_id; + if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) + !== 1) { self::markReplica((int) $replica->host_id, - (string) $template->id, 'uncertain', - 'operation_not_found'); - } else { + (string) $template->id, $operationId, ['pending'], + 'uncertain', 'operation_id_invalid'); + continue; + } + try { + $operation = CloudAgentSettingsService::clientForHost( + (int) $replica->host_id + )->request('GET', '/v1/operations/' + . rawurlencode($operationId), $actor); + self::applyImportOperation($template, + (int) $replica->host_id, $operationId, $operation); + } catch (CloudAgentError $error) { self::markReplica((int) $replica->host_id, - (string) $template->id, 'pending', - 'agent_unavailable'); + (string) $template->id, $operationId, ['pending'], + $error->httpStatus === 404 + ? 'uncertain' : 'pending', + $error->httpStatus === 404 + ? 'operation_not_found' : 'agent_unavailable'); + } catch (\Throwable $error) { + self::markReplica((int) $replica->host_id, + (string) $template->id, $operationId, ['pending'], + 'uncertain', 'panel_persist_failed'); } } } @@ -288,8 +291,9 @@ final class CloudTemplateCatalogService $requestActor, ['operation_id' => $operationId]); self::applyDeleteOperation($replica, $operation); } catch (CloudAgentError $error) { - self::markReplica($hostId, $id, - $error->retryable + self::markReplica($hostId, $id, $operationId, + ['delete_sending'], + $error->retryable && !$error->safePreDispatchFailure ? 'delete_running' : 'delete_failed', $error->agentCode); } @@ -311,7 +315,8 @@ final class CloudTemplateCatalogService ->orderBy('host_id')->get() as $replica) { $operationId = (string) $replica->operation_id; if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) { - self::markReplica((int) $replica->host_id, $id, + self::markReplica((int) $replica->host_id, $id, $operationId, + ['delete_running'], 'delete_uncertain', 'operation_id_invalid'); continue; } @@ -322,7 +327,8 @@ final class CloudTemplateCatalogService . rawurlencode($operationId), $actor); self::applyDeleteOperation($replica, $operation); } catch (CloudAgentError $error) { - self::markReplica((int) $replica->host_id, $id, + self::markReplica((int) $replica->host_id, $id, $operationId, + ['delete_running'], $error->agentHttpStatus === 404 ? 'delete_uncertain' : 'delete_running', $error->agentHttpStatus === 404 @@ -336,15 +342,18 @@ final class CloudTemplateCatalogService { $hostId = (int) $replica->host_id; $id = (string) $replica->template_id; + $operationId = (string) $replica->operation_id; if (($operation['operation_id'] ?? null) - !== (string) $replica->operation_id) { - self::markReplica($hostId, $id, 'delete_uncertain', + !== $operationId) { + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], 'delete_uncertain', 'operation_id_mismatch'); return; } $status = strtolower(trim((string) ($operation['status'] ?? ''))); if ($status === 'succeeded') { - self::markReplica($hostId, $id, + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], CloudAgentCatalogDeletion::confirms( $operation, 'template', $id ) ? 'delete_succeeded' : 'delete_uncertain', @@ -358,21 +367,26 @@ final class CloudTemplateCatalogService if ($status === 'failed') { $confirmedMissing = $code === 'template_not_found' && ($operation['retryable'] ?? null) === false; - self::markReplica($hostId, $id, + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], $confirmedMissing ? 'delete_succeeded' : 'delete_failed', $code); return; } if ($status === 'uncertain') { - self::markReplica($hostId, $id, 'delete_uncertain', $code); + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], 'delete_uncertain', + $code); return; } if (in_array($status, ['reserved', 'pending', 'running', 'dispatching'], true)) { - self::markReplica($hostId, $id, 'delete_running', ''); + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], 'delete_running', ''); return; } - self::markReplica($hostId, $id, 'delete_uncertain', + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], 'delete_uncertain', 'agent_status_invalid'); } @@ -832,15 +846,26 @@ final class CloudTemplateCatalogService $operation = CloudAgentSettingsService::clientForHost( $hostId )->request('POST', '/v1/templates', $requestActor, $body); - self::applyImportOperation($template, $hostId, $operationId, - $operation); } catch (CloudAgentError $error) { self::markReplica($hostId, (string) $template->id, - $error->retryable ? 'pending' : 'failed', + $operationId, ['pending'], + $error->retryable && !$error->safePreDispatchFailure + ? 'pending' : 'failed', $error->agentCode); + continue; } catch (\Throwable $error) { - self::markReplica($hostId, (string) $template->id, 'failed', + self::markReplica($hostId, (string) $template->id, + $operationId, ['pending'], 'failed', 'import_dispatch_failed'); + continue; + } + try { + self::applyImportOperation($template, $hostId, $operationId, + $operation); + } catch (\Throwable $error) { + self::markReplica($hostId, (string) $template->id, + $operationId, ['pending'], 'uncertain', + 'panel_persist_failed'); } } self::refreshImportState($template); @@ -851,6 +876,11 @@ final class CloudTemplateCatalogService private static function applyImportOperation(CloudTemplate $template, int $hostId, string $operationId, array $operation): void { + if (($operation['operation_id'] ?? null) !== $operationId) { + self::markReplica($hostId, (string) $template->id, $operationId, + ['pending'], 'uncertain', 'operation_id_mismatch'); + return; + } $status = strtolower(trim((string) ($operation['status'] ?? ''))); if ($status === 'succeeded') { self::acceptImportResult($template, $hostId, $operationId, @@ -858,21 +888,25 @@ final class CloudTemplateCatalogService return; } if ($status === 'failed') { - self::markReplica($hostId, (string) $template->id, 'failed', + self::markReplica($hostId, (string) $template->id, $operationId, + ['pending'], 'failed', self::operationErrorCode($operation, 'agent_import_failed')); return; } if (in_array($status, ['uncertain', 'dispatching'], true)) { - self::markReplica($hostId, (string) $template->id, 'uncertain', + self::markReplica($hostId, (string) $template->id, $operationId, + ['pending'], 'uncertain', self::operationErrorCode($operation, 'agent_import_uncertain')); return; } if (in_array($status, ['reserved', 'pending', 'running'], true)) { - self::markReplica($hostId, (string) $template->id, 'pending', ''); + self::markReplica($hostId, (string) $template->id, $operationId, + ['pending'], 'pending', ''); return; } - self::markReplica($hostId, (string) $template->id, 'uncertain', + self::markReplica($hostId, (string) $template->id, $operationId, + ['pending'], 'uncertain', 'agent_status_invalid'); } @@ -897,18 +931,65 @@ final class CloudTemplateCatalogService && !hash_equals((string) $template->sha256, $digest)) || ((int) $template->image_bytes > 0 && (int) $template->image_bytes !== (int) $bytes)) { - self::markReplica($hostId, (string) $template->id, 'uncertain', + self::markReplica($hostId, (string) $template->id, $operationId, + ['pending'], 'uncertain', 'agent_result_mismatch'); return; } - if ((string) $template->sha256 === '') { - $template->sha256 = $digest; - $template->image_bytes = (int) $bytes; - $template->updated_at = time(); - $template->save(); - } - self::writeReplica($hostId, $template, 'active', $operationId, '', - (int) $bytes, $digest); + $template->getConnection()->transaction(static function () use ( + $template, $hostId, $operationId, $digest, $bytes + ): void { + $locked = CloudTemplate::query()->where('id', + (string) $template->id)->whereIn('state', + ['importing', 'error'])->lockForUpdate()->first(); + if ($locked === null) { + return; + } + $replica = CloudTemplateReplica::query() + ->where('host_id', $hostId) + ->where('template_id', (string) $template->id) + ->where('operation_id', $operationId) + ->where('state', 'pending')->lockForUpdate()->first(); + if ($replica === null) { + return; + } + if ((int) $locked->revision !== (int) $template->revision + || (string) $locked->generation_token + !== (string) $template->generation_token + || ((string) $locked->sha256 !== '' + && !hash_equals((string) $locked->sha256, $digest)) + || ((int) $locked->image_bytes > 0 + && (int) $locked->image_bytes !== (int) $bytes)) { + self::markReplica($hostId, (string) $template->id, + $operationId, ['pending'], 'uncertain', + 'agent_result_mismatch'); + return; + } + if ((string) $locked->sha256 === '') { + $locked->sha256 = $digest; + $locked->image_bytes = (int) $bytes; + $locked->updated_at = time(); + $locked->save(); + } + $updated = CloudTemplateReplica::query() + ->where('host_id', $hostId) + ->where('template_id', (string) $template->id) + ->where('operation_id', $operationId) + ->where('state', 'pending')->update([ + 'revision' => (int) $locked->revision, + 'generation_token' => (string) $locked->generation_token, + 'state' => 'active', + 'image_bytes' => (int) $bytes, + 'sha256' => $digest, + 'error_code' => '', + 'updated_at' => time(), + ]); + if ($updated !== 1) { + throw new \RuntimeException( + 'Template replica import transition was lost' + ); + } + }, 3); } private static function refreshImportState(CloudTemplate $template): void @@ -929,8 +1010,11 @@ final class CloudTemplateCatalogService } else { $template->state = 'importing'; } - $template->updated_at = time(); - $template->save(); + CloudTemplate::query()->where('id', (string) $template->id) + ->whereIn('state', ['importing', 'error'])->update([ + 'state' => $template->state, + 'updated_at' => time(), + ]); } private static function importJobDto(CloudTemplate $template): array @@ -964,14 +1048,20 @@ final class CloudTemplateCatalogService } private static function markReplica(int $hostId, string $templateId, - string $state, string $errorCode): void + string $operationId, array $allowedStates, string $state, + string $errorCode): bool { - CloudTemplateReplica::query()->where('host_id', $hostId) - ->where('template_id', $templateId)->update([ + if ($allowedStates === []) { + return false; + } + return CloudTemplateReplica::query()->where('host_id', $hostId) + ->where('template_id', $templateId) + ->where('operation_id', $operationId) + ->whereIn('state', $allowedStates)->update([ 'state' => $state, 'error_code' => substr($errorCode, 0, 64), 'updated_at' => time(), - ]); + ]) === 1; } private static function operationErrorCode(array $operation, diff --git a/web/app/Services/Cloud/CloudUserIsoService.php b/web/app/Services/Cloud/CloudUserIsoService.php index e9b53545..6b16f52b 100644 --- a/web/app/Services/Cloud/CloudUserIsoService.php +++ b/web/app/Services/Cloud/CloudUserIsoService.php @@ -83,15 +83,26 @@ final class CloudUserIsoService 'name' => $name, 'source_url' => $url, ]); - self::applyImportOperation($iso, $hostId, $operationId, - $operation); } catch (CloudAgentError $error) { self::markReplica($hostId, (string) $iso->id, - $error->retryable ? 'pending' : 'failed', + $operationId, ['pending'], + $error->retryable && !$error->safePreDispatchFailure + ? 'pending' : 'failed', $error->agentCode); + continue; } catch (\Throwable $error) { - self::markReplica($hostId, (string) $iso->id, 'failed', + self::markReplica($hostId, (string) $iso->id, + $operationId, ['pending'], 'failed', 'import_dispatch_failed'); + continue; + } + try { + self::applyImportOperation($iso, $hostId, $operationId, + $operation); + } catch (\Throwable $error) { + self::markReplica($hostId, (string) $iso->id, + $operationId, ['pending'], 'uncertain', + 'panel_persist_failed'); } } self::refreshCatalogState($iso); @@ -109,40 +120,42 @@ final class CloudUserIsoService if ($iso === null) { throw new CloudError('ISO 下载任务不存在', 404); } - if (in_array((string) $iso->state, ['importing', 'failed'], true) - && CloudUserIsoReplica::query()->where('user_iso_id', + if (in_array((string) $iso->state, ['importing', 'failed'], true)) { + if (CloudUserIsoReplica::query()->where('user_iso_id', (string) $iso->id)->where('state', 'pending')->exists()) { - foreach (CloudUserIsoReplica::query() - ->where('user_iso_id', (string) $iso->id) - ->where('state', '<>', 'active')->orderBy('host_id')->get() - as $replica) { - $replicaOperationId = (string) $replica->operation_id; - if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', - $replicaOperationId) !== 1) { - self::markReplica((int) $replica->host_id, - (string) $iso->id, 'uncertain', - 'operation_id_invalid'); - continue; - } - $requestActor = $actor; - $requestActor['can_manage'] = true; - try { - $operation = CloudAgentSettingsService::clientForHost( - (int) $replica->host_id - )->request('GET', '/v1/operations/' - . rawurlencode($replicaOperationId), $requestActor); - self::applyImportOperation($iso, - (int) $replica->host_id, $replicaOperationId, - $operation); - } catch (CloudAgentError $error) { - if ($error->httpStatus !== 404) { + foreach (CloudUserIsoReplica::query() + ->where('user_iso_id', (string) $iso->id) + ->where('state', 'pending')->orderBy('host_id')->get() + as $replica) { + $replicaOperationId = (string) $replica->operation_id; + if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', + $replicaOperationId) !== 1) { + self::markReplica((int) $replica->host_id, + (string) $iso->id, $replicaOperationId, ['pending'], + 'uncertain', 'operation_id_invalid'); + continue; + } + $requestActor = $actor; + $requestActor['can_manage'] = true; + try { + $operation = CloudAgentSettingsService::clientForHost( + (int) $replica->host_id + )->request('GET', '/v1/operations/' + . rawurlencode($replicaOperationId), $requestActor); + self::applyImportOperation($iso, + (int) $replica->host_id, $replicaOperationId, + $operation); + } catch (CloudAgentError $error) { self::markReplica((int) $replica->host_id, - (string) $iso->id, 'pending', - 'agent_unavailable'); - } else { + (string) $iso->id, $replicaOperationId, ['pending'], + $error->httpStatus === 404 + ? 'uncertain' : 'pending', + $error->httpStatus === 404 + ? 'operation_not_found' : 'agent_unavailable'); + } catch (\Throwable $error) { self::markReplica((int) $replica->host_id, - (string) $iso->id, 'uncertain', - 'operation_not_found'); + (string) $iso->id, $replicaOperationId, ['pending'], + 'uncertain', 'panel_persist_failed'); } } } @@ -254,7 +267,8 @@ final class CloudUserIsoService self::applyDeleteOperation($iso, $replica, $operation); } catch (CloudAgentError $error) { self::markReplica((int) $replica->host_id, $id, - $error->retryable + $operationId, ['delete_sending'], + $error->retryable && !$error->safePreDispatchFailure ? 'delete_running' : 'delete_failed', $error->agentCode); } @@ -278,6 +292,7 @@ final class CloudUserIsoService $operationId = (string) $replica->operation_id; if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) { self::markReplica((int) $replica->host_id, $id, + $operationId, ['delete_running'], 'delete_uncertain', 'operation_id_invalid'); continue; } @@ -291,6 +306,7 @@ final class CloudUserIsoService self::applyDeleteOperation($iso, $replica, $operation); } catch (CloudAgentError $error) { self::markReplica((int) $replica->host_id, $id, + $operationId, ['delete_running'], $error->agentHttpStatus === 404 ? 'delete_uncertain' : 'delete_running', $error->agentHttpStatus === 404 @@ -304,9 +320,11 @@ final class CloudUserIsoService { $hostId = (int) $replica->host_id; $id = (string) $iso->id; + $operationId = (string) $replica->operation_id; if (($operation['operation_id'] ?? null) - !== (string) $replica->operation_id) { - self::markReplica($hostId, $id, 'delete_uncertain', + !== $operationId) { + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], 'delete_uncertain', 'operation_id_mismatch'); return; } @@ -315,7 +333,8 @@ final class CloudUserIsoService $confirmed = CloudAgentCatalogDeletion::confirms( $operation, 'user_iso', $id, (int) $iso->user_id ); - self::markReplica($hostId, $id, + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], $confirmed ? 'delete_succeeded' : 'delete_uncertain', $confirmed ? '' : 'delete_evidence_missing'); return; @@ -325,21 +344,26 @@ final class CloudUserIsoService if ($status === 'failed') { $confirmedMissing = $code === 'user_iso_not_found' && ($operation['retryable'] ?? null) === false; - self::markReplica($hostId, $id, + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], $confirmedMissing ? 'delete_succeeded' : 'delete_failed', $code); return; } if ($status === 'uncertain') { - self::markReplica($hostId, $id, 'delete_uncertain', $code); + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], 'delete_uncertain', + $code); return; } if (in_array($status, ['reserved', 'pending', 'running', 'dispatching'], true)) { - self::markReplica($hostId, $id, 'delete_running', ''); + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], 'delete_running', ''); return; } - self::markReplica($hostId, $id, 'delete_uncertain', + self::markReplica($hostId, $id, $operationId, + ['delete_sending', 'delete_running'], 'delete_uncertain', 'agent_status_invalid'); } @@ -412,6 +436,11 @@ final class CloudUserIsoService private static function applyImportOperation(CloudUserIso $iso, int $hostId, string $operationId, array $operation): void { + if (($operation['operation_id'] ?? null) !== $operationId) { + self::markReplica($hostId, (string) $iso->id, $operationId, + ['pending'], 'uncertain', 'operation_id_mismatch'); + return; + } $status = strtolower(trim((string) ($operation['status'] ?? ''))); if ($status === 'succeeded') { self::acceptReplicaResult($iso, $hostId, $operationId, @@ -419,21 +448,25 @@ final class CloudUserIsoService return; } if ($status === 'failed') { - self::markReplica($hostId, (string) $iso->id, 'failed', + self::markReplica($hostId, (string) $iso->id, $operationId, + ['pending'], 'failed', self::operationErrorCode($operation, 'agent_import_failed')); return; } if (in_array($status, ['uncertain', 'dispatching'], true)) { - self::markReplica($hostId, (string) $iso->id, 'uncertain', + self::markReplica($hostId, (string) $iso->id, $operationId, + ['pending'], 'uncertain', self::operationErrorCode($operation, 'agent_import_uncertain')); return; } if (in_array($status, ['reserved', 'pending', 'running'], true)) { - self::markReplica($hostId, (string) $iso->id, 'pending', ''); + self::markReplica($hostId, (string) $iso->id, $operationId, + ['pending'], 'pending', ''); return; } - self::markReplica($hostId, (string) $iso->id, 'uncertain', + self::markReplica($hostId, (string) $iso->id, $operationId, + ['pending'], 'uncertain', 'agent_status_invalid'); } @@ -454,22 +487,68 @@ final class CloudUserIsoService || $bytes === false || ((string) $iso->sha256 !== '' && !hash_equals((string) $iso->sha256, $digest))) { - self::markReplica($hostId, (string) $iso->id, 'uncertain', + self::markReplica($hostId, (string) $iso->id, $operationId, + ['pending'], 'uncertain', 'agent_result_mismatch'); return; } - if ((string) $iso->sha256 === '') { - $iso->sha256 = $digest; - $iso->size_bytes = (int) $bytes; - $iso->updated_at = time(); - $iso->save(); - } - self::writeReplica($hostId, $iso, 'active', $operationId, '', - (int) $bytes, $digest); + $iso->getConnection()->transaction(static function () use ( + $iso, $hostId, $operationId, $digest, $bytes + ): void { + $locked = CloudUserIso::query()->where('id', (string) $iso->id) + ->whereIn('state', ['importing', 'failed']) + ->lockForUpdate()->first(); + if ($locked === null) { + return; + } + $replica = CloudUserIsoReplica::query() + ->where('host_id', $hostId) + ->where('user_iso_id', (string) $iso->id) + ->where('operation_id', $operationId) + ->where('state', 'pending')->lockForUpdate()->first(); + if ($replica === null) { + return; + } + if ((int) $locked->user_id !== (int) $iso->user_id + || ((string) $locked->sha256 !== '' + && !hash_equals((string) $locked->sha256, $digest))) { + self::markReplica($hostId, (string) $iso->id, $operationId, + ['pending'], 'uncertain', 'agent_result_mismatch'); + return; + } + if ((string) $locked->sha256 === '') { + $locked->sha256 = $digest; + $locked->size_bytes = (int) $bytes; + $locked->updated_at = time(); + $locked->save(); + } + $updated = CloudUserIsoReplica::query() + ->where('host_id', $hostId) + ->where('user_iso_id', (string) $iso->id) + ->where('operation_id', $operationId) + ->where('state', 'pending')->update([ + 'user_id' => (int) $locked->user_id, + 'state' => 'active', + 'size_bytes' => (int) $bytes, + 'sha256' => $digest, + 'error_code' => '', + 'updated_at' => time(), + ]); + if ($updated !== 1) { + throw new \RuntimeException( + 'User ISO replica import transition was lost' + ); + } + }, 3); } private static function refreshCatalogState(CloudUserIso $iso): void { + $current = CloudUserIso::query()->where('id', (string) $iso->id) + ->first(); + if ($current === null) { + return; + } $replicas = CloudUserIsoReplica::query()->where( 'user_iso_id', (string) $iso->id )->get(); @@ -484,15 +563,20 @@ final class CloudUserIsoService } elseif (!$replicas->contains(static fn ($row): bool => (string) $row->state !== 'active')) { $used = (int) CloudUserIso::query()->where('user_id', - (int) $iso->user_id)->where('state', 'active')->sum('size_bytes'); - if ($used + (int) $iso->size_bytes > self::MAX_TOTAL_BYTES) { + (int) $current->user_id)->where('state', 'active') + ->sum('size_bytes'); + if ($used + (int) $current->size_bytes + > self::MAX_TOTAL_BYTES) { $iso->state = 'failed'; } else { $iso->state = 'active'; } } - $iso->updated_at = time(); - $iso->save(); + CloudUserIso::query()->where('id', (string) $iso->id) + ->whereIn('state', ['importing', 'failed'])->update([ + 'state' => $iso->state, + 'updated_at' => time(), + ]); } private static function writeReplica(int $hostId, CloudUserIso $iso, @@ -526,14 +610,20 @@ final class CloudUserIsoService } private static function markReplica(int $hostId, string $isoId, - string $state, string $errorCode): void + string $operationId, array $allowedStates, string $state, + string $errorCode): bool { - CloudUserIsoReplica::query()->where('host_id', $hostId) - ->where('user_iso_id', $isoId)->update([ + if ($allowedStates === []) { + return false; + } + return CloudUserIsoReplica::query()->where('host_id', $hostId) + ->where('user_iso_id', $isoId) + ->where('operation_id', $operationId) + ->whereIn('state', $allowedStates)->update([ 'state' => $state, - 'error_code' => $errorCode, + 'error_code' => substr($errorCode, 0, 64), 'updated_at' => time(), - ]); + ]) === 1; } private static function dto(CloudUserIso $iso): array diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 1ae11c97..33b55d36 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -333,6 +333,27 @@ final class CloudWorkerService . get_class($finishError)); } } + if ($initialDispatch && $error instanceof CloudAgentError + && $error->safePreDispatchFailure) { + // Both authenticated hops proved that the request never + // reached the Agent API handler or operation journal. Restore + // only this leased row to the persistent queue. Timeouts and + // unmarked disconnects remain uncertain and are never replayed. + $requeued = CloudJob::where('id', $job->id) + ->where('lease_token', $job->lease_token) + ->whereNotNull('dispatched_at')->update([ + 'status' => 'queued', + 'dispatched_at' => null, + 'error_code' => $error->agentCode, + 'lease_until' => 0, + 'lease_token' => null, + 'available_at' => time() + 5, + 'updated_at' => time(), + ]); + if ($requeued === 1) { + return true; + } + } if (!$dispatched) { // Decryption/configuration/local validation failed before an // Agent request was sent, so create did not happen and an diff --git a/web/config/.config.php.example b/web/config/.config.php.example index f4fbff4e..b885124e 100644 --- a/web/config/.config.php.example +++ b/web/config/.config.php.example @@ -42,12 +42,11 @@ $System_Config['cloud_data_key'] = ''; # 在线更新重启的 PHP-FPM unit;使用系统默认名称时无需修改。 $System_Config['cloud_update_php_fpm_unit'] = 'php-fpm.service'; # 必须与 runtime/update.json 的 service_secret 完全一致,仅供本机 Unix Socket HMAC。 -$System_Config['cloud_update_helper_secret'] = 'REPLACE_WITH_AT_LEAST_32_RANDOM_BYTES'; -# 必须与 runtime/console-relay.json 的 panel_shared_secret 完全一致。 -$System_Config['cloud_console_relay_secret'] = 'REPLACE_WITH_A_DIFFERENT_32_BYTE_SECRET'; -# VNC 持续检查 PHP 会话。设为面板服务器本机可访问的网站 URL,不要指向宿主机。 -$System_Config['cloud_introspection_url'] = 'http://127.0.0.1/api/internal/cloud/session-introspection'; -$System_Config['cloud_introspection_host'] = ''; +$System_Config['cloud_update_helper_secret'] = 'REPLACE_ME'; +# 必须与 runtime/agent-gateway.json 的 panel_shared_secret 完全一致。 +$System_Config['cloud_agent_gateway_secret'] = 'REPLACE_ME'; +# PHP 只通过本机 Agent Gateway 向当前已连接的宿主机下发请求。 +$System_Config['cloud_agent_gateway_url'] = 'http://127.0.0.1:8001/internal/agent/request'; # 单台实例最多排队的未来续费账期,防止免费套餐或自动化请求无限写入数据库。 $System_Config['cloud_max_renewal_periods_ahead'] = 24; # redis diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index efbe9654..9d6f30e1 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -2,6 +2,7 @@ // Included by routes.php after the Slim application has been constructed. $app->get('/api/v1/session', 'App\Controllers\CloudApiController:session'); $app->post('/api/internal/cloud/session-introspection', 'App\Controllers\CloudInternalController:introspection'); +$app->post('/api/internal/cloud/agent-auth', 'App\Controllers\CloudInternalController:agentAuth'); $app->get('/api/internal/cloud/upload-staging/{id:[0-9a-f]{32}}', 'App\Controllers\CloudUploadStagingController:download'); $app->group('/api/v1', function () { diff --git a/web/public/cloud/agent-settings.js b/web/public/cloud/agent-settings.js index 0abe7dc6..6af0b0a2 100644 --- a/web/public/cloud/agent-settings.js +++ b/web/public/cloud/agent-settings.js @@ -41,7 +41,7 @@ function renderList() { for (const agent of state.agents) { const button = text('button', 'cloud-admin-host-item', agent.host_name || `宿主机 #${agent.host_id}`); button.type = 'button'; button.dataset.active = String(agent.host_id) === String(state.selected) ? '1' : '0'; - button.append(text('small', '', `${agent.region || '地区未设置'} · ${agent.endpoint || 'Agent 未配置'}`)); + button.append(text('small', '', `${agent.region || '地区未设置'} · Agent ID ${agent.agent_id || agent.host_id}`)); button.addEventListener('click', () => { state.selected = String(agent.host_id); renderList(); fill(agent); }); list.append(button); } @@ -63,8 +63,8 @@ function renderHealth(agent) { function fill(agent) { byId('agent-settings-host-id').value = agent?.host_id || ''; byId('agent-settings-revision').value = String(agent?.revision || 0); - byId('agent-settings-title').textContent = agent ? `Agent API · ${agent.host_name}` : '选择计算节点'; - byId('agent-settings-endpoint').value = agent?.endpoint || ''; + byId('agent-settings-title').textContent = agent ? `Agent 上行连接 · ${agent.host_name}` : '选择计算节点'; + byId('agent-settings-agent-id').value = agent?.agent_id || agent?.host_id || ''; byId('agent-settings-secret').value = ''; byId('agent-settings-secret').placeholder = agent?.secret_configured ? '已配置,留空保留现有密钥' : '首次配置必须填写至少 32 位密钥'; byId('agent-settings-enabled').checked = agent?.enabled === true; @@ -91,7 +91,6 @@ async function save(event) { event.preventDefault(); if (!canManage || state.busy) return; const hostId = byId('agent-settings-host-id').value; if (!hostId) return; const body = { - endpoint: byId('agent-settings-endpoint').value.trim(), enabled: byId('agent-settings-enabled').checked, api_secret: byId('agent-settings-secret').value, expected_revision: Number(byId('agent-settings-revision').value), @@ -114,7 +113,7 @@ async function save(event) { async function testConnection() { if (!canManage || state.busy) return; const agent = selectedAgent(); if (!agent) return; - state.busy = true; fill(agent); notice('正在验证 Agent 签名连接…'); + state.busy = true; fill(agent); notice('正在通过上行 WSS 验证 Agent 连接…'); try { const tested = await api(`/api/v1/admin/agents/${encodeURIComponent(agent.host_id)}/test`, { method: 'POST', body: { expected_revision: agent.revision } }); const index = state.agents.findIndex(item => item.host_id === agent.host_id); if (index >= 0) state.agents[index] = tested; diff --git a/web/resources/views/venix/admin/cloud/agents.tpl b/web/resources/views/venix/admin/cloud/agents.tpl index 9b3d353c..2ca077dd 100644 --- a/web/resources/views/venix/admin/cloud/agents.tpl +++ b/web/resources/views/venix/admin/cloud/agents.tpl @@ -8,7 +8,7 @@

Host API

Agent API 设置

-

为每台计算节点配置独立的 Agent 地址和 HMAC 密钥。普通管理员可查看状态,只有超级管理员可以修改或测试连接。

+

为每台计算节点配置独立身份和密钥;Agent 使用面板地址主动建立 WSS 连接。普通管理员可查看状态,只有超级管理员可以修改或测试连接。

正在加载…

@@ -17,7 +17,7 @@
- API 密钥使用 PHP 的 cloud_data_key 派生用途密钥密封保存,不会回显。任何地址、密钥或启用状态变更都会产生新配置版本;新版本通过健康与能力测试前不参与调度。 + API 密钥使用 PHP 的 cloud_data_key 派生用途密钥密封保存,不会回显。宿主机配置只需面板 HTTPS 地址、下方 Agent ID 和密钥;不再开放 Agent 监听端口或部署宿主机服务端证书。
@@ -37,7 +37,7 @@

Connection

选择计算节点
-

请求由面板 PHP 直接签名并发送到所选宿主机 Agent。

+

PHP 持久化并签名任务,本机 Go Gateway 通过 Agent 主动建立的 WSS 连接下发。

@@ -46,9 +46,9 @@
- - -
仅接受固定 IP HTTPS 地址;证书必须包含该 IP 的 SAN。不允许账号、路径、查询参数、回环或链路本地地址。
+ + +
把此 ID 与面板 HTTPS 地址、下面保存的同一密钥写入该宿主机 Agent 配置。
@@ -56,13 +56,13 @@
密钥不会从服务器返回到浏览器。
- +