From ab385f7b865d91efa9636a0cb13104edec07f0fc Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 16 Oct 2026 04:14:46 +0800 Subject: [PATCH] feat: harden php agent cutover and reconcile --- agent/internal/api/server.go | 54 ++- agent/internal/api/server_power_test.go | 70 ++++ deploy/virt-cloud-agent.env.example | 9 +- docs/GO_AGENT_API_V1.md | 29 +- docs/GO_AGENT_DEPLOYMENT.md | 19 +- docs/GO_AGENT_MIGRATION.md | 4 +- docs/GO_AGENT_OPERATION_MATRIX.md | 2 +- docs/GO_AGENT_PHP_SHADOW.md | 38 +- docs/GO_AGENT_PYTHON_DELETION_PLAN.md | 26 +- docs/GO_AGENT_SETUP.md | 8 +- web/app/Command/CloudAgentSetup.php | 12 +- web/app/Controllers/CloudApiController.php | 131 ++++-- web/app/Middleware/CloudApi.php | 6 +- web/app/Models/CloudAgentOperation.php | 2 + .../Cloud/CloudAgentActionService.php | 134 ++++++ web/app/Services/Cloud/CloudAgentClient.php | 4 + .../Cloud/CloudAgentCreatePlanService.php | 61 ++- .../Cloud/CloudAgentDispatchUncertain.php | 30 ++ .../Cloud/CloudAgentFileSecretResolver.php | 168 ++++++++ web/app/Services/Cloud/CloudAgentHandle.php | 14 +- .../Cloud/CloudAgentHostClientResolver.php | 28 +- .../Cloud/CloudAgentObservationService.php | 87 +++- .../CloudAgentOperationHistoryService.php | 17 +- .../Cloud/CloudAgentOperationIdentity.php | 75 ++++ .../CloudAgentOperationReconcileService.php | 385 ++++++++++++++++++ .../Cloud/CloudAgentOperationRepository.php | 194 ++++++++- .../CloudAgentOperationStatusService.php | 48 ++- .../Cloud/CloudAgentRegistryHealthService.php | 60 ++- .../Services/Cloud/CloudAgentVmGateway.php | 151 +++++-- .../Cloud/CloudInstanceRuntimeService.php | 26 ++ .../Cloud/CloudInstanceRuntimeSyncService.php | 64 ++- .../Cloud/CloudRuntimeHealthService.php | 16 +- web/config/routes_cloud.php | 5 + web/public/cloud/admin-operations.js | 2 +- web/public/cloud/runtime.js | 100 ++++- .../views/venix/admin/cloud/operations.tpl | 2 +- web/sql/cloud_agent_v1.sql | 20 +- .../CloudAgentActionCutoverContractTest.php | 83 ++++ ...loudAgentCreatePlanServiceContractTest.php | 4 + ...oudAgentFileSecretResolverContractTest.php | 80 ++++ ...oudAgentObservationServiceContractTest.php | 4 + ...CloudAgentOperationHistoryContractTest.php | 6 + ...oudAgentOperationReconcileContractTest.php | 76 ++++ ...udAgentOperationRepositoryContractTest.php | 9 +- ...gentOperationStatusServiceContractTest.php | 6 +- .../CloudAgentRegistryHealthContractTest.php | 3 + .../CloudAgentRuntimeHealthContractTest.php | 34 ++ web/tests/CloudAgentSchemaContractTest.php | 5 + .../CloudApiVmCacheCutoverContractTest.php | 41 ++ ...udInstanceRuntimeSyncAgentContractTest.php | 11 +- 50 files changed, 2256 insertions(+), 207 deletions(-) create mode 100644 web/app/Services/Cloud/CloudAgentActionService.php create mode 100644 web/app/Services/Cloud/CloudAgentDispatchUncertain.php create mode 100644 web/app/Services/Cloud/CloudAgentFileSecretResolver.php create mode 100644 web/app/Services/Cloud/CloudAgentOperationIdentity.php create mode 100644 web/app/Services/Cloud/CloudAgentOperationReconcileService.php create mode 100644 web/tests/CloudAgentActionCutoverContractTest.php create mode 100644 web/tests/CloudAgentFileSecretResolverContractTest.php create mode 100644 web/tests/CloudAgentOperationReconcileContractTest.php create mode 100644 web/tests/CloudAgentRuntimeHealthContractTest.php create mode 100644 web/tests/CloudApiVmCacheCutoverContractTest.php diff --git a/agent/internal/api/server.go b/agent/internal/api/server.go index e8f3cf18..0c7f8d27 100644 --- a/agent/internal/api/server.go +++ b/agent/internal/api/server.go @@ -1494,16 +1494,14 @@ func (s *Server) reconcileOperation(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusNotFound, errorResponse{Error: "operation not found"}) return } - // Reconciliation of planning/validation operations is another read-only - // capability or artifact observation. Allow it through the control-plane - // gate while the host readiness bit is degraded; otherwise an - // unavailable/policy result could never be durably re-observed. Mutating or - // lifecycle reconciliation keeps the normal fail-closed host readiness gate. - if isPlanningKind(record.Kind) { - if !s.preflightAllowed(w) { - return - } - } else if !s.writeAllowed(w, r.Context()) { + // Every reconcile handler below is a read-back operation. It may update + // the local journal, but it must never replay the original host mutation. + // Therefore it uses the control-plane gate only and deliberately does not + // require a fresh Probe()/hostReady bit. A failed/stale probe is exactly + // the state in which an operator needs to read back an already ambiguous + // operation. Lifecycle requests created through POST /operations remain + // behind writeAllowed and still fail closed while the host is degraded. + if !s.readbackAllowed(w) { return } if record.Status != operation.StatusUncertain { @@ -2020,35 +2018,49 @@ func (s *Server) writeAllowed(w http.ResponseWriter, ctx context.Context) bool { return false } -// preflightAllowed is the narrower gate for read-only planning/validation -// operations. These operations exist to obtain fresh capability or artifact -// observations, so requiring hostReady here would make a failed/stale host -// probe impossible to diagnose or reconcile. Transport authentication is -// enforced by Handler; this method covers the remaining local control-plane -// invariants and keeps the durable journal mandatory. -func (s *Server) preflightAllowed(w http.ResponseWriter) bool { +// readbackAllowed is the control-plane gate for read-only planning, +// validation, and reconciliation operations. These operations may obtain a +// fresh capability/artifact/domain observation and commit that observation to +// the local journal, but they must not require a successful host Probe(). A +// failed/stale probe is precisely when an ambiguous operation needs a later +// read-back. Transport authentication is enforced by Handler; this method +// covers the remaining local control-plane invariants and keeps the durable +// journal mandatory. +func (s *Server) readbackAllowed(w http.ResponseWriter) bool { + return s.controlPlaneReadOnlyAllowed(w, "read-back") +} + +func (s *Server) controlPlaneReadOnlyAllowed(w http.ResponseWriter, operation string) bool { if s == nil || s.store == nil || !s.store.Writable() || s.cfg.AgentID == "" { - writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent control plane is degraded and preflight is disabled"}) + writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent control plane is degraded and " + operation + " is disabled"}) return false } s.stateMu.RLock() tlsReady := s.tlsReady s.stateMu.RUnlock() if s.cfg.RequireMTLS && (!s.cfg.TLSComplete() || !tlsReady) { - writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent TLS is not ready for preflight"}) + writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent TLS is not ready for " + operation}) return false } if s.cfg.RequireMTLS && len(s.cfg.ClientCertFingerprints) == 0 { - writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent client certificate policy is not ready for preflight"}) + writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent client certificate policy is not ready for " + operation}) return false } if s.cfg.RequireSigned && (s.auth == nil || !s.auth.Ready()) { - writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent request signing is not ready for preflight"}) + writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "agent request signing is not ready for " + operation}) return false } return true } +// preflightAllowed is kept as a narrow compatibility name for the create +// planning path. Both planning and reconciliation are read-only control-plane +// operations and intentionally share the same host-readiness-independent +// checks, while retaining the preflight-specific error wording. +func (s *Server) preflightAllowed(w http.ResponseWriter) bool { + return s.controlPlaneReadOnlyAllowed(w, "preflight") +} + func (s *Server) authorized(r *http.Request) bool { if !s.cfg.RequireMTLS && s.cfg.IsLoopbackListen() { return true diff --git a/agent/internal/api/server_power_test.go b/agent/internal/api/server_power_test.go index 3676e664..c926f67a 100644 --- a/agent/internal/api/server_power_test.go +++ b/agent/internal/api/server_power_test.go @@ -114,9 +114,15 @@ type reconcilePowerBackend struct { *hypervisor.FakeBackend powerCalls int readCalls int + probeCalls int readErr error } +func (b *reconcilePowerBackend) Probe(ctx context.Context) (hypervisor.ProbeResult, error) { + b.probeCalls++ + return b.FakeBackend.Probe(ctx) +} + func (b *reconcilePowerBackend) PowerDomain(ctx context.Context, uuid string, action hypervisor.PowerAction) (hypervisor.PowerResult, error) { b.powerCalls++ return b.FakeBackend.PowerDomain(ctx, uuid, action) @@ -223,6 +229,70 @@ func TestDomainPowerReconcileReadsStateWithoutReplayingAction(t *testing.T) { } } +func TestDomainPowerReconcileReadsWhenHostReadinessIsDegraded(t *testing.T) { + store := operation.NewMemory() + fake := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) + if _, err := fake.DefineDomain(context.Background(), hypervisor.DefineRequest{ + UUID: testPowerUUID, + XML: []byte(` ` + testPowerUUID + `hvm`), + }); err != nil { + t.Fatal(err) + } + // Keep the host readiness bit degraded and make a fresh Probe fail. The + // domain state read itself remains available, which is the boundary a + // reconciliation request must support. + fake.SetProbeError(context.DeadlineExceeded) + backend := &reconcilePowerBackend{FakeBackend: fake} + c := config.Defaults() + c.AgentID = "host-103" + s, err := New(Options{ + Config: c, + Store: store, + Capabilities: capability.BackendProvider{Backend: backend, AgentID: c.AgentID, ProtocolVersion: c.ProtocolVersion}, + Backend: backend, + HostBackendReady: false, + }) + if err != nil { + t.Fatal(err) + } + if s.readiness() != config.ReadinessDegraded { + t.Fatalf("initial readiness = %q, want degraded", s.readiness()) + } + + const operationID = "00000000-0000-4000-8000-000000000028" + // A shut-off domain satisfies a shutdown read-back. If reconcile were to + // replay the lifecycle request, powerCalls would become non-zero; the + // read-only contract requires only ReadDomainPowerState. + seedUncertainPowerOperation(t, store, operationID, testPowerUUID, hypervisor.PowerActionShutdown) + recorder := httptest.NewRecorder() + s.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/v1/operations/"+operationID+"/reconcile", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("degraded reconcile status = %d, body=%s; read-back must remain available", recorder.Code, recorder.Body.String()) + } + record, err := store.Get(operationID) + if err != nil { + t.Fatal(err) + } + if record.Status != operation.StatusSucceeded || record.CommitBoundary != "power_readback_confirmed" { + t.Fatalf("degraded reconcile record = %+v", record) + } + if backend.readCalls != 1 || backend.powerCalls != 0 || backend.probeCalls != 0 { + t.Fatalf("degraded reconcile calls: read=%d power=%d probe=%d; want read=1 power=0 probe=0", backend.readCalls, backend.powerCalls, backend.probeCalls) + } + + // The relaxed gate is limited to reconciliation. A new lifecycle request + // must still fail closed while the host readiness probe is degraded. + writeBody := []byte(`{"operation_id":"00000000-0000-4000-8000-000000000029","idempotency_key":"idem-degraded-write-0029","kind":"domain_power","target":"` + testPowerUUID + `","payload":{"uuid":"` + testPowerUUID + `","action":"start"}}`) + write := httptest.NewRecorder() + s.Handler().ServeHTTP(write, httptest.NewRequest(http.MethodPost, "/v1/operations", bytes.NewReader(writeBody))) + if write.Code != http.StatusServiceUnavailable { + t.Fatalf("degraded lifecycle status = %d, body=%s; new mutation must remain blocked", write.Code, write.Body.String()) + } + if backend.powerCalls != 0 { + t.Fatalf("degraded lifecycle unexpectedly replayed power action: power=%d", backend.powerCalls) + } +} + func TestDomainPowerReconcileReadFailureKeepsUncertain(t *testing.T) { store := operation.NewMemory() fake := hypervisor.NewFakeBackend(hypervisor.DefaultCapabilitySnapshot()) diff --git a/deploy/virt-cloud-agent.env.example b/deploy/virt-cloud-agent.env.example index 34231b4c..24821b80 100644 --- a/deploy/virt-cloud-agent.env.example +++ b/deploy/virt-cloud-agent.env.example @@ -22,10 +22,13 @@ VIRT_CLOUD_AGENT_LIBVIRT_URI=qemu:///system # File paths only. Provision the actual files out of band with root-only # ownership; the HMAC key specifically must be a regular 0400/0600 file with -# no group/other permission bits. Never paste key contents here. +# no group/other permission bits. Never paste key contents here. This is the +# Agent-side verifier file only; the PHP panel uses its separate per-agent +# root-only secret directory (`.hmac/.ca.pem/.crt/.key`). # request.hmac must contain canonical standard base64 for exactly 32 random raw -# bytes (44 ASCII characters, optionally followed by one newline). PHP -# CloudSecrets and Go auth.FromFile decode the same format. +# bytes (44 ASCII characters, optionally followed by one newline). Go +# auth.FromFile decodes it; do not point PHP's `cloud_agent_secret_dir` at this +# host-local directory or copy the path into the panel database. VIRT_CLOUD_AGENT_TLS_CERT=/etc/virt-cloud-agent/tls/agent.crt VIRT_CLOUD_AGENT_TLS_KEY=/etc/virt-cloud-agent/tls/agent.key VIRT_CLOUD_AGENT_TLS_CA=/etc/virt-cloud-agent/tls/panel-ca.crt diff --git a/docs/GO_AGENT_API_V1.md b/docs/GO_AGENT_API_V1.md index da866a97..6982c4c5 100644 --- a/docs/GO_AGENT_API_V1.md +++ b/docs/GO_AGENT_API_V1.md @@ -11,14 +11,21 @@ 并将客户端证书指纹绑定到 `cloud_agents.client_certificate_fingerprint`;轮换期 使用 `cloud_agent_client_certificates` 的受控重叠记录。 - Agent 不连接面板数据库,不持有 PHP 数据库密码,也不接受 SSH 私钥。 -- `VIRT_CLOUD_AGENT_HMAC_KEY_FILE` 与 PHP `CloudSecrets` 使用同一文件协议:文件内容是 - canonical standard base64 文本,解码后必须恰好 32 个 raw bytes;文件末尾可有一个换行。 - Go `FromFile` 会拒绝 symlink、非 regular file、group/other 权限、原始字节、嵌入空白、 - 非 canonical padding 和其他长度;权限错误只让 Agent 保持 degraded,不会放行写请求。 - 密钥路径的祖先目录也会逐级检查;仅内核 sticky 保护的标准共享临时根(例如 - `/tmp`)可作为测试边界,生产 secret/journal 应放在单 Agent 独占的私有目录。 - PHP `CloudSecrets::key('engine_hmac_key')` 返回解码后的 raw bytes。`CloudAgentClient` 只接受 - 该 raw key,不能把 44 字符的 base64 文本直接拿来做 HMAC。 +- `VIRT_CLOUD_AGENT_HMAC_KEY_FILE` 是宿主 Agent 本机的 verifier 文件;它只供 Go Agent + 读取,不能把这个路径或宿主文件直接配置到 PHP。文件内容是 canonical standard base64 + 文本,解码后必须恰好 32 个 raw bytes;文件末尾可有一个换行。Go `FromFile` 会拒绝 + symlink、非 regular file、group/other 权限、原始字节、嵌入空白、非 canonical padding + 和其他长度;权限错误只让 Agent 保持 degraded,不会放行写请求。密钥路径的祖先目录也 + 会逐级检查;仅内核 sticky 保护的标准共享临时根(例如 `/tmp`)可作为测试边界,生产 + secret/journal 应放在单 Agent 独占的私有目录。 +- PHP 面板使用独立的 root-only secret provider。默认目录为 + `/run/virt-cloud/agents/`,每个已登记 `agent_id` 必须有四个文件: + `.hmac`(同样的 canonical base64 HMAC key)、`.ca.pem`、 + `.crt`、`.key`。PHP `CloudAgentFileSecretResolver` 每次请求都会 + 校验这些文件及其祖先目录的 regular/non-symlink、owner 和权限边界,并只把解码后的 + 32-byte key 与文件路径交给 `CloudAgentClient`;这些文件不存入数据库,也不由 + `CloudSecrets`/旧 `cloud_engine_hmac_file` 解析。面板端的四文件必须通过受控部署流程 + 写入,不能把宿主机的 `request.hmac` 路径当作面板 secret directory。 - `GET /v1/health` 是 liveness;配置 mode/owner 宽松时仍返回 200,但 `readiness=degraded`。除只读 `domain_preflight` 外,写操作在宿主 readiness 降级时返回 503;preflight 仍必须通过认证传输、Agent 身份和 durable journal @@ -38,6 +45,12 @@ | `X-Cloud-Agent-ID` | 必须等于当前 Agent 的注册身份,并参与 HMAC canonical 文本;防止同一 transport secret 被发往错误宿主 | | client certificate | 绑定已注册 Agent 调用方/面板传输身份;`X-Cloud-Actor` 当前只作签名审计元数据,业务权限仍由 PHP 发起前校验 | +面板生成新操作时可以只提交 `idempotency_key`,由 PHP 按该键稳定生成 +`operation_id`。如果调用方显式提交 `operation_id`,必须同时提交同一次请求的 +`idempotency_key`(请求头和请求体若同时出现也必须一致);operation UUID 本身 +不携带原始幂等键,面板不会从 UUID 派生或猜测幂等键。仅有 `operation_id` 的 +重试在进入 Agent 或 journal 之前以 422 拒绝,避免把未知重试变成第二个操作。 + 响应必须包含 operation 状态。网络超时不是失败证明;面板应以 `GET /v1/operations/{operation_id}` 和专用 reconcile 端点核对,而不是自动 重放 delete/reinstall/power 等破坏性操作。 diff --git a/docs/GO_AGENT_DEPLOYMENT.md b/docs/GO_AGENT_DEPLOYMENT.md index cc792188..4853fb5b 100644 --- a/docs/GO_AGENT_DEPLOYMENT.md +++ b/docs/GO_AGENT_DEPLOYMENT.md @@ -102,11 +102,22 @@ VIRT_CLOUD_AGENT_STORAGE_ROOTS=/var/lib/libvirt/images/virt-cloud symlink。Agent 每次 define 前都会重新检查根、文件和 realpath;目录替换或路径 竞态会使 operation 保持失败/不确定,而不会扩大访问范围;读回 XML 时会再次执行 同一 allowlist 检查,防止宿主外部改写定义后被误报为 Agent 成功。 -`request.hmac` 文件必须写入 canonical standard base64 的 32 个随机 raw bytes -(44 个 ASCII 字符,可带一个末尾换行),且必须是 owner 可读、无 group/other +宿主机的 `request.hmac` 文件必须写入 canonical standard base64 的 32 个随机 raw +bytes(44 个 ASCII 字符,可带一个末尾换行),且必须是 owner 可读、无 group/other 权限位的 regular file(`0400` 或 `0600`,不能是 symlink);Go Agent 启动时严格 -解码,PHP `CloudAgentFileSecretResolver` 和 `CloudAgentClient` 使用同一解码结果。 -不要把原始二进制或 44 字符 base64 文本直接当作 HMAC key 传给客户端。实际证书、私钥、CA 和 HMAC key 不应由 Git、面板数据库或日志分发。远程部署 +解码。该文件是宿主 Agent 的本地 verifier secret,不是 PHP 面板的 secret 路径;不要 +把它的路径写入面板配置或把宿主机目录直接暴露给 PHP。 + +面板侧另行准备 root-only 的 `cloud_agent_secret_dir`(默认 +`/run/virt-cloud/agents`),为每个登记的 `agent_id` 写入四个文件: +`.hmac`、`.ca.pem`、`.crt`、`.key`。 +其中 `.hmac` 使用与 Agent `request.hmac` 相同的 canonical base64 文件协议,`.ca.pem` +是面板信任的 Agent CA,`.crt`/`.key` 是面板调用 Agent 的客户端证书和私钥。PHP +`CloudAgentFileSecretResolver` 每次解析都会检查四个文件及祖先目录的 owner、regular +file、非 symlink、权限和大小,然后把解码后的 raw key 交给 `CloudAgentClient`;四个文件 +不进数据库、不由旧 `CloudSecrets` 读取,也不应提交到 Git。不要把原始二进制或 44 字符 +base64 文本直接当作 HMAC key 传给客户端。实际证书、私钥、CA 和 HMAC key 不应由 Git、 +面板数据库或日志分发。远程部署 还应先完成面板 CA 签发、客户端证书指纹登记和时间同步,再开放管理地址端口。 密钥、journal 和 loader 路径的祖先目录默认必须不可由 group/other 写入;为支持 只读冒烟和临时运行目录,内核 sticky 保护的标准共享临时根(例如 `/tmp` 的 diff --git a/docs/GO_AGENT_MIGRATION.md b/docs/GO_AGENT_MIGRATION.md index c8cf8503..04ffc2ba 100644 --- a/docs/GO_AGENT_MIGRATION.md +++ b/docs/GO_AGENT_MIGRATION.md @@ -35,7 +35,9 @@ desired/observed JSON 和 operation 记录;新 schema 不包含旧的 `*_ciphe Agent 请求签名密钥的文件协议固定为 canonical standard base64(解码后恰好 32 个 raw bytes,允许文件末尾换行)。Go `auth.FromFile` 负责严格解码;PHP -`CloudSecrets` 负责同样的解码并把 raw key 交给 `CloudAgentClient`。测试或内存调用 +面板侧 `CloudAgentFileSecretResolver` 负责读取每个 `agent_id` 的 `.hmac/.ca.pem/.crt/.key` +四文件,执行同样的 HMAC 解码并把 raw key 与 TLS 文件路径交给 `CloudAgentClient`;宿主 +侧 `VIRT_CLOUD_AGENT_HMAC_KEY_FILE` 则只由 Go Agent 本地 verifier 读取。测试或内存调用 可以使用明确标注的 raw-key constructor,但不得把配置文件中的 base64 文本直接用于 HMAC。 diff --git a/docs/GO_AGENT_OPERATION_MATRIX.md b/docs/GO_AGENT_OPERATION_MATRIX.md index 0be1b10e..f52174c7 100644 --- a/docs/GO_AGENT_OPERATION_MATRIX.md +++ b/docs/GO_AGENT_OPERATION_MATRIX.md @@ -21,7 +21,7 @@ | 快照创建/恢复/删除 | 未实现 | libvirt job 状态、卷身份、冻结/解冻和取消后的恢复证据 | | 防火墙规则 | 未实现 | nft/firewalld owner marker、revision/digest、只读 live 规则回读 | | 网络挂载/服务暂停恢复 | 未实现 | 新网络架构的宿主数据面方案、跨宿主验收;本阶段不写 WireGuard/L3 | -| 删除与失败清理 | 未实现 | 删除 commit boundary、cleanup manifest、未知结果禁止重放 | +| 删除与失败清理 | Agent 仅实现 inactive 域 `undefine` 读回边界;PHP 完整删除尚未映射 | 删除 commit boundary、cleanup manifest、磁盘/seed/网络/账务清理和未知结果禁止重放 | | 控制台票据 / WebSocket | 未实现 | 短期票据、域身份绑定、TLS/WSS 和撤销测试 | | 宿主扫描/保存/SSH | 不属于 Agent 运行时 | 改为 enrollment + mTLS + capability probe;切换完成后删除旧 SSH 路径 | diff --git a/docs/GO_AGENT_PHP_SHADOW.md b/docs/GO_AGENT_PHP_SHADOW.md index a1cb1a81..5055d935 100644 --- a/docs/GO_AGENT_PHP_SHADOW.md +++ b/docs/GO_AGENT_PHP_SHADOW.md @@ -1,12 +1,11 @@ -# PHP 面板 Go Agent shadow 边界 +# PHP 面板 Go Agent 连接边界 -本阶段增加两个仍未接管生产路由的边界:只读 `cloud_agents` registry health -shadow `CloudAgentRegistryHealthService`,以及按 host 解析 mTLS/HMAC client 的 -`CloudAgentHostClientResolver`。后者是后续 Agent gateway 的唯一 client 构造入口; -它不替换现有 `CloudRuntimeHealthService` 的 Python/Engine 检查,不改变任何 VM -写路由,也不把 shadow 结果当作宿主可写 readiness。 -当前没有把它接入生产路由;在完成 enrollment、secret provider 和真实 mTLS -握手验收前,默认面板响应保持不变。 +本阶段使用只读 `cloud_agents` registry health +`CloudAgentRegistryHealthService`,并将按 host 解析 mTLS/HMAC client 的 +`CloudAgentHostClientResolver` 接入两条明确的页面边界:实时 VM 状态的 +`domain_observe` 和电源动作的 `domain_power`。其余页面仍在迁移清单中;未实现的 +Agent kind 不会回退 Python/SSH。enrollment、secret provider 和真实 mTLS 握手 +验收完成前,Agent 路由缺配置会 fail-closed,不把 liveness/缓存状态当作生产成功。 ## 身份与密钥边界 @@ -22,12 +21,16 @@ shadow `CloudAgentRegistryHealthService`,以及按 host 解析 mTLS/HMAC clien 明确的 `resolveReadOnly()` 调用,避免把只读降级状态误用于电源/生命周期写操作。 它还核对请求的 `host_id`、`protocol_version=v1`、小写 SHA-256 客户端证书指纹, 并把本地 cert 文件的实际指纹与 registry 比对;HMAC/TLS 路径只存在于 resolver - 的瞬时调用中,不进入 `CloudAgentHandle`。只读 handle 在类型上拒绝 - `operation()`/`reconcile()`,不会因为调用方误用而向 degraded Agent 发写请求。 + 的瞬时调用中,不进入 `CloudAgentHandle`。只读 handle 在类型上只允许 + `domain_read`/`domain_observe` 操作;其他 operation/reconcile 仍会被拒绝, + 不会因为调用方误用而向 degraded Agent 发写请求。 - Agent operation 响应进入新 journal 前必须经过 `CloudAgentOperationEnvelope`:逐字段 比对 `operation_id`、`idempotency_key`、`request_hash`、`kind`、`target`,并校验 `status`、`version`、`commit_boundary` 和受限 result;Agent 的 `error_message` 不会 持久化。后续 claim/finalize 仍须用 `version + lease_owner + attempt_count` 做 CAS。 +- 新操作可以只带幂等键(PHP 按键生成 UUID);一旦请求显式带 + `operation_id`,必须同时带原始幂等键。UUID 不可逆推出旧幂等键,operation-only + 重试在控制器和服务边界直接 422,不能靠猜测/派生键重放。 - `key_file` 必须是 regular、非 symlink、非 group/other writable 且没有任何 group/other 权限(通常为 `0600`/`0400`);CA/cert 至少拒绝 group/other 写和 超大/空文件。最终文件的祖先目录也必须是不可被 group/other 写入的真实目录; @@ -40,11 +43,20 @@ shadow `CloudAgentRegistryHealthService`,以及按 host 解析 mTLS/HMAC clien `CloudAgentEndpointPolicy` 只接受无凭据、无路径/query、HTTPS endpoint,并要求 注入 `CloudAgentHostResolver`。解析结果必须全部是 public、非保留 IP;PHP 不使用 -ambient DNS。shadow 会把同一次显式解析结果传给 `CloudAgentClient`,由 Guzzle +ambient DNS。探测会把同一次显式解析结果传给 `CloudAgentClient`,由 Guzzle 的 `resolve` 选项固定请求目的地址,避免校验后再次走环境 DNS。endpoint 主机名仍 作为 TLS SNI/Host 身份,因此部署仍需完成证书 SAN、出口策略和真实 mTLS 握手; -在这些生产证据齐全前,shadow 只能作为诊断信号,不能用于调度、授权或切换旧 -Engine。 +在这些生产证据齐全前,未迁移的调度/写操作仍不能切换旧 Engine;已接入的 +`domain_power` 仅在 Agent resolver、fresh journal、CAS 和 read-back 全部成功时才会 +返回成功,失败或超时保留可轮询的 `uncertain` 记录。 + +## 实例身份与观察保留 + +- 每条操作记录同时绑定 `agent_id`、`host_id`、数据库 `instance_id` 和 UUID;状态与 + 操作历史必须按这组快照回读,避免宿主重绑或删除后复用 UUID/数字 ID 时串到旧任务。 +- 运行观察按当前实例 ID 做 CAS revision 写入;删除、重绑或并发生命周期写入赢得 CAS + 时,迟到的 Agent 响应只能被丢弃,不能覆盖较新的缓存。高频只读观察只保留有限诊断 + 窗口,`pending/running/uncertain` 记录永不由维护清理。 ## 验收边界 diff --git a/docs/GO_AGENT_PYTHON_DELETION_PLAN.md b/docs/GO_AGENT_PYTHON_DELETION_PLAN.md index 3d30340a..32e8ed72 100644 --- a/docs/GO_AGENT_PYTHON_DELETION_PLAN.md +++ b/docs/GO_AGENT_PYTHON_DELETION_PLAN.md @@ -5,8 +5,8 @@ Python 已经可以删除,也不表示当前 Agent 已经覆盖生产能力。 ## 删除范围和保留边界 -当前仓库有 54 个已跟踪 Python 文件:26 个 Engine 运行时文件、27 个 Engine -测试文件和 1 个前端手工 fixture。只有在 PHP 路由全部切换到 Go Agent、真实 +当前仓库有 54 个已跟踪 Python 文件:其中 `engine/` 有 53 个(26 个 Engine +运行时文件、27 个 Engine 测试文件),另有 1 个 `web/` 前端手工 fixture。只有在 PHP 路由全部切换到 Go Agent、真实 宿主回读和回滚窗口验收完成后,才删除下面列出的 Engine 源码和测试。删除提交 应单独完成,便于回滚;不要把行为迁移、数据库重写和大规模删除混在一个提交。 @@ -104,7 +104,7 @@ test_users.py | PHP 区域 | 迁移要求 | | --- | --- | -| `CloudApiController`、`CloudPageController` | 保持用户页面和响应 DTO;VM 读写统一使用 Agent typed operation | +| `CloudApiController`、`CloudPageController` | 保持用户页面和响应 DTO;VM 读写统一使用 Agent typed operation。`vmStatus?live=1` 已切到 `domain_observe`,电源动作已切到 `domain_power`;其余读写仍按下方门槛逐项切换 | | `CloudAdminController`、`CloudAdminInstanceController`、`CloudAdminRuntimeController` | 管理页面保留;宿主扫描/测试/保存改为 enrollment、health、capability probe | | `CloudAdminHostNetworkService` | IP 账本仍由 PHP;网络实际变更由 Agent `domain_network`,必须有 revision/read-back | | `CloudAdminOperationsService`、`CloudLifecycleService` | PHP 负责授权、CAS、租约和展示;Agent 负责执行与 reconcile | @@ -130,7 +130,7 @@ test_users.py | 1 | `capability_probe` | 宿主扫描、EFI/Q35、存储/QGA 能力 | 已实现首版 | | 2 | `domain_read` | VM 详情、运行状态、XML 摘要 | 已实现首版 | | 3 | `domain_create` | 创建、镜像/磁盘绑定、seed、define | 由三阶段协议组成,完整执行未实现 | -| 4 | `domain_power` | 启动、关机、强制关机、重启、暂停/恢复 | 已实现首版 | +| 4 | `domain_power` | 启动、关机、强制关机、重启、暂停/恢复 | 已实现首版;PHP 电源路由已切换,`reset` 仍显式 501 | | 5 | `domain_network` | 网卡挂载、私网/公网状态、服务暂停恢复 | 未实现 | | 6 | `domain_settings` | CPU、内存、磁盘和设备设置 | 未实现 | | 7 | `domain_reinstall` | 重装、旧卷/新卷切换、seed 重建 | 未实现 | @@ -164,3 +164,21 @@ test_users.py 任一门槛未满足时,只能继续迁移或修复 Agent/PHP 契约,不能删除 Python。删除 动作应作为独立提交并在发布记录中列出精确路径,避免误删未跟踪 cleanup 文件、 依赖环境或缓存。 + +## 当前切换记录(2026-10-16) + +- `/api/v1/vms/{id}/actions` 已由 PHP 直接调用 Go Agent `domain_power`,不再创建 + `cloud_jobs` 或访问 Python Engine;页面继续收到 `job_id/status_url`,其中新任务号是 + UUID。`poweroff` 在 PHP 边界映射为 Agent `destroy`;未实现的 `reset` 在入库前返回 + 501。浏览器按实例 UUID+动作保存有界幂等键(sessionStorage 失败时保留进程内副本), + `uncertain`/超时只提示核对,不自动生成第二个电源操作。 +- `/api/v1/jobs/{uuid}`、`reinstall-jobs`、`rescue-jobs` 的 UUID 路由投影新表状态,旧 + 数字任务路由仍保留给迁移窗口。Agent 操作结果经过身份校验、CAS journal 和公开字段 + allowlist;超时保留 `uncertain`,不会自动重放。 +- 新表的 `cloud_agent_instance_state` 在 reservation 事务中作为单实例 mutation mutex, + 身份键为 `agent_id + host_id + instance_id + instance_uuid`; + `pending/running/uncertain` 会阻止同一 Agent/宿主/数据库实例代际的新破坏性操作。该锁和真实宿主回读 + 仍需在 CI/MariaDB/宿主机验收,不能据静态合同宣称生产可用。 +- 这只是第一条写路由的切片,不是 Python 删除批准。创建、重装、删除、网络、设置、 + 快照、防火墙、救援、控制台、指标和后台宿主流程仍有旧调用点;在这些能力完成并通过 + 第四节门槛前,不得删除 `engine/`、Python systemd unit 或旧 PHP 代码。 diff --git a/docs/GO_AGENT_SETUP.md b/docs/GO_AGENT_SETUP.md index 7ff7a86a..b01accde 100644 --- a/docs/GO_AGENT_SETUP.md +++ b/docs/GO_AGENT_SETUP.md @@ -1,7 +1,8 @@ # Go Agent 控制面初始化 Go Agent 使用独立的 `web/sql/cloud_agent_v1.sql` 表集合(文件名对应 HTTP -v1;其中 fresh-schema marker 已是 v2,包含 operation CAS `version`)。它是新的控制面 +v1;其中 fresh-schema marker 已是 v4,包含 operation CAS `version` 以及绑定 +宿主与数据库实例代际的 mutation mutex)。它是新的控制面 边界,不是旧 Python Engine 的升级脚本:不会读取、迁移、解密或导入旧表、旧 operation ID、SSH 私钥或 `engine.json`。 @@ -18,7 +19,8 @@ php web/xcat cloudAgentInit - `cloud_agent_operations` - `cloud_agent_instance_state` - `cloud_agent_artifacts` -- `cloud_agent_schema_meta`(v2 版本与 fresh-schema 指纹;含 operation CAS `version`) +- `cloud_agent_schema_meta`(v4 版本与 fresh-schema 指纹;含 operation CAS `version` + 和 `agent_id + host_id + instance_id + instance_uuid` 实例锁身份) 它不会调用 `cloudInit`,也不会运行 `CloudSetup::exportEngineConfig()`;旧 `php web/xcat cloudInit` 和 `cloudExportConfig` 命令保持原样,便于旧 Python @@ -30,6 +32,6 @@ Engine 在迁移窗口内继续回滚。 libvirt 已可用;后续仍需 enrollment、证书指纹和 `/v1/health`、`/v1/capabilities` 回读验收。 -完整表集合还必须包含 `cloud_agent_schema_meta` 中的 v2 marker 和固定指纹;同名 +完整表集合还必须包含 `cloud_agent_schema_meta` 中的 v4 marker 和固定指纹;同名 但列定义/版本不明的旧表不会被当作 Go Agent schema 接受。该检查只保护新的 Agent 控制面边界,不读取或迁移任何旧 Python 表。 diff --git a/web/app/Command/CloudAgentSetup.php b/web/app/Command/CloudAgentSetup.php index 2048502b..a9a781a0 100644 --- a/web/app/Command/CloudAgentSetup.php +++ b/web/app/Command/CloudAgentSetup.php @@ -26,11 +26,11 @@ final class CloudAgentSetup ]; private const SCHEMA_NAME = 'virt-cloud-agent'; - private const SCHEMA_VERSION = 2; - private const SCHEMA_FINGERPRINT = '252088fbb05f7d2d1224f7f7da62b2a2cca2b719a394bd84fe295a2d1fdac61c'; + private const SCHEMA_VERSION = 4; + private const SCHEMA_FINGERPRINT = '87a45391989171906c485383fa1f57fabad828adcb23ff01fe67cae99e7ed6e4'; /** - * Create the v2 Agent schema on the configured default database. The + * Create the v4 Agent schema on the configured default database. The * installer filename remains cloud_agent_v1.sql because the HTTP protocol * is still v1; the marker prevents accepting the earlier pre-CAS shape. * @@ -61,7 +61,7 @@ final class CloudAgentSetup if (count($existing) === count(self::TABLES)) { self::assertSchemaMarker($connection); - echo "Go Agent v2 控制面表已存在,未执行重复建表。\n"; + echo "Go Agent v4 控制面表已存在,未执行重复建表。\n"; return true; } if ($existing !== []) { @@ -88,7 +88,7 @@ final class CloudAgentSetup self::assertSchemaMarker($connection); - echo "Go Agent v2 控制面表已初始化;未调用旧 CloudSetup、未读取旧引擎密钥、未连接宿主机。\n"; + echo "Go Agent v4 控制面表已初始化;未调用旧 CloudSetup、未读取旧引擎密钥、未连接宿主机。\n"; return true; } @@ -114,7 +114,7 @@ final class CloudAgentSetup $path = BASE_PATH . '/sql/cloud_agent_v1.sql'; $sql = file_get_contents($path); if ($sql === false || trim($sql) === '') { - throw new \RuntimeException('缺少 Go Agent v2 建表脚本:' . $path); + throw new \RuntimeException('缺少 Go Agent v4 建表脚本:' . $path); } return $sql; } diff --git a/web/app/Controllers/CloudApiController.php b/web/app/Controllers/CloudApiController.php index bbbd1d1e..c7f38c17 100644 --- a/web/app/Controllers/CloudApiController.php +++ b/web/app/Controllers/CloudApiController.php @@ -18,6 +18,8 @@ use App\Services\Cloud\CloudAgentObservationService; use App\Services\Cloud\CloudAgentDispatchUncertain; use App\Services\Cloud\CloudAgentOperationException; use App\Services\Cloud\CloudAgentOperationHistoryService; +use App\Services\Cloud\CloudAgentOperationIdentity; +use App\Services\Cloud\CloudAgentOperationReconcileService; use App\Services\Cloud\CloudAgentOperationRepository; use App\Services\Cloud\CloudAgentOperationStatusService; use App\Services\Cloud\CloudAgentVmGateway; @@ -88,6 +90,12 @@ final class CloudApiController )); } + /** Build the explicit, non-replaying Agent operation reconciliation seam. */ + private function agentOperationReconcileService(): CloudAgentOperationReconcileService + { + return CloudAgentOperationReconcileService::fromConfig(); + } + /** Build the fresh, database-only Agent operation-history projection. */ private function agentOperationHistoryService(): CloudAgentOperationHistoryService { @@ -447,6 +455,37 @@ final class CloudApiController (bool) ($actor['is_admin'] ?? false))); } + /** + * Reconcile a UUID Agent operation by host read-back, never by replaying + * the original VM lifecycle request. Numeric legacy jobs intentionally do + * not expose this endpoint. + */ + public function reconcileJob($request, $response, $args) + { + $actor = $this->actor($request); + $operationId = (string) ($args['id'] ?? ''); + CloudAgentOperationIdentity::validateUuid($operationId); + try { + $this->agentOperationReconcileService()->reconcile($operationId, $actor); + } catch (CloudAgentOperationException $error) { + // Another reconciler may already hold the lease. Return the + // owner-scoped uncertain DTO instead of turning a harmless + // duplicate click into a generic 503 or a new host request. + if (in_array($error->errorCode, ['lease_busy', 'uncertain', 'lease_lost'], true)) { + $status = CloudAgentOperationStatusService::forActor($operationId, $actor); + $httpStatus = ($status['status'] ?? null) === 'uncertain' ? 202 : 200; + return $this->json($response, $status, $httpStatus); + } + throw new CloudError( + self::publicAgentActionError($error->errorCode), + $error->httpStatus, + ); + } + $status = CloudAgentOperationStatusService::forActor($operationId, $actor); + $httpStatus = ($status['status'] ?? null) === 'uncertain' ? 202 : 200; + return $this->json($response, $status, $httpStatus); + } + private function unavailableVm(CloudInstance $instance): array { return ['name' => $instance->name, 'state' => 'unknown', @@ -608,9 +647,10 @@ final class CloudApiController /** * Resolve list-only VM dependencies in fixed-size database queries and - * targeted engine reads. The database collection is authoritative; a - * live row is requested only for the UUID already present in the - * actor-scoped instance collection, so this path never inventories a host. + * the PHP-owned runtime observation cache. The cache is populated by + * CloudInstanceRuntimeSyncService through the Go Agent; this page path + * never opens a legacy Engine connection and never treats a stale cache + * row as a live host response. */ private function vmListContexts($instances, array $actor): array { @@ -637,22 +677,34 @@ final class CloudApiController } $liveByInstance = []; - $engine = new CloudEngineClient(); + $runtimeByInstance = []; + try { + $runtimeByInstance = CloudInstanceRuntimeService::forInstances($instances); + } catch (\Throwable $error) { + // The runtime table is optional during a rolling fresh-schema + // install. An unavailable cache must degrade to the explicit + // "host unavailable" DTO, not resurrect the retired Engine. + $runtimeByInstance = []; + } foreach ($instances as $instance) { $instanceId = (int) $instance->id; $liveByInstance[$instanceId] = $this->unavailableVm($instance); - try { - $uuid = rawurlencode((string) $instance->uuid); - $live = $engine->host((int) $instance->host_id, - 'GET', 'vms/' . $uuid, $actor); - if (is_array($live) && is_string($live['uuid'] ?? null) - && hash_equals(strtolower((string) $instance->uuid), - strtolower(trim($live['uuid'])))) { - $liveByInstance[$instanceId] = $live; - } - } catch (CloudError $error) { - // Preserve a useful database-backed row when this host or - // targeted UUID is unavailable; other instances remain visible. + $runtime = is_array($runtimeByInstance[(string) $instanceId] ?? null) + ? $runtimeByInstance[(string) $instanceId] : []; + $runtimeUuid = is_string($runtime['uuid'] ?? null) + ? strtolower(trim($runtime['uuid'])) : ''; + $instanceUuid = strtolower(trim((string) $instance->uuid)); + $runtimeState = CloudInstanceRuntimeService::normalizeState( + $runtime['state'] ?? null + ); + if (($runtime['state_fresh'] ?? false) === true + && $runtimeState !== 'unknown' + && preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $runtimeUuid) === 1 + && hash_equals($instanceUuid, $runtimeUuid)) { + $liveByInstance[$instanceId] = [ + 'uuid' => $runtimeUuid, + 'state' => $runtimeState, + ]; } } @@ -681,11 +733,23 @@ final class CloudApiController $spec = is_array($decodedSpec) && !array_is_list($decodedSpec) ? $decodedSpec : []; if ($context === null) { + $vm = $this->unavailableVm($instance); try { - $vm = (new CloudEngineClient())->host((int) $instance->host_id, 'GET', - 'vms/' . rawurlencode($instance->uuid), $actor); - } catch (CloudError $error) { - $vm = $this->unavailableVm($instance); + $runtime = CloudInstanceRuntimeService::forInstances([$instance])[ + (string) $instance->id + ] ?? []; + $runtimeUuid = is_string($runtime['uuid'] ?? null) + ? strtolower(trim($runtime['uuid'])) : ''; + $runtimeState = CloudInstanceRuntimeService::normalizeState( + $runtime['state'] ?? null + ); + if (($runtime['state_fresh'] ?? false) === true + && $runtimeState !== 'unknown' + && hash_equals(strtolower((string) $instance->uuid), $runtimeUuid)) { + $vm = ['uuid' => $runtimeUuid, 'state' => $runtimeState]; + } + } catch (\Throwable $error) { + // Keep the explicit unavailable DTO when the cache is absent. } $period = $instance->current_period_id === null ? null : CloudServicePeriod::where('id', $instance->current_period_id) @@ -856,7 +920,8 @@ final class CloudApiController foreach ($this->vmSnapshotsByHost($snapshots, $targets) as $hostId => $group) { foreach ($group as $snapshot) { $uuid = strtolower((string) $snapshot['uuid']); - $observationTargets[(int) $hostId . ':' . $uuid] = [ + $observationTargets[(string) $snapshot['id']] = [ + 'id' => (int) $snapshot['id'], 'host_id' => (int) $hostId, 'uuid' => $uuid, ]; @@ -872,7 +937,7 @@ final class CloudApiController foreach ($snapshots as $snapshot) { $uuid = strtolower((string) $snapshot['uuid']); $target = $targets[(int) $snapshot['id']] ?? null; - $key = is_array($target) ? (int) $target['host_id'] . ':' . $uuid : ''; + $key = is_array($target) ? (string) $snapshot['id'] : ''; $statuses[] = CloudInstancePageService::powerStatus( $snapshot, $live[$key] ?? [] ); @@ -912,10 +977,21 @@ final class CloudApiController // Live state remains useful during a rolling schema // update, but an unclaimed response must not write back. } + if ($runtimeRevision === null) { + // The instance was deleted/rebound or the runtime row + // could not be claimed. Do not ask the Agent for a + // response that can no longer be tied to this panel + // generation; render the already-authorised cache. + return $this->json($response, + CloudInstancePageService::status($snapshot, [])); + } $observed = $this->agentObservationService()->observe( (int) $instance->host_id, (string) $snapshot['uuid'], $actor, + null, + null, + (int) $instance->id, ); $record = is_array($observed['record'] ?? null) ? $observed['record'] : []; @@ -1185,7 +1261,7 @@ final class CloudApiController $body = CloudInput::object($request); // These two fields are control-plane identity, not Agent payload // fields. They are optional for the unchanged browser UI; when a - // caller supplies an idempotency key, retries derive the same UUID. + // caller supplies an idempotency key, the panel derives the UUID. CloudInput::only($body, ['action', 'idempotency_key', 'operation_id']); $action = $body['action'] ?? null; if (!is_string($action) || !CloudOperationCatalog::isPowerAction($action)) { @@ -1225,6 +1301,15 @@ final class CloudApiController if ($operationId !== null && !is_string($operationId)) { throw new CloudError('面板 Agent operation_id 无效', 422); } + if ($operationId !== null && $operationId !== '' + && ($idempotency === null || $idempotency === '')) { + // UUIDs are deliberately opaque. The panel cannot reconstruct + // the original key from an operation_id, so accepting this shape + // would turn a retry into a new operation or an idempotency + // conflict. Require both values whenever a caller pins UUID. + throw new CloudError( + '面板 Agent 重试必须同时提供 operation_id 和幂等键', 422); + } try { $dispatch = $this->agentActionService()->dispatch( $instance, diff --git a/web/app/Middleware/CloudApi.php b/web/app/Middleware/CloudApi.php index 1e4b99c9..35b749f3 100644 --- a/web/app/Middleware/CloudApi.php +++ b/web/app/Middleware/CloudApi.php @@ -13,8 +13,9 @@ final class CloudApi try { $path = (string) $request->getUri()->getPath(); // PHP-owned snapshots and operations diagnostics need no engine - // session. Health may call the signed, session-independent engine - // probe; authenticating these routes must not create cloud_sessions. + // session. Agent registry health may call a signed, + // session-independent liveness probe; authenticating these routes + // must not create cloud_sessions. $databaseOnly = preg_match('#^/api/v1/invoices(?:/|$)#D', $path) === 1 || preg_match('#^/api/v1/admin/billing(?:/|$)#D', $path) === 1 || preg_match('#^/api/v1/admin/instances(?:/|$)#D', $path) === 1 @@ -26,6 +27,7 @@ final class CloudApi || preg_match('#^/api/v1/ssh-keys(?:/|$)#D', $path) === 1 || preg_match('#^/api/v1/(?:jobs|reinstall-jobs|rescue-jobs)/[0-9]+$#D', $path) === 1 || preg_match('#^/api/v1/(?:jobs|reinstall-jobs|rescue-jobs)/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$#D', $path) === 1 + || preg_match('#^/api/v1/jobs/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/reconcile$#D', $path) === 1 || preg_match('#^/api/v1/vms/[0-9]+/operations$#D', $path) === 1 || preg_match('#^/api/v1/vms/[0-9]+/firewall$#D', $path) === 1 || preg_match('#^/api/v1/vms/[0-9]+/snapshots(?:/[0-9a-f]{32}(?:/restore)?)?$#D', $path) === 1 diff --git a/web/app/Models/CloudAgentOperation.php b/web/app/Models/CloudAgentOperation.php index 054fc511..a461bf78 100644 --- a/web/app/Models/CloudAgentOperation.php +++ b/web/app/Models/CloudAgentOperation.php @@ -24,6 +24,8 @@ final class CloudAgentOperation extends Model protected $casts = [ 'id' => 'int', + 'host_id' => 'int', + 'instance_id' => 'int', 'version' => 'int', 'attempt_count' => 'int', 'lease_expires_at' => 'int', diff --git a/web/app/Services/Cloud/CloudAgentActionService.php b/web/app/Services/Cloud/CloudAgentActionService.php new file mode 100644 index 00000000..4928a011 --- /dev/null +++ b/web/app/Services/Cloud/CloudAgentActionService.php @@ -0,0 +1,134 @@ + $actor + * @return array{record:array,replayed:bool,panel_kind:string,agent_kind:string,operation_id:string,idempotency_key:string,panel_action:string} + */ + public function dispatch( + CloudInstance $instance, + array $actor, + string $panelAction, + ?string $idempotencyKey = null, + ?string $operationId = null, + ): array { + if (!CloudOperationCatalog::isPowerAction($panelAction)) { + throw new CloudError('操作无效', 422); + } + // Do not gate idempotent retries on the panel's cached status. A + // replay may arrive after the guest has transitioned (or after a + // previous request was fenced uncertain); the durable Agent journal + // and host readback are the authority for the operation outcome. + $uuid = strtolower(trim((string) $instance->uuid)); + if (preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $uuid) !== 1) { + throw new CloudError('虚拟机标识无效', 409); + } + if ($operationId !== null && $operationId !== '') { + $operationId = CloudAgentOperationIdentity::validateUuid($operationId); + if ($idempotencyKey === null || $idempotencyKey === '') { + // An operation UUID does not contain, and cannot safely + // recover, the original idempotency key. Deriving a new key + // would make a retry look like a different Agent operation. + throw new CloudError( + '面板 Agent 重试必须同时提供 operation_id 和幂等键', 422); + } + $idempotencyKey = CloudAgentOperationIdentity::validateIdempotencyKey($idempotencyKey); + } else { + $idempotencyKey = $idempotencyKey === null || $idempotencyKey === '' + ? CloudAgentOperationIdentity::idempotencyKey('action') + : CloudAgentOperationIdentity::validateIdempotencyKey($idempotencyKey); + $operationId = CloudAgentOperationIdentity::uuidForIdempotencyKey($idempotencyKey); + } + + $dispatched = $this->gateway->dispatch( + (int) $instance->host_id, + 'action', + $operationId, + $idempotencyKey, + $uuid, + ['uuid' => $uuid, 'action' => $panelAction], + $actor, + $uuid, + (int) $instance->id, + ); + return $dispatched + [ + 'operation_id' => $operationId, + 'idempotency_key' => $idempotencyKey, + 'panel_action' => $panelAction, + ]; + } + + /** + * Convert a safe journal record into the DTO consumed by the existing + * browser poller. The record has already passed repository validation; + * this second projection keeps the response independent of storage shape. + * + * @param array $record + * @param array $actor + * @return array + */ + public static function acknowledgement( + array $record, + CloudInstance $instance, + array $actor, + string $panelAction, + string $idempotencyKey, + ): array { + $operationId = is_string($record['operation_id'] ?? null) + ? $record['operation_id'] : ''; + CloudAgentOperationIdentity::validateUuid($operationId); + $status = CloudOperationErrorCatalog::userStatus($record['status'] ?? null) ?? 'unknown'; + $administrator = (bool) ($actor['is_admin'] ?? false); + $errorCode = CloudOperationErrorCatalog::publicCode($record['error_code'] ?? null); + $rawResult = is_array($record['result'] ?? null) ? $record['result'] : null; + $result = $rawResult === null ? null : CloudOperationErrorCatalog::result( + $rawResult, $administrator, 'action', $status, false, $panelAction, + ); + $phase = is_array($result) ? ($result['phase'] ?? null) : null; + $phase = $administrator + ? CloudOperationErrorCatalog::adminSummary('action', $status, $phase, false, $panelAction) + : CloudOperationErrorCatalog::userPhase($phase, $status, 'action'); + $error = $administrator + ? CloudOperationErrorCatalog::adminSummary('action', $status, + $record['error_code'] ?? null, false, $panelAction) + : CloudOperationErrorCatalog::userMessage('action', $status, $errorCode, + false, $panelAction); + $labels = CloudOperationCatalog::labels('action', $panelAction); + return [ + 'id' => $operationId, + 'job_id' => $operationId, + 'operation_id' => $operationId, + 'idempotency_key' => $idempotencyKey, + 'instance_id' => (string) $instance->id, + 'kind' => 'action', + 'status' => $status, + 'error' => $error, + 'error_code' => $errorCode, + 'attempts' => max(0, (int) ($record['attempt_count'] ?? 0)), + 'created_at' => max(0, (int) ($record['created_at'] ?? 0)), + 'updated_at' => max(0, (int) ($record['updated_at'] ?? 0)), + 'action' => $panelAction, + 'kind_label' => $labels['kind_label'], + 'action_label' => $labels['action_label'], + 'phase' => is_string($phase) ? mb_substr($phase, 0, 120) : null, + 'result' => $result, + ]; + } +} diff --git a/web/app/Services/Cloud/CloudAgentClient.php b/web/app/Services/Cloud/CloudAgentClient.php index f96af07d..b9ed6987 100644 --- a/web/app/Services/Cloud/CloudAgentClient.php +++ b/web/app/Services/Cloud/CloudAgentClient.php @@ -25,6 +25,7 @@ final class CloudAgentClient private const SUPPORTED_OPERATION_KINDS = [ 'capability_probe', 'domain_read', + 'domain_observe', 'domain_power', 'seed_prepare', 'domain_preflight', @@ -571,6 +572,9 @@ final class CloudAgentClient || !function_exists('openssl_x509_fingerprint')) { return null; } + if (!CloudAgentHostClientResolver::validateCredentialPath($path, false)) { + return null; + } $stat = @lstat($path); if (!is_array($stat) || (($stat['mode'] ?? 0) & 0170000) !== 0100000 diff --git a/web/app/Services/Cloud/CloudAgentCreatePlanService.php b/web/app/Services/Cloud/CloudAgentCreatePlanService.php index 0079ba93..31ccdecf 100644 --- a/web/app/Services/Cloud/CloudAgentCreatePlanService.php +++ b/web/app/Services/Cloud/CloudAgentCreatePlanService.php @@ -71,12 +71,19 @@ final class CloudAgentCreatePlanService ): array { $request = self::validateRequest($request); $domainUuid = $request['domain']['uuid']; - $idempotencyKey = $idempotencyKey === null || $idempotencyKey === '' - ? CloudAgentOperationIdentity::idempotencyKey('create-plan') - : CloudAgentOperationIdentity::validateIdempotencyKey($idempotencyKey); - $operationId = $operationId === null || $operationId === '' - ? CloudAgentOperationIdentity::uuidForIdempotencyKey($idempotencyKey) - : CloudAgentOperationIdentity::validateUuid($operationId); + if ($operationId !== null && $operationId !== '') { + $operationId = CloudAgentOperationIdentity::validateUuid($operationId); + if ($idempotencyKey === null || $idempotencyKey === '') { + throw new CloudError( + '面板 Agent 计划重试必须同时提供 operation_id 和幂等键', 422); + } + $idempotencyKey = CloudAgentOperationIdentity::validateIdempotencyKey($idempotencyKey); + } else { + $idempotencyKey = $idempotencyKey === null || $idempotencyKey === '' + ? CloudAgentOperationIdentity::idempotencyKey('create-plan') + : CloudAgentOperationIdentity::validateIdempotencyKey($idempotencyKey); + $operationId = CloudAgentOperationIdentity::uuidForIdempotencyKey($idempotencyKey); + } $requestHash = self::requestHash( $operationId, $idempotencyKey, @@ -88,11 +95,16 @@ final class CloudAgentCreatePlanService // a degraded read-only handle is intentionally not allowed to report // a create plan because its capability snapshot may be stale. $handle = $this->resolver->resolve($hostId); + if ($handle->hostId !== $hostId) { + throw new CloudError('宿主机 Agent 身份不匹配', 503); + } $reservation = $this->operations->reserve([ 'operation_id' => $operationId, 'idempotency_key' => $idempotencyKey, 'agent_id' => $handle->agentId, + 'host_id' => $hostId, 'instance_uuid' => $domainUuid, + 'instance_id' => null, 'kind' => self::KIND, 'target' => $domainUuid, 'request_hash' => $requestHash, @@ -133,18 +145,19 @@ final class CloudAgentCreatePlanService $actor, ); if (self::isReducedFailure($response, $operationId)) { - $status = $response['status']; $errorCode = is_string($response['error_code'] ?? null) && $response['error_code'] !== '' ? $response['error_code'] : 'agent_create_plan_rejected'; - if ($status === 'failed') { + if (self::isDefinitiveReducedRejection($response)) { $record = $this->operations->finalize( $claim, 'failed', null, $errorCode, null, ); } else { - // A non-terminal reduced response does not prove whether - // the Agent completed its capability observation. - $record = $this->operations->markUncertain($claim, $errorCode); + // A 5xx or non-terminal reduced response does not prove + // whether the Agent completed its capability observation. + $record = $this->operations->markUncertain( + $claim, 'agent_create_plan_transport_uncertain', + ); } } else { $envelope = CloudAgentOperationEnvelope::fromResponse( @@ -185,9 +198,19 @@ final class CloudAgentCreatePlanService // caller must surface the operation ID and reconcile explicitly. throw $error; } catch (CloudError $error) { - // Planning is read-only, but a transport interruption still does - // not prove whether the host capability call completed. Keep the - // durable row uncertain; never silently retry it here. + // CloudAgentClient normalizes only transport/malformed-response + // ambiguity to 502. Deterministic local/configuration or explicit + // Agent rejection must be terminal failure, not "待核对". + if ($error->httpStatus !== 502) { + try { + $this->operations->finalize( + $claim, 'failed', null, 'agent_request_rejected', null, + ); + } catch (CloudAgentOperationException) { + throw $error; + } + throw $error; + } try { $record = $this->operations->markUncertain( $claim, 'agent_create_plan_transport_uncertain' @@ -363,7 +386,7 @@ final class CloudAgentCreatePlanService && $response['_agent_http_status'] >= 400 && $response['_agent_http_status'] <= 599 && is_string($response['status'] ?? null) - && in_array($response['status'], ['pending', 'running', 'failed', 'uncertain'], true) + && in_array($response['status'], ['pending', 'running', 'succeeded', 'failed', 'uncertain'], true) && !array_key_exists('idempotency_key', $response) && !array_key_exists('result', $response) && !array_key_exists('commit_boundary', $response) @@ -372,6 +395,14 @@ final class CloudAgentCreatePlanService && !array_key_exists('target', $response); } + /** @param array $response */ + private static function isDefinitiveReducedRejection(array $response): bool + { + return ($response['status'] ?? null) === 'failed' + && in_array((int) ($response['_agent_http_status'] ?? 0), + [400, 401, 403, 404, 413, 422, 501], true); + } + /** @param array $value @param list $required @param list $optional */ private static function keys(array $value, array $required, array $optional = []): void { diff --git a/web/app/Services/Cloud/CloudAgentDispatchUncertain.php b/web/app/Services/Cloud/CloudAgentDispatchUncertain.php new file mode 100644 index 00000000..93a24223 --- /dev/null +++ b/web/app/Services/Cloud/CloudAgentDispatchUncertain.php @@ -0,0 +1,30 @@ + $record */ + public function __construct( + public readonly string $operationId, + public readonly array $record, + int $httpStatus = 502, + ) { + if (preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $operationId) !== 1) { + throw new \InvalidArgumentException('Uncertain Agent operation id is invalid'); + } + if ($httpStatus < 400 || $httpStatus > 599) { + throw new \InvalidArgumentException('Uncertain Agent HTTP status is invalid'); + } + $this->httpStatus = $httpStatus; + parent::__construct('宿主机 Agent 响应不确定,请按操作编号核对结果'); + } +} diff --git a/web/app/Services/Cloud/CloudAgentFileSecretResolver.php b/web/app/Services/Cloud/CloudAgentFileSecretResolver.php new file mode 100644 index 00000000..c5dfdf28 --- /dev/null +++ b/web/app/Services/Cloud/CloudAgentFileSecretResolver.php @@ -0,0 +1,168 @@ +fileReader = \Closure::fromCallable($fileReader ?? [self::class, 'readSecureFile']); + } + + /** @var \Closure(string):string|null */ + private \Closure $fileReader; + + /** + * Files use the following fixed names per Agent: + * + * .hmac canonical standard-base64, 32 raw bytes + * .ca.pem panel trust anchor + * .crt panel client certificate + * .key panel client private key + * + * Returning paths instead of certificate contents keeps the TLS library's + * normal certificate handling and lets CloudAgentClient re-check the + * registered certificate fingerprint before every request. + * + * @return array{hmac_key:string,ca_file:string,cert_file:string,key_file:string}|null + */ + public function resolve(string $agentId): ?array + { + if (preg_match(self::AGENT_ID_PATTERN, $agentId) !== 1) { + return null; + } + $base = rtrim($this->directory, '/') . '/' . $agentId; + $hmac = $this->readHmac($base . '.hmac'); + if ($hmac === null) { + return null; + } + $paths = [ + 'ca_file' => $base . '.ca.pem', + 'cert_file' => $base . '.crt', + 'key_file' => $base . '.key', + ]; + foreach ($paths as $path) { + if (($this->fileReader)($path) === null) { + return null; + } + } + return ['hmac_key' => $hmac] + $paths; + } + + private function readHmac(string $path): ?string + { + $contents = ($this->fileReader)($path); + if (!is_string($contents)) { + return null; + } + // Permit exactly one conventional final LF/CRLF, but do not trim + // arbitrary whitespace: accepting it would make the file protocol + // ambiguous across PHP and Go. + if (str_ends_with($contents, "\r\n")) { + $contents = substr($contents, 0, -2); + } elseif (str_ends_with($contents, "\n")) { + $contents = substr($contents, 0, -1); + } + if ($contents === '' || preg_match('/^[A-Za-z0-9+\/=]+$/D', $contents) !== 1) { + return null; + } + $decoded = base64_decode($contents, true); + if (!is_string($decoded) || strlen($decoded) !== 32 + || !hash_equals($contents, base64_encode($decoded))) { + return null; + } + return $decoded; + } + + private static function validAbsoluteDirectory(string $directory): bool + { + if ($directory === '' || $directory === '/' || $directory !== trim($directory) + || $directory[0] !== '/' || preg_match('/[\x00-\x1F\x7F]/', $directory) === 1 + || str_contains($directory, '//')) { + return false; + } + $parts = explode('/', trim($directory, '/')); + foreach ($parts as $part) { + if ($part === '' || $part === '.' || $part === '..' + || preg_match('/^[A-Za-z0-9_.:-]+$/D', $part) !== 1) { + return false; + } + } + return true; + } + + /** + * Read a secret file only when its leaf and every parent are real, + * non-writable filesystem objects. The injected reader used by contract + * tests is intentionally the sole bypass; production construction uses + * this method by default. + */ + private static function readSecureFile(string $path): ?string + { + if ($path === '' || $path[0] !== '/' || preg_match('/[\x00-\x1F\x7F]/', $path) === 1) { + return null; + } + // Reuse the transport resolver's ownership/symlink policy before + // reading the HMAC, CA, certificate or private key. The bounded + // size check below remains stricter for secret files. + // HMAC material is a bearer signing secret just like the private key; + // group/other-readable files are rejected even though CA/cert PEM + // files may use the looser non-writable certificate mode. + $privateKey = str_ends_with($path, '.key') || str_ends_with($path, '.hmac'); + if (!CloudAgentHostClientResolver::validateCredentialPath($path, $privateKey)) { + return null; + } + $stat = @lstat($path); + $maxBytes = str_ends_with($path, '.hmac') + ? self::MAX_KEY_FILE_BYTES : self::MAX_TLS_FILE_BYTES; + if (!is_array($stat) + || (($stat['mode'] ?? 0) & 0170000) !== 0100000 + || (($stat['mode'] ?? 0) & 0022) !== 0 + || (int) ($stat['size'] ?? 0) < 1 + || (int) ($stat['size'] ?? 0) > $maxBytes) { + return null; + } + $directory = dirname($path); + while ($directory !== '/') { + $directoryStat = @lstat($directory); + $mode = is_array($directoryStat) ? (int) ($directoryStat['mode'] ?? 0) : 0; + if (!is_array($directoryStat) + || ($mode & 0170000) !== 0040000 + || ($mode & 0022) !== 0) { + return null; + } + $parent = dirname($directory); + if ($parent === $directory) { + return null; + } + $directory = $parent; + } + $value = @file_get_contents($path); + return is_string($value) && strlen($value) <= $maxBytes + ? $value : null; + } +} diff --git a/web/app/Services/Cloud/CloudAgentHandle.php b/web/app/Services/Cloud/CloudAgentHandle.php index 4d96bee8..6c90f543 100644 --- a/web/app/Services/Cloud/CloudAgentHandle.php +++ b/web/app/Services/Cloud/CloudAgentHandle.php @@ -7,8 +7,10 @@ namespace App\Services\Cloud; * * The handle deliberately exposes no endpoint secret or TLS material. An * active handle may submit a whitelisted operation; a degraded handle is a - * read-only observation handle and rejects operation/reconcile calls. Registry - * identity remains available for audit and operation routing. + * read-only observation handle. Reconciliation is deliberately allowed on a + * degraded handle because it only reads the Agent journal/host state and is + * the recovery path for an operation whose transport result was lost. + * Registry identity remains available for audit and operation routing. */ final class CloudAgentHandle { @@ -83,7 +85,7 @@ final class CloudAgentHandle array $payload, array $actor, ): array { - if ($this->readOnly) { + if ($this->readOnly && !in_array($kind, ['domain_read', 'domain_observe'], true)) { throw new CloudError('降级宿主机 Agent 仅允许只读观察', 503); } return $this->client->operation($operationId, $idempotencyKey, $kind, $target, $payload, $actor); @@ -96,9 +98,9 @@ final class CloudAgentHandle public function reconcile(string $operationId, array $actor): array { - if ($this->readOnly) { - throw new CloudError('降级宿主机 Agent 仅允许只读观察', 503); - } + // Reconcile is a typed read-back endpoint, not a replay of the + // original mutation. It must remain available when the Agent is + // degraded; otherwise an uncertain operation can never be resolved. return $this->client->reconcile($operationId, $actor); } } diff --git a/web/app/Services/Cloud/CloudAgentHostClientResolver.php b/web/app/Services/Cloud/CloudAgentHostClientResolver.php index d6ca5214..e9c8631d 100644 --- a/web/app/Services/Cloud/CloudAgentHostClientResolver.php +++ b/web/app/Services/Cloud/CloudAgentHostClientResolver.php @@ -11,8 +11,9 @@ use App\Models\CloudAgent; * client. It never consults CloudHost, old engine settings, encrypted * database values or SSH credentials. Registry rows are identity metadata; * HMAC/TLS material is supplied by a separate root-only secret provider. - * CloudAgentRegistryHealthService remains an explicitly isolated shadow until - * its wiring is migrated to this resolver. + * CloudAgentRegistryHealthService uses the same resolver policies for health + * probes, but keeps its bounded liveness response separate from operation + * dispatch. */ final class CloudAgentHostClientResolver { @@ -241,7 +242,12 @@ final class CloudAgentHostClientResolver return is_array($row) ? $row : []; } - private static function validateCredentialPath(string $path, bool $privateKey = false): bool + /** + * Validate a credential path before any TLS library opens it. This is + * public so health probes and other request-local assembly paths cannot + * accidentally bypass the same ownership/symlink boundary. + */ + public static function validateCredentialPath(string $path, bool $privateKey = false): bool { if ($path === '' || $path[0] !== '/' || trim($path) !== $path || preg_match('/[\x00-\x1F\x7F]/', $path) === 1) { @@ -249,10 +255,15 @@ final class CloudAgentHostClientResolver } $stat = @lstat($path); $mode = is_array($stat) ? (int) ($stat['mode'] ?? 0) : 0; + $effectiveUid = function_exists('posix_geteuid') ? (int) posix_geteuid() : null; + $ownerUid = is_array($stat) && isset($stat['uid']) ? (int) $stat['uid'] : null; + $ownerAllowed = $ownerUid !== null + && ($ownerUid === 0 || ($effectiveUid !== null && $ownerUid === $effectiveUid)); if (!is_array($stat) || ($mode & 0170000) !== 0100000 || ($mode & 006000) !== 0 || ($mode & 0022) !== 0 + || !$ownerAllowed || !isset($stat['size']) || (int) $stat['size'] < 1 || (int) $stat['size'] > 1024 * 1024) { return false; @@ -260,12 +271,6 @@ final class CloudAgentHostClientResolver if ($privateKey && ($mode & 0077) !== 0) { return false; } - if ($privateKey && function_exists('posix_geteuid') - && isset($stat['uid']) && (int) $stat['uid'] !== 0 - && (int) $stat['uid'] !== (int) posix_geteuid()) { - return false; - } - // Do not trust a safe-looking leaf below a group/world-writable or // symlinked directory. This keeps a later Guzzle open from being // redirected by a replaceable parent during certificate rotation. @@ -276,7 +281,10 @@ final class CloudAgentHostClientResolver if (!is_array($directoryStat) || ($directoryMode & 0170000) !== 0040000 || ($directoryMode & 006000) !== 0 - || ($directoryMode & 0022) !== 0) { + || ($directoryMode & 0022) !== 0 + || !isset($directoryStat['uid']) + || !((int) $directoryStat['uid'] === 0 + || ($effectiveUid !== null && (int) $directoryStat['uid'] === $effectiveUid))) { return false; } $parent = dirname($directory); diff --git a/web/app/Services/Cloud/CloudAgentObservationService.php b/web/app/Services/Cloud/CloudAgentObservationService.php index ab302a0e..dbb9a0df 100644 --- a/web/app/Services/Cloud/CloudAgentObservationService.php +++ b/web/app/Services/Cloud/CloudAgentObservationService.php @@ -14,6 +14,10 @@ namespace App\Services\Cloud; */ final class CloudAgentObservationService { + private const MAX_BATCH_TARGETS = 20; + + private const MAX_BATCH_SECONDS = 20.0; + private const LEASE_SECONDS = 60; /** @var \Closure():string */ @@ -43,10 +47,25 @@ final class CloudAgentObservationService array $actor, ?string $operationId = null, ?string $idempotencyKey = null, + ?int $instanceId = null, ): array { $uuid = self::uuid($uuid); - $operationId ??= self::newUuid(); - $idempotencyKey ??= 'observe-' . bin2hex(random_bytes(16)); + if ($instanceId !== null && $instanceId < 1) { + throw new CloudError('虚拟机编号无效', 422); + } + if ($operationId !== null && $operationId !== '') { + $operationId = CloudAgentOperationIdentity::validateUuid($operationId); + if ($idempotencyKey === null || $idempotencyKey === '') { + throw new CloudError( + '面板 Agent 观察重试必须同时提供 operation_id 和幂等键', 422); + } + $idempotencyKey = CloudAgentOperationIdentity::validateIdempotencyKey($idempotencyKey); + } else { + $idempotencyKey = $idempotencyKey === null || $idempotencyKey === '' + ? CloudAgentOperationIdentity::idempotencyKey('observe') + : CloudAgentOperationIdentity::validateIdempotencyKey($idempotencyKey); + $operationId = CloudAgentOperationIdentity::uuidForIdempotencyKey($idempotencyKey); + } self::assertOperationIdentity($operationId, $idempotencyKey); $payload = ['uuid' => $uuid]; $requestHash = self::requestHash($operationId, $idempotencyKey, $uuid, $payload); @@ -54,11 +73,16 @@ final class CloudAgentObservationService // A degraded registration is sufficient for a read-only observation; // the resolver still requires a pinned endpoint, certificate and key. $handle = $this->resolver->resolveReadOnly($hostId); + if ($handle->hostId !== $hostId) { + throw new CloudError('宿主机 Agent 身份不匹配', 503); + } $reservation = $this->operations->reserve([ 'operation_id' => $operationId, 'idempotency_key' => $idempotencyKey, 'agent_id' => $handle->agentId, + 'host_id' => $hostId, 'instance_uuid' => $uuid, + 'instance_id' => $instanceId, 'kind' => 'domain_observe', 'target' => $uuid, 'request_hash' => $requestHash, @@ -101,16 +125,17 @@ final class CloudAgentObservationService // HTTP rejection. Preserve that deterministic failure in the // panel journal; only transport/unknown outcomes are uncertain. if (self::isReducedFailure($response, $operationId)) { - $status = $response['status']; $errorCode = is_string($response['error_code'] ?? null) && $response['error_code'] !== '' ? $response['error_code'] : 'agent_observation_rejected'; - if ($status === 'failed') { + if (self::isDefinitiveReducedRejection($response)) { $record = $this->operations->finalize( $claim, 'failed', null, $errorCode, null, ); } else { - $record = $this->operations->markUncertain($claim, $errorCode); + $record = $this->operations->markUncertain( + $claim, 'agent_observation_transport_uncertain', + ); } } else { $envelope = CloudAgentOperationEnvelope::fromResponse( @@ -148,6 +173,20 @@ final class CloudAgentObservationService } catch (CloudAgentOperationException $error) { throw $error; } catch (CloudError $error) { + // CloudAgentClient uses 502 for transport or malformed-response + // ambiguity. Other statuses are local/configuration or explicit + // request rejection and can be finalized as failed without + // manufacturing a "待核对" host outcome. + if ($error->httpStatus !== 502) { + try { + $this->operations->finalize( + $claim, 'failed', null, 'agent_request_rejected', null, + ); + } catch (CloudAgentOperationException) { + throw $error; + } + throw $error; + } try { $this->operations->markUncertain($claim, 'agent_observation_transport_uncertain'); } catch (CloudAgentOperationException) { @@ -181,26 +220,39 @@ final class CloudAgentObservationService * a successful state. Keys are caller-provided host/UUID identities and * are never taken from Agent response data. * - * @param array $targets + * @param array $targets * @param array $actor * @return array> */ public function observeMany(array $targets, array $actor): array { $live = []; + $started = microtime(true); + $processed = 0; foreach ($targets as $key => $target) { + if ($processed >= self::MAX_BATCH_TARGETS + || microtime(true) - $started >= self::MAX_BATCH_SECONDS) { + break; + } if (!is_string($key) || !is_array($target) || !is_int($target['host_id'] ?? null) || $target['host_id'] < 1 || !is_string($target['uuid'] ?? null)) { continue; } + $instanceId = $target['id'] ?? null; + if ($instanceId !== null && (!is_int($instanceId) || $instanceId < 1)) { + continue; + } $uuid = strtolower(trim($target['uuid'])); if (preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $uuid) !== 1) { continue; } try { - $observed = $this->observe($target['host_id'], $uuid, $actor); + $processed++; + $observed = $this->observe( + $target['host_id'], $uuid, $actor, null, null, $instanceId + ); $record = is_array($observed['record'] ?? null) ? $observed['record'] : []; if (($record['status'] ?? null) !== 'succeeded') { @@ -258,14 +310,6 @@ final class CloudAgentObservationService return $uuid; } - private static function newUuid(): string - { - $bytes = random_bytes(16); - $bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40); - $bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80); - return vsprintf('%s%s-%s-%s-%s-%s%s%s', str_split(bin2hex($bytes), 4)); - } - private static function assertOperationIdentity(string $operationId, string $idempotencyKey): void { if (preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $operationId) !== 1 @@ -278,14 +322,27 @@ final class CloudAgentObservationService private static function isReducedFailure(array $response, string $operationId): bool { return ($response['operation_id'] ?? null) === $operationId + && isset($response['_agent_http_status']) + && is_int($response['_agent_http_status']) + && $response['_agent_http_status'] >= 400 + && $response['_agent_http_status'] <= 599 && is_string($response['status'] ?? null) && in_array($response['status'], [ 'pending', 'running', 'succeeded', 'failed', 'uncertain', ], true) + && !array_key_exists('result', $response) && !array_key_exists('idempotency_key', $response) && !array_key_exists('request_hash', $response) && !array_key_exists('kind', $response) && !array_key_exists('target', $response) && !array_key_exists('commit_boundary', $response); } + + /** @param array $response */ + private static function isDefinitiveReducedRejection(array $response): bool + { + return ($response['status'] ?? null) === 'failed' + && in_array((int) ($response['_agent_http_status'] ?? 0), + [400, 401, 403, 404, 413, 422, 501], true); + } } diff --git a/web/app/Services/Cloud/CloudAgentOperationHistoryService.php b/web/app/Services/Cloud/CloudAgentOperationHistoryService.php index 87ff4bf8..ead1f9df 100644 --- a/web/app/Services/Cloud/CloudAgentOperationHistoryService.php +++ b/web/app/Services/Cloud/CloudAgentOperationHistoryService.php @@ -2,6 +2,7 @@ namespace App\Services\Cloud; +use App\Models\CloudAgent; use App\Models\CloudInstance; /** @@ -31,7 +32,9 @@ final class CloudAgentOperationHistoryService * Authorization is intentionally performed by CloudAccess::instance in * the controller before this service is called. That keeps the existing * owner/admin visibility rule in one place while this service remains a - * database-only projection. + * database-only projection. The repository additionally binds rows to the + * current host snapshot and numeric instance ID so a deleted/recreated + * UUID cannot inherit another VM's history. * * @return array{items:list>,pagination:array} */ @@ -44,12 +47,24 @@ final class CloudAgentOperationHistoryService if (preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $uuid) !== 1) { throw new CloudError('虚拟机标识无效', 503); } + try { + $agent = CloudAgent::where('host_id', (int) ($instance->host_id ?? 0))->first(); + $agentId = is_string($agent->agent_id ?? null) ? trim($agent->agent_id) : ''; + } catch (\Throwable) { + throw new CloudError('Agent 注册表暂时不可用', 503); + } + if ($agentId === '') { + throw new CloudError('Agent 注册身份不存在', 503); + } try { $history = $this->operations->historyForInstance( $uuid, self::HISTORY_KINDS, $page, $perPage, + $agentId, + (int) ($instance->host_id ?? 0), + (int) ($instance->id ?? 0), ); } catch (CloudAgentOperationException $error) { $status = in_array($error->httpStatus, [400, 422], true) diff --git a/web/app/Services/Cloud/CloudAgentOperationIdentity.php b/web/app/Services/Cloud/CloudAgentOperationIdentity.php new file mode 100644 index 00000000..53bd8ec9 --- /dev/null +++ b/web/app/Services/Cloud/CloudAgentOperationIdentity.php @@ -0,0 +1,75 @@ + 900) { + throw new \InvalidArgumentException('Agent reconcile lease is outside the supported range'); + } + $this->leaseOwnerFactory = \Closure::fromCallable( + $leaseOwnerFactory ?? static fn (): string => 'php-agent-reconcile-' . bin2hex(random_bytes(8)), + ); + } + + /** + * Build the production service from the fresh Agent registry and secret + * directory. No legacy engine or SSH client is consulted. + */ + public static function fromConfig( + ?callable $leaseOwnerFactory = null, + int $leaseSeconds = self::LEASE_SECONDS, + ): self { + return new self( + CloudAgentHostClientResolverFactory::fromConfig(), + new CloudAgentOperationRepository(), + $leaseOwnerFactory, + $leaseSeconds, + ); + } + + /** + * Reconcile one tenant-visible domain power operation. + * + * Only `domain_power` is exposed by the current panel job DTO. Keeping + * this allow-list explicit prevents an internal observation/plan UUID + * from becoming a generic Agent proxy. Other operation kinds need their + * own owner binding and presentation contract before being added here. + * + * @param array $actor + * @return array fresh repository record + */ + public function reconcile(string $operationId, array $actor): array + { + $operationId = CloudAgentOperationIdentity::validateUuid($operationId); + $record = $this->authorizedRecord($operationId, $actor); + $status = (string) ($record['status'] ?? ''); + if (in_array($status, ['succeeded', 'failed'], true)) { + return $record; + } + if ($status !== 'uncertain') { + throw new CloudError('任务仍在处理中,请稍后再核对', 409); + } + if (($record['kind'] ?? null) !== 'domain_power') { + throw new CloudError('该 Agent 操作暂不支持面板核对', 409); + } + + $agentId = is_string($record['agent_id'] ?? null) ? $record['agent_id'] : ''; + $hostId = self::positiveInt($record['host_id'] ?? null); + if ($agentId === '' || $hostId < 1) { + throw new CloudError('任务所属 Agent 身份已损坏', 503); + } + + // Resolve the current endpoint, then fence it to the immutable Agent + // identity stored on the operation. A host re-enrollment must never + // let an old operation be queried against a different Agent. + // Reconciliation is read-only and must remain possible while the + // Agent registry reports degraded. Mutating dispatches still use + // resolve(), which only admits active registrations. + $handle = $this->resolver->resolveReadOnly($hostId); + if ($handle->hostId !== $hostId || !hash_equals($agentId, $handle->agentId)) { + throw new CloudError('任务所属 Agent 身份已变化,请联系管理员核对', 409); + } + + $owner = ($this->leaseOwnerFactory)(); + if (!is_string($owner) + || preg_match('/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,95}$/D', $owner) !== 1) { + throw new CloudError('面板 Agent 核对租约身份无效', 503); + } + $claim = $this->claimReconcile($operationId, $owner); + $record = $claim['record'] ?? $record; + $claimToken = $claim['claim'] ?? null; + if (!$claimToken instanceof CloudAgentOperationClaim) { + throw new CloudError('面板 Agent 核对未能取得租约', 503); + } + + $getFailure = null; + try { + $first = null; + try { + $first = $handle->getOperation($operationId, $actor); + } catch (CloudError $error) { + $getFailure = $error; + } + + if (is_array($first)) { + $envelope = $this->envelope($first, $record); + if ($envelope !== null && self::terminal($envelope->status)) { + return $this->finalizeEnvelope($claimToken, $envelope); + } + } + + // The Agent endpoint is a typed read-back operation. It must not + // call the original PowerDomain/lifecycle method again. + $reconciled = null; + try { + $reconciled = $handle->reconcile($operationId, $actor); + } catch (CloudError $error) { + $getFailure = $error; + } + if (is_array($reconciled)) { + $envelope = $this->envelope($reconciled, $record); + if ($envelope !== null) { + if (self::terminal($envelope->status)) { + return $this->finalizeEnvelope($claimToken, $envelope); + } + return $this->retainUncertain( + $claimToken, + self::safeErrorCode($envelope->errorCode, 'agent_reconcile_incomplete'), + self::boundary($envelope->commitBoundary), + $envelope->result, + ); + } + } + + // Non-2xx responses are intentionally reduced by CloudAgentClient + // and cannot prove identity/terminal state. One final GET may + // obtain the complete record after a successful Agent read-back; + // otherwise the panel remains explicitly uncertain. + if (is_array($reconciled) || $getFailure !== null) { + try { + $last = $handle->getOperation($operationId, $actor); + $envelope = $this->envelope($last, $record); + if ($envelope !== null && self::terminal($envelope->status)) { + return $this->finalizeEnvelope($claimToken, $envelope); + } + } catch (CloudError $error) { + $getFailure = $error; + } + } + + $fallback = is_array($reconciled) && is_string($reconciled['error_code'] ?? null) + ? $reconciled['error_code'] : ''; + $fallback = self::safeErrorCode($fallback, + $getFailure === null ? 'agent_reconcile_uncertain' : 'agent_reconcile_transport_uncertain'); + return $this->retainUncertain($claimToken, $fallback); + } catch (CloudAgentOperationException $error) { + // A lost local lease/CAS is not a host result. Do not overwrite a + // newer claimant; surface the conflict for a later operation-ID + // retry. + // A malformed terminal envelope can fail repository validation + // before its CAS write (for example a sensitive result field or + // an unproven commit boundary). Release our reconcile lease by + // retaining uncertain state, but never downgrade lease/storage + // conflicts into a host result. + if (in_array($error->errorCode, ['invalid_request', 'invalid_transition'], true)) { + try { + return $this->retainUncertain( + $claimToken, + 'agent_reconcile_result_invalid', + ); + } catch (CloudAgentOperationException) { + throw $error; + } + } + throw $error; + } catch (CloudError $error) { + // Malformed or mismatched Agent envelopes are never terminal + // evidence. Keep the row uncertain and retain the safe error code. + try { + return $this->retainUncertain( + $claimToken, + $error->httpStatus === 502 + ? 'agent_reconcile_response_invalid' + : 'agent_reconcile_transport_uncertain', + ); + } catch (CloudAgentOperationException) { + throw $error; + } + } catch (\Throwable) { + try { + return $this->retainUncertain($claimToken, 'agent_reconcile_response_invalid'); + } catch (CloudAgentOperationException) { + throw new CloudError('Agent 核对结果无法保存,请稍后重试', 503); + } + } + } + + /** @return array{record:array,claim:CloudAgentOperationClaim} */ + private function claimReconcile(string $operationId, string $owner): array + { + try { + return $this->operations->claim( + $operationId, + $owner, + $this->leaseSeconds, + 'reconcile', + ); + } catch (CloudAgentOperationException $error) { + // claim() fences an expired running worker to uncertain and throws + // after committing that fence. Retry exactly once in reconcile + // mode; never dispatch the original operation again. + if ($error->errorCode !== 'uncertain') { + throw $error; + } + return $this->operations->claim( + $operationId, + $owner, + $this->leaseSeconds, + 'reconcile', + ); + } + } + + /** @param array $record */ + private function envelope(array $response, array $record): ?CloudAgentOperationEnvelope + { + try { + return CloudAgentOperationEnvelope::fromResponse( + $response, + (string) $record['operation_id'], + (string) $record['idempotency_key'], + (string) $record['request_hash'], + (string) $record['kind'], + (string) $record['target'], + ); + } catch (CloudError) { + // Reduced non-2xx envelopes and malformed identity are both + // non-terminal. The caller will retain uncertain state. + return null; + } + } + + private function finalizeEnvelope( + CloudAgentOperationClaim $claim, + CloudAgentOperationEnvelope $envelope, + ): array { + if ($envelope->status === 'succeeded') { + return $this->operations->finalize( + $claim, + 'succeeded', + $envelope->result, + '', + self::boundary($envelope->commitBoundary), + ); + } + if ($envelope->status === 'failed') { + return $this->operations->finalize( + $claim, + 'failed', + $envelope->result, + self::safeErrorCode($envelope->errorCode, 'agent_reconcile_failed'), + self::boundary($envelope->commitBoundary), + ); + } + return $this->retainUncertain( + $claim, + self::safeErrorCode($envelope->errorCode, 'agent_reconcile_incomplete'), + self::boundary($envelope->commitBoundary), + $envelope->result, + ); + } + + /** @param array|null $result */ + private function retainUncertain( + CloudAgentOperationClaim $claim, + string $errorCode, + ?string $boundary = null, + ?array $result = null, + ): array { + return $this->operations->finalize( + $claim, + 'uncertain', + $result, + self::safeErrorCode($errorCode, 'agent_reconcile_uncertain'), + $boundary, + ); + } + + /** @param array $actor @return array */ + private function authorizedRecord(string $operationId, array $actor): array + { + try { + $record = $this->operations->get($operationId); + } catch (CloudAgentOperationException $error) { + if ($error->errorCode === 'not_found') { + throw new CloudError('任务不存在', 404); + } + throw new CloudError('Agent 操作记录暂时不可用', 503); + } + $uuid = is_string($record['instance_uuid'] ?? null) + ? strtolower(trim($record['instance_uuid'])) : ''; + $hostId = self::positiveInt($record['host_id'] ?? null); + $instanceId = self::positiveInt($record['instance_id'] ?? null); + if ($hostId < 1 || $instanceId < 1 + || preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $uuid) !== 1 + || !is_string($record['agent_id'] ?? null) || trim($record['agent_id']) === '') { + throw new CloudError('任务身份已损坏', 503); + } + try { + // Query the numeric generation first, then compare UUID in PHP so + // a case-insensitive database collation cannot bridge identities. + $instance = CloudInstance::where('host_id', $hostId) + ->where('id', $instanceId)->first(); + } catch (\Throwable) { + throw new CloudError('任务所属实例暂时不可用', 503); + } + $currentUuid = $instance === null + ? '' : strtolower(trim((string) ($instance->uuid ?? ''))); + if ($instance === null || $currentUuid === '' || !hash_equals($uuid, $currentUuid) + || (!(bool) ($actor['is_admin'] ?? false) + && (int) ($instance->user_id ?? 0) !== (int) ($actor['user_id'] ?? 0))) { + throw new CloudError('任务不存在', 404); + } + $operationCreated = self::positiveInt($record['created_at'] ?? null); + $instanceCreated = self::positiveInt($instance->created_at ?? null); + if ($operationCreated > 0 && $instanceCreated > 0 && $operationCreated < $instanceCreated) { + // A reused numeric id/UUID must not let an old uncertain row be + // reconciled against a newly provisioned VM generation. + throw new CloudError('任务实例代际已变化,请重新查询', 409); + } + if (($record['kind'] ?? null) === 'domain_power' + && !hash_equals($uuid, strtolower(trim((string) ($record['target'] ?? ''))))) { + // The numeric instance lookup alone is not sufficient: a corrupt + // or manually inserted operation target could otherwise ask the + // Agent to read back another UUID under this owner's VM. + throw new CloudError('任务实例目标已损坏', 503); + } + return $record; + } + + private static function terminal(string $status): bool + { + return in_array($status, ['succeeded', 'failed'], true); + } + + private static function boundary(?string $boundary): ?string + { + // A local uncertain row may already be fenced to unknown. Never ask + // the repository to roll that evidence back to not_started. + return $boundary === null || $boundary === 'not_started' ? null : $boundary; + } + + private static function safeErrorCode(string $value, string $fallback): string + { + return preg_match('/^[a-z][a-z0-9_.-]{0,63}$/D', $value) === 1 + ? $value : $fallback; + } + + private static function positiveInt(mixed $value): int + { + if (is_int($value)) { + return $value > 0 ? $value : 0; + } + return is_string($value) && preg_match('/^[1-9][0-9]*$/D', $value) === 1 + ? (int) $value : 0; + } +} diff --git a/web/app/Services/Cloud/CloudAgentOperationRepository.php b/web/app/Services/Cloud/CloudAgentOperationRepository.php index b79fa49e..af44fb9c 100644 --- a/web/app/Services/Cloud/CloudAgentOperationRepository.php +++ b/web/app/Services/Cloud/CloudAgentOperationRepository.php @@ -22,10 +22,17 @@ final class CloudAgentOperationRepository private const MAX_DEPTH = 16; + private const READONLY_KINDS = [ + 'capability_probe', + 'domain_read', + 'domain_observe', + ]; + /** @var list */ private const KINDS = [ 'capability_probe', 'domain_read', + 'domain_observe', 'domain_power', 'seed_prepare', 'domain_preflight', @@ -33,6 +40,12 @@ final class CloudAgentOperationRepository 'domain_create_validate', ]; + /** Host/artifact mutations are serialized; capability/readback probes are not. */ + private const MUTATING_KINDS = [ + 'domain_power', + 'seed_prepare', + ]; + /** @var object */ private object $connection; @@ -76,11 +89,37 @@ final class CloudAgentOperationRepository return $this->replayOrConflict($existing, $request); } + // The fresh instance-state row is the durable mutex for + // mutating operations on one guest. Read-only probes may + // run beside a mutation, but a second lifecycle/create + // request must wait for a terminal record or explicit + // reconciliation instead of racing libvirt. + if ($request['instance_uuid'] !== null + && in_array($request['kind'], self::MUTATING_KINDS, true)) { + $this->lockInstanceState( + $request['agent_id'], $request['host_id'], $request['instance_id'], + $request['instance_uuid'], $now, + ); + $active = $table->where('agent_id', $request['agent_id']) + ->where('host_id', $request['host_id']) + ->where('instance_id', $request['instance_id']) + ->where('instance_uuid', $request['instance_uuid']) + ->where('operation_id', '<>', $request['operation_id']) + ->whereIn('status', ['pending', 'running', 'uncertain']) + ->lockForUpdate()->exists(); + if ($active) { + throw new CloudAgentOperationException( + 'instance_busy', '虚拟机已有 Agent 操作,请先核对结果', 409); + } + } + $table->insert([ 'operation_id' => $request['operation_id'], 'idempotency_key' => $request['idempotency_key'], 'agent_id' => $request['agent_id'], + 'host_id' => $request['host_id'], 'instance_uuid' => $request['instance_uuid'], + 'instance_id' => $request['instance_id'], 'kind' => $request['kind'], 'target' => $request['target'], 'request_hash' => $request['request_hash'], @@ -104,6 +143,18 @@ final class CloudAgentOperationRepository throw new CloudAgentOperationException( 'storage_unavailable', 'Agent 操作记录未能读取', 503); } + if ($request['instance_uuid'] !== null + && in_array($request['kind'], self::MUTATING_KINDS, true)) { + $this->connection->table('cloud_agent_instance_state') + ->where('agent_id', $request['agent_id']) + ->where('host_id', $request['host_id']) + ->where('instance_id', $request['instance_id']) + ->where('instance_uuid', $request['instance_uuid']) + ->update([ + 'last_operation_id' => $request['operation_id'], + 'updated_at' => $now, + ]); + } return [ 'record' => self::publicRecord($created), 'replay' => false, @@ -495,6 +546,37 @@ final class CloudAgentOperationRepository return self::publicRecord($row); } + /** + * Retain a bounded diagnostic window for high-frequency read-only + * observations. Terminal probe rows are not user-visible operations and + * may be pruned after the retention boundary; pending/running/uncertain + * rows are never removed because they may still require reconciliation. + */ + public function pruneReadOnly(int $before, int $limit = 2000): int + { + if ($before < 1 || $limit < 1 || $limit > 10000) { + return 0; + } + try { + $table = $this->connection->table(self::TABLE); + $ids = $table->whereIn('kind', self::READONLY_KINDS) + ->whereIn('status', ['succeeded', 'failed']) + ->where('created_at', '<', $before) + ->orderBy('id') + ->limit($limit) + ->pluck('id') + ->all(); + if ($ids === []) { + return 0; + } + return (int) $table->whereIn('id', $ids)->delete(); + } catch (\Throwable) { + // Journal pruning is maintenance, never a reason to fail a live + // observation tick or to alter its last-known runtime state. + return 0; + } + } + /** * Read a bounded, validated operation history for one fresh-schema VM. * @@ -512,8 +594,24 @@ final class CloudAgentOperationRepository array $kinds, int $page = 1, int $perPage = 10, + ?string $agentId = null, + ?int $hostId = null, + ?int $instanceId = null, ): array { $instanceUuid = self::instanceUuid($instanceUuid); + if ($agentId === null) { + throw new CloudAgentOperationException( + 'invalid_request', 'Agent 操作历史必须绑定 Agent 身份', 422); + } + $agentId = self::agentId($agentId); + if ($hostId === null || $hostId < 1) { + throw new CloudAgentOperationException( + 'invalid_request', 'Agent 操作历史必须绑定宿主身份', 422); + } + if ($instanceId === null || $instanceId < 1) { + throw new CloudAgentOperationException( + 'invalid_request', 'Agent 操作历史必须绑定实例身份', 422); + } if ($page < 1 || $perPage < 1 || $perPage > 50) { throw new CloudAgentOperationException( 'invalid_request', 'Agent 操作历史分页参数无效', 422); @@ -536,6 +634,9 @@ final class CloudAgentOperationRepository try { $query = $this->connection->table(self::TABLE) ->where('instance_uuid', $instanceUuid) + ->where('agent_id', $agentId) + ->where('host_id', $hostId) + ->where('instance_id', $instanceId) ->whereIn('kind', $normalizedKinds) // `id` is only a local tie-breaker. It is never exposed as // the operation identity returned to the browser. @@ -573,22 +674,75 @@ final class CloudAgentOperationRepository $same = (string) ($row->operation_id ?? '') === $request['operation_id'] && (string) ($row->idempotency_key ?? '') === $request['idempotency_key'] && (string) ($row->agent_id ?? '') === $request['agent_id'] + && self::positiveInt($row->host_id ?? null) === $request['host_id'] && (string) ($row->request_hash ?? '') === $request['request_hash'] && (string) ($row->kind ?? '') === $request['kind'] && (string) ($row->target ?? '') === $request['target'] - && self::nullableString($row->instance_uuid ?? null) === $request['instance_uuid']; + && self::nullableString($row->instance_uuid ?? null) === $request['instance_uuid'] + && self::nullablePositiveInt($row->instance_id ?? null) === $request['instance_id']; if (!$same) { throw new CloudAgentOperationException('conflict', 'Agent 幂等键或操作身份已被占用', 409); } return ['record' => self::publicRecord($row), 'replay' => true]; } + /** + * Lock or create the fresh per-instance state row used as a mutation + * mutex. The caller is already inside the reservation transaction. + */ + private function lockInstanceState( + string $agentId, + int $hostId, + int $instanceId, + string $instanceUuid, + int $now, + ): void { + $state = $this->connection->table('cloud_agent_instance_state') + ->where('agent_id', $agentId) + ->where('host_id', $hostId) + ->where('instance_id', $instanceId) + ->where('instance_uuid', $instanceUuid) + ->lockForUpdate()->first(); + if ($state !== null) { + return; + } + $this->connection->table('cloud_agent_instance_state')->insert([ + 'agent_id' => $agentId, + 'host_id' => $hostId, + 'instance_id' => $instanceId, + 'instance_uuid' => $instanceUuid, + 'desired_revision' => 1, + 'desired_json' => '{"state":"unknown"}', + 'observed_revision' => 0, + 'observed_json' => null, + 'convergence_status' => 'unknown', + 'last_operation_id' => null, + 'last_error_code' => '', + 'created_at' => $now, + 'updated_at' => $now, + ]); + $state = $this->connection->table('cloud_agent_instance_state') + ->where('agent_id', $agentId) + ->where('host_id', $hostId) + ->where('instance_id', $instanceId) + ->where('instance_uuid', $instanceUuid) + ->lockForUpdate()->first(); + if ($state === null) { + throw new CloudAgentOperationException( + 'storage_unavailable', 'Agent 实例状态锁不可用', 503); + } + } + /** @param array $input @return array */ private static function normalizeReservation(array $input): array { $operationId = self::operationId($input['operation_id'] ?? null); $idempotencyKey = self::idempotencyKey($input['idempotency_key'] ?? null); $agentId = self::agentId($input['agent_id'] ?? null); + $hostId = self::positiveInt($input['host_id'] ?? null); + if ($hostId < 1) { + throw new CloudAgentOperationException('invalid_request', 'Agent 宿主身份无效', 422); + } $kind = self::kind($input['kind'] ?? null); $target = self::target($input['target'] ?? ''); $requestHash = self::hash($input['request_hash'] ?? null); @@ -596,6 +750,18 @@ final class CloudAgentOperationRepository if ($instanceUuid !== null) { $instanceUuid = self::operationId($instanceUuid); } + $instanceId = $input['instance_id'] ?? null; + if ($instanceId !== null) { + $instanceId = self::positiveInt($instanceId); + if ($instanceId < 1 || $instanceUuid === null) { + throw new CloudAgentOperationException('invalid_request', 'Agent 实例身份不完整', 422); + } + } + if (in_array($kind, self::MUTATING_KINDS, true) + && ($instanceUuid === null || $instanceId === null)) { + throw new CloudAgentOperationException( + 'invalid_request', 'Agent 变更操作必须绑定实例身份', 422); + } if (!array_key_exists('desired', $input) || !is_array($input['desired']) || array_is_list($input['desired'])) { throw new CloudAgentOperationException('invalid_request', 'Agent desired state 必须是对象', 422); @@ -604,7 +770,9 @@ final class CloudAgentOperationRepository 'operation_id' => $operationId, 'idempotency_key' => $idempotencyKey, 'agent_id' => $agentId, + 'host_id' => $hostId, 'instance_uuid' => $instanceUuid, + 'instance_id' => $instanceId, 'kind' => $kind, 'target' => $target, 'request_hash' => $requestHash, @@ -862,6 +1030,15 @@ final class CloudAgentOperationRepository ? (int) $value : null; } + private static function nullablePositiveInt(mixed $value): ?int + { + if ($value === null) { + return null; + } + $normalized = self::positiveInt($value); + return $normalized > 0 ? $normalized : null; + } + private static function nullableString(mixed $value): ?string { return $value === null ? null : (is_string($value) ? $value : null); @@ -882,7 +1059,9 @@ final class CloudAgentOperationRepository 'operation_id' => (string) ($row->operation_id ?? ''), 'idempotency_key' => (string) ($row->idempotency_key ?? ''), 'agent_id' => (string) ($row->agent_id ?? ''), + 'host_id' => self::positiveInt($row->host_id ?? null), 'instance_uuid' => self::nullableString($row->instance_uuid ?? null), + 'instance_id' => self::nullablePositiveInt($row->instance_id ?? null), 'kind' => (string) ($row->kind ?? ''), 'target' => (string) ($row->target ?? ''), 'request_hash' => (string) ($row->request_hash ?? ''), @@ -906,6 +1085,7 @@ final class CloudAgentOperationRepository $operationId = $row->operation_id ?? null; $idempotencyKey = $row->idempotency_key ?? null; $agentId = $row->agent_id ?? null; + $hostId = self::positiveInt($row->host_id ?? null); $kind = $row->kind ?? null; $target = $row->target ?? null; $requestHash = $row->request_hash ?? null; @@ -915,6 +1095,7 @@ final class CloudAgentOperationRepository || preg_match('/^[A-Za-z0-9][A-Za-z0-9_-]{7,63}$/D', $idempotencyKey) !== 1 || !is_string($agentId) || preg_match('/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,95}$/D', $agentId) !== 1 + || $hostId < 1 || !is_string($kind) || !in_array($kind, self::KINDS, true) || !is_string($target) @@ -931,6 +1112,17 @@ final class CloudAgentOperationRepository || preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $instanceUuid) !== 1)) { throw new CloudAgentOperationException('storage_corrupt', 'Agent 实例身份已损坏', 503); } + $instanceId = $row->instance_id ?? null; + if ($instanceId !== null && self::nullablePositiveInt($instanceId) === null) { + throw new CloudAgentOperationException('storage_corrupt', 'Agent 实例编号已损坏', 503); + } + if ($instanceId !== null && $instanceUuid === null) { + throw new CloudAgentOperationException('storage_corrupt', 'Agent 实例身份不完整', 503); + } + if (in_array($kind, self::MUTATING_KINDS, true) + && ($instanceUuid === null || $instanceId === null)) { + throw new CloudAgentOperationException('storage_corrupt', 'Agent 变更操作实例身份不完整', 503); + } $status = $row->status ?? null; if (!is_string($status) || !in_array($status, self::statuses(), true) || self::positiveInt($row->version ?? null) < 1 diff --git a/web/app/Services/Cloud/CloudAgentOperationStatusService.php b/web/app/Services/Cloud/CloudAgentOperationStatusService.php index 6dc0d8d5..01fbbdfb 100644 --- a/web/app/Services/Cloud/CloudAgentOperationStatusService.php +++ b/web/app/Services/Cloud/CloudAgentOperationStatusService.php @@ -2,6 +2,7 @@ namespace App\Services\Cloud; +use App\Models\CloudAgent; use App\Models\CloudAgentOperation; use App\Models\CloudInstance; @@ -31,12 +32,34 @@ final class CloudAgentOperationStatusService || preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $row->instance_uuid) !== 1) { throw new CloudError('任务不存在', 404); } + $agentId = is_string($row->agent_id ?? null) ? trim($row->agent_id) : ''; + $hostId = filter_var($row->host_id ?? null, FILTER_VALIDATE_INT, + ['options' => ['min_range' => 1]]); + $rawInstanceId = $row->instance_id ?? null; + $instanceId = null; + if ($rawInstanceId !== null) { + $instanceId = filter_var($rawInstanceId, FILTER_VALIDATE_INT, + ['options' => ['min_range' => 1]]); + if ($instanceId === false) { + throw new CloudError('任务不存在', 404); + } + } + if ($agentId === '' || $hostId === false) { + throw new CloudError('任务不存在', 404); + } try { - // Keep the tombstone visible for the terminal delete operation. - // Authorization still uses the owner on the retained instance - // row; hiding `deleted` here would make a successful/uncertain - // delete job impossible to poll after the worker marks it done. - $instance = CloudInstance::where('uuid', $row->instance_uuid)->first(); + // The operation stores an immutable host snapshot. The current + // registry must still point the Agent at that same host; a + // re-enrollment/rebind can never make an old UUID operation cross + // into a different host's instance. + $agent = CloudAgent::where('agent_id', $agentId) + ->where('host_id', (int) $hostId)->first(); + $instance = $agent === null + ? null + : CloudInstance::where('host_id', (int) $hostId) + ->when($instanceId !== null, + static fn ($query) => $query->where('id', (int) $instanceId)) + ->where('uuid', $row->instance_uuid)->first(); } catch (\Throwable) { throw new CloudError('任务所属实例暂时不可用', 503); } @@ -56,8 +79,19 @@ final class CloudAgentOperationStatusService } throw new CloudError('Agent 操作记录暂时不可用', 503); } + if ((int) ($record['host_id'] ?? 0) !== (int) $hostId + || (($instanceId === null && ($record['instance_id'] ?? null) !== null) + || ($instanceId !== null && (int) ($record['instance_id'] ?? 0) !== (int) $instanceId)) { + throw new CloudError('任务身份已变化,请重新查询', 409); + } $kind = (string) ($record['kind'] ?? ''); $panelKind = self::panelKind($kind); + if ($panelKind === null) { + // Internal observations/plans are not tenant-visible jobs. A + // caller who knows a UUID still must not turn a capability probe + // into a browser DTO. + throw new CloudError('任务不存在', 404); + } $status = CloudOperationErrorCatalog::userStatus($record['status'] ?? null); if ($status === null) { // The repository validates this invariant. Keep the adapter @@ -117,9 +151,9 @@ final class CloudAgentOperationStatusService ]; } - private static function panelKind(string $agentKind): string + private static function panelKind(string $agentKind): ?string { - return $agentKind === 'domain_power' ? 'action' : $agentKind; + return $agentKind === 'domain_power' ? 'action' : null; } private static function panelAction(?string $agentAction): ?string diff --git a/web/app/Services/Cloud/CloudAgentRegistryHealthService.php b/web/app/Services/Cloud/CloudAgentRegistryHealthService.php index 3bcf9e7c..f56951da 100644 --- a/web/app/Services/Cloud/CloudAgentRegistryHealthService.php +++ b/web/app/Services/Cloud/CloudAgentRegistryHealthService.php @@ -3,10 +3,12 @@ namespace App\Services\Cloud; use App\Models\CloudAgent; +use App\Services\Config; /** - * Optional, read-only health shadow for the greenfield cloud_agents registry. - * It never replaces CloudRuntimeHealthService's legacy Engine/Python check. + * Read-only liveness probe for the greenfield cloud_agents registry. + * It reports Agent transport/readiness only; a healthy liveness response does + * not prove that every registered host or VM has converged. * A host resolver is required for DNS names; the service never uses ambient * DNS or follows redirects, so deployment must inject an explicit allowlist. */ @@ -14,6 +16,8 @@ final class CloudAgentRegistryHealthService { private const REGISTRY_STATUSES = ['enrolling', 'active', 'degraded', 'disabled', 'revoked']; private const HEALTH_STATUSES = ['ok', 'degraded']; + private const MAX_REGISTRY_ROWS = 100; + private const MAX_SNAPSHOT_SECONDS = 10.0; /** @var \Closure():iterable */ private \Closure $registry; @@ -30,13 +34,37 @@ final class CloudAgentRegistryHealthService }); } + /** + * Assemble the health probe from the fresh deployment configuration. + * Secrets remain in the root-only Agent directory; this helper never reads + * legacy Engine settings or encrypted database fields. + */ + public static function fromConfig(): self + { + $directory = Config::get('cloud_agent_secret_dir', '/run/virt-cloud/agents'); + $allowlist = Config::get('cloud_agent_address_allowlist', []); + if (!is_string($directory) || !is_array($allowlist)) { + throw new \InvalidArgumentException('Go Agent health configuration is invalid'); + } + return new self( + null, + new CloudAgentFileSecretResolver($directory), + new CloudAgentStaticHostResolver($allowlist), + ); + } + /** @return array{key:string,label:string,status:string,message:string,checked_at:int,agents:list} */ public function snapshot(?int $now = null): array { $now ??= time(); $rows = []; try { + $count = 0; foreach (($this->registry)() as $row) { + $count++; + if ($count > self::MAX_REGISTRY_ROWS) { + return $this->result('unknown', 'Agent 注册表条目过多,已停止探测。', [], $now); + } $rows[] = is_object($row) ? (method_exists($row, 'toArray') ? $row->toArray() : get_object_vars($row)) : (is_array($row) ? $row : []); @@ -58,19 +86,20 @@ final class CloudAgentRegistryHealthService $seenAgents[$agentId] = true; } $agents = []; + $deadline = microtime(true) + self::MAX_SNAPSHOT_SECONDS; foreach ($rows as $row) { - $agents[] = $this->probe($row, $now); + $agents[] = $this->probe($row, $now, $deadline); } $statuses = array_column($agents, 'status'); $aggregate = in_array('unknown', $statuses, true) ? 'unknown' : (in_array('degraded', $statuses, true) ? 'degraded' : (in_array('disabled', $statuses, true) ? 'disabled' : 'ok')); return $this->result($aggregate, - $aggregate === 'ok' ? '已完成 Agent liveness shadow;不替代引擎检查。' : '部分 Agent 未通过安全探测。', + $aggregate === 'ok' ? 'Agent liveness 正常;这不代表所有宿主机均已收敛。' : '部分 Agent 未通过安全探测。', $agents, $now); } - private function probe(array $row, int $now): array + private function probe(array $row, int $now, float $deadline): array { $agentId = trim((string) ($row['agent_id'] ?? '')); $base = ['host_id' => (int) ($row['host_id'] ?? 0), 'agent_id' => $agentId, @@ -83,6 +112,9 @@ final class CloudAgentRegistryHealthService ? 'disabled' : 'unknown'; return $base + ['status' => $status, 'message' => 'Agent 尚未进入可探测状态,未发起网络请求。', 'checked_at' => $now]; } + if (microtime(true) >= $deadline) { + return $base + ['status' => 'unknown', 'message' => 'Agent 探测达到时间上限,稍后重试。', 'checked_at' => $now]; + } try { $resolvedAddresses = []; $endpoint = CloudAgentEndpointPolicy::validate((string) ($row['endpoint'] ?? ''), @@ -100,9 +132,21 @@ final class CloudAgentRegistryHealthService if (!$tlsReady) { return $base + ['status' => 'unknown', 'message' => 'Agent TLS/密钥 resolver 未配置,未执行探测。', 'checked_at' => $now]; } + $fingerprint = strtolower(trim((string) ($row['client_certificate_fingerprint'] ?? ''))); + if (preg_match('/^[0-9a-f]{64}$/D', $fingerprint) !== 1) { + return $base + ['status' => 'unknown', 'message' => 'Agent 客户端证书指纹未登记,未执行探测。', 'checked_at' => $now]; + } + foreach (['ca_file', 'cert_file', 'key_file'] as $tlsKey) { + if (!is_string($material[$tlsKey] ?? null) + || !CloudAgentHostClientResolver::validateCredentialPath( + $material[$tlsKey], $tlsKey === 'key_file' + )) { + return $base + ['status' => 'unknown', 'message' => 'Agent mTLS 文件权限不符合安全策略,未执行探测。', 'checked_at' => $now]; + } + } $client = new CloudAgentClient($endpoint, (string) ($material['hmac_key'] ?? ''), null, array_intersect_key($material, array_flip(['ca_file', 'cert_file', 'key_file'])), 4.0, - $agentId, $resolvedAddresses); + $agentId, $resolvedAddresses, $fingerprint); $health = $client->health(); $status = in_array($health['status'] ?? null, self::HEALTH_STATUSES, true) ? $health['status'] : 'unknown'; return $base + ['status' => $status, 'message' => $status === 'ok' ? 'Agent liveness 正常。' : 'Agent readiness 需关注。', 'checked_at' => $now]; @@ -113,7 +157,9 @@ final class CloudAgentRegistryHealthService private function result(string $status, string $message, array $agents, int $now): array { - return ['key' => 'cloud_agent_shadow', 'label' => 'Go Agent(shadow)', 'status' => $status, + // Keep the key stable for existing admin-page consumers during the + // fresh-schema cutover; the implementation is no longer a shadow. + return ['key' => 'cloud_agent_shadow', 'label' => 'Go Agent', 'status' => $status, 'message' => $message, 'checked_at' => $now, 'agents' => $agents]; } } diff --git a/web/app/Services/Cloud/CloudAgentVmGateway.php b/web/app/Services/Cloud/CloudAgentVmGateway.php index e614e4ff..29df3b1d 100644 --- a/web/app/Services/Cloud/CloudAgentVmGateway.php +++ b/web/app/Services/Cloud/CloudAgentVmGateway.php @@ -125,9 +125,16 @@ final class CloudAgentVmGateway array $payload, array $actor, ?string $instanceUuid = null, + ?int $instanceId = null, ): array { $agentKind = self::agentKindForPanelKind($panelKind); self::assertOperationIdentity($operationId, $idempotencyKey, $target, $instanceUuid); + if ($instanceId !== null && $instanceId < 1) { + throw new CloudError('虚拟机编号无效', 422); + } + if ($instanceId !== null && $instanceUuid === null) { + throw new CloudError('虚拟机身份不完整', 422); + } if ($agentKind === 'domain_power' && ($instanceUuid === null || $instanceUuid !== $target)) { throw new CloudError('电源操作实例身份与目标不一致', 422); @@ -138,11 +145,16 @@ final class CloudAgentVmGateway // Resolve before reserving so an operation cannot be written against // a host whose current enrollment or key material is unusable. $handle = $this->resolver->resolve($hostId); + if ($handle->hostId !== $hostId) { + throw new CloudError('宿主机 Agent 身份不匹配', 503); + } $reservation = $this->operations->reserve([ 'operation_id' => $operationId, 'idempotency_key' => $idempotencyKey, 'agent_id' => $handle->agentId, + 'host_id' => $hostId, 'instance_uuid' => $instanceUuid, + 'instance_id' => $instanceId, 'kind' => $agentKind, 'target' => $target, 'request_hash' => $requestHash, @@ -180,57 +192,98 @@ final class CloudAgentVmGateway $payload, $actor, ); - $envelope = CloudAgentOperationEnvelope::fromResponse( - $response, - $operationId, - $idempotencyKey, - $requestHash, - $agentKind, - $target, - ); - if ($envelope->status === 'succeeded') { - $record = $this->operations->finalize( - $claim, - 'succeeded', - $envelope->result, - '', - $envelope->commitBoundary, - ); - } elseif ($envelope->status === 'failed') { - $record = $this->operations->finalize( - $claim, - 'failed', - $envelope->result, - $envelope->errorCode, - $envelope->commitBoundary, - ); + // For a definitive 4xx/5xx rejection CloudAgentClient returns a + // deliberately reduced envelope (operation_id/status/version/ + // error_code) so host error text cannot cross the boundary. The + // Agent validates the payload before journal execution in this + // case, therefore finalize it as failed instead of misclassifying + // a deterministic rejection as an uncertain host mutation. + if (self::isReducedFailure($response, $operationId)) { + $errorCode = is_string($response['error_code'] ?? null) + && $response['error_code'] !== '' + ? $response['error_code'] : 'agent_operation_rejected'; + if (self::isDefinitiveReducedRejection($response)) { + $record = $this->operations->finalize( + $claim, 'failed', null, $errorCode, null, + ); + } else { + // A 5xx/timeout-like response, or any non-terminal + // reduced envelope, cannot prove that the host did not + // commit. Preserve the operation for reconciliation. + $record = $this->operations->markUncertain( + $claim, 'agent_operation_transport_uncertain', + ); + } } else { - $record = $this->operations->markUncertain( - $claim, - $envelope->errorCode !== '' ? $envelope->errorCode : 'agent_operation_incomplete', - $envelope->commitBoundary, + $envelope = CloudAgentOperationEnvelope::fromResponse( + $response, + $operationId, + $idempotencyKey, + $requestHash, + $agentKind, + $target, ); + if ($envelope->status === 'succeeded') { + $record = $this->operations->finalize( + $claim, + 'succeeded', + $envelope->result, + '', + $envelope->commitBoundary, + ); + } elseif ($envelope->status === 'failed') { + $record = $this->operations->finalize( + $claim, + 'failed', + $envelope->result, + $envelope->errorCode, + $envelope->commitBoundary, + ); + } else { + $record = $this->operations->markUncertain( + $claim, + $envelope->errorCode !== '' ? $envelope->errorCode : 'agent_operation_incomplete', + $envelope->commitBoundary, + ); + } } } catch (CloudAgentOperationException $error) { // A database CAS failure must not be converted into a transport // result. The caller must surface it and reconcile by operation ID. throw $error; } catch (CloudError $error) { - // A timeout, reset, or incomplete envelope does not prove whether - // the host acted. Keep the row uncertain and never replay blindly. + // Local request rejection (bad mTLS material, unsupported payload, + // or an explicit 4xx/501) happened before a host-side commit and + // is a terminal journal failure. CloudAgentClient normalizes + // transport/ambiguous response failures to 502; only that class + // is fenced as uncertain and requires operation-ID reconciliation. + if ($error->httpStatus !== 502) { + try { + $this->operations->finalize( + $claim, 'failed', null, 'agent_request_rejected', null, + ); + } catch (CloudAgentOperationException) { + throw $error; + } + throw $error; + } try { $record = $this->operations->markUncertain($claim, 'agent_transport_uncertain'); } catch (CloudAgentOperationException) { throw $error; } - throw new CloudError('宿主机 Agent 响应不确定,请按操作编号核对结果', $error->httpStatus); + throw new CloudAgentDispatchUncertain( + $operationId, + $record, + $error->httpStatus, + ); } catch (\Throwable $error) { try { $record = $this->operations->markUncertain($claim, 'agent_response_invalid'); } catch (CloudAgentOperationException) { throw new CloudError('宿主机 Agent 响应无效,请按操作编号核对结果', 502); } - throw new CloudError('宿主机 Agent 响应无效,请按操作编号核对结果', 502); + throw new CloudAgentDispatchUncertain($operationId, $record, 502); } return [ @@ -301,6 +354,40 @@ final class CloudAgentVmGateway throw new CloudError('宿主机 Agent 操作 journal 映射无效', 501); } + /** + * CloudAgentClient uses this bounded shape only for non-2xx responses. + * It is intentionally stricter than a normal envelope: no result or + * arbitrary response fields are accepted on this path. + * + * @param array $response + */ + private static function isReducedFailure(array $response, string $operationId): bool + { + if (($response['operation_id'] ?? null) !== $operationId + || !isset($response['_agent_http_status']) + || !is_int($response['_agent_http_status']) + || $response['_agent_http_status'] < 400 + || $response['_agent_http_status'] > 599 + || !is_string($response['status'] ?? null) + || !in_array($response['status'], ['pending', 'running', 'succeeded', 'failed', 'uncertain'], true) + || array_key_exists('result', $response) + || array_key_exists('commit_boundary', $response) + || array_key_exists('request_hash', $response) + || array_key_exists('kind', $response) + || array_key_exists('target', $response)) { + return false; + } + return !array_key_exists('idempotency_key', $response); + } + + /** @param array $response */ + private static function isDefinitiveReducedRejection(array $response): bool + { + return ($response['status'] ?? null) === 'failed' + && in_array((int) ($response['_agent_http_status'] ?? 0), + [400, 401, 403, 404, 413, 422, 501], true); + } + private static function assertOperationIdentity( string $operationId, string $idempotencyKey, diff --git a/web/app/Services/Cloud/CloudInstanceRuntimeService.php b/web/app/Services/Cloud/CloudInstanceRuntimeService.php index 7f77a124..0cd2cefe 100644 --- a/web/app/Services/Cloud/CloudInstanceRuntimeService.php +++ b/web/app/Services/Cloud/CloudInstanceRuntimeService.php @@ -196,6 +196,26 @@ final class CloudInstanceRuntimeService return (int) $query->increment('revision', 1, $values); } + /** + * Advance the scheduler cursor for a row that was selected but not sent + * before the soft Agent batch deadline. This does not alter the last + * known power state or claim that a host request happened; it prevents a + * large offline host group from starving later hosts on every tick. + */ + public static function deferObservation( + int $instanceId, + int $hostId, + string $uuid, + ?int $now = null, + ): void { + $now = self::now($now); + [$instanceId, $hostId, $uuid] = self::identity($instanceId, $hostId, $uuid); + CloudInstanceRuntime::where('instance_id', $instanceId) + ->where('host_id', $hostId) + ->where('uuid', $uuid) + ->update(['last_attempt_at' => $now, 'updated_at' => $now]); + } + /** Remove an observation when its panel instance is irreversibly deleted. */ public static function forget(int $instanceId): int { @@ -329,6 +349,12 @@ final class CloudInstanceRuntimeService $query->where('revision', max(0, $revision)); } $updated = $query->increment('revision', 1, $values); + if ($revision !== null && (int) $updated !== 1) { + // A concurrent writer or delete won the generation race. Never + // return the newer row as if this stale host response had been + // committed; callers must treat it as skipped. + throw new RuntimeException('Runtime observation CAS lost'); + } $runtime = CloudInstanceRuntime::where('instance_id', $instanceId)->first(); if ($runtime !== null) { // A zero-row conditional update means a newer writer won the diff --git a/web/app/Services/Cloud/CloudInstanceRuntimeSyncService.php b/web/app/Services/Cloud/CloudInstanceRuntimeSyncService.php index 112a2385..53f4ef8a 100644 --- a/web/app/Services/Cloud/CloudInstanceRuntimeSyncService.php +++ b/web/app/Services/Cloud/CloudInstanceRuntimeSyncService.php @@ -16,6 +16,11 @@ use App\Models\CloudInstanceRuntime; */ final class CloudInstanceRuntimeSyncService { + // A lifecycle tick must not spend hours serially waiting on an offline + // host. The cursor remains in the runtime table, so the next tick can + // resume the bounded rotation. + private const MAX_SYNC_SECONDS = 25.0; + /** @var \Closure():CloudAgentObservationService */ private \Closure $agentFactory; @@ -119,18 +124,47 @@ final class CloudInstanceRuntimeSyncService } } + $deadline = microtime(true) + self::MAX_SYNC_SECONDS; foreach ($groups as $hostId => $group) { - $this->syncHost((int) $hostId, $group, $actor, $result); + if (microtime(true) >= $deadline) { + foreach ($group as $row) { + $this->defer($row); + } + $result['skipped'] += count($group); + continue; + } + $this->syncHost((int) $hostId, $group, $actor, $result, $deadline); } + // Routine observations are high-frequency internal readbacks, not + // tenant-visible operation history. Keep a bounded diagnostic window + // while retaining every pending/running/uncertain row for explicit + // reconciliation. + (new CloudAgentOperationRepository())->pruneReadOnly( + time() - 86400, + 2000, + ); return $result; } - private function syncHost(int $hostId, array $group, array $actor, array &$result): void + private function syncHost( + int $hostId, + array $group, + array $actor, + array &$result, + float $deadline, + ): void { // Claim each row immediately before its Agent request. A successful // worker operation advances the same revision, making the response // stale without allowing it to overwrite fresher state. - foreach ($group as $row) { + foreach ($group as $index => $row) { + if (microtime(true) >= $deadline) { + foreach (array_slice($group, (int) $index) as $deferred) { + $this->defer($deferred); + } + $result['skipped'] += max(1, count($group) - (int) $index); + break; + } $instanceId = (int) $row['id']; $uuid = (string) $row['uuid']; try { @@ -142,7 +176,14 @@ final class CloudInstanceRuntimeSyncService continue; } try { - $observed = $this->agent()->observe($hostId, $uuid, $actor); + $observed = $this->agent()->observe( + $hostId, + $uuid, + $actor, + null, + null, + $instanceId, + ); $record = is_array($observed['record'] ?? null) ? $observed['record'] : []; $candidate = is_array($observed['observation'] ?? null) @@ -193,6 +234,21 @@ final class CloudInstanceRuntimeSyncService return $this->agent ??= ($this->agentFactory)(); } + /** @param array{id:int,host_id:int,uuid:string} $row */ + private function defer(array $row): void + { + try { + CloudInstanceRuntimeService::deferObservation( + (int) ($row['id'] ?? 0), + (int) ($row['host_id'] ?? 0), + (string) ($row['uuid'] ?? ''), + ); + } catch (\Throwable) { + // Scheduling metadata is best-effort; never turn a soft deadline + // into a lifecycle failure or recreate a deleted runtime row. + } + } + /** @param array{observed:int,unavailable:int,skipped:int,errors:list} $result */ private function markUnavailable( int $instanceId, diff --git a/web/app/Services/Cloud/CloudRuntimeHealthService.php b/web/app/Services/Cloud/CloudRuntimeHealthService.php index 889ebcb0..c5d69258 100644 --- a/web/app/Services/Cloud/CloudRuntimeHealthService.php +++ b/web/app/Services/Cloud/CloudRuntimeHealthService.php @@ -19,8 +19,13 @@ final class CloudRuntimeHealthService ?callable $redisProbe = null, ?callable $heartbeatProbe = null) { $this->databaseProbe = \Closure::fromCallable($databaseProbe ?? [self::class, 'probeDatabase']); + // Keep the constructor injection name for existing tests/callers, but + // the production default is now the fresh Agent registry. The + // injected probe is only a test seam; production construction uses + // the Go Agent registry and never falls back to Python or SSH. $this->engineProbe = \Closure::fromCallable($engineProbe ?? static function (array $actor): bool { - $result = (new CloudEngineClient(null, 4.0))->request('GET', '/internal/v1/health', $actor); + unset($actor); + $result = CloudAgentRegistryHealthService::fromConfig()->snapshot(); return ($result['status'] ?? null) === 'ok'; }); $this->redisProbe = \Closure::fromCallable($redisProbe ?? static function (): bool { @@ -59,10 +64,13 @@ final class CloudRuntimeHealthService $checks[] = $this->row('log_database', '日志数据库', 'disabled', '未启用日志数据库;虚拟化指标依赖该配置。'); } - $checks[] = $this->check('engine', '私有引擎', + // Keep the historical `engine` key so the existing admin page and + // stored UI tests remain stable; its implementation is now the Go + // Agent registry/liveness probe, not the retired Python service. + $checks[] = $this->check('engine', 'Go Agent', fn (): bool => ($this->engineProbe)($actor) === true, - '签名接口及引擎存储检查通过;未探测宿主机或虚拟机。', - '引擎检查失败或超时,请检查引擎服务、签名配置及存储。'); + 'Agent 签名接口与注册表 liveness 检查通过;未探测宿主机或虚拟机。', + 'Go Agent 检查失败或未完成安全探测,请检查 Agent、mTLS、签名密钥和注册表。'); if (trim((string) Config::get('redis_host', '')) !== '') { $checks[] = $this->check('redis', 'Redis', fn (): bool => ($this->redisProbe)() === true, diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index 4c830ec0..232a2292 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -22,6 +22,11 @@ $app->group('/api/v1', function () { $this->get('/' . $name . '/{id:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}', 'App\Controllers\CloudApiController:job'); } + // Explicit Agent read-back endpoint. It is deliberately POST so callers + // cannot mistake a normal status poll for permission to trigger a host + // probe; the controller still authorizes the operation owner/admin. + $this->post('/jobs/{id:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}/reconcile', + 'App\Controllers\CloudApiController:reconcileJob'); $this->get('/resources', 'App\Controllers\CloudApiController:resources'); $this->map(['GET', 'POST'], '/user-isos', 'App\Controllers\CloudApiController:userIsos'); $this->get('/user-isos/import-jobs/{job_id:[A-Za-z0-9_-]{8,80}}', diff --git a/web/public/cloud/admin-operations.js b/web/public/cloud/admin-operations.js index 9d8e53dd..b997f568 100644 --- a/web/public/cloud/admin-operations.js +++ b/web/public/cloud/admin-operations.js @@ -247,7 +247,7 @@ function renderHealth(payload) { body.append(column); } state.health = payload; - healthNotice(`检查于 ${date(payload.checked_at)} · 页面可见时每 30 秒检查;引擎连通不代表宿主机均正常。`); + healthNotice(`检查于 ${date(payload.checked_at)} · 页面可见时每 30 秒检查;Agent liveness 不代表所有宿主机均正常。`); } function stopHealth() { diff --git a/web/public/cloud/runtime.js b/web/public/cloud/runtime.js index 0d1531e1..f40c7819 100644 --- a/web/public/cloud/runtime.js +++ b/web/public/cloud/runtime.js @@ -8,6 +8,8 @@ let vmStatusRequest = null; let vmStatusGeneration = 0; const POWER_STATE_CONVERGENCE_INTERVAL_MS = 1500; const POWER_STATE_CONVERGENCE_TIMEOUT_MS = 60000; +const POWER_OPERATION_STORAGE_PREFIX = 'virt-cloud:power-operation:'; +const powerOperationMemory = new Map(); // Keep the historical storage key so an interrupted deployment made just // before an online update can still reconcile to the same opening invoice. const PROVISION_INVOICE_STORAGE_PREFIX = 'virt-cloud:pending-order:'; @@ -386,18 +388,71 @@ async function waitForShutdownState(vmId) { } return false; } +function powerOperationStorageKey(vmId, action, vmUuid = '') { + const identity = String(vmUuid || vmId); + return `${POWER_OPERATION_STORAGE_PREFIX}${encodeURIComponent(identity)}:${encodeURIComponent(String(action))}`; +} +function newPowerOperationKey() { + if (!globalThis.crypto?.getRandomValues) throw new Error('当前浏览器不支持安全操作标识,请更新浏览器。'); + const bytes = new Uint8Array(16); + crypto.getRandomValues(bytes); + return `action-${Array.from(bytes, value => value.toString(16).padStart(2, '0')).join('')}`; +} +function powerOperationKey(vmId, action, vmUuid = '') { + const storageKey = powerOperationStorageKey(vmId, action, vmUuid); + let saved = ''; + try { saved = sessionStorage.getItem(storageKey) || ''; } catch {} + if (/^[A-Za-z0-9][A-Za-z0-9_-]{7,63}$/.test(saved)) return saved; + const inMemory = powerOperationMemory.get(storageKey); + if (/^[A-Za-z0-9][A-Za-z0-9_-]{7,63}$/.test(inMemory || '')) return inMemory; + const key = newPowerOperationKey(); + powerOperationMemory.set(storageKey, key); + while (powerOperationMemory.size > 128) powerOperationMemory.delete(powerOperationMemory.keys().next().value); + try { sessionStorage.setItem(storageKey, key); } catch {} + return key; +} +function clearPowerOperationKey(vmId, action, vmUuid = '') { + const storageKey = powerOperationStorageKey(vmId, action, vmUuid); + powerOperationMemory.delete(storageKey); + try { sessionStorage.removeItem(storageKey); } catch {} +} +function retainPowerOperationKey(error) { + if (error?.job?.status) return error.job.status === 'uncertain'; + const status = Number(error?.status || 0); + // A timeout, 409 lease/operation conflict, rate limit, or 5xx response + // cannot prove that the host did not commit. Keep the same key so a later + // click/reload reconciles the original operation instead of creating one. + return status === 0 || status === 409 || status === 429 || status >= 500; +} async function powerAction(action, dangerous = false) { const vm = selectedVm(); if (!vm?.can_operate) return toast('实例服务已到期或正在执行策略,日常操作暂不可用。', 'error'); if (state.mutationBusy || state.powerMutation) return; if (dangerous && !confirm('此操作会强制断电,可能丢失未保存的数据。确认继续?')) return; const mutation = { vmId: vm.api_id, action }; + const idempotencyKey = powerOperationKey(vm.api_id, action, vm.uuid); state.powerMutation = mutation; renderVmDetail(vm); try { - const result = await api(`${vmPath(vm)}/actions`, { method: 'POST', body: { action } }); + const result = await api(`${vmPath(vm)}/actions`, { + method: 'POST', + headers: { 'Idempotency-Key': idempotencyKey }, + body: { action, idempotency_key: idempotencyKey }, + }); + let job = null; + if (result.job_id) job = await pollJob('power', result.job_id, undefined, undefined, { allowUncertain: true, reconcile: true }); + if (result?.status === 'uncertain' || job?.status === 'uncertain') { + toast('结果待核对,请按任务编号查询;不要重复提交。', 'warning', '电源任务'); + void loadVmStatus(vm.api_id, { live: true, force: true }); + return; + } + if (!result.job_id && !['succeeded', 'completed', 'done'].includes(result?.status)) { + toast('电源命令已提交,但任务编号缺失;请到实例详情核查,暂勿重复提交。', 'warning', '电源任务'); + void loadVmStatus(vm.api_id, { live: true, force: true }); + return; + } + clearPowerOperationKey(vm.api_id, action, vm.uuid); toast(`${result?.action_label || '电源'}命令发送成功`); - if (result.job_id) await pollJob('power', result.job_id); if (action === 'shutdown') { const stopped = await waitForShutdownState(vm.api_id); if (stopped) toast('已确认实例完成关机。', 'success'); @@ -406,7 +461,12 @@ async function powerAction(action, dangerous = false) { await loadVmStatus(vm.api_id, { live: true, force: true }); } } catch (error) { - toast(error.message, 'error'); + if (!retainPowerOperationKey(error)) clearPowerOperationKey(vm.api_id, action, vm.uuid); + if (retainPowerOperationKey(error)) { + toast('电源请求结果待核对,请按任务编号查询;不要重复提交。', 'warning', '电源任务'); + } else { + toast(error.message, 'error'); + } void loadVmStatus(vm.api_id, { live: true, force: true }); } finally { if (state.powerMutation === mutation) state.powerMutation = null; @@ -976,7 +1036,39 @@ async function openReinstall(vm, bootstrap = null) { } async function pollJob(kind, id, phaseId, errorId, options = {}) { const path = `/api/v1/jobs/${encodeURIComponent(id)}`; - for (let attempt = 0; attempt < 600; attempt++) { if (document.visibilityState === 'hidden') await new Promise(resolve => setTimeout(resolve, 1000)); const job = await api(path); if (phaseId) text(phaseId, job.phase || job.status || '正在处理…'); if (['completed', 'succeeded', 'done'].includes(job.status)) return job; if (job.status === 'uncertain' && options.allowUncertain === true) return job; if (['failed', 'cancelled', 'needs_attention', 'uncertain'].includes(job.status)) throw new Error(job.error || job.message || '任务失败,请核查虚拟机状态。'); await new Promise(resolve => setTimeout(resolve, 2000)); } + let reconciled = false; + for (let attempt = 0; attempt < 600; attempt++) { + if (document.visibilityState === 'hidden') await new Promise(resolve => setTimeout(resolve, 1000)); + const job = await api(path); + if (phaseId) text(phaseId, job.phase || job.status || '正在处理…'); + if (['completed', 'succeeded', 'done'].includes(job.status)) return job; + if (job.status === 'uncertain' && options.reconcile === true && !reconciled) { + // This is a read-back request, never a replay of the original action. + // A failed/timeout reconcile leaves the original job object intact so + // the caller keeps its idempotency key and shows the existing warning. + reconciled = true; + try { + const checked = await api(`${path}/reconcile`, { method: 'POST', body: {} }); + if (['completed', 'succeeded', 'done'].includes(checked?.status)) return checked; + if (['failed', 'cancelled', 'needs_attention'].includes(checked?.status)) { + const error = new Error(checked.error || checked.message || '任务失败,请核查虚拟机状态。'); + error.job = checked; + throw error; + } + if (checked?.status === 'uncertain') return checked; + } catch (error) { + if (error?.job) throw error; + } + if (options.allowUncertain === true) return job; + } + if (job.status === 'uncertain' && options.allowUncertain === true) return job; + if (['failed', 'cancelled', 'needs_attention', 'uncertain'].includes(job.status)) { + const error = new Error(job.error || job.message || '任务失败,请核查虚拟机状态。'); + error.job = job; + throw error; + } + await new Promise(resolve => setTimeout(resolve, 2000)); + } throw new Error('任务仍在处理,可稍后刷新详情。不要重复提交。'); } async function reinstall(event) { diff --git a/web/resources/views/venix/admin/cloud/operations.tpl b/web/resources/views/venix/admin/cloud/operations.tpl index 486a7156..d9da5928 100644 --- a/web/resources/views/venix/admin/cloud/operations.tpl +++ b/web/resources/views/venix/admin/cloud/operations.tpl @@ -10,7 +10,7 @@

运行状态

尚未检查。运行状态单独加载,不影响任务列表。

-

引擎连通不代表所有宿主机正常;未收到的后台心跳会标为未知。

+

Agent liveness 不代表所有宿主机正常;未收到的后台心跳会标为未知。

{foreach ['total'=>['任务总数','mdi-format-list-bulleted','primary'],'failed'=>['失败任务','mdi-alert-circle-outline','danger'],'uncertain'=>['待核对','mdi-help-circle-outline','warning'],'delayed'=>['排队或租约异常','mdi-clock-alert-outline','warning']] as $key=>$card} diff --git a/web/sql/cloud_agent_v1.sql b/web/sql/cloud_agent_v1.sql index 975e5698..0753d3af 100644 --- a/web/sql/cloud_agent_v1.sql +++ b/web/sql/cloud_agent_v1.sql @@ -1,7 +1,8 @@ --- Virt Cloud Go Agent control-plane schema v2. +-- Virt Cloud Go Agent control-plane schema v4. -- The filename remains v1 because it is the v1 API installer; the marker --- version is v2 after adding the operation CAS token and must not accept an --- older pre-CAS database. +-- version is v4 after adding operation CAS and host/instance identity fences; +-- it must not accept an +-- older pre-CAS database or a UUID-only instance mutex. -- -- This is a fresh schema for the new panel <-> host-agent protocol. It is -- intentionally not an upgrade script: it does not read, decrypt, rename, or @@ -54,7 +55,9 @@ CREATE TABLE cloud_agent_operations ( operation_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, idempotency_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, agent_id VARCHAR(96) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + host_id BIGINT UNSIGNED NOT NULL, instance_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL, + instance_id BIGINT UNSIGNED NULL, kind VARCHAR(48) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, target VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', request_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, @@ -76,7 +79,8 @@ CREATE TABLE cloud_agent_operations ( completed_at BIGINT UNSIGNED NULL, UNIQUE KEY cloud_agent_operation_id (operation_id), UNIQUE KEY cloud_agent_operation_idempotency (agent_id, idempotency_key), - KEY cloud_agent_operation_instance (instance_uuid, created_at), + KEY cloud_agent_operation_instance (host_id, instance_id, instance_uuid, created_at), + KEY cloud_agent_operation_readonly_retention (kind, status, created_at, id), KEY cloud_agent_operation_status (agent_id, status, updated_at), KEY cloud_agent_operation_lease (status, lease_expires_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; @@ -84,6 +88,8 @@ CREATE TABLE cloud_agent_operations ( CREATE TABLE cloud_agent_instance_state ( id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, agent_id VARCHAR(96) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + host_id BIGINT UNSIGNED NOT NULL, + instance_id BIGINT UNSIGNED NOT NULL, instance_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, desired_revision BIGINT UNSIGNED NOT NULL DEFAULT 1, desired_json JSON NOT NULL, @@ -94,7 +100,7 @@ CREATE TABLE cloud_agent_instance_state ( last_error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', created_at BIGINT UNSIGNED NOT NULL, updated_at BIGINT UNSIGNED NOT NULL, - UNIQUE KEY cloud_agent_instance_identity (agent_id, instance_uuid), + UNIQUE KEY cloud_agent_instance_identity (agent_id, host_id, instance_id, instance_uuid), KEY cloud_agent_instance_convergence (agent_id, convergence_status, updated_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; @@ -132,6 +138,6 @@ CREATE TABLE cloud_agent_schema_meta ( INSERT INTO cloud_agent_schema_meta (schema_name, schema_version, schema_fingerprint, created_at) VALUES - ('virt-cloud-agent', 2, - '252088fbb05f7d2d1224f7f7da62b2a2cca2b719a394bd84fe295a2d1fdac61c', + ('virt-cloud-agent', 4, + '87a45391989171906c485383fa1f57fabad828adcb23ff01fe67cae99e7ed6e4', UNIX_TIMESTAMP()); diff --git a/web/tests/CloudAgentActionCutoverContractTest.php b/web/tests/CloudAgentActionCutoverContractTest.php new file mode 100644 index 00000000..5b2b4232 --- /dev/null +++ b/web/tests/CloudAgentActionCutoverContractTest.php @@ -0,0 +1,83 @@ +dispatch') + && str_contains($actionBody, 'CloudAgentDispatchUncertain') + && str_contains($actionBody, "'/api/v1/jobs/'"), + 'Action endpoint is not wired to the Agent operation/status boundary.'); +foreach (['CloudJobService::enqueue', 'CloudEngineClient', 'ssh', 'SSH', 'shell_exec', 'curl_exec'] as $forbidden) { + $assert(!str_contains($actionBody, $forbidden), + 'Action endpoint still contains a legacy transport/queue reference: ' . $forbidden); +} +$assert(str_contains($service, 'CloudAgentOperationIdentity::uuidForIdempotencyKey') + && str_contains($service, '重试必须同时提供 operation_id 和幂等键') + && !str_contains($service, 'idempotencyKeyForOperationId') + && str_contains($service, 'CloudAgentVmGateway') + && str_contains($service, "'action'") + && str_contains($service, "'instance_id' => (string)"), + 'Action service lost typed identity, gateway dispatch, or browser DTO projection.'); +$assert(str_contains($identity, 'random_bytes(16)') + && str_contains($identity, 'uuidForIdempotencyKey') + && str_contains($identity, 'validateIdempotencyKey') + && !str_contains($identity, 'idempotencyKeyForOperationId'), + 'Agent operation identity must use UUIDs and bounded idempotency keys without deriving a key from an opaque operation UUID.'); +$assert(str_contains($gateway, "'poweroff' => 'destroy'") + && str_contains($gateway, 'isReducedFailure') + && str_contains($gateway, 'isDefinitiveReducedRejection') + && str_contains($gateway, 'agent_operation_transport_uncertain'), + 'Gateway lost the panel-to-Agent power mapping or reduced-error handling.'); +$assert(str_contains($repository, 'lockInstanceState') + && str_contains($repository, "'instance_busy'") + && str_contains($repository, "whereIn('status', ['pending', 'running', 'uncertain'])") + && str_contains($repository, 'in_array($request[\'kind\'], self::MUTATING_KINDS, true)'), + 'Agent mutations lost the per-instance durable serialization gate.'); +$assert(str_contains($actionBody, 'bodyIdempotency !== $headerIdempotency') + && str_contains($actionBody, 'bodyOperationId !== $headerOperationId') + && str_contains($actionBody, '重试必须同时提供 operation_id 和幂等键') + && str_contains($actionBody, 'CloudAgentOperationException'), + 'Action endpoint must reject split identity headers and preserve deterministic repository errors.'); +$assert(str_contains($runtime, 'POWER_OPERATION_STORAGE_PREFIX') + && str_contains($runtime, 'powerOperationStorageKey(vmId, action, vmUuid = \'\')') + && str_contains($runtime, 'powerOperationMemory') + && str_contains($runtime, "body: { action, idempotency_key: idempotencyKey }") + && str_contains($runtime, "headers: { 'Idempotency-Key': idempotencyKey }") + && str_contains($runtime, 'allowUncertain: true') + && str_contains($runtime, 'reconcile: true') + && str_contains($runtime, '${path}/reconcile') + && str_contains($runtime, '结果待核对') + && str_contains($runtime, 'clearPowerOperationKey'), + 'Power UI must persist one bounded idempotency key and surface uncertain Agent results without retrying.'); + +fwrite(STDOUT, "Cloud Agent action cutover contract passed.\n"); diff --git a/web/tests/CloudAgentCreatePlanServiceContractTest.php b/web/tests/CloudAgentCreatePlanServiceContractTest.php index fb27bb1d..f2674a31 100644 --- a/web/tests/CloudAgentCreatePlanServiceContractTest.php +++ b/web/tests/CloudAgentCreatePlanServiceContractTest.php @@ -22,7 +22,11 @@ $assert(str_contains($service, 'final class CloudAgentCreatePlanService') && str_contains($service, 'CloudAgentOperationRepository') && str_contains($service, 'public static function fromConfig(') && str_contains($service, 'CloudAgentOperationEnvelope::fromResponse(') + && str_contains($service, '重试必须同时提供 operation_id 和幂等键') + && !str_contains($service, 'idempotencyKeyForOperationId') && str_contains($service, 'CloudAgentDispatchUncertain') + && str_contains($service, 'isDefinitiveReducedRejection') + && str_contains($service, 'agent_create_plan_transport_uncertain') && str_contains($service, 'markUncertain') && str_contains($service, "'succeeded'") && str_contains($service, 'never a create acknowledgement'), diff --git a/web/tests/CloudAgentFileSecretResolverContractTest.php b/web/tests/CloudAgentFileSecretResolverContractTest.php new file mode 100644 index 00000000..4904d0d7 --- /dev/null +++ b/web/tests/CloudAgentFileSecretResolverContractTest.php @@ -0,0 +1,80 @@ + $encodedKey . "\n", + '/run/virt-cloud/agents/agent-103.ca.pem' => 'CA', + '/run/virt-cloud/agents/agent-103.crt' => 'CERT', + '/run/virt-cloud/agents/agent-103.key' => 'KEY', +]; +$resolver = new CloudAgentFileSecretResolver( + '/run/virt-cloud/agents', + static fn (string $path): ?string => $files[$path] ?? null, +); +$material = $resolver->resolve('agent-103'); +$assert(is_array($material) && hash_equals($rawKey, $material['hmac_key'] ?? ''), + 'The resolver did not decode the canonical base64 key into raw bytes.'); +$assert(($material['cert_file'] ?? '') === '/run/virt-cloud/agents/agent-103.crt' + && ($material['key_file'] ?? '') === '/run/virt-cloud/agents/agent-103.key', + 'The resolver returned unexpected TLS paths.'); + +foreach (['agent/103', '../agent-103', ' agent-103', '', 'agent-103\n'] as $invalid) { + $assert($resolver->resolve($invalid) === null, + 'Invalid Agent ID was accepted: ' . json_encode($invalid)); +} + +$badFiles = $files; +$badFiles['/run/virt-cloud/agents/agent-103.hmac'] = $encodedKey . " \n"; +$bad = new CloudAgentFileSecretResolver( + '/run/virt-cloud/agents', + static fn (string $path): ?string => $badFiles[$path] ?? null, +); +$assert($bad->resolve('agent-103') === null, + 'The resolver accepted ambiguous whitespace in the key file.'); + +$nonCanonical = $files; +$nonCanonical['/run/virt-cloud/agents/agent-103.hmac'] = rtrim($encodedKey, '=') . "\n"; +$nonCanonicalResolver = new CloudAgentFileSecretResolver( + '/run/virt-cloud/agents', + static fn (string $path): ?string => $nonCanonical[$path] ?? null, +); +$assert($nonCanonicalResolver->resolve('agent-103') === null, + 'The resolver accepted non-canonical Base64 padding.'); + +foreach (['relative/agents', '/', '/run//virt-cloud/agents', '/run/../agents', '/run/virt cloud/agents'] as $directory) { + try { + new CloudAgentFileSecretResolver($directory); + throw new RuntimeException('Unsafe secret directory was accepted: ' . $directory); + } catch (InvalidArgumentException) { + // expected + } +} + +$source = file_get_contents(dirname(__DIR__) . '/app/Services/Cloud/CloudAgentFileSecretResolver.php'); +$assert(is_string($source) + && str_contains($source, 'base64_decode') + && str_contains($source, 'lstat') + && str_contains($source, 'validAbsoluteDirectory') + && str_contains($source, 'hash_equals') + && str_contains($source, 'CloudAgentSecretResolver') + && !str_contains($source, 'CloudSecrets') + && !str_contains($source, 'CloudEngineClient') + && !str_contains($source, 'ssh'), + 'The file resolver crossed the legacy secret or SSH boundary.'); + +fwrite(STDOUT, "Cloud Agent file secret resolver contract passed.\n"); diff --git a/web/tests/CloudAgentObservationServiceContractTest.php b/web/tests/CloudAgentObservationServiceContractTest.php index 5a88708f..c161be5d 100644 --- a/web/tests/CloudAgentObservationServiceContractTest.php +++ b/web/tests/CloudAgentObservationServiceContractTest.php @@ -24,6 +24,10 @@ $assert(is_string($service) && str_contains($service, 'public function observeMany(') && str_contains($service, 'CloudInstanceMetadata::state') && str_contains($service, 'isReducedFailure') + && str_contains($service, 'isDefinitiveReducedRejection') + && str_contains($service, 'agent_observation_transport_uncertain') + && str_contains($service, '重试必须同时提供 operation_id 和幂等键') + && !str_contains($service, 'idempotencyKeyForOperationId') && !str_contains($service, 'CloudEngineClient') && !str_contains($service, 'ssh'), 'Observation service crossed the legacy Python/SSH boundary or lost its journal gate.'); diff --git a/web/tests/CloudAgentOperationHistoryContractTest.php b/web/tests/CloudAgentOperationHistoryContractTest.php index 1dee4e6c..2ca08427 100644 --- a/web/tests/CloudAgentOperationHistoryContractTest.php +++ b/web/tests/CloudAgentOperationHistoryContractTest.php @@ -31,6 +31,9 @@ $method = $start === false $assert(str_contains($repository, 'function historyForInstance(') && str_contains($repository, 'where(\'instance_uuid\', $instanceUuid)') + && str_contains($repository, 'where(\'agent_id\', $agentId)') + && str_contains($repository, 'where(\'host_id\', $hostId)') + && str_contains($repository, 'where(\'instance_id\', $instanceId)') && str_contains($repository, 'whereIn(\'kind\', $normalizedKinds)') && str_contains($repository, 'self::publicRecord($row)'), 'Agent operation repository lost the validated per-instance history query.'); @@ -40,6 +43,9 @@ $assert(str_contains($repository, 'if ($page < 1 || $perPage < 1 || $perPage > 5 $assert(str_contains($service, "private const HISTORY_KINDS = ['domain_power']") && str_contains($service, 'historyForInstance(') + && str_contains($service, '$agentId') + && str_contains($service, '(int) ($instance->host_id ?? 0)') + && str_contains($service, '(int) ($instance->id ?? 0)') && str_contains($service, 'return $kind === \'domain_power\' ? \'action\' : null') && str_contains($service, 'if ($action === \'destroy\')') && str_contains($service, 'CloudOperationErrorCatalog::result(') diff --git a/web/tests/CloudAgentOperationReconcileContractTest.php b/web/tests/CloudAgentOperationReconcileContractTest.php new file mode 100644 index 00000000..f5cacce8 --- /dev/null +++ b/web/tests/CloudAgentOperationReconcileContractTest.php @@ -0,0 +1,76 @@ +reconcile(', + "'reconcile'", + 'CloudAgentOperationRepository', + '->finalize(', + 'agent_reconcile_result_invalid', + 'hash_equals', + 'resolveReadOnly', + "->where('host_id'", + "->where('id'", + 'instance_uuid', + "record['target']", +] as $required) { + $assert(str_contains($service, $required), + 'Reconcile service is missing a required safety boundary: ' . $required); +} + +foreach ([ + 'CloudEngineClient', 'CloudJobService', 'CloudWorkerService', + 'shell_exec', 'curl_exec', 'CloudAgentVmGateway', '->operation(', +] as $forbidden) { + $assert(!str_contains($service, $forbidden), + 'Reconcile service crossed a legacy or mutation boundary: ' . $forbidden); +} + +$uuid = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'; +$assert(str_contains($routes, "/jobs/{id:{$uuid}}/reconcile") + && str_contains($routes, 'CloudApiController:reconcileJob'), + 'The explicit UUID reconcile route is missing.'); +$assert(str_contains($middleware, '/api/v1/jobs/[0-9a-f]{8}-[0-9a-f]{4}') + && str_contains($middleware, '/reconcile$'), + 'The reconcile route is not classified as database-only by CloudApi middleware.'); + +$methodOffset = strpos($controller, 'public function reconcileJob('); +$assert($methodOffset !== false, 'CloudApiController has no reconcileJob endpoint.'); +$method = substr($controller, $methodOffset); +$assert(str_contains($method, 'CloudAgentOperationIdentity::validateUuid') + && str_contains($method, 'CloudAgentOperationReconcileService') + && str_contains($method, 'CloudAgentOperationStatusService::forActor') + && str_contains($method, "=== 'uncertain' ? 202 : 200"), + 'reconcileJob does not validate, authorize through the service, and project the safe status DTO.'); +$assert(str_contains($handle, 'public function reconcile(') + && str_contains($handle, 'typed read-back endpoint') + && !str_contains($handle, "if (\$this->readOnly) {\n throw new CloudError('降级宿主机 Agent 仅允许只读观察', 503);\n }\n return \$this->client->reconcile"), + 'A degraded read-only Agent handle must keep the non-replaying reconcile endpoint available.'); + +fwrite(STDOUT, "Cloud Agent operation reconcile contract passed.\n"); diff --git a/web/tests/CloudAgentOperationRepositoryContractTest.php b/web/tests/CloudAgentOperationRepositoryContractTest.php index 7d8deaae..d1e3a641 100644 --- a/web/tests/CloudAgentOperationRepositoryContractTest.php +++ b/web/tests/CloudAgentOperationRepositoryContractTest.php @@ -31,7 +31,8 @@ $assert(str_contains($repository, "private const TABLE = 'cloud_agent_operations && str_contains($repository, "function setCommitBoundary") && str_contains($repository, "function renew") && str_contains($repository, "function markUncertain") - && str_contains($repository, "function get"), + && str_contains($repository, "function get") + && str_contains($repository, "function pruneReadOnly"), 'Agent operation repository is missing a required state transition.'); foreach ([ @@ -72,6 +73,12 @@ $assert(str_contains($repository, 'array_is_list($input[\'desired\'])') 'Agent operation desired/result JSON needs an object and key-shape boundary.'); $assert(str_contains($schema, 'version BIGINT UNSIGNED NOT NULL DEFAULT 1') && str_contains($schema, 'attempt_count INT UNSIGNED') + && str_contains($schema, 'host_id BIGINT UNSIGNED NOT NULL') + && str_contains($schema, 'instance_id BIGINT UNSIGNED NULL') + && str_contains($repository, '->where(\'instance_id\', $request[\'instance_id\'])') + && str_contains($repository, "'host_id' => $hostId") + && str_contains($repository, "'instance_id' => $instanceId") + && str_contains($schema, 'UNIQUE KEY cloud_agent_instance_identity (agent_id, host_id, instance_id, instance_uuid)') && str_contains($schema, 'lease_owner VARCHAR(96)') && str_contains($schema, 'lease_expires_at BIGINT UNSIGNED'), 'Fresh schema lacks operation CAS/lease columns.'); diff --git a/web/tests/CloudAgentOperationStatusServiceContractTest.php b/web/tests/CloudAgentOperationStatusServiceContractTest.php index 7b952bdb..0658cc81 100644 --- a/web/tests/CloudAgentOperationStatusServiceContractTest.php +++ b/web/tests/CloudAgentOperationStatusServiceContractTest.php @@ -42,7 +42,11 @@ $assert($jobOffset !== false && $agentOffset !== false && $legacyOffset !== fals 'The job controller must dispatch UUIDs to the Agent projection before the numeric CloudJob query.'); $assert(str_contains($service, 'CloudAgentOperationRepository') - && str_contains($service, 'CloudInstance::where(\'uuid\', $row->instance_uuid)') + && str_contains($service, 'CloudAgent::where(\'agent_id\', $agentId)') + && str_contains($service, '->where(\'host_id\', (int) $hostId)') + && str_contains($service, "CloudInstance::where('host_id', $hostId)") + && str_contains($service, "->where('id', (int) $instanceId)") + && str_contains($service, "->where('uuid', $row->instance_uuid)") && str_contains($service, 'CloudOperationErrorCatalog::result(') && str_contains($service, "'result' => $result"), 'Agent status projection lost repository validation, owner lookup, or result redaction.'); diff --git a/web/tests/CloudAgentRegistryHealthContractTest.php b/web/tests/CloudAgentRegistryHealthContractTest.php index 9311c784..961b8ad0 100644 --- a/web/tests/CloudAgentRegistryHealthContractTest.php +++ b/web/tests/CloudAgentRegistryHealthContractTest.php @@ -99,6 +99,9 @@ $assert($source !== false && str_contains($source, 'CloudAgentSecretResolver') && str_contains($source, 'CloudAgentHostResolver') && str_contains($source, '$resolvedAddresses') && str_contains($source, 'CloudAgentClient') + && str_contains($source, 'validateCredentialPath') + && str_contains($source, 'MAX_REGISTRY_ROWS') + && str_contains($source, 'MAX_SNAPSHOT_SECONDS') && !str_contains($source, 'CloudEngineClient') && !str_contains($source, 'CloudSecrets') && !str_contains($source, 'ssh'), 'Registry shadow crossed the legacy secret or SSH boundary.'); diff --git a/web/tests/CloudAgentRuntimeHealthContractTest.php b/web/tests/CloudAgentRuntimeHealthContractTest.php new file mode 100644 index 00000000..640c0e1e --- /dev/null +++ b/web/tests/CloudAgentRuntimeHealthContractTest.php @@ -0,0 +1,34 @@ +check('engine', 'Go Agent'") + && !str_contains($runtime, 'CloudEngineClient') + && !str_contains($runtime, 'shell_exec') + && !str_contains($runtime, 'ssh'), + 'Runtime health still depends on the legacy Engine/SSH transport.'); +$assert(is_string($registry) + && str_contains($registry, 'fromConfig()') + && str_contains($registry, 'client_certificate_fingerprint') + && str_contains($registry, '$fingerprint'), + 'Agent health registry lost fresh configuration or certificate pinning.'); +$assert(is_string($javascript) + && str_contains($javascript, 'Agent liveness') + && !str_contains($javascript, '引擎连通不代表'), + 'Administrator health UI still describes the retired Engine probe.'); + +fwrite(STDOUT, "Cloud Agent runtime health contract passed.\n"); diff --git a/web/tests/CloudAgentSchemaContractTest.php b/web/tests/CloudAgentSchemaContractTest.php index 6d255a8d..c1455860 100644 --- a/web/tests/CloudAgentSchemaContractTest.php +++ b/web/tests/CloudAgentSchemaContractTest.php @@ -58,6 +58,11 @@ foreach ([ 'virt-cloud-agent', 'schema_fingerprint CHAR(64)', 'UNIQUE KEY cloud_agent_operation_idempotency', + 'host_id BIGINT UNSIGNED NOT NULL', + 'instance_id BIGINT UNSIGNED NULL', + 'instance_id BIGINT UNSIGNED NOT NULL', + 'UNIQUE KEY cloud_agent_instance_identity (agent_id, host_id, instance_id, instance_uuid)', + 'cloud_agent_operation_readonly_retention', 'version BIGINT UNSIGNED NOT NULL DEFAULT 1', 'desired_json JSON NOT NULL', 'observed_json JSON NULL', diff --git a/web/tests/CloudApiVmCacheCutoverContractTest.php b/web/tests/CloudApiVmCacheCutoverContractTest.php new file mode 100644 index 00000000..0fdc3956 --- /dev/null +++ b/web/tests/CloudApiVmCacheCutoverContractTest.php @@ -0,0 +1,41 @@ +host(', 'vms/'] as $forbidden) { + $assert(!str_contains($list, $forbidden), + 'VM list context still contains a legacy Engine read: ' . $forbidden); +} + +$start = strpos($source, 'private function vmData('); +$end = strpos($source, ' public function vms(', $start === false ? 0 : $start); +$detail = ($start !== false && $end !== false) ? substr($source, $start, $end - $start) : ''; +$assert(str_contains($detail, 'CloudInstanceRuntimeService::forInstances') + && str_contains($detail, 'state_fresh'), + 'VM detail does not use the runtime cache when no preloaded context exists.'); +foreach (['CloudEngineClient', '->host(', 'vms/'] as $forbidden) { + $assert(!str_contains($detail, $forbidden), + 'VM detail still contains a legacy Engine read: ' . $forbidden); +} + +fwrite(STDOUT, "Cloud API VM cache cutover contract passed.\n"); diff --git a/web/tests/CloudInstanceRuntimeSyncAgentContractTest.php b/web/tests/CloudInstanceRuntimeSyncAgentContractTest.php index 59338209..2d8f59c7 100644 --- a/web/tests/CloudInstanceRuntimeSyncAgentContractTest.php +++ b/web/tests/CloudInstanceRuntimeSyncAgentContractTest.php @@ -17,10 +17,13 @@ $assert(is_string($service) && is_string($observation), 'Runtime sync sources ar $assert(str_contains($service, 'CloudAgentObservationService') && str_contains($service, 'CloudAgentHostClientResolverFactory::fromConfig()') - && str_contains($service, '->observe($hostId, $uuid, $actor)') + && str_contains($service, '->observe(') + && str_contains($service, '$instanceId,') && str_contains($service, 'CloudInstanceRuntimeService::beginObservation(') && str_contains($service, 'CloudInstanceRuntimeService::markObserved(') && str_contains($service, 'CloudInstanceRuntimeService::markUnavailable(') + && str_contains($service, 'pruneReadOnly(') + && str_contains($service, 'MAX_SYNC_SECONDS') && str_contains($service, 'hash_equals($uuid, $candidateUuid)') && !str_contains($service, 'CloudEngineClient') && !str_contains($service, 'vm-states') @@ -34,4 +37,10 @@ $assert(str_contains($observation, 'resolveReadOnly($hostId)') && str_contains($observation, 'observeMany'), 'Agent observation service must keep read-only resolver, typed operation and uncertain journal semantics.'); +$runtime = file_get_contents($root . '/app/Services/Cloud/CloudInstanceRuntimeService.php'); +$assert(is_string($runtime) + && str_contains($runtime, 'Runtime observation CAS lost') + && str_contains($runtime, '$revision !== null && (int) $updated !== 1'), + 'Runtime cache writes must reject stale revision tokens instead of reporting an observation as committed.'); + echo "Cloud Agent runtime sync contract passed.\n";