From 12e20d150f4627eb0353a74dfd6b1be0d2fb2ad3 Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 4 Oct 2026 04:39:11 +0800 Subject: [PATCH] feat: add host inventory diagnostics and admin history --- docs/ADMIN_OPERATIONS.md | 2 + docs/HOST_CAPACITY_INVENTORY_SYNC.md | 4 +- docs/HOST_INVENTORY_DIAGNOSTICS.md | 69 +++ docs/VALIDATION.md | 13 + engine/tests/test_inventory_diagnostics.py | 406 +++++++++++++++++ engine/virsh_panel/app.py | 48 +- engine/virsh_panel/internal.py | 66 ++- engine/virsh_panel/inventory_diagnostics.py | 370 +++++++++++++++ engine/virsh_panel/libvirt_service.py | 75 ++-- .../Controllers/Admin/CloudPageController.php | 36 ++ .../CloudAdminHostInventoryController.php | 58 +++ web/app/Middleware/CloudApi.php | 1 + web/app/Models/CloudHostInventoryAttempt.php | 18 + web/app/Services/Cloud/CloudEngineClient.php | 164 +++++++ .../Cloud/CloudHostInventoryDiagnostics.php | 423 ++++++++++++++++++ .../Cloud/CloudHostInventorySyncService.php | 68 ++- .../Cloud/CloudInventoryRequestError.php | 15 + web/config/routes.php | 1 + web/config/routes_cloud.php | 5 + web/public/cloud/admin-host-inventory.js | 312 +++++++++++++ web/public/cloud/admin.js | 4 +- .../venix/admin/cloud/host_inventory.tpl | 42 ++ .../venix/admin/cloud/host_inventory_rows.tpl | 11 + .../views/venix/admin/cloud/operations.tpl | 1 + .../views/venix/admin/hosts/index.tpl | 1 + .../views/venix/admin/static_header.tpl | 1 + web/resources/views/venix/layout_assets.tpl | 2 +- web/sql/cloud.sql | 26 ++ web/tests/AdminHostInventoryBehaviorTest.mjs | 171 +++++++ web/tests/AdminHostInventoryViewTest.php | 123 +++++ .../CloudHostInventoryDiagnosticsFixture.php | 31 ++ .../CloudHostInventoryDiagnosticsTest.php | 210 +++++++++ .../CloudHostInventorySyncContractTest.php | 4 +- .../CloudHostInventorySyncServiceTest.php | 68 ++- web/tests/CloudInventoryEngineClientTest.php | 122 +++++ 35 files changed, 2886 insertions(+), 85 deletions(-) create mode 100644 docs/HOST_INVENTORY_DIAGNOSTICS.md create mode 100644 engine/tests/test_inventory_diagnostics.py create mode 100644 engine/virsh_panel/inventory_diagnostics.py create mode 100644 web/app/Controllers/CloudAdminHostInventoryController.php create mode 100644 web/app/Models/CloudHostInventoryAttempt.php create mode 100644 web/app/Services/Cloud/CloudHostInventoryDiagnostics.php create mode 100644 web/app/Services/Cloud/CloudInventoryRequestError.php create mode 100644 web/public/cloud/admin-host-inventory.js create mode 100644 web/resources/views/venix/admin/cloud/host_inventory.tpl create mode 100644 web/resources/views/venix/admin/cloud/host_inventory_rows.tpl create mode 100644 web/tests/AdminHostInventoryBehaviorTest.mjs create mode 100644 web/tests/AdminHostInventoryViewTest.php create mode 100644 web/tests/CloudHostInventoryDiagnosticsFixture.php create mode 100644 web/tests/CloudHostInventoryDiagnosticsTest.php create mode 100644 web/tests/CloudInventoryEngineClientTest.php diff --git a/docs/ADMIN_OPERATIONS.md b/docs/ADMIN_OPERATIONS.md index 18611132..689bd79b 100644 --- a/docs/ADMIN_OPERATIONS.md +++ b/docs/ADMIN_OPERATIONS.md @@ -2,6 +2,8 @@ 管理员入口:`/admin/operations`。沿用 Venix 管理主题,任务列表由业务数据库直接渲染;运行检查随后异步加载,不等待引擎才显示任务。 +宿主机容量/IP 盘点的逐次成功、失败、超时与阶段线索使用独立 [盘点诊断页](HOST_INVENTORY_DIAGNOSTICS.md)(`/admin/host-inventory`)。系统运行页提供跳转;它与下面的后台进程心跳不是同一种健康信号。盘点诊断新增历史表,须按其文档运行 `cloudInit`,不适用本页心跳部分的“无需数据库迁移”。 + ## 查看范围 - 覆盖创建、电源操作、重装、救援、删除及到期、流量策略任务。 diff --git a/docs/HOST_CAPACITY_INVENTORY_SYNC.md b/docs/HOST_CAPACITY_INVENTORY_SYNC.md index a84119f4..92bc58c9 100644 --- a/docs/HOST_CAPACITY_INVENTORY_SYNC.md +++ b/docs/HOST_CAPACITY_INVENTORY_SYNC.md @@ -29,12 +29,14 @@ ## 失败语义 +逐次尝试、分层失败和超时后的引擎线索见 [宿主机盘点诊断](HOST_INVENTORY_DIAGNOSTICS.md)。下表是原有健康/调度状态,不是底层错误的精确分类;多数引擎 HTTP 错误经旧客户端规则折叠后仍会显示 `unavailable`。 + 任何请求、响应校验或事务失败都保留最后一次容量数值与全部 IP 行,不写零值、不根据缺失响应推断资源已经释放: | 场景 | host health | 固定诊断码 | | --- | --- | --- | | 连接、超时、限流或临时 5xx | `unavailable` | `host_inventory_unavailable` | -| 引擎拒绝读取 | `degraded` | `host_inventory_engine_rejected` | +| 保留为 4xx 的引擎拒绝(例如有效 JSON 404) | `degraded` | `host_inventory_engine_rejected` | | 容量合同无效 | `degraded` | `host_inventory_capacity_invalid` | | 网络/IP 合同无效或计数不一致 | `degraded` | `host_inventory_network_invalid` | | 地址已属于另一宿主机 | `degraded` | `host_inventory_ip_conflict` | diff --git a/docs/HOST_INVENTORY_DIAGNOSTICS.md b/docs/HOST_INVENTORY_DIAGNOSTICS.md new file mode 100644 index 00000000..bbe99c42 --- /dev/null +++ b/docs/HOST_INVENTORY_DIAGNOSTICS.md @@ -0,0 +1,69 @@ +# 宿主机盘点诊断 + +管理员入口:`/admin/host-inventory`。计算节点列表、资源调度抽屉、系统运行页和管理员导航均提供入口。页面只读取记录,不会因打开、筛选、翻页或刷新而执行 SSH/libvirt 盘点。手动同步仍使用原有的宿主机“手动同步”操作。 + +## 记录与结果 + +每次真正开始的自动或手动同步生成一个独立的 32 位 `attempt_id`。业务库 `cloud_host_inventory_attempts` 保存开始/结束时间、来源、代际、实际请求预算、PHP 阶段耗时、固定错误码、上游 HTTP 状态和容量是否已提交;不复制容量响应或 IP 清单。摘要按宿主机统计,结果筛选仅影响列表。 + +- `succeeded`:两个响应通过校验且容量/IP 事务已经提交。 +- `failed` / `timeout`:面板本次尝试未成功;不等于物理宿主机离线。 +- `superseded`:新的同步代际已取代该结果,不能覆盖较新的容量或健康状态。 +- `skipped`:尝试开始后发现宿主机不再允许同步,未发布容量。 +- 超过 5 分钟未写终态的记录显示“中断或结果未知”,不伪造结束时间,也不推断容量一定未提交。 + +面板成功、失败时间由面板记录;耗时使用本进程单调时钟。引擎时间单独展示,不用跨机器时间差推断网络耗时。面板诊断版本是已加载代码的协议标识,不是磁盘上的 Git HEAD。 + +历史写入故障不会更改原有盘点结果;缺表或数据库不可写时,用固定请求编号/阶段/错误码的 PHP 服务日志补充诊断。面板完全不可用时仍需通过服务日志排查。 + +## 超时后的引擎线索 + +原有 `host/capacity`、`host/network-options` 请求附带 `inventory_attempt_id` 查询参数,整个查询参数受原有 HMAC 保护。一个面板尝试对应容量、网络两份引擎摘要。 + +引擎在进入可能阻塞的阶段前原子保存私有 JSON 摘要,覆盖:宿主机配置、全局锁等待、在线更新检查、ISO/镜像清理、libvirt 连接、宿主资源、VM 枚举/扫描、存储池、公网池目录及占用。嵌套阶段耗时包含内部阶段,不能直接相加。 + +详情页“读取引擎线索”仅调用独立的轻量查询: + +```text +GET /api/v1/admin/host-inventory/{attempt_id}/engine +GET /internal/v1/hosts/{host_id}/host/inventory-diagnostics/{attempt_id} +``` + +查询有独立 1 秒引擎请求预算,绕过宿主应用加载、全局更新锁与远程清理;不重试完整盘点。PHP 仅接受经过白名单校验的固定字段。普通用户不可访问,响应禁止缓存。 + +PHP 超时和引擎最终结果分开展示。HTTP 等待结束不意味着底层工作停止;工作线程晚到的成功只补充引擎证据,不修改面板的超时记录或容量。同步盘点 worker 尚未启动就取消的请求标记为中断,结束时间保持未知;当时仍在执行的清理线程保留最后确认阶段,不声称线程已停止。引擎重启后的未终结摘要同样标记为中断/未知。 + +只有开始记录、没有结束记录时,页面显示的是“最后确认阶段”,不是已证明的根因。没有对应摘要可能是引擎未收到请求、版本未启用诊断、诊断存储不可用或超过保留期限,不应猜测为宿主故障。 + +## 错误判读 + +| 证据 | 能确认的范围 | +| --- | --- | +| `engine_http_timeout` | cURL 明确报告超时,或上游返回 408/504;内部原因需结合引擎摘要 | +| `engine_connection_or_timeout` | 客户端没有可可靠区分连接失败与超时的信息,例如部分非 cURL handler | +| `engine_auth_rejected` | 上游 401/403;检查内部身份/HMAC,不应直接认定宿主离线 | +| `engine_configuration_locked` / `engine_request_conflict` | 上游 423/409;结合阶段排查配置或状态冲突 | +| `storage_scan` + 404 | 失败涉及存储读取;不能把业务 404 当作接口版本不支持并重扫 | +| `capacity_validate` / `network_validate` | 面板未接受引擎返回的数据合同或容量/IP 一致性 | +| `database_apply` | 容量/IP 校验后的本地提交阶段失败或结果已被更新代际取代 | + +历史健康映射保持不变,所以宿主上的通用 `unavailable` 与诊断页的细分类别可能不同。诊断不会放宽调度、覆盖预留、延长快照有效期,或重试创建/删除操作。 + +## 保留与开销 + +- 面板历史约保留 7 天,每宿主约 25000 条;在后续尝试中每次有界分批清理,不通过浏览页面清理。停止盘点的环境不会主动触发这一清理。 +- 引擎全局最多 4000 份请求摘要、最长 7 天、每份最多 32 KiB;数量和时间先到者为准,因此旧面板记录可能已无引擎细节。目录为 `data_dir/inventory-diagnostics`,目录 `0700`、文件 `0600`,不增加第三个生产数据库。 +- 不保存凭据、HMAC、libvirt URI、原始异常、完整响应、VM XML 或地址清单。 +- 阶段持久化带来少量本地 I/O;应使用可靠的本地数据目录并实机测量开销。记录写入异常采用固定码降级,不吞掉原业务异常。 + +## 部署与验收 + +1. 使用现有安全更新流程部署引擎和 PHP;手工升级时在项目 `web/` 下运行 `php xcat cloudInit` 创建新历史表。不要覆盖已有业务数据库。 +2. 按既有维护流程重启引擎、lifecycle,并确保 PHP-FPM/OPcache 加载新代码。不要为升级诊断强行打断在途破坏性操作。只执行 `git pull` 不能保证常驻进程已更新。 +3. 打开诊断页。历史从启用后开始记录,不追溯补造过去失败。缺表时页面明确提示数据库升级,盘点本身仍沿用旧行为。 +4. 对选定测试宿主手动同步,检查成功记录、两份引擎摘要、共同请求编号及容量已更新标志。 +5. 在受控测试中验证慢请求/锁等待、认证失败、存储池错误、响应格式错误、数据库失败,以及 HTTP 超时后引擎晚到成功。失败时不能清空已有容量或释放已占用 IP。 + +本阶段保留原来两次 HTTP 请求、每请求 8 秒预算和全部扫描/调度行为;单连接轻量盘点属于后续依据诊断证据决定的优化。 + +测试资产:`CloudHostInventoryDiagnosticsTest.php`、`CloudInventoryEngineClientTest.php`、`CloudHostInventorySyncServiceTest.php`、`AdminHostInventoryViewTest.php`、`AdminHostInventoryBehaviorTest.mjs` 和 `engine/tests/test_inventory_diagnostics.py`。模拟和 SQLite/WASM 测试不替代真实 MariaDB、PHP-FPM/systemd 与 SSH/libvirt 联调。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 3428b5eb..0e17413a 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,18 @@ # 本版验证记录 +## 2026-10-04:宿主机盘点诊断 + +本轮只增加诊断历史、管理员页面和跨 PHP/引擎阶段证据;每请求 8 秒预算、容量/IP 提交代际和调度判定不变。部署步骤见 [宿主机盘点诊断](HOST_INVENTORY_DIAGNOSTICS.md)。 + +- Python 3.12 全量隔离回归:**747 passed、10 skipped、68 subtests passed、1 failed**。新增诊断用例覆盖签名/权限、锁等待、存储失败降级、嵌套阶段、取消请求与 worker 晚完成、重启和保留上限。 +- 唯一 Python 失败为 `test_app.py::test_native_public_pool_blocks_silent_default_network_change`:测试创建公网池返回 409“公网地址池目录存储不可用”,预期 200。在修改前的 `fad3b4e` 独立源码快照中复现同一失败;未在本轮扩大范围修改。 +- 隔离 PHP 8.5.10/WASM、精确 `composer.lock` 依赖与 SQLite 测试通过:`CloudInventoryEngineClientTest`、`CloudHostInventoryDiagnosticsTest`、`CloudHostInventorySyncServiceTest`、`CloudHostInventorySyncContractTest`、`CloudInventoryRelationshipContractTest`、`CloudNetworkIpLedgerContractTest`、`AdminHostInventoryViewTest`、`AdminHostSchedulingContractTest`、`AdminOperationsViewTest`、`CloudRuntimeHeartbeatTest`。覆盖固定错误分类、HMAC 查询参数、历史缺表降级、超时留痕、过时代际、只读页面和字段脱敏。缺表用例的固定诊断降级日志为预期输出。 +- 扩展 PHP 检查另发现两项既有失败,均在 `fad3b4e` 快照复现:`CloudProtocolTest.php:85` 的源码断言插值 `$actor`;`CloudAdminOperationsServiceTest.php:245` 的查询字段断言与原服务选择 `payload_cipher` 不一致。本轮未修改这两个测试或其既有服务。 +- 新页面行为测试 `AdminHostInventoryBehaviorTest.mjs`、相关 PHP 语法、两份 JS 语法及 `git diff --check` 通过。Chrome 合成数据检查 1440/390 px 页面和详情抽屉:无页面错误或横向溢出;首次展示不调用 API,不扫描宿主机,只有显式点击才读取引擎线索。摘要、列表、详情统一使用面板时区。 +- 未连接真实 MariaDB 或远程宿主机,没有验证生产 PHP-FPM/systemd、cURL 传输、SSH/libvirt 延迟或诊断持久化的实际 I/O 开销;不能据此声称用户当前 `unavailable` 的根因已确定。真实环境启用后,应依据新增记录验证失败层和最后确认阶段。 + +## 2026-09-27:集成基线 + 2026-09-27,集成代码的隔离验证,不代表生产验收。 - 当前 macOS 工作区 Python 3.12 回归:515 passed、10 skipped、55 subtests passed。覆盖原虚拟化逻辑及私有 API、签名/重放、多宿主机归属、持久幂等、一次性 VNC、退出后断权、带宽 XML、救援上传摘要,以及到期断网/恢复、同实例变更串行、多网卡 live+config 断网/恢复、退出救援保持计费断网、流量累计检查点和破坏性操作结果核对。生产仍按部署文档使用 Python 3.12 或更新的受支持版本复测。 diff --git a/engine/tests/test_inventory_diagnostics.py b/engine/tests/test_inventory_diagnostics.py new file mode 100644 index 00000000..434bafa6 --- /dev/null +++ b/engine/tests/test_inventory_diagnostics.py @@ -0,0 +1,406 @@ +"""Durable inventory evidence must not mutate or rescan the observed host.""" +from __future__ import annotations + +import asyncio +import base64 +import json +import os +import stat +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +from test_engine_boundary import ( + ADMIN, SECRET, USER, FakeUpdater, MemoryStore, boundary, call, + cloud_engine_request, +) +from virsh_panel.internal import EngineApplication +from virsh_panel.inventory_diagnostics import ( + CURRENT, MAX_BYTES, MAX_STAGES, InventoryDiagnostics, inventory_result, inventory_stage, +) +from virsh_panel.libvirt_service import VMError, VMService + +ATTEMPT = "a" * 32 +OTHER = "b" * 32 +CAPACITY = "/internal/v1/hosts/hk/host/capacity" +DIAGNOSTICS = f"/internal/v1/hosts/hk/host/inventory-diagnostics/{ATTEMPT}" + + +def test_endpoint_authentication_and_host_isolation(boundary): + client, engine, *_ = boundary + assert client.get(DIAGNOSTICS).status_code == 401 + assert call(client, "GET", DIAGNOSTICS, USER).status_code == 403 + assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, USER).status_code == 403 + assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == [] + + assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200 + assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200 + result = call(client, "GET", DIAGNOSTICS, ADMIN).json() + assert result["diagnostics_available"] is True + assert [item["resource"] for item in result["traces"]] == ["capacity", "network"] + assert all(item["state"] == "succeeded" for item in result["traces"]) + assert all(item["http_status"] == 200 for item in result["traces"]) + assert "public_ip_claims" in [item["stage"] for item in result["traces"][0]["stages"]] + for secret in ("host.example.test", "SHA256:example", "/fake/config", "_process", "_started"): + assert secret not in json.dumps(result) + assert call(client, "GET", DIAGNOSTICS.replace("/hk/", "/other/"), ADMIN).json()["traces"] == [] + + +@pytest.mark.parametrize("query", ["", "A" * 32, "../bad", "a" * 31, ATTEMPT + "&inventory_attempt_id=" + OTHER]) +def test_invalid_correlation_is_rejected_after_authentication(boundary, query): + client, engine, *_ = boundary + assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + query, ADMIN).status_code == 400 + assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == [] + + +def test_query_cannot_be_added_to_unsigned_target(boundary): + client, engine, *_ = boundary + from virsh_panel.engine_security import signed_headers + headers = signed_headers(SECRET, "GET", CAPACITY, b"", ADMIN) + result = client.get(CAPACITY + "?inventory_attempt_id=" + ATTEMPT, headers=headers) + assert result.status_code == 401 + assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == [] + + +def test_read_bypasses_registry_lock_and_gc(boundary): + client, engine, *_ = boundary + with patch.object(engine.registry, "get", side_effect=AssertionError("must not load host")), \ + patch.object(engine, "_update_lock", side_effect=AssertionError("must not wait")), \ + patch.object(engine, "_cleanup_host_user_iso_tombstones", side_effect=AssertionError("must not gc")), \ + patch.object(engine, "_cleanup_host_cloud_template_tombstones", side_effect=AssertionError("must not gc")): + assert call(client, "GET", DIAGNOSTICS, ADMIN).status_code == 200 + + +def test_failed_profile_records_fixed_stage_without_secret(boundary): + client, engine, *_ = boundary + assert call(client, "GET", CAPACITY, ADMIN).status_code == 200 + engine.registry.components["hk"].manager.profile["key_locked"] = True + assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 423 + trace = call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"][0] + assert (trace["state"], trace["stage"], trace["error_code"], trace["http_status"]) == ( + "failed", "profile", "profile_failed", 423) + assert trace["stages"][-1]["state"] == "failed" + + +def make_engine(tmp_path, app): + registry = SimpleNamespace(get=AsyncMock(return_value=app)) + engine = EngineApplication( + {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}, + MemoryStore(), MemoryStore(), registry=registry, updater=FakeUpdater(), + ) + engine._cleanup_host_user_iso_tombstones = AsyncMock() + engine._cleanup_host_cloud_template_tombstones = AsyncMock() + return engine + + +def test_lock_wait_is_persisted_before_capacity_endpoint(tmp_path): + async def scenario(): + called = False + async def endpoint(scope, receive, send): + nonlocal called + called = True + await send({"type": "http.response.start", "status": 200}) + await send({"type": "http.response.body", "body": b"{}"}) + engine = make_engine(tmp_path, endpoint) + lock = engine._update_lock() + await lock.acquire() + request = asyncio.create_task(cloud_engine_request( + engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN)) + try: + for _ in range(200): + traces = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] + if traces and traces[0]["stage"] == "update_lock_wait": + break + await asyncio.sleep(0.001) + else: + pytest.fail("lock wait not persisted") + status, result = await asyncio.wait_for(cloud_engine_request( + engine, "GET", DIAGNOSTICS, b"", ADMIN), timeout=1) + assert status == 200 + assert result["traces"][0]["state"] == "running" + assert result["traces"][0]["stages"][-1]["state"] == "running" + assert called is False + finally: + lock.release() + await request + assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]["state"] == "succeeded" + asyncio.run(scenario()) + + +def test_cancellation_during_lock_wait_is_interrupted_not_forever_running(tmp_path): + async def scenario(): + endpoint = AsyncMock() + engine = make_engine(tmp_path, endpoint) + lock = engine._update_lock() + await lock.acquire() + request = asyncio.create_task(cloud_engine_request( + engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN)) + try: + for _ in range(200): + traces = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] + if traces and traces[0]["stage"] == "update_lock_wait": + break + await asyncio.sleep(0.001) + else: + pytest.fail("lock wait not persisted") + request.cancel() + with pytest.raises(asyncio.CancelledError): + await request + result = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0] + assert (result["state"], result["error_code"]) == ("interrupted", "request_interrupted") + assert result["stage"] == "update_lock_wait" + assert result["stages"][-1]["state"] == "interrupted" + assert result["finished_at"] is None and result["http_status"] is None + endpoint.assert_not_called() + finally: + lock.release() + asyncio.run(scenario()) + + +def test_cancellation_during_gc_does_not_claim_worker_stopped(tmp_path): + async def scenario(): + started, release, completed = threading.Event(), threading.Event(), threading.Event() + def gc_worker(): + try: + with inventory_stage("libvirt_connect"): + started.set() + assert release.wait(2) + finally: + completed.set() + engine = make_engine(tmp_path, AsyncMock()) + async def cleanup(_host): + await asyncio.to_thread(gc_worker) + engine._cleanup_host_user_iso_tombstones = cleanup + request = asyncio.create_task(cloud_engine_request( + engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN)) + try: + assert await asyncio.to_thread(started.wait, 1) + request.cancel() + with pytest.raises(asyncio.CancelledError): + await request + result = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0] + assert result["state"] == "interrupted" + assert result["finished_at"] is None and result["http_status"] is None + assert result["stage"] == "libvirt_connect" + assert result["stages"][-1]["state"] == "running" + assert result["stages"][-2]["state"] == "interrupted" + assert not completed.is_set() + finally: + release.set() + assert await asyncio.to_thread(completed.wait, 1) + result = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0] + assert result["state"] == "interrupted" # GC completion is not inventory success + assert result["finished_at"] is None + assert result["stages"][-1]["state"] == "succeeded" + asyncio.run(scenario()) + + +@pytest.mark.parametrize("name", ["registry_profile", "iso_gc", "template_gc"]) +def test_pre_dispatch_failures_record_only_fixed_codes(tmp_path, name): + async def scenario(): + engine = make_engine(tmp_path, AsyncMock()) + target = engine.registry if name == "registry_profile" else engine + attribute = {"registry_profile": "get", "iso_gc": "_cleanup_host_user_iso_tombstones", + "template_gc": "_cleanup_host_cloud_template_tombstones"}[name] + setattr(target, attribute, AsyncMock(side_effect=RuntimeError("SECRET-password-/private/key"))) + status, _ = await cloud_engine_request( + engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN) + assert status == 503 + trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0] + assert trace["stage"] == name + assert trace["error_code"] == name + "_failed" + assert "SECRET" not in json.dumps(trace) + asyncio.run(scenario()) + + +def test_worker_finishes_after_http_waiter_cancelled(tmp_path): + async def scenario(): + started, release = threading.Event(), threading.Event() + @inventory_result + def worker(): + with inventory_stage("domain_scan"): + started.set() + assert release.wait(2) + return {} + async def endpoint(scope, receive, send): + await asyncio.to_thread(worker) + await send({"type": "http.response.start", "status": 200}) + engine = make_engine(tmp_path, endpoint) + request = asyncio.create_task(cloud_engine_request( + engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN)) + try: + assert await asyncio.to_thread(started.wait, 1) + request.cancel() + with pytest.raises(asyncio.CancelledError): + await request + trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0] + assert (trace["state"], trace["stage"]) == ("running", "domain_scan") + finally: + release.set() + for _ in range(200): + trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0] + if trace["state"] == "succeeded": + break + await asyncio.sleep(0.001) + assert trace["state"] == "succeeded" + asyncio.run(scenario()) + + +def test_queued_worker_starting_after_waiter_cancelled_records_real_result(tmp_path): + store = InventoryDiagnostics(tmp_path, "unknown") + trace = store.begin("hk", ATTEMPT, "capacity") + trace.waiter_cancelled() + assert trace.value["state"] == "interrupted" + token = CURRENT.set(trace) + try: + @inventory_result + def worker(): + with inventory_stage("host_info"): + assert trace.value["state"] == "running" + return {} + worker() + finally: + CURRENT.reset(token) + result = store.read("hk", ATTEMPT)["traces"][0] + assert result["state"] == "succeeded" + assert result["error_code"] == "" + assert result["finished_at"] is not None + + +def test_atomic_summary_permissions_retention_and_restart(tmp_path): + store = InventoryDiagnostics(tmp_path, "a" * 40, max_traces=2) + first = store.begin("hk", ATTEMPT, "capacity") + first.start("domain_scan") + path = store._path("hk", ATTEMPT, "capacity") + assert stat.S_IMODE(store.directory.stat().st_mode) == 0o700 + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert path.stat().st_size < MAX_BYTES + assert store.begin("hk", ATTEMPT, "capacity") is None + restarted = InventoryDiagnostics(tmp_path, "b" * 40, process_id="another-process") + trace = restarted.read("hk", ATTEMPT)["traces"][0] + assert trace["state"] == "interrupted" + assert trace["error_code"] == "request_interrupted" + assert trace["finished_at"] is None + assert trace["engine_version"] == "a" * 40 + store.begin("hk", OTHER, "capacity").finish() + store.begin("other", OTHER, "capacity").finish() + assert len(list(store.directory.glob("*.json"))) == 2 + assert store.read("hk", ATTEMPT)["traces"] == [] + first.finish() + assert not path.exists() # evicted late work cannot exceed retention bound + for current in store.directory.glob("*.json"): + os.utime(current, (1, 1)) + store.last_pruned = 0 + store.begin("hk", "c" * 32, "capacity").finish() + assert len(list(store.directory.glob("*.json"))) == 1 + + +def test_storage_failure_does_not_fail_inventory(boundary): + client, engine, *_ = boundary + with patch.object(engine.inventory_diagnostics, "_write", side_effect=OSError("SECRET path")): + result = call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN) + assert result.status_code == 200 + assert call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"] == [] + + +def test_real_service_stages_and_translated_http_status(tmp_path): + from virsh_panel import libvirt_service + connection = SimpleNamespace( + getInfo=lambda: ("x86_64", 8192, 4), listAllDomains=lambda _: [], + storagePoolLookupByName=lambda _: SimpleNamespace(isActive=lambda: True, info=lambda: (0, 100, 0, 100)), + close=lambda: None, + ) + store = InventoryDiagnostics(tmp_path, "unknown") + trace = store.begin("hk", ATTEMPT, "capacity") + token = CURRENT.set(trace) + try: + with patch.object(libvirt_service.libvirt, "open", return_value=connection, create=True): + VMService("SECRET-uri", "images", "iso", "default").host_capacity() + trace.finish() + finally: + CURRENT.reset(token) + stages = [item["stage"] for item in store.read("hk", ATTEMPT)["traces"][0]["stages"]] + assert stages == ["libvirt_connect", "host_info", "domain_list", "domain_scan", "storage_scan"] + trace = store.begin("hk", OTHER, "capacity") + token = CURRENT.set(trace) + try: + @inventory_result + def fail(): + VMService("SECRET-uri", "images", "iso", "default").host_capacity() + with patch.object(libvirt_service.libvirt, "open", side_effect=libvirt_service.libvirt.libvirtError("SECRET"), create=True): + with pytest.raises(VMError): + fail() + finally: + CURRENT.reset(token) + result = store.read("hk", OTHER)["traces"][0] + assert (result["stage"], result["http_status"]) == ("libvirt_connect", 502) + assert "SECRET" not in json.dumps(result) + + +def test_nested_failed_span_does_not_fail_caught_gc_request(tmp_path): + store = InventoryDiagnostics(tmp_path, "unknown") + trace = store.begin("hk", ATTEMPT, "capacity") + token = CURRENT.set(trace) + try: + with inventory_stage("iso_gc"): + try: + with inventory_stage("libvirt_connect"): + running = store.read("hk", ATTEMPT)["traces"][0] + assert [item["state"] for item in running["stages"]] == ["running", "running"] + raise VMError("secret connection error", 502) + except VMError: + pass + assert trace.value["state"] == "running" + with inventory_stage("host_info"): + pass + trace.finish(200) + finally: + CURRENT.reset(token) + result = store.read("hk", ATTEMPT)["traces"][0] + assert result["state"] == "succeeded" + assert result["error_code"] == "" + assert [item["state"] for item in result["stages"]] == ["succeeded", "failed", "succeeded"] + assert result["stages"][0]["duration_ms"] >= result["stages"][1]["duration_ms"] + + +def test_escaping_nested_failure_preserves_deepest_stage(tmp_path): + store = InventoryDiagnostics(tmp_path, "unknown") + trace = store.begin("hk", ATTEMPT, "capacity") + token = CURRENT.set(trace) + try: + with pytest.raises(VMError): + with inventory_stage("iso_gc"): + try: + with inventory_stage("libvirt_connect"): + raise RuntimeError("secret") + except RuntimeError as exc: + raise VMError("redacted", 502) from exc + trace.finish(502) + finally: + CURRENT.reset(token) + result = store.read("hk", ATTEMPT)["traces"][0] + assert result["state"] == "failed" + assert result["stage"] == "libvirt_connect" + assert [item["state"] for item in result["stages"]] == ["failed", "failed"] + + +def test_stage_limit_keeps_latest_blocking_evidence(tmp_path): + store = InventoryDiagnostics(tmp_path, "unknown") + trace = store.begin("hk", ATTEMPT, "capacity") + token = CURRENT.set(trace) + try: + with inventory_stage("iso_gc"): + for _ in range(MAX_STAGES + 5): + with inventory_stage("libvirt_connect"): + pass + item = trace.start("storage_scan") + result = store.read("hk", ATTEMPT)["traces"][0] + assert len(result["stages"]) == MAX_STAGES + assert result["stages"][-1]["stage"] == "storage_scan" + assert result["stages"][-1]["state"] == "running" + trace.end(item) + trace.finish(200) + finally: + CURRENT.reset(token) diff --git a/engine/virsh_panel/app.py b/engine/virsh_panel/app.py index 859cdc00..51d51685 100644 --- a/engine/virsh_panel/app.py +++ b/engine/virsh_panel/app.py @@ -42,6 +42,7 @@ from .console import ConsoleRelayError, relay_console from .database import DatabaseError, make_components from .database_bootstrap import DatabaseBootstrap, DatabaseSetupError from .libvirt_service import VMError, VMService +from .inventory_diagnostics import inventory_result, inventory_stage from .metrics import (MAX_QUERY_POINTS, RETENTION_SECONDS, RANGES, MetricsCollector, host_scope, query_interval_seconds) from .panel_state import PanelState, SetupError @@ -754,14 +755,17 @@ def create_app( raise VMError("公网地址池中没有可分配的 IP,请检查已用和保留地址", 409) def public_network_options(profile: dict, service: VMService) -> dict: - pools, compatibility_mode = public_network_pools(profile) + with inventory_stage("public_pool_catalog"): + pools, compatibility_mode = public_network_pools(profile) existing_vms = service.list_vms() if pools else [] - claimed_addresses = public_ip_claim_addresses(profile) - inventories = [public_network_inventory( - profile, pool, service, - existing_vms=existing_vms, - claimed_addresses=claimed_addresses, - ) for pool in pools] + with inventory_stage("public_ip_claims"): + claimed_addresses = public_ip_claim_addresses(profile) + with inventory_stage("public_ip_scan"): + inventories = [public_network_inventory( + profile, pool, service, + existing_vms=existing_vms, + claimed_addresses=claimed_addresses, + ) for pool in pools] public_items = [] for inventory in inventories: available_ips = [ @@ -809,14 +813,17 @@ def create_app( } def public_ip_capacity(profile: dict, service: VMService) -> dict: - pools, compatibility_mode = public_network_pools(profile) + with inventory_stage("public_pool_catalog"): + pools, compatibility_mode = public_network_pools(profile) existing_vms = service.list_vms() if pools else [] - claimed_addresses = public_ip_claim_addresses(profile) - inventories = [public_network_inventory( - profile, pool, service, - existing_vms=existing_vms, - claimed_addresses=claimed_addresses, - ) for pool in pools] + with inventory_stage("public_ip_claims"): + claimed_addresses = public_ip_claim_addresses(profile) + with inventory_stage("public_ip_scan"): + inventories = [public_network_inventory( + profile, pool, service, + existing_vms=existing_vms, + claimed_addresses=claimed_addresses, + ) for pool in pools] public_items = [{ "pool_key": item["pool_key"], "revision": item["revision"], @@ -1885,9 +1892,12 @@ def create_app( return _public_host(manager.current()) @app.get("/api/v1/host/network-options") + @inventory_result def host_network_options(_session: Session = Depends(require_admin)): - profile = current_profile() - return public_network_options(profile, current_service()) + with inventory_stage("profile"): + profile = current_profile() + service = current_service() + return public_network_options(profile, service) @app.get("/api/v1/host/network-pools") def host_network_pools(_session: Session = Depends(require_admin)): @@ -1925,10 +1935,12 @@ def create_app( } @app.get("/api/v1/host/capacity") + @inventory_result def host_capacity(_session: Session = Depends(require_admin)): """Return the scheduler's non-secret observation for this host.""" - profile = current_profile() - service = new_service(profile) + with inventory_stage("profile"): + profile = current_profile() + service = new_service(profile) capacity = service.host_capacity() return { "health": "healthy", diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index 47044885..8a71d4c1 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -47,6 +47,10 @@ from .engine_context import (EngineComponents, HostStore, LogStore, MainStore, TRUSTED_ACTOR, actor_session) from .engine_security import (EngineAuthError, authenticate, decode_secret, signed_headers) +from .inventory_diagnostics import ( + ATTEMPT_ID, CURRENT as INVENTORY_TRACE, InventoryDiagnostics, + engine_version, inventory_stage, +) from .updater import UpdateError, Updater from .user_isos import UserISORuntime @@ -59,6 +63,7 @@ JSON_LIMIT = 1024 * 1024 CLOUD_UPLOAD_IDLE_SECONDS = 60 CLOUD_UPLOAD_WRITE_BYTES = 1024 * 1024 READ_PATHS = ( + r"host/inventory-diagnostics/[0-9a-f]{32}", r"host", r"host/network-options", r"host/network-pools", r"host/capacity", r"resources", r"vm-templates", r"rescue-templates", r"user-isos", r"vms", r"vm-states", r"vm-metadata", r"user-isos/[0-9a-f]{32}", @@ -277,6 +282,8 @@ class EngineApplication: self.registry = registry or HostRegistry(config, store, log_store) self.introspection = introspection project_dir = Path(config.get("project_dir", Path(__file__).resolve().parents[2])) + self.inventory_diagnostics = InventoryDiagnostics( + Path(config["data_dir"]), engine_version(project_dir)) self.updater = updater or Updater( project_dir, store, worker_active=self._update_service_active) self.update_trigger = update_trigger or self._start_update_service @@ -322,11 +329,15 @@ class EngineApplication: if scope["type"] != "http": return token = None + inventory_token = None + inventory_trace = None response_started = False async def tracked_send(message): nonlocal response_started if message["type"] == "http.response.start": response_started = True + if inventory_trace: + inventory_trace.finish(message["status"]) await send(message) try: # Public reverse proxies must never forward these routes. Reject @@ -395,18 +406,47 @@ class EngineApplication: return await receive() child_receive = replay_receive token = TRUSTED_ACTOR.set(delegated) + inventory_match = re.fullmatch( + r"/internal/v1/hosts/([A-Za-z0-9][A-Za-z0-9_-]{0,63})/host/(capacity|network-options)", + scope["path"], + ) + if scope["method"] == "GET" and inventory_match: + # The query is part of the authenticated raw target. Never + # let unauthenticated or non-admin callers inject evidence. + self._authorize_route("GET", "host/" + inventory_match[2], delegated) + query = parse_qsl(scope.get("query_string", b"").decode("ascii"), keep_blank_values=True) + identifiers = [value for key, value in query if key == "inventory_attempt_id"] + if identifiers: + if len(identifiers) != 1 or not ATTEMPT_ID.fullmatch(identifiers[0]): + raise EngineAuthError("盘点请求编号无效", 400) + inventory_trace = self.inventory_diagnostics.begin( + inventory_match[1], identifiers[0], + "capacity" if inventory_match[2] == "capacity" else "network") + inventory_token = INVENTORY_TRACE.set(inventory_trace) await self._dispatch(scope, child_receive, tracked_send, body, delegated) + except asyncio.CancelledError: + if inventory_trace: + inventory_trace.waiter_cancelled() + raise except EngineAuthError as exc: + if inventory_trace: + inventory_trace.finish(exc.status) if not response_started: await json_response(send, exc.status, {"error": str(exc)}) except UpdateError as exc: + if inventory_trace: + inventory_trace.finish(exc.status) if not response_started: await json_response(send, exc.status, {"error": str(exc)}) except Exception: + if inventory_trace: + inventory_trace.finish(503) LOG.exception("Private engine request failed") if not response_started: await json_response(send, 503, {"error": "引擎服务暂时不可用,请检查引擎日志"}) finally: + if inventory_token is not None: + INVENTORY_TRACE.reset(inventory_token) if token is not None: TRUSTED_ACTOR.reset(token) @@ -520,15 +560,31 @@ class EngineApplication: # catalog appear to have host-local semantics. raise EngineAuthError("内部资源不存在", 404) self._authorize_route(scope["method"], resource, delegated) - app = await self.registry.get(host_id) + if scope["method"] == "GET" and resource.startswith("host/inventory-diagnostics/"): + # Independent evidence retrieval must work while registry restore, + # maintenance locks or libvirt are blocked. It never scans a host. + attempt_id = resource.rsplit("/", 1)[1] + result = await asyncio.to_thread(self.inventory_diagnostics.read, host_id, attempt_id) + await json_response(send, 200, result) + return + with inventory_stage("registry_profile"): + app = await self.registry.get(host_id) if scope["method"] == "GET": # Reads normally bypass the update coordination lock, but lazy # cache GC mutates libvirt. Coordinate that maintenance work with # online update and skip it while an update is already active. - async with self._update_lock(): - if not await asyncio.to_thread(self.updater.active_job): - await self._cleanup_host_user_iso_tombstones(host_id) - await self._cleanup_host_cloud_template_tombstones(host_id) + with inventory_stage("update_lock_wait"): + await self._update_lock().acquire() + try: + with inventory_stage("update_check"): + update_active = await asyncio.to_thread(self.updater.active_job) + if not update_active: + with inventory_stage("iso_gc"): + await self._cleanup_host_user_iso_tombstones(host_id) + with inventory_stage("template_gc"): + await self._cleanup_host_cloud_template_tombstones(host_id) + finally: + self._update_lock().release() else: # Every host write reached this method while holding _update_lock # and after the active-update gate in _dispatch(). diff --git a/engine/virsh_panel/inventory_diagnostics.py b/engine/virsh_panel/inventory_diagnostics.py new file mode 100644 index 00000000..3ffa2527 --- /dev/null +++ b/engine/virsh_panel/inventory_diagnostics.py @@ -0,0 +1,370 @@ +"""Bounded, non-secret inventory evidence, independent of HTTP caller lifetime. + +Only a signed administrator request can install CURRENT. No resource names, +addresses, exception text or response bodies are accepted by this recorder. +Atomic private summaries publish complete stage transitions before blocking work. +""" +from __future__ import annotations + +import asyncio +import contextvars +import functools +import hashlib +import inspect +import json +import logging +import os +import re +import stat +import subprocess +import tempfile +import threading +import time +import uuid +from collections import deque +from contextlib import contextmanager +from pathlib import Path + +LOG = logging.getLogger(__name__) +ATTEMPT_ID = re.compile(r"[0-9a-f]{32}\Z") +STAGES = frozenset({ + "request", "registry_profile", "update_lock_wait", "update_check", + "iso_gc", "template_gc", "profile", "libvirt_connect", "host_info", + "domain_list", "domain_scan", "storage_scan", "public_pool_catalog", + "public_ip_claims", "public_ip_scan", "response", +}) +PROCESS_ID = uuid.uuid4().hex +CURRENT: contextvars.ContextVar = contextvars.ContextVar("inventory_trace", default=None) +MAX_TRACES = 4000 +RETENTION_SECONDS = 7 * 86400 +MAX_STAGES = 48 +MAX_BYTES = 32768 + + +def engine_version(project_dir: Path) -> str: + """Capture the loaded process revision once, never reread on each request.""" + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=project_dir, capture_output=True, + text=True, timeout=1, check=True, + ).stdout.strip() + return result if re.fullmatch(r"[0-9a-f]{40,64}", result) else "unknown" + except (OSError, ValueError, subprocess.SubprocessError): + return "unknown" + + +class InventoryDiagnostics: + def __init__(self, data_dir: Path, version: str, *, process_id=PROCESS_ID, + max_traces=MAX_TRACES, retention_seconds=RETENTION_SECONDS): + self.directory = data_dir / "inventory-diagnostics" + self.version = version + self.process_id = process_id + self.max_traces = max_traces + self.retention_seconds = retention_seconds + self.lock = threading.RLock() + self.records = deque() + self.last_pruned = 0.0 + self.ready = False + try: + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + info = self.directory.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid(): + raise OSError("unsafe directory") + self.directory.chmod(0o700) + self._prune(self.max_traces) + self.ready = True + except Exception: + LOG.warning("inventory_diagnostics_storage_unavailable") + + def _path(self, host, attempt, resource): + key = hashlib.sha256((host + "\0" + attempt + "\0" + resource).encode()).hexdigest() + return self.directory / (key + ".json") + + def _prune(self, limit): + records = [] + now = time.time() + for path in self.directory.iterdir(): + if not re.fullmatch(r"[0-9a-f]{64}\.json", path.name): + # Clean only our interrupted atomic-write temporary files. + if path.name.startswith(".inventory-") and path.lstat().st_mtime < now - 3600: + path.unlink(missing_ok=True) + continue + info = path.lstat() + if stat.S_ISREG(info.st_mode): + if info.st_mtime < now - self.retention_seconds: + path.unlink(missing_ok=True) + else: + records.append((info.st_mtime, path)) + for _, path in sorted(records, reverse=True)[limit:]: + path.unlink(missing_ok=True) + self.records = deque(path for _, path in sorted(records, reverse=True)[:limit][::-1]) + self.last_pruned = time.monotonic() + + def _write(self, path, value): + payload = json.dumps(value, separators=(",", ":")).encode() + if len(payload) > MAX_BYTES: + raise ValueError("summary too large") + descriptor, temporary = tempfile.mkstemp(prefix=".inventory-", dir=self.directory) + try: + with os.fdopen(descriptor, "wb") as output: + os.fchmod(output.fileno(), 0o600) + output.write(payload) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + finally: + Path(temporary).unlink(missing_ok=True) + + def begin(self, host: str, attempt: str, resource: str): + if not self.ready: + return None + trace = InventoryTrace(self, host, attempt, resource) + try: + with self.lock: + path = self._path(host, attempt, resource) + # Never overwrite evidence from a replayed attempt/resource. + if path.exists() or path.is_symlink(): + return None + # The directory is engine-private and has one supported + # writer process. Keep a bounded queue rather than scanning + # thousands of files before every inventory request. + if time.monotonic() - self.last_pruned >= 60: + self._prune(self.max_traces) + while len(self.records) >= self.max_traces: + self.records.popleft().unlink(missing_ok=True) + self._write(path, trace.value) + self.records.append(path) + return trace + except Exception: + LOG.warning("inventory_diagnostics_write_failed attempt=%s resource=%s", attempt, resource) + return None + + def save(self, trace): + try: + with self.lock: + if trace.value["state"] == "running": + trace.value["duration_ms"] = max(0, int((time.monotonic() - trace.value["_started"]) * 1000)) + path = self._path(trace.host, trace.attempt, trace.value["resource"]) + # A retention-evicted request cannot resurrect its old record. + if path.exists() and not path.is_symlink(): + self._write(path, trace.value) + except Exception: + LOG.warning("inventory_diagnostics_write_failed attempt=%s resource=%s stage=%s", + trace.attempt, trace.value["resource"], trace.value["stage"]) + + def read(self, host: str, attempt: str) -> dict: + if not self.ready: + return {"attempt_id": attempt, "traces": [], "diagnostics_available": False} + try: + values = [] + with self.lock: + for resource in ("capacity", "network"): + path = self._path(host, attempt, resource) + try: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except FileNotFoundError: + continue + with os.fdopen(descriptor, "rb") as source: + info = os.fstat(source.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_BYTES: + raise ValueError("invalid summary") + if info.st_mtime < time.time() - self.retention_seconds: + continue + values.append(self._public(json.loads(source.read(MAX_BYTES + 1)))) + return {"attempt_id": attempt, "traces": values, "diagnostics_available": True} + except Exception: + LOG.warning("inventory_diagnostics_read_failed attempt=%s", attempt) + return {"attempt_id": attempt, "traces": [], "diagnostics_available": False} + + def _public(self, value): + if value["state"] in {"running", "interrupted"} and value.get("_process") != self.process_id: + # A restart is evidence of interruption, not evidence of the old + # thread's completion time. Do not invent a finish timestamp. + value.update(state="interrupted", error_code="request_interrupted") + for item in value["stages"]: + if item["state"] == "running": + item["state"] = "interrupted" + elif value["state"] == "running": + now = time.monotonic() + value["duration_ms"] = max(0, int((now - value["_started"]) * 1000)) + for item in value["stages"]: + if item["state"] == "running": + item["duration_ms"] = max(0, int((now - item["_started"]) * 1000)) + return {key: ([{k: v for k, v in item.items() if not k.startswith("_")} for item in item_value] + if key == "stages" else item_value) + for key, item_value in value.items() if not key.startswith("_")} + + +class InventoryTrace: + def __init__(self, store, host, attempt, resource): + self.store, self.host, self.attempt = store, host, attempt + self.lock = threading.RLock() + self.active_stages = [] + self.failure_stage = None + self.failure_exception = None + self.endpoint_started = False + self.value = { + "resource": resource, "state": "running", "stage": "request", + "error_code": "", "http_status": None, "started_at": int(time.time()), + "finished_at": None, "duration_ms": 0, "stages": [], + "engine_version": store.version, "_started": time.monotonic(), + "_process": store.process_id, + } + + def start(self, name): + with self.lock: + if name not in STAGES or self.value["state"] != "running": + return None + self.failure_stage = None + self.failure_exception = None + self.value["stage"] = name + if len(self.value["stages"]) >= MAX_STAGES: + # Repeated tombstone cleanup can contain many connections. + # Keep the newest evidence bounded without losing the stage + # that is about to block, or removing an active parent span. + completed = next((item for item in self.value["stages"] + if item["state"] != "running"), None) + if completed is None: + return None + self.value["stages"].remove(completed) + item = {"stage": name, "state": "running", "duration_ms": 0, "_started": time.monotonic()} + try: + asyncio.get_running_loop() + item["_async"] = True + except RuntimeError: + item["_async"] = False + self.value["stages"].append(item) + self.active_stages.append(item) + self.store.save(self) + return item + + def _close_stage(self, state): + for item in self.active_stages: + if item["state"] == "running": + item.update(state=state, duration_ms=max(0, int((time.monotonic() - item["_started"]) * 1000))) + + def end(self, item): + with self.lock: + if item is not None and item["state"] == "running": + item.update(state="succeeded", duration_ms=max(0, int((time.monotonic() - item["_started"]) * 1000))) + self.active_stages.remove(item) + if self.active_stages and self.value["state"] == "running": + self.value["stage"] = self.active_stages[-1]["stage"] + elif self.value["state"] == "running": + self.value["stage"] = item["stage"] + self.store.save(self) + + def fail_stage(self, item, exception): + with self.lock: + # A GC helper may catch a failed remote call and still allow the + # inventory to proceed. A failed span is NOT a failed request. + if item is not None and item["state"] == "running": + item.update(state="failed", duration_ms=max(0, int((time.monotonic() - item["_started"]) * 1000))) + self.active_stages.remove(item) + chained = exception + matches = False + seen = set() + while chained is not None and id(chained) not in seen: + if chained is self.failure_exception: + matches = True + break + seen.add(id(chained)) + chained = chained.__cause__ or chained.__context__ + if not matches: + self.failure_stage = item["stage"] if item else self.value["stage"] + self.failure_exception = exception + self.value["stage"] = self.failure_stage + self.store.save(self) + + def worker_started(self): + with self.lock: + self.endpoint_started = True + # A queued worker can start just after its ASGI waiter was + # cancelled. Its actual completion remains authoritative engine + # evidence; the cancelled PHP attempt is still separate. + if self.value["state"] == "interrupted" and self.value.get("_waiter_cancelled"): + self.value.update(state="running", error_code="") + self.store.save(self) + + def waiter_cancelled(self): + with self.lock: + if self.value["state"] != "running": + return + self.value["_waiter_cancelled"] = True + if self.endpoint_started: + # Cancelling an await cannot stop the sync endpoint worker. + # Its decorator, not this waiter, will record its result. + self.store.save(self) + return + now = time.monotonic() + self.value.update( + state="interrupted", error_code="request_interrupted", + http_status=None, finished_at=None, + duration_ms=max(0, int((now - self.value["_started"]) * 1000)), + ) + for item in tuple(self.active_stages): + if item["state"] == "running" and item.get("_async"): + item.update(state="interrupted", duration_ms=max(0, int((now - item["_started"]) * 1000))) + self.active_stages.remove(item) + # A pre-endpoint GC thread may still run. Preserve its last + # confirmed span as running, with no invented finish timestamp. + self.store.save(self) + + def finish(self, status=200, *, interrupted=False): + with self.lock: + if self.value["state"] != "running": + # A lower-level libvirt exception is translated into VMError + # by the service. Preserve its original stage, but use the + # final endpoint status when that translation is known. + if self.value["state"] == "failed" and status >= 400: + self.value["http_status"] = status + self.store.save(self) + return + state = "interrupted" if interrupted else "succeeded" if 200 <= status < 300 else "failed" + if state == "failed" and self.failure_stage: + self.value["stage"] = self.failure_stage + self._close_stage(state) + self.value.update( + state=state, http_status=status, finished_at=int(time.time()), + duration_ms=max(0, int((time.monotonic() - self.value["_started"]) * 1000)), + error_code=("request_interrupted" if interrupted else "" if state == "succeeded" else self.value["stage"] + "_failed"), + ) + self.store.save(self) + + +@contextmanager +def inventory_stage(name): + trace = CURRENT.get() + item = trace.start(name) if trace else None + try: + yield + except Exception as exc: + if trace: + trace.fail_stage(item, exc) + raise + else: + if trace: + trace.end(item) + + +def inventory_result(function): + """Finalize in the actual worker, even if its HTTP waiter was cancelled.""" + @functools.wraps(function) + def wrapper(*args, **kwargs): + trace = CURRENT.get() + if trace: + trace.worker_started() + try: + result = function(*args, **kwargs) + except Exception as exc: + if trace: + status = getattr(exc, "status", getattr(exc, "status_code", 503)) + trace.finish(status if isinstance(status, int) and 400 <= status <= 599 else 503) + raise + if trace: + trace.finish(200) + return result + # FastAPI resolves forward annotations using the callable's globals; + # this wrapper lives in another module, so resolve them at decoration. + wrapper.__signature__ = inspect.signature(function, eval_str=True) + return wrapper diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index 835c2513..71f85821 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -30,6 +30,7 @@ from virsh_panel.cloud_init import ( validate_guest_hostname, ) from virsh_panel.metrics import RawVMStats +from virsh_panel.inventory_diagnostics import inventory_stage from virsh_panel.user_isos import ( MAX_USER_ISO_BYTES, MAX_USER_ISOS, MAX_USER_ISO_TOTAL_BYTES, USER_ISO_ID_RE, USER_ISO_VOLUME_RE, verify_user_iso, @@ -109,9 +110,10 @@ class VMService: @contextmanager def _connect(self) -> Iterator[Any]: try: - conn = libvirt.open(self.uri) - if conn is None: - raise VMError("无法连接虚拟化宿主机", 502) + with inventory_stage("libvirt_connect"): + conn = libvirt.open(self.uri) + if conn is None: + raise VMError("无法连接虚拟化宿主机", 502) except libvirt.libvirtError as exc: LOG.warning("libvirt connection failed: %s", exc) raise VMError("无法连接虚拟化宿主机", 502) from exc @@ -601,10 +603,13 @@ class VMService: def list_vms(self) -> list[dict[str, Any]]: with self._connect() as conn: - return sorted( - (self._vm_dict(domain) for domain in conn.listAllDomains(0)), - key=lambda row: row["name"].casefold(), - ) + with inventory_stage("domain_list"): + domains = conn.listAllDomains(0) + with inventory_stage("domain_scan"): + return sorted( + (self._vm_dict(domain) for domain in domains), + key=lambda row: row["name"].casefold(), + ) @staticmethod def _vm_state_dict(domain: Any) -> dict[str, Any]: @@ -875,12 +880,13 @@ class VMService: counters so callers do not lose precision by rounding to GiB. """ with self._connect() as conn: - node = conn.getInfo() - if (not isinstance(node, (tuple, list)) or len(node) < 3 - or isinstance(node[1], bool) or not isinstance(node[1], int) - or isinstance(node[2], bool) or not isinstance(node[2], int) - or node[1] < 0 or node[2] < 0): - raise VMError("宿主机容量信息无效", 502) + with inventory_stage("host_info"): + node = conn.getInfo() + if (not isinstance(node, (tuple, list)) or len(node) < 3 + or isinstance(node[1], bool) or not isinstance(node[1], int) + or isinstance(node[2], bool) or not isinstance(node[2], int) + or node[1] < 0 or node[2] < 0): + raise VMError("宿主机容量信息无效", 502) cpu_total = node[2] # libvirt-python converts virNodeInfo.memory from KiB to MiB @@ -891,26 +897,29 @@ class VMService: cpu_allocated = 0 memory_allocated_mib = 0 running = 0 - domains = conn.listAllDomains(0) - for domain in domains: - info = domain.info() - if (not isinstance(info, (tuple, list)) or len(info) < 4 - or isinstance(info[1], bool) or not isinstance(info[1], int) - or isinstance(info[3], bool) or not isinstance(info[3], int) - or info[1] < 0 or info[3] < 0): - raise VMError("宿主机虚拟机容量信息无效", 502) - memory_allocated_mib += info[1] // 1024 - cpu_allocated += info[3] - if info[0] in {1, 2, 3, 7}: - running += 1 - - disk_pool = self._active_pool(conn, self.storage_pool) - pool_info = disk_pool.info() - if (not isinstance(pool_info, (tuple, list)) or len(pool_info) < 4 - or any(isinstance(value, bool) or not isinstance(value, int) - or value < 0 for value in pool_info[1:4])): - raise VMError("宿主机存储容量信息无效", 502) - storage_total, storage_allocated, storage_available = pool_info[1:4] + with inventory_stage("domain_list"): + domains = conn.listAllDomains(0) + with inventory_stage("domain_scan"): + for domain in domains: + info = domain.info() + if (not isinstance(info, (tuple, list)) or len(info) < 4 + or isinstance(info[1], bool) or not isinstance(info[1], int) + or isinstance(info[3], bool) or not isinstance(info[3], int) + or info[1] < 0 or info[3] < 0): + raise VMError("宿主机虚拟机容量信息无效", 502) + memory_allocated_mib += info[1] // 1024 + cpu_allocated += info[3] + if info[0] in {1, 2, 3, 7}: + running += 1 + + with inventory_stage("storage_scan"): + disk_pool = self._active_pool(conn, self.storage_pool) + pool_info = disk_pool.info() + if (not isinstance(pool_info, (tuple, list)) or len(pool_info) < 4 + or any(isinstance(value, bool) or not isinstance(value, int) + or value < 0 for value in pool_info[1:4])): + raise VMError("宿主机存储容量信息无效", 502) + storage_total, storage_allocated, storage_available = pool_info[1:4] return { "instances": {"total": len(domains), "running": running}, diff --git a/web/app/Controllers/Admin/CloudPageController.php b/web/app/Controllers/Admin/CloudPageController.php index 34ad0eda..e76b3dad 100644 --- a/web/app/Controllers/Admin/CloudPageController.php +++ b/web/app/Controllers/Admin/CloudPageController.php @@ -10,6 +10,7 @@ use App\Services\Cloud\CloudAdminOperationsService; use App\Services\Cloud\CloudAdminResourceService; use App\Services\Cloud\CloudCountryCatalog; use App\Services\Cloud\CloudError; +use App\Services\Cloud\CloudHostInventoryDiagnostics; /** Admin pages use database-only first paints; JSON APIs repeat authorisation. */ final class CloudPageController extends AdminController @@ -177,4 +178,39 @@ final class CloudPageController extends AdminController { return $this->view()->display('admin/update.tpl'); } + + public function hostInventory($request, $response, $args) + { + try { + $actor = CloudAccess::actor(false); + } catch (CloudError $exception) { + return $response->withStatus($exception->httpStatus) + ->withHeader('Cache-Control', 'private, no-store'); + } + if (!($actor['is_admin'] ?? false)) { + return $response->withStatus(403)->withHeader('Cache-Control', 'private, no-store'); + } + $payload = null; + $error = ''; + $status = 200; + try { + $payload = CloudHostInventoryDiagnostics::page($actor, $request->getQueryParams()); + } catch (CloudError $exception) { + $error = $exception->getMessage(); + $status = $exception->httpStatus; + } catch (\Throwable $exception) { + error_log('Admin inventory history failed: ' . get_class($exception)); + $error = '盘点记录暂时无法读取,请确认数据库已升级后重试。'; + $status = 503; + } + $smarty = $this->view()->assign([ + 'inventory_page' => $payload, 'inventory_error' => $error, + 'inventory_timezone' => date_default_timezone_get(), + 'inventory_bootstrap_json' => json_encode($payload, + JSON_HEX_TAG | JSON_HEX_AMP | JSON_HEX_APOS | JSON_HEX_QUOT + | JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE | JSON_THROW_ON_ERROR), + ]); + $response->getBody()->write($smarty->fetch('admin/cloud/host_inventory.tpl')); + return $response->withStatus($status)->withHeader('Cache-Control', 'private, no-store'); + } } diff --git a/web/app/Controllers/CloudAdminHostInventoryController.php b/web/app/Controllers/CloudAdminHostInventoryController.php new file mode 100644 index 00000000..54a462ec --- /dev/null +++ b/web/app/Controllers/CloudAdminHostInventoryController.php @@ -0,0 +1,58 @@ +withJson(CloudHostInventoryDiagnostics::page( + $request->getAttribute('cloud_actor') ?? CloudAccess::actor(false), + $request->getQueryParams() + ))->withHeader('Cache-Control', 'private, no-store'); + } + + public function detail($request, $response, $args) + { + return $response->withJson(CloudHostInventoryDiagnostics::detail( + $request->getAttribute('cloud_actor') ?? CloudAccess::actor(false), + (string) ($args['id'] ?? '') + ))->withHeader('Cache-Control', 'private, no-store'); + } + + public function engine($request, $response, $args) + { + $actor = $request->getAttribute('cloud_actor') ?? CloudAccess::actor(false); + $attemptId = (string) ($args['id'] ?? ''); + // This authorizes the read and binds its host to a persisted attempt. + // Never accept an arbitrary host or call the capacity/network endpoints. + $detail = CloudHostInventoryDiagnostics::detail($actor, $attemptId); + try { + $result = (new CloudEngineClient(null, 1.0))->inventoryDiagnostics( + (int) $detail['attempt']['host_id'], $actor, $attemptId + ); + $available = ($result['diagnostics_available'] ?? true) === true; + $result = ['available' => $available, 'attempt_id' => $attemptId, + 'traces' => $result['traces'], + 'message' => $available + ? '仅查询引擎已记录的线索;不会发起盘点,也不会改变面板同步结果。' + : '引擎诊断存储暂不可用;无法取得内部阶段线索,不代表宿主机离线。']; + } catch (CloudInventoryRequestError $exception) { + $result = ['available' => false, 'attempt_id' => $attemptId, + 'traces' => [], 'error_code' => $exception->diagnosticCode, + 'message' => '引擎线索暂不可读取或尚未记录;无法据此判断宿主机是否正常。']; + } catch (CloudError $exception) { + $result = ['available' => false, 'attempt_id' => $attemptId, + 'traces' => [], 'error_code' => 'diagnostics_unavailable', + 'message' => '引擎线索暂不可读取;请检查引擎版本、认证及连接配置。']; + } + return $response->withJson($result)->withHeader('Cache-Control', 'private, no-store'); + } +} diff --git a/web/app/Middleware/CloudApi.php b/web/app/Middleware/CloudApi.php index 739ede39..49aab4cf 100644 --- a/web/app/Middleware/CloudApi.php +++ b/web/app/Middleware/CloudApi.php @@ -19,6 +19,7 @@ final class CloudApi || preg_match('#^/api/v1/admin/billing(?:/|$)#D', $path) === 1 || preg_match('#^/api/v1/admin/instances(?:/|$)#D', $path) === 1 || preg_match('#^/api/v1/admin/operations(?:/|$)#D', $path) === 1 + || preg_match('#^/api/v1/admin/host-inventory(?:/|$)#D', $path) === 1 || preg_match('#^/api/v1/admin/(?:resources|regions|network-lines|resource-pools|resource-reservations)(?:/|$)#D', $path) === 1 || preg_match('#^/api/v1/admin/hosts/[0-9]+/(?:scheduling|capacity-sync)$#D', $path) === 1 || preg_match('#^/api/v1/ssh-keys(?:/|$)#D', $path) === 1 diff --git a/web/app/Models/CloudHostInventoryAttempt.php b/web/app/Models/CloudHostInventoryAttempt.php new file mode 100644 index 00000000..0a27d7a3 --- /dev/null +++ b/web/app/Models/CloudHostInventoryAttempt.php @@ -0,0 +1,18 @@ + 'int', 'host_id' => 'int', 'generation' => 'int', 'started_at' => 'int', + 'finished_at' => 'int', 'updated_at' => 'int', 'duration_ms' => 'int', + 'http_status' => 'int', 'applied' => 'boolean', 'timeout_ms' => 'int', + ]; +} diff --git a/web/app/Services/Cloud/CloudEngineClient.php b/web/app/Services/Cloud/CloudEngineClient.php index 14a54a89..81543488 100644 --- a/web/app/Services/Cloud/CloudEngineClient.php +++ b/web/app/Services/Cloud/CloudEngineClient.php @@ -6,11 +6,14 @@ use GuzzleHttp\Client; final class CloudEngineClient { + private float $timeoutSeconds; + public function __construct(private ?Client $client = null, float $timeoutSeconds = 20.0) { if ($timeoutSeconds < 1 || $timeoutSeconds > 900) { throw new \InvalidArgumentException('Engine timeout is outside the supported range'); } + $this->timeoutSeconds = $timeoutSeconds; $this->client ??= new Client(['connect_timeout' => 3, 'timeout' => $timeoutSeconds, 'http_errors' => false, 'allow_redirects' => false]); } @@ -343,4 +346,165 @@ final class CloudEngineClient return $this->request($method, '/internal/v1/hosts/' . $hostId . '/' . $resource, $actor, $payload, $expectedOperationId); } + + /** Read-only inventory keeps diagnostic categories without changing mutation errors. */ + public function inventoryTimeoutSeconds(): float + { + return $this->timeoutSeconds; + } + + public function hostInventory(int $hostId, string $resource, array $actor, + string $attemptId): array + { + if (!in_array($resource, ['host/capacity', 'host/network-options'], true)) { + throw new \InvalidArgumentException('Unsupported inventory resource'); + } + $this->validateInventoryIdentity($hostId, $attemptId); + // The correlation ID is covered by the existing HMAC canonical target. + return $this->inventoryRequest('/internal/v1/hosts/' . $hostId . '/' + . $resource . '?inventory_attempt_id=' . $attemptId, $actor, + $this->timeoutSeconds); + } + + /** Bounded local evidence lookup: never runs inventory or touches a hypervisor. */ + public function inventoryDiagnostics(int $hostId, array $actor, + string $attemptId): array + { + $this->validateInventoryIdentity($hostId, $attemptId); + $value = $this->inventoryRequest('/internal/v1/hosts/' . $hostId + . '/host/inventory-diagnostics/' . $attemptId, $actor, 1.0); + return self::inventoryEvidence($value, $attemptId); + } + + private function validateInventoryIdentity(int $hostId, string $attemptId): void + { + if ($hostId < 1 || preg_match('/^[0-9a-f]{32}$/D', $attemptId) !== 1) { + throw new \InvalidArgumentException('Invalid inventory identity'); + } + } + + private function inventoryRequest(string $path, array $actor, float $timeout): array + { + try { + $base = $this->base($path); + $headers = self::headers('GET', $path, '', $actor); + } catch (\Throwable $error) { + throw new CloudInventoryRequestError('engine_client_configuration_invalid', null, 503); + } + try { + $response = $this->client->request('GET', $base . $path, [ + 'headers' => $headers, 'body' => '', 'timeout' => $timeout, + 'connect_timeout' => min(3.0, $timeout), 'allow_redirects' => false, + 'http_errors' => false, 'proxy' => '', + ]); + } catch (\GuzzleHttp\Exception\TransferException $error) { + $context = method_exists($error, 'getHandlerContext') + ? $error->getHandlerContext() : []; + // cURL 28 proves a timeout, but not which remote stage caused it. + $code = (int) ($context['errno'] ?? 0) === 28 + ? 'engine_http_timeout' + : ($error instanceof \GuzzleHttp\Exception\ConnectException + ? (in_array((int) ($context['errno'] ?? 0), [5, 6, 7], true) + ? 'engine_connect_failed' : 'engine_connection_or_timeout') + : 'engine_transport_failed'); + throw new CloudInventoryRequestError($code); + } catch (\Throwable $error) { + throw new CloudInventoryRequestError('engine_transport_failed'); + } + $status = $response->getStatusCode(); + try { + $body = $response->getBody()->read(2097153); + $value = strlen($body) <= 2097152 + ? json_decode($body, true, 64, JSON_THROW_ON_ERROR) : null; + } catch (\Throwable $error) { + $value = null; + } + if ($status >= 400) { + $code = match ($status) { + 401, 403 => 'engine_auth_rejected', + 404 => 'engine_resource_missing', + 408, 504 => 'engine_http_timeout', + 409 => 'engine_request_conflict', + 423 => 'engine_configuration_locked', + 429 => 'engine_rate_limited', + default => $status >= 500 ? 'engine_service_error' : 'engine_request_rejected', + }; + // Preserve the old scheduling classification: a non-JSON error + // was an uncertain 502, including a non-JSON upstream 404. + $legacyStatus = $status === 404 && is_array($value) ? 404 : 502; + throw new CloudInventoryRequestError($code, $status, $legacyStatus); + } + if (!is_array($value)) { + throw new CloudInventoryRequestError('engine_response_invalid', $status); + } + return $value; + } + + /** Allowlisted evidence only: never pass arbitrary engine strings to the admin UI. */ + private static function inventoryEvidence(array $value, string $attemptId): array + { + $stages = ['request', 'registry_profile', 'update_lock_wait', 'update_check', 'iso_gc', + 'template_gc', 'profile', 'libvirt_connect', 'host_info', 'domain_list', + 'domain_scan', 'storage_scan', 'public_pool_catalog', 'public_ip_claims', + 'public_ip_scan', 'response']; + $states = ['running', 'succeeded', 'failed', 'interrupted']; + $codes = array_merge(['', 'request_failed', 'request_interrupted'], + array_map(static fn (string $stage): string => $stage . '_failed', $stages)); + if (($value['attempt_id'] ?? null) !== $attemptId + || !is_array($value['traces'] ?? null) || count($value['traces']) > 2 + || (isset($value['diagnostics_available']) && !is_bool($value['diagnostics_available']))) { + throw new CloudInventoryRequestError('engine_response_invalid'); + } + $number = static function ($input, bool $nullable = false): ?int { + if ($input === null && $nullable) { + return null; + } + if (!is_int($input) || $input < 0) { + throw new CloudInventoryRequestError('engine_response_invalid'); + } + return $input; + }; + $traces = []; + $seen = []; + foreach ($value['traces'] as $trace) { + if (!is_array($trace) + || !in_array($trace['resource'] ?? null, ['capacity', 'network'], true) + || isset($seen[$trace['resource']]) + || !in_array($trace['state'] ?? null, $states, true) + || !in_array($trace['stage'] ?? null, $stages, true) + || !is_array($trace['stages'] ?? null) || count($trace['stages']) > 64) { + throw new CloudInventoryRequestError('engine_response_invalid'); + } + $seen[$trace['resource']] = true; + $code = $trace['error_code'] ?? ''; + if (!is_string($code) || (!in_array($code, $codes, true) + && preg_match('/^http_[45][0-9]{2}$/D', $code) !== 1)) { + $code = 'request_failed'; + } + $http = $number($trace['http_status'] ?? null, true); + if ($http !== null && ($http < 100 || $http > 599)) { + throw new CloudInventoryRequestError('engine_response_invalid'); + } + $version = $trace['engine_version'] ?? ''; + $version = is_string($version) + && preg_match('/^[0-9a-f]{7,64}$/D', $version) === 1 ? $version : 'unknown'; + $rows = []; + foreach ($trace['stages'] as $row) { + if (!is_array($row) || !in_array($row['stage'] ?? null, $stages, true) + || !in_array($row['state'] ?? null, $states, true)) { + throw new CloudInventoryRequestError('engine_response_invalid'); + } + $rows[] = ['stage' => $row['stage'], 'state' => $row['state'], + 'duration_ms' => $number($row['duration_ms'] ?? null)]; + } + $traces[] = ['resource' => $trace['resource'], 'state' => $trace['state'], + 'stage' => $trace['stage'], 'error_code' => $code, 'http_status' => $http, + 'started_at' => $number($trace['started_at'] ?? null), + 'finished_at' => $number($trace['finished_at'] ?? null, true), + 'duration_ms' => $number($trace['duration_ms'] ?? null), + 'stages' => $rows, 'engine_version' => $version]; + } + return ['attempt_id' => $attemptId, 'traces' => $traces, + 'diagnostics_available' => $value['diagnostics_available'] ?? true]; + } } diff --git a/web/app/Services/Cloud/CloudHostInventoryDiagnostics.php b/web/app/Services/Cloud/CloudHostInventoryDiagnostics.php new file mode 100644 index 00000000..52285604 --- /dev/null +++ b/web/app/Services/Cloud/CloudHostInventoryDiagnostics.php @@ -0,0 +1,423 @@ + '进行中', 'succeeded' => '成功', 'failed' => '失败', + 'timeout' => '请求超时', 'skipped' => '跳过', 'superseded' => '已被更新的尝试取代', + 'interrupted' => '中断或结果未知', + ]; + private const STAGES = [ + 'admission' => '面板准备盘点', 'capacity_request' => '请求引擎容量', + 'capacity_validate' => '校验容量响应', 'network_request' => '请求公网 IP 信息', + 'network_validate' => '校验公网 IP 响应', 'database_apply' => '提交容量与 IP 数据', + 'failure_marker' => '记录同步失败状态', + ]; + private const ERRORS = [ + 'attempt_start_failed' => '面板无法创建容量同步代际', + 'engine_http_timeout' => '等待引擎响应超时;内部阶段需查看引擎证据', + 'engine_connect_failed' => '面板无法连接引擎', + 'engine_connection_or_timeout' => '连接失败或请求超时;客户端未提供可确认的细分原因', + 'engine_transport_failed' => '面板与引擎之间的传输失败', + 'engine_response_invalid' => '引擎响应无法解析或不符合协议', + 'engine_auth_rejected' => '引擎拒绝内部请求认证', + 'engine_configuration_locked' => '引擎配置被锁定或正在更新', + 'engine_request_conflict' => '引擎报告状态或配置冲突', + 'engine_resource_missing' => '引擎报告请求资源不存在', + 'engine_rate_limited' => '引擎请求受到频率或并发限制', + 'engine_service_error' => '引擎处理请求失败;内部阶段需查看引擎证据', + 'engine_request_rejected' => '引擎拒绝此次盘点请求', + 'engine_client_configuration_invalid' => '面板内部引擎客户端配置无效', + 'host_inventory_capacity_invalid' => '容量响应校验未通过', + 'host_inventory_network_invalid' => '公网 IP 响应校验未通过', + 'host_inventory_ip_conflict' => '公网 IP 或地址池记录冲突', + 'host_inventory_write_failed' => '容量和 IP 数据事务提交失败', + 'host_inventory_unavailable' => '本次盘点未能取得可用的引擎响应', + 'host_inventory_engine_rejected' => '引擎未接受本次盘点', + 'unexpected_error' => '发生未分类错误,未记录原始异常内容', + ]; + + private array $record; + private array $stages = []; + private int $startedMonotonic; + private int $stageStartedMonotonic; + private bool $stored = false; + private bool $finished = false; + private bool $fallbackLogged = false; + private static array $lastCleanup = []; + + private function __construct(int $hostId, string $source, int $startedAt, + float $timeoutSeconds) + { + $this->startedMonotonic = hrtime(true); + $this->stageStartedMonotonic = $this->startedMonotonic; + try { + $id = bin2hex(random_bytes(16)); + } catch (\Throwable $ignored) { + // Correlation only; this identifier is never an authentication token. + $id = substr(hash('sha256', $hostId . ':' . $this->startedMonotonic + . ':' . getmypid() . ':' . uniqid('', true)), 0, 32); + } + $this->record = [ + 'attempt_id' => $id, 'host_id' => max(1, $hostId), + 'source' => $source === 'manual' ? 'manual' : 'auto', 'generation' => 0, + 'started_at' => max(1, $startedAt), 'finished_at' => null, + 'updated_at' => max(1, $startedAt), 'duration_ms' => 0, + 'outcome' => 'running', 'stage' => 'admission', 'error_code' => '', + 'http_status' => null, 'applied' => false, + 'timeout_ms' => is_finite($timeoutSeconds) + ? max(1, min(86400000, (int) round($timeoutSeconds * 1000))) : 8000, + 'panel_version' => self::PANEL_VERSION, + 'php_version' => self::version(PHP_VERSION), 'stages_json' => '[]', + ]; + } + + public static function start(int $hostId, string $source, int $startedAt, + float $timeoutSeconds): self + { + $attempt = new self($hostId, $source, $startedAt, $timeoutSeconds); + try { + CloudHostInventoryAttempt::query()->insert($attempt->record); + $attempt->stored = true; + self::cleanup($hostId, $attempt->record['started_at']); + } catch (\Throwable $ignored) { + $attempt->fallback(); + } + return $attempt; + } + + public function id(): string + { + return $this->record['attempt_id']; + } + + public function generation(int $generation): void + { + if (!$this->finished) { + $this->record['generation'] = max(0, $generation); + $this->persist(); + } + } + + public function stage(string $stage): void + { + if ($this->finished || !isset(self::STAGES[$stage]) || $stage === $this->record['stage']) { + return; + } + $this->closeStage(); + $this->record['stage'] = $stage; + $this->persist(); + } + + public function finish(string $outcome, string $errorCode = '', + ?int $httpStatus = null, bool $applied = false): void + { + if ($this->finished) { + return; + } + $this->finished = true; + $this->closeStage(); + $this->record['outcome'] = isset(self::OUTCOMES[$outcome]) && $outcome !== 'running' + ? $outcome : 'failed'; + $this->record['error_code'] = self::errorCode($errorCode); + $this->record['http_status'] = $httpStatus !== null && $httpStatus >= 100 + && $httpStatus <= 599 ? $httpStatus : null; + $this->record['applied'] = $outcome === 'succeeded' && $applied; + // Match the injected observation clock's wall-time origin, but measure + // elapsed time solely within this process using the monotonic clock. + $this->record['finished_at'] = $this->record['started_at'] + + (int) floor($this->record['duration_ms'] / 1000); + $this->persist(); + } + + private function closeStage(): void + { + $now = hrtime(true); + if (count($this->stages) < 32) { + $this->stages[] = ['stage' => $this->record['stage'], + 'duration_ms' => max(0, (int) floor(($now - $this->stageStartedMonotonic) / 1000000))]; + } + $this->stageStartedMonotonic = $now; + $this->record['duration_ms'] = max(0, + (int) floor(($now - $this->startedMonotonic) / 1000000)); + } + + private function persist(): void + { + try { + if (!$this->stored) { + $this->fallback(); + return; + } + $this->record['updated_at'] = $this->record['started_at'] + + (int) floor($this->record['duration_ms'] / 1000); + $this->record['stages_json'] = json_encode($this->stages, JSON_THROW_ON_ERROR); + CloudHostInventoryAttempt::where('attempt_id', $this->id()) + ->where('outcome', 'running')->update($this->record); + } catch (\Throwable $ignored) { + $this->fallback(); + } + } + + private function fallback(): void + { + if ($this->fallbackLogged && !$this->finished) { + return; + } + $this->fallbackLogged = true; + try { + error_log('cloud_inventory_diagnostics_unavailable attempt_id=' . $this->id() + . ' host_id=' . $this->record['host_id'] . ' outcome=' . $this->record['outcome'] + . ' stage=' . $this->record['stage'] . ' error_code=' . $this->record['error_code']); + } catch (\Throwable $ignored) { + // Failure of diagnostic storage or logging cannot change scheduling. + } + } + + /** Bounded best-effort cleanup on attempts only, never a page-view side effect. */ + private static function cleanup(int $hostId, int $now): void + { + if (($now - (self::$lastCleanup[$hostId] ?? 0)) < 60) { + return; + } + if (count(self::$lastCleanup) > 512) { + self::$lastCleanup = []; + } + self::$lastCleanup[$hostId] = $now; + try { + $expired = CloudHostInventoryAttempt::where('started_at', '<', $now - self::RETENTION_SECONDS) + ->orderBy('started_at')->orderBy('sequence_id')->limit(self::CLEANUP_BATCH) + ->pluck('attempt_id')->all(); + if ($expired !== []) { + CloudHostInventoryAttempt::whereIn('attempt_id', $expired)->delete(); + } + $overflow = CloudHostInventoryAttempt::where('host_id', $hostId) + ->orderByDesc('started_at')->orderByDesc('sequence_id')->offset(self::HOST_RECORD_LIMIT) + ->limit(self::CLEANUP_BATCH)->pluck('attempt_id')->all(); + if ($overflow !== []) { + CloudHostInventoryAttempt::whereIn('attempt_id', $overflow)->delete(); + } + } catch (\Throwable $ignored) { + // History cleanup must not fail the current observation. + } + } + + public static function page(array $actor, array $input = []): array + { + self::authorize($actor); + $filters = self::filters($input); + try { + return self::readPage($filters); + } catch (\Throwable $ignored) { + throw new CloudError('盘点诊断历史不可读取,请确认数据库已初始化或升级', 503); + } + } + + private static function readPage(array $filters): array + { + $now = time(); + $base = CloudHostInventoryAttempt::query(); + if ($filters['host_id'] !== null) { + $base->where('host_id', (int) $filters['host_id']); + } + $summaryRow = (clone $base)->selectRaw('COUNT(*) AS total, MAX(started_at) AS last_attempt_at, ' + . "MAX(CASE WHEN outcome = 'succeeded' AND applied = 1 THEN finished_at ELSE NULL END) AS last_success_at, " + . "MAX(CASE WHEN outcome = 'succeeded' AND applied = 1 THEN sequence_id ELSE NULL END) AS last_success_sequence, " + . "MAX(CASE WHEN outcome IN ('failed', 'timeout') THEN finished_at ELSE NULL END) AS last_failure_at, " + . "COALESCE(SUM(CASE WHEN outcome = 'running' AND updated_at >= ? THEN 1 ELSE 0 END), 0) AS in_progress, " + . "COALESCE(SUM(CASE WHEN outcome = 'failed' THEN 1 ELSE 0 END), 0) AS failed, " + . "COALESCE(SUM(CASE WHEN outcome = 'timeout' THEN 1 ELSE 0 END), 0) AS timeout", + [$now - self::INTERRUPTED_SECONDS])->first(); + $summary = []; + foreach (['total', 'in_progress', 'failed', 'timeout'] as $field) { + $summary[$field] = max(0, (int) ($summaryRow?->{$field} ?? 0)); + } + foreach (['last_attempt_at', 'last_success_at', 'last_failure_at'] as $field) { + $summary[$field] = self::positiveTime($summaryRow?->{$field}); + } + $summary['consecutive_failures_scope'] = $filters['host_id'] === null + ? 'all_hosts' : 'selected_host'; + $summary['consecutive_failures'] = null; + if ($filters['host_id'] !== null) { + $failed = (clone $base)->whereIn('outcome', ['failed', 'timeout']); + if ($summaryRow?->last_success_sequence !== null) { + $failed->where('sequence_id', '>', (int) $summaryRow->last_success_sequence); + } + // Interrupted or skipped attempts are not evidence of another failure. + $summary['consecutive_failures'] = $failed->count(); + } + $query = (clone $base); + if ($filters['outcome'] === 'attention') { + $query->where(static function ($q) use ($now): void { + $q->whereIn('outcome', ['failed', 'timeout', 'interrupted']) + ->orWhere(static function ($q) use ($now): void { + $q->where('outcome', 'running')->where('updated_at', '<', $now - self::INTERRUPTED_SECONDS); + }); + }); + } elseif ($filters['outcome'] === 'interrupted') { + $query->where(static function ($q) use ($now): void { + $q->where('outcome', 'interrupted')->orWhere(static function ($q) use ($now): void { + $q->where('outcome', 'running')->where('updated_at', '<', $now - self::INTERRUPTED_SECONDS); + }); + }); + } elseif ($filters['outcome'] === 'running') { + $query->where('outcome', 'running')->where('updated_at', '>=', $now - self::INTERRUPTED_SECONDS); + } elseif ($filters['outcome'] !== 'all') { + $query->where('outcome', $filters['outcome']); + } + $total = $query->count(); + $pages = max(1, (int) ceil($total / $filters['per_page'])); + $filters['page'] = min($filters['page'], $pages); + $records = $query->orderByDesc('started_at')->orderByDesc('sequence_id') + ->offset(($filters['page'] - 1) * $filters['per_page'])->limit($filters['per_page'])->get(); + $hosts = []; + $hostNames = []; + foreach (CloudHost::orderBy('name')->orderBy('id')->get(['id', 'name']) as $host) { + $name = self::text((string) $host->name, 80); + $hosts[] = ['id' => (string) $host->id, 'name' => $name]; + $hostNames[(int) $host->id] = $name; + } + $items = []; + foreach ($records as $record) { + $items[] = self::dto($record, $hostNames[(int) $record->host_id] ?? '宿主机已删除', $now); + } + $from = $total === 0 ? 0 : ($filters['page'] - 1) * $filters['per_page'] + 1; + return ['items' => $items, 'filters' => $filters, 'hosts' => $hosts, + 'summary' => $summary, 'pagination' => ['page' => $filters['page'], + 'per_page' => $filters['per_page'], 'total' => $total, 'total_pages' => $pages, + 'from' => $from, 'to' => $from === 0 ? 0 : $from + count($items) - 1], + 'generated_at' => $now, 'retention_days' => 7]; + } + + public static function detail(array $actor, string $attemptId): array + { + self::authorize($actor); + if (preg_match('/^[a-f0-9]{32}$/D', $attemptId) !== 1) { + throw new CloudError('盘点请求编号无效', 400); + } + try { + $record = CloudHostInventoryAttempt::find($attemptId); + $host = $record === null ? null : CloudHost::find((int) $record->host_id, ['id', 'name']); + } catch (\Throwable $ignored) { + throw new CloudError('盘点诊断历史不可读取,请确认数据库已初始化或升级', 503); + } + if ($record === null) { + throw new CloudError('盘点记录不存在或已超过保留期限', 404); + } + return ['attempt' => self::dto($record, $host === null ? '宿主机已删除' + : self::text((string) $host->name, 80), time()), 'generated_at' => time(), 'notes' => [ + '面板结果与引擎最终结果独立;引擎晚到成功不代表容量已经提交。', + '超时只确认面板未及时收到响应,不能据此认定物理宿主机离线。', + '最后确认阶段不是已证实的故障根因;超过 5 分钟未结束的记录按中断或未知展示。', + '版本字段为运行时代码的诊断协议标识,不是 Git 提交号;部署后应重启常驻进程。', + '历史保留约 7 天且每宿主上限 25000 条,分批清理;本页不执行任何盘点。', + ]]; + } + + private static function dto(CloudHostInventoryAttempt $record, string $hostName, int $now): array + { + $outcome = isset(self::OUTCOMES[(string) $record->outcome]) + ? (string) $record->outcome : 'interrupted'; + $interrupted = $outcome === 'running' && (int) $record->updated_at < $now - self::INTERRUPTED_SECONDS; + if ($interrupted) { + $outcome = 'interrupted'; + } + $stage = isset(self::STAGES[(string) $record->stage]) ? (string) $record->stage : 'admission'; + $error = self::errorCode((string) $record->error_code); + $stages = []; + $storedStages = strlen((string) $record->stages_json) <= 16384 + ? json_decode((string) $record->stages_json, true) : null; + foreach (is_array($storedStages) ? array_slice($storedStages, 0, 32) : [] as $item) { + if (is_array($item) && is_string($item['stage'] ?? null) + && isset(self::STAGES[$item['stage']]) && is_int($item['duration_ms'] ?? null)) { + $stages[] = ['stage' => $item['stage'], 'label' => self::STAGES[$item['stage']], + 'duration_ms' => max(0, min(86400000, $item['duration_ms']))]; + } + } + $status = (int) $record->http_status; + return ['attempt_id' => (string) $record->attempt_id, 'host_id' => (int) $record->host_id, + 'host_name' => $hostName, 'source' => $record->source === 'manual' ? 'manual' : 'auto', + 'source_label' => $record->source === 'manual' ? '管理员手动同步' : '自动盘点', + 'started_at' => (int) $record->started_at, 'finished_at' => self::positiveTime($record->finished_at), + 'duration_ms' => max(0, (int) $record->duration_ms), 'duration_is_final' => $record->finished_at !== null, + 'outcome' => $outcome, 'outcome_label' => self::OUTCOMES[$outcome], + 'stage' => $stage, 'stage_label' => self::STAGES[$stage], 'last_confirmed_stage' => $stage, + 'error_code' => $error, 'error_message' => $error === '' ? '' : self::ERRORS[$error], + 'http_status' => $status >= 100 && $status <= 599 ? $status : null, + 'applied' => $outcome === 'succeeded' && (bool) $record->applied, + 'timeout_ms' => max(0, (int) $record->timeout_ms), 'panel_version' => self::version((string) $record->panel_version), + 'php_version' => self::version((string) $record->php_version), 'generation' => max(0, (int) $record->generation), + 'stages' => $stages, 'interrupted' => $interrupted || $outcome === 'interrupted']; + } + + public static function filters(array $input): array + { + foreach ($input as $key => $value) { + if (!in_array($key, ['host_id', 'outcome', 'page', 'per_page'], true) + || (!is_string($value) && !is_int($value))) { + throw new CloudError('盘点诊断筛选参数无效', 400); + } + } + $outcome = (string) ($input['outcome'] ?? 'all'); + if (!in_array($outcome, array_merge(['all', 'attention'], array_keys(self::OUTCOMES)), true)) { + throw new CloudError('盘点结果筛选无效', 400); + } + $host = (string) ($input['host_id'] ?? ''); + $page = self::positiveId((string) ($input['page'] ?? '1')); + $perPage = self::positiveId((string) ($input['per_page'] ?? '20')); + if (!in_array($perPage, [20, 50, 100], true)) { + throw new CloudError('每页条数只能为 20、50 或 100', 400); + } + return ['host_id' => $host === '' ? null : (string) self::positiveId($host), + 'outcome' => $outcome, 'page' => $page, 'per_page' => $perPage]; + } + + private static function positiveId(string $value): int + { + if (preg_match('/^[1-9][0-9]{0,17}$/D', $value) !== 1) { + throw new CloudError('编号或页码无效', 400); + } + return (int) $value; + } + + private static function authorize(array $actor): void + { + if (!($actor['is_admin'] ?? false)) { + throw new CloudError('需要管理员权限', 403); + } + } + + private static function errorCode(string $value): string + { + return $value === '' || isset(self::ERRORS[$value]) ? $value : 'unexpected_error'; + } + + private static function version(string $value): string + { + return strlen($value) <= 64 && preg_match('/^[A-Za-z0-9][A-Za-z0-9._+-]*$/D', $value) === 1 + ? $value : 'unknown'; + } + + private static function positiveTime($value): ?int + { + return $value === null || (int) $value <= 0 ? null : (int) $value; + } + + private static function text(string $value, int $limit): string + { + return mb_substr(preg_replace('/[\x00-\x1f\x7f]/u', '', $value) ?? '', 0, $limit, 'UTF-8'); + } +} diff --git a/web/app/Services/Cloud/CloudHostInventorySyncService.php b/web/app/Services/Cloud/CloudHostInventorySyncService.php index c687f779..2289d4b5 100644 --- a/web/app/Services/Cloud/CloudHostInventorySyncService.php +++ b/web/app/Services/Cloud/CloudHostInventorySyncService.php @@ -85,7 +85,7 @@ final class CloudHostInventorySyncService foreach ($hosts as $host) { $result['checked']++; - $this->pollHost((int) $host->id, $actor, $result); + $this->pollHost((int) $host->id, $actor, $result, 'auto'); } $result['errors'] = array_values(array_unique($result['errors'])); return $result; @@ -118,67 +118,88 @@ final class CloudHostInventorySyncService return $result; } $result['checked'] = 1; - $this->pollHost($hostId, $actor, $result); + $this->pollHost($hostId, $actor, $result, 'manual'); $result['errors'] = array_values(array_unique($result['errors'])); return $result; } - private function pollHost(int $hostId, array $actor, array &$result): void + private function pollHost(int $hostId, array $actor, array &$result, + string $source): void { // This local lower bound is authoritative for comparing database // reservations with the engine snapshot. The engine's observed_at is // retained for diagnostics only because host clocks may legitimately // differ from the PHP/database clock. $attemptedAt = $this->now(); + $diagnostics = CloudHostInventoryDiagnostics::start( + $hostId, $source, $attemptedAt, $this->engine->inventoryTimeoutSeconds() + ); try { $attemptGeneration = $this->beginAttempt($hostId, $attemptedAt); } catch (\Throwable $error) { $result['errors'][] = 'host_inventory_sync_failed'; + $diagnostics->finish('failed', 'attempt_start_failed'); return; } if ($attemptGeneration === null) { $result['skipped']++; + $diagnostics->finish('skipped'); return; } + $diagnostics->generation($attemptGeneration); + $diagnostics->stage('capacity_request'); try { - $capacityPayload = $this->engine->host( - $hostId, 'GET', 'host/capacity', $actor + $capacityPayload = $this->engine->hostInventory( + $hostId, 'host/capacity', $actor, $diagnostics->id() ); + } catch (CloudInventoryRequestError $error) { + $this->failed($hostId, $this->engineFailure($error), $result, + $attemptedAt, $attemptGeneration, $diagnostics, + $error->diagnosticCode, $error->upstreamHttpStatus); + return; } catch (CloudError $error) { $this->failed($hostId, $this->engineFailure($error), $result, - $attemptedAt, $attemptGeneration); + $attemptedAt, $attemptGeneration, $diagnostics); return; } catch (\Throwable $error) { $this->failed($hostId, ['unavailable', self::ERROR_UNAVAILABLE], $result, - $attemptedAt, $attemptGeneration); + $attemptedAt, $attemptGeneration, $diagnostics); return; } + $diagnostics->stage('capacity_validate'); try { $capacity = self::capacity($capacityPayload, $this->now()); } catch (\UnexpectedValueException $error) { $this->failed($hostId, ['degraded', self::ERROR_INVALID_CAPACITY], $result, - $attemptedAt, $attemptGeneration); + $attemptedAt, $attemptGeneration, $diagnostics); return; } + $diagnostics->stage('network_request'); try { - $networkPayload = $this->engine->host( - $hostId, 'GET', 'host/network-options', $actor + $networkPayload = $this->engine->hostInventory( + $hostId, 'host/network-options', $actor, $diagnostics->id() ); + } catch (CloudInventoryRequestError $error) { + $this->failed($hostId, $this->engineFailure($error), $result, + $attemptedAt, $attemptGeneration, $diagnostics, + $error->diagnosticCode, $error->upstreamHttpStatus); + return; } catch (CloudError $error) { $this->failed($hostId, $this->engineFailure($error), $result, - $attemptedAt, $attemptGeneration); + $attemptedAt, $attemptGeneration, $diagnostics); return; } catch (\Throwable $error) { $this->failed($hostId, ['unavailable', self::ERROR_UNAVAILABLE], $result, - $attemptedAt, $attemptGeneration); + $attemptedAt, $attemptGeneration, $diagnostics); return; } + $diagnostics->stage('network_validate'); try { $network = self::network($networkPayload); [$capacity, $pools] = self::reconcilePublicInventory( @@ -187,10 +208,11 @@ final class CloudHostInventorySyncService } catch (\UnexpectedValueException $error) { $this->failed($hostId, ['degraded', self::ERROR_INVALID_NETWORK], $result, - $attemptedAt, $attemptGeneration); + $attemptedAt, $attemptGeneration, $diagnostics); return; } + $diagnostics->stage('database_apply'); try { $applied = $this->apply( $hostId, $capacity, $pools, $attemptedAt, @@ -198,32 +220,39 @@ final class CloudHostInventorySyncService ); if (!$applied) { $result['skipped']++; + $diagnostics->finish('superseded'); return; } $result['succeeded']++; + $diagnostics->finish('succeeded', '', null, true); } catch (\UnexpectedValueException $error) { $this->failed($hostId, ['degraded', self::ERROR_INVALID_NETWORK], $result, - $attemptedAt, $attemptGeneration); + $attemptedAt, $attemptGeneration, $diagnostics); } catch (\DomainException $error) { $this->failed($hostId, ['degraded', self::ERROR_INVENTORY_CONFLICT], $result, - $attemptedAt, $attemptGeneration); + $attemptedAt, $attemptGeneration, $diagnostics); } catch (\Throwable $error) { $this->failed($hostId, ['degraded', self::ERROR_WRITE_FAILED], $result, - $attemptedAt, $attemptGeneration); + $attemptedAt, $attemptGeneration, $diagnostics); } } private function failed(int $hostId, array $failure, array &$result, - int $attemptedAt, int $attemptGeneration): void + int $attemptedAt, int $attemptGeneration, + CloudHostInventoryDiagnostics $diagnostics, string $diagnosticCode = '', + ?int $upstreamStatus = null): void { [$health, $code] = $failure; + $diagnosticCode = $diagnosticCode === '' ? $code : $diagnosticCode; + $outcome = $diagnosticCode === 'engine_http_timeout' ? 'timeout' : 'failed'; try { if (!$this->markFailure($hostId, $health, $code, $attemptedAt, $attemptGeneration)) { $result['skipped']++; + $diagnostics->finish('superseded', $diagnosticCode, $upstreamStatus); return; } $result[$health]++; @@ -234,8 +263,9 @@ final class CloudHostInventorySyncService // The lifecycle pass must continue to other hosts and billing // policy. Never log an upstream error body from this boundary. error_log('cloud host inventory failure marker unavailable: ' - . $code . ' host=' . $hostId); + . $code . ' host=' . $hostId . ' attempt=' . $diagnostics->id()); } + $diagnostics->finish($outcome, $diagnosticCode, $upstreamStatus); } /** Claim a monotonic per-host observation generation before network I/O. */ @@ -265,7 +295,7 @@ final class CloudHostInventorySyncService ); } - private function engineFailure(CloudError $error): array + private function engineFailure(CloudError|CloudInventoryRequestError $error): array { if (in_array($error->httpStatus, [408, 429, 502, 503, 504], true)) { return ['unavailable', self::ERROR_UNAVAILABLE]; diff --git a/web/app/Services/Cloud/CloudInventoryRequestError.php b/web/app/Services/Cloud/CloudInventoryRequestError.php new file mode 100644 index 00000000..0784224a --- /dev/null +++ b/web/app/Services/Cloud/CloudInventoryRequestError.php @@ -0,0 +1,15 @@ +group('/admin', function () { $this->get('/resources', 'App\Controllers\Admin\CloudPageController:resources'); // Host preparation (admin only; backend integration is not enabled yet) $this->get('/hosts', 'App\Controllers\Admin\CloudPageController:hosts'); + $this->get('/host-inventory', 'App\Controllers\Admin\CloudPageController:hostInventory'); $this->get('/cloud/templates', 'App\Controllers\Admin\CloudPageController:templates'); $this->get('/cloud/rescue', 'App\Controllers\Admin\CloudPageController:rescue'); $this->get('/cloud/instances', 'App\Controllers\Admin\CloudPageController:instances'); diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index e8016221..933d9b65 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -68,6 +68,11 @@ $app->group('/api/v1', function () { $this->get('/admin/operations', 'App\Controllers\CloudAdminOperationsController:index'); $this->get('/admin/operations/health', 'App\Controllers\CloudAdminRuntimeController:index'); $this->get('/admin/operations/jobs/{id:[0-9]+}', 'App\Controllers\CloudAdminOperationsController:detail'); + $this->get('/admin/host-inventory', 'App\Controllers\CloudAdminHostInventoryController:index'); + $this->get('/admin/host-inventory/{id:[0-9a-f]{32}}', + 'App\Controllers\CloudAdminHostInventoryController:detail'); + $this->get('/admin/host-inventory/{id:[0-9a-f]{32}}/engine', + 'App\Controllers\CloudAdminHostInventoryController:engine'); $this->get('/admin/billing/invoices', 'App\Controllers\CloudAdminBillingController:invoices'); $this->get('/admin/billing/invoices/{id:[0-9]+}', diff --git a/web/public/cloud/admin-host-inventory.js b/web/public/cloud/admin-host-inventory.js new file mode 100644 index 00000000..87511e4c --- /dev/null +++ b/web/public/cloud/admin-host-inventory.js @@ -0,0 +1,312 @@ +const root = document.querySelector('[data-admin-host-inventory]'); +const byId = id => document.getElementById(id); +const filterNames = ['host_id', 'outcome', 'per_page']; +const columns = ['宿主机 / 来源', '开始 / 结束', '耗时 / 预算', '面板结果', '阶段 / 错误码', '容量生效', '操作']; +const stageLabels = { + request: '引擎接收请求', registry_profile: '解析宿主机配置', + update_lock_wait: '等待全局锁', update_check: '检查在线更新状态', + iso_gc: '清理 ISO 待删除资源', template_gc: '清理镜像待删除资源', + profile: '装载宿主机凭据', libvirt_connect: '连接 libvirt', host_info: '读取宿主机资源', + domain_list: '枚举虚拟机', domain_scan: '扫描虚拟机', storage_scan: '读取存储池', + public_pool_catalog: '读取公网池目录', public_ip_claims: '读取公网 IP 占用', + public_ip_scan: '盘点公网 IP', response: '返回响应', +}; +const traceStates = { running: '进行中', succeeded: '采集成功', failed: '失败', interrupted: '中断 / 未知' }; +const state = { page: null, url: null, request: null, loading: false, + detailId: null, detailRequest: null, engineRequest: null, closing: false, scroll: null, opener: null }; + +function el(tag, className = '', value) { + const node = document.createElement(tag); node.className = className; + if (value !== undefined) node.textContent = String(value); + return node; +} +function text(value, fallback = '未记录') { + return value === null || value === undefined || String(value).trim() === '' ? fallback : String(value); +} +function date(value) { + if (!value || !Number.isFinite(Number(value))) return '尚未记录'; + const parsed = new Date(Number(value) * 1000); + if (!Number.isFinite(parsed.getTime())) return '时间无效'; + // Match the server-rendered rows even when the administrator's browser uses + // another time zone. Elapsed durations still come from each process's clock. + return parsed.toLocaleString('zh-CN', { hour12: false, timeZone: root?.dataset.timezone || 'UTC' }); +} +function duration(value) { + return value === null || value === undefined || !Number.isFinite(Number(value)) ? '未确认' : `${Number(value).toLocaleString('zh-CN')} ms`; +} +function applied(attempt) { + if (attempt.applied === true) return '容量已更新'; + if (attempt.outcome === 'superseded') return '结果已被取代'; + if (attempt.outcome === 'interrupted') return '是否生效待核对'; + return attempt.outcome === 'running' ? '尚未确认' : '容量未更新'; +} +function stack(primary, ...secondary) { + const node = el('span', 'admin-checkout-user'); + node.append(primary instanceof Node ? primary : el('strong', '', primary)); + for (const value of secondary.filter(Boolean)) node.append(value instanceof Node ? value : el('span', '', value)); + return node; +} +function infoList(entries) { + const list = el('dl', 'admin-instance-details'); + for (const [label, value] of entries) list.append(el('dt', '', label), el('dd', 'text-break', text(value))); + return list; +} +function notice(message, error = false) { + const node = byId('inventory-notice'); node.textContent = message; + node.className = `admin-config-status ${error ? 'is-error' : 'is-success'} mb-3`; + node.setAttribute('role', error ? 'alert' : 'status'); +} +function detailNotice(message, error = false) { + const node = byId('inventory-detail-state'); node.textContent = message; node.hidden = !message; + node.className = `alert alert-${error ? 'danger' : 'info'}`; + node.setAttribute('role', error ? 'alert' : 'status'); +} +async function api(path, controller, timeoutMs = 15000) { + const timer = setTimeout(() => controller.abort('timeout'), timeoutMs); + try { + const response = await fetch(path, { headers: { Accept: 'application/json' }, + credentials: 'same-origin', cache: 'no-store', signal: controller.signal }); + if (response.status === 401) throw new Error('登录已过期,请重新登录。'); + if (response.status === 403) throw new Error('当前账号没有查看权限。'); + let payload; + try { payload = await response.json(); } catch { throw new Error('服务器响应格式无效,请稍后重试。'); } + if (!response.ok) throw new Error(typeof payload?.detail === 'string' ? payload.detail : '诊断记录暂不可读取,请稍后重试。'); + return payload; + } catch (error) { + if (controller.signal.reason === 'timeout') throw new Error('读取诊断超时;未触发新的盘点。'); + throw error; + } finally { clearTimeout(timer); } +} +function listUrl(input = new URL(location.href)) { + const url = new URL('/admin/host-inventory', location.origin); + for (const key of [...filterNames, 'page']) { + const value = input.searchParams.get(key); + if (value) url.searchParams.set(key, value); + } + return url; +} +function readForm(url) { + const form = byId('inventory-filter'); + for (const key of filterNames) { + const control = form.elements.namedItem(key); + const value = url.searchParams.get(key) || ({ outcome: 'all', per_page: '20' })[key] || ''; + if (value && !Array.from(control.options).some(option => option.value === value)) control.add(new Option(value, value)); + control.value = value; + } +} +function formUrl() { + const url = new URL('/admin/host-inventory', location.origin); + for (const key of filterNames) { + const value = byId('inventory-filter').elements.namedItem(key).value; + if (value) url.searchParams.set(key, value); + } + return url; +} +function pagination() { + const page = state.page?.pagination; + byId('inventory-count').textContent = page ? `共 ${page.total} 条` : '数据未获取'; + byId('inventory-pagination').hidden = !page?.total; + byId('inventory-page-label').textContent = page ? `第 ${page.page} / ${page.total_pages} 页 · 共 ${page.total} 条` : '记录暂不可用'; + byId('inventory-previous').disabled = state.loading || !page || page.page <= 1; + byId('inventory-next').disabled = state.loading || !page || page.page >= page.total_pages; + byId('inventory-refresh').disabled = state.loading; + byId('inventory-body').setAttribute('aria-busy', String(state.loading)); +} +function renderRows() { + const body = byId('inventory-body'); body.replaceChildren(); + for (const attempt of state.page.items) { + const row = el('tr'); const button = el('button', 'btn btn-primary btn-sm', '查看详情'); + button.type = 'button'; button.dataset.inventoryDetail = String(attempt.attempt_id); + const statusClass = attempt.outcome === 'succeeded' ? 'text-success' + : (['failed', 'timeout'].includes(attempt.outcome) ? 'text-danger' : (attempt.outcome === 'interrupted' ? 'text-warning' : '')); + const cells = [ + stack(`${text(attempt.host_name, '宿主机已移除')} · #${attempt.host_id}`, attempt.source_label), + stack(date(attempt.started_at), attempt.finished_at ? date(attempt.finished_at) : (attempt.outcome === 'running' ? '尚未结束' : '结束时间未确认')), + stack(`${attempt.duration_is_final === false ? '已记录 ' : ''}${duration(attempt.duration_ms)}`, `每请求 ${duration(attempt.timeout_ms)}`), + stack(el('strong', statusClass, text(attempt.outcome_label)), attempt.error_message ? el('span', 'text-danger', attempt.error_message) : ''), + stack(text(attempt.stage_label, '阶段未确认'), text(attempt.error_code, '未记录错误码')), + applied(attempt), button, + ]; + cells.forEach((value, index) => { + const cell = el('td'); cell.dataset.label = columns[index]; + cell.append(value instanceof Node ? value : el('span', '', value)); row.append(cell); + }); + body.append(row); + } + if (!state.page.items.length) { + const row = el('tr'); const cell = el('td', 'text-center text-muted py-4', '没有符合条件的记录;新功能只记录启用后的盘点尝试。'); + cell.colSpan = columns.length; row.append(cell); body.append(row); + } +} +function applyPage(page, replaceRows = true) { + if (!Array.isArray(page?.items) || !page.pagination || !page.filters || !page.summary) throw new Error('盘点记录格式无效,请重试。'); + state.page = page; + const url = new URL('/admin/host-inventory', location.origin); + for (const key of [...filterNames, 'page']) { + const value = ['page', 'per_page'].includes(key) ? page.pagination[key] : page.filters[key]; + if (value !== null && value !== undefined && value !== '') url.searchParams.set(key, String(value)); + } + state.url = url; + const host = byId('inventory-host'); host.replaceChildren(new Option('全部宿主机', '')); + for (const item of page.hosts || []) host.add(new Option(`${item.name} · #${item.id}`, item.id)); + readForm(url); + for (const key of ['last_attempt_at', 'last_success_at', 'last_failure_at']) byId(`inventory-summary-${key}`).textContent = date(page.summary[key]); + byId('inventory-summary-state').textContent = `记录中的进行中:${page.summary.in_progress} 次。` + + (page.filters.host_id ? `该宿主机连续失败:${page.summary.consecutive_failures} 次。` : '选择宿主机可查看连续失败次数。'); + if (replaceRows) renderRows(); + pagination(); +} +async function loadPage(url, historyMode = 'push') { + state.request?.abort(); const controller = new AbortController(); state.request = controller; + state.loading = true; pagination(); notice('正在读取数据库记录…'); + try { + const payload = await api(`/api/v1/admin/host-inventory${url.search}`, controller); + if (controller.signal.aborted || state.request !== controller) return; + applyPage(payload); + if (historyMode === 'push') history.pushState(history.state, '', state.url); + notice(`已读取 ${payload.pagination.total} 条盘点记录;未执行宿主机扫描。`); + } catch (error) { + if (state.request !== controller || (controller.signal.aborted && controller.signal.reason !== 'timeout')) return; + notice(`${error.message}${state.page ? ' 下方保留上次成功读取的记录。' : ''}`, true); + } finally { + if (state.request === controller) { state.request = null; state.loading = false; pagination(); } + } +} +function renderStages(stages, engine = false) { + const list = el('ul', 'list-unstyled mb-0'); + for (const item of (Array.isArray(stages) ? stages : [])) { + const label = engine ? (stageLabels[item.stage] || text(item.stage, '阶段未确认')) : text(item.label, item.stage); + const status = engine ? ` · ${traceStates[item.state] || text(item.state)}` : ''; + list.append(el('li', 'text-break py-1', `${label}${status} · ${duration(item.duration_ms)}`)); + } + if (!list.children.length) list.append(el('li', 'text-muted', '未记录阶段耗时。')); + return list; +} +function renderDetail(payload) { + const attempt = payload?.attempt; + if (!attempt || String(attempt.attempt_id) !== state.detailId) throw new Error('记录与请求不一致,请重新打开。'); + byId('inventory-detail-title').textContent = `${text(attempt.host_name, '宿主机已移除')} · ${text(attempt.outcome_label)}`; + const body = byId('inventory-detail-body'); + body.replaceChildren(el('h3', 'h6', '面板同步结果'), infoList([ + ['尝试编号', attempt.attempt_id], ['宿主机', `${text(attempt.host_name)} · #${attempt.host_id}`], + ['来源', attempt.source_label], ['面板结果', attempt.outcome_label], ['容量是否生效', applied(attempt)], + ['开始时间', date(attempt.started_at)], ['结束时间', attempt.finished_at ? date(attempt.finished_at) : '尚未确认'], + [attempt.duration_is_final === false ? '已记录耗时(非最终)' : '总耗时', duration(attempt.duration_ms)], ['每个引擎请求预算', duration(attempt.timeout_ms)], + ['面板记录阶段', attempt.stage_label], ['错误码', attempt.error_code], + ['脱敏说明', text(attempt.error_message, '未记录错误')], ['引擎 HTTP 状态', attempt.http_status], + ['面板诊断版本', attempt.panel_version], ['PHP 版本', attempt.php_version], ['同步代际', attempt.generation], + ]), el('h3', 'h6 mt-4', 'PHP 阶段耗时'), renderStages(attempt.stages)); + for (const note of payload.notes || []) body.append(el('p', 'text-muted fs-13 mt-3 mb-0', note)); + byId('inventory-engine').hidden = false; + byId('inventory-engine-refresh').disabled = false; +} +function renderEvidence(payload) { + if (!payload || payload.attempt_id !== state.detailId || !Array.isArray(payload.traces)) throw new Error('引擎线索与请求不一致。'); + const box = byId('inventory-engine-body'); box.replaceChildren(); + byId('inventory-engine-state').textContent = text(payload.message, + payload.available ? '已读取引擎记录;不改变面板同步结果。' : '无法取得引擎线索,内部阶段尚未确认。'); + if (!payload.available) { + if (payload.error_code) box.append(el('p', 'text-muted text-break', `诊断查询错误码:${payload.error_code}`)); + return; + } + if (!payload.traces.length) { box.append(el('p', 'text-muted', '没有对应引擎记录:可能尚未接收、未启用记录或已过保留期;不能据此认定宿主机故障。')); return; } + for (const trace of payload.traces) { + const section = el('section', 'border-top pt-3 mt-3'); + section.append(el('h4', 'h6', ({ capacity: '容量请求', network: '公网网络请求' })[trace.resource] || '引擎请求'), infoList([ + ['引擎结果(独立)', traceStates[trace.state] || text(trace.state)], + ['最后确认阶段', stageLabels[trace.stage] || text(trace.stage, '尚未确认')], + ['引擎开始时间', date(trace.started_at)], ['引擎结束时间', trace.finished_at ? date(trace.finished_at) : '尚未确认'], + ['引擎耗时', duration(trace.duration_ms)], ['引擎 HTTP 状态', trace.http_status], + ['引擎错误码', trace.error_code], ['引擎运行版本', trace.engine_version], + ]), renderStages(trace.stages, true)); + box.append(section); + } + box.append(el('p', 'text-muted fs-13 mt-3', '各进程独立计时,不能直接相减面板与引擎的时间。引擎父阶段包含子阶段耗时,不能相加计算总时长。只有阶段开始记录时,不能将该阶段直接认定为故障根因。')); +} +async function readEvidence() { + if (!state.detailId || state.engineRequest || state.closing) return; + const id = state.detailId; const controller = new AbortController(); state.engineRequest = controller; + byId('inventory-engine-refresh').disabled = true; + byId('inventory-engine-state').textContent = '正在读取引擎已有记录,不会扫描宿主机…'; + byId('inventory-engine-body').replaceChildren(); + try { + const payload = await api(`/api/v1/admin/host-inventory/${id}/engine`, controller, 5000); + if (controller.signal.aborted || state.detailId !== id || state.engineRequest !== controller) return; + renderEvidence(payload); + } catch (error) { + if (state.detailId !== id || state.engineRequest !== controller || (controller.signal.aborted && controller.signal.reason !== 'timeout')) return; + byId('inventory-engine-state').textContent = `${error.message} 引擎内部阶段尚未确认。`; + } finally { + if (state.engineRequest === controller) { state.engineRequest = null; byId('inventory-engine-refresh').disabled = false; } + } +} +async function openDetail(id, opener) { + if (!/^[0-9a-f]{32}$/.test(id) || state.closing) return; + state.detailRequest?.abort(); state.engineRequest?.abort(); state.engineRequest = null; + state.detailId = id; state.opener = opener; + const controller = new AbortController(); state.detailRequest = controller; + byId('inventory-detail-title').textContent = '盘点诊断'; byId('inventory-detail-body').replaceChildren(); + byId('inventory-engine').hidden = true; byId('inventory-engine-body').replaceChildren(); + byId('inventory-engine-state').textContent = '尚未查询。仅读取已存在的线索,不会发起盘点。'; + detailNotice('正在读取数据库记录…'); + if (!byId('inventory-detail').open) { + state.scroll = { x: window.scrollX, y: window.scrollY }; + document.body.style.setProperty('--admin-instance-scroll-top', `${-state.scroll.y}px`); + document.documentElement.classList.add('admin-instance-detail-open'); + window.VenixDrawerMotion.openDrawerWithMotion(byId('inventory-detail')); + } + try { + const payload = await api(`/api/v1/admin/host-inventory/${id}`, controller); + if (controller.signal.aborted || state.detailId !== id || state.detailRequest !== controller) return; + renderDetail(payload); detailNotice(''); + } catch (error) { + if (state.detailRequest !== controller || (controller.signal.aborted && controller.signal.reason !== 'timeout')) return; + detailNotice(`${error.message} 请关闭后重试。`, true); + } finally { if (state.detailRequest === controller) state.detailRequest = null; } +} +function closeDetail() { + if (state.closing || !byId('inventory-detail').open) return; + state.detailRequest?.abort(); state.engineRequest?.abort(); + state.detailRequest = null; state.engineRequest = null; state.detailId = null; state.closing = true; + void window.VenixDrawerMotion.closeDrawerWithMotion(byId('inventory-detail')).then(closed => { + state.closing = false; if (!closed) return; + document.documentElement.classList.remove('admin-instance-detail-open'); + document.body.style.removeProperty('--admin-instance-scroll-top'); + if (state.scroll) window.scrollTo({ left: state.scroll.x, top: state.scroll.y, behavior: 'instant' }); + state.scroll = null; if (state.opener?.isConnected) state.opener.focus({ preventScroll: true }); state.opener = null; + }); +} + +if (root) { + try { + const payload = JSON.parse(byId('admin-inventory-bootstrap').textContent); + if (payload) applyPage(payload, false); else readForm(listUrl()); + } catch { notice('首屏诊断记录无效,请点击刷新记录重试。', true); readForm(listUrl()); } + byId('inventory-filter').addEventListener('submit', event => { event.preventDefault(); void loadPage(formUrl()); }); + byId('inventory-reset').addEventListener('click', event => { + if (event.button !== 0 || event.ctrlKey || event.metaKey || event.altKey || event.shiftKey) return; + event.preventDefault(); void loadPage(new URL('/admin/host-inventory', location.origin)); + }); + byId('inventory-refresh').addEventListener('click', () => { void loadPage(listUrl(), 'none'); }); + for (const [id, delta] of [['inventory-previous', -1], ['inventory-next', 1]]) byId(id).addEventListener('click', () => { + if (!state.page || state.loading) return; + const url = new URL(state.url); url.searchParams.set('page', state.page.pagination.page + delta); void loadPage(url); + }); + root.addEventListener('click', event => { + const button = event.target.closest('[data-inventory-detail]'); + if (button) void openDetail(button.dataset.inventoryDetail, button); + }); + byId('inventory-detail-close').addEventListener('click', closeDetail); + byId('inventory-engine-refresh').addEventListener('click', () => { void readEvidence(); }); + const dialog = byId('inventory-detail'); + dialog.addEventListener('cancel', event => { event.preventDefault(); closeDetail(); }); + dialog.addEventListener('click', event => { + if (event.target !== dialog) return; + const rect = dialog.getBoundingClientRect(); + if (event.clientX < rect.left || event.clientX > rect.right || event.clientY < rect.top || event.clientY > rect.bottom) closeDetail(); + }); + window.addEventListener('popstate', () => { closeDetail(); readForm(listUrl()); void loadPage(listUrl(), 'none'); }); + window.addEventListener('pagehide', () => { state.request?.abort(); state.detailRequest?.abort(); state.engineRequest?.abort(); closeDetail(); }); + pagination(); + // Intentionally no initial fetch, timer, visibility refresh or automatic engine query. +} diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js index 1f0cac26..80bea276 100644 --- a/web/public/cloud/admin.js +++ b/web/public/cloud/admin.js @@ -98,7 +98,9 @@ function renderHosts() { scheduling.type = 'button'; scheduling.addEventListener('click', () => openHostScheduling(host.id, scheduling)); const networkPools = element('button', 'btn btn-outline-primary btn-sm', '公网地址池'); networkPools.type = 'button'; networkPools.addEventListener('click', () => openHostNetworkPools(host.id, networkPools)); - actions.append(scheduling, networkPools); row.append(button, actions); list.append(row); + const inventory = element('a', 'btn btn-outline-secondary btn-sm', '盘点诊断'); + inventory.href = `/admin/host-inventory?host_id=${encodeURIComponent(host.id)}`; + actions.append(scheduling, networkPools, inventory); row.append(button, actions); list.append(row); } } function schedulerLabel(value) { return ({ active: '调度启用', draining: '正在排空', disabled: '调度停用' })[value] || '调度未配置'; } diff --git a/web/resources/views/venix/admin/cloud/host_inventory.tpl b/web/resources/views/venix/admin/cloud/host_inventory.tpl new file mode 100644 index 00000000..ccec80e2 --- /dev/null +++ b/web/resources/views/venix/admin/cloud/host_inventory.tpl @@ -0,0 +1,42 @@ +{assign var=admin_title value='宿主机盘点诊断'} +{assign var=admin_active value='host-inventory'} +{include file='admin/static_header.tpl'} +{assign var=inventory_filters value=$inventory_page.filters|default:[]} +{assign var=inventory_outcomes value=['all'=>'全部结果','attention'=>'失败 / 超时 / 中断','running'=>'进行中','succeeded'=>'成功','failed'=>'失败','timeout'=>'超时','interrupted'=>'中断 / 未知','skipped'=>'跳过','superseded'=>'已被更新尝试取代']} +
+
+

宿主机盘点诊断

逐次查看自动与手动盘点的结果、失败阶段和耗时。刷新此页只读取记录,不会扫描宿主机。

+
宿主机设置
+
+

{if $inventory_error|default:''}{$inventory_error|escape}{else}已读取 {$inventory_page.pagination.total|default:0|escape} 条盘点记录{/if}

+
+ {foreach ['last_attempt_at'=>'最近尝试','last_success_at'=>'最近成功','last_failure_at'=>'最近失败'] as $key=>$label} +

{$label}

{if !$inventory_page}—{elseif $inventory_page.summary[$key]}{$inventory_page.summary[$key]|date_format:'%Y-%m-%d %H:%M:%S'|escape}{else}尚未记录{/if}
+ {/foreach} +

{if !$inventory_page}盘点摘要暂不可用。{else}记录中的进行中:{$inventory_page.summary.in_progress|default:0|escape} 次。{if $inventory_filters.host_id|default:''}该宿主机连续失败:{$inventory_page.summary.consecutive_failures|default:0|escape} 次。{else}选择宿主机可查看连续失败次数。{/if}{/if}

+
+
+

筛选记录

摘要按所选宿主机统计,不受结果筛选影响。时间采用面板时区 {$inventory_timezone|default:'UTC'|escape};历史记录有保留上限。

+
+ + + +
清除筛选
+
+
+
+

盘点尝试

成功表示面板校验与容量提交已完成;引擎晚到的成功不等于容量已更新。

{if $inventory_page}共 {$inventory_page.pagination.total|escape} 条{else}数据未获取{/if}
+
{foreach ['宿主机 / 来源','开始 / 结束','耗时 / 预算','面板结果','阶段 / 错误码','容量生效','操作'] as $column}{/foreach}{include file='admin/cloud/host_inventory_rows.tpl'}
管理员宿主机盘点历史
{$column}
+ +
+
+ +
+ +

盘点尝试详情

盘点诊断

+

正在读取记录…

+
最后确认阶段不是已证明的根因;引擎结果不会修改面板的同步结果。
+
+ + +{include file='admin/static_footer.tpl'} diff --git a/web/resources/views/venix/admin/cloud/host_inventory_rows.tpl b/web/resources/views/venix/admin/cloud/host_inventory_rows.tpl new file mode 100644 index 00000000..8416e980 --- /dev/null +++ b/web/resources/views/venix/admin/cloud/host_inventory_rows.tpl @@ -0,0 +1,11 @@ +{foreach $inventory_page.items|default:[] as $attempt} + + {$attempt.host_name|default:'宿主机已移除'|escape} · #{$attempt.host_id|escape}{$attempt.source_label|escape} + {$attempt.started_at|date_format:'%Y-%m-%d %H:%M:%S'|escape}{if $attempt.finished_at}{$attempt.finished_at|date_format:'%Y-%m-%d %H:%M:%S'|escape}{elseif $attempt.outcome == 'running'}尚未结束{else}结束时间未确认{/if} + {if !$attempt.duration_is_final|default:false}已记录 {/if}{if $attempt.duration_ms !== null}{$attempt.duration_ms|escape} ms{else}未确认{/if}每请求 {$attempt.timeout_ms|escape} ms + {$attempt.outcome_label|escape}{if $attempt.error_message}{$attempt.error_message|escape}{/if} + {$attempt.stage_label|default:'阶段未确认'|escape}{$attempt.error_code|default:'未记录错误码'|escape} + {if $attempt.applied}容量已更新{elseif $attempt.outcome == 'superseded'}结果已被取代{elseif $attempt.outcome == 'interrupted'}是否生效待核对{elseif $attempt.outcome == 'running'}尚未确认{else}容量未更新{/if} + + +{foreachelse}{if $inventory_error|default:''}盘点记录暂不可读取,请确认数据库已升级后重试。{else}没有符合条件的记录;新功能只记录启用后的盘点尝试。{/if}{/foreach} diff --git a/web/resources/views/venix/admin/cloud/operations.tpl b/web/resources/views/venix/admin/cloud/operations.tpl index 46ac4a26..ffda1686 100644 --- a/web/resources/views/venix/admin/cloud/operations.tpl +++ b/web/resources/views/venix/admin/cloud/operations.tpl @@ -4,6 +4,7 @@ {assign var=operations_filters value=$operations_page.filters|default:[]} {assign var=operations_statuses value=['attention'=>'需要关注','all'=>'全部状态','queued'=>'已排队','running'=>'执行中','uncertain'=>'待核对','failed'=>'失败','succeeded'=>'已完成']}
+

查看宿主机盘点诊断

系统运行与异常任务

查看后台运行状态与全局任务记录;仅供核对,不会重试、取消任务或触发退款。

{if $operations_error|default:''}{$operations_error|escape}{else}已读取 {$operations_page.pagination.total|default:0|escape} 条任务{/if}
diff --git a/web/resources/views/venix/admin/hosts/index.tpl b/web/resources/views/venix/admin/hosts/index.tpl index a4006a1a..dc57c7e3 100644 --- a/web/resources/views/venix/admin/hosts/index.tpl +++ b/web/resources/views/venix/admin/hosts/index.tpl @@ -11,6 +11,7 @@

管理计算节点连接、存储目录和公网网络配置。

+ 盘点诊断

正在加载…

diff --git a/web/resources/views/venix/admin/static_header.tpl b/web/resources/views/venix/admin/static_header.tpl index 271b5adb..a9eb72bb 100644 --- a/web/resources/views/venix/admin/static_header.tpl +++ b/web/resources/views/venix/admin/static_header.tpl @@ -163,6 +163,7 @@
  • 救援模版
  • 虚拟机管理
  • 系统运行与异常任务
  • +
  • 宿主机盘点诊断
  • {if $admin_can_manage|default:false}
  • 在线更新
  • {/if} diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 38f4cf92..429803a8 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026100401' scope='global'} +{assign var=venix_asset_version value='2026100402' scope='global'} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 3694c12f..3a64e711 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -247,6 +247,32 @@ CREATE TABLE IF NOT EXISTS cloud_host_capacity_snapshots ( KEY cloud_host_capacity_sync (sync_state, last_attempt_at, host_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +-- Independent, bounded diagnostic history. These rows never authorize placement +-- and an engine finishing after HTTP timeout cannot rewrite the panel outcome. +CREATE TABLE IF NOT EXISTS cloud_host_inventory_attempts ( + attempt_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL PRIMARY KEY, + sequence_id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE, + host_id BIGINT UNSIGNED NOT NULL, + source VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + generation BIGINT UNSIGNED NOT NULL DEFAULT 0, + started_at BIGINT UNSIGNED NOT NULL, + finished_at BIGINT UNSIGNED DEFAULT NULL, + updated_at BIGINT UNSIGNED NOT NULL, + duration_ms BIGINT UNSIGNED NOT NULL DEFAULT 0, + outcome VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'running', + stage VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'admission', + error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + http_status SMALLINT UNSIGNED DEFAULT NULL, + applied TINYINT(1) NOT NULL DEFAULT 0, + timeout_ms INT UNSIGNED NOT NULL, + panel_version VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + php_version VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + stages_json TEXT NOT NULL, + KEY cloud_inventory_host_time (host_id, started_at, sequence_id), + KEY cloud_inventory_outcome_time (outcome, started_at, sequence_id), + KEY cloud_inventory_retention (started_at, sequence_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + -- PHP reserves one concrete address before a paid provisioning job is queued. -- The private engine still validates the claim on the selected host. An -- uncertain operation keeps the address unavailable until reconciliation. diff --git a/web/tests/AdminHostInventoryBehaviorTest.mjs b/web/tests/AdminHostInventoryBehaviorTest.mjs new file mode 100644 index 00000000..7ddaddac --- /dev/null +++ b/web/tests/AdminHostInventoryBehaviorTest.mjs @@ -0,0 +1,171 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import vm from 'node:vm'; + +const source = await readFile(new URL('../public/cloud/admin-host-inventory.js', import.meta.url), 'utf8'); +class Node { + constructor(tag = 'div') { + this.tagName = tag.toUpperCase(); this.children = []; this.dataset = {}; this.attributes = new Map(); + this.listeners = new Map(); this.hidden = false; this.disabled = false; this.isConnected = true; + this.open = false; this.value = ''; this._text = ''; this.style = { setProperty() {}, removeProperty() {} }; + const classes = new Set(); this.classList = { add: x => classes.add(x), remove: x => classes.delete(x) }; + } + set textContent(value) { this._text = String(value); this.children = []; } + get textContent() { return this._text + this.children.map(item => item.textContent).join(''); } + set innerHTML(_) { throw new Error('Never interpret diagnostic content as HTML.'); } + get options() { return this.children; } + append(...values) { this.children.push(...values); } + add(option) { this.children.push(option); } + replaceChildren(...values) { this._text = ''; this.children = values; } + setAttribute(key, value) { this.attributes.set(key, String(value)); } + addEventListener(key, handler) { const listeners = this.listeners.get(key) || []; listeners.push(handler); this.listeners.set(key, listeners); } + dispatch(key, fields = {}) { + const event = { target: this, button: 0, defaultPrevented: false, preventDefault() { this.defaultPrevented = true; }, ...fields }; + for (const listener of this.listeners.get(key) || []) listener(event); + } + closest(selector) { return selector === '[data-inventory-detail]' && this.dataset.inventoryDetail ? this : null; } + focus() {} + getBoundingClientRect() { return { left: 0, right: 500, top: 0, bottom: 800 }; } +} +class Option extends Node { constructor(label, value) { super('option'); this.textContent = label; this.value = String(value); } } +const flush = async () => { for (let i = 0; i < 20; i += 1) await Promise.resolve(); }; +const firstId = 'a'.repeat(32); +const secondId = 'b'.repeat(32); +function attempt(id = firstId, changes = {}) { + return { attempt_id: id, host_id: 7, host_name: '示例宿主机', source: 'manual', source_label: '管理员手动同步', + started_at: 1234, finished_at: 1242, duration_ms: 8000, duration_is_final: true, + outcome: 'timeout', outcome_label: '超时', stage: 'capacity_request', stage_label: '请求容量数据', + error_code: 'engine_http_timeout', error_message: '超时;引擎阶段未确认', applied: false, + timeout_ms: 8000, http_status: null, panel_version: 'inventory-v1', php_version: '8.5.0', generation: 1, stages: [], ...changes }; +} +function page(items = [attempt()], pageNumber = 1) { + return { items, pagination: { page: pageNumber, per_page: 20, total: 21, total_pages: 2 }, + filters: { host_id: '7', outcome: 'all', page: pageNumber, per_page: 20 }, + hosts: [{ id: '7', name: '示例宿主机' }], + summary: { last_attempt_at: 1234, last_success_at: 1230, last_failure_at: 1242, in_progress: 0, consecutive_failures: 1 } }; +} +function fixture(initial = page()) { + const ids = [...source.matchAll(/byId\('([^']+)'\)/g)].map(match => match[1]); + ids.push('inventory-previous', 'inventory-next', 'inventory-outcome', 'inventory-per-page', ...['last_attempt_at', 'last_success_at', 'last_failure_at'].map(key => `inventory-summary-${key}`)); + const nodes = Object.fromEntries([...new Set(ids)].map(id => [id, new Node()])); + nodes['admin-inventory-bootstrap'].textContent = JSON.stringify(initial); + nodes['inventory-body'].textContent = 'SSR records'; + nodes['inventory-filter'].elements = { namedItem: key => nodes[({ host_id: 'inventory-host', outcome: 'inventory-outcome', per_page: 'inventory-per-page' })[key]] }; + const root = new Node(); const document = new Node(); document.documentElement = new Node(); document.body = new Node(); + document.getElementById = id => nodes[id]; document.createElement = tag => new Node(tag); + document.querySelector = selector => selector === '[data-admin-host-inventory]' ? root : null; + const window = new Node(); window.scrollX = 0; window.scrollY = 100; window.scrollTo = () => {}; + window.VenixDrawerMotion = { + openDrawerWithMotion: dialog => { dialog.open = true; }, + closeDrawerWithMotion: dialog => { dialog.open = false; return Promise.resolve(true); }, + }; + const location = { origin: 'https://example.test', href: 'https://example.test/admin/host-inventory?host_id=7' }; + const history = { state: null, pushState(value, _, url) { this.state = value; location.href = String(url); } }; + let now = 0; let lastTimer = 0; const timers = new Map(); const requests = []; + const setTimeout = (callback, delay) => { const id = ++lastTimer; timers.set(id, { callback, due: now + delay }); return id; }; + const fetch = (url, options) => new Promise((resolve, reject) => { requests.push({ url, options, resolve, reject, settled: false }); }); + vm.runInNewContext(source, { document, window, location, history, URL, Node, Option, Date, AbortController, + fetch, setTimeout, clearTimeout: id => timers.delete(id), console }, { filename: 'admin-host-inventory.js' }); + return { + nodes, root, window, requests, location, + async click(id) { if (!nodes[id].disabled) nodes[id].dispatch('click'); await flush(); }, + async open(id = firstId) { const button = new Node('button'); button.dataset.inventoryDetail = id; root.dispatch('click', { target: button }); await flush(); }, + async respond(payload, request = requests.find(item => !item.settled), status = 200) { + assert.ok(request); request.settled = true; request.resolve({ ok: status < 300, status, json: async () => payload }); await flush(); + }, + async advance(ms) { + const end = now + ms; + while (true) { + const next = [...timers.entries()].filter(([, timer]) => timer.due <= end).sort((a, b) => a[1].due - b[1].due)[0]; + if (!next) break; + now = next[1].due; timers.delete(next[0]); next[1].callback(); await flush(); + } + now = end; await flush(); + }, + }; +} + +// SSR and idle pages never trigger requests. Refresh reads only history, and +// opening a detail does not retrieve engine evidence until the explicit click. +{ + const test = fixture(); await test.advance(120000); + assert.equal(test.requests.length, 0, 'The SSR page must not refetch or poll.'); + assert.equal(test.nodes['inventory-body'].textContent, 'SSR records'); + await test.click('inventory-refresh'); assert.equal(test.requests.length, 1); + assert.match(test.requests[0].url, /^\/api\/v1\/admin\/host-inventory\?host_id=7$/); + const unsafe = attempt(firstId, { host_name: '', error_message: '' }); + await test.respond(page([unsafe])); + assert.match(test.nodes['inventory-body'].textContent, /'; +$unsafe['items'][0]['error_message'] = ''; +$unsafe['hosts'][0]['name'] = '" onmouseover="alert(2)'; +$unsafeHtml = $render($unsafe); +$dom->loadHTML('' . $unsafeHtml); +$xpath = new DOMXPath($dom); +$assert($xpath->query('//*[@onerror or @onmouseover]')->length === 0 + && !str_contains($unsafeHtml, '') + && str_contains($dom->textContent, ''), + 'Database labels must be escaped in both templates and JSON bootstraps.'); +$empty = $page; +$empty['items'] = []; +$empty['pagination']['total'] = 0; +$assert(str_contains($render($empty), '新功能只记录启用后的盘点尝试'), 'Empty history requires an honest explanation.'); +$failedHtml = $render(null, '数据库暂不可读'); +$assert(str_contains($failedHtml, '数据库暂不可读') && !str_contains($failedHtml, '已读取 0 条'), + 'Unavailable records must not be rendered as an empty successful list.'); +$unknown = $page; +$unknown['items'][0] = array_replace($attempt, ['outcome' => 'interrupted', + 'outcome_label' => '中断 / 未知', 'finished_at' => null, 'duration_is_final' => false]); +$assert(str_contains($render($unknown), '是否生效待核对') && str_contains($render($unknown), '已记录 8010 ms'), + 'Interrupted attempts must not claim final duration or definite non-application.'); + +$root = dirname(__DIR__); +$controller = file_get_contents($root . '/app/Controllers/CloudAdminHostInventoryController.php'); +$pageController = file_get_contents($root . '/app/Controllers/Admin/CloudPageController.php'); +$routes = file_get_contents($root . '/config/routes_cloud.php'); +$middleware = file_get_contents($root . '/app/Middleware/CloudApi.php'); +$js = file_get_contents($root . '/public/cloud/admin-host-inventory.js'); +$assert(str_contains($routes, "'/admin/host-inventory/{id:[0-9a-f]{32}}/engine'") + && str_contains($middleware, '^/api/v1/admin/host-inventory(?:/|$)') + && str_contains($controller, 'CloudAccess::actor(false)'), + 'Diagnostic APIs need signed-in admin identity without creating engine sessions.'); +$assert(str_contains($pageController, "'inventory_bootstrap_json'") + && str_contains($controller, 'CloudHostInventoryDiagnostics::detail($actor, $attemptId)'), + 'Engine evidence must authorize and bind host identity through a persisted attempt.'); +$assert(str_contains($controller, 'new CloudEngineClient(null, 1.0)') + && str_contains($controller, '->inventoryDiagnostics(') + && !str_contains($controller, '->inventory(') && !str_contains($controller, 'syncOne('), + 'Evidence retrieval must be a bounded read-only lookup, never another inventory.'); +$assert(!str_contains($js, 'innerHTML') && !str_contains($js, "method: 'POST'") + && !str_contains($js, '/capacity-sync') && !str_contains($js, 'setInterval('), + 'UI must be text-only and must not poll or run inventory.'); +echo "Administrator inventory SSR, escaping, read-only routes and diagnostic boundaries passed.\n"; diff --git a/web/tests/CloudHostInventoryDiagnosticsFixture.php b/web/tests/CloudHostInventoryDiagnosticsFixture.php new file mode 100644 index 00000000..9b2d53cd --- /dev/null +++ b/web/tests/CloudHostInventoryDiagnosticsFixture.php @@ -0,0 +1,31 @@ +create('cloud_host_inventory_attempts', static function (Blueprint $table): void { + $table->increments('sequence_id'); + $table->string('attempt_id', 32)->unique(); + $table->integer('host_id'); + $table->string('source', 16); + $table->integer('generation')->default(0); + $table->integer('started_at'); + $table->integer('finished_at')->nullable(); + $table->integer('updated_at'); + $table->integer('duration_ms')->default(0); + $table->string('outcome', 16)->default('running'); + $table->string('stage', 32)->default('admission'); + $table->string('error_code', 64)->default(''); + $table->integer('http_status')->nullable(); + $table->boolean('applied')->default(false); + $table->integer('timeout_ms'); + $table->string('panel_version', 64); + $table->string('php_version', 32); + $table->text('stages_json'); + $table->index(['host_id', 'started_at', 'sequence_id']); + $table->index(['outcome', 'started_at', 'sequence_id']); + $table->index(['started_at', 'sequence_id']); + }); +} diff --git a/web/tests/CloudHostInventoryDiagnosticsTest.php b/web/tests/CloudHostInventoryDiagnosticsTest.php new file mode 100644 index 00000000..40467023 --- /dev/null +++ b/web/tests/CloudHostInventoryDiagnosticsTest.php @@ -0,0 +1,210 @@ +httpStatus === $status, 'Unexpected error status.'); + return; + } + throw new \RuntimeException('Expected request rejection.'); + }; + $capsule = new Capsule(); + $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema(); + $schema->create('cloud_hosts', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('name'); + $table->string('credential_cipher')->default('never-read-this-secret'); + }); + createCloudHostInventoryAttemptTable($schema); + Capsule::table('cloud_hosts')->insert([ + ['id' => 1, 'name' => 'Host One'], ['id' => 2, 'name' => 'Host Two'], + ]); + $actor = ['is_admin' => true]; + $now = time(); + + $success = Diagnostics::start(1, 'auto', $now, 8.0); + $assert(preg_match('/^[a-f0-9]{32}$/D', $success->id()) === 1, 'Attempt IDs must be stable opaque identifiers.'); + $success->generation(10); + $success->stage('capacity_request'); + $success->stage('capacity_validate'); + $success->stage('network_request'); + $success->stage('network_validate'); + $success->stage('database_apply'); + $success->finish('succeeded', '', null, true); + $beforeLateFinish = CloudHostInventoryAttempt::find($success->id())->toArray(); + $success->finish('failed', 'unexpected_error'); + $success->stage('capacity_request'); + $success->generation(500); + $assert(CloudHostInventoryAttempt::find($success->id())->toArray() === $beforeLateFinish, + 'Terminal outcomes must not change on a second finish or a late phase.'); + $dto = Diagnostics::detail($actor, $success->id())['attempt']; + $assert($dto['outcome'] === 'succeeded' && $dto['applied'] && $dto['generation'] === 10, + 'Successful commit metadata was lost.'); + $assert($dto['timeout_ms'] === 8000 && $dto['finished_at'] >= $dto['started_at'] + && $dto['duration_ms'] >= 0 && count($dto['stages']) === 6, + 'Timeout and monotonic phase durations must be recorded.'); + $assert($dto['panel_version'] === 'inventory-diagnostics-v1', 'Never pretend runtime version is current disk Git HEAD.'); + + $timeout = Diagnostics::start(1, 'manual', $now, 8.0); + $timeout->generation(11); + $timeout->stage('capacity_request'); + $timeout->finish('timeout', 'engine_http_timeout', null, false); + $failed = Diagnostics::start(1, 'auto', $now, 8.0); + $failed->generation(12); + $failed->stage('network_request'); + $failed->finish('failed', 'engine_resource_missing', 404, false); + $unrelated = Diagnostics::start(2, 'auto', $now, 8.0); + $unrelated->finish('succeeded', '', null, true); + $summary = Diagnostics::page($actor, ['host_id' => '1'])['summary']; + $assert($summary['consecutive_failures'] === 2, + 'Same-second failures must not disappear behind the earlier success timestamp.'); + $assert($summary['last_success_at'] !== null && $summary['last_failure_at'] !== null, + 'The last success/failure times must survive subsequent attempts.'); + $assert(Diagnostics::page($actor)['summary']['consecutive_failures'] === null, + 'A global failure streak across unrelated hosts is misleading.'); + + $superseded = Diagnostics::start(1, 'manual', $now, 8.0); + $superseded->finish('superseded'); + $assert(Diagnostics::page($actor, ['host_id' => 1])['summary']['consecutive_failures'] === 2, + 'Superseded/skipped attempts cannot be counted as confirmed failures.'); + $running = Diagnostics::start(1, 'auto', $now, 8.0); + $running->stage('capacity_request'); + $assert(Diagnostics::page($actor, ['host_id' => 1])['summary']['in_progress'] === 1, + 'Recent running attempt should be visible.'); + CloudHostInventoryAttempt::where('attempt_id', $running->id())->update(['updated_at' => $now - 301]); + $interrupted = Diagnostics::detail($actor, $running->id())['attempt']; + $assert($interrupted['outcome'] === 'interrupted' && $interrupted['finished_at'] === null + && $interrupted['last_confirmed_stage'] === 'capacity_request', + 'Abandoned attempts must display unknown without inventing a completion or failure cause.'); + $assert(CloudHostInventoryAttempt::find($running->id())->outcome === 'running', + 'Read-side interpretation must not mutate diagnostic history.'); + $assert(Diagnostics::page($actor, ['host_id' => 1, 'outcome' => 'attention'])['pagination']['total'] === 3, + 'Attention filter should include failed, timed-out, and interrupted records.'); + $assert(Diagnostics::page($actor, ['outcome' => 'running'])['pagination']['total'] === 0, + 'Interrupted records must not appear as actively running.'); + $assert(Diagnostics::page($actor, ['outcome' => 'interrupted'])['pagination']['total'] === 1, + 'Interrupted filter must match stale ongoing records.'); + + $malicious = Diagnostics::start(1, 'Bearer never-store-source-secret', $now, 8.0); + $malicious->stage('password=never-store-stage-secret'); + $malicious->finish('failed', 'token=never-store-error-secret', 999, true); + $maliciousDto = Diagnostics::detail($actor, $malicious->id())['attempt']; + $assert($maliciousDto['error_code'] === 'unexpected_error' && $maliciousDto['stage'] === 'admission' + && $maliciousDto['source'] === 'auto' && $maliciousDto['http_status'] === null && !$maliciousDto['applied'], + 'Unknown raw diagnostics must be reduced to fixed safe values.'); + CloudHostInventoryAttempt::where('attempt_id', $malicious->id())->update([ + 'stages_json' => '[{"stage":"password=never-display-stage-secret","duration_ms":4},' + . '{"stage":"capacity_request","duration_ms":5,"password":"never-display-extra-secret"}]', + 'error_code' => 'token=never-display-error-secret', + ]); + $clean = json_encode(Diagnostics::detail($actor, $malicious->id()), JSON_THROW_ON_ERROR); + $assert(!str_contains($clean, 'never-') && !str_contains($clean, 'credential'), + 'Only allowlisted diagnostic keys may be exposed by the DTO.'); + $assert(count(Diagnostics::detail($actor, $malicious->id())['attempt']['stages']) === 1, + 'Unsafe stage entries must be discarded, extra keys stripped.'); + + $reject(static fn () => Diagnostics::page(['is_admin' => false]), 403); + $reject(static fn () => Diagnostics::detail([], $success->id()), 403); + $reject(static fn () => Diagnostics::detail($actor, '../secret'), 400); + $reject(static fn () => Diagnostics::detail($actor, str_repeat('0', 32)), 404); + foreach ([['host_id' => '-1'], ['outcome' => 'injected'], ['page' => '0'], + ['per_page' => '10000'], ['host_id' => []], ['q' => 'not-supported']] as $input) { + $reject(static fn () => Diagnostics::page($actor, $input), 400); + } + $lastPage = Diagnostics::page($actor, ['page' => '900']); + $assert($lastPage['pagination']['page'] === $lastPage['pagination']['total_pages'], + 'Out-of-range pages should clamp to the last page.'); + + // Capture only page/detail queries: they must never write or load host secrets. + $connection = $capsule->getConnection('default'); + $connection->enableQueryLog(); + Diagnostics::page($actor, ['host_id' => '1']); + Diagnostics::detail($actor, $timeout->id()); + foreach ($connection->getQueryLog() as $query) { + $assert(str_starts_with(strtolower(ltrim($query['query'])), 'select'), + 'Viewing diagnostic records must be database-read-only.'); + $assert(!str_contains($query['query'], 'credential_cipher'), 'History must not select host credentials.'); + } + $connection->disableQueryLog(); + + // Old history is removed in a bounded batch on a new attempt, not on a read. + $seed = CloudHostInventoryAttempt::find($timeout->id())->getAttributes(); + unset($seed['sequence_id']); + for ($i = 1; $i <= 300; $i++) { + $row = $seed; + $row['attempt_id'] = sprintf('%032x', $i); + $row['started_at'] = $now - 8 * 86400; + $row['updated_at'] = $row['started_at']; + $row['finished_at'] = $row['started_at']; + CloudHostInventoryAttempt::query()->insert($row); + } + Diagnostics::start(3, 'auto', $now, 8.0)->finish('skipped'); + $assert(CloudHostInventoryAttempt::where('started_at', '<', $now - 7 * 86400)->count() === 44, + 'Retention cleanup must be bounded rather than deleting unbounded history per sync.'); + + // The per-host cap is independent of the age cutoff, including a burst of + // same-second attempts. Bulk fixtures avoid one query per synthetic row. + $rows = []; + for ($i = 10000; $i < 35020; $i++) { + $row = $seed; + $row['attempt_id'] = sprintf('%032x', $i); + $row['host_id'] = 4; + $row['started_at'] = $now - 1; + $row['updated_at'] = $now - 1; + $row['finished_at'] = $now - 1; + $rows[] = $row; + if (count($rows) === 100) { + CloudHostInventoryAttempt::query()->insert($rows); + $rows = []; + } + } + if ($rows !== []) { + CloudHostInventoryAttempt::query()->insert($rows); + } + $capAttempt = Diagnostics::start(4, 'auto', $now, 8.0); + $capAttempt->finish('skipped'); + $assert(CloudHostInventoryAttempt::where('host_id', 4)->count() === 25000 + && CloudHostInventoryAttempt::find($capAttempt->id()) !== null, + 'The per-host cap must prune older rows while preserving the current attempt.'); + + // Diagnostic storage failure is fail-open and cannot abort the actual inventory. + $schema->drop('cloud_host_inventory_attempts'); + $fallback = Diagnostics::start(1, 'manual', $now, 8.0); + $fallback->stage('capacity_request'); + $fallback->generation(20); + $fallback->finish('timeout', 'engine_http_timeout'); + $assert(strlen($fallback->id()) === 32, 'An ID is still needed for engine correlation when DB logging fails.'); + $reject(static fn () => Diagnostics::page($actor), 503); + $reject(static fn () => Diagnostics::detail($actor, $fallback->id()), 503); + + echo "Cloud inventory diagnostic history tests passed.\n"; +} diff --git a/web/tests/CloudHostInventorySyncContractTest.php b/web/tests/CloudHostInventorySyncContractTest.php index 85fff526..df88bf3c 100644 --- a/web/tests/CloudHostInventorySyncContractTest.php +++ b/web/tests/CloudHostInventorySyncContractTest.php @@ -18,8 +18,8 @@ $assert = static function ($condition, string $message): void { } }; -$assert(str_contains($service, "'GET', 'host/capacity'") - && str_contains($service, "'GET', 'host/network-options'") +$assert(str_contains($service, "'host/capacity', \$actor, \$diagnostics->id()") + && str_contains($service, "'host/network-options', \$actor, \$diagnostics->id()") && str_contains($service, 'private const ENGINE_TIMEOUT_SECONDS = 8.0;') && str_contains($service, 'public_ip_available_count') && str_contains($service, "['available', 'disabled']") diff --git a/web/tests/CloudHostInventorySyncServiceTest.php b/web/tests/CloudHostInventorySyncServiceTest.php index 0c601741..c54e4388 100644 --- a/web/tests/CloudHostInventorySyncServiceTest.php +++ b/web/tests/CloudHostInventorySyncServiceTest.php @@ -27,9 +27,11 @@ namespace App\Services\Cloud { namespace { require dirname(__DIR__) . '/vendor/autoload.php'; + require __DIR__ . '/CloudHostInventoryDiagnosticsFixture.php'; use App\Models\CloudHost; use App\Models\CloudHostCapacitySnapshot; + use App\Models\CloudHostInventoryAttempt; use App\Models\CloudHostNetworkAttachment; use App\Models\CloudHostResourcePoolMembership; use App\Models\CloudPublicIp; @@ -163,6 +165,7 @@ namespace { $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); + createCloudHostInventoryAttemptTable($schema); $schema->create('user', function (Blueprint $table): void { $table->increments('id'); $table->integer('enable')->default(1); @@ -447,14 +450,27 @@ namespace { ->public_ip_pool_id === (int) $publicPools[$poolB]->id, 'A new address was written without its public-pool association.'); $assert(count($history) === 2 - && (string) $history[0]['request']->getUri() - === 'http://127.0.0.1:8000/internal/v1/hosts/1/host/capacity' - && (string) $history[1]['request']->getUri() - === 'http://127.0.0.1:8000/internal/v1/hosts/1/host/network-options', + && $history[0]['request']->getUri()->getPath() + === '/internal/v1/hosts/1/host/capacity' + && $history[1]['request']->getUri()->getPath() + === '/internal/v1/hosts/1/host/network-options' + && preg_match('/^inventory_attempt_id=[0-9a-f]{32}$/D', + $history[0]['request']->getUri()->getQuery()) === 1 + && $history[0]['request']->getUri()->getQuery() + === $history[1]['request']->getUri()->getQuery(), 'Lifecycle inventory did not use both bounded host endpoints.'); $actorHeader = $history[0]['request']->getHeaderLine('X-Cloud-Actor'); $actorHeader .= str_repeat('=', (4 - strlen($actorHeader) % 4) % 4); $actor = json_decode(base64_decode(strtr($actorHeader, '-_', '+/'), true), true); + $record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail(); + $assert($record->outcome === 'succeeded' && $record->applied + && $record->source === 'auto' && $record->stage === 'database_apply' + && (int) $record->timeout_ms === 3000 + && (int) $record->started_at === 1000 + && (int) $record->finished_at >= 1000 + && $history[0]['request']->getUri()->getQuery() + === 'inventory_attempt_id=' . $record->attempt_id, + 'A full committed inventory needs its own successful correlated history record.'); $assert(($actor['user_id'] ?? null) === 99 && ($actor['session_id'] ?? null) === 'cloud-host-inventory-sync' && ($actor['is_super_admin'] ?? null) === true, @@ -500,6 +516,11 @@ namespace { && $snapshot->last_error_code === 'host_inventory_network_invalid' && $states === $beforeStates && $afterPools === $beforePools, 'Failure cleared capacity, pool metadata or concrete IP state.'); + $record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail(); + $assert($record->outcome === 'failed' && !$record->applied + && $record->source === 'manual' && $record->stage === 'network_validate' + && $record->error_code === 'host_inventory_network_invalid', + 'Network validation failure must retain its layer and outcome.'); $now = 1020; $unavailableClient = $client([ @@ -520,6 +541,11 @@ namespace { && $host->last_seen_at === 1000 && $host->health_checked_at === 1020 && !str_contains($persisted, 'private engine path'), 'Engine failure cleared last success or persisted an upstream error.'); + $record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail(); + $assert($record->error_code === 'engine_service_error' && (int) $record->http_status === 503 + && $record->stage === 'capacity_request' && !$record->applied + && !str_contains(json_encode($record->toArray()), 'private engine path'), + 'Transport and upstream HTTP evidence must survive safe classification.'); $reflection = new ReflectionClass(CloudHostInventorySyncService::class); $beginAttempt = $reflection->getMethod('beginAttempt'); @@ -736,5 +762,37 @@ namespace { && $host->scheduler_state === 'disabled', 'Clearing the authoritative membership recreated a stale scalar link.'); - echo "Cloud host multi-pool inventory and membership checks passed.\n"; + $now = 1060; + $timeoutClient = $client([new \GuzzleHttp\Exception\ConnectException('SECRET timeout', + new \GuzzleHttp\Psr7\Request('GET', 'http://127.0.0.1'), null, ['errno' => 28])]); + $timedOut = (new CloudHostInventorySyncService($timeoutClient, + static function () use (&$now): int { return $now; }))->syncOne(1); + $record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail(); + $assert($timedOut['unavailable'] === 1 && $record->outcome === 'timeout' + && $record->error_code === 'engine_http_timeout' && !$record->applied + && $record->stage === 'capacity_request' && $record->http_status === null, + 'A timeout needs durable evidence without inventing an engine failure phase.'); + + $now = 1070; + $supersededClient = $client([ + static function () use ($capsule, $capacity, $json, &$now) { + $capsule->getConnection()->table('cloud_host_capacity_snapshots') + ->where('host_id', 1)->increment('attempt_generation'); + return new Response(200, [], $json($capacity($now))); + }, + new Response(200, [], $json($network())), + ]); + $superseded = (new CloudHostInventorySyncService($supersededClient, + static function () use (&$now): int { return $now; }))->syncOne(1); + $record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail(); + $assert($superseded['skipped'] === 1 && $record->outcome === 'superseded' && !$record->applied, + 'A stale generation must be retained as superseded, never successful.'); + + $schema->drop('cloud_host_inventory_attempts'); + $withoutHistory = (new CloudHostInventorySyncService($client([ + new Response(503, [], '{"error":"SECRET"}'), + ]), static function () use (&$now): int { return $now; }))->syncOne(1); + $assert($withoutHistory['unavailable'] === 1, + 'Unavailable diagnostic storage must not prevent ordinary failure handling.'); + echo "Cloud host multi-pool inventory, diagnostic history and membership checks passed.\n"; } diff --git a/web/tests/CloudInventoryEngineClientTest.php b/web/tests/CloudInventoryEngineClientTest.php new file mode 100644 index 00000000..9c29cbcf --- /dev/null +++ b/web/tests/CloudInventoryEngineClientTest.php @@ -0,0 +1,122 @@ +push(Middleware::history($history)); + return new CloudEngineClient(new Client(['handler' => $handler]), 8.0); + }; + $actor = ['user_id' => 1, 'is_admin' => true, 'is_super_admin' => true]; + $id = str_repeat('a', 32); + $history = []; + $engine = $client([new Response(200, [], '{"health":"healthy"}')], $history); + $assert($engine->hostInventory(1, 'host/capacity', $actor, $id)['health'] === 'healthy', + 'Inventory response changed'); + $request = $history[0]['request']; + $target = '/internal/v1/hosts/1/host/capacity?inventory_attempt_id=' . $id; + $expected = hash_hmac('sha256', CloudEngineClient::canonical('GET', $target, '', + $request->getHeaderLine('X-Cloud-Timestamp'), $request->getHeaderLine('X-Cloud-Nonce'), + $request->getHeaderLine('X-Cloud-Actor')), str_repeat('k', 32)); + $assert($request->getRequestTarget() === $target + && hash_equals($expected, $request->getHeaderLine('X-Cloud-Signature')), + 'Correlation ID must be authenticated with the complete query'); + $assert($history[0]['options']['timeout'] === 8.0 + && $history[0]['options']['connect_timeout'] === 3.0 + && $history[0]['options']['allow_redirects'] === false, + 'Effective bounded inventory request options missing'); + + $cases = [ + [new ConnectException('SECRET', new Request('GET', 'http://localhost'), null, + ['errno' => 28]), 'engine_http_timeout', null, 502], + [new ConnectException('SECRET', new Request('GET', 'http://localhost'), null, + ['errno' => 7]), 'engine_connect_failed', null, 502], + [new ConnectException('SECRET timed out', new Request('GET', 'http://localhost')), + 'engine_connection_or_timeout', null, 502], + [new Response(401, [], 'SECRET'), 'engine_auth_rejected', 401, 502], + [new Response(403, [], 'SECRET'), 'engine_auth_rejected', 403, 502], + [new Response(404, [], 'SECRET'), 'engine_resource_missing', 404, 502], + [new Response(404, [], '{"error":"SECRET"}'), 'engine_resource_missing', 404, 404], + [new Response(409, [], 'SECRET'), 'engine_request_conflict', 409, 502], + [new Response(423, [], 'SECRET'), 'engine_configuration_locked', 423, 502], + [new Response(429, [], 'SECRET'), 'engine_rate_limited', 429, 502], + [new Response(500, [], 'SECRET'), 'engine_service_error', 500, 502], + [new Response(504, [], 'SECRET'), 'engine_http_timeout', 504, 502], + [new Response(200, [], 'SECRET'), 'engine_response_invalid', 200, 502], + ]; + foreach ($cases as [$reply, $code, $upstream, $legacy]) { + $engine = $client([$reply], $history); + try { + $engine->hostInventory(1, 'host/network-options', $actor, $id); + throw new \RuntimeException('Expected inventory failure'); + } catch (CloudInventoryRequestError $error) { + $assert($error->diagnosticCode === $code && $error->upstreamHttpStatus === $upstream + && $error->httpStatus === $legacy && !str_contains($error->getMessage(), 'SECRET'), + 'Incorrect safe inventory error classification: ' . $code); + $assert(count($history) === 1, 'Errors must not trigger fallback scans'); + } + } + + $trace = ['resource' => 'capacity', 'state' => 'running', 'stage' => 'update_lock_wait', + 'error_code' => '', 'http_status' => null, 'started_at' => time(), 'finished_at' => null, + 'duration_ms' => 8001, 'engine_version' => 'SECRET', 'raw_error' => 'SECRET', + 'stages' => [['stage' => 'update_lock_wait', 'state' => 'running', 'duration_ms' => 8000]]]; + $engine = $client([new Response(200, [], json_encode(['attempt_id' => $id, + 'traces' => [$trace], 'password' => 'SECRET']))], $history); + $evidence = $engine->inventoryDiagnostics(1, $actor, $id); + $assert($evidence['traces'][0]['stage'] === 'update_lock_wait' + && !str_contains(json_encode($evidence), 'SECRET') + && $history[0]['options']['timeout'] === 1.0 + && $history[0]['options']['connect_timeout'] === 1.0 + && $history[0]['request']->getUri()->getPath() + === '/internal/v1/hosts/1/host/inventory-diagnostics/' . $id, + 'Evidence reads must be bounded, sanitized and separate from inventory'); + foreach ([['attempt_id' => str_repeat('b', 32), 'traces' => [$trace]], + ['attempt_id' => $id, 'traces' => [array_replace($trace, ['stage' => 'SECRET'])]]] as $invalid) { + $engine = $client([new Response(200, [], json_encode($invalid))], $history); + try { + $engine->inventoryDiagnostics(1, $actor, $id); + throw new \RuntimeException('Invalid evidence accepted'); + } catch (CloudInventoryRequestError $error) { + $assert($error->diagnosticCode === 'engine_response_invalid', 'Evidence contract failed'); + } + } + // General VM mutation handling must retain its original error boundary. + $engine = $client([new Response(423, [], '{"error":"SECRET"}')], $history); + try { + $engine->host(1, 'POST', 'vms/example/start', $actor, []); + throw new \RuntimeException('Expected mutation error'); + } catch (CloudError $error) { + $assert($error->httpStatus === 502 && !str_contains($error->getMessage(), 'SECRET'), + 'Inventory diagnostics changed mutation error semantics'); + } + echo "Inventory engine diagnostics client checks passed.\n"; +}