From 82c1e533bf64e247040c9cb26f396ec908d71f9d Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 9 Oct 2026 05:39:10 +0800 Subject: [PATCH] Record Fabric remote failure phases --- engine/tests/test_engine_boundary.py | 19 ++ engine/tests/test_fabric.py | 245 +++++++++++++++++- engine/virsh_panel/fabric.py | 113 +++++--- engine/virsh_panel/internal.py | 9 +- .../Cloud/CloudAdminFabricService.php | 21 +- web/public/cloud/admin-fabric.js | 13 + web/resources/views/venix/layout_assets.tpl | 2 +- web/tests/AdminFabricContractTest.php | 16 ++ .../AdminFabricReadinessWorkflowTest.php | 49 +++- web/tests/AdminFabricWizardBehaviorTest.mjs | 30 +++ web/tests/CloudFirewallUiContractTest.php | 2 +- .../CloudHostFabricEndpointContractTest.php | 2 +- 12 files changed, 480 insertions(+), 41 deletions(-) diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index e3d61579..7a1fc4d5 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -29,6 +29,7 @@ from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, Hos LogStore, MainStore, TRUSTED_ACTOR) from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor, signed_headers) +from virsh_panel.fabric import FabricError from virsh_panel.internal import EngineApplication, HostRegistry from virsh_panel.libvirt_service import ( FIREWALL_FILTER_COMPILER_VERSION, VMError, build_firewall_filter_xml, @@ -353,6 +354,24 @@ def test_fabric_is_engine_global_and_super_admin_write_only(boundary): invalid = call(client, "POST", "/internal/v1/fabric/apply", SUPER, {}) assert invalid.status_code == 400 assert invalid.json()["error_code"] == "fabric_invalid" + assert "failure_phase" not in invalid.json() + + async def failed_identity(_value): + raise FabricError( + "宿主机 Fabric 命令失败", 502, + "fabric_remote_command_failed", "identity", + ) + + with patch.object(engine.fabric, "apply", failed_identity): + identity_failed = call( + client, "POST", "/internal/v1/fabric/apply", SUPER, {}, + ) + assert identity_failed.status_code == 502 + assert identity_failed.json() == { + "error": "宿主机 Fabric 命令失败", + "error_code": "fabric_remote_command_failed", + "failure_phase": "identity", + } assert call( client, "POST", "/internal/v1/fabric/probe", USER, {"host_ids": ["host-a"]}, diff --git a/engine/tests/test_fabric.py b/engine/tests/test_fabric.py index 03eacdf7..aa745bae 100644 --- a/engine/tests/test_fabric.py +++ b/engine/tests/test_fabric.py @@ -70,6 +70,26 @@ def plan(): }) +def plan_payload(current=None): + current = current or plan() + return { + "fabric_id": current.fabric_id, + "region": current.region, + "overlay_cidr": current.overlay_cidr, + "revision": current.revision, + "listen_port": current.listen_port, + "nodes": [ + { + "host_id": node.host_id, + "overlay_ip": node.overlay_ip, + "endpoint": node.endpoint, + "listen_port": node.listen_port, + } + for node in current.nodes + ], + } + + def test_plan_is_canonical_and_rejects_wide_or_tenant_routes(): first = plan() reordered = parse_fabric_plan({ @@ -4592,6 +4612,40 @@ def test_adapter_readback_requires_exact_peer_route_and_persistence(): )["transport_verified"] is True +@pytest.mark.parametrize("failure_phase", ["identity", "apply", "observe"]) +def test_adapter_remote_failure_exposes_only_whitelisted_phase(failure_phase): + class FailedRemoteManager(FakeManager): + def _exec(self, argv, *, timeout, input_text=None): + self.scripts.append(input_text) + return subprocess.CompletedProcess( + argv, 71, "", "secret /root/path private-key-material") + + current = plan() + adapter = FabricHostAdapter(FailedRemoteManager(KEY1)) + with pytest.raises(FabricError) as error: + if failure_phase == "identity": + adapter.identity() + elif failure_phase == "apply": + adapter.apply( + current, current.nodes[0], + {"host-a": KEY1, "host-b": KEY2}) + else: + adapter.observe( + current, current.nodes[0], + {"host-a": KEY1, "host-b": KEY2}) + + assert error.value.error_code == "fabric_remote_command_failed" + assert error.value.failure_phase == failure_phase + safe_error = json.dumps({ + "message": str(error.value), + "error_code": error.value.error_code, + "failure_phase": error.value.failure_phase, + }) + assert "secret" not in safe_error + assert "/root/path" not in safe_error + assert "private-key-material" not in safe_error + + def test_adapter_pins_firewalld_mode_through_apply_and_observe(): current = plan() manager = FakeManager(KEY1, firewall_mode=FIREWALL_MODE_FIREWALLD) @@ -4770,6 +4824,7 @@ class MemoryStore: class FakeAdapter: failures = set() + observe_failures = set() def __init__(self, manager, profile=None): self.manager = manager @@ -4782,7 +4837,8 @@ class FakeAdapter: raise FabricError("offline", 502, "fabric_remote_command_failed") def observe(self, current, host, keys): - if self.manager.host_id in self.failures: + if (self.manager.host_id in self.failures + or self.manager.host_id in self.observe_failures): raise FabricError("offline", 502, "fabric_remote_command_failed") return { "state": "active", "transport_verified": True, @@ -5347,6 +5403,12 @@ def test_coordinator_keeps_successful_hosts_and_blocks_next_stage(monkeypatch): assert result["next_stage_allowed"] is False states = {node["host_id"]: node["state"] for node in result["nodes"]} assert states == {"host-a": "active", "host-b": "uncertain"} + failed = next( + node for node in result["nodes"] if node["host_id"] == "host-b") + assert failed["last_error_phase"] == "apply" + persisted = json.loads(store.get( + "fabric_node", f"{result['fabric_id']}.host-b")) + assert persisted["last_error_phase"] == "apply" assert managers["host-a"].reprobe_calls == 1 assert managers["host-b"].reprobe_calls == 1 serialized = "\n".join(store.values.values()) @@ -5355,6 +5417,52 @@ def test_coordinator_keeps_successful_hosts_and_blocks_next_stage(monkeypatch): FakeAdapter.failures = set() +def test_initial_apply_identity_failure_carries_safe_phase_without_record( + monkeypatch): + current = plan() + store = MemoryStore() + components = _fabric_components(current) + + class IdentityFailureAdapter(FakeAdapter): + def identity(self): + raise FabricError( + "safe", 502, "fabric_remote_command_failed") + + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", IdentityFailureAdapter) + + with pytest.raises(FabricError) as error: + asyncio.run(FabricCoordinator( + store, Registry(components)).apply(plan_payload(current))) + + assert error.value.error_code == "fabric_remote_command_failed" + assert error.value.failure_phase == "identity" + assert store.get("fabric_desired", current.fabric_id) is None + + +def test_apply_records_post_apply_observe_failure_phase(monkeypatch): + current = plan() + store = MemoryStore() + components = _fabric_components(current) + monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", FakeAdapter) + FakeAdapter.failures = set() + FakeAdapter.observe_failures = {"host-a"} + try: + result = asyncio.run(FabricCoordinator( + store, Registry(components)).apply(plan_payload(current))) + finally: + FakeAdapter.observe_failures = set() + + nodes = {node["host_id"]: node for node in result["nodes"]} + assert nodes["host-a"]["last_error_code"] == ( + "fabric_remote_command_failed") + assert nodes["host-a"]["last_error_phase"] == "observe" + assert "last_error_phase" not in nodes["host-b"] + persisted = json.loads(store.get( + "fabric_node", f"{current.fabric_id}.host-a")) + assert persisted["last_error_phase"] == "observe" + + def test_recover_does_not_replay_an_interrupted_write(): store = MemoryStore() desired = {"fabric_id": "hk-fabric", "state": "applying", @@ -5389,6 +5497,8 @@ def test_public_status_does_not_echo_nested_secrets_or_commands(): "command": "rm -rf /", "nodes": [{ "host_id": "host-a", "public_key": KEY1, + "last_error_code": "fabric_remote_command_failed", + "last_error_phase": "apply", "private_key": "secret", "observed": { "interface": FABRIC_INTERFACE, @@ -5396,13 +5506,26 @@ def test_public_status_does_not_echo_nested_secrets_or_commands(): "peers": [(KEY2, "203.0.113.2:51820", "100.64.1.2/32")], "handshakes": {KEY2: 123}, }, + }, { + "host_id": "host-b", "public_key": KEY2, + "last_error_phase": "observe", + }, { + "host_id": "host-c", "public_key": KEY3, + "last_error_code": "fabric_remote_command_failed", + "last_error_phase": "stderr:/secret/path", }], }) serialized = json.dumps(public) assert public["listen_port"] == 51820 assert "private_key" not in serialized - assert "command" not in serialized + assert "command" not in public + assert "command" not in public["nodes"][0]["observed"] assert KEY1 in serialized and KEY2 in serialized + assert public["nodes"][0]["last_error_phase"] == "apply" + assert "last_error_phase" not in public["nodes"][1] + assert "last_error_phase" not in public["nodes"][2] + assert FabricError( + "safe", failure_phase="stderr:/secret/path").failure_phase is None def test_reconcile_rejects_a_tampered_plan_digest(): @@ -5565,6 +5688,30 @@ def _recovery_components(current, *, configured=False): return components +def test_successful_apply_clears_stale_failure_phase(): + current = plan() + store = MemoryStore() + record = _active_fabric_record(current) + for node in record["nodes"]: + node["last_error_code"] = "fabric_remote_command_failed" + node["last_error_phase"] = "observe" + store.set("fabric_desired", current.fabric_id, json.dumps(record)) + coordinator = FabricCoordinator( + store, Registry(_fabric_components(current))) + + result = asyncio.run(coordinator.apply(plan_payload(current))) + + assert result["state"] == "active" + assert all("last_error_phase" not in node for node in result["nodes"]) + durable = json.loads(store.get("fabric_desired", current.fabric_id)) + assert all("last_error_phase" not in node for node in durable["nodes"]) + for host in current.nodes: + persisted = json.loads(store.get( + "fabric_node", f"{current.fabric_id}.{host.host_id}")) + assert "last_error_phase" not in persisted + assert persisted["last_error_code"] is None + + def test_reconcile_recovers_when_incomplete_host_observe_initially_fails( monkeypatch): current = plan() @@ -5584,6 +5731,7 @@ def test_reconcile_recovers_when_incomplete_host_observe_initially_fails( assert result["state"] == "active" assert result["next_stage_allowed"] is True + assert all("last_error_phase" not in node for node in result["nodes"]) assert all(component.manager.reconcile_identity_calls == 1 for component in components.values()) assert all(component.manager.reconcile_apply_calls == 1 @@ -5591,6 +5739,94 @@ def test_reconcile_recovers_when_incomplete_host_observe_initially_fails( assert first.reconcile_observe_calls == 2 +def test_reconcile_persists_post_apply_observe_failure_phase(monkeypatch): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _recovery_components(current) + failed = components["host-a"].manager + + class PostApplyObserveFailureAdapter(ReconcileRecoveryAdapter): + def observe(self, plan_value, host, keys): + if self.manager is failed and self.manager.reconcile_configured: + self.manager.reconcile_observe_calls += 1 + raise FabricError( + "incomplete", 502, "fabric_remote_command_failed") + return super().observe(plan_value, host, keys) + + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", + PostApplyObserveFailureAdapter) + + result = asyncio.run(coordinator.reconcile(current.fabric_id)) + + nodes = {node["host_id"]: node for node in result["nodes"]} + assert result["state"] == "uncertain" + assert nodes["host-a"]["last_error_code"] == ( + "fabric_remote_command_failed") + assert nodes["host-a"]["last_error_phase"] == "observe" + persisted = json.loads(store.get( + "fabric_node", f"{current.fabric_id}.host-a")) + assert persisted["last_error_phase"] == "observe" + assert failed.reconcile_observe_calls == 2 + assert failed.reconcile_apply_calls == 1 + + +def test_reconcile_identity_gate_keeps_other_host_observe_phase(monkeypatch): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _recovery_components(current) + observed_failed = components["host-a"].manager + identity_failed = components["host-b"].manager + observed_failed.reconcile_observe_error = FabricError( + "incomplete", 502, "fabric_remote_command_failed") + identity_failed.reconcile_identity_error = FabricError( + "offline", 502, "fabric_remote_command_failed") + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) + + result = asyncio.run(coordinator.reconcile(current.fabric_id)) + + nodes = {node["host_id"]: node for node in result["nodes"]} + assert nodes["host-a"]["last_error_phase"] == "observe" + assert nodes["host-b"]["last_error_phase"] == "identity" + assert all(component.manager.reconcile_apply_calls == 0 + for component in components.values()) + + +def test_successful_reconcile_observation_clears_stale_failure_phase( + monkeypatch): + current = plan() + store = MemoryStore() + record = _active_fabric_record(current) + for node in record["nodes"]: + node["last_error_code"] = "fabric_remote_command_failed" + node["last_error_phase"] = "apply" + store.set("fabric_desired", current.fabric_id, json.dumps(record)) + components = _recovery_components(current, configured=True) + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) + + result = asyncio.run(coordinator.reconcile(current.fabric_id)) + + assert result["state"] == "active" + assert all(node["last_error_code"] is None for node in result["nodes"]) + assert all("last_error_phase" not in node for node in result["nodes"]) + durable = json.loads(store.get("fabric_desired", current.fabric_id)) + assert all("last_error_phase" not in node for node in durable["nodes"]) + for host in current.nodes: + persisted = json.loads(store.get( + "fabric_node", f"{current.fabric_id}.{host.host_id}")) + assert "last_error_phase" not in persisted + assert persisted["last_error_code"] is None + + def test_reconcile_recovers_key_and_emergency_only_then_is_idempotent( monkeypatch): current = plan() @@ -5712,6 +5948,10 @@ def test_reconcile_identity_failure_blocks_all_host_writes( assert result["state"] == "uncertain" assert result["next_stage_allowed"] is False assert nodes["host-b"]["last_error_code"] == expected_error + assert nodes["host-b"]["last_error_phase"] == "identity" + persisted = json.loads(store.get( + "fabric_node", f"{current.fabric_id}.host-b")) + assert persisted["last_error_phase"] == "identity" for component in components.values(): manager = component.manager assert manager.reconcile_identity_calls == 1 @@ -5738,6 +5978,7 @@ def test_reconcile_apply_failure_on_one_node_does_not_stop_other_node( assert result["state"] == "uncertain" assert nodes["host-a"]["last_error_code"] == ( "fabric_remote_command_failed") + assert nodes["host-a"]["last_error_phase"] == "apply" assert nodes["host-b"]["state"] == "active" assert nodes["host-b"]["observed_plan_digest"] == current.digest assert all(component.manager.reconcile_identity_calls == 1 diff --git a/engine/virsh_panel/fabric.py b/engine/virsh_panel/fabric.py index 7d0fc052..15b9561f 100644 --- a/engine/virsh_panel/fabric.py +++ b/engine/virsh_panel/fabric.py @@ -62,6 +62,7 @@ FIREWALL_MODE_FIREWALLD = "firewalld_nft" FABRIC_FIREWALL_MODES = frozenset(( FIREWALL_MODE_NATIVE, FIREWALL_MODE_FIREWALLD, )) +FABRIC_FAILURE_PHASES = frozenset(("identity", "observe", "apply")) IFUP_PATHS = frozenset(("/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup")) IFDOWN_PATHS = frozenset(("/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown")) IFQUERY_PATHS = frozenset(("/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery")) @@ -710,10 +711,25 @@ class FabricError(Exception): """Safe error returned by the host-only fabric boundary.""" def __init__(self, message: str, status: int = 400, - error_code: str = "fabric_invalid"): + error_code: str = "fabric_invalid", + failure_phase: str | None = None): super().__init__(message) self.status = status self.error_code = error_code + self.failure_phase = _fabric_failure_phase(failure_phase) + + def at_phase(self, failure_phase: str) -> FabricError: + """Attach one bounded operation phase without replacing a prior one.""" + if self.failure_phase is None: + self.failure_phase = _fabric_failure_phase(failure_phase) + return self + + +def _fabric_failure_phase(value: Any) -> str | None: + """Return only a public-safe Fabric operation phase.""" + if isinstance(value, str) and value in FABRIC_FAILURE_PHASES: + return value + return None @dataclass(frozen=True) @@ -8888,19 +8904,22 @@ class FabricHostAdapter: # generation halfway through could configure a different host. self.profile = dict(profile) if isinstance(profile, Mapping) else None - def _run(self, script: str) -> str: + def _run(self, script: str, *, failure_phase: str) -> str: + safe_phase = _fabric_failure_phase(failure_phase) try: current = self.profile if self.profile is not None else self.manager.current() except RemoteHostError as exc: raise FabricError( "宿主机 SSH 配置不可用", 502, - "fabric_host_connection_failed") from exc + "fabric_host_connection_failed", safe_phase) from exc if not current: raise FabricError( - "宿主机尚未配置", 409, "fabric_host_not_configured") + "宿主机尚未配置", 409, "fabric_host_not_configured", + safe_phase) if current.get("key_locked"): raise FabricError( - "宿主机 SSH 私钥仍锁定", 409, "fabric_host_key_locked") + "宿主机 SSH 私钥仍锁定", 409, "fabric_host_key_locked", + safe_phase) argv = [ self.manager.ssh_binary, "-F", current["ssh_config_file"], "-T", "-o", "ConnectTimeout=10", current["ssh_target"], "/bin/sh -s", @@ -8911,28 +8930,29 @@ class FabricHostAdapter: except TypeError as exc: raise FabricError( "宿主机 SSH runner 不支持 Fabric 命令输入", 503, - "fabric_runner_unsupported") from exc + "fabric_runner_unsupported", safe_phase) from exc except RemoteHostError as exc: raise FabricError( "宿主机 Fabric SSH 请求失败", 502, - "fabric_host_connection_failed") from exc + "fabric_host_connection_failed", safe_phase) from exc if result.returncode != 0: # stderr may contain remote paths or attacker-controlled data and # is deliberately not returned or persisted. raise FabricError( "宿主机 Fabric 命令失败", 502, - "fabric_remote_command_failed") + "fabric_remote_command_failed", safe_phase) return result.stdout def identity(self, *, create_key: bool = True) -> str: parsed = _parse_markers(self._run( - _identity_script(create_key=create_key))) + _identity_script(create_key=create_key), + failure_phase="identity")) try: return validate_wireguard_key(parsed["public_key"]) except KeyError as exc: raise FabricError( "宿主机未返回 WireGuard 公钥", 502, - "fabric_identity_invalid") from exc + "fabric_identity_invalid", "identity") from exc def _deployment_mode(self) -> str: return _profile_network_mode( @@ -8962,11 +8982,11 @@ class FabricHostAdapter: plan, host, public_keys, deployment_mode=mode, firewall_mode=firewall_mode, ifup_path=ifup_path, ifquery_path=ifquery_path, - ifdown_path=ifdown_path)) + ifdown_path=ifdown_path), failure_phase="apply") if "__VC_FABRIC_APPLIED__" not in output: raise FabricError( "宿主机未确认 Fabric 应用", 502, - "fabric_apply_unconfirmed") + "fabric_apply_unconfirmed", "apply") def observe(self, plan: FabricPlan, host: FabricNode, public_keys: Mapping[str, str]) -> dict[str, Any]: @@ -8984,7 +9004,8 @@ class FabricHostAdapter: connection_uuid=_fabric_connection_uuid(plan, host), host_id=host.host_id, fabric_id=plan.fabric_id, overlay_cidr=plan.overlay_cidr, - underlay_endpoint=host.endpoint.rsplit(":", 1)[0]))) + underlay_endpoint=host.endpoint.rsplit(":", 1)[0]), + failure_phase="observe")) expected_peers = { (public_keys[peer.host_id], peer.endpoint, peer.allowed_ip) for peer in plan.nodes if peer.host_id != host.host_id @@ -9839,6 +9860,12 @@ class FabricCoordinator: "last_error_code"): if key in raw: node[key] = raw[key] + failure_phase = _fabric_failure_phase( + raw.get("last_error_phase")) + if (failure_phase is not None + and isinstance(raw.get("last_error_code"), str) + and raw.get("last_error_code")): + node["last_error_phase"] = failure_phase observed = raw.get("observed") if isinstance(observed, Mapping): safe_observed: dict[str, Any] = {} @@ -10264,13 +10291,16 @@ class FabricCoordinator: public_keys: dict[str, str] = {} for node in plan.nodes: component, profile = components[node.host_id] - public_keys[node.host_id] = await asyncio.to_thread( - FabricHostAdapter( - component.manager, profile).identity) + try: + public_keys[node.host_id] = await asyncio.to_thread( + FabricHostAdapter( + component.manager, profile).identity) + except FabricError as exc: + raise exc.at_phase("identity") if len(set(public_keys.values())) != len(public_keys): raise FabricError( "Fabric 宿主机公钥不能重复", 409, - "fabric_identity_conflict") + "fabric_identity_conflict", "identity") if (previous is not None and plan.revision == previous.get("revision")): previous_keys = previous.get("public_keys") @@ -10278,7 +10308,7 @@ class FabricCoordinator: or previous_keys != public_keys): raise FabricError( "相同 revision 的宿主机公钥已变化,请提升 revision", - 409, "fabric_identity_conflict") + 409, "fabric_identity_conflict", "identity") desired: dict[str, Any] = { "schema_version": FABRIC_SCHEMA_VERSION, @@ -10317,9 +10347,11 @@ class FabricCoordinator: node = next( item for item in desired["nodes"] if item["host_id"] == host.host_id) + failure_phase = "apply" try: await asyncio.to_thread( adapter.apply, plan, host, public_keys) + failure_phase = "observe" observed = await asyncio.to_thread( adapter.observe, plan, host, public_keys) node.update( @@ -10332,15 +10364,13 @@ class FabricCoordinator: "config_digest"), last_error_code=None, ) + node.pop("last_error_phase", None) except FabricError as exc: # Continue with other nodes. A healthy host is never rolled # back merely because a different participant is offline. - node.update( - state="uncertain", - transport_verified=False, - observed_revision=None, - observed_plan_digest=None, - last_error_code=exc.error_code, + self._record_node_error( + node, exc.error_code, + failure_phase=failure_phase, ) self._finish(desired) await asyncio.to_thread( @@ -10400,6 +10430,12 @@ class FabricCoordinator: "last_error_code": node.get("last_error_code"), "updated_at": updated_at, } + failure_phase = _fabric_failure_phase( + node.get("last_error_phase")) + if (failure_phase is not None + and isinstance(node.get("last_error_code"), str) + and node.get("last_error_code")): + value["last_error_phase"] = failure_phase self._save( "fabric_node", self._node_key( @@ -10500,6 +10536,7 @@ class FabricCoordinator: observed_plan_digest=details.get("config_digest"), last_error_code=None, ) + node.pop("last_error_phase", None) @staticmethod def _reconcile_observation_is_exact( @@ -10517,9 +10554,11 @@ class FabricCoordinator: ) @staticmethod - def _record_reconcile_error( + def _record_node_error( node: dict[str, Any], error_code: str, - *, clear_observation: bool = False) -> None: + *, failure_phase: str, + clear_observation: bool = False) -> None: + safe_phase = _fabric_failure_phase(failure_phase) update: dict[str, Any] = { "state": "uncertain", "transport_verified": False, @@ -10527,9 +10566,13 @@ class FabricCoordinator: "observed_plan_digest": None, "last_error_code": error_code, } + if safe_phase is not None: + update["last_error_phase"] = safe_phase if clear_observation: update["observed"] = {} node.update(update) + if safe_phase is None: + node.pop("last_error_phase", None) async def reconcile(self, fabric_id: str) -> dict[str, Any]: if not _FABRIC_KEY.fullmatch(fabric_id): @@ -10582,8 +10625,10 @@ class FabricCoordinator: observed, plan, host, public_keys): repair_targets.append((host, node, adapter)) except FabricError as exc: - self._record_reconcile_error( - node, exc.error_code, clear_observation=True) + self._record_node_error( + node, exc.error_code, + failure_phase="observe", + clear_observation=True) # A missing or incomplete deployment can make the broad # observer fail before it can emit markers. The narrow # identity probe is the recovery precondition: if that @@ -10608,17 +10653,20 @@ class FabricCoordinator: 409, "fabric_identity_conflict") except FabricError as exc: identities_valid = False - self._record_reconcile_error( - node, exc.error_code) + self._record_node_error( + node, exc.error_code, + failure_phase="identity") if identities_valid: for host, node, adapter in repair_targets: + failure_phase = "apply" try: # An admitted SSH mutation continues under the # per-Fabric lock even if the HTTP caller disconnects. # asyncio.to_thread cannot cancel its worker itself. await _await_mutation_completion(asyncio.to_thread( adapter.apply, plan, host, public_keys)) + failure_phase = "observe" observed = await asyncio.to_thread( adapter.observe, plan, host, public_keys) self._record_reconcile_observation(node, observed) @@ -10626,8 +10674,9 @@ class FabricCoordinator: # All identities were proven before the first write; # an apply/readback failure on one node must not stop # the remaining nodes from converging. - self._record_reconcile_error( - node, exc.error_code) + self._record_node_error( + node, exc.error_code, + failure_phase=failure_phase) # A VPC/Fabric catalog write may have completed while remote # observation was in progress. Re-enter the shared gate and # recheck resource exclusivity before publishing active. Drain diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index 6d08a63a..4530891e 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -1090,9 +1090,14 @@ class EngineApplication: return raise FabricError("内部资源不存在", 404) except FabricError as exc: + payload = { + "error": str(exc), + "error_code": exc.error_code, + } + if exc.failure_phase is not None: + payload["failure_phase"] = exc.failure_phase await json_response( - send, exc.status, - {"error": str(exc), "error_code": exc.error_code}, + send, exc.status, payload, ) async def _vpc_dispatch( diff --git a/web/app/Services/Cloud/CloudAdminFabricService.php b/web/app/Services/Cloud/CloudAdminFabricService.php index ea54218a..e1866316 100644 --- a/web/app/Services/Cloud/CloudAdminFabricService.php +++ b/web/app/Services/Cloud/CloudAdminFabricService.php @@ -35,6 +35,7 @@ final class CloudAdminFabricService private const MAX_READINESS_HOSTS = 128; private const HOST_STATES = ['active', 'draining', 'disabled']; private const ENDPOINT_SOURCES = ['configured', 'ssh_host_ipv4']; + private const ERROR_PHASES = ['identity', 'observe', 'apply']; private const ERROR_CODES = [ 'fabric_apply_unconfirmed', 'fabric_host_capability_missing', 'fabric_host_capability_probe_failed', 'fabric_host_capability_probe_invalid', @@ -1138,6 +1139,7 @@ final class CloudAdminFabricService 'transport_verified' => $observedNode['transport_verified'] ?? false, 'observed_revision' => $observedNode['observed_revision'] ?? null, 'last_error_code' => $observedNode['last_error_code'] ?? null, + 'last_error_phase' => $observedNode['last_error_phase'] ?? null, 'readiness_state' => $state, 'reasons' => $reasons, ]); @@ -1995,6 +1997,11 @@ final class CloudAdminFabricService continue; } $hostId = self::safeText($node['host_id'] ?? null, '宿主机标识', 64); + $lastErrorCode = self::errorCode($node['last_error_code'] ?? null); + $lastErrorPhase = self::errorPhase($node['last_error_phase'] ?? null); + if ($lastErrorPhase !== null && $lastErrorCode === null) { + throw new CloudError('Fabric 失败阶段缺少对应错误代码', 502); + } $row = [ 'host_id' => $hostId, 'host_name' => $hostNames[$hostId]['name'] ?? null, @@ -2007,7 +2014,8 @@ final class CloudAdminFabricService 'transport_verified' => ($node['transport_verified'] ?? false) === true, 'observed_revision' => max(0, (int) ($node['observed_revision'] ?? 0)), 'observed_plan_digest' => self::nullableDigest($node['observed_plan_digest'] ?? null), - 'last_error_code' => self::errorCode($node['last_error_code'] ?? null), + 'last_error_code' => $lastErrorCode, + 'last_error_phase' => $lastErrorPhase, 'observed' => self::observed($node['observed'] ?? []), ]; $result['nodes'][] = $row; @@ -2346,4 +2354,15 @@ final class CloudAdminFabricService } return $value; } + + private static function errorPhase($value): ?string + { + if ($value === null || $value === '') { + return null; + } + if (!is_string($value) || !in_array($value, self::ERROR_PHASES, true)) { + throw new CloudError('Fabric 失败阶段回读无效', 502); + } + return $value; + } } diff --git a/web/public/cloud/admin-fabric.js b/web/public/cloud/admin-fabric.js index f2b1da88..873a6969 100644 --- a/web/public/cloud/admin-fabric.js +++ b/web/public/cloud/admin-fabric.js @@ -84,8 +84,14 @@ const errorLabels = { fabric_region_conflict: 'Fabric 地区不能直接变更,请走迁移流程', fabric_membership_conflict: '参与宿主机集合冲突', fabric_readback_invalid: 'Fabric 回读无效', + fabric_remote_command_failed: '宿主机远程配置命令失败', fabric_revision_conflict: 'Fabric revision 冲突', }; +const lastErrorPhaseLabels = { + identity: 'identity(身份)', + observe: 'observe(回读)', + apply: 'apply(应用)', +}; const reasonLabels = { host_disabled: '宿主机未启用', host_record_missing: '面板宿主机记录缺失', @@ -203,6 +209,10 @@ function endpointSource(value) { return value === 'configured' ? '宿主机专用配置' : value === 'ssh_host_ipv4' ? 'SSH IPv4 自动回退' : '未记录'; } +function lastErrorPhaseLabel(value) { + return Object.prototype.hasOwnProperty.call(lastErrorPhaseLabels, value) + ? lastErrorPhaseLabels[value] : null; +} function sysctl(value, enabledValue) { if (value === null || value === undefined || value === '' || typeof value === 'boolean') { return '未检测'; @@ -643,11 +653,14 @@ function renderHosts(region) { '出口核验:' + flag(capabilities.public_verified, '通过', '未通过'), ])); const fabric = el('td'); + const lastErrorPhase = host.last_error_code + ? lastErrorPhaseLabel(host.last_error_phase) : null; fabric.append(compactLines([ '成员:' + flag(host.fabric_member), '状态:' + stateLabel(host.fabric_state), 'Transport:' + flag(host.transport_verified, '已验证', '未验证'), 'Observed revision:' + text(host.observed_revision), + ...(lastErrorPhase ? ['上次 Fabric 失败阶段:' + lastErrorPhase] : []), ])); row.append(identity, platform, probe, manager, wireguard, publicPath, fabric, renderReadinessCell(host)); diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 2036a54a..a71b5ed6 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026100902' scope='global'} +{assign var=venix_asset_version value='2026100903' scope='global'} diff --git a/web/tests/AdminFabricContractTest.php b/web/tests/AdminFabricContractTest.php index 1d43afd8..43d07174 100644 --- a/web/tests/AdminFabricContractTest.php +++ b/web/tests/AdminFabricContractTest.php @@ -167,6 +167,22 @@ $assert(!str_contains($service, "'private_key'") && !str_contains($service, "'ssh_command'") && !str_contains($service, "'raw_output'"), 'Fabric admin projection must not accept or expose private credentials or raw commands.'); +$assert(str_contains($service, + "private const ERROR_PHASES = ['identity', 'observe', 'apply'];") + && str_contains($service, + "\$lastErrorPhase = self::errorPhase(\$node['last_error_phase'] ?? null)") + && str_contains($service, "'last_error_phase' => \$lastErrorPhase") + && str_contains($service, + "'last_error_phase' => \$observedNode['last_error_phase'] ?? null") + && str_contains($script, + "fabric_remote_command_failed: '宿主机远程配置命令失败'") + && str_contains($script, "identity: 'identity(身份)'") + && str_contains($script, "observe: 'observe(回读)'") + && str_contains($script, "apply: 'apply(应用)'") + && str_contains($script, 'const lastErrorPhase = host.last_error_code') + && str_contains($script, "'上次 Fabric 失败阶段:' + lastErrorPhase") + && !str_contains($script, '本次探测失败阶段'), + 'Fabric failure phases are not safely projected as persisted diagnostics.'); $assert(str_contains($script, "iproute2_missing: '缺少 ip 命令(Debian 包 iproute2 / AlmaLinux 包 iproute)'") && str_contains($script, diff --git a/web/tests/AdminFabricReadinessWorkflowTest.php b/web/tests/AdminFabricReadinessWorkflowTest.php index bb016488..288bdde3 100644 --- a/web/tests/AdminFabricReadinessWorkflowTest.php +++ b/web/tests/AdminFabricReadinessWorkflowTest.php @@ -564,6 +564,47 @@ $assert(is_array($orphan) && $hk['ready'] === false && $hk['host_set_matches'] === false, 'A scheduler/Fabric membership drift was hidden or promoted to ready.'); +$legacyPhaseHost = $hostById($hk['hosts'], 1); +$assert(is_array($legacyPhaseHost) + && array_key_exists('last_error_phase', $legacyPhaseHost) + && $legacyPhaseHost['last_error_phase'] === null, + 'A mixed-version Fabric response without a failure phase was rejected.'); +$phaseStatus = $driftStatus; +foreach (['identity', 'observe', 'apply'] as $index => $phase) { + $phaseStatus['nodes'][$index]['last_error_code'] = 'fabric_remote_command_failed'; + $phaseStatus['nodes'][$index]['last_error_phase'] = $phase; +} +$phaseHistory = []; +$phaseProjection = CloudAdminFabricService::listing($actor, $engine([ + $response([$phaseStatus]), +], $phaseHistory)); +$phaseRegion = $findById($phaseProjection['regions'], 1); +$assert(is_array($phaseRegion) + && ($hostById($phaseRegion['hosts'], 1)['last_error_phase'] ?? null) === 'identity' + && ($hostById($phaseRegion['hosts'], 2)['last_error_phase'] ?? null) === 'observe' + && ($hostById($phaseRegion['hosts'], 6)['last_error_phase'] ?? null) === 'apply', + 'Reviewed Fabric failure phases were not projected to their host rows.'); +$invalidPhaseStatus = $driftStatus; +$invalidPhaseStatus['nodes'][0]['last_error_code'] = 'fabric_remote_command_failed'; +$invalidPhaseStatus['nodes'][0]['last_error_phase'] = 'probe'; +$invalidPhaseHistory = []; +$assertCloudError( + static fn () => CloudAdminFabricService::listing($actor, $engine([ + $response([$invalidPhaseStatus]), + ], $invalidPhaseHistory)), + 502, + 'An unreviewed Fabric failure phase escaped the Engine response boundary' +); +$orphanPhaseStatus = $driftStatus; +$orphanPhaseStatus['nodes'][0]['last_error_phase'] = 'apply'; +$orphanPhaseHistory = []; +$assertCloudError( + static fn () => CloudAdminFabricService::listing($actor, $engine([ + $response([$orphanPhaseStatus]), + ], $orphanPhaseHistory)), + 502, + 'A Fabric failure phase without an error code escaped the response boundary' +); $assert(($hk['workflow']['stage'] ?? null) === 'migration_required' && ($hk['workflow']['can_apply'] ?? true) === false && ($hk['workflow']['can_reconcile'] ?? true) === false @@ -1126,12 +1167,13 @@ Capsule::table('cloud_vpcs')->where('id', 299)->delete(); $uncertainStatus = $status; $uncertainStatus['state'] = 'uncertain'; $uncertainStatus['next_stage_allowed'] = false; -foreach ($uncertainStatus['nodes'] as &$uncertainNode) { +foreach ($uncertainStatus['nodes'] as $uncertainIndex => &$uncertainNode) { $uncertainNode['state'] = 'uncertain'; $uncertainNode['transport_verified'] = false; $uncertainNode['observed_revision'] = null; $uncertainNode['observed_plan_digest'] = null; $uncertainNode['last_error_code'] = 'fabric_remote_command_failed'; + $uncertainNode['last_error_phase'] = $uncertainIndex === 0 ? 'apply' : 'observe'; } unset($uncertainNode); $recoveryProbePayload = ['hosts' => [ @@ -1144,6 +1186,8 @@ $partialRecovery = CloudAdminFabricService::probeRegion('1', $actor, $engine([ $response($recoveryProbePayload), ], $history)); $partialWorkflow = $partialRecovery['region']['workflow'] ?? null; +$partialHostOne = $hostById($partialRecovery['region']['hosts'] ?? [], 1); +$partialHostTwo = $hostById($partialRecovery['region']['hosts'] ?? [], 2); $assert(($partialWorkflow['stage'] ?? null) === 'needs_reconcile' && ($partialWorkflow['can_apply'] ?? true) === false && ($partialWorkflow['can_reconcile'] ?? false) === true @@ -1152,6 +1196,9 @@ $assert(($partialWorkflow['stage'] ?? null) === 'needs_reconcile' $partialWorkflow['blocking_reasons'] ?? [], true) && ($partialRecovery['region']['plan_preview']['mode'] ?? null) === 'existing', 'A partial automatic apply was not projected as a reconcile-only recovery.'); +$assert(($partialHostOne['last_error_phase'] ?? null) === 'apply' + && ($partialHostTwo['last_error_phase'] ?? null) === 'observe', + 'A partial Fabric recovery lost the persisted failure phase diagnostic.'); $history = []; $client = $engine([ diff --git a/web/tests/AdminFabricWizardBehaviorTest.mjs b/web/tests/AdminFabricWizardBehaviorTest.mjs index 3bf77df8..9066f9e8 100644 --- a/web/tests/AdminFabricWizardBehaviorTest.mjs +++ b/web/tests/AdminFabricWizardBehaviorTest.mjs @@ -213,6 +213,36 @@ function fixture(initialRegions, canManage = true) { assert.match(copy, /wg-fabric0:未检测/); } +// A persisted Engine failure phase is shown as historical Fabric evidence, +// not as the outcome of the fresh capability probe rendered beside it. Older +// Engine responses without the optional field remain quiet. +{ + const failedHost = { + host_id: '1', host_name: 'failed', engine_region: 'hk', + fabric_member: true, fabric_state: 'uncertain', transport_verified: false, + observed_revision: 0, readiness_state: 'uncertain', + last_error_code: 'fabric_remote_command_failed', last_error_phase: 'apply', + reasons: ['fabric_remote_command_failed'], capabilities: {}, + }; + const legacyHost = { ...failedHost, host_id: '2', host_name: 'legacy' }; + delete legacyHost.last_error_phase; + const staleSuccessHost = { + ...failedHost, host_id: '3', host_name: 'active', fabric_state: 'active', + transport_verified: true, last_error_code: null, last_error_phase: 'apply', + reasons: [], + }; + const test = fixture([region(1, 'needs_reconcile', { + hosts: [failedHost, legacyHost, staleSuccessHost], + plan_preview: { ...preview(), mode: 'existing' }, + })]); + await test.respond(test.requests[0], { regions: test.initialRegions, engine_available: true }); + const copy = test.nodes['fabric-hosts-body'].textContent; + assert.equal((copy.match(/上次 Fabric 失败阶段:/g) || []).length, 1); + assert.match(copy, /上次 Fabric 失败阶段:apply(应用)/); + assert.match(copy, /宿主机远程配置命令失败/); + assert.doesNotMatch(copy, /本次探测失败阶段/); +} + // Double activation emits one apply only. A 200 response that has not reached // the authoritative active stage is never painted as success and advances to // the server-selected reconcile step. diff --git a/web/tests/CloudFirewallUiContractTest.php b/web/tests/CloudFirewallUiContractTest.php index 655d6d14..fe21a8d3 100644 --- a/web/tests/CloudFirewallUiContractTest.php +++ b/web/tests/CloudFirewallUiContractTest.php @@ -18,7 +18,7 @@ $assert(str_contains($template, 'data-instance-setting-tab="firewall"') && str_contains($template, 'id="vm-firewall-rules"'), 'Instance settings must expose the integrated firewall tab and whole-policy editor.'); -$assert(str_contains($assets, "venix_asset_version value='2026100902'"), +$assert(str_contains($assets, "venix_asset_version value='2026100903'"), 'Shared Venix asset cache key must publish the firewall JavaScript and CSS.'); $assert(str_contains($template, '反欺骗保护始终开启') diff --git a/web/tests/CloudHostFabricEndpointContractTest.php b/web/tests/CloudHostFabricEndpointContractTest.php index 1b62a64e..0e763f85 100644 --- a/web/tests/CloudHostFabricEndpointContractTest.php +++ b/web/tests/CloudHostFabricEndpointContractTest.php @@ -97,7 +97,7 @@ $assert(str_contains($fabricClient, 'function endpointSource(value)') && str_contains($fabricClient, 'firewall_competing_service_unknown') && str_contains($fabricClient, 'firewall_manager_conflict'), 'Fabric readiness UI does not distinguish explicit and SSH-IPv4 fallback endpoints.'); -$assert(str_contains($assets, "venix_asset_version value='2026100902'"), +$assert(str_contains($assets, "venix_asset_version value='2026100903'"), 'Changed host/Fabric browser assets did not advance the shared cache key.'); echo "Cloud host Fabric endpoint contracts passed.\n";