Record Fabric remote failure phases

venix
admin 1 month ago
parent 369201caf9
commit 82c1e533bf

@ -29,6 +29,7 @@ from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, Hos
LogStore, MainStore, TRUSTED_ACTOR)
from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor,
signed_headers)
from virsh_panel.fabric import FabricError
from virsh_panel.internal import EngineApplication, HostRegistry
from virsh_panel.libvirt_service import (
FIREWALL_FILTER_COMPILER_VERSION, VMError, build_firewall_filter_xml,
@ -353,6 +354,24 @@ def test_fabric_is_engine_global_and_super_admin_write_only(boundary):
invalid = call(client, "POST", "/internal/v1/fabric/apply", SUPER, {})
assert invalid.status_code == 400
assert invalid.json()["error_code"] == "fabric_invalid"
assert "failure_phase" not in invalid.json()
async def failed_identity(_value):
raise FabricError(
"宿主机 Fabric 命令失败", 502,
"fabric_remote_command_failed", "identity",
)
with patch.object(engine.fabric, "apply", failed_identity):
identity_failed = call(
client, "POST", "/internal/v1/fabric/apply", SUPER, {},
)
assert identity_failed.status_code == 502
assert identity_failed.json() == {
"error": "宿主机 Fabric 命令失败",
"error_code": "fabric_remote_command_failed",
"failure_phase": "identity",
}
assert call(
client, "POST", "/internal/v1/fabric/probe", USER,
{"host_ids": ["host-a"]},

@ -70,6 +70,26 @@ def plan():
})
def plan_payload(current=None):
current = current or plan()
return {
"fabric_id": current.fabric_id,
"region": current.region,
"overlay_cidr": current.overlay_cidr,
"revision": current.revision,
"listen_port": current.listen_port,
"nodes": [
{
"host_id": node.host_id,
"overlay_ip": node.overlay_ip,
"endpoint": node.endpoint,
"listen_port": node.listen_port,
}
for node in current.nodes
],
}
def test_plan_is_canonical_and_rejects_wide_or_tenant_routes():
first = plan()
reordered = parse_fabric_plan({
@ -4592,6 +4612,40 @@ def test_adapter_readback_requires_exact_peer_route_and_persistence():
)["transport_verified"] is True
@pytest.mark.parametrize("failure_phase", ["identity", "apply", "observe"])
def test_adapter_remote_failure_exposes_only_whitelisted_phase(failure_phase):
class FailedRemoteManager(FakeManager):
def _exec(self, argv, *, timeout, input_text=None):
self.scripts.append(input_text)
return subprocess.CompletedProcess(
argv, 71, "", "secret /root/path private-key-material")
current = plan()
adapter = FabricHostAdapter(FailedRemoteManager(KEY1))
with pytest.raises(FabricError) as error:
if failure_phase == "identity":
adapter.identity()
elif failure_phase == "apply":
adapter.apply(
current, current.nodes[0],
{"host-a": KEY1, "host-b": KEY2})
else:
adapter.observe(
current, current.nodes[0],
{"host-a": KEY1, "host-b": KEY2})
assert error.value.error_code == "fabric_remote_command_failed"
assert error.value.failure_phase == failure_phase
safe_error = json.dumps({
"message": str(error.value),
"error_code": error.value.error_code,
"failure_phase": error.value.failure_phase,
})
assert "secret" not in safe_error
assert "/root/path" not in safe_error
assert "private-key-material" not in safe_error
def test_adapter_pins_firewalld_mode_through_apply_and_observe():
current = plan()
manager = FakeManager(KEY1, firewall_mode=FIREWALL_MODE_FIREWALLD)
@ -4770,6 +4824,7 @@ class MemoryStore:
class FakeAdapter:
failures = set()
observe_failures = set()
def __init__(self, manager, profile=None):
self.manager = manager
@ -4782,7 +4837,8 @@ class FakeAdapter:
raise FabricError("offline", 502, "fabric_remote_command_failed")
def observe(self, current, host, keys):
if self.manager.host_id in self.failures:
if (self.manager.host_id in self.failures
or self.manager.host_id in self.observe_failures):
raise FabricError("offline", 502, "fabric_remote_command_failed")
return {
"state": "active", "transport_verified": True,
@ -5347,6 +5403,12 @@ def test_coordinator_keeps_successful_hosts_and_blocks_next_stage(monkeypatch):
assert result["next_stage_allowed"] is False
states = {node["host_id"]: node["state"] for node in result["nodes"]}
assert states == {"host-a": "active", "host-b": "uncertain"}
failed = next(
node for node in result["nodes"] if node["host_id"] == "host-b")
assert failed["last_error_phase"] == "apply"
persisted = json.loads(store.get(
"fabric_node", f"{result['fabric_id']}.host-b"))
assert persisted["last_error_phase"] == "apply"
assert managers["host-a"].reprobe_calls == 1
assert managers["host-b"].reprobe_calls == 1
serialized = "\n".join(store.values.values())
@ -5355,6 +5417,52 @@ def test_coordinator_keeps_successful_hosts_and_blocks_next_stage(monkeypatch):
FakeAdapter.failures = set()
def test_initial_apply_identity_failure_carries_safe_phase_without_record(
monkeypatch):
current = plan()
store = MemoryStore()
components = _fabric_components(current)
class IdentityFailureAdapter(FakeAdapter):
def identity(self):
raise FabricError(
"safe", 502, "fabric_remote_command_failed")
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", IdentityFailureAdapter)
with pytest.raises(FabricError) as error:
asyncio.run(FabricCoordinator(
store, Registry(components)).apply(plan_payload(current)))
assert error.value.error_code == "fabric_remote_command_failed"
assert error.value.failure_phase == "identity"
assert store.get("fabric_desired", current.fabric_id) is None
def test_apply_records_post_apply_observe_failure_phase(monkeypatch):
current = plan()
store = MemoryStore()
components = _fabric_components(current)
monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", FakeAdapter)
FakeAdapter.failures = set()
FakeAdapter.observe_failures = {"host-a"}
try:
result = asyncio.run(FabricCoordinator(
store, Registry(components)).apply(plan_payload(current)))
finally:
FakeAdapter.observe_failures = set()
nodes = {node["host_id"]: node for node in result["nodes"]}
assert nodes["host-a"]["last_error_code"] == (
"fabric_remote_command_failed")
assert nodes["host-a"]["last_error_phase"] == "observe"
assert "last_error_phase" not in nodes["host-b"]
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.host-a"))
assert persisted["last_error_phase"] == "observe"
def test_recover_does_not_replay_an_interrupted_write():
store = MemoryStore()
desired = {"fabric_id": "hk-fabric", "state": "applying",
@ -5389,6 +5497,8 @@ def test_public_status_does_not_echo_nested_secrets_or_commands():
"command": "rm -rf /",
"nodes": [{
"host_id": "host-a", "public_key": KEY1,
"last_error_code": "fabric_remote_command_failed",
"last_error_phase": "apply",
"private_key": "secret",
"observed": {
"interface": FABRIC_INTERFACE,
@ -5396,13 +5506,26 @@ def test_public_status_does_not_echo_nested_secrets_or_commands():
"peers": [(KEY2, "203.0.113.2:51820", "100.64.1.2/32")],
"handshakes": {KEY2: 123},
},
}, {
"host_id": "host-b", "public_key": KEY2,
"last_error_phase": "observe",
}, {
"host_id": "host-c", "public_key": KEY3,
"last_error_code": "fabric_remote_command_failed",
"last_error_phase": "stderr:/secret/path",
}],
})
serialized = json.dumps(public)
assert public["listen_port"] == 51820
assert "private_key" not in serialized
assert "command" not in serialized
assert "command" not in public
assert "command" not in public["nodes"][0]["observed"]
assert KEY1 in serialized and KEY2 in serialized
assert public["nodes"][0]["last_error_phase"] == "apply"
assert "last_error_phase" not in public["nodes"][1]
assert "last_error_phase" not in public["nodes"][2]
assert FabricError(
"safe", failure_phase="stderr:/secret/path").failure_phase is None
def test_reconcile_rejects_a_tampered_plan_digest():
@ -5565,6 +5688,30 @@ def _recovery_components(current, *, configured=False):
return components
def test_successful_apply_clears_stale_failure_phase():
current = plan()
store = MemoryStore()
record = _active_fabric_record(current)
for node in record["nodes"]:
node["last_error_code"] = "fabric_remote_command_failed"
node["last_error_phase"] = "observe"
store.set("fabric_desired", current.fabric_id, json.dumps(record))
coordinator = FabricCoordinator(
store, Registry(_fabric_components(current)))
result = asyncio.run(coordinator.apply(plan_payload(current)))
assert result["state"] == "active"
assert all("last_error_phase" not in node for node in result["nodes"])
durable = json.loads(store.get("fabric_desired", current.fabric_id))
assert all("last_error_phase" not in node for node in durable["nodes"])
for host in current.nodes:
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.{host.host_id}"))
assert "last_error_phase" not in persisted
assert persisted["last_error_code"] is None
def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
monkeypatch):
current = plan()
@ -5584,6 +5731,7 @@ def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
assert result["state"] == "active"
assert result["next_stage_allowed"] is True
assert all("last_error_phase" not in node for node in result["nodes"])
assert all(component.manager.reconcile_identity_calls == 1
for component in components.values())
assert all(component.manager.reconcile_apply_calls == 1
@ -5591,6 +5739,94 @@ def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
assert first.reconcile_observe_calls == 2
def test_reconcile_persists_post_apply_observe_failure_phase(monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
failed = components["host-a"].manager
class PostApplyObserveFailureAdapter(ReconcileRecoveryAdapter):
def observe(self, plan_value, host, keys):
if self.manager is failed and self.manager.reconcile_configured:
self.manager.reconcile_observe_calls += 1
raise FabricError(
"incomplete", 502, "fabric_remote_command_failed")
return super().observe(plan_value, host, keys)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter",
PostApplyObserveFailureAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
nodes = {node["host_id"]: node for node in result["nodes"]}
assert result["state"] == "uncertain"
assert nodes["host-a"]["last_error_code"] == (
"fabric_remote_command_failed")
assert nodes["host-a"]["last_error_phase"] == "observe"
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.host-a"))
assert persisted["last_error_phase"] == "observe"
assert failed.reconcile_observe_calls == 2
assert failed.reconcile_apply_calls == 1
def test_reconcile_identity_gate_keeps_other_host_observe_phase(monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
observed_failed = components["host-a"].manager
identity_failed = components["host-b"].manager
observed_failed.reconcile_observe_error = FabricError(
"incomplete", 502, "fabric_remote_command_failed")
identity_failed.reconcile_identity_error = FabricError(
"offline", 502, "fabric_remote_command_failed")
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
nodes = {node["host_id"]: node for node in result["nodes"]}
assert nodes["host-a"]["last_error_phase"] == "observe"
assert nodes["host-b"]["last_error_phase"] == "identity"
assert all(component.manager.reconcile_apply_calls == 0
for component in components.values())
def test_successful_reconcile_observation_clears_stale_failure_phase(
monkeypatch):
current = plan()
store = MemoryStore()
record = _active_fabric_record(current)
for node in record["nodes"]:
node["last_error_code"] = "fabric_remote_command_failed"
node["last_error_phase"] = "apply"
store.set("fabric_desired", current.fabric_id, json.dumps(record))
components = _recovery_components(current, configured=True)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "active"
assert all(node["last_error_code"] is None for node in result["nodes"])
assert all("last_error_phase" not in node for node in result["nodes"])
durable = json.loads(store.get("fabric_desired", current.fabric_id))
assert all("last_error_phase" not in node for node in durable["nodes"])
for host in current.nodes:
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.{host.host_id}"))
assert "last_error_phase" not in persisted
assert persisted["last_error_code"] is None
def test_reconcile_recovers_key_and_emergency_only_then_is_idempotent(
monkeypatch):
current = plan()
@ -5712,6 +5948,10 @@ def test_reconcile_identity_failure_blocks_all_host_writes(
assert result["state"] == "uncertain"
assert result["next_stage_allowed"] is False
assert nodes["host-b"]["last_error_code"] == expected_error
assert nodes["host-b"]["last_error_phase"] == "identity"
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.host-b"))
assert persisted["last_error_phase"] == "identity"
for component in components.values():
manager = component.manager
assert manager.reconcile_identity_calls == 1
@ -5738,6 +5978,7 @@ def test_reconcile_apply_failure_on_one_node_does_not_stop_other_node(
assert result["state"] == "uncertain"
assert nodes["host-a"]["last_error_code"] == (
"fabric_remote_command_failed")
assert nodes["host-a"]["last_error_phase"] == "apply"
assert nodes["host-b"]["state"] == "active"
assert nodes["host-b"]["observed_plan_digest"] == current.digest
assert all(component.manager.reconcile_identity_calls == 1

@ -62,6 +62,7 @@ FIREWALL_MODE_FIREWALLD = "firewalld_nft"
FABRIC_FIREWALL_MODES = frozenset((
FIREWALL_MODE_NATIVE, FIREWALL_MODE_FIREWALLD,
))
FABRIC_FAILURE_PHASES = frozenset(("identity", "observe", "apply"))
IFUP_PATHS = frozenset(("/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup"))
IFDOWN_PATHS = frozenset(("/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown"))
IFQUERY_PATHS = frozenset(("/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery"))
@ -710,10 +711,25 @@ class FabricError(Exception):
"""Safe error returned by the host-only fabric boundary."""
def __init__(self, message: str, status: int = 400,
error_code: str = "fabric_invalid"):
error_code: str = "fabric_invalid",
failure_phase: str | None = None):
super().__init__(message)
self.status = status
self.error_code = error_code
self.failure_phase = _fabric_failure_phase(failure_phase)
def at_phase(self, failure_phase: str) -> FabricError:
"""Attach one bounded operation phase without replacing a prior one."""
if self.failure_phase is None:
self.failure_phase = _fabric_failure_phase(failure_phase)
return self
def _fabric_failure_phase(value: Any) -> str | None:
"""Return only a public-safe Fabric operation phase."""
if isinstance(value, str) and value in FABRIC_FAILURE_PHASES:
return value
return None
@dataclass(frozen=True)
@ -8888,19 +8904,22 @@ class FabricHostAdapter:
# generation halfway through could configure a different host.
self.profile = dict(profile) if isinstance(profile, Mapping) else None
def _run(self, script: str) -> str:
def _run(self, script: str, *, failure_phase: str) -> str:
safe_phase = _fabric_failure_phase(failure_phase)
try:
current = self.profile if self.profile is not None else self.manager.current()
except RemoteHostError as exc:
raise FabricError(
"宿主机 SSH 配置不可用", 502,
"fabric_host_connection_failed") from exc
"fabric_host_connection_failed", safe_phase) from exc
if not current:
raise FabricError(
"宿主机尚未配置", 409, "fabric_host_not_configured")
"宿主机尚未配置", 409, "fabric_host_not_configured",
safe_phase)
if current.get("key_locked"):
raise FabricError(
"宿主机 SSH 私钥仍锁定", 409, "fabric_host_key_locked")
"宿主机 SSH 私钥仍锁定", 409, "fabric_host_key_locked",
safe_phase)
argv = [
self.manager.ssh_binary, "-F", current["ssh_config_file"], "-T",
"-o", "ConnectTimeout=10", current["ssh_target"], "/bin/sh -s",
@ -8911,28 +8930,29 @@ class FabricHostAdapter:
except TypeError as exc:
raise FabricError(
"宿主机 SSH runner 不支持 Fabric 命令输入", 503,
"fabric_runner_unsupported") from exc
"fabric_runner_unsupported", safe_phase) from exc
except RemoteHostError as exc:
raise FabricError(
"宿主机 Fabric SSH 请求失败", 502,
"fabric_host_connection_failed") from exc
"fabric_host_connection_failed", safe_phase) from exc
if result.returncode != 0:
# stderr may contain remote paths or attacker-controlled data and
# is deliberately not returned or persisted.
raise FabricError(
"宿主机 Fabric 命令失败", 502,
"fabric_remote_command_failed")
"fabric_remote_command_failed", safe_phase)
return result.stdout
def identity(self, *, create_key: bool = True) -> str:
parsed = _parse_markers(self._run(
_identity_script(create_key=create_key)))
_identity_script(create_key=create_key),
failure_phase="identity"))
try:
return validate_wireguard_key(parsed["public_key"])
except KeyError as exc:
raise FabricError(
"宿主机未返回 WireGuard 公钥", 502,
"fabric_identity_invalid") from exc
"fabric_identity_invalid", "identity") from exc
def _deployment_mode(self) -> str:
return _profile_network_mode(
@ -8962,11 +8982,11 @@ class FabricHostAdapter:
plan, host, public_keys, deployment_mode=mode,
firewall_mode=firewall_mode,
ifup_path=ifup_path, ifquery_path=ifquery_path,
ifdown_path=ifdown_path))
ifdown_path=ifdown_path), failure_phase="apply")
if "__VC_FABRIC_APPLIED__" not in output:
raise FabricError(
"宿主机未确认 Fabric 应用", 502,
"fabric_apply_unconfirmed")
"fabric_apply_unconfirmed", "apply")
def observe(self, plan: FabricPlan, host: FabricNode,
public_keys: Mapping[str, str]) -> dict[str, Any]:
@ -8984,7 +9004,8 @@ class FabricHostAdapter:
connection_uuid=_fabric_connection_uuid(plan, host),
host_id=host.host_id, fabric_id=plan.fabric_id,
overlay_cidr=plan.overlay_cidr,
underlay_endpoint=host.endpoint.rsplit(":", 1)[0])))
underlay_endpoint=host.endpoint.rsplit(":", 1)[0]),
failure_phase="observe"))
expected_peers = {
(public_keys[peer.host_id], peer.endpoint, peer.allowed_ip)
for peer in plan.nodes if peer.host_id != host.host_id
@ -9839,6 +9860,12 @@ class FabricCoordinator:
"last_error_code"):
if key in raw:
node[key] = raw[key]
failure_phase = _fabric_failure_phase(
raw.get("last_error_phase"))
if (failure_phase is not None
and isinstance(raw.get("last_error_code"), str)
and raw.get("last_error_code")):
node["last_error_phase"] = failure_phase
observed = raw.get("observed")
if isinstance(observed, Mapping):
safe_observed: dict[str, Any] = {}
@ -10264,13 +10291,16 @@ class FabricCoordinator:
public_keys: dict[str, str] = {}
for node in plan.nodes:
component, profile = components[node.host_id]
public_keys[node.host_id] = await asyncio.to_thread(
FabricHostAdapter(
component.manager, profile).identity)
try:
public_keys[node.host_id] = await asyncio.to_thread(
FabricHostAdapter(
component.manager, profile).identity)
except FabricError as exc:
raise exc.at_phase("identity")
if len(set(public_keys.values())) != len(public_keys):
raise FabricError(
"Fabric 宿主机公钥不能重复", 409,
"fabric_identity_conflict")
"fabric_identity_conflict", "identity")
if (previous is not None
and plan.revision == previous.get("revision")):
previous_keys = previous.get("public_keys")
@ -10278,7 +10308,7 @@ class FabricCoordinator:
or previous_keys != public_keys):
raise FabricError(
"相同 revision 的宿主机公钥已变化,请提升 revision",
409, "fabric_identity_conflict")
409, "fabric_identity_conflict", "identity")
desired: dict[str, Any] = {
"schema_version": FABRIC_SCHEMA_VERSION,
@ -10317,9 +10347,11 @@ class FabricCoordinator:
node = next(
item for item in desired["nodes"]
if item["host_id"] == host.host_id)
failure_phase = "apply"
try:
await asyncio.to_thread(
adapter.apply, plan, host, public_keys)
failure_phase = "observe"
observed = await asyncio.to_thread(
adapter.observe, plan, host, public_keys)
node.update(
@ -10332,15 +10364,13 @@ class FabricCoordinator:
"config_digest"),
last_error_code=None,
)
node.pop("last_error_phase", None)
except FabricError as exc:
# Continue with other nodes. A healthy host is never rolled
# back merely because a different participant is offline.
node.update(
state="uncertain",
transport_verified=False,
observed_revision=None,
observed_plan_digest=None,
last_error_code=exc.error_code,
self._record_node_error(
node, exc.error_code,
failure_phase=failure_phase,
)
self._finish(desired)
await asyncio.to_thread(
@ -10400,6 +10430,12 @@ class FabricCoordinator:
"last_error_code": node.get("last_error_code"),
"updated_at": updated_at,
}
failure_phase = _fabric_failure_phase(
node.get("last_error_phase"))
if (failure_phase is not None
and isinstance(node.get("last_error_code"), str)
and node.get("last_error_code")):
value["last_error_phase"] = failure_phase
self._save(
"fabric_node",
self._node_key(
@ -10500,6 +10536,7 @@ class FabricCoordinator:
observed_plan_digest=details.get("config_digest"),
last_error_code=None,
)
node.pop("last_error_phase", None)
@staticmethod
def _reconcile_observation_is_exact(
@ -10517,9 +10554,11 @@ class FabricCoordinator:
)
@staticmethod
def _record_reconcile_error(
def _record_node_error(
node: dict[str, Any], error_code: str,
*, clear_observation: bool = False) -> None:
*, failure_phase: str,
clear_observation: bool = False) -> None:
safe_phase = _fabric_failure_phase(failure_phase)
update: dict[str, Any] = {
"state": "uncertain",
"transport_verified": False,
@ -10527,9 +10566,13 @@ class FabricCoordinator:
"observed_plan_digest": None,
"last_error_code": error_code,
}
if safe_phase is not None:
update["last_error_phase"] = safe_phase
if clear_observation:
update["observed"] = {}
node.update(update)
if safe_phase is None:
node.pop("last_error_phase", None)
async def reconcile(self, fabric_id: str) -> dict[str, Any]:
if not _FABRIC_KEY.fullmatch(fabric_id):
@ -10582,8 +10625,10 @@ class FabricCoordinator:
observed, plan, host, public_keys):
repair_targets.append((host, node, adapter))
except FabricError as exc:
self._record_reconcile_error(
node, exc.error_code, clear_observation=True)
self._record_node_error(
node, exc.error_code,
failure_phase="observe",
clear_observation=True)
# A missing or incomplete deployment can make the broad
# observer fail before it can emit markers. The narrow
# identity probe is the recovery precondition: if that
@ -10608,17 +10653,20 @@ class FabricCoordinator:
409, "fabric_identity_conflict")
except FabricError as exc:
identities_valid = False
self._record_reconcile_error(
node, exc.error_code)
self._record_node_error(
node, exc.error_code,
failure_phase="identity")
if identities_valid:
for host, node, adapter in repair_targets:
failure_phase = "apply"
try:
# An admitted SSH mutation continues under the
# per-Fabric lock even if the HTTP caller disconnects.
# asyncio.to_thread cannot cancel its worker itself.
await _await_mutation_completion(asyncio.to_thread(
adapter.apply, plan, host, public_keys))
failure_phase = "observe"
observed = await asyncio.to_thread(
adapter.observe, plan, host, public_keys)
self._record_reconcile_observation(node, observed)
@ -10626,8 +10674,9 @@ class FabricCoordinator:
# All identities were proven before the first write;
# an apply/readback failure on one node must not stop
# the remaining nodes from converging.
self._record_reconcile_error(
node, exc.error_code)
self._record_node_error(
node, exc.error_code,
failure_phase=failure_phase)
# A VPC/Fabric catalog write may have completed while remote
# observation was in progress. Re-enter the shared gate and
# recheck resource exclusivity before publishing active. Drain

@ -1090,9 +1090,14 @@ class EngineApplication:
return
raise FabricError("内部资源不存在", 404)
except FabricError as exc:
payload = {
"error": str(exc),
"error_code": exc.error_code,
}
if exc.failure_phase is not None:
payload["failure_phase"] = exc.failure_phase
await json_response(
send, exc.status,
{"error": str(exc), "error_code": exc.error_code},
send, exc.status, payload,
)
async def _vpc_dispatch(

@ -35,6 +35,7 @@ final class CloudAdminFabricService
private const MAX_READINESS_HOSTS = 128;
private const HOST_STATES = ['active', 'draining', 'disabled'];
private const ENDPOINT_SOURCES = ['configured', 'ssh_host_ipv4'];
private const ERROR_PHASES = ['identity', 'observe', 'apply'];
private const ERROR_CODES = [
'fabric_apply_unconfirmed', 'fabric_host_capability_missing',
'fabric_host_capability_probe_failed', 'fabric_host_capability_probe_invalid',
@ -1138,6 +1139,7 @@ final class CloudAdminFabricService
'transport_verified' => $observedNode['transport_verified'] ?? false,
'observed_revision' => $observedNode['observed_revision'] ?? null,
'last_error_code' => $observedNode['last_error_code'] ?? null,
'last_error_phase' => $observedNode['last_error_phase'] ?? null,
'readiness_state' => $state,
'reasons' => $reasons,
]);
@ -1995,6 +1997,11 @@ final class CloudAdminFabricService
continue;
}
$hostId = self::safeText($node['host_id'] ?? null, '宿主机标识', 64);
$lastErrorCode = self::errorCode($node['last_error_code'] ?? null);
$lastErrorPhase = self::errorPhase($node['last_error_phase'] ?? null);
if ($lastErrorPhase !== null && $lastErrorCode === null) {
throw new CloudError('Fabric 失败阶段缺少对应错误代码', 502);
}
$row = [
'host_id' => $hostId,
'host_name' => $hostNames[$hostId]['name'] ?? null,
@ -2007,7 +2014,8 @@ final class CloudAdminFabricService
'transport_verified' => ($node['transport_verified'] ?? false) === true,
'observed_revision' => max(0, (int) ($node['observed_revision'] ?? 0)),
'observed_plan_digest' => self::nullableDigest($node['observed_plan_digest'] ?? null),
'last_error_code' => self::errorCode($node['last_error_code'] ?? null),
'last_error_code' => $lastErrorCode,
'last_error_phase' => $lastErrorPhase,
'observed' => self::observed($node['observed'] ?? []),
];
$result['nodes'][] = $row;
@ -2346,4 +2354,15 @@ final class CloudAdminFabricService
}
return $value;
}
private static function errorPhase($value): ?string
{
if ($value === null || $value === '') {
return null;
}
if (!is_string($value) || !in_array($value, self::ERROR_PHASES, true)) {
throw new CloudError('Fabric 失败阶段回读无效', 502);
}
return $value;
}
}

@ -84,8 +84,14 @@ const errorLabels = {
fabric_region_conflict: 'Fabric 地区不能直接变更,请走迁移流程',
fabric_membership_conflict: '参与宿主机集合冲突',
fabric_readback_invalid: 'Fabric 回读无效',
fabric_remote_command_failed: '宿主机远程配置命令失败',
fabric_revision_conflict: 'Fabric revision 冲突',
};
const lastErrorPhaseLabels = {
identity: 'identity(身份)',
observe: 'observe(回读)',
apply: 'apply(应用)',
};
const reasonLabels = {
host_disabled: '宿主机未启用',
host_record_missing: '面板宿主机记录缺失',
@ -203,6 +209,10 @@ function endpointSource(value) {
return value === 'configured' ? '宿主机专用配置'
: value === 'ssh_host_ipv4' ? 'SSH IPv4 自动回退' : '未记录';
}
function lastErrorPhaseLabel(value) {
return Object.prototype.hasOwnProperty.call(lastErrorPhaseLabels, value)
? lastErrorPhaseLabels[value] : null;
}
function sysctl(value, enabledValue) {
if (value === null || value === undefined || value === '' || typeof value === 'boolean') {
return '未检测';
@ -643,11 +653,14 @@ function renderHosts(region) {
'出口核验:' + flag(capabilities.public_verified, '通过', '未通过'),
]));
const fabric = el('td');
const lastErrorPhase = host.last_error_code
? lastErrorPhaseLabel(host.last_error_phase) : null;
fabric.append(compactLines([
'成员:' + flag(host.fabric_member),
'状态:' + stateLabel(host.fabric_state),
'Transport:' + flag(host.transport_verified, '已验证', '未验证'),
'Observed revision:' + text(host.observed_revision),
...(lastErrorPhase ? ['上次 Fabric 失败阶段:' + lastErrorPhase] : []),
]));
row.append(identity, platform, probe, manager, wireguard, publicPath, fabric,
renderReadinessCell(host));

@ -1,2 +1,2 @@
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
{assign var=venix_asset_version value='2026100902' scope='global'}
{assign var=venix_asset_version value='2026100903' scope='global'}

@ -167,6 +167,22 @@ $assert(!str_contains($service, "'private_key'")
&& !str_contains($service, "'ssh_command'")
&& !str_contains($service, "'raw_output'"),
'Fabric admin projection must not accept or expose private credentials or raw commands.');
$assert(str_contains($service,
"private const ERROR_PHASES = ['identity', 'observe', 'apply'];")
&& str_contains($service,
"\$lastErrorPhase = self::errorPhase(\$node['last_error_phase'] ?? null)")
&& str_contains($service, "'last_error_phase' => \$lastErrorPhase")
&& str_contains($service,
"'last_error_phase' => \$observedNode['last_error_phase'] ?? null")
&& str_contains($script,
"fabric_remote_command_failed: '宿主机远程配置命令失败'")
&& str_contains($script, "identity: 'identity(身份)'")
&& str_contains($script, "observe: 'observe(回读)'")
&& str_contains($script, "apply: 'apply(应用)'")
&& str_contains($script, 'const lastErrorPhase = host.last_error_code')
&& str_contains($script, "'上次 Fabric 失败阶段:' + lastErrorPhase")
&& !str_contains($script, '本次探测失败阶段'),
'Fabric failure phases are not safely projected as persisted diagnostics.');
$assert(str_contains($script,
"iproute2_missing: '缺少 ip 命令(Debian 包 iproute2 / AlmaLinux 包 iproute)'")
&& str_contains($script,

@ -564,6 +564,47 @@ $assert(is_array($orphan)
&& $hk['ready'] === false
&& $hk['host_set_matches'] === false,
'A scheduler/Fabric membership drift was hidden or promoted to ready.');
$legacyPhaseHost = $hostById($hk['hosts'], 1);
$assert(is_array($legacyPhaseHost)
&& array_key_exists('last_error_phase', $legacyPhaseHost)
&& $legacyPhaseHost['last_error_phase'] === null,
'A mixed-version Fabric response without a failure phase was rejected.');
$phaseStatus = $driftStatus;
foreach (['identity', 'observe', 'apply'] as $index => $phase) {
$phaseStatus['nodes'][$index]['last_error_code'] = 'fabric_remote_command_failed';
$phaseStatus['nodes'][$index]['last_error_phase'] = $phase;
}
$phaseHistory = [];
$phaseProjection = CloudAdminFabricService::listing($actor, $engine([
$response([$phaseStatus]),
], $phaseHistory));
$phaseRegion = $findById($phaseProjection['regions'], 1);
$assert(is_array($phaseRegion)
&& ($hostById($phaseRegion['hosts'], 1)['last_error_phase'] ?? null) === 'identity'
&& ($hostById($phaseRegion['hosts'], 2)['last_error_phase'] ?? null) === 'observe'
&& ($hostById($phaseRegion['hosts'], 6)['last_error_phase'] ?? null) === 'apply',
'Reviewed Fabric failure phases were not projected to their host rows.');
$invalidPhaseStatus = $driftStatus;
$invalidPhaseStatus['nodes'][0]['last_error_code'] = 'fabric_remote_command_failed';
$invalidPhaseStatus['nodes'][0]['last_error_phase'] = 'probe';
$invalidPhaseHistory = [];
$assertCloudError(
static fn () => CloudAdminFabricService::listing($actor, $engine([
$response([$invalidPhaseStatus]),
], $invalidPhaseHistory)),
502,
'An unreviewed Fabric failure phase escaped the Engine response boundary'
);
$orphanPhaseStatus = $driftStatus;
$orphanPhaseStatus['nodes'][0]['last_error_phase'] = 'apply';
$orphanPhaseHistory = [];
$assertCloudError(
static fn () => CloudAdminFabricService::listing($actor, $engine([
$response([$orphanPhaseStatus]),
], $orphanPhaseHistory)),
502,
'A Fabric failure phase without an error code escaped the response boundary'
);
$assert(($hk['workflow']['stage'] ?? null) === 'migration_required'
&& ($hk['workflow']['can_apply'] ?? true) === false
&& ($hk['workflow']['can_reconcile'] ?? true) === false
@ -1126,12 +1167,13 @@ Capsule::table('cloud_vpcs')->where('id', 299)->delete();
$uncertainStatus = $status;
$uncertainStatus['state'] = 'uncertain';
$uncertainStatus['next_stage_allowed'] = false;
foreach ($uncertainStatus['nodes'] as &$uncertainNode) {
foreach ($uncertainStatus['nodes'] as $uncertainIndex => &$uncertainNode) {
$uncertainNode['state'] = 'uncertain';
$uncertainNode['transport_verified'] = false;
$uncertainNode['observed_revision'] = null;
$uncertainNode['observed_plan_digest'] = null;
$uncertainNode['last_error_code'] = 'fabric_remote_command_failed';
$uncertainNode['last_error_phase'] = $uncertainIndex === 0 ? 'apply' : 'observe';
}
unset($uncertainNode);
$recoveryProbePayload = ['hosts' => [
@ -1144,6 +1186,8 @@ $partialRecovery = CloudAdminFabricService::probeRegion('1', $actor, $engine([
$response($recoveryProbePayload),
], $history));
$partialWorkflow = $partialRecovery['region']['workflow'] ?? null;
$partialHostOne = $hostById($partialRecovery['region']['hosts'] ?? [], 1);
$partialHostTwo = $hostById($partialRecovery['region']['hosts'] ?? [], 2);
$assert(($partialWorkflow['stage'] ?? null) === 'needs_reconcile'
&& ($partialWorkflow['can_apply'] ?? true) === false
&& ($partialWorkflow['can_reconcile'] ?? false) === true
@ -1152,6 +1196,9 @@ $assert(($partialWorkflow['stage'] ?? null) === 'needs_reconcile'
$partialWorkflow['blocking_reasons'] ?? [], true)
&& ($partialRecovery['region']['plan_preview']['mode'] ?? null) === 'existing',
'A partial automatic apply was not projected as a reconcile-only recovery.');
$assert(($partialHostOne['last_error_phase'] ?? null) === 'apply'
&& ($partialHostTwo['last_error_phase'] ?? null) === 'observe',
'A partial Fabric recovery lost the persisted failure phase diagnostic.');
$history = [];
$client = $engine([

@ -213,6 +213,36 @@ function fixture(initialRegions, canManage = true) {
assert.match(copy, /wg-fabric0:未检测/);
}
// A persisted Engine failure phase is shown as historical Fabric evidence,
// not as the outcome of the fresh capability probe rendered beside it. Older
// Engine responses without the optional field remain quiet.
{
const failedHost = {
host_id: '1', host_name: 'failed', engine_region: 'hk',
fabric_member: true, fabric_state: 'uncertain', transport_verified: false,
observed_revision: 0, readiness_state: 'uncertain',
last_error_code: 'fabric_remote_command_failed', last_error_phase: 'apply',
reasons: ['fabric_remote_command_failed'], capabilities: {},
};
const legacyHost = { ...failedHost, host_id: '2', host_name: 'legacy' };
delete legacyHost.last_error_phase;
const staleSuccessHost = {
...failedHost, host_id: '3', host_name: 'active', fabric_state: 'active',
transport_verified: true, last_error_code: null, last_error_phase: 'apply',
reasons: [],
};
const test = fixture([region(1, 'needs_reconcile', {
hosts: [failedHost, legacyHost, staleSuccessHost],
plan_preview: { ...preview(), mode: 'existing' },
})]);
await test.respond(test.requests[0], { regions: test.initialRegions, engine_available: true });
const copy = test.nodes['fabric-hosts-body'].textContent;
assert.equal((copy.match(/上次 Fabric 失败阶段:/g) || []).length, 1);
assert.match(copy, /上次 Fabric 失败阶段:apply(应用)/);
assert.match(copy, /宿主机远程配置命令失败/);
assert.doesNotMatch(copy, /本次探测失败阶段/);
}
// Double activation emits one apply only. A 200 response that has not reached
// the authoritative active stage is never painted as success and advances to
// the server-selected reconcile step.

@ -18,7 +18,7 @@ $assert(str_contains($template, 'data-instance-setting-tab="firewall"')
&& str_contains($template, 'id="vm-firewall-rules"'),
'Instance settings must expose the integrated firewall tab and whole-policy editor.');
$assert(str_contains($assets, "venix_asset_version value='2026100902'"),
$assert(str_contains($assets, "venix_asset_version value='2026100903'"),
'Shared Venix asset cache key must publish the firewall JavaScript and CSS.');
$assert(str_contains($template, '反欺骗保护始终开启')

@ -97,7 +97,7 @@ $assert(str_contains($fabricClient, 'function endpointSource(value)')
&& str_contains($fabricClient, 'firewall_competing_service_unknown')
&& str_contains($fabricClient, 'firewall_manager_conflict'),
'Fabric readiness UI does not distinguish explicit and SSH-IPv4 fallback endpoints.');
$assert(str_contains($assets, "venix_asset_version value='2026100902'"),
$assert(str_contains($assets, "venix_asset_version value='2026100903'"),
'Changed host/Fabric browser assets did not advance the shared cache key.');
echo "Cloud host Fabric endpoint contracts passed.\n";

Loading…
Cancel
Save