From 369201caf9c74ae642dc6216616d787ac8192e45 Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 9 Oct 2026 05:05:10 +0800 Subject: [PATCH] Repair fixed Fabric plans during reconcile --- engine/tests/test_fabric.py | 443 +++++++++++++++++- engine/virsh_panel/fabric.py | 266 ++++++++--- .../AdminFabricReadinessWorkflowTest.php | 87 +++- 3 files changed, 707 insertions(+), 89 deletions(-) diff --git a/engine/tests/test_fabric.py b/engine/tests/test_fabric.py index 019272a0..03eacdf7 100644 --- a/engine/tests/test_fabric.py +++ b/engine/tests/test_fabric.py @@ -51,6 +51,7 @@ from virsh_panel.remote_host import RemoteHostError KEY1 = base64.b64encode(bytes([1]) * 32).decode() KEY2 = base64.b64encode(bytes([2]) * 32).decode() +KEY3 = base64.b64encode(bytes([3]) * 32).decode() def plan(): @@ -158,9 +159,11 @@ def test_wireguard_key_rejects_the_all_zero_reserved_key(): def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys(): current = plan() + read_only_identity = _identity_script(create_key=False) scripts = ( _BOOT_HELPER, _identity_script(), + read_only_identity, _observe_script(("100.64.1.2",), "100.64.1.1", ("203.0.113.2",), "networkmanager_nmcli", "12345678-1234-5678-1234-567812345678"), @@ -176,7 +179,14 @@ def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys(): assert result.returncode == 0, result.stderr assert "br0" not in script assert "PRIVATE KEY" not in script - observe = scripts[2] + assert "genkey" not in read_only_identity + assert 'mktemp "$BASE/.wg-key.' not in read_only_identity + assert 'mv -f "$_tmp_key" "$KEY"' not in read_only_identity + assert "mkdir /run/virt-cloud" in read_only_identity + assert ': > "$LOCK"' in read_only_identity + assert 'exec 9>>"$LOCK"' in read_only_identity + assert '[ -d "$BASE" ]' in read_only_identity + observe = scripts[3] assert "__VC_FABRIC_ADMIN_STATE__" in observe assert "__VC_FABRIC_SYSCTL__ip_forward" in observe assert "__VC_FABRIC_SYSCTL__rp_filter_default" in observe @@ -188,8 +198,13 @@ def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys(): assert "list set inet virt_cloud_wg_fabric vpc_tenant_ips" in observe assert 'table!="main" && table!="local"' in observe apply = scripts[-1] - assert KEY2 in apply and KEY1 not in apply + assert KEY2 in apply and KEY1 in apply assert 'KEY="$BASE/wg-fabric0.key"' in apply + assert "genkey" not in apply + identity_check = '[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ]' + assert identity_check in apply + assert apply.index(identity_check) < apply.index( + "vc_fabric_quarantine down") assert f"SYSCTL_FILE={FABRIC_SYSCTL}" in apply assert _sysctl_text(current).rstrip() in apply assert "wg-fabric0" in apply @@ -258,6 +273,22 @@ def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys(): }, "ifupdown", "/tmp/ifup") +def test_reconcile_identity_probe_recreates_only_volatile_runtime_lock(): + script = _identity_script(create_key=False) + + # /run is empty again after reboot, including after a pre-journal apply + # failure. Recreate its lock so key validation remains serialized, while + # never manufacturing or replacing the persistent WireGuard identity. + assert "mkdir /run/virt-cloud" in script + assert ': > "$LOCK"' in script + assert '"$FLOCK" -x 9' in script + assert "genkey" not in script + assert 'mktemp "$BASE/.wg-key.' not in script + assert 'mv -f "$_tmp_key" "$KEY"' not in script + assert '[ -d "$BASE" ]' in script + assert '[ -f "$KEY" ]' in script + + def test_firewalld_scripts_use_only_panel_manifest_and_trusted_overlay_devices(): current = plan() apply = _apply_script( @@ -955,7 +986,6 @@ def test_fixed_artifacts_publish_desired_snapshot_before_atomic_install(): ("$_tmp_owner", "$OWNER", "owner"), ("$_tmp_nft", "$NFT_RULES", "nft_rules"), ("$_tmp_nft_main", "$NFT_MANIFEST", "nft_main"), - ("$_tmp_key", "$KEY", "key"), ("$_tmp_helper", "$HELPER", "helper"), ("$_tmp_nm", "$NM_PROFILE", "nm_profile"), ("$_tmp_conf", "$CONF", "conf"), @@ -967,6 +997,7 @@ def test_fixed_artifacts_publish_desired_snapshot_before_atomic_install(): assert ( f'publish_desired_file "{source}" "{target}" {name}' in apply ) + assert 'publish_desired_file "$_tmp_key" "$KEY" key' not in apply assert 'journal_file_matches "$restore_target"' in _BOOT_HELPER assert '"$restore_name.desired"' in _BOOT_HELPER @@ -4378,8 +4409,10 @@ class FakeManager: def __init__(self, public_key=KEY1, extra_address=False, self_route=False, config_digest=None, ping_ok=True, endpoint_device="eth0", host=None, fabric_endpoint_ip="", - firewall_mode=FIREWALL_MODE_NATIVE): + firewall_mode=FIREWALL_MODE_NATIVE, + live_public_key=None): self.public_key = public_key + self.live_public_key = live_public_key or public_key self.extra_address = extra_address self.self_route = self_route self.config_digest = config_digest @@ -4470,6 +4503,7 @@ class FakeManager: if input_text.startswith("#!/bin/sh\nset -eu") and "__VC_FABRIC_PUBLIC_KEY__=" in input_text: output = f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n" elif "__VC_FABRIC_APPLIED__" in input_text: + self.live_public_key = self.public_key output = "__VC_FABRIC_APPLIED__\n" else: now = int(time.time()) @@ -4504,6 +4538,7 @@ class FakeManager: f"__VC_FABRIC_CONFIG_ROUTE__={target}/32\n" f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n" "__VC_FABRIC_INTERFACE__=wireguard\n" + f"__VC_FABRIC_LIVE_PUBLIC_KEY__={self.live_public_key}\n" "__VC_FABRIC_LISTEN__=51820\n" f"__VC_FABRIC_ADDRESS__={address}\n" f"{extra_address}" @@ -4699,7 +4734,23 @@ def test_adapter_rejects_endpoint_egress_drift_to_fabric_or_missing_device(): observed = FabricHostAdapter(failed_ping).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) + assert observed["config_ok"] is True + assert observed["state"] == "active" + assert observed["transport_verified"] is False + + +def test_adapter_rejects_live_public_key_drift_from_persistent_key(): + current = plan() + observed = FabricHostAdapter(FakeManager( + KEY1, live_public_key=KEY3, + )).observe( + current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, + ) + + assert observed["public_key"] == KEY1 + assert observed["observed"]["live_public_key"] == KEY3 assert observed["config_ok"] is False + assert observed["state"] == "degraded" class MemoryStore: @@ -5420,10 +5471,13 @@ def _fabric_components(current): return components -def _reconcile_observation(current): +def _reconcile_observation(current, host): + public_key = KEY1 if host.host_id == "host-a" else KEY2 return { "state": "active", + "config_ok": True, "transport_verified": True, + "public_key": public_key, "observed": { "interface": FABRIC_INTERFACE, "config_revision": current.revision, @@ -5432,6 +5486,369 @@ def _reconcile_observation(current): } +class ReconcileRecoveryAdapter: + """Stateful host stub for fixed-plan reconcile recovery tests.""" + + def __init__(self, manager, _profile=None): + self.manager = manager + + def observe(self, current, host, _keys): + self.manager.reconcile_observe_calls += 1 + if self.manager.reconcile_observe_error is not None: + error = self.manager.reconcile_observe_error + if self.manager.reconcile_observe_error_once: + self.manager.reconcile_observe_error = None + raise error + if self.manager.reconcile_configured: + result = _reconcile_observation(current, host) + result["public_key"] = self.manager.public_key + result["transport_verified"] = ( + self.manager.reconcile_transport_verified) + return result + return { + "state": "degraded", + "config_ok": False, + "transport_verified": False, + "public_key": self.manager.public_key, + "observed": { + "interface": None, + "config_revision": None, + "config_digest": None, + "emergency_state": "owned", + }, + } + + def identity(self, *, create_key=True): + self.manager.reconcile_identity_calls += 1 + self.manager.reconcile_identity_create_flags.append(create_key) + entered = self.manager.reconcile_identity_entered + release = self.manager.reconcile_identity_release + if entered is not None: + entered.set() + if release is not None: + assert release.wait(timeout=2) + if self.manager.reconcile_identity_error is not None: + raise self.manager.reconcile_identity_error + return self.manager.public_key + + def apply(self, *_args): + self.manager.reconcile_apply_calls += 1 + entered = self.manager.reconcile_apply_entered + release = self.manager.reconcile_apply_release + if entered is not None: + entered.set() + if release is not None: + assert release.wait(timeout=2) + if self.manager.reconcile_apply_error is not None: + raise self.manager.reconcile_apply_error + self.manager.reconcile_configured = True + + +def _recovery_components(current, *, configured=False): + components = _fabric_components(current) + for component in components.values(): + manager = component.manager + manager.reconcile_configured = configured + manager.reconcile_transport_verified = True + manager.reconcile_observe_calls = 0 + manager.reconcile_observe_error = None + manager.reconcile_observe_error_once = False + manager.reconcile_identity_calls = 0 + manager.reconcile_identity_create_flags = [] + manager.reconcile_identity_error = None + manager.reconcile_identity_entered = None + manager.reconcile_identity_release = None + manager.reconcile_apply_calls = 0 + manager.reconcile_apply_error = None + manager.reconcile_apply_entered = None + manager.reconcile_apply_release = None + return components + + +def test_reconcile_recovers_when_incomplete_host_observe_initially_fails( + monkeypatch): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _recovery_components(current) + first = components["host-a"].manager + first.reconcile_observe_error = FabricError( + "incomplete", 502, "fabric_remote_command_failed") + first.reconcile_observe_error_once = True + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) + + result = asyncio.run(coordinator.reconcile(current.fabric_id)) + + assert result["state"] == "active" + assert result["next_stage_allowed"] is True + assert all(component.manager.reconcile_identity_calls == 1 + for component in components.values()) + assert all(component.manager.reconcile_apply_calls == 1 + for component in components.values()) + assert first.reconcile_observe_calls == 2 + + +def test_reconcile_recovers_key_and_emergency_only_then_is_idempotent( + monkeypatch): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _recovery_components(current) + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) + + first = asyncio.run(coordinator.reconcile(current.fabric_id)) + second = asyncio.run(coordinator.reconcile(current.fabric_id)) + + assert first["state"] == "active" + assert first["next_stage_allowed"] is True + assert second["state"] == "active" + assert second["next_stage_allowed"] is True + for component in components.values(): + manager = component.manager + assert manager.reconcile_apply_calls == 1 + assert manager.reconcile_identity_calls == 1 + assert manager.reconcile_identity_create_flags == [False] + # initial degraded + post-apply exact + second reconcile exact + assert manager.reconcile_observe_calls == 3 + + +def test_reconcile_does_not_reapply_exact_config_for_stale_transport( + monkeypatch): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _recovery_components(current, configured=True) + for component in components.values(): + component.manager.reconcile_transport_verified = False + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) + + result = asyncio.run(coordinator.reconcile(current.fabric_id)) + + assert result["state"] == "degraded" + assert result["next_stage_allowed"] is False + for component in components.values(): + manager = component.manager + assert manager.reconcile_apply_calls == 0 + assert manager.reconcile_identity_calls == 0 + assert manager.reconcile_observe_calls == 1 + + +def test_reconcile_does_not_reapply_exact_config_for_failed_ping(): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _fabric_components(current) + for component in components.values(): + component.manager.ping_ok = False + coordinator = FabricCoordinator(store, Registry(components)) + + result = asyncio.run(coordinator.reconcile(current.fabric_id)) + + assert result["state"] == "degraded" + assert result["next_stage_allowed"] is False + assert all(node["state"] == "active" for node in result["nodes"]) + assert all(node["transport_verified"] is False + for node in result["nodes"]) + # One script is the initial observe. An identity probe or apply would add + # another script and prove reconcile mistook liveness for config drift. + assert all(len(component.manager.scripts) == 1 + for component in components.values()) + + +def test_reconcile_repairs_live_public_key_drift(): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _fabric_components(current) + drifted = components["host-a"].manager + drifted.live_public_key = KEY3 + coordinator = FabricCoordinator(store, Registry(components)) + + result = asyncio.run(coordinator.reconcile(current.fabric_id)) + + assert result["state"] == "active" + assert result["next_stage_allowed"] is True + assert drifted.live_public_key == KEY1 + # observe -> fixed-key identity -> apply -> exact post-observe + assert len(drifted.scripts) == 4 + # The already exact peer receives only its initial observation. + assert len(components["host-b"].manager.scripts) == 1 + + +@pytest.mark.parametrize("identity_failure", ["drift", "unavailable"]) +def test_reconcile_identity_failure_blocks_all_host_writes( + monkeypatch, identity_failure): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _recovery_components(current) + failed = components["host-b"].manager + expected_error = "fabric_identity_conflict" + if identity_failure == "drift": + failed.public_key = KEY3 + else: + expected_error = "fabric_remote_command_failed" + failed.reconcile_identity_error = FabricError( + "offline", 502, expected_error) + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) + + result = asyncio.run(coordinator.reconcile(current.fabric_id)) + + nodes = {node["host_id"]: node for node in result["nodes"]} + assert result["state"] == "uncertain" + assert result["next_stage_allowed"] is False + assert nodes["host-b"]["last_error_code"] == expected_error + for component in components.values(): + manager = component.manager + assert manager.reconcile_identity_calls == 1 + assert manager.reconcile_identity_create_flags == [False] + assert manager.reconcile_apply_calls == 0 + + +def test_reconcile_apply_failure_on_one_node_does_not_stop_other_node( + monkeypatch): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _recovery_components(current) + components["host-a"].manager.reconcile_apply_error = FabricError( + "offline", 502, "fabric_remote_command_failed") + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) + + result = asyncio.run(coordinator.reconcile(current.fabric_id)) + + nodes = {node["host_id"]: node for node in result["nodes"]} + assert result["state"] == "uncertain" + assert nodes["host-a"]["last_error_code"] == ( + "fabric_remote_command_failed") + assert nodes["host-b"]["state"] == "active" + assert nodes["host-b"]["observed_plan_digest"] == current.digest + assert all(component.manager.reconcile_identity_calls == 1 + for component in components.values()) + assert all(component.manager.reconcile_apply_calls == 1 + for component in components.values()) + + +def test_cancelled_reconcile_drains_admitted_host_apply_before_unlock( + monkeypatch): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _recovery_components(current) + entered = threading.Event() + release = threading.Event() + first = components["host-a"].manager + first.reconcile_apply_entered = entered + first.reconcile_apply_release = release + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) + + async def exercise(): + reconcile_task = asyncio.create_task( + coordinator.reconcile(current.fabric_id)) + for _attempt in range(100): + if entered.is_set(): + break + await asyncio.sleep(0.01) + assert entered.is_set() + + lock_entered = asyncio.Event() + + async def wait_for_fabric_lock(): + async with coordinator._locks[current.fabric_id]: + lock_entered.set() + + waiter = asyncio.create_task(wait_for_fabric_lock()) + reconcile_task.cancel() + await asyncio.sleep(0) + assert reconcile_task.done() is False + assert lock_entered.is_set() is False + release.set() + with pytest.raises(asyncio.CancelledError): + await reconcile_task + await asyncio.wait_for(waiter, timeout=1) + assert lock_entered.is_set() is True + + try: + asyncio.run(exercise()) + finally: + release.set() + assert first.reconcile_apply_calls == 1 + assert first.reconcile_configured is True + assert components["host-b"].manager.reconcile_apply_calls == 0 + + +def test_cancelled_reconcile_drains_runtime_identity_lock_before_unlock( + monkeypatch): + current = plan() + store = MemoryStore() + store.set("fabric_desired", current.fabric_id, json.dumps( + _active_fabric_record(current))) + components = _recovery_components(current) + entered = threading.Event() + release = threading.Event() + first = components["host-a"].manager + first.reconcile_identity_entered = entered + first.reconcile_identity_release = release + coordinator = FabricCoordinator(store, Registry(components)) + monkeypatch.setattr( + "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) + + async def exercise(): + reconcile_task = asyncio.create_task( + coordinator.reconcile(current.fabric_id)) + for _attempt in range(100): + if entered.is_set(): + break + await asyncio.sleep(0.01) + assert entered.is_set() + + lock_entered = asyncio.Event() + + async def wait_for_fabric_lock(): + async with coordinator._locks[current.fabric_id]: + lock_entered.set() + + waiter = asyncio.create_task(wait_for_fabric_lock()) + reconcile_task.cancel() + await asyncio.sleep(0) + assert reconcile_task.done() is False + assert lock_entered.is_set() is False + release.set() + with pytest.raises(asyncio.CancelledError): + await reconcile_task + await asyncio.wait_for(waiter, timeout=1) + assert lock_entered.is_set() is True + + try: + asyncio.run(exercise()) + finally: + release.set() + assert first.reconcile_identity_calls == 1 + assert first.reconcile_apply_calls == 0 + assert components["host-b"].manager.reconcile_identity_calls == 0 + assert components["host-b"].manager.reconcile_apply_calls == 0 + + def _vpc_payload_for_fabric(current, *, vpc_id="vpc-race"): return { "schema_version": 1, @@ -5487,10 +5904,10 @@ def test_reconcile_waits_for_admitted_vpc_apply_before_invalidating( def __init__(self, _manager, _profile=None): pass - def observe(self, *_args): + def observe(self, _plan, host, _keys): observe_entered.set() assert observe_release.wait(timeout=2) - return _reconcile_observation(current) + return _reconcile_observation(current, host) vpc._apply_locked = blocked_vpc_apply monkeypatch.setattr( @@ -5559,10 +5976,10 @@ def test_reconcile_first_durable_invalidation_blocks_vpc_host_mutation( def __init__(self, _manager, _profile=None): pass - def observe(self, *_args): + def observe(self, _plan, host, _keys): observe_entered.set() assert observe_release.wait(timeout=2) - return _reconcile_observation(current) + return _reconcile_observation(current, host) store = BlockingInvalidationStore() store.set("fabric_desired", current.fabric_id, json.dumps( @@ -5648,7 +6065,7 @@ def test_reconcile_releases_catalog_during_remote_observe_and_relocks_publish( def __init__(self, _manager, _profile=None): pass - def observe(self, *_args): + def observe(self, _plan, host, _keys): nonlocal observation_count observe_entered.set() assert observe_release.wait(timeout=2) @@ -5656,7 +6073,7 @@ def test_reconcile_releases_catalog_during_remote_observe_and_relocks_publish( observation_count += 1 if observation_count == len(current.nodes): all_observed.set() - return _reconcile_observation(current) + return _reconcile_observation(current, host) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter) @@ -5719,8 +6136,8 @@ def test_cancelled_reconcile_drains_root_save_before_catalog_unlock( def __init__(self, _manager, _profile=None): pass - def observe(self, *_args): - return _reconcile_observation(current) + def observe(self, _plan, host, _keys): + return _reconcile_observation(current, host) store = BlockingRootStore() store.set("fabric_desired", current.fabric_id, json.dumps( diff --git a/engine/virsh_panel/fabric.py b/engine/virsh_panel/fabric.py index 3bf2ce3f..7d0fc052 100644 --- a/engine/virsh_panel/fabric.py +++ b/engine/virsh_panel/fabric.py @@ -6794,7 +6794,58 @@ def _nm_profile_template(plan: FabricPlan, host: FabricNode, return "\n".join(lines) + "\n" -def _identity_script() -> str: +def _identity_script(*, create_key: bool = True) -> str: + if create_key: + runtime_lock_setup = '''if [ -e /run/virt-cloud ]; then + [ -d /run/virt-cloud ] && [ ! -L /run/virt-cloud ] || exit 71 + [ "$($STAT -c '%u:%a' /run/virt-cloud 2>/dev/null)" = 0:700 ] || exit 71 +else + mkdir /run/virt-cloud + chmod 700 /run/virt-cloud +fi +[ ! -L "$LOCK" ] || exit 71 +if [ ! -e "$LOCK" ]; then : > "$LOCK"; chmod 600 "$LOCK"; fi +[ -f "$LOCK" ] && [ "$($STAT -c '%u:%a' "$LOCK" 2>/dev/null)" = 0:600 ] || exit 71 +exec 9>>"$LOCK"''' + lock_command = '"$FLOCK" -x 9 || exit 79' + base_setup = '''mkdir -p "$BASE" +chmod 700 "$BASE"''' + key_setup = '''if [ ! -f "$KEY" ]; then + umask 077 + _tmp_key=$(mktemp "$BASE/.wg-key.XXXXXX") + "$WG" genkey > "$_tmp_key" + chmod 600 "$_tmp_key" + mv -f "$_tmp_key" "$KEY" + _tmp_key= +else + [ "$($STAT -c '%u' "$KEY" 2>/dev/null)" = 0 ] || exit 71 + [ "$($STAT -c '%a' "$KEY" 2>/dev/null)" = 600 ] || exit 71 +fi''' + else: + # Reconcile already owns a durable public-key set. Verifying that set + # must never manufacture a replacement private key: a missing or + # changed identity is drift and blocks host writes for that node. + # /run is volatile. A reboot after a pre-journal failure retains the + # fixed private key but loses its serialization metadata, so it is + # safe and necessary to recreate only the root-owned runtime lock. + # BASE and KEY remain strictly read-only in this mode. + runtime_lock_setup = '''if [ -e /run/virt-cloud ]; then + [ -d /run/virt-cloud ] && [ ! -L /run/virt-cloud ] || exit 71 + [ "$($STAT -c '%u:%a' /run/virt-cloud 2>/dev/null)" = 0:700 ] || exit 71 +else + mkdir /run/virt-cloud + chmod 700 /run/virt-cloud +fi +[ ! -L "$LOCK" ] || exit 71 +if [ ! -e "$LOCK" ]; then : > "$LOCK"; chmod 600 "$LOCK"; fi +[ -f "$LOCK" ] && [ "$($STAT -c '%u:%a' "$LOCK" 2>/dev/null)" = 0:600 ] || exit 71 +exec 9>>"$LOCK"''' + lock_command = '"$FLOCK" -x 9 || exit 79' + base_setup = '''[ -d "$BASE" ] && [ ! -L "$BASE" ] || exit 71 +[ "$($STAT -c '%u:%a' "$BASE" 2>/dev/null)" = 0:700 ] || exit 71''' + key_setup = '''[ -f "$KEY" ] && [ ! -L "$KEY" ] || exit 71 +[ "$($STAT -c '%u' "$KEY" 2>/dev/null)" = 0 ] || exit 71 +[ "$($STAT -c '%a' "$KEY" 2>/dev/null)" = 600 ] || exit 71''' return f'''#!/bin/sh set -eu umask 077 @@ -6817,18 +6868,8 @@ STAT=$(find_tool /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat) || exit 70 AWK=$(find_tool /usr/bin/awk /bin/awk /usr/sbin/awk /sbin/awk) || exit 70 FLOCK=$(find_tool /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock) || exit 70 if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi -if [ -e /run/virt-cloud ]; then - [ -d /run/virt-cloud ] && [ ! -L /run/virt-cloud ] || exit 71 - [ "$($STAT -c '%u:%a' /run/virt-cloud 2>/dev/null)" = 0:700 ] || exit 71 -else - mkdir /run/virt-cloud - chmod 700 /run/virt-cloud -fi -[ ! -L "$LOCK" ] || exit 71 -if [ ! -e "$LOCK" ]; then : > "$LOCK"; chmod 600 "$LOCK"; fi -[ -f "$LOCK" ] && [ "$($STAT -c '%u:%a' "$LOCK" 2>/dev/null)" = 0:600 ] || exit 71 -exec 9>>"$LOCK" -"$FLOCK" -x 9 || exit 79 +{runtime_lock_setup} +{lock_command} if [ -f "$OWNER" ] && [ "$(cat "$OWNER")" != {_q(FABRIC_OWNER_MARKER)} ]; then exit 71; fi if [ -f "$OWNER" ]; then [ "$($STAT -c '%u' "$OWNER" 2>/dev/null)" = 0 ] || exit 71 @@ -6838,20 +6879,9 @@ if "$IP" link show dev "$IFACE" >/dev/null 2>&1; then [ -f "$OWNER" ] && [ "$(cat "$OWNER")" = {_q(FABRIC_OWNER_MARKER)} ] || exit 71 "$WG" show "$IFACE" >/dev/null 2>&1 || exit 71 fi -mkdir -p "$BASE" -chmod 700 "$BASE" +{base_setup} if [ -L "$KEY" ]; then exit 71; fi -if [ ! -f "$KEY" ]; then - umask 077 - _tmp_key=$(mktemp "$BASE/.wg-key.XXXXXX") - "$WG" genkey > "$_tmp_key" - chmod 600 "$_tmp_key" - mv -f "$_tmp_key" "$KEY" - _tmp_key= -else - [ "$($STAT -c '%u' "$KEY" 2>/dev/null)" = 0 ] || exit 71 - [ "$($STAT -c '%a' "$KEY" 2>/dev/null)" = 600 ] || exit 71 -fi +{key_setup} printf '%s\\n' "__VC_FABRIC_PUBLIC_KEY__=$($WG pubkey < "$KEY")" ''' @@ -7271,6 +7301,7 @@ SYSCTL_FILE={_q(FABRIC_SYSCTL)} NFT_MAIN= PLAN_DIGEST={_q(plan.digest)} EXPECTED_DIGEST="$PLAN_DIGEST" +EXPECTED_PUBLIC_KEY={_q(public_keys[host.host_id])} DEPLOYMENT_MODE={_q(deployment_mode)} FIREWALL_MODE={_q(firewall_mode)} CONNECTION={_q(FABRIC_CONNECTION)} @@ -7317,6 +7348,12 @@ if [ ! -e "$LOCK" ]; then : > "$LOCK"; chmod 600 "$LOCK"; fi exec 9>>"$LOCK" "$FLOCK" -x 9 || exit 79 [ ! -L "$LOCK" ] && [ "$($STAT -c '%u:%a' "$LOCK" 2>/dev/null)" = 0:600 ] || exit 71 +# Identity collection always precedes apply. Revalidate the fixed local +# public key under the same host lock before quarantine or any persistent +# mutation; apply must never manufacture a replacement identity. +[ -f "$KEY" ] && [ ! -L "$KEY" ] || exit 71 +[ "$($STAT -c '%u:%a' "$KEY" 2>/dev/null)" = 0:600 ] || exit 71 +[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ] || exit 71 # Close a proven panel-owned overlay before mutable firewall, sysctl, config, # artifact, or nft preflight. NetworkManager devices are made unmanaged first # so the daemon cannot race link-down with reactivation. @@ -7604,15 +7641,9 @@ if [ "$FIREWALL_MODE" = firewalld_nft ]; then [ "$(vc_firewall_rule_state permanent "$OLD_UNDERLAY_ZONE" "$OLD_UNDERLAY_RULE")" = no ] || exit 71 fi fi -if [ ! -f "$KEY" ]; then - umask 077 - _tmp_key=$(mktemp "$BASE/.wg-key.XXXXXX") - "$WG" genkey > "$_tmp_key" - chmod 600 "$_tmp_key" - publish_desired_file "$_tmp_key" "$KEY" key || exit 76 - _tmp_key= -fi -chmod 600 "$KEY" +[ -f "$KEY" ] && [ ! -L "$KEY" ] || exit 71 +[ "$($STAT -c '%u:%a' "$KEY" 2>/dev/null)" = 0:600 ] || exit 71 +[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ] || exit 71 if [ "$DEPLOYMENT_MODE" = networkmanager_nmcli ]; then [ ! -L /etc/NetworkManager ] && [ ! -L /etc/NetworkManager/system-connections ] || exit 71 [ ! -e "$IFUPDOWN_FRAGMENT" ] && [ ! -L "$IFUPDOWN_FRAGMENT" ] || exit 71 @@ -8454,6 +8485,7 @@ fi [ -f "$KEY" ] && printf '%s\\n' "__VC_FABRIC_PUBLIC_KEY__=$($WG pubkey < "$KEY")" if "$IP" link show dev "$IFACE" >/dev/null 2>&1 && "$WG" show "$IFACE" >/dev/null 2>&1; then printf '%s\\n' '__VC_FABRIC_INTERFACE__=wireguard' + printf '%s\\n' "__VC_FABRIC_LIVE_PUBLIC_KEY__=$($WG show "$IFACE" public-key 2>/dev/null)" printf '%s\\n' "__VC_FABRIC_LISTEN__=$($WG show "$IFACE" listen-port 2>/dev/null)" $IP -o -4 addr show dev "$IFACE" 2>/dev/null | awk '{{for (i=1;i<=NF;i++) if ($i=="inet") {{split($(i+1),a,"/"); print a[1]}}}}' | while IFS= read -r address; do [ -n "$address" ] || continue @@ -8668,6 +8700,8 @@ def _parse_markers(output: str) -> dict[str, Any]: "fabric_readback_invalid") if line.startswith("__VC_FABRIC_PUBLIC_KEY__="): result["public_key"] = line.split("=", 1)[1] + elif line.startswith("__VC_FABRIC_LIVE_PUBLIC_KEY__="): + result["live_public_key"] = line.split("=", 1)[1] elif line.startswith("__VC_FABRIC_OWNER__="): result["owner"] = line.split("=", 1)[1] elif line.startswith("__VC_FABRIC_CONFIG_FABRIC__="): @@ -8890,8 +8924,9 @@ class FabricHostAdapter: "fabric_remote_command_failed") return result.stdout - def identity(self) -> str: - parsed = _parse_markers(self._run(_identity_script())) + def identity(self, *, create_key: bool = True) -> str: + parsed = _parse_markers(self._run( + _identity_script(create_key=create_key))) try: return validate_wireguard_key(parsed["public_key"]) except KeyError as exc: @@ -9177,6 +9212,7 @@ class FabricHostAdapter: and parsed.get("config_ok") == "yes" and parsed.get("interface") == "wireguard" and parsed.get("public_key") == public_keys[host.host_id] + and parsed.get("live_public_key") == public_keys[host.host_id] and str(parsed.get("listen_port", "")) == str(host.listen_port) and actual_addresses == expected_addresses and overlay_peers == expected_peers @@ -9187,7 +9223,6 @@ class FabricHostAdapter: and actual_routes == expected_routes and actual_paths == expected_paths and endpoint_paths_ok - and actual_pings == expected_pings and actual_route_overlaps == expected_route_overlaps and parsed.get("enabled") == "enabled" # Apply releases its flock and synchronously runs the independent @@ -9227,10 +9262,14 @@ class FabricHostAdapter: handshakes[peer_key] = int(raw) except (TypeError, ValueError): handshakes[peer_key] = 0 - transport_ok = bool(expected_peers) and all( - 0 < handshakes.get(peer_key, 0) <= now + 60 - and now - handshakes.get(peer_key, 0) <= HANDSHAKE_MAX_AGE - for peer_key, _endpoint, _allowed in expected_peers + transport_ok = ( + bool(expected_peers) + and actual_pings == expected_pings + and all( + 0 < handshakes.get(peer_key, 0) <= now + 60 + and now - handshakes.get(peer_key, 0) <= HANDSHAKE_MAX_AGE + for peer_key, _endpoint, _allowed in expected_peers + ) ) return { "state": "active" if config_ok else "degraded", @@ -9249,6 +9288,7 @@ class FabricHostAdapter: "config_overlay": parsed.get("config_overlay"), "config_ok": parsed.get("config_ok"), "interface": parsed.get("interface"), + "live_public_key": parsed.get("live_public_key"), "listen_port": parsed.get("listen_port"), "addresses": sorted(actual_addresses), "peers": sorted(actual_peers), @@ -9805,7 +9845,7 @@ class FabricCoordinator: for key in ( "owner", "config_fabric", "config_revision", "config_digest", "config_network", "config_overlay", - "config_ok", "interface", + "config_ok", "interface", "live_public_key", "listen_port", "enabled", "service_active", "timer_enabled", "timer_active", "nft_guard", "key_secure", @@ -10436,6 +10476,61 @@ class FabricCoordinator: await asyncio.to_thread( self._save, "fabric_desired", plan.fabric_id, desired) + @staticmethod + def _record_reconcile_observation( + node: dict[str, Any], observed: Mapping[str, Any]) -> None: + if not isinstance(observed, Mapping): + raise FabricError( + "宿主机 Fabric 回读结果无效", 502, + "fabric_readback_invalid") + details = observed.get("observed") + state = observed.get("state") + transport_verified = observed.get("transport_verified") + if (state not in {"active", "degraded"} + or not isinstance(transport_verified, bool) + or not isinstance(details, Mapping)): + raise FabricError( + "宿主机 Fabric 回读结果无效", 502, + "fabric_readback_invalid") + node.update( + state=state, + transport_verified=transport_verified, + observed=dict(details), + observed_revision=details.get("config_revision"), + observed_plan_digest=details.get("config_digest"), + last_error_code=None, + ) + + @staticmethod + def _reconcile_observation_is_exact( + observed: Mapping[str, Any], plan: FabricPlan, + host: FabricNode, public_keys: Mapping[str, str]) -> bool: + details = observed.get("observed") + return bool( + observed.get("state") == "active" + and observed.get("config_ok") is True + and observed.get("public_key") == public_keys[host.host_id] + and isinstance(details, Mapping) + and str(details.get("config_revision", "")) + == str(plan.revision) + and details.get("config_digest") == plan.digest + ) + + @staticmethod + def _record_reconcile_error( + node: dict[str, Any], error_code: str, + *, clear_observation: bool = False) -> None: + update: dict[str, Any] = { + "state": "uncertain", + "transport_verified": False, + "observed_revision": None, + "observed_plan_digest": None, + "last_error_code": error_code, + } + if clear_observation: + update["observed"] = {} + node.update(update) + async def reconcile(self, fabric_id: str) -> dict[str, Any]: if not _FABRIC_KEY.fullmatch(fabric_id): raise FabricError("Fabric 标识无效") @@ -10460,36 +10555,79 @@ class FabricCoordinator: host_id: validate_wireguard_key(key) for host_id, key in public_keys.items() } + if len(set(public_keys.values())) != len(public_keys): + raise FabricError( + "Fabric 公钥记录包含重复身份", 500, + "fabric_record_corrupt") + # Reconcile is a recovery operation for the already fixed plan, + # not a second plan generator. First read every host. An exact + # active configuration (including the fixed local public key) is + # never rewritten merely because its peer handshake is stale. + repair_targets: list[ + tuple[FabricNode, dict[str, Any], FabricHostAdapter] + ] = [] for host in plan.nodes: node = next( item for item in desired["nodes"] if item["host_id"] == host.host_id) try: component, profile = components[host.host_id] + adapter = FabricHostAdapter( + component.manager, profile) observed = await asyncio.to_thread( - FabricHostAdapter( - component.manager, profile).observe, - plan, host, public_keys) - node.update( - state=observed["state"], - transport_verified=observed[ - "transport_verified"], - observed=observed["observed"], - observed_revision=observed["observed"].get( - "config_revision"), - observed_plan_digest=observed["observed"].get( - "config_digest"), - last_error_code=None, - ) + adapter.observe, plan, host, public_keys) + self._record_reconcile_observation(node, observed) + if not self._reconcile_observation_is_exact( + observed, plan, host, public_keys): + repair_targets.append((host, node, adapter)) + except FabricError as exc: + self._record_reconcile_error( + node, exc.error_code, clear_observation=True) + # A missing or incomplete deployment can make the broad + # observer fail before it can emit markers. The narrow + # identity probe is the recovery precondition: if that + # succeeds for every repair target, the apply script can + # safely validate and converge its owned emergency state. + repair_targets.append((host, node, adapter)) + + # Verify every repair candidate without creating a replacement + # key before the first dataplane/config mutation. An unreachable + # or changed identity therefore prevents every dataplane/config + # apply. The probe may recreate only its volatile root-owned + # runtime lock. + identities_valid = True + for host, node, adapter in repair_targets: + try: + public_key = await _await_mutation_completion( + asyncio.to_thread( + adapter.identity, create_key=False)) + if public_key != public_keys[host.host_id]: + raise FabricError( + "宿主机 WireGuard 身份与固定计划不一致", + 409, "fabric_identity_conflict") except FabricError as exc: - node.update( - state="uncertain", - transport_verified=False, - observed_revision=None, - observed_plan_digest=None, - last_error_code=exc.error_code, - ) + identities_valid = False + self._record_reconcile_error( + node, exc.error_code) + + if identities_valid: + for host, node, adapter in repair_targets: + try: + # An admitted SSH mutation continues under the + # per-Fabric lock even if the HTTP caller disconnects. + # asyncio.to_thread cannot cancel its worker itself. + await _await_mutation_completion(asyncio.to_thread( + adapter.apply, plan, host, public_keys)) + observed = await asyncio.to_thread( + adapter.observe, plan, host, public_keys) + self._record_reconcile_observation(node, observed) + except FabricError as exc: + # All identities were proven before the first write; + # an apply/readback failure on one node must not stop + # the remaining nodes from converging. + self._record_reconcile_error( + node, exc.error_code) # A VPC/Fabric catalog write may have completed while remote # observation was in progress. Re-enter the shared gate and # recheck resource exclusivity before publishing active. Drain diff --git a/web/tests/AdminFabricReadinessWorkflowTest.php b/web/tests/AdminFabricReadinessWorkflowTest.php index 377d2740..bb016488 100644 --- a/web/tests/AdminFabricReadinessWorkflowTest.php +++ b/web/tests/AdminFabricReadinessWorkflowTest.php @@ -1123,13 +1123,25 @@ Capsule::table('cloud_vpcs')->where('id', 299)->delete(); // A deterministic desired Fabric left by a partial apply is recoverable only // through the region reconcile action after another fresh detection. +$uncertainStatus = $status; +$uncertainStatus['state'] = 'uncertain'; +$uncertainStatus['next_stage_allowed'] = false; +foreach ($uncertainStatus['nodes'] as &$uncertainNode) { + $uncertainNode['state'] = 'uncertain'; + $uncertainNode['transport_verified'] = false; + $uncertainNode['observed_revision'] = null; + $uncertainNode['observed_plan_digest'] = null; + $uncertainNode['last_error_code'] = 'fabric_remote_command_failed'; +} +unset($uncertainNode); +$recoveryProbePayload = ['hosts' => [ + $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), + $probeHost(2), +]]; $history = []; $partialRecovery = CloudAdminFabricService::probeRegion('1', $actor, $engine([ - $response([$status]), - $response(['hosts' => [ - $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), - $probeHost(2), - ]]), + $response([$uncertainStatus]), + $response($recoveryProbePayload), ], $history)); $partialWorkflow = $partialRecovery['region']['workflow'] ?? null; $assert(($partialWorkflow['stage'] ?? null) === 'needs_reconcile' @@ -1147,7 +1159,7 @@ $client = $engine([ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), - $response([$status]), + $response([$uncertainStatus]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), @@ -1167,6 +1179,61 @@ $assert($partialApplyPaths === [ && CloudRegionFabric::where('region_id', 1)->first() === null, 'Partial-success apply recovery reached an engine write or created a mapping.'); +// Reconcile may safely replay only the fixed desired plan, but an uncertain +// engine response and exact readback still cannot publish a platform mapping. +$history = []; +$client = $engine([ + $response($uncertainStatus), + $response([$uncertainStatus]), + $response($recoveryProbePayload), + $response($uncertainStatus), + $response($uncertainStatus), +], $history); +$assertCloudError( + static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), + 409, + 'An uncertain replay result published an active region mapping' +); +$uncertainPaths = array_map(static fn (array $entry): string => + $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), + $history); +$assert($uncertainPaths === [ + 'GET /internal/v1/fabric/fabric-hk', + 'GET /internal/v1/fabric', + 'POST /internal/v1/fabric/probe', + 'POST /internal/v1/fabric/fabric-hk/reconcile', + 'GET /internal/v1/fabric/fabric-hk', + ] + && !in_array('POST /internal/v1/fabric/apply', $uncertainPaths, true) + && CloudRegionFabric::where('region_id', 1)->first() === null, + 'Uncertain replay either repeated apply or created a platform mapping.'); + +$history = []; +$client = $engine([ + $response($uncertainStatus), + $response([$uncertainStatus]), + $response($recoveryProbePayload), + new ConnectException('timeout', new Request( + 'POST', 'http://127.0.0.1:8000/internal/v1/fabric/fabric-hk/reconcile' + )), +], $history); +$assertCloudError( + static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), + 502, + 'A lost replay response published an active region mapping' +); +$unknownReplayPaths = array_map(static fn (array $entry): string => + $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), + $history); +$assert($unknownReplayPaths === [ + 'GET /internal/v1/fabric/fabric-hk', + 'GET /internal/v1/fabric', + 'POST /internal/v1/fabric/probe', + 'POST /internal/v1/fabric/fabric-hk/reconcile', + ] + && CloudRegionFabric::where('region_id', 1)->first() === null, + 'Unknown replay outcome created a platform mapping.'); + // A deterministic name is not proof of ownership. With no panel mapping, // recovery must reject every foreign desired-plan dimension before issuing // the host-writing reconcile request. @@ -1179,10 +1246,6 @@ $foreignOverlay['nodes'][0]['overlay_ip'] = $foreignOverlayBase . '1'; $foreignOverlay['nodes'][1]['overlay_ip'] = $foreignOverlayBase . '2'; $foreignEndpoint = $status; $foreignEndpoint['nodes'][0]['endpoint'] = '203.0.113.250:51820'; -$recoveryProbePayload = ['hosts' => [ - $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), - $probeHost(2), -]]; $foreignCases = [ 'revision' => [$foreignRevision, false], 'overlay' => [$foreignOverlay, true], @@ -1220,8 +1283,8 @@ foreach ($foreignCases as $label => [$foreign, $needsListAndProbe]) { $history = []; $configured = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([ - $response($status), - $response([$status]), + $response($uncertainStatus), + $response([$uncertainStatus]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2),