Repair fixed Fabric plans during reconcile

venix
admin 1 month ago
parent d1c0ac6749
commit 369201caf9

@ -51,6 +51,7 @@ from virsh_panel.remote_host import RemoteHostError
KEY1 = base64.b64encode(bytes([1]) * 32).decode()
KEY2 = base64.b64encode(bytes([2]) * 32).decode()
KEY3 = base64.b64encode(bytes([3]) * 32).decode()
def plan():
@ -158,9 +159,11 @@ def test_wireguard_key_rejects_the_all_zero_reserved_key():
def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
current = plan()
read_only_identity = _identity_script(create_key=False)
scripts = (
_BOOT_HELPER,
_identity_script(),
read_only_identity,
_observe_script(("100.64.1.2",), "100.64.1.1",
("203.0.113.2",), "networkmanager_nmcli",
"12345678-1234-5678-1234-567812345678"),
@ -176,7 +179,14 @@ def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
assert result.returncode == 0, result.stderr
assert "br0" not in script
assert "PRIVATE KEY" not in script
observe = scripts[2]
assert "genkey" not in read_only_identity
assert 'mktemp "$BASE/.wg-key.' not in read_only_identity
assert 'mv -f "$_tmp_key" "$KEY"' not in read_only_identity
assert "mkdir /run/virt-cloud" in read_only_identity
assert ': > "$LOCK"' in read_only_identity
assert 'exec 9>>"$LOCK"' in read_only_identity
assert '[ -d "$BASE" ]' in read_only_identity
observe = scripts[3]
assert "__VC_FABRIC_ADMIN_STATE__" in observe
assert "__VC_FABRIC_SYSCTL__ip_forward" in observe
assert "__VC_FABRIC_SYSCTL__rp_filter_default" in observe
@ -188,8 +198,13 @@ def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
assert "list set inet virt_cloud_wg_fabric vpc_tenant_ips" in observe
assert 'table!="main" && table!="local"' in observe
apply = scripts[-1]
assert KEY2 in apply and KEY1 not in apply
assert KEY2 in apply and KEY1 in apply
assert 'KEY="$BASE/wg-fabric0.key"' in apply
assert "genkey" not in apply
identity_check = '[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ]'
assert identity_check in apply
assert apply.index(identity_check) < apply.index(
"vc_fabric_quarantine down")
assert f"SYSCTL_FILE={FABRIC_SYSCTL}" in apply
assert _sysctl_text(current).rstrip() in apply
assert "wg-fabric0" in apply
@ -258,6 +273,22 @@ def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
}, "ifupdown", "/tmp/ifup")
def test_reconcile_identity_probe_recreates_only_volatile_runtime_lock():
script = _identity_script(create_key=False)
# /run is empty again after reboot, including after a pre-journal apply
# failure. Recreate its lock so key validation remains serialized, while
# never manufacturing or replacing the persistent WireGuard identity.
assert "mkdir /run/virt-cloud" in script
assert ': > "$LOCK"' in script
assert '"$FLOCK" -x 9' in script
assert "genkey" not in script
assert 'mktemp "$BASE/.wg-key.' not in script
assert 'mv -f "$_tmp_key" "$KEY"' not in script
assert '[ -d "$BASE" ]' in script
assert '[ -f "$KEY" ]' in script
def test_firewalld_scripts_use_only_panel_manifest_and_trusted_overlay_devices():
current = plan()
apply = _apply_script(
@ -955,7 +986,6 @@ def test_fixed_artifacts_publish_desired_snapshot_before_atomic_install():
("$_tmp_owner", "$OWNER", "owner"),
("$_tmp_nft", "$NFT_RULES", "nft_rules"),
("$_tmp_nft_main", "$NFT_MANIFEST", "nft_main"),
("$_tmp_key", "$KEY", "key"),
("$_tmp_helper", "$HELPER", "helper"),
("$_tmp_nm", "$NM_PROFILE", "nm_profile"),
("$_tmp_conf", "$CONF", "conf"),
@ -967,6 +997,7 @@ def test_fixed_artifacts_publish_desired_snapshot_before_atomic_install():
assert (
f'publish_desired_file "{source}" "{target}" {name}' in apply
)
assert 'publish_desired_file "$_tmp_key" "$KEY" key' not in apply
assert 'journal_file_matches "$restore_target"' in _BOOT_HELPER
assert '"$restore_name.desired"' in _BOOT_HELPER
@ -4378,8 +4409,10 @@ class FakeManager:
def __init__(self, public_key=KEY1, extra_address=False, self_route=False,
config_digest=None, ping_ok=True, endpoint_device="eth0",
host=None, fabric_endpoint_ip="",
firewall_mode=FIREWALL_MODE_NATIVE):
firewall_mode=FIREWALL_MODE_NATIVE,
live_public_key=None):
self.public_key = public_key
self.live_public_key = live_public_key or public_key
self.extra_address = extra_address
self.self_route = self_route
self.config_digest = config_digest
@ -4470,6 +4503,7 @@ class FakeManager:
if input_text.startswith("#!/bin/sh\nset -eu") and "__VC_FABRIC_PUBLIC_KEY__=" in input_text:
output = f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n"
elif "__VC_FABRIC_APPLIED__" in input_text:
self.live_public_key = self.public_key
output = "__VC_FABRIC_APPLIED__\n"
else:
now = int(time.time())
@ -4504,6 +4538,7 @@ class FakeManager:
f"__VC_FABRIC_CONFIG_ROUTE__={target}/32\n"
f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n"
"__VC_FABRIC_INTERFACE__=wireguard\n"
f"__VC_FABRIC_LIVE_PUBLIC_KEY__={self.live_public_key}\n"
"__VC_FABRIC_LISTEN__=51820\n"
f"__VC_FABRIC_ADDRESS__={address}\n"
f"{extra_address}"
@ -4699,7 +4734,23 @@ def test_adapter_rejects_endpoint_egress_drift_to_fabric_or_missing_device():
observed = FabricHostAdapter(failed_ping).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is True
assert observed["state"] == "active"
assert observed["transport_verified"] is False
def test_adapter_rejects_live_public_key_drift_from_persistent_key():
current = plan()
observed = FabricHostAdapter(FakeManager(
KEY1, live_public_key=KEY3,
)).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
assert observed["public_key"] == KEY1
assert observed["observed"]["live_public_key"] == KEY3
assert observed["config_ok"] is False
assert observed["state"] == "degraded"
class MemoryStore:
@ -5420,10 +5471,13 @@ def _fabric_components(current):
return components
def _reconcile_observation(current):
def _reconcile_observation(current, host):
public_key = KEY1 if host.host_id == "host-a" else KEY2
return {
"state": "active",
"config_ok": True,
"transport_verified": True,
"public_key": public_key,
"observed": {
"interface": FABRIC_INTERFACE,
"config_revision": current.revision,
@ -5432,6 +5486,369 @@ def _reconcile_observation(current):
}
class ReconcileRecoveryAdapter:
"""Stateful host stub for fixed-plan reconcile recovery tests."""
def __init__(self, manager, _profile=None):
self.manager = manager
def observe(self, current, host, _keys):
self.manager.reconcile_observe_calls += 1
if self.manager.reconcile_observe_error is not None:
error = self.manager.reconcile_observe_error
if self.manager.reconcile_observe_error_once:
self.manager.reconcile_observe_error = None
raise error
if self.manager.reconcile_configured:
result = _reconcile_observation(current, host)
result["public_key"] = self.manager.public_key
result["transport_verified"] = (
self.manager.reconcile_transport_verified)
return result
return {
"state": "degraded",
"config_ok": False,
"transport_verified": False,
"public_key": self.manager.public_key,
"observed": {
"interface": None,
"config_revision": None,
"config_digest": None,
"emergency_state": "owned",
},
}
def identity(self, *, create_key=True):
self.manager.reconcile_identity_calls += 1
self.manager.reconcile_identity_create_flags.append(create_key)
entered = self.manager.reconcile_identity_entered
release = self.manager.reconcile_identity_release
if entered is not None:
entered.set()
if release is not None:
assert release.wait(timeout=2)
if self.manager.reconcile_identity_error is not None:
raise self.manager.reconcile_identity_error
return self.manager.public_key
def apply(self, *_args):
self.manager.reconcile_apply_calls += 1
entered = self.manager.reconcile_apply_entered
release = self.manager.reconcile_apply_release
if entered is not None:
entered.set()
if release is not None:
assert release.wait(timeout=2)
if self.manager.reconcile_apply_error is not None:
raise self.manager.reconcile_apply_error
self.manager.reconcile_configured = True
def _recovery_components(current, *, configured=False):
components = _fabric_components(current)
for component in components.values():
manager = component.manager
manager.reconcile_configured = configured
manager.reconcile_transport_verified = True
manager.reconcile_observe_calls = 0
manager.reconcile_observe_error = None
manager.reconcile_observe_error_once = False
manager.reconcile_identity_calls = 0
manager.reconcile_identity_create_flags = []
manager.reconcile_identity_error = None
manager.reconcile_identity_entered = None
manager.reconcile_identity_release = None
manager.reconcile_apply_calls = 0
manager.reconcile_apply_error = None
manager.reconcile_apply_entered = None
manager.reconcile_apply_release = None
return components
def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
first = components["host-a"].manager
first.reconcile_observe_error = FabricError(
"incomplete", 502, "fabric_remote_command_failed")
first.reconcile_observe_error_once = True
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "active"
assert result["next_stage_allowed"] is True
assert all(component.manager.reconcile_identity_calls == 1
for component in components.values())
assert all(component.manager.reconcile_apply_calls == 1
for component in components.values())
assert first.reconcile_observe_calls == 2
def test_reconcile_recovers_key_and_emergency_only_then_is_idempotent(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
first = asyncio.run(coordinator.reconcile(current.fabric_id))
second = asyncio.run(coordinator.reconcile(current.fabric_id))
assert first["state"] == "active"
assert first["next_stage_allowed"] is True
assert second["state"] == "active"
assert second["next_stage_allowed"] is True
for component in components.values():
manager = component.manager
assert manager.reconcile_apply_calls == 1
assert manager.reconcile_identity_calls == 1
assert manager.reconcile_identity_create_flags == [False]
# initial degraded + post-apply exact + second reconcile exact
assert manager.reconcile_observe_calls == 3
def test_reconcile_does_not_reapply_exact_config_for_stale_transport(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current, configured=True)
for component in components.values():
component.manager.reconcile_transport_verified = False
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
for component in components.values():
manager = component.manager
assert manager.reconcile_apply_calls == 0
assert manager.reconcile_identity_calls == 0
assert manager.reconcile_observe_calls == 1
def test_reconcile_does_not_reapply_exact_config_for_failed_ping():
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _fabric_components(current)
for component in components.values():
component.manager.ping_ok = False
coordinator = FabricCoordinator(store, Registry(components))
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert all(node["state"] == "active" for node in result["nodes"])
assert all(node["transport_verified"] is False
for node in result["nodes"])
# One script is the initial observe. An identity probe or apply would add
# another script and prove reconcile mistook liveness for config drift.
assert all(len(component.manager.scripts) == 1
for component in components.values())
def test_reconcile_repairs_live_public_key_drift():
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _fabric_components(current)
drifted = components["host-a"].manager
drifted.live_public_key = KEY3
coordinator = FabricCoordinator(store, Registry(components))
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "active"
assert result["next_stage_allowed"] is True
assert drifted.live_public_key == KEY1
# observe -> fixed-key identity -> apply -> exact post-observe
assert len(drifted.scripts) == 4
# The already exact peer receives only its initial observation.
assert len(components["host-b"].manager.scripts) == 1
@pytest.mark.parametrize("identity_failure", ["drift", "unavailable"])
def test_reconcile_identity_failure_blocks_all_host_writes(
monkeypatch, identity_failure):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
failed = components["host-b"].manager
expected_error = "fabric_identity_conflict"
if identity_failure == "drift":
failed.public_key = KEY3
else:
expected_error = "fabric_remote_command_failed"
failed.reconcile_identity_error = FabricError(
"offline", 502, expected_error)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
nodes = {node["host_id"]: node for node in result["nodes"]}
assert result["state"] == "uncertain"
assert result["next_stage_allowed"] is False
assert nodes["host-b"]["last_error_code"] == expected_error
for component in components.values():
manager = component.manager
assert manager.reconcile_identity_calls == 1
assert manager.reconcile_identity_create_flags == [False]
assert manager.reconcile_apply_calls == 0
def test_reconcile_apply_failure_on_one_node_does_not_stop_other_node(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
components["host-a"].manager.reconcile_apply_error = FabricError(
"offline", 502, "fabric_remote_command_failed")
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
nodes = {node["host_id"]: node for node in result["nodes"]}
assert result["state"] == "uncertain"
assert nodes["host-a"]["last_error_code"] == (
"fabric_remote_command_failed")
assert nodes["host-b"]["state"] == "active"
assert nodes["host-b"]["observed_plan_digest"] == current.digest
assert all(component.manager.reconcile_identity_calls == 1
for component in components.values())
assert all(component.manager.reconcile_apply_calls == 1
for component in components.values())
def test_cancelled_reconcile_drains_admitted_host_apply_before_unlock(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
entered = threading.Event()
release = threading.Event()
first = components["host-a"].manager
first.reconcile_apply_entered = entered
first.reconcile_apply_release = release
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
async def exercise():
reconcile_task = asyncio.create_task(
coordinator.reconcile(current.fabric_id))
for _attempt in range(100):
if entered.is_set():
break
await asyncio.sleep(0.01)
assert entered.is_set()
lock_entered = asyncio.Event()
async def wait_for_fabric_lock():
async with coordinator._locks[current.fabric_id]:
lock_entered.set()
waiter = asyncio.create_task(wait_for_fabric_lock())
reconcile_task.cancel()
await asyncio.sleep(0)
assert reconcile_task.done() is False
assert lock_entered.is_set() is False
release.set()
with pytest.raises(asyncio.CancelledError):
await reconcile_task
await asyncio.wait_for(waiter, timeout=1)
assert lock_entered.is_set() is True
try:
asyncio.run(exercise())
finally:
release.set()
assert first.reconcile_apply_calls == 1
assert first.reconcile_configured is True
assert components["host-b"].manager.reconcile_apply_calls == 0
def test_cancelled_reconcile_drains_runtime_identity_lock_before_unlock(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
entered = threading.Event()
release = threading.Event()
first = components["host-a"].manager
first.reconcile_identity_entered = entered
first.reconcile_identity_release = release
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
async def exercise():
reconcile_task = asyncio.create_task(
coordinator.reconcile(current.fabric_id))
for _attempt in range(100):
if entered.is_set():
break
await asyncio.sleep(0.01)
assert entered.is_set()
lock_entered = asyncio.Event()
async def wait_for_fabric_lock():
async with coordinator._locks[current.fabric_id]:
lock_entered.set()
waiter = asyncio.create_task(wait_for_fabric_lock())
reconcile_task.cancel()
await asyncio.sleep(0)
assert reconcile_task.done() is False
assert lock_entered.is_set() is False
release.set()
with pytest.raises(asyncio.CancelledError):
await reconcile_task
await asyncio.wait_for(waiter, timeout=1)
assert lock_entered.is_set() is True
try:
asyncio.run(exercise())
finally:
release.set()
assert first.reconcile_identity_calls == 1
assert first.reconcile_apply_calls == 0
assert components["host-b"].manager.reconcile_identity_calls == 0
assert components["host-b"].manager.reconcile_apply_calls == 0
def _vpc_payload_for_fabric(current, *, vpc_id="vpc-race"):
return {
"schema_version": 1,
@ -5487,10 +5904,10 @@ def test_reconcile_waits_for_admitted_vpc_apply_before_invalidating(
def __init__(self, _manager, _profile=None):
pass
def observe(self, *_args):
def observe(self, _plan, host, _keys):
observe_entered.set()
assert observe_release.wait(timeout=2)
return _reconcile_observation(current)
return _reconcile_observation(current, host)
vpc._apply_locked = blocked_vpc_apply
monkeypatch.setattr(
@ -5559,10 +5976,10 @@ def test_reconcile_first_durable_invalidation_blocks_vpc_host_mutation(
def __init__(self, _manager, _profile=None):
pass
def observe(self, *_args):
def observe(self, _plan, host, _keys):
observe_entered.set()
assert observe_release.wait(timeout=2)
return _reconcile_observation(current)
return _reconcile_observation(current, host)
store = BlockingInvalidationStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
@ -5648,7 +6065,7 @@ def test_reconcile_releases_catalog_during_remote_observe_and_relocks_publish(
def __init__(self, _manager, _profile=None):
pass
def observe(self, *_args):
def observe(self, _plan, host, _keys):
nonlocal observation_count
observe_entered.set()
assert observe_release.wait(timeout=2)
@ -5656,7 +6073,7 @@ def test_reconcile_releases_catalog_during_remote_observe_and_relocks_publish(
observation_count += 1
if observation_count == len(current.nodes):
all_observed.set()
return _reconcile_observation(current)
return _reconcile_observation(current, host)
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter)
@ -5719,8 +6136,8 @@ def test_cancelled_reconcile_drains_root_save_before_catalog_unlock(
def __init__(self, _manager, _profile=None):
pass
def observe(self, *_args):
return _reconcile_observation(current)
def observe(self, _plan, host, _keys):
return _reconcile_observation(current, host)
store = BlockingRootStore()
store.set("fabric_desired", current.fabric_id, json.dumps(

@ -6794,7 +6794,58 @@ def _nm_profile_template(plan: FabricPlan, host: FabricNode,
return "\n".join(lines) + "\n"
def _identity_script() -> str:
def _identity_script(*, create_key: bool = True) -> str:
if create_key:
runtime_lock_setup = '''if [ -e /run/virt-cloud ]; then
[ -d /run/virt-cloud ] && [ ! -L /run/virt-cloud ] || exit 71
[ "$($STAT -c '%u:%a' /run/virt-cloud 2>/dev/null)" = 0:700 ] || exit 71
else
mkdir /run/virt-cloud
chmod 700 /run/virt-cloud
fi
[ ! -L "$LOCK" ] || exit 71
if [ ! -e "$LOCK" ]; then : > "$LOCK"; chmod 600 "$LOCK"; fi
[ -f "$LOCK" ] && [ "$($STAT -c '%u:%a' "$LOCK" 2>/dev/null)" = 0:600 ] || exit 71
exec 9>>"$LOCK"'''
lock_command = '"$FLOCK" -x 9 || exit 79'
base_setup = '''mkdir -p "$BASE"
chmod 700 "$BASE"'''
key_setup = '''if [ ! -f "$KEY" ]; then
umask 077
_tmp_key=$(mktemp "$BASE/.wg-key.XXXXXX")
"$WG" genkey > "$_tmp_key"
chmod 600 "$_tmp_key"
mv -f "$_tmp_key" "$KEY"
_tmp_key=
else
[ "$($STAT -c '%u' "$KEY" 2>/dev/null)" = 0 ] || exit 71
[ "$($STAT -c '%a' "$KEY" 2>/dev/null)" = 600 ] || exit 71
fi'''
else:
# Reconcile already owns a durable public-key set. Verifying that set
# must never manufacture a replacement private key: a missing or
# changed identity is drift and blocks host writes for that node.
# /run is volatile. A reboot after a pre-journal failure retains the
# fixed private key but loses its serialization metadata, so it is
# safe and necessary to recreate only the root-owned runtime lock.
# BASE and KEY remain strictly read-only in this mode.
runtime_lock_setup = '''if [ -e /run/virt-cloud ]; then
[ -d /run/virt-cloud ] && [ ! -L /run/virt-cloud ] || exit 71
[ "$($STAT -c '%u:%a' /run/virt-cloud 2>/dev/null)" = 0:700 ] || exit 71
else
mkdir /run/virt-cloud
chmod 700 /run/virt-cloud
fi
[ ! -L "$LOCK" ] || exit 71
if [ ! -e "$LOCK" ]; then : > "$LOCK"; chmod 600 "$LOCK"; fi
[ -f "$LOCK" ] && [ "$($STAT -c '%u:%a' "$LOCK" 2>/dev/null)" = 0:600 ] || exit 71
exec 9>>"$LOCK"'''
lock_command = '"$FLOCK" -x 9 || exit 79'
base_setup = '''[ -d "$BASE" ] && [ ! -L "$BASE" ] || exit 71
[ "$($STAT -c '%u:%a' "$BASE" 2>/dev/null)" = 0:700 ] || exit 71'''
key_setup = '''[ -f "$KEY" ] && [ ! -L "$KEY" ] || exit 71
[ "$($STAT -c '%u' "$KEY" 2>/dev/null)" = 0 ] || exit 71
[ "$($STAT -c '%a' "$KEY" 2>/dev/null)" = 600 ] || exit 71'''
return f'''#!/bin/sh
set -eu
umask 077
@ -6817,18 +6868,8 @@ STAT=$(find_tool /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat) || exit 70
AWK=$(find_tool /usr/bin/awk /bin/awk /usr/sbin/awk /sbin/awk) || exit 70
FLOCK=$(find_tool /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock) || exit 70
if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi
if [ -e /run/virt-cloud ]; then
[ -d /run/virt-cloud ] && [ ! -L /run/virt-cloud ] || exit 71
[ "$($STAT -c '%u:%a' /run/virt-cloud 2>/dev/null)" = 0:700 ] || exit 71
else
mkdir /run/virt-cloud
chmod 700 /run/virt-cloud
fi
[ ! -L "$LOCK" ] || exit 71
if [ ! -e "$LOCK" ]; then : > "$LOCK"; chmod 600 "$LOCK"; fi
[ -f "$LOCK" ] && [ "$($STAT -c '%u:%a' "$LOCK" 2>/dev/null)" = 0:600 ] || exit 71
exec 9>>"$LOCK"
"$FLOCK" -x 9 || exit 79
{runtime_lock_setup}
{lock_command}
if [ -f "$OWNER" ] && [ "$(cat "$OWNER")" != {_q(FABRIC_OWNER_MARKER)} ]; then exit 71; fi
if [ -f "$OWNER" ]; then
[ "$($STAT -c '%u' "$OWNER" 2>/dev/null)" = 0 ] || exit 71
@ -6838,20 +6879,9 @@ if "$IP" link show dev "$IFACE" >/dev/null 2>&1; then
[ -f "$OWNER" ] && [ "$(cat "$OWNER")" = {_q(FABRIC_OWNER_MARKER)} ] || exit 71
"$WG" show "$IFACE" >/dev/null 2>&1 || exit 71
fi
mkdir -p "$BASE"
chmod 700 "$BASE"
{base_setup}
if [ -L "$KEY" ]; then exit 71; fi
if [ ! -f "$KEY" ]; then
umask 077
_tmp_key=$(mktemp "$BASE/.wg-key.XXXXXX")
"$WG" genkey > "$_tmp_key"
chmod 600 "$_tmp_key"
mv -f "$_tmp_key" "$KEY"
_tmp_key=
else
[ "$($STAT -c '%u' "$KEY" 2>/dev/null)" = 0 ] || exit 71
[ "$($STAT -c '%a' "$KEY" 2>/dev/null)" = 600 ] || exit 71
fi
{key_setup}
printf '%s\\n' "__VC_FABRIC_PUBLIC_KEY__=$($WG pubkey < "$KEY")"
'''
@ -7271,6 +7301,7 @@ SYSCTL_FILE={_q(FABRIC_SYSCTL)}
NFT_MAIN=
PLAN_DIGEST={_q(plan.digest)}
EXPECTED_DIGEST="$PLAN_DIGEST"
EXPECTED_PUBLIC_KEY={_q(public_keys[host.host_id])}
DEPLOYMENT_MODE={_q(deployment_mode)}
FIREWALL_MODE={_q(firewall_mode)}
CONNECTION={_q(FABRIC_CONNECTION)}
@ -7317,6 +7348,12 @@ if [ ! -e "$LOCK" ]; then : > "$LOCK"; chmod 600 "$LOCK"; fi
exec 9>>"$LOCK"
"$FLOCK" -x 9 || exit 79
[ ! -L "$LOCK" ] && [ "$($STAT -c '%u:%a' "$LOCK" 2>/dev/null)" = 0:600 ] || exit 71
# Identity collection always precedes apply. Revalidate the fixed local
# public key under the same host lock before quarantine or any persistent
# mutation; apply must never manufacture a replacement identity.
[ -f "$KEY" ] && [ ! -L "$KEY" ] || exit 71
[ "$($STAT -c '%u:%a' "$KEY" 2>/dev/null)" = 0:600 ] || exit 71
[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ] || exit 71
# Close a proven panel-owned overlay before mutable firewall, sysctl, config,
# artifact, or nft preflight. NetworkManager devices are made unmanaged first
# so the daemon cannot race link-down with reactivation.
@ -7604,15 +7641,9 @@ if [ "$FIREWALL_MODE" = firewalld_nft ]; then
[ "$(vc_firewall_rule_state permanent "$OLD_UNDERLAY_ZONE" "$OLD_UNDERLAY_RULE")" = no ] || exit 71
fi
fi
if [ ! -f "$KEY" ]; then
umask 077
_tmp_key=$(mktemp "$BASE/.wg-key.XXXXXX")
"$WG" genkey > "$_tmp_key"
chmod 600 "$_tmp_key"
publish_desired_file "$_tmp_key" "$KEY" key || exit 76
_tmp_key=
fi
chmod 600 "$KEY"
[ -f "$KEY" ] && [ ! -L "$KEY" ] || exit 71
[ "$($STAT -c '%u:%a' "$KEY" 2>/dev/null)" = 0:600 ] || exit 71
[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ] || exit 71
if [ "$DEPLOYMENT_MODE" = networkmanager_nmcli ]; then
[ ! -L /etc/NetworkManager ] && [ ! -L /etc/NetworkManager/system-connections ] || exit 71
[ ! -e "$IFUPDOWN_FRAGMENT" ] && [ ! -L "$IFUPDOWN_FRAGMENT" ] || exit 71
@ -8454,6 +8485,7 @@ fi
[ -f "$KEY" ] && printf '%s\\n' "__VC_FABRIC_PUBLIC_KEY__=$($WG pubkey < "$KEY")"
if "$IP" link show dev "$IFACE" >/dev/null 2>&1 && "$WG" show "$IFACE" >/dev/null 2>&1; then
printf '%s\\n' '__VC_FABRIC_INTERFACE__=wireguard'
printf '%s\\n' "__VC_FABRIC_LIVE_PUBLIC_KEY__=$($WG show "$IFACE" public-key 2>/dev/null)"
printf '%s\\n' "__VC_FABRIC_LISTEN__=$($WG show "$IFACE" listen-port 2>/dev/null)"
$IP -o -4 addr show dev "$IFACE" 2>/dev/null | awk '{{for (i=1;i<=NF;i++) if ($i=="inet") {{split($(i+1),a,"/"); print a[1]}}}}' | while IFS= read -r address; do
[ -n "$address" ] || continue
@ -8668,6 +8700,8 @@ def _parse_markers(output: str) -> dict[str, Any]:
"fabric_readback_invalid")
if line.startswith("__VC_FABRIC_PUBLIC_KEY__="):
result["public_key"] = line.split("=", 1)[1]
elif line.startswith("__VC_FABRIC_LIVE_PUBLIC_KEY__="):
result["live_public_key"] = line.split("=", 1)[1]
elif line.startswith("__VC_FABRIC_OWNER__="):
result["owner"] = line.split("=", 1)[1]
elif line.startswith("__VC_FABRIC_CONFIG_FABRIC__="):
@ -8890,8 +8924,9 @@ class FabricHostAdapter:
"fabric_remote_command_failed")
return result.stdout
def identity(self) -> str:
parsed = _parse_markers(self._run(_identity_script()))
def identity(self, *, create_key: bool = True) -> str:
parsed = _parse_markers(self._run(
_identity_script(create_key=create_key)))
try:
return validate_wireguard_key(parsed["public_key"])
except KeyError as exc:
@ -9177,6 +9212,7 @@ class FabricHostAdapter:
and parsed.get("config_ok") == "yes"
and parsed.get("interface") == "wireguard"
and parsed.get("public_key") == public_keys[host.host_id]
and parsed.get("live_public_key") == public_keys[host.host_id]
and str(parsed.get("listen_port", "")) == str(host.listen_port)
and actual_addresses == expected_addresses
and overlay_peers == expected_peers
@ -9187,7 +9223,6 @@ class FabricHostAdapter:
and actual_routes == expected_routes
and actual_paths == expected_paths
and endpoint_paths_ok
and actual_pings == expected_pings
and actual_route_overlaps == expected_route_overlaps
and parsed.get("enabled") == "enabled"
# Apply releases its flock and synchronously runs the independent
@ -9227,10 +9262,14 @@ class FabricHostAdapter:
handshakes[peer_key] = int(raw)
except (TypeError, ValueError):
handshakes[peer_key] = 0
transport_ok = bool(expected_peers) and all(
0 < handshakes.get(peer_key, 0) <= now + 60
and now - handshakes.get(peer_key, 0) <= HANDSHAKE_MAX_AGE
for peer_key, _endpoint, _allowed in expected_peers
transport_ok = (
bool(expected_peers)
and actual_pings == expected_pings
and all(
0 < handshakes.get(peer_key, 0) <= now + 60
and now - handshakes.get(peer_key, 0) <= HANDSHAKE_MAX_AGE
for peer_key, _endpoint, _allowed in expected_peers
)
)
return {
"state": "active" if config_ok else "degraded",
@ -9249,6 +9288,7 @@ class FabricHostAdapter:
"config_overlay": parsed.get("config_overlay"),
"config_ok": parsed.get("config_ok"),
"interface": parsed.get("interface"),
"live_public_key": parsed.get("live_public_key"),
"listen_port": parsed.get("listen_port"),
"addresses": sorted(actual_addresses),
"peers": sorted(actual_peers),
@ -9805,7 +9845,7 @@ class FabricCoordinator:
for key in (
"owner", "config_fabric", "config_revision",
"config_digest", "config_network", "config_overlay",
"config_ok", "interface",
"config_ok", "interface", "live_public_key",
"listen_port", "enabled",
"service_active", "timer_enabled",
"timer_active", "nft_guard", "key_secure",
@ -10436,6 +10476,61 @@ class FabricCoordinator:
await asyncio.to_thread(
self._save, "fabric_desired", plan.fabric_id, desired)
@staticmethod
def _record_reconcile_observation(
node: dict[str, Any], observed: Mapping[str, Any]) -> None:
if not isinstance(observed, Mapping):
raise FabricError(
"宿主机 Fabric 回读结果无效", 502,
"fabric_readback_invalid")
details = observed.get("observed")
state = observed.get("state")
transport_verified = observed.get("transport_verified")
if (state not in {"active", "degraded"}
or not isinstance(transport_verified, bool)
or not isinstance(details, Mapping)):
raise FabricError(
"宿主机 Fabric 回读结果无效", 502,
"fabric_readback_invalid")
node.update(
state=state,
transport_verified=transport_verified,
observed=dict(details),
observed_revision=details.get("config_revision"),
observed_plan_digest=details.get("config_digest"),
last_error_code=None,
)
@staticmethod
def _reconcile_observation_is_exact(
observed: Mapping[str, Any], plan: FabricPlan,
host: FabricNode, public_keys: Mapping[str, str]) -> bool:
details = observed.get("observed")
return bool(
observed.get("state") == "active"
and observed.get("config_ok") is True
and observed.get("public_key") == public_keys[host.host_id]
and isinstance(details, Mapping)
and str(details.get("config_revision", ""))
== str(plan.revision)
and details.get("config_digest") == plan.digest
)
@staticmethod
def _record_reconcile_error(
node: dict[str, Any], error_code: str,
*, clear_observation: bool = False) -> None:
update: dict[str, Any] = {
"state": "uncertain",
"transport_verified": False,
"observed_revision": None,
"observed_plan_digest": None,
"last_error_code": error_code,
}
if clear_observation:
update["observed"] = {}
node.update(update)
async def reconcile(self, fabric_id: str) -> dict[str, Any]:
if not _FABRIC_KEY.fullmatch(fabric_id):
raise FabricError("Fabric 标识无效")
@ -10460,36 +10555,79 @@ class FabricCoordinator:
host_id: validate_wireguard_key(key)
for host_id, key in public_keys.items()
}
if len(set(public_keys.values())) != len(public_keys):
raise FabricError(
"Fabric 公钥记录包含重复身份", 500,
"fabric_record_corrupt")
# Reconcile is a recovery operation for the already fixed plan,
# not a second plan generator. First read every host. An exact
# active configuration (including the fixed local public key) is
# never rewritten merely because its peer handshake is stale.
repair_targets: list[
tuple[FabricNode, dict[str, Any], FabricHostAdapter]
] = []
for host in plan.nodes:
node = next(
item for item in desired["nodes"]
if item["host_id"] == host.host_id)
try:
component, profile = components[host.host_id]
adapter = FabricHostAdapter(
component.manager, profile)
observed = await asyncio.to_thread(
FabricHostAdapter(
component.manager, profile).observe,
plan, host, public_keys)
node.update(
state=observed["state"],
transport_verified=observed[
"transport_verified"],
observed=observed["observed"],
observed_revision=observed["observed"].get(
"config_revision"),
observed_plan_digest=observed["observed"].get(
"config_digest"),
last_error_code=None,
)
adapter.observe, plan, host, public_keys)
self._record_reconcile_observation(node, observed)
if not self._reconcile_observation_is_exact(
observed, plan, host, public_keys):
repair_targets.append((host, node, adapter))
except FabricError as exc:
self._record_reconcile_error(
node, exc.error_code, clear_observation=True)
# A missing or incomplete deployment can make the broad
# observer fail before it can emit markers. The narrow
# identity probe is the recovery precondition: if that
# succeeds for every repair target, the apply script can
# safely validate and converge its owned emergency state.
repair_targets.append((host, node, adapter))
# Verify every repair candidate without creating a replacement
# key before the first dataplane/config mutation. An unreachable
# or changed identity therefore prevents every dataplane/config
# apply. The probe may recreate only its volatile root-owned
# runtime lock.
identities_valid = True
for host, node, adapter in repair_targets:
try:
public_key = await _await_mutation_completion(
asyncio.to_thread(
adapter.identity, create_key=False))
if public_key != public_keys[host.host_id]:
raise FabricError(
"宿主机 WireGuard 身份与固定计划不一致",
409, "fabric_identity_conflict")
except FabricError as exc:
node.update(
state="uncertain",
transport_verified=False,
observed_revision=None,
observed_plan_digest=None,
last_error_code=exc.error_code,
)
identities_valid = False
self._record_reconcile_error(
node, exc.error_code)
if identities_valid:
for host, node, adapter in repair_targets:
try:
# An admitted SSH mutation continues under the
# per-Fabric lock even if the HTTP caller disconnects.
# asyncio.to_thread cannot cancel its worker itself.
await _await_mutation_completion(asyncio.to_thread(
adapter.apply, plan, host, public_keys))
observed = await asyncio.to_thread(
adapter.observe, plan, host, public_keys)
self._record_reconcile_observation(node, observed)
except FabricError as exc:
# All identities were proven before the first write;
# an apply/readback failure on one node must not stop
# the remaining nodes from converging.
self._record_reconcile_error(
node, exc.error_code)
# A VPC/Fabric catalog write may have completed while remote
# observation was in progress. Re-enter the shared gate and
# recheck resource exclusivity before publishing active. Drain

@ -1123,13 +1123,25 @@ Capsule::table('cloud_vpcs')->where('id', 299)->delete();
// A deterministic desired Fabric left by a partial apply is recoverable only
// through the region reconcile action after another fresh detection.
$uncertainStatus = $status;
$uncertainStatus['state'] = 'uncertain';
$uncertainStatus['next_stage_allowed'] = false;
foreach ($uncertainStatus['nodes'] as &$uncertainNode) {
$uncertainNode['state'] = 'uncertain';
$uncertainNode['transport_verified'] = false;
$uncertainNode['observed_revision'] = null;
$uncertainNode['observed_plan_digest'] = null;
$uncertainNode['last_error_code'] = 'fabric_remote_command_failed';
}
unset($uncertainNode);
$recoveryProbePayload = ['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]];
$history = [];
$partialRecovery = CloudAdminFabricService::probeRegion('1', $actor, $engine([
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response([$uncertainStatus]),
$response($recoveryProbePayload),
], $history));
$partialWorkflow = $partialRecovery['region']['workflow'] ?? null;
$assert(($partialWorkflow['stage'] ?? null) === 'needs_reconcile'
@ -1147,7 +1159,7 @@ $client = $engine([
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response([$status]),
$response([$uncertainStatus]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
@ -1167,6 +1179,61 @@ $assert($partialApplyPaths === [
&& CloudRegionFabric::where('region_id', 1)->first() === null,
'Partial-success apply recovery reached an engine write or created a mapping.');
// Reconcile may safely replay only the fixed desired plan, but an uncertain
// engine response and exact readback still cannot publish a platform mapping.
$history = [];
$client = $engine([
$response($uncertainStatus),
$response([$uncertainStatus]),
$response($recoveryProbePayload),
$response($uncertainStatus),
$response($uncertainStatus),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'An uncertain replay result published an active region mapping'
);
$uncertainPaths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($uncertainPaths === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric/fabric-hk',
]
&& !in_array('POST /internal/v1/fabric/apply', $uncertainPaths, true)
&& CloudRegionFabric::where('region_id', 1)->first() === null,
'Uncertain replay either repeated apply or created a platform mapping.');
$history = [];
$client = $engine([
$response($uncertainStatus),
$response([$uncertainStatus]),
$response($recoveryProbePayload),
new ConnectException('timeout', new Request(
'POST', 'http://127.0.0.1:8000/internal/v1/fabric/fabric-hk/reconcile'
)),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
502,
'A lost replay response published an active region mapping'
);
$unknownReplayPaths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($unknownReplayPaths === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
]
&& CloudRegionFabric::where('region_id', 1)->first() === null,
'Unknown replay outcome created a platform mapping.');
// A deterministic name is not proof of ownership. With no panel mapping,
// recovery must reject every foreign desired-plan dimension before issuing
// the host-writing reconcile request.
@ -1179,10 +1246,6 @@ $foreignOverlay['nodes'][0]['overlay_ip'] = $foreignOverlayBase . '1';
$foreignOverlay['nodes'][1]['overlay_ip'] = $foreignOverlayBase . '2';
$foreignEndpoint = $status;
$foreignEndpoint['nodes'][0]['endpoint'] = '203.0.113.250:51820';
$recoveryProbePayload = ['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]];
$foreignCases = [
'revision' => [$foreignRevision, false],
'overlay' => [$foreignOverlay, true],
@ -1220,8 +1283,8 @@ foreach ($foreignCases as $label => [$foreign, $needsListAndProbe]) {
$history = [];
$configured = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([
$response($status),
$response([$status]),
$response($uncertainStatus),
$response([$uncertainStatus]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),

Loading…
Cancel
Save