|
|
|
|
@ -51,6 +51,7 @@ from virsh_panel.remote_host import RemoteHostError
|
|
|
|
|
|
|
|
|
|
KEY1 = base64.b64encode(bytes([1]) * 32).decode()
|
|
|
|
|
KEY2 = base64.b64encode(bytes([2]) * 32).decode()
|
|
|
|
|
KEY3 = base64.b64encode(bytes([3]) * 32).decode()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def plan():
|
|
|
|
|
@ -158,9 +159,11 @@ def test_wireguard_key_rejects_the_all_zero_reserved_key():
|
|
|
|
|
|
|
|
|
|
def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
|
|
|
|
|
current = plan()
|
|
|
|
|
read_only_identity = _identity_script(create_key=False)
|
|
|
|
|
scripts = (
|
|
|
|
|
_BOOT_HELPER,
|
|
|
|
|
_identity_script(),
|
|
|
|
|
read_only_identity,
|
|
|
|
|
_observe_script(("100.64.1.2",), "100.64.1.1",
|
|
|
|
|
("203.0.113.2",), "networkmanager_nmcli",
|
|
|
|
|
"12345678-1234-5678-1234-567812345678"),
|
|
|
|
|
@ -176,7 +179,14 @@ def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
|
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
assert "br0" not in script
|
|
|
|
|
assert "PRIVATE KEY" not in script
|
|
|
|
|
observe = scripts[2]
|
|
|
|
|
assert "genkey" not in read_only_identity
|
|
|
|
|
assert 'mktemp "$BASE/.wg-key.' not in read_only_identity
|
|
|
|
|
assert 'mv -f "$_tmp_key" "$KEY"' not in read_only_identity
|
|
|
|
|
assert "mkdir /run/virt-cloud" in read_only_identity
|
|
|
|
|
assert ': > "$LOCK"' in read_only_identity
|
|
|
|
|
assert 'exec 9>>"$LOCK"' in read_only_identity
|
|
|
|
|
assert '[ -d "$BASE" ]' in read_only_identity
|
|
|
|
|
observe = scripts[3]
|
|
|
|
|
assert "__VC_FABRIC_ADMIN_STATE__" in observe
|
|
|
|
|
assert "__VC_FABRIC_SYSCTL__ip_forward" in observe
|
|
|
|
|
assert "__VC_FABRIC_SYSCTL__rp_filter_default" in observe
|
|
|
|
|
@ -188,8 +198,13 @@ def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
|
|
|
|
|
assert "list set inet virt_cloud_wg_fabric vpc_tenant_ips" in observe
|
|
|
|
|
assert 'table!="main" && table!="local"' in observe
|
|
|
|
|
apply = scripts[-1]
|
|
|
|
|
assert KEY2 in apply and KEY1 not in apply
|
|
|
|
|
assert KEY2 in apply and KEY1 in apply
|
|
|
|
|
assert 'KEY="$BASE/wg-fabric0.key"' in apply
|
|
|
|
|
assert "genkey" not in apply
|
|
|
|
|
identity_check = '[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ]'
|
|
|
|
|
assert identity_check in apply
|
|
|
|
|
assert apply.index(identity_check) < apply.index(
|
|
|
|
|
"vc_fabric_quarantine down")
|
|
|
|
|
assert f"SYSCTL_FILE={FABRIC_SYSCTL}" in apply
|
|
|
|
|
assert _sysctl_text(current).rstrip() in apply
|
|
|
|
|
assert "wg-fabric0" in apply
|
|
|
|
|
@ -258,6 +273,22 @@ def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
|
|
|
|
|
}, "ifupdown", "/tmp/ifup")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_identity_probe_recreates_only_volatile_runtime_lock():
|
|
|
|
|
script = _identity_script(create_key=False)
|
|
|
|
|
|
|
|
|
|
# /run is empty again after reboot, including after a pre-journal apply
|
|
|
|
|
# failure. Recreate its lock so key validation remains serialized, while
|
|
|
|
|
# never manufacturing or replacing the persistent WireGuard identity.
|
|
|
|
|
assert "mkdir /run/virt-cloud" in script
|
|
|
|
|
assert ': > "$LOCK"' in script
|
|
|
|
|
assert '"$FLOCK" -x 9' in script
|
|
|
|
|
assert "genkey" not in script
|
|
|
|
|
assert 'mktemp "$BASE/.wg-key.' not in script
|
|
|
|
|
assert 'mv -f "$_tmp_key" "$KEY"' not in script
|
|
|
|
|
assert '[ -d "$BASE" ]' in script
|
|
|
|
|
assert '[ -f "$KEY" ]' in script
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_firewalld_scripts_use_only_panel_manifest_and_trusted_overlay_devices():
|
|
|
|
|
current = plan()
|
|
|
|
|
apply = _apply_script(
|
|
|
|
|
@ -955,7 +986,6 @@ def test_fixed_artifacts_publish_desired_snapshot_before_atomic_install():
|
|
|
|
|
("$_tmp_owner", "$OWNER", "owner"),
|
|
|
|
|
("$_tmp_nft", "$NFT_RULES", "nft_rules"),
|
|
|
|
|
("$_tmp_nft_main", "$NFT_MANIFEST", "nft_main"),
|
|
|
|
|
("$_tmp_key", "$KEY", "key"),
|
|
|
|
|
("$_tmp_helper", "$HELPER", "helper"),
|
|
|
|
|
("$_tmp_nm", "$NM_PROFILE", "nm_profile"),
|
|
|
|
|
("$_tmp_conf", "$CONF", "conf"),
|
|
|
|
|
@ -967,6 +997,7 @@ def test_fixed_artifacts_publish_desired_snapshot_before_atomic_install():
|
|
|
|
|
assert (
|
|
|
|
|
f'publish_desired_file "{source}" "{target}" {name}' in apply
|
|
|
|
|
)
|
|
|
|
|
assert 'publish_desired_file "$_tmp_key" "$KEY" key' not in apply
|
|
|
|
|
assert 'journal_file_matches "$restore_target"' in _BOOT_HELPER
|
|
|
|
|
assert '"$restore_name.desired"' in _BOOT_HELPER
|
|
|
|
|
|
|
|
|
|
@ -4378,8 +4409,10 @@ class FakeManager:
|
|
|
|
|
def __init__(self, public_key=KEY1, extra_address=False, self_route=False,
|
|
|
|
|
config_digest=None, ping_ok=True, endpoint_device="eth0",
|
|
|
|
|
host=None, fabric_endpoint_ip="",
|
|
|
|
|
firewall_mode=FIREWALL_MODE_NATIVE):
|
|
|
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
|
|
|
live_public_key=None):
|
|
|
|
|
self.public_key = public_key
|
|
|
|
|
self.live_public_key = live_public_key or public_key
|
|
|
|
|
self.extra_address = extra_address
|
|
|
|
|
self.self_route = self_route
|
|
|
|
|
self.config_digest = config_digest
|
|
|
|
|
@ -4470,6 +4503,7 @@ class FakeManager:
|
|
|
|
|
if input_text.startswith("#!/bin/sh\nset -eu") and "__VC_FABRIC_PUBLIC_KEY__=" in input_text:
|
|
|
|
|
output = f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n"
|
|
|
|
|
elif "__VC_FABRIC_APPLIED__" in input_text:
|
|
|
|
|
self.live_public_key = self.public_key
|
|
|
|
|
output = "__VC_FABRIC_APPLIED__\n"
|
|
|
|
|
else:
|
|
|
|
|
now = int(time.time())
|
|
|
|
|
@ -4504,6 +4538,7 @@ class FakeManager:
|
|
|
|
|
f"__VC_FABRIC_CONFIG_ROUTE__={target}/32\n"
|
|
|
|
|
f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n"
|
|
|
|
|
"__VC_FABRIC_INTERFACE__=wireguard\n"
|
|
|
|
|
f"__VC_FABRIC_LIVE_PUBLIC_KEY__={self.live_public_key}\n"
|
|
|
|
|
"__VC_FABRIC_LISTEN__=51820\n"
|
|
|
|
|
f"__VC_FABRIC_ADDRESS__={address}\n"
|
|
|
|
|
f"{extra_address}"
|
|
|
|
|
@ -4699,7 +4734,23 @@ def test_adapter_rejects_endpoint_egress_drift_to_fabric_or_missing_device():
|
|
|
|
|
observed = FabricHostAdapter(failed_ping).observe(
|
|
|
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
|
|
|
)
|
|
|
|
|
assert observed["config_ok"] is True
|
|
|
|
|
assert observed["state"] == "active"
|
|
|
|
|
assert observed["transport_verified"] is False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_adapter_rejects_live_public_key_drift_from_persistent_key():
|
|
|
|
|
current = plan()
|
|
|
|
|
observed = FabricHostAdapter(FakeManager(
|
|
|
|
|
KEY1, live_public_key=KEY3,
|
|
|
|
|
)).observe(
|
|
|
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
assert observed["public_key"] == KEY1
|
|
|
|
|
assert observed["observed"]["live_public_key"] == KEY3
|
|
|
|
|
assert observed["config_ok"] is False
|
|
|
|
|
assert observed["state"] == "degraded"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MemoryStore:
|
|
|
|
|
@ -5420,10 +5471,13 @@ def _fabric_components(current):
|
|
|
|
|
return components
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _reconcile_observation(current):
|
|
|
|
|
def _reconcile_observation(current, host):
|
|
|
|
|
public_key = KEY1 if host.host_id == "host-a" else KEY2
|
|
|
|
|
return {
|
|
|
|
|
"state": "active",
|
|
|
|
|
"config_ok": True,
|
|
|
|
|
"transport_verified": True,
|
|
|
|
|
"public_key": public_key,
|
|
|
|
|
"observed": {
|
|
|
|
|
"interface": FABRIC_INTERFACE,
|
|
|
|
|
"config_revision": current.revision,
|
|
|
|
|
@ -5432,6 +5486,369 @@ def _reconcile_observation(current):
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ReconcileRecoveryAdapter:
|
|
|
|
|
"""Stateful host stub for fixed-plan reconcile recovery tests."""
|
|
|
|
|
|
|
|
|
|
def __init__(self, manager, _profile=None):
|
|
|
|
|
self.manager = manager
|
|
|
|
|
|
|
|
|
|
def observe(self, current, host, _keys):
|
|
|
|
|
self.manager.reconcile_observe_calls += 1
|
|
|
|
|
if self.manager.reconcile_observe_error is not None:
|
|
|
|
|
error = self.manager.reconcile_observe_error
|
|
|
|
|
if self.manager.reconcile_observe_error_once:
|
|
|
|
|
self.manager.reconcile_observe_error = None
|
|
|
|
|
raise error
|
|
|
|
|
if self.manager.reconcile_configured:
|
|
|
|
|
result = _reconcile_observation(current, host)
|
|
|
|
|
result["public_key"] = self.manager.public_key
|
|
|
|
|
result["transport_verified"] = (
|
|
|
|
|
self.manager.reconcile_transport_verified)
|
|
|
|
|
return result
|
|
|
|
|
return {
|
|
|
|
|
"state": "degraded",
|
|
|
|
|
"config_ok": False,
|
|
|
|
|
"transport_verified": False,
|
|
|
|
|
"public_key": self.manager.public_key,
|
|
|
|
|
"observed": {
|
|
|
|
|
"interface": None,
|
|
|
|
|
"config_revision": None,
|
|
|
|
|
"config_digest": None,
|
|
|
|
|
"emergency_state": "owned",
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def identity(self, *, create_key=True):
|
|
|
|
|
self.manager.reconcile_identity_calls += 1
|
|
|
|
|
self.manager.reconcile_identity_create_flags.append(create_key)
|
|
|
|
|
entered = self.manager.reconcile_identity_entered
|
|
|
|
|
release = self.manager.reconcile_identity_release
|
|
|
|
|
if entered is not None:
|
|
|
|
|
entered.set()
|
|
|
|
|
if release is not None:
|
|
|
|
|
assert release.wait(timeout=2)
|
|
|
|
|
if self.manager.reconcile_identity_error is not None:
|
|
|
|
|
raise self.manager.reconcile_identity_error
|
|
|
|
|
return self.manager.public_key
|
|
|
|
|
|
|
|
|
|
def apply(self, *_args):
|
|
|
|
|
self.manager.reconcile_apply_calls += 1
|
|
|
|
|
entered = self.manager.reconcile_apply_entered
|
|
|
|
|
release = self.manager.reconcile_apply_release
|
|
|
|
|
if entered is not None:
|
|
|
|
|
entered.set()
|
|
|
|
|
if release is not None:
|
|
|
|
|
assert release.wait(timeout=2)
|
|
|
|
|
if self.manager.reconcile_apply_error is not None:
|
|
|
|
|
raise self.manager.reconcile_apply_error
|
|
|
|
|
self.manager.reconcile_configured = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _recovery_components(current, *, configured=False):
|
|
|
|
|
components = _fabric_components(current)
|
|
|
|
|
for component in components.values():
|
|
|
|
|
manager = component.manager
|
|
|
|
|
manager.reconcile_configured = configured
|
|
|
|
|
manager.reconcile_transport_verified = True
|
|
|
|
|
manager.reconcile_observe_calls = 0
|
|
|
|
|
manager.reconcile_observe_error = None
|
|
|
|
|
manager.reconcile_observe_error_once = False
|
|
|
|
|
manager.reconcile_identity_calls = 0
|
|
|
|
|
manager.reconcile_identity_create_flags = []
|
|
|
|
|
manager.reconcile_identity_error = None
|
|
|
|
|
manager.reconcile_identity_entered = None
|
|
|
|
|
manager.reconcile_identity_release = None
|
|
|
|
|
manager.reconcile_apply_calls = 0
|
|
|
|
|
manager.reconcile_apply_error = None
|
|
|
|
|
manager.reconcile_apply_entered = None
|
|
|
|
|
manager.reconcile_apply_release = None
|
|
|
|
|
return components
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _recovery_components(current)
|
|
|
|
|
first = components["host-a"].manager
|
|
|
|
|
first.reconcile_observe_error = FabricError(
|
|
|
|
|
"incomplete", 502, "fabric_remote_command_failed")
|
|
|
|
|
first.reconcile_observe_error_once = True
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
assert result["state"] == "active"
|
|
|
|
|
assert result["next_stage_allowed"] is True
|
|
|
|
|
assert all(component.manager.reconcile_identity_calls == 1
|
|
|
|
|
for component in components.values())
|
|
|
|
|
assert all(component.manager.reconcile_apply_calls == 1
|
|
|
|
|
for component in components.values())
|
|
|
|
|
assert first.reconcile_observe_calls == 2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_recovers_key_and_emergency_only_then_is_idempotent(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _recovery_components(current)
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
|
|
|
|
|
|
first = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
second = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
assert first["state"] == "active"
|
|
|
|
|
assert first["next_stage_allowed"] is True
|
|
|
|
|
assert second["state"] == "active"
|
|
|
|
|
assert second["next_stage_allowed"] is True
|
|
|
|
|
for component in components.values():
|
|
|
|
|
manager = component.manager
|
|
|
|
|
assert manager.reconcile_apply_calls == 1
|
|
|
|
|
assert manager.reconcile_identity_calls == 1
|
|
|
|
|
assert manager.reconcile_identity_create_flags == [False]
|
|
|
|
|
# initial degraded + post-apply exact + second reconcile exact
|
|
|
|
|
assert manager.reconcile_observe_calls == 3
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_does_not_reapply_exact_config_for_stale_transport(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _recovery_components(current, configured=True)
|
|
|
|
|
for component in components.values():
|
|
|
|
|
component.manager.reconcile_transport_verified = False
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
assert result["state"] == "degraded"
|
|
|
|
|
assert result["next_stage_allowed"] is False
|
|
|
|
|
for component in components.values():
|
|
|
|
|
manager = component.manager
|
|
|
|
|
assert manager.reconcile_apply_calls == 0
|
|
|
|
|
assert manager.reconcile_identity_calls == 0
|
|
|
|
|
assert manager.reconcile_observe_calls == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_does_not_reapply_exact_config_for_failed_ping():
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _fabric_components(current)
|
|
|
|
|
for component in components.values():
|
|
|
|
|
component.manager.ping_ok = False
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
assert result["state"] == "degraded"
|
|
|
|
|
assert result["next_stage_allowed"] is False
|
|
|
|
|
assert all(node["state"] == "active" for node in result["nodes"])
|
|
|
|
|
assert all(node["transport_verified"] is False
|
|
|
|
|
for node in result["nodes"])
|
|
|
|
|
# One script is the initial observe. An identity probe or apply would add
|
|
|
|
|
# another script and prove reconcile mistook liveness for config drift.
|
|
|
|
|
assert all(len(component.manager.scripts) == 1
|
|
|
|
|
for component in components.values())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_repairs_live_public_key_drift():
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _fabric_components(current)
|
|
|
|
|
drifted = components["host-a"].manager
|
|
|
|
|
drifted.live_public_key = KEY3
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
assert result["state"] == "active"
|
|
|
|
|
assert result["next_stage_allowed"] is True
|
|
|
|
|
assert drifted.live_public_key == KEY1
|
|
|
|
|
# observe -> fixed-key identity -> apply -> exact post-observe
|
|
|
|
|
assert len(drifted.scripts) == 4
|
|
|
|
|
# The already exact peer receives only its initial observation.
|
|
|
|
|
assert len(components["host-b"].manager.scripts) == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("identity_failure", ["drift", "unavailable"])
|
|
|
|
|
def test_reconcile_identity_failure_blocks_all_host_writes(
|
|
|
|
|
monkeypatch, identity_failure):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _recovery_components(current)
|
|
|
|
|
failed = components["host-b"].manager
|
|
|
|
|
expected_error = "fabric_identity_conflict"
|
|
|
|
|
if identity_failure == "drift":
|
|
|
|
|
failed.public_key = KEY3
|
|
|
|
|
else:
|
|
|
|
|
expected_error = "fabric_remote_command_failed"
|
|
|
|
|
failed.reconcile_identity_error = FabricError(
|
|
|
|
|
"offline", 502, expected_error)
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
|
|
|
assert result["state"] == "uncertain"
|
|
|
|
|
assert result["next_stage_allowed"] is False
|
|
|
|
|
assert nodes["host-b"]["last_error_code"] == expected_error
|
|
|
|
|
for component in components.values():
|
|
|
|
|
manager = component.manager
|
|
|
|
|
assert manager.reconcile_identity_calls == 1
|
|
|
|
|
assert manager.reconcile_identity_create_flags == [False]
|
|
|
|
|
assert manager.reconcile_apply_calls == 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_apply_failure_on_one_node_does_not_stop_other_node(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _recovery_components(current)
|
|
|
|
|
components["host-a"].manager.reconcile_apply_error = FabricError(
|
|
|
|
|
"offline", 502, "fabric_remote_command_failed")
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
|
|
|
assert result["state"] == "uncertain"
|
|
|
|
|
assert nodes["host-a"]["last_error_code"] == (
|
|
|
|
|
"fabric_remote_command_failed")
|
|
|
|
|
assert nodes["host-b"]["state"] == "active"
|
|
|
|
|
assert nodes["host-b"]["observed_plan_digest"] == current.digest
|
|
|
|
|
assert all(component.manager.reconcile_identity_calls == 1
|
|
|
|
|
for component in components.values())
|
|
|
|
|
assert all(component.manager.reconcile_apply_calls == 1
|
|
|
|
|
for component in components.values())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_cancelled_reconcile_drains_admitted_host_apply_before_unlock(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _recovery_components(current)
|
|
|
|
|
entered = threading.Event()
|
|
|
|
|
release = threading.Event()
|
|
|
|
|
first = components["host-a"].manager
|
|
|
|
|
first.reconcile_apply_entered = entered
|
|
|
|
|
first.reconcile_apply_release = release
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
|
|
|
|
|
|
async def exercise():
|
|
|
|
|
reconcile_task = asyncio.create_task(
|
|
|
|
|
coordinator.reconcile(current.fabric_id))
|
|
|
|
|
for _attempt in range(100):
|
|
|
|
|
if entered.is_set():
|
|
|
|
|
break
|
|
|
|
|
await asyncio.sleep(0.01)
|
|
|
|
|
assert entered.is_set()
|
|
|
|
|
|
|
|
|
|
lock_entered = asyncio.Event()
|
|
|
|
|
|
|
|
|
|
async def wait_for_fabric_lock():
|
|
|
|
|
async with coordinator._locks[current.fabric_id]:
|
|
|
|
|
lock_entered.set()
|
|
|
|
|
|
|
|
|
|
waiter = asyncio.create_task(wait_for_fabric_lock())
|
|
|
|
|
reconcile_task.cancel()
|
|
|
|
|
await asyncio.sleep(0)
|
|
|
|
|
assert reconcile_task.done() is False
|
|
|
|
|
assert lock_entered.is_set() is False
|
|
|
|
|
release.set()
|
|
|
|
|
with pytest.raises(asyncio.CancelledError):
|
|
|
|
|
await reconcile_task
|
|
|
|
|
await asyncio.wait_for(waiter, timeout=1)
|
|
|
|
|
assert lock_entered.is_set() is True
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
asyncio.run(exercise())
|
|
|
|
|
finally:
|
|
|
|
|
release.set()
|
|
|
|
|
assert first.reconcile_apply_calls == 1
|
|
|
|
|
assert first.reconcile_configured is True
|
|
|
|
|
assert components["host-b"].manager.reconcile_apply_calls == 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_cancelled_reconcile_drains_runtime_identity_lock_before_unlock(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _recovery_components(current)
|
|
|
|
|
entered = threading.Event()
|
|
|
|
|
release = threading.Event()
|
|
|
|
|
first = components["host-a"].manager
|
|
|
|
|
first.reconcile_identity_entered = entered
|
|
|
|
|
first.reconcile_identity_release = release
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
|
|
|
|
|
|
async def exercise():
|
|
|
|
|
reconcile_task = asyncio.create_task(
|
|
|
|
|
coordinator.reconcile(current.fabric_id))
|
|
|
|
|
for _attempt in range(100):
|
|
|
|
|
if entered.is_set():
|
|
|
|
|
break
|
|
|
|
|
await asyncio.sleep(0.01)
|
|
|
|
|
assert entered.is_set()
|
|
|
|
|
|
|
|
|
|
lock_entered = asyncio.Event()
|
|
|
|
|
|
|
|
|
|
async def wait_for_fabric_lock():
|
|
|
|
|
async with coordinator._locks[current.fabric_id]:
|
|
|
|
|
lock_entered.set()
|
|
|
|
|
|
|
|
|
|
waiter = asyncio.create_task(wait_for_fabric_lock())
|
|
|
|
|
reconcile_task.cancel()
|
|
|
|
|
await asyncio.sleep(0)
|
|
|
|
|
assert reconcile_task.done() is False
|
|
|
|
|
assert lock_entered.is_set() is False
|
|
|
|
|
release.set()
|
|
|
|
|
with pytest.raises(asyncio.CancelledError):
|
|
|
|
|
await reconcile_task
|
|
|
|
|
await asyncio.wait_for(waiter, timeout=1)
|
|
|
|
|
assert lock_entered.is_set() is True
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
asyncio.run(exercise())
|
|
|
|
|
finally:
|
|
|
|
|
release.set()
|
|
|
|
|
assert first.reconcile_identity_calls == 1
|
|
|
|
|
assert first.reconcile_apply_calls == 0
|
|
|
|
|
assert components["host-b"].manager.reconcile_identity_calls == 0
|
|
|
|
|
assert components["host-b"].manager.reconcile_apply_calls == 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _vpc_payload_for_fabric(current, *, vpc_id="vpc-race"):
|
|
|
|
|
return {
|
|
|
|
|
"schema_version": 1,
|
|
|
|
|
@ -5487,10 +5904,10 @@ def test_reconcile_waits_for_admitted_vpc_apply_before_invalidating(
|
|
|
|
|
def __init__(self, _manager, _profile=None):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
def observe(self, *_args):
|
|
|
|
|
def observe(self, _plan, host, _keys):
|
|
|
|
|
observe_entered.set()
|
|
|
|
|
assert observe_release.wait(timeout=2)
|
|
|
|
|
return _reconcile_observation(current)
|
|
|
|
|
return _reconcile_observation(current, host)
|
|
|
|
|
|
|
|
|
|
vpc._apply_locked = blocked_vpc_apply
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
@ -5559,10 +5976,10 @@ def test_reconcile_first_durable_invalidation_blocks_vpc_host_mutation(
|
|
|
|
|
def __init__(self, _manager, _profile=None):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
def observe(self, *_args):
|
|
|
|
|
def observe(self, _plan, host, _keys):
|
|
|
|
|
observe_entered.set()
|
|
|
|
|
assert observe_release.wait(timeout=2)
|
|
|
|
|
return _reconcile_observation(current)
|
|
|
|
|
return _reconcile_observation(current, host)
|
|
|
|
|
|
|
|
|
|
store = BlockingInvalidationStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
@ -5648,7 +6065,7 @@ def test_reconcile_releases_catalog_during_remote_observe_and_relocks_publish(
|
|
|
|
|
def __init__(self, _manager, _profile=None):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
def observe(self, *_args):
|
|
|
|
|
def observe(self, _plan, host, _keys):
|
|
|
|
|
nonlocal observation_count
|
|
|
|
|
observe_entered.set()
|
|
|
|
|
assert observe_release.wait(timeout=2)
|
|
|
|
|
@ -5656,7 +6073,7 @@ def test_reconcile_releases_catalog_during_remote_observe_and_relocks_publish(
|
|
|
|
|
observation_count += 1
|
|
|
|
|
if observation_count == len(current.nodes):
|
|
|
|
|
all_observed.set()
|
|
|
|
|
return _reconcile_observation(current)
|
|
|
|
|
return _reconcile_observation(current, host)
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter)
|
|
|
|
|
@ -5719,8 +6136,8 @@ def test_cancelled_reconcile_drains_root_save_before_catalog_unlock(
|
|
|
|
|
def __init__(self, _manager, _profile=None):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
def observe(self, *_args):
|
|
|
|
|
return _reconcile_observation(current)
|
|
|
|
|
def observe(self, _plan, host, _keys):
|
|
|
|
|
return _reconcile_observation(current, host)
|
|
|
|
|
|
|
|
|
|
store = BlockingRootStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
|