|
|
|
|
@ -70,6 +70,26 @@ def plan():
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def plan_payload(current=None):
|
|
|
|
|
current = current or plan()
|
|
|
|
|
return {
|
|
|
|
|
"fabric_id": current.fabric_id,
|
|
|
|
|
"region": current.region,
|
|
|
|
|
"overlay_cidr": current.overlay_cidr,
|
|
|
|
|
"revision": current.revision,
|
|
|
|
|
"listen_port": current.listen_port,
|
|
|
|
|
"nodes": [
|
|
|
|
|
{
|
|
|
|
|
"host_id": node.host_id,
|
|
|
|
|
"overlay_ip": node.overlay_ip,
|
|
|
|
|
"endpoint": node.endpoint,
|
|
|
|
|
"listen_port": node.listen_port,
|
|
|
|
|
}
|
|
|
|
|
for node in current.nodes
|
|
|
|
|
],
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_plan_is_canonical_and_rejects_wide_or_tenant_routes():
|
|
|
|
|
first = plan()
|
|
|
|
|
reordered = parse_fabric_plan({
|
|
|
|
|
@ -4592,6 +4612,40 @@ def test_adapter_readback_requires_exact_peer_route_and_persistence():
|
|
|
|
|
)["transport_verified"] is True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("failure_phase", ["identity", "apply", "observe"])
|
|
|
|
|
def test_adapter_remote_failure_exposes_only_whitelisted_phase(failure_phase):
|
|
|
|
|
class FailedRemoteManager(FakeManager):
|
|
|
|
|
def _exec(self, argv, *, timeout, input_text=None):
|
|
|
|
|
self.scripts.append(input_text)
|
|
|
|
|
return subprocess.CompletedProcess(
|
|
|
|
|
argv, 71, "", "secret /root/path private-key-material")
|
|
|
|
|
|
|
|
|
|
current = plan()
|
|
|
|
|
adapter = FabricHostAdapter(FailedRemoteManager(KEY1))
|
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
|
|
|
if failure_phase == "identity":
|
|
|
|
|
adapter.identity()
|
|
|
|
|
elif failure_phase == "apply":
|
|
|
|
|
adapter.apply(
|
|
|
|
|
current, current.nodes[0],
|
|
|
|
|
{"host-a": KEY1, "host-b": KEY2})
|
|
|
|
|
else:
|
|
|
|
|
adapter.observe(
|
|
|
|
|
current, current.nodes[0],
|
|
|
|
|
{"host-a": KEY1, "host-b": KEY2})
|
|
|
|
|
|
|
|
|
|
assert error.value.error_code == "fabric_remote_command_failed"
|
|
|
|
|
assert error.value.failure_phase == failure_phase
|
|
|
|
|
safe_error = json.dumps({
|
|
|
|
|
"message": str(error.value),
|
|
|
|
|
"error_code": error.value.error_code,
|
|
|
|
|
"failure_phase": error.value.failure_phase,
|
|
|
|
|
})
|
|
|
|
|
assert "secret" not in safe_error
|
|
|
|
|
assert "/root/path" not in safe_error
|
|
|
|
|
assert "private-key-material" not in safe_error
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_adapter_pins_firewalld_mode_through_apply_and_observe():
|
|
|
|
|
current = plan()
|
|
|
|
|
manager = FakeManager(KEY1, firewall_mode=FIREWALL_MODE_FIREWALLD)
|
|
|
|
|
@ -4770,6 +4824,7 @@ class MemoryStore:
|
|
|
|
|
|
|
|
|
|
class FakeAdapter:
|
|
|
|
|
failures = set()
|
|
|
|
|
observe_failures = set()
|
|
|
|
|
|
|
|
|
|
def __init__(self, manager, profile=None):
|
|
|
|
|
self.manager = manager
|
|
|
|
|
@ -4782,7 +4837,8 @@ class FakeAdapter:
|
|
|
|
|
raise FabricError("offline", 502, "fabric_remote_command_failed")
|
|
|
|
|
|
|
|
|
|
def observe(self, current, host, keys):
|
|
|
|
|
if self.manager.host_id in self.failures:
|
|
|
|
|
if (self.manager.host_id in self.failures
|
|
|
|
|
or self.manager.host_id in self.observe_failures):
|
|
|
|
|
raise FabricError("offline", 502, "fabric_remote_command_failed")
|
|
|
|
|
return {
|
|
|
|
|
"state": "active", "transport_verified": True,
|
|
|
|
|
@ -5347,6 +5403,12 @@ def test_coordinator_keeps_successful_hosts_and_blocks_next_stage(monkeypatch):
|
|
|
|
|
assert result["next_stage_allowed"] is False
|
|
|
|
|
states = {node["host_id"]: node["state"] for node in result["nodes"]}
|
|
|
|
|
assert states == {"host-a": "active", "host-b": "uncertain"}
|
|
|
|
|
failed = next(
|
|
|
|
|
node for node in result["nodes"] if node["host_id"] == "host-b")
|
|
|
|
|
assert failed["last_error_phase"] == "apply"
|
|
|
|
|
persisted = json.loads(store.get(
|
|
|
|
|
"fabric_node", f"{result['fabric_id']}.host-b"))
|
|
|
|
|
assert persisted["last_error_phase"] == "apply"
|
|
|
|
|
assert managers["host-a"].reprobe_calls == 1
|
|
|
|
|
assert managers["host-b"].reprobe_calls == 1
|
|
|
|
|
serialized = "\n".join(store.values.values())
|
|
|
|
|
@ -5355,6 +5417,52 @@ def test_coordinator_keeps_successful_hosts_and_blocks_next_stage(monkeypatch):
|
|
|
|
|
FakeAdapter.failures = set()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_initial_apply_identity_failure_carries_safe_phase_without_record(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
components = _fabric_components(current)
|
|
|
|
|
|
|
|
|
|
class IdentityFailureAdapter(FakeAdapter):
|
|
|
|
|
def identity(self):
|
|
|
|
|
raise FabricError(
|
|
|
|
|
"safe", 502, "fabric_remote_command_failed")
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", IdentityFailureAdapter)
|
|
|
|
|
|
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
|
|
|
asyncio.run(FabricCoordinator(
|
|
|
|
|
store, Registry(components)).apply(plan_payload(current)))
|
|
|
|
|
|
|
|
|
|
assert error.value.error_code == "fabric_remote_command_failed"
|
|
|
|
|
assert error.value.failure_phase == "identity"
|
|
|
|
|
assert store.get("fabric_desired", current.fabric_id) is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_apply_records_post_apply_observe_failure_phase(monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
components = _fabric_components(current)
|
|
|
|
|
monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", FakeAdapter)
|
|
|
|
|
FakeAdapter.failures = set()
|
|
|
|
|
FakeAdapter.observe_failures = {"host-a"}
|
|
|
|
|
try:
|
|
|
|
|
result = asyncio.run(FabricCoordinator(
|
|
|
|
|
store, Registry(components)).apply(plan_payload(current)))
|
|
|
|
|
finally:
|
|
|
|
|
FakeAdapter.observe_failures = set()
|
|
|
|
|
|
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
|
|
|
assert nodes["host-a"]["last_error_code"] == (
|
|
|
|
|
"fabric_remote_command_failed")
|
|
|
|
|
assert nodes["host-a"]["last_error_phase"] == "observe"
|
|
|
|
|
assert "last_error_phase" not in nodes["host-b"]
|
|
|
|
|
persisted = json.loads(store.get(
|
|
|
|
|
"fabric_node", f"{current.fabric_id}.host-a"))
|
|
|
|
|
assert persisted["last_error_phase"] == "observe"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_recover_does_not_replay_an_interrupted_write():
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
desired = {"fabric_id": "hk-fabric", "state": "applying",
|
|
|
|
|
@ -5389,6 +5497,8 @@ def test_public_status_does_not_echo_nested_secrets_or_commands():
|
|
|
|
|
"command": "rm -rf /",
|
|
|
|
|
"nodes": [{
|
|
|
|
|
"host_id": "host-a", "public_key": KEY1,
|
|
|
|
|
"last_error_code": "fabric_remote_command_failed",
|
|
|
|
|
"last_error_phase": "apply",
|
|
|
|
|
"private_key": "secret",
|
|
|
|
|
"observed": {
|
|
|
|
|
"interface": FABRIC_INTERFACE,
|
|
|
|
|
@ -5396,13 +5506,26 @@ def test_public_status_does_not_echo_nested_secrets_or_commands():
|
|
|
|
|
"peers": [(KEY2, "203.0.113.2:51820", "100.64.1.2/32")],
|
|
|
|
|
"handshakes": {KEY2: 123},
|
|
|
|
|
},
|
|
|
|
|
}, {
|
|
|
|
|
"host_id": "host-b", "public_key": KEY2,
|
|
|
|
|
"last_error_phase": "observe",
|
|
|
|
|
}, {
|
|
|
|
|
"host_id": "host-c", "public_key": KEY3,
|
|
|
|
|
"last_error_code": "fabric_remote_command_failed",
|
|
|
|
|
"last_error_phase": "stderr:/secret/path",
|
|
|
|
|
}],
|
|
|
|
|
})
|
|
|
|
|
serialized = json.dumps(public)
|
|
|
|
|
assert public["listen_port"] == 51820
|
|
|
|
|
assert "private_key" not in serialized
|
|
|
|
|
assert "command" not in serialized
|
|
|
|
|
assert "command" not in public
|
|
|
|
|
assert "command" not in public["nodes"][0]["observed"]
|
|
|
|
|
assert KEY1 in serialized and KEY2 in serialized
|
|
|
|
|
assert public["nodes"][0]["last_error_phase"] == "apply"
|
|
|
|
|
assert "last_error_phase" not in public["nodes"][1]
|
|
|
|
|
assert "last_error_phase" not in public["nodes"][2]
|
|
|
|
|
assert FabricError(
|
|
|
|
|
"safe", failure_phase="stderr:/secret/path").failure_phase is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_rejects_a_tampered_plan_digest():
|
|
|
|
|
@ -5565,6 +5688,30 @@ def _recovery_components(current, *, configured=False):
|
|
|
|
|
return components
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_successful_apply_clears_stale_failure_phase():
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
record = _active_fabric_record(current)
|
|
|
|
|
for node in record["nodes"]:
|
|
|
|
|
node["last_error_code"] = "fabric_remote_command_failed"
|
|
|
|
|
node["last_error_phase"] = "observe"
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(record))
|
|
|
|
|
coordinator = FabricCoordinator(
|
|
|
|
|
store, Registry(_fabric_components(current)))
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.apply(plan_payload(current)))
|
|
|
|
|
|
|
|
|
|
assert result["state"] == "active"
|
|
|
|
|
assert all("last_error_phase" not in node for node in result["nodes"])
|
|
|
|
|
durable = json.loads(store.get("fabric_desired", current.fabric_id))
|
|
|
|
|
assert all("last_error_phase" not in node for node in durable["nodes"])
|
|
|
|
|
for host in current.nodes:
|
|
|
|
|
persisted = json.loads(store.get(
|
|
|
|
|
"fabric_node", f"{current.fabric_id}.{host.host_id}"))
|
|
|
|
|
assert "last_error_phase" not in persisted
|
|
|
|
|
assert persisted["last_error_code"] is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
@ -5584,6 +5731,7 @@ def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
|
|
|
|
|
|
|
|
|
|
assert result["state"] == "active"
|
|
|
|
|
assert result["next_stage_allowed"] is True
|
|
|
|
|
assert all("last_error_phase" not in node for node in result["nodes"])
|
|
|
|
|
assert all(component.manager.reconcile_identity_calls == 1
|
|
|
|
|
for component in components.values())
|
|
|
|
|
assert all(component.manager.reconcile_apply_calls == 1
|
|
|
|
|
@ -5591,6 +5739,94 @@ def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
|
|
|
|
|
assert first.reconcile_observe_calls == 2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_persists_post_apply_observe_failure_phase(monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _recovery_components(current)
|
|
|
|
|
failed = components["host-a"].manager
|
|
|
|
|
|
|
|
|
|
class PostApplyObserveFailureAdapter(ReconcileRecoveryAdapter):
|
|
|
|
|
def observe(self, plan_value, host, keys):
|
|
|
|
|
if self.manager is failed and self.manager.reconcile_configured:
|
|
|
|
|
self.manager.reconcile_observe_calls += 1
|
|
|
|
|
raise FabricError(
|
|
|
|
|
"incomplete", 502, "fabric_remote_command_failed")
|
|
|
|
|
return super().observe(plan_value, host, keys)
|
|
|
|
|
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter",
|
|
|
|
|
PostApplyObserveFailureAdapter)
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
|
|
|
assert result["state"] == "uncertain"
|
|
|
|
|
assert nodes["host-a"]["last_error_code"] == (
|
|
|
|
|
"fabric_remote_command_failed")
|
|
|
|
|
assert nodes["host-a"]["last_error_phase"] == "observe"
|
|
|
|
|
persisted = json.loads(store.get(
|
|
|
|
|
"fabric_node", f"{current.fabric_id}.host-a"))
|
|
|
|
|
assert persisted["last_error_phase"] == "observe"
|
|
|
|
|
assert failed.reconcile_observe_calls == 2
|
|
|
|
|
assert failed.reconcile_apply_calls == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_identity_gate_keeps_other_host_observe_phase(monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
|
|
|
_active_fabric_record(current)))
|
|
|
|
|
components = _recovery_components(current)
|
|
|
|
|
observed_failed = components["host-a"].manager
|
|
|
|
|
identity_failed = components["host-b"].manager
|
|
|
|
|
observed_failed.reconcile_observe_error = FabricError(
|
|
|
|
|
"incomplete", 502, "fabric_remote_command_failed")
|
|
|
|
|
identity_failed.reconcile_identity_error = FabricError(
|
|
|
|
|
"offline", 502, "fabric_remote_command_failed")
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
|
|
|
assert nodes["host-a"]["last_error_phase"] == "observe"
|
|
|
|
|
assert nodes["host-b"]["last_error_phase"] == "identity"
|
|
|
|
|
assert all(component.manager.reconcile_apply_calls == 0
|
|
|
|
|
for component in components.values())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_successful_reconcile_observation_clears_stale_failure_phase(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
record = _active_fabric_record(current)
|
|
|
|
|
for node in record["nodes"]:
|
|
|
|
|
node["last_error_code"] = "fabric_remote_command_failed"
|
|
|
|
|
node["last_error_phase"] = "apply"
|
|
|
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(record))
|
|
|
|
|
components = _recovery_components(current, configured=True)
|
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
|
|
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
|
|
|
|
|
|
assert result["state"] == "active"
|
|
|
|
|
assert all(node["last_error_code"] is None for node in result["nodes"])
|
|
|
|
|
assert all("last_error_phase" not in node for node in result["nodes"])
|
|
|
|
|
durable = json.loads(store.get("fabric_desired", current.fabric_id))
|
|
|
|
|
assert all("last_error_phase" not in node for node in durable["nodes"])
|
|
|
|
|
for host in current.nodes:
|
|
|
|
|
persisted = json.loads(store.get(
|
|
|
|
|
"fabric_node", f"{current.fabric_id}.{host.host_id}"))
|
|
|
|
|
assert "last_error_phase" not in persisted
|
|
|
|
|
assert persisted["last_error_code"] is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_recovers_key_and_emergency_only_then_is_idempotent(
|
|
|
|
|
monkeypatch):
|
|
|
|
|
current = plan()
|
|
|
|
|
@ -5712,6 +5948,10 @@ def test_reconcile_identity_failure_blocks_all_host_writes(
|
|
|
|
|
assert result["state"] == "uncertain"
|
|
|
|
|
assert result["next_stage_allowed"] is False
|
|
|
|
|
assert nodes["host-b"]["last_error_code"] == expected_error
|
|
|
|
|
assert nodes["host-b"]["last_error_phase"] == "identity"
|
|
|
|
|
persisted = json.loads(store.get(
|
|
|
|
|
"fabric_node", f"{current.fabric_id}.host-b"))
|
|
|
|
|
assert persisted["last_error_phase"] == "identity"
|
|
|
|
|
for component in components.values():
|
|
|
|
|
manager = component.manager
|
|
|
|
|
assert manager.reconcile_identity_calls == 1
|
|
|
|
|
@ -5738,6 +5978,7 @@ def test_reconcile_apply_failure_on_one_node_does_not_stop_other_node(
|
|
|
|
|
assert result["state"] == "uncertain"
|
|
|
|
|
assert nodes["host-a"]["last_error_code"] == (
|
|
|
|
|
"fabric_remote_command_failed")
|
|
|
|
|
assert nodes["host-a"]["last_error_phase"] == "apply"
|
|
|
|
|
assert nodes["host-b"]["state"] == "active"
|
|
|
|
|
assert nodes["host-b"]["observed_plan_digest"] == current.digest
|
|
|
|
|
assert all(component.manager.reconcile_identity_calls == 1
|
|
|
|
|
|