You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_fabric_apply_safe_log.py

243 lines
10 KiB
Python

"""Execute rendered async journal plumbing without running host mutations.
The runner, update state machine, logger, helper EXIT trap and committed
rollback branch are production-rendered code. Only stat's platform-specific
ownership report and sync are external-tool doubles inside temporary files.
"""
from __future__ import annotations
import base64
import os
from pathlib import Path
import shlex
import subprocess
import sys
import time
import pytest
from virsh_panel.fabric import (
FABRIC_OPERATION_ROOT,
_TRANSACTION_SCRIPT,
_apply_script,
_fabric_operation_runner_script,
_fabric_operation_update_script,
parse_fabric_plan,
)
_SECRET = "NEVER_LOG_raw_private_fixture_or_sensitive_path"
_OPERATION = "a" * 32
_DIGEST = "b" * 64
def _write(path: Path, body: str, mode: int = 0o700):
path.write_text(body, encoding="utf-8")
path.chmod(mode)
def _render(manager="ifupdown", firewall="native_nftables"):
plan = parse_fabric_plan({
"fabric_id": "safe-log-test", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": "a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
})
keys = {node.host_id: base64.b64encode(bytes([index]) * 32).decode()
for index, node in enumerate(plan.nodes, 1)}
return _apply_script(plan, plan.nodes[0], keys,
deployment_mode=manager, firewall_mode=firewall)
def _apply_logger(rendered):
start = rendered.index("vc_fabric_apply_failure_log() {")
return rendered[start:rendered.index("vc_fabric_operation_checkpoint() {", start)]
def _helper_entry(rendered):
# This is the helper actually embedded in apply, not a retyped logger.
start = rendered.index("#!/bin/sh\n# virt-cloud-wg-fabric-v1\n# fabric-plan-digest=")
end = rendered.index("BASE=/etc/virt-cloud\n", start)
prefix = rendered[start:end]
assert "__VC_FABRIC_DIGEST__" not in prefix
assert "trap cleanup EXIT HUP INT TERM\n" in prefix
return prefix
def _run(script, *, env=None, args=()):
return subprocess.run(["/bin/sh", "-eu", "-c", script, "safe-log-test", *args],
capture_output=True, text=True, check=False,
env={**os.environ, **(env or {})})
def _journal_wrapper(body):
return ("exec 3>&1\nexec >/dev/null 2>&1\n"
"VC_FABRIC_SAFE_LOG=1\nexport VC_FABRIC_SAFE_LOG\n" + body)
@pytest.mark.parametrize("detail", [
"entry", "final_acl", "final_tenants", "final_identity", "final_link",
"final_addresses", "final_peers", "final_endpoints", "final_allowed_ips",
"final_routes", "final_overlay_paths", "commit", "convergence",
"complete",
])
def test_rendered_apply_logger_has_one_fixed_fd3_record(detail):
script = _journal_wrapper(_apply_logger(_render()) + (
f"VC_FABRIC_APPLY_DETAIL={shlex.quote(detail)}\n"
f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
"vc_fabric_apply_failure_log 71\n"
))
result = _run(script)
assert result.returncode == 0
assert result.stdout == (
f"virt-cloud-fabric apply_state=failed apply_check={detail} exit_status=71\n")
assert result.stderr == ""
@pytest.mark.parametrize("logger", ["apply", "helper"])
@pytest.mark.parametrize("status", ["", "-1", "256", "9" * 100, _SECRET])
def test_unknown_phase_and_invalid_status_are_sanitized(logger, status):
rendered = _render()
phase = "unknown\n" + _SECRET + "=/sensitive/path"
if logger == "apply":
body = (_apply_logger(rendered)
+ f"VC_FABRIC_APPLY_DETAIL={shlex.quote(phase)}\n"
+ f"vc_fabric_apply_failure_log {shlex.quote(status)}\n")
line = "virt-cloud-fabric apply_state=failed apply_check=unknown exit_status=1\n"
else:
body = (_helper_entry(rendered)
+ f"VC_FABRIC_HELPER_PHASE={shlex.quote(phase)}\n"
+ f"vc_fabric_helper_failure_log {shlex.quote(status)}\nexit 0\n")
line = "virt-cloud-fabric helper_state=failed helper_phase=unknown exit_status=1\n"
result = _run(_journal_wrapper(body))
assert result.returncode == 0
assert result.stdout == line
assert result.stderr == ""
assert _SECRET not in result.stdout
@pytest.mark.parametrize("status", [1, 70, 71, 79, 127, 143, 255])
def test_rendered_child_helper_preserves_exit_status_without_duplicate_fd3_log(status):
body = (_helper_entry(_render()) + "VC_FABRIC_HELPER_PHASE=final\n"
+ f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
+ f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
+ f"exit {status}\n")
result = _run(_journal_wrapper(body))
assert result.returncode == status
assert result.stdout == (
f"virt-cloud-fabric helper_state=failed helper_phase=final exit_status={status}\n")
assert result.stderr == ""
def _runner_fixture(tmp_path: Path, apply_body: str):
directory = tmp_path / _OPERATION
directory.mkdir(mode=0o700)
stat_tool = tmp_path / "stat"
_write(stat_tool, f"#!{sys.executable}\n" + (
"import os, stat, sys\n"
"assert sys.argv[1:3] == ['-c', '%u:%a']\n"
"print('0:' + format(stat.S_IMODE(os.stat(sys.argv[3]).st_mode), 'o'))\n"
))
sync_tool = tmp_path / "sync"
_write(sync_tool, "#!/bin/sh\nexit 0\n")
updater = _fabric_operation_update_script(_OPERATION, _DIGEST).replace(
FABRIC_OPERATION_ROOT, str(tmp_path)).replace(
"stat -c", shlex.quote(str(stat_tool)) + " -c").replace(
"\nsync\n", "\n" + shlex.quote(str(sync_tool)) + "\n")
_write(directory / "update", updater)
_write(directory / "apply", "#!/bin/sh\n" + apply_body)
_write(directory / "manifest", (
f"schema=1\noperation_id={_OPERATION}\nplan_digest={_DIGEST}\n"), 0o600)
now = int(time.time())
_write(directory / "status", (
f"schema=1\noperation_id={_OPERATION}\nstatus=queued\nphase=apply\n"
"checkpoint=dispatch\nreason=none\n"
f"created_at={now}\nstarted_at=0\nupdated_at={now}\nfinished_at=0\n"), 0o600)
runner = _fabric_operation_runner_script(_OPERATION, _DIGEST).replace(
FABRIC_OPERATION_ROOT, str(tmp_path))
return runner, directory
@pytest.mark.parametrize("status", [0, 71, 79])
def test_rendered_runner_propagates_safe_descriptor_and_never_raw_output(tmp_path, status):
helper = tmp_path / "child-helper"
_write(helper, _helper_entry(_render()) + (
"VC_FABRIC_HELPER_PHASE=gate\n"
f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\nexit {status}\n"
))
body = (f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
+ f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
+ f"exec /bin/sh {shlex.quote(str(helper))}\n")
runner, directory = _runner_fixture(tmp_path, body)
result = _run(runner)
assert result.returncode == (0 if status == 0 else 1)
assert result.stderr == ""
assert _SECRET not in result.stdout
assert str(tmp_path) not in result.stdout
lines = result.stdout.splitlines()
assert len(lines) == (2 if status == 0 else 3)
assert lines[0] == (
f"virt-cloud-fabric operation={_OPERATION} state=running checkpoint=dispatch reason=none")
if status:
assert lines[1] == (
f"virt-cloud-fabric helper_state=failed helper_phase=gate exit_status={status}")
state = (directory / "status").read_text()
assert ("status=succeeded\n" if status == 0 else "status=failed\n") in state
assert lines[-1].startswith(f"virt-cloud-fabric operation={_OPERATION} state=")
@pytest.mark.parametrize("status", [71, 79])
def test_first_apply_failure_precedes_secondary_rollback_cleanup_record(tmp_path, status):
rendered = _render()
helper = tmp_path / "secondary-helper"
_write(helper, _helper_entry(rendered) + "VC_FABRIC_HELPER_PHASE=gate\nexit 79\n")
start = _TRANSACTION_SCRIPT.index("rollback() {")
end = _TRANSACTION_SCRIPT.index("\ntrap rollback EXIT HUP INT TERM", start)
rollback = _TRANSACTION_SCRIPT[start:end]
body = ("COMMITTED=yes\n" + _apply_logger(rendered) + rollback + (
"\ncleanup() {\n"
f" printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
f" /bin/sh {shlex.quote(str(helper))} >/dev/null 2>&1 || true\n"
"}\ntrap rollback EXIT HUP INT TERM\n"
f"VC_FABRIC_APPLY_DETAIL=final_routes\nexit {status}\n"
))
result = _run(_journal_wrapper(body))
assert result.returncode == status
assert result.stderr == ""
assert result.stdout.splitlines() == [
f"virt-cloud-fabric apply_state=failed apply_check=final_routes exit_status={status}",
"virt-cloud-fabric helper_state=failed helper_phase=gate exit_status=79",
]
@pytest.mark.parametrize("manager", ["ifupdown", "networkmanager_nmcli"])
def test_rendered_timer_start_strictly_follows_durable_commit(manager):
rendered = _render(manager)
timer_command = '"$SYSTEMCTL" start virt-cloud-wg-fabric-reconcile.timer >/dev/null'
assert rendered.count(timer_command) == 1
commit_marker = rendered.rindex('committed > "$_txn_state_tmp"')
publish = rendered.index('mv -f "$_txn_state_tmp" "$TXN/state"', commit_marker)
durable_sync = rendered.index("\nsync\n", publish)
committed_flag = rendered.index("COMMITTED=yes", durable_sync)
timer_start = rendered.index(timer_command)
unlock = rendered.index("exec 9>&-", timer_start)
assert commit_marker < publish < durable_sync < committed_flag < timer_start < unlock
def test_success_and_disabled_safe_logging_do_not_invent_failure_records():
logger = _apply_logger(_render())
result = _run(_journal_wrapper(logger + "VC_FABRIC_APPLY_DETAIL=complete\n"
"vc_fabric_apply_failure_log 0\n"))
assert result.returncode == 0
assert result.stdout == result.stderr == ""
result = _run(_journal_wrapper(logger + "VC_FABRIC_SAFE_LOG=0\n"
"vc_fabric_apply_failure_log 71\n"))
assert result.returncode == 0
assert result.stdout == result.stderr == ""