You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
243 lines
10 KiB
Python
243 lines
10 KiB
Python
"""Execute rendered async journal plumbing without running host mutations.
|
|
|
|
The runner, update state machine, logger, helper EXIT trap and committed
|
|
rollback branch are production-rendered code. Only stat's platform-specific
|
|
ownership report and sync are external-tool doubles inside temporary files.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.fabric import (
|
|
FABRIC_OPERATION_ROOT,
|
|
_TRANSACTION_SCRIPT,
|
|
_apply_script,
|
|
_fabric_operation_runner_script,
|
|
_fabric_operation_update_script,
|
|
parse_fabric_plan,
|
|
)
|
|
|
|
|
|
_SECRET = "NEVER_LOG_raw_private_fixture_or_sensitive_path"
|
|
_OPERATION = "a" * 32
|
|
_DIGEST = "b" * 64
|
|
|
|
|
|
def _write(path: Path, body: str, mode: int = 0o700):
|
|
path.write_text(body, encoding="utf-8")
|
|
path.chmod(mode)
|
|
|
|
|
|
def _render(manager="ifupdown", firewall="native_nftables"):
|
|
plan = parse_fabric_plan({
|
|
"fabric_id": "safe-log-test", "region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
})
|
|
keys = {node.host_id: base64.b64encode(bytes([index]) * 32).decode()
|
|
for index, node in enumerate(plan.nodes, 1)}
|
|
return _apply_script(plan, plan.nodes[0], keys,
|
|
deployment_mode=manager, firewall_mode=firewall)
|
|
|
|
|
|
def _apply_logger(rendered):
|
|
start = rendered.index("vc_fabric_apply_failure_log() {")
|
|
return rendered[start:rendered.index("vc_fabric_operation_checkpoint() {", start)]
|
|
|
|
|
|
def _helper_entry(rendered):
|
|
# This is the helper actually embedded in apply, not a retyped logger.
|
|
start = rendered.index("#!/bin/sh\n# virt-cloud-wg-fabric-v1\n# fabric-plan-digest=")
|
|
end = rendered.index("BASE=/etc/virt-cloud\n", start)
|
|
prefix = rendered[start:end]
|
|
assert "__VC_FABRIC_DIGEST__" not in prefix
|
|
assert "trap cleanup EXIT HUP INT TERM\n" in prefix
|
|
return prefix
|
|
|
|
|
|
def _run(script, *, env=None, args=()):
|
|
return subprocess.run(["/bin/sh", "-eu", "-c", script, "safe-log-test", *args],
|
|
capture_output=True, text=True, check=False,
|
|
env={**os.environ, **(env or {})})
|
|
|
|
|
|
def _journal_wrapper(body):
|
|
return ("exec 3>&1\nexec >/dev/null 2>&1\n"
|
|
"VC_FABRIC_SAFE_LOG=1\nexport VC_FABRIC_SAFE_LOG\n" + body)
|
|
|
|
|
|
@pytest.mark.parametrize("detail", [
|
|
"entry", "final_acl", "final_tenants", "final_identity", "final_link",
|
|
"final_addresses", "final_peers", "final_endpoints", "final_allowed_ips",
|
|
"final_routes", "final_overlay_paths", "commit", "convergence",
|
|
"complete",
|
|
])
|
|
def test_rendered_apply_logger_has_one_fixed_fd3_record(detail):
|
|
script = _journal_wrapper(_apply_logger(_render()) + (
|
|
f"VC_FABRIC_APPLY_DETAIL={shlex.quote(detail)}\n"
|
|
f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
|
|
f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
|
|
"vc_fabric_apply_failure_log 71\n"
|
|
))
|
|
result = _run(script)
|
|
assert result.returncode == 0
|
|
assert result.stdout == (
|
|
f"virt-cloud-fabric apply_state=failed apply_check={detail} exit_status=71\n")
|
|
assert result.stderr == ""
|
|
|
|
|
|
@pytest.mark.parametrize("logger", ["apply", "helper"])
|
|
@pytest.mark.parametrize("status", ["", "-1", "256", "9" * 100, _SECRET])
|
|
def test_unknown_phase_and_invalid_status_are_sanitized(logger, status):
|
|
rendered = _render()
|
|
phase = "unknown\n" + _SECRET + "=/sensitive/path"
|
|
if logger == "apply":
|
|
body = (_apply_logger(rendered)
|
|
+ f"VC_FABRIC_APPLY_DETAIL={shlex.quote(phase)}\n"
|
|
+ f"vc_fabric_apply_failure_log {shlex.quote(status)}\n")
|
|
line = "virt-cloud-fabric apply_state=failed apply_check=unknown exit_status=1\n"
|
|
else:
|
|
body = (_helper_entry(rendered)
|
|
+ f"VC_FABRIC_HELPER_PHASE={shlex.quote(phase)}\n"
|
|
+ f"vc_fabric_helper_failure_log {shlex.quote(status)}\nexit 0\n")
|
|
line = "virt-cloud-fabric helper_state=failed helper_phase=unknown exit_status=1\n"
|
|
result = _run(_journal_wrapper(body))
|
|
assert result.returncode == 0
|
|
assert result.stdout == line
|
|
assert result.stderr == ""
|
|
assert _SECRET not in result.stdout
|
|
|
|
|
|
@pytest.mark.parametrize("status", [1, 70, 71, 79, 127, 143, 255])
|
|
def test_rendered_child_helper_preserves_exit_status_without_duplicate_fd3_log(status):
|
|
body = (_helper_entry(_render()) + "VC_FABRIC_HELPER_PHASE=final\n"
|
|
+ f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
|
|
+ f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
|
|
+ f"exit {status}\n")
|
|
result = _run(_journal_wrapper(body))
|
|
assert result.returncode == status
|
|
assert result.stdout == (
|
|
f"virt-cloud-fabric helper_state=failed helper_phase=final exit_status={status}\n")
|
|
assert result.stderr == ""
|
|
|
|
|
|
def _runner_fixture(tmp_path: Path, apply_body: str):
|
|
directory = tmp_path / _OPERATION
|
|
directory.mkdir(mode=0o700)
|
|
stat_tool = tmp_path / "stat"
|
|
_write(stat_tool, f"#!{sys.executable}\n" + (
|
|
"import os, stat, sys\n"
|
|
"assert sys.argv[1:3] == ['-c', '%u:%a']\n"
|
|
"print('0:' + format(stat.S_IMODE(os.stat(sys.argv[3]).st_mode), 'o'))\n"
|
|
))
|
|
sync_tool = tmp_path / "sync"
|
|
_write(sync_tool, "#!/bin/sh\nexit 0\n")
|
|
updater = _fabric_operation_update_script(_OPERATION, _DIGEST).replace(
|
|
FABRIC_OPERATION_ROOT, str(tmp_path)).replace(
|
|
"stat -c", shlex.quote(str(stat_tool)) + " -c").replace(
|
|
"\nsync\n", "\n" + shlex.quote(str(sync_tool)) + "\n")
|
|
_write(directory / "update", updater)
|
|
_write(directory / "apply", "#!/bin/sh\n" + apply_body)
|
|
_write(directory / "manifest", (
|
|
f"schema=1\noperation_id={_OPERATION}\nplan_digest={_DIGEST}\n"), 0o600)
|
|
now = int(time.time())
|
|
_write(directory / "status", (
|
|
f"schema=1\noperation_id={_OPERATION}\nstatus=queued\nphase=apply\n"
|
|
"checkpoint=dispatch\nreason=none\n"
|
|
f"created_at={now}\nstarted_at=0\nupdated_at={now}\nfinished_at=0\n"), 0o600)
|
|
runner = _fabric_operation_runner_script(_OPERATION, _DIGEST).replace(
|
|
FABRIC_OPERATION_ROOT, str(tmp_path))
|
|
return runner, directory
|
|
|
|
|
|
@pytest.mark.parametrize("status", [0, 71, 79])
|
|
def test_rendered_runner_propagates_safe_descriptor_and_never_raw_output(tmp_path, status):
|
|
helper = tmp_path / "child-helper"
|
|
_write(helper, _helper_entry(_render()) + (
|
|
"VC_FABRIC_HELPER_PHASE=gate\n"
|
|
f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
|
|
f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\nexit {status}\n"
|
|
))
|
|
body = (f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
|
|
+ f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
|
|
+ f"exec /bin/sh {shlex.quote(str(helper))}\n")
|
|
runner, directory = _runner_fixture(tmp_path, body)
|
|
result = _run(runner)
|
|
assert result.returncode == (0 if status == 0 else 1)
|
|
assert result.stderr == ""
|
|
assert _SECRET not in result.stdout
|
|
assert str(tmp_path) not in result.stdout
|
|
lines = result.stdout.splitlines()
|
|
assert len(lines) == (2 if status == 0 else 3)
|
|
assert lines[0] == (
|
|
f"virt-cloud-fabric operation={_OPERATION} state=running checkpoint=dispatch reason=none")
|
|
if status:
|
|
assert lines[1] == (
|
|
f"virt-cloud-fabric helper_state=failed helper_phase=gate exit_status={status}")
|
|
state = (directory / "status").read_text()
|
|
assert ("status=succeeded\n" if status == 0 else "status=failed\n") in state
|
|
assert lines[-1].startswith(f"virt-cloud-fabric operation={_OPERATION} state=")
|
|
|
|
|
|
@pytest.mark.parametrize("status", [71, 79])
|
|
def test_first_apply_failure_precedes_secondary_rollback_cleanup_record(tmp_path, status):
|
|
rendered = _render()
|
|
helper = tmp_path / "secondary-helper"
|
|
_write(helper, _helper_entry(rendered) + "VC_FABRIC_HELPER_PHASE=gate\nexit 79\n")
|
|
start = _TRANSACTION_SCRIPT.index("rollback() {")
|
|
end = _TRANSACTION_SCRIPT.index("\ntrap rollback EXIT HUP INT TERM", start)
|
|
rollback = _TRANSACTION_SCRIPT[start:end]
|
|
body = ("COMMITTED=yes\n" + _apply_logger(rendered) + rollback + (
|
|
"\ncleanup() {\n"
|
|
f" printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
|
|
f" /bin/sh {shlex.quote(str(helper))} >/dev/null 2>&1 || true\n"
|
|
"}\ntrap rollback EXIT HUP INT TERM\n"
|
|
f"VC_FABRIC_APPLY_DETAIL=final_routes\nexit {status}\n"
|
|
))
|
|
result = _run(_journal_wrapper(body))
|
|
assert result.returncode == status
|
|
assert result.stderr == ""
|
|
assert result.stdout.splitlines() == [
|
|
f"virt-cloud-fabric apply_state=failed apply_check=final_routes exit_status={status}",
|
|
"virt-cloud-fabric helper_state=failed helper_phase=gate exit_status=79",
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("manager", ["ifupdown", "networkmanager_nmcli"])
|
|
def test_rendered_timer_start_strictly_follows_durable_commit(manager):
|
|
rendered = _render(manager)
|
|
timer_command = '"$SYSTEMCTL" start virt-cloud-wg-fabric-reconcile.timer >/dev/null'
|
|
assert rendered.count(timer_command) == 1
|
|
commit_marker = rendered.rindex('committed > "$_txn_state_tmp"')
|
|
publish = rendered.index('mv -f "$_txn_state_tmp" "$TXN/state"', commit_marker)
|
|
durable_sync = rendered.index("\nsync\n", publish)
|
|
committed_flag = rendered.index("COMMITTED=yes", durable_sync)
|
|
timer_start = rendered.index(timer_command)
|
|
unlock = rendered.index("exec 9>&-", timer_start)
|
|
assert commit_marker < publish < durable_sync < committed_flag < timer_start < unlock
|
|
|
|
|
|
def test_success_and_disabled_safe_logging_do_not_invent_failure_records():
|
|
logger = _apply_logger(_render())
|
|
result = _run(_journal_wrapper(logger + "VC_FABRIC_APPLY_DETAIL=complete\n"
|
|
"vc_fabric_apply_failure_log 0\n"))
|
|
assert result.returncode == 0
|
|
assert result.stdout == result.stderr == ""
|
|
result = _run(_journal_wrapper(logger + "VC_FABRIC_SAFE_LOG=0\n"
|
|
"vc_fabric_apply_failure_log 71\n"))
|
|
assert result.returncode == 0
|
|
assert result.stdout == result.stderr == ""
|