You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_fabric_gate_retry.py

263 lines
9.1 KiB
Python

"""Execute the bounded dependency retry shell against structured job evidence."""
from __future__ import annotations
import json
import shlex
import stat
import subprocess
import sys
from pathlib import Path
import pytest
from virsh_panel.fabric import FABRIC_LIBVIRT_GATE_UNITS, FABRIC_SERVICE, _BOOT_HELPER
BOOT = "b" * 32
DAEMON = "libvirtd.service"
def _event(**changes):
event = {
"__CURSOR": "s=abc;i=1;b=" + BOOT + ";m=c8;t=1;x=1",
"__REALTIME_TIMESTAMP": "1000",
"__MONOTONIC_TIMESTAMP": "200",
"_BOOT_ID": BOOT,
"_PID": "1",
"UNIT": DAEMON,
"JOB_TYPE": "start",
"JOB_RESULT": "dependency",
}
event.update(changes)
return event
FAKE_CONTROL = r'''import json, os, sys
from pathlib import Path
path = Path(os.environ["GATE_TEST_STATE"])
state = json.loads(path.read_text())
args = sys.argv[1:]
if args[0] == "--no-block":
state["no_block"] = True
args = args[1:]
if args[0] == "--job-mode=fail":
state["job_mode_fail"] = True
args = args[1:]
action = args[0]
unit = args[-1]
primary = unit == "virt-cloud-wg-fabric.service"
if action == "show":
properties = [args[i+1] for i, arg in enumerate(args) if arg == "-p"]
if primary:
values = {"ActiveState": state["primary"], "StateChangeTimestampMonotonic": "100"}
elif unit == "libvirtd.service":
values = {"LoadState": "loaded", "ActiveState": state.get("daemon_state", "inactive"),
"Job": state.get("daemon_job", ""),
"Requires": state.get("requires", "virt-cloud-wg-fabric.service"),
"After": state.get("after", "virt-cloud-wg-fabric.service")}
else:
values = {"LoadState": "not-found", "ActiveState": "inactive", "Job": "", "Requires": "", "After": ""}
for prop in properties:
print(prop + "=" + values.get(prop, ""))
elif action == "is-active":
print(state["primary"] if primary else state.get("daemon_state", "inactive"))
elif action == "is-failed":
sys.exit(0 if primary and state["primary"] == "failed" else 1)
elif action == "reset-failed":
state["actions"].append([action, unit])
if not primary:
sys.exit(99)
elif action == "start":
state["actions"].append([action, unit])
if primary:
state["primary"] = "active"
state.update(state.get("on_primary_start", {}))
else:
state["daemon_state"] = "active"
else:
sys.exit(98)
path.write_text(json.dumps(state))
'''
FAKE_JOURNAL = r'''import json, os, sys
from pathlib import Path
path = Path(os.environ["GATE_TEST_STATE"])
state = json.loads(path.read_text())
args = sys.argv[1:]
assert "-b" in args and "_PID=1" in args and "-n" in args
assert args[args.index("-n")+1] == "1"
assert not any(a.startswith("JOB_RESULT=") for a in args)
assert "--output-fields=UNIT,JOB_TYPE,JOB_RESULT,_PID" in args
unit = next(a[5:] for a in args if a.startswith("UNIT="))
if unit != "libvirtd.service":
sys.exit(0)
reads = state.get("journal_reads", 0) + 1
state["journal_reads"] = reads
event = state.get("event")
if reads >= state.get("change_at_read", 999):
event = state.get("changed_event", event)
path.write_text(json.dumps(state))
if state.get("journal_failure"):
sys.exit(1)
if state.get("raw_export") is not None:
sys.stdout.write(state["raw_export"])
elif event:
for key, value in event.items():
if value is not None:
print(key + "=" + value)
if state.get("duplicate_result"):
print("JOB_RESULT=dependency")
print()
'''
def _write_command(path: Path, body: str):
path.write_text("#!" + sys.executable + "\n" + body)
path.chmod(0o700)
def _harness(tmp_path: Path, **changes):
runtime = tmp_path / "runtime"
runtime.mkdir(mode=0o700)
boot = tmp_path / "boot_id"
boot.write_text("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb\n")
state_path = tmp_path / "state.json"
state = {"primary": "failed", "event": _event(), "actions": []}
state.update(changes)
state_path.write_text(json.dumps(state))
control = tmp_path / "systemctl"
journal = tmp_path / "journalctl"
stat_command = tmp_path / "stat"
_write_command(control, FAKE_CONTROL)
_write_command(journal, FAKE_JOURNAL)
_write_command(stat_command, "import os,stat,sys\np=sys.argv[-1]\ns=os.stat(p)\nprint(s.st_size if sys.argv[-2]=='%s' else '0:0:'+oct(stat.S_IMODE(s.st_mode))[2:])\n")
timeout = tmp_path / "timeout"
timeout.write_text("#!/bin/sh\nshift 3\nexec \"$@\"\n")
timeout.chmod(0o700)
flock = tmp_path / "flock"
flock.write_text("#!/bin/sh\nexit 0\n")
flock.chmod(0o700)
start = _BOOT_HELPER.index('if [ "${1:-}" = --retry-gated-units ]; then')
end = _BOOT_HELPER.index("__VC_NMCLI_FILENAME_RESOLVER__", start)
branch = _BOOT_HELPER[start:end].replace(
"__VC_LIBVIRT_GATE_UNITS__", " ".join(FABRIC_LIBVIRT_GATE_UNITS),
).replace("/run/virt-cloud", str(runtime)).replace(
"/proc/sys/kernel/random/boot_id", str(boot),
)
script = f'''set -eu
umask 077
export GATE_TEST_STATE={shlex.quote(str(state_path))}
SYSTEMCTL={shlex.quote(str(control))}
STAT={shlex.quote(str(stat_command))}
find_tool() {{
case "${{1##*/}}" in
journalctl) printf '%s' {shlex.quote(str(journal))} ;;
timeout) printf '%s' {shlex.quote(str(timeout))} ;;
flock) printf '%s' {shlex.quote(str(flock))} ;;
*) return 1 ;;
esac
}}
vc_primary_inactive_retry_ready() {{ return 1; }}
vc_legacy_firewalld_state_detected() {{ return 1; }}
vc_fabric_rpf_repair_needed() {{ return 1; }}
{branch}
'''
def run():
completed = subprocess.run(
["/bin/sh", "-s", "--", "--retry-gated-units"],
input=script, text=True, capture_output=True, check=False,
)
assert completed.returncode == 0, completed.stderr
return json.loads(state_path.read_text())
return run, runtime, state_path
def _daemon_starts(state):
return [action for action in state["actions"] if action == ["start", DAEMON]]
def test_current_boot_dependency_is_claimed_once_and_started_after_primary(tmp_path):
run, runtime, state_path = _harness(tmp_path)
first = run()
assert first["actions"] == [
["reset-failed", FABRIC_SERVICE], ["start", FABRIC_SERVICE], ["start", DAEMON],
]
assert first["job_mode_fail"] is True
assert first["no_block"] is True
marker = runtime / "fabric-gate-retry" / DAEMON
assert stat.S_IMODE(marker.stat().st_mode) == 0o600
assert marker.read_text().splitlines()[:2] == ["virt-cloud-fabric-gate-retry-v1", DAEMON]
# Even a renewed primary failure and unchanged old evidence cannot replay
# the same daemon's already consumed claim in this boot.
first.update(primary="failed", daemon_state="inactive")
state_path.write_text(json.dumps(first))
assert len(_daemon_starts(run())) == 1
@pytest.mark.parametrize("changes", [
{"event": _event(JOB_TYPE="stop", JOB_RESULT="done")},
{"event": _event(JOB_RESULT=None)},
{"event": _event(JOB_RESULT="failed")},
{"event": _event(__MONOTONIC_TIMESTAMP="99")},
{"event": _event(_BOOT_ID="a" * 32)},
{"event": _event(_PID="20")},
{"event": _event(UNIT="virtqemud.service")},
{"event": _event(__CURSOR="")},
{"duplicate_result": True},
{"journal_failure": True},
{"raw_export": "X" * 4097},
{"requires": "unrelated.service"},
{"after": "unrelated.service"},
{"daemon_state": "failed"},
{"daemon_job": "3"},
{"event": None},
])
def test_unproved_or_superseded_job_never_restarts_daemon(tmp_path, changes):
run, _, _ = _harness(tmp_path, **changes)
state = run()
assert _daemon_starts(state) == []
assert ["start", FABRIC_SERVICE] in state["actions"]
assert not any(action == ["reset-failed", DAEMON] for action in state["actions"])
@pytest.mark.parametrize("change", [
{"event": _event(JOB_TYPE="stop", JOB_RESULT="done")},
{"event": _event(JOB_RESULT="failed")},
{"daemon_job": "9"},
{"daemon_state": "active"},
])
def test_new_job_while_primary_is_repaired_invalidates_claim(tmp_path, change):
run, _, _ = _harness(tmp_path, on_primary_start=change)
assert _daemon_starts(run()) == []
def test_job_changes_between_last_two_state_reads_are_not_replayed(tmp_path):
run, _, _ = _harness(
tmp_path, change_at_read=3, changed_event=_event(JOB_TYPE="stop", JOB_RESULT="done"),
)
assert _daemon_starts(run()) == []
@pytest.mark.parametrize("kind", ["symlink", "unknown_file", "wrong_mode"])
def test_unknown_retry_inventory_is_preserved_and_primary_still_repairs(tmp_path, kind):
run, runtime, _ = _harness(tmp_path)
retry = runtime / "fabric-gate-retry"
retry.mkdir(mode=0o700)
target = tmp_path / "external"
target.write_text("preserve\n")
if kind == "symlink":
(retry / DAEMON).symlink_to(target)
elif kind == "unknown_file":
(retry / "unknown").write_text("preserve\n")
else:
retry.chmod(0o755)
state = run()
assert _daemon_starts(state) == []
assert ["start", FABRIC_SERVICE] in state["actions"]
assert target.read_text() == "preserve\n"
if kind == "unknown_file":
assert (retry / "unknown").read_text() == "preserve\n"