You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
263 lines
9.1 KiB
Python
263 lines
9.1 KiB
Python
"""Execute the bounded dependency retry shell against structured job evidence."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import shlex
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.fabric import FABRIC_LIBVIRT_GATE_UNITS, FABRIC_SERVICE, _BOOT_HELPER
|
|
|
|
|
|
BOOT = "b" * 32
|
|
DAEMON = "libvirtd.service"
|
|
|
|
|
|
def _event(**changes):
|
|
event = {
|
|
"__CURSOR": "s=abc;i=1;b=" + BOOT + ";m=c8;t=1;x=1",
|
|
"__REALTIME_TIMESTAMP": "1000",
|
|
"__MONOTONIC_TIMESTAMP": "200",
|
|
"_BOOT_ID": BOOT,
|
|
"_PID": "1",
|
|
"UNIT": DAEMON,
|
|
"JOB_TYPE": "start",
|
|
"JOB_RESULT": "dependency",
|
|
}
|
|
event.update(changes)
|
|
return event
|
|
|
|
|
|
FAKE_CONTROL = r'''import json, os, sys
|
|
from pathlib import Path
|
|
path = Path(os.environ["GATE_TEST_STATE"])
|
|
state = json.loads(path.read_text())
|
|
args = sys.argv[1:]
|
|
if args[0] == "--no-block":
|
|
state["no_block"] = True
|
|
args = args[1:]
|
|
if args[0] == "--job-mode=fail":
|
|
state["job_mode_fail"] = True
|
|
args = args[1:]
|
|
action = args[0]
|
|
unit = args[-1]
|
|
primary = unit == "virt-cloud-wg-fabric.service"
|
|
if action == "show":
|
|
properties = [args[i+1] for i, arg in enumerate(args) if arg == "-p"]
|
|
if primary:
|
|
values = {"ActiveState": state["primary"], "StateChangeTimestampMonotonic": "100"}
|
|
elif unit == "libvirtd.service":
|
|
values = {"LoadState": "loaded", "ActiveState": state.get("daemon_state", "inactive"),
|
|
"Job": state.get("daemon_job", ""),
|
|
"Requires": state.get("requires", "virt-cloud-wg-fabric.service"),
|
|
"After": state.get("after", "virt-cloud-wg-fabric.service")}
|
|
else:
|
|
values = {"LoadState": "not-found", "ActiveState": "inactive", "Job": "", "Requires": "", "After": ""}
|
|
for prop in properties:
|
|
print(prop + "=" + values.get(prop, ""))
|
|
elif action == "is-active":
|
|
print(state["primary"] if primary else state.get("daemon_state", "inactive"))
|
|
elif action == "is-failed":
|
|
sys.exit(0 if primary and state["primary"] == "failed" else 1)
|
|
elif action == "reset-failed":
|
|
state["actions"].append([action, unit])
|
|
if not primary:
|
|
sys.exit(99)
|
|
elif action == "start":
|
|
state["actions"].append([action, unit])
|
|
if primary:
|
|
state["primary"] = "active"
|
|
state.update(state.get("on_primary_start", {}))
|
|
else:
|
|
state["daemon_state"] = "active"
|
|
else:
|
|
sys.exit(98)
|
|
path.write_text(json.dumps(state))
|
|
'''
|
|
|
|
|
|
FAKE_JOURNAL = r'''import json, os, sys
|
|
from pathlib import Path
|
|
path = Path(os.environ["GATE_TEST_STATE"])
|
|
state = json.loads(path.read_text())
|
|
args = sys.argv[1:]
|
|
assert "-b" in args and "_PID=1" in args and "-n" in args
|
|
assert args[args.index("-n")+1] == "1"
|
|
assert not any(a.startswith("JOB_RESULT=") for a in args)
|
|
assert "--output-fields=UNIT,JOB_TYPE,JOB_RESULT,_PID" in args
|
|
unit = next(a[5:] for a in args if a.startswith("UNIT="))
|
|
if unit != "libvirtd.service":
|
|
sys.exit(0)
|
|
reads = state.get("journal_reads", 0) + 1
|
|
state["journal_reads"] = reads
|
|
event = state.get("event")
|
|
if reads >= state.get("change_at_read", 999):
|
|
event = state.get("changed_event", event)
|
|
path.write_text(json.dumps(state))
|
|
if state.get("journal_failure"):
|
|
sys.exit(1)
|
|
if state.get("raw_export") is not None:
|
|
sys.stdout.write(state["raw_export"])
|
|
elif event:
|
|
for key, value in event.items():
|
|
if value is not None:
|
|
print(key + "=" + value)
|
|
if state.get("duplicate_result"):
|
|
print("JOB_RESULT=dependency")
|
|
print()
|
|
'''
|
|
|
|
|
|
def _write_command(path: Path, body: str):
|
|
path.write_text("#!" + sys.executable + "\n" + body)
|
|
path.chmod(0o700)
|
|
|
|
|
|
def _harness(tmp_path: Path, **changes):
|
|
runtime = tmp_path / "runtime"
|
|
runtime.mkdir(mode=0o700)
|
|
boot = tmp_path / "boot_id"
|
|
boot.write_text("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb\n")
|
|
state_path = tmp_path / "state.json"
|
|
state = {"primary": "failed", "event": _event(), "actions": []}
|
|
state.update(changes)
|
|
state_path.write_text(json.dumps(state))
|
|
control = tmp_path / "systemctl"
|
|
journal = tmp_path / "journalctl"
|
|
stat_command = tmp_path / "stat"
|
|
_write_command(control, FAKE_CONTROL)
|
|
_write_command(journal, FAKE_JOURNAL)
|
|
_write_command(stat_command, "import os,stat,sys\np=sys.argv[-1]\ns=os.stat(p)\nprint(s.st_size if sys.argv[-2]=='%s' else '0:0:'+oct(stat.S_IMODE(s.st_mode))[2:])\n")
|
|
timeout = tmp_path / "timeout"
|
|
timeout.write_text("#!/bin/sh\nshift 3\nexec \"$@\"\n")
|
|
timeout.chmod(0o700)
|
|
flock = tmp_path / "flock"
|
|
flock.write_text("#!/bin/sh\nexit 0\n")
|
|
flock.chmod(0o700)
|
|
start = _BOOT_HELPER.index('if [ "${1:-}" = --retry-gated-units ]; then')
|
|
end = _BOOT_HELPER.index("__VC_NMCLI_FILENAME_RESOLVER__", start)
|
|
branch = _BOOT_HELPER[start:end].replace(
|
|
"__VC_LIBVIRT_GATE_UNITS__", " ".join(FABRIC_LIBVIRT_GATE_UNITS),
|
|
).replace("/run/virt-cloud", str(runtime)).replace(
|
|
"/proc/sys/kernel/random/boot_id", str(boot),
|
|
)
|
|
script = f'''set -eu
|
|
umask 077
|
|
export GATE_TEST_STATE={shlex.quote(str(state_path))}
|
|
SYSTEMCTL={shlex.quote(str(control))}
|
|
STAT={shlex.quote(str(stat_command))}
|
|
find_tool() {{
|
|
case "${{1##*/}}" in
|
|
journalctl) printf '%s' {shlex.quote(str(journal))} ;;
|
|
timeout) printf '%s' {shlex.quote(str(timeout))} ;;
|
|
flock) printf '%s' {shlex.quote(str(flock))} ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}}
|
|
vc_primary_inactive_retry_ready() {{ return 1; }}
|
|
vc_legacy_firewalld_state_detected() {{ return 1; }}
|
|
vc_fabric_rpf_repair_needed() {{ return 1; }}
|
|
{branch}
|
|
'''
|
|
|
|
def run():
|
|
completed = subprocess.run(
|
|
["/bin/sh", "-s", "--", "--retry-gated-units"],
|
|
input=script, text=True, capture_output=True, check=False,
|
|
)
|
|
assert completed.returncode == 0, completed.stderr
|
|
return json.loads(state_path.read_text())
|
|
|
|
return run, runtime, state_path
|
|
|
|
|
|
def _daemon_starts(state):
|
|
return [action for action in state["actions"] if action == ["start", DAEMON]]
|
|
|
|
|
|
def test_current_boot_dependency_is_claimed_once_and_started_after_primary(tmp_path):
|
|
run, runtime, state_path = _harness(tmp_path)
|
|
first = run()
|
|
assert first["actions"] == [
|
|
["reset-failed", FABRIC_SERVICE], ["start", FABRIC_SERVICE], ["start", DAEMON],
|
|
]
|
|
assert first["job_mode_fail"] is True
|
|
assert first["no_block"] is True
|
|
marker = runtime / "fabric-gate-retry" / DAEMON
|
|
assert stat.S_IMODE(marker.stat().st_mode) == 0o600
|
|
assert marker.read_text().splitlines()[:2] == ["virt-cloud-fabric-gate-retry-v1", DAEMON]
|
|
# Even a renewed primary failure and unchanged old evidence cannot replay
|
|
# the same daemon's already consumed claim in this boot.
|
|
first.update(primary="failed", daemon_state="inactive")
|
|
state_path.write_text(json.dumps(first))
|
|
assert len(_daemon_starts(run())) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("changes", [
|
|
{"event": _event(JOB_TYPE="stop", JOB_RESULT="done")},
|
|
{"event": _event(JOB_RESULT=None)},
|
|
{"event": _event(JOB_RESULT="failed")},
|
|
{"event": _event(__MONOTONIC_TIMESTAMP="99")},
|
|
{"event": _event(_BOOT_ID="a" * 32)},
|
|
{"event": _event(_PID="20")},
|
|
{"event": _event(UNIT="virtqemud.service")},
|
|
{"event": _event(__CURSOR="")},
|
|
{"duplicate_result": True},
|
|
{"journal_failure": True},
|
|
{"raw_export": "X" * 4097},
|
|
{"requires": "unrelated.service"},
|
|
{"after": "unrelated.service"},
|
|
{"daemon_state": "failed"},
|
|
{"daemon_job": "3"},
|
|
{"event": None},
|
|
])
|
|
def test_unproved_or_superseded_job_never_restarts_daemon(tmp_path, changes):
|
|
run, _, _ = _harness(tmp_path, **changes)
|
|
state = run()
|
|
assert _daemon_starts(state) == []
|
|
assert ["start", FABRIC_SERVICE] in state["actions"]
|
|
assert not any(action == ["reset-failed", DAEMON] for action in state["actions"])
|
|
|
|
|
|
@pytest.mark.parametrize("change", [
|
|
{"event": _event(JOB_TYPE="stop", JOB_RESULT="done")},
|
|
{"event": _event(JOB_RESULT="failed")},
|
|
{"daemon_job": "9"},
|
|
{"daemon_state": "active"},
|
|
])
|
|
def test_new_job_while_primary_is_repaired_invalidates_claim(tmp_path, change):
|
|
run, _, _ = _harness(tmp_path, on_primary_start=change)
|
|
assert _daemon_starts(run()) == []
|
|
|
|
|
|
def test_job_changes_between_last_two_state_reads_are_not_replayed(tmp_path):
|
|
run, _, _ = _harness(
|
|
tmp_path, change_at_read=3, changed_event=_event(JOB_TYPE="stop", JOB_RESULT="done"),
|
|
)
|
|
assert _daemon_starts(run()) == []
|
|
|
|
|
|
@pytest.mark.parametrize("kind", ["symlink", "unknown_file", "wrong_mode"])
|
|
def test_unknown_retry_inventory_is_preserved_and_primary_still_repairs(tmp_path, kind):
|
|
run, runtime, _ = _harness(tmp_path)
|
|
retry = runtime / "fabric-gate-retry"
|
|
retry.mkdir(mode=0o700)
|
|
target = tmp_path / "external"
|
|
target.write_text("preserve\n")
|
|
if kind == "symlink":
|
|
(retry / DAEMON).symlink_to(target)
|
|
elif kind == "unknown_file":
|
|
(retry / "unknown").write_text("preserve\n")
|
|
else:
|
|
retry.chmod(0o755)
|
|
state = run()
|
|
assert _daemon_starts(state) == []
|
|
assert ["start", FABRIC_SERVICE] in state["actions"]
|
|
assert target.read_text() == "preserve\n"
|
|
if kind == "unknown_file":
|
|
assert (retry / "unknown").read_text() == "preserve\n"
|