You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_update_worker.py

381 lines
17 KiB
Python

"""Offline contracts for the code-only privileged update worker."""
from __future__ import annotations
import unittest
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from virsh_panel import update_worker
class FakeStore:
def __init__(self, job_id: str | None):
self.job_id = job_id
def get_value(self, name: str) -> str | None:
if name == "update.active_job":
return self.job_id
raise AssertionError(name)
class FakeUpdater:
def __init__(self, job: dict | None):
self.job = job
self.repo_dir = Path("/tmp")
self.failed: list[tuple[str, str]] = []
def get_job(self, _job_id: str) -> dict | None:
return self.job
def _checkout_head(self, *, require_clean: bool):
if not require_clean:
raise AssertionError("recovery must require a clean checkout")
return "a" * 40
def fail_job(self, job_id: str, message: str) -> None:
self.failed.append((job_id, message))
if self.job is not None:
self.job["status"] = "failed"
class UpdateWorkerTests(unittest.TestCase):
def test_service_consumers_start_only_after_engine_is_healthy(self):
calls = []
deployment = object.__new__(update_worker.DeploymentTransaction)
deployment.repo_dir = Path("/tmp")
deployment.config = {"service_secret": "unused"}
deployment.units = update_worker._service_units(deployment.config)
def start(units, *, message):
calls.append(("start", units, message))
def wait(_repo_dir, units):
calls.append(("active", units))
with patch.object(deployment, "_start_services", side_effect=start), \
patch.object(update_worker, "_wait_units", side_effect=wait), \
patch.object(update_worker, "_wait_engine_health",
side_effect=lambda config: calls.append(("health", config))):
deployment.start_and_check()
engine = ("virt-cloud-engine.service",)
consumers = (
"virt-cloud-worker.service",
"virt-cloud-vpc-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
)
web = ("php-fpm.service",)
self.assertEqual(calls[0][:2], ("start", web))
self.assertEqual(calls[1], ("active", web))
self.assertEqual(calls[2][:2], ("start", engine))
self.assertEqual(calls[3], ("active", engine))
self.assertEqual(calls[4], ("health", deployment.config))
self.assertEqual(calls[5][:2], ("start", consumers))
self.assertEqual(calls[6], ("active", consumers))
self.assertIn("PHP-FPM", calls[0][2])
self.assertIn("私有引擎", calls[2][2])
self.assertIn("后台服务", calls[5][2])
def test_engine_health_failure_restores_gated_consumers_before_raising(self):
calls = []
deployment = object.__new__(update_worker.DeploymentTransaction)
deployment.repo_dir = Path("/tmp")
deployment.config = {"service_secret": "unused"}
deployment.units = update_worker._service_units(deployment.config)
def start(units, *, message):
calls.append(("start", units, message))
def unhealthy(config):
calls.append(("health", config))
raise update_worker.UpdateError("引擎未就绪", 500)
with patch.object(deployment, "_start_services", side_effect=start), \
patch.object(update_worker, "_wait_units",
side_effect=lambda _repo, units: calls.append(("active", units))), \
patch.object(update_worker, "_wait_engine_health",
side_effect=unhealthy):
with self.assertRaisesRegex(update_worker.UpdateError, "引擎未就绪"):
deployment.start_and_check()
self.assertEqual(calls[0][:2], ("start", ("php-fpm.service",)))
self.assertEqual(calls[2][:2], ("start", ("virt-cloud-engine.service",)))
self.assertEqual(calls[4], ("health", deployment.config))
self.assertEqual(calls[5][:2], ("start", (
"virt-cloud-worker.service",
"virt-cloud-vpc-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
)))
def test_start_services_uses_only_the_explicit_unit_group(self):
deployment = object.__new__(update_worker.DeploymentTransaction)
deployment.repo_dir = Path("/tmp")
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker, "_run") as run:
deployment._start_services(
("virt-cloud-engine.service",), message="engine start failed"
)
run.assert_called_once_with(
["/usr/bin/systemctl", "start", "virt-cloud-engine.service"],
cwd=Path("/tmp"),
timeout=180,
message="engine start failed",
environment={"PATH": update_worker._EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
)
def test_running_job_is_marked_failed_once(self):
store = FakeStore("job-1234567890123456")
updater = FakeUpdater({"status": "running"})
self.assertTrue(update_worker.finalize_incomplete_job(updater, store))
self.assertEqual(updater.job["status"], "failed")
self.assertEqual(len(updater.failed), 1)
self.assertIn("日志", updater.failed[0][1])
self.assertFalse(update_worker.finalize_incomplete_job(updater, store))
self.assertEqual(len(updater.failed), 1)
def test_queued_job_is_marked_failed(self):
store = FakeStore("job-1234567890123456")
updater = FakeUpdater({"status": "queued"})
self.assertTrue(update_worker.finalize_incomplete_job(updater, store))
self.assertEqual(updater.job["status"], "failed")
def test_terminal_or_missing_job_is_not_modified(self):
for status in ("succeeded", "failed", "blocked"):
with self.subTest(status=status):
updater = FakeUpdater({"status": status})
self.assertFalse(update_worker.finalize_incomplete_job(
updater, FakeStore("job-1234567890123456")))
self.assertEqual(updater.failed, [])
self.assertFalse(update_worker.finalize_incomplete_job(
FakeUpdater(None), FakeStore("job-1234567890123456")))
self.assertFalse(update_worker.finalize_incomplete_job(
FakeUpdater({"status": "running"}), FakeStore(None)))
def test_cli_finalize_mode_recovers_services_only_after_schema_checkpoint(self):
store = FakeStore("job-1234567890123456")
updater = FakeUpdater({
"status": "running",
"deployment_state": "schema_ready",
"new_commit": "a" * 40,
})
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
patch.object(update_worker, "_managed_service_units", return_value=(
"virt-cloud-engine.service", "php-fpm.service")), \
patch.object(update_worker.DeploymentTransaction, "start_and_check") as start:
self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0)
self.assertEqual(updater.job["status"], "failed")
start.assert_called_once_with()
def test_cli_finalize_mode_leaves_services_stopped_without_safe_checkpoint(self):
for state in ("code_switched", "schema_migrating"):
with self.subTest(state=state):
store = FakeStore("job-1234567890123456")
updater = FakeUpdater({"status": "running", "deployment_state": state})
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
patch.object(update_worker.DeploymentTransaction, "start_and_check") as start:
self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0)
start.assert_not_called()
def test_cli_recover_blocked_runs_forward_migration_and_health_sequence(self):
calls = []
class RecoveryUpdater(FakeUpdater):
def _apply_lock(self):
calls.append("lock")
return nullcontext()
def begin_blocked_recovery(self):
calls.append("begin")
return {"job_id": "job-1234567890123456"}
def recovery_checkpoint(self, job_id, state, phase):
calls.append(("checkpoint", job_id, state, phase))
def complete_blocked_recovery(self, job_id):
calls.append(("complete", job_id))
class RecoveryDeployment:
def __init__(self, repo_dir, config, *, recovery=False):
calls.append(("deployment", repo_dir, config, recovery))
def quiesce(self):
calls.append("quiesce")
def activate(self, mark_schema_ready):
calls.append("cloudInit")
mark_schema_ready()
calls.append("services-healthy")
store = FakeStore("job-1234567890123456")
updater = RecoveryUpdater({"status": "blocked"})
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
patch.object(update_worker, "DeploymentTransaction", RecoveryDeployment):
self.assertEqual(update_worker.main(["--recover-blocked"]), 0)
self.assertEqual(calls[0][0], "deployment")
self.assertTrue(calls[0][3])
self.assertEqual(calls[1:4], ["lock", "begin", "quiesce"])
self.assertEqual(calls[4][:3], (
"checkpoint", "job-1234567890123456", "schema_migrating"))
self.assertEqual(calls[5], "cloudInit")
self.assertEqual(calls[6][:3], (
"checkpoint", "job-1234567890123456", "schema_ready"))
self.assertEqual(calls[7:], [
"services-healthy", ("complete", "job-1234567890123456")])
def test_update_worker_is_a_separate_hardened_oneshot(self):
deploy = Path(__file__).resolve().parents[2] / "deploy"
engine_unit = (deploy / "virt-cloud-engine.service").read_text(encoding="utf-8")
update_unit = (deploy / "virt-cloud-update.service").read_text(encoding="utf-8")
self.assertIn("virsh_panel.internal:create_engine", engine_unit)
engine_exec = next(
line for line in engine_unit.splitlines() if line.startswith("ExecStart=")
)
self.assertIn(" --workers 1 ", engine_exec)
self.assertEqual(engine_exec.split().count("--workers"), 1)
self.assertNotIn("update_worker", engine_unit)
for value in (
"Type=oneshot",
"User=root",
"WorkingDirectory=/home/virt-cloud/engine",
"CLOUD_ENGINE_CONFIG=/home/virt-cloud/runtime/engine.json",
"ExecStart=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker",
"ExecStopPost=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker --finalize-incomplete",
"TimeoutStartSec=45min",
"UMask=0077",
"NoNewPrivileges=true",
"PrivateTmp=true",
"ProtectSystem=strict",
"ProtectHome=false",
"ReadWritePaths=/home/virt-cloud",
"LimitCORE=0",
):
self.assertIn(value, update_unit)
self.assertNotIn("[Install]", update_unit)
self.assertNotIn("/home/virsh-panel", update_unit)
self.assertNotIn("virsh-panel.service", update_unit)
self.assertNotIn("/etc/systemd/system", update_unit)
self.assertNotIn("ExecStopPost=-/usr/bin/systemctl start", update_unit)
def test_online_worker_is_code_only_and_uses_configured_fpm(self):
source = Path(update_worker.__file__).read_text(encoding="utf-8")
for forbidden in (
"daemon-reload",
"/etc/systemd/system",
'"pip", "install"',
'_executable("composer")',
'[composer, "install"',
"update-staging",
"web/vendor",
"tempfile.",
"def _git_file(",
"prepare=",
):
self.assertNotIn(forbidden, source)
self.assertIn("mark_schema_ready()", source)
units = update_worker._service_units({"update_php_fpm_unit": "php85-php-fpm.service"})
self.assertIn("php85-php-fpm.service", units)
self.assertNotIn("php-fpm.service", units)
with self.assertRaisesRegex(Exception, "服务名"):
update_worker._service_units({"update_php_fpm_unit": "bad;service"})
def test_normal_update_preserves_running_installed_units_and_skips_missing_optional(self):
states = {
"virt-cloud-engine.service": ("loaded", "active"),
"virt-cloud-worker.service": ("loaded", "active"),
"virt-cloud-vpc-worker.service": ("not-found", "inactive"),
"virt-cloud-lifecycle.service": ("not-found", "inactive"),
"virt-cloud-policy-worker@1.service": ("not-found", "inactive"),
"virt-cloud-policy-worker@2.service": ("not-found", "inactive"),
"php-fpm.service": ("loaded", "active"),
}
def systemctl(argv, **_kwargs):
load, active = states[argv[-1]]
return SimpleNamespace(
returncode=0,
stdout=f"LoadState={load}\nActiveState={active}\n",
)
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=False)
self.assertEqual(units, (
"virt-cloud-engine.service",
"virt-cloud-worker.service",
"php-fpm.service",
))
def test_normal_update_quiesces_only_a_running_vpc_worker(self):
vpc_unit = "virt-cloud-vpc-worker.service"
for vpc_state in ("active", "inactive"):
with self.subTest(vpc_state=vpc_state):
inspected = []
def systemctl(argv, **_kwargs):
unit = argv[-1]
inspected.append(unit)
active = vpc_state if unit == vpc_unit else "active"
return SimpleNamespace(
returncode=0,
stdout=f"LoadState=loaded\nActiveState={active}\n",
)
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker.subprocess, "run", side_effect=systemctl), \
patch.object(update_worker, "_run") as run:
deployment = update_worker.DeploymentTransaction(Path("/tmp"), {})
deployment.quiesce()
self.assertIn(vpc_unit, inspected)
self.assertEqual(vpc_unit in deployment.units, vpc_state == "active")
run.assert_called_once()
command = run.call_args.args[0]
self.assertEqual(command[:2], ["/usr/bin/systemctl", "stop"])
self.assertEqual(command[2:], list(deployment.units))
self.assertEqual(vpc_unit in command, vpc_state == "active")
def test_recovery_starts_installed_units_even_when_they_are_stopped(self):
states = {
unit: ("loaded", "inactive") for unit in update_worker._service_units({})
}
def systemctl(argv, **_kwargs):
load, active = states[argv[-1]]
return SimpleNamespace(
returncode=0,
stdout=f"LoadState={load}\nActiveState={active}\n",
)
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=True)
self.assertEqual(units, update_worker._service_units({}))
self.assertIn("virt-cloud-vpc-worker.service", units)
def test_update_refuses_before_stopping_when_required_service_is_missing(self):
def systemctl(argv, **_kwargs):
unit = argv[-1]
load = "not-found" if unit == "php-fpm.service" else "loaded"
return SimpleNamespace(
returncode=0,
stdout=f"LoadState={load}\nActiveState=active\n",
)
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
with self.assertRaisesRegex(Exception, "php-fpm.service 未安装"):
update_worker._managed_service_units(Path("/tmp"), {}, recovery=False)
if __name__ == "__main__":
unittest.main()