You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
381 lines
17 KiB
Python
381 lines
17 KiB
Python
"""Offline contracts for the code-only privileged update worker."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unittest
|
|
from contextlib import nullcontext
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from virsh_panel import update_worker
|
|
|
|
|
|
class FakeStore:
|
|
def __init__(self, job_id: str | None):
|
|
self.job_id = job_id
|
|
|
|
def get_value(self, name: str) -> str | None:
|
|
if name == "update.active_job":
|
|
return self.job_id
|
|
raise AssertionError(name)
|
|
|
|
|
|
class FakeUpdater:
|
|
def __init__(self, job: dict | None):
|
|
self.job = job
|
|
self.repo_dir = Path("/tmp")
|
|
self.failed: list[tuple[str, str]] = []
|
|
|
|
def get_job(self, _job_id: str) -> dict | None:
|
|
return self.job
|
|
|
|
def _checkout_head(self, *, require_clean: bool):
|
|
if not require_clean:
|
|
raise AssertionError("recovery must require a clean checkout")
|
|
return "a" * 40
|
|
|
|
def fail_job(self, job_id: str, message: str) -> None:
|
|
self.failed.append((job_id, message))
|
|
if self.job is not None:
|
|
self.job["status"] = "failed"
|
|
|
|
|
|
class UpdateWorkerTests(unittest.TestCase):
|
|
def test_service_consumers_start_only_after_engine_is_healthy(self):
|
|
calls = []
|
|
deployment = object.__new__(update_worker.DeploymentTransaction)
|
|
deployment.repo_dir = Path("/tmp")
|
|
deployment.config = {"service_secret": "unused"}
|
|
deployment.units = update_worker._service_units(deployment.config)
|
|
|
|
def start(units, *, message):
|
|
calls.append(("start", units, message))
|
|
|
|
def wait(_repo_dir, units):
|
|
calls.append(("active", units))
|
|
|
|
with patch.object(deployment, "_start_services", side_effect=start), \
|
|
patch.object(update_worker, "_wait_units", side_effect=wait), \
|
|
patch.object(update_worker, "_wait_engine_health",
|
|
side_effect=lambda config: calls.append(("health", config))):
|
|
deployment.start_and_check()
|
|
|
|
engine = ("virt-cloud-engine.service",)
|
|
consumers = (
|
|
"virt-cloud-worker.service",
|
|
"virt-cloud-vpc-worker.service",
|
|
"virt-cloud-lifecycle.service",
|
|
"virt-cloud-policy-worker@1.service",
|
|
"virt-cloud-policy-worker@2.service",
|
|
)
|
|
web = ("php-fpm.service",)
|
|
self.assertEqual(calls[0][:2], ("start", web))
|
|
self.assertEqual(calls[1], ("active", web))
|
|
self.assertEqual(calls[2][:2], ("start", engine))
|
|
self.assertEqual(calls[3], ("active", engine))
|
|
self.assertEqual(calls[4], ("health", deployment.config))
|
|
self.assertEqual(calls[5][:2], ("start", consumers))
|
|
self.assertEqual(calls[6], ("active", consumers))
|
|
self.assertIn("PHP-FPM", calls[0][2])
|
|
self.assertIn("私有引擎", calls[2][2])
|
|
self.assertIn("后台服务", calls[5][2])
|
|
|
|
def test_engine_health_failure_restores_gated_consumers_before_raising(self):
|
|
calls = []
|
|
deployment = object.__new__(update_worker.DeploymentTransaction)
|
|
deployment.repo_dir = Path("/tmp")
|
|
deployment.config = {"service_secret": "unused"}
|
|
deployment.units = update_worker._service_units(deployment.config)
|
|
|
|
def start(units, *, message):
|
|
calls.append(("start", units, message))
|
|
|
|
def unhealthy(config):
|
|
calls.append(("health", config))
|
|
raise update_worker.UpdateError("引擎未就绪", 500)
|
|
|
|
with patch.object(deployment, "_start_services", side_effect=start), \
|
|
patch.object(update_worker, "_wait_units",
|
|
side_effect=lambda _repo, units: calls.append(("active", units))), \
|
|
patch.object(update_worker, "_wait_engine_health",
|
|
side_effect=unhealthy):
|
|
with self.assertRaisesRegex(update_worker.UpdateError, "引擎未就绪"):
|
|
deployment.start_and_check()
|
|
|
|
self.assertEqual(calls[0][:2], ("start", ("php-fpm.service",)))
|
|
self.assertEqual(calls[2][:2], ("start", ("virt-cloud-engine.service",)))
|
|
self.assertEqual(calls[4], ("health", deployment.config))
|
|
self.assertEqual(calls[5][:2], ("start", (
|
|
"virt-cloud-worker.service",
|
|
"virt-cloud-vpc-worker.service",
|
|
"virt-cloud-lifecycle.service",
|
|
"virt-cloud-policy-worker@1.service",
|
|
"virt-cloud-policy-worker@2.service",
|
|
)))
|
|
|
|
def test_start_services_uses_only_the_explicit_unit_group(self):
|
|
deployment = object.__new__(update_worker.DeploymentTransaction)
|
|
deployment.repo_dir = Path("/tmp")
|
|
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
|
|
patch.object(update_worker, "_run") as run:
|
|
deployment._start_services(
|
|
("virt-cloud-engine.service",), message="engine start failed"
|
|
)
|
|
|
|
run.assert_called_once_with(
|
|
["/usr/bin/systemctl", "start", "virt-cloud-engine.service"],
|
|
cwd=Path("/tmp"),
|
|
timeout=180,
|
|
message="engine start failed",
|
|
environment={"PATH": update_worker._EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
|
|
)
|
|
|
|
def test_running_job_is_marked_failed_once(self):
|
|
store = FakeStore("job-1234567890123456")
|
|
updater = FakeUpdater({"status": "running"})
|
|
self.assertTrue(update_worker.finalize_incomplete_job(updater, store))
|
|
self.assertEqual(updater.job["status"], "failed")
|
|
self.assertEqual(len(updater.failed), 1)
|
|
self.assertIn("日志", updater.failed[0][1])
|
|
self.assertFalse(update_worker.finalize_incomplete_job(updater, store))
|
|
self.assertEqual(len(updater.failed), 1)
|
|
|
|
def test_queued_job_is_marked_failed(self):
|
|
store = FakeStore("job-1234567890123456")
|
|
updater = FakeUpdater({"status": "queued"})
|
|
self.assertTrue(update_worker.finalize_incomplete_job(updater, store))
|
|
self.assertEqual(updater.job["status"], "failed")
|
|
|
|
def test_terminal_or_missing_job_is_not_modified(self):
|
|
for status in ("succeeded", "failed", "blocked"):
|
|
with self.subTest(status=status):
|
|
updater = FakeUpdater({"status": status})
|
|
self.assertFalse(update_worker.finalize_incomplete_job(
|
|
updater, FakeStore("job-1234567890123456")))
|
|
self.assertEqual(updater.failed, [])
|
|
self.assertFalse(update_worker.finalize_incomplete_job(
|
|
FakeUpdater(None), FakeStore("job-1234567890123456")))
|
|
self.assertFalse(update_worker.finalize_incomplete_job(
|
|
FakeUpdater({"status": "running"}), FakeStore(None)))
|
|
|
|
def test_cli_finalize_mode_recovers_services_only_after_schema_checkpoint(self):
|
|
store = FakeStore("job-1234567890123456")
|
|
updater = FakeUpdater({
|
|
"status": "running",
|
|
"deployment_state": "schema_ready",
|
|
"new_commit": "a" * 40,
|
|
})
|
|
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
|
|
patch.object(update_worker, "_managed_service_units", return_value=(
|
|
"virt-cloud-engine.service", "php-fpm.service")), \
|
|
patch.object(update_worker.DeploymentTransaction, "start_and_check") as start:
|
|
self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0)
|
|
self.assertEqual(updater.job["status"], "failed")
|
|
start.assert_called_once_with()
|
|
|
|
def test_cli_finalize_mode_leaves_services_stopped_without_safe_checkpoint(self):
|
|
for state in ("code_switched", "schema_migrating"):
|
|
with self.subTest(state=state):
|
|
store = FakeStore("job-1234567890123456")
|
|
updater = FakeUpdater({"status": "running", "deployment_state": state})
|
|
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
|
|
patch.object(update_worker.DeploymentTransaction, "start_and_check") as start:
|
|
self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0)
|
|
start.assert_not_called()
|
|
|
|
def test_cli_recover_blocked_runs_forward_migration_and_health_sequence(self):
|
|
calls = []
|
|
|
|
class RecoveryUpdater(FakeUpdater):
|
|
def _apply_lock(self):
|
|
calls.append("lock")
|
|
return nullcontext()
|
|
|
|
def begin_blocked_recovery(self):
|
|
calls.append("begin")
|
|
return {"job_id": "job-1234567890123456"}
|
|
|
|
def recovery_checkpoint(self, job_id, state, phase):
|
|
calls.append(("checkpoint", job_id, state, phase))
|
|
|
|
def complete_blocked_recovery(self, job_id):
|
|
calls.append(("complete", job_id))
|
|
|
|
class RecoveryDeployment:
|
|
def __init__(self, repo_dir, config, *, recovery=False):
|
|
calls.append(("deployment", repo_dir, config, recovery))
|
|
|
|
def quiesce(self):
|
|
calls.append("quiesce")
|
|
|
|
def activate(self, mark_schema_ready):
|
|
calls.append("cloudInit")
|
|
mark_schema_ready()
|
|
calls.append("services-healthy")
|
|
|
|
store = FakeStore("job-1234567890123456")
|
|
updater = RecoveryUpdater({"status": "blocked"})
|
|
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
|
|
patch.object(update_worker, "DeploymentTransaction", RecoveryDeployment):
|
|
self.assertEqual(update_worker.main(["--recover-blocked"]), 0)
|
|
|
|
self.assertEqual(calls[0][0], "deployment")
|
|
self.assertTrue(calls[0][3])
|
|
self.assertEqual(calls[1:4], ["lock", "begin", "quiesce"])
|
|
self.assertEqual(calls[4][:3], (
|
|
"checkpoint", "job-1234567890123456", "schema_migrating"))
|
|
self.assertEqual(calls[5], "cloudInit")
|
|
self.assertEqual(calls[6][:3], (
|
|
"checkpoint", "job-1234567890123456", "schema_ready"))
|
|
self.assertEqual(calls[7:], [
|
|
"services-healthy", ("complete", "job-1234567890123456")])
|
|
|
|
def test_update_worker_is_a_separate_hardened_oneshot(self):
|
|
deploy = Path(__file__).resolve().parents[2] / "deploy"
|
|
engine_unit = (deploy / "virt-cloud-engine.service").read_text(encoding="utf-8")
|
|
update_unit = (deploy / "virt-cloud-update.service").read_text(encoding="utf-8")
|
|
self.assertIn("virsh_panel.internal:create_engine", engine_unit)
|
|
engine_exec = next(
|
|
line for line in engine_unit.splitlines() if line.startswith("ExecStart=")
|
|
)
|
|
self.assertIn(" --workers 1 ", engine_exec)
|
|
self.assertEqual(engine_exec.split().count("--workers"), 1)
|
|
self.assertNotIn("update_worker", engine_unit)
|
|
for value in (
|
|
"Type=oneshot",
|
|
"User=root",
|
|
"WorkingDirectory=/home/virt-cloud/engine",
|
|
"CLOUD_ENGINE_CONFIG=/home/virt-cloud/runtime/engine.json",
|
|
"ExecStart=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker",
|
|
"ExecStopPost=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker --finalize-incomplete",
|
|
"TimeoutStartSec=45min",
|
|
"UMask=0077",
|
|
"NoNewPrivileges=true",
|
|
"PrivateTmp=true",
|
|
"ProtectSystem=strict",
|
|
"ProtectHome=false",
|
|
"ReadWritePaths=/home/virt-cloud",
|
|
"LimitCORE=0",
|
|
):
|
|
self.assertIn(value, update_unit)
|
|
self.assertNotIn("[Install]", update_unit)
|
|
self.assertNotIn("/home/virsh-panel", update_unit)
|
|
self.assertNotIn("virsh-panel.service", update_unit)
|
|
self.assertNotIn("/etc/systemd/system", update_unit)
|
|
self.assertNotIn("ExecStopPost=-/usr/bin/systemctl start", update_unit)
|
|
|
|
def test_online_worker_is_code_only_and_uses_configured_fpm(self):
|
|
source = Path(update_worker.__file__).read_text(encoding="utf-8")
|
|
for forbidden in (
|
|
"daemon-reload",
|
|
"/etc/systemd/system",
|
|
'"pip", "install"',
|
|
'_executable("composer")',
|
|
'[composer, "install"',
|
|
"update-staging",
|
|
"web/vendor",
|
|
"tempfile.",
|
|
"def _git_file(",
|
|
"prepare=",
|
|
):
|
|
self.assertNotIn(forbidden, source)
|
|
self.assertIn("mark_schema_ready()", source)
|
|
units = update_worker._service_units({"update_php_fpm_unit": "php85-php-fpm.service"})
|
|
self.assertIn("php85-php-fpm.service", units)
|
|
self.assertNotIn("php-fpm.service", units)
|
|
with self.assertRaisesRegex(Exception, "服务名"):
|
|
update_worker._service_units({"update_php_fpm_unit": "bad;service"})
|
|
|
|
def test_normal_update_preserves_running_installed_units_and_skips_missing_optional(self):
|
|
states = {
|
|
"virt-cloud-engine.service": ("loaded", "active"),
|
|
"virt-cloud-worker.service": ("loaded", "active"),
|
|
"virt-cloud-vpc-worker.service": ("not-found", "inactive"),
|
|
"virt-cloud-lifecycle.service": ("not-found", "inactive"),
|
|
"virt-cloud-policy-worker@1.service": ("not-found", "inactive"),
|
|
"virt-cloud-policy-worker@2.service": ("not-found", "inactive"),
|
|
"php-fpm.service": ("loaded", "active"),
|
|
}
|
|
|
|
def systemctl(argv, **_kwargs):
|
|
load, active = states[argv[-1]]
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout=f"LoadState={load}\nActiveState={active}\n",
|
|
)
|
|
|
|
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
|
|
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
|
|
units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=False)
|
|
self.assertEqual(units, (
|
|
"virt-cloud-engine.service",
|
|
"virt-cloud-worker.service",
|
|
"php-fpm.service",
|
|
))
|
|
|
|
def test_normal_update_quiesces_only_a_running_vpc_worker(self):
|
|
vpc_unit = "virt-cloud-vpc-worker.service"
|
|
for vpc_state in ("active", "inactive"):
|
|
with self.subTest(vpc_state=vpc_state):
|
|
inspected = []
|
|
|
|
def systemctl(argv, **_kwargs):
|
|
unit = argv[-1]
|
|
inspected.append(unit)
|
|
active = vpc_state if unit == vpc_unit else "active"
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout=f"LoadState=loaded\nActiveState={active}\n",
|
|
)
|
|
|
|
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
|
|
patch.object(update_worker.subprocess, "run", side_effect=systemctl), \
|
|
patch.object(update_worker, "_run") as run:
|
|
deployment = update_worker.DeploymentTransaction(Path("/tmp"), {})
|
|
deployment.quiesce()
|
|
|
|
self.assertIn(vpc_unit, inspected)
|
|
self.assertEqual(vpc_unit in deployment.units, vpc_state == "active")
|
|
run.assert_called_once()
|
|
command = run.call_args.args[0]
|
|
self.assertEqual(command[:2], ["/usr/bin/systemctl", "stop"])
|
|
self.assertEqual(command[2:], list(deployment.units))
|
|
self.assertEqual(vpc_unit in command, vpc_state == "active")
|
|
|
|
def test_recovery_starts_installed_units_even_when_they_are_stopped(self):
|
|
states = {
|
|
unit: ("loaded", "inactive") for unit in update_worker._service_units({})
|
|
}
|
|
|
|
def systemctl(argv, **_kwargs):
|
|
load, active = states[argv[-1]]
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout=f"LoadState={load}\nActiveState={active}\n",
|
|
)
|
|
|
|
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
|
|
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
|
|
units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=True)
|
|
self.assertEqual(units, update_worker._service_units({}))
|
|
self.assertIn("virt-cloud-vpc-worker.service", units)
|
|
|
|
def test_update_refuses_before_stopping_when_required_service_is_missing(self):
|
|
def systemctl(argv, **_kwargs):
|
|
unit = argv[-1]
|
|
load = "not-found" if unit == "php-fpm.service" else "loaded"
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout=f"LoadState={load}\nActiveState=active\n",
|
|
)
|
|
|
|
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
|
|
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
|
|
with self.assertRaisesRegex(Exception, "php-fpm.service 未安装"):
|
|
update_worker._managed_service_units(Path("/tmp"), {}, recovery=False)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|