"""Offline contracts for the code-only privileged update worker.""" from __future__ import annotations import unittest from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace from unittest.mock import patch from virsh_panel import update_worker class FakeStore: def __init__(self, job_id: str | None): self.job_id = job_id def get_value(self, name: str) -> str | None: if name == "update.active_job": return self.job_id raise AssertionError(name) class FakeUpdater: def __init__(self, job: dict | None): self.job = job self.repo_dir = Path("/tmp") self.failed: list[tuple[str, str]] = [] def get_job(self, _job_id: str) -> dict | None: return self.job def _checkout_head(self, *, require_clean: bool): if not require_clean: raise AssertionError("recovery must require a clean checkout") return "a" * 40 def fail_job(self, job_id: str, message: str) -> None: self.failed.append((job_id, message)) if self.job is not None: self.job["status"] = "failed" class UpdateWorkerTests(unittest.TestCase): def test_service_consumers_start_only_after_engine_is_healthy(self): calls = [] deployment = object.__new__(update_worker.DeploymentTransaction) deployment.repo_dir = Path("/tmp") deployment.config = {"service_secret": "unused"} deployment.units = ( "virt-cloud-engine.service", "virt-cloud-worker.service", "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", "virt-cloud-policy-worker@2.service", "php-fpm.service", ) def start(units, *, message): calls.append(("start", units, message)) def wait(_repo_dir, units): calls.append(("active", units)) with patch.object(deployment, "_start_services", side_effect=start), \ patch.object(update_worker, "_wait_units", side_effect=wait), \ patch.object(update_worker, "_wait_engine_health", side_effect=lambda config: calls.append(("health", config))): deployment.start_and_check() engine = ("virt-cloud-engine.service",) consumers = ( "virt-cloud-worker.service", "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", "virt-cloud-policy-worker@2.service", ) web = ("php-fpm.service",) self.assertEqual(calls[0][:2], ("start", web)) self.assertEqual(calls[1], ("active", web)) self.assertEqual(calls[2][:2], ("start", engine)) self.assertEqual(calls[3], ("active", engine)) self.assertEqual(calls[4], ("health", deployment.config)) self.assertEqual(calls[5][:2], ("start", consumers)) self.assertEqual(calls[6], ("active", consumers)) self.assertIn("PHP-FPM", calls[0][2]) self.assertIn("私有引擎", calls[2][2]) self.assertIn("后台服务", calls[5][2]) def test_engine_health_failure_restores_gated_consumers_before_raising(self): calls = [] deployment = object.__new__(update_worker.DeploymentTransaction) deployment.repo_dir = Path("/tmp") deployment.config = {"service_secret": "unused"} deployment.units = ( "virt-cloud-engine.service", "virt-cloud-worker.service", "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", "virt-cloud-policy-worker@2.service", "php-fpm.service", ) def start(units, *, message): calls.append(("start", units, message)) def unhealthy(config): calls.append(("health", config)) raise update_worker.UpdateError("引擎未就绪", 500) with patch.object(deployment, "_start_services", side_effect=start), \ patch.object(update_worker, "_wait_units", side_effect=lambda _repo, units: calls.append(("active", units))), \ patch.object(update_worker, "_wait_engine_health", side_effect=unhealthy): with self.assertRaisesRegex(update_worker.UpdateError, "引擎未就绪"): deployment.start_and_check() self.assertEqual(calls[0][:2], ("start", ("php-fpm.service",))) self.assertEqual(calls[2][:2], ("start", ("virt-cloud-engine.service",))) self.assertEqual(calls[4], ("health", deployment.config)) self.assertEqual(calls[5][:2], ("start", ( "virt-cloud-worker.service", "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", "virt-cloud-policy-worker@2.service", ))) def test_start_services_uses_only_the_explicit_unit_group(self): deployment = object.__new__(update_worker.DeploymentTransaction) deployment.repo_dir = Path("/tmp") with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \ patch.object(update_worker, "_run") as run: deployment._start_services( ("virt-cloud-engine.service",), message="engine start failed" ) run.assert_called_once_with( ["/usr/bin/systemctl", "start", "virt-cloud-engine.service"], cwd=Path("/tmp"), timeout=180, message="engine start failed", environment={"PATH": update_worker._EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"}, ) def test_running_job_is_marked_failed_once(self): store = FakeStore("job-1234567890123456") updater = FakeUpdater({"status": "running"}) self.assertTrue(update_worker.finalize_incomplete_job(updater, store)) self.assertEqual(updater.job["status"], "failed") self.assertEqual(len(updater.failed), 1) self.assertIn("日志", updater.failed[0][1]) self.assertFalse(update_worker.finalize_incomplete_job(updater, store)) self.assertEqual(len(updater.failed), 1) def test_queued_job_is_marked_failed(self): store = FakeStore("job-1234567890123456") updater = FakeUpdater({"status": "queued"}) self.assertTrue(update_worker.finalize_incomplete_job(updater, store)) self.assertEqual(updater.job["status"], "failed") def test_terminal_or_missing_job_is_not_modified(self): for status in ("succeeded", "failed", "blocked"): with self.subTest(status=status): updater = FakeUpdater({"status": status}) self.assertFalse(update_worker.finalize_incomplete_job( updater, FakeStore("job-1234567890123456"))) self.assertEqual(updater.failed, []) self.assertFalse(update_worker.finalize_incomplete_job( FakeUpdater(None), FakeStore("job-1234567890123456"))) self.assertFalse(update_worker.finalize_incomplete_job( FakeUpdater({"status": "running"}), FakeStore(None))) def test_cli_finalize_mode_recovers_services_only_after_schema_checkpoint(self): store = FakeStore("job-1234567890123456") updater = FakeUpdater({ "status": "running", "deployment_state": "schema_ready", "new_commit": "a" * 40, }) with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \ patch.object(update_worker, "_managed_service_units", return_value=( "virt-cloud-engine.service", "php-fpm.service")), \ patch.object(update_worker.DeploymentTransaction, "start_and_check") as start: self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0) self.assertEqual(updater.job["status"], "failed") start.assert_called_once_with() def test_cli_finalize_mode_leaves_services_stopped_without_safe_checkpoint(self): for state in ("code_switched", "schema_migrating"): with self.subTest(state=state): store = FakeStore("job-1234567890123456") updater = FakeUpdater({"status": "running", "deployment_state": state}) with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \ patch.object(update_worker.DeploymentTransaction, "start_and_check") as start: self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0) start.assert_not_called() def test_cli_recover_blocked_runs_forward_migration_and_health_sequence(self): calls = [] class RecoveryUpdater(FakeUpdater): def _apply_lock(self): calls.append("lock") return nullcontext() def begin_blocked_recovery(self): calls.append("begin") return {"job_id": "job-1234567890123456"} def recovery_checkpoint(self, job_id, state, phase): calls.append(("checkpoint", job_id, state, phase)) def complete_blocked_recovery(self, job_id): calls.append(("complete", job_id)) class RecoveryDeployment: def __init__(self, repo_dir, config, *, recovery=False): calls.append(("deployment", repo_dir, config, recovery)) def quiesce(self): calls.append("quiesce") def activate(self, mark_schema_ready): calls.append("cloudInit") mark_schema_ready() calls.append("services-healthy") store = FakeStore("job-1234567890123456") updater = RecoveryUpdater({"status": "blocked"}) with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \ patch.object(update_worker, "DeploymentTransaction", RecoveryDeployment): self.assertEqual(update_worker.main(["--recover-blocked"]), 0) self.assertEqual(calls[0][0], "deployment") self.assertTrue(calls[0][3]) self.assertEqual(calls[1:4], ["lock", "begin", "quiesce"]) self.assertEqual(calls[4][:3], ( "checkpoint", "job-1234567890123456", "schema_migrating")) self.assertEqual(calls[5], "cloudInit") self.assertEqual(calls[6][:3], ( "checkpoint", "job-1234567890123456", "schema_ready")) self.assertEqual(calls[7:], [ "services-healthy", ("complete", "job-1234567890123456")]) def test_update_worker_is_a_separate_hardened_oneshot(self): deploy = Path(__file__).resolve().parents[2] / "deploy" engine_unit = (deploy / "virt-cloud-engine.service").read_text(encoding="utf-8") update_unit = (deploy / "virt-cloud-update.service").read_text(encoding="utf-8") self.assertIn("virsh_panel.internal:create_engine", engine_unit) engine_exec = next( line for line in engine_unit.splitlines() if line.startswith("ExecStart=") ) self.assertIn(" --workers 1 ", engine_exec) self.assertEqual(engine_exec.split().count("--workers"), 1) self.assertNotIn("update_worker", engine_unit) for value in ( "Type=oneshot", "User=root", "WorkingDirectory=/home/virt-cloud/engine", "CLOUD_ENGINE_CONFIG=/home/virt-cloud/runtime/engine.json", "ExecStart=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker", "ExecStopPost=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker --finalize-incomplete", "TimeoutStartSec=45min", "UMask=0077", "NoNewPrivileges=true", "PrivateTmp=true", "ProtectSystem=strict", "ProtectHome=false", "ReadWritePaths=/home/virt-cloud", "LimitCORE=0", ): self.assertIn(value, update_unit) self.assertNotIn("[Install]", update_unit) self.assertNotIn("/home/virsh-panel", update_unit) self.assertNotIn("virsh-panel.service", update_unit) self.assertNotIn("/etc/systemd/system", update_unit) self.assertNotIn("ExecStopPost=-/usr/bin/systemctl start", update_unit) def test_online_worker_is_code_only_and_uses_configured_fpm(self): source = Path(update_worker.__file__).read_text(encoding="utf-8") for forbidden in ( "daemon-reload", "/etc/systemd/system", '"pip", "install"', '_executable("composer")', '[composer, "install"', "update-staging", "web/vendor", "tempfile.", "def _git_file(", "prepare=", ): self.assertNotIn(forbidden, source) self.assertIn("mark_schema_ready()", source) units = update_worker._service_units({"update_php_fpm_unit": "php85-php-fpm.service"}) self.assertIn("php85-php-fpm.service", units) self.assertNotIn("php-fpm.service", units) with self.assertRaisesRegex(Exception, "服务名"): update_worker._service_units({"update_php_fpm_unit": "bad;service"}) def test_normal_update_preserves_running_installed_units_and_skips_missing_optional(self): states = { "virt-cloud-engine.service": ("loaded", "active"), "virt-cloud-worker.service": ("loaded", "active"), "virt-cloud-lifecycle.service": ("not-found", "inactive"), "virt-cloud-policy-worker@1.service": ("not-found", "inactive"), "virt-cloud-policy-worker@2.service": ("not-found", "inactive"), "php-fpm.service": ("loaded", "active"), } def systemctl(argv, **_kwargs): load, active = states[argv[-1]] return SimpleNamespace( returncode=0, stdout=f"LoadState={load}\nActiveState={active}\n", ) with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \ patch.object(update_worker.subprocess, "run", side_effect=systemctl): units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=False) self.assertEqual(units, ( "virt-cloud-engine.service", "virt-cloud-worker.service", "php-fpm.service", )) def test_recovery_starts_installed_units_even_when_they_are_stopped(self): states = { unit: ("loaded", "inactive") for unit in update_worker._service_units({}) } def systemctl(argv, **_kwargs): load, active = states[argv[-1]] return SimpleNamespace( returncode=0, stdout=f"LoadState={load}\nActiveState={active}\n", ) with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \ patch.object(update_worker.subprocess, "run", side_effect=systemctl): units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=True) self.assertEqual(units, update_worker._service_units({})) def test_update_refuses_before_stopping_when_required_service_is_missing(self): def systemctl(argv, **_kwargs): unit = argv[-1] load = "not-found" if unit == "php-fpm.service" else "loaded" return SimpleNamespace( returncode=0, stdout=f"LoadState={load}\nActiveState=active\n", ) with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \ patch.object(update_worker.subprocess, "run", side_effect=systemctl): with self.assertRaisesRegex(Exception, "php-fpm.service 未安装"): update_worker._managed_service_units(Path("/tmp"), {}, recovery=False) if __name__ == "__main__": unittest.main()