diff --git a/engine/tests/test_update_worker.py b/engine/tests/test_update_worker.py index f2679d1f..ca6e5da8 100644 --- a/engine/tests/test_update_worker.py +++ b/engine/tests/test_update_worker.py @@ -5,6 +5,7 @@ from __future__ import annotations import unittest from contextlib import nullcontext from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch from virsh_panel import update_worker @@ -77,6 +78,8 @@ class FinalizeIncompleteTests(unittest.TestCase): "new_commit": "a" * 40, }) with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \ + patch.object(update_worker, "_managed_service_units", return_value=( + "virt-cloud-engine.service", "php-fpm.service")), \ patch.object(update_worker.DeploymentTransaction, "start_and_check") as start: self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0) self.assertEqual(updater.job["status"], "failed") @@ -111,8 +114,8 @@ class FinalizeIncompleteTests(unittest.TestCase): calls.append(("complete", job_id)) class RecoveryDeployment: - def __init__(self, repo_dir, config): - calls.append(("deployment", repo_dir, config)) + def __init__(self, repo_dir, config, *, recovery=False): + calls.append(("deployment", repo_dir, config, recovery)) def quiesce(self): calls.append("quiesce") @@ -129,6 +132,7 @@ class FinalizeIncompleteTests(unittest.TestCase): self.assertEqual(update_worker.main(["--recover-blocked"]), 0) self.assertEqual(calls[0][0], "deployment") + self.assertTrue(calls[0][3]) self.assertEqual(calls[1:4], ["lock", "begin", "quiesce"]) self.assertEqual(calls[4][:3], ( "checkpoint", "job-1234567890123456", "schema_migrating")) @@ -189,6 +193,63 @@ class FinalizeIncompleteTests(unittest.TestCase): with self.assertRaisesRegex(Exception, "服务名"): update_worker._service_units({"update_php_fpm_unit": "bad;service"}) + def test_normal_update_preserves_running_installed_units_and_skips_missing_optional(self): + states = { + "virt-cloud-engine.service": ("loaded", "active"), + "virt-cloud-worker.service": ("loaded", "active"), + "virt-cloud-lifecycle.service": ("not-found", "inactive"), + "virt-cloud-policy-worker@1.service": ("not-found", "inactive"), + "virt-cloud-policy-worker@2.service": ("not-found", "inactive"), + "php-fpm.service": ("loaded", "active"), + } + + def systemctl(argv, **_kwargs): + load, active = states[argv[-1]] + return SimpleNamespace( + returncode=0, + stdout=f"LoadState={load}\nActiveState={active}\n", + ) + + with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \ + patch.object(update_worker.subprocess, "run", side_effect=systemctl): + units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=False) + self.assertEqual(units, ( + "virt-cloud-engine.service", + "virt-cloud-worker.service", + "php-fpm.service", + )) + + def test_recovery_starts_installed_units_even_when_they_are_stopped(self): + states = { + unit: ("loaded", "inactive") for unit in update_worker._service_units({}) + } + + def systemctl(argv, **_kwargs): + load, active = states[argv[-1]] + return SimpleNamespace( + returncode=0, + stdout=f"LoadState={load}\nActiveState={active}\n", + ) + + with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \ + patch.object(update_worker.subprocess, "run", side_effect=systemctl): + units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=True) + self.assertEqual(units, update_worker._service_units({})) + + def test_update_refuses_before_stopping_when_required_service_is_missing(self): + def systemctl(argv, **_kwargs): + unit = argv[-1] + load = "not-found" if unit == "php-fpm.service" else "loaded" + return SimpleNamespace( + returncode=0, + stdout=f"LoadState={load}\nActiveState=active\n", + ) + + with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \ + patch.object(update_worker.subprocess, "run", side_effect=systemctl): + with self.assertRaisesRegex(Exception, "php-fpm.service 未安装"): + update_worker._managed_service_units(Path("/tmp"), {}, recovery=False) + if __name__ == "__main__": unittest.main() diff --git a/engine/virsh_panel/update_worker.py b/engine/virsh_panel/update_worker.py index 97eae88e..c0991030 100644 --- a/engine/virsh_panel/update_worker.py +++ b/engine/virsh_panel/update_worker.py @@ -42,6 +42,7 @@ _CORE_SERVICE_UNITS = ( _EXECUTABLE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" _COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z") _UNIT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.@:-]{0,126}\.service\Z") +_RUNNING_UNIT_STATES = {"active", "activating", "reloading"} def _run(argv: list[str], *, cwd: Path, timeout: int, message: str, @@ -78,6 +79,58 @@ def _service_units(config: dict) -> tuple[str, ...]: return tuple(dict.fromkeys((*_CORE_SERVICE_UNITS, php_fpm))) +def _managed_service_units(repo_dir: Path, config: dict, *, recovery: bool) -> tuple[str, ...]: + """Resolve services before stopping anything. + + Deployments created before lifecycle/policy workers existed may not have + those optional unit files installed. Passing a missing unit together with + PHP-FPM to one ``systemctl stop`` command can stop the website and then + return an error, leaving it offline. Normal updates preserve the exact + running service set; crash recovery starts every installed panel unit so a + previously stopped core service can be brought back. + """ + systemctl = _executable("systemctl") + candidates = _service_units(config) + php_fpm = str(config.get("update_php_fpm_unit", "php-fpm.service")) + required = {"virt-cloud-engine.service", php_fpm} + installed: list[str] = [] + running: list[str] = [] + for unit in candidates: + try: + result = subprocess.run( + [systemctl, "show", "--property=LoadState", "--property=ActiveState", unit], + cwd=repo_dir, + env={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"}, + capture_output=True, + text=True, + timeout=5, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise UpdateError("无法在停服前检查面板服务状态;在线更新已取消", 500) from exc + properties = {} + for line in result.stdout.splitlines(): + name, separator, value = line.partition("=") + if separator: + properties[name] = value + if result.returncode != 0 or properties.get("LoadState") != "loaded": + if unit in required: + raise UpdateError(f"在线更新所需服务 {unit} 未安装;尚未停止任何服务", 409) + continue + installed.append(unit) + if properties.get("ActiveState") in _RUNNING_UNIT_STATES: + running.append(unit) + + if not recovery: + inactive_required = sorted(required.difference(running)) + if inactive_required: + raise UpdateError( + f"在线更新所需服务 {inactive_required[0]} 未运行;尚未停止任何服务", 409 + ) + return tuple(running) + return tuple(installed) + + def _wait_units(repo_dir: Path, units: tuple[str, ...]) -> None: systemctl = _executable("systemctl") deadline = time.monotonic() + 60 @@ -136,10 +189,10 @@ def _wait_engine_health(config: dict) -> None: class DeploymentTransaction: """Quiesce, migrate and restart around the updater's source switch.""" - def __init__(self, repo_dir: Path, config: dict) -> None: + def __init__(self, repo_dir: Path, config: dict, *, recovery: bool = False) -> None: self.repo_dir = repo_dir self.config = config - self.units = _service_units(config) + self.units = _managed_service_units(repo_dir, config, recovery=recovery) def quiesce(self) -> None: _run( @@ -241,7 +294,7 @@ def main(argv: list[str] | None = None) -> int: try: updater, store, config = _load_updater() if options.recover_blocked: - deployment = DeploymentTransaction(updater.repo_dir, config) + deployment = DeploymentTransaction(updater.repo_dir, config, recovery=True) # This is an explicit root/SSH recovery path, never invoked by the # web service or ExecStopPost. Holding the Git lock makes a killed # recovery safely resumable without overlapping another process. @@ -278,7 +331,7 @@ def main(argv: list[str] | None = None) -> int: if (isinstance(job, dict) and job.get("status") != "succeeded" and _safe_to_start_services(updater, job)): LOG.warning("durable update checkpoint permits service recovery") - DeploymentTransaction(updater.repo_dir, config).start_and_check() + DeploymentTransaction(updater.repo_dir, config, recovery=True).start_and_check() elif isinstance(job, dict) and job.get("deployment_state") in { "quiescing", "services_stopped", "code_switched", "schema_migrating"}: LOG.error(