Prevent online update service outage

dev
admin 2 months ago
parent 9689e386f0
commit 9809b2cee2

@ -5,6 +5,7 @@ from __future__ import annotations
import unittest
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from virsh_panel import update_worker
@ -77,6 +78,8 @@ class FinalizeIncompleteTests(unittest.TestCase):
"new_commit": "a" * 40,
})
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
patch.object(update_worker, "_managed_service_units", return_value=(
"virt-cloud-engine.service", "php-fpm.service")), \
patch.object(update_worker.DeploymentTransaction, "start_and_check") as start:
self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0)
self.assertEqual(updater.job["status"], "failed")
@ -111,8 +114,8 @@ class FinalizeIncompleteTests(unittest.TestCase):
calls.append(("complete", job_id))
class RecoveryDeployment:
def __init__(self, repo_dir, config):
calls.append(("deployment", repo_dir, config))
def __init__(self, repo_dir, config, *, recovery=False):
calls.append(("deployment", repo_dir, config, recovery))
def quiesce(self):
calls.append("quiesce")
@ -129,6 +132,7 @@ class FinalizeIncompleteTests(unittest.TestCase):
self.assertEqual(update_worker.main(["--recover-blocked"]), 0)
self.assertEqual(calls[0][0], "deployment")
self.assertTrue(calls[0][3])
self.assertEqual(calls[1:4], ["lock", "begin", "quiesce"])
self.assertEqual(calls[4][:3], (
"checkpoint", "job-1234567890123456", "schema_migrating"))
@ -189,6 +193,63 @@ class FinalizeIncompleteTests(unittest.TestCase):
with self.assertRaisesRegex(Exception, "服务名"):
update_worker._service_units({"update_php_fpm_unit": "bad;service"})
def test_normal_update_preserves_running_installed_units_and_skips_missing_optional(self):
states = {
"virt-cloud-engine.service": ("loaded", "active"),
"virt-cloud-worker.service": ("loaded", "active"),
"virt-cloud-lifecycle.service": ("not-found", "inactive"),
"virt-cloud-policy-worker@1.service": ("not-found", "inactive"),
"virt-cloud-policy-worker@2.service": ("not-found", "inactive"),
"php-fpm.service": ("loaded", "active"),
}
def systemctl(argv, **_kwargs):
load, active = states[argv[-1]]
return SimpleNamespace(
returncode=0,
stdout=f"LoadState={load}\nActiveState={active}\n",
)
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=False)
self.assertEqual(units, (
"virt-cloud-engine.service",
"virt-cloud-worker.service",
"php-fpm.service",
))
def test_recovery_starts_installed_units_even_when_they_are_stopped(self):
states = {
unit: ("loaded", "inactive") for unit in update_worker._service_units({})
}
def systemctl(argv, **_kwargs):
load, active = states[argv[-1]]
return SimpleNamespace(
returncode=0,
stdout=f"LoadState={load}\nActiveState={active}\n",
)
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=True)
self.assertEqual(units, update_worker._service_units({}))
def test_update_refuses_before_stopping_when_required_service_is_missing(self):
def systemctl(argv, **_kwargs):
unit = argv[-1]
load = "not-found" if unit == "php-fpm.service" else "loaded"
return SimpleNamespace(
returncode=0,
stdout=f"LoadState={load}\nActiveState=active\n",
)
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
with self.assertRaisesRegex(Exception, "php-fpm.service 未安装"):
update_worker._managed_service_units(Path("/tmp"), {}, recovery=False)
if __name__ == "__main__":
unittest.main()

@ -42,6 +42,7 @@ _CORE_SERVICE_UNITS = (
_EXECUTABLE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
_COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z")
_UNIT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.@:-]{0,126}\.service\Z")
_RUNNING_UNIT_STATES = {"active", "activating", "reloading"}
def _run(argv: list[str], *, cwd: Path, timeout: int, message: str,
@ -78,6 +79,58 @@ def _service_units(config: dict) -> tuple[str, ...]:
return tuple(dict.fromkeys((*_CORE_SERVICE_UNITS, php_fpm)))
def _managed_service_units(repo_dir: Path, config: dict, *, recovery: bool) -> tuple[str, ...]:
"""Resolve services before stopping anything.
Deployments created before lifecycle/policy workers existed may not have
those optional unit files installed. Passing a missing unit together with
PHP-FPM to one ``systemctl stop`` command can stop the website and then
return an error, leaving it offline. Normal updates preserve the exact
running service set; crash recovery starts every installed panel unit so a
previously stopped core service can be brought back.
"""
systemctl = _executable("systemctl")
candidates = _service_units(config)
php_fpm = str(config.get("update_php_fpm_unit", "php-fpm.service"))
required = {"virt-cloud-engine.service", php_fpm}
installed: list[str] = []
running: list[str] = []
for unit in candidates:
try:
result = subprocess.run(
[systemctl, "show", "--property=LoadState", "--property=ActiveState", unit],
cwd=repo_dir,
env={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
capture_output=True,
text=True,
timeout=5,
check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise UpdateError("无法在停服前检查面板服务状态;在线更新已取消", 500) from exc
properties = {}
for line in result.stdout.splitlines():
name, separator, value = line.partition("=")
if separator:
properties[name] = value
if result.returncode != 0 or properties.get("LoadState") != "loaded":
if unit in required:
raise UpdateError(f"在线更新所需服务 {unit} 未安装;尚未停止任何服务", 409)
continue
installed.append(unit)
if properties.get("ActiveState") in _RUNNING_UNIT_STATES:
running.append(unit)
if not recovery:
inactive_required = sorted(required.difference(running))
if inactive_required:
raise UpdateError(
f"在线更新所需服务 {inactive_required[0]} 未运行;尚未停止任何服务", 409
)
return tuple(running)
return tuple(installed)
def _wait_units(repo_dir: Path, units: tuple[str, ...]) -> None:
systemctl = _executable("systemctl")
deadline = time.monotonic() + 60
@ -136,10 +189,10 @@ def _wait_engine_health(config: dict) -> None:
class DeploymentTransaction:
"""Quiesce, migrate and restart around the updater's source switch."""
def __init__(self, repo_dir: Path, config: dict) -> None:
def __init__(self, repo_dir: Path, config: dict, *, recovery: bool = False) -> None:
self.repo_dir = repo_dir
self.config = config
self.units = _service_units(config)
self.units = _managed_service_units(repo_dir, config, recovery=recovery)
def quiesce(self) -> None:
_run(
@ -241,7 +294,7 @@ def main(argv: list[str] | None = None) -> int:
try:
updater, store, config = _load_updater()
if options.recover_blocked:
deployment = DeploymentTransaction(updater.repo_dir, config)
deployment = DeploymentTransaction(updater.repo_dir, config, recovery=True)
# This is an explicit root/SSH recovery path, never invoked by the
# web service or ExecStopPost. Holding the Git lock makes a killed
# recovery safely resumable without overlapping another process.
@ -278,7 +331,7 @@ def main(argv: list[str] | None = None) -> int:
if (isinstance(job, dict) and job.get("status") != "succeeded"
and _safe_to_start_services(updater, job)):
LOG.warning("durable update checkpoint permits service recovery")
DeploymentTransaction(updater.repo_dir, config).start_and_check()
DeploymentTransaction(updater.repo_dir, config, recovery=True).start_and_check()
elif isinstance(job, dict) and job.get("deployment_state") in {
"quiescing", "services_stopped", "code_switched", "schema_migrating"}:
LOG.error(

Loading…
Cancel
Save