You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/virsh_panel/update_worker.py

408 lines
17 KiB
Python

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Privileged one-shot worker for a persisted Virt Cloud update job.
The private engine may only enqueue a job and start the fixed systemd unit.
This module is the only process allowed to quiesce the application, run the
idempotent cloud schema initializer after the updater fast-forwards the source,
and restart the configured service set. Dependencies and deployment-specific
systemd units are deliberately never changed online.
"""
from __future__ import annotations
import argparse
import json
import logging
import os
import re
import shutil
import subprocess
import time
from pathlib import Path
from urllib.error import URLError
from urllib.request import ProxyHandler, Request, build_opener
from .database import DatabaseError
from .engine_context import MainStore
from .engine_security import decode_secret, signed_headers
from .internal import NoRedirects, master_key, private_config
from .updater import UpdateError, Updater
LOG = logging.getLogger(__name__)
_CONFIG_ENV = "CLOUD_ENGINE_CONFIG"
_DEFAULT_CONFIG = Path(__file__).resolve().parents[2] / "runtime" / "engine.json"
_ENGINE_HEALTH_URL = "http://127.0.0.1:8000/internal/v1/health"
_ENGINE_SERVICE_UNIT = "virt-cloud-engine.service"
_CORE_SERVICE_UNITS = (
_ENGINE_SERVICE_UNIT,
"virt-cloud-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
)
_EXECUTABLE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
_COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z")
_UNIT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.@:-]{0,126}\.service\Z")
_RUNNING_UNIT_STATES = {"active", "activating", "reloading"}
def _run(argv: list[str], *, cwd: Path, timeout: int, message: str,
environment: dict[str, str] | None = None) -> None:
"""Run one fixed command without exposing its environment or output."""
try:
result = subprocess.run(
argv,
cwd=cwd,
env=environment,
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise UpdateError(message, 500) from exc
if result.returncode != 0:
raise UpdateError(message, 500)
def _executable(name: str) -> str:
path = shutil.which(name, path=_EXECUTABLE_PATH)
if not path:
raise UpdateError(f"服务器缺少在线更新所需的 {name} 命令", 409)
return path
def _service_units(config: dict) -> tuple[str, ...]:
php_fpm = str(config.get("update_php_fpm_unit", "php-fpm.service"))
if (not _UNIT.fullmatch(php_fpm) or php_fpm == "virt-cloud-update.service"
or php_fpm in _CORE_SERVICE_UNITS):
raise UpdateError("在线更新的 PHP-FPM 服务名配置无效", 409)
return tuple(dict.fromkeys((*_CORE_SERVICE_UNITS, php_fpm)))
def _managed_service_units(repo_dir: Path, config: dict, *, recovery: bool) -> tuple[str, ...]:
"""Resolve services before stopping anything.
Deployments created before lifecycle/policy workers existed may not have
those optional unit files installed. Passing a missing unit together with
PHP-FPM to one ``systemctl stop`` command can stop the website and then
return an error, leaving it offline. Normal updates preserve the exact
running service set; crash recovery starts every installed panel unit so a
previously stopped core service can be brought back.
"""
systemctl = _executable("systemctl")
candidates = _service_units(config)
php_fpm = str(config.get("update_php_fpm_unit", "php-fpm.service"))
required = {"virt-cloud-engine.service", php_fpm}
installed: list[str] = []
running: list[str] = []
for unit in candidates:
try:
result = subprocess.run(
[systemctl, "show", "--property=LoadState", "--property=ActiveState", unit],
cwd=repo_dir,
env={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
capture_output=True,
text=True,
timeout=5,
check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise UpdateError("无法在停服前检查面板服务状态;在线更新已取消", 500) from exc
properties = {}
for line in result.stdout.splitlines():
name, separator, value = line.partition("=")
if separator:
properties[name] = value
if result.returncode != 0 or properties.get("LoadState") != "loaded":
if unit in required:
raise UpdateError(f"在线更新所需服务 {unit} 未安装;尚未停止任何服务", 409)
continue
installed.append(unit)
if properties.get("ActiveState") in _RUNNING_UNIT_STATES:
running.append(unit)
if not recovery:
inactive_required = sorted(required.difference(running))
if inactive_required:
raise UpdateError(
f"在线更新所需服务 {inactive_required[0]} 未运行;尚未停止任何服务", 409
)
return tuple(running)
return tuple(installed)
def _wait_units(repo_dir: Path, units: tuple[str, ...]) -> None:
systemctl = _executable("systemctl")
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
all_active = True
for unit in units:
try:
result = subprocess.run(
[systemctl, "is-active", "--quiet", unit],
cwd=repo_dir,
env={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
capture_output=True,
text=True,
timeout=5,
check=False,
)
except (OSError, subprocess.TimeoutExpired):
all_active = False
break
if result.returncode != 0:
all_active = False
break
if all_active:
return
time.sleep(1)
raise UpdateError("更新后有服务未进入运行状态,请检查 systemd 日志", 500)
def _wait_engine_health(config: dict) -> None:
target = "/internal/v1/health"
actor = {
"user_id": 1,
"is_admin": True,
"is_super_admin": True,
"can_manage": True,
"session_id": "",
}
secret = decode_secret(config["service_secret"])
deadline = time.monotonic() + 45
opener = build_opener(ProxyHandler({}), NoRedirects())
while time.monotonic() < deadline:
headers = signed_headers(secret, "GET", target, b"", actor)
try:
with opener.open(Request(_ENGINE_HEALTH_URL, headers=headers, method="GET"), timeout=2) as response:
raw = response.read(8193)
payload = json.loads(raw)
if (response.status == 200 and len(raw) <= 8192
and isinstance(payload, dict) and payload.get("status") == "ok"):
return
except (OSError, URLError, ValueError):
pass
time.sleep(1)
raise UpdateError("新版本私有引擎未通过本机健康检查,请检查引擎日志", 500)
class DeploymentTransaction:
"""Quiesce, migrate and restart around the updater's source switch."""
def __init__(self, repo_dir: Path, config: dict, *, recovery: bool = False) -> None:
self.repo_dir = repo_dir
self.config = config
self.units = _managed_service_units(repo_dir, config, recovery=recovery)
def quiesce(self) -> None:
_run(
[_executable("systemctl"), "stop", *self.units],
cwd=self.repo_dir,
timeout=180,
message="无法暂停面板服务;代码尚未切换",
environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
)
def activate(self, mark_schema_ready) -> None:
web_dir = self.repo_dir / "web"
if not web_dir.is_dir() or (web_dir / "xcat").is_symlink() or not (web_dir / "xcat").is_file():
raise UpdateError("更新后的 PHP 项目目录不完整", 500)
_run(
[_executable("php"), "xcat", "cloudInit"],
cwd=web_dir,
timeout=300,
message="数据库升级或引擎配置导出失败;请查看更新服务日志并人工核查",
environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
)
# This durable updater checkpoint is the sole proof that it is safe
# for this process or ExecStopPost to start services on the new code.
mark_schema_ready()
self.start_and_check()
def _start_services(self, units: tuple[str, ...], *, message: str) -> None:
if not units:
return
_run(
[_executable("systemctl"), "start", *units],
cwd=self.repo_dir,
timeout=180,
message=message,
environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
)
def start_and_check(self) -> None:
engine_units = tuple(unit for unit in self.units if unit == _ENGINE_SERVICE_UNIT)
if engine_units != (_ENGINE_SERVICE_UNIT,):
raise UpdateError("更新服务集合缺少私有引擎,已拒绝启动其他服务", 500)
php_fpm = str(self.config.get("update_php_fpm_unit", "php-fpm.service"))
web_units = tuple(unit for unit in self.units if unit == php_fpm)
if web_units != (php_fpm,):
raise UpdateError("更新服务集合缺少 PHP-FPM,已拒绝启动其他服务", 500)
consumer_units = tuple(
unit for unit in self.units if unit not in {_ENGINE_SERVICE_UNIT, php_fpm}
)
# ``systemctl is-active`` for a Type=exec unit only proves that uvicorn
# was executed; it does not prove that application startup completed
# or that the signed internal API is accepting requests. Starting the
# lifecycle worker in the same systemctl command used to let its first
# readiness check race that startup window and mark hosts unavailable.
# Keep every consumer stopped until the engine's signed
# health endpoint confirms actual readiness.
self._start_services(
web_units,
message="新版本 PHP-FPM 启动失败,请检查 systemd 日志",
)
_wait_units(self.repo_dir, web_units)
self._start_services(
engine_units,
message="新版本私有引擎启动失败,请检查 systemd 日志",
)
_wait_units(self.repo_dir, engine_units)
try:
_wait_engine_health(self.config)
except Exception:
# The website must remain available to report and retry a failed
# update. Restore the consumers too: the target checkout's PHP
# daemons have their own bounded engine-readiness gate, so they do
# not query a half-started engine or claim VM work. Re-raise
# the original health failure after systemd has a recoverable
# service topology instead of leaving the panel SSH-only.
self._start_services(
consumer_units,
message="引擎未就绪且面板后台服务恢复失败,请检查 systemd 日志",
)
_wait_units(self.repo_dir, consumer_units)
raise
self._start_services(
consumer_units,
message="新版本面板后台服务启动失败,请检查 systemd 日志",
)
_wait_units(self.repo_dir, consumer_units)
def _clean_recorded_checkout(updater: Updater, job: dict, field: str) -> bool:
commit = job.get(field)
if not isinstance(commit, str) or not _COMMIT.fullmatch(commit):
return False
try:
return updater._checkout_head(require_clean=True) == commit
except (OSError, UpdateError):
return False
def _safe_to_start_services(updater: Updater, job: dict) -> bool:
"""Require durable proof before starting a possibly migrated checkout."""
state = job.get("deployment_state")
if state in {"schema_ready", "services_ready"}:
return _clean_recorded_checkout(updater, job, "new_commit")
if state in {"quiescing", "services_stopped", "code_switched"}:
return _clean_recorded_checkout(updater, job, "old_commit")
# In particular, schema_migrating means MariaDB may have committed only a
# prefix of the migration. A source rollback alone cannot prove safety.
# Unknown states also fail closed.
return False
def _load_updater() -> tuple[Updater, MainStore, dict]:
config_path = Path(os.environ.get(_CONFIG_ENV, str(_DEFAULT_CONFIG)))
config = private_config(config_path)
key = master_key(Path(config["master_key_file"]))
store = MainStore(config["main_database"], key)
source_root = Path(__file__).resolve().parents[2]
configured_root = Path(config.get("project_dir", source_root))
if not configured_root.is_absolute() or configured_root.resolve() != source_root.resolve():
raise UpdateError("引擎配置中的项目目录与当前程序目录不一致", 409)
return Updater(source_root, store), store, config
def finalize_incomplete_job(updater: Updater, store: MainStore) -> bool:
"""Mark a queued/running job failed after an interrupted one-shot."""
job_id = store.get_value("update.active_job")
if not job_id:
return False
job = updater.get_job(job_id)
if job is None or job.get("status") not in {"queued", "running"}:
return False
updater.fail_job(
job_id,
"独立更新服务异常退出或超时;请检查更新服务日志和当前 Git 版本,勿直接重复提交",
)
return True
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Run or finalize the Virt Cloud update job")
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--finalize-incomplete", action="store_true")
mode.add_argument("--recover-blocked", action="store_true")
options = parser.parse_args(argv)
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
try:
updater, store, config = _load_updater()
if options.recover_blocked:
deployment = DeploymentTransaction(updater.repo_dir, config, recovery=True)
# This is an explicit root/SSH recovery path, never invoked by the
# web service or ExecStopPost. Holding the Git lock makes a killed
# recovery safely resumable without overlapping another process.
with updater._apply_lock():
job = updater.begin_blocked_recovery()
job_id = job["job_id"]
try:
deployment.quiesce()
updater.recovery_checkpoint(
job_id,
"schema_migrating",
"SSH 人工恢复正在执行数据库初始化",
)
deployment.activate(lambda: updater.recovery_checkpoint(
job_id,
"schema_ready",
"SSH 人工恢复数据库初始化完成,正在启动服务",
))
updater.complete_blocked_recovery(job_id)
except Exception:
updater.fail_job(
job_id,
"SSH 人工恢复未完成;维护锁是否保留取决于最后一个持久安全检查点",
)
raise
LOG.info("blocked panel update recovered: job=%s", job_id)
return 0
if options.finalize_incomplete:
changed = finalize_incomplete_job(updater, store)
if changed:
LOG.warning("interrupted panel update marked failed")
job_id = store.get_value("update.active_job")
job = updater.get_job(job_id) if job_id else None
if (isinstance(job, dict) and job.get("status") != "succeeded"
and _safe_to_start_services(updater, job)):
LOG.warning("durable update checkpoint permits service recovery")
DeploymentTransaction(updater.repo_dir, config, recovery=True).start_and_check()
elif isinstance(job, dict) and job.get("deployment_state") in {
"quiescing", "services_stopped", "code_switched", "schema_migrating"}:
LOG.error(
"services remain stopped: update checkpoint is not safe for automatic recovery"
)
return 0
deployment = DeploymentTransaction(updater.repo_dir, config)
result = updater.run_pending_job(
before_switch=deployment.quiesce,
restart=deployment.activate,
)
except (UpdateError, DatabaseError, OSError, ValueError) as exc:
# Never log DB configuration, the Gitea PAT, subprocess output or the
# environment. Every public UpdateError is deliberately credential-free.
LOG.error("panel update failed: %s", exc)
return 1
LOG.info(
"panel update finished: job=%s old=%s new=%s updated=%s",
result["job_id"], result.get("old_commit"), result.get("new_commit"), result.get("updated"),
)
return 0
if __name__ == "__main__":
raise SystemExit(main())