"""Privileged one-shot worker for a persisted Virt Cloud update job. The private engine may only enqueue a job and start the fixed systemd unit. This module is the only process allowed to quiesce the application, run the idempotent cloud schema initializer after the updater fast-forwards the source, and restart the configured service set. Dependencies and deployment-specific systemd units are deliberately never changed online. """ from __future__ import annotations import argparse import json import logging import os import re import shutil import subprocess import time from pathlib import Path from urllib.error import URLError from urllib.request import ProxyHandler, Request, build_opener from .database import DatabaseError from .engine_context import MainStore from .engine_security import decode_secret, signed_headers from .internal import NoRedirects, master_key, private_config from .updater import UpdateError, Updater LOG = logging.getLogger(__name__) _CONFIG_ENV = "CLOUD_ENGINE_CONFIG" _DEFAULT_CONFIG = Path(__file__).resolve().parents[2] / "runtime" / "engine.json" _ENGINE_HEALTH_URL = "http://127.0.0.1:8000/internal/v1/health" _ENGINE_SERVICE_UNIT = "virt-cloud-engine.service" _CORE_SERVICE_UNITS = ( _ENGINE_SERVICE_UNIT, "virt-cloud-worker.service", "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", "virt-cloud-policy-worker@2.service", ) _EXECUTABLE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" _COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z") _UNIT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.@:-]{0,126}\.service\Z") _RUNNING_UNIT_STATES = {"active", "activating", "reloading"} def _run(argv: list[str], *, cwd: Path, timeout: int, message: str, environment: dict[str, str] | None = None) -> None: """Run one fixed command without exposing its environment or output.""" try: result = subprocess.run( argv, cwd=cwd, env=environment, capture_output=True, text=True, timeout=timeout, check=False, ) except (OSError, subprocess.TimeoutExpired) as exc: raise UpdateError(message, 500) from exc if result.returncode != 0: raise UpdateError(message, 500) def _executable(name: str) -> str: path = shutil.which(name, path=_EXECUTABLE_PATH) if not path: raise UpdateError(f"服务器缺少在线更新所需的 {name} 命令", 409) return path def _service_units(config: dict) -> tuple[str, ...]: php_fpm = str(config.get("update_php_fpm_unit", "php-fpm.service")) if (not _UNIT.fullmatch(php_fpm) or php_fpm == "virt-cloud-update.service" or php_fpm in _CORE_SERVICE_UNITS): raise UpdateError("在线更新的 PHP-FPM 服务名配置无效", 409) return tuple(dict.fromkeys((*_CORE_SERVICE_UNITS, php_fpm))) def _managed_service_units(repo_dir: Path, config: dict, *, recovery: bool) -> tuple[str, ...]: """Resolve services before stopping anything. Deployments created before lifecycle/policy workers existed may not have those optional unit files installed. Passing a missing unit together with PHP-FPM to one ``systemctl stop`` command can stop the website and then return an error, leaving it offline. Normal updates preserve the exact running service set; crash recovery starts every installed panel unit so a previously stopped core service can be brought back. """ systemctl = _executable("systemctl") candidates = _service_units(config) php_fpm = str(config.get("update_php_fpm_unit", "php-fpm.service")) required = {"virt-cloud-engine.service", php_fpm} installed: list[str] = [] running: list[str] = [] for unit in candidates: try: result = subprocess.run( [systemctl, "show", "--property=LoadState", "--property=ActiveState", unit], cwd=repo_dir, env={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"}, capture_output=True, text=True, timeout=5, check=False, ) except (OSError, subprocess.TimeoutExpired) as exc: raise UpdateError("无法在停服前检查面板服务状态;在线更新已取消", 500) from exc properties = {} for line in result.stdout.splitlines(): name, separator, value = line.partition("=") if separator: properties[name] = value if result.returncode != 0 or properties.get("LoadState") != "loaded": if unit in required: raise UpdateError(f"在线更新所需服务 {unit} 未安装;尚未停止任何服务", 409) continue installed.append(unit) if properties.get("ActiveState") in _RUNNING_UNIT_STATES: running.append(unit) if not recovery: inactive_required = sorted(required.difference(running)) if inactive_required: raise UpdateError( f"在线更新所需服务 {inactive_required[0]} 未运行;尚未停止任何服务", 409 ) return tuple(running) return tuple(installed) def _wait_units(repo_dir: Path, units: tuple[str, ...]) -> None: systemctl = _executable("systemctl") deadline = time.monotonic() + 60 while time.monotonic() < deadline: all_active = True for unit in units: try: result = subprocess.run( [systemctl, "is-active", "--quiet", unit], cwd=repo_dir, env={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"}, capture_output=True, text=True, timeout=5, check=False, ) except (OSError, subprocess.TimeoutExpired): all_active = False break if result.returncode != 0: all_active = False break if all_active: return time.sleep(1) raise UpdateError("更新后有服务未进入运行状态,请检查 systemd 日志", 500) def _wait_engine_health(config: dict) -> None: target = "/internal/v1/health" actor = { "user_id": 1, "is_admin": True, "is_super_admin": True, "can_manage": True, "session_id": "", } secret = decode_secret(config["service_secret"]) deadline = time.monotonic() + 45 opener = build_opener(ProxyHandler({}), NoRedirects()) while time.monotonic() < deadline: headers = signed_headers(secret, "GET", target, b"", actor) try: with opener.open(Request(_ENGINE_HEALTH_URL, headers=headers, method="GET"), timeout=2) as response: raw = response.read(8193) payload = json.loads(raw) if (response.status == 200 and len(raw) <= 8192 and isinstance(payload, dict) and payload.get("status") == "ok"): return except (OSError, URLError, ValueError): pass time.sleep(1) raise UpdateError("新版本私有引擎未通过本机健康检查,请检查引擎日志", 500) class DeploymentTransaction: """Quiesce, migrate and restart around the updater's source switch.""" def __init__(self, repo_dir: Path, config: dict, *, recovery: bool = False) -> None: self.repo_dir = repo_dir self.config = config self.units = _managed_service_units(repo_dir, config, recovery=recovery) def quiesce(self) -> None: _run( [_executable("systemctl"), "stop", *self.units], cwd=self.repo_dir, timeout=180, message="无法暂停面板服务;代码尚未切换", environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"}, ) def activate(self, mark_schema_ready) -> None: web_dir = self.repo_dir / "web" if not web_dir.is_dir() or (web_dir / "xcat").is_symlink() or not (web_dir / "xcat").is_file(): raise UpdateError("更新后的 PHP 项目目录不完整", 500) _run( [_executable("php"), "xcat", "cloudInit"], cwd=web_dir, timeout=300, message="数据库升级或引擎配置导出失败;请查看更新服务日志并人工核查", environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"}, ) # This durable updater checkpoint is the sole proof that it is safe # for this process or ExecStopPost to start services on the new code. mark_schema_ready() self.start_and_check() def _start_services(self, units: tuple[str, ...], *, message: str) -> None: if not units: return _run( [_executable("systemctl"), "start", *units], cwd=self.repo_dir, timeout=180, message=message, environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"}, ) def start_and_check(self) -> None: engine_units = tuple(unit for unit in self.units if unit == _ENGINE_SERVICE_UNIT) if engine_units != (_ENGINE_SERVICE_UNIT,): raise UpdateError("更新服务集合缺少私有引擎,已拒绝启动其他服务", 500) php_fpm = str(self.config.get("update_php_fpm_unit", "php-fpm.service")) web_units = tuple(unit for unit in self.units if unit == php_fpm) if web_units != (php_fpm,): raise UpdateError("更新服务集合缺少 PHP-FPM,已拒绝启动其他服务", 500) consumer_units = tuple( unit for unit in self.units if unit not in {_ENGINE_SERVICE_UNIT, php_fpm} ) # ``systemctl is-active`` for a Type=exec unit only proves that uvicorn # was executed; it does not prove that application startup completed # or that the signed internal API is accepting requests. Starting the # lifecycle worker in the same systemctl command used to let its first # readiness check race that startup window and mark hosts unavailable. # Keep every consumer stopped until the engine's signed # health endpoint confirms actual readiness. self._start_services( web_units, message="新版本 PHP-FPM 启动失败,请检查 systemd 日志", ) _wait_units(self.repo_dir, web_units) self._start_services( engine_units, message="新版本私有引擎启动失败,请检查 systemd 日志", ) _wait_units(self.repo_dir, engine_units) try: _wait_engine_health(self.config) except Exception: # The website must remain available to report and retry a failed # update. Restore the consumers too: the target checkout's PHP # daemons have their own bounded engine-readiness gate, so they do # not query a half-started engine or claim VM work. Re-raise # the original health failure after systemd has a recoverable # service topology instead of leaving the panel SSH-only. self._start_services( consumer_units, message="引擎未就绪且面板后台服务恢复失败,请检查 systemd 日志", ) _wait_units(self.repo_dir, consumer_units) raise self._start_services( consumer_units, message="新版本面板后台服务启动失败,请检查 systemd 日志", ) _wait_units(self.repo_dir, consumer_units) def _clean_recorded_checkout(updater: Updater, job: dict, field: str) -> bool: commit = job.get(field) if not isinstance(commit, str) or not _COMMIT.fullmatch(commit): return False try: return updater._checkout_head(require_clean=True) == commit except (OSError, UpdateError): return False def _safe_to_start_services(updater: Updater, job: dict) -> bool: """Require durable proof before starting a possibly migrated checkout.""" state = job.get("deployment_state") if state in {"schema_ready", "services_ready"}: return _clean_recorded_checkout(updater, job, "new_commit") if state in {"quiescing", "services_stopped", "code_switched"}: return _clean_recorded_checkout(updater, job, "old_commit") # In particular, schema_migrating means MariaDB may have committed only a # prefix of the migration. A source rollback alone cannot prove safety. # Unknown states also fail closed. return False def _load_updater() -> tuple[Updater, MainStore, dict]: config_path = Path(os.environ.get(_CONFIG_ENV, str(_DEFAULT_CONFIG))) config = private_config(config_path) key = master_key(Path(config["master_key_file"])) store = MainStore(config["main_database"], key) source_root = Path(__file__).resolve().parents[2] configured_root = Path(config.get("project_dir", source_root)) if not configured_root.is_absolute() or configured_root.resolve() != source_root.resolve(): raise UpdateError("引擎配置中的项目目录与当前程序目录不一致", 409) return Updater(source_root, store), store, config def finalize_incomplete_job(updater: Updater, store: MainStore) -> bool: """Mark a queued/running job failed after an interrupted one-shot.""" job_id = store.get_value("update.active_job") if not job_id: return False job = updater.get_job(job_id) if job is None or job.get("status") not in {"queued", "running"}: return False updater.fail_job( job_id, "独立更新服务异常退出或超时;请检查更新服务日志和当前 Git 版本,勿直接重复提交", ) return True def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Run or finalize the Virt Cloud update job") mode = parser.add_mutually_exclusive_group() mode.add_argument("--finalize-incomplete", action="store_true") mode.add_argument("--recover-blocked", action="store_true") options = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") try: updater, store, config = _load_updater() if options.recover_blocked: deployment = DeploymentTransaction(updater.repo_dir, config, recovery=True) # This is an explicit root/SSH recovery path, never invoked by the # web service or ExecStopPost. Holding the Git lock makes a killed # recovery safely resumable without overlapping another process. with updater._apply_lock(): job = updater.begin_blocked_recovery() job_id = job["job_id"] try: deployment.quiesce() updater.recovery_checkpoint( job_id, "schema_migrating", "SSH 人工恢复正在执行数据库初始化", ) deployment.activate(lambda: updater.recovery_checkpoint( job_id, "schema_ready", "SSH 人工恢复数据库初始化完成,正在启动服务", )) updater.complete_blocked_recovery(job_id) except Exception: updater.fail_job( job_id, "SSH 人工恢复未完成;维护锁是否保留取决于最后一个持久安全检查点", ) raise LOG.info("blocked panel update recovered: job=%s", job_id) return 0 if options.finalize_incomplete: changed = finalize_incomplete_job(updater, store) if changed: LOG.warning("interrupted panel update marked failed") job_id = store.get_value("update.active_job") job = updater.get_job(job_id) if job_id else None if (isinstance(job, dict) and job.get("status") != "succeeded" and _safe_to_start_services(updater, job)): LOG.warning("durable update checkpoint permits service recovery") DeploymentTransaction(updater.repo_dir, config, recovery=True).start_and_check() elif isinstance(job, dict) and job.get("deployment_state") in { "quiescing", "services_stopped", "code_switched", "schema_migrating"}: LOG.error( "services remain stopped: update checkpoint is not safe for automatic recovery" ) return 0 deployment = DeploymentTransaction(updater.repo_dir, config) result = updater.run_pending_job( before_switch=deployment.quiesce, restart=deployment.activate, ) except (UpdateError, DatabaseError, OSError, ValueError) as exc: # Never log DB configuration, the Gitea PAT, subprocess output or the # environment. Every public UpdateError is deliberately credential-free. LOG.error("panel update failed: %s", exc) return 1 LOG.info( "panel update finished: job=%s old=%s new=%s updated=%s", result["job_id"], result.get("old_commit"), result.get("new_commit"), result.get("updated"), ) return 0 if __name__ == "__main__": raise SystemExit(main())