|
|
"""Privileged one-shot worker for a persisted Virt Cloud update job.
|
|
|
|
|
|
The private engine may only enqueue a job and start the fixed systemd unit.
|
|
|
This module is the only process allowed to quiesce the application, run the
|
|
|
idempotent cloud schema initializer after the updater fast-forwards the source,
|
|
|
and restart the configured service set. Dependencies and deployment-specific
|
|
|
systemd units are deliberately never changed online.
|
|
|
"""
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
import argparse
|
|
|
import json
|
|
|
import logging
|
|
|
import os
|
|
|
import re
|
|
|
import shutil
|
|
|
import subprocess
|
|
|
import time
|
|
|
from pathlib import Path
|
|
|
from urllib.error import URLError
|
|
|
from urllib.request import ProxyHandler, Request, build_opener
|
|
|
|
|
|
from .database import DatabaseError
|
|
|
from .engine_context import MainStore
|
|
|
from .engine_security import decode_secret, signed_headers
|
|
|
from .internal import NoRedirects, master_key, private_config
|
|
|
from .updater import UpdateError, Updater
|
|
|
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
_CONFIG_ENV = "CLOUD_ENGINE_CONFIG"
|
|
|
_DEFAULT_CONFIG = Path(__file__).resolve().parents[2] / "runtime" / "engine.json"
|
|
|
_ENGINE_HEALTH_URL = "http://127.0.0.1:8000/internal/v1/health"
|
|
|
_ENGINE_SERVICE_UNIT = "virt-cloud-engine.service"
|
|
|
_CORE_SERVICE_UNITS = (
|
|
|
_ENGINE_SERVICE_UNIT,
|
|
|
"virt-cloud-worker.service",
|
|
|
"virt-cloud-lifecycle.service",
|
|
|
"virt-cloud-policy-worker@1.service",
|
|
|
"virt-cloud-policy-worker@2.service",
|
|
|
)
|
|
|
_EXECUTABLE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
|
_COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z")
|
|
|
_UNIT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.@:-]{0,126}\.service\Z")
|
|
|
_RUNNING_UNIT_STATES = {"active", "activating", "reloading"}
|
|
|
|
|
|
|
|
|
def _run(argv: list[str], *, cwd: Path, timeout: int, message: str,
|
|
|
environment: dict[str, str] | None = None) -> None:
|
|
|
"""Run one fixed command without exposing its environment or output."""
|
|
|
try:
|
|
|
result = subprocess.run(
|
|
|
argv,
|
|
|
cwd=cwd,
|
|
|
env=environment,
|
|
|
capture_output=True,
|
|
|
text=True,
|
|
|
timeout=timeout,
|
|
|
check=False,
|
|
|
)
|
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
|
raise UpdateError(message, 500) from exc
|
|
|
if result.returncode != 0:
|
|
|
raise UpdateError(message, 500)
|
|
|
|
|
|
|
|
|
def _executable(name: str) -> str:
|
|
|
path = shutil.which(name, path=_EXECUTABLE_PATH)
|
|
|
if not path:
|
|
|
raise UpdateError(f"服务器缺少在线更新所需的 {name} 命令", 409)
|
|
|
return path
|
|
|
|
|
|
|
|
|
def _service_units(config: dict) -> tuple[str, ...]:
|
|
|
php_fpm = str(config.get("update_php_fpm_unit", "php-fpm.service"))
|
|
|
if (not _UNIT.fullmatch(php_fpm) or php_fpm == "virt-cloud-update.service"
|
|
|
or php_fpm in _CORE_SERVICE_UNITS):
|
|
|
raise UpdateError("在线更新的 PHP-FPM 服务名配置无效", 409)
|
|
|
return tuple(dict.fromkeys((*_CORE_SERVICE_UNITS, php_fpm)))
|
|
|
|
|
|
|
|
|
def _managed_service_units(repo_dir: Path, config: dict, *, recovery: bool) -> tuple[str, ...]:
|
|
|
"""Resolve services before stopping anything.
|
|
|
|
|
|
Deployments created before lifecycle/policy workers existed may not have
|
|
|
those optional unit files installed. Passing a missing unit together with
|
|
|
PHP-FPM to one ``systemctl stop`` command can stop the website and then
|
|
|
return an error, leaving it offline. Normal updates preserve the exact
|
|
|
running service set; crash recovery starts every installed panel unit so a
|
|
|
previously stopped core service can be brought back.
|
|
|
"""
|
|
|
systemctl = _executable("systemctl")
|
|
|
candidates = _service_units(config)
|
|
|
php_fpm = str(config.get("update_php_fpm_unit", "php-fpm.service"))
|
|
|
required = {"virt-cloud-engine.service", php_fpm}
|
|
|
installed: list[str] = []
|
|
|
running: list[str] = []
|
|
|
for unit in candidates:
|
|
|
try:
|
|
|
result = subprocess.run(
|
|
|
[systemctl, "show", "--property=LoadState", "--property=ActiveState", unit],
|
|
|
cwd=repo_dir,
|
|
|
env={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
|
|
|
capture_output=True,
|
|
|
text=True,
|
|
|
timeout=5,
|
|
|
check=False,
|
|
|
)
|
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
|
raise UpdateError("无法在停服前检查面板服务状态;在线更新已取消", 500) from exc
|
|
|
properties = {}
|
|
|
for line in result.stdout.splitlines():
|
|
|
name, separator, value = line.partition("=")
|
|
|
if separator:
|
|
|
properties[name] = value
|
|
|
if result.returncode != 0 or properties.get("LoadState") != "loaded":
|
|
|
if unit in required:
|
|
|
raise UpdateError(f"在线更新所需服务 {unit} 未安装;尚未停止任何服务", 409)
|
|
|
continue
|
|
|
installed.append(unit)
|
|
|
if properties.get("ActiveState") in _RUNNING_UNIT_STATES:
|
|
|
running.append(unit)
|
|
|
|
|
|
if not recovery:
|
|
|
inactive_required = sorted(required.difference(running))
|
|
|
if inactive_required:
|
|
|
raise UpdateError(
|
|
|
f"在线更新所需服务 {inactive_required[0]} 未运行;尚未停止任何服务", 409
|
|
|
)
|
|
|
return tuple(running)
|
|
|
return tuple(installed)
|
|
|
|
|
|
|
|
|
def _wait_units(repo_dir: Path, units: tuple[str, ...]) -> None:
|
|
|
systemctl = _executable("systemctl")
|
|
|
deadline = time.monotonic() + 60
|
|
|
while time.monotonic() < deadline:
|
|
|
all_active = True
|
|
|
for unit in units:
|
|
|
try:
|
|
|
result = subprocess.run(
|
|
|
[systemctl, "is-active", "--quiet", unit],
|
|
|
cwd=repo_dir,
|
|
|
env={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
|
|
|
capture_output=True,
|
|
|
text=True,
|
|
|
timeout=5,
|
|
|
check=False,
|
|
|
)
|
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
|
all_active = False
|
|
|
break
|
|
|
if result.returncode != 0:
|
|
|
all_active = False
|
|
|
break
|
|
|
if all_active:
|
|
|
return
|
|
|
time.sleep(1)
|
|
|
raise UpdateError("更新后有服务未进入运行状态,请检查 systemd 日志", 500)
|
|
|
|
|
|
|
|
|
def _wait_engine_health(config: dict) -> None:
|
|
|
target = "/internal/v1/health"
|
|
|
actor = {
|
|
|
"user_id": 1,
|
|
|
"is_admin": True,
|
|
|
"is_super_admin": True,
|
|
|
"can_manage": True,
|
|
|
"session_id": "",
|
|
|
}
|
|
|
secret = decode_secret(config["service_secret"])
|
|
|
deadline = time.monotonic() + 45
|
|
|
opener = build_opener(ProxyHandler({}), NoRedirects())
|
|
|
while time.monotonic() < deadline:
|
|
|
headers = signed_headers(secret, "GET", target, b"", actor)
|
|
|
try:
|
|
|
with opener.open(Request(_ENGINE_HEALTH_URL, headers=headers, method="GET"), timeout=2) as response:
|
|
|
raw = response.read(8193)
|
|
|
payload = json.loads(raw)
|
|
|
if (response.status == 200 and len(raw) <= 8192
|
|
|
and isinstance(payload, dict) and payload.get("status") == "ok"):
|
|
|
return
|
|
|
except (OSError, URLError, ValueError):
|
|
|
pass
|
|
|
time.sleep(1)
|
|
|
raise UpdateError("新版本私有引擎未通过本机健康检查,请检查引擎日志", 500)
|
|
|
|
|
|
|
|
|
class DeploymentTransaction:
|
|
|
"""Quiesce, migrate and restart around the updater's source switch."""
|
|
|
|
|
|
def __init__(self, repo_dir: Path, config: dict, *, recovery: bool = False) -> None:
|
|
|
self.repo_dir = repo_dir
|
|
|
self.config = config
|
|
|
self.units = _managed_service_units(repo_dir, config, recovery=recovery)
|
|
|
|
|
|
def quiesce(self) -> None:
|
|
|
_run(
|
|
|
[_executable("systemctl"), "stop", *self.units],
|
|
|
cwd=self.repo_dir,
|
|
|
timeout=180,
|
|
|
message="无法暂停面板服务;代码尚未切换",
|
|
|
environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
|
|
|
)
|
|
|
|
|
|
def activate(self, mark_schema_ready) -> None:
|
|
|
web_dir = self.repo_dir / "web"
|
|
|
if not web_dir.is_dir() or (web_dir / "xcat").is_symlink() or not (web_dir / "xcat").is_file():
|
|
|
raise UpdateError("更新后的 PHP 项目目录不完整", 500)
|
|
|
_run(
|
|
|
[_executable("php"), "xcat", "cloudInit"],
|
|
|
cwd=web_dir,
|
|
|
timeout=300,
|
|
|
message="数据库升级或引擎配置导出失败;请查看更新服务日志并人工核查",
|
|
|
environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
|
|
|
)
|
|
|
# This durable updater checkpoint is the sole proof that it is safe
|
|
|
# for this process or ExecStopPost to start services on the new code.
|
|
|
mark_schema_ready()
|
|
|
self.start_and_check()
|
|
|
|
|
|
def _start_services(self, units: tuple[str, ...], *, message: str) -> None:
|
|
|
if not units:
|
|
|
return
|
|
|
_run(
|
|
|
[_executable("systemctl"), "start", *units],
|
|
|
cwd=self.repo_dir,
|
|
|
timeout=180,
|
|
|
message=message,
|
|
|
environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
|
|
|
)
|
|
|
|
|
|
def start_and_check(self) -> None:
|
|
|
engine_units = tuple(unit for unit in self.units if unit == _ENGINE_SERVICE_UNIT)
|
|
|
if engine_units != (_ENGINE_SERVICE_UNIT,):
|
|
|
raise UpdateError("更新服务集合缺少私有引擎,已拒绝启动其他服务", 500)
|
|
|
php_fpm = str(self.config.get("update_php_fpm_unit", "php-fpm.service"))
|
|
|
web_units = tuple(unit for unit in self.units if unit == php_fpm)
|
|
|
if web_units != (php_fpm,):
|
|
|
raise UpdateError("更新服务集合缺少 PHP-FPM,已拒绝启动其他服务", 500)
|
|
|
consumer_units = tuple(
|
|
|
unit for unit in self.units if unit not in {_ENGINE_SERVICE_UNIT, php_fpm}
|
|
|
)
|
|
|
|
|
|
# ``systemctl is-active`` for a Type=exec unit only proves that uvicorn
|
|
|
# was executed; it does not prove that application startup completed
|
|
|
# or that the signed internal API is accepting requests. Starting the
|
|
|
# lifecycle worker in the same systemctl command used to let its first
|
|
|
# readiness check race that startup window and mark hosts unavailable.
|
|
|
# Keep every consumer stopped until the engine's signed
|
|
|
# health endpoint confirms actual readiness.
|
|
|
self._start_services(
|
|
|
web_units,
|
|
|
message="新版本 PHP-FPM 启动失败,请检查 systemd 日志",
|
|
|
)
|
|
|
_wait_units(self.repo_dir, web_units)
|
|
|
self._start_services(
|
|
|
engine_units,
|
|
|
message="新版本私有引擎启动失败,请检查 systemd 日志",
|
|
|
)
|
|
|
_wait_units(self.repo_dir, engine_units)
|
|
|
try:
|
|
|
_wait_engine_health(self.config)
|
|
|
except Exception:
|
|
|
# The website must remain available to report and retry a failed
|
|
|
# update. Restore the consumers too: the target checkout's PHP
|
|
|
# daemons have their own bounded engine-readiness gate, so they do
|
|
|
# not query a half-started engine or claim VM work. Re-raise
|
|
|
# the original health failure after systemd has a recoverable
|
|
|
# service topology instead of leaving the panel SSH-only.
|
|
|
self._start_services(
|
|
|
consumer_units,
|
|
|
message="引擎未就绪且面板后台服务恢复失败,请检查 systemd 日志",
|
|
|
)
|
|
|
_wait_units(self.repo_dir, consumer_units)
|
|
|
raise
|
|
|
self._start_services(
|
|
|
consumer_units,
|
|
|
message="新版本面板后台服务启动失败,请检查 systemd 日志",
|
|
|
)
|
|
|
_wait_units(self.repo_dir, consumer_units)
|
|
|
|
|
|
|
|
|
def _clean_recorded_checkout(updater: Updater, job: dict, field: str) -> bool:
|
|
|
commit = job.get(field)
|
|
|
if not isinstance(commit, str) or not _COMMIT.fullmatch(commit):
|
|
|
return False
|
|
|
try:
|
|
|
return updater._checkout_head(require_clean=True) == commit
|
|
|
except (OSError, UpdateError):
|
|
|
return False
|
|
|
|
|
|
|
|
|
def _safe_to_start_services(updater: Updater, job: dict) -> bool:
|
|
|
"""Require durable proof before starting a possibly migrated checkout."""
|
|
|
state = job.get("deployment_state")
|
|
|
if state in {"schema_ready", "services_ready"}:
|
|
|
return _clean_recorded_checkout(updater, job, "new_commit")
|
|
|
if state in {"quiescing", "services_stopped", "code_switched"}:
|
|
|
return _clean_recorded_checkout(updater, job, "old_commit")
|
|
|
# In particular, schema_migrating means MariaDB may have committed only a
|
|
|
# prefix of the migration. A source rollback alone cannot prove safety.
|
|
|
# Unknown states also fail closed.
|
|
|
return False
|
|
|
|
|
|
|
|
|
def _load_updater() -> tuple[Updater, MainStore, dict]:
|
|
|
config_path = Path(os.environ.get(_CONFIG_ENV, str(_DEFAULT_CONFIG)))
|
|
|
config = private_config(config_path)
|
|
|
key = master_key(Path(config["master_key_file"]))
|
|
|
store = MainStore(config["main_database"], key)
|
|
|
source_root = Path(__file__).resolve().parents[2]
|
|
|
configured_root = Path(config.get("project_dir", source_root))
|
|
|
if not configured_root.is_absolute() or configured_root.resolve() != source_root.resolve():
|
|
|
raise UpdateError("引擎配置中的项目目录与当前程序目录不一致", 409)
|
|
|
return Updater(source_root, store), store, config
|
|
|
|
|
|
|
|
|
def finalize_incomplete_job(updater: Updater, store: MainStore) -> bool:
|
|
|
"""Mark a queued/running job failed after an interrupted one-shot."""
|
|
|
job_id = store.get_value("update.active_job")
|
|
|
if not job_id:
|
|
|
return False
|
|
|
job = updater.get_job(job_id)
|
|
|
if job is None or job.get("status") not in {"queued", "running"}:
|
|
|
return False
|
|
|
updater.fail_job(
|
|
|
job_id,
|
|
|
"独立更新服务异常退出或超时;请检查更新服务日志和当前 Git 版本,勿直接重复提交",
|
|
|
)
|
|
|
return True
|
|
|
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
|
parser = argparse.ArgumentParser(description="Run or finalize the Virt Cloud update job")
|
|
|
mode = parser.add_mutually_exclusive_group()
|
|
|
mode.add_argument("--finalize-incomplete", action="store_true")
|
|
|
mode.add_argument("--recover-blocked", action="store_true")
|
|
|
options = parser.parse_args(argv)
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
|
|
try:
|
|
|
updater, store, config = _load_updater()
|
|
|
if options.recover_blocked:
|
|
|
deployment = DeploymentTransaction(updater.repo_dir, config, recovery=True)
|
|
|
# This is an explicit root/SSH recovery path, never invoked by the
|
|
|
# web service or ExecStopPost. Holding the Git lock makes a killed
|
|
|
# recovery safely resumable without overlapping another process.
|
|
|
with updater._apply_lock():
|
|
|
job = updater.begin_blocked_recovery()
|
|
|
job_id = job["job_id"]
|
|
|
try:
|
|
|
deployment.quiesce()
|
|
|
updater.recovery_checkpoint(
|
|
|
job_id,
|
|
|
"schema_migrating",
|
|
|
"SSH 人工恢复正在执行数据库初始化",
|
|
|
)
|
|
|
deployment.activate(lambda: updater.recovery_checkpoint(
|
|
|
job_id,
|
|
|
"schema_ready",
|
|
|
"SSH 人工恢复数据库初始化完成,正在启动服务",
|
|
|
))
|
|
|
updater.complete_blocked_recovery(job_id)
|
|
|
except Exception:
|
|
|
updater.fail_job(
|
|
|
job_id,
|
|
|
"SSH 人工恢复未完成;维护锁是否保留取决于最后一个持久安全检查点",
|
|
|
)
|
|
|
raise
|
|
|
LOG.info("blocked panel update recovered: job=%s", job_id)
|
|
|
return 0
|
|
|
if options.finalize_incomplete:
|
|
|
changed = finalize_incomplete_job(updater, store)
|
|
|
if changed:
|
|
|
LOG.warning("interrupted panel update marked failed")
|
|
|
job_id = store.get_value("update.active_job")
|
|
|
job = updater.get_job(job_id) if job_id else None
|
|
|
if (isinstance(job, dict) and job.get("status") != "succeeded"
|
|
|
and _safe_to_start_services(updater, job)):
|
|
|
LOG.warning("durable update checkpoint permits service recovery")
|
|
|
DeploymentTransaction(updater.repo_dir, config, recovery=True).start_and_check()
|
|
|
elif isinstance(job, dict) and job.get("deployment_state") in {
|
|
|
"quiescing", "services_stopped", "code_switched", "schema_migrating"}:
|
|
|
LOG.error(
|
|
|
"services remain stopped: update checkpoint is not safe for automatic recovery"
|
|
|
)
|
|
|
return 0
|
|
|
deployment = DeploymentTransaction(updater.repo_dir, config)
|
|
|
result = updater.run_pending_job(
|
|
|
before_switch=deployment.quiesce,
|
|
|
restart=deployment.activate,
|
|
|
)
|
|
|
except (UpdateError, DatabaseError, OSError, ValueError) as exc:
|
|
|
# Never log DB configuration, the Gitea PAT, subprocess output or the
|
|
|
# environment. Every public UpdateError is deliberately credential-free.
|
|
|
LOG.error("panel update failed: %s", exc)
|
|
|
return 1
|
|
|
LOG.info(
|
|
|
"panel update finished: job=%s old=%s new=%s updated=%s",
|
|
|
result["job_id"], result.get("old_commit"), result.get("new_commit"), result.get("updated"),
|
|
|
)
|
|
|
return 0
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
raise SystemExit(main())
|