|
|
"""Conservative Git updater for Virt Cloud's own, fixed Gitea repository.
|
|
|
|
|
|
The web process may inspect a checkout and enqueue a job. ``run_pending_job``
|
|
|
must run in a separate, privileged one-shot service: the regular web service
|
|
|
has a read-only code directory, and restarting it from within a request is
|
|
|
unreliable. The caller supplies the restart/health-check callback.
|
|
|
|
|
|
The personal access token is stored by the injected encrypted database store.
|
|
|
It never appears in a Git URL, command argument, temporary file, or returned
|
|
|
API data. A private, short-lived askpass program receives it only in the
|
|
|
child's environment while an HTTPS Git operation is in progress.
|
|
|
"""
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
import fcntl
|
|
|
import json
|
|
|
import os
|
|
|
import re
|
|
|
import secrets
|
|
|
import stat
|
|
|
import subprocess
|
|
|
import tempfile
|
|
|
import threading
|
|
|
import time
|
|
|
from contextlib import contextmanager
|
|
|
from pathlib import Path
|
|
|
from typing import Callable, Protocol
|
|
|
|
|
|
|
|
|
REPOSITORY_URL = "https://g.luoli.live/root/virt.git"
|
|
|
BRANCH = "venix"
|
|
|
_TOKEN_KEY = "update.gitea_access_token"
|
|
|
_ACTIVE_JOB_KEY = "update.active_job"
|
|
|
_JOB_PREFIX = "update.job."
|
|
|
_HEX_COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z")
|
|
|
_JOB_ID = re.compile(r"[A-Za-z0-9_-]{16,64}\Z")
|
|
|
_ACTIVE_JOB_STATUSES = {"queued", "running", "blocked"}
|
|
|
_QUEUED_JOB_TIMEOUT = 60
|
|
|
_RUNNING_JOB_TIMEOUT = 35 * 60
|
|
|
|
|
|
|
|
|
class UpdateStore(Protocol):
|
|
|
"""Small persistence surface supplied by the encrypted database layer."""
|
|
|
|
|
|
def get_secret(self, name: str) -> str | None: ...
|
|
|
|
|
|
def set_secret(self, name: str, value: str) -> None: ...
|
|
|
|
|
|
def delete_secret(self, name: str) -> None: ...
|
|
|
|
|
|
def get_value(self, name: str) -> str | None: ...
|
|
|
|
|
|
def set_value(self, name: str, value: str) -> None: ...
|
|
|
|
|
|
|
|
|
class UpdateError(Exception):
|
|
|
"""A safe error suitable for returning to the authenticated operator."""
|
|
|
|
|
|
def __init__(self, message: str, status: int = 400) -> None:
|
|
|
super().__init__(message)
|
|
|
self.status = status
|
|
|
|
|
|
|
|
|
class Updater:
|
|
|
"""Read/check/advance only the pinned ``venix`` branch of the panel repo."""
|
|
|
|
|
|
def __init__(
|
|
|
self,
|
|
|
repo_dir: Path,
|
|
|
secret_store: UpdateStore,
|
|
|
*,
|
|
|
git_binary: str = "git",
|
|
|
runner: Callable[..., subprocess.CompletedProcess[str]] | None = None,
|
|
|
worker_active: Callable[[], bool | None] | None = None,
|
|
|
) -> None:
|
|
|
self.repo_dir = Path(repo_dir).absolute()
|
|
|
self.store = secret_store
|
|
|
self.git_binary = git_binary
|
|
|
self._run = runner or subprocess.run
|
|
|
self._worker_active = worker_active
|
|
|
self._lock = threading.RLock()
|
|
|
|
|
|
def settings(self) -> dict:
|
|
|
"""Return only public configuration, never the stored access token."""
|
|
|
with self._lock:
|
|
|
active_id = self.store.get_value(_ACTIVE_JOB_KEY)
|
|
|
last_job = self.get_job(active_id) if active_id else None
|
|
|
active_job = self._active_job_record(active_id)
|
|
|
result = {
|
|
|
"repository": REPOSITORY_URL,
|
|
|
"branch": BRANCH,
|
|
|
"token_configured": bool(self.store.get_secret(_TOKEN_KEY)),
|
|
|
"checkout_ready": False,
|
|
|
"current_commit": None,
|
|
|
"last_job_id": active_id if last_job is not None else None,
|
|
|
"active_job_id": (
|
|
|
active_id if active_job is not None
|
|
|
and active_job.get("status") in _ACTIVE_JOB_STATUSES else None
|
|
|
),
|
|
|
}
|
|
|
try:
|
|
|
result["current_commit"] = self._checkout_head(require_clean=True)
|
|
|
result["checkout_ready"] = True
|
|
|
except UpdateError as exc:
|
|
|
result["checkout_error"] = str(exc)
|
|
|
return result
|
|
|
|
|
|
def active_job(self) -> dict | None:
|
|
|
"""Return the queued, running or recovery-blocked maintenance job."""
|
|
|
with self._lock:
|
|
|
return self._active_job_record(self.store.get_value(_ACTIVE_JOB_KEY))
|
|
|
|
|
|
def _active_job_record(self, job_id: str | None) -> dict | None:
|
|
|
if not job_id:
|
|
|
return None
|
|
|
job = self.get_job(job_id)
|
|
|
if job is None or job.get("status") not in _ACTIVE_JOB_STATUSES:
|
|
|
return None
|
|
|
if job.get("status") == "blocked":
|
|
|
if self._abandoned_job_is_safe_to_release(job):
|
|
|
job.update(
|
|
|
status="failed",
|
|
|
phase="人工恢复后维护锁已解除",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
self._save_job(job)
|
|
|
return None
|
|
|
return job
|
|
|
# ``--recover-blocked`` is an explicit SSH process, not the systemd
|
|
|
# one-shot unit. Its durable recovery flag is the maintenance lease;
|
|
|
# never infer that it died merely because the update unit is inactive.
|
|
|
# A killed recovery stays fail-closed and can be resumed under the Git
|
|
|
# flock by running the same SSH command again.
|
|
|
if job.get("status") == "running" and job.get("recovery_requested") is True:
|
|
|
return job
|
|
|
now = int(time.time())
|
|
|
timestamp = job.get("created_at") if job["status"] == "queued" else job.get("started_at")
|
|
|
timeout = _QUEUED_JOB_TIMEOUT if job["status"] == "queued" else _RUNNING_JOB_TIMEOUT
|
|
|
stale = (isinstance(timestamp, int) and not isinstance(timestamp, bool)
|
|
|
and now >= timestamp and now - timestamp > timeout)
|
|
|
try:
|
|
|
worker_state = self._worker_active() if self._worker_active is not None else None
|
|
|
except Exception:
|
|
|
# An unavailable service manager is not proof that a privileged
|
|
|
# updater has stopped. Keep maintenance mode in that case.
|
|
|
worker_state = None
|
|
|
abandoned = (
|
|
|
job["status"] == "running" and worker_state is False
|
|
|
) or (
|
|
|
stale and worker_state is not True
|
|
|
and (self._worker_active is None or worker_state is False)
|
|
|
)
|
|
|
if abandoned:
|
|
|
if job["status"] == "running" and not self._abandoned_job_is_safe_to_release(job):
|
|
|
message = (
|
|
|
"独立更新服务已停止,但代码或数据库状态尚不能确认;"
|
|
|
"维护锁已保留,请通过 SSH 检查更新服务、Git 版本和数据库初始化状态"
|
|
|
)
|
|
|
if job.get("phase") != message or job.get("error") != message:
|
|
|
job.update(phase=message, error=message)
|
|
|
self._save_job(job)
|
|
|
return job
|
|
|
job.update(
|
|
|
status="failed",
|
|
|
phase="更新任务执行器已停止",
|
|
|
error="独立更新服务已停止或未在限定时间内启动;请检查更新服务日志和当前 Git 版本",
|
|
|
finished_at=now,
|
|
|
)
|
|
|
self._save_job(job)
|
|
|
return None
|
|
|
return job
|
|
|
|
|
|
def _abandoned_job_is_safe_to_release(self, job: dict) -> bool:
|
|
|
"""Only release maintenance when code/schema compatibility is known.
|
|
|
|
|
|
An interrupted worker may have stopped the application and switched
|
|
|
source code without completing the schema migration. Automatically
|
|
|
dropping the maintenance gate in that state could let a manually
|
|
|
restarted service write with an incompatible schema.
|
|
|
"""
|
|
|
state = job.get("deployment_state")
|
|
|
if state in {"schema_ready", "services_ready"}:
|
|
|
target = job.get("new_commit")
|
|
|
if not isinstance(target, str) or not _HEX_COMMIT.fullmatch(target):
|
|
|
return False
|
|
|
try:
|
|
|
return self._checkout_head(require_clean=True) == target
|
|
|
except UpdateError:
|
|
|
return False
|
|
|
if state not in {"preparing", "quiescing", "services_stopped", "code_switched"}:
|
|
|
return False
|
|
|
# A clean rollback to the recorded old commit is safe regardless of
|
|
|
# the last observed phase. This is also the SSH recovery exit for a
|
|
|
# ``blocked`` job after a failed migration.
|
|
|
old_commit = job.get("old_commit") or job.get("expected_current_commit")
|
|
|
if not isinstance(old_commit, str) or not _HEX_COMMIT.fullmatch(old_commit):
|
|
|
return False
|
|
|
try:
|
|
|
return self._checkout_head(require_clean=True) == old_commit
|
|
|
except UpdateError:
|
|
|
return False
|
|
|
|
|
|
def set_token(self, token: str) -> None:
|
|
|
if (
|
|
|
not isinstance(token, str)
|
|
|
or not 16 <= len(token) <= 4096
|
|
|
or token != token.strip()
|
|
|
or any(ord(char) < 33 or ord(char) > 126 for char in token)
|
|
|
):
|
|
|
raise UpdateError("Gitea 访问令牌格式无效")
|
|
|
with self._lock:
|
|
|
self._assert_no_running_job_for_token_change()
|
|
|
self.store.set_secret(_TOKEN_KEY, token)
|
|
|
|
|
|
def clear_token(self) -> None:
|
|
|
with self._lock:
|
|
|
self._assert_no_running_job_for_token_change()
|
|
|
self.store.delete_secret(_TOKEN_KEY)
|
|
|
|
|
|
def _assert_no_running_job_for_token_change(self) -> None:
|
|
|
active = self._active_job_record(self.store.get_value(_ACTIVE_JOB_KEY))
|
|
|
if active is not None:
|
|
|
raise UpdateError("更新任务执行期间不能修改访问令牌", 409)
|
|
|
|
|
|
def _environment(self) -> dict[str, str]:
|
|
|
# Do not inherit GIT_TRACE, Git configuration, proxy overrides or
|
|
|
# GIT_SSL_NO_VERIFY from the panel service environment.
|
|
|
return {
|
|
|
"PATH": os.defpath,
|
|
|
"LANG": "C",
|
|
|
"LC_ALL": "C",
|
|
|
"GIT_CONFIG_NOSYSTEM": "1",
|
|
|
"GIT_CONFIG_GLOBAL": os.devnull,
|
|
|
"GIT_TERMINAL_PROMPT": "0",
|
|
|
"GIT_OPTIONAL_LOCKS": "0",
|
|
|
"GIT_ALLOW_PROTOCOL": "https",
|
|
|
}
|
|
|
|
|
|
@contextmanager
|
|
|
def _git_auth(self):
|
|
|
token = self.store.get_secret(_TOKEN_KEY)
|
|
|
if not token:
|
|
|
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
|
|
|
with tempfile.TemporaryDirectory(prefix="virt-cloud-askpass-") as directory:
|
|
|
askpass = Path(directory) / "askpass"
|
|
|
# This script contains no credential. Git passes a prompt as one
|
|
|
# argument; the token is handed to the Git child only in memory.
|
|
|
askpass.write_text(
|
|
|
"#!/bin/sh\n"
|
|
|
"case \"$1\" in\n"
|
|
|
" *sername*|*SERNAM*) printf '%s\\n' \"$VIRT_CLOUD_GIT_USERNAME\" ;;\n"
|
|
|
" *assword*|*ASSWORD*) printf '%s\\n' \"$VIRT_CLOUD_GIT_TOKEN\" ;;\n"
|
|
|
" *) exit 1 ;;\n"
|
|
|
"esac\n",
|
|
|
encoding="ascii",
|
|
|
)
|
|
|
askpass.chmod(0o700)
|
|
|
env = self._environment()
|
|
|
env.update({
|
|
|
"GIT_ASKPASS": str(askpass),
|
|
|
"VIRT_CLOUD_GIT_USERNAME": "root",
|
|
|
"VIRT_CLOUD_GIT_TOKEN": token,
|
|
|
})
|
|
|
yield env
|
|
|
|
|
|
def _git(
|
|
|
self,
|
|
|
*args: str,
|
|
|
timeout: float = 15,
|
|
|
environment: dict[str, str] | None = None,
|
|
|
error: str = "Git 操作失败,请检查面板日志",
|
|
|
allowed_codes: tuple[int, ...] = (0,),
|
|
|
) -> subprocess.CompletedProcess[str]:
|
|
|
argv = [
|
|
|
self.git_binary,
|
|
|
"-c", "core.hooksPath=/dev/null",
|
|
|
"-c", "http.followRedirects=false",
|
|
|
"-c", "http.sslVerify=true",
|
|
|
"-c", "http.extraHeader=",
|
|
|
"-c", "credential.helper=",
|
|
|
"-c", "fetch.recurseSubmodules=false",
|
|
|
*args,
|
|
|
]
|
|
|
try:
|
|
|
result = self._run(
|
|
|
argv,
|
|
|
cwd=self.repo_dir,
|
|
|
env=environment or self._environment(),
|
|
|
capture_output=True,
|
|
|
text=True,
|
|
|
timeout=timeout,
|
|
|
check=False,
|
|
|
)
|
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
|
# Never include stderr, a URL, or a token in a public exception.
|
|
|
raise UpdateError(error, 502) from exc
|
|
|
if result.returncode not in allowed_codes:
|
|
|
raise UpdateError(error, 502)
|
|
|
return result
|
|
|
|
|
|
def _checkout_head(self, *, require_clean: bool) -> str:
|
|
|
if self.repo_dir.is_symlink() or not self.repo_dir.is_dir():
|
|
|
raise UpdateError("面板目录不是有效的 Git 检出目录", 409)
|
|
|
repo_info = self.repo_dir.lstat()
|
|
|
if repo_info.st_uid != os.geteuid() or repo_info.st_mode & 0o022:
|
|
|
raise UpdateError("面板目录必须由当前服务账号拥有,且不能允许组或其他用户写入", 409)
|
|
|
git_dir = self.repo_dir / ".git"
|
|
|
if git_dir.is_symlink() or not git_dir.is_dir():
|
|
|
raise UpdateError("面板必须从完整 Git 仓库部署,不能使用压缩包或工作树", 409)
|
|
|
try:
|
|
|
info = git_dir.lstat()
|
|
|
except OSError as exc:
|
|
|
raise UpdateError("无法读取面板 Git 仓库", 409) from exc
|
|
|
if not stat.S_ISDIR(info.st_mode):
|
|
|
raise UpdateError("面板 Git 仓库格式无效", 409)
|
|
|
if info.st_uid != os.geteuid() or info.st_mode & 0o022:
|
|
|
raise UpdateError("面板 Git 仓库权限不安全,已拒绝在线更新", 409)
|
|
|
|
|
|
top = self._git("rev-parse", "--show-toplevel", error="无法识别面板 Git 仓库").stdout.strip()
|
|
|
if Path(top).resolve() != self.repo_dir.resolve():
|
|
|
raise UpdateError("Git 仓库根目录与面板程序目录不一致", 409)
|
|
|
branch = self._git(
|
|
|
"symbolic-ref", "--quiet", "--short", "HEAD",
|
|
|
error=f"面板 Git 仓库当前不是 {BRANCH} 分支",
|
|
|
).stdout.strip()
|
|
|
if branch != BRANCH:
|
|
|
raise UpdateError(f"面板只允许从 {BRANCH} 分支更新", 409)
|
|
|
configured_urls = self._git(
|
|
|
"config", "--local", "--get-all", "remote.origin.url",
|
|
|
error="面板 Git 仓库缺少 origin 地址",
|
|
|
).stdout.splitlines()
|
|
|
if configured_urls != [REPOSITORY_URL]:
|
|
|
raise UpdateError("Git origin 必须是已指定的 Gitea 私有仓库,且不得包含凭据", 409)
|
|
|
# url.*.insteadOf could silently rewrite our explicit HTTPS URL to a
|
|
|
# different server and disclose the access token. Refuse it outright.
|
|
|
rewrites = self._git(
|
|
|
"config", "--local", "--name-only", "--get-regexp", r"^url\..*\.insteadof$",
|
|
|
allowed_codes=(0, 1),
|
|
|
error="无法检查 Git URL 重写配置",
|
|
|
)
|
|
|
if rewrites.returncode == 0:
|
|
|
raise UpdateError("Git 仓库配置了 URL 重写,无法安全更新", 409)
|
|
|
head = self._git("rev-parse", "HEAD", error="无法读取当前面板版本").stdout.strip()
|
|
|
if not _HEX_COMMIT.fullmatch(head):
|
|
|
raise UpdateError("当前 Git 提交标识无效", 409)
|
|
|
if require_clean:
|
|
|
status = self._git(
|
|
|
"status", "--porcelain=v1", "--untracked-files=all",
|
|
|
error="无法检查面板文件变更",
|
|
|
).stdout
|
|
|
if status.strip():
|
|
|
raise UpdateError("面板目录存在未提交或未跟踪文件;为防止覆盖,请先处理这些文件", 409)
|
|
|
return head
|
|
|
|
|
|
def _remote_head(self) -> str:
|
|
|
with self._git_auth() as env:
|
|
|
output = self._git(
|
|
|
"ls-remote", "--exit-code", "--", REPOSITORY_URL, f"refs/heads/{BRANCH}",
|
|
|
environment=env,
|
|
|
timeout=45,
|
|
|
error="无法读取 Gitea 私有仓库;请检查访问令牌、TLS 证书和网络",
|
|
|
).stdout.strip()
|
|
|
fields = output.split()
|
|
|
if len(fields) != 2 or fields[1] != f"refs/heads/{BRANCH}" or not _HEX_COMMIT.fullmatch(fields[0]):
|
|
|
raise UpdateError("Gitea 返回了无效的分支信息", 502)
|
|
|
return fields[0]
|
|
|
|
|
|
def check(self) -> dict:
|
|
|
"""Inspect the remote head without altering the local checkout."""
|
|
|
with self._lock:
|
|
|
current = self._checkout_head(require_clean=True)
|
|
|
remote = self._remote_head()
|
|
|
return {
|
|
|
"repository": REPOSITORY_URL,
|
|
|
"branch": BRANCH,
|
|
|
"current_commit": current,
|
|
|
"remote_commit": remote,
|
|
|
"status": "current" if current == remote else "update_available",
|
|
|
}
|
|
|
|
|
|
def _fetch_remote(self) -> str:
|
|
|
with self._git_auth() as env:
|
|
|
self._git(
|
|
|
"fetch", "--no-tags", "--no-recurse-submodules", "--",
|
|
|
REPOSITORY_URL, f"refs/heads/{BRANCH}",
|
|
|
environment=env,
|
|
|
timeout=300,
|
|
|
error="无法从 Gitea 拉取更新;请检查访问令牌、TLS 证书和网络",
|
|
|
)
|
|
|
head = self._git("rev-parse", "FETCH_HEAD", error="无法识别已下载的更新版本").stdout.strip()
|
|
|
if not _HEX_COMMIT.fullmatch(head):
|
|
|
raise UpdateError("下载的 Git 提交标识无效", 502)
|
|
|
return head
|
|
|
|
|
|
def _assert_safe_tree(self, target: str) -> None:
|
|
|
protected_paths = (
|
|
|
".gitmodules",
|
|
|
"runtime",
|
|
|
"web/config/config.php",
|
|
|
"web/config/.config.php",
|
|
|
# Ticket images are private runtime data. Keep the path ignored
|
|
|
# and refuse a future target commit that tries to vendor files
|
|
|
# over an existing attachment directory.
|
|
|
"web/storage/ticket-attachments",
|
|
|
"web/vendor",
|
|
|
"engine/.venv",
|
|
|
"engine/master.key",
|
|
|
"master.key",
|
|
|
)
|
|
|
listing = self._git(
|
|
|
"ls-tree", "-r", "--name-only", target, "--", *protected_paths,
|
|
|
error="无法检查更新文件清单",
|
|
|
).stdout.strip()
|
|
|
if listing:
|
|
|
raise UpdateError("更新包含运行数据、私有配置、已安装依赖或子模块配置,已拒绝覆盖", 409)
|
|
|
|
|
|
# ``ls-tree -r`` does not report a gitlink's descendants. Refuse any
|
|
|
# mode 160000 entry independently of its path so a repository update
|
|
|
# cannot introduce executable code through an unreviewed submodule.
|
|
|
modes = self._git(
|
|
|
"ls-tree", "-r", target,
|
|
|
error="无法检查更新子模块清单",
|
|
|
).stdout.splitlines()
|
|
|
if any(line.startswith("160000 ") for line in modes):
|
|
|
raise UpdateError("更新包含 Git 子模块,已拒绝自动更新", 409)
|
|
|
if any(line.startswith("120000 ") for line in modes):
|
|
|
raise UpdateError("更新包含符号链接,已拒绝自动更新", 409)
|
|
|
|
|
|
def _normalize_checkout_permissions(self, target: str) -> None:
|
|
|
"""Make root-updated application code readable by PHP-FPM.
|
|
|
|
|
|
The privileged updater deliberately runs with a restrictive umask so
|
|
|
credentials and lock files stay private. Git also applies that umask
|
|
|
to replacement worktree files, however, which can turn PHP sources
|
|
|
into mode 0600 and take an apache/php-fpm deployment offline. Restore
|
|
|
only paths tracked by the verified target tree; ignored runtime,
|
|
|
vendor and private configuration files are never touched.
|
|
|
"""
|
|
|
listing = self._git(
|
|
|
"ls-tree", "-r", "-z", target,
|
|
|
error="无法读取更新文件权限清单",
|
|
|
).stdout
|
|
|
files: list[tuple[Path, int]] = []
|
|
|
directories: set[Path] = {self.repo_dir}
|
|
|
for record in listing.split("\0"):
|
|
|
if record == "":
|
|
|
continue
|
|
|
metadata, separator, relative = record.partition("\t")
|
|
|
fields = metadata.split()
|
|
|
parts = Path(relative).parts
|
|
|
if (not separator or len(fields) != 3 or fields[0] not in {"100644", "100755"}
|
|
|
or fields[1] != "blob" or not parts or relative.startswith("/")
|
|
|
or any(part in {"", ".", ".."} for part in parts)):
|
|
|
raise UpdateError("更新文件权限清单无效", 500)
|
|
|
local = self.repo_dir.joinpath(*parts)
|
|
|
try:
|
|
|
info = local.lstat()
|
|
|
except OSError as exc:
|
|
|
raise UpdateError("更新后的程序文件不完整", 500) from exc
|
|
|
if not stat.S_ISREG(info.st_mode):
|
|
|
raise UpdateError("更新后的程序文件类型无效", 500)
|
|
|
files.append((local, 0o755 if fields[0] == "100755" else 0o644))
|
|
|
parent = local.parent
|
|
|
while parent != self.repo_dir.parent:
|
|
|
directories.add(parent)
|
|
|
if parent == self.repo_dir:
|
|
|
break
|
|
|
parent = parent.parent
|
|
|
try:
|
|
|
for directory in sorted(directories, key=lambda value: len(value.parts)):
|
|
|
info = directory.lstat()
|
|
|
if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode):
|
|
|
raise UpdateError("更新后的程序目录类型无效", 500)
|
|
|
directory.chmod(0o755)
|
|
|
for local, mode in files:
|
|
|
local.chmod(mode)
|
|
|
except UpdateError:
|
|
|
raise
|
|
|
except OSError as exc:
|
|
|
raise UpdateError("无法恢复更新后程序文件的读取权限", 500) from exc
|
|
|
|
|
|
def _assert_online_compatible(self, current: str, target: str) -> None:
|
|
|
"""Refuse changes that require privileged/manual deployment work.
|
|
|
|
|
|
Python requirements are not hash locked, so installing them as root is
|
|
|
neither reproducible nor safe for a browser-triggered update. Unit
|
|
|
files may contain deployment-specific paths and users and must never be
|
|
|
overwritten by an online update.
|
|
|
"""
|
|
|
changed = self._git(
|
|
|
"diff", "--name-only", f"{current}..{target}", "--",
|
|
|
"engine/requirements.txt", "web/composer.json", "web/composer.lock", "deploy",
|
|
|
error="无法检查更新的部署兼容性",
|
|
|
).stdout.splitlines()
|
|
|
if "engine/requirements.txt" in changed:
|
|
|
raise UpdateError("此版本修改了 Python 依赖,请通过 SSH 按部署文档升级", 409)
|
|
|
if any(path in {"web/composer.json", "web/composer.lock"} for path in changed):
|
|
|
raise UpdateError("此版本修改了 PHP 依赖,请通过 SSH 按部署文档升级", 409)
|
|
|
if any(path == "deploy" or path.startswith("deploy/") for path in changed):
|
|
|
raise UpdateError("此版本修改了 systemd 服务文件,请通过 SSH 按部署文档升级", 409)
|
|
|
|
|
|
@contextmanager
|
|
|
def _apply_lock(self):
|
|
|
git_dir = self.repo_dir / ".git"
|
|
|
flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0)
|
|
|
try:
|
|
|
descriptor = os.open(git_dir / "virt-cloud-update.lock", flags, 0o600)
|
|
|
except OSError as exc:
|
|
|
raise UpdateError("无法获取面板更新锁;请确认独立更新服务有代码目录写权限", 500) from exc
|
|
|
try:
|
|
|
info = os.fstat(descriptor)
|
|
|
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077:
|
|
|
raise UpdateError("面板更新锁文件权限不安全", 500)
|
|
|
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
|
except BlockingIOError as exc:
|
|
|
os.close(descriptor)
|
|
|
raise UpdateError("另一个更新任务正在运行", 409) from exc
|
|
|
except BaseException:
|
|
|
os.close(descriptor)
|
|
|
raise
|
|
|
try:
|
|
|
yield
|
|
|
finally:
|
|
|
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
|
os.close(descriptor)
|
|
|
|
|
|
def apply(
|
|
|
self, *, progress: Callable[[str], None] | None = None,
|
|
|
expected_current_commit: str | None = None,
|
|
|
expected_remote_commit: str | None = None,
|
|
|
before_switch: Callable[[], None] | None = None,
|
|
|
checkpoint: Callable[[str, dict[str, str]], None] | None = None,
|
|
|
) -> dict:
|
|
|
"""Prepare an update, quiesce services, then fast-forward the code."""
|
|
|
report = progress or (lambda _phase: None)
|
|
|
with self._lock:
|
|
|
# Validate before creating a lock inside .git.
|
|
|
current = self._checkout_head(require_clean=True)
|
|
|
with self._apply_lock():
|
|
|
# Re-check after acquiring the cross-process lock.
|
|
|
current = self._checkout_head(require_clean=True)
|
|
|
if expected_current_commit is not None and current != expected_current_commit:
|
|
|
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
|
|
|
report("正在从 Gitea 下载更新")
|
|
|
target = self._fetch_remote()
|
|
|
if expected_remote_commit is not None and target != expected_remote_commit:
|
|
|
raise UpdateError(f"Gitea {BRANCH} 分支已变化,请重新检查更新后再执行", 409)
|
|
|
self._assert_safe_tree(target)
|
|
|
ancestry = self._git(
|
|
|
"merge-base", "--is-ancestor", current, target,
|
|
|
allowed_codes=(0, 1),
|
|
|
error="无法验证更新提交历史",
|
|
|
)
|
|
|
if ancestry.returncode != 0:
|
|
|
raise UpdateError(f"远端 {BRANCH} 分支与当前版本已分叉,已拒绝自动覆盖", 409)
|
|
|
if target == current:
|
|
|
return {"updated": False, "old_commit": current, "new_commit": target}
|
|
|
self._assert_online_compatible(current, target)
|
|
|
# Keep an immutable rollback pointer for manual recovery. A
|
|
|
# forced reset is never part of the automatic update path.
|
|
|
rollback_ref = f"refs/virt-cloud/rollback/{int(time.time())}-{secrets.token_hex(4)}"
|
|
|
self._git(
|
|
|
"update-ref", rollback_ref, current,
|
|
|
error="无法保存更新前版本,已取消切换",
|
|
|
)
|
|
|
# Files may have been edited while the remote update was fetched.
|
|
|
self._checkout_head(require_clean=True)
|
|
|
if before_switch is not None:
|
|
|
report("正在暂停面板服务")
|
|
|
if checkpoint is not None:
|
|
|
checkpoint("quiescing", {
|
|
|
"old_commit": current,
|
|
|
"new_commit": target,
|
|
|
"rollback_ref": rollback_ref,
|
|
|
})
|
|
|
before_switch()
|
|
|
if checkpoint is not None:
|
|
|
checkpoint("services_stopped", {
|
|
|
"old_commit": current,
|
|
|
"new_commit": target,
|
|
|
"rollback_ref": rollback_ref,
|
|
|
})
|
|
|
report("正在切换到新版本")
|
|
|
self._git(
|
|
|
"merge", "--ff-only", "--no-stat", target,
|
|
|
timeout=120,
|
|
|
error="无法快进更新代码;请检查面板目录与 Git 日志",
|
|
|
)
|
|
|
self._normalize_checkout_permissions(target)
|
|
|
installed = self._checkout_head(require_clean=True)
|
|
|
if installed != target:
|
|
|
raise UpdateError("更新后的 Git 提交与预期不符,请人工核查", 500)
|
|
|
if checkpoint is not None:
|
|
|
checkpoint("code_switched", {
|
|
|
"old_commit": current,
|
|
|
"new_commit": target,
|
|
|
"rollback_ref": rollback_ref,
|
|
|
})
|
|
|
return {
|
|
|
"updated": True,
|
|
|
"old_commit": current,
|
|
|
"new_commit": target,
|
|
|
"rollback_ref": rollback_ref,
|
|
|
}
|
|
|
|
|
|
def _job_key(self, job_id: str) -> str:
|
|
|
if not isinstance(job_id, str) or not _JOB_ID.fullmatch(job_id):
|
|
|
raise UpdateError("更新任务编号无效", 404)
|
|
|
return _JOB_PREFIX + job_id
|
|
|
|
|
|
def get_job(self, job_id: str) -> dict | None:
|
|
|
raw = self.store.get_value(self._job_key(job_id))
|
|
|
if raw is None:
|
|
|
return None
|
|
|
try:
|
|
|
job = json.loads(raw)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise UpdateError("更新任务记录损坏", 500) from exc
|
|
|
if not isinstance(job, dict) or job.get("job_id") != job_id:
|
|
|
raise UpdateError("更新任务记录损坏", 500)
|
|
|
return job
|
|
|
|
|
|
def _save_job(self, job: dict) -> None:
|
|
|
self.store.set_value(self._job_key(job["job_id"]), json.dumps(job, ensure_ascii=False))
|
|
|
|
|
|
def create_job(
|
|
|
self, *, expected_current_commit: str | None = None,
|
|
|
expected_remote_commit: str | None = None,
|
|
|
) -> dict:
|
|
|
"""Enqueue one persisted job; the web layer then starts the worker."""
|
|
|
with self._lock:
|
|
|
current = self._checkout_head(require_clean=True)
|
|
|
if expected_current_commit is not None:
|
|
|
if not _HEX_COMMIT.fullmatch(expected_current_commit):
|
|
|
raise UpdateError("要更新的当前版本标识无效", 422)
|
|
|
if current != expected_current_commit:
|
|
|
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
|
|
|
if not self.store.get_secret(_TOKEN_KEY):
|
|
|
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
|
|
|
if expected_remote_commit is not None:
|
|
|
if not _HEX_COMMIT.fullmatch(expected_remote_commit):
|
|
|
raise UpdateError("要更新的远端版本标识无效", 422)
|
|
|
if self._remote_head() != expected_remote_commit:
|
|
|
raise UpdateError(f"Gitea {BRANCH} 分支已变化,请重新检查更新后再执行", 409)
|
|
|
active = self._active_job_record(self.store.get_value(_ACTIVE_JOB_KEY))
|
|
|
if active is not None:
|
|
|
raise UpdateError("已有更新任务正在执行", 409)
|
|
|
job_id = secrets.token_urlsafe(18)
|
|
|
job = {
|
|
|
"job_id": job_id,
|
|
|
"status": "queued",
|
|
|
"phase": "等待独立更新服务启动",
|
|
|
"created_at": int(time.time()),
|
|
|
}
|
|
|
if expected_current_commit is not None:
|
|
|
job["expected_current_commit"] = expected_current_commit
|
|
|
if expected_remote_commit is not None:
|
|
|
job["expected_remote_commit"] = expected_remote_commit
|
|
|
self._save_job(job)
|
|
|
self.store.set_value(_ACTIVE_JOB_KEY, job_id)
|
|
|
return job.copy()
|
|
|
|
|
|
def fail_job(self, job_id: str, message: str) -> dict:
|
|
|
"""Fail a safe job, or retain a maintenance lock if state is uncertain."""
|
|
|
with self._lock:
|
|
|
job = self.get_job(job_id)
|
|
|
if job is None:
|
|
|
raise UpdateError("更新任务不存在", 404)
|
|
|
if job["status"] not in {"queued", "running"}:
|
|
|
return job
|
|
|
safe_to_release = (
|
|
|
job["status"] == "queued" or self._abandoned_job_is_safe_to_release(job)
|
|
|
)
|
|
|
if safe_to_release:
|
|
|
job.update(
|
|
|
status="failed",
|
|
|
phase="更新失败",
|
|
|
error=str(message)[:300],
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
else:
|
|
|
error = (
|
|
|
f"{str(message)[:180]};代码或数据库状态尚未确认,维护锁已保留。"
|
|
|
"请通过 SSH 核查 Git 版本并完成数据库初始化或回滚后再恢复服务"
|
|
|
)
|
|
|
job.update(
|
|
|
status="blocked",
|
|
|
phase="更新中断,需要通过 SSH 人工恢复",
|
|
|
error=error[:300],
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
self._save_job(job)
|
|
|
return job.copy()
|
|
|
|
|
|
def begin_blocked_recovery(self) -> dict:
|
|
|
"""Claim a blocked job for an explicit SSH forward recovery.
|
|
|
|
|
|
Recovery never fetches or switches source. It is permitted only when
|
|
|
the checkout is clean and still at the exact target commit recorded by
|
|
|
the interrupted job. The caller must hold ``_apply_lock`` throughout
|
|
|
migration and service health verification.
|
|
|
"""
|
|
|
with self._lock:
|
|
|
job_id = self.store.get_value(_ACTIVE_JOB_KEY)
|
|
|
if not job_id:
|
|
|
raise UpdateError("没有需要人工恢复的在线更新任务", 409)
|
|
|
job = self.get_job(job_id)
|
|
|
resumable = (
|
|
|
isinstance(job, dict)
|
|
|
and job.get("status") == "running"
|
|
|
and job.get("recovery_requested") is True
|
|
|
)
|
|
|
if not isinstance(job, dict) or (job.get("status") != "blocked" and not resumable):
|
|
|
raise UpdateError("当前更新任务不处于可人工恢复状态", 409)
|
|
|
target = job.get("new_commit")
|
|
|
if not isinstance(target, str) or not _HEX_COMMIT.fullmatch(target):
|
|
|
raise UpdateError("更新任务缺少可验证的目标提交", 409)
|
|
|
if self._checkout_head(require_clean=True) != target:
|
|
|
raise UpdateError("人工恢复要求干净工作树保持在任务记录的新提交", 409)
|
|
|
job.pop("error", None)
|
|
|
job.pop("finished_at", None)
|
|
|
job.update(
|
|
|
status="running",
|
|
|
phase="SSH 人工恢复正在准备数据库初始化",
|
|
|
recovery_requested=True,
|
|
|
recovery_started_at=int(time.time()),
|
|
|
)
|
|
|
self._save_job(job)
|
|
|
return job.copy()
|
|
|
|
|
|
def recovery_checkpoint(self, job_id: str, state: str, phase: str) -> dict:
|
|
|
if state not in {"schema_migrating", "schema_ready"}:
|
|
|
raise UpdateError("人工恢复检查点无效", 500)
|
|
|
with self._lock:
|
|
|
if self.store.get_value(_ACTIVE_JOB_KEY) != job_id:
|
|
|
raise UpdateError("人工恢复任务已变化", 409)
|
|
|
job = self.get_job(job_id)
|
|
|
if (job is None or job.get("status") != "running"
|
|
|
or job.get("recovery_requested") is not True):
|
|
|
raise UpdateError("人工恢复任务状态无效", 409)
|
|
|
target = job.get("new_commit")
|
|
|
if not isinstance(target, str) or self._checkout_head(require_clean=True) != target:
|
|
|
raise UpdateError("人工恢复期间 Git 版本发生变化,维护锁继续保留", 409)
|
|
|
job.update(deployment_state=state, phase=str(phase)[:200])
|
|
|
self._save_job(job)
|
|
|
return job.copy()
|
|
|
|
|
|
def complete_blocked_recovery(self, job_id: str) -> dict:
|
|
|
with self._lock:
|
|
|
if self.store.get_value(_ACTIVE_JOB_KEY) != job_id:
|
|
|
raise UpdateError("人工恢复任务已变化", 409)
|
|
|
job = self.get_job(job_id)
|
|
|
if (job is None or job.get("status") != "running"
|
|
|
or job.get("recovery_requested") is not True
|
|
|
or job.get("deployment_state") != "schema_ready"):
|
|
|
raise UpdateError("人工恢复尚未完成数据库与服务检查", 409)
|
|
|
target = job.get("new_commit")
|
|
|
if not isinstance(target, str) or self._checkout_head(require_clean=True) != target:
|
|
|
raise UpdateError("人工恢复期间 Git 版本发生变化,维护锁继续保留", 409)
|
|
|
job.pop("error", None)
|
|
|
job.update(
|
|
|
status="succeeded",
|
|
|
deployment_state="services_ready",
|
|
|
phase="SSH 人工恢复完成",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
self._save_job(job)
|
|
|
return job.copy()
|
|
|
|
|
|
def run_pending_job(
|
|
|
self, *, restart: Callable[[Callable[[], None]], None] | None = None,
|
|
|
before_switch: Callable[[], None] | None = None,
|
|
|
) -> dict:
|
|
|
"""Run the queued job in a separate service and persist every phase.
|
|
|
|
|
|
``restart`` should restart the web unit and, ideally, verify its health.
|
|
|
If it fails, the job records that code may already have changed and
|
|
|
exposes the old commit/rollback ref for operator-led recovery.
|
|
|
"""
|
|
|
with self._lock:
|
|
|
job_id = self.store.get_value(_ACTIVE_JOB_KEY)
|
|
|
if not job_id:
|
|
|
raise UpdateError("没有待执行的更新任务", 404)
|
|
|
job = self.get_job(job_id)
|
|
|
if job is None or job.get("status") != "queued":
|
|
|
raise UpdateError("更新任务未处于等待状态;请人工核查后重新提交", 409)
|
|
|
job.update(
|
|
|
status="running",
|
|
|
phase="正在检查本地仓库",
|
|
|
deployment_state="preparing",
|
|
|
started_at=int(time.time()),
|
|
|
)
|
|
|
self._save_job(job)
|
|
|
|
|
|
def progress(phase: str) -> None:
|
|
|
with self._lock:
|
|
|
job["phase"] = str(phase)[:200]
|
|
|
self._save_job(job)
|
|
|
|
|
|
def checkpoint(state: str, details: dict[str, str] | None = None) -> None:
|
|
|
if state not in {
|
|
|
"quiescing", "services_stopped", "code_switched",
|
|
|
"schema_migrating", "schema_ready", "services_ready",
|
|
|
}:
|
|
|
raise UpdateError("更新任务检查点无效", 500)
|
|
|
safe_details = details or {}
|
|
|
if set(safe_details) - {"old_commit", "new_commit", "rollback_ref"}:
|
|
|
raise UpdateError("更新任务检查点内容无效", 500)
|
|
|
for name, value in safe_details.items():
|
|
|
if name in {"old_commit", "new_commit"} and not _HEX_COMMIT.fullmatch(value):
|
|
|
raise UpdateError("更新任务检查点提交标识无效", 500)
|
|
|
if name == "rollback_ref" and not value.startswith("refs/virt-cloud/rollback/"):
|
|
|
raise UpdateError("更新任务检查点回滚引用无效", 500)
|
|
|
with self._lock:
|
|
|
job.update(safe_details)
|
|
|
job["deployment_state"] = state
|
|
|
self._save_job(job)
|
|
|
|
|
|
applied: dict | None = None
|
|
|
try:
|
|
|
applied = self.apply(
|
|
|
progress=progress,
|
|
|
expected_current_commit=job.get("expected_current_commit"),
|
|
|
expected_remote_commit=job.get("expected_remote_commit"),
|
|
|
before_switch=before_switch,
|
|
|
checkpoint=checkpoint,
|
|
|
)
|
|
|
job.update(applied)
|
|
|
self._save_job(job)
|
|
|
if applied["updated"] and restart is not None:
|
|
|
progress("正在重启面板并检查服务")
|
|
|
# Persist this before invoking PHP. MariaDB DDL may commit one
|
|
|
# statement at a time, so a killed migration is not proven
|
|
|
# safe merely by rolling the source checkout back afterward.
|
|
|
checkpoint("schema_migrating")
|
|
|
def mark_schema_ready() -> None:
|
|
|
progress("数据库初始化完成,正在启动面板服务")
|
|
|
checkpoint("schema_ready")
|
|
|
|
|
|
restart(mark_schema_ready)
|
|
|
checkpoint("services_ready")
|
|
|
elif applied["updated"]:
|
|
|
# A worker that cannot restart the web service must not
|
|
|
# report a successful one-click update.
|
|
|
raise UpdateError("代码已更新,但未配置自动重启;请人工重启面板服务", 500)
|
|
|
job.update(status="succeeded", phase="更新完成" if applied["updated"] else "已是最新版本",
|
|
|
finished_at=int(time.time()))
|
|
|
self._save_job(job)
|
|
|
return job.copy()
|
|
|
except Exception as exc:
|
|
|
if isinstance(exc, UpdateError):
|
|
|
error = str(exc)
|
|
|
else:
|
|
|
error = "更新失败,请查看独立更新服务日志"
|
|
|
if applied is not None and applied.get("updated"):
|
|
|
error += ";程序代码可能已切换,请人工核查并重启或按回滚引用恢复"
|
|
|
safe_to_release = self._abandoned_job_is_safe_to_release(job)
|
|
|
if safe_to_release:
|
|
|
job.update(
|
|
|
status="failed", phase="更新失败", error=error,
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
else:
|
|
|
error += ";代码或数据库状态尚未确认,维护锁已保留,请通过 SSH 人工恢复"
|
|
|
job.update(
|
|
|
status="blocked",
|
|
|
phase="更新中断,需要通过 SSH 人工恢复",
|
|
|
error=error,
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
self._save_job(job)
|
|
|
if isinstance(exc, UpdateError):
|
|
|
raise
|
|
|
raise UpdateError(error, 500) from exc
|
|
|
|
|
|
|
|
|
__all__ = ["BRANCH", "REPOSITORY_URL", "UpdateError", "UpdateStore", "Updater"]
|