You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/virsh_panel/updater.py

879 lines
40 KiB
Python

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Conservative Git updater for Virt Cloud's own, fixed Gitea repository.
The web process may inspect a checkout and enqueue a job. ``run_pending_job``
must run in a separate, privileged one-shot service: the regular web service
has a read-only code directory, and restarting it from within a request is
unreliable. The caller supplies the restart/health-check callback.
The personal access token is stored by the injected encrypted database store.
It never appears in a Git URL, command argument, temporary file, or returned
API data. A private, short-lived askpass program receives it only in the
child's environment while an HTTPS Git operation is in progress.
"""
from __future__ import annotations
import fcntl
import json
import os
import re
import secrets
import stat
import subprocess
import tempfile
import threading
import time
from contextlib import contextmanager
from pathlib import Path
from typing import Callable, Protocol
REPOSITORY_URL = "https://g.luoli.live/root/virt.git"
BRANCH = "venix"
_TOKEN_KEY = "update.gitea_access_token"
_ACTIVE_JOB_KEY = "update.active_job"
_JOB_PREFIX = "update.job."
_HEX_COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z")
_JOB_ID = re.compile(r"[A-Za-z0-9_-]{16,64}\Z")
_ACTIVE_JOB_STATUSES = {"queued", "running", "blocked"}
_QUEUED_JOB_TIMEOUT = 60
_RUNNING_JOB_TIMEOUT = 35 * 60
class UpdateStore(Protocol):
"""Small persistence surface supplied by the encrypted database layer."""
def get_secret(self, name: str) -> str | None: ...
def set_secret(self, name: str, value: str) -> None: ...
def delete_secret(self, name: str) -> None: ...
def get_value(self, name: str) -> str | None: ...
def set_value(self, name: str, value: str) -> None: ...
class UpdateError(Exception):
"""A safe error suitable for returning to the authenticated operator."""
def __init__(self, message: str, status: int = 400) -> None:
super().__init__(message)
self.status = status
class Updater:
"""Read/check/advance only the pinned ``venix`` branch of the panel repo."""
def __init__(
self,
repo_dir: Path,
secret_store: UpdateStore,
*,
git_binary: str = "git",
runner: Callable[..., subprocess.CompletedProcess[str]] | None = None,
worker_active: Callable[[], bool | None] | None = None,
) -> None:
self.repo_dir = Path(repo_dir).absolute()
self.store = secret_store
self.git_binary = git_binary
self._run = runner or subprocess.run
self._worker_active = worker_active
self._lock = threading.RLock()
def settings(self) -> dict:
"""Return only public configuration, never the stored access token."""
with self._lock:
active_id = self.store.get_value(_ACTIVE_JOB_KEY)
last_job = self.get_job(active_id) if active_id else None
active_job = self._active_job_record(active_id)
result = {
"repository": REPOSITORY_URL,
"branch": BRANCH,
"token_configured": bool(self.store.get_secret(_TOKEN_KEY)),
"checkout_ready": False,
"current_commit": None,
"last_job_id": active_id if last_job is not None else None,
"active_job_id": (
active_id if active_job is not None
and active_job.get("status") in _ACTIVE_JOB_STATUSES else None
),
}
try:
result["current_commit"] = self._checkout_head(require_clean=True)
result["checkout_ready"] = True
except UpdateError as exc:
result["checkout_error"] = str(exc)
return result
def active_job(self) -> dict | None:
"""Return the queued, running or recovery-blocked maintenance job."""
with self._lock:
return self._active_job_record(self.store.get_value(_ACTIVE_JOB_KEY))
def _active_job_record(self, job_id: str | None) -> dict | None:
if not job_id:
return None
job = self.get_job(job_id)
if job is None or job.get("status") not in _ACTIVE_JOB_STATUSES:
return None
if job.get("status") == "blocked":
if self._abandoned_job_is_safe_to_release(job):
job.update(
status="failed",
phase="人工恢复后维护锁已解除",
finished_at=int(time.time()),
)
self._save_job(job)
return None
return job
# ``--recover-blocked`` is an explicit SSH process, not the systemd
# one-shot unit. Its durable recovery flag is the maintenance lease;
# never infer that it died merely because the update unit is inactive.
# A killed recovery stays fail-closed and can be resumed under the Git
# flock by running the same SSH command again.
if job.get("status") == "running" and job.get("recovery_requested") is True:
return job
now = int(time.time())
timestamp = job.get("created_at") if job["status"] == "queued" else job.get("started_at")
timeout = _QUEUED_JOB_TIMEOUT if job["status"] == "queued" else _RUNNING_JOB_TIMEOUT
stale = (isinstance(timestamp, int) and not isinstance(timestamp, bool)
and now >= timestamp and now - timestamp > timeout)
try:
worker_state = self._worker_active() if self._worker_active is not None else None
except Exception:
# An unavailable service manager is not proof that a privileged
# updater has stopped. Keep maintenance mode in that case.
worker_state = None
abandoned = (
job["status"] == "running" and worker_state is False
) or (
stale and worker_state is not True
and (self._worker_active is None or worker_state is False)
)
if abandoned:
if job["status"] == "running" and not self._abandoned_job_is_safe_to_release(job):
message = (
"独立更新服务已停止,但代码或数据库状态尚不能确认;"
"维护锁已保留,请通过 SSH 检查更新服务、Git 版本和数据库初始化状态"
)
if job.get("phase") != message or job.get("error") != message:
job.update(phase=message, error=message)
self._save_job(job)
return job
job.update(
status="failed",
phase="更新任务执行器已停止",
error="独立更新服务已停止或未在限定时间内启动;请检查更新服务日志和当前 Git 版本",
finished_at=now,
)
self._save_job(job)
return None
return job
def _abandoned_job_is_safe_to_release(self, job: dict) -> bool:
"""Only release maintenance when code/schema compatibility is known.
An interrupted worker may have stopped the application and switched
source code without completing the schema migration. Automatically
dropping the maintenance gate in that state could let a manually
restarted service write with an incompatible schema.
"""
state = job.get("deployment_state")
if state in {"schema_ready", "services_ready"}:
target = job.get("new_commit")
if not isinstance(target, str) or not _HEX_COMMIT.fullmatch(target):
return False
try:
return self._checkout_head(require_clean=True) == target
except UpdateError:
return False
if state not in {"preparing", "quiescing", "services_stopped", "code_switched"}:
return False
# A clean rollback to the recorded old commit is safe regardless of
# the last observed phase. This is also the SSH recovery exit for a
# ``blocked`` job after a failed migration.
old_commit = job.get("old_commit") or job.get("expected_current_commit")
if not isinstance(old_commit, str) or not _HEX_COMMIT.fullmatch(old_commit):
return False
try:
return self._checkout_head(require_clean=True) == old_commit
except UpdateError:
return False
def set_token(self, token: str) -> None:
if (
not isinstance(token, str)
or not 16 <= len(token) <= 4096
or token != token.strip()
or any(ord(char) < 33 or ord(char) > 126 for char in token)
):
raise UpdateError("Gitea 访问令牌格式无效")
with self._lock:
self._assert_no_running_job_for_token_change()
self.store.set_secret(_TOKEN_KEY, token)
def clear_token(self) -> None:
with self._lock:
self._assert_no_running_job_for_token_change()
self.store.delete_secret(_TOKEN_KEY)
def _assert_no_running_job_for_token_change(self) -> None:
active = self._active_job_record(self.store.get_value(_ACTIVE_JOB_KEY))
if active is not None:
raise UpdateError("更新任务执行期间不能修改访问令牌", 409)
def _environment(self) -> dict[str, str]:
# Do not inherit GIT_TRACE, Git configuration, proxy overrides or
# GIT_SSL_NO_VERIFY from the panel service environment.
return {
"PATH": os.defpath,
"LANG": "C",
"LC_ALL": "C",
"GIT_CONFIG_NOSYSTEM": "1",
"GIT_CONFIG_GLOBAL": os.devnull,
"GIT_TERMINAL_PROMPT": "0",
"GIT_OPTIONAL_LOCKS": "0",
"GIT_ALLOW_PROTOCOL": "https",
}
@contextmanager
def _git_auth(self):
token = self.store.get_secret(_TOKEN_KEY)
if not token:
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
with tempfile.TemporaryDirectory(prefix="virt-cloud-askpass-") as directory:
askpass = Path(directory) / "askpass"
# This script contains no credential. Git passes a prompt as one
# argument; the token is handed to the Git child only in memory.
askpass.write_text(
"#!/bin/sh\n"
"case \"$1\" in\n"
" *sername*|*SERNAM*) printf '%s\\n' \"$VIRT_CLOUD_GIT_USERNAME\" ;;\n"
" *assword*|*ASSWORD*) printf '%s\\n' \"$VIRT_CLOUD_GIT_TOKEN\" ;;\n"
" *) exit 1 ;;\n"
"esac\n",
encoding="ascii",
)
askpass.chmod(0o700)
env = self._environment()
env.update({
"GIT_ASKPASS": str(askpass),
"VIRT_CLOUD_GIT_USERNAME": "root",
"VIRT_CLOUD_GIT_TOKEN": token,
})
yield env
def _git(
self,
*args: str,
timeout: float = 15,
environment: dict[str, str] | None = None,
error: str = "Git 操作失败,请检查面板日志",
allowed_codes: tuple[int, ...] = (0,),
) -> subprocess.CompletedProcess[str]:
argv = [
self.git_binary,
"-c", "core.hooksPath=/dev/null",
"-c", "http.followRedirects=false",
"-c", "http.sslVerify=true",
"-c", "http.extraHeader=",
"-c", "credential.helper=",
"-c", "fetch.recurseSubmodules=false",
*args,
]
try:
result = self._run(
argv,
cwd=self.repo_dir,
env=environment or self._environment(),
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
# Never include stderr, a URL, or a token in a public exception.
raise UpdateError(error, 502) from exc
if result.returncode not in allowed_codes:
raise UpdateError(error, 502)
return result
def _checkout_head(self, *, require_clean: bool) -> str:
if self.repo_dir.is_symlink() or not self.repo_dir.is_dir():
raise UpdateError("面板目录不是有效的 Git 检出目录", 409)
repo_info = self.repo_dir.lstat()
if repo_info.st_uid != os.geteuid() or repo_info.st_mode & 0o022:
raise UpdateError("面板目录必须由当前服务账号拥有,且不能允许组或其他用户写入", 409)
git_dir = self.repo_dir / ".git"
if git_dir.is_symlink() or not git_dir.is_dir():
raise UpdateError("面板必须从完整 Git 仓库部署,不能使用压缩包或工作树", 409)
try:
info = git_dir.lstat()
except OSError as exc:
raise UpdateError("无法读取面板 Git 仓库", 409) from exc
if not stat.S_ISDIR(info.st_mode):
raise UpdateError("面板 Git 仓库格式无效", 409)
if info.st_uid != os.geteuid() or info.st_mode & 0o022:
raise UpdateError("面板 Git 仓库权限不安全,已拒绝在线更新", 409)
top = self._git("rev-parse", "--show-toplevel", error="无法识别面板 Git 仓库").stdout.strip()
if Path(top).resolve() != self.repo_dir.resolve():
raise UpdateError("Git 仓库根目录与面板程序目录不一致", 409)
branch = self._git(
"symbolic-ref", "--quiet", "--short", "HEAD",
error=f"面板 Git 仓库当前不是 {BRANCH} 分支",
).stdout.strip()
if branch != BRANCH:
raise UpdateError(f"面板只允许从 {BRANCH} 分支更新", 409)
configured_urls = self._git(
"config", "--local", "--get-all", "remote.origin.url",
error="面板 Git 仓库缺少 origin 地址",
).stdout.splitlines()
if configured_urls != [REPOSITORY_URL]:
raise UpdateError("Git origin 必须是已指定的 Gitea 私有仓库,且不得包含凭据", 409)
# url.*.insteadOf could silently rewrite our explicit HTTPS URL to a
# different server and disclose the access token. Refuse it outright.
rewrites = self._git(
"config", "--local", "--name-only", "--get-regexp", r"^url\..*\.insteadof$",
allowed_codes=(0, 1),
error="无法检查 Git URL 重写配置",
)
if rewrites.returncode == 0:
raise UpdateError("Git 仓库配置了 URL 重写,无法安全更新", 409)
head = self._git("rev-parse", "HEAD", error="无法读取当前面板版本").stdout.strip()
if not _HEX_COMMIT.fullmatch(head):
raise UpdateError("当前 Git 提交标识无效", 409)
if require_clean:
status = self._git(
"status", "--porcelain=v1", "--untracked-files=all",
error="无法检查面板文件变更",
).stdout
if status.strip():
raise UpdateError("面板目录存在未提交或未跟踪文件;为防止覆盖,请先处理这些文件", 409)
return head
def _remote_head(self) -> str:
with self._git_auth() as env:
output = self._git(
"ls-remote", "--exit-code", "--", REPOSITORY_URL, f"refs/heads/{BRANCH}",
environment=env,
timeout=45,
error="无法读取 Gitea 私有仓库;请检查访问令牌、TLS 证书和网络",
).stdout.strip()
fields = output.split()
if len(fields) != 2 or fields[1] != f"refs/heads/{BRANCH}" or not _HEX_COMMIT.fullmatch(fields[0]):
raise UpdateError("Gitea 返回了无效的分支信息", 502)
return fields[0]
def check(self) -> dict:
"""Inspect the remote head without altering the local checkout."""
with self._lock:
current = self._checkout_head(require_clean=True)
remote = self._remote_head()
return {
"repository": REPOSITORY_URL,
"branch": BRANCH,
"current_commit": current,
"remote_commit": remote,
"status": "current" if current == remote else "update_available",
}
def _fetch_remote(self) -> str:
with self._git_auth() as env:
self._git(
"fetch", "--no-tags", "--no-recurse-submodules", "--",
REPOSITORY_URL, f"refs/heads/{BRANCH}",
environment=env,
timeout=300,
error="无法从 Gitea 拉取更新;请检查访问令牌、TLS 证书和网络",
)
head = self._git("rev-parse", "FETCH_HEAD", error="无法识别已下载的更新版本").stdout.strip()
if not _HEX_COMMIT.fullmatch(head):
raise UpdateError("下载的 Git 提交标识无效", 502)
return head
def _assert_safe_tree(self, target: str) -> None:
protected_paths = (
".gitmodules",
"runtime",
"web/config/config.php",
"web/config/.config.php",
# Ticket images are private runtime data. Keep the path ignored
# and refuse a future target commit that tries to vendor files
# over an existing attachment directory.
"web/storage/ticket-attachments",
"web/vendor",
"engine/.venv",
"engine/master.key",
"master.key",
)
listing = self._git(
"ls-tree", "-r", "--name-only", target, "--", *protected_paths,
error="无法检查更新文件清单",
).stdout.strip()
if listing:
raise UpdateError("更新包含运行数据、私有配置、已安装依赖或子模块配置,已拒绝覆盖", 409)
# ``ls-tree -r`` does not report a gitlink's descendants. Refuse any
# mode 160000 entry independently of its path so a repository update
# cannot introduce executable code through an unreviewed submodule.
modes = self._git(
"ls-tree", "-r", target,
error="无法检查更新子模块清单",
).stdout.splitlines()
if any(line.startswith("160000 ") for line in modes):
raise UpdateError("更新包含 Git 子模块,已拒绝自动更新", 409)
if any(line.startswith("120000 ") for line in modes):
raise UpdateError("更新包含符号链接,已拒绝自动更新", 409)
def _normalize_checkout_permissions(self, target: str) -> None:
"""Make root-updated application code readable by PHP-FPM.
The privileged updater deliberately runs with a restrictive umask so
credentials and lock files stay private. Git also applies that umask
to replacement worktree files, however, which can turn PHP sources
into mode 0600 and take an apache/php-fpm deployment offline. Restore
only paths tracked by the verified target tree; ignored runtime,
vendor and private configuration files are never touched.
"""
listing = self._git(
"ls-tree", "-r", "-z", target,
error="无法读取更新文件权限清单",
).stdout
files: list[tuple[Path, int]] = []
directories: set[Path] = {self.repo_dir}
for record in listing.split("\0"):
if record == "":
continue
metadata, separator, relative = record.partition("\t")
fields = metadata.split()
parts = Path(relative).parts
if (not separator or len(fields) != 3 or fields[0] not in {"100644", "100755"}
or fields[1] != "blob" or not parts or relative.startswith("/")
or any(part in {"", ".", ".."} for part in parts)):
raise UpdateError("更新文件权限清单无效", 500)
local = self.repo_dir.joinpath(*parts)
try:
info = local.lstat()
except OSError as exc:
raise UpdateError("更新后的程序文件不完整", 500) from exc
if not stat.S_ISREG(info.st_mode):
raise UpdateError("更新后的程序文件类型无效", 500)
files.append((local, 0o755 if fields[0] == "100755" else 0o644))
parent = local.parent
while parent != self.repo_dir.parent:
directories.add(parent)
if parent == self.repo_dir:
break
parent = parent.parent
try:
for directory in sorted(directories, key=lambda value: len(value.parts)):
info = directory.lstat()
if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode):
raise UpdateError("更新后的程序目录类型无效", 500)
directory.chmod(0o755)
for local, mode in files:
local.chmod(mode)
except UpdateError:
raise
except OSError as exc:
raise UpdateError("无法恢复更新后程序文件的读取权限", 500) from exc
def _assert_online_compatible(self, current: str, target: str) -> None:
"""Refuse changes that require privileged/manual deployment work.
Python requirements are not hash locked, so installing them as root is
neither reproducible nor safe for a browser-triggered update. Unit
files may contain deployment-specific paths and users and must never be
overwritten by an online update.
"""
changed = self._git(
"diff", "--name-only", f"{current}..{target}", "--",
"engine/requirements.txt", "web/composer.json", "web/composer.lock", "deploy",
error="无法检查更新的部署兼容性",
).stdout.splitlines()
if "engine/requirements.txt" in changed:
raise UpdateError("此版本修改了 Python 依赖,请通过 SSH 按部署文档升级", 409)
if any(path in {"web/composer.json", "web/composer.lock"} for path in changed):
raise UpdateError("此版本修改了 PHP 依赖,请通过 SSH 按部署文档升级", 409)
if any(path == "deploy" or path.startswith("deploy/") for path in changed):
raise UpdateError("此版本修改了 systemd 服务文件,请通过 SSH 按部署文档升级", 409)
@contextmanager
def _apply_lock(self):
git_dir = self.repo_dir / ".git"
flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0)
try:
descriptor = os.open(git_dir / "virt-cloud-update.lock", flags, 0o600)
except OSError as exc:
raise UpdateError("无法获取面板更新锁;请确认独立更新服务有代码目录写权限", 500) from exc
try:
info = os.fstat(descriptor)
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077:
raise UpdateError("面板更新锁文件权限不安全", 500)
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as exc:
os.close(descriptor)
raise UpdateError("另一个更新任务正在运行", 409) from exc
except BaseException:
os.close(descriptor)
raise
try:
yield
finally:
fcntl.flock(descriptor, fcntl.LOCK_UN)
os.close(descriptor)
def apply(
self, *, progress: Callable[[str], None] | None = None,
expected_current_commit: str | None = None,
expected_remote_commit: str | None = None,
before_switch: Callable[[], None] | None = None,
checkpoint: Callable[[str, dict[str, str]], None] | None = None,
) -> dict:
"""Prepare an update, quiesce services, then fast-forward the code."""
report = progress or (lambda _phase: None)
with self._lock:
# Validate before creating a lock inside .git.
current = self._checkout_head(require_clean=True)
with self._apply_lock():
# Re-check after acquiring the cross-process lock.
current = self._checkout_head(require_clean=True)
if expected_current_commit is not None and current != expected_current_commit:
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
report("正在从 Gitea 下载更新")
target = self._fetch_remote()
if expected_remote_commit is not None and target != expected_remote_commit:
raise UpdateError(f"Gitea {BRANCH} 分支已变化,请重新检查更新后再执行", 409)
self._assert_safe_tree(target)
ancestry = self._git(
"merge-base", "--is-ancestor", current, target,
allowed_codes=(0, 1),
error="无法验证更新提交历史",
)
if ancestry.returncode != 0:
raise UpdateError(f"远端 {BRANCH} 分支与当前版本已分叉,已拒绝自动覆盖", 409)
if target == current:
return {"updated": False, "old_commit": current, "new_commit": target}
self._assert_online_compatible(current, target)
# Keep an immutable rollback pointer for manual recovery. A
# forced reset is never part of the automatic update path.
rollback_ref = f"refs/virt-cloud/rollback/{int(time.time())}-{secrets.token_hex(4)}"
self._git(
"update-ref", rollback_ref, current,
error="无法保存更新前版本,已取消切换",
)
# Files may have been edited while the remote update was fetched.
self._checkout_head(require_clean=True)
if before_switch is not None:
report("正在暂停面板服务")
if checkpoint is not None:
checkpoint("quiescing", {
"old_commit": current,
"new_commit": target,
"rollback_ref": rollback_ref,
})
before_switch()
if checkpoint is not None:
checkpoint("services_stopped", {
"old_commit": current,
"new_commit": target,
"rollback_ref": rollback_ref,
})
report("正在切换到新版本")
self._git(
"merge", "--ff-only", "--no-stat", target,
timeout=120,
error="无法快进更新代码;请检查面板目录与 Git 日志",
)
self._normalize_checkout_permissions(target)
installed = self._checkout_head(require_clean=True)
if installed != target:
raise UpdateError("更新后的 Git 提交与预期不符,请人工核查", 500)
if checkpoint is not None:
checkpoint("code_switched", {
"old_commit": current,
"new_commit": target,
"rollback_ref": rollback_ref,
})
return {
"updated": True,
"old_commit": current,
"new_commit": target,
"rollback_ref": rollback_ref,
}
def _job_key(self, job_id: str) -> str:
if not isinstance(job_id, str) or not _JOB_ID.fullmatch(job_id):
raise UpdateError("更新任务编号无效", 404)
return _JOB_PREFIX + job_id
def get_job(self, job_id: str) -> dict | None:
raw = self.store.get_value(self._job_key(job_id))
if raw is None:
return None
try:
job = json.loads(raw)
except (TypeError, ValueError) as exc:
raise UpdateError("更新任务记录损坏", 500) from exc
if not isinstance(job, dict) or job.get("job_id") != job_id:
raise UpdateError("更新任务记录损坏", 500)
return job
def _save_job(self, job: dict) -> None:
self.store.set_value(self._job_key(job["job_id"]), json.dumps(job, ensure_ascii=False))
def create_job(
self, *, expected_current_commit: str | None = None,
expected_remote_commit: str | None = None,
) -> dict:
"""Enqueue one persisted job; the web layer then starts the worker."""
with self._lock:
current = self._checkout_head(require_clean=True)
if expected_current_commit is not None:
if not _HEX_COMMIT.fullmatch(expected_current_commit):
raise UpdateError("要更新的当前版本标识无效", 422)
if current != expected_current_commit:
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
if not self.store.get_secret(_TOKEN_KEY):
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
if expected_remote_commit is not None:
if not _HEX_COMMIT.fullmatch(expected_remote_commit):
raise UpdateError("要更新的远端版本标识无效", 422)
if self._remote_head() != expected_remote_commit:
raise UpdateError(f"Gitea {BRANCH} 分支已变化,请重新检查更新后再执行", 409)
active = self._active_job_record(self.store.get_value(_ACTIVE_JOB_KEY))
if active is not None:
raise UpdateError("已有更新任务正在执行", 409)
job_id = secrets.token_urlsafe(18)
job = {
"job_id": job_id,
"status": "queued",
"phase": "等待独立更新服务启动",
"created_at": int(time.time()),
}
if expected_current_commit is not None:
job["expected_current_commit"] = expected_current_commit
if expected_remote_commit is not None:
job["expected_remote_commit"] = expected_remote_commit
self._save_job(job)
self.store.set_value(_ACTIVE_JOB_KEY, job_id)
return job.copy()
def fail_job(self, job_id: str, message: str) -> dict:
"""Fail a safe job, or retain a maintenance lock if state is uncertain."""
with self._lock:
job = self.get_job(job_id)
if job is None:
raise UpdateError("更新任务不存在", 404)
if job["status"] not in {"queued", "running"}:
return job
safe_to_release = (
job["status"] == "queued" or self._abandoned_job_is_safe_to_release(job)
)
if safe_to_release:
job.update(
status="failed",
phase="更新失败",
error=str(message)[:300],
finished_at=int(time.time()),
)
else:
error = (
f"{str(message)[:180]};代码或数据库状态尚未确认,维护锁已保留。"
"请通过 SSH 核查 Git 版本并完成数据库初始化或回滚后再恢复服务"
)
job.update(
status="blocked",
phase="更新中断,需要通过 SSH 人工恢复",
error=error[:300],
finished_at=int(time.time()),
)
self._save_job(job)
return job.copy()
def begin_blocked_recovery(self) -> dict:
"""Claim a blocked job for an explicit SSH forward recovery.
Recovery never fetches or switches source. It is permitted only when
the checkout is clean and still at the exact target commit recorded by
the interrupted job. The caller must hold ``_apply_lock`` throughout
migration and service health verification.
"""
with self._lock:
job_id = self.store.get_value(_ACTIVE_JOB_KEY)
if not job_id:
raise UpdateError("没有需要人工恢复的在线更新任务", 409)
job = self.get_job(job_id)
resumable = (
isinstance(job, dict)
and job.get("status") == "running"
and job.get("recovery_requested") is True
)
if not isinstance(job, dict) or (job.get("status") != "blocked" and not resumable):
raise UpdateError("当前更新任务不处于可人工恢复状态", 409)
target = job.get("new_commit")
if not isinstance(target, str) or not _HEX_COMMIT.fullmatch(target):
raise UpdateError("更新任务缺少可验证的目标提交", 409)
if self._checkout_head(require_clean=True) != target:
raise UpdateError("人工恢复要求干净工作树保持在任务记录的新提交", 409)
job.pop("error", None)
job.pop("finished_at", None)
job.update(
status="running",
phase="SSH 人工恢复正在准备数据库初始化",
recovery_requested=True,
recovery_started_at=int(time.time()),
)
self._save_job(job)
return job.copy()
def recovery_checkpoint(self, job_id: str, state: str, phase: str) -> dict:
if state not in {"schema_migrating", "schema_ready"}:
raise UpdateError("人工恢复检查点无效", 500)
with self._lock:
if self.store.get_value(_ACTIVE_JOB_KEY) != job_id:
raise UpdateError("人工恢复任务已变化", 409)
job = self.get_job(job_id)
if (job is None or job.get("status") != "running"
or job.get("recovery_requested") is not True):
raise UpdateError("人工恢复任务状态无效", 409)
target = job.get("new_commit")
if not isinstance(target, str) or self._checkout_head(require_clean=True) != target:
raise UpdateError("人工恢复期间 Git 版本发生变化,维护锁继续保留", 409)
job.update(deployment_state=state, phase=str(phase)[:200])
self._save_job(job)
return job.copy()
def complete_blocked_recovery(self, job_id: str) -> dict:
with self._lock:
if self.store.get_value(_ACTIVE_JOB_KEY) != job_id:
raise UpdateError("人工恢复任务已变化", 409)
job = self.get_job(job_id)
if (job is None or job.get("status") != "running"
or job.get("recovery_requested") is not True
or job.get("deployment_state") != "schema_ready"):
raise UpdateError("人工恢复尚未完成数据库与服务检查", 409)
target = job.get("new_commit")
if not isinstance(target, str) or self._checkout_head(require_clean=True) != target:
raise UpdateError("人工恢复期间 Git 版本发生变化,维护锁继续保留", 409)
job.pop("error", None)
job.update(
status="succeeded",
deployment_state="services_ready",
phase="SSH 人工恢复完成",
finished_at=int(time.time()),
)
self._save_job(job)
return job.copy()
def run_pending_job(
self, *, restart: Callable[[Callable[[], None]], None] | None = None,
before_switch: Callable[[], None] | None = None,
) -> dict:
"""Run the queued job in a separate service and persist every phase.
``restart`` should restart the web unit and, ideally, verify its health.
If it fails, the job records that code may already have changed and
exposes the old commit/rollback ref for operator-led recovery.
"""
with self._lock:
job_id = self.store.get_value(_ACTIVE_JOB_KEY)
if not job_id:
raise UpdateError("没有待执行的更新任务", 404)
job = self.get_job(job_id)
if job is None or job.get("status") != "queued":
raise UpdateError("更新任务未处于等待状态;请人工核查后重新提交", 409)
job.update(
status="running",
phase="正在检查本地仓库",
deployment_state="preparing",
started_at=int(time.time()),
)
self._save_job(job)
def progress(phase: str) -> None:
with self._lock:
job["phase"] = str(phase)[:200]
self._save_job(job)
def checkpoint(state: str, details: dict[str, str] | None = None) -> None:
if state not in {
"quiescing", "services_stopped", "code_switched",
"schema_migrating", "schema_ready", "services_ready",
}:
raise UpdateError("更新任务检查点无效", 500)
safe_details = details or {}
if set(safe_details) - {"old_commit", "new_commit", "rollback_ref"}:
raise UpdateError("更新任务检查点内容无效", 500)
for name, value in safe_details.items():
if name in {"old_commit", "new_commit"} and not _HEX_COMMIT.fullmatch(value):
raise UpdateError("更新任务检查点提交标识无效", 500)
if name == "rollback_ref" and not value.startswith("refs/virt-cloud/rollback/"):
raise UpdateError("更新任务检查点回滚引用无效", 500)
with self._lock:
job.update(safe_details)
job["deployment_state"] = state
self._save_job(job)
applied: dict | None = None
try:
applied = self.apply(
progress=progress,
expected_current_commit=job.get("expected_current_commit"),
expected_remote_commit=job.get("expected_remote_commit"),
before_switch=before_switch,
checkpoint=checkpoint,
)
job.update(applied)
self._save_job(job)
if applied["updated"] and restart is not None:
progress("正在重启面板并检查服务")
# Persist this before invoking PHP. MariaDB DDL may commit one
# statement at a time, so a killed migration is not proven
# safe merely by rolling the source checkout back afterward.
checkpoint("schema_migrating")
def mark_schema_ready() -> None:
progress("数据库初始化完成,正在启动面板服务")
checkpoint("schema_ready")
restart(mark_schema_ready)
checkpoint("services_ready")
elif applied["updated"]:
# A worker that cannot restart the web service must not
# report a successful one-click update.
raise UpdateError("代码已更新,但未配置自动重启;请人工重启面板服务", 500)
job.update(status="succeeded", phase="更新完成" if applied["updated"] else "已是最新版本",
finished_at=int(time.time()))
self._save_job(job)
return job.copy()
except Exception as exc:
if isinstance(exc, UpdateError):
error = str(exc)
else:
error = "更新失败,请查看独立更新服务日志"
if applied is not None and applied.get("updated"):
error += ";程序代码可能已切换,请人工核查并重启或按回滚引用恢复"
safe_to_release = self._abandoned_job_is_safe_to_release(job)
if safe_to_release:
job.update(
status="failed", phase="更新失败", error=error,
finished_at=int(time.time()),
)
else:
error += ";代码或数据库状态尚未确认,维护锁已保留,请通过 SSH 人工恢复"
job.update(
status="blocked",
phase="更新中断,需要通过 SSH 人工恢复",
error=error,
finished_at=int(time.time()),
)
self._save_job(job)
if isinstance(exc, UpdateError):
raise
raise UpdateError(error, 500) from exc
__all__ = ["BRANCH", "REPOSITORY_URL", "UpdateError", "UpdateStore", "Updater"]