You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
660 lines
29 KiB
Python
660 lines
29 KiB
Python
"""Offline checks for the private-repository update safety boundary."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from virsh_panel.updater import BRANCH, REPOSITORY_URL, UpdateError, Updater
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self):
|
|
self.secrets: dict[str, str] = {}
|
|
self.values: dict[str, str] = {}
|
|
self.deployment_states: list[str] = []
|
|
|
|
def get_secret(self, name):
|
|
return self.secrets.get(name)
|
|
|
|
def set_secret(self, name, value):
|
|
self.secrets[name] = value
|
|
|
|
def delete_secret(self, name):
|
|
self.secrets.pop(name, None)
|
|
|
|
def get_value(self, name):
|
|
return self.values.get(name)
|
|
|
|
def set_value(self, name, value):
|
|
self.values[name] = value
|
|
if name.startswith("update.job."):
|
|
payload = json.loads(value)
|
|
state = payload.get("deployment_state")
|
|
if state and (not self.deployment_states or self.deployment_states[-1] != state):
|
|
self.deployment_states.append(state)
|
|
|
|
|
|
def git(path: Path, *args: str) -> str:
|
|
result = subprocess.run(
|
|
["git", *args], cwd=path, capture_output=True, text=True, check=True,
|
|
)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def make_repo(tmp_path: Path) -> Path:
|
|
source = tmp_path / "source"
|
|
source.mkdir()
|
|
git(source, "init", "-b", BRANCH)
|
|
git(source, "config", "user.name", "Updater Test")
|
|
git(source, "config", "user.email", "updater@example.test")
|
|
(source / "README.md").write_text("old\n", encoding="utf-8")
|
|
(source / "engine").mkdir()
|
|
(source / "engine" / "requirements.txt").write_text("example-package==1.0\n", encoding="utf-8")
|
|
(source / ".gitignore").write_text(
|
|
"engine/.venv/\nruntime/\nweb/vendor/\nweb/storage/ticket-attachments/\n",
|
|
encoding="utf-8",
|
|
)
|
|
git(source, "add", "README.md", "engine/requirements.txt", ".gitignore")
|
|
git(source, "commit", "-m", "initial")
|
|
checkout = tmp_path / "panel"
|
|
subprocess.run(
|
|
["git", "clone", "--branch", BRANCH, str(source), str(checkout)],
|
|
capture_output=True, text=True, check=True,
|
|
)
|
|
git(checkout, "remote", "set-url", "origin", REPOSITORY_URL)
|
|
return checkout
|
|
|
|
|
|
def advance_source(source: Path, *, filename: str = "README.md") -> str:
|
|
(source / filename).parent.mkdir(parents=True, exist_ok=True)
|
|
(source / filename).write_text("new\n", encoding="utf-8")
|
|
# Protected runtime paths are normally ignored; forcing them lets the
|
|
# updater test prove that a malicious target commit is still rejected.
|
|
git(source, "add", "-f", filename)
|
|
git(source, "commit", "-m", "next")
|
|
return git(source, "rev-parse", "HEAD")
|
|
|
|
|
|
def local_fetch(updater: Updater, source: Path):
|
|
def fetch() -> str:
|
|
git(updater.repo_dir, "fetch", "--no-tags", str(source), f"refs/heads/{BRANCH}")
|
|
return git(updater.repo_dir, "rev-parse", "FETCH_HEAD")
|
|
|
|
updater._fetch_remote = fetch
|
|
|
|
|
|
class UpdaterTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory(prefix="virsh-panel-updater-test-")
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.tmp_path = Path(self.temp.name)
|
|
|
|
def test_settings_and_token_never_reveal_secret(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
store = MemoryStore()
|
|
updater = Updater(checkout, store)
|
|
updater.set_token("a-test-access-token-123")
|
|
settings = updater.settings()
|
|
self.assertEqual(settings["repository"], REPOSITORY_URL)
|
|
self.assertTrue(settings["token_configured"])
|
|
self.assertTrue(settings["checkout_ready"])
|
|
self.assertIsNone(settings["active_job_id"])
|
|
self.assertIsNone(settings["last_job_id"])
|
|
self.assertNotIn("a-test-access-token-123", repr(settings))
|
|
with updater._git_auth() as env:
|
|
self.assertEqual(env["VIRT_CLOUD_GIT_TOKEN"], "a-test-access-token-123")
|
|
self.assertNotIn("a-test-access-token-123", Path(env["GIT_ASKPASS"]).read_text())
|
|
updater.clear_token()
|
|
self.assertFalse(updater.settings()["token_configured"])
|
|
|
|
def test_check_is_read_only_and_reports_new_version(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
updater = Updater(checkout, MemoryStore())
|
|
current = git(checkout, "rev-parse", "HEAD")
|
|
updater._remote_head = lambda: current
|
|
self.assertEqual(updater.check()["status"], "current")
|
|
updater._remote_head = lambda: "a" * 40
|
|
self.assertEqual(updater.check()["status"], "update_available")
|
|
self.assertEqual(git(checkout, "rev-parse", "HEAD"), current)
|
|
|
|
def test_zip_or_dirty_checkout_is_rejected(self):
|
|
plain = self.tmp_path / "plain"
|
|
plain.mkdir()
|
|
with self.assertRaisesRegex(UpdateError, "Git"):
|
|
Updater(plain, MemoryStore()).check()
|
|
|
|
checkout = make_repo(self.tmp_path)
|
|
(checkout / "README.md").write_text("user changes\n", encoding="utf-8")
|
|
updater = Updater(checkout, MemoryStore())
|
|
with self.assertRaisesRegex(UpdateError, "未提交"):
|
|
updater.create_job()
|
|
with self.assertRaisesRegex(UpdateError, "未提交"):
|
|
updater.apply()
|
|
|
|
def test_project_ignores_ticket_attachment_runtime_data(self):
|
|
project_root = Path(__file__).resolve().parents[2]
|
|
self.assertIn(
|
|
"/web/storage/ticket-attachments/",
|
|
(project_root / ".gitignore").read_text(encoding="utf-8"),
|
|
)
|
|
self.assertIn(
|
|
"storage/ticket-attachments/",
|
|
(project_root / "web" / ".gitignore").read_text(encoding="utf-8"),
|
|
)
|
|
|
|
def test_ignored_ticket_attachments_do_not_make_checkout_dirty(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
attachment = checkout / "web" / "storage" / "ticket-attachments" / "ticket-image.png"
|
|
attachment.parent.mkdir(parents=True)
|
|
attachment.write_bytes(b"private runtime image")
|
|
updater = Updater(checkout, MemoryStore())
|
|
updater._remote_head = lambda: git(checkout, "rev-parse", "HEAD")
|
|
|
|
self.assertTrue(updater.settings()["checkout_ready"])
|
|
self.assertEqual(updater.check()["status"], "current")
|
|
self.assertEqual(git(checkout, "status", "--porcelain=v1", "--untracked-files=all"), "")
|
|
|
|
source = self.tmp_path / "source"
|
|
target = advance_source(source)
|
|
local_fetch(updater, source)
|
|
applied = updater.apply()
|
|
self.assertEqual(applied["new_commit"], target)
|
|
self.assertEqual(attachment.read_bytes(), b"private runtime image")
|
|
|
|
def test_target_commit_cannot_vendor_ticket_attachments(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
target = source / "web" / "storage" / "ticket-attachments" / "bundled.png"
|
|
target.parent.mkdir(parents=True)
|
|
target.write_bytes(b"must remain runtime-only")
|
|
git(source, "add", "-f", str(target.relative_to(source)))
|
|
git(source, "commit", "-m", "attempt to vendor ticket image")
|
|
updater = Updater(checkout, MemoryStore())
|
|
local_fetch(updater, source)
|
|
|
|
with self.assertRaisesRegex(UpdateError, "运行数据"):
|
|
updater.apply()
|
|
|
|
def test_fast_forward_job_is_persisted_and_calls_restart(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
old = git(checkout, "rev-parse", "HEAD")
|
|
target = advance_source(source)
|
|
store = MemoryStore()
|
|
updater = Updater(checkout, store)
|
|
updater.set_token("a-test-access-token-123")
|
|
local_fetch(updater, source)
|
|
queued = updater.create_job()
|
|
self.assertEqual(updater.settings()["active_job_id"], queued["job_id"])
|
|
restarted = []
|
|
|
|
def restart(mark_schema_ready):
|
|
restarted.append(True)
|
|
mark_schema_ready()
|
|
|
|
result = updater.run_pending_job(
|
|
before_switch=lambda: None,
|
|
restart=restart,
|
|
)
|
|
self.assertEqual(result["status"], "succeeded")
|
|
self.assertEqual(result["old_commit"], old)
|
|
self.assertEqual(result["new_commit"], target)
|
|
self.assertTrue(result["rollback_ref"].startswith("refs/virt-cloud/rollback/"))
|
|
self.assertEqual(updater.get_job(queued["job_id"]), result)
|
|
self.assertEqual(updater.settings()["last_job_id"], queued["job_id"])
|
|
self.assertIsNone(updater.settings()["active_job_id"])
|
|
self.assertEqual(restarted, [True])
|
|
self.assertEqual(store.deployment_states, [
|
|
"preparing",
|
|
"quiescing",
|
|
"services_stopped",
|
|
"code_switched",
|
|
"schema_migrating",
|
|
"schema_ready",
|
|
"services_ready",
|
|
])
|
|
self.assertEqual(result["deployment_state"], "services_ready")
|
|
self.assertEqual(git(checkout, "rev-parse", "HEAD"), target)
|
|
self.assertEqual(git(checkout, "rev-parse", result["rollback_ref"]), old)
|
|
|
|
def test_apply_restores_tracked_modes_without_exposing_ignored_secrets(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
nested = source / "web" / "app" / "NewFeature.php"
|
|
nested.parent.mkdir(parents=True)
|
|
nested.write_text("<?php\n", encoding="utf-8")
|
|
executable = source / "tools" / "task"
|
|
executable.parent.mkdir()
|
|
executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
|
executable.chmod(0o755)
|
|
git(source, "add", "web/app/NewFeature.php", "tools/task")
|
|
git(source, "commit", "-m", "add application files")
|
|
|
|
private = checkout / "runtime" / "engine" / "master.key"
|
|
private.parent.mkdir(parents=True)
|
|
private.write_text("secret\n", encoding="utf-8")
|
|
private.parent.chmod(0o700)
|
|
private.chmod(0o600)
|
|
|
|
updater = Updater(checkout, MemoryStore())
|
|
local_fetch(updater, source)
|
|
previous_umask = os.umask(0o077)
|
|
try:
|
|
updater.apply()
|
|
finally:
|
|
os.umask(previous_umask)
|
|
|
|
self.assertEqual(stat.S_IMODE((checkout / "web/app/NewFeature.php").stat().st_mode), 0o644)
|
|
self.assertEqual(stat.S_IMODE((checkout / "tools/task").stat().st_mode), 0o755)
|
|
self.assertEqual(stat.S_IMODE((checkout / "web/app").stat().st_mode), 0o755)
|
|
self.assertEqual(stat.S_IMODE(private.stat().st_mode), 0o600)
|
|
self.assertEqual(stat.S_IMODE(private.parent.stat().st_mode), 0o700)
|
|
|
|
def test_noop_apply_restores_tracked_modes_without_quiescing_or_exposing_secrets(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
nested = source / "web" / "app" / "CurrentFeature.php"
|
|
nested.parent.mkdir(parents=True)
|
|
nested.write_text("<?php\n", encoding="utf-8")
|
|
executable = source / "tools" / "task"
|
|
executable.parent.mkdir()
|
|
executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
|
executable.chmod(0o755)
|
|
git(source, "add", "web/app/CurrentFeature.php", "tools/task")
|
|
git(source, "commit", "-m", "add current application files")
|
|
git(checkout, "fetch", str(source), BRANCH)
|
|
git(checkout, "merge", "--ff-only", "FETCH_HEAD")
|
|
current = git(checkout, "rev-parse", "HEAD")
|
|
|
|
local_source = checkout / "web/app/CurrentFeature.php"
|
|
local_executable = checkout / "tools/task"
|
|
local_source.chmod(0o600)
|
|
local_executable.chmod(0o700)
|
|
local_source.parent.chmod(0o700)
|
|
local_executable.parent.chmod(0o700)
|
|
private = checkout / "runtime" / "engine" / "master.key"
|
|
private.parent.mkdir(parents=True)
|
|
private.parent.chmod(0o700)
|
|
private.write_text("secret\n", encoding="utf-8")
|
|
private.chmod(0o600)
|
|
|
|
updater = Updater(checkout, MemoryStore())
|
|
local_fetch(updater, source)
|
|
quiesced = []
|
|
result = updater.apply(before_switch=lambda: quiesced.append(True))
|
|
|
|
self.assertEqual(result, {"updated": False, "old_commit": current, "new_commit": current})
|
|
self.assertEqual(git(checkout, "rev-parse", "HEAD"), current)
|
|
self.assertEqual(git(checkout, "status", "--porcelain"), "")
|
|
self.assertEqual(quiesced, [])
|
|
self.assertEqual(stat.S_IMODE(local_source.stat().st_mode), 0o644)
|
|
self.assertEqual(stat.S_IMODE(local_executable.stat().st_mode), 0o755)
|
|
self.assertEqual(stat.S_IMODE(local_source.parent.stat().st_mode), 0o755)
|
|
self.assertEqual(stat.S_IMODE(local_executable.parent.stat().st_mode), 0o755)
|
|
self.assertEqual(stat.S_IMODE(private.stat().st_mode), 0o600)
|
|
self.assertEqual(stat.S_IMODE(private.parent.stat().st_mode), 0o700)
|
|
self.assertEqual(private.read_text(encoding="utf-8"), "secret\n")
|
|
|
|
def test_target_tree_with_symlink_is_rejected_before_quiesce(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
(source / "linked-config").symlink_to("/etc/passwd")
|
|
git(source, "add", "linked-config")
|
|
git(source, "commit", "-m", "add forbidden symlink")
|
|
updater = Updater(checkout, MemoryStore())
|
|
local_fetch(updater, source)
|
|
quiesced = []
|
|
with self.assertRaisesRegex(UpdateError, "符号链接"):
|
|
updater.apply(before_switch=lambda: quiesced.append(True))
|
|
self.assertEqual(quiesced, [])
|
|
|
|
def test_remote_history_rewrite_never_overwrites_local_code(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
target = advance_source(source)
|
|
git(checkout, "config", "user.name", "Updater Test")
|
|
git(checkout, "config", "user.email", "updater@example.test")
|
|
(checkout / "README.md").write_text("local fork\n", encoding="utf-8")
|
|
git(checkout, "add", "README.md")
|
|
git(checkout, "commit", "-m", "local fork")
|
|
local = git(checkout, "rev-parse", "HEAD")
|
|
updater = Updater(checkout, MemoryStore())
|
|
local_fetch(updater, source)
|
|
with self.assertRaisesRegex(UpdateError, "分叉"):
|
|
updater.apply()
|
|
self.assertEqual(git(checkout, "rev-parse", "HEAD"), local)
|
|
self.assertNotEqual(local, target)
|
|
|
|
def test_dependency_and_deployment_changes_require_manual_upgrade_before_quiesce(self):
|
|
for filename, message in (
|
|
("engine/requirements.txt", "Python 依赖"),
|
|
("web/composer.json", "PHP 依赖"),
|
|
("web/composer.lock", "PHP 依赖"),
|
|
("deploy/virt-cloud-engine.service", "systemd 服务"),
|
|
):
|
|
with self.subTest(filename=filename):
|
|
case = self.tmp_path / filename.replace("/", "-")
|
|
case.mkdir()
|
|
checkout = make_repo(case)
|
|
source = case / "source"
|
|
advance_source(source, filename=filename)
|
|
updater = Updater(checkout, MemoryStore())
|
|
local_fetch(updater, source)
|
|
old = git(checkout, "rev-parse", "HEAD")
|
|
quiesced = []
|
|
with self.assertRaisesRegex(UpdateError, message):
|
|
updater.apply(before_switch=lambda: quiesced.append(True))
|
|
self.assertEqual(quiesced, [])
|
|
self.assertEqual(git(checkout, "rev-parse", "HEAD"), old)
|
|
|
|
def test_stale_unclaimed_job_is_failed_and_no_longer_blocks(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
store = MemoryStore()
|
|
job_id = "stale-job-1234567890"
|
|
store.set_value("update.active_job", job_id)
|
|
store.set_value("update.job." + job_id, json.dumps({
|
|
"job_id": job_id,
|
|
"status": "queued",
|
|
"phase": "等待独立更新服务启动",
|
|
"created_at": int(time.time()) - 120,
|
|
}))
|
|
updater = Updater(checkout, store)
|
|
self.assertIsNone(updater.active_job())
|
|
self.assertEqual(updater.get_job(job_id)["status"], "failed")
|
|
self.assertEqual(updater.settings()["last_job_id"], job_id)
|
|
|
|
def test_running_job_stays_active_while_worker_is_running(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
store = MemoryStore()
|
|
job_id = "running-job-123456789"
|
|
current = git(checkout, "rev-parse", "HEAD")
|
|
store.set_value("update.active_job", job_id)
|
|
store.set_value("update.job." + job_id, json.dumps({
|
|
"job_id": job_id,
|
|
"status": "running",
|
|
"phase": "正在准备",
|
|
"deployment_state": "preparing",
|
|
"expected_current_commit": current,
|
|
"created_at": int(time.time()) - 5000,
|
|
"started_at": int(time.time()) - 5000,
|
|
}))
|
|
running = Updater(checkout, store, worker_active=lambda: True)
|
|
self.assertIsNotNone(running.active_job())
|
|
self.assertEqual(running.get_job(job_id)["status"], "running")
|
|
|
|
def test_inactive_worker_keeps_maintenance_lock_for_ambiguous_deployment(self):
|
|
for index, state in enumerate((None, "unknown", "code_switched")):
|
|
with self.subTest(state=state):
|
|
case = self.tmp_path / f"ambiguous-{index}-{state or 'missing'}"
|
|
case.mkdir()
|
|
checkout = make_repo(case)
|
|
store = MemoryStore()
|
|
job_id = "running-job-123456789"
|
|
job = {
|
|
"job_id": job_id,
|
|
"status": "running",
|
|
"phase": "正在更新",
|
|
"started_at": int(time.time()) - 5000,
|
|
}
|
|
if state is not None:
|
|
job["deployment_state"] = state
|
|
if state == "unknown":
|
|
job["old_commit"] = git(checkout, "rev-parse", "HEAD")
|
|
store.set_value("update.active_job", job_id)
|
|
store.set_value("update.job." + job_id, json.dumps(job))
|
|
stopped = Updater(checkout, store, worker_active=lambda: False)
|
|
active = stopped.active_job()
|
|
self.assertIsNotNone(active)
|
|
self.assertEqual(active["status"], "running")
|
|
self.assertIn("维护锁已保留", active["phase"])
|
|
self.assertIn("维护锁已保留", active["error"])
|
|
|
|
def test_inactive_worker_releases_only_known_safe_deployment_states(self):
|
|
for state in (
|
|
"preparing", "quiescing", "services_stopped", "code_switched",
|
|
"schema_ready", "services_ready",
|
|
):
|
|
with self.subTest(state=state):
|
|
case = self.tmp_path / state
|
|
case.mkdir()
|
|
checkout = make_repo(case)
|
|
store = MemoryStore()
|
|
job_id = "running-job-123456789"
|
|
current = git(checkout, "rev-parse", "HEAD")
|
|
store.set_value("update.active_job", job_id)
|
|
job = {
|
|
"job_id": job_id,
|
|
"status": "running",
|
|
"phase": "正在更新",
|
|
"deployment_state": state,
|
|
"old_commit": current,
|
|
"started_at": int(time.time()) - 5000,
|
|
}
|
|
if state in {"schema_ready", "services_ready"}:
|
|
job["new_commit"] = current
|
|
store.set_value("update.job." + job_id, json.dumps(job))
|
|
stopped = Updater(checkout, store, worker_active=lambda: False)
|
|
self.assertIsNone(stopped.active_job())
|
|
self.assertEqual(stopped.get_job(job_id)["status"], "failed")
|
|
|
|
def test_inactive_worker_keeps_lock_when_early_state_checkout_is_not_clean_old_commit(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
store = MemoryStore()
|
|
job_id = "running-job-123456789"
|
|
current = git(checkout, "rev-parse", "HEAD")
|
|
store.set_value("update.active_job", job_id)
|
|
store.set_value("update.job." + job_id, json.dumps({
|
|
"job_id": job_id,
|
|
"status": "running",
|
|
"phase": "正在暂停服务",
|
|
"deployment_state": "services_stopped",
|
|
"old_commit": current,
|
|
"started_at": int(time.time()) - 5000,
|
|
}))
|
|
(checkout / "README.md").write_text("dirty\n", encoding="utf-8")
|
|
stopped = Updater(checkout, store, worker_active=lambda: False)
|
|
active = stopped.active_job()
|
|
self.assertIsNotNone(active)
|
|
self.assertEqual(active["status"], "running")
|
|
self.assertIn("维护锁已保留", active["phase"])
|
|
|
|
def test_inactive_worker_keeps_lock_during_schema_migration_after_source_rollback(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
store = MemoryStore()
|
|
job_id = "running-job-123456789"
|
|
current = git(checkout, "rev-parse", "HEAD")
|
|
store.set_value("update.active_job", job_id)
|
|
store.set_value("update.job." + job_id, json.dumps({
|
|
"job_id": job_id,
|
|
"status": "running",
|
|
"phase": "正在执行数据库迁移",
|
|
"deployment_state": "schema_migrating",
|
|
"old_commit": current,
|
|
"started_at": int(time.time()) - 5000,
|
|
}))
|
|
stopped = Updater(checkout, store, worker_active=lambda: False)
|
|
active = stopped.active_job()
|
|
self.assertIsNotNone(active)
|
|
self.assertEqual(active["status"], "running")
|
|
self.assertIn("维护锁已保留", active["phase"])
|
|
|
|
def test_git_url_rewrite_is_rejected_before_sending_token(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
git(checkout, "config", "url.https://evil.example/.insteadOf", "https://g.luoli.live/")
|
|
updater = Updater(checkout, MemoryStore())
|
|
updater.set_token("a-test-access-token-123")
|
|
with self.assertRaisesRegex(UpdateError, "URL 重写"):
|
|
updater.check()
|
|
|
|
def test_new_code_cannot_track_private_data_or_submodules(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
target = advance_source(source, filename="runtime/engine/master.key")
|
|
updater = Updater(checkout, MemoryStore())
|
|
local_fetch(updater, source)
|
|
old = git(checkout, "rev-parse", "HEAD")
|
|
with self.assertRaisesRegex(UpdateError, "运行数据"):
|
|
updater.apply()
|
|
self.assertEqual(git(checkout, "rev-parse", "HEAD"), old)
|
|
self.assertNotEqual(old, target)
|
|
|
|
def test_failed_restart_is_persisted_without_hiding_applied_commit(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
target = advance_source(source)
|
|
updater = Updater(checkout, MemoryStore())
|
|
updater.set_token("a-test-access-token-123")
|
|
local_fetch(updater, source)
|
|
job = updater.create_job()
|
|
|
|
def fail_restart(_mark_schema_ready):
|
|
raise UpdateError("重启失败", 500)
|
|
|
|
with self.assertRaisesRegex(UpdateError, "重启失败"):
|
|
updater.run_pending_job(restart=fail_restart)
|
|
stored = updater.get_job(job["job_id"])
|
|
self.assertEqual(stored["status"], "blocked")
|
|
self.assertEqual(stored["deployment_state"], "schema_migrating")
|
|
self.assertEqual(stored["new_commit"], target)
|
|
self.assertTrue(stored["rollback_ref"])
|
|
self.assertIn("代码可能已切换", stored["error"])
|
|
self.assertIn("维护锁已保留", stored["error"])
|
|
self.assertEqual(updater.settings()["active_job_id"], job["job_id"])
|
|
|
|
# Once migration has started, a source rollback cannot prove that
|
|
# partially committed DDL is compatible with the old application.
|
|
git(checkout, "reset", "--hard", stored["old_commit"])
|
|
active = updater.active_job()
|
|
self.assertIsNotNone(active)
|
|
self.assertEqual(active["status"], "blocked")
|
|
self.assertEqual(updater.settings()["active_job_id"], job["job_id"])
|
|
|
|
def test_explicit_forward_recovery_clears_old_error_only_after_safe_completion(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
old = git(checkout, "rev-parse", "HEAD")
|
|
target = advance_source(source)
|
|
store = MemoryStore()
|
|
updater = Updater(checkout, store)
|
|
local_fetch(updater, source)
|
|
applied = updater.apply()
|
|
self.assertEqual(applied["new_commit"], target)
|
|
|
|
job_id = "blocked-job-123456789"
|
|
store.set_value("update.active_job", job_id)
|
|
store.set_value("update.job." + job_id, json.dumps({
|
|
"job_id": job_id,
|
|
"status": "blocked",
|
|
"phase": "更新中断",
|
|
"deployment_state": "schema_migrating",
|
|
"old_commit": old,
|
|
"new_commit": target,
|
|
"rollback_ref": applied["rollback_ref"],
|
|
"error": "旧失败原因",
|
|
"finished_at": 123,
|
|
}))
|
|
|
|
claimed = updater.begin_blocked_recovery()
|
|
self.assertEqual(claimed["status"], "running")
|
|
self.assertTrue(claimed["recovery_requested"])
|
|
self.assertNotIn("error", claimed)
|
|
self.assertNotIn("finished_at", claimed)
|
|
migrating = updater.recovery_checkpoint(
|
|
job_id, "schema_migrating", "正在重新执行数据库初始化")
|
|
self.assertEqual(migrating["deployment_state"], "schema_migrating")
|
|
ready = updater.recovery_checkpoint(
|
|
job_id, "schema_ready", "数据库初始化完成")
|
|
self.assertEqual(ready["deployment_state"], "schema_ready")
|
|
# SSH recovery is not the systemd update unit. Starting the new engine
|
|
# for its health check must not let that engine release maintenance
|
|
# before the recovery CLI records successful completion.
|
|
observing_engine = Updater(checkout, store, worker_active=lambda: False)
|
|
observed = observing_engine.active_job()
|
|
self.assertIsNotNone(observed)
|
|
self.assertEqual(observed["status"], "running")
|
|
self.assertTrue(observed["recovery_requested"])
|
|
completed = updater.complete_blocked_recovery(job_id)
|
|
self.assertEqual(completed["status"], "succeeded")
|
|
self.assertEqual(completed["deployment_state"], "services_ready")
|
|
self.assertEqual(completed["phase"], "SSH 人工恢复完成")
|
|
self.assertNotIn("error", completed)
|
|
self.assertIsNone(updater.active_job())
|
|
|
|
def test_forward_recovery_rejects_old_checkout_and_failed_migration_keeps_lock(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
old = git(checkout, "rev-parse", "HEAD")
|
|
target = advance_source(source)
|
|
store = MemoryStore()
|
|
updater = Updater(checkout, store)
|
|
local_fetch(updater, source)
|
|
applied = updater.apply()
|
|
job_id = "blocked-job-123456789"
|
|
store.set_value("update.active_job", job_id)
|
|
store.set_value("update.job." + job_id, json.dumps({
|
|
"job_id": job_id,
|
|
"status": "blocked",
|
|
"phase": "更新中断",
|
|
"deployment_state": "schema_migrating",
|
|
"old_commit": old,
|
|
"new_commit": target,
|
|
"rollback_ref": applied["rollback_ref"],
|
|
"error": "迁移中断",
|
|
}))
|
|
|
|
git(checkout, "reset", "--hard", old)
|
|
with self.assertRaisesRegex(UpdateError, "新提交"):
|
|
updater.begin_blocked_recovery()
|
|
self.assertEqual(updater.get_job(job_id)["status"], "blocked")
|
|
|
|
git(checkout, "reset", "--hard", target)
|
|
updater.begin_blocked_recovery()
|
|
updater.recovery_checkpoint(job_id, "schema_migrating", "正在迁移")
|
|
failed = updater.fail_job(job_id, "人工恢复失败")
|
|
self.assertEqual(failed["status"], "blocked")
|
|
self.assertEqual(failed["deployment_state"], "schema_migrating")
|
|
self.assertIn("维护锁已保留", failed["error"])
|
|
self.assertIsNotNone(updater.active_job())
|
|
|
|
def test_checked_remote_commit_is_pinned_through_worker_execution(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
source = self.tmp_path / "source"
|
|
old = git(checkout, "rev-parse", "HEAD")
|
|
checked = advance_source(source)
|
|
store = MemoryStore()
|
|
updater = Updater(checkout, store)
|
|
updater.set_token("a-test-access-token-123")
|
|
updater._remote_head = lambda: checked
|
|
local_fetch(updater, source)
|
|
queued = updater.create_job(
|
|
expected_current_commit=old,
|
|
expected_remote_commit=checked,
|
|
)
|
|
advance_source(source, filename="VERSION")
|
|
with self.assertRaisesRegex(UpdateError, f"{BRANCH} 分支已变化"):
|
|
updater.run_pending_job(restart=lambda mark_schema_ready: mark_schema_ready())
|
|
self.assertEqual(git(checkout, "rev-parse", "HEAD"), old)
|
|
self.assertEqual(updater.get_job(queued["job_id"])["status"], "failed")
|
|
|
|
def test_create_job_rejects_changed_remote_since_check(self):
|
|
checkout = make_repo(self.tmp_path)
|
|
updater = Updater(checkout, MemoryStore())
|
|
updater.set_token("a-test-access-token-123")
|
|
updater._remote_head = lambda: "b" * 40
|
|
with self.assertRaisesRegex(UpdateError, f"{BRANCH} 分支已变化"):
|
|
updater.create_job(
|
|
expected_current_commit=git(checkout, "rev-parse", "HEAD"),
|
|
expected_remote_commit="a" * 40,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|