救援操作会中断服务。面板先将救援 ISO 预挂载为下一次完整开机的启动介质,再请求正常关机并开机。若正常关机失败或超时,只有救援页显示“已预挂载”时,才能确认风险后点击“强制重启进入救援”:这会强制断电并冷启动,可能丢失未保存数据。也可返回详情页,手动“强制断电”后再“开机”。若预挂载未成功,请先核查错误,切勿直接强制重启。普通“强制重置”不保证读取新保存的启动配置。退出救援会卸载 ISO 并恢复原系统启动;若退出时关机超时,可通过救援系统的 VNC 控制台手动关机,待显示“已关机”后重试退出。面板不会自动强制断电,也不会删除或格式化原系统盘。
+
救援系统的登录方式和密码由该 ISO 自身决定;面板不会注入原虚拟机的 root 密码或 SSH 公钥。通常请先通过 VNC 控制台操作。
+
+
+
+
ACCOUNT SSH KEYS
公钥管理
保存自己的 SSH 公钥,创建云镜像虚拟机时直接选择。每个用户只能管理自己的公钥。
@@ -387,6 +405,19 @@
+
+
RESCUE IMAGE CATALOG
救援模版
仅管理员上传救援 ISO;虚拟机所有者可在详情页选择已导入的镜像。
+
+
+
上传救援 ISO
文件将上传到面板并导入虚拟化宿主机;上传期间请保持页面打开。
+
+
+
✚
已上传模版
删除模版前,请确认没有虚拟机正在使用。
正在加载…
+
+
PANEL UPDATE
网站更新
从当前项目的私有 Gitea 仓库检查并安装新版本。
diff --git a/static/styles.css b/static/styles.css
index 48143070..6ebfec21 100644
--- a/static/styles.css
+++ b/static/styles.css
@@ -175,3 +175,18 @@ tbody tr:hover{background:#fcfefe}
@media(max-width:510px){.register-code-row{flex-wrap:wrap}.register-code-row input{flex-basis:100%}.register-code-row .button{width:100%}.pending-user-row{align-items:flex-start;flex-direction:column}.pending-user-row .button{width:100%}.smtp-card .form-grid{grid-template-columns:1fr}}
@media(max-width:510px){.ssh-key-row{align-items:flex-start;flex-direction:column}.ssh-key-actions{width:100%}.ssh-key-actions .button{flex:1}}
@media(max-width:510px){.vm-detail-heading .heading-actions{justify-content:space-between}.vm-detail-heading .heading-actions .button{flex:0 0 auto}.vm-detail-facts{grid-template-columns:1fr}.vm-detail-facts>div{padding:12px 0}.vm-detail-actions .row-button{flex:1 1 135px}.vm-table td{padding:10px 11px}.vm-identity{min-width:0}.vm-name-link{max-width:calc(100vw - 130px)}}
+.rescue-status{margin:8px 0 17px;padding:12px 14px;border:1px solid #c8e9e8;border-radius:9px;background:#f0fbfa;color:#236d71;font-size:12px;line-height:1.6}
+.rescue-login-note{margin:0 0 19px;color:#557085;font-size:12px;line-height:1.7}
+#rescue-status-title{margin:0;color:#2c465a;font-size:16px}
+.rescue-template-cards{display:grid;grid-template-columns:repeat(auto-fill,minmax(180px,1fr));gap:12px;margin:0 0 21px}
+.rescue-template-card{display:flex;flex-direction:column;align-items:center;justify-content:center;gap:8px;min-height:130px;padding:16px 13px;border:1px solid #dce7ec;border-radius:11px;background:#fff;color:#314b5c;text-align:center;cursor:pointer;overflow-wrap:anywhere}
+.rescue-template-card:hover,.rescue-template-card:focus-visible{border-color:#38aeb7;outline:none;box-shadow:0 0 0 3px rgba(25,179,187,.1)}
+.rescue-template-card[aria-pressed="true"]{border-color:#28a8b0;background:#eaf8f8;box-shadow:0 0 0 2px rgba(25,179,187,.12)}
+.rescue-template-card:disabled{cursor:wait;opacity:.7}
+.rescue-template-symbol{display:grid;place-items:center;width:43px;height:43px;border-radius:10px;background:#e2f6f5;color:#168d94;font-size:24px;font-weight:800}
+.rescue-template-card strong{font-size:12px}.rescue-template-card small{font-size:11px;color:#8798a5}
+.rescue-actions{display:flex;flex-wrap:wrap;gap:10px;margin-top:12px}.rescue-actions .button{min-width:190px}
+.rescue-template-row{display:flex;align-items:center;justify-content:space-between;gap:12px;padding:13px 14px;border:1px solid #e0e9ee;border-radius:9px;background:#fbfdfd}
+.rescue-template-row-label{display:grid;gap:5px;min-width:0;overflow-wrap:anywhere}.rescue-template-row-label strong{font-size:12px;color:#34495c}.rescue-template-row-label small{font-size:11px;color:#8a9caa}
+#rescue-template-file{height:auto;padding:10px 12px}
+@media(max-width:510px){.rescue-template-cards{grid-template-columns:repeat(2,minmax(0,1fr))}.rescue-actions .button{width:100%}.rescue-template-row{align-items:flex-start;flex-direction:column}}
diff --git a/tests/test_app.py b/tests/test_app.py
index f3ec13df..5c227f26 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -82,6 +82,8 @@ class FakeVMService:
self.list_error = False
self.reinstall_gate: threading.Event | None = None
self.reinstall_started = threading.Event()
+ self.rescue_enter_error: VMError | None = None
+ self.rescue_pending_manual_restart = False
self.vm = {
"uuid": VM_ID, "name": "alma-test", "state": "running", "vcpu": 2,
"memory_mib": 2048, "disks": [], "autostart": False,
@@ -146,8 +148,60 @@ class FakeVMService:
return {"volume_name": volume_name, "virtual_size": virtual_size,
"image_bytes": image_path.stat().st_size}
+ def import_rescue_iso(self, image_path, volume_name):
+ self.calls.append(("import_rescue_iso", volume_name, image_path.stat().st_size))
+ return {"volume_name": volume_name, "image_bytes": image_path.stat().st_size}
+
+ def delete_rescue_iso(self, volume_name):
+ self.calls.append(("delete_rescue_iso", volume_name))
+ return {"volume_name": volume_name, "deleted": True}
+
+ def prepare_rescue_vm(self, vm_id, volume_name):
+ self.calls.append(("prepare_rescue_vm", vm_id, volume_name))
+ if self.vm.get("rescue_mode") or self.vm.get("rescue_staged"):
+ raise VMError("已经处于救援模式", 409)
+ return f'{self.vm["name"]}'
+
+ def enter_rescue_vm(self, vm_id, volume_name, original_xml):
+ self.calls.append(("enter_rescue_vm", vm_id, volume_name, original_xml))
+ if self.rescue_enter_error is not None:
+ raise self.rescue_enter_error
+ if self.rescue_pending_manual_restart:
+ self.vm["rescue_staged"] = True
+ self.vm["rescue_mode"] = False
+ return {"vm": self.vm, "pending_manual_restart": True,
+ "warning": "救援 ISO 已配置为下次冷启动;请手动强制断电后开机"}
+ self.vm["rescue_mode"] = True
+ self.vm["rescue_staged"] = False
+ self.vm["rescue_volume_name"] = volume_name
+ return {"vm": self.vm}
+
+ def exit_rescue_vm(self, vm_id, original_xml):
+ self.calls.append(("exit_rescue_vm", vm_id, original_xml))
+ self.vm["rescue_mode"] = False
+ self.vm["rescue_staged"] = False
+ self.vm["rescue_volume_name"] = None
+ self.vm["state"] = "running"
+ return {"vm": self.vm}
+
+ def force_start_staged_rescue_vm(self, vm_id, original_xml):
+ self.calls.append(("force_start_staged_rescue_vm", vm_id, original_xml))
+ if not self.vm.get("rescue_staged"):
+ raise VMError("救援 ISO 尚未预挂载", 409)
+ self.vm["state"] = "running"
+ self.vm["rescue_staged"] = False
+ self.vm["rescue_mode"] = True
+ return {"vm": self.vm}
+
def action(self, vm_id, action):
self.calls.append(("action", vm_id, action))
+ if self.vm.get("rescue_staged") and action == "poweroff":
+ self.vm["state"] = "shutoff"
+ self.vm["rescue_mode"] = True
+ elif self.vm.get("rescue_staged") and action == "start":
+ self.vm["state"] = "running"
+ self.vm["rescue_mode"] = True
+ self.vm["rescue_staged"] = False
return self.vm
def delete_vm(self, vm_id, delete_disks):
@@ -669,7 +723,8 @@ def test_vm_crud_actions_and_error_mapping(web):
assert deleted.status_code == 200 and deleted.json()["deleted"] is True
assert service.calls == [
("resources",), ("list_vms",), ("get_vm", VM_ID), ("get_vm", "missing"),
- ("create_vm", CREATE_BODY), ("action", VM_ID, "reboot"),
+ ("create_vm", CREATE_BODY), ("get_vm", VM_ID),
+ ("action", VM_ID, "reboot"), ("get_vm", VM_ID),
("delete_vm", VM_ID, True),
]
assert all("command=/fake/ssh-wrapper" in p["libvirt_uri"] for p in seen)
@@ -689,6 +744,250 @@ def test_vm_create_uses_vcpu_without_a_separate_nic_queue_field(web):
headers=headers).status_code == 422
+def test_rescue_iso_upload_enter_exit_and_safe_lifecycle(web):
+ client, _state, token, service, _manager, _seen = web
+ csrf = setup_admin(client, token)
+ configure_host(client, csrf)
+ headers = {"X-CSRF-Token": csrf, "Content-Type": "application/octet-stream"}
+ json_headers = {"X-CSRF-Token": csrf}
+ assert client.get("/api/v1/rescue-templates").json() == {"templates": []}
+
+ iso = bytearray(1024 * 1024)
+ for sector, kind in ((16, 1), (17, 0), (18, 255)):
+ offset = sector * 2048
+ iso[offset] = kind
+ iso[offset + 1:offset + 6] = b"CD001"
+ iso[offset + 6] = 1
+ iso[17 * 2048 + 7:17 * 2048 + 30] = b"EL TORITO SPECIFICATION"
+ iso[17 * 2048 + 71:17 * 2048 + 75] = (20).to_bytes(4, "little")
+ assert client.post("/api/v1/rescue-templates?name=Rescue", content=iso).status_code == 403
+ invalid = client.post("/api/v1/rescue-templates?name=Invalid",
+ content=b"invalid", headers=headers)
+ assert invalid.status_code in (413, 422)
+ queued = client.post("/api/v1/rescue-templates?name=Rescue",
+ content=bytes(iso), headers=headers)
+ assert queued.status_code == 202, queued.text
+ import_job_id = queued.json()["job_id"]
+ for _ in range(100):
+ job = client.get(f"/api/v1/rescue-templates/import-jobs/{import_job_id}").json()
+ if job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert job["status"] == "completed", job
+ template = client.get("/api/v1/rescue-templates").json()["templates"][0]
+ assert template["name"] == "Rescue"
+ assert ("import_rescue_iso", template["volume_name"], len(iso)) in service.calls
+
+ rescue_url = f"/api/v1/vms/{VM_ID}/rescue"
+ wrong = client.post(rescue_url, json={"template_id": template["id"],
+ "confirmation_name": "wrong"}, headers=json_headers)
+ assert wrong.status_code == 422
+ started = client.post(rescue_url, json={"template_id": template["id"],
+ "confirmation_name": "alma-test"}, headers=json_headers)
+ assert started.status_code == 202, started.text
+ for _ in range(100):
+ rescue_job = client.get(
+ f"/api/v1/rescue-jobs/{started.json()['job_id']}").json()
+ if rescue_job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert rescue_job["status"] == "completed", rescue_job
+ assert client.get(rescue_url).json()["active"] is True
+ assert client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "reboot"},
+ headers=json_headers).status_code == 409
+ assert client.delete(f"/api/v1/rescue-templates/{template['id']}",
+ headers=headers).status_code == 409
+
+ exited = client.post(f"{rescue_url}/exit", json={"confirmation_name": "alma-test"},
+ headers=json_headers)
+ assert exited.status_code == 202, exited.text
+ for _ in range(100):
+ exit_job = client.get(f"/api/v1/rescue-jobs/{exited.json()['job_id']}").json()
+ if exit_job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert exit_job["status"] == "completed", exit_job
+ assert client.get(rescue_url).json()["active"] is False
+
+ # If a worker dies after saving its recovery record but the original
+ # persistent definition is still present, reading status safely clears
+ # the stale record and allows ordinary VM operations again.
+ restarted = client.post(rescue_url, json={"template_id": template["id"],
+ "confirmation_name": "alma-test"},
+ headers=json_headers)
+ assert restarted.status_code == 202, restarted.text
+ for _ in range(100):
+ rescue_job = client.get(
+ f"/api/v1/rescue-jobs/{restarted.json()['job_id']}").json()
+ if rescue_job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert rescue_job["status"] == "completed", rescue_job
+ service.vm["rescue_mode"] = False
+ service.vm["rescue_volume_name"] = None
+ assert client.get(rescue_url).json()["status"] is None
+ assert client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "reboot"},
+ headers=json_headers).status_code == 200
+
+ # A staged rescue definition survives a graceful-shutdown timeout. The
+ # owner can power off and cold-start it, while reset/reboot stay blocked.
+ service.rescue_pending_manual_restart = True
+ staged_request = client.post(rescue_url, json={"template_id": template["id"],
+ "confirmation_name": "alma-test"},
+ headers=json_headers)
+ assert staged_request.status_code == 202, staged_request.text
+ for _ in range(100):
+ staged_job = client.get(
+ f"/api/v1/rescue-jobs/{staged_request.json()['job_id']}").json()
+ if staged_job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert staged_job["status"] == "completed", staged_job
+ assert staged_job["pending_manual_restart"] is True
+ staged_state = client.get(rescue_url).json()
+ assert staged_state["staged"] is True and staged_state["active"] is False
+ assert staged_state["can_exit"] is True
+ assert client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "reset"},
+ headers=json_headers).status_code == 409
+ assert client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"},
+ headers=json_headers).status_code == 200
+ assert client.get(rescue_url).json()["staged"] is True
+ assert client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "start"},
+ headers=json_headers).status_code == 200
+ assert client.get(rescue_url).json()["active"] is True
+ assert client.get(rescue_url).json()["staged"] is False
+ restored = client.post(f"{rescue_url}/exit",
+ json={"confirmation_name": "alma-test"}, headers=json_headers)
+ assert restored.status_code == 202, restored.text
+ for _ in range(100):
+ restored_job = client.get(
+ f"/api/v1/rescue-jobs/{restored.json()['job_id']}").json()
+ if restored_job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert restored_job["status"] == "completed", restored_job
+
+ staged_request = client.post(rescue_url, json={"template_id": template["id"],
+ "confirmation_name": "alma-test"},
+ headers=json_headers)
+ assert staged_request.status_code == 202, staged_request.text
+ for _ in range(100):
+ staged_job = client.get(
+ f"/api/v1/rescue-jobs/{staged_request.json()['job_id']}").json()
+ if staged_job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert staged_job["status"] == "completed", staged_job
+ force_url = f"{rescue_url}/force-start"
+ assert client.post(force_url, json={"confirmation_name": "alma-test"},
+ headers=json_headers).status_code == 422
+ assert client.post(force_url,
+ json={"confirmation_name": "wrong", "ack_data_loss": True},
+ headers=json_headers).status_code == 422
+ forced = client.post(force_url,
+ json={"confirmation_name": "alma-test", "ack_data_loss": True},
+ headers=json_headers)
+ assert forced.status_code == 202, forced.text
+ for _ in range(100):
+ force_job = client.get(
+ f"/api/v1/rescue-jobs/{forced.json()['job_id']}").json()
+ if force_job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert force_job["status"] == "completed", force_job
+ assert force_job["operation"] == "force"
+ assert client.get(rescue_url).json()["active"] is True
+ assert client.post(force_url,
+ json={"confirmation_name": "alma-test", "ack_data_loss": True},
+ headers=json_headers).status_code == 409
+ restored = client.post(f"{rescue_url}/exit",
+ json={"confirmation_name": "alma-test"}, headers=json_headers)
+ assert restored.status_code == 202, restored.text
+ for _ in range(100):
+ restored_job = client.get(
+ f"/api/v1/rescue-jobs/{restored.json()['job_id']}").json()
+ if restored_job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert restored_job["status"] == "completed", restored_job
+ assert client.delete(f"/api/v1/rescue-templates/{template['id']}",
+ headers=headers).json() == {"deleted": True}
+
+
+def test_orphan_rescue_marker_blocks_vm_actions_even_without_database_record(web):
+ client, _state, token, service, _manager, _seen = web
+ csrf = setup_admin(client, token)
+ configure_host(client, csrf)
+ service.vm["rescue_mode"] = True
+ headers = {"X-CSRF-Token": csrf}
+ assert client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "reboot"},
+ headers=headers).status_code == 409
+ assert client.delete(f"/api/v1/vms/{VM_ID}", headers=headers).status_code == 409
+ service.vm["rescue_mode"] = False
+ service.vm["rescue_staged"] = True
+ assert client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "reboot"},
+ headers=headers).status_code == 409
+ assert client.delete(f"/api/v1/vms/{VM_ID}", headers=headers).status_code == 409
+
+
+def test_orphan_staged_rescue_marker_blocks_reinstall(web):
+ client, _state, _token, service, _manager, _seen = web
+ csrf, template_id, _volume_name = configure_reinstall_case(web)
+ service.vm["rescue_staged"] = True
+ response = client.post(
+ f"/api/v1/vms/{VM_ID}/reinstall",
+ json={"template_id": template_id, "confirmation_name": "cloud-vm",
+ "root_password": GUEST_PASSWORD},
+ headers={"X-CSRF-Token": csrf},
+ )
+ assert response.status_code == 409, response.text
+
+
+def test_rescue_premount_failure_does_not_enable_force_restart(web):
+ client, _state, token, service, _manager, _seen = web
+ csrf = setup_admin(client, token)
+ configure_host(client, csrf)
+ headers = {"X-CSRF-Token": csrf}
+ iso = bytearray(1024 * 1024)
+ for sector, kind in ((16, 1), (17, 0), (18, 255)):
+ offset = sector * 2048
+ iso[offset] = kind
+ iso[offset + 1:offset + 6] = b"CD001"
+ iso[offset + 6] = 1
+ iso[17 * 2048 + 7:17 * 2048 + 30] = b"EL TORITO SPECIFICATION"
+ iso[17 * 2048 + 71:17 * 2048 + 75] = (20).to_bytes(4, "little")
+ imported = client.post("/api/v1/rescue-templates?name=Rescue",
+ content=bytes(iso), headers=headers)
+ assert imported.status_code == 202, imported.text
+ for _ in range(100):
+ import_job = client.get(
+ f"/api/v1/rescue-templates/import-jobs/{imported.json()['job_id']}").json()
+ if import_job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert import_job["status"] == "completed", import_job
+ template = client.get("/api/v1/rescue-templates").json()["templates"][0]
+ service.rescue_enter_error = VMError(
+ "救援 ISO 预挂载失败,原虚拟机未受影响", 502)
+ started = client.post(f"/api/v1/vms/{VM_ID}/rescue",
+ json={"template_id": template["id"],
+ "confirmation_name": "alma-test"}, headers=headers)
+ assert started.status_code == 202, started.text
+ for _ in range(100):
+ job = client.get(f"/api/v1/rescue-jobs/{started.json()['job_id']}").json()
+ if job["status"] in {"completed", "failed"}:
+ break
+ time.sleep(0.01)
+ assert job["status"] == "failed", job
+ assert "救援 ISO 预挂载失败" in job["error"]
+ assert client.get(f"/api/v1/vms/{VM_ID}/rescue").json()["status"] is None
+ assert client.post(f"/api/v1/vms/{VM_ID}/rescue/force-start",
+ json={"confirmation_name": "alma-test", "ack_data_loss": True},
+ headers=headers).status_code == 409
+ assert client.post(f"/api/v1/vms/{VM_ID}/actions",
+ json={"action": "poweroff"}, headers=headers).status_code == 200
+
+
def test_cloud_template_and_creation_routes_enforce_auth_csrf_and_fields(web):
client, _, token, service, _, _ = web
import_body = {
diff --git a/tests/test_frontend_page_layout.py b/tests/test_frontend_page_layout.py
index 73c88051..bb5408f6 100644
--- a/tests/test_frontend_page_layout.py
+++ b/tests/test_frontend_page_layout.py
@@ -97,3 +97,83 @@ def test_every_html_id_is_unique(page) -> None:
_, by_id = page
duplicates = sorted(element_id for element_id, elements in by_id.items() if len(elements) > 1)
assert duplicates == []
+
+
+def test_rescue_pages_are_separate_dashboard_pages(page) -> None:
+ _, by_id = page
+ rescue = by_id["rescue-view"][0]
+ assert {ancestor.element_id for ancestor in rescue.ancestors} >= {
+ "dashboard-view", "vm-workspace",
+ }
+ assert "hidden" in rescue.attrs
+ form = by_id["rescue-form"][0]
+ assert "rescue-view" in {ancestor.element_id for ancestor in form.ancestors}
+ assert "modal-backdrop" not in {
+ token
+ for ancestor in (*form.ancestors, form)
+ for token in (ancestor.attrs.get("class") or "").split()
+ }
+
+ catalog = by_id["rescue-templates-workspace"][0]
+ assert "dashboard-view" in {ancestor.element_id for ancestor in catalog.ancestors}
+ assert "hidden" in catalog.attrs
+ upload = by_id["rescue-template-upload-form"][0]
+ assert "rescue-templates-workspace" in {
+ ancestor.element_id for ancestor in upload.ancestors
+ }
+ assert by_id["rescue-template-file"][0].attrs["type"] == "file"
+
+
+def test_rescue_frontend_uses_streaming_upload_and_job_status(page) -> None:
+ _, by_id = page
+ assert by_id["rescue-templates-button"][0].tag == "button"
+ assert by_id["rescue-templates-mobile-button"][0].tag == "button"
+ script = (STATIC_DIR / "app.js").read_text(encoding="utf-8")
+ assert '"rescue-templates-button", "rescue-templates-mobile-button"' in script
+ assert 'rawBody: file, contentType: "application/octet-stream"' in script
+ assert "file.text()" not in script.split(
+ 'byId("rescue-template-upload-form").addEventListener', 1
+ )[1].split("async function startRescueVmAction", 1)[0]
+ assert '"/api/v1/rescue-templates"' in script
+ assert '/rescue/exit`' in script
+ assert 'pollRescueImport(jobId' in script
+ assert 'pollVmRescueJob(jobId' in script
+ assert 'actionButton("紧急救援"' in script
+ assert 'const rescueMode = vm.rescue_mode === true;' in script
+ assert 'const rescueStaged = vm.rescue_staged === true;' in script
+ assert 'const rescueGuard = rescueMode || rescueStaged;' in script
+ assert 'typeof job.warning === "string"' in script
+ html = (STATIC_DIR / "index.html").read_text(encoding="utf-8")
+ assert "救援系统的登录方式和密码由该 ISO 自身决定" in html
+
+
+def test_rescue_timeout_advises_manual_power_cycle_only_when_iso_is_staged(page) -> None:
+ _, by_id = page
+ assert "rescue-view" in by_id
+ script = (STATIC_DIR / "app.js").read_text(encoding="utf-8")
+ html = (STATIC_DIR / "index.html").read_text(encoding="utf-8")
+ assert "正常关机失败或超时" in html
+ assert "只有救援页显示“已预挂载”时" in html
+ assert "点击“强制重启进入救援”" in html
+ assert "强制断电并冷启动" in html
+ assert "“强制重置”不保证读取新保存的启动配置" in html
+ assert "若预挂载未成功,请先核查错误" in html
+ assert 'shutdownFailed && job.operation === "enter"' in script
+ assert 'shutdownFailed && job.operation === "exit"' in script
+ assert 'current?.staged === true && job.operation === "enter"' in script
+ assert 'state.rescueStaged = result?.staged === true;' in script
+ assert 'state.rescuePowerState = String(result?.power_state || "unknown");' in script
+ assert 'job.pending_manual_restart === true' in script
+ assert "正常关机失败或超时,救援 ISO 已预挂载但尚未进入救援系统" in script
+ assert '/rescue/force-start`' in script
+ assert 'ack_data_loss: true' in script
+ assert 'byId("rescue-force-start").addEventListener' in script
+ assert by_id["rescue-force-start"][0].attrs["type"] == "button"
+ assert 'disabled: busy || (rescueMode && !rescueStaged)' in script
+ assert 'title: rescueStaged ? "从已预挂载的救援 ISO 开机"' in script
+ assert "“强制重置”不保证读取新启动配置" in script
+ assert "面板不会自动强制断电" in script
+ assert 'else if (shuttingDown)' in script
+ assert '() => openConfirm(vm, "poweroff")' in script
+ assert "客服" not in html
+ assert "客服" not in script
diff --git a/tests/test_libvirt_service.py b/tests/test_libvirt_service.py
index edc351b3..7b38d387 100644
--- a/tests/test_libvirt_service.py
+++ b/tests/test_libvirt_service.py
@@ -17,6 +17,7 @@ from unittest.mock import patch
SERVICE_PATH = pathlib.Path(__file__).resolve().parents[1] / "virsh_panel" / "libvirt_service.py"
TEMPLATE_NAME = "panel-template-alma10-12345678-0123456789abcdef.qcow2"
+RESCUE_NAME = "panel-rescue-0123456789abcdef0123456789abcdef.iso"
class FakeLibvirtError(Exception):
@@ -251,10 +252,10 @@ class FakeDomain:
return int(self.auto_start)
def setAutostart(self, enabled):
- assert enabled == 1
+ assert enabled in (0, 1)
if not self.persistent or self.backend.fail_domain_autostart:
raise FakeLibvirtError("domain autostart failed")
- self.auto_start = True
+ self.auto_start = bool(enabled)
self.backend.domain_autostart_changed.append(self._uuid)
def isActive(self):
@@ -303,6 +304,7 @@ class FakeDomain:
def destroy(self):
if self.backend.fail_destroy:
raise FakeLibvirtError("domain destroy failed")
+ self.backend.destroy_calls += 1
self.running = False
self.live_root = None
if not self.persistent:
@@ -473,6 +475,7 @@ class FakeBackend:
self.deleted_volumes = []
self.domain_starts = 0
self.shutdown_calls = 0
+ self.destroy_calls = 0
self.shutdown_stalls = False
self.streams = []
self.network_active = True
@@ -1690,6 +1693,420 @@ class VMServiceTests(unittest.TestCase):
self.assertNotIn(TEMPLATE_NAME, self.backend.pools["images"].volumes)
self.assertEqual(self.backend.opened, self.backend.closed)
+ def rescue_volume(self):
+ pool = self.backend.pools["iso"]
+ pool.volumes[RESCUE_NAME] = FakeVolume(pool, RESCUE_NAME,
+ capacity=1024 * 1024, fmt="raw")
+ return pool.volumes[RESCUE_NAME]
+
+ def test_rescue_iso_import_rejects_invalid_file_and_never_overwrites(self):
+ with tempfile.TemporaryDirectory() as directory:
+ image = pathlib.Path(directory) / "rescue.iso"
+ image.write_bytes(b"not an ISO" * 10000)
+ with self.assertRaises(self.error_type) as invalid:
+ self.service.import_rescue_iso(image, RESCUE_NAME)
+ self.assertEqual(invalid.exception.status, 422)
+ self.assertNotIn(RESCUE_NAME, self.backend.pools["iso"].volumes)
+
+ payload = bytearray(1024 * 1024)
+ payload[32769:32774] = b"CD001"
+ image.write_bytes(payload)
+ imported = self.service.import_rescue_iso(image, RESCUE_NAME)
+ self.assertEqual(imported, {"volume_name": RESCUE_NAME,
+ "image_bytes": len(payload)})
+ self.assertEqual(bytes(self.backend.pools["iso"].volumes[RESCUE_NAME].contents),
+ payload)
+ with self.assertRaises(self.error_type) as exists:
+ self.service.import_rescue_iso(image, RESCUE_NAME)
+ self.assertEqual(exists.exception.status, 409)
+
+ def test_rescue_iso_import_stream_failure_removes_only_new_volume(self):
+ with tempfile.TemporaryDirectory() as directory:
+ image = pathlib.Path(directory) / "rescue.iso"
+ payload = bytearray(1024 * 1024)
+ payload[32769:32774] = b"CD001"
+ image.write_bytes(payload)
+ self.backend.fail_send = True
+ with self.assertRaises(self.error_type):
+ self.service.import_rescue_iso(image, RESCUE_NAME)
+ self.assertNotIn(RESCUE_NAME, self.backend.pools["iso"].volumes)
+ self.assertTrue(self.backend.streams[-1].aborted)
+
+ def test_rescue_iso_delete_can_clean_corrupt_unreferenced_managed_volume(self):
+ pool = self.backend.pools["iso"]
+ pool.volumes[RESCUE_NAME] = FakeVolume(pool, RESCUE_NAME,
+ capacity=1024, fmt="qcow2")
+ self.assertTrue(self.service.delete_rescue_iso(RESCUE_NAME)["deleted"])
+ self.assertNotIn(RESCUE_NAME, pool.volumes)
+
+ def test_rescue_boot_preserves_seed_cdrom_and_exit_restores_original_xml(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ original_disk_names = set(self.backend.pools["images"].volumes)
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+
+ entered = service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertTrue(entered["vm"]["rescue_mode"])
+ self.assertFalse(entered["vm"]["rescue_staged"])
+ self.assertEqual(entered["vm"]["rescue_volume_name"], RESCUE_NAME)
+ self.assertTrue(self.backend.domains[0].autostart())
+ rescue_root = self.backend.domains[0].root
+ self.assertEqual(len(rescue_root.findall("./devices/disk[@device='cdrom']")), 2)
+ self.assertIsNone(rescue_root.find("./os/boot"))
+ seed = rescue_root.findall("./devices/disk[@device='cdrom']")[0]
+ rescue = rescue_root.findall("./devices/disk[@device='cdrom']")[1]
+ self.assertIn("panel-seed-", seed.find("source").get("file"))
+ self.assertEqual(rescue.find("source").get("file"),
+ self.backend.pools["iso"].volumes[RESCUE_NAME].path())
+ self.assertEqual(rescue.find("boot").get("order"), "1")
+ self.assertEqual(rescue_root.find("./devices/disk[@device='disk']/boot").get("order"),
+ "2")
+ with self.assertRaises(self.error_type) as in_use:
+ service.delete_rescue_iso(RESCUE_NAME)
+ self.assertEqual(in_use.exception.status, 409)
+
+ exited = service.exit_rescue_vm(created["uuid"], original_xml)
+ self.assertFalse(exited["vm"]["rescue_mode"])
+ self.assertTrue(self.backend.domains[0].autostart())
+ self.assertEqual(len(self.backend.domains[0].root.findall(
+ "./devices/disk[@device='cdrom']")), 1)
+ self.assertEqual(set(self.backend.pools["images"].volumes), original_disk_names)
+ self.assertEqual(self.backend.shutdown_calls, 2)
+ self.assertEqual(self.backend.destroyed, [])
+ self.assertTrue(service.delete_rescue_iso(RESCUE_NAME)["deleted"])
+
+ def test_stopped_vm_can_start_directly_in_rescue_mode(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ domain = self.backend.domains[0]
+ domain.shutdown()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ before = self.backend.shutdown_calls
+ result = service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertTrue(result["vm"]["rescue_mode"])
+ self.assertEqual(self.backend.shutdown_calls, before)
+ self.assertEqual(self.backend.destroyed, [])
+
+ def test_rescue_entry_rejects_changed_xml_before_shutdown(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ changed = ET.fromstring(original_xml)
+ changed.find("./memory").text = "8192"
+ self.backend.domains[0].define(ET.tostring(changed, encoding="unicode"))
+ with self.assertRaises(self.error_type) as mismatch:
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertEqual(mismatch.exception.status, 409)
+ self.assertEqual(self.backend.shutdown_calls, 0)
+
+ def test_rescue_rejects_ambiguous_multi_disk_boot_before_shutdown(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ domain = self.backend.domains[0]
+ original = ET.fromstring(domain.XMLDesc(1))
+ disk = original.find("./devices/disk[@device='disk']")
+ disk.find("target").set("dev", "vdb")
+ extra = copy.deepcopy(disk)
+ extra.find("target").set("dev", "vdc")
+ original.find("devices").append(extra)
+ domain.define(ET.tostring(original, encoding="unicode"))
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ with self.assertRaises(self.error_type) as ambiguous:
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertEqual(ambiguous.exception.status, 409)
+ self.assertEqual(self.backend.shutdown_calls, 0)
+
+ def test_rescue_honors_explicit_boot_disk_in_multi_disk_vm(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ domain = self.backend.domains[0]
+ original = ET.fromstring(domain.XMLDesc(1))
+ os_node = original.find("os")
+ for boot in list(os_node.findall("boot")):
+ os_node.remove(boot)
+ disk = original.find("./devices/disk[@device='disk']")
+ ET.SubElement(disk, "boot", order="2")
+ extra = copy.deepcopy(disk)
+ extra.find("target").set("dev", "vdb")
+ extra.find("boot").set("order", "1")
+ original.find("devices").append(extra)
+ domain.define(ET.tostring(original, encoding="unicode"))
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ booted_disks = [node.find("target").get("dev") for node in
+ domain.root.findall("./devices/disk[@device='disk']")
+ if node.find("boot") is not None]
+ self.assertEqual(booted_disks, ["vdb"])
+
+ def test_rescue_shutdown_timeout_never_forces_poweroff(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ self.backend.shutdown_stalls = True
+ with patch.object(self.service_module, "REINSTALL_SHUTDOWN_TIMEOUT", 0):
+ result = service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertTrue(result["pending_manual_restart"])
+ self.assertIn("正常关机超时", result["warning"])
+ self.assertIn("强制重启进入救援", result["warning"])
+ self.assertIn("强制断电", result["warning"])
+ self.assertIn("强制重置", result["warning"])
+ self.assertNotIn("联系客服", result["warning"])
+ self.assertTrue(self.backend.domains[0].running)
+ self.assertEqual(self.backend.destroyed, [])
+ domain = self.backend.domains[0]
+ self.assertFalse(result["vm"]["rescue_mode"])
+ self.assertTrue(result["vm"]["rescue_staged"])
+ self.assertEqual(result["vm"]["rescue_volume_name"], RESCUE_NAME)
+ self.assertIsNone(domain.root.find(
+ f"./metadata/{{{self.service_module.PANEL_NS}}}rescue"))
+ self.assertIsNotNone(domain.inactive_root.find(
+ f"./metadata/{{{self.service_module.PANEL_NS}}}rescue"))
+
+ # A QEMU reset uses its live configuration; a cold power cycle reads
+ # the durable rescue definition saved before the failed shutdown.
+ service.action(created["uuid"], "reset")
+ self.assertTrue(service.get_vm(created["uuid"])["rescue_staged"])
+ before_starts = self.backend.domain_starts
+ forced = service.force_start_staged_rescue_vm(created["uuid"], original_xml)
+ self.assertEqual(self.backend.destroy_calls, 1)
+ self.assertEqual(self.backend.domain_starts, before_starts + 1)
+ self.assertTrue(forced["vm"]["rescue_mode"])
+ self.assertTrue(service.get_vm(created["uuid"])["rescue_mode"])
+ self.assertFalse(service.get_vm(created["uuid"])["rescue_staged"])
+ self.backend.shutdown_stalls = False
+ service.exit_rescue_vm(created["uuid"], original_xml)
+ self.assertFalse(service.get_vm(created["uuid"])["rescue_mode"])
+
+ def test_rescue_shutdown_request_failure_instructs_manual_poweroff(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ domain = self.backend.domains[0]
+ with patch.object(domain, "shutdown", side_effect=FakeLibvirtError("ACPI unavailable")):
+ result = service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertTrue(result["pending_manual_restart"])
+ self.assertIn("无法请求虚拟机正常关机", result["warning"])
+ self.assertIn("强制重启进入救援", result["warning"])
+ self.assertNotIn("联系客服", result["warning"])
+ self.assertTrue(domain.running)
+ self.assertEqual(self.backend.destroyed, [])
+ self.assertFalse(service.get_vm(created["uuid"])["rescue_mode"])
+ self.assertTrue(service.get_vm(created["uuid"])["rescue_staged"])
+
+ def test_rescue_shutdown_error_after_external_cold_start_reports_active(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ domain = self.backend.domains[0]
+
+ def cold_start_then_error():
+ domain.running = False
+ domain.live_root = None
+ domain.create()
+ raise FakeLibvirtError("shutdown result lost")
+
+ with patch.object(domain, "shutdown", side_effect=cold_start_then_error):
+ result = service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertNotIn("pending_manual_restart", result)
+ self.assertTrue(result["vm"]["rescue_mode"])
+ self.assertFalse(result["vm"]["rescue_staged"])
+ self.assertIn("VNC", result["warning"])
+
+ def test_rescue_staged_exit_cancels_boot_without_stopping_running_vm(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ domain = self.backend.domains[0]
+ self.backend.shutdown_stalls = True
+ with patch.object(self.service_module, "REINSTALL_SHUTDOWN_TIMEOUT", 0):
+ result = service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertTrue(result["pending_manual_restart"])
+ before_shutdowns = self.backend.shutdown_calls
+ before_starts = self.backend.domain_starts
+ exited = service.exit_rescue_vm(created["uuid"], original_xml)
+ self.assertFalse(exited["vm"]["rescue_mode"])
+ self.assertFalse(exited["vm"]["rescue_staged"])
+ self.assertTrue(domain.running)
+ self.assertEqual(self.backend.shutdown_calls, before_shutdowns)
+ self.assertEqual(self.backend.domain_starts, before_starts)
+ self.assertTrue(service.delete_rescue_iso(RESCUE_NAME)["deleted"])
+
+ def test_force_staged_rescue_rejects_wrong_backup_without_poweroff(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ self.backend.shutdown_stalls = True
+ with patch.object(self.service_module, "REINSTALL_SHUTDOWN_TIMEOUT", 0):
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ tampered = original_xml.replace("cloud-vm", "other-vm")
+ with self.assertRaises(self.error_type) as error:
+ service.force_start_staged_rescue_vm(created["uuid"], tampered)
+ self.assertEqual(error.exception.status, 409)
+ self.assertEqual(self.backend.destroy_calls, 0)
+ self.assertTrue(self.backend.domains[0].running)
+
+ def test_force_staged_rescue_start_failure_keeps_iso_for_retry(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ self.backend.shutdown_stalls = True
+ with patch.object(self.service_module, "REINSTALL_SHUTDOWN_TIMEOUT", 0):
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.backend.fail_domain_create = True
+ with self.assertRaises(self.error_type) as error:
+ service.force_start_staged_rescue_vm(created["uuid"], original_xml)
+ self.assertEqual(error.exception.status, 502)
+ self.assertEqual(self.backend.destroy_calls, 1)
+ self.assertFalse(self.backend.domains[0].running)
+ self.assertTrue(service.get_vm(created["uuid"])["rescue_staged"])
+ self.backend.fail_domain_create = False
+ retried = service.force_start_staged_rescue_vm(created["uuid"], original_xml)
+ self.assertTrue(retried["vm"]["rescue_mode"])
+ self.assertFalse(retried["vm"]["rescue_staged"])
+ self.assertEqual(self.backend.destroy_calls, 1)
+
+ def test_force_staged_rescue_poweroff_failure_keeps_original_running(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ self.backend.shutdown_stalls = True
+ with patch.object(self.service_module, "REINSTALL_SHUTDOWN_TIMEOUT", 0):
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ before_starts = self.backend.domain_starts
+ self.backend.fail_destroy = True
+ with self.assertRaises(self.error_type) as error:
+ service.force_start_staged_rescue_vm(created["uuid"], original_xml)
+ self.assertEqual(error.exception.status, 502)
+ self.assertEqual(self.backend.domain_starts, before_starts)
+ self.assertTrue(self.backend.domains[0].running)
+ self.assertTrue(service.get_vm(created["uuid"])["rescue_staged"])
+
+ def test_rescue_staged_exit_after_manual_poweroff_starts_original_system(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ self.backend.shutdown_stalls = True
+ with patch.object(self.service_module, "REINSTALL_SHUTDOWN_TIMEOUT", 0):
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ service.action(created["uuid"], "poweroff")
+ self.assertTrue(service.get_vm(created["uuid"])["rescue_staged"])
+ exited = service.exit_rescue_vm(created["uuid"], original_xml)
+ self.assertEqual(exited["vm"]["state"], "running")
+ self.assertFalse(exited["vm"]["rescue_mode"])
+ self.assertFalse(exited["vm"]["rescue_staged"])
+
+ def test_rescue_start_failure_restores_original_vm(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ self.backend.fail_define = True
+ with self.assertRaises(self.error_type) as failure:
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertEqual(failure.exception.status, 502)
+ self.assertTrue(self.backend.domains[0].running)
+ self.assertFalse(service.get_vm(created["uuid"])["rescue_mode"])
+ self.assertEqual(self.backend.destroyed, [])
+
+ def test_rescue_guest_start_failure_restores_original_vm(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ original_create = FakeDomain.create
+ namespace = self.service_module.PANEL_NS
+
+ def fail_rescue_start(self):
+ if self.inactive_root.find(
+ f"./metadata/{{{namespace}}}rescue") is not None:
+ raise FakeLibvirtError("rescue start failed")
+ return original_create(self)
+
+ with patch.object(FakeDomain, "create", fail_rescue_start):
+ with self.assertRaises(self.error_type) as failure:
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertEqual(failure.exception.status, 502)
+ self.assertTrue(self.backend.domains[0].running)
+ self.assertFalse(service.get_vm(created["uuid"])["rescue_mode"])
+
+ def test_rescue_start_transport_error_after_boot_keeps_rescue_config(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ self.backend.fail_domain_create_after_start = True
+ result = service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertIn("warning", result)
+ self.assertTrue(result["vm"]["rescue_mode"])
+ self.assertTrue(self.backend.domains[0].running)
+
+ def test_rescue_exit_rejects_wrong_backup_without_shutdown(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ tampered = original_xml.replace("cloud-vm", "other-vm")
+ before = self.backend.shutdown_calls
+ with self.assertRaises(self.error_type) as failure:
+ service.exit_rescue_vm(created["uuid"], tampered)
+ self.assertEqual(failure.exception.status, 409)
+ self.assertEqual(self.backend.shutdown_calls, before)
+
+ def test_rescue_exit_start_failure_restores_rescue_vm(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ original_create = FakeDomain.create
+ namespace = self.service_module.PANEL_NS
+
+ def fail_normal_start(self):
+ if self.inactive_root.find(f"./metadata/{{{namespace}}}rescue") is None:
+ raise FakeLibvirtError("normal system start failed")
+ return original_create(self)
+
+ with patch.object(FakeDomain, "create", fail_normal_start):
+ with self.assertRaises(self.error_type) as failure:
+ service.exit_rescue_vm(created["uuid"], original_xml)
+ self.assertEqual(failure.exception.status, 502)
+ self.assertTrue(self.backend.domains[0].running)
+ self.assertTrue(service.get_vm(created["uuid"])["rescue_mode"])
+ self.assertEqual(self.backend.destroyed, [])
+
+ def test_rescue_redefinition_restores_original_autostart_setting(self):
+ service = self.cloud_service()
+ created = self.create_cloud_vm(service, self.cloud_spec())
+ self.rescue_volume()
+ domain = self.backend.domains[0]
+ domain.setAutostart(0)
+ original_xml = service.prepare_rescue_vm(created["uuid"], RESCUE_NAME)
+ original_define = FakeDomain.define
+
+ def resetting_define(self, xml):
+ original_define(self, xml)
+ self.auto_start = True
+
+ with patch.object(FakeDomain, "define", resetting_define):
+ service.enter_rescue_vm(created["uuid"], RESCUE_NAME, original_xml)
+ self.assertFalse(domain.autostart())
+ service.exit_rescue_vm(created["uuid"], original_xml)
+ self.assertFalse(domain.autostart())
+
def test_metric_counters_use_bulk_libvirt_stats_for_running_domains(self):
constants = {
"VIR_DOMAIN_STATS_CPU_TOTAL": 2,
diff --git a/tests/test_rescue_templates.py b/tests/test_rescue_templates.py
new file mode 100644
index 00000000..ce7bfa41
--- /dev/null
+++ b/tests/test_rescue_templates.py
@@ -0,0 +1,63 @@
+"""Rescue ISO catalog and bootable-file boundary tests."""
+
+from __future__ import annotations
+
+import pytest
+
+from virsh_panel.cloud_templates import TemplateError
+from virsh_panel.rescue_templates import (
+ RescueTemplateStore, rescue_scope, rescue_volume_name, verify_bootable_iso,
+)
+
+
+def _profile(iso_pool="iso"):
+ return {
+ "host": "vmhost.example", "port": 2222, "fingerprint": "SHA256:vmhost",
+ "iso_pool": iso_pool,
+ }
+
+
+def _iso_bytes(bootable=True):
+ image = bytearray(1024 * 1024)
+ for sector, kind in ((16, 1), (17, 0 if bootable else 2), (18, 255)):
+ offset = sector * 2048
+ image[offset] = kind
+ image[offset + 1:offset + 6] = b"CD001"
+ image[offset + 6] = 1
+ if bootable:
+ image[17 * 2048 + 7:17 * 2048 + 30] = b"EL TORITO SPECIFICATION"
+ image[17 * 2048 + 71:17 * 2048 + 75] = (20).to_bytes(4, "little")
+ return bytes(image)
+
+
+def test_rescue_catalog_is_scoped_to_iso_pool_and_rejects_duplicate_names():
+ catalog = RescueTemplateStore(None)
+ first_scope = rescue_scope(_profile())
+ second_scope = rescue_scope(_profile("other-iso"))
+ assert first_scope != second_scope
+ template_id = "a" * 32
+ record = {
+ "id": template_id, "name": "System Rescue",
+ "volume_name": rescue_volume_name(template_id),
+ "size_bytes": 1024 * 1024, "sha256": "b" * 64,
+ }
+ catalog.add(first_scope, record)
+ assert catalog.list(second_scope) == []
+ assert catalog.get(first_scope, template_id)["name"] == "System Rescue"
+ with pytest.raises(TemplateError):
+ catalog.add(first_scope, {**record, "id": "c" * 32,
+ "volume_name": rescue_volume_name("c" * 32)})
+ catalog.delete(first_scope, template_id)
+ assert catalog.list(first_scope) == []
+
+
+def test_rescue_iso_must_include_el_torito_boot_record(tmp_path):
+ path = tmp_path / "rescue.iso"
+ path.write_bytes(_iso_bytes())
+ verify_bootable_iso(path)
+ path.write_bytes(_iso_bytes(bootable=False))
+ with pytest.raises(TemplateError, match="El Torito"):
+ verify_bootable_iso(path)
+ path.write_bytes(b"not an ISO")
+ with pytest.raises(TemplateError):
+ verify_bootable_iso(path)
diff --git a/virsh_panel/app.py b/virsh_panel/app.py
index f3e551ef..d0315a46 100644
--- a/virsh_panel/app.py
+++ b/virsh_panel/app.py
@@ -3,16 +3,21 @@
from __future__ import annotations
import asyncio
+import hashlib
import hmac
import ipaddress
import json
import logging
+import os
import subprocess
+import tempfile
import threading
import time
import uuid
+import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager, suppress
+from pathlib import Path
from typing import Any, Callable, Literal
from urllib.parse import urlsplit
@@ -36,6 +41,10 @@ from .metrics import (MAX_QUERY_POINTS, RETENTION_SECONDS, RANGES, MetricsCollec
host_scope, query_interval_seconds)
from .panel_state import PanelState, SetupError
from .remote_host import HostProfile, RemoteHostError, RemoteHostManager
+from .rescue_templates import (
+ MAX_RESCUE_ISO_BYTES, RescueTemplateStore, rescue_scope,
+ rescue_volume_name, verify_bootable_iso,
+)
from .ssh_keys import SSHKeyError
from .smtp_service import SMTPError, SMTPSettingsStore, send_verification_code
from .updater import BRANCH, REPOSITORY_URL, UpdateError, Updater
@@ -224,6 +233,26 @@ class ReinstallBody(BaseModel):
confirmation_name: str = Field(min_length=1, max_length=63)
+class EnterRescueBody(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ template_id: str = Field(pattern=r"^[0-9a-f]{32}$")
+ confirmation_name: str = Field(min_length=1, max_length=63)
+
+
+class ExitRescueBody(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ confirmation_name: str = Field(min_length=1, max_length=63)
+
+
+class ForceRescueBody(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ confirmation_name: str = Field(min_length=1, max_length=63)
+ ack_data_loss: Literal[True]
+
+
class ActionBody(BaseModel):
model_config = ConfigDict(extra="forbid")
@@ -346,10 +375,20 @@ def create_app(
import_lock = threading.RLock()
import_jobs: dict[str, dict] = {}
pending_templates: set[tuple[str, str]] = set()
+ rescue_catalog = RescueTemplateStore(data_store)
+ rescue_import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="rescue-iso-import")
+ rescue_import_lock = threading.RLock()
+ rescue_import_jobs: dict[str, dict] = {}
+ pending_rescue_imports: set[str] = set()
reinstall_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-reinstall")
reinstall_lock = threading.RLock()
reinstall_jobs: dict[str, dict] = {}
pending_reinstalls: set[tuple[str, str]] = set()
+ rescue_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-rescue")
+ rescue_lock = threading.RLock()
+ rescue_jobs: dict[str, dict] = {}
+ pending_rescues: set[tuple[str, str]] = set()
+ test_rescue_states: dict[str, str] = {}
# The production store encrypts each credential record with a fresh AES-GCM
# nonce. The in-memory fallback is only for injected, database-free tests.
test_credentials: dict[str, str] = {}
@@ -507,6 +546,79 @@ def create_app(
else:
test_credentials.pop(key, None)
+ def rescue_state_key(profile: dict, vm_id: str) -> str:
+ return credential_key(profile, vm_id)
+
+ def rescue_xml_matches(first: str, second: str) -> bool:
+ try:
+ return ET.canonicalize(first) == ET.canonicalize(second)
+ except (TypeError, ET.ParseError):
+ return False
+
+ def load_rescue_state(profile: dict, vm_id: str) -> dict | None:
+ key = rescue_state_key(profile, vm_id)
+ raw = (data_store.get("vm_rescue", key) if data_store is not None
+ else test_rescue_states.get(key))
+ if raw is None:
+ return None
+ try:
+ record = json.loads(raw)
+ except (TypeError, ValueError) as exc:
+ raise DatabaseError("虚拟机救援记录损坏") from exc
+ if (not isinstance(record, dict)
+ or not isinstance(record.get("original_xml"), str)
+ or not record["original_xml"].startswith(" None:
+ key = rescue_state_key(profile, vm_id)
+ value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
+ if data_store is not None:
+ data_store.set("vm_rescue", key, value)
+ else:
+ test_rescue_states[key] = value
+
+ def delete_rescue_state(profile: dict, vm_id: str) -> None:
+ key = rescue_state_key(profile, vm_id)
+ if data_store is not None:
+ data_store.delete("vm_rescue", key)
+ else:
+ test_rescue_states.pop(key, None)
+
+ def current_rescue_state(profile: dict, vm_id: str,
+ service: VMService | None = None) -> dict | None:
+ """Clear a stale record only when the original inactive XML is intact.
+
+ An interrupted worker may have stopped before changing the VM. The
+ database job is marked failed on restart, but its recovery record must
+ not indefinitely block normal operations in that safe case.
+ Callers hold rescue_lock so this cannot race a live rescue worker.
+ """
+ record = load_rescue_state(profile, vm_id)
+ if record is None or (host_scope(profile), vm_id) in pending_rescues:
+ return record
+ service = service or new_service(profile)
+ current_vm = service.get_vm(vm_id)
+ if current_vm.get("rescue_mode") or current_vm.get("rescue_staged"):
+ return record
+ try:
+ original_still_defined = rescue_xml_matches(
+ service.prepare_rescue_vm(vm_id, record["volume_name"]),
+ record["original_xml"],
+ )
+ except VMError:
+ return record
+ if original_still_defined:
+ delete_rescue_state(profile, vm_id)
+ return None
+ return record
+
def parse_credential(raw: str | None) -> dict | None:
if raw is None:
return None
@@ -560,7 +672,9 @@ def create_app(
if data_store is None:
return
for kind, destination in (("import_job", import_jobs),
- ("reinstall_job", reinstall_jobs)):
+ ("reinstall_job", reinstall_jobs),
+ ("rescue_import_job", rescue_import_jobs),
+ ("rescue_job", rescue_jobs)):
for job_id, raw in data_store.list(kind).items():
try:
job = json.loads(raw)
@@ -605,6 +719,8 @@ def create_app(
metric_collector.close()
reinstall_executor.shutdown(wait=True, cancel_futures=False)
import_executor.shutdown(wait=True, cancel_futures=False)
+ rescue_executor.shutdown(wait=True, cancel_futures=False)
+ rescue_import_executor.shutdown(wait=True, cancel_futures=False)
close_manager = getattr(manager, "close", None) if manager is not None else None
if callable(close_manager):
close_manager()
@@ -895,7 +1011,7 @@ def create_app(
@app.post("/api/v1/database/setup")
def setup_database(body: DatabaseSetupBody, request: Request):
- nonlocal state, manager, templates, data_store, updater, sessions
+ nonlocal state, manager, templates, data_store, updater, sessions, rescue_catalog
if bootstrap is None:
raise DatabaseSetupError("数据库初始化仅供正常部署使用", 409)
peer = _peer(request)
@@ -926,6 +1042,7 @@ def create_app(
setup_limiter.failed(peer)
raise
state, manager, templates, data_store = components
+ rescue_catalog = RescueTemplateStore(data_store)
sessions = DatabaseSessionStore(data_store)
restore_jobs()
setup_limiter.succeeded(peer)
@@ -1169,9 +1286,10 @@ def create_app(
@app.post("/api/v1/update/apply", status_code=202)
def apply_update(body: UpdateApplyBody, _session: Session = Depends(require_admin_write)):
- with import_lock, reinstall_lock:
- if pending_templates or pending_reinstalls:
- raise UpdateError("云镜像导入或虚拟机重装正在执行,请等待完成后再更新面板", 409)
+ with import_lock, reinstall_lock, rescue_lock, rescue_import_lock:
+ if (pending_templates or pending_reinstalls or pending_rescues
+ or pending_rescue_imports):
+ raise UpdateError("镜像导入或虚拟机任务正在执行,请等待完成后再更新面板", 409)
try:
unit = subprocess.run(
["/usr/bin/systemctl", "is-active", "virsh-panel-update.service"],
@@ -1252,9 +1370,12 @@ def create_app(
def save_host(body: HostBody, _session: Session = Depends(require_admin_write)):
# A running reinstall uses the saved SSH host identity and paths. Do
# not replace them while its libvirt operation is in flight.
- with reinstall_lock:
+ with reinstall_lock, rescue_lock, rescue_import_lock:
forbid_long_job_during_update()
- if pending_reinstalls or any(
+ if (pending_reinstalls or pending_rescues or pending_rescue_imports
+ or (data_store is not None and data_store.list("vm_rescue"))):
+ raise VMError("请先完成救援或重装任务并退出救援模式,再修改宿主机配置", 409)
+ if any(
job.get("status") == "succeeded" and "generated_password" in job
for job in reinstall_jobs.values()):
raise VMError("请先完成重装任务并确认保存自动生成的密码,再修改宿主机配置", 409)
@@ -1386,6 +1507,172 @@ def create_app(
raise TemplateError("导入任务不存在", 404)
return {key: value for key, value in job.items() if key != "scope"}
+ @app.get("/api/v1/rescue-templates")
+ def list_rescue_templates(_session: Session = Depends(require_approved)):
+ return {"templates": rescue_catalog.list(rescue_scope(current_profile()))}
+
+ def run_rescue_import(job_id: str, profile: dict, scope: str,
+ record: dict, image_path: Path) -> None:
+ uploaded = False
+ cataloged = False
+ try:
+ with rescue_import_lock:
+ rescue_import_jobs[job_id]["status"] = "running"
+ rescue_import_jobs[job_id]["phase"] = "正在传输 ISO 到宿主机"
+ save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
+ new_service(profile).import_rescue_iso(image_path, record["volume_name"])
+ uploaded = True
+ rescue_catalog.add(scope, record)
+ cataloged = True
+ with rescue_import_lock:
+ rescue_import_jobs[job_id] = {
+ "scope": scope, "status": "completed", "phase": "救援模版已导入",
+ "template": record,
+ }
+ save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
+ except Exception as exc:
+ if isinstance(exc, (TemplateError, VMError)):
+ error = str(exc)
+ else:
+ LOG.exception("Rescue ISO import failed: job=%s", job_id)
+ error = "救援 ISO 导入失败,请查看面板日志"
+ if cataloged:
+ try:
+ rescue_catalog.delete(scope, record["id"])
+ except Exception:
+ LOG.exception("Could not clean failed rescue catalog entry: job=%s", job_id)
+ error += ";数据库中可能留有救援模版记录,请管理员检查"
+ if uploaded:
+ try:
+ new_service(profile).delete_rescue_iso(record["volume_name"])
+ except Exception:
+ LOG.exception("Could not clean failed rescue ISO import: job=%s", job_id)
+ error += ";宿主机上可能留有救援 ISO 卷,请管理员检查"
+ with rescue_import_lock:
+ rescue_import_jobs[job_id] = {
+ "scope": scope, "status": "failed", "phase": "导入失败", "error": error,
+ }
+ try:
+ save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
+ except DatabaseError:
+ LOG.exception("Could not persist rescue import failure: job=%s", job_id)
+ finally:
+ image_path.unlink(missing_ok=True)
+ with rescue_import_lock:
+ pending_rescue_imports.discard(scope)
+
+ @app.post("/api/v1/rescue-templates", status_code=202)
+ async def upload_rescue_template(
+ request: Request,
+ name: str = Query(min_length=1, max_length=64),
+ _session: Session = Depends(require_admin_write),
+ ):
+ name = validate_template_name(name)
+ profile = current_profile()
+ scope = rescue_scope(profile)
+ with rescue_import_lock:
+ forbid_long_job_during_update()
+ if scope in pending_rescue_imports:
+ raise TemplateError("已有救援 ISO 正在导入,请等待完成", 409)
+ if any(row["name"].casefold() == name.casefold()
+ for row in rescue_catalog.list(scope)):
+ raise TemplateError("当前宿主机已有同名救援模版", 409)
+ pending_rescue_imports.add(scope)
+ image_path: Path | None = None
+ handed_off = False
+ try:
+ declared_size = request.headers.get("content-length")
+ if declared_size is not None:
+ try:
+ if int(declared_size) > MAX_RESCUE_ISO_BYTES:
+ raise TemplateError("救援 ISO 超过 8 GiB 限制", 413)
+ except ValueError as exc:
+ raise TemplateError("上传长度无效", 400) from exc
+ fd, raw_path = tempfile.mkstemp(prefix=".rescue-upload-", suffix=".iso",
+ dir=settings.data_dir)
+ image_path = Path(raw_path)
+ os.fchmod(fd, 0o600)
+ checksum = hashlib.sha256()
+ size = 0
+ with os.fdopen(fd, "wb") as output:
+ async for chunk in request.stream():
+ size += len(chunk)
+ if size > MAX_RESCUE_ISO_BYTES:
+ raise TemplateError("救援 ISO 超过 8 GiB 限制", 413)
+ output.write(chunk)
+ checksum.update(chunk)
+ output.flush()
+ os.fsync(output.fileno())
+ verify_bootable_iso(image_path)
+ template_id = uuid.uuid4().hex
+ record = {
+ "id": template_id, "name": name,
+ "volume_name": rescue_volume_name(template_id),
+ "size_bytes": size, "sha256": checksum.hexdigest(),
+ }
+ job_id = str(uuid.uuid4())
+ with rescue_import_lock:
+ rescue_import_jobs[job_id] = {
+ "scope": scope, "status": "queued", "phase": "等待传输到宿主机",
+ }
+ save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
+ try:
+ rescue_import_executor.submit(
+ run_rescue_import, job_id, profile, scope, record, image_path,
+ )
+ except RuntimeError as exc:
+ rescue_import_jobs.pop(job_id, None)
+ delete_job("rescue_import_job", job_id)
+ raise TemplateError("面板正在停止,无法导入救援 ISO", 503) from exc
+ LOG.info("rescue ISO import queued: job=%s admin=%s", job_id, _session.username)
+ handed_off = True # The worker now owns and removes the temporary file.
+ return {"job_id": job_id, "status": "queued"}
+ finally:
+ if not handed_off:
+ if image_path is not None:
+ image_path.unlink(missing_ok=True)
+ with rescue_import_lock:
+ pending_rescue_imports.discard(scope)
+
+ @app.get("/api/v1/rescue-templates/import-jobs/{job_id}")
+ def rescue_import_job(job_id: str, _session: Session = Depends(require_admin)):
+ scope = rescue_scope(current_profile())
+ with rescue_import_lock:
+ job = rescue_import_jobs.get(job_id)
+ if job is None or job["scope"] != scope:
+ raise TemplateError("救援模版导入任务不存在", 404)
+ return {key: value for key, value in job.items() if key != "scope"}
+
+ @app.delete("/api/v1/rescue-templates/{template_id}")
+ def delete_rescue_template(template_id: str,
+ _session: Session = Depends(require_admin_write)):
+ profile = current_profile()
+ scope = rescue_scope(profile)
+ with rescue_lock, rescue_import_lock:
+ forbid_long_job_during_update()
+ if scope in pending_rescue_imports:
+ raise TemplateError("救援 ISO 正在导入,请稍后删除", 409)
+ record = rescue_catalog.get(scope, template_id)
+ states = (data_store.list("vm_rescue") if data_store is not None
+ else test_rescue_states.copy())
+ for raw in states.values():
+ try:
+ state_record = json.loads(raw)
+ except (TypeError, ValueError) as exc:
+ raise DatabaseError("虚拟机救援记录损坏") from exc
+ if not isinstance(state_record, dict):
+ raise DatabaseError("虚拟机救援记录损坏")
+ if state_record.get("volume_name") == record["volume_name"]:
+ raise TemplateError("此救援模版正被虚拟机使用,须先退出救援模式", 409)
+ try:
+ new_service(profile).delete_rescue_iso(record["volume_name"])
+ except VMError as exc:
+ if exc.status != 404:
+ raise
+ rescue_catalog.delete(scope, template_id)
+ LOG.info("rescue ISO deleted: id=%s admin=%s", template_id, _session.username)
+ return {"deleted": True}
+
@app.get("/api/v1/vms")
def list_vms(_session: Session = Depends(require_approved)):
profile = current_profile()
@@ -1683,13 +1970,22 @@ def create_app(
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
- with reinstall_lock:
+ with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
require_vm_access(_session, profile, vm_id)
+ canonical_id = str(uuid.UUID(vm_id))
+ service = new_service(profile)
+ rescue_record = current_rescue_state(profile, canonical_id, service)
+ rescue_vm = service.get_vm(canonical_id)
+ if ((host_scope(profile), canonical_id) in pending_rescues
+ or rescue_record is not None
+ or rescue_vm.get("rescue_mode")
+ or rescue_vm.get("rescue_staged")):
+ raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
scope = template_scope(profile)
record = templates.get(scope, body.template_id)
- vm = new_service(profile).get_vm(vm_id)
+ vm = service.get_vm(vm_id)
if not vm.get("managed") or not vm.get("public_ip_cidr"):
raise VMError("只有面板创建的公网云镜像虚拟机支持自动重装", 409)
if body.confirmation_name != vm["name"]:
@@ -1753,9 +2049,292 @@ def create_app(
save_job("reinstall_job", job_id, job)
return {"acknowledged": True}
+ def run_rescue_job(job_id: str, profile: dict, vm_id: str,
+ operation: Literal["enter", "exit", "force"]) -> None:
+ vm_scope = host_scope(profile)
+ key = (vm_scope, vm_id)
+ completed = False
+ try:
+ with rescue_lock:
+ job = rescue_jobs[job_id]
+ job["status"] = "running"
+ job["phase"] = (
+ "正在预挂载救援 ISO 并等待正常关机" if operation == "enter"
+ else "正在强制断电并从救援 ISO 冷启动" if operation == "force"
+ else "正在正常关机并恢复原系统启动配置"
+ )
+ save_job("rescue_job", job_id, job)
+ state_record = load_rescue_state(profile, vm_id)
+ if state_record is None:
+ raise VMError("救援恢复记录丢失,已停止操作", 409)
+ state_record["status"] = "exiting" if operation == "exit" else "running"
+ save_rescue_state(profile, vm_id, state_record)
+ service = new_service(profile)
+ if operation == "enter":
+ result = service.enter_rescue_vm(vm_id, state_record["volume_name"],
+ state_record["original_xml"])
+ elif operation == "force":
+ result = service.force_start_staged_rescue_vm(
+ vm_id, state_record["original_xml"])
+ else:
+ result = service.exit_rescue_vm(vm_id, state_record["original_xml"])
+ completed = True
+ staged = (operation == "enter" and isinstance(result, dict)
+ and result.get("pending_manual_restart") is True)
+ with rescue_lock:
+ if operation in {"enter", "force"}:
+ state_record["status"] = "staged" if staged else "active"
+ save_rescue_state(profile, vm_id, state_record)
+ else:
+ delete_rescue_state(profile, vm_id)
+ job = rescue_jobs[job_id]
+ job["status"] = "completed"
+ job["phase"] = ("救援 ISO 已预挂载,等待手动强制断电后开机" if staged
+ else "救援系统已启动" if operation in {"enter", "force"}
+ else "已退出救援模式,原系统已启动")
+ if staged:
+ job["pending_manual_restart"] = True
+ if isinstance(result, dict) and result.get("warning"):
+ job["warning"] = str(result["warning"])
+ save_job("rescue_job", job_id, job)
+ LOG.info("vm rescue %s succeeded: job=%s uuid=%s", operation, job_id, vm_id)
+ except Exception as exc:
+ if isinstance(exc, (VMError, TemplateError)):
+ error = str(exc)
+ else:
+ LOG.exception("VM rescue %s failed: job=%s uuid=%s", operation, job_id, vm_id)
+ error = "紧急救援操作失败,请查看面板和宿主机日志"
+ with rescue_lock:
+ try:
+ state_record = load_rescue_state(profile, vm_id)
+ if state_record is not None:
+ # If the original inactive definition survived, the
+ # failed job cannot strand an otherwise normal VM in
+ # a database-only "rescue" state.
+ service = new_service(profile)
+ actual_vm = service.get_vm(vm_id)
+ if (not actual_vm.get("rescue_mode")
+ and rescue_xml_matches(
+ service.prepare_rescue_vm(
+ vm_id, state_record["volume_name"]),
+ state_record["original_xml"],
+ )):
+ delete_rescue_state(profile, vm_id)
+ else:
+ state_record["status"] = "failed"
+ save_rescue_state(profile, vm_id, state_record)
+ except (DatabaseError, VMError, TemplateError):
+ LOG.exception("Could not save rescue recovery state: job=%s", job_id)
+ job = rescue_jobs[job_id]
+ job["status"] = "failed"
+ job["phase"] = "救援操作失败"
+ if "正常关机超时" in error or "无法请求虚拟机正常关机" in error:
+ job["error"] = error
+ else:
+ job["error"] = error + ";请确认虚拟机实际运行状态,必要时联系管理员"
+ if completed:
+ job["error"] = "虚拟机操作已完成,但数据库状态保存失败;请管理员检查救援记录"
+ try:
+ save_job("rescue_job", job_id, job)
+ except DatabaseError:
+ LOG.exception("Could not persist rescue job failure: job=%s", job_id)
+ finally:
+ with rescue_lock:
+ pending_rescues.discard(key)
+
+ @app.get("/api/v1/vms/{vm_id}/rescue")
+ def get_vm_rescue(vm_id: str, _session: Session = Depends(require_approved)):
+ profile = current_profile()
+ canonical_id = require_vm_access(_session, profile, vm_id)
+ with rescue_lock:
+ service = new_service(profile)
+ vm = service.get_vm(canonical_id)
+ record = current_rescue_state(profile, vm["uuid"], service)
+ staged = bool(vm.get("rescue_staged"))
+ active = bool(vm.get("rescue_mode")) and not staged
+ if (record is not None
+ and (host_scope(profile), vm["uuid"]) not in pending_rescues):
+ if staged and record["status"] in {"queued", "running", "failed"}:
+ record["status"] = "staged"
+ save_rescue_state(profile, vm["uuid"], record)
+ elif active and record["status"] in {"queued", "running", "staged"}:
+ record["status"] = "active"
+ save_rescue_state(profile, vm["uuid"], record)
+ template_name = None
+ template_id = record["template_id"] if record is not None else None
+ if record is not None:
+ try:
+ template_name = rescue_catalog.get(
+ rescue_scope(profile), record["template_id"])["name"]
+ except TemplateError:
+ pass
+ return {
+ "active": active,
+ "staged": staged,
+ "power_state": vm.get("state", "unknown"),
+ "template_id": template_id,
+ "template_name": template_name,
+ "status": record["status"] if record is not None else None,
+ "job_id": record.get("job_id") if record is not None else None,
+ "can_exit": (active or staged) and record is not None,
+ }
+
+ @app.post("/api/v1/vms/{vm_id}/rescue", status_code=202)
+ def enter_vm_rescue(vm_id: str, body: EnterRescueBody,
+ _session: Session = Depends(require_approved_write)):
+ with reinstall_lock, rescue_lock:
+ forbid_long_job_during_update()
+ profile = current_profile()
+ canonical_id = require_vm_access(_session, profile, vm_id)
+ vm_scope = host_scope(profile)
+ key = (vm_scope, canonical_id)
+ if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
+ raise VMError("该虚拟机已有救援或重装任务,暂时不能重复操作", 409)
+ service = new_service(profile)
+ vm = service.get_vm(canonical_id)
+ if body.confirmation_name != vm["name"]:
+ raise VMError("确认名称与虚拟机名称不一致", 422)
+ if vm.get("rescue_mode") or vm.get("rescue_staged"):
+ raise VMError("虚拟机已预挂载救援 ISO 或处于救援模式,请先退出", 409)
+ record = rescue_catalog.get(rescue_scope(profile), body.template_id)
+ original_xml = service.prepare_rescue_vm(canonical_id, record["volume_name"])
+ prior = load_rescue_state(profile, canonical_id)
+ if prior is not None:
+ if not rescue_xml_matches(prior["original_xml"], original_xml):
+ raise VMError("虚拟机仍有未完成的救援记录,请管理员检查后再操作", 409)
+ delete_rescue_state(profile, canonical_id)
+ job_id = str(uuid.uuid4())
+ state_record = {
+ "template_id": record["id"], "volume_name": record["volume_name"],
+ "original_xml": original_xml, "status": "queued", "job_id": job_id,
+ }
+ save_rescue_state(profile, canonical_id, state_record)
+ rescue_jobs[job_id] = {
+ "scope": vm_scope, "owner": _session.username,
+ "vm_id": canonical_id, "operation": "enter",
+ "status": "queued", "phase": "等待救援任务开始",
+ }
+ try:
+ save_job("rescue_job", job_id, rescue_jobs[job_id])
+ pending_rescues.add(key)
+ rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "enter")
+ except Exception:
+ pending_rescues.discard(key)
+ rescue_jobs.pop(job_id, None)
+ delete_job("rescue_job", job_id)
+ delete_rescue_state(profile, canonical_id)
+ raise
+ LOG.info("vm rescue enter queued: job=%s uuid=%s user=%s",
+ job_id, canonical_id, _session.username)
+ return {"job_id": job_id, "status": "queued"}
+
+ @app.post("/api/v1/vms/{vm_id}/rescue/force-start", status_code=202)
+ def force_start_vm_rescue(vm_id: str, body: ForceRescueBody,
+ _session: Session = Depends(require_approved_write)):
+ """Explicit, confirmed power cycle into an already staged rescue ISO."""
+ with reinstall_lock, rescue_lock:
+ forbid_long_job_during_update()
+ profile = current_profile()
+ canonical_id = require_vm_access(_session, profile, vm_id)
+ vm_scope = host_scope(profile)
+ key = (vm_scope, canonical_id)
+ if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
+ raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409)
+ record = load_rescue_state(profile, canonical_id)
+ if record is None:
+ raise VMError("缺少原始启动配置备份,不能安全强制重启进入救援", 409)
+ vm = new_service(profile).get_vm(canonical_id)
+ if body.confirmation_name != vm["name"]:
+ raise VMError("确认名称与虚拟机名称不一致", 422)
+ if not vm.get("rescue_staged"):
+ raise VMError("救援 ISO 尚未预挂载,或虚拟机已经进入救援模式", 409)
+ job_id = str(uuid.uuid4())
+ rescue_jobs[job_id] = {
+ "scope": vm_scope, "owner": _session.username,
+ "vm_id": canonical_id, "operation": "force",
+ "status": "queued", "phase": "等待手动确认的救援冷启动",
+ }
+ previous_status = record["status"]
+ previous_job_id = record.get("job_id")
+ try:
+ save_job("rescue_job", job_id, rescue_jobs[job_id])
+ record["status"] = "queued"
+ record["job_id"] = job_id
+ save_rescue_state(profile, canonical_id, record)
+ pending_rescues.add(key)
+ rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "force")
+ except Exception:
+ pending_rescues.discard(key)
+ rescue_jobs.pop(job_id, None)
+ delete_job("rescue_job", job_id)
+ record["status"] = previous_status
+ record["job_id"] = previous_job_id
+ save_rescue_state(profile, canonical_id, record)
+ raise
+ LOG.warning("vm rescue force-start queued: job=%s uuid=%s user=%s",
+ job_id, canonical_id, _session.username)
+ return {"job_id": job_id, "status": "queued"}
+
+ @app.post("/api/v1/vms/{vm_id}/rescue/exit", status_code=202)
+ def exit_vm_rescue(vm_id: str, body: ExitRescueBody,
+ _session: Session = Depends(require_approved_write)):
+ with reinstall_lock, rescue_lock:
+ forbid_long_job_during_update()
+ profile = current_profile()
+ canonical_id = require_vm_access(_session, profile, vm_id)
+ vm_scope = host_scope(profile)
+ key = (vm_scope, canonical_id)
+ if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
+ raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409)
+ record = load_rescue_state(profile, canonical_id)
+ if record is None:
+ raise VMError("找不到原始启动配置,无法安全退出救援模式", 409)
+ vm = new_service(profile).get_vm(canonical_id)
+ if body.confirmation_name != vm["name"]:
+ raise VMError("确认名称与虚拟机名称不一致", 422)
+ if not (vm.get("rescue_mode") or vm.get("rescue_staged")):
+ raise VMError("虚拟机当前并非救援模式,请管理员核对配置", 409)
+ job_id = str(uuid.uuid4())
+ rescue_jobs[job_id] = {
+ "scope": vm_scope, "owner": _session.username,
+ "vm_id": canonical_id, "operation": "exit",
+ "status": "queued", "phase": "等待退出救援模式",
+ }
+ save_job("rescue_job", job_id, rescue_jobs[job_id])
+ record["status"] = "exiting"
+ record["job_id"] = job_id
+ save_rescue_state(profile, canonical_id, record)
+ pending_rescues.add(key)
+ try:
+ rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "exit")
+ except RuntimeError as exc:
+ pending_rescues.discard(key)
+ record["status"] = "active"
+ save_rescue_state(profile, canonical_id, record)
+ rescue_jobs.pop(job_id, None)
+ delete_job("rescue_job", job_id)
+ raise VMError("面板正在停止,无法退出救援模式", 503) from exc
+ LOG.info("vm rescue exit queued: job=%s uuid=%s user=%s",
+ job_id, canonical_id, _session.username)
+ return {"job_id": job_id, "status": "queued"}
+
+ @app.get("/api/v1/rescue-jobs/{job_id}")
+ def get_rescue_job(job_id: str, _session: Session = Depends(require_approved)):
+ profile = current_profile()
+ vm_scope = host_scope(profile)
+ with rescue_lock:
+ job = rescue_jobs.get(job_id)
+ if (job is None or job["scope"] != vm_scope
+ or (job["owner"] != _session.username
+ and not account_for_session(_session)["admin"])):
+ raise VMError("救援任务不存在", 404)
+ require_vm_access(_session, profile, job["vm_id"])
+ return {key: value for key, value in job.items()
+ if key not in {"scope", "owner", "vm_id"}}
+
@app.post("/api/v1/vms/{vm_id}/actions")
def action(vm_id: str, body: ActionBody, _session: Session = Depends(require_approved_write)):
- with reinstall_lock:
+ with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
require_vm_access(_session, profile, vm_id)
@@ -1765,7 +2344,25 @@ def create_app(
normalized_id = vm_id
if (template_scope(profile), normalized_id) in pending_reinstalls:
raise VMError("虚拟机正在重装系统,暂时不能执行其他操作", 409)
- vm = new_service(profile).action(vm_id, body.action)
+ service = new_service(profile)
+ if (host_scope(profile), normalized_id) in pending_rescues:
+ raise VMError("虚拟机正在切换救援模式,请等待任务完成", 409)
+ rescue_record = current_rescue_state(profile, normalized_id, service)
+ current_vm = service.get_vm(normalized_id)
+ staged = bool(current_vm.get("rescue_staged"))
+ if staged and rescue_record is not None:
+ if body.action not in {"poweroff", "start"}:
+ raise VMError(
+ "救援 ISO 已预挂载;请强制断电后开机进入救援,或取消救援配置", 409)
+ elif rescue_record is not None or current_vm.get("rescue_mode") or staged:
+ raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
+ vm = service.action(vm_id, body.action)
+ if staged and body.action == "start":
+ if not vm.get("rescue_mode") or vm.get("rescue_staged"):
+ raise VMError(
+ "虚拟机已发送开机指令,但未确认从救援 ISO 启动;请检查 VNC 控制台", 502)
+ rescue_record["status"] = "active"
+ save_rescue_state(profile, normalized_id, rescue_record)
LOG.info("vm action: uuid=%s action=%s admin=%s", vm.get("uuid"), body.action, _session.username)
return vm
@@ -1775,7 +2372,7 @@ def create_app(
delete_disks: bool = False,
_session: Session = Depends(require_approved_write),
):
- with reinstall_lock:
+ with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
require_vm_access(_session, profile, vm_id)
@@ -1785,7 +2382,15 @@ def create_app(
normalized_id = vm_id
if (template_scope(profile), normalized_id) in pending_reinstalls:
raise VMError("虚拟机正在重装系统,暂时不能删除", 409)
- result = new_service(profile).delete_vm(vm_id, delete_disks)
+ service = new_service(profile)
+ rescue_record = current_rescue_state(profile, normalized_id, service)
+ rescue_vm = service.get_vm(normalized_id)
+ if ((host_scope(profile), normalized_id) in pending_rescues
+ or rescue_record is not None
+ or rescue_vm.get("rescue_mode")
+ or rescue_vm.get("rescue_staged")):
+ raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
+ result = service.delete_vm(vm_id, delete_disks)
if database_mode:
try:
state.users.delete_vm_owner(host_scope(profile), result["uuid"])
diff --git a/virsh_panel/libvirt_service.py b/virsh_panel/libvirt_service.py
index 80633bb4..638573a0 100644
--- a/virsh_panel/libvirt_service.py
+++ b/virsh_panel/libvirt_service.py
@@ -33,9 +33,12 @@ PANEL_NS = "urn:virsh-panel:1"
NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,62}\Z")
BRIDGE_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,14}\Z")
TEMPLATE_VOLUME_RE = re.compile(r"panel-template-alma(?:9|10)-[0-9a-f]{8}-[0-9a-f]{16}\.qcow2\Z")
+RESCUE_VOLUME_RE = re.compile(r"panel-rescue-[0-9a-f]{32}\.iso\Z")
_CREATE_LOCK = threading.RLock()
REINSTALL_SHUTDOWN_TIMEOUT = 120
REINSTALL_SHUTDOWN_POLL_INTERVAL = 1
+MIN_RESCUE_ISO_BYTES = 1024 * 1024
+MAX_RESCUE_ISO_BYTES = 8 * GIB
STATES = {
0: "nostate",
1: "running",
@@ -310,6 +313,20 @@ class VMService:
if isinstance(capacity, int) and capacity > 0:
disk["capacity_gib"] = round(capacity / GIB, 2)
managed = root.find(f"./metadata/{{{PANEL_NS}}}managed")
+ rescue = root.find(f"./metadata/{{{PANEL_NS}}}rescue")
+ rescue_staged = False
+ inactive_rescue = None
+ if domain.isPersistent():
+ try:
+ inactive_root = ET.fromstring(
+ domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE))
+ except ET.ParseError as exc:
+ raise VMError("宿主机返回的虚拟机持久配置 XML 无效", 502) from exc
+ inactive_rescue = inactive_root.find(f"./metadata/{{{PANEL_NS}}}rescue")
+ # Defining the next boot does not replace the live QEMU process.
+ # A stopped VM with a rescue definition is staged until first boot.
+ rescue_staged = inactive_rescue is not None and (
+ not domain.isActive() or rescue is None)
os_source = managed.get("os-source") if managed is not None else None
if os_source not in {"cloud_template", "iso"}:
os_source = "unknown"
@@ -332,6 +349,11 @@ class VMService:
"os_name": managed.get("os-name") if os_source == "cloud_template" else None,
"os_source": os_source,
"install_iso": managed.get("install-iso") if os_source == "iso" else None,
+ "rescue_mode": rescue is not None,
+ "rescue_staged": rescue_staged,
+ "rescue_volume_name": (
+ rescue.get("volume") if rescue is not None else
+ inactive_rescue.get("volume") if inactive_rescue is not None else None),
"console_port": self._vnc_port(root),
}
@@ -694,6 +716,530 @@ class VMService:
LOG.warning("Template upload failed for %s: %s", volume_name, exc)
raise VMError("模板镜像上传到宿主机失败", 502) from exc
+ @staticmethod
+ def _check_rescue_volume_name(volume_name: str) -> None:
+ if not isinstance(volume_name, str) or not RESCUE_VOLUME_RE.fullmatch(volume_name):
+ raise VMError("救援镜像卷名称无效")
+
+ def _lookup_rescue_volume(self, conn: Any, volume_name: str) -> tuple[Any, Any]:
+ self._check_rescue_volume_name(volume_name)
+ pool = self._active_pool(conn, self.iso_pool)
+ self._refresh_directory_pool(pool)
+ if volume_name not in pool.listVolumes():
+ raise VMError("救援镜像在宿主机上不存在,请重新上传", 404)
+ volume = pool.storageVolLookupByName(volume_name)
+ self._volume_source(pool, volume)
+ return pool, volume
+
+ def _rescue_volume(self, conn: Any, volume_name: str) -> tuple[Any, Any]:
+ pool, volume = self._lookup_rescue_volume(conn, volume_name)
+ try:
+ root = ET.fromstring(volume.XMLDesc(0))
+ fmt = root.find("./target/format")
+ size = int(volume.info()[1])
+ except (ET.ParseError, TypeError, ValueError) as exc:
+ raise VMError("救援镜像卷信息无效", 502) from exc
+ if (fmt is None or fmt.get("type") != "raw"
+ or not MIN_RESCUE_ISO_BYTES <= size <= MAX_RESCUE_ISO_BYTES):
+ raise VMError("救援镜像必须是有效大小的 raw ISO 卷", 409)
+ return pool, volume
+
+ def import_rescue_iso(self, image_path: Path, volume_name: str) -> dict[str, Any]:
+ """Upload one administrator-supplied ISO to the host ISO pool."""
+ self._check_rescue_volume_name(volume_name)
+ image_path = Path(image_path)
+ if image_path.is_symlink() or not image_path.is_file():
+ raise VMError("救援镜像文件无效")
+ image_size = image_path.stat().st_size
+ if not MIN_RESCUE_ISO_BYTES <= image_size <= MAX_RESCUE_ISO_BYTES:
+ raise VMError("救援 ISO 大小必须在 1 MiB 至 8 GiB 之间", 422)
+ with image_path.open("rb") as image:
+ image.seek(32769)
+ if image.read(5) != b"CD001":
+ raise VMError("上传的文件不是 ISO 9660 镜像", 422)
+ with self._connect() as conn:
+ pool = self._active_pool(conn, self.iso_pool)
+ self._refresh_directory_pool(pool)
+ if volume_name in pool.listVolumes():
+ raise VMError("同名救援镜像已存在,不会覆盖", 409)
+ if image_size > pool.info()[3]:
+ raise VMError("宿主机 ISO 存储池空间不足", 409)
+ volume = pool.createXML(self._volume_xml_bytes(
+ volume_name, image_size, "raw"), 0)
+ if volume is None:
+ raise VMError("宿主机未能创建救援镜像卷", 502)
+ try:
+ def read_chunks() -> Iterator[bytes]:
+ with image_path.open("rb") as image:
+ while chunk := image.read(1024 * 1024):
+ yield chunk
+
+ self._upload_volume(conn, volume, read_chunks(), image_size)
+ self._rescue_volume(conn, volume_name)
+ return {"volume_name": volume_name, "image_bytes": image_size}
+ except Exception as exc:
+ try:
+ volume.delete(0)
+ except Exception:
+ LOG.exception("Rescue ISO upload failed and volume %s remains", volume_name)
+ raise VMError(
+ f"救援镜像上传失败且宿主机卷 {volume_name} 未能清理,请人工检查", 502,
+ ) from exc
+ if isinstance(exc, VMError):
+ raise
+ LOG.warning("Rescue ISO upload failed for %s: %s", volume_name, exc)
+ raise VMError("救援镜像上传到宿主机失败", 502) from exc
+
+ def delete_rescue_iso(self, volume_name: str) -> dict[str, Any]:
+ """Delete only a panel-named rescue ISO unused by every domain."""
+ with _CREATE_LOCK, self._connect() as conn:
+ pool, volume = self._lookup_rescue_volume(conn, volume_name)
+ if self._other_domain_uses_volume(
+ conn, "", pool.name(), volume_name, volume.path()):
+ raise VMError("救援镜像仍被虚拟机引用,不能删除", 409)
+ volume.delete(0)
+ return {"volume_name": volume_name, "deleted": True}
+
+ @staticmethod
+ def _rescue_original_hash(original_xml: str) -> str:
+ if not isinstance(original_xml, str) or not original_xml:
+ raise VMError("原虚拟机配置备份无效", 409)
+ return hashlib.sha256(original_xml.encode("utf-8")).hexdigest()
+
+ @staticmethod
+ def _rescue_xml_matches(first: str, second: str) -> bool:
+ try:
+ return ET.canonicalize(first) == ET.canonicalize(second)
+ except ET.ParseError as exc:
+ raise VMError("虚拟机配置 XML 无效", 502) from exc
+
+ @staticmethod
+ def _rescue_vm_preflight(domain: Any) -> str:
+ if not domain.isPersistent():
+ raise VMError("只有持久化虚拟机可以进入救援模式", 409)
+ if getattr(domain, "hasManagedSaveImage", lambda _flags: 0)(0):
+ raise VMError("虚拟机存在托管保存状态,请先处理后再救援", 409)
+ original_xml = domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)
+ try:
+ root = ET.fromstring(original_xml)
+ except ET.ParseError as exc:
+ raise VMError("虚拟机配置 XML 无效", 502) from exc
+ if root.find(f"./metadata/{{{PANEL_NS}}}rescue") is not None:
+ raise VMError("虚拟机已处于救援模式", 409)
+ if root.find("./devices/disk[@device='disk']") is None:
+ raise VMError("虚拟机没有系统磁盘,不能进入救援模式", 409)
+ return original_xml
+
+ @staticmethod
+ def _rescue_xml(original_xml: str, vm_uuid: str, volume_name: str,
+ source: tuple[str, dict[str, str]], autostart: bool) -> str:
+ try:
+ root = ET.fromstring(original_xml)
+ except ET.ParseError as exc:
+ raise VMError("原虚拟机配置备份无效", 409) from exc
+ if root.findtext("uuid") != vm_uuid:
+ raise VMError("原虚拟机配置备份与目标虚拟机不匹配", 409)
+ os_node = root.find("os")
+ devices = root.find("devices")
+ if os_node is None or devices is None:
+ raise VMError("原虚拟机配置缺少启动或设备信息", 409)
+ disks = [disk for disk in devices.findall("disk")
+ if disk.get("device") == "disk"]
+ if not disks:
+ raise VMError("虚拟机没有系统磁盘,不能进入救援模式", 409)
+ explicitly_booted = [disk for disk in disks
+ if (disk.find("boot") is not None
+ and disk.find("boot").get("order") == "1")]
+ if len(explicitly_booted) == 1:
+ primary = explicitly_booted[0]
+ else:
+ primary = next((disk for disk in disks
+ if disk.find("target") is not None
+ and disk.find("target").get("dev") == "vda"), None)
+ if primary is None and len(disks) == 1:
+ primary = disks[0]
+ if primary is None:
+ raise VMError("无法确定多磁盘虚拟机的系统盘,请先确认原启动顺序", 409)
+ # A cloud-init seed ISO often occupies the existing CD-ROM. Booting
+ # "cdrom" as a class could select the wrong image, so target the new
+ # device explicitly and keep every original device unchanged in the
+ # saved XML used on exit.
+ for boot in list(os_node.findall("boot")):
+ os_node.remove(boot)
+ for device in devices:
+ for boot in list(device.findall("boot")):
+ device.remove(boot)
+ ET.SubElement(primary, "boot", order="2")
+ used_targets = {target.get("dev") for target in devices.findall("./disk/target")}
+ rescue_target = next((f"sd{chr(code)}" for code in range(ord("a"), ord("z") + 1)
+ if f"sd{chr(code)}" not in used_targets), None)
+ if rescue_target is None:
+ raise VMError("虚拟机没有可用的 SATA 光驱目标位置", 409)
+ cdrom = ET.SubElement(devices, "disk", type=source[0], device="cdrom")
+ ET.SubElement(cdrom, "driver", name="qemu", type="raw")
+ ET.SubElement(cdrom, "source", **source[1])
+ ET.SubElement(cdrom, "target", dev=rescue_target, bus="sata")
+ ET.SubElement(cdrom, "readonly")
+ ET.SubElement(cdrom, "boot", order="1")
+ metadata = root.find("metadata")
+ if metadata is None:
+ metadata = ET.SubElement(root, "metadata")
+ ET.SubElement(metadata, f"{{{PANEL_NS}}}rescue",
+ volume=volume_name,
+ **{"original-sha256": VMService._rescue_original_hash(original_xml),
+ "original-autostart": "1" if autostart else "0"})
+ return ET.tostring(root, encoding="unicode")
+
+ @staticmethod
+ def _restore_rescue_autostart(domain: Any, enabled: bool) -> None:
+ if bool(domain.autostart()) != enabled:
+ domain.setAutostart(int(enabled))
+
+ @staticmethod
+ def _wait_for_rescue_shutdown(domain: Any, phase: str) -> None:
+ if not domain.isActive():
+ return
+ if phase == "救援启动":
+ next_step = (
+ "救援 ISO 已写入下次冷启动配置。请先刷新虚拟机状态;如仍在运行,"
+ "可在救援页面点击“强制重启进入救援”(强制断电后从救援配置开机),"
+ "也可以手动“强制断电”后再点“开机”。"
+ "普通“强制重置”只重置运行中的 QEMU,不会加载新的启动配置;"
+ "强制断电可能导致未保存数据丢失"
+ )
+ else:
+ next_step = (
+ "原系统启动配置尚未恢复。请先刷新虚拟机状态;如仍在运行,"
+ "可通过救援系统的 VNC 控制台手动关机,待显示已关机后重试退出救援"
+ )
+ try:
+ domain.shutdown()
+ except libvirt.libvirtError as exc:
+ raise VMError(
+ f"无法请求虚拟机正常关机,{phase}尚未完成;面板未强制断电。{next_step}",
+ 409,
+ ) from exc
+ deadline = time.monotonic() + REINSTALL_SHUTDOWN_TIMEOUT
+ while domain.isActive():
+ if time.monotonic() >= deadline:
+ raise VMError(
+ f"正常关机超时,{phase}尚未完成;虚拟机可能随后自行关机。"
+ f"面板未强制断电。{next_step}", 409)
+ time.sleep(REINSTALL_SHUTDOWN_POLL_INTERVAL)
+
+ def prepare_rescue_vm(self, vm_id: str, rescue_volume_name: str) -> str:
+ """Return inactive XML for durable storage before changing this VM."""
+ with _CREATE_LOCK, self._connect() as conn:
+ self._rescue_volume(conn, rescue_volume_name)
+ return self._rescue_vm_preflight(self._domain(conn, vm_id))
+
+ def enter_rescue_vm(self, vm_id: str, rescue_volume_name: str,
+ original_xml: str, *,
+ progress: Callable[[str], None] | None = None) -> dict[str, Any]:
+ """Stage a durable rescue boot, gracefully stop, then start the ISO."""
+ def phase(value: str) -> None:
+ if progress is not None:
+ progress(value)
+
+ with _CREATE_LOCK, self._connect() as conn:
+ vm_uuid = self._uuid(vm_id)
+ domain = self._domain(conn, vm_uuid)
+ current_xml = self._rescue_vm_preflight(domain)
+ if not self._rescue_xml_matches(current_xml, original_xml):
+ raise VMError("虚拟机配置已变化,请重新提交救援操作", 409)
+ pool, volume = self._rescue_volume(conn, rescue_volume_name)
+ original_autostart = bool(domain.autostart())
+ rescue_xml = self._rescue_xml(
+ original_xml, vm_uuid, rescue_volume_name,
+ self._volume_source(pool, volume), original_autostart)
+ was_running = bool(domain.isActive())
+ switched = False
+ rescue_domain = None
+ try:
+ phase("mounting")
+ rescue_domain = conn.defineXML(rescue_xml)
+ if rescue_domain is None:
+ raise VMError("宿主机未能保存救援启动配置", 502)
+ switched = True
+ self._restore_rescue_autostart(rescue_domain, original_autostart)
+ staged_xml = rescue_domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)
+ except Exception as exc:
+ restored = True
+ if switched:
+ try:
+ restored_domain = conn.defineXML(original_xml)
+ if restored_domain is None:
+ raise VMError("宿主机未能恢复原虚拟机配置", 502)
+ self._restore_rescue_autostart(restored_domain, original_autostart)
+ except Exception:
+ LOG.exception("Could not restore original VM XML for %s", vm_uuid)
+ restored = False
+ if not restored:
+ raise VMError(
+ "救援 ISO 预挂载失败且原虚拟机持久配置未能恢复;"
+ "没有强制断电,请检查宿主机", 502,
+ ) from exc
+ raise VMError("救援 ISO 预挂载失败,原虚拟机未受影响", 502) from exc
+
+ phase("shutting_down")
+ try:
+ self._wait_for_rescue_shutdown(domain, "救援启动")
+ except VMError as exc:
+ if domain.isActive():
+ vm = self._vm_dict(domain)
+ if vm["rescue_mode"] and not vm["rescue_staged"]:
+ # The guest may have shut down and been cold-started
+ # by an external actor while the request timed out.
+ return {"vm": vm, "warning":
+ "救援配置已运行,但宿主机关机状态返回异常;请通过 VNC 确认已进入救援系统"}
+ # Never discard the persistent rescue boot here: the user
+ # can safely decide whether to power-cycle manually.
+ phase("pending_manual_restart")
+ return {"vm": vm,
+ "pending_manual_restart": True,
+ "warning": str(exc)}
+ # The guest stopped after the error; continue with a cold
+ # start from the already-staged rescue definition.
+ latest_xml = domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)
+ if not self._rescue_xml_matches(latest_xml, staged_xml):
+ raise VMError("关机后救援启动配置已变化,救援已停止;请检查虚拟机状态", 409)
+ try:
+ phase("starting")
+ rescue_domain.create()
+ except Exception as exc:
+ if switched and rescue_domain is not None:
+ try:
+ if rescue_domain.isActive() and self._vm_dict(rescue_domain)["rescue_mode"]:
+ phase("completed")
+ return {"vm": self._vm_dict(rescue_domain),
+ "warning": "救援系统可能已启动,但宿主机启动返回错误;请检查控制台"}
+ except Exception:
+ LOG.exception("Could not inspect rescue VM %s after start error", vm_uuid)
+ restored = True
+ restart_domain = domain
+ if switched:
+ try:
+ restored_domain = conn.defineXML(original_xml)
+ if restored_domain is None:
+ raise VMError("宿主机未能恢复原虚拟机配置", 502)
+ self._restore_rescue_autostart(restored_domain, original_autostart)
+ restart_domain = restored_domain
+ except Exception:
+ LOG.exception("Could not restore original VM XML for %s", vm_uuid)
+ restored = False
+ if restored and was_running:
+ try:
+ restart_domain.create()
+ except Exception:
+ LOG.exception("Could not restart original VM %s after rescue failure", vm_uuid)
+ restored = False
+ if not restored:
+ raise VMError(
+ "救援启动失败且原虚拟机未能完全恢复;没有强制断电,请检查宿主机", 502,
+ ) from exc
+ raise VMError("救援启动失败,已恢复原虚拟机配置及运行状态", 502) from exc
+ phase("completed")
+ return {"vm": self._vm_dict(rescue_domain)}
+
+ def force_start_staged_rescue_vm(self, vm_id: str,
+ original_xml: str) -> dict[str, Any]:
+ """Cold power-cycle only a verified, pre-staged rescue VM.
+
+ libvirt's reset() acts on the live QEMU process and cannot load a
+ changed inactive boot definition. destroy()+create() deliberately
+ stops that process and starts a new one from the persistent XML.
+ """
+ with _CREATE_LOCK, self._connect() as conn:
+ vm_uuid = self._uuid(vm_id)
+ domain = self._domain(conn, vm_uuid)
+ if not domain.isPersistent():
+ raise VMError("救援虚拟机不是持久化定义,不能强制重启", 409)
+ try:
+ original_root = ET.fromstring(original_xml)
+ except (TypeError, ET.ParseError) as exc:
+ raise VMError("原虚拟机配置备份无效", 409) from exc
+ if (original_root.findtext("uuid") != vm_uuid
+ or original_root.find(f"./metadata/{{{PANEL_NS}}}rescue") is not None):
+ raise VMError("原虚拟机配置备份与目标虚拟机不匹配", 409)
+ try:
+ staged_root = ET.fromstring(
+ domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE))
+ except ET.ParseError as exc:
+ raise VMError("救援虚拟机持久配置 XML 无效", 502) from exc
+ marker = staged_root.find(f"./metadata/{{{PANEL_NS}}}rescue")
+ if (marker is None or marker.get("original-sha256") !=
+ self._rescue_original_hash(original_xml)
+ or not RESCUE_VOLUME_RE.fullmatch(marker.get("volume", ""))
+ or marker.get("original-autostart") not in {"0", "1"}):
+ raise VMError("救援预挂载记录与原配置备份不匹配,已拒绝强制重启", 409)
+ if domain.isActive() and self._vm_xml(domain).find(
+ f"./metadata/{{{PANEL_NS}}}rescue") is not None:
+ raise VMError("虚拟机已从救援配置运行,无需再次强制重启", 409)
+ pool, volume = self._rescue_volume(conn, marker.get("volume"))
+ source_type, source_attrs = self._volume_source(pool, volume)
+ if not any(
+ disk.get("type") == source_type
+ and disk.find("source") is not None
+ and all(disk.find("source").get(key) == value
+ for key, value in source_attrs.items())
+ and disk.find("boot") is not None
+ and disk.find("boot").get("order") == "1"
+ for disk in staged_root.findall("./devices/disk[@device='cdrom']")):
+ raise VMError("救援 ISO 启动设备与预挂载记录不匹配,已拒绝强制重启", 409)
+ if domain.isActive():
+ try:
+ domain.destroy()
+ except libvirt.libvirtError as exc:
+ if domain.isActive():
+ raise VMError("强制断电失败;救援配置已保留,请检查虚拟机状态", 502) from exc
+ try:
+ domain.create()
+ except libvirt.libvirtError as exc:
+ try:
+ vm = self._vm_dict(domain)
+ if (vm["state"] == "running" and vm["rescue_mode"]
+ and not vm["rescue_staged"]):
+ return {"vm": vm,
+ "warning": "救援系统可能已启动,但宿主机返回错误;请检查控制台"}
+ except Exception:
+ LOG.exception("Could not inspect staged rescue VM %s after start error", vm_uuid)
+ raise VMError(
+ "救援系统强制重启后未能启动;救援 ISO 配置已保留,"
+ "可重试开机或取消救援。强制断电可能导致未保存数据丢失", 502,
+ ) from exc
+ vm = self._vm_dict(domain)
+ if not domain.isActive() or not vm["rescue_mode"] or vm["rescue_staged"]:
+ raise VMError(
+ "已尝试强制启动,但救援系统未进入运行状态;"
+ "救援 ISO 配置已保留,请检查控制台或重试开机", 502,
+ )
+ return {"vm": vm}
+
+ def exit_rescue_vm(self, vm_id: str, original_xml: str, *,
+ progress: Callable[[str], None] | None = None) -> dict[str, Any]:
+ """Gracefully stop rescue mode, restore original XML, and start OS."""
+ def phase(value: str) -> None:
+ if progress is not None:
+ progress(value)
+
+ with _CREATE_LOCK, self._connect() as conn:
+ vm_uuid = self._uuid(vm_id)
+ domain = self._domain(conn, vm_uuid)
+ if not domain.isPersistent():
+ raise VMError("救援虚拟机不是持久化定义,无法恢复", 409)
+ try:
+ original_root = ET.fromstring(original_xml)
+ except (TypeError, ET.ParseError) as exc:
+ raise VMError("原虚拟机配置备份无效", 409) from exc
+ if (original_root.findtext("uuid") != vm_uuid
+ or original_root.find(f"./metadata/{{{PANEL_NS}}}rescue") is not None):
+ raise VMError("原虚拟机配置备份与目标虚拟机不匹配", 409)
+ rescue_xml = domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)
+ try:
+ rescue_root = ET.fromstring(rescue_xml)
+ except ET.ParseError as exc:
+ raise VMError("救援虚拟机配置 XML 无效", 502) from exc
+ marker = rescue_root.find(f"./metadata/{{{PANEL_NS}}}rescue")
+ if (marker is None or marker.get("original-sha256") !=
+ self._rescue_original_hash(original_xml)
+ or not RESCUE_VOLUME_RE.fullmatch(marker.get("volume", ""))
+ or marker.get("original-autostart") not in {"0", "1"}):
+ raise VMError("虚拟机救援标记与原配置备份不匹配,已停止恢复", 409)
+ staged_running = False
+ if domain.isActive():
+ live_marker = self._vm_xml(domain).find(
+ f"./metadata/{{{PANEL_NS}}}rescue")
+ staged_running = live_marker is None
+ if (live_marker is not None
+ and (live_marker.get("original-sha256") !=
+ marker.get("original-sha256")
+ or live_marker.get("volume") != marker.get("volume"))):
+ raise VMError("虚拟机当前并未从已记录的救援配置运行,已停止恢复", 409)
+ original_autostart = marker.get("original-autostart") == "1"
+ rescue_autostart = bool(domain.autostart())
+ if staged_running:
+ # Rescue was saved for the next cold boot, but the original
+ # guest is still running. Cancelling the staged boot must not
+ # shut down or restart that guest.
+ switched = False
+ try:
+ phase("restoring")
+ normal_domain = conn.defineXML(original_xml)
+ if normal_domain is None:
+ raise VMError("宿主机未能恢复原虚拟机配置", 502)
+ switched = True
+ self._restore_rescue_autostart(normal_domain, original_autostart)
+ except Exception as exc:
+ if switched:
+ try:
+ rescue_domain = conn.defineXML(rescue_xml)
+ if rescue_domain is None:
+ raise VMError("宿主机未能恢复救援启动配置", 502)
+ self._restore_rescue_autostart(
+ rescue_domain, rescue_autostart)
+ except Exception:
+ LOG.exception("Could not restore staged rescue XML for %s", vm_uuid)
+ raise VMError(
+ "取消救援预挂载失败,原虚拟机仍在运行,但持久配置状态不确定;"
+ "请检查宿主机", 502,
+ ) from exc
+ raise VMError("取消救援预挂载失败,原虚拟机仍在运行", 502) from exc
+ phase("completed")
+ return {"vm": self._vm_dict(normal_domain)}
+ was_running = bool(domain.isActive())
+ phase("shutting_down")
+ self._wait_for_rescue_shutdown(domain, "退出救援")
+ latest_xml = domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)
+ if not self._rescue_xml_matches(latest_xml, rescue_xml):
+ raise VMError("关机后救援配置已变化,退出救援已停止;请检查虚拟机状态", 409)
+ switched = False
+ normal_domain = None
+ try:
+ phase("starting")
+ normal_domain = conn.defineXML(original_xml)
+ if normal_domain is None:
+ raise VMError("宿主机未能恢复原虚拟机配置", 502)
+ switched = True
+ self._restore_rescue_autostart(normal_domain, original_autostart)
+ normal_domain.create()
+ except Exception as exc:
+ if switched and normal_domain is not None:
+ try:
+ if normal_domain.isActive() and not self._vm_dict(normal_domain)["rescue_mode"]:
+ phase("completed")
+ return {"vm": self._vm_dict(normal_domain),
+ "warning": "原系统可能已启动,但宿主机启动返回错误;请检查控制台"}
+ except Exception:
+ LOG.exception("Could not inspect VM %s after rescue exit error", vm_uuid)
+ restored = True
+ restart_domain = domain
+ if switched:
+ try:
+ restored_domain = conn.defineXML(rescue_xml)
+ if restored_domain is None:
+ raise VMError("宿主机未能恢复救援启动配置", 502)
+ self._restore_rescue_autostart(restored_domain, rescue_autostart)
+ restart_domain = restored_domain
+ except Exception:
+ LOG.exception("Could not restore rescue VM XML for %s", vm_uuid)
+ restored = False
+ if restored and was_running:
+ try:
+ restart_domain.create()
+ except Exception:
+ LOG.exception("Could not restart rescue VM %s after exit failure", vm_uuid)
+ restored = False
+ if not restored:
+ raise VMError(
+ "退出救援失败且救援虚拟机未能完全恢复;没有强制断电,请检查宿主机", 502,
+ ) from exc
+ raise VMError("退出救援失败,已恢复救援启动配置及运行状态", 502) from exc
+ phase("completed")
+ vm = self._vm_dict(normal_domain)
+ if vm["rescue_mode"]:
+ raise VMError("原系统已启动但仍显示救援模式,请检查虚拟机配置", 502)
+ return {"vm": vm}
+
@staticmethod
def _mac_for_new_vm(conn: Any) -> str:
used = {
diff --git a/virsh_panel/rescue_templates.py b/virsh_panel/rescue_templates.py
new file mode 100644
index 00000000..38276e8d
--- /dev/null
+++ b/virsh_panel/rescue_templates.py
@@ -0,0 +1,142 @@
+"""Host-scoped catalog for administrator-uploaded rescue ISO images.
+
+The ISO bytes live in the remote libvirt ISO pool. In production only the
+catalog metadata is stored here, encrypted by the MariaDB record store.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import threading
+from pathlib import Path
+from typing import Any, Mapping
+
+from .cloud_templates import TemplateError, validate_template_name
+
+
+RESCUE_ID_RE = re.compile(r"[0-9a-f]{32}\Z")
+RESCUE_VOLUME_RE = re.compile(r"panel-rescue-[0-9a-f]{32}\.iso\Z")
+MAX_RESCUE_ISO_BYTES = 8 * 1024**3
+
+
+def rescue_scope(profile: Mapping[str, Any]) -> str:
+ """Bind ISO records to a verified SSH host and its selected ISO pool."""
+ try:
+ identity = [profile[key] for key in ("host", "port", "fingerprint", "iso_pool")]
+ except (KeyError, TypeError) as exc:
+ raise TemplateError("宿主机资料不完整,无法选择救援模版", 409) from exc
+ if (not isinstance(identity[0], str) or isinstance(identity[1], bool)
+ or not isinstance(identity[1], int)
+ or not isinstance(identity[2], str) or not isinstance(identity[3], str)):
+ raise TemplateError("宿主机资料不完整,无法选择救援模版", 409)
+ raw = json.dumps(identity, ensure_ascii=False, separators=(",", ":"))
+ return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+def rescue_volume_name(template_id: str) -> str:
+ if not isinstance(template_id, str) or not RESCUE_ID_RE.fullmatch(template_id):
+ raise TemplateError("救援模版标识无效")
+ return f"panel-rescue-{template_id}.iso"
+
+
+def verify_bootable_iso(path: Path) -> None:
+ """Reject ordinary data ISOs, including cloud-init seed images.
+
+ El Torito boot records are required for x86 BIOS/UEFI optical boot. This
+ checks the disc descriptor, not that its bundled operating system works.
+ """
+ size = path.stat().st_size
+ if not 1024 * 1024 <= size <= MAX_RESCUE_ISO_BYTES:
+ raise TemplateError("救援 ISO 大小必须在 1 MiB 至 8 GiB 之间", 413)
+ bootable = False
+ with path.open("rb") as image:
+ for sector in range(16, 80):
+ image.seek(sector * 2048)
+ descriptor = image.read(2048)
+ if len(descriptor) != 2048 or descriptor[1:6] != b"CD001":
+ raise TemplateError("上传文件不是有效的 ISO 9660 镜像", 422)
+ if (descriptor[0] == 0
+ and descriptor[7:30].rstrip(b" ") == b"EL TORITO SPECIFICATION"
+ and int.from_bytes(descriptor[71:75], "little") > 0):
+ bootable = True
+ if descriptor[0] == 255:
+ break
+ if not bootable:
+ raise TemplateError("ISO 未包含可识别的 El Torito 启动记录,无法用作救援模版", 422)
+
+
+class RescueTemplateStore:
+ """Small catalog with the same API in production and isolated HTTP tests."""
+
+ def __init__(self, store: Any | None) -> None:
+ self.store = store
+ self._memory: dict[str, str] = {}
+ self._lock = threading.RLock()
+
+ def _list(self, scope: str) -> list[dict[str, Any]]:
+ source = (self.store.list("rescue_template") if self.store is not None
+ else self._memory.copy())
+ records = []
+ for key, payload in source.items():
+ if not key.startswith(f"{scope}:"):
+ continue
+ try:
+ record = json.loads(payload)
+ except (TypeError, ValueError) as exc:
+ raise TemplateError("救援模版数据库记录损坏", 500) from exc
+ if (not isinstance(record, dict)
+ or not RESCUE_ID_RE.fullmatch(str(record.get("id", "")))
+ or record.get("volume_name") != rescue_volume_name(record["id"])
+ or not isinstance(record.get("name"), str)
+ or not isinstance(record.get("size_bytes"), int)):
+ raise TemplateError("救援模版数据库记录损坏", 500)
+ records.append(record)
+ return sorted(records, key=lambda item: item["name"].casefold())
+
+ def list(self, scope: str) -> list[dict[str, Any]]:
+ with self._lock:
+ return self._list(scope)
+
+ def get(self, scope: str, template_id: str) -> dict[str, Any]:
+ rescue_volume_name(template_id)
+ with self._lock:
+ for record in self._list(scope):
+ if record["id"] == template_id:
+ return record
+ raise TemplateError("救援模版不存在或不属于当前宿主机", 404)
+
+ def add(self, scope: str, record: dict[str, Any]) -> dict[str, Any]:
+ template_id = record.get("id")
+ if record.get("volume_name") != rescue_volume_name(template_id):
+ raise TemplateError("救援模版记录无效")
+ name = validate_template_name(record.get("name"))
+ if (isinstance(record.get("size_bytes"), bool)
+ or not isinstance(record.get("size_bytes"), int)
+ or not 0 < record["size_bytes"] <= MAX_RESCUE_ISO_BYTES):
+ raise TemplateError("救援 ISO 大小无效")
+ record = {**record, "name": name}
+ key = f"{scope}:{template_id}"
+ value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
+ with self._lock:
+ existing = self._list(scope)
+ if any(row["id"] == template_id for row in existing):
+ raise TemplateError("此救援模版已存在", 409)
+ if any(row["name"].casefold() == name.casefold() for row in existing):
+ raise TemplateError("当前宿主机已有同名救援模版", 409)
+ if self.store is not None:
+ if not self.store.insert("rescue_template", key, value):
+ raise TemplateError("此救援模版已存在", 409)
+ else:
+ self._memory[key] = value
+ return record
+
+ def delete(self, scope: str, template_id: str) -> None:
+ self.get(scope, template_id)
+ key = f"{scope}:{template_id}"
+ with self._lock:
+ if self.store is not None:
+ self.store.delete("rescue_template", key)
+ else:
+ self._memory.pop(key, None)