You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_engine_boundary.py

1072 lines
50 KiB
Python

"""Private engine tests: no PHP cookies, no real DB and no real hypervisor."""
from __future__ import annotations
import base64
import asyncio
import hashlib
import json
import threading
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from test_app import CREATE_BODY, FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID
from virsh_panel.app import create_app
from virsh_panel.config import Settings
from virsh_panel.database import DatabaseTemplateStore
from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore,
LogStore, MainStore, TRUSTED_ACTOR)
from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor,
signed_headers)
from virsh_panel.internal import EngineApplication, HostRegistry
from virsh_panel.metrics import host_scope
SECRET = b"s" * 32
USER = {"user_id": 41, "is_admin": False, "is_super_admin": False,
"can_manage": True, "session_id": "test-session"}
SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True}
ADMIN = {**SUPER, "is_super_admin": False}
class MemoryStore:
def __init__(self):
self.records = {}
self.nonces = set()
self.lock = threading.RLock()
self.daily_traffic_calls = []
def get(self, kind, key):
with self.lock:
return self.records.get((kind, key))
def set(self, kind, key, value):
with self.lock:
self.records[kind, key] = value
def insert(self, kind, key, value):
with self.lock:
if (kind, key) in self.records:
return False
self.records[kind, key] = value
return True
def delete(self, kind, key):
with self.lock:
self.records.pop((kind, key), None)
def list(self, kind):
with self.lock:
return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind}
def consume_nonce(self, nonce, expires_at, now):
with self.lock:
if nonce in self.nonces:
return False
self.nonces.add(nonce)
return True
def query_metric_samples(self, *args, **kwargs):
return [{"timestamp": 123, "cpu_percent": 20.0}]
def query_daily_traffic(self, scope, vm_id, **kwargs):
self.daily_traffic_calls.append((scope, vm_id, kwargs))
day_start = kwargs["until"] - kwargs["until"] % 86400
return [{"day_start": day_start, "download_bytes": 4096,
"upload_bytes": 1024, "total_bytes": 5120}]
def get_value(self, key):
return self.get("value", key)
def set_value(self, key, value):
return self.set("value", key, value)
def delete_value(self, key):
return self.delete("value", key)
def get_secret(self, key):
return self.get("secret", key)
def set_secret(self, key, value):
return self.set("secret", key, value)
def delete_secret(self, key):
return self.delete("secret", key)
class FakeUpdater:
def __init__(self):
self.token = None
self.active = None
self.jobs = {}
def settings(self):
return {
"repository": "https://g.luoli.live/root/virt.git",
"branch": "venix",
"token_configured": self.token is not None,
"checkout_ready": True,
"current_commit": "a" * 40,
"last_job_id": None,
"active_job_id": self.active["job_id"] if self.active else None,
}
def set_token(self, token):
self.token = token
def clear_token(self):
self.token = None
def active_job(self):
return self.active
def check(self):
return {"repository": "https://g.luoli.live/root/virt.git", "branch": "venix",
"current_commit": "a" * 40, "remote_commit": "b" * 40,
"status": "update_available"}
def create_job(self, *, expected_current_commit=None, expected_remote_commit=None):
if self.token is None:
from virsh_panel.updater import UpdateError
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
if expected_current_commit != "a" * 40 or expected_remote_commit != "b" * 40:
from virsh_panel.updater import UpdateError
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
job = {"job_id": "update-job-1234567890", "status": "queued",
"phase": "等待独立更新服务启动", "created_at": 1,
"private": self.token}
self.jobs[job["job_id"]] = job
self.active = job
return job.copy()
def get_job(self, job_id):
return self.jobs.get(job_id)
def fail_job(self, job_id, message):
self.jobs[job_id].update(status="failed", error=message)
self.active = None
return self.jobs[job_id].copy()
class Service(FakeVMService):
def __init__(self):
super().__init__()
self.create_failure = False
self.absent = False
self.vm.update(managed=True, os_source="iso")
def create_vm(self, spec):
self.calls.append(("create_vm", spec))
if self.create_failure:
from virsh_panel.libvirt_service import VMError
raise VMError("宿主机失联,回滚状态不明", 502)
self.vm.update(uuid=spec["vm_uuid"], name=spec["name"])
return self.vm.copy()
def provisioning_absent(self, vm_uuid, name):
return self.absent
@pytest.fixture
def boundary(tmp_path):
store = MemoryStore()
logs = MemoryStore()
config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}
services = {}
def factory(host_id):
(tmp_path / host_id).mkdir(mode=0o700, exist_ok=True)
scoped = HostStore(store, host_id)
manager = FakeRemoteManager()
manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22,
"fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images",
"iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default",
"libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config",
"ssh_known_hosts_file": "/fake/known_hosts"}
service = Service()
services[host_id] = service
components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager,
DatabaseTemplateStore(scoped), scoped, logs)
app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components,
service_factory=lambda profile: service)
return components, app
registry = HostRegistry(config, store, logs, factory=factory)
async def active(*args):
return True
engine = EngineApplication(config, store, logs, registry=registry, introspection=active)
engine._service = lambda profile: services[profile["host_id"]]
with TestClient(engine, base_url="http://127.0.0.1") as client:
yield client, engine, store, logs, services
def call(client, method, path, actor=USER, value=None, raw=None, headers=None):
body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode()
signed = signed_headers(SECRET, method, path, body, actor)
if value is not None:
signed["Content-Type"] = "application/json"
signed.update(headers or {})
return client.request(method, path, content=body, headers=signed)
def test_hmac_binds_actor_payload_target_and_rejects_replay():
store = MemoryStore()
body = b"{}"
target = "/internal/v1/hosts/hk/vms?range=1h"
headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER,
timestamp=1000, nonce="n" * 24).items()}
assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
changed = {**headers, "x-cloud-actor": encode_actor(SUPER)}
with pytest.raises(EngineAuthError):
authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061)
invalid_cleanup = {**USER, "system_cleanup": "true"}
invalid_headers = {key.lower(): value for key, value in signed_headers(
SECRET, "POST", target, body, invalid_cleanup,
timestamp=1000, nonce="c" * 24,
).items()}
with pytest.raises(EngineAuthError):
authenticate(
invalid_headers, SECRET, "POST", target,
hashlib.sha256(body).hexdigest(), store, now=1000,
)
def test_private_route_allowlist_and_no_browser_admin_cookie(boundary):
client, *_ = boundary
for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin",
"/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"):
assert call(client, "POST", path, SUPER, {}).status_code == 404
assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401
assert TRUSTED_ACTOR.get() is None
def test_internal_update_routes_are_super_admin_only_and_never_echo_token(boundary):
client, engine, *_ = boundary
updater = FakeUpdater()
triggered = []
engine.updater = updater
def trigger():
triggered.append(True)
updater.active["status"] = "running"
engine.update_trigger = trigger
settings_path = "/internal/v1/update/settings"
assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401
assert call(client, "GET", settings_path, USER).status_code == 403
assert call(client, "GET", settings_path, ADMIN).status_code == 403
assert call(client, "GET", settings_path, {**SUPER, "can_manage": False}).status_code == 403
token = "gitea-private-token-123456"
saved = call(client, "PUT", settings_path, SUPER, {"access_token": token})
assert saved.status_code == 200
assert saved.json()["token_configured"] is True
assert saved.json()["repository"] == "https://g.luoli.live/root/virt.git"
assert saved.json()["branch"] == "venix"
assert token not in saved.text
assert token not in call(client, "GET", settings_path, SUPER).text
checked = call(client, "POST", "/internal/v1/update/check", SUPER, {})
assert checked.status_code == 200
assert checked.json()["status"] == "update_available"
applied = call(client, "POST", "/internal/v1/update/apply", SUPER, {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert applied.status_code == 202
assert applied.json()["job_id"] == "update-job-1234567890"
assert "private" not in applied.json() and token not in applied.text
assert triggered == [True]
job = call(client, "GET", "/internal/v1/update/jobs/update-job-1234567890", SUPER)
assert job.status_code == 200 and token not in job.text and "private" not in job.json()
# While the persisted update job is active, reads remain available but no
# host/VM mutation may begin and overlap the code/service restart.
assert call(client, "GET", "/internal/v1/health", SUPER).status_code == 200
assert call(client, "POST", "/internal/v1/hosts/scan", SUPER,
{"host": "host.example.test", "port": 22}).status_code == 409
def test_internal_update_validation_clear_and_trigger_failure(boundary):
client, engine, *_ = boundary
updater = FakeUpdater()
engine.updater = updater
settings_path = "/internal/v1/update/settings"
assert call(client, "PUT", settings_path, SUPER, {}).status_code == 422
assert call(client, "PUT", settings_path, SUPER,
{"access_token": "x", "unexpected": True}).status_code == 422
assert call(client, "POST", "/internal/v1/update/check", SUPER,
{"repository": "https://evil.example/repo.git"}).status_code == 422
assert call(client, "POST", "/internal/v1/update/apply", SUPER,
{"expected_current_commit": "a" * 40}).status_code == 422
assert call(client, "GET", "/internal/v1/update/jobs/missing-job-123456", SUPER).status_code == 404
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
cleared = call(client, "PUT", settings_path, SUPER, {"clear_token": True})
assert cleared.status_code == 200 and cleared.json()["token_configured"] is False
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
def fail_trigger():
from virsh_panel.updater import UpdateError
raise UpdateError("独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503)
engine.update_trigger = fail_trigger
failed = call(client, "POST", "/internal/v1/update/apply", SUPER, {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert failed.status_code == 503
job = updater.jobs["update-job-1234567890"]
assert job["status"] == "failed"
assert "令牌" not in job["error"]
@pytest.mark.parametrize(("state", "expected"), [
("activating", True),
("active", True),
("deactivating", True),
("inactive", False),
("failed", False),
("maintenance", None),
])
def test_update_oneshot_transitional_states_keep_maintenance_gate(state, expected):
with patch("virsh_panel.internal.subprocess.run",
return_value=SimpleNamespace(returncode=0, stdout=state + "\n")):
assert EngineApplication._update_service_active() is expected
def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary):
client, engine, *_ = boundary
path = "/internal/v1/hosts/hk"
assert call(client, "GET", path + "/host").status_code == 403
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200
assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200
assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404
assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == []
assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False},
{"action": "reboot"}).status_code == 403
def test_existing_unmanaged_vms_are_claimed_once_for_configuring_admin(boundary):
client, engine, _, _, services = boundary
path = "/internal/v1/hosts/hk"
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
manual_uuid = "22222222-2222-4222-8222-222222222222"
orphaned_managed_uuid = "33333333-3333-4333-8333-333333333333"
services["hk"].other_vms = [
{"uuid": manual_uuid, "name": "manual-vm", "state": "shutoff",
"vcpu": 2, "memory_mib": 2048, "disks": [], "managed": False},
{"uuid": orphaned_managed_uuid, "name": "unknown-panel-vm", "state": "shutoff",
"vcpu": 2, "memory_mib": 2048, "disks": [], "managed": True},
]
body = {"known_owners": {VM_ID: 41}}
assert call(client, "POST", path + "/host/discover-vms", ADMIN, body).status_code == 403
discovered = call(client, "POST", path + "/host/discover-vms", SUPER, body)
assert discovered.status_code == 200, discovered.text
rows = {row["uuid"]: row for row in discovered.json()["items"]}
assert rows[VM_ID]["owner_user_id"] == 41
assert rows[manual_uuid]["owner_user_id"] == SUPER["user_id"]
assert orphaned_managed_uuid not in rows
assert discovered.json()["claimed_count"] == 2
assert discovered.json()["untracked_managed_count"] == 1
assert discovered.json()["total_count"] == 3
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"]
adopted = call(client, "POST", path + "/host/discover-vms", SUPER,
{**body, "claim_unowned_managed": True})
assert adopted.status_code == 200
adopted_rows = {row["uuid"]: row for row in adopted.json()["items"]}
assert adopted_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"]
assert adopted.json()["claimed_count"] == 1
assert adopted.json()["untracked_managed_count"] == 0
repeated = call(client, "POST", path + "/host/discover-vms",
{**SUPER, "user_id": 2}, {**body, "claim_unowned_managed": True})
assert repeated.status_code == 200
repeated_rows = {row["uuid"]: row for row in repeated.json()["items"]}
assert repeated_rows[manual_uuid]["owner_user_id"] == SUPER["user_id"]
assert repeated_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"]
assert repeated.json()["claimed_count"] == 0
assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"]
assert {row["uuid"] for row in call(
client, "GET", path + "/vms", {**USER, "user_id": SUPER["user_id"]}
).json()["items"]} == {manual_uuid, orphaned_managed_uuid}
assert call(client, "GET", path + "/vms", {**USER, "user_id": 2}).json()["items"] == []
def test_stable_host_identity_and_namespaces():
store = MemoryStore()
first, second = HostStore(store, "a"), HostStore(store, "b")
first.set("host", "current", "one")
second.set("host", "current", "two")
assert first.get("host", "current") == "one"
assert second.get("host", "current") == "two"
a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"}
assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"})
assert host_scope(a) != host_scope({**a, "host_id": "b"})
assert MainStore.RECORD_TABLE == "engine_records"
assert LogStore.METRIC_TABLE == "engine_vm_metric_samples"
def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403
provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"}
created = call(client, "POST", path, provisioning, CREATE_BODY)
assert created.status_code == 201, created.text
vm_id = created.json()["uuid"]
assert created.json()["result"]["uuid"] == vm_id
assert created.json()["status"] == "succeeded"
component = engine.registry.components["hk"]
assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41)
again = call(client, "POST", path, provisioning, CREATE_BODY)
assert again.status_code == 201 and again.json()["uuid"] == vm_id
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409
operation = call(client, "GET", "/internal/v1/operations/order-000001").json()
assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id
assert operation["safe_to_refund"] is False
assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404
def test_uncertain_creation_never_refunds_or_recreates(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
call(client, "GET", path)
services["hk"].create_failure = True
actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"}
failed = call(client, "POST", path, actor, CREATE_BODY)
assert failed.status_code == 502
assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"]
record = json.loads(store.get("operation", "order-uncertain"))
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
assert component.state.users.owns_vm(scope, record["vm_uuid"], 41)
component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password")
component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]}))
again = call(client, "POST", path, actor, CREATE_BODY)
assert again.status_code == 202 and again.json()["status"] == "uncertain"
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
services["hk"].absent = True
operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json()
assert operation["status"] == "failed" and operation["safe_to_refund"] is True
assert not component.state.users.owns_vm(scope, record["vm_uuid"], 41)
assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") is None
assert component.store.list("ip_claim") == {}
@pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"],
{**CREATE_BODY, "guest_ssh_key_id": 3}])
def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload):
client, engine, _, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
actor = {**USER, "can_provision": True, "operation_id": "rejected-create"}
response = call(client, "POST", path, actor, payload)
assert response.status_code == 422
# Simulate a dispatcher losing the POST response and querying after restart.
engine.active_operations.clear()
saved = call(client, "GET", "/internal/v1/operations/rejected-create").json()
assert saved["status"] == "failed" and saved["safe_to_refund"] is True
assert call(client, "POST", path, actor, payload).status_code == 422
assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409
assert not any(row[0] == "create_vm" for row in services["hk"].calls)
def test_admin_created_vm_also_has_external_owner(boundary):
client, engine, *_ = boundary
actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"}
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
assert response.status_code == 201
components = engine.registry.components["hk"]
assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1)
def test_power_action_has_persistent_idempotency(boundary):
client, engine, _, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403
actor = {**USER, "operation_id": "reboot-operation"}
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1
assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409
operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json()
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary):
client, engine, _, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
rescue_key = f"{scope}.{VM_ID}"
components.store.set("vm_rescue", rescue_key, json.dumps({
"original_xml": "<domain><name>alma-test</name></domain>",
"template_id": "0123456789abcdef0123456789abcdef",
"volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso",
"status": "active",
}))
services["hk"].vm.update(
state="shutoff", rescue_mode=True, rescue_staged=True,
rescue_volume_name="panel-rescue-0123456789abcdef0123456789abcdef.iso",
)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
ordinary = call(
client, "DELETE", path,
{**USER, "operation_id": "ordinary-rescue-delete", "system_cleanup": False},
)
assert ordinary.status_code == 409
assert components.store.get("vm_rescue", rescue_key) is not None
assert not any(item[0] == "delete_vm" for item in services["hk"].calls)
cleanup = call(
client, "DELETE", path,
{**USER, "operation_id": "expiry-rescue-delete", "system_cleanup": True},
)
assert cleanup.status_code == 200, cleanup.text
assert cleanup.json()["status"] == "succeeded"
assert services["hk"].calls[-1] == ("delete_vm", VM_ID, True)
assert components.store.get("vm_rescue", rescue_key) is None
assert components.state.users.owner_of_vm(scope, VM_ID) is None
@pytest.mark.parametrize(("route", "method", "target"), [
("service-suspend", "service_suspend", {"state": "shutoff", "autostart": False}),
("service-resume", "service_resume", {"autostart": True}),
("network-suspend", "network_suspend", {"network_suspended": True}),
("network-resume", "network_resume", {"network_suspended": False}),
])
def test_private_target_state_operations_are_owned_idempotent_mutations(
boundary, route, method, target):
client, engine, _, _, services = boundary
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
assert call(client, "POST", base, USER, {}).status_code == 403
assert call(client, "POST", base, {**USER, "can_manage": False,
"operation_id": f"blocked-{route}"}, {}).status_code == 403
assert call(client, "POST", base, {**USER, "user_id": 42,
"operation_id": f"foreign-{route}"}, {}).status_code == 404
actor = {**USER, "operation_id": f"target-{route}"}
first = call(client, "POST", base, actor, {})
assert first.status_code == 200, first.text
assert first.json()["status"] == "succeeded"
assert all(first.json()[key] == value for key, value in target.items())
repeated = call(client, "POST", base, actor, {})
assert repeated.status_code == 200
assert len([item for item in services["hk"].calls if item[0] == method]) == 1
@pytest.mark.parametrize(("route", "method", "drift"), [
("service-suspend", "service_suspend", {"state": "running", "autostart": True}),
("service-resume", "service_resume", {"autostart": False}),
("network-suspend", "network_suspend", {"network_suspended": False}),
("network-resume", "network_resume", {"network_suspended": True}),
])
def test_target_state_operation_replays_after_lost_response(
boundary, route, method, drift):
client, engine, store, _, services = boundary
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
actor = {**USER, "operation_id": f"replay-{route}"}
assert call(client, "POST", base, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(drift)
engine.active_operations.clear()
replayed = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
actor)
assert replayed.status_code == 200, replayed.text
assert replayed.json()["status"] == "succeeded"
assert len([item for item in services["hk"].calls if item[0] == method]) == 2
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost again")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(drift)
replayed_post = call(client, "POST", base, actor, {})
assert replayed_post.status_code == 200, replayed_post.text
assert replayed_post.json()["status"] == "succeeded"
assert len([item for item in services["hk"].calls if item[0] == method]) == 3
def test_non_reconciling_operation_lookup_never_replays_target_state(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend"
actor = {**USER, "operation_id": "peek-target-state"}
assert call(client, "POST", path, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm["network_suspended"] = False
engine.active_operations.clear()
before = len(services["hk"].calls)
peeked = call(client, "GET",
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
assert peeked.status_code == 200
assert peeked.json()["status"] == "uncertain"
assert peeked.json()["executor_active"] is False
assert len(services["hk"].calls) == before
assert services["hk"].vm["network_suspended"] is False
engine.active_operations.add(actor["operation_id"])
active = call(client, "GET",
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
assert active.status_code == 200
assert active.json()["executor_active"] is True
assert len(services["hk"].calls) == before
engine.active_operations.discard(actor["operation_id"])
def test_target_state_lock_serializes_same_vm_and_not_other_vms(boundary):
_, engine, *_ = boundary
async def exercise():
first = engine._target_lock("hk", VM_ID)
same = engine._target_lock("hk", VM_ID)
other = engine._target_lock("hk", "22222222-2222-4222-8222-222222222222")
assert first is same
assert first is not other
entered = []
release = asyncio.Event()
async def owner():
async with first:
entered.append("owner")
await release.wait()
entered.append("owner-finished")
async def waiter():
async with same:
entered.append("waiter")
owner_task = asyncio.create_task(owner())
await asyncio.sleep(0)
waiter_task = asyncio.create_task(waiter())
await asyncio.sleep(0)
assert entered == ["owner"]
release.set()
await asyncio.gather(owner_task, waiter_task)
assert entered == ["owner", "owner-finished", "waiter"]
errors = []
def run_in_isolated_loop():
try:
asyncio.run(exercise())
except BaseException as exc: # propagate assertion failures to pytest
errors.append(exc)
thread = threading.Thread(target=run_in_isolated_loop)
thread.start()
thread.join(timeout=5)
assert not thread.is_alive()
if errors:
raise errors[0]
def test_expiry_target_waits_for_inflight_user_mutation_on_same_vm(boundary):
client, engine, _, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
service = services["hk"]
action_entered = threading.Event()
release_action = threading.Event()
network_entered = threading.Event()
original_action = service.action
original_network_suspend = service.network_suspend
def slow_action(vm_id, action):
action_entered.set()
assert release_action.wait(timeout=5), "test did not release user mutation"
return original_action(vm_id, action)
def observed_network_suspend(vm_id):
network_entered.set()
return original_network_suspend(vm_id)
service.action = slow_action
service.network_suspend = observed_network_suspend
responses = {}
def user_request():
responses["action"] = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions",
{**USER, "operation_id": "concurrent-user-action"}, {"action": "reset"})
def policy_request():
responses["policy"] = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
{**USER, "operation_id": "concurrent-expiry-policy"}, {})
first = threading.Thread(target=user_request)
first.start()
assert action_entered.wait(timeout=5)
second = threading.Thread(target=policy_request)
second.start()
assert not network_entered.wait(timeout=0.2)
release_action.set()
first.join(timeout=5)
second.join(timeout=5)
assert not first.is_alive() and not second.is_alive()
assert responses["action"].status_code == 200
assert responses["policy"].status_code == 200
assert network_entered.is_set()
mutations = [item[0] for item in service.calls
if item[0] in {"action", "network_suspend"}]
assert mutations == ["action", "network_suspend"]
assert service.vm["network_suspended"] is True
def test_target_state_reconciliation_does_not_bypass_original_owner(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
store.set("operation", "unsafe-target", json.dumps({
"operation_id": "unsafe-target", "host_id": "hk", "user_id": 42,
"kind": "mutation", "resource": f"vms/{VM_ID}/network-suspend",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"actor_can_manage": True, "actor_is_admin": False,
"safe_to_refund": False,
}))
response = call(client, "GET", "/internal/v1/operations/unsafe-target",
{**USER, "user_id": 42})
assert response.status_code == 200
assert response.json()["status"] == "failed"
assert not any(item[0] == "network_suspend" for item in services["hk"].calls)
def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID,
"status": "running", "phase": "正在救援"}
components.store.set("rescue_job", "job-abc", json.dumps(job))
operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID,
"status": "uncertain", "result": None, "safe_to_refund": False}
store.set("operation", "lost-rescue-job", json.dumps(operation))
recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID
job.update(status="completed", phase="救援系统启动")
components.store.set("rescue_job", "job-abc", json.dumps(job))
completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
assert completed["status"] == "succeeded"
assert not completed["safe_to_refund"]
def test_interrupted_async_job_becomes_terminal_without_replay(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.store.set("reinstall_job", "job-abcd", json.dumps({
"operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID,
"status": "failed", "phase": "面板重启,任务已中断"}))
store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job",
"host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall",
"vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False}))
recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json()
assert recovered["status"] == "failed"
assert "中断" in recovered["error"]
@pytest.mark.parametrize("resource", [
f"vms/{VM_ID}/actions",
f"vms/{VM_ID}",
f"vms/{VM_ID}/reinstall",
f"vms/{VM_ID}/rescue",
])
def test_orphaned_mutation_becomes_terminal_without_replay(boundary, resource):
client, _, store, *_ = boundary
operation_id = "orphan-" + resource.rsplit("/", 1)[-1].replace("?", "-")
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": resource, "vm_uuid": VM_ID,
"status": "running", "result": None, "safe_to_refund": False,
"created_at": 1,
}))
recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}").json()
assert recovered["status"] == "failed"
assert "不会自动重放" in recovered["error"]
def test_service_resume_replays_exact_start_target_after_lost_response(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
services["hk"].vm.update(state="shutoff", autostart=False)
actor = {**USER, "operation_id": "resume-start-target"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume"
first = call(client, "POST", path, actor, {"start_if_suspended": True})
assert first.status_code == 200, first.text
assert services["hk"].vm["state"] == "running"
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(state="shutoff", autostart=False)
engine.active_operations.clear()
replayed = call(client, "GET", "/internal/v1/operations/resume-start-target", actor)
assert replayed.status_code == 200
assert replayed.json()["status"] == "succeeded"
assert services["hk"].vm["state"] == "running"
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
def test_service_suspend_replay_preserves_original_running_state(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
actor = {**USER, "operation_id": "suspend-running-hint"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend"
assert call(client, "POST", path, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
assert saved["service_was_running_hint"] is True
# Model a hard crash after libvirt stopped the guest but before the
# successful result was durable. Reconciliation now observes shutoff, yet
# must retain the original running state for the billing resume policy.
saved.update(status="uncertain", result=None, error="engine crashed")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(state="shutoff", autostart=False)
engine.active_operations.clear()
reconciled = call(client, "GET",
"/internal/v1/operations/suspend-running-hint", actor)
assert reconciled.status_code == 200
assert reconciled.json()["status"] == "succeeded"
assert reconciled.json()["result"]["service_was_running"] is True
def test_service_suspend_replay_without_running_hint_fails_closed(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
operation_id = "suspend-missing-running-hint"
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id,
"host_id": "hk",
"user_id": USER["user_id"],
"vm_uuid": VM_ID,
"kind": "mutation",
"resource": f"vms/{VM_ID}/service-suspend",
"method": "POST",
"status": "uncertain",
"actor_can_manage": True,
"actor_is_admin": False,
"created_at": 1,
}))
before_calls = list(services["hk"].calls)
response = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
assert response.status_code == 200
assert response.json()["status"] == "failed"
assert "已取消服务暂停" in response.json()["error"]
assert services["hk"].calls == before_calls
def test_service_suspend_does_not_stop_vm_when_running_hint_cannot_persist(
boundary, monkeypatch):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
operation_id = "suspend-hint-write-failed"
original_set = store.set
def fail_operation_write(kind, key, value):
if kind == "operation" and key == operation_id:
raise RuntimeError("database unavailable")
return original_set(kind, key, value)
monkeypatch.setattr(store, "set", fail_operation_write)
failed = call(
client, "POST",
f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend",
{**USER, "operation_id": operation_id}, {})
assert failed.status_code == 503
assert services["hk"].vm["state"] == "running"
assert not any(
item[0] == "service_suspend" for item in services["hk"].calls)
monkeypatch.setattr(store, "set", original_set)
recovered = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
assert recovered.status_code == 200
assert recovered.json()["status"] == "failed"
assert services["hk"].vm["state"] == "running"
assert not any(
item[0] == "service_suspend" for item in services["hk"].calls)
def test_metrics_query_uses_log_store_and_owner_permission(boundary):
client, engine, _, logs, _ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
component.state.users.assign_vm(scope, VM_ID, 41)
response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d")
assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20
with patch("virsh_panel.app.time.time", return_value=3_000_000):
traffic = call(
client, "GET",
f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily?days=2",
)
assert traffic.status_code == 200
assert traffic.json()["points"][-1] == {
"date": "1970-02-04", "day_start": 2_937_600,
"download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120,
}
assert logs.daily_traffic_calls == [(scope, VM_ID, {
"host_id": "hk", "user_id": 41,
"since": 2_851_200, "until": 3_000_000, "limit": 2,
})]
denied = call(
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily",
{**USER, "user_id": 42},
)
assert denied.status_code == 404
assert len(logs.daily_traffic_calls) == 1
def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
assert issued.status_code == 200, issued.text
assert issued.json()["websocket_path"] == "/engine-console"
ticket = issued.json()["ticket"]
key = hashlib.sha256(ticket.encode()).hexdigest()
record = json.loads(store.get("console_ticket", key))
assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session"
assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket",
{**USER, "user_id": 42}, {}).status_code == 404
from starlette.websockets import WebSocketDisconnect
with pytest.raises(WebSocketDisconnect):
with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"},
subprotocols=["cloud-ticket." + ticket]):
pass
assert store.get("console_ticket", key) is not None
def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
checks = 0
async def active(*args):
nonlocal checks
checks += 1
return checks <= 2 # Ticket issuance + handshake; then PHP revokes.
engine.introspection = active
engine.console_check_interval = 0.01
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
ticket = issued.json()["ticket"]
protocol = "cloud-ticket." + ticket
async def relay(socket, **kwargs):
await socket.send_bytes(b"vnc")
await asyncio.Event().wait()
from starlette.websockets import WebSocketDisconnect
with patch("virsh_panel.console.relay_console", relay):
with client.websocket_connect("wss://panel.example.test/engine-console",
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket:
assert socket.accepted_subprotocol == protocol
assert socket.receive_bytes() == b"vnc"
with pytest.raises(WebSocketDisconnect):
socket.receive_bytes()
key = hashlib.sha256(ticket.encode()).hexdigest()
assert store.get("console_ticket", key) is None
assert store.get("ticket_used", key) == "1"
with pytest.raises(WebSocketDisconnect):
with client.websocket_connect("wss://panel.example.test/engine-console",
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]):
pass
def test_streamed_iso_digest_must_match_before_import_is_queued(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/rescue-templates?name=rescue"
signed_body = b"A" * (1024 * 1024)
wrong_body = b"B" * len(signed_body)
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest()
signed["Content-Type"] = "application/octet-stream"
response = client.post(path, content=wrong_body, headers=signed)
assert response.status_code == 401, response.text
assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls)
assert engine.registry.components["hk"].store.list("rescue_import_job") == {}
def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables():
connection = MagicMock()
cursor = connection.cursor.return_value.__enter__.return_value
config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test",
"ssl_ca": "/private/test/ca.pem"}
with patch("virsh_panel.database.pymysql.connect", return_value=connection):
main = MainStore(config, b"m" * 32)
statements = [call.args[0] for call in cursor.execute.call_args_list]
assert any("engine_records" in statement for statement in statements)
assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements)
assert not any("vm_metric_samples" in statement for statement in statements)
assert main._config["ssl_verify_identity"] is True
cursor.reset_mock()
LogStore({**config, "database": "logs"}, b"m" * 32)
statements = [call.args[0] for call in cursor.execute.call_args_list]
assert "engine_vm_metric_samples" in statements[0]
assert "engine_vm_traffic_totals" in statements[1]
assert "engine_vm_traffic_checkpoints" in statements[2]
assert all("ALTER TABLE engine_vm_metric_samples" in statement
for statement in statements[3:])
assert LogStore.TRAFFIC_TABLE == "engine_vm_traffic_totals"
assert LogStore.TRAFFIC_CHECKPOINT_TABLE == "engine_vm_traffic_checkpoints"