You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_engine_boundary.py

414 lines
21 KiB
Python

"""Private engine tests: no PHP cookies, no real DB and no real hypervisor."""
from __future__ import annotations
import base64
import asyncio
import hashlib
import json
import threading
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from test_app import CREATE_BODY, FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID
from virsh_panel.app import create_app
from virsh_panel.config import Settings
from virsh_panel.database import DatabaseTemplateStore
from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore,
LogStore, MainStore, TRUSTED_ACTOR)
from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor,
signed_headers)
from virsh_panel.internal import EngineApplication, HostRegistry
from virsh_panel.metrics import host_scope
SECRET = b"s" * 32
USER = {"user_id": 41, "is_admin": False, "is_super_admin": False,
"can_manage": True, "session_id": "test-session"}
SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True}
ADMIN = {**SUPER, "is_super_admin": False}
class MemoryStore:
def __init__(self):
self.records = {}
self.nonces = set()
self.lock = threading.RLock()
def get(self, kind, key):
with self.lock:
return self.records.get((kind, key))
def set(self, kind, key, value):
with self.lock:
self.records[kind, key] = value
def insert(self, kind, key, value):
with self.lock:
if (kind, key) in self.records:
return False
self.records[kind, key] = value
return True
def delete(self, kind, key):
with self.lock:
self.records.pop((kind, key), None)
def list(self, kind):
with self.lock:
return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind}
def consume_nonce(self, nonce, expires_at, now):
with self.lock:
if nonce in self.nonces:
return False
self.nonces.add(nonce)
return True
def query_metric_samples(self, *args, **kwargs):
return [{"timestamp": 123, "cpu_percent": 20.0}]
class Service(FakeVMService):
def __init__(self):
super().__init__()
self.create_failure = False
self.absent = False
self.vm.update(managed=True, os_source="iso")
def create_vm(self, spec):
self.calls.append(("create_vm", spec))
if self.create_failure:
from virsh_panel.libvirt_service import VMError
raise VMError("宿主机失联,回滚状态不明", 502)
self.vm.update(uuid=spec["vm_uuid"], name=spec["name"])
return self.vm.copy()
def provisioning_absent(self, vm_uuid, name):
return self.absent
@pytest.fixture
def boundary(tmp_path):
store = MemoryStore()
logs = MemoryStore()
config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}
services = {}
def factory(host_id):
(tmp_path / host_id).mkdir(mode=0o700, exist_ok=True)
scoped = HostStore(store, host_id)
manager = FakeRemoteManager()
manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22,
"fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images",
"iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default",
"libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config",
"ssh_known_hosts_file": "/fake/known_hosts"}
service = Service()
services[host_id] = service
components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager,
DatabaseTemplateStore(scoped), scoped, logs)
app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components,
service_factory=lambda profile: service)
return components, app
registry = HostRegistry(config, store, logs, factory=factory)
async def active(*args):
return True
engine = EngineApplication(config, store, logs, registry=registry, introspection=active)
engine._service = lambda profile: services[profile["host_id"]]
with TestClient(engine, base_url="http://127.0.0.1") as client:
yield client, engine, store, logs, services
def call(client, method, path, actor=USER, value=None, raw=None, headers=None):
body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode()
signed = signed_headers(SECRET, method, path, body, actor)
if value is not None:
signed["Content-Type"] = "application/json"
signed.update(headers or {})
return client.request(method, path, content=body, headers=signed)
def test_hmac_binds_actor_payload_target_and_rejects_replay():
store = MemoryStore()
body = b"{}"
target = "/internal/v1/hosts/hk/vms?range=1h"
headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER,
timestamp=1000, nonce="n" * 24).items()}
assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
changed = {**headers, "x-cloud-actor": encode_actor(SUPER)}
with pytest.raises(EngineAuthError):
authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061)
def test_private_route_allowlist_and_no_browser_admin_cookie(boundary):
client, *_ = boundary
for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin",
"/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"):
assert call(client, "POST", path, SUPER, {}).status_code == 404
assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401
assert TRUSTED_ACTOR.get() is None
def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary):
client, engine, *_ = boundary
path = "/internal/v1/hosts/hk"
assert call(client, "GET", path + "/host").status_code == 403
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200
assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200
assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404
assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == []
assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False},
{"action": "reboot"}).status_code == 403
def test_stable_host_identity_and_namespaces():
store = MemoryStore()
first, second = HostStore(store, "a"), HostStore(store, "b")
first.set("host", "current", "one")
second.set("host", "current", "two")
assert first.get("host", "current") == "one"
assert second.get("host", "current") == "two"
a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"}
assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"})
assert host_scope(a) != host_scope({**a, "host_id": "b"})
assert MainStore.RECORD_TABLE == "engine_records"
assert LogStore.METRIC_TABLE == "engine_vm_metric_samples"
def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403
provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"}
created = call(client, "POST", path, provisioning, CREATE_BODY)
assert created.status_code == 201, created.text
vm_id = created.json()["uuid"]
assert created.json()["result"]["uuid"] == vm_id
assert created.json()["status"] == "succeeded"
component = engine.registry.components["hk"]
assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41)
again = call(client, "POST", path, provisioning, CREATE_BODY)
assert again.status_code == 201 and again.json()["uuid"] == vm_id
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409
operation = call(client, "GET", "/internal/v1/operations/order-000001").json()
assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id
assert operation["safe_to_refund"] is False
assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404
def test_uncertain_creation_never_refunds_or_recreates(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
call(client, "GET", path)
services["hk"].create_failure = True
actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"}
failed = call(client, "POST", path, actor, CREATE_BODY)
assert failed.status_code == 502
assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"]
record = json.loads(store.get("operation", "order-uncertain"))
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
assert component.state.users.owns_vm(scope, record["vm_uuid"], 41)
component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password")
component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]}))
again = call(client, "POST", path, actor, CREATE_BODY)
assert again.status_code == 202 and again.json()["status"] == "uncertain"
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
services["hk"].absent = True
operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json()
assert operation["status"] == "failed" and operation["safe_to_refund"] is True
assert not component.state.users.owns_vm(scope, record["vm_uuid"], 41)
assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") is None
assert component.store.list("ip_claim") == {}
@pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"],
{**CREATE_BODY, "guest_ssh_key_id": 3}])
def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload):
client, engine, _, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
actor = {**USER, "can_provision": True, "operation_id": "rejected-create"}
response = call(client, "POST", path, actor, payload)
assert response.status_code == 422
# Simulate a dispatcher losing the POST response and querying after restart.
engine.active_operations.clear()
saved = call(client, "GET", "/internal/v1/operations/rejected-create").json()
assert saved["status"] == "failed" and saved["safe_to_refund"] is True
assert call(client, "POST", path, actor, payload).status_code == 422
assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409
assert not any(row[0] == "create_vm" for row in services["hk"].calls)
def test_admin_created_vm_also_has_external_owner(boundary):
client, engine, *_ = boundary
actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"}
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
assert response.status_code == 201
components = engine.registry.components["hk"]
assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1)
def test_power_action_has_persistent_idempotency(boundary):
client, engine, _, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403
actor = {**USER, "operation_id": "reboot-operation"}
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1
assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409
operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json()
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID,
"status": "running", "phase": "正在救援"}
components.store.set("rescue_job", "job-abc", json.dumps(job))
operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID,
"status": "uncertain", "result": None, "safe_to_refund": False}
store.set("operation", "lost-rescue-job", json.dumps(operation))
recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID
job.update(status="completed", phase="救援系统启动")
components.store.set("rescue_job", "job-abc", json.dumps(job))
completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
assert completed["status"] == "succeeded"
assert not completed["safe_to_refund"]
def test_interrupted_async_job_stays_uncertain(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.store.set("reinstall_job", "job-abcd", json.dumps({
"operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID,
"status": "failed", "phase": "面板重启,任务已中断"}))
store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job",
"host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall",
"vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False}))
recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json()
assert recovered["status"] == "uncertain"
def test_metrics_query_uses_log_store_and_owner_permission(boundary):
client, engine, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d")
assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20
def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
assert issued.status_code == 200, issued.text
assert issued.json()["websocket_path"] == "/engine-console"
ticket = issued.json()["ticket"]
key = hashlib.sha256(ticket.encode()).hexdigest()
record = json.loads(store.get("console_ticket", key))
assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session"
assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket",
{**USER, "user_id": 42}, {}).status_code == 404
from starlette.websockets import WebSocketDisconnect
with pytest.raises(WebSocketDisconnect):
with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"},
subprotocols=["cloud-ticket." + ticket]):
pass
assert store.get("console_ticket", key) is not None
def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
checks = 0
async def active(*args):
nonlocal checks
checks += 1
return checks <= 2 # Ticket issuance + handshake; then PHP revokes.
engine.introspection = active
engine.console_check_interval = 0.01
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
ticket = issued.json()["ticket"]
protocol = "cloud-ticket." + ticket
async def relay(socket, **kwargs):
await socket.send_bytes(b"vnc")
await asyncio.Event().wait()
from starlette.websockets import WebSocketDisconnect
with patch("virsh_panel.console.relay_console", relay):
with client.websocket_connect("wss://panel.example.test/engine-console",
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket:
assert socket.accepted_subprotocol == protocol
assert socket.receive_bytes() == b"vnc"
with pytest.raises(WebSocketDisconnect):
socket.receive_bytes()
key = hashlib.sha256(ticket.encode()).hexdigest()
assert store.get("console_ticket", key) is None
assert store.get("ticket_used", key) == "1"
with pytest.raises(WebSocketDisconnect):
with client.websocket_connect("wss://panel.example.test/engine-console",
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]):
pass
def test_streamed_iso_digest_must_match_before_import_is_queued(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/rescue-templates?name=rescue"
signed_body = b"A" * (1024 * 1024)
wrong_body = b"B" * len(signed_body)
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest()
signed["Content-Type"] = "application/octet-stream"
response = client.post(path, content=wrong_body, headers=signed)
assert response.status_code == 401, response.text
assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls)
assert engine.registry.components["hk"].store.list("rescue_import_job") == {}
def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables():
connection = MagicMock()
cursor = connection.cursor.return_value.__enter__.return_value
config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test",
"ssl_ca": "/private/test/ca.pem"}
with patch("virsh_panel.database.pymysql.connect", return_value=connection):
main = MainStore(config, b"m" * 32)
statements = [call.args[0] for call in cursor.execute.call_args_list]
assert any("engine_records" in statement for statement in statements)
assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements)
assert not any("vm_metric_samples" in statement for statement in statements)
assert main._config["ssl_verify_identity"] is True
cursor.reset_mock()
LogStore({**config, "database": "logs"}, b"m" * 32)
statements = [call.args[0] for call in cursor.execute.call_args_list]
assert len(statements) == 1 and "engine_vm_metric_samples" in statements[0]