You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
414 lines
21 KiB
Python
414 lines
21 KiB
Python
"""Private engine tests: no PHP cookies, no real DB and no real hypervisor."""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import threading
|
|
from contextlib import contextmanager
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from test_app import CREATE_BODY, FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID
|
|
from virsh_panel.app import create_app
|
|
from virsh_panel.config import Settings
|
|
from virsh_panel.database import DatabaseTemplateStore
|
|
from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore,
|
|
LogStore, MainStore, TRUSTED_ACTOR)
|
|
from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor,
|
|
signed_headers)
|
|
from virsh_panel.internal import EngineApplication, HostRegistry
|
|
from virsh_panel.metrics import host_scope
|
|
|
|
|
|
SECRET = b"s" * 32
|
|
USER = {"user_id": 41, "is_admin": False, "is_super_admin": False,
|
|
"can_manage": True, "session_id": "test-session"}
|
|
SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True}
|
|
ADMIN = {**SUPER, "is_super_admin": False}
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self):
|
|
self.records = {}
|
|
self.nonces = set()
|
|
self.lock = threading.RLock()
|
|
|
|
def get(self, kind, key):
|
|
with self.lock:
|
|
return self.records.get((kind, key))
|
|
|
|
def set(self, kind, key, value):
|
|
with self.lock:
|
|
self.records[kind, key] = value
|
|
|
|
def insert(self, kind, key, value):
|
|
with self.lock:
|
|
if (kind, key) in self.records:
|
|
return False
|
|
self.records[kind, key] = value
|
|
return True
|
|
|
|
def delete(self, kind, key):
|
|
with self.lock:
|
|
self.records.pop((kind, key), None)
|
|
|
|
def list(self, kind):
|
|
with self.lock:
|
|
return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind}
|
|
|
|
def consume_nonce(self, nonce, expires_at, now):
|
|
with self.lock:
|
|
if nonce in self.nonces:
|
|
return False
|
|
self.nonces.add(nonce)
|
|
return True
|
|
|
|
def query_metric_samples(self, *args, **kwargs):
|
|
return [{"timestamp": 123, "cpu_percent": 20.0}]
|
|
|
|
|
|
class Service(FakeVMService):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.create_failure = False
|
|
self.absent = False
|
|
self.vm.update(managed=True, os_source="iso")
|
|
|
|
def create_vm(self, spec):
|
|
self.calls.append(("create_vm", spec))
|
|
if self.create_failure:
|
|
from virsh_panel.libvirt_service import VMError
|
|
raise VMError("宿主机失联,回滚状态不明", 502)
|
|
self.vm.update(uuid=spec["vm_uuid"], name=spec["name"])
|
|
return self.vm.copy()
|
|
|
|
def provisioning_absent(self, vm_uuid, name):
|
|
return self.absent
|
|
|
|
|
|
@pytest.fixture
|
|
def boundary(tmp_path):
|
|
store = MemoryStore()
|
|
logs = MemoryStore()
|
|
config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}
|
|
services = {}
|
|
def factory(host_id):
|
|
(tmp_path / host_id).mkdir(mode=0o700, exist_ok=True)
|
|
scoped = HostStore(store, host_id)
|
|
manager = FakeRemoteManager()
|
|
manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22,
|
|
"fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images",
|
|
"iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default",
|
|
"libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config",
|
|
"ssh_known_hosts_file": "/fake/known_hosts"}
|
|
service = Service()
|
|
services[host_id] = service
|
|
components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager,
|
|
DatabaseTemplateStore(scoped), scoped, logs)
|
|
app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components,
|
|
service_factory=lambda profile: service)
|
|
return components, app
|
|
registry = HostRegistry(config, store, logs, factory=factory)
|
|
async def active(*args):
|
|
return True
|
|
engine = EngineApplication(config, store, logs, registry=registry, introspection=active)
|
|
engine._service = lambda profile: services[profile["host_id"]]
|
|
with TestClient(engine, base_url="http://127.0.0.1") as client:
|
|
yield client, engine, store, logs, services
|
|
|
|
|
|
def call(client, method, path, actor=USER, value=None, raw=None, headers=None):
|
|
body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode()
|
|
signed = signed_headers(SECRET, method, path, body, actor)
|
|
if value is not None:
|
|
signed["Content-Type"] = "application/json"
|
|
signed.update(headers or {})
|
|
return client.request(method, path, content=body, headers=signed)
|
|
|
|
|
|
def test_hmac_binds_actor_payload_target_and_rejects_replay():
|
|
store = MemoryStore()
|
|
body = b"{}"
|
|
target = "/internal/v1/hosts/hk/vms?range=1h"
|
|
headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER,
|
|
timestamp=1000, nonce="n" * 24).items()}
|
|
assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
changed = {**headers, "x-cloud-actor": encode_actor(SUPER)}
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061)
|
|
|
|
|
|
def test_private_route_allowlist_and_no_browser_admin_cookie(boundary):
|
|
client, *_ = boundary
|
|
for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin",
|
|
"/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"):
|
|
assert call(client, "POST", path, SUPER, {}).status_code == 404
|
|
assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401
|
|
assert TRUSTED_ACTOR.get() is None
|
|
|
|
|
|
def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary):
|
|
client, engine, *_ = boundary
|
|
path = "/internal/v1/hosts/hk"
|
|
assert call(client, "GET", path + "/host").status_code == 403
|
|
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
|
|
assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
|
|
assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200
|
|
assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200
|
|
assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404
|
|
assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == []
|
|
assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False},
|
|
{"action": "reboot"}).status_code == 403
|
|
|
|
|
|
def test_stable_host_identity_and_namespaces():
|
|
store = MemoryStore()
|
|
first, second = HostStore(store, "a"), HostStore(store, "b")
|
|
first.set("host", "current", "one")
|
|
second.set("host", "current", "two")
|
|
assert first.get("host", "current") == "one"
|
|
assert second.get("host", "current") == "two"
|
|
a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"}
|
|
assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"})
|
|
assert host_scope(a) != host_scope({**a, "host_id": "b"})
|
|
assert MainStore.RECORD_TABLE == "engine_records"
|
|
assert LogStore.METRIC_TABLE == "engine_vm_metric_samples"
|
|
|
|
|
|
def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403
|
|
provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"}
|
|
created = call(client, "POST", path, provisioning, CREATE_BODY)
|
|
assert created.status_code == 201, created.text
|
|
vm_id = created.json()["uuid"]
|
|
assert created.json()["result"]["uuid"] == vm_id
|
|
assert created.json()["status"] == "succeeded"
|
|
component = engine.registry.components["hk"]
|
|
assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41)
|
|
again = call(client, "POST", path, provisioning, CREATE_BODY)
|
|
assert again.status_code == 201 and again.json()["uuid"] == vm_id
|
|
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
|
|
assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409
|
|
operation = call(client, "GET", "/internal/v1/operations/order-000001").json()
|
|
assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id
|
|
assert operation["safe_to_refund"] is False
|
|
assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404
|
|
|
|
|
|
def test_uncertain_creation_never_refunds_or_recreates(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
call(client, "GET", path)
|
|
services["hk"].create_failure = True
|
|
actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"}
|
|
failed = call(client, "POST", path, actor, CREATE_BODY)
|
|
assert failed.status_code == 502
|
|
assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"]
|
|
record = json.loads(store.get("operation", "order-uncertain"))
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
assert component.state.users.owns_vm(scope, record["vm_uuid"], 41)
|
|
component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password")
|
|
component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]}))
|
|
again = call(client, "POST", path, actor, CREATE_BODY)
|
|
assert again.status_code == 202 and again.json()["status"] == "uncertain"
|
|
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
|
|
services["hk"].absent = True
|
|
operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json()
|
|
assert operation["status"] == "failed" and operation["safe_to_refund"] is True
|
|
assert not component.state.users.owns_vm(scope, record["vm_uuid"], 41)
|
|
assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") is None
|
|
assert component.store.list("ip_claim") == {}
|
|
|
|
|
|
@pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"],
|
|
{**CREATE_BODY, "guest_ssh_key_id": 3}])
|
|
def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload):
|
|
client, engine, _, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
actor = {**USER, "can_provision": True, "operation_id": "rejected-create"}
|
|
response = call(client, "POST", path, actor, payload)
|
|
assert response.status_code == 422
|
|
# Simulate a dispatcher losing the POST response and querying after restart.
|
|
engine.active_operations.clear()
|
|
saved = call(client, "GET", "/internal/v1/operations/rejected-create").json()
|
|
assert saved["status"] == "failed" and saved["safe_to_refund"] is True
|
|
assert call(client, "POST", path, actor, payload).status_code == 422
|
|
assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409
|
|
assert not any(row[0] == "create_vm" for row in services["hk"].calls)
|
|
|
|
|
|
def test_admin_created_vm_also_has_external_owner(boundary):
|
|
client, engine, *_ = boundary
|
|
actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"}
|
|
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
|
|
assert response.status_code == 201
|
|
components = engine.registry.components["hk"]
|
|
assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1)
|
|
|
|
|
|
def test_power_action_has_persistent_idempotency(boundary):
|
|
client, engine, _, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
|
assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403
|
|
actor = {**USER, "operation_id": "reboot-operation"}
|
|
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
|
|
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
|
|
assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1
|
|
assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409
|
|
operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json()
|
|
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
|
|
|
|
|
|
def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID,
|
|
"status": "running", "phase": "正在救援"}
|
|
components.store.set("rescue_job", "job-abc", json.dumps(job))
|
|
operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID,
|
|
"status": "uncertain", "result": None, "safe_to_refund": False}
|
|
store.set("operation", "lost-rescue-job", json.dumps(operation))
|
|
recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
|
|
assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID
|
|
job.update(status="completed", phase="救援系统启动")
|
|
components.store.set("rescue_job", "job-abc", json.dumps(job))
|
|
completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
|
|
assert completed["status"] == "succeeded"
|
|
assert not completed["safe_to_refund"]
|
|
|
|
|
|
def test_interrupted_async_job_stays_uncertain(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.store.set("reinstall_job", "job-abcd", json.dumps({
|
|
"operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID,
|
|
"status": "failed", "phase": "面板重启,任务已中断"}))
|
|
store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job",
|
|
"host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall",
|
|
"vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False}))
|
|
recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json()
|
|
assert recovered["status"] == "uncertain"
|
|
|
|
|
|
def test_metrics_query_uses_log_store_and_owner_permission(boundary):
|
|
client, engine, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d")
|
|
assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20
|
|
|
|
|
|
def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
|
|
assert issued.status_code == 200, issued.text
|
|
assert issued.json()["websocket_path"] == "/engine-console"
|
|
ticket = issued.json()["ticket"]
|
|
key = hashlib.sha256(ticket.encode()).hexdigest()
|
|
record = json.loads(store.get("console_ticket", key))
|
|
assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session"
|
|
assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket",
|
|
{**USER, "user_id": 42}, {}).status_code == 404
|
|
from starlette.websockets import WebSocketDisconnect
|
|
with pytest.raises(WebSocketDisconnect):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"},
|
|
subprotocols=["cloud-ticket." + ticket]):
|
|
pass
|
|
assert store.get("console_ticket", key) is not None
|
|
|
|
|
|
def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
checks = 0
|
|
async def active(*args):
|
|
nonlocal checks
|
|
checks += 1
|
|
return checks <= 2 # Ticket issuance + handshake; then PHP revokes.
|
|
engine.introspection = active
|
|
engine.console_check_interval = 0.01
|
|
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
|
|
ticket = issued.json()["ticket"]
|
|
protocol = "cloud-ticket." + ticket
|
|
async def relay(socket, **kwargs):
|
|
await socket.send_bytes(b"vnc")
|
|
await asyncio.Event().wait()
|
|
from starlette.websockets import WebSocketDisconnect
|
|
with patch("virsh_panel.console.relay_console", relay):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console",
|
|
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket:
|
|
assert socket.accepted_subprotocol == protocol
|
|
assert socket.receive_bytes() == b"vnc"
|
|
with pytest.raises(WebSocketDisconnect):
|
|
socket.receive_bytes()
|
|
key = hashlib.sha256(ticket.encode()).hexdigest()
|
|
assert store.get("console_ticket", key) is None
|
|
assert store.get("ticket_used", key) == "1"
|
|
with pytest.raises(WebSocketDisconnect):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console",
|
|
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]):
|
|
pass
|
|
|
|
|
|
def test_streamed_iso_digest_must_match_before_import_is_queued(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/rescue-templates?name=rescue"
|
|
signed_body = b"A" * (1024 * 1024)
|
|
wrong_body = b"B" * len(signed_body)
|
|
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
|
|
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest()
|
|
signed["Content-Type"] = "application/octet-stream"
|
|
response = client.post(path, content=wrong_body, headers=signed)
|
|
assert response.status_code == 401, response.text
|
|
assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls)
|
|
assert engine.registry.components["hk"].store.list("rescue_import_job") == {}
|
|
|
|
|
|
def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables():
|
|
connection = MagicMock()
|
|
cursor = connection.cursor.return_value.__enter__.return_value
|
|
config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test",
|
|
"ssl_ca": "/private/test/ca.pem"}
|
|
with patch("virsh_panel.database.pymysql.connect", return_value=connection):
|
|
main = MainStore(config, b"m" * 32)
|
|
statements = [call.args[0] for call in cursor.execute.call_args_list]
|
|
assert any("engine_records" in statement for statement in statements)
|
|
assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements)
|
|
assert not any("vm_metric_samples" in statement for statement in statements)
|
|
assert main._config["ssl_verify_identity"] is True
|
|
cursor.reset_mock()
|
|
LogStore({**config, "database": "logs"}, b"m" * 32)
|
|
statements = [call.args[0] for call in cursor.execute.call_args_list]
|
|
assert len(statements) == 1 and "engine_vm_metric_samples" in statements[0]
|