You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1072 lines
50 KiB
Python
1072 lines
50 KiB
Python
"""Private engine tests: no PHP cookies, no real DB and no real hypervisor."""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import threading
|
|
from contextlib import contextmanager
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from test_app import CREATE_BODY, FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID
|
|
from virsh_panel.app import create_app
|
|
from virsh_panel.config import Settings
|
|
from virsh_panel.database import DatabaseTemplateStore
|
|
from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore,
|
|
LogStore, MainStore, TRUSTED_ACTOR)
|
|
from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor,
|
|
signed_headers)
|
|
from virsh_panel.internal import EngineApplication, HostRegistry
|
|
from virsh_panel.metrics import host_scope
|
|
|
|
|
|
SECRET = b"s" * 32
|
|
USER = {"user_id": 41, "is_admin": False, "is_super_admin": False,
|
|
"can_manage": True, "session_id": "test-session"}
|
|
SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True}
|
|
ADMIN = {**SUPER, "is_super_admin": False}
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self):
|
|
self.records = {}
|
|
self.nonces = set()
|
|
self.lock = threading.RLock()
|
|
self.daily_traffic_calls = []
|
|
|
|
def get(self, kind, key):
|
|
with self.lock:
|
|
return self.records.get((kind, key))
|
|
|
|
def set(self, kind, key, value):
|
|
with self.lock:
|
|
self.records[kind, key] = value
|
|
|
|
def insert(self, kind, key, value):
|
|
with self.lock:
|
|
if (kind, key) in self.records:
|
|
return False
|
|
self.records[kind, key] = value
|
|
return True
|
|
|
|
def delete(self, kind, key):
|
|
with self.lock:
|
|
self.records.pop((kind, key), None)
|
|
|
|
def list(self, kind):
|
|
with self.lock:
|
|
return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind}
|
|
|
|
def consume_nonce(self, nonce, expires_at, now):
|
|
with self.lock:
|
|
if nonce in self.nonces:
|
|
return False
|
|
self.nonces.add(nonce)
|
|
return True
|
|
|
|
def query_metric_samples(self, *args, **kwargs):
|
|
return [{"timestamp": 123, "cpu_percent": 20.0}]
|
|
|
|
def query_daily_traffic(self, scope, vm_id, **kwargs):
|
|
self.daily_traffic_calls.append((scope, vm_id, kwargs))
|
|
day_start = kwargs["until"] - kwargs["until"] % 86400
|
|
return [{"day_start": day_start, "download_bytes": 4096,
|
|
"upload_bytes": 1024, "total_bytes": 5120}]
|
|
|
|
def get_value(self, key):
|
|
return self.get("value", key)
|
|
|
|
def set_value(self, key, value):
|
|
return self.set("value", key, value)
|
|
|
|
def delete_value(self, key):
|
|
return self.delete("value", key)
|
|
|
|
def get_secret(self, key):
|
|
return self.get("secret", key)
|
|
|
|
def set_secret(self, key, value):
|
|
return self.set("secret", key, value)
|
|
|
|
def delete_secret(self, key):
|
|
return self.delete("secret", key)
|
|
|
|
|
|
class FakeUpdater:
|
|
def __init__(self):
|
|
self.token = None
|
|
self.active = None
|
|
self.jobs = {}
|
|
|
|
def settings(self):
|
|
return {
|
|
"repository": "https://g.luoli.live/root/virt.git",
|
|
"branch": "venix",
|
|
"token_configured": self.token is not None,
|
|
"checkout_ready": True,
|
|
"current_commit": "a" * 40,
|
|
"last_job_id": None,
|
|
"active_job_id": self.active["job_id"] if self.active else None,
|
|
}
|
|
|
|
def set_token(self, token):
|
|
self.token = token
|
|
|
|
def clear_token(self):
|
|
self.token = None
|
|
|
|
def active_job(self):
|
|
return self.active
|
|
|
|
def check(self):
|
|
return {"repository": "https://g.luoli.live/root/virt.git", "branch": "venix",
|
|
"current_commit": "a" * 40, "remote_commit": "b" * 40,
|
|
"status": "update_available"}
|
|
|
|
def create_job(self, *, expected_current_commit=None, expected_remote_commit=None):
|
|
if self.token is None:
|
|
from virsh_panel.updater import UpdateError
|
|
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
|
|
if expected_current_commit != "a" * 40 or expected_remote_commit != "b" * 40:
|
|
from virsh_panel.updater import UpdateError
|
|
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
|
|
job = {"job_id": "update-job-1234567890", "status": "queued",
|
|
"phase": "等待独立更新服务启动", "created_at": 1,
|
|
"private": self.token}
|
|
self.jobs[job["job_id"]] = job
|
|
self.active = job
|
|
return job.copy()
|
|
|
|
def get_job(self, job_id):
|
|
return self.jobs.get(job_id)
|
|
|
|
def fail_job(self, job_id, message):
|
|
self.jobs[job_id].update(status="failed", error=message)
|
|
self.active = None
|
|
return self.jobs[job_id].copy()
|
|
|
|
|
|
class Service(FakeVMService):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.create_failure = False
|
|
self.absent = False
|
|
self.vm.update(managed=True, os_source="iso")
|
|
|
|
def create_vm(self, spec):
|
|
self.calls.append(("create_vm", spec))
|
|
if self.create_failure:
|
|
from virsh_panel.libvirt_service import VMError
|
|
raise VMError("宿主机失联,回滚状态不明", 502)
|
|
self.vm.update(uuid=spec["vm_uuid"], name=spec["name"])
|
|
return self.vm.copy()
|
|
|
|
def provisioning_absent(self, vm_uuid, name):
|
|
return self.absent
|
|
|
|
|
|
@pytest.fixture
|
|
def boundary(tmp_path):
|
|
store = MemoryStore()
|
|
logs = MemoryStore()
|
|
config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}
|
|
services = {}
|
|
def factory(host_id):
|
|
(tmp_path / host_id).mkdir(mode=0o700, exist_ok=True)
|
|
scoped = HostStore(store, host_id)
|
|
manager = FakeRemoteManager()
|
|
manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22,
|
|
"fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images",
|
|
"iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default",
|
|
"libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config",
|
|
"ssh_known_hosts_file": "/fake/known_hosts"}
|
|
service = Service()
|
|
services[host_id] = service
|
|
components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager,
|
|
DatabaseTemplateStore(scoped), scoped, logs)
|
|
app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components,
|
|
service_factory=lambda profile: service)
|
|
return components, app
|
|
registry = HostRegistry(config, store, logs, factory=factory)
|
|
async def active(*args):
|
|
return True
|
|
engine = EngineApplication(config, store, logs, registry=registry, introspection=active)
|
|
engine._service = lambda profile: services[profile["host_id"]]
|
|
with TestClient(engine, base_url="http://127.0.0.1") as client:
|
|
yield client, engine, store, logs, services
|
|
|
|
|
|
def call(client, method, path, actor=USER, value=None, raw=None, headers=None):
|
|
body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode()
|
|
signed = signed_headers(SECRET, method, path, body, actor)
|
|
if value is not None:
|
|
signed["Content-Type"] = "application/json"
|
|
signed.update(headers or {})
|
|
return client.request(method, path, content=body, headers=signed)
|
|
|
|
|
|
def test_hmac_binds_actor_payload_target_and_rejects_replay():
|
|
store = MemoryStore()
|
|
body = b"{}"
|
|
target = "/internal/v1/hosts/hk/vms?range=1h"
|
|
headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER,
|
|
timestamp=1000, nonce="n" * 24).items()}
|
|
assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
changed = {**headers, "x-cloud-actor": encode_actor(SUPER)}
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061)
|
|
|
|
invalid_cleanup = {**USER, "system_cleanup": "true"}
|
|
invalid_headers = {key.lower(): value for key, value in signed_headers(
|
|
SECRET, "POST", target, body, invalid_cleanup,
|
|
timestamp=1000, nonce="c" * 24,
|
|
).items()}
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(
|
|
invalid_headers, SECRET, "POST", target,
|
|
hashlib.sha256(body).hexdigest(), store, now=1000,
|
|
)
|
|
|
|
|
|
def test_private_route_allowlist_and_no_browser_admin_cookie(boundary):
|
|
client, *_ = boundary
|
|
for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin",
|
|
"/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"):
|
|
assert call(client, "POST", path, SUPER, {}).status_code == 404
|
|
assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401
|
|
assert TRUSTED_ACTOR.get() is None
|
|
|
|
|
|
def test_internal_update_routes_are_super_admin_only_and_never_echo_token(boundary):
|
|
client, engine, *_ = boundary
|
|
updater = FakeUpdater()
|
|
triggered = []
|
|
engine.updater = updater
|
|
def trigger():
|
|
triggered.append(True)
|
|
updater.active["status"] = "running"
|
|
engine.update_trigger = trigger
|
|
settings_path = "/internal/v1/update/settings"
|
|
|
|
assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401
|
|
assert call(client, "GET", settings_path, USER).status_code == 403
|
|
assert call(client, "GET", settings_path, ADMIN).status_code == 403
|
|
assert call(client, "GET", settings_path, {**SUPER, "can_manage": False}).status_code == 403
|
|
|
|
token = "gitea-private-token-123456"
|
|
saved = call(client, "PUT", settings_path, SUPER, {"access_token": token})
|
|
assert saved.status_code == 200
|
|
assert saved.json()["token_configured"] is True
|
|
assert saved.json()["repository"] == "https://g.luoli.live/root/virt.git"
|
|
assert saved.json()["branch"] == "venix"
|
|
assert token not in saved.text
|
|
assert token not in call(client, "GET", settings_path, SUPER).text
|
|
|
|
checked = call(client, "POST", "/internal/v1/update/check", SUPER, {})
|
|
assert checked.status_code == 200
|
|
assert checked.json()["status"] == "update_available"
|
|
applied = call(client, "POST", "/internal/v1/update/apply", SUPER, {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
})
|
|
assert applied.status_code == 202
|
|
assert applied.json()["job_id"] == "update-job-1234567890"
|
|
assert "private" not in applied.json() and token not in applied.text
|
|
assert triggered == [True]
|
|
job = call(client, "GET", "/internal/v1/update/jobs/update-job-1234567890", SUPER)
|
|
assert job.status_code == 200 and token not in job.text and "private" not in job.json()
|
|
|
|
# While the persisted update job is active, reads remain available but no
|
|
# host/VM mutation may begin and overlap the code/service restart.
|
|
assert call(client, "GET", "/internal/v1/health", SUPER).status_code == 200
|
|
assert call(client, "POST", "/internal/v1/hosts/scan", SUPER,
|
|
{"host": "host.example.test", "port": 22}).status_code == 409
|
|
|
|
|
|
def test_internal_update_validation_clear_and_trigger_failure(boundary):
|
|
client, engine, *_ = boundary
|
|
updater = FakeUpdater()
|
|
engine.updater = updater
|
|
settings_path = "/internal/v1/update/settings"
|
|
assert call(client, "PUT", settings_path, SUPER, {}).status_code == 422
|
|
assert call(client, "PUT", settings_path, SUPER,
|
|
{"access_token": "x", "unexpected": True}).status_code == 422
|
|
assert call(client, "POST", "/internal/v1/update/check", SUPER,
|
|
{"repository": "https://evil.example/repo.git"}).status_code == 422
|
|
assert call(client, "POST", "/internal/v1/update/apply", SUPER,
|
|
{"expected_current_commit": "a" * 40}).status_code == 422
|
|
assert call(client, "GET", "/internal/v1/update/jobs/missing-job-123456", SUPER).status_code == 404
|
|
|
|
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
|
|
cleared = call(client, "PUT", settings_path, SUPER, {"clear_token": True})
|
|
assert cleared.status_code == 200 and cleared.json()["token_configured"] is False
|
|
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
|
|
|
|
def fail_trigger():
|
|
from virsh_panel.updater import UpdateError
|
|
raise UpdateError("独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503)
|
|
|
|
engine.update_trigger = fail_trigger
|
|
failed = call(client, "POST", "/internal/v1/update/apply", SUPER, {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
})
|
|
assert failed.status_code == 503
|
|
job = updater.jobs["update-job-1234567890"]
|
|
assert job["status"] == "failed"
|
|
assert "令牌" not in job["error"]
|
|
|
|
|
|
@pytest.mark.parametrize(("state", "expected"), [
|
|
("activating", True),
|
|
("active", True),
|
|
("deactivating", True),
|
|
("inactive", False),
|
|
("failed", False),
|
|
("maintenance", None),
|
|
])
|
|
def test_update_oneshot_transitional_states_keep_maintenance_gate(state, expected):
|
|
with patch("virsh_panel.internal.subprocess.run",
|
|
return_value=SimpleNamespace(returncode=0, stdout=state + "\n")):
|
|
assert EngineApplication._update_service_active() is expected
|
|
|
|
|
|
def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary):
|
|
client, engine, *_ = boundary
|
|
path = "/internal/v1/hosts/hk"
|
|
assert call(client, "GET", path + "/host").status_code == 403
|
|
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
|
|
assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
|
|
assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200
|
|
assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200
|
|
assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404
|
|
assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == []
|
|
assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False},
|
|
{"action": "reboot"}).status_code == 403
|
|
|
|
|
|
def test_existing_unmanaged_vms_are_claimed_once_for_configuring_admin(boundary):
|
|
client, engine, _, _, services = boundary
|
|
path = "/internal/v1/hosts/hk"
|
|
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
|
|
manual_uuid = "22222222-2222-4222-8222-222222222222"
|
|
orphaned_managed_uuid = "33333333-3333-4333-8333-333333333333"
|
|
services["hk"].other_vms = [
|
|
{"uuid": manual_uuid, "name": "manual-vm", "state": "shutoff",
|
|
"vcpu": 2, "memory_mib": 2048, "disks": [], "managed": False},
|
|
{"uuid": orphaned_managed_uuid, "name": "unknown-panel-vm", "state": "shutoff",
|
|
"vcpu": 2, "memory_mib": 2048, "disks": [], "managed": True},
|
|
]
|
|
body = {"known_owners": {VM_ID: 41}}
|
|
assert call(client, "POST", path + "/host/discover-vms", ADMIN, body).status_code == 403
|
|
discovered = call(client, "POST", path + "/host/discover-vms", SUPER, body)
|
|
assert discovered.status_code == 200, discovered.text
|
|
rows = {row["uuid"]: row for row in discovered.json()["items"]}
|
|
assert rows[VM_ID]["owner_user_id"] == 41
|
|
assert rows[manual_uuid]["owner_user_id"] == SUPER["user_id"]
|
|
assert orphaned_managed_uuid not in rows
|
|
assert discovered.json()["claimed_count"] == 2
|
|
assert discovered.json()["untracked_managed_count"] == 1
|
|
assert discovered.json()["total_count"] == 3
|
|
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"]
|
|
adopted = call(client, "POST", path + "/host/discover-vms", SUPER,
|
|
{**body, "claim_unowned_managed": True})
|
|
assert adopted.status_code == 200
|
|
adopted_rows = {row["uuid"]: row for row in adopted.json()["items"]}
|
|
assert adopted_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"]
|
|
assert adopted.json()["claimed_count"] == 1
|
|
assert adopted.json()["untracked_managed_count"] == 0
|
|
repeated = call(client, "POST", path + "/host/discover-vms",
|
|
{**SUPER, "user_id": 2}, {**body, "claim_unowned_managed": True})
|
|
assert repeated.status_code == 200
|
|
repeated_rows = {row["uuid"]: row for row in repeated.json()["items"]}
|
|
assert repeated_rows[manual_uuid]["owner_user_id"] == SUPER["user_id"]
|
|
assert repeated_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"]
|
|
assert repeated.json()["claimed_count"] == 0
|
|
assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"]
|
|
assert {row["uuid"] for row in call(
|
|
client, "GET", path + "/vms", {**USER, "user_id": SUPER["user_id"]}
|
|
).json()["items"]} == {manual_uuid, orphaned_managed_uuid}
|
|
assert call(client, "GET", path + "/vms", {**USER, "user_id": 2}).json()["items"] == []
|
|
|
|
|
|
def test_stable_host_identity_and_namespaces():
|
|
store = MemoryStore()
|
|
first, second = HostStore(store, "a"), HostStore(store, "b")
|
|
first.set("host", "current", "one")
|
|
second.set("host", "current", "two")
|
|
assert first.get("host", "current") == "one"
|
|
assert second.get("host", "current") == "two"
|
|
a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"}
|
|
assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"})
|
|
assert host_scope(a) != host_scope({**a, "host_id": "b"})
|
|
assert MainStore.RECORD_TABLE == "engine_records"
|
|
assert LogStore.METRIC_TABLE == "engine_vm_metric_samples"
|
|
|
|
|
|
def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403
|
|
provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"}
|
|
created = call(client, "POST", path, provisioning, CREATE_BODY)
|
|
assert created.status_code == 201, created.text
|
|
vm_id = created.json()["uuid"]
|
|
assert created.json()["result"]["uuid"] == vm_id
|
|
assert created.json()["status"] == "succeeded"
|
|
component = engine.registry.components["hk"]
|
|
assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41)
|
|
again = call(client, "POST", path, provisioning, CREATE_BODY)
|
|
assert again.status_code == 201 and again.json()["uuid"] == vm_id
|
|
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
|
|
assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409
|
|
operation = call(client, "GET", "/internal/v1/operations/order-000001").json()
|
|
assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id
|
|
assert operation["safe_to_refund"] is False
|
|
assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404
|
|
|
|
|
|
def test_uncertain_creation_never_refunds_or_recreates(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
call(client, "GET", path)
|
|
services["hk"].create_failure = True
|
|
actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"}
|
|
failed = call(client, "POST", path, actor, CREATE_BODY)
|
|
assert failed.status_code == 502
|
|
assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"]
|
|
record = json.loads(store.get("operation", "order-uncertain"))
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
assert component.state.users.owns_vm(scope, record["vm_uuid"], 41)
|
|
component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password")
|
|
component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]}))
|
|
again = call(client, "POST", path, actor, CREATE_BODY)
|
|
assert again.status_code == 202 and again.json()["status"] == "uncertain"
|
|
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
|
|
services["hk"].absent = True
|
|
operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json()
|
|
assert operation["status"] == "failed" and operation["safe_to_refund"] is True
|
|
assert not component.state.users.owns_vm(scope, record["vm_uuid"], 41)
|
|
assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") is None
|
|
assert component.store.list("ip_claim") == {}
|
|
|
|
|
|
@pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"],
|
|
{**CREATE_BODY, "guest_ssh_key_id": 3}])
|
|
def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload):
|
|
client, engine, _, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
actor = {**USER, "can_provision": True, "operation_id": "rejected-create"}
|
|
response = call(client, "POST", path, actor, payload)
|
|
assert response.status_code == 422
|
|
# Simulate a dispatcher losing the POST response and querying after restart.
|
|
engine.active_operations.clear()
|
|
saved = call(client, "GET", "/internal/v1/operations/rejected-create").json()
|
|
assert saved["status"] == "failed" and saved["safe_to_refund"] is True
|
|
assert call(client, "POST", path, actor, payload).status_code == 422
|
|
assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409
|
|
assert not any(row[0] == "create_vm" for row in services["hk"].calls)
|
|
|
|
|
|
def test_admin_created_vm_also_has_external_owner(boundary):
|
|
client, engine, *_ = boundary
|
|
actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"}
|
|
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
|
|
assert response.status_code == 201
|
|
components = engine.registry.components["hk"]
|
|
assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1)
|
|
|
|
|
|
def test_power_action_has_persistent_idempotency(boundary):
|
|
client, engine, _, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
|
assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403
|
|
actor = {**USER, "operation_id": "reboot-operation"}
|
|
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
|
|
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
|
|
assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1
|
|
assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409
|
|
operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json()
|
|
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
|
|
|
|
|
|
def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary):
|
|
client, engine, _, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
rescue_key = f"{scope}.{VM_ID}"
|
|
components.store.set("vm_rescue", rescue_key, json.dumps({
|
|
"original_xml": "<domain><name>alma-test</name></domain>",
|
|
"template_id": "0123456789abcdef0123456789abcdef",
|
|
"volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso",
|
|
"status": "active",
|
|
}))
|
|
services["hk"].vm.update(
|
|
state="shutoff", rescue_mode=True, rescue_staged=True,
|
|
rescue_volume_name="panel-rescue-0123456789abcdef0123456789abcdef.iso",
|
|
)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
|
|
|
|
ordinary = call(
|
|
client, "DELETE", path,
|
|
{**USER, "operation_id": "ordinary-rescue-delete", "system_cleanup": False},
|
|
)
|
|
assert ordinary.status_code == 409
|
|
assert components.store.get("vm_rescue", rescue_key) is not None
|
|
assert not any(item[0] == "delete_vm" for item in services["hk"].calls)
|
|
|
|
cleanup = call(
|
|
client, "DELETE", path,
|
|
{**USER, "operation_id": "expiry-rescue-delete", "system_cleanup": True},
|
|
)
|
|
assert cleanup.status_code == 200, cleanup.text
|
|
assert cleanup.json()["status"] == "succeeded"
|
|
assert services["hk"].calls[-1] == ("delete_vm", VM_ID, True)
|
|
assert components.store.get("vm_rescue", rescue_key) is None
|
|
assert components.state.users.owner_of_vm(scope, VM_ID) is None
|
|
|
|
|
|
@pytest.mark.parametrize(("route", "method", "target"), [
|
|
("service-suspend", "service_suspend", {"state": "shutoff", "autostart": False}),
|
|
("service-resume", "service_resume", {"autostart": True}),
|
|
("network-suspend", "network_suspend", {"network_suspended": True}),
|
|
("network-resume", "network_resume", {"network_suspended": False}),
|
|
])
|
|
def test_private_target_state_operations_are_owned_idempotent_mutations(
|
|
boundary, route, method, target):
|
|
client, engine, _, _, services = boundary
|
|
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
|
|
assert call(client, "POST", base, USER, {}).status_code == 403
|
|
assert call(client, "POST", base, {**USER, "can_manage": False,
|
|
"operation_id": f"blocked-{route}"}, {}).status_code == 403
|
|
assert call(client, "POST", base, {**USER, "user_id": 42,
|
|
"operation_id": f"foreign-{route}"}, {}).status_code == 404
|
|
|
|
actor = {**USER, "operation_id": f"target-{route}"}
|
|
first = call(client, "POST", base, actor, {})
|
|
assert first.status_code == 200, first.text
|
|
assert first.json()["status"] == "succeeded"
|
|
assert all(first.json()[key] == value for key, value in target.items())
|
|
repeated = call(client, "POST", base, actor, {})
|
|
assert repeated.status_code == 200
|
|
assert len([item for item in services["hk"].calls if item[0] == method]) == 1
|
|
|
|
|
|
@pytest.mark.parametrize(("route", "method", "drift"), [
|
|
("service-suspend", "service_suspend", {"state": "running", "autostart": True}),
|
|
("service-resume", "service_resume", {"autostart": False}),
|
|
("network-suspend", "network_suspend", {"network_suspended": False}),
|
|
("network-resume", "network_resume", {"network_suspended": True}),
|
|
])
|
|
def test_target_state_operation_replays_after_lost_response(
|
|
boundary, route, method, drift):
|
|
client, engine, store, _, services = boundary
|
|
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
actor = {**USER, "operation_id": f"replay-{route}"}
|
|
assert call(client, "POST", base, actor, {}).status_code == 200
|
|
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(drift)
|
|
engine.active_operations.clear()
|
|
|
|
replayed = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
|
|
actor)
|
|
assert replayed.status_code == 200, replayed.text
|
|
assert replayed.json()["status"] == "succeeded"
|
|
assert len([item for item in services["hk"].calls if item[0] == method]) == 2
|
|
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost again")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(drift)
|
|
replayed_post = call(client, "POST", base, actor, {})
|
|
assert replayed_post.status_code == 200, replayed_post.text
|
|
assert replayed_post.json()["status"] == "succeeded"
|
|
assert len([item for item in services["hk"].calls if item[0] == method]) == 3
|
|
|
|
|
|
def test_non_reconciling_operation_lookup_never_replays_target_state(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend"
|
|
actor = {**USER, "operation_id": "peek-target-state"}
|
|
assert call(client, "POST", path, actor, {}).status_code == 200
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm["network_suspended"] = False
|
|
engine.active_operations.clear()
|
|
before = len(services["hk"].calls)
|
|
|
|
peeked = call(client, "GET",
|
|
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
|
|
assert peeked.status_code == 200
|
|
assert peeked.json()["status"] == "uncertain"
|
|
assert peeked.json()["executor_active"] is False
|
|
assert len(services["hk"].calls) == before
|
|
assert services["hk"].vm["network_suspended"] is False
|
|
|
|
engine.active_operations.add(actor["operation_id"])
|
|
active = call(client, "GET",
|
|
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
|
|
assert active.status_code == 200
|
|
assert active.json()["executor_active"] is True
|
|
assert len(services["hk"].calls) == before
|
|
engine.active_operations.discard(actor["operation_id"])
|
|
|
|
|
|
def test_target_state_lock_serializes_same_vm_and_not_other_vms(boundary):
|
|
_, engine, *_ = boundary
|
|
|
|
async def exercise():
|
|
first = engine._target_lock("hk", VM_ID)
|
|
same = engine._target_lock("hk", VM_ID)
|
|
other = engine._target_lock("hk", "22222222-2222-4222-8222-222222222222")
|
|
assert first is same
|
|
assert first is not other
|
|
entered = []
|
|
release = asyncio.Event()
|
|
|
|
async def owner():
|
|
async with first:
|
|
entered.append("owner")
|
|
await release.wait()
|
|
entered.append("owner-finished")
|
|
|
|
async def waiter():
|
|
async with same:
|
|
entered.append("waiter")
|
|
|
|
owner_task = asyncio.create_task(owner())
|
|
await asyncio.sleep(0)
|
|
waiter_task = asyncio.create_task(waiter())
|
|
await asyncio.sleep(0)
|
|
assert entered == ["owner"]
|
|
release.set()
|
|
await asyncio.gather(owner_task, waiter_task)
|
|
assert entered == ["owner", "owner-finished", "waiter"]
|
|
|
|
errors = []
|
|
def run_in_isolated_loop():
|
|
try:
|
|
asyncio.run(exercise())
|
|
except BaseException as exc: # propagate assertion failures to pytest
|
|
errors.append(exc)
|
|
|
|
thread = threading.Thread(target=run_in_isolated_loop)
|
|
thread.start()
|
|
thread.join(timeout=5)
|
|
assert not thread.is_alive()
|
|
if errors:
|
|
raise errors[0]
|
|
|
|
|
|
def test_expiry_target_waits_for_inflight_user_mutation_on_same_vm(boundary):
|
|
client, engine, _, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
service = services["hk"]
|
|
action_entered = threading.Event()
|
|
release_action = threading.Event()
|
|
network_entered = threading.Event()
|
|
original_action = service.action
|
|
original_network_suspend = service.network_suspend
|
|
|
|
def slow_action(vm_id, action):
|
|
action_entered.set()
|
|
assert release_action.wait(timeout=5), "test did not release user mutation"
|
|
return original_action(vm_id, action)
|
|
|
|
def observed_network_suspend(vm_id):
|
|
network_entered.set()
|
|
return original_network_suspend(vm_id)
|
|
|
|
service.action = slow_action
|
|
service.network_suspend = observed_network_suspend
|
|
responses = {}
|
|
|
|
def user_request():
|
|
responses["action"] = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions",
|
|
{**USER, "operation_id": "concurrent-user-action"}, {"action": "reset"})
|
|
|
|
def policy_request():
|
|
responses["policy"] = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
|
|
{**USER, "operation_id": "concurrent-expiry-policy"}, {})
|
|
|
|
first = threading.Thread(target=user_request)
|
|
first.start()
|
|
assert action_entered.wait(timeout=5)
|
|
second = threading.Thread(target=policy_request)
|
|
second.start()
|
|
assert not network_entered.wait(timeout=0.2)
|
|
release_action.set()
|
|
first.join(timeout=5)
|
|
second.join(timeout=5)
|
|
assert not first.is_alive() and not second.is_alive()
|
|
assert responses["action"].status_code == 200
|
|
assert responses["policy"].status_code == 200
|
|
assert network_entered.is_set()
|
|
mutations = [item[0] for item in service.calls
|
|
if item[0] in {"action", "network_suspend"}]
|
|
assert mutations == ["action", "network_suspend"]
|
|
assert service.vm["network_suspended"] is True
|
|
|
|
|
|
def test_target_state_reconciliation_does_not_bypass_original_owner(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
store.set("operation", "unsafe-target", json.dumps({
|
|
"operation_id": "unsafe-target", "host_id": "hk", "user_id": 42,
|
|
"kind": "mutation", "resource": f"vms/{VM_ID}/network-suspend",
|
|
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
|
|
"actor_can_manage": True, "actor_is_admin": False,
|
|
"safe_to_refund": False,
|
|
}))
|
|
response = call(client, "GET", "/internal/v1/operations/unsafe-target",
|
|
{**USER, "user_id": 42})
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "failed"
|
|
assert not any(item[0] == "network_suspend" for item in services["hk"].calls)
|
|
|
|
|
|
def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID,
|
|
"status": "running", "phase": "正在救援"}
|
|
components.store.set("rescue_job", "job-abc", json.dumps(job))
|
|
operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID,
|
|
"status": "uncertain", "result": None, "safe_to_refund": False}
|
|
store.set("operation", "lost-rescue-job", json.dumps(operation))
|
|
recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
|
|
assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID
|
|
job.update(status="completed", phase="救援系统启动")
|
|
components.store.set("rescue_job", "job-abc", json.dumps(job))
|
|
completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
|
|
assert completed["status"] == "succeeded"
|
|
assert not completed["safe_to_refund"]
|
|
|
|
|
|
def test_interrupted_async_job_becomes_terminal_without_replay(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.store.set("reinstall_job", "job-abcd", json.dumps({
|
|
"operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID,
|
|
"status": "failed", "phase": "面板重启,任务已中断"}))
|
|
store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job",
|
|
"host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall",
|
|
"vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False}))
|
|
recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json()
|
|
assert recovered["status"] == "failed"
|
|
assert "中断" in recovered["error"]
|
|
|
|
|
|
@pytest.mark.parametrize("resource", [
|
|
f"vms/{VM_ID}/actions",
|
|
f"vms/{VM_ID}",
|
|
f"vms/{VM_ID}/reinstall",
|
|
f"vms/{VM_ID}/rescue",
|
|
])
|
|
def test_orphaned_mutation_becomes_terminal_without_replay(boundary, resource):
|
|
client, _, store, *_ = boundary
|
|
operation_id = "orphan-" + resource.rsplit("/", 1)[-1].replace("?", "-")
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation", "resource": resource, "vm_uuid": VM_ID,
|
|
"status": "running", "result": None, "safe_to_refund": False,
|
|
"created_at": 1,
|
|
}))
|
|
recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}").json()
|
|
assert recovered["status"] == "failed"
|
|
assert "不会自动重放" in recovered["error"]
|
|
|
|
|
|
def test_service_resume_replays_exact_start_target_after_lost_response(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
services["hk"].vm.update(state="shutoff", autostart=False)
|
|
actor = {**USER, "operation_id": "resume-start-target"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume"
|
|
|
|
first = call(client, "POST", path, actor, {"start_if_suspended": True})
|
|
assert first.status_code == 200, first.text
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
|
|
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(state="shutoff", autostart=False)
|
|
engine.active_operations.clear()
|
|
replayed = call(client, "GET", "/internal/v1/operations/resume-start-target", actor)
|
|
assert replayed.status_code == 200
|
|
assert replayed.json()["status"] == "succeeded"
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
|
|
|
|
|
|
def test_service_suspend_replay_preserves_original_running_state(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
actor = {**USER, "operation_id": "suspend-running-hint"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend"
|
|
assert call(client, "POST", path, actor, {}).status_code == 200
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
assert saved["service_was_running_hint"] is True
|
|
|
|
# Model a hard crash after libvirt stopped the guest but before the
|
|
# successful result was durable. Reconciliation now observes shutoff, yet
|
|
# must retain the original running state for the billing resume policy.
|
|
saved.update(status="uncertain", result=None, error="engine crashed")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(state="shutoff", autostart=False)
|
|
engine.active_operations.clear()
|
|
reconciled = call(client, "GET",
|
|
"/internal/v1/operations/suspend-running-hint", actor)
|
|
assert reconciled.status_code == 200
|
|
assert reconciled.json()["status"] == "succeeded"
|
|
assert reconciled.json()["result"]["service_was_running"] is True
|
|
|
|
|
|
def test_service_suspend_replay_without_running_hint_fails_closed(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
operation_id = "suspend-missing-running-hint"
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id,
|
|
"host_id": "hk",
|
|
"user_id": USER["user_id"],
|
|
"vm_uuid": VM_ID,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/service-suspend",
|
|
"method": "POST",
|
|
"status": "uncertain",
|
|
"actor_can_manage": True,
|
|
"actor_is_admin": False,
|
|
"created_at": 1,
|
|
}))
|
|
before_calls = list(services["hk"].calls)
|
|
|
|
response = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "failed"
|
|
assert "已取消服务暂停" in response.json()["error"]
|
|
assert services["hk"].calls == before_calls
|
|
|
|
|
|
def test_service_suspend_does_not_stop_vm_when_running_hint_cannot_persist(
|
|
boundary, monkeypatch):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
operation_id = "suspend-hint-write-failed"
|
|
original_set = store.set
|
|
|
|
def fail_operation_write(kind, key, value):
|
|
if kind == "operation" and key == operation_id:
|
|
raise RuntimeError("database unavailable")
|
|
return original_set(kind, key, value)
|
|
|
|
monkeypatch.setattr(store, "set", fail_operation_write)
|
|
failed = call(
|
|
client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend",
|
|
{**USER, "operation_id": operation_id}, {})
|
|
|
|
assert failed.status_code == 503
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert not any(
|
|
item[0] == "service_suspend" for item in services["hk"].calls)
|
|
|
|
monkeypatch.setattr(store, "set", original_set)
|
|
recovered = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
|
|
assert recovered.status_code == 200
|
|
assert recovered.json()["status"] == "failed"
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert not any(
|
|
item[0] == "service_suspend" for item in services["hk"].calls)
|
|
|
|
def test_metrics_query_uses_log_store_and_owner_permission(boundary):
|
|
client, engine, _, logs, _ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
component.state.users.assign_vm(scope, VM_ID, 41)
|
|
response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d")
|
|
assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20
|
|
|
|
with patch("virsh_panel.app.time.time", return_value=3_000_000):
|
|
traffic = call(
|
|
client, "GET",
|
|
f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily?days=2",
|
|
)
|
|
assert traffic.status_code == 200
|
|
assert traffic.json()["points"][-1] == {
|
|
"date": "1970-02-04", "day_start": 2_937_600,
|
|
"download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120,
|
|
}
|
|
assert logs.daily_traffic_calls == [(scope, VM_ID, {
|
|
"host_id": "hk", "user_id": 41,
|
|
"since": 2_851_200, "until": 3_000_000, "limit": 2,
|
|
})]
|
|
denied = call(
|
|
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily",
|
|
{**USER, "user_id": 42},
|
|
)
|
|
assert denied.status_code == 404
|
|
assert len(logs.daily_traffic_calls) == 1
|
|
|
|
|
|
def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
|
|
assert issued.status_code == 200, issued.text
|
|
assert issued.json()["websocket_path"] == "/engine-console"
|
|
ticket = issued.json()["ticket"]
|
|
key = hashlib.sha256(ticket.encode()).hexdigest()
|
|
record = json.loads(store.get("console_ticket", key))
|
|
assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session"
|
|
assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket",
|
|
{**USER, "user_id": 42}, {}).status_code == 404
|
|
from starlette.websockets import WebSocketDisconnect
|
|
with pytest.raises(WebSocketDisconnect):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"},
|
|
subprotocols=["cloud-ticket." + ticket]):
|
|
pass
|
|
assert store.get("console_ticket", key) is not None
|
|
|
|
|
|
def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
checks = 0
|
|
async def active(*args):
|
|
nonlocal checks
|
|
checks += 1
|
|
return checks <= 2 # Ticket issuance + handshake; then PHP revokes.
|
|
engine.introspection = active
|
|
engine.console_check_interval = 0.01
|
|
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
|
|
ticket = issued.json()["ticket"]
|
|
protocol = "cloud-ticket." + ticket
|
|
async def relay(socket, **kwargs):
|
|
await socket.send_bytes(b"vnc")
|
|
await asyncio.Event().wait()
|
|
from starlette.websockets import WebSocketDisconnect
|
|
with patch("virsh_panel.console.relay_console", relay):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console",
|
|
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket:
|
|
assert socket.accepted_subprotocol == protocol
|
|
assert socket.receive_bytes() == b"vnc"
|
|
with pytest.raises(WebSocketDisconnect):
|
|
socket.receive_bytes()
|
|
key = hashlib.sha256(ticket.encode()).hexdigest()
|
|
assert store.get("console_ticket", key) is None
|
|
assert store.get("ticket_used", key) == "1"
|
|
with pytest.raises(WebSocketDisconnect):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console",
|
|
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]):
|
|
pass
|
|
|
|
|
|
def test_streamed_iso_digest_must_match_before_import_is_queued(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/rescue-templates?name=rescue"
|
|
signed_body = b"A" * (1024 * 1024)
|
|
wrong_body = b"B" * len(signed_body)
|
|
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
|
|
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest()
|
|
signed["Content-Type"] = "application/octet-stream"
|
|
response = client.post(path, content=wrong_body, headers=signed)
|
|
assert response.status_code == 401, response.text
|
|
assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls)
|
|
assert engine.registry.components["hk"].store.list("rescue_import_job") == {}
|
|
|
|
|
|
def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables():
|
|
connection = MagicMock()
|
|
cursor = connection.cursor.return_value.__enter__.return_value
|
|
config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test",
|
|
"ssl_ca": "/private/test/ca.pem"}
|
|
with patch("virsh_panel.database.pymysql.connect", return_value=connection):
|
|
main = MainStore(config, b"m" * 32)
|
|
statements = [call.args[0] for call in cursor.execute.call_args_list]
|
|
assert any("engine_records" in statement for statement in statements)
|
|
assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements)
|
|
assert not any("vm_metric_samples" in statement for statement in statements)
|
|
assert main._config["ssl_verify_identity"] is True
|
|
cursor.reset_mock()
|
|
LogStore({**config, "database": "logs"}, b"m" * 32)
|
|
statements = [call.args[0] for call in cursor.execute.call_args_list]
|
|
assert "engine_vm_metric_samples" in statements[0]
|
|
assert "engine_vm_traffic_totals" in statements[1]
|
|
assert "engine_vm_traffic_checkpoints" in statements[2]
|
|
assert all("ALTER TABLE engine_vm_metric_samples" in statement
|
|
for statement in statements[3:])
|
|
assert LogStore.TRAFFIC_TABLE == "engine_vm_traffic_totals"
|
|
assert LogStore.TRAFFIC_CHECKPOINT_TABLE == "engine_vm_traffic_checkpoints"
|