You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_engine_boundary.py

2625 lines
113 KiB
Python

"""Private engine tests: no PHP cookies, no real DB and no real hypervisor."""
from __future__ import annotations
import base64
import asyncio
import hashlib
import json
import os
import struct
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from urllib.parse import urlencode
import pytest
from fastapi.testclient import TestClient
from test_app import (CLOUD_CREATE_BODY, CREATE_BODY, OFFICIAL_CLOUD_URL,
FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID)
from virsh_panel.app import create_app
from virsh_panel.cloud_templates import (TemplateError, template_id_for,
template_scope, template_volume_name)
from virsh_panel.config import Settings
from virsh_panel.database import DatabaseError, DatabaseTemplateStore
from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore,
LogStore, MainStore, TRUSTED_ACTOR)
from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor,
signed_headers)
from virsh_panel.internal import EngineApplication, HostRegistry
from virsh_panel.libvirt_service import VMError
from virsh_panel.metrics import host_scope
from virsh_panel.user_isos import user_iso_scope
SECRET = b"s" * 32
USER = {"user_id": 41, "is_admin": False, "is_super_admin": False,
"can_manage": True, "session_id": "test-session"}
SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True}
ADMIN = {**SUPER, "is_super_admin": False}
class MemoryStore:
def __init__(self):
self.records = {}
self.nonces = set()
self.lock = threading.RLock()
self.daily_traffic_calls = []
def get(self, kind, key):
with self.lock:
return self.records.get((kind, key))
def set(self, kind, key, value):
with self.lock:
self.records[kind, key] = value
def insert(self, kind, key, value):
with self.lock:
if (kind, key) in self.records:
return False
self.records[kind, key] = value
return True
def delete(self, kind, key):
with self.lock:
self.records.pop((kind, key), None)
def list(self, kind):
with self.lock:
return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind}
def consume_nonce(self, nonce, expires_at, now):
with self.lock:
if nonce in self.nonces:
return False
self.nonces.add(nonce)
return True
def query_metric_samples(self, *args, **kwargs):
return [{"timestamp": 123, "cpu_percent": 20.0}]
def query_daily_traffic(self, scope, vm_id, **kwargs):
self.daily_traffic_calls.append((scope, vm_id, kwargs))
day_start = kwargs["until"] - kwargs["until"] % 86400
return [{"day_start": day_start, "download_bytes": 4096,
"upload_bytes": 1024, "total_bytes": 5120}]
def get_value(self, key):
return self.get("value", key)
def set_value(self, key, value):
return self.set("value", key, value)
def delete_value(self, key):
return self.delete("value", key)
def get_secret(self, key):
return self.get("secret", key)
def set_secret(self, key, value):
return self.set("secret", key, value)
def delete_secret(self, key):
return self.delete("secret", key)
class FakeUpdater:
def __init__(self):
self.token = None
self.active = None
self.jobs = {}
def settings(self):
return {
"repository": "https://g.luoli.live/root/virt.git",
"branch": "venix",
"token_configured": self.token is not None,
"checkout_ready": True,
"current_commit": "a" * 40,
"last_job_id": None,
"active_job_id": self.active["job_id"] if self.active else None,
}
def set_token(self, token):
self.token = token
def clear_token(self):
self.token = None
def active_job(self):
return self.active
def check(self):
return {"repository": "https://g.luoli.live/root/virt.git", "branch": "venix",
"current_commit": "a" * 40, "remote_commit": "b" * 40,
"status": "update_available"}
def create_job(self, *, expected_current_commit=None, expected_remote_commit=None):
if self.token is None:
from virsh_panel.updater import UpdateError
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
if expected_current_commit != "a" * 40 or expected_remote_commit != "b" * 40:
from virsh_panel.updater import UpdateError
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
job = {"job_id": "update-job-1234567890", "status": "queued",
"phase": "等待独立更新服务启动", "created_at": 1,
"private": self.token}
self.jobs[job["job_id"]] = job
self.active = job
return job.copy()
def get_job(self, job_id):
return self.jobs.get(job_id)
def fail_job(self, job_id, message):
self.jobs[job_id].update(status="failed", error=message)
self.active = None
return self.jobs[job_id].copy()
class Service(FakeVMService):
def __init__(self):
super().__init__()
self.create_failure = False
self.absent = False
self.absent_checks = []
self.cloud_template_delete_error = None
self.vm.update(managed=True, os_source="iso")
def create_vm(self, spec):
self.calls.append(("create_vm", spec))
if self.create_failure:
from virsh_panel.libvirt_service import VMError
raise VMError("宿主机失联,回滚状态不明", 502)
self.vm.update(uuid=spec["vm_uuid"], name=spec["name"])
return self.vm.copy()
def provisioning_absent(self, vm_uuid, name, *, uuid_resources=False,
legacy_name_collision=False):
self.absent_checks.append(
(vm_uuid, name, uuid_resources, legacy_name_collision))
return self.absent
def delete_cloud_template_replicas(self, source_volume_name):
self.calls.append(("delete_cloud_template_replicas", source_volume_name))
if self.cloud_template_delete_error is not None:
raise self.cloud_template_delete_error
return {"source_volume_name": source_volume_name, "deleted": True}
@pytest.fixture
def boundary(tmp_path):
store = MemoryStore()
logs = MemoryStore()
config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}
services = {}
def factory(host_id, user_isos, cloud_templates):
(tmp_path / host_id).mkdir(mode=0o700, exist_ok=True)
scoped = HostStore(store, host_id)
manager = FakeRemoteManager()
manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22,
"fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images",
"iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default",
"libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config",
"ssh_known_hosts_file": "/fake/known_hosts"}
service = Service()
services[host_id] = service
components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager,
DatabaseTemplateStore(scoped), scoped, logs, user_isos,
cloud_templates)
app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components,
service_factory=lambda profile: service)
return components, app
registry = HostRegistry(config, store, logs, factory=factory)
async def active(*args):
return True
engine = EngineApplication(config, store, logs, registry=registry, introspection=active)
engine._service = lambda profile: services[profile["host_id"]]
with TestClient(engine, base_url="http://127.0.0.1") as client:
yield client, engine, store, logs, services
def call(client, method, path, actor=USER, value=None, raw=None, headers=None):
body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode()
signed = signed_headers(SECRET, method, path, body, actor)
if value is not None:
signed["Content-Type"] = "application/json"
signed.update(headers or {})
return client.request(method, path, content=body, headers=signed)
def import_user_iso_for_test(client, tmp_path, name, *, digest="e" * 64):
download = tmp_path / (hashlib.sha256(name.encode()).hexdigest() + ".iso")
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, digest)):
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
"url": "https://downloads.example.test/os.iso",
"name": name,
"sha256": "",
})
assert queued.status_code == 202, queued.text
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
return job["iso"]["id"]
def import_cloud_template_for_test(client, tmp_path, name="Delete Cloud Image"):
payload = name.encode("utf-8")
digest = hashlib.sha256(payload).hexdigest()
def fake_download(_url, _sha256, directory, **_kwargs):
source = directory / ".cloud-image-delete-boundary.qcow2"
source.write_bytes(payload)
source.chmod(0o600)
return source, 2 * 1024**3, digest
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=fake_download):
queued = call(client, "POST", "/internal/v1/vm-templates/import",
SUPER, {"url": OFFICIAL_CLOUD_URL,
"name": name, "sha256": ""})
assert queued.status_code == 202, queued.text
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET",
f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER,
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
return job["template"], payload
def test_global_cloud_template_is_visible_and_cached_once_per_host(
boundary, tmp_path):
client, engine, store, _logs, services = boundary
payload = b"verified-global-cloud-template"
digest = hashlib.sha256(payload).hexdigest()
def fake_download(_url, _sha256, directory, **_kwargs):
source = directory / ".cloud-image-boundary.qcow2"
source.write_bytes(payload)
source.chmod(0o600)
return source, 2 * 1024**3, digest
denied = call(client, "POST", "/internal/v1/vm-templates/import",
USER, {"url": OFFICIAL_CLOUD_URL,
"name": "Not Allowed", "sha256": ""})
assert denied.status_code == 403
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=fake_download):
queued = call(client, "POST", "/internal/v1/vm-templates/import",
SUPER, {"url": OFFICIAL_CLOUD_URL,
"name": "AlmaLinux Global", "sha256": ""})
assert queued.status_code == 202, queued.text
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET",
f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER,
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
template_id = job["template"]["id"]
assert services == {}
assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == {
"templates": [job["template"]],
}
provisioning = {**USER, "can_provision": True}
body = {**CLOUD_CREATE_BODY, "template_id": template_id}
hk = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0001"}, body,
)
assert hk.status_code == 201, hk.text
jp = call(
client, "POST", "/internal/v1/hosts/jp/vms",
{**provisioning, "operation_id": "global-template-jp-0001"},
{**body, "name": "cloud-vm-jp", "public_ip_cidr": "8.8.8.19/24"},
)
assert jp.status_code == 201, jp.text
first_markers = {
host_id: json.loads(HostStore(store, host_id).get(
"cloud_template_replica", template_id))
for host_id in ("hk", "jp")
}
# The public catalog intentionally omits the private backing filename.
source_volume_name = template_volume_name(template_id)
for host_id, marker in first_markers.items():
assert marker["status"] == "ready"
assert marker["sha256"] == digest
assert len(marker["replica_identity"]) == 64
assert marker["replica_volume_name"] != source_volume_name
assert marker["replica_volume_name"].startswith(
source_volume_name.removesuffix(".qcow2") + "-")
created = [row for row in services[host_id].calls
if row[0] == "create_cloud_vm"]
assert created[-1][2] == marker["replica_volume_name"]
observed_markers = []
original_hk_cache = services["hk"].cache_template
def inspect_hk_marker(*args, **kwargs):
marker = json.loads(HostStore(store, "hk").get(
"cloud_template_replica", template_id))
observed_markers.append(marker)
return original_hk_cache(*args, **kwargs)
services["hk"].cache_template = inspect_hk_marker
hk_again = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0002"},
{**body, "name": "cloud-vm-hk-2", "public_ip_cidr": "8.8.8.20/24"},
)
assert hk_again.status_code == 201, hk_again.text
services["hk"].cache_template = original_hk_cache
assert [marker["status"] for marker in observed_markers] == ["syncing"]
fallback_volume_name = observed_markers[0]["replica_volume_name"]
assert fallback_volume_name != first_markers["hk"]["replica_volume_name"]
hk_cache = [row for row in services["hk"].calls
if row[0] == "cache_template"]
jp_cache = [row for row in services["jp"].calls
if row[0] == "cache_template"]
assert hk_cache[0][4:] == (None, None)
assert hk_cache[1][1] == fallback_volume_name
assert hk_cache[1][4] == first_markers["hk"]["replica_volume_name"]
assert hk_cache[1][5] == first_markers["hk"]["replica_identity"]
assert jp_cache[0][4:] == (None, None)
for host_id in ("hk", "jp"):
marker = json.loads(HostStore(store, host_id).get(
"cloud_template_replica", template_id))
assert marker["status"] == "ready"
assert marker["sha256"] == digest
assert len(marker["replica_identity"]) == 64
# A successful trusted reuse keeps the proven prior generation even
# though the durable syncing marker contained a fresh fallback name.
assert marker["replica_volume_name"] == (
first_markers[host_id]["replica_volume_name"])
# Rebinding the logical host record to a different SSH host identity must
# invalidate its ready proof even when the pool name is unchanged.
engine.registry.components["hk"].manager.profile["fingerprint"] = (
"SHA256:replacement-host")
rebound = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0003"},
{**body, "name": "cloud-vm-hk-3", "public_ip_cidr": "8.8.8.21/24"},
)
assert rebound.status_code == 201, rebound.text
hk_cache = [row for row in services["hk"].calls
if row[0] == "cache_template"]
assert hk_cache[-1][4:] == (None, None)
rebound_marker = json.loads(HostStore(store, "hk").get(
"cloud_template_replica", template_id))
assert rebound_marker["status"] == "ready"
assert rebound_marker["replica_volume_name"] == hk_cache[-1][1]
assert (rebound_marker["replica_volume_name"]
!= first_markers["hk"]["replica_volume_name"])
# A crash/failure after the syncing marker was stored must retry that
# exact planned generation. Allocating yet another name would leak the
# possibly partial remote volume and make recovery non-deterministic.
failed_calls = []
def fail_hk_cache(image_path, volume_name, virtual_size, **kwargs):
failed_calls.append((volume_name, kwargs, json.loads(
HostStore(store, "hk").get(
"cloud_template_replica", template_id))))
raise VMError("云镜像缓存同步中断", 502)
services["hk"].cache_template = fail_hk_cache
interrupted = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0004"},
{**body, "name": "cloud-vm-hk-4", "public_ip_cidr": "8.8.8.22/24"},
)
assert interrupted.status_code == 502, interrupted.text
assert len(failed_calls) == 1
planned_volume_name, failed_kwargs, failed_marker = failed_calls[0]
assert failed_marker["status"] == "syncing"
assert failed_marker["replica_volume_name"] == planned_volume_name
assert failed_kwargs["trusted_replica_volume_name"] == (
rebound_marker["replica_volume_name"])
assert failed_kwargs["trusted_replica_identity"] == (
rebound_marker["replica_identity"])
services["hk"].cache_template = original_hk_cache
retried = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0005"},
{**body, "name": "cloud-vm-hk-5", "public_ip_cidr": "8.8.8.23/24"},
)
assert retried.status_code == 201, retried.text
hk_cache = [row for row in services["hk"].calls
if row[0] == "cache_template"]
assert hk_cache[-1][1] == planned_volume_name
assert hk_cache[-1][4:] == (None, None)
recovered_marker = json.loads(HostStore(store, "hk").get(
"cloud_template_replica", template_id))
assert recovered_marker["status"] == "ready"
assert recovered_marker["replica_volume_name"] == planned_volume_name
assert [row for row in services["hk"].calls
if row[0] == "create_cloud_vm"][-1][2] == planned_volume_name
assert (tmp_path / "cloud-templates"
/ engine.registry.cloud_templates.get(template_id)["volume_name"]).is_file()
def test_legacy_host_template_id_remains_usable_for_an_already_queued_order(
boundary):
client, engine, store, _logs, services = boundary
# Initialize the original target host, then seed the same host-scoped
# catalog record an order created before the global-catalog upgrade holds.
assert call(client, "GET", "/internal/v1/hosts/legacy/vms").status_code == 200
profile = engine.registry.components["legacy"].manager.profile
scope = template_scope(profile)
digest = "d" * 64
template_id = template_id_for(scope, digest, "10")
DatabaseTemplateStore(HostStore(store, "legacy")).add(scope, {
"id": template_id,
"name": "Legacy AlmaLinux",
"os_family": "AlmaLinux",
"os_version": "10",
"sha256": digest,
"volume_name": template_volume_name(template_id),
"virtual_size": 2 * 1024**3,
"image_bytes": 1024,
"created_at": 1,
})
response = call(
client, "POST", "/internal/v1/hosts/legacy/vms",
{**USER, "can_provision": True,
"operation_id": "legacy-template-order-0001"},
{**CLOUD_CREATE_BODY, "template_id": template_id,
"name": "legacy-paid-order", "public_ip_cidr": "8.8.8.22/24"},
)
assert response.status_code == 201, response.text
assert not any(row[0] == "cache_template"
for row in services["legacy"].calls)
create = next(row for row in services["legacy"].calls
if row[0] == "create_cloud_vm")
assert create[2] == template_volume_name(template_id)
def test_global_cloud_template_delete_is_idempotent_and_retries_host_gc(
boundary, tmp_path):
client, engine, store, _logs, services = boundary
imported, payload = import_cloud_template_for_test(client, tmp_path)
template_id = imported["id"]
template_created_at = imported["created_at"]
source_volume_name = template_volume_name(template_id)
source_path = tmp_path / "cloud-templates" / source_volume_name
assert source_path.is_file()
for host_id in ("hk", "jp"):
assert call(
client, "GET", f"/internal/v1/hosts/{host_id}/resources"
).status_code == 200
HostStore(store, host_id).set(
"cloud_template_replica", template_id, "ready-marker")
assert call(
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
ADMIN, {"created_at": template_created_at},
).status_code == 403
services["hk"].cloud_template_delete_error = VMError(
"云镜像仍被虚拟机引用,不能删除", 409)
original_store_delete = store.delete
failed_catalog_delete = False
def fail_first_catalog_delete(kind, key):
nonlocal failed_catalog_delete
if (kind == "cloud_template" and key == template_id
and not failed_catalog_delete):
failed_catalog_delete = True
raise RuntimeError("catalog delete failed before commit")
return original_store_delete(kind, key)
store.delete = fail_first_catalog_delete
try:
deleted = call(
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
SUPER, {"created_at": template_created_at},
)
finally:
store.delete = original_store_delete
assert failed_catalog_delete is True
assert deleted.status_code == 200, deleted.text
assert deleted.json() == {
"deleted": True, "cache_cleanup_pending": True,
}
assert not source_path.exists()
assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == {
"templates": [],
}
for _ in range(100):
tombstone = json.loads(store.get(
"cloud_template_tombstone", template_id))
if tombstone["pending_hosts"] == ["hk"]:
break
time.sleep(0.01)
assert tombstone["source_deleted"] is True
assert tombstone["source_volume_name"] == source_volume_name
assert tombstone["pending_hosts"] == ["hk"]
assert HostStore(store, "jp").get(
"cloud_template_replica", template_id) is None
assert HostStore(store, "hk").get(
"cloud_template_replica", template_id) is not None
services["hk"].cloud_template_delete_error = VMError(
"存储池 images 不存在", 404)
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
still_pending = json.loads(store.get(
"cloud_template_tombstone", template_id))
assert still_pending["pending_hosts"] == ["hk"]
assert HostStore(store, "hk").get(
"cloud_template_replica", template_id) is not None
services["hk"].cloud_template_delete_error = None
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
terminal = json.loads(store.get("cloud_template_tombstone", template_id))
assert terminal["pending_hosts"] == []
assert HostStore(store, "hk").get(
"cloud_template_replica", template_id) is None
repeated = call(
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
SUPER, {"created_at": template_created_at})
assert repeated.status_code == 200
assert repeated.json() == {
"deleted": True, "cache_cleanup_pending": False,
}
# A duplicate task from the old deletion must not touch a later deletion
# generation (or the same-ID image/replica that may exist between them).
old_deletion_id = terminal["deletion_id"]
engine.registry.cloud_templates._prepare_reimport(template_id)
reimported = {
**imported,
"volume_name": source_volume_name,
"created_at": template_created_at + 1,
}
temporary = engine.registry.cloud_templates.files.directory() \
/ ".cloud-image-reimport-boundary.qcow2"
temporary.write_bytes(payload)
temporary.chmod(0o600)
engine.registry.cloud_templates.files.publish(
temporary, template_id, expected_size=len(payload))
engine.registry.cloud_templates.catalog.add(reimported)
stale_delete = call(
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
SUPER, {"created_at": template_created_at})
assert stale_delete.status_code == 409
assert engine.registry.cloud_templates.get(
template_id)["created_at"] == template_created_at + 1
next_tombstone = {
**terminal,
"deletion_id": "b" * 32,
"template_created_at": template_created_at + 1,
"pending_hosts": ["hk"],
}
store.set("cloud_template_tombstone", template_id,
json.dumps(next_tombstone))
calls_before = list(services["hk"].calls)
engine._cleanup_cloud_template_tombstone_on_host_sync(
template_id, old_deletion_id, "hk", services["hk"],
engine.registry.components["hk"],
)
assert services["hk"].calls == calls_before
assert json.loads(store.get(
"cloud_template_tombstone", template_id))["pending_hosts"] == ["hk"]
def test_user_iso_routes_are_owner_scoped_and_create_uses_opaque_id(boundary, tmp_path):
client, _engine, _store, _logs, services = boundary
download = tmp_path / "download.iso"
download.write_bytes(b"i" * (1024 * 1024))
def fake_download(_url, _sha256, _directory, *, max_bytes):
assert max_bytes >= download.stat().st_size
return download, download.stat().st_size, "a" * 64
with patch("virsh_panel.user_isos.download_user_iso", side_effect=fake_download):
queued = call(client, "POST", "/internal/v1/user-isos/import",
USER, {"url": "https://downloads.example.test/os.iso",
"name": "My Installer", "sha256": ""})
assert queued.status_code == 202
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
iso_id = job["iso"]["id"]
assert "volume_name" not in job["iso"]
listed = call(client, "GET", "/internal/v1/user-isos").json()
assert [row["id"] for row in listed["isos"]] == [iso_id]
assert "volume_name" not in listed["isos"][0]
global_source = (tmp_path / "user-isos" / str(USER["user_id"])
/ f"panel-user-iso-{iso_id}.iso")
assert global_source.is_file()
assert services == {}
assert call(
client, "GET", "/internal/v1/hosts/hk/user-isos", USER,
).status_code == 404
assert call(
client, "POST", "/internal/v1/hosts/hk/user-isos/import", USER, {
"url": "https://downloads.example.test/alias.iso",
"name": "Host Alias Must Fail", "sha256": "",
},
).status_code == 404
assert call(
client, "DELETE", f"/internal/v1/hosts/hk/user-isos/{iso_id}", USER,
).status_code == 404
assert global_source.is_file()
other = {**USER, "user_id": 42}
assert call(client, "GET", f"/internal/v1/user-isos/{iso_id}",
other).status_code == 404
assert call(client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}",
other).status_code == 404
assert call(client, "DELETE", f"/internal/v1/user-isos/{iso_id}",
other).status_code == 404
provisioning = {**USER, "can_provision": True,
"operation_id": "create-user-iso-0001"}
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
body["user_iso_id"] = iso_id
created = call(client, "POST", "/internal/v1/hosts/hk/vms",
provisioning, body)
assert created.status_code == 201, created.text
create_calls = [row for row in services["hk"].calls
if row[0] == "create_user_iso_vm"]
assert len(create_calls) == 1
assert create_calls[0][2] == USER["user_id"]
assert create_calls[0][4] == iso_id
cache_calls = [row for row in services["hk"].calls
if row[0] == "cache_user_iso"]
assert len(cache_calls) == 1
assert cache_calls[0][4] is False
other_host_actor = {
**provisioning, "operation_id": "create-user-iso-0003",
}
other_host_body = {**body, "name": "other-host-installer"}
other_host = call(client, "POST", "/internal/v1/hosts/jp/vms",
other_host_actor, other_host_body)
assert other_host.status_code == 201, other_host.text
assert len([row for row in services["jp"].calls
if row[0] == "cache_user_iso"]) == 1
assert global_source.is_file()
ambiguous_actor = {**provisioning, "operation_id": "create-user-iso-0002"}
ambiguous = call(client, "POST", "/internal/v1/hosts/hk/vms",
ambiguous_actor, {**body, "iso": "legacy.iso"})
assert ambiguous.status_code == 422
def test_user_iso_writes_require_manage_capability(boundary):
client, *_rest = boundary
actor = {**USER, "can_manage": False}
response = call(client, "POST", "/internal/v1/user-isos/import",
actor, {"url": "https://downloads.example.test/os.iso",
"name": "Installer", "sha256": ""})
assert response.status_code == 403
def test_concurrent_first_use_serializes_replica_marker_and_cache(boundary, tmp_path):
client, _engine, _store, _logs, services = boundary
download = tmp_path / "concurrent.iso"
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, "1" * 64)):
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
"url": "https://downloads.example.test/concurrent.iso",
"name": "Concurrent Installer", "sha256": "",
})
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
iso_id = job["iso"]["id"]
# Materialize the target child app before starting concurrent requests.
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
service = services["hk"]
release = threading.Event()
service.user_iso_cache_release = release
base = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
base["user_iso_id"] = iso_id
def create(number):
actor = {**USER, "can_provision": True,
"operation_id": f"concurrent-user-iso-{number:04d}"}
return call(client, "POST", "/internal/v1/hosts/hk/vms", actor,
{**base, "name": f"concurrent-vm-{number}"})
with ThreadPoolExecutor(max_workers=2) as executor:
first = executor.submit(create, 1)
assert service.user_iso_cache_started.wait(timeout=2)
second = executor.submit(create, 2)
time.sleep(0.05)
release.set()
responses = [first.result(timeout=5), second.result(timeout=5)]
assert [response.status_code for response in responses] == [201, 201]
cache_calls = [row for row in service.calls if row[0] == "cache_user_iso"]
assert [row[4] for row in cache_calls] == [False, True]
def test_user_iso_source_stays_in_use_through_vm_create(boundary, tmp_path):
client, engine, _store, _logs, services = boundary
iso_id = import_user_iso_for_test(client, tmp_path, "Create Lease Installer")
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
service = services["hk"]
release = threading.Event()
service.user_iso_create_release = release
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
body["user_iso_id"] = iso_id
actor = {**USER, "can_provision": True,
"operation_id": "user-iso-source-lease-0001"}
with ThreadPoolExecutor(max_workers=1) as executor:
creating = executor.submit(
call, client, "POST", "/internal/v1/hosts/hk/vms", actor, body)
assert service.user_iso_create_started.wait(timeout=2)
with pytest.raises(TemplateError, match="正在同步") as busy:
engine.registry.user_isos.delete_for(USER["user_id"], iso_id)
assert busy.value.status == 409
release.set()
created = creating.result(timeout=5)
assert created.status_code == 201, created.text
assert engine.registry.user_isos.get_for(USER["user_id"], iso_id)["id"] == iso_id
def test_ready_replica_marker_survives_failed_reuse(boundary, tmp_path):
client, engine, store, _logs, services = boundary
iso_id = import_user_iso_for_test(client, tmp_path, "Ready Marker Installer")
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
body["user_iso_id"] = iso_id
first = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**USER, "can_provision": True,
"operation_id": "ready-marker-create-0001"}, body)
assert first.status_code == 201, first.text
marker_key = f"{USER['user_id']}:{iso_id}"
scoped = HostStore(store, "hk")
assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready"
with patch.object(
engine.registry.user_isos, "_verify_source",
side_effect=TemplateError("ISO 源文件的 SHA-256 校验值不匹配", 500)):
failed = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**USER, "can_provision": True,
"operation_id": "ready-marker-create-0002"},
{**body, "name": "ready-marker-failed"})
assert failed.status_code == 500
assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready"
recovered = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**USER, "can_provision": True,
"operation_id": "ready-marker-create-0003"},
{**body, "name": "ready-marker-recovered"})
assert recovered.status_code == 201, recovered.text
cache_calls = [row for row in services["hk"].calls
if row[0] == "cache_user_iso"]
assert cache_calls[-1][4] is True
def test_global_delete_keeps_tombstone_until_host_cache_is_reclaimed(
boundary, tmp_path):
client, _engine, store, _logs, services = boundary
download = tmp_path / "delete.iso"
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, "2" * 64)):
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
"url": "https://downloads.example.test/delete.iso",
"name": "Delete Installer", "sha256": "",
})
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
iso_id = job["iso"]["id"]
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
body["user_iso_id"] = iso_id
created = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**USER, "can_provision": True,
"operation_id": "delete-user-iso-cache-0001"}, body)
assert created.status_code == 201, created.text
from virsh_panel.libvirt_service import VMError
services["hk"].user_iso_delete_error = VMError(
"用户 ISO 仍被虚拟机安装过程引用,不能删除", 409)
deleted = call(
client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
assert deleted.status_code == 200
assert deleted.json() == {"deleted": True, "cache_cleanup_pending": True}
assert call(client, "GET", "/internal/v1/user-isos", USER).json()["isos"] == []
tombstone_key = f"{USER['user_id']}:{iso_id}"
for _ in range(100):
if any(row[0] == "delete_user_iso" for row in services["hk"].calls):
break
time.sleep(0.01)
assert store.get("user_iso_tombstone", tombstone_key) is not None
scoped = HostStore(store, "hk")
assert scoped.get("user_iso_replica", tombstone_key) is not None
services["hk"].user_iso_delete_error = None
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
assert store.get("user_iso_tombstone", tombstone_key) is None
assert scoped.get("user_iso_replica", tombstone_key) is None
def test_concurrent_global_deletes_preserve_committed_tombstone(boundary, tmp_path):
client, engine, store, _logs, services = boundary
iso_id = import_user_iso_for_test(client, tmp_path, "Concurrent Delete Installer")
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
record = engine.registry.user_isos.get_for(USER["user_id"], iso_id)
marker_key = f"{USER['user_id']}:{iso_id}"
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
"status": "ready",
"volume_name": record["volume_name"],
"size_bytes": record["size_bytes"],
"sha256": record["sha256"],
}))
from virsh_panel.libvirt_service import VMError
services["hk"].user_iso_volumes.add((USER["user_id"], record["volume_name"]))
services["hk"].user_iso_delete_error = VMError(
"用户 ISO 仍被虚拟机安装过程引用,不能删除", 409)
original_delete = engine.registry.user_isos.delete_for
started = threading.Event()
release = threading.Event()
def blocking_delete(user_id, selected_iso_id):
started.set()
assert release.wait(timeout=5)
return original_delete(user_id, selected_iso_id)
try:
with patch.object(engine.registry.user_isos, "delete_for",
side_effect=blocking_delete) as delete_for:
with ThreadPoolExecutor(max_workers=2) as executor:
first = executor.submit(
call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
assert started.wait(timeout=2)
second = executor.submit(
call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
time.sleep(0.05)
release.set()
responses = [first.result(timeout=5), second.result(timeout=5)]
assert delete_for.call_count == 1
finally:
release.set()
assert [response.status_code for response in responses] == [200, 200]
tombstone = json.loads(store.get("user_iso_tombstone", marker_key))
assert tombstone["source_deleted"] is True
assert tombstone["pending_hosts"] == ["hk"]
def test_user_iso_cache_cleanup_blocks_online_update(boundary, tmp_path):
client, engine, store, _logs, services = boundary
iso_id = import_user_iso_for_test(client, tmp_path, "Update Gate Installer")
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
record = engine.registry.user_isos.get_for(USER["user_id"], iso_id)
marker_key = f"{USER['user_id']}:{iso_id}"
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
"status": "ready",
"volume_name": record["volume_name"],
"size_bytes": record["size_bytes"],
"sha256": record["sha256"],
}))
service = services["hk"]
service.user_iso_volumes.add((USER["user_id"], record["volume_name"]))
release = threading.Event()
service.user_iso_delete_release = release
updater = FakeUpdater()
updater.token = "valid-token-123456"
engine.updater = updater
try:
deleted = call(
client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
assert deleted.status_code == 200
assert service.user_iso_delete_started.wait(timeout=2)
blocked = call(client, "POST", "/internal/v1/update/apply", SUPER, {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert blocked.status_code == 409
assert updater.active is None
finally:
release.set()
for _ in range(100):
if not engine.maintenance_tasks:
break
time.sleep(0.01)
assert not engine.maintenance_tasks
def test_host_get_skips_cache_cleanup_during_active_update(boundary):
client, engine, store, _logs, services = boundary
iso_id = "f" * 32
volume_name = f"panel-user-iso-{iso_id}.iso"
marker_key = f"{USER['user_id']}:{iso_id}"
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
"status": "ready", "volume_name": volume_name,
"size_bytes": 1024 * 1024, "sha256": "f" * 64,
}))
store.set("user_iso_tombstone", marker_key, json.dumps({
"user_id": USER["user_id"], "iso_id": iso_id,
"volume_name": volume_name, "pending_hosts": ["hk"],
"source_deleted": True, "created_at": 1,
}))
updater = FakeUpdater()
updater.active = {"job_id": "update-job-1234567890", "status": "running"}
engine.updater = updater
response = call(client, "GET", "/internal/v1/hosts/hk/resources")
assert response.status_code == 200
assert not any(row[0] == "delete_user_iso" for row in services["hk"].calls)
assert store.get("user_iso_tombstone", marker_key) is not None
def test_slow_tombstone_cleanup_does_not_block_engine_startup(tmp_path):
class StartupRegistry:
def __init__(self):
self.components = {"offline": object()}
self.user_isos = SimpleNamespace(has_active_jobs=lambda: False)
self.started = False
self.closed = False
async def start(self):
self.started = True
async def close(self):
self.closed = True
async def scenario():
registry = StartupRegistry()
updater = FakeUpdater()
engine = EngineApplication(
{"service_secret": base64.b64encode(SECRET).decode(),
"data_dir": str(tmp_path)},
MemoryStore(), MemoryStore(), registry=registry, updater=updater,
)
cleanup_started = asyncio.Event()
cleanup_release = asyncio.Event()
async def slow_cleanup(host_id):
assert host_id == "offline"
cleanup_started.set()
await cleanup_release.wait()
engine._cleanup_host_user_iso_tombstones = slow_cleanup
incoming = asyncio.Queue()
outgoing = asyncio.Queue()
async def receive():
return await incoming.get()
async def send(message):
await outgoing.put(message)
await incoming.put({"type": "lifespan.startup"})
lifespan = asyncio.create_task(engine._lifespan(receive, send))
startup = await asyncio.wait_for(outgoing.get(), timeout=0.5)
assert startup == {"type": "lifespan.startup.complete"}
await asyncio.wait_for(cleanup_started.wait(), timeout=0.5)
assert registry.started is True
assert engine.maintenance_tasks
assert engine._long_running_host_job() is True
cleanup_release.set()
await incoming.put({"type": "lifespan.shutdown"})
shutdown = await asyncio.wait_for(outgoing.get(), timeout=0.5)
assert shutdown == {"type": "lifespan.shutdown.complete"}
await asyncio.wait_for(lifespan, timeout=0.5)
assert registry.closed is True
asyncio.run(scenario())
def test_user_iso_global_import_does_not_contact_a_host(boundary, tmp_path):
client, _engine, _store, _logs, services = boundary
download = tmp_path / "global.iso"
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, "f" * 64)):
response = call(client, "POST", "/internal/v1/user-isos/import",
USER, {"url": "https://downloads.example.test/os.iso",
"name": "Installer", "sha256": ""})
assert response.status_code == 202
job_id = response.json()["job_id"]
for _ in range(100):
job = call(
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
assert services == {}
def test_user_iso_download_is_bounded_by_panel_storage(boundary, tmp_path):
client, engine, _store, _logs, _services = boundary
download = tmp_path / "bounded.iso"
download.write_bytes(b"i" * (1024 * 1024))
def bounded_download(_url, _sha256, _directory, *, max_bytes):
assert max_bytes == 2 * 1024 * 1024
return download, download.stat().st_size, "d" * 64
with patch.object(engine.registry.user_isos.files, "available_bytes",
return_value=2 * 1024 * 1024), patch(
"virsh_panel.user_isos.download_user_iso", side_effect=bounded_download):
response = call(client, "POST", "/internal/v1/user-isos/import",
USER, {"url": "https://downloads.example.test/os.iso",
"name": "Bounded Installer", "sha256": ""})
assert response.status_code == 202
job_id = response.json()["job_id"]
for _ in range(100):
job = call(client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
def test_user_iso_job_hides_internal_paths_in_failures(boundary):
client, _engine, _store, _logs, _services = boundary
with patch("virsh_panel.user_isos.download_user_iso",
side_effect=TemplateError("目录 /private/data/user-isos/41 损坏", 500)):
response = call(client, "POST", "/internal/v1/user-isos/import",
USER, {"url": "https://downloads.example.test/os.iso",
"name": "Error Installer", "sha256": ""})
assert response.status_code == 202
job_id = response.json()["job_id"]
for _ in range(100):
job = call(client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "failed"
assert job["error"] == "ISO 下载或保存失败,请联系管理员查看日志"
assert "/private/data" not in json.dumps(job, ensure_ascii=False)
def test_user_iso_import_persistence_failure_does_not_stick_pending(boundary, tmp_path):
client, _engine, store, _logs, _services = boundary
assert call(client, "GET", "/internal/v1/user-isos").status_code == 200
original_set = store.set
def fail_user_iso_job(kind, key, value):
if kind == "user_iso_import_job":
raise DatabaseError()
return original_set(kind, key, value)
body = {"url": "https://downloads.example.test/os.iso",
"name": "Installer", "sha256": ""}
with patch.object(store, "set", side_effect=fail_user_iso_job):
failed = call(client, "POST", "/internal/v1/user-isos/import",
USER, body)
assert failed.status_code == 503
download = tmp_path / "retry.iso"
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, "b" * 64)):
retried = call(client, "POST", "/internal/v1/user-isos/import",
USER, body)
assert retried.status_code == 202, retried.text
for _ in range(100):
job = call(
client, "GET",
f"/internal/v1/user-isos/import-jobs/{retried.json()['job_id']}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
def test_interrupted_user_iso_import_is_restored_as_failed(boundary):
client, _engine, store, _logs, _services = boundary
job_id = "interrupted-user-iso"
scope = user_iso_scope(USER["user_id"])
store.set("user_iso_import_job", job_id, json.dumps({
"job_id": job_id, "scope": scope, "owner_user_id": USER["user_id"],
"status": "running", "phase": "正在安全下载 ISO", "created_at": 1,
}))
_engine.registry.user_isos._restored = False
_engine.registry.user_isos.restore_jobs()
restored = call(
client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}",
)
assert restored.status_code == 200
assert restored.json()["status"] == "failed"
assert "重启" in restored.json()["phase"]
def test_user_iso_list_exposes_only_owners_active_import(boundary, tmp_path):
client, engine, _store, _logs, _services = boundary
download = tmp_path / "active.iso"
download.write_bytes(b"i" * (1024 * 1024))
started = threading.Event()
release = threading.Event()
def blocking_download(_url, _sha256, _directory, *, max_bytes):
assert max_bytes >= download.stat().st_size
started.set()
assert release.wait(timeout=5)
return download, download.stat().st_size, "c" * 64
body = {"url": "https://downloads.example.test/os.iso",
"name": "Active Installer", "sha256": ""}
try:
with patch("virsh_panel.user_isos.download_user_iso", side_effect=blocking_download):
queued = call(client, "POST", "/internal/v1/user-isos/import",
USER, body)
assert queued.status_code == 202
assert started.wait(timeout=2)
assert engine._long_running_host_job() is True
own = call(client, "GET", "/internal/v1/user-isos", USER).json()
active = own["active_import_job"]
assert active["job_id"] == queued.json()["job_id"]
assert active["status"] == "running"
assert set(active) <= {"job_id", "status", "phase", "created_at",
"started_at", "finished_at", "iso", "error"}
assert "scope" not in active and "owner_user_id" not in active
other = call(client, "GET", "/internal/v1/user-isos",
{**USER, "user_id": 42}).json()
assert "active_import_job" not in other
release.set()
for _ in range(100):
job = call(
client, "GET",
f"/internal/v1/user-isos/import-jobs/{active['job_id']}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
finally:
release.set()
assert job["status"] == "completed", job
assert "active_import_job" not in call(
client, "GET", "/internal/v1/user-isos", USER).json()
def test_hmac_binds_actor_payload_target_and_rejects_replay():
store = MemoryStore()
body = b"{}"
target = "/internal/v1/hosts/hk/vms?range=1h"
headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER,
timestamp=1000, nonce="n" * 24).items()}
assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
changed = {**headers, "x-cloud-actor": encode_actor(SUPER)}
with pytest.raises(EngineAuthError):
authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061)
invalid_cleanup = {**USER, "system_cleanup": "true"}
invalid_headers = {key.lower(): value for key, value in signed_headers(
SECRET, "POST", target, body, invalid_cleanup,
timestamp=1000, nonce="c" * 24,
).items()}
with pytest.raises(EngineAuthError):
authenticate(
invalid_headers, SECRET, "POST", target,
hashlib.sha256(body).hexdigest(), store, now=1000,
)
def test_private_route_allowlist_and_no_browser_admin_cookie(boundary):
client, *_ = boundary
for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin",
"/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"):
assert call(client, "POST", path, SUPER, {}).status_code == 404
assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401
assert TRUSTED_ACTOR.get() is None
def test_internal_update_routes_are_super_admin_only_and_never_echo_token(boundary):
client, engine, *_ = boundary
updater = FakeUpdater()
triggered = []
engine.updater = updater
def trigger():
triggered.append(True)
updater.active["status"] = "running"
engine.update_trigger = trigger
settings_path = "/internal/v1/update/settings"
assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401
assert call(client, "GET", settings_path, USER).status_code == 403
assert call(client, "GET", settings_path, ADMIN).status_code == 403
assert call(client, "GET", settings_path, {**SUPER, "can_manage": False}).status_code == 403
token = "gitea-private-token-123456"
saved = call(client, "PUT", settings_path, SUPER, {"access_token": token})
assert saved.status_code == 200
assert saved.json()["token_configured"] is True
assert saved.json()["repository"] == "https://g.luoli.live/root/virt.git"
assert saved.json()["branch"] == "venix"
assert token not in saved.text
assert token not in call(client, "GET", settings_path, SUPER).text
checked = call(client, "POST", "/internal/v1/update/check", SUPER, {})
assert checked.status_code == 200
assert checked.json()["status"] == "update_available"
applied = call(client, "POST", "/internal/v1/update/apply", SUPER, {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert applied.status_code == 202
assert applied.json()["job_id"] == "update-job-1234567890"
assert "private" not in applied.json() and token not in applied.text
assert triggered == [True]
job = call(client, "GET", "/internal/v1/update/jobs/update-job-1234567890", SUPER)
assert job.status_code == 200 and token not in job.text and "private" not in job.json()
# While the persisted update job is active, reads remain available but no
# host/VM mutation may begin and overlap the code/service restart.
assert call(client, "GET", "/internal/v1/health", SUPER).status_code == 200
assert call(client, "POST", "/internal/v1/hosts/scan", SUPER,
{"host": "host.example.test", "port": 22}).status_code == 409
def test_internal_update_validation_clear_and_trigger_failure(boundary):
client, engine, *_ = boundary
updater = FakeUpdater()
engine.updater = updater
settings_path = "/internal/v1/update/settings"
assert call(client, "PUT", settings_path, SUPER, {}).status_code == 422
assert call(client, "PUT", settings_path, SUPER,
{"access_token": "x", "unexpected": True}).status_code == 422
assert call(client, "POST", "/internal/v1/update/check", SUPER,
{"repository": "https://evil.example/repo.git"}).status_code == 422
assert call(client, "POST", "/internal/v1/update/apply", SUPER,
{"expected_current_commit": "a" * 40}).status_code == 422
assert call(client, "GET", "/internal/v1/update/jobs/missing-job-123456", SUPER).status_code == 404
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
cleared = call(client, "PUT", settings_path, SUPER, {"clear_token": True})
assert cleared.status_code == 200 and cleared.json()["token_configured"] is False
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
def fail_trigger():
from virsh_panel.updater import UpdateError
raise UpdateError("独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503)
engine.update_trigger = fail_trigger
failed = call(client, "POST", "/internal/v1/update/apply", SUPER, {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert failed.status_code == 503
job = updater.jobs["update-job-1234567890"]
assert job["status"] == "failed"
assert "令牌" not in job["error"]
@pytest.mark.parametrize(("state", "expected"), [
("activating", True),
("active", True),
("deactivating", True),
("inactive", False),
("failed", False),
("maintenance", None),
])
def test_update_oneshot_transitional_states_keep_maintenance_gate(state, expected):
with patch("virsh_panel.internal.subprocess.run",
return_value=SimpleNamespace(returncode=0, stdout=state + "\n")):
assert EngineApplication._update_service_active() is expected
def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary):
client, engine, *_ = boundary
path = "/internal/v1/hosts/hk"
assert call(client, "GET", path + "/host").status_code == 403
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200
assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200
assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404
assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == []
assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False},
{"action": "reboot"}).status_code == 403
def test_existing_unmanaged_vms_are_claimed_once_for_configuring_admin(boundary):
client, engine, _, _, services = boundary
path = "/internal/v1/hosts/hk"
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
manual_uuid = "22222222-2222-4222-8222-222222222222"
orphaned_managed_uuid = "33333333-3333-4333-8333-333333333333"
services["hk"].other_vms = [
{"uuid": manual_uuid, "name": "manual-vm", "state": "shutoff",
"vcpu": 2, "memory_mib": 2048, "disks": [], "managed": False},
{"uuid": orphaned_managed_uuid, "name": "unknown-panel-vm", "state": "shutoff",
"vcpu": 2, "memory_mib": 2048, "disks": [], "managed": True},
]
body = {"known_owners": {VM_ID: 41}}
assert call(client, "POST", path + "/host/discover-vms", ADMIN, body).status_code == 403
discovered = call(client, "POST", path + "/host/discover-vms", SUPER, body)
assert discovered.status_code == 200, discovered.text
rows = {row["uuid"]: row for row in discovered.json()["items"]}
assert rows[VM_ID]["owner_user_id"] == 41
assert rows[manual_uuid]["owner_user_id"] == SUPER["user_id"]
assert orphaned_managed_uuid not in rows
assert discovered.json()["claimed_count"] == 2
assert discovered.json()["untracked_managed_count"] == 1
assert discovered.json()["total_count"] == 3
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"]
adopted = call(client, "POST", path + "/host/discover-vms", SUPER,
{**body, "claim_unowned_managed": True})
assert adopted.status_code == 200
adopted_rows = {row["uuid"]: row for row in adopted.json()["items"]}
assert adopted_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"]
assert adopted.json()["claimed_count"] == 1
assert adopted.json()["untracked_managed_count"] == 0
repeated = call(client, "POST", path + "/host/discover-vms",
{**SUPER, "user_id": 2}, {**body, "claim_unowned_managed": True})
assert repeated.status_code == 200
repeated_rows = {row["uuid"]: row for row in repeated.json()["items"]}
assert repeated_rows[manual_uuid]["owner_user_id"] == SUPER["user_id"]
assert repeated_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"]
assert repeated.json()["claimed_count"] == 0
assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"]
assert {row["uuid"] for row in call(
client, "GET", path + "/vms", {**USER, "user_id": SUPER["user_id"]}
).json()["items"]} == {manual_uuid, orphaned_managed_uuid}
assert call(client, "GET", path + "/vms", {**USER, "user_id": 2}).json()["items"] == []
def test_stable_host_identity_and_namespaces():
store = MemoryStore()
first, second = HostStore(store, "a"), HostStore(store, "b")
first.set("host", "current", "one")
second.set("host", "current", "two")
assert first.get("host", "current") == "one"
assert second.get("host", "current") == "two"
a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"}
assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"})
assert host_scope(a) != host_scope({**a, "host_id": "b"})
assert MainStore.RECORD_TABLE == "engine_records"
assert LogStore.METRIC_TABLE == "engine_vm_metric_samples"
def test_lightweight_vm_state_reads_are_allowlisted(boundary):
client, _engine, _store, _logs, services = boundary
listed = call(client, "GET", "/internal/v1/hosts/hk/vm-states", SUPER)
assert listed.status_code == 200, listed.text
assert listed.json()["items"] == [{
"uuid": VM_ID, "name": "alma-test", "state": "running",
}]
single = call(
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/state", SUPER,
)
assert single.status_code == 200, single.text
assert single.json() == {
"uuid": VM_ID, "name": "alma-test", "state": "running",
}
assert ("list_vm_states",) in services["hk"].calls
assert ("get_vm_state", VM_ID) in services["hk"].calls
def test_selected_vm_metadata_is_allowlisted_and_owner_scoped(boundary):
client, engine, _store, _logs, services = boundary
path = f"/internal/v1/hosts/hk/vm-metadata?ids={VM_ID}"
admin = call(client, "GET", path, SUPER)
assert admin.status_code == 200, admin.text
assert admin.json()["items"][0]["uuid"] == VM_ID
assert services["hk"].calls[-1] == ("list_vm_metadata", (VM_ID,))
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
component.state.users.assign_vm(scope, VM_ID, USER["user_id"])
owner = call(client, "GET", path, USER)
assert owner.status_code == 200, owner.text
calls_after_owner = services["hk"].calls.count(("list_vm_metadata", (VM_ID,)))
assert calls_after_owner == 2
stranger = call(client, "GET", path, {**USER, "user_id": 42})
assert stranger.status_code == 404
assert stranger.json()["error"] == "虚拟机不存在"
assert services["hk"].calls.count(("list_vm_metadata", (VM_ID,))) == calls_after_owner
def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403
provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"}
created = call(client, "POST", path, provisioning, CREATE_BODY)
assert created.status_code == 201, created.text
vm_id = created.json()["uuid"]
assert created.json()["result"]["uuid"] == vm_id
assert created.json()["status"] == "succeeded"
component = engine.registry.components["hk"]
assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41)
again = call(client, "POST", path, provisioning, CREATE_BODY)
assert again.status_code == 201 and again.json()["uuid"] == vm_id
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409
operation = call(client, "GET", "/internal/v1/operations/order-000001").json()
assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id
assert operation["safe_to_refund"] is False
assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404
def test_uncertain_creation_never_refunds_or_recreates(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
call(client, "GET", path)
services["hk"].create_failure = True
actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"}
failed = call(client, "POST", path, actor, CREATE_BODY)
assert failed.status_code == 502
assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"]
record = json.loads(store.get("operation", "order-uncertain"))
assert record["resource_naming"] == "uuid-v1"
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
assert component.state.users.owns_vm(scope, record["vm_uuid"], 41)
component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password")
component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]}))
again = call(client, "POST", path, actor, CREATE_BODY)
assert again.status_code == 202 and again.json()["status"] == "uncertain"
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
services["hk"].absent = True
operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json()
assert operation["status"] == "failed" and operation["safe_to_refund"] is True
assert services["hk"].absent_checks[-1] == (
record["vm_uuid"], CREATE_BODY["name"], True, False)
assert not component.state.users.owns_vm(scope, record["vm_uuid"], 41)
assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") is None
assert component.store.list("ip_claim") == {}
def test_legacy_name_collision_operation_uses_legacy_absent_reconciliation(boundary):
client, _engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
services["hk"].absent = True
legacy_vm_id = "00000000-0000-4000-8000-000000000002"
store.set("operation", "legacy-name-collision", json.dumps({
"operation_id": "legacy-name-collision",
"host_id": "hk",
"user_id": USER["user_id"],
"request_hash": "legacy",
"vm_uuid": legacy_vm_id,
"vm_name": CREATE_BODY["name"],
"kind": "create",
"resource": "vms",
"method": "POST",
"status": "uncertain",
"result": None,
"error": "虚拟机名称已存在",
"safe_to_refund": False,
"created_at": 1,
}))
operation = call(
client, "GET", "/internal/v1/operations/legacy-name-collision", USER,
).json()
assert operation["status"] == "failed"
assert operation["safe_to_refund"] is True
assert services["hk"].absent_checks[-1] == (
legacy_vm_id, CREATE_BODY["name"], False, True)
@pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"],
{**CREATE_BODY, "guest_ssh_key_id": 3}])
def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload):
client, engine, _, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
actor = {**USER, "can_provision": True, "operation_id": "rejected-create"}
response = call(client, "POST", path, actor, payload)
assert response.status_code == 422
# Simulate a dispatcher losing the POST response and querying after restart.
engine.active_operations.clear()
saved = call(client, "GET", "/internal/v1/operations/rejected-create").json()
assert saved["status"] == "failed" and saved["safe_to_refund"] is True
assert call(client, "POST", path, actor, payload).status_code == 422
assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409
assert not any(row[0] == "create_vm" for row in services["hk"].calls)
def test_admin_created_vm_also_has_external_owner(boundary):
client, engine, *_ = boundary
actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"}
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
assert response.status_code == 201
components = engine.registry.components["hk"]
assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1)
def test_power_action_has_persistent_idempotency(boundary):
client, engine, _, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403
actor = {**USER, "operation_id": "reboot-operation"}
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1
assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409
operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json()
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary):
client, engine, _, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
rescue_key = f"{scope}.{VM_ID}"
components.store.set("vm_rescue", rescue_key, json.dumps({
"original_xml": "<domain><name>alma-test</name></domain>",
"template_id": "0123456789abcdef0123456789abcdef",
"volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso",
"status": "active",
}))
services["hk"].vm.update(
state="shutoff", rescue_mode=True, rescue_staged=True,
rescue_volume_name="panel-rescue-0123456789abcdef0123456789abcdef.iso",
)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
ordinary = call(
client, "DELETE", path,
{**USER, "operation_id": "ordinary-rescue-delete", "system_cleanup": False},
)
assert ordinary.status_code == 409
assert components.store.get("vm_rescue", rescue_key) is not None
assert not any(item[0] == "delete_vm" for item in services["hk"].calls)
cleanup = call(
client, "DELETE", path,
{**USER, "operation_id": "expiry-rescue-delete", "system_cleanup": True},
)
assert cleanup.status_code == 200, cleanup.text
assert cleanup.json()["status"] == "succeeded"
assert services["hk"].calls[-1] == ("delete_vm", VM_ID, True)
assert components.store.get("vm_rescue", rescue_key) is None
assert components.state.users.owner_of_vm(scope, VM_ID) is None
@pytest.mark.parametrize(("route", "method", "target"), [
("service-suspend", "service_suspend", {"state": "shutoff", "autostart": False}),
("service-resume", "service_resume", {"autostart": True}),
("network-suspend", "network_suspend", {"network_suspended": True}),
("network-resume", "network_resume", {"network_suspended": False}),
])
def test_private_target_state_operations_are_owned_idempotent_mutations(
boundary, route, method, target):
client, engine, _, _, services = boundary
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
assert call(client, "POST", base, USER, {}).status_code == 403
assert call(client, "POST", base, {**USER, "can_manage": False,
"operation_id": f"blocked-{route}"}, {}).status_code == 403
assert call(client, "POST", base, {**USER, "user_id": 42,
"operation_id": f"foreign-{route}"}, {}).status_code == 404
actor = {**USER, "operation_id": f"target-{route}"}
first = call(client, "POST", base, actor, {})
assert first.status_code == 200, first.text
assert first.json()["status"] == "succeeded"
assert all(first.json()[key] == value for key, value in target.items())
repeated = call(client, "POST", base, actor, {})
assert repeated.status_code == 200
assert len([item for item in services["hk"].calls if item[0] == method]) == 1
@pytest.mark.parametrize(("route", "method", "drift"), [
("service-suspend", "service_suspend", {"state": "running", "autostart": True}),
("service-resume", "service_resume", {"autostart": False}),
("network-suspend", "network_suspend", {"network_suspended": False}),
("network-resume", "network_resume", {"network_suspended": True}),
])
def test_target_state_operation_replays_after_lost_response(
boundary, route, method, drift):
client, engine, store, _, services = boundary
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
actor = {**USER, "operation_id": f"replay-{route}"}
assert call(client, "POST", base, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(drift)
engine.active_operations.clear()
replayed = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
actor)
assert replayed.status_code == 200, replayed.text
assert replayed.json()["status"] == "succeeded"
assert len([item for item in services["hk"].calls if item[0] == method]) == 2
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost again")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(drift)
replayed_post = call(client, "POST", base, actor, {})
assert replayed_post.status_code == 200, replayed_post.text
assert replayed_post.json()["status"] == "succeeded"
assert len([item for item in services["hk"].calls if item[0] == method]) == 3
def test_non_reconciling_operation_lookup_never_replays_target_state(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend"
actor = {**USER, "operation_id": "peek-target-state"}
assert call(client, "POST", path, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm["network_suspended"] = False
engine.active_operations.clear()
before = len(services["hk"].calls)
peeked = call(client, "GET",
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
assert peeked.status_code == 200
assert peeked.json()["status"] == "uncertain"
assert peeked.json()["executor_active"] is False
assert len(services["hk"].calls) == before
assert services["hk"].vm["network_suspended"] is False
engine.active_operations.add(actor["operation_id"])
active = call(client, "GET",
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
assert active.status_code == 200
assert active.json()["executor_active"] is True
assert len(services["hk"].calls) == before
engine.active_operations.discard(actor["operation_id"])
def test_target_state_lock_serializes_same_vm_and_not_other_vms(boundary):
_, engine, *_ = boundary
async def exercise():
first = engine._target_lock("hk", VM_ID)
same = engine._target_lock("hk", VM_ID)
other = engine._target_lock("hk", "22222222-2222-4222-8222-222222222222")
assert first is same
assert first is not other
entered = []
release = asyncio.Event()
async def owner():
async with first:
entered.append("owner")
await release.wait()
entered.append("owner-finished")
async def waiter():
async with same:
entered.append("waiter")
owner_task = asyncio.create_task(owner())
await asyncio.sleep(0)
waiter_task = asyncio.create_task(waiter())
await asyncio.sleep(0)
assert entered == ["owner"]
release.set()
await asyncio.gather(owner_task, waiter_task)
assert entered == ["owner", "owner-finished", "waiter"]
errors = []
def run_in_isolated_loop():
try:
asyncio.run(exercise())
except BaseException as exc: # propagate assertion failures to pytest
errors.append(exc)
thread = threading.Thread(target=run_in_isolated_loop)
thread.start()
thread.join(timeout=5)
assert not thread.is_alive()
if errors:
raise errors[0]
def test_expiry_target_waits_for_inflight_user_mutation_on_same_vm(boundary):
client, engine, _, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
service = services["hk"]
action_entered = threading.Event()
release_action = threading.Event()
network_entered = threading.Event()
original_action = service.action
original_network_suspend = service.network_suspend
def slow_action(vm_id, action):
action_entered.set()
assert release_action.wait(timeout=5), "test did not release user mutation"
return original_action(vm_id, action)
def observed_network_suspend(vm_id):
network_entered.set()
return original_network_suspend(vm_id)
service.action = slow_action
service.network_suspend = observed_network_suspend
responses = {}
def user_request():
responses["action"] = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions",
{**USER, "operation_id": "concurrent-user-action"}, {"action": "reset"})
def policy_request():
responses["policy"] = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
{**USER, "operation_id": "concurrent-expiry-policy"}, {})
first = threading.Thread(target=user_request)
first.start()
assert action_entered.wait(timeout=5)
second = threading.Thread(target=policy_request)
second.start()
assert not network_entered.wait(timeout=0.2)
release_action.set()
first.join(timeout=5)
second.join(timeout=5)
assert not first.is_alive() and not second.is_alive()
assert responses["action"].status_code == 200
assert responses["policy"].status_code == 200
assert network_entered.is_set()
mutations = [item[0] for item in service.calls
if item[0] in {"action", "network_suspend"}]
assert mutations == ["action", "network_suspend"]
assert service.vm["network_suspended"] is True
def test_target_state_reconciliation_does_not_bypass_original_owner(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
store.set("operation", "unsafe-target", json.dumps({
"operation_id": "unsafe-target", "host_id": "hk", "user_id": 42,
"kind": "mutation", "resource": f"vms/{VM_ID}/network-suspend",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"actor_can_manage": True, "actor_is_admin": False,
"safe_to_refund": False,
}))
response = call(client, "GET", "/internal/v1/operations/unsafe-target",
{**USER, "user_id": 42})
assert response.status_code == 200
assert response.json()["status"] == "failed"
assert not any(item[0] == "network_suspend" for item in services["hk"].calls)
def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID,
"status": "running", "phase": "正在救援"}
components.store.set("rescue_job", "job-abc", json.dumps(job))
operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID,
"status": "uncertain", "result": None, "safe_to_refund": False}
store.set("operation", "lost-rescue-job", json.dumps(operation))
recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID
job.update(status="completed", phase="救援系统启动")
components.store.set("rescue_job", "job-abc", json.dumps(job))
completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
assert completed["status"] == "succeeded"
assert not completed["safe_to_refund"]
def test_interrupted_async_job_becomes_terminal_without_replay(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.store.set("reinstall_job", "job-abcd", json.dumps({
"operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID,
"status": "failed", "phase": "面板重启,任务已中断"}))
store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job",
"host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall",
"vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False}))
recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json()
assert recovered["status"] == "failed"
assert "中断" in recovered["error"]
@pytest.mark.parametrize("resource", [
f"vms/{VM_ID}/actions",
f"vms/{VM_ID}",
f"vms/{VM_ID}/reinstall",
f"vms/{VM_ID}/rescue",
])
def test_orphaned_mutation_becomes_terminal_without_replay(boundary, resource):
client, _, store, *_ = boundary
operation_id = "orphan-" + resource.rsplit("/", 1)[-1].replace("?", "-")
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": resource, "vm_uuid": VM_ID,
"status": "running", "result": None, "safe_to_refund": False,
"created_at": 1,
}))
recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}").json()
assert recovered["status"] == "failed"
assert "不会自动重放" in recovered["error"]
def test_service_resume_replays_exact_start_target_after_lost_response(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
services["hk"].vm.update(state="shutoff", autostart=False)
actor = {**USER, "operation_id": "resume-start-target"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume"
first = call(client, "POST", path, actor, {"start_if_suspended": True})
assert first.status_code == 200, first.text
assert services["hk"].vm["state"] == "running"
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(state="shutoff", autostart=False)
engine.active_operations.clear()
replayed = call(client, "GET", "/internal/v1/operations/resume-start-target", actor)
assert replayed.status_code == 200
assert replayed.json()["status"] == "succeeded"
assert services["hk"].vm["state"] == "running"
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
def test_service_suspend_replay_preserves_original_running_state(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
actor = {**USER, "operation_id": "suspend-running-hint"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend"
assert call(client, "POST", path, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
assert saved["service_was_running_hint"] is True
# Model a hard crash after libvirt stopped the guest but before the
# successful result was durable. Reconciliation now observes shutoff, yet
# must retain the original running state for the billing resume policy.
saved.update(status="uncertain", result=None, error="engine crashed")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(state="shutoff", autostart=False)
engine.active_operations.clear()
reconciled = call(client, "GET",
"/internal/v1/operations/suspend-running-hint", actor)
assert reconciled.status_code == 200
assert reconciled.json()["status"] == "succeeded"
assert reconciled.json()["result"]["service_was_running"] is True
def test_service_suspend_replay_without_running_hint_fails_closed(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
operation_id = "suspend-missing-running-hint"
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id,
"host_id": "hk",
"user_id": USER["user_id"],
"vm_uuid": VM_ID,
"kind": "mutation",
"resource": f"vms/{VM_ID}/service-suspend",
"method": "POST",
"status": "uncertain",
"actor_can_manage": True,
"actor_is_admin": False,
"created_at": 1,
}))
before_calls = list(services["hk"].calls)
response = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
assert response.status_code == 200
assert response.json()["status"] == "failed"
assert "已取消服务暂停" in response.json()["error"]
assert services["hk"].calls == before_calls
def test_service_suspend_does_not_stop_vm_when_running_hint_cannot_persist(
boundary, monkeypatch):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
operation_id = "suspend-hint-write-failed"
original_set = store.set
def fail_operation_write(kind, key, value):
if kind == "operation" and key == operation_id:
raise RuntimeError("database unavailable")
return original_set(kind, key, value)
monkeypatch.setattr(store, "set", fail_operation_write)
failed = call(
client, "POST",
f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend",
{**USER, "operation_id": operation_id}, {})
assert failed.status_code == 503
assert services["hk"].vm["state"] == "running"
assert not any(
item[0] == "service_suspend" for item in services["hk"].calls)
monkeypatch.setattr(store, "set", original_set)
recovered = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
assert recovered.status_code == 200
assert recovered.json()["status"] == "failed"
assert services["hk"].vm["state"] == "running"
assert not any(
item[0] == "service_suspend" for item in services["hk"].calls)
def test_metrics_query_uses_log_store_and_owner_permission(boundary):
client, engine, _, logs, _ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
component.state.users.assign_vm(scope, VM_ID, 41)
response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d")
assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20
with patch("virsh_panel.app.time.time", return_value=3_000_000):
traffic = call(
client, "GET",
f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily?days=2",
)
assert traffic.status_code == 200
assert traffic.json()["points"][-1] == {
"date": "1970-02-04", "day_start": 2_937_600,
"download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120,
}
assert logs.daily_traffic_calls == [(scope, VM_ID, {
"host_id": "hk", "user_id": 41,
"since": 2_851_200, "until": 3_000_000, "limit": 2,
})]
denied = call(
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily",
{**USER, "user_id": 42},
)
assert denied.status_code == 404
assert len(logs.daily_traffic_calls) == 1
def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
assert issued.status_code == 200, issued.text
assert issued.json()["websocket_path"] == "/engine-console"
ticket = issued.json()["ticket"]
key = hashlib.sha256(ticket.encode()).hexdigest()
record = json.loads(store.get("console_ticket", key))
assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session"
assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket",
{**USER, "user_id": 42}, {}).status_code == 404
from starlette.websockets import WebSocketDisconnect
with pytest.raises(WebSocketDisconnect):
with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"},
subprotocols=["cloud-ticket." + ticket]):
pass
assert store.get("console_ticket", key) is not None
def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
checks = 0
async def active(*args):
nonlocal checks
checks += 1
return checks <= 2 # Ticket issuance + handshake; then PHP revokes.
engine.introspection = active
engine.console_check_interval = 0.01
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
ticket = issued.json()["ticket"]
protocol = "cloud-ticket." + ticket
async def relay(socket, **kwargs):
await socket.send_bytes(b"vnc")
await asyncio.Event().wait()
from starlette.websockets import WebSocketDisconnect
with patch("virsh_panel.console.relay_console", relay):
with client.websocket_connect("wss://panel.example.test/engine-console",
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket:
assert socket.accepted_subprotocol == protocol
assert socket.receive_bytes() == b"vnc"
with pytest.raises(WebSocketDisconnect):
socket.receive_bytes()
key = hashlib.sha256(ticket.encode()).hexdigest()
assert store.get("console_ticket", key) is None
assert store.get("ticket_used", key) == "1"
with pytest.raises(WebSocketDisconnect):
with client.websocket_connect("wss://panel.example.test/engine-console",
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]):
pass
def test_streamed_iso_digest_must_match_before_import_is_queued(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/rescue-templates?name=rescue"
signed_body = b"A" * (1024 * 1024)
wrong_body = b"B" * len(signed_body)
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest()
signed["Content-Type"] = "application/octet-stream"
response = client.post(path, content=wrong_body, headers=signed)
assert response.status_code == 401, response.text
assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls)
assert engine.registry.components["hk"].store.list("rescue_import_job") == {}
def cloud_upload_fixture(size=1024 * 1024 + 257):
header = bytearray(104)
header[:4] = b"QFI\xfb"
header[4:8] = struct.pack(">I", 3)
header[20:24] = struct.pack(">I", 16)
header[24:32] = struct.pack(">Q", 20 * 1024**3)
header[100:104] = struct.pack(">I", 104)
return bytes(header) + b"\0" * (size - len(header))
def wait_for_cloud_upload_job(client, job_id):
path = f"/internal/v1/vm-templates/import-jobs/{job_id}"
for _ in range(200):
response = call(client, "GET", path, SUPER)
assert response.status_code == 200, response.text
job = response.json()
if job["status"] in {"completed", "failed"}:
return job
time.sleep(0.01)
raise AssertionError("uploaded cloud template import did not finish")
def cloud_engine_scope(method, target, body, actor, *, headers=None):
path, separator, query = target.partition("?")
request_headers = signed_headers(SECRET, method, target, body, actor)
request_headers.update(headers or {})
return {
"type": "http",
"asgi": {"version": "3.0"},
"http_version": "1.1",
"method": method,
"scheme": "http",
"path": path,
"raw_path": path.encode("ascii"),
"query_string": query.encode("ascii") if separator else b"",
"headers": [
(name.lower().encode("ascii"), value.encode("latin-1"))
for name, value in request_headers.items()
],
"client": ("testclient", 50000),
"server": ("127.0.0.1", 80),
}
async def cloud_engine_request(engine, method, target, body, actor, *,
headers=None, receive=None):
scope = cloud_engine_scope(
method, target, body, actor, headers=headers)
delivered = False
async def receive_body():
nonlocal delivered
if delivered:
return {"type": "http.disconnect"}
delivered = True
return {"type": "http.request", "body": body, "more_body": False}
messages = []
async def send(message):
messages.append(message)
await engine(scope, receive or receive_body, send)
status = next(message["status"] for message in messages
if message["type"] == "http.response.start")
raw = b"".join(message.get("body", b"") for message in messages
if message["type"] == "http.response.body")
return status, json.loads(raw)
def test_global_cloud_template_upload_streams_past_json_limit(boundary):
client, engine, *_ = boundary
query = urlencode({
"name": "Debian 12 fixed",
"os_family": "Debian",
"os_version": "12",
})
path = "/internal/v1/vm-templates/upload?" + query
payload = cloud_upload_fixture()
with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"):
response = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
})
assert response.status_code == 202, response.text
job = wait_for_cloud_upload_job(client, response.json()["job_id"])
assert job["status"] == "completed", job
assert job["template"]["name"] == "Debian 12 fixed"
assert job["template"]["os_family"] == "Debian"
assert job["template"]["os_version"] == "12"
assert job["template"]["virtual_size"] == 20 * 1024**3
with engine.registry.cloud_templates.source(
job["template"]["id"]) as (_record, source):
assert source.stat().st_size == len(payload)
assert hashlib.sha256(source.read_bytes()).digest() == hashlib.sha256(
payload).digest()
def test_global_cloud_template_upload_digest_must_match_before_queue(boundary):
client, engine, *_ = boundary
path = ("/internal/v1/vm-templates/upload?"
"name=Debian+12&os_family=Debian&os_version=12")
signed_body = cloud_upload_fixture()
wrong_body = signed_body[:-1] + b"x"
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(
signed_body).hexdigest()
signed["Content-Type"] = "application/octet-stream"
response = client.post(path, content=wrong_body, headers=signed)
assert response.status_code == 401, response.text
assert engine.registry.cloud_templates.jobs == {}
assert engine.registry.cloud_templates.pending == set()
root = engine.registry.cloud_templates.files.directory()
assert not list(root.glob(".cloud-image-upload-*.qcow2"))
@pytest.mark.parametrize(
("family", "version"),
[
("debian", "12"),
("Debian", "14"),
("CentOS", "10"),
("Ubuntu", "24"),
],
)
def test_global_cloud_template_upload_rejects_noncanonical_metadata(
boundary, family, version):
client, engine, *_ = boundary
path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": "Invalid metadata",
"os_family": family,
"os_version": version,
})
payload = cloud_upload_fixture()
response = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
})
assert response.status_code == 422, response.text
assert engine.registry.cloud_templates.jobs == {}
def test_global_cloud_template_upload_requires_super_admin_and_octet_stream(
boundary):
client, engine, *_ = boundary
path = ("/internal/v1/vm-templates/upload?"
"name=Debian+12&os_family=Debian&os_version=12")
payload = cloud_upload_fixture()
digest = hashlib.sha256(payload).hexdigest()
forbidden = call(client, "POST", path, ADMIN, raw=payload, headers={
"X-Cloud-Content-SHA256": digest,
"Content-Type": "application/octet-stream",
})
assert forbidden.status_code == 403, forbidden.text
wrong_type = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": digest,
"Content-Type": "multipart/form-data",
})
assert wrong_type.status_code == 415, wrong_type.text
assert engine.registry.cloud_templates.jobs == {}
def test_global_cloud_template_upload_enforces_minimum_size(boundary):
client, engine, *_ = boundary
path = ("/internal/v1/vm-templates/upload?"
"name=Too+Small&os_family=Debian&os_version=12")
payload = cloud_upload_fixture(104)
response = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
})
assert response.status_code == 422, response.text
assert engine.registry.cloud_templates.jobs == {}
assert engine.registry.cloud_templates.pending == set()
def test_global_cloud_template_upload_timeout_releases_reservation(
boundary, monkeypatch):
_client, engine, *_ = boundary
query = b"name=Timeout&os_family=Debian&os_version=12"
scope = {
"query_string": query,
"headers": [
(b"content-type", b"application/octet-stream"),
(b"content-length", str(1024 * 1024).encode()),
],
"engine_upload_validation": {"complete": False, "valid": False},
"engine_upload_digest": "0" * 64,
}
async def stalled_receive():
await asyncio.sleep(1)
return {"type": "http.request", "body": b"", "more_body": False}
async def send(_message):
raise AssertionError("timeout must not start a response")
monkeypatch.setattr("virsh_panel.internal.CLOUD_UPLOAD_IDLE_SECONDS", 0.01)
with pytest.raises(TemplateError, match="上传超时") as timeout:
asyncio.run(engine._upload_cloud_template(scope, stalled_receive, send))
assert timeout.value.status == 408
assert engine.registry.cloud_templates.pending == set()
assert engine.registry.cloud_templates.upload_reservations == {}
root = engine.registry.cloud_templates.files.directory()
assert not list(root.glob(".cloud-image-upload-*.qcow2"))
def test_stalled_cloud_upload_does_not_block_other_engine_writes(
boundary, monkeypatch):
_client, engine, *_ = boundary
payload = cloud_upload_fixture()
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": "Stalled upload",
"os_family": "Debian",
"os_version": "12",
})
upload_started = asyncio.Event()
release_upload = asyncio.Event()
class FastRemoteHostManager:
def __init__(self, *_args, **_kwargs):
pass
@staticmethod
def scan(host, port):
return {"host": host, "port": port, "reachable": True}
@staticmethod
def close():
pass
monkeypatch.setattr(
"virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager)
async def stalled_receive():
upload_started.set()
await release_upload.wait()
return {"type": "http.disconnect"}
async def scenario():
upload = asyncio.create_task(cloud_engine_request(
engine,
"POST",
upload_path,
payload,
SUPER,
headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
"Content-Length": str(len(payload)),
},
receive=stalled_receive,
))
await asyncio.wait_for(upload_started.wait(), timeout=1)
assert engine.registry.cloud_templates.pending
scan_body = json.dumps({
"host": "host.example.test", "port": 22,
}, separators=(",", ":")).encode()
status, result = await asyncio.wait_for(cloud_engine_request(
engine,
"POST",
"/internal/v1/hosts/scan",
scan_body,
SUPER,
headers={"Content-Type": "application/json"},
), timeout=0.5)
assert status == 200
assert result["host"] == "host.example.test"
release_upload.set()
upload_status, _result = await asyncio.wait_for(upload, timeout=1)
assert upload_status == 400
asyncio.run(scenario())
assert engine.registry.cloud_templates.pending == set()
assert engine.registry.cloud_templates.upload_reservations == {}
def test_cloud_upload_fsync_does_not_block_event_loop_or_engine_writes(
boundary, monkeypatch):
_client, engine, *_ = boundary
payload = cloud_upload_fixture()
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": "Slow fsync upload",
"os_family": "Debian",
"os_version": "12",
})
fsync_started = threading.Event()
release_fsync = threading.Event()
original_fsync = os.fsync
def slow_fsync(descriptor):
fsync_started.set()
assert release_fsync.wait(timeout=2), "test did not release fsync"
return original_fsync(descriptor)
class FastRemoteHostManager:
def __init__(self, *_args, **_kwargs):
pass
@staticmethod
def scan(host, port):
return {"host": host, "port": port, "reachable": True}
@staticmethod
def close():
pass
monkeypatch.setattr("virsh_panel.internal.os.fsync", slow_fsync)
monkeypatch.setattr(
"virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager)
monkeypatch.setattr(
"virsh_panel.cloud_templates._verify_qcow2_with_qemu",
lambda *_args: None)
async def scenario():
safety_release = threading.Timer(1.5, release_fsync.set)
safety_release.start()
started_at = time.monotonic()
try:
upload = asyncio.create_task(cloud_engine_request(
engine,
"POST",
upload_path,
payload,
SUPER,
headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
"Content-Length": str(len(payload)),
},
))
while not fsync_started.is_set():
await asyncio.sleep(0.001)
scan_body = json.dumps({
"host": "host.example.test", "port": 22,
}, separators=(",", ":")).encode()
status, _result = await asyncio.wait_for(cloud_engine_request(
engine,
"POST",
"/internal/v1/hosts/scan",
scan_body,
SUPER,
headers={"Content-Type": "application/json"},
), timeout=0.5)
assert status == 200
assert time.monotonic() - started_at < 0.5
assert not release_fsync.is_set()
release_fsync.set()
upload_status, _result = await asyncio.wait_for(upload, timeout=2)
assert upload_status == 202
finally:
release_fsync.set()
safety_release.cancel()
asyncio.run(scenario())
def test_cloud_upload_rechecks_update_state_before_queue(
boundary, monkeypatch):
_client, engine, *_ = boundary
updater = FakeUpdater()
engine.updater = updater
payload = cloud_upload_fixture()
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": "Update race upload",
"os_family": "Debian",
"os_version": "12",
})
upload_started = asyncio.Event()
release_upload = asyncio.Event()
delivered = False
monkeypatch.setattr(
"virsh_panel.cloud_templates._verify_qcow2_with_qemu",
lambda *_args: None)
async def delayed_receive():
nonlocal delivered
if delivered:
return {"type": "http.disconnect"}
delivered = True
upload_started.set()
await release_upload.wait()
return {"type": "http.request", "body": payload, "more_body": False}
async def scenario():
upload = asyncio.create_task(cloud_engine_request(
engine,
"POST",
upload_path,
payload,
SUPER,
headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
"Content-Length": str(len(payload)),
},
receive=delayed_receive,
))
await asyncio.wait_for(upload_started.wait(), timeout=1)
assert engine.registry.cloud_templates.pending
updater.active = {
"job_id": "update-job-1234567890", "status": "running",
}
release_upload.set()
status, result = await asyncio.wait_for(upload, timeout=2)
assert status == 409
assert result["error"] == "面板正在更新,暂时不能执行虚拟化写操作"
try:
asyncio.run(scenario())
finally:
updater.active = None
assert engine.registry.cloud_templates.pending == set()
assert engine.registry.cloud_templates.upload_reservations == {}
assert engine.registry.cloud_templates.jobs == {}
def test_upload_same_os_version_keeps_existing_template(boundary):
client, engine, *_ = boundary
def upload(name, payload):
path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": name,
"os_family": "Debian",
"os_version": "12",
})
response = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
})
assert response.status_code == 202, response.text
job = wait_for_cloud_upload_job(client, response.json()["job_id"])
assert job["status"] == "completed", job
return job["template"]
official_payload = cloud_upload_fixture()[:-1] + b"o"
fixed_payload = cloud_upload_fixture()[:-1] + b"f"
with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"):
official = upload("Debian 12 official", official_payload)
fixed = upload("Debian 12 fixed", fixed_payload)
assert official["id"] != fixed["id"]
templates = call(
client, "GET", "/internal/v1/vm-templates", SUPER).json()["templates"]
assert {row["id"] for row in templates} == {official["id"], fixed["id"]}
with engine.registry.cloud_templates.source(
official["id"]) as (_record, source):
assert hashlib.sha256(source.read_bytes()).hexdigest() == official["sha256"]
def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables():
connection = MagicMock()
cursor = connection.cursor.return_value.__enter__.return_value
config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test",
"ssl_ca": "/private/test/ca.pem"}
with patch("virsh_panel.database.pymysql.connect", return_value=connection):
main = MainStore(config, b"m" * 32)
statements = [call.args[0] for call in cursor.execute.call_args_list]
assert any("engine_records" in statement for statement in statements)
assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements)
assert not any("vm_metric_samples" in statement for statement in statements)
assert main._config["ssl_verify_identity"] is True
cursor.reset_mock()
LogStore({**config, "database": "logs"}, b"m" * 32)
statements = [call.args[0] for call in cursor.execute.call_args_list]
assert "engine_vm_metric_samples" in statements[0]
assert "engine_vm_traffic_totals" in statements[1]
assert "engine_vm_traffic_checkpoints" in statements[2]
assert all("ALTER TABLE engine_vm_metric_samples" in statement
for statement in statements[3:])
assert LogStore.TRAFFIC_TABLE == "engine_vm_traffic_totals"
assert LogStore.TRAFFIC_CHECKPOINT_TABLE == "engine_vm_traffic_checkpoints"