You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
3216 lines
135 KiB
Python
3216 lines
135 KiB
Python
"""Private engine tests: no PHP cookies, no real DB and no real hypervisor."""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import struct
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import contextmanager
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
from urllib.parse import urlencode
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from test_app import (CLOUD_CREATE_BODY, CREATE_BODY, OFFICIAL_CLOUD_URL,
|
|
FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID)
|
|
from virsh_panel.app import create_app
|
|
from virsh_panel.cloud_templates import (TemplateError, template_id_for,
|
|
template_scope, template_volume_name)
|
|
from virsh_panel.config import Settings
|
|
from virsh_panel.database import DatabaseError, DatabaseTemplateStore
|
|
from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore,
|
|
LogStore, MainStore, TRUSTED_ACTOR)
|
|
from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor,
|
|
signed_headers)
|
|
from virsh_panel.internal import EngineApplication, HostRegistry
|
|
from virsh_panel.libvirt_service import VMError
|
|
from virsh_panel.metrics import host_scope
|
|
from virsh_panel.user_isos import user_iso_scope
|
|
|
|
|
|
SECRET = b"s" * 32
|
|
USER = {"user_id": 41, "is_admin": False, "is_super_admin": False,
|
|
"can_manage": True, "session_id": "test-session"}
|
|
SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True}
|
|
ADMIN = {**SUPER, "is_super_admin": False}
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self):
|
|
self.records = {}
|
|
self.nonces = set()
|
|
self.lock = threading.RLock()
|
|
self.daily_traffic_calls = []
|
|
|
|
def get(self, kind, key):
|
|
with self.lock:
|
|
return self.records.get((kind, key))
|
|
|
|
def set(self, kind, key, value):
|
|
with self.lock:
|
|
self.records[kind, key] = value
|
|
|
|
def insert(self, kind, key, value):
|
|
with self.lock:
|
|
if (kind, key) in self.records:
|
|
return False
|
|
self.records[kind, key] = value
|
|
return True
|
|
|
|
def delete(self, kind, key):
|
|
with self.lock:
|
|
self.records.pop((kind, key), None)
|
|
|
|
def list(self, kind):
|
|
with self.lock:
|
|
return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind}
|
|
|
|
def consume_nonce(self, nonce, expires_at, now):
|
|
with self.lock:
|
|
if nonce in self.nonces:
|
|
return False
|
|
self.nonces.add(nonce)
|
|
return True
|
|
|
|
def query_metric_samples(self, *args, **kwargs):
|
|
return [{"timestamp": 123, "cpu_percent": 20.0}]
|
|
|
|
def query_daily_traffic(self, scope, vm_id, **kwargs):
|
|
self.daily_traffic_calls.append((scope, vm_id, kwargs))
|
|
day_start = kwargs["until"] - kwargs["until"] % 86400
|
|
return [{"day_start": day_start, "download_bytes": 4096,
|
|
"upload_bytes": 1024, "total_bytes": 5120}]
|
|
|
|
def get_value(self, key):
|
|
return self.get("value", key)
|
|
|
|
def set_value(self, key, value):
|
|
return self.set("value", key, value)
|
|
|
|
def delete_value(self, key):
|
|
return self.delete("value", key)
|
|
|
|
def get_secret(self, key):
|
|
return self.get("secret", key)
|
|
|
|
def set_secret(self, key, value):
|
|
return self.set("secret", key, value)
|
|
|
|
def delete_secret(self, key):
|
|
return self.delete("secret", key)
|
|
|
|
|
|
class FakeUpdater:
|
|
def __init__(self):
|
|
self.token = None
|
|
self.active = None
|
|
self.jobs = {}
|
|
|
|
def settings(self):
|
|
return {
|
|
"repository": "https://g.luoli.live/root/virt.git",
|
|
"branch": "venix",
|
|
"token_configured": self.token is not None,
|
|
"checkout_ready": True,
|
|
"current_commit": "a" * 40,
|
|
"last_job_id": None,
|
|
"active_job_id": self.active["job_id"] if self.active else None,
|
|
}
|
|
|
|
def set_token(self, token):
|
|
self.token = token
|
|
|
|
def clear_token(self):
|
|
self.token = None
|
|
|
|
def active_job(self):
|
|
return self.active
|
|
|
|
def check(self):
|
|
return {"repository": "https://g.luoli.live/root/virt.git", "branch": "venix",
|
|
"current_commit": "a" * 40, "remote_commit": "b" * 40,
|
|
"status": "update_available"}
|
|
|
|
def create_job(self, *, expected_current_commit=None, expected_remote_commit=None):
|
|
if self.token is None:
|
|
from virsh_panel.updater import UpdateError
|
|
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
|
|
if expected_current_commit != "a" * 40 or expected_remote_commit != "b" * 40:
|
|
from virsh_panel.updater import UpdateError
|
|
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
|
|
job = {"job_id": "update-job-1234567890", "status": "queued",
|
|
"phase": "等待独立更新服务启动", "created_at": 1,
|
|
"private": self.token}
|
|
self.jobs[job["job_id"]] = job
|
|
self.active = job
|
|
return job.copy()
|
|
|
|
def get_job(self, job_id):
|
|
return self.jobs.get(job_id)
|
|
|
|
def fail_job(self, job_id, message):
|
|
self.jobs[job_id].update(status="failed", error=message)
|
|
self.active = None
|
|
return self.jobs[job_id].copy()
|
|
|
|
|
|
class Service(FakeVMService):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.create_failure = False
|
|
self.absent = False
|
|
self.absent_checks = []
|
|
self.cloud_template_delete_error = None
|
|
self.vm.update(managed=True, os_source="iso")
|
|
|
|
def create_vm(self, spec):
|
|
self.calls.append(("create_vm", spec))
|
|
if self.create_failure:
|
|
from virsh_panel.libvirt_service import VMError
|
|
raise VMError("宿主机失联,回滚状态不明", 502)
|
|
self.vm.update(uuid=spec["vm_uuid"], name=spec["name"])
|
|
return self.vm.copy()
|
|
|
|
def provisioning_absent(self, vm_uuid, name, *, uuid_resources=False,
|
|
legacy_name_collision=False):
|
|
self.absent_checks.append(
|
|
(vm_uuid, name, uuid_resources, legacy_name_collision))
|
|
return self.absent
|
|
|
|
def change_cloud_vm_network(self, vm_id, network, *, confirm_restart=False):
|
|
self.calls.append((
|
|
"change_cloud_vm_network", vm_id,
|
|
network["public_ip_cidr"], confirm_restart,
|
|
))
|
|
if vm_id != self.vm["uuid"]:
|
|
raise VMError("虚拟机不存在", 404)
|
|
self.vm["public_ip_cidr"] = network["public_ip_cidr"]
|
|
self.vm["public_pool_key"] = network.get("public_pool_key")
|
|
return {
|
|
"vm": self.vm.copy(),
|
|
"public_pool_key": network.get("public_pool_key"),
|
|
"public_ip_cidr": network["public_ip_cidr"],
|
|
"network_changed": True,
|
|
}
|
|
|
|
def delete_cloud_template_replicas(self, source_volume_name):
|
|
self.calls.append(("delete_cloud_template_replicas", source_volume_name))
|
|
if self.cloud_template_delete_error is not None:
|
|
raise self.cloud_template_delete_error
|
|
return {"source_volume_name": source_volume_name, "deleted": True}
|
|
|
|
|
|
@pytest.fixture
|
|
def boundary(tmp_path):
|
|
store = MemoryStore()
|
|
logs = MemoryStore()
|
|
config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}
|
|
services = {}
|
|
def factory(host_id, user_isos, cloud_templates):
|
|
(tmp_path / host_id).mkdir(mode=0o700, exist_ok=True)
|
|
scoped = HostStore(store, host_id)
|
|
manager = FakeRemoteManager()
|
|
manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22,
|
|
"fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images",
|
|
"iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default",
|
|
"libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config",
|
|
"ssh_known_hosts_file": "/fake/known_hosts"}
|
|
service = Service()
|
|
services[host_id] = service
|
|
components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager,
|
|
DatabaseTemplateStore(scoped), scoped, logs, user_isos,
|
|
cloud_templates)
|
|
app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components,
|
|
service_factory=lambda profile: service)
|
|
return components, app
|
|
registry = HostRegistry(config, store, logs, factory=factory)
|
|
async def active(*args):
|
|
return True
|
|
engine = EngineApplication(config, store, logs, registry=registry, introspection=active)
|
|
engine._service = lambda profile: services[profile["host_id"]]
|
|
with TestClient(engine, base_url="http://127.0.0.1") as client:
|
|
yield client, engine, store, logs, services
|
|
|
|
|
|
def call(client, method, path, actor=USER, value=None, raw=None, headers=None):
|
|
body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode()
|
|
signed = signed_headers(SECRET, method, path, body, actor)
|
|
if value is not None:
|
|
signed["Content-Type"] = "application/json"
|
|
signed.update(headers or {})
|
|
return client.request(method, path, content=body, headers=signed)
|
|
|
|
|
|
def test_host_capacity_is_admin_only_and_does_not_expose_host_secrets(boundary):
|
|
client, _engine, _store, _logs, _services = boundary
|
|
|
|
denied = call(client, "GET", "/internal/v1/hosts/hk/host/capacity", USER)
|
|
assert denied.status_code == 403
|
|
|
|
response = call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER,
|
|
)
|
|
assert response.status_code == 200, response.text
|
|
payload = response.json()
|
|
assert payload["health"] == "healthy"
|
|
assert isinstance(payload["observed_at"], int)
|
|
assert payload["cpu"] == {"total": 16, "allocated": 2, "available": 14}
|
|
assert {key: payload["public_ipv4"][key] for key in (
|
|
"configured", "total", "reserved", "allocated", "available",
|
|
)} == {
|
|
"configured": False,
|
|
"total": 0,
|
|
"reserved": 0,
|
|
"allocated": 0,
|
|
"available": 0,
|
|
}
|
|
assert payload["public_ipv4"]["pools"] == []
|
|
assert payload["public_ipv4"]["compatibility_mode"] is False
|
|
for secret_value in (
|
|
"host.example.test", "/fake/config", "/fake/known_hosts", "SHA256:example",
|
|
):
|
|
assert secret_value not in response.text
|
|
|
|
|
|
def import_user_iso_for_test(client, tmp_path, name, *, digest="e" * 64):
|
|
download = tmp_path / (hashlib.sha256(name.encode()).hexdigest() + ".iso")
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, digest)):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
|
|
"url": "https://downloads.example.test/os.iso",
|
|
"name": name,
|
|
"sha256": "",
|
|
})
|
|
assert queued.status_code == 202, queued.text
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
return job["iso"]["id"]
|
|
|
|
|
|
def test_host_network_options_is_admin_read_only_and_bounded(boundary):
|
|
client, _engine, _store, _logs, _services = boundary
|
|
denied = call(client, "GET", "/internal/v1/hosts/hk/host/network-options", USER)
|
|
assert denied.status_code == 403
|
|
response = call(client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER)
|
|
assert response.status_code == 200, response.text
|
|
payload = response.json()
|
|
assert payload["configuration_state"] == "not_configured"
|
|
assert payload["available_ips"] == []
|
|
assert "private_key" not in response.text
|
|
|
|
|
|
def test_host_public_network_pools_are_revisioned_idempotent_and_aggregated(
|
|
boundary):
|
|
client, _engine, _store, _logs, services = boundary
|
|
# Materialize the host child app before configuring its host-scoped
|
|
# encrypted pool catalog.
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER,
|
|
).json() == {
|
|
"items": [], "count": 0, "maximum": 32,
|
|
"compatibility_mode": False,
|
|
}
|
|
|
|
first_key = "a" * 32
|
|
second_key = "b" * 32
|
|
first_path = f"/internal/v1/hosts/hk/host/network-pools/{first_key}"
|
|
first = {
|
|
"revision": 1,
|
|
"state": "active",
|
|
"network_selector": "default",
|
|
"cidr": "8.8.8.0/29",
|
|
"gateway": "8.8.8.1",
|
|
"dns_servers": ["1.1.1.1"],
|
|
"reserved_ips": ["8.8.8.2"],
|
|
"guest_prefix_length": 29,
|
|
}
|
|
assert call(client, "PUT", first_path, USER, first).status_code == 403
|
|
created = call(client, "PUT", first_path, SUPER, first)
|
|
assert created.status_code == 200, created.text
|
|
assert created.json()["applied"] is True
|
|
assert created.json()["pool"]["pool_key"] == first_key
|
|
|
|
repeated = call(client, "PUT", first_path, SUPER, first)
|
|
assert repeated.status_code == 200
|
|
assert repeated.json()["idempotent"] is True
|
|
conflict = call(client, "PUT", first_path, SUPER, {
|
|
**first, "gateway": "8.8.8.2",
|
|
})
|
|
assert conflict.status_code == 409
|
|
|
|
second_path = f"/internal/v1/hosts/hk/host/network-pools/{second_key}"
|
|
second = {
|
|
**first,
|
|
"cidr": "9.9.9.0/29",
|
|
"gateway": "9.9.9.1",
|
|
"reserved_ips": ["9.9.9.2"],
|
|
}
|
|
assert call(client, "PUT", second_path, SUPER, second).status_code == 200
|
|
advanced = call(client, "PUT", first_path, SUPER, {
|
|
**first, "revision": 2, "state": "draining",
|
|
})
|
|
assert advanced.status_code == 200
|
|
assert advanced.json()["pool"]["state"] == "draining"
|
|
assert call(client, "PUT", first_path, SUPER, first).status_code == 409
|
|
overlapping = call(client, "PUT", f"/internal/v1/hosts/hk/host/network-pools/{'c' * 32}",
|
|
SUPER, {**first, "cidr": "8.8.8.0/30"})
|
|
assert overlapping.status_code == 400
|
|
|
|
catalog = call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER,
|
|
).json()
|
|
assert catalog["count"] == 2
|
|
assert [item["pool_key"] for item in catalog["items"]] == [
|
|
first_key, second_key,
|
|
]
|
|
assert catalog["compatibility_mode"] is False
|
|
|
|
services["hk"].calls.clear()
|
|
options = call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER,
|
|
).json()
|
|
assert services["hk"].calls.count(("list_vms",)) == 1
|
|
assert len(options["pools"]) == 2
|
|
assert options["pools"][0]["reserved_ips"] == ["8.8.8.2"]
|
|
assert options["pools"][1]["reserved_ips"] == ["9.9.9.2"]
|
|
services["hk"].calls.clear()
|
|
capacity = call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER,
|
|
).json()["public_ipv4"]
|
|
assert services["hk"].calls.count(("list_vms",)) == 1
|
|
assert capacity["total"] == sum(item["total"] for item in capacity["pools"])
|
|
assert capacity["reserved"] == sum(
|
|
item["reserved"] for item in capacity["pools"])
|
|
assert capacity["allocated"] == sum(
|
|
item["allocated"] for item in capacity["pools"])
|
|
assert capacity["available"] == sum(
|
|
item["available"] for item in capacity["pools"])
|
|
|
|
disabled = call(client, "PUT", second_path, SUPER, {
|
|
**second, "revision": 2, "state": "disabled",
|
|
})
|
|
assert disabled.status_code == 200
|
|
no_active = call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER,
|
|
).json()
|
|
assert no_active["configuration_state"] == "not_configured"
|
|
assert len(no_active["pools"]) == 2
|
|
disabled_capacity = call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER,
|
|
).json()["public_ipv4"]
|
|
assert disabled_capacity["configured"] is True
|
|
|
|
|
|
def test_first_native_pool_cannot_orphan_occupied_legacy_network(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.4.4",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/29",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 29,
|
|
})
|
|
services["hk"].vm["public_ip_cidr"] = "8.8.8.3/29"
|
|
pool_key = "d" * 32
|
|
path = f"/internal/v1/hosts/hk/host/network-pools/{pool_key}"
|
|
base = {
|
|
"revision": 1,
|
|
"state": "active",
|
|
"network_selector": "bridge:br0",
|
|
"dns_servers": ["1.1.1.1"],
|
|
"reserved_ips": ["9.9.9.2"],
|
|
"guest_prefix_length": 29,
|
|
}
|
|
rejected = call(client, "PUT", path, SUPER, {
|
|
**base, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1",
|
|
})
|
|
assert rejected.status_code == 409
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER,
|
|
).json()["compatibility_mode"] is True
|
|
|
|
adopted = call(client, "PUT", path, SUPER, {
|
|
**base,
|
|
"cidr": "8.8.8.0/29",
|
|
"gateway": "8.8.8.1",
|
|
"reserved_ips": ["8.8.8.2"],
|
|
})
|
|
assert adopted.status_code == 200, adopted.text
|
|
assert adopted.json()["compatibility_mode"] is False
|
|
|
|
|
|
def import_cloud_template_for_test(client, tmp_path, name="Delete Cloud Image"):
|
|
payload = name.encode("utf-8")
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
|
|
def fake_download(_url, _sha256, directory, **_kwargs):
|
|
source = directory / ".cloud-image-delete-boundary.qcow2"
|
|
source.write_bytes(payload)
|
|
source.chmod(0o600)
|
|
return source, 2 * 1024**3, digest
|
|
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=fake_download):
|
|
queued = call(client, "POST", "/internal/v1/vm-templates/import",
|
|
SUPER, {"url": OFFICIAL_CLOUD_URL,
|
|
"name": name, "sha256": ""})
|
|
assert queued.status_code == 202, queued.text
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET",
|
|
f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER,
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
return job["template"], payload
|
|
|
|
|
|
def test_paid_scheduler_can_claim_exact_host_pool_ip(boundary, tmp_path):
|
|
client, engine, _store, _logs, services = boundary
|
|
template, _payload = import_cloud_template_for_test(
|
|
client, tmp_path, "Scheduled Address Image",
|
|
)
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/resources", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
service = services["hk"]
|
|
service.vm["public_ip_cidr"] = "8.8.8.4/28"
|
|
scope = host_scope(profile)
|
|
occupied_claim = f"{scope}.8.8.8.5"
|
|
components.store.set("ip_claim", occupied_claim, json.dumps({
|
|
"vm_uuid": "00000000-0000-4000-8000-000000000005",
|
|
}))
|
|
base = {
|
|
key: value for key, value in CLOUD_CREATE_BODY.items()
|
|
if key not in {"gateway", "dns_servers"}
|
|
}
|
|
base["template_id"] = template["id"]
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
|
|
unauthorized = call(client, "POST", path, {
|
|
**USER, "operation_id": "scheduled-ip-unpaid-0001",
|
|
}, {**base, "name": "unpaid-exact-ip", "public_ip_cidr": "8.8.8.8/28"})
|
|
assert unauthorized.status_code == 403
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.8") is None
|
|
|
|
service.absent = True
|
|
outside = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-outside-0001",
|
|
}, {**base, "name": "outside-exact-ip", "public_ip_cidr": "8.8.8.20/28"})
|
|
assert outside.status_code == 422
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.20") is None
|
|
|
|
occupied_vm = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-vm-used-0001",
|
|
}, {**base, "name": "vm-used-exact-ip", "public_ip_cidr": "8.8.8.4/28"})
|
|
assert occupied_vm.status_code == 409
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.4") is None
|
|
|
|
occupied_task = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-claim-used-0001",
|
|
}, {**base, "name": "claim-used-exact-ip", "public_ip_cidr": "8.8.8.5/28"})
|
|
assert occupied_task.status_code == 409
|
|
assert components.store.get("ip_claim", occupied_claim) is not None
|
|
|
|
created = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-success-0001",
|
|
}, {**base, "name": "scheduled-exact-ip", "public_ip_cidr": "8.8.8.6/28"})
|
|
assert created.status_code == 201, created.text
|
|
created_id = created.json()["uuid"]
|
|
created_call = [
|
|
row for row in service.calls if row[0] == "create_cloud_vm"
|
|
][-1]
|
|
assert created_call[1]["public_ip_cidr"] == "8.8.8.6/28"
|
|
assert created_call[1]["public_pool_key"] == "legacy-default"
|
|
assert created_call[1]["gateway"] == "8.8.8.1"
|
|
assert created_call[1]["dns_servers"] == ["1.1.1.1"]
|
|
created_claim = json.loads(components.store.get(
|
|
"ip_claim", f"{scope}.8.8.8.6",
|
|
))
|
|
assert created_claim["vm_uuid"] == created_id
|
|
assert created_claim["pool_key"] == "legacy-default"
|
|
|
|
def rolled_back_cloud_create(_spec, _volume_name):
|
|
raise VMError("模拟创建失败,宿主机已完成回滚", 502)
|
|
|
|
service.create_cloud_vm = rolled_back_cloud_create
|
|
failed = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-rollback-0001",
|
|
}, {**base, "name": "rolled-back-exact-ip", "public_ip_cidr": "8.8.8.7/28"})
|
|
assert failed.status_code == 502
|
|
assert failed.json()["status"] == "failed"
|
|
assert failed.json()["safe_to_refund"] is True
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.7") is None
|
|
|
|
|
|
def test_paid_scheduler_claims_the_requested_native_public_pool(
|
|
boundary, tmp_path):
|
|
client, engine, _store, _logs, services = boundary
|
|
template, _payload = import_cloud_template_for_test(
|
|
client, tmp_path, "Native Pool Address Image",
|
|
)
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources", SUPER).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile["default_network"] = "bridge:br0"
|
|
first_key = "1" * 32
|
|
second_key = "2" * 32
|
|
common = {
|
|
"revision": 1,
|
|
"state": "active",
|
|
"network_selector": "bridge:br0",
|
|
"dns_servers": ["1.1.1.1"],
|
|
"reserved_ips": [],
|
|
"guest_prefix_length": 29,
|
|
}
|
|
assert call(
|
|
client, "PUT",
|
|
f"/internal/v1/hosts/hk/host/network-pools/{first_key}", SUPER,
|
|
{**common, "cidr": "8.8.8.0/29", "gateway": "8.8.8.1"},
|
|
).status_code == 200
|
|
assert call(
|
|
client, "PUT",
|
|
f"/internal/v1/hosts/hk/host/network-pools/{second_key}", SUPER,
|
|
{**common, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1"},
|
|
).status_code == 200
|
|
base = {
|
|
key: value for key, value in CLOUD_CREATE_BODY.items()
|
|
if key not in {"gateway", "dns_servers"}
|
|
}
|
|
base["template_id"] = template["id"]
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
provisioning = {**USER, "can_provision": True}
|
|
|
|
wrong = call(client, "POST", path, {
|
|
**provisioning, "operation_id": "native-pool-mismatch-0001",
|
|
}, {
|
|
**base, "name": "wrong-native-pool",
|
|
"public_pool_key": first_key,
|
|
"public_ip_cidr": "9.9.9.3/29",
|
|
})
|
|
assert wrong.status_code == 422
|
|
|
|
created = call(client, "POST", path, {
|
|
**provisioning, "operation_id": "native-pool-success-0001",
|
|
}, {
|
|
**base, "name": "right-native-pool",
|
|
"public_pool_key": second_key,
|
|
"public_ip_cidr": "9.9.9.3/29",
|
|
})
|
|
assert created.status_code == 201, created.text
|
|
create_call = [
|
|
row for row in services["hk"].calls if row[0] == "create_cloud_vm"
|
|
][-1]
|
|
assert create_call[1]["public_pool_key"] == second_key
|
|
assert create_call[1]["gateway"] == "9.9.9.1"
|
|
scope = host_scope(profile)
|
|
claim = json.loads(components.store.get("ip_claim", f"{scope}.9.9.9.3"))
|
|
assert claim["pool_key"] == second_key
|
|
|
|
|
|
def test_global_cloud_template_is_visible_and_cached_once_per_host(
|
|
boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
payload = b"verified-global-cloud-template"
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
|
|
def fake_download(_url, _sha256, directory, **_kwargs):
|
|
source = directory / ".cloud-image-boundary.qcow2"
|
|
source.write_bytes(payload)
|
|
source.chmod(0o600)
|
|
return source, 2 * 1024**3, digest
|
|
|
|
denied = call(client, "POST", "/internal/v1/vm-templates/import",
|
|
USER, {"url": OFFICIAL_CLOUD_URL,
|
|
"name": "Not Allowed", "sha256": ""})
|
|
assert denied.status_code == 403
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=fake_download):
|
|
queued = call(client, "POST", "/internal/v1/vm-templates/import",
|
|
SUPER, {"url": OFFICIAL_CLOUD_URL,
|
|
"name": "AlmaLinux Global", "sha256": ""})
|
|
assert queued.status_code == 202, queued.text
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET",
|
|
f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER,
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
template_id = job["template"]["id"]
|
|
assert services == {}
|
|
assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == {
|
|
"templates": [job["template"]],
|
|
}
|
|
|
|
provisioning = {**USER, "can_provision": True}
|
|
body = {**CLOUD_CREATE_BODY, "template_id": template_id}
|
|
hk = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0001"}, body,
|
|
)
|
|
assert hk.status_code == 201, hk.text
|
|
jp = call(
|
|
client, "POST", "/internal/v1/hosts/jp/vms",
|
|
{**provisioning, "operation_id": "global-template-jp-0001"},
|
|
{**body, "name": "cloud-vm-jp", "public_ip_cidr": "8.8.8.19/24"},
|
|
)
|
|
assert jp.status_code == 201, jp.text
|
|
first_markers = {
|
|
host_id: json.loads(HostStore(store, host_id).get(
|
|
"cloud_template_replica", template_id))
|
|
for host_id in ("hk", "jp")
|
|
}
|
|
# The public catalog intentionally omits the private backing filename.
|
|
source_volume_name = template_volume_name(template_id)
|
|
for host_id, marker in first_markers.items():
|
|
assert marker["status"] == "ready"
|
|
assert marker["sha256"] == digest
|
|
assert len(marker["replica_identity"]) == 64
|
|
assert marker["replica_volume_name"] != source_volume_name
|
|
assert marker["replica_volume_name"].startswith(
|
|
source_volume_name.removesuffix(".qcow2") + "-")
|
|
created = [row for row in services[host_id].calls
|
|
if row[0] == "create_cloud_vm"]
|
|
assert created[-1][2] == marker["replica_volume_name"]
|
|
|
|
observed_markers = []
|
|
original_hk_cache = services["hk"].cache_template
|
|
|
|
def inspect_hk_marker(*args, **kwargs):
|
|
marker = json.loads(HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id))
|
|
observed_markers.append(marker)
|
|
return original_hk_cache(*args, **kwargs)
|
|
|
|
services["hk"].cache_template = inspect_hk_marker
|
|
hk_again = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0002"},
|
|
{**body, "name": "cloud-vm-hk-2", "public_ip_cidr": "8.8.8.20/24"},
|
|
)
|
|
assert hk_again.status_code == 201, hk_again.text
|
|
services["hk"].cache_template = original_hk_cache
|
|
assert [marker["status"] for marker in observed_markers] == ["syncing"]
|
|
fallback_volume_name = observed_markers[0]["replica_volume_name"]
|
|
assert fallback_volume_name != first_markers["hk"]["replica_volume_name"]
|
|
|
|
hk_cache = [row for row in services["hk"].calls
|
|
if row[0] == "cache_template"]
|
|
jp_cache = [row for row in services["jp"].calls
|
|
if row[0] == "cache_template"]
|
|
assert hk_cache[0][4:] == (None, None)
|
|
assert hk_cache[1][1] == fallback_volume_name
|
|
assert hk_cache[1][4] == first_markers["hk"]["replica_volume_name"]
|
|
assert hk_cache[1][5] == first_markers["hk"]["replica_identity"]
|
|
assert jp_cache[0][4:] == (None, None)
|
|
for host_id in ("hk", "jp"):
|
|
marker = json.loads(HostStore(store, host_id).get(
|
|
"cloud_template_replica", template_id))
|
|
assert marker["status"] == "ready"
|
|
assert marker["sha256"] == digest
|
|
assert len(marker["replica_identity"]) == 64
|
|
# A successful trusted reuse keeps the proven prior generation even
|
|
# though the durable syncing marker contained a fresh fallback name.
|
|
assert marker["replica_volume_name"] == (
|
|
first_markers[host_id]["replica_volume_name"])
|
|
|
|
# Rebinding the logical host record to a different SSH host identity must
|
|
# invalidate its ready proof even when the pool name is unchanged.
|
|
engine.registry.components["hk"].manager.profile["fingerprint"] = (
|
|
"SHA256:replacement-host")
|
|
rebound = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0003"},
|
|
{**body, "name": "cloud-vm-hk-3", "public_ip_cidr": "8.8.8.21/24"},
|
|
)
|
|
assert rebound.status_code == 201, rebound.text
|
|
hk_cache = [row for row in services["hk"].calls
|
|
if row[0] == "cache_template"]
|
|
assert hk_cache[-1][4:] == (None, None)
|
|
rebound_marker = json.loads(HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id))
|
|
assert rebound_marker["status"] == "ready"
|
|
assert rebound_marker["replica_volume_name"] == hk_cache[-1][1]
|
|
assert (rebound_marker["replica_volume_name"]
|
|
!= first_markers["hk"]["replica_volume_name"])
|
|
|
|
# A crash/failure after the syncing marker was stored must retry that
|
|
# exact planned generation. Allocating yet another name would leak the
|
|
# possibly partial remote volume and make recovery non-deterministic.
|
|
failed_calls = []
|
|
|
|
def fail_hk_cache(image_path, volume_name, virtual_size, **kwargs):
|
|
failed_calls.append((volume_name, kwargs, json.loads(
|
|
HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id))))
|
|
raise VMError("云镜像缓存同步中断", 502)
|
|
|
|
services["hk"].cache_template = fail_hk_cache
|
|
interrupted = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0004"},
|
|
{**body, "name": "cloud-vm-hk-4", "public_ip_cidr": "8.8.8.22/24"},
|
|
)
|
|
assert interrupted.status_code == 502, interrupted.text
|
|
assert len(failed_calls) == 1
|
|
planned_volume_name, failed_kwargs, failed_marker = failed_calls[0]
|
|
assert failed_marker["status"] == "syncing"
|
|
assert failed_marker["replica_volume_name"] == planned_volume_name
|
|
assert failed_kwargs["trusted_replica_volume_name"] == (
|
|
rebound_marker["replica_volume_name"])
|
|
assert failed_kwargs["trusted_replica_identity"] == (
|
|
rebound_marker["replica_identity"])
|
|
|
|
services["hk"].cache_template = original_hk_cache
|
|
retried = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0005"},
|
|
{**body, "name": "cloud-vm-hk-5", "public_ip_cidr": "8.8.8.23/24"},
|
|
)
|
|
assert retried.status_code == 201, retried.text
|
|
hk_cache = [row for row in services["hk"].calls
|
|
if row[0] == "cache_template"]
|
|
assert hk_cache[-1][1] == planned_volume_name
|
|
assert hk_cache[-1][4:] == (None, None)
|
|
recovered_marker = json.loads(HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id))
|
|
assert recovered_marker["status"] == "ready"
|
|
assert recovered_marker["replica_volume_name"] == planned_volume_name
|
|
assert [row for row in services["hk"].calls
|
|
if row[0] == "create_cloud_vm"][-1][2] == planned_volume_name
|
|
assert (tmp_path / "cloud-templates"
|
|
/ engine.registry.cloud_templates.get(template_id)["volume_name"]).is_file()
|
|
|
|
|
|
def test_legacy_host_template_id_remains_usable_for_an_already_queued_order(
|
|
boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
# Initialize the original target host, then seed the same host-scoped
|
|
# catalog record an order created before the global-catalog upgrade holds.
|
|
assert call(client, "GET", "/internal/v1/hosts/legacy/vms").status_code == 200
|
|
profile = engine.registry.components["legacy"].manager.profile
|
|
scope = template_scope(profile)
|
|
digest = "d" * 64
|
|
template_id = template_id_for(scope, digest, "10")
|
|
DatabaseTemplateStore(HostStore(store, "legacy")).add(scope, {
|
|
"id": template_id,
|
|
"name": "Legacy AlmaLinux",
|
|
"os_family": "AlmaLinux",
|
|
"os_version": "10",
|
|
"sha256": digest,
|
|
"volume_name": template_volume_name(template_id),
|
|
"virtual_size": 2 * 1024**3,
|
|
"image_bytes": 1024,
|
|
"created_at": 1,
|
|
})
|
|
|
|
response = call(
|
|
client, "POST", "/internal/v1/hosts/legacy/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "legacy-template-order-0001"},
|
|
{**CLOUD_CREATE_BODY, "template_id": template_id,
|
|
"name": "legacy-paid-order", "public_ip_cidr": "8.8.8.22/24"},
|
|
)
|
|
|
|
assert response.status_code == 201, response.text
|
|
assert not any(row[0] == "cache_template"
|
|
for row in services["legacy"].calls)
|
|
create = next(row for row in services["legacy"].calls
|
|
if row[0] == "create_cloud_vm")
|
|
assert create[2] == template_volume_name(template_id)
|
|
|
|
|
|
def test_global_cloud_template_delete_is_idempotent_and_retries_host_gc(
|
|
boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
imported, payload = import_cloud_template_for_test(client, tmp_path)
|
|
template_id = imported["id"]
|
|
template_created_at = imported["created_at"]
|
|
source_volume_name = template_volume_name(template_id)
|
|
source_path = tmp_path / "cloud-templates" / source_volume_name
|
|
assert source_path.is_file()
|
|
|
|
for host_id in ("hk", "jp"):
|
|
assert call(
|
|
client, "GET", f"/internal/v1/hosts/{host_id}/resources"
|
|
).status_code == 200
|
|
HostStore(store, host_id).set(
|
|
"cloud_template_replica", template_id, "ready-marker")
|
|
|
|
assert call(
|
|
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
|
|
ADMIN, {"created_at": template_created_at},
|
|
).status_code == 403
|
|
services["hk"].cloud_template_delete_error = VMError(
|
|
"云镜像仍被虚拟机引用,不能删除", 409)
|
|
original_store_delete = store.delete
|
|
failed_catalog_delete = False
|
|
|
|
def fail_first_catalog_delete(kind, key):
|
|
nonlocal failed_catalog_delete
|
|
if (kind == "cloud_template" and key == template_id
|
|
and not failed_catalog_delete):
|
|
failed_catalog_delete = True
|
|
raise RuntimeError("catalog delete failed before commit")
|
|
return original_store_delete(kind, key)
|
|
|
|
store.delete = fail_first_catalog_delete
|
|
try:
|
|
deleted = call(
|
|
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
|
|
SUPER, {"created_at": template_created_at},
|
|
)
|
|
finally:
|
|
store.delete = original_store_delete
|
|
assert failed_catalog_delete is True
|
|
assert deleted.status_code == 200, deleted.text
|
|
assert deleted.json() == {
|
|
"deleted": True, "cache_cleanup_pending": True,
|
|
}
|
|
assert not source_path.exists()
|
|
assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == {
|
|
"templates": [],
|
|
}
|
|
|
|
for _ in range(100):
|
|
tombstone = json.loads(store.get(
|
|
"cloud_template_tombstone", template_id))
|
|
if tombstone["pending_hosts"] == ["hk"]:
|
|
break
|
|
time.sleep(0.01)
|
|
assert tombstone["source_deleted"] is True
|
|
assert tombstone["source_volume_name"] == source_volume_name
|
|
assert tombstone["pending_hosts"] == ["hk"]
|
|
assert HostStore(store, "jp").get(
|
|
"cloud_template_replica", template_id) is None
|
|
assert HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id) is not None
|
|
|
|
services["hk"].cloud_template_delete_error = VMError(
|
|
"存储池 images 不存在", 404)
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
still_pending = json.loads(store.get(
|
|
"cloud_template_tombstone", template_id))
|
|
assert still_pending["pending_hosts"] == ["hk"]
|
|
assert HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id) is not None
|
|
|
|
services["hk"].cloud_template_delete_error = None
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
terminal = json.loads(store.get("cloud_template_tombstone", template_id))
|
|
assert terminal["pending_hosts"] == []
|
|
assert HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id) is None
|
|
|
|
repeated = call(
|
|
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
|
|
SUPER, {"created_at": template_created_at})
|
|
assert repeated.status_code == 200
|
|
assert repeated.json() == {
|
|
"deleted": True, "cache_cleanup_pending": False,
|
|
}
|
|
|
|
# A duplicate task from the old deletion must not touch a later deletion
|
|
# generation (or the same-ID image/replica that may exist between them).
|
|
old_deletion_id = terminal["deletion_id"]
|
|
engine.registry.cloud_templates._prepare_reimport(template_id)
|
|
reimported = {
|
|
**imported,
|
|
"volume_name": source_volume_name,
|
|
"created_at": template_created_at + 1,
|
|
}
|
|
temporary = engine.registry.cloud_templates.files.directory() \
|
|
/ ".cloud-image-reimport-boundary.qcow2"
|
|
temporary.write_bytes(payload)
|
|
temporary.chmod(0o600)
|
|
engine.registry.cloud_templates.files.publish(
|
|
temporary, template_id, expected_size=len(payload))
|
|
engine.registry.cloud_templates.catalog.add(reimported)
|
|
stale_delete = call(
|
|
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
|
|
SUPER, {"created_at": template_created_at})
|
|
assert stale_delete.status_code == 409
|
|
assert engine.registry.cloud_templates.get(
|
|
template_id)["created_at"] == template_created_at + 1
|
|
|
|
next_tombstone = {
|
|
**terminal,
|
|
"deletion_id": "b" * 32,
|
|
"template_created_at": template_created_at + 1,
|
|
"pending_hosts": ["hk"],
|
|
}
|
|
store.set("cloud_template_tombstone", template_id,
|
|
json.dumps(next_tombstone))
|
|
calls_before = list(services["hk"].calls)
|
|
engine._cleanup_cloud_template_tombstone_on_host_sync(
|
|
template_id, old_deletion_id, "hk", services["hk"],
|
|
engine.registry.components["hk"],
|
|
)
|
|
assert services["hk"].calls == calls_before
|
|
assert json.loads(store.get(
|
|
"cloud_template_tombstone", template_id))["pending_hosts"] == ["hk"]
|
|
|
|
|
|
def test_user_iso_routes_are_owner_scoped_and_create_uses_opaque_id(boundary, tmp_path):
|
|
client, _engine, _store, _logs, services = boundary
|
|
download = tmp_path / "download.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
|
|
def fake_download(_url, _sha256, _directory, *, max_bytes):
|
|
assert max_bytes >= download.stat().st_size
|
|
return download, download.stat().st_size, "a" * 64
|
|
|
|
with patch("virsh_panel.user_isos.download_user_iso", side_effect=fake_download):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "My Installer", "sha256": ""})
|
|
assert queued.status_code == 202
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
iso_id = job["iso"]["id"]
|
|
assert "volume_name" not in job["iso"]
|
|
listed = call(client, "GET", "/internal/v1/user-isos").json()
|
|
assert [row["id"] for row in listed["isos"]] == [iso_id]
|
|
assert "volume_name" not in listed["isos"][0]
|
|
global_source = (tmp_path / "user-isos" / str(USER["user_id"])
|
|
/ f"panel-user-iso-{iso_id}.iso")
|
|
assert global_source.is_file()
|
|
assert services == {}
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/user-isos", USER,
|
|
).status_code == 404
|
|
assert call(
|
|
client, "POST", "/internal/v1/hosts/hk/user-isos/import", USER, {
|
|
"url": "https://downloads.example.test/alias.iso",
|
|
"name": "Host Alias Must Fail", "sha256": "",
|
|
},
|
|
).status_code == 404
|
|
assert call(
|
|
client, "DELETE", f"/internal/v1/hosts/hk/user-isos/{iso_id}", USER,
|
|
).status_code == 404
|
|
assert global_source.is_file()
|
|
|
|
other = {**USER, "user_id": 42}
|
|
assert call(client, "GET", f"/internal/v1/user-isos/{iso_id}",
|
|
other).status_code == 404
|
|
assert call(client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
other).status_code == 404
|
|
assert call(client, "DELETE", f"/internal/v1/user-isos/{iso_id}",
|
|
other).status_code == 404
|
|
|
|
provisioning = {**USER, "can_provision": True,
|
|
"operation_id": "create-user-iso-0001"}
|
|
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
body["user_iso_id"] = iso_id
|
|
created = call(client, "POST", "/internal/v1/hosts/hk/vms",
|
|
provisioning, body)
|
|
assert created.status_code == 201, created.text
|
|
create_calls = [row for row in services["hk"].calls
|
|
if row[0] == "create_user_iso_vm"]
|
|
assert len(create_calls) == 1
|
|
assert create_calls[0][2] == USER["user_id"]
|
|
assert create_calls[0][4] == iso_id
|
|
cache_calls = [row for row in services["hk"].calls
|
|
if row[0] == "cache_user_iso"]
|
|
assert len(cache_calls) == 1
|
|
assert cache_calls[0][4] is False
|
|
|
|
other_host_actor = {
|
|
**provisioning, "operation_id": "create-user-iso-0003",
|
|
}
|
|
other_host_body = {**body, "name": "other-host-installer"}
|
|
other_host = call(client, "POST", "/internal/v1/hosts/jp/vms",
|
|
other_host_actor, other_host_body)
|
|
assert other_host.status_code == 201, other_host.text
|
|
assert len([row for row in services["jp"].calls
|
|
if row[0] == "cache_user_iso"]) == 1
|
|
assert global_source.is_file()
|
|
|
|
ambiguous_actor = {**provisioning, "operation_id": "create-user-iso-0002"}
|
|
ambiguous = call(client, "POST", "/internal/v1/hosts/hk/vms",
|
|
ambiguous_actor, {**body, "iso": "legacy.iso"})
|
|
assert ambiguous.status_code == 422
|
|
|
|
|
|
def test_user_iso_writes_require_manage_capability(boundary):
|
|
client, *_rest = boundary
|
|
actor = {**USER, "can_manage": False}
|
|
response = call(client, "POST", "/internal/v1/user-isos/import",
|
|
actor, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Installer", "sha256": ""})
|
|
assert response.status_code == 403
|
|
|
|
|
|
def test_concurrent_first_use_serializes_replica_marker_and_cache(boundary, tmp_path):
|
|
client, _engine, _store, _logs, services = boundary
|
|
download = tmp_path / "concurrent.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, "1" * 64)):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
|
|
"url": "https://downloads.example.test/concurrent.iso",
|
|
"name": "Concurrent Installer", "sha256": "",
|
|
})
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
iso_id = job["iso"]["id"]
|
|
|
|
# Materialize the target child app before starting concurrent requests.
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
service = services["hk"]
|
|
release = threading.Event()
|
|
service.user_iso_cache_release = release
|
|
base = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
base["user_iso_id"] = iso_id
|
|
|
|
def create(number):
|
|
actor = {**USER, "can_provision": True,
|
|
"operation_id": f"concurrent-user-iso-{number:04d}"}
|
|
return call(client, "POST", "/internal/v1/hosts/hk/vms", actor,
|
|
{**base, "name": f"concurrent-vm-{number}"})
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
first = executor.submit(create, 1)
|
|
assert service.user_iso_cache_started.wait(timeout=2)
|
|
second = executor.submit(create, 2)
|
|
time.sleep(0.05)
|
|
release.set()
|
|
responses = [first.result(timeout=5), second.result(timeout=5)]
|
|
|
|
assert [response.status_code for response in responses] == [201, 201]
|
|
cache_calls = [row for row in service.calls if row[0] == "cache_user_iso"]
|
|
assert [row[4] for row in cache_calls] == [False, True]
|
|
|
|
|
|
def test_user_iso_source_stays_in_use_through_vm_create(boundary, tmp_path):
|
|
client, engine, _store, _logs, services = boundary
|
|
iso_id = import_user_iso_for_test(client, tmp_path, "Create Lease Installer")
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
service = services["hk"]
|
|
release = threading.Event()
|
|
service.user_iso_create_release = release
|
|
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
body["user_iso_id"] = iso_id
|
|
actor = {**USER, "can_provision": True,
|
|
"operation_id": "user-iso-source-lease-0001"}
|
|
|
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
creating = executor.submit(
|
|
call, client, "POST", "/internal/v1/hosts/hk/vms", actor, body)
|
|
assert service.user_iso_create_started.wait(timeout=2)
|
|
with pytest.raises(TemplateError, match="正在同步") as busy:
|
|
engine.registry.user_isos.delete_for(USER["user_id"], iso_id)
|
|
assert busy.value.status == 409
|
|
release.set()
|
|
created = creating.result(timeout=5)
|
|
|
|
assert created.status_code == 201, created.text
|
|
assert engine.registry.user_isos.get_for(USER["user_id"], iso_id)["id"] == iso_id
|
|
|
|
|
|
def test_ready_replica_marker_survives_failed_reuse(boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
iso_id = import_user_iso_for_test(client, tmp_path, "Ready Marker Installer")
|
|
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
body["user_iso_id"] = iso_id
|
|
first = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "ready-marker-create-0001"}, body)
|
|
assert first.status_code == 201, first.text
|
|
marker_key = f"{USER['user_id']}:{iso_id}"
|
|
scoped = HostStore(store, "hk")
|
|
assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready"
|
|
|
|
with patch.object(
|
|
engine.registry.user_isos, "_verify_source",
|
|
side_effect=TemplateError("ISO 源文件的 SHA-256 校验值不匹配", 500)):
|
|
failed = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "ready-marker-create-0002"},
|
|
{**body, "name": "ready-marker-failed"})
|
|
assert failed.status_code == 500
|
|
assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready"
|
|
|
|
recovered = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "ready-marker-create-0003"},
|
|
{**body, "name": "ready-marker-recovered"})
|
|
assert recovered.status_code == 201, recovered.text
|
|
cache_calls = [row for row in services["hk"].calls
|
|
if row[0] == "cache_user_iso"]
|
|
assert cache_calls[-1][4] is True
|
|
|
|
|
|
def test_global_delete_keeps_tombstone_until_host_cache_is_reclaimed(
|
|
boundary, tmp_path):
|
|
client, _engine, store, _logs, services = boundary
|
|
download = tmp_path / "delete.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, "2" * 64)):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
|
|
"url": "https://downloads.example.test/delete.iso",
|
|
"name": "Delete Installer", "sha256": "",
|
|
})
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
iso_id = job["iso"]["id"]
|
|
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
body["user_iso_id"] = iso_id
|
|
created = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "delete-user-iso-cache-0001"}, body)
|
|
assert created.status_code == 201, created.text
|
|
|
|
from virsh_panel.libvirt_service import VMError
|
|
services["hk"].user_iso_delete_error = VMError(
|
|
"用户 ISO 仍被虚拟机安装过程引用,不能删除", 409)
|
|
deleted = call(
|
|
client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
|
|
assert deleted.status_code == 200
|
|
assert deleted.json() == {"deleted": True, "cache_cleanup_pending": True}
|
|
assert call(client, "GET", "/internal/v1/user-isos", USER).json()["isos"] == []
|
|
|
|
tombstone_key = f"{USER['user_id']}:{iso_id}"
|
|
for _ in range(100):
|
|
if any(row[0] == "delete_user_iso" for row in services["hk"].calls):
|
|
break
|
|
time.sleep(0.01)
|
|
assert store.get("user_iso_tombstone", tombstone_key) is not None
|
|
scoped = HostStore(store, "hk")
|
|
assert scoped.get("user_iso_replica", tombstone_key) is not None
|
|
|
|
services["hk"].user_iso_delete_error = None
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
assert store.get("user_iso_tombstone", tombstone_key) is None
|
|
assert scoped.get("user_iso_replica", tombstone_key) is None
|
|
|
|
|
|
def test_concurrent_global_deletes_preserve_committed_tombstone(boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
iso_id = import_user_iso_for_test(client, tmp_path, "Concurrent Delete Installer")
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
record = engine.registry.user_isos.get_for(USER["user_id"], iso_id)
|
|
marker_key = f"{USER['user_id']}:{iso_id}"
|
|
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
|
|
"status": "ready",
|
|
"volume_name": record["volume_name"],
|
|
"size_bytes": record["size_bytes"],
|
|
"sha256": record["sha256"],
|
|
}))
|
|
from virsh_panel.libvirt_service import VMError
|
|
services["hk"].user_iso_volumes.add((USER["user_id"], record["volume_name"]))
|
|
services["hk"].user_iso_delete_error = VMError(
|
|
"用户 ISO 仍被虚拟机安装过程引用,不能删除", 409)
|
|
|
|
original_delete = engine.registry.user_isos.delete_for
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
|
|
def blocking_delete(user_id, selected_iso_id):
|
|
started.set()
|
|
assert release.wait(timeout=5)
|
|
return original_delete(user_id, selected_iso_id)
|
|
|
|
try:
|
|
with patch.object(engine.registry.user_isos, "delete_for",
|
|
side_effect=blocking_delete) as delete_for:
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
first = executor.submit(
|
|
call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
|
|
assert started.wait(timeout=2)
|
|
second = executor.submit(
|
|
call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
|
|
time.sleep(0.05)
|
|
release.set()
|
|
responses = [first.result(timeout=5), second.result(timeout=5)]
|
|
assert delete_for.call_count == 1
|
|
finally:
|
|
release.set()
|
|
|
|
assert [response.status_code for response in responses] == [200, 200]
|
|
tombstone = json.loads(store.get("user_iso_tombstone", marker_key))
|
|
assert tombstone["source_deleted"] is True
|
|
assert tombstone["pending_hosts"] == ["hk"]
|
|
|
|
|
|
def test_user_iso_cache_cleanup_blocks_online_update(boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
iso_id = import_user_iso_for_test(client, tmp_path, "Update Gate Installer")
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
record = engine.registry.user_isos.get_for(USER["user_id"], iso_id)
|
|
marker_key = f"{USER['user_id']}:{iso_id}"
|
|
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
|
|
"status": "ready",
|
|
"volume_name": record["volume_name"],
|
|
"size_bytes": record["size_bytes"],
|
|
"sha256": record["sha256"],
|
|
}))
|
|
service = services["hk"]
|
|
service.user_iso_volumes.add((USER["user_id"], record["volume_name"]))
|
|
release = threading.Event()
|
|
service.user_iso_delete_release = release
|
|
updater = FakeUpdater()
|
|
updater.token = "valid-token-123456"
|
|
engine.updater = updater
|
|
try:
|
|
deleted = call(
|
|
client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
|
|
assert deleted.status_code == 200
|
|
assert service.user_iso_delete_started.wait(timeout=2)
|
|
blocked = call(client, "POST", "/internal/v1/update/apply", SUPER, {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
})
|
|
assert blocked.status_code == 409
|
|
assert updater.active is None
|
|
finally:
|
|
release.set()
|
|
for _ in range(100):
|
|
if not engine.maintenance_tasks:
|
|
break
|
|
time.sleep(0.01)
|
|
assert not engine.maintenance_tasks
|
|
|
|
|
|
def test_host_get_skips_cache_cleanup_during_active_update(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
iso_id = "f" * 32
|
|
volume_name = f"panel-user-iso-{iso_id}.iso"
|
|
marker_key = f"{USER['user_id']}:{iso_id}"
|
|
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
|
|
"status": "ready", "volume_name": volume_name,
|
|
"size_bytes": 1024 * 1024, "sha256": "f" * 64,
|
|
}))
|
|
store.set("user_iso_tombstone", marker_key, json.dumps({
|
|
"user_id": USER["user_id"], "iso_id": iso_id,
|
|
"volume_name": volume_name, "pending_hosts": ["hk"],
|
|
"source_deleted": True, "created_at": 1,
|
|
}))
|
|
updater = FakeUpdater()
|
|
updater.active = {"job_id": "update-job-1234567890", "status": "running"}
|
|
engine.updater = updater
|
|
|
|
response = call(client, "GET", "/internal/v1/hosts/hk/resources")
|
|
|
|
assert response.status_code == 200
|
|
assert not any(row[0] == "delete_user_iso" for row in services["hk"].calls)
|
|
assert store.get("user_iso_tombstone", marker_key) is not None
|
|
|
|
|
|
def test_slow_tombstone_cleanup_does_not_block_engine_startup(tmp_path):
|
|
class StartupRegistry:
|
|
def __init__(self):
|
|
self.components = {"offline": object()}
|
|
self.user_isos = SimpleNamespace(has_active_jobs=lambda: False)
|
|
self.started = False
|
|
self.closed = False
|
|
|
|
async def start(self):
|
|
self.started = True
|
|
|
|
async def close(self):
|
|
self.closed = True
|
|
|
|
async def scenario():
|
|
registry = StartupRegistry()
|
|
updater = FakeUpdater()
|
|
engine = EngineApplication(
|
|
{"service_secret": base64.b64encode(SECRET).decode(),
|
|
"data_dir": str(tmp_path)},
|
|
MemoryStore(), MemoryStore(), registry=registry, updater=updater,
|
|
)
|
|
cleanup_started = asyncio.Event()
|
|
cleanup_release = asyncio.Event()
|
|
|
|
async def slow_cleanup(host_id):
|
|
assert host_id == "offline"
|
|
cleanup_started.set()
|
|
await cleanup_release.wait()
|
|
|
|
engine._cleanup_host_user_iso_tombstones = slow_cleanup
|
|
incoming = asyncio.Queue()
|
|
outgoing = asyncio.Queue()
|
|
|
|
async def receive():
|
|
return await incoming.get()
|
|
|
|
async def send(message):
|
|
await outgoing.put(message)
|
|
|
|
await incoming.put({"type": "lifespan.startup"})
|
|
lifespan = asyncio.create_task(engine._lifespan(receive, send))
|
|
startup = await asyncio.wait_for(outgoing.get(), timeout=0.5)
|
|
assert startup == {"type": "lifespan.startup.complete"}
|
|
await asyncio.wait_for(cleanup_started.wait(), timeout=0.5)
|
|
assert registry.started is True
|
|
assert engine.maintenance_tasks
|
|
assert engine._long_running_host_job() is True
|
|
|
|
cleanup_release.set()
|
|
await incoming.put({"type": "lifespan.shutdown"})
|
|
shutdown = await asyncio.wait_for(outgoing.get(), timeout=0.5)
|
|
assert shutdown == {"type": "lifespan.shutdown.complete"}
|
|
await asyncio.wait_for(lifespan, timeout=0.5)
|
|
assert registry.closed is True
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_user_iso_global_import_does_not_contact_a_host(boundary, tmp_path):
|
|
client, _engine, _store, _logs, services = boundary
|
|
download = tmp_path / "global.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, "f" * 64)):
|
|
response = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Installer", "sha256": ""})
|
|
assert response.status_code == 202
|
|
job_id = response.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
assert services == {}
|
|
|
|
|
|
def test_user_iso_download_is_bounded_by_panel_storage(boundary, tmp_path):
|
|
client, engine, _store, _logs, _services = boundary
|
|
download = tmp_path / "bounded.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
|
|
def bounded_download(_url, _sha256, _directory, *, max_bytes):
|
|
assert max_bytes == 2 * 1024 * 1024
|
|
return download, download.stat().st_size, "d" * 64
|
|
|
|
with patch.object(engine.registry.user_isos.files, "available_bytes",
|
|
return_value=2 * 1024 * 1024), patch(
|
|
"virsh_panel.user_isos.download_user_iso", side_effect=bounded_download):
|
|
response = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Bounded Installer", "sha256": ""})
|
|
assert response.status_code == 202
|
|
job_id = response.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
|
|
|
|
def test_user_iso_job_hides_internal_paths_in_failures(boundary):
|
|
client, _engine, _store, _logs, _services = boundary
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
side_effect=TemplateError("目录 /private/data/user-isos/41 损坏", 500)):
|
|
response = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Error Installer", "sha256": ""})
|
|
assert response.status_code == 202
|
|
job_id = response.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "failed"
|
|
assert job["error"] == "ISO 下载或保存失败,请联系管理员查看日志"
|
|
assert "/private/data" not in json.dumps(job, ensure_ascii=False)
|
|
|
|
|
|
def test_user_iso_import_persistence_failure_does_not_stick_pending(boundary, tmp_path):
|
|
client, _engine, store, _logs, _services = boundary
|
|
assert call(client, "GET", "/internal/v1/user-isos").status_code == 200
|
|
original_set = store.set
|
|
|
|
def fail_user_iso_job(kind, key, value):
|
|
if kind == "user_iso_import_job":
|
|
raise DatabaseError()
|
|
return original_set(kind, key, value)
|
|
|
|
body = {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Installer", "sha256": ""}
|
|
with patch.object(store, "set", side_effect=fail_user_iso_job):
|
|
failed = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, body)
|
|
assert failed.status_code == 503
|
|
|
|
download = tmp_path / "retry.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, "b" * 64)):
|
|
retried = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, body)
|
|
assert retried.status_code == 202, retried.text
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{retried.json()['job_id']}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
|
|
|
|
def test_interrupted_user_iso_import_is_restored_as_failed(boundary):
|
|
client, _engine, store, _logs, _services = boundary
|
|
job_id = "interrupted-user-iso"
|
|
scope = user_iso_scope(USER["user_id"])
|
|
store.set("user_iso_import_job", job_id, json.dumps({
|
|
"job_id": job_id, "scope": scope, "owner_user_id": USER["user_id"],
|
|
"status": "running", "phase": "正在安全下载 ISO", "created_at": 1,
|
|
}))
|
|
_engine.registry.user_isos._restored = False
|
|
_engine.registry.user_isos.restore_jobs()
|
|
restored = call(
|
|
client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
)
|
|
assert restored.status_code == 200
|
|
assert restored.json()["status"] == "failed"
|
|
assert "重启" in restored.json()["phase"]
|
|
|
|
|
|
def test_user_iso_list_exposes_only_owners_active_import(boundary, tmp_path):
|
|
client, engine, _store, _logs, _services = boundary
|
|
download = tmp_path / "active.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
|
|
def blocking_download(_url, _sha256, _directory, *, max_bytes):
|
|
assert max_bytes >= download.stat().st_size
|
|
started.set()
|
|
assert release.wait(timeout=5)
|
|
return download, download.stat().st_size, "c" * 64
|
|
|
|
body = {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Active Installer", "sha256": ""}
|
|
try:
|
|
with patch("virsh_panel.user_isos.download_user_iso", side_effect=blocking_download):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, body)
|
|
assert queued.status_code == 202
|
|
assert started.wait(timeout=2)
|
|
assert engine._long_running_host_job() is True
|
|
own = call(client, "GET", "/internal/v1/user-isos", USER).json()
|
|
active = own["active_import_job"]
|
|
assert active["job_id"] == queued.json()["job_id"]
|
|
assert active["status"] == "running"
|
|
assert set(active) <= {"job_id", "status", "phase", "created_at",
|
|
"started_at", "finished_at", "iso", "error"}
|
|
assert "scope" not in active and "owner_user_id" not in active
|
|
other = call(client, "GET", "/internal/v1/user-isos",
|
|
{**USER, "user_id": 42}).json()
|
|
assert "active_import_job" not in other
|
|
release.set()
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{active['job_id']}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
finally:
|
|
release.set()
|
|
assert job["status"] == "completed", job
|
|
assert "active_import_job" not in call(
|
|
client, "GET", "/internal/v1/user-isos", USER).json()
|
|
|
|
|
|
def test_hmac_binds_actor_payload_target_and_rejects_replay():
|
|
store = MemoryStore()
|
|
body = b"{}"
|
|
target = "/internal/v1/hosts/hk/vms?range=1h"
|
|
headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER,
|
|
timestamp=1000, nonce="n" * 24).items()}
|
|
assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
changed = {**headers, "x-cloud-actor": encode_actor(SUPER)}
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061)
|
|
|
|
invalid_cleanup = {**USER, "system_cleanup": "true"}
|
|
invalid_headers = {key.lower(): value for key, value in signed_headers(
|
|
SECRET, "POST", target, body, invalid_cleanup,
|
|
timestamp=1000, nonce="c" * 24,
|
|
).items()}
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(
|
|
invalid_headers, SECRET, "POST", target,
|
|
hashlib.sha256(body).hexdigest(), store, now=1000,
|
|
)
|
|
|
|
|
|
def test_private_route_allowlist_and_no_browser_admin_cookie(boundary):
|
|
client, *_ = boundary
|
|
for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin",
|
|
"/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"):
|
|
assert call(client, "POST", path, SUPER, {}).status_code == 404
|
|
assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401
|
|
assert TRUSTED_ACTOR.get() is None
|
|
|
|
|
|
def test_internal_update_routes_are_super_admin_only_and_never_echo_token(boundary):
|
|
client, engine, *_ = boundary
|
|
updater = FakeUpdater()
|
|
triggered = []
|
|
engine.updater = updater
|
|
def trigger():
|
|
triggered.append(True)
|
|
updater.active["status"] = "running"
|
|
engine.update_trigger = trigger
|
|
settings_path = "/internal/v1/update/settings"
|
|
|
|
assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401
|
|
assert call(client, "GET", settings_path, USER).status_code == 403
|
|
assert call(client, "GET", settings_path, ADMIN).status_code == 403
|
|
assert call(client, "GET", settings_path, {**SUPER, "can_manage": False}).status_code == 403
|
|
|
|
token = "gitea-private-token-123456"
|
|
saved = call(client, "PUT", settings_path, SUPER, {"access_token": token})
|
|
assert saved.status_code == 200
|
|
assert saved.json()["token_configured"] is True
|
|
assert saved.json()["repository"] == "https://g.luoli.live/root/virt.git"
|
|
assert saved.json()["branch"] == "venix"
|
|
assert token not in saved.text
|
|
assert token not in call(client, "GET", settings_path, SUPER).text
|
|
|
|
checked = call(client, "POST", "/internal/v1/update/check", SUPER, {})
|
|
assert checked.status_code == 200
|
|
assert checked.json()["status"] == "update_available"
|
|
applied = call(client, "POST", "/internal/v1/update/apply", SUPER, {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
})
|
|
assert applied.status_code == 202
|
|
assert applied.json()["job_id"] == "update-job-1234567890"
|
|
assert "private" not in applied.json() and token not in applied.text
|
|
assert triggered == [True]
|
|
job = call(client, "GET", "/internal/v1/update/jobs/update-job-1234567890", SUPER)
|
|
assert job.status_code == 200 and token not in job.text and "private" not in job.json()
|
|
|
|
# While the persisted update job is active, reads remain available but no
|
|
# host/VM mutation may begin and overlap the code/service restart.
|
|
assert call(client, "GET", "/internal/v1/health", SUPER).status_code == 200
|
|
assert call(client, "POST", "/internal/v1/hosts/scan", SUPER,
|
|
{"host": "host.example.test", "port": 22}).status_code == 409
|
|
|
|
|
|
def test_internal_update_validation_clear_and_trigger_failure(boundary):
|
|
client, engine, *_ = boundary
|
|
updater = FakeUpdater()
|
|
engine.updater = updater
|
|
settings_path = "/internal/v1/update/settings"
|
|
assert call(client, "PUT", settings_path, SUPER, {}).status_code == 422
|
|
assert call(client, "PUT", settings_path, SUPER,
|
|
{"access_token": "x", "unexpected": True}).status_code == 422
|
|
assert call(client, "POST", "/internal/v1/update/check", SUPER,
|
|
{"repository": "https://evil.example/repo.git"}).status_code == 422
|
|
assert call(client, "POST", "/internal/v1/update/apply", SUPER,
|
|
{"expected_current_commit": "a" * 40}).status_code == 422
|
|
assert call(client, "GET", "/internal/v1/update/jobs/missing-job-123456", SUPER).status_code == 404
|
|
|
|
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
|
|
cleared = call(client, "PUT", settings_path, SUPER, {"clear_token": True})
|
|
assert cleared.status_code == 200 and cleared.json()["token_configured"] is False
|
|
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
|
|
|
|
def fail_trigger():
|
|
from virsh_panel.updater import UpdateError
|
|
raise UpdateError("独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503)
|
|
|
|
engine.update_trigger = fail_trigger
|
|
failed = call(client, "POST", "/internal/v1/update/apply", SUPER, {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
})
|
|
assert failed.status_code == 503
|
|
job = updater.jobs["update-job-1234567890"]
|
|
assert job["status"] == "failed"
|
|
assert "令牌" not in job["error"]
|
|
|
|
|
|
@pytest.mark.parametrize(("state", "expected"), [
|
|
("activating", True),
|
|
("active", True),
|
|
("deactivating", True),
|
|
("inactive", False),
|
|
("failed", False),
|
|
("maintenance", None),
|
|
])
|
|
def test_update_oneshot_transitional_states_keep_maintenance_gate(state, expected):
|
|
with patch("virsh_panel.internal.subprocess.run",
|
|
return_value=SimpleNamespace(returncode=0, stdout=state + "\n")):
|
|
assert EngineApplication._update_service_active() is expected
|
|
|
|
|
|
def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary):
|
|
client, engine, *_ = boundary
|
|
path = "/internal/v1/hosts/hk"
|
|
assert call(client, "GET", path + "/host").status_code == 403
|
|
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
|
|
assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
|
|
assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200
|
|
assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200
|
|
assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404
|
|
assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == []
|
|
assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False},
|
|
{"action": "reboot"}).status_code == 403
|
|
|
|
|
|
def test_existing_unmanaged_vms_are_claimed_once_for_configuring_admin(boundary):
|
|
client, engine, _, _, services = boundary
|
|
path = "/internal/v1/hosts/hk"
|
|
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
|
|
manual_uuid = "22222222-2222-4222-8222-222222222222"
|
|
orphaned_managed_uuid = "33333333-3333-4333-8333-333333333333"
|
|
services["hk"].other_vms = [
|
|
{"uuid": manual_uuid, "name": "manual-vm", "state": "shutoff",
|
|
"vcpu": 2, "memory_mib": 2048, "disks": [], "managed": False},
|
|
{"uuid": orphaned_managed_uuid, "name": "unknown-panel-vm", "state": "shutoff",
|
|
"vcpu": 2, "memory_mib": 2048, "disks": [], "managed": True},
|
|
]
|
|
body = {"known_owners": {VM_ID: 41}}
|
|
assert call(client, "POST", path + "/host/discover-vms", ADMIN, body).status_code == 403
|
|
discovered = call(client, "POST", path + "/host/discover-vms", SUPER, body)
|
|
assert discovered.status_code == 200, discovered.text
|
|
rows = {row["uuid"]: row for row in discovered.json()["items"]}
|
|
assert rows[VM_ID]["owner_user_id"] == 41
|
|
assert rows[manual_uuid]["owner_user_id"] == SUPER["user_id"]
|
|
assert orphaned_managed_uuid not in rows
|
|
assert discovered.json()["claimed_count"] == 2
|
|
assert discovered.json()["untracked_managed_count"] == 1
|
|
assert discovered.json()["total_count"] == 3
|
|
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"]
|
|
adopted = call(client, "POST", path + "/host/discover-vms", SUPER,
|
|
{**body, "claim_unowned_managed": True})
|
|
assert adopted.status_code == 200
|
|
adopted_rows = {row["uuid"]: row for row in adopted.json()["items"]}
|
|
assert adopted_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"]
|
|
assert adopted.json()["claimed_count"] == 1
|
|
assert adopted.json()["untracked_managed_count"] == 0
|
|
repeated = call(client, "POST", path + "/host/discover-vms",
|
|
{**SUPER, "user_id": 2}, {**body, "claim_unowned_managed": True})
|
|
assert repeated.status_code == 200
|
|
repeated_rows = {row["uuid"]: row for row in repeated.json()["items"]}
|
|
assert repeated_rows[manual_uuid]["owner_user_id"] == SUPER["user_id"]
|
|
assert repeated_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"]
|
|
assert repeated.json()["claimed_count"] == 0
|
|
assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"]
|
|
assert {row["uuid"] for row in call(
|
|
client, "GET", path + "/vms", {**USER, "user_id": SUPER["user_id"]}
|
|
).json()["items"]} == {manual_uuid, orphaned_managed_uuid}
|
|
assert call(client, "GET", path + "/vms", {**USER, "user_id": 2}).json()["items"] == []
|
|
|
|
|
|
def test_stable_host_identity_and_namespaces():
|
|
store = MemoryStore()
|
|
first, second = HostStore(store, "a"), HostStore(store, "b")
|
|
first.set("host", "current", "one")
|
|
second.set("host", "current", "two")
|
|
assert first.get("host", "current") == "one"
|
|
assert second.get("host", "current") == "two"
|
|
a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"}
|
|
assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"})
|
|
assert host_scope(a) != host_scope({**a, "host_id": "b"})
|
|
assert MainStore.RECORD_TABLE == "engine_records"
|
|
assert LogStore.METRIC_TABLE == "engine_vm_metric_samples"
|
|
|
|
|
|
def test_lightweight_vm_state_reads_are_allowlisted(boundary):
|
|
client, _engine, _store, _logs, services = boundary
|
|
listed = call(client, "GET", "/internal/v1/hosts/hk/vm-states", SUPER)
|
|
assert listed.status_code == 200, listed.text
|
|
assert listed.json()["items"] == [{
|
|
"uuid": VM_ID, "name": "alma-test", "state": "running",
|
|
}]
|
|
single = call(
|
|
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/state", SUPER,
|
|
)
|
|
assert single.status_code == 200, single.text
|
|
assert single.json() == {
|
|
"uuid": VM_ID, "name": "alma-test", "state": "running",
|
|
}
|
|
assert ("list_vm_states",) in services["hk"].calls
|
|
assert ("get_vm_state", VM_ID) in services["hk"].calls
|
|
|
|
|
|
def test_selected_vm_metadata_is_allowlisted_and_owner_scoped(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
path = f"/internal/v1/hosts/hk/vm-metadata?ids={VM_ID}"
|
|
|
|
admin = call(client, "GET", path, SUPER)
|
|
assert admin.status_code == 200, admin.text
|
|
assert admin.json()["items"][0]["uuid"] == VM_ID
|
|
assert services["hk"].calls[-1] == ("list_vm_metadata", (VM_ID,))
|
|
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
component.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
owner = call(client, "GET", path, USER)
|
|
assert owner.status_code == 200, owner.text
|
|
calls_after_owner = services["hk"].calls.count(("list_vm_metadata", (VM_ID,)))
|
|
assert calls_after_owner == 2
|
|
|
|
stranger = call(client, "GET", path, {**USER, "user_id": 42})
|
|
assert stranger.status_code == 404
|
|
assert stranger.json()["error"] == "虚拟机不存在"
|
|
assert services["hk"].calls.count(("list_vm_metadata", (VM_ID,))) == calls_after_owner
|
|
|
|
|
|
def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403
|
|
provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"}
|
|
created = call(client, "POST", path, provisioning, CREATE_BODY)
|
|
assert created.status_code == 201, created.text
|
|
vm_id = created.json()["uuid"]
|
|
assert created.json()["result"]["uuid"] == vm_id
|
|
assert created.json()["status"] == "succeeded"
|
|
component = engine.registry.components["hk"]
|
|
assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41)
|
|
again = call(client, "POST", path, provisioning, CREATE_BODY)
|
|
assert again.status_code == 201 and again.json()["uuid"] == vm_id
|
|
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
|
|
assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409
|
|
operation = call(client, "GET", "/internal/v1/operations/order-000001").json()
|
|
assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id
|
|
assert operation["safe_to_refund"] is False
|
|
assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404
|
|
|
|
|
|
def test_uncertain_creation_never_refunds_or_recreates(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
call(client, "GET", path)
|
|
services["hk"].create_failure = True
|
|
actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"}
|
|
failed = call(client, "POST", path, actor, CREATE_BODY)
|
|
assert failed.status_code == 502
|
|
assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"]
|
|
record = json.loads(store.get("operation", "order-uncertain"))
|
|
assert record["resource_naming"] == "uuid-v1"
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
assert component.state.users.owns_vm(scope, record["vm_uuid"], 41)
|
|
component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password")
|
|
component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]}))
|
|
again = call(client, "POST", path, actor, CREATE_BODY)
|
|
assert again.status_code == 202 and again.json()["status"] == "uncertain"
|
|
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
|
|
services["hk"].absent = True
|
|
operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json()
|
|
assert operation["status"] == "failed" and operation["safe_to_refund"] is True
|
|
assert services["hk"].absent_checks[-1] == (
|
|
record["vm_uuid"], CREATE_BODY["name"], True, False)
|
|
assert not component.state.users.owns_vm(scope, record["vm_uuid"], 41)
|
|
assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") is None
|
|
assert component.store.list("ip_claim") == {}
|
|
|
|
|
|
def test_legacy_name_collision_operation_uses_legacy_absent_reconciliation(boundary):
|
|
client, _engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
services["hk"].absent = True
|
|
legacy_vm_id = "00000000-0000-4000-8000-000000000002"
|
|
store.set("operation", "legacy-name-collision", json.dumps({
|
|
"operation_id": "legacy-name-collision",
|
|
"host_id": "hk",
|
|
"user_id": USER["user_id"],
|
|
"request_hash": "legacy",
|
|
"vm_uuid": legacy_vm_id,
|
|
"vm_name": CREATE_BODY["name"],
|
|
"kind": "create",
|
|
"resource": "vms",
|
|
"method": "POST",
|
|
"status": "uncertain",
|
|
"result": None,
|
|
"error": "虚拟机名称已存在",
|
|
"safe_to_refund": False,
|
|
"created_at": 1,
|
|
}))
|
|
|
|
operation = call(
|
|
client, "GET", "/internal/v1/operations/legacy-name-collision", USER,
|
|
).json()
|
|
|
|
assert operation["status"] == "failed"
|
|
assert operation["safe_to_refund"] is True
|
|
assert services["hk"].absent_checks[-1] == (
|
|
legacy_vm_id, CREATE_BODY["name"], False, True)
|
|
|
|
|
|
@pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"],
|
|
{**CREATE_BODY, "guest_ssh_key_id": 3}])
|
|
def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload):
|
|
client, engine, _, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
actor = {**USER, "can_provision": True, "operation_id": "rejected-create"}
|
|
response = call(client, "POST", path, actor, payload)
|
|
assert response.status_code == 422
|
|
# Simulate a dispatcher losing the POST response and querying after restart.
|
|
engine.active_operations.clear()
|
|
saved = call(client, "GET", "/internal/v1/operations/rejected-create").json()
|
|
assert saved["status"] == "failed" and saved["safe_to_refund"] is True
|
|
assert call(client, "POST", path, actor, payload).status_code == 422
|
|
assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409
|
|
assert not any(row[0] == "create_vm" for row in services["hk"].calls)
|
|
|
|
|
|
def test_admin_created_vm_also_has_external_owner(boundary):
|
|
client, engine, *_ = boundary
|
|
actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"}
|
|
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
|
|
assert response.status_code == 201
|
|
components = engine.registry.components["hk"]
|
|
assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1)
|
|
|
|
|
|
def test_power_action_has_persistent_idempotency(boundary):
|
|
client, engine, _, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
|
assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403
|
|
actor = {**USER, "operation_id": "reboot-operation"}
|
|
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
|
|
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
|
|
assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1
|
|
assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409
|
|
operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json()
|
|
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
|
|
|
|
|
|
def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary):
|
|
client, engine, _, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
rescue_key = f"{scope}.{VM_ID}"
|
|
components.store.set("vm_rescue", rescue_key, json.dumps({
|
|
"original_xml": "<domain><name>alma-test</name></domain>",
|
|
"template_id": "0123456789abcdef0123456789abcdef",
|
|
"volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso",
|
|
"status": "active",
|
|
}))
|
|
services["hk"].vm.update(
|
|
state="shutoff", rescue_mode=True, rescue_staged=True,
|
|
rescue_volume_name="panel-rescue-0123456789abcdef0123456789abcdef.iso",
|
|
)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
|
|
|
|
ordinary = call(
|
|
client, "DELETE", path,
|
|
{**USER, "operation_id": "ordinary-rescue-delete", "system_cleanup": False},
|
|
)
|
|
assert ordinary.status_code == 409
|
|
assert components.store.get("vm_rescue", rescue_key) is not None
|
|
assert not any(item[0] == "delete_vm" for item in services["hk"].calls)
|
|
|
|
cleanup = call(
|
|
client, "DELETE", path,
|
|
{**USER, "operation_id": "expiry-rescue-delete", "system_cleanup": True},
|
|
)
|
|
assert cleanup.status_code == 200, cleanup.text
|
|
assert cleanup.json()["status"] == "succeeded"
|
|
assert services["hk"].calls[-1] == ("delete_vm", VM_ID, True)
|
|
assert components.store.get("vm_rescue", rescue_key) is None
|
|
assert components.state.users.owner_of_vm(scope, VM_ID) is None
|
|
|
|
|
|
@pytest.mark.parametrize(("route", "method", "target"), [
|
|
("service-suspend", "service_suspend", {"state": "shutoff", "autostart": False}),
|
|
("service-resume", "service_resume", {"autostart": True}),
|
|
("network-suspend", "network_suspend", {"network_suspended": True}),
|
|
("network-resume", "network_resume", {"network_suspended": False}),
|
|
])
|
|
def test_private_target_state_operations_are_owned_idempotent_mutations(
|
|
boundary, route, method, target):
|
|
client, engine, _, _, services = boundary
|
|
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
|
|
assert call(client, "POST", base, USER, {}).status_code == 403
|
|
assert call(client, "POST", base, {**USER, "can_manage": False,
|
|
"operation_id": f"blocked-{route}"}, {}).status_code == 403
|
|
assert call(client, "POST", base, {**USER, "user_id": 42,
|
|
"operation_id": f"foreign-{route}"}, {}).status_code == 404
|
|
|
|
actor = {**USER, "operation_id": f"target-{route}"}
|
|
first = call(client, "POST", base, actor, {})
|
|
assert first.status_code == 200, first.text
|
|
assert first.json()["status"] == "succeeded"
|
|
assert all(first.json()[key] == value for key, value in target.items())
|
|
repeated = call(client, "POST", base, actor, {})
|
|
assert repeated.status_code == 200
|
|
assert len([item for item in services["hk"].calls if item[0] == method]) == 1
|
|
|
|
|
|
@pytest.mark.parametrize(("route", "method", "drift"), [
|
|
("service-suspend", "service_suspend", {"state": "running", "autostart": True}),
|
|
("service-resume", "service_resume", {"autostart": False}),
|
|
("network-suspend", "network_suspend", {"network_suspended": False}),
|
|
("network-resume", "network_resume", {"network_suspended": True}),
|
|
])
|
|
def test_target_state_operation_replays_after_lost_response(
|
|
boundary, route, method, drift):
|
|
client, engine, store, _, services = boundary
|
|
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
actor = {**USER, "operation_id": f"replay-{route}"}
|
|
assert call(client, "POST", base, actor, {}).status_code == 200
|
|
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(drift)
|
|
engine.active_operations.clear()
|
|
|
|
replayed = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
|
|
actor)
|
|
assert replayed.status_code == 200, replayed.text
|
|
assert replayed.json()["status"] == "succeeded"
|
|
assert len([item for item in services["hk"].calls if item[0] == method]) == 2
|
|
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost again")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(drift)
|
|
replayed_post = call(client, "POST", base, actor, {})
|
|
assert replayed_post.status_code == 200, replayed_post.text
|
|
assert replayed_post.json()["status"] == "succeeded"
|
|
assert len([item for item in services["hk"].calls if item[0] == method]) == 3
|
|
|
|
|
|
def test_non_reconciling_operation_lookup_never_replays_target_state(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend"
|
|
actor = {**USER, "operation_id": "peek-target-state"}
|
|
assert call(client, "POST", path, actor, {}).status_code == 200
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm["network_suspended"] = False
|
|
engine.active_operations.clear()
|
|
before = len(services["hk"].calls)
|
|
|
|
peeked = call(client, "GET",
|
|
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
|
|
assert peeked.status_code == 200
|
|
assert peeked.json()["status"] == "uncertain"
|
|
assert peeked.json()["executor_active"] is False
|
|
assert len(services["hk"].calls) == before
|
|
assert services["hk"].vm["network_suspended"] is False
|
|
|
|
engine.active_operations.add(actor["operation_id"])
|
|
active = call(client, "GET",
|
|
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
|
|
assert active.status_code == 200
|
|
assert active.json()["executor_active"] is True
|
|
assert len(services["hk"].calls) == before
|
|
engine.active_operations.discard(actor["operation_id"])
|
|
|
|
|
|
def test_target_state_lock_serializes_same_vm_and_not_other_vms(boundary):
|
|
_, engine, *_ = boundary
|
|
|
|
async def exercise():
|
|
first = engine._target_lock("hk", VM_ID)
|
|
same = engine._target_lock("hk", VM_ID)
|
|
other = engine._target_lock("hk", "22222222-2222-4222-8222-222222222222")
|
|
assert first is same
|
|
assert first is not other
|
|
entered = []
|
|
release = asyncio.Event()
|
|
|
|
async def owner():
|
|
async with first:
|
|
entered.append("owner")
|
|
await release.wait()
|
|
entered.append("owner-finished")
|
|
|
|
async def waiter():
|
|
async with same:
|
|
entered.append("waiter")
|
|
|
|
owner_task = asyncio.create_task(owner())
|
|
await asyncio.sleep(0)
|
|
waiter_task = asyncio.create_task(waiter())
|
|
await asyncio.sleep(0)
|
|
assert entered == ["owner"]
|
|
release.set()
|
|
await asyncio.gather(owner_task, waiter_task)
|
|
assert entered == ["owner", "owner-finished", "waiter"]
|
|
|
|
errors = []
|
|
def run_in_isolated_loop():
|
|
try:
|
|
asyncio.run(exercise())
|
|
except BaseException as exc: # propagate assertion failures to pytest
|
|
errors.append(exc)
|
|
|
|
thread = threading.Thread(target=run_in_isolated_loop)
|
|
thread.start()
|
|
thread.join(timeout=5)
|
|
assert not thread.is_alive()
|
|
if errors:
|
|
raise errors[0]
|
|
|
|
|
|
def test_expiry_target_waits_for_inflight_user_mutation_on_same_vm(boundary):
|
|
client, engine, _, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
service = services["hk"]
|
|
action_entered = threading.Event()
|
|
release_action = threading.Event()
|
|
network_entered = threading.Event()
|
|
original_action = service.action
|
|
original_network_suspend = service.network_suspend
|
|
|
|
def slow_action(vm_id, action):
|
|
action_entered.set()
|
|
assert release_action.wait(timeout=5), "test did not release user mutation"
|
|
return original_action(vm_id, action)
|
|
|
|
def observed_network_suspend(vm_id):
|
|
network_entered.set()
|
|
return original_network_suspend(vm_id)
|
|
|
|
service.action = slow_action
|
|
service.network_suspend = observed_network_suspend
|
|
responses = {}
|
|
|
|
def user_request():
|
|
responses["action"] = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions",
|
|
{**USER, "operation_id": "concurrent-user-action"}, {"action": "reset"})
|
|
|
|
def policy_request():
|
|
responses["policy"] = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
|
|
{**USER, "operation_id": "concurrent-expiry-policy"}, {})
|
|
|
|
first = threading.Thread(target=user_request)
|
|
first.start()
|
|
assert action_entered.wait(timeout=5)
|
|
second = threading.Thread(target=policy_request)
|
|
second.start()
|
|
assert not network_entered.wait(timeout=0.2)
|
|
release_action.set()
|
|
first.join(timeout=5)
|
|
second.join(timeout=5)
|
|
assert not first.is_alive() and not second.is_alive()
|
|
assert responses["action"].status_code == 200
|
|
assert responses["policy"].status_code == 200
|
|
assert network_entered.is_set()
|
|
mutations = [item[0] for item in service.calls
|
|
if item[0] in {"action", "network_suspend"}]
|
|
assert mutations == ["action", "network_suspend"]
|
|
assert service.vm["network_suspended"] is True
|
|
|
|
|
|
def test_target_state_reconciliation_does_not_bypass_original_owner(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
store.set("operation", "unsafe-target", json.dumps({
|
|
"operation_id": "unsafe-target", "host_id": "hk", "user_id": 42,
|
|
"kind": "mutation", "resource": f"vms/{VM_ID}/network-suspend",
|
|
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
|
|
"actor_can_manage": True, "actor_is_admin": False,
|
|
"safe_to_refund": False,
|
|
}))
|
|
response = call(client, "GET", "/internal/v1/operations/unsafe-target",
|
|
{**USER, "user_id": 42})
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "failed"
|
|
assert not any(item[0] == "network_suspend" for item in services["hk"].calls)
|
|
|
|
|
|
def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID,
|
|
"status": "running", "phase": "正在救援"}
|
|
components.store.set("rescue_job", "job-abc", json.dumps(job))
|
|
operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID,
|
|
"status": "uncertain", "result": None, "safe_to_refund": False}
|
|
store.set("operation", "lost-rescue-job", json.dumps(operation))
|
|
recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
|
|
assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID
|
|
job.update(status="completed", phase="救援系统启动")
|
|
components.store.set("rescue_job", "job-abc", json.dumps(job))
|
|
completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
|
|
assert completed["status"] == "succeeded"
|
|
assert not completed["safe_to_refund"]
|
|
|
|
|
|
def test_interrupted_async_job_becomes_terminal_without_replay(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.store.set("reinstall_job", "job-abcd", json.dumps({
|
|
"operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID,
|
|
"status": "failed", "phase": "面板重启,任务已中断"}))
|
|
store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job",
|
|
"host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall",
|
|
"vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False}))
|
|
recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json()
|
|
assert recovered["status"] == "failed"
|
|
assert "中断" in recovered["error"]
|
|
|
|
|
|
@pytest.mark.parametrize("resource", [
|
|
f"vms/{VM_ID}/actions",
|
|
f"vms/{VM_ID}/reinstall",
|
|
f"vms/{VM_ID}/rescue",
|
|
])
|
|
def test_orphaned_mutation_becomes_terminal_without_replay(boundary, resource):
|
|
client, _, store, *_ = boundary
|
|
operation_id = "orphan-" + resource.rsplit("/", 1)[-1].replace("?", "-")
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation", "resource": resource, "vm_uuid": VM_ID,
|
|
"status": "running", "result": None, "safe_to_refund": False,
|
|
"created_at": 1,
|
|
}))
|
|
recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}").json()
|
|
assert recovered["status"] == "failed"
|
|
assert "不会自动重放" in recovered["error"]
|
|
|
|
|
|
def test_orphaned_delete_remains_uncertain_without_replay(boundary):
|
|
client, _engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
operation_id = "orphan-destructive-delete"
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id,
|
|
"host_id": "hk",
|
|
"user_id": 41,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}",
|
|
"method": "DELETE",
|
|
"vm_uuid": VM_ID,
|
|
"status": "running",
|
|
"result": None,
|
|
"safe_to_refund": False,
|
|
"delete_ip_claim_keys": [],
|
|
"created_at": 1,
|
|
}))
|
|
|
|
before = list(services["hk"].calls)
|
|
first = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
|
|
).json()
|
|
second = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
|
|
).json()
|
|
|
|
assert first["status"] == second["status"] == "uncertain"
|
|
assert "原操作" in second["error"]
|
|
assert services["hk"].calls == before
|
|
|
|
|
|
def test_network_change_lost_response_keeps_both_addresses_uncertain(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True,
|
|
os_source="cloud_template",
|
|
public_ip_cidr="8.8.8.4/28",
|
|
)
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
actor = {**USER, "operation_id": "network-change-lost-response"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network"
|
|
|
|
changed = call(client, "POST", path, actor, {
|
|
"public_ip_cidr": "8.8.8.6/28",
|
|
"confirm_restart": True,
|
|
})
|
|
assert changed.status_code == 200, changed.text
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
assert saved["network_source_cidr"] == "8.8.8.4/28"
|
|
assert saved["network_target_cidr"] == "8.8.8.6/28"
|
|
|
|
# Model a lost response after libvirt may already have switched. A single
|
|
# metadata read cannot prove that XML, cloud-init and nwfilter all agree,
|
|
# so automatic reconciliation must retain both address claims.
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
components.store.set("ip_claim", f"{scope}.8.8.8.4", json.dumps({
|
|
"vm_uuid": VM_ID,
|
|
}))
|
|
engine.active_operations.clear()
|
|
reconciled = call(
|
|
client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
|
|
actor,
|
|
)
|
|
assert reconciled.status_code == 200
|
|
assert reconciled.json()["status"] == "uncertain"
|
|
assert "人工核查" in reconciled.json()["error"]
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.4") is not None
|
|
|
|
source_record = {
|
|
**saved,
|
|
"operation_id": "network-change-still-source",
|
|
"network_source_cidr": "8.8.8.6/28",
|
|
"network_target_cidr": "8.8.8.7/28",
|
|
"status": "uncertain",
|
|
}
|
|
store.set("operation", source_record["operation_id"],
|
|
json.dumps(source_record))
|
|
components.store.set("ip_claim", f"{scope}.8.8.8.7", json.dumps({
|
|
"vm_uuid": VM_ID,
|
|
}))
|
|
source = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{source_record['operation_id']}", actor,
|
|
)
|
|
assert source.json()["status"] == "uncertain"
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.7") is not None
|
|
|
|
service.vm["public_ip_cidr"] = "8.8.8.8/28"
|
|
drift_record = {
|
|
**source_record,
|
|
"operation_id": "network-change-third-address",
|
|
"status": "uncertain",
|
|
}
|
|
store.set("operation", drift_record["operation_id"],
|
|
json.dumps(drift_record))
|
|
drift = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{drift_record['operation_id']}", actor,
|
|
)
|
|
assert drift.json()["status"] == "uncertain"
|
|
assert "人工核查" in drift.json()["error"]
|
|
|
|
|
|
def test_network_change_can_move_between_cidrs_on_the_same_selector(boundary):
|
|
client, engine, store, _, services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile["default_network"] = "bridge:br0"
|
|
first_key = "3" * 32
|
|
second_key = "4" * 32
|
|
common = {
|
|
"revision": 1,
|
|
"state": "active",
|
|
"network_selector": "bridge:br0",
|
|
"dns_servers": ["1.1.1.1"],
|
|
"reserved_ips": [],
|
|
"guest_prefix_length": 29,
|
|
}
|
|
assert call(
|
|
client, "PUT",
|
|
f"/internal/v1/hosts/hk/host/network-pools/{first_key}", SUPER,
|
|
{**common, "cidr": "8.8.8.0/29", "gateway": "8.8.8.1"},
|
|
).status_code == 200
|
|
assert call(
|
|
client, "PUT",
|
|
f"/internal/v1/hosts/hk/host/network-pools/{second_key}", SUPER,
|
|
{**common, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1"},
|
|
).status_code == 200
|
|
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True,
|
|
os_source="cloud_template",
|
|
public_ip_cidr="8.8.8.3/29",
|
|
public_pool_key=first_key,
|
|
)
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
actor = {**USER, "operation_id": "network-cross-cidr-0001"}
|
|
changed = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
|
|
actor, {
|
|
"public_pool_key": second_key,
|
|
"public_ip_cidr": "9.9.9.3/29",
|
|
"confirm_restart": True,
|
|
},
|
|
)
|
|
assert changed.status_code == 200, changed.text
|
|
assert changed.json()["public_pool_key"] == second_key
|
|
operation = json.loads(store.get("operation", actor["operation_id"]))
|
|
assert operation["network_source_pool_key"] == first_key
|
|
assert operation["network_target_pool_key"] == second_key
|
|
claim = json.loads(components.store.get(
|
|
"ip_claim", f"{scope}.9.9.9.3"))
|
|
assert claim["pool_key"] == second_key
|
|
|
|
|
|
def test_deleted_vm_retries_only_public_ip_claim_cleanup(boundary):
|
|
client, engine, store, _, _services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
claim_key = f"{scope}.8.8.8.4"
|
|
components.store.set("ip_claim", claim_key, json.dumps({
|
|
"vm_uuid": VM_ID,
|
|
}))
|
|
operation = {
|
|
"operation_id": "delete-claim-cleanup",
|
|
"host_id": "hk",
|
|
"user_id": USER["user_id"],
|
|
"vm_uuid": VM_ID,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}",
|
|
"method": "DELETE",
|
|
"status": "uncertain",
|
|
"result": None,
|
|
"safe_to_refund": False,
|
|
"cleanup_pending": "public_ip_claim",
|
|
"delete_ip_claim_keys": [claim_key],
|
|
"created_at": 1,
|
|
}
|
|
store.set("operation", operation["operation_id"], json.dumps(operation))
|
|
|
|
recovered = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{operation['operation_id']}", USER,
|
|
)
|
|
assert recovered.status_code == 200
|
|
assert recovered.json()["status"] == "succeeded"
|
|
assert recovered.json()["result"]["public_ip_claims_released"] is True
|
|
assert components.store.get("ip_claim", claim_key) is None
|
|
|
|
foreign_key = f"{scope}.8.8.8.5"
|
|
components.store.set("ip_claim", foreign_key, json.dumps({
|
|
"vm_uuid": "22222222-2222-4222-8222-222222222222",
|
|
}))
|
|
foreign = {
|
|
**operation,
|
|
"operation_id": "delete-foreign-claim",
|
|
"status": "uncertain",
|
|
"cleanup_pending": "public_ip_claim",
|
|
"delete_ip_claim_keys": [foreign_key],
|
|
}
|
|
store.set("operation", foreign["operation_id"], json.dumps(foreign))
|
|
blocked = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{foreign['operation_id']}", USER,
|
|
)
|
|
assert blocked.json()["status"] == "uncertain"
|
|
assert components.store.get("ip_claim", foreign_key) is not None
|
|
|
|
|
|
def test_service_resume_replays_exact_start_target_after_lost_response(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
services["hk"].vm.update(state="shutoff", autostart=False)
|
|
actor = {**USER, "operation_id": "resume-start-target"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume"
|
|
|
|
first = call(client, "POST", path, actor, {"start_if_suspended": True})
|
|
assert first.status_code == 200, first.text
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
|
|
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(state="shutoff", autostart=False)
|
|
engine.active_operations.clear()
|
|
replayed = call(client, "GET", "/internal/v1/operations/resume-start-target", actor)
|
|
assert replayed.status_code == 200
|
|
assert replayed.json()["status"] == "succeeded"
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
|
|
|
|
|
|
def test_service_suspend_replay_preserves_original_running_state(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
actor = {**USER, "operation_id": "suspend-running-hint"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend"
|
|
assert call(client, "POST", path, actor, {}).status_code == 200
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
assert saved["service_was_running_hint"] is True
|
|
|
|
# Model a hard crash after libvirt stopped the guest but before the
|
|
# successful result was durable. Reconciliation now observes shutoff, yet
|
|
# must retain the original running state for the billing resume policy.
|
|
saved.update(status="uncertain", result=None, error="engine crashed")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(state="shutoff", autostart=False)
|
|
engine.active_operations.clear()
|
|
reconciled = call(client, "GET",
|
|
"/internal/v1/operations/suspend-running-hint", actor)
|
|
assert reconciled.status_code == 200
|
|
assert reconciled.json()["status"] == "succeeded"
|
|
assert reconciled.json()["result"]["service_was_running"] is True
|
|
|
|
|
|
def test_service_suspend_replay_without_running_hint_fails_closed(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
operation_id = "suspend-missing-running-hint"
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id,
|
|
"host_id": "hk",
|
|
"user_id": USER["user_id"],
|
|
"vm_uuid": VM_ID,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/service-suspend",
|
|
"method": "POST",
|
|
"status": "uncertain",
|
|
"actor_can_manage": True,
|
|
"actor_is_admin": False,
|
|
"created_at": 1,
|
|
}))
|
|
before_calls = list(services["hk"].calls)
|
|
|
|
response = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "failed"
|
|
assert "已取消服务暂停" in response.json()["error"]
|
|
assert services["hk"].calls == before_calls
|
|
|
|
|
|
def test_service_suspend_does_not_stop_vm_when_running_hint_cannot_persist(
|
|
boundary, monkeypatch):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
operation_id = "suspend-hint-write-failed"
|
|
original_set = store.set
|
|
|
|
def fail_operation_write(kind, key, value):
|
|
if kind == "operation" and key == operation_id:
|
|
raise RuntimeError("database unavailable")
|
|
return original_set(kind, key, value)
|
|
|
|
monkeypatch.setattr(store, "set", fail_operation_write)
|
|
failed = call(
|
|
client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend",
|
|
{**USER, "operation_id": operation_id}, {})
|
|
|
|
assert failed.status_code == 503
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert not any(
|
|
item[0] == "service_suspend" for item in services["hk"].calls)
|
|
|
|
monkeypatch.setattr(store, "set", original_set)
|
|
recovered = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
|
|
assert recovered.status_code == 200
|
|
assert recovered.json()["status"] == "failed"
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert not any(
|
|
item[0] == "service_suspend" for item in services["hk"].calls)
|
|
|
|
def test_metrics_query_uses_log_store_and_owner_permission(boundary):
|
|
client, engine, _, logs, _ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
component.state.users.assign_vm(scope, VM_ID, 41)
|
|
response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d")
|
|
assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20
|
|
|
|
with patch("virsh_panel.app.time.time", return_value=3_000_000):
|
|
traffic = call(
|
|
client, "GET",
|
|
f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily?days=2",
|
|
)
|
|
assert traffic.status_code == 200
|
|
assert traffic.json()["points"][-1] == {
|
|
"date": "1970-02-04", "day_start": 2_937_600,
|
|
"download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120,
|
|
}
|
|
assert logs.daily_traffic_calls == [(scope, VM_ID, {
|
|
"host_id": "hk", "user_id": 41,
|
|
"since": 2_851_200, "until": 3_000_000, "limit": 2,
|
|
})]
|
|
denied = call(
|
|
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily",
|
|
{**USER, "user_id": 42},
|
|
)
|
|
assert denied.status_code == 404
|
|
assert len(logs.daily_traffic_calls) == 1
|
|
|
|
|
|
def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
|
|
assert issued.status_code == 200, issued.text
|
|
assert issued.json()["websocket_path"] == "/engine-console"
|
|
ticket = issued.json()["ticket"]
|
|
key = hashlib.sha256(ticket.encode()).hexdigest()
|
|
record = json.loads(store.get("console_ticket", key))
|
|
assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session"
|
|
assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket",
|
|
{**USER, "user_id": 42}, {}).status_code == 404
|
|
from starlette.websockets import WebSocketDisconnect
|
|
with pytest.raises(WebSocketDisconnect):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"},
|
|
subprotocols=["cloud-ticket." + ticket]):
|
|
pass
|
|
assert store.get("console_ticket", key) is not None
|
|
|
|
|
|
def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
checks = 0
|
|
async def active(*args):
|
|
nonlocal checks
|
|
checks += 1
|
|
return checks <= 2 # Ticket issuance + handshake; then PHP revokes.
|
|
engine.introspection = active
|
|
engine.console_check_interval = 0.01
|
|
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
|
|
ticket = issued.json()["ticket"]
|
|
protocol = "cloud-ticket." + ticket
|
|
async def relay(socket, **kwargs):
|
|
await socket.send_bytes(b"vnc")
|
|
await asyncio.Event().wait()
|
|
from starlette.websockets import WebSocketDisconnect
|
|
with patch("virsh_panel.console.relay_console", relay):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console",
|
|
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket:
|
|
assert socket.accepted_subprotocol == protocol
|
|
assert socket.receive_bytes() == b"vnc"
|
|
with pytest.raises(WebSocketDisconnect):
|
|
socket.receive_bytes()
|
|
key = hashlib.sha256(ticket.encode()).hexdigest()
|
|
assert store.get("console_ticket", key) is None
|
|
assert store.get("ticket_used", key) == "1"
|
|
with pytest.raises(WebSocketDisconnect):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console",
|
|
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]):
|
|
pass
|
|
|
|
|
|
def test_streamed_iso_digest_must_match_before_import_is_queued(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/rescue-templates?name=rescue"
|
|
signed_body = b"A" * (1024 * 1024)
|
|
wrong_body = b"B" * len(signed_body)
|
|
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
|
|
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest()
|
|
signed["Content-Type"] = "application/octet-stream"
|
|
response = client.post(path, content=wrong_body, headers=signed)
|
|
assert response.status_code == 401, response.text
|
|
assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls)
|
|
assert engine.registry.components["hk"].store.list("rescue_import_job") == {}
|
|
|
|
|
|
def cloud_upload_fixture(size=1024 * 1024 + 257):
|
|
header = bytearray(104)
|
|
header[:4] = b"QFI\xfb"
|
|
header[4:8] = struct.pack(">I", 3)
|
|
header[20:24] = struct.pack(">I", 16)
|
|
header[24:32] = struct.pack(">Q", 20 * 1024**3)
|
|
header[100:104] = struct.pack(">I", 104)
|
|
return bytes(header) + b"\0" * (size - len(header))
|
|
|
|
|
|
def wait_for_cloud_upload_job(client, job_id):
|
|
path = f"/internal/v1/vm-templates/import-jobs/{job_id}"
|
|
for _ in range(200):
|
|
response = call(client, "GET", path, SUPER)
|
|
assert response.status_code == 200, response.text
|
|
job = response.json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
return job
|
|
time.sleep(0.01)
|
|
raise AssertionError("uploaded cloud template import did not finish")
|
|
|
|
|
|
def cloud_engine_scope(method, target, body, actor, *, headers=None):
|
|
path, separator, query = target.partition("?")
|
|
request_headers = signed_headers(SECRET, method, target, body, actor)
|
|
request_headers.update(headers or {})
|
|
return {
|
|
"type": "http",
|
|
"asgi": {"version": "3.0"},
|
|
"http_version": "1.1",
|
|
"method": method,
|
|
"scheme": "http",
|
|
"path": path,
|
|
"raw_path": path.encode("ascii"),
|
|
"query_string": query.encode("ascii") if separator else b"",
|
|
"headers": [
|
|
(name.lower().encode("ascii"), value.encode("latin-1"))
|
|
for name, value in request_headers.items()
|
|
],
|
|
"client": ("testclient", 50000),
|
|
"server": ("127.0.0.1", 80),
|
|
}
|
|
|
|
|
|
async def cloud_engine_request(engine, method, target, body, actor, *,
|
|
headers=None, receive=None):
|
|
scope = cloud_engine_scope(
|
|
method, target, body, actor, headers=headers)
|
|
delivered = False
|
|
|
|
async def receive_body():
|
|
nonlocal delivered
|
|
if delivered:
|
|
return {"type": "http.disconnect"}
|
|
delivered = True
|
|
return {"type": "http.request", "body": body, "more_body": False}
|
|
|
|
messages = []
|
|
|
|
async def send(message):
|
|
messages.append(message)
|
|
|
|
await engine(scope, receive or receive_body, send)
|
|
status = next(message["status"] for message in messages
|
|
if message["type"] == "http.response.start")
|
|
raw = b"".join(message.get("body", b"") for message in messages
|
|
if message["type"] == "http.response.body")
|
|
return status, json.loads(raw)
|
|
|
|
|
|
def test_global_cloud_template_upload_streams_past_json_limit(boundary):
|
|
client, engine, *_ = boundary
|
|
query = urlencode({
|
|
"name": "Debian 12 fixed",
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
path = "/internal/v1/vm-templates/upload?" + query
|
|
payload = cloud_upload_fixture()
|
|
with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"):
|
|
response = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert response.status_code == 202, response.text
|
|
job = wait_for_cloud_upload_job(client, response.json()["job_id"])
|
|
assert job["status"] == "completed", job
|
|
assert job["template"]["name"] == "Debian 12 fixed"
|
|
assert job["template"]["os_family"] == "Debian"
|
|
assert job["template"]["os_version"] == "12"
|
|
assert job["template"]["virtual_size"] == 20 * 1024**3
|
|
with engine.registry.cloud_templates.source(
|
|
job["template"]["id"]) as (_record, source):
|
|
assert source.stat().st_size == len(payload)
|
|
assert hashlib.sha256(source.read_bytes()).digest() == hashlib.sha256(
|
|
payload).digest()
|
|
|
|
|
|
def test_global_cloud_template_upload_digest_must_match_before_queue(boundary):
|
|
client, engine, *_ = boundary
|
|
path = ("/internal/v1/vm-templates/upload?"
|
|
"name=Debian+12&os_family=Debian&os_version=12")
|
|
signed_body = cloud_upload_fixture()
|
|
wrong_body = signed_body[:-1] + b"x"
|
|
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
|
|
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(
|
|
signed_body).hexdigest()
|
|
signed["Content-Type"] = "application/octet-stream"
|
|
response = client.post(path, content=wrong_body, headers=signed)
|
|
assert response.status_code == 401, response.text
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
root = engine.registry.cloud_templates.files.directory()
|
|
assert not list(root.glob(".cloud-image-upload-*.qcow2"))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("family", "version"),
|
|
[
|
|
("debian", "12"),
|
|
("Debian", "14"),
|
|
("CentOS", "10"),
|
|
("Ubuntu", "24"),
|
|
],
|
|
)
|
|
def test_global_cloud_template_upload_rejects_noncanonical_metadata(
|
|
boundary, family, version):
|
|
client, engine, *_ = boundary
|
|
path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": "Invalid metadata",
|
|
"os_family": family,
|
|
"os_version": version,
|
|
})
|
|
payload = cloud_upload_fixture()
|
|
response = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert response.status_code == 422, response.text
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
|
|
|
|
def test_global_cloud_template_upload_requires_super_admin_and_octet_stream(
|
|
boundary):
|
|
client, engine, *_ = boundary
|
|
path = ("/internal/v1/vm-templates/upload?"
|
|
"name=Debian+12&os_family=Debian&os_version=12")
|
|
payload = cloud_upload_fixture()
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
forbidden = call(client, "POST", path, ADMIN, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": digest,
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert forbidden.status_code == 403, forbidden.text
|
|
wrong_type = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": digest,
|
|
"Content-Type": "multipart/form-data",
|
|
})
|
|
assert wrong_type.status_code == 415, wrong_type.text
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
|
|
|
|
def test_global_cloud_template_upload_enforces_minimum_size(boundary):
|
|
client, engine, *_ = boundary
|
|
path = ("/internal/v1/vm-templates/upload?"
|
|
"name=Too+Small&os_family=Debian&os_version=12")
|
|
payload = cloud_upload_fixture(104)
|
|
response = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert response.status_code == 422, response.text
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
|
|
|
|
def test_global_cloud_template_upload_timeout_releases_reservation(
|
|
boundary, monkeypatch):
|
|
_client, engine, *_ = boundary
|
|
query = b"name=Timeout&os_family=Debian&os_version=12"
|
|
scope = {
|
|
"query_string": query,
|
|
"headers": [
|
|
(b"content-type", b"application/octet-stream"),
|
|
(b"content-length", str(1024 * 1024).encode()),
|
|
],
|
|
"engine_upload_validation": {"complete": False, "valid": False},
|
|
"engine_upload_digest": "0" * 64,
|
|
}
|
|
|
|
async def stalled_receive():
|
|
await asyncio.sleep(1)
|
|
return {"type": "http.request", "body": b"", "more_body": False}
|
|
|
|
async def send(_message):
|
|
raise AssertionError("timeout must not start a response")
|
|
|
|
monkeypatch.setattr("virsh_panel.internal.CLOUD_UPLOAD_IDLE_SECONDS", 0.01)
|
|
with pytest.raises(TemplateError, match="上传超时") as timeout:
|
|
asyncio.run(engine._upload_cloud_template(scope, stalled_receive, send))
|
|
assert timeout.value.status == 408
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
assert engine.registry.cloud_templates.upload_reservations == {}
|
|
root = engine.registry.cloud_templates.files.directory()
|
|
assert not list(root.glob(".cloud-image-upload-*.qcow2"))
|
|
|
|
|
|
def test_stalled_cloud_upload_does_not_block_other_engine_writes(
|
|
boundary, monkeypatch):
|
|
_client, engine, *_ = boundary
|
|
payload = cloud_upload_fixture()
|
|
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": "Stalled upload",
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
upload_started = asyncio.Event()
|
|
release_upload = asyncio.Event()
|
|
|
|
class FastRemoteHostManager:
|
|
def __init__(self, *_args, **_kwargs):
|
|
pass
|
|
|
|
@staticmethod
|
|
def scan(host, port):
|
|
return {"host": host, "port": port, "reachable": True}
|
|
|
|
@staticmethod
|
|
def close():
|
|
pass
|
|
|
|
monkeypatch.setattr(
|
|
"virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager)
|
|
|
|
async def stalled_receive():
|
|
upload_started.set()
|
|
await release_upload.wait()
|
|
return {"type": "http.disconnect"}
|
|
|
|
async def scenario():
|
|
upload = asyncio.create_task(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
upload_path,
|
|
payload,
|
|
SUPER,
|
|
headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
"Content-Length": str(len(payload)),
|
|
},
|
|
receive=stalled_receive,
|
|
))
|
|
await asyncio.wait_for(upload_started.wait(), timeout=1)
|
|
assert engine.registry.cloud_templates.pending
|
|
|
|
scan_body = json.dumps({
|
|
"host": "host.example.test", "port": 22,
|
|
}, separators=(",", ":")).encode()
|
|
status, result = await asyncio.wait_for(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
"/internal/v1/hosts/scan",
|
|
scan_body,
|
|
SUPER,
|
|
headers={"Content-Type": "application/json"},
|
|
), timeout=0.5)
|
|
assert status == 200
|
|
assert result["host"] == "host.example.test"
|
|
|
|
release_upload.set()
|
|
upload_status, _result = await asyncio.wait_for(upload, timeout=1)
|
|
assert upload_status == 400
|
|
|
|
asyncio.run(scenario())
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
assert engine.registry.cloud_templates.upload_reservations == {}
|
|
|
|
|
|
def test_cloud_upload_fsync_does_not_block_event_loop_or_engine_writes(
|
|
boundary, monkeypatch):
|
|
_client, engine, *_ = boundary
|
|
payload = cloud_upload_fixture()
|
|
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": "Slow fsync upload",
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
fsync_started = threading.Event()
|
|
release_fsync = threading.Event()
|
|
original_fsync = os.fsync
|
|
|
|
def slow_fsync(descriptor):
|
|
fsync_started.set()
|
|
assert release_fsync.wait(timeout=2), "test did not release fsync"
|
|
return original_fsync(descriptor)
|
|
|
|
class FastRemoteHostManager:
|
|
def __init__(self, *_args, **_kwargs):
|
|
pass
|
|
|
|
@staticmethod
|
|
def scan(host, port):
|
|
return {"host": host, "port": port, "reachable": True}
|
|
|
|
@staticmethod
|
|
def close():
|
|
pass
|
|
|
|
monkeypatch.setattr("virsh_panel.internal.os.fsync", slow_fsync)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.cloud_templates._verify_qcow2_with_qemu",
|
|
lambda *_args: None)
|
|
|
|
async def scenario():
|
|
safety_release = threading.Timer(1.5, release_fsync.set)
|
|
safety_release.start()
|
|
started_at = time.monotonic()
|
|
try:
|
|
upload = asyncio.create_task(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
upload_path,
|
|
payload,
|
|
SUPER,
|
|
headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
"Content-Length": str(len(payload)),
|
|
},
|
|
))
|
|
while not fsync_started.is_set():
|
|
await asyncio.sleep(0.001)
|
|
|
|
scan_body = json.dumps({
|
|
"host": "host.example.test", "port": 22,
|
|
}, separators=(",", ":")).encode()
|
|
status, _result = await asyncio.wait_for(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
"/internal/v1/hosts/scan",
|
|
scan_body,
|
|
SUPER,
|
|
headers={"Content-Type": "application/json"},
|
|
), timeout=0.5)
|
|
assert status == 200
|
|
assert time.monotonic() - started_at < 0.5
|
|
assert not release_fsync.is_set()
|
|
release_fsync.set()
|
|
upload_status, _result = await asyncio.wait_for(upload, timeout=2)
|
|
assert upload_status == 202
|
|
finally:
|
|
release_fsync.set()
|
|
safety_release.cancel()
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_cloud_upload_rechecks_update_state_before_queue(
|
|
boundary, monkeypatch):
|
|
_client, engine, *_ = boundary
|
|
updater = FakeUpdater()
|
|
engine.updater = updater
|
|
payload = cloud_upload_fixture()
|
|
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": "Update race upload",
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
upload_started = asyncio.Event()
|
|
release_upload = asyncio.Event()
|
|
delivered = False
|
|
monkeypatch.setattr(
|
|
"virsh_panel.cloud_templates._verify_qcow2_with_qemu",
|
|
lambda *_args: None)
|
|
|
|
async def delayed_receive():
|
|
nonlocal delivered
|
|
if delivered:
|
|
return {"type": "http.disconnect"}
|
|
delivered = True
|
|
upload_started.set()
|
|
await release_upload.wait()
|
|
return {"type": "http.request", "body": payload, "more_body": False}
|
|
|
|
async def scenario():
|
|
upload = asyncio.create_task(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
upload_path,
|
|
payload,
|
|
SUPER,
|
|
headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
"Content-Length": str(len(payload)),
|
|
},
|
|
receive=delayed_receive,
|
|
))
|
|
await asyncio.wait_for(upload_started.wait(), timeout=1)
|
|
assert engine.registry.cloud_templates.pending
|
|
updater.active = {
|
|
"job_id": "update-job-1234567890", "status": "running",
|
|
}
|
|
release_upload.set()
|
|
status, result = await asyncio.wait_for(upload, timeout=2)
|
|
assert status == 409
|
|
assert result["error"] == "面板正在更新,暂时不能执行虚拟化写操作"
|
|
|
|
try:
|
|
asyncio.run(scenario())
|
|
finally:
|
|
updater.active = None
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
assert engine.registry.cloud_templates.upload_reservations == {}
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
|
|
|
|
def test_upload_same_os_version_keeps_existing_template(boundary):
|
|
client, engine, *_ = boundary
|
|
|
|
def upload(name, payload):
|
|
path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": name,
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
response = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert response.status_code == 202, response.text
|
|
job = wait_for_cloud_upload_job(client, response.json()["job_id"])
|
|
assert job["status"] == "completed", job
|
|
return job["template"]
|
|
|
|
official_payload = cloud_upload_fixture()[:-1] + b"o"
|
|
fixed_payload = cloud_upload_fixture()[:-1] + b"f"
|
|
with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"):
|
|
official = upload("Debian 12 official", official_payload)
|
|
fixed = upload("Debian 12 fixed", fixed_payload)
|
|
|
|
assert official["id"] != fixed["id"]
|
|
templates = call(
|
|
client, "GET", "/internal/v1/vm-templates", SUPER).json()["templates"]
|
|
assert {row["id"] for row in templates} == {official["id"], fixed["id"]}
|
|
with engine.registry.cloud_templates.source(
|
|
official["id"]) as (_record, source):
|
|
assert hashlib.sha256(source.read_bytes()).hexdigest() == official["sha256"]
|
|
|
|
|
|
def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables():
|
|
connection = MagicMock()
|
|
cursor = connection.cursor.return_value.__enter__.return_value
|
|
config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test",
|
|
"ssl_ca": "/private/test/ca.pem"}
|
|
with patch("virsh_panel.database.pymysql.connect", return_value=connection):
|
|
main = MainStore(config, b"m" * 32)
|
|
statements = [call.args[0] for call in cursor.execute.call_args_list]
|
|
assert any("engine_records" in statement for statement in statements)
|
|
assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements)
|
|
assert not any("vm_metric_samples" in statement for statement in statements)
|
|
assert main._config["ssl_verify_identity"] is True
|
|
cursor.reset_mock()
|
|
LogStore({**config, "database": "logs"}, b"m" * 32)
|
|
statements = [call.args[0] for call in cursor.execute.call_args_list]
|
|
assert "engine_vm_metric_samples" in statements[0]
|
|
assert "engine_vm_traffic_totals" in statements[1]
|
|
assert "engine_vm_traffic_checkpoints" in statements[2]
|
|
assert all("ALTER TABLE engine_vm_metric_samples" in statement
|
|
for statement in statements[3:])
|
|
assert LogStore.TRAFFIC_TABLE == "engine_vm_traffic_totals"
|
|
assert LogStore.TRAFFIC_CHECKPOINT_TABLE == "engine_vm_traffic_checkpoints"
|