You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_engine_boundary.py

6359 lines
258 KiB
Python

"""Private engine tests: no PHP cookies, no real DB and no real hypervisor."""
from __future__ import annotations
import base64
import asyncio
import hashlib
import json
import os
import struct
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager, contextmanager
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from urllib.parse import urlencode
import pytest
from fastapi.testclient import TestClient
from test_app import (CLOUD_CREATE_BODY, CREATE_BODY, OFFICIAL_CLOUD_URL,
FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID)
from virsh_panel.app import create_app
from virsh_panel.cloud_templates import (TemplateError, template_id_for,
template_scope, template_volume_name)
from virsh_panel.config import Settings
from virsh_panel.coordination import mark_remote_write_started
from virsh_panel.database import DatabaseError, DatabaseTemplateStore
from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore,
LogStore, MainStore, TRUSTED_ACTOR)
from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor,
signed_headers)
from virsh_panel.fabric import FabricError
from virsh_panel.internal import EngineApplication, HostRegistry
from virsh_panel.libvirt_service import (
FIREWALL_FILTER_COMPILER_VERSION, VMError, build_firewall_filter_xml,
firewall_policy_digest,
snapshot_volume_name,
)
from virsh_panel.metrics import host_scope
from virsh_panel.shutdown import ManagedThreadPoolExecutor
from virsh_panel.user_isos import user_iso_scope
from virsh_panel.vpc import parse_vpc_delete_plan
SECRET = b"s" * 32
USER = {"user_id": 41, "is_admin": False, "is_super_admin": False,
"can_manage": True, "session_id": "test-session"}
SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True}
ADMIN = {**SUPER, "is_super_admin": False}
class MemoryStore:
def __init__(self):
self.records = {}
self.nonces = set()
self.lock = threading.RLock()
self.daily_traffic_calls = []
def get(self, kind, key):
with self.lock:
return self.records.get((kind, key))
def set(self, kind, key, value):
with self.lock:
self.records[kind, key] = value
def insert(self, kind, key, value):
with self.lock:
if (kind, key) in self.records:
return False
self.records[kind, key] = value
return True
def compare_and_set(self, kind, key, expected, value):
with self.lock:
if self.records.get((kind, key)) != expected:
return False
self.records[kind, key] = value
return True
def delete(self, kind, key):
with self.lock:
self.records.pop((kind, key), None)
def list(self, kind):
with self.lock:
return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind}
def consume_nonce(self, nonce, expires_at, now):
with self.lock:
if nonce in self.nonces:
return False
self.nonces.add(nonce)
return True
def query_metric_samples(self, *args, **kwargs):
return [{"timestamp": 123, "cpu_percent": 20.0}]
def query_daily_traffic(self, scope, vm_id, **kwargs):
self.daily_traffic_calls.append((scope, vm_id, kwargs))
day_start = kwargs["until"] - kwargs["until"] % 86400
return [{"day_start": day_start, "download_bytes": 4096,
"upload_bytes": 1024, "total_bytes": 5120}]
def get_value(self, key):
return self.get("value", key)
def set_value(self, key, value):
return self.set("value", key, value)
def delete_value(self, key):
return self.delete("value", key)
def get_secret(self, key):
return self.get("secret", key)
def set_secret(self, key, value):
return self.set("secret", key, value)
def delete_secret(self, key):
return self.delete("secret", key)
class FakeUpdater:
def __init__(self):
self.token = None
self.active = None
self.jobs = {}
def settings(self):
return {
"repository": "https://g.luoli.live/root/virt.git",
"branch": "venix",
"token_configured": self.token is not None,
"checkout_ready": True,
"current_commit": "a" * 40,
"last_job_id": None,
"active_job_id": self.active["job_id"] if self.active else None,
}
def set_token(self, token):
self.token = token
def clear_token(self):
self.token = None
def active_job(self):
return self.active
def check(self):
return {"repository": "https://g.luoli.live/root/virt.git", "branch": "venix",
"current_commit": "a" * 40, "remote_commit": "b" * 40,
"status": "update_available"}
def preflight(self, *, expected_current_commit=None, expected_remote_commit=None):
if self.token is None:
from virsh_panel.updater import UpdateError
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
if expected_current_commit != "a" * 40 or expected_remote_commit != "b" * 40:
from virsh_panel.updater import UpdateError
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
def create_job(self, *, expected_current_commit=None, expected_remote_commit=None,
job_id=None):
from virsh_panel.updater import UpdateError, UpdateNotQueued
try:
self.preflight(expected_current_commit=expected_current_commit,
expected_remote_commit=expected_remote_commit)
except UpdateError as exc:
raise UpdateNotQueued(str(exc), exc.status) from exc
job = {"job_id": job_id or "update-job-1234567890", "status": "queued",
"phase": "等待独立更新服务启动", "created_at": 1,
"private": self.token}
self.jobs[job["job_id"]] = job
self.active = job
return job.copy()
def get_job(self, job_id):
return self.jobs.get(job_id)
def fail_job(self, job_id, message):
self.jobs[job_id].update(status="failed", error=message)
self.active = None
return self.jobs[job_id].copy()
def test_engine_application_wires_one_catalog_mutation_lock(tmp_path):
engine = EngineApplication(
{
"service_secret": base64.b64encode(SECRET).decode(),
"data_dir": str(tmp_path),
},
MemoryStore(),
MemoryStore(),
registry=SimpleNamespace(),
updater=FakeUpdater(),
)
try:
application_lock = engine._network_catalog_mutation_lock()
assert isinstance(application_lock, asyncio.Lock)
assert engine.fabric.catalog_mutation_lock() is application_lock
assert engine.vpc._catalog_mutation_lock() is application_lock
finally:
engine.fabric.close()
class Service(FakeVMService):
def __init__(self):
super().__init__()
self.create_failure = False
self.absent = False
self.absent_checks = []
self.cloud_template_delete_error = None
self.vm.update(managed=True, os_source="iso")
def create_vm(self, spec):
self.calls.append(("create_vm", spec))
# This adapter skips the RPC transport; model its write boundary so
# post-dispatch failures remain distinct from no-write preflights.
mark_remote_write_started()
if self.create_failure:
from virsh_panel.libvirt_service import VMError
raise VMError("宿主机失联,回滚状态不明", 502)
self.vm.update(uuid=spec["vm_uuid"], name=spec["name"])
return self.vm.copy()
def delete_vm(self, vm_id, delete_disks):
mark_remote_write_started()
return super().delete_vm(vm_id, delete_disks)
def provisioning_absent(self, vm_uuid, name, *, uuid_resources=False,
legacy_name_collision=False):
self.absent_checks.append(
(vm_uuid, name, uuid_resources, legacy_name_collision))
return self.absent
def change_cloud_vm_network(self, vm_id, network, *, confirm_restart=False):
self.calls.append((
"change_cloud_vm_network", vm_id,
network["public_ip_cidr"], confirm_restart,
))
if vm_id != self.vm["uuid"]:
raise VMError("虚拟机不存在", 404)
self.vm["public_ip_cidr"] = network["public_ip_cidr"]
self.vm["public_pool_key"] = network.get("public_pool_key")
return {
"vm": self.vm.copy(),
"public_pool_key": network.get("public_pool_key"),
"public_ip_cidr": network["public_ip_cidr"],
"network_changed": True,
}
def delete_cloud_template_replicas(self, source_volume_name):
self.calls.append(("delete_cloud_template_replicas", source_volume_name))
if self.cloud_template_delete_error is not None:
raise self.cloud_template_delete_error
return {"source_volume_name": source_volume_name, "deleted": True}
@pytest.fixture
def boundary(tmp_path):
store = MemoryStore()
logs = MemoryStore()
config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}
services = {}
def factory(host_id, user_isos, cloud_templates):
(tmp_path / host_id).mkdir(mode=0o700, exist_ok=True)
scoped = HostStore(store, host_id)
manager = FakeRemoteManager()
manager.host_id = host_id
manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22,
"fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images",
"iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default",
"libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config",
"ssh_known_hosts_file": "/fake/known_hosts"}
service = Service()
services[host_id] = service
components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager,
DatabaseTemplateStore(scoped), scoped, logs, user_isos,
cloud_templates)
app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components,
service_factory=lambda profile: service)
return components, app
registry = HostRegistry(config, store, logs, factory=factory)
async def active(*args):
return True
engine = EngineApplication(config, store, logs, registry=registry, introspection=active)
engine._service = lambda profile: services[profile["host_id"]]
with TestClient(engine, base_url="http://127.0.0.1") as client:
yield client, engine, store, logs, services
def call(client, method, path, actor=USER, value=None, raw=None, headers=None):
body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode()
signed = signed_headers(SECRET, method, path, body, actor)
if value is not None:
signed["Content-Type"] = "application/json"
signed.update(headers or {})
return client.request(method, path, content=body, headers=signed)
FIREWALL_RULES = [
{
"protocol": "icmp",
"source_cidr": "0.0.0.0/0",
"port_from": None,
"port_to": None,
"description": "Ping",
},
{
"protocol": "tcp",
"source_cidr": "203.0.113.55/32",
"port_from": 22,
"port_to": 22,
"description": "SSH",
},
]
def firewall_body(*, revision=1, enabled=True, rules=None):
rules = FIREWALL_RULES if rules is None else rules
return {
"enabled": enabled,
"revision": revision,
"policy_digest": firewall_policy_digest(enabled, rules),
"rules": rules,
}
def test_legacy_firewall_compiler_digest_fixture_is_stable():
body = firewall_body()
assert build_firewall_filter_xml(
VM_ID, body["enabled"], body["rules"], compiler_version=1,
)[3] == "42e1cd7c3e410dc6a0d80740753b19b6b4a67ca00c4b8265b2f0bb201f9af793"
def test_host_capacity_endpoint_is_removed(boundary):
client, _engine, _store, _logs, _services = boundary
assert call(client, "GET", "/internal/v1/hosts/hk/host/capacity", USER).status_code == 404
assert call(client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER).status_code == 404
def import_user_iso_for_test(client, tmp_path, name, *, digest="e" * 64):
download = tmp_path / (hashlib.sha256(name.encode()).hexdigest() + ".iso")
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, digest)):
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
"url": "https://downloads.example.test/os.iso",
"name": name,
"sha256": "",
})
assert queued.status_code == 202, queued.text
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
return job["iso"]["id"]
def test_host_network_options_endpoint_is_removed(boundary):
client, _engine, _store, _logs, _services = boundary
assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options", USER).status_code == 404
assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER).status_code == 404
def test_fabric_is_engine_global_and_super_admin_write_only(boundary):
client, engine, store, _logs, _services = boundary
assert call(client, "GET", "/internal/v1/fabric", USER).status_code == 403
assert call(client, "GET", "/internal/v1/fabric", ADMIN).json() == []
assert call(client, "POST", "/internal/v1/fabric/apply", USER, {}).status_code == 403
assert call(client, "POST", "/internal/v1/fabric/apply", ADMIN, {}).status_code == 403
invalid = call(client, "POST", "/internal/v1/fabric/apply", SUPER, {})
assert invalid.status_code == 400
assert invalid.json()["error_code"] == "fabric_invalid"
assert "failure_phase" not in invalid.json()
async def failed_identity(_value):
raise FabricError(
"宿主机 Fabric 命令失败", 502,
"fabric_remote_command_failed", "identity",
)
with patch.object(engine.fabric, "enqueue_apply", failed_identity):
identity_failed = call(
client, "POST", "/internal/v1/fabric/apply", SUPER, {},
)
assert identity_failed.status_code == 502
assert identity_failed.json() == {
"error": "宿主机 Fabric 命令失败",
"error_code": "fabric_remote_command_failed",
"failure_phase": "identity",
}
receipt = {
"fabric_id": "fabric-hk",
"operation": {
"id": "a" * 32,
"kind": "apply",
"status": "queued",
"started_at": None,
"updated_at": 1,
"finished_at": None,
},
}
async def accepted_apply(_value):
return receipt
with patch.object(engine.fabric, "enqueue_apply", accepted_apply):
accepted = call(
client, "POST", "/internal/v1/fabric/apply", SUPER, {},
)
assert accepted.status_code == 202
assert accepted.json() == receipt
reconcile_receipt = {
**receipt,
"operation": {**receipt["operation"], "kind": "reconcile"},
}
async def accepted_reconcile(fabric_id):
assert fabric_id == "fabric-hk"
return reconcile_receipt
with patch.object(
engine.fabric, "enqueue_reconcile", accepted_reconcile):
accepted = call(
client, "POST",
"/internal/v1/fabric/fabric-hk/reconcile", SUPER, {},
)
assert accepted.status_code == 202
assert accepted.json() == reconcile_receipt
assert call(
client, "POST", "/internal/v1/fabric/probe", USER,
{"host_ids": ["host-a"]},
).status_code == 403
assert call(
client, "POST", "/internal/v1/fabric/probe", ADMIN,
{"host_ids": ["host-a"]},
).status_code == 403
assert call(
client, "POST", "/internal/v1/fabric/probe",
{**SUPER, "can_manage": False}, {"host_ids": ["host-a"]},
).status_code == 403
malformed = call(
client, "POST", "/internal/v1/fabric/probe", SUPER,
raw=b'{"host_ids":',
)
assert malformed.status_code == 422
assert malformed.json()["error_code"] == "fabric_probe_invalid"
extra = call(
client, "POST", "/internal/v1/fabric/probe", SUPER,
{"host_ids": ["host-a"], "unexpected": True},
)
assert extra.status_code == 422
assert extra.json()["error_code"] == "fabric_probe_invalid"
store.set(
"engine_host", "host-a", json.dumps({"host_id": "host-a"}),
)
unsupported = call(
client, "POST", "/internal/v1/fabric/probe", SUPER,
{"host_ids": ["host-a"]},
)
assert unsupported.status_code == 200
assert unsupported.json()["hosts"][0]["error_code"] == (
"fabric_host_capability_probe_unsupported")
manager = engine.registry.components["host-a"].manager
manager.profile.update({
"host_id": "host-a", "host": "203.0.113.80", "region": "hk",
})
capabilities = {
"vpc_ready": True,
"reasons": [],
"os": {"id": "debian", "version_id": "13"},
"network": {
"network_manager": "ifupdown",
"network_manager_expected": "ifupdown",
"network_manager_supported": True,
"firewall_manager": "nftables",
"firewall_mode": "native_nftables",
"ip_forward": 1,
"rp_filter": 2,
"rp_filter_default": 2,
"rp_filter_default_effective": 2,
"rp_filter_interfaces": {"br0": 2},
"local_ipv4_addresses": ["203.0.113.80"],
"local_ipv4_interfaces": {"203.0.113.80": ["br0"]},
},
"wireguard": {
"deployment_mode": "ifupdown", "ready": True,
"owned_by_panel": False,
},
"nftables": {
"ready": True, "service_active": True,
"service_persistent": True,
},
"network_roles": {
"public": {"selector": "default", "verified": True},
},
}
manager.reprobe_current_snapshot = lambda: (manager.profile, capabilities)
probed = call(
client, "POST", "/internal/v1/fabric/probe", SUPER,
{"host_ids": ["host-a"]},
)
assert probed.status_code == 200
readiness = probed.json()["hosts"][0]
assert readiness["configured"] is True
assert readiness["reachable"] is True
assert readiness["vpc_ready"] is True
assert readiness["endpoint_ip"] == "203.0.113.80"
assert readiness["capabilities"]["network_manager"] == "ifupdown"
assert call(
client, "GET", "/internal/v1/hosts/hk/fabric", SUPER
).status_code == 404
def test_vpc_control_plane_is_global_and_does_not_use_host_network_change(boundary):
client, engine, store, _logs, _services = boundary
assert engine.vpc_reconcile_task is not None
assert not engine.vpc_reconcile_task.done()
assert call(client, "GET", "/internal/v1/vpcs", USER).status_code == 403
assert call(client, "GET", "/internal/v1/vpcs", ADMIN).json() == []
assert call(client, "POST", "/internal/v1/vpcs/apply", USER, {}).status_code == 400
assert call(client, "POST", "/internal/v1/vpcs/apply", ADMIN, {}).status_code == 400
assert call(client, "POST", "/internal/v1/vpcs/apply",
{**USER, "can_manage": False}, {}).status_code == 400
invalid = call(client, "POST", "/internal/v1/vpcs/apply", SUPER, {})
assert invalid.status_code == 400
assert invalid.json()["error_code"] == "vpc_operation_required"
valid = {
"schema_version": 1,
"vpc_id": "vpc-boundary-1",
"region": "hk",
"fabric_id": "fabric-boundary",
"fabric_revision": 1,
"cidr": "10.30.0.0/24",
"desired_revision": 1,
"operation_id": "vpc-boundary-op1",
"subnets": [{"subnet_id": "subnet-a", "cidr": "10.30.0.0/25"}],
"hosts": [{"host_id": "host-a", "members": ["member-a"]}],
"members": [{
"member_id": "member-a",
"instance_uuid": "11111111-1111-4111-8111-111111111111",
"host_id": "host-a",
"private_ip_cidr": "10.30.0.10/32",
"subnet_id": "subnet-a",
}],
"routes": [],
"acls": [],
}
applied = call(client, "POST", "/internal/v1/vpcs/apply",
{**USER, "can_manage": False}, valid)
assert applied.status_code == 200, applied.text
assert applied.json()["vpc_id"] == "vpc-boundary-1"
assert applied.json()["next_stage_allowed"] is False
assert "fabric_gate" in applied.json()["observed"]
# The independent PHP VPC worker has a tenant actor and no browser
# session. It must recover the stored receipt before replaying an apply.
worker = {**USER, "session_id": ""}
recovered = call(client, "GET", "/internal/v1/vpcs/vpc-boundary-1", worker)
assert recovered.status_code == 200, recovered.text
for field in ("vpc_id", "operation_id", "desired_revision", "desired_digest", "observed"):
assert recovered.json()[field] == applied.json()[field]
assert call(client, "GET", "/internal/v1/vpcs/missing-vpc", worker).status_code == 404
# A single signed PHP-authorized target does not grant inventory or
# infrastructure administration access to the tenant worker.
for path in ("/internal/v1/vpcs", "/internal/v1/fabric",
"/internal/v1/hosts/hk/host",
"/internal/v1/vm-templates/template-one/startup-script"):
assert call(client, "GET", path, worker).status_code == 403
assert client.get("/internal/v1/vpcs/vpc-boundary-1").status_code == 401
assert call(client, "GET", "/internal/v1/vpcs", ADMIN).json()[0][
"vpc_id"] == "vpc-boundary-1"
assert call(client, "GET", "/internal/v1/vpc", SUPER).status_code == 404
assert call(client, "GET", "/internal/v1/hosts/hk/vpcs", SUPER).status_code == 404
assert not store.list("operation")
def test_vpc_root_delete_endpoint_returns_exact_durable_receipt(boundary):
client, _engine, store, _logs, _services = boundary
target = {
"schema_version": 1,
"vpc_id": "vpc-empty-delete",
"region": "hk",
"fabric_id": "fabric-boundary",
"fabric_revision": 1,
"cidr": "10.31.0.0/24",
"desired_revision": 3,
"operation_id": "vpc-empty-delete-op-0001",
"subnets": [],
"hosts": [],
"members": [],
"routes": [],
"acls": [],
"retiring_members": [],
}
target["desired_digest"] = parse_vpc_delete_plan(target)[0].desired_digest
path = "/internal/v1/vpcs/vpc-empty-delete/delete"
deleted = call(
client, "POST", path, {**USER, "can_manage": False}, target)
assert deleted.status_code == 200, deleted.text
receipt = deleted.json()
assert receipt == {
"vpc_id": "vpc-empty-delete",
"operation_id": "vpc-empty-delete-op-0001",
"evidence_version": 2,
"desired_revision": 3,
"desired_digest": target["desired_digest"],
"state": "deleted",
"deleted": True,
"readback": "confirmed",
"removed_host_ids": [],
"completed_at": receipt["completed_at"],
}
assert receipt["completed_at"] > 0
assert call(client, "POST", path, SUPER, target).json() == receipt
assert store.get("vpc_deleted", "vpc-empty-delete") is not None
assert store.get("vpc_desired", "vpc-empty-delete") is None
missing_manifest = dict(target)
missing_manifest.pop("retiring_members")
invalid = call(
client, "POST", "/internal/v1/vpcs/missing-manifest/delete",
SUPER, {**missing_manifest, "vpc_id": "missing-manifest"})
assert invalid.status_code == 422
assert invalid.json()["error_code"] == "vpc_delete_manifest_invalid"
def test_host_public_network_pools_are_revisioned_idempotent_and_aggregated(
boundary):
client, _engine, _store, _logs, services = boundary
# Materialize the host child app before configuring its host-scoped
# encrypted pool catalog.
assert call(
client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER,
).json() == {
"items": [], "count": 0, "maximum": 32,
"compatibility_mode": False,
}
first_key = "a" * 32
second_key = "b" * 32
first_path = f"/internal/v1/hosts/hk/host/network-pools/{first_key}"
first = {
"revision": 1,
"state": "active",
"network_selector": "default",
"cidr": "8.8.8.0/29",
"gateway": "8.8.8.1",
"dns_servers": ["1.1.1.1"],
"reserved_ips": ["8.8.8.2"],
"guest_prefix_length": 29,
}
assert call(client, "PUT", first_path, USER, first).status_code == 403
created = call(client, "PUT", first_path, SUPER, first)
assert created.status_code == 200, created.text
assert created.json()["applied"] is True
assert created.json()["pool"]["pool_key"] == first_key
repeated = call(client, "PUT", first_path, SUPER, first)
assert repeated.status_code == 200
assert repeated.json()["idempotent"] is True
conflict = call(client, "PUT", first_path, SUPER, {
**first, "gateway": "8.8.8.2",
})
assert conflict.status_code == 409
second_path = f"/internal/v1/hosts/hk/host/network-pools/{second_key}"
second = {
**first,
"cidr": "9.9.9.0/29",
"gateway": "9.9.9.1",
"reserved_ips": ["9.9.9.2"],
}
assert call(client, "PUT", second_path, SUPER, second).status_code == 200
advanced = call(client, "PUT", first_path, SUPER, {
**first, "revision": 2, "state": "draining",
})
assert advanced.status_code == 200
assert advanced.json()["pool"]["state"] == "draining"
assert call(client, "PUT", first_path, SUPER, first).status_code == 409
overlapping = call(client, "PUT", f"/internal/v1/hosts/hk/host/network-pools/{'c' * 32}",
SUPER, {**first, "cidr": "8.8.8.0/30"})
assert overlapping.status_code == 400
catalog = call(
client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER,
).json()
assert catalog["count"] == 2
assert [item["pool_key"] for item in catalog["items"]] == [
first_key, second_key,
]
assert catalog["compatibility_mode"] is False
assert call(
client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER,
).status_code == 404
assert call(
client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER,
).status_code == 404
disabled = call(client, "PUT", second_path, SUPER, {
**second, "revision": 2, "state": "disabled",
})
assert disabled.status_code == 200
assert call(
client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER,
).status_code == 404
assert call(
client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER,
).status_code == 404
def test_first_native_pool_cannot_orphan_occupied_legacy_network(boundary):
client, engine, _store, _logs, services = boundary
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
components = engine.registry.components["hk"]
profile = components.manager.profile
profile.update({
"host": "8.8.4.4",
"default_network": "bridge:br0",
"public_pool_cidr": "8.8.8.0/29",
"public_gateway": "8.8.8.1",
"public_dns_servers": ["1.1.1.1"],
"public_reserved_ips": ["8.8.8.2"],
"public_vm_prefix": 29,
})
scope = host_scope(profile)
components.store.set("ip_claim", f"{scope}.8.8.8.3", json.dumps({
"vm_uuid": "00000000-0000-4000-8000-000000000003",
}))
pool_key = "d" * 32
path = f"/internal/v1/hosts/hk/host/network-pools/{pool_key}"
base = {
"revision": 1,
"state": "active",
"network_selector": "bridge:br0",
"dns_servers": ["1.1.1.1"],
"reserved_ips": ["9.9.9.2"],
"guest_prefix_length": 29,
}
rejected = call(client, "PUT", path, SUPER, {
**base, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1",
})
assert rejected.status_code == 409
assert call(
client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER,
).json()["compatibility_mode"] is True
adopted = call(client, "PUT", path, SUPER, {
**base,
"cidr": "8.8.8.0/29",
"gateway": "8.8.8.1",
"reserved_ips": ["8.8.8.2"],
})
assert adopted.status_code == 200, adopted.text
assert adopted.json()["compatibility_mode"] is False
def import_cloud_template_for_test(client, tmp_path, name="Delete Cloud Image"):
payload = name.encode("utf-8")
digest = hashlib.sha256(payload).hexdigest()
def fake_download(_url, _sha256, directory, **_kwargs):
source = directory / ".cloud-image-delete-boundary.qcow2"
source.write_bytes(payload)
source.chmod(0o600)
return source, 2 * 1024**3, digest
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=fake_download):
queued = call(client, "POST", "/internal/v1/vm-templates/import",
SUPER, {"url": OFFICIAL_CLOUD_URL,
"name": name, "sha256": ""})
assert queued.status_code == 202, queued.text
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET",
f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER,
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
return job["template"], payload
def test_paid_scheduler_can_claim_exact_host_pool_ip(boundary, tmp_path):
client, engine, _store, _logs, services = boundary
template, _payload = import_cloud_template_for_test(
client, tmp_path, "Scheduled Address Image",
)
assert call(
client, "GET", "/internal/v1/hosts/hk/resources", SUPER,
).status_code == 200
components = engine.registry.components["hk"]
profile = components.manager.profile
profile.update({
"host": "8.8.8.3",
"default_network": "bridge:br0",
"public_pool_cidr": "8.8.8.0/28",
"public_gateway": "8.8.8.1",
"public_dns_servers": ["1.1.1.1"],
"public_reserved_ips": ["8.8.8.2"],
"public_vm_prefix": 28,
})
service = services["hk"]
scope = host_scope(profile)
components.store.set("ip_claim", f"{scope}.8.8.8.4", json.dumps({
"vm_uuid": "00000000-0000-4000-8000-000000000004",
}))
occupied_claim = f"{scope}.8.8.8.5"
components.store.set("ip_claim", occupied_claim, json.dumps({
"vm_uuid": "00000000-0000-4000-8000-000000000005",
}))
base = {
key: value for key, value in CLOUD_CREATE_BODY.items()
if key not in {"gateway", "dns_servers"}
}
base["template_id"] = template["id"]
path = "/internal/v1/hosts/hk/vms"
unauthorized = call(client, "POST", path, {
**USER, "operation_id": "scheduled-ip-unpaid-0001",
}, {**base, "name": "unpaid-exact-ip", "public_ip_cidr": "8.8.8.8/28"})
assert unauthorized.status_code == 403
assert components.store.get("ip_claim", f"{scope}.8.8.8.8") is None
service.absent = True
outside = call(client, "POST", path, {
**USER, "can_provision": True,
"operation_id": "scheduled-ip-outside-0001",
}, {**base, "name": "outside-exact-ip", "public_ip_cidr": "8.8.8.20/28"})
assert outside.status_code == 422
assert components.store.get("ip_claim", f"{scope}.8.8.8.20") is None
occupied_vm = call(client, "POST", path, {
**USER, "can_provision": True,
"operation_id": "scheduled-ip-vm-used-0001",
}, {**base, "name": "vm-used-exact-ip", "public_ip_cidr": "8.8.8.4/28"})
assert occupied_vm.status_code == 409
vm_claim = json.loads(components.store.get(
"ip_claim", f"{scope}.8.8.8.4",
))
assert vm_claim == {
"vm_uuid": "00000000-0000-4000-8000-000000000004",
}
occupied_task = call(client, "POST", path, {
**USER, "can_provision": True,
"operation_id": "scheduled-ip-claim-used-0001",
}, {**base, "name": "claim-used-exact-ip", "public_ip_cidr": "8.8.8.5/28"})
assert occupied_task.status_code == 409
assert components.store.get("ip_claim", occupied_claim) is not None
created = call(client, "POST", path, {
**USER, "can_provision": True,
"operation_id": "scheduled-ip-success-0001",
}, {**base, "name": "scheduled-exact-ip", "public_ip_cidr": "8.8.8.6/28"})
assert created.status_code == 201, created.text
created_id = created.json()["uuid"]
created_call = [
row for row in service.calls if row[0] == "create_cloud_vm"
][-1]
assert created_call[1]["public_ip_cidr"] == "8.8.8.6/28"
assert created_call[1]["public_pool_key"] == "legacy-default"
assert created_call[1]["gateway"] == "8.8.8.1"
assert created_call[1]["dns_servers"] == ["1.1.1.1"]
assert not any(row[0] == "list_vms" for row in service.calls)
created_claim = json.loads(components.store.get(
"ip_claim", f"{scope}.8.8.8.6",
))
assert created_claim["vm_uuid"] == created_id
assert created_claim["pool_key"] == "legacy-default"
def rolled_back_cloud_create(_spec, _volume_name):
raise VMError("模拟创建失败,宿主机已完成回滚", 502)
service.create_cloud_vm = rolled_back_cloud_create
failed = call(client, "POST", path, {
**USER, "can_provision": True,
"operation_id": "scheduled-ip-rollback-0001",
}, {**base, "name": "rolled-back-exact-ip", "public_ip_cidr": "8.8.8.7/28"})
assert failed.status_code == 502
assert failed.json()["status"] == "failed"
assert failed.json()["safe_to_refund"] is True
assert components.store.get("ip_claim", f"{scope}.8.8.8.7") is None
def test_exact_public_ip_claim_is_atomic_without_inventory_scan(
boundary, tmp_path):
client, engine, _store, _logs, services = boundary
template, _payload = import_cloud_template_for_test(
client, tmp_path, "Concurrent Scheduled Address Image",
)
assert call(
client, "GET", "/internal/v1/hosts/hk/resources", SUPER,
).status_code == 200
components = engine.registry.components["hk"]
service = services["hk"]
# The exact-claim path must use only the panel-owned claim ledger. Any
# fallback to a domain inventory scan turns this guard into a test failure.
def forbidden_inventory_scan():
raise AssertionError("exact IP claim must not enumerate libvirt VMs")
service.list_vms = forbidden_inventory_scan
service.calls.clear()
profile = components.manager.profile
profile.update({
"host": "8.8.8.3",
"default_network": "bridge:br0",
"public_pool_cidr": "8.8.8.0/28",
"public_gateway": "8.8.8.1",
"public_dns_servers": ["1.1.1.1"],
"public_reserved_ips": ["8.8.8.2"],
"public_vm_prefix": 28,
})
base = {
key: value for key, value in CLOUD_CREATE_BODY.items()
if key not in {"gateway", "dns_servers"}
}
base["template_id"] = template["id"]
path = "/internal/v1/hosts/hk/vms"
barrier = threading.Barrier(2)
def create(number):
barrier.wait(timeout=2)
return call(client, "POST", path, {
**USER, "can_provision": True,
"operation_id": f"scheduled-ip-concurrent-{number:04d}",
}, {
**base, "name": f"scheduled-concurrent-{number}",
"public_ip_cidr": "8.8.8.6/28",
})
with ThreadPoolExecutor(max_workers=2) as executor:
responses = [
future.result(timeout=5)
for future in (
executor.submit(create, 1),
executor.submit(create, 2),
)
]
assert sorted(response.status_code for response in responses) == [201, 409]
successful = next(response for response in responses
if response.status_code == 201)
scope = host_scope(profile)
claim = json.loads(components.store.get(
"ip_claim", f"{scope}.8.8.8.6",
))
assert claim["vm_uuid"] == successful.json()["uuid"]
assert not any(row[0] == "list_vms" for row in service.calls)
def test_paid_scheduler_claims_the_requested_native_public_pool(
boundary, tmp_path):
client, engine, _store, _logs, services = boundary
template, _payload = import_cloud_template_for_test(
client, tmp_path, "Native Pool Address Image",
)
assert call(client, "GET", "/internal/v1/hosts/hk/resources", SUPER).status_code == 200
components = engine.registry.components["hk"]
profile = components.manager.profile
profile["default_network"] = "bridge:br0"
first_key = "1" * 32
second_key = "2" * 32
common = {
"revision": 1,
"state": "active",
"network_selector": "bridge:br0",
"dns_servers": ["1.1.1.1"],
"reserved_ips": [],
"guest_prefix_length": 29,
}
assert call(
client, "PUT",
f"/internal/v1/hosts/hk/host/network-pools/{first_key}", SUPER,
{**common, "cidr": "8.8.8.0/29", "gateway": "8.8.8.1"},
).status_code == 200
assert call(
client, "PUT",
f"/internal/v1/hosts/hk/host/network-pools/{second_key}", SUPER,
{**common, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1"},
).status_code == 200
base = {
key: value for key, value in CLOUD_CREATE_BODY.items()
if key not in {"gateway", "dns_servers"}
}
base["template_id"] = template["id"]
path = "/internal/v1/hosts/hk/vms"
provisioning = {**USER, "can_provision": True}
wrong = call(client, "POST", path, {
**provisioning, "operation_id": "native-pool-mismatch-0001",
}, {
**base, "name": "wrong-native-pool",
"public_pool_key": first_key,
"public_ip_cidr": "9.9.9.3/29",
})
assert wrong.status_code == 422
created = call(client, "POST", path, {
**provisioning, "operation_id": "native-pool-success-0001",
}, {
**base, "name": "right-native-pool",
"public_pool_key": second_key,
"public_ip_cidr": "9.9.9.3/29",
})
assert created.status_code == 201, created.text
create_call = [
row for row in services["hk"].calls if row[0] == "create_cloud_vm"
][-1]
assert create_call[1]["public_pool_key"] == second_key
assert create_call[1]["gateway"] == "9.9.9.1"
scope = host_scope(profile)
claim = json.loads(components.store.get("ip_claim", f"{scope}.9.9.9.3"))
assert claim["pool_key"] == second_key
def test_global_cloud_template_is_visible_and_cached_once_per_host(
boundary, tmp_path):
client, engine, store, _logs, services = boundary
payload = b"verified-global-cloud-template"
digest = hashlib.sha256(payload).hexdigest()
def fake_download(_url, _sha256, directory, **_kwargs):
source = directory / ".cloud-image-boundary.qcow2"
source.write_bytes(payload)
source.chmod(0o600)
return source, 2 * 1024**3, digest
denied = call(client, "POST", "/internal/v1/vm-templates/import",
USER, {"url": OFFICIAL_CLOUD_URL,
"name": "Not Allowed", "sha256": ""})
assert denied.status_code == 403
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=fake_download):
queued = call(client, "POST", "/internal/v1/vm-templates/import",
SUPER, {"url": OFFICIAL_CLOUD_URL,
"name": "AlmaLinux Global", "sha256": ""})
assert queued.status_code == 202, queued.text
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET",
f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER,
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
template_id = job["template"]["id"]
assert services == {}
assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == {
"templates": [job["template"]],
}
provisioning = {**USER, "can_provision": True}
body = {**CLOUD_CREATE_BODY, "template_id": template_id}
hk = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0001"}, body,
)
assert hk.status_code == 201, hk.text
jp = call(
client, "POST", "/internal/v1/hosts/jp/vms",
{**provisioning, "operation_id": "global-template-jp-0001"},
{**body, "name": "cloud-vm-jp", "public_ip_cidr": "8.8.8.19/24"},
)
assert jp.status_code == 201, jp.text
first_markers = {
host_id: json.loads(HostStore(store, host_id).get(
"cloud_template_replica", template_id))
for host_id in ("hk", "jp")
}
# The public catalog intentionally omits the private backing filename.
source_volume_name = template_volume_name(template_id)
for host_id, marker in first_markers.items():
assert marker["status"] == "ready"
assert marker["sha256"] == digest
assert len(marker["replica_identity"]) == 64
assert marker["replica_volume_name"] != source_volume_name
assert marker["replica_volume_name"].startswith(
source_volume_name.removesuffix(".qcow2") + "-")
created = [row for row in services[host_id].calls
if row[0] == "create_cloud_vm"]
assert created[-1][2] == marker["replica_volume_name"]
observed_markers = []
original_hk_cache = services["hk"].cache_template
def inspect_hk_marker(*args, **kwargs):
marker = json.loads(HostStore(store, "hk").get(
"cloud_template_replica", template_id))
observed_markers.append(marker)
return original_hk_cache(*args, **kwargs)
services["hk"].cache_template = inspect_hk_marker
hk_again = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0002"},
{**body, "name": "cloud-vm-hk-2", "public_ip_cidr": "8.8.8.20/24"},
)
assert hk_again.status_code == 201, hk_again.text
services["hk"].cache_template = original_hk_cache
assert [marker["status"] for marker in observed_markers] == ["syncing"]
fallback_volume_name = observed_markers[0]["replica_volume_name"]
assert fallback_volume_name != first_markers["hk"]["replica_volume_name"]
hk_cache = [row for row in services["hk"].calls
if row[0] == "cache_template"]
jp_cache = [row for row in services["jp"].calls
if row[0] == "cache_template"]
assert hk_cache[0][4:] == (None, None)
assert hk_cache[1][1] == fallback_volume_name
assert hk_cache[1][4] == first_markers["hk"]["replica_volume_name"]
assert hk_cache[1][5] == first_markers["hk"]["replica_identity"]
assert jp_cache[0][4:] == (None, None)
for host_id in ("hk", "jp"):
marker = json.loads(HostStore(store, host_id).get(
"cloud_template_replica", template_id))
assert marker["status"] == "ready"
assert marker["sha256"] == digest
assert len(marker["replica_identity"]) == 64
# A successful trusted reuse keeps the proven prior generation even
# though the durable syncing marker contained a fresh fallback name.
assert marker["replica_volume_name"] == (
first_markers[host_id]["replica_volume_name"])
# Rebinding the logical host record to a different SSH host identity must
# invalidate its ready proof even when the pool name is unchanged.
engine.registry.components["hk"].manager.profile["fingerprint"] = (
"SHA256:replacement-host")
rebound = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0003"},
{**body, "name": "cloud-vm-hk-3", "public_ip_cidr": "8.8.8.21/24"},
)
assert rebound.status_code == 201, rebound.text
hk_cache = [row for row in services["hk"].calls
if row[0] == "cache_template"]
assert hk_cache[-1][4:] == (None, None)
rebound_marker = json.loads(HostStore(store, "hk").get(
"cloud_template_replica", template_id))
assert rebound_marker["status"] == "ready"
assert rebound_marker["replica_volume_name"] == hk_cache[-1][1]
assert (rebound_marker["replica_volume_name"]
!= first_markers["hk"]["replica_volume_name"])
# A crash/failure after the syncing marker was stored must retry that
# exact planned generation. Allocating yet another name would leak the
# possibly partial remote volume and make recovery non-deterministic.
failed_calls = []
def fail_hk_cache(image_path, volume_name, virtual_size, **kwargs):
failed_calls.append((volume_name, kwargs, json.loads(
HostStore(store, "hk").get(
"cloud_template_replica", template_id))))
raise VMError("云镜像缓存同步中断", 502)
services["hk"].cache_template = fail_hk_cache
interrupted = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0004"},
{**body, "name": "cloud-vm-hk-4", "public_ip_cidr": "8.8.8.22/24"},
)
assert interrupted.status_code == 502, interrupted.text
assert len(failed_calls) == 1
planned_volume_name, failed_kwargs, failed_marker = failed_calls[0]
assert failed_marker["status"] == "syncing"
assert failed_marker["replica_volume_name"] == planned_volume_name
assert failed_kwargs["trusted_replica_volume_name"] == (
rebound_marker["replica_volume_name"])
assert failed_kwargs["trusted_replica_identity"] == (
rebound_marker["replica_identity"])
services["hk"].cache_template = original_hk_cache
retried = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**provisioning, "operation_id": "global-template-hk-0005"},
{**body, "name": "cloud-vm-hk-5", "public_ip_cidr": "8.8.8.23/24"},
)
assert retried.status_code == 201, retried.text
hk_cache = [row for row in services["hk"].calls
if row[0] == "cache_template"]
assert hk_cache[-1][1] == planned_volume_name
assert hk_cache[-1][4:] == (None, None)
recovered_marker = json.loads(HostStore(store, "hk").get(
"cloud_template_replica", template_id))
assert recovered_marker["status"] == "ready"
assert recovered_marker["replica_volume_name"] == planned_volume_name
assert [row for row in services["hk"].calls
if row[0] == "create_cloud_vm"][-1][2] == planned_volume_name
assert (tmp_path / "cloud-templates"
/ engine.registry.cloud_templates.get(template_id)["volume_name"]).is_file()
def test_legacy_host_template_id_remains_usable_for_an_already_queued_order(
boundary):
client, engine, store, _logs, services = boundary
# Initialize the original target host, then seed the same host-scoped
# catalog record an order created before the global-catalog upgrade holds.
assert call(client, "GET", "/internal/v1/hosts/legacy/vms").status_code == 200
profile = engine.registry.components["legacy"].manager.profile
scope = template_scope(profile)
digest = "d" * 64
template_id = template_id_for(scope, digest, "10")
DatabaseTemplateStore(HostStore(store, "legacy")).add(scope, {
"id": template_id,
"name": "Legacy AlmaLinux",
"os_family": "AlmaLinux",
"os_version": "10",
"sha256": digest,
"volume_name": template_volume_name(template_id),
"virtual_size": 2 * 1024**3,
"image_bytes": 1024,
"created_at": 1,
})
response = call(
client, "POST", "/internal/v1/hosts/legacy/vms",
{**USER, "can_provision": True,
"operation_id": "legacy-template-order-0001"},
{**CLOUD_CREATE_BODY, "template_id": template_id,
"name": "legacy-paid-order", "public_ip_cidr": "8.8.8.22/24"},
)
assert response.status_code == 201, response.text
assert not any(row[0] == "cache_template"
for row in services["legacy"].calls)
create = next(row for row in services["legacy"].calls
if row[0] == "create_cloud_vm")
assert create[2] == template_volume_name(template_id)
def test_global_cloud_template_delete_is_idempotent_and_retries_host_gc(
boundary, tmp_path):
client, engine, store, _logs, services = boundary
imported, payload = import_cloud_template_for_test(client, tmp_path)
template_id = imported["id"]
template_created_at = imported["created_at"]
source_volume_name = template_volume_name(template_id)
source_path = tmp_path / "cloud-templates" / source_volume_name
assert source_path.is_file()
for host_id in ("hk", "jp"):
assert call(
client, "GET", f"/internal/v1/hosts/{host_id}/resources"
).status_code == 200
HostStore(store, host_id).set(
"cloud_template_replica", template_id, "ready-marker")
assert call(
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
ADMIN, {"created_at": template_created_at},
).status_code == 403
services["hk"].cloud_template_delete_error = VMError(
"云镜像仍被虚拟机引用,不能删除", 409)
original_store_delete = store.delete
failed_catalog_delete = False
def fail_first_catalog_delete(kind, key):
nonlocal failed_catalog_delete
if (kind == "cloud_template" and key == template_id
and not failed_catalog_delete):
failed_catalog_delete = True
raise RuntimeError("catalog delete failed before commit")
return original_store_delete(kind, key)
store.delete = fail_first_catalog_delete
try:
deleted = call(
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
SUPER, {"created_at": template_created_at},
)
finally:
store.delete = original_store_delete
assert failed_catalog_delete is True
assert deleted.status_code == 200, deleted.text
assert deleted.json() == {
"deleted": True, "cache_cleanup_pending": True,
}
assert not source_path.exists()
assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == {
"templates": [],
}
for _ in range(100):
tombstone = json.loads(store.get(
"cloud_template_tombstone", template_id))
if tombstone["pending_hosts"] == ["hk"]:
break
time.sleep(0.01)
assert tombstone["source_deleted"] is True
assert tombstone["source_volume_name"] == source_volume_name
assert tombstone["pending_hosts"] == ["hk"]
assert HostStore(store, "jp").get(
"cloud_template_replica", template_id) is None
assert HostStore(store, "hk").get(
"cloud_template_replica", template_id) is not None
# Catalog and host reads are side-effect free. The durable delete event's
# maintenance supervisor retries independently of inventory/read traffic.
calls_before = len([
row for row in services["hk"].calls
if row[0] == "delete_cloud_template_replicas"
])
services["hk"].cloud_template_delete_error = VMError(
"存储池 images 不存在", 404)
with patch.object(
engine, "_background",
side_effect=AssertionError("catalog reads must not schedule cache GC")):
assert call(
client, "GET", "/internal/v1/vm-templates", USER
).status_code == 200
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
assert len([
row for row in services["hk"].calls
if row[0] == "delete_cloud_template_replicas"
]) == calls_before
still_pending = json.loads(store.get(
"cloud_template_tombstone", template_id))
assert still_pending["pending_hosts"] == ["hk"]
assert HostStore(store, "hk").get(
"cloud_template_replica", template_id) is not None
engine.cache_cleanup_retry_seconds = 0.02
engine._wake_cache_cleanup()
for _ in range(100):
if len([
row for row in services["hk"].calls
if row[0] == "delete_cloud_template_replicas"
]) > calls_before:
break
time.sleep(0.01)
still_pending = json.loads(store.get(
"cloud_template_tombstone", template_id))
assert still_pending["pending_hosts"] == ["hk"]
services["hk"].cloud_template_delete_error = VMError(
"云镜像宿主机缓存不存在", 404,
error_code="cloud_template_cache_not_found",
)
engine._wake_cache_cleanup()
for _ in range(100):
terminal = json.loads(store.get(
"cloud_template_tombstone", template_id))
if terminal["pending_hosts"] == []:
break
time.sleep(0.01)
terminal = json.loads(store.get("cloud_template_tombstone", template_id))
assert terminal["pending_hosts"] == []
assert HostStore(store, "hk").get(
"cloud_template_replica", template_id) is None
repeated = call(
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
SUPER, {"created_at": template_created_at})
assert repeated.status_code == 200
assert repeated.json() == {
"deleted": True, "cache_cleanup_pending": False,
}
# A duplicate task from the old deletion must not touch a later deletion
# generation (or the same-ID image/replica that may exist between them).
old_deletion_id = terminal["deletion_id"]
engine.registry.cloud_templates._prepare_reimport(template_id)
reimported = {
**imported,
"volume_name": source_volume_name,
"created_at": template_created_at + 1,
}
temporary = engine.registry.cloud_templates.files.directory() \
/ ".cloud-image-reimport-boundary.qcow2"
temporary.write_bytes(payload)
temporary.chmod(0o600)
engine.registry.cloud_templates.files.publish(
temporary, template_id, expected_size=len(payload))
engine.registry.cloud_templates.catalog.add(reimported)
stale_delete = call(
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
SUPER, {"created_at": template_created_at})
assert stale_delete.status_code == 409
assert engine.registry.cloud_templates.get(
template_id)["created_at"] == template_created_at + 1
next_tombstone = {
**terminal,
"deletion_id": "b" * 32,
"template_created_at": template_created_at + 1,
"pending_hosts": ["hk"],
}
store.set("cloud_template_tombstone", template_id,
json.dumps(next_tombstone))
calls_before = list(services["hk"].calls)
engine._cleanup_cloud_template_tombstone_on_host_sync(
template_id, old_deletion_id, "hk", services["hk"],
engine.registry.components["hk"],
)
assert services["hk"].calls == calls_before
assert json.loads(store.get(
"cloud_template_tombstone", template_id))["pending_hosts"] == ["hk"]
def test_user_iso_routes_are_owner_scoped_and_create_uses_opaque_id(boundary, tmp_path):
client, _engine, _store, _logs, services = boundary
download = tmp_path / "download.iso"
download.write_bytes(b"i" * (1024 * 1024))
def fake_download(_url, _sha256, _directory, *, max_bytes):
assert max_bytes >= download.stat().st_size
return download, download.stat().st_size, "a" * 64
with patch("virsh_panel.user_isos.download_user_iso", side_effect=fake_download):
queued = call(client, "POST", "/internal/v1/user-isos/import",
USER, {"url": "https://downloads.example.test/os.iso",
"name": "My Installer", "sha256": ""})
assert queued.status_code == 202
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
iso_id = job["iso"]["id"]
assert "volume_name" not in job["iso"]
listed = call(client, "GET", "/internal/v1/user-isos").json()
assert [row["id"] for row in listed["isos"]] == [iso_id]
assert "volume_name" not in listed["isos"][0]
global_source = (tmp_path / "user-isos" / str(USER["user_id"])
/ f"panel-user-iso-{iso_id}.iso")
assert global_source.is_file()
assert services == {}
assert call(
client, "GET", "/internal/v1/hosts/hk/user-isos", USER,
).status_code == 404
assert call(
client, "POST", "/internal/v1/hosts/hk/user-isos/import", USER, {
"url": "https://downloads.example.test/alias.iso",
"name": "Host Alias Must Fail", "sha256": "",
},
).status_code == 404
assert call(
client, "DELETE", f"/internal/v1/hosts/hk/user-isos/{iso_id}", USER,
).status_code == 404
assert global_source.is_file()
other = {**USER, "user_id": 42}
assert call(client, "GET", f"/internal/v1/user-isos/{iso_id}",
other).status_code == 404
assert call(client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}",
other).status_code == 404
assert call(client, "DELETE", f"/internal/v1/user-isos/{iso_id}",
other).status_code == 404
provisioning = {**USER, "can_provision": True,
"operation_id": "create-user-iso-0001"}
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
body["user_iso_id"] = iso_id
created = call(client, "POST", "/internal/v1/hosts/hk/vms",
provisioning, body)
assert created.status_code == 201, created.text
create_calls = [row for row in services["hk"].calls
if row[0] == "create_user_iso_vm"]
assert len(create_calls) == 1
assert create_calls[0][2] == USER["user_id"]
assert create_calls[0][4] == iso_id
cache_calls = [row for row in services["hk"].calls
if row[0] == "cache_user_iso"]
assert len(cache_calls) == 1
assert cache_calls[0][4] is False
other_host_actor = {
**provisioning, "operation_id": "create-user-iso-0003",
}
other_host_body = {**body, "name": "other-host-installer"}
other_host = call(client, "POST", "/internal/v1/hosts/jp/vms",
other_host_actor, other_host_body)
assert other_host.status_code == 201, other_host.text
assert len([row for row in services["jp"].calls
if row[0] == "cache_user_iso"]) == 1
assert global_source.is_file()
ambiguous_actor = {**provisioning, "operation_id": "create-user-iso-0002"}
ambiguous = call(client, "POST", "/internal/v1/hosts/hk/vms",
ambiguous_actor, {**body, "iso": "legacy.iso"})
assert ambiguous.status_code == 422
def test_user_iso_writes_require_manage_capability(boundary):
client, *_rest = boundary
actor = {**USER, "can_manage": False}
response = call(client, "POST", "/internal/v1/user-isos/import",
actor, {"url": "https://downloads.example.test/os.iso",
"name": "Installer", "sha256": ""})
assert response.status_code == 403
def test_concurrent_first_use_serializes_replica_marker_and_cache(boundary, tmp_path):
client, _engine, _store, _logs, services = boundary
download = tmp_path / "concurrent.iso"
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, "1" * 64)):
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
"url": "https://downloads.example.test/concurrent.iso",
"name": "Concurrent Installer", "sha256": "",
})
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
iso_id = job["iso"]["id"]
# Materialize the target child app before starting concurrent requests.
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
service = services["hk"]
release = threading.Event()
service.user_iso_cache_release = release
base = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
base["user_iso_id"] = iso_id
def create(number):
actor = {**USER, "can_provision": True,
"operation_id": f"concurrent-user-iso-{number:04d}"}
return call(client, "POST", "/internal/v1/hosts/hk/vms", actor,
{**base, "name": f"concurrent-vm-{number}"})
with ThreadPoolExecutor(max_workers=2) as executor:
first = executor.submit(create, 1)
assert service.user_iso_cache_started.wait(timeout=2)
second = executor.submit(create, 2)
time.sleep(0.05)
release.set()
responses = [first.result(timeout=5), second.result(timeout=5)]
assert [response.status_code for response in responses] == [201, 201]
cache_calls = [row for row in service.calls if row[0] == "cache_user_iso"]
assert [row[4] for row in cache_calls] == [False, True]
def test_user_iso_source_stays_in_use_through_vm_create(boundary, tmp_path):
client, engine, _store, _logs, services = boundary
iso_id = import_user_iso_for_test(client, tmp_path, "Create Lease Installer")
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
service = services["hk"]
release = threading.Event()
service.user_iso_create_release = release
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
body["user_iso_id"] = iso_id
actor = {**USER, "can_provision": True,
"operation_id": "user-iso-source-lease-0001"}
with ThreadPoolExecutor(max_workers=1) as executor:
creating = executor.submit(
call, client, "POST", "/internal/v1/hosts/hk/vms", actor, body)
assert service.user_iso_create_started.wait(timeout=2)
with pytest.raises(TemplateError, match="正在同步") as busy:
engine.registry.user_isos.delete_for(USER["user_id"], iso_id)
assert busy.value.status == 409
release.set()
created = creating.result(timeout=5)
assert created.status_code == 201, created.text
assert engine.registry.user_isos.get_for(USER["user_id"], iso_id)["id"] == iso_id
def test_ready_replica_marker_survives_failed_reuse(boundary, tmp_path):
client, engine, store, _logs, services = boundary
iso_id = import_user_iso_for_test(client, tmp_path, "Ready Marker Installer")
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
body["user_iso_id"] = iso_id
first = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**USER, "can_provision": True,
"operation_id": "ready-marker-create-0001"}, body)
assert first.status_code == 201, first.text
marker_key = f"{USER['user_id']}:{iso_id}"
scoped = HostStore(store, "hk")
assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready"
with patch.object(
engine.registry.user_isos, "_verify_source",
side_effect=TemplateError("ISO 源文件的 SHA-256 校验值不匹配", 500)):
failed = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**USER, "can_provision": True,
"operation_id": "ready-marker-create-0002"},
{**body, "name": "ready-marker-failed"})
assert failed.status_code == 500
assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready"
recovered = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**USER, "can_provision": True,
"operation_id": "ready-marker-create-0003"},
{**body, "name": "ready-marker-recovered"})
assert recovered.status_code == 201, recovered.text
cache_calls = [row for row in services["hk"].calls
if row[0] == "cache_user_iso"]
assert cache_calls[-1][4] is True
def test_global_delete_retries_tombstone_until_host_cache_is_reclaimed(
boundary, tmp_path):
client, engine, store, _logs, services = boundary
download = tmp_path / "delete.iso"
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, "2" * 64)):
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
"url": "https://downloads.example.test/delete.iso",
"name": "Delete Installer", "sha256": "",
})
job_id = queued.json()["job_id"]
for _ in range(100):
job = call(
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
iso_id = job["iso"]["id"]
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
body["user_iso_id"] = iso_id
created = call(
client, "POST", "/internal/v1/hosts/hk/vms",
{**USER, "can_provision": True,
"operation_id": "delete-user-iso-cache-0001"}, body)
assert created.status_code == 201, created.text
from virsh_panel.libvirt_service import VMError
services["hk"].user_iso_delete_error = VMError(
"存储池 iso 不存在", 404)
deleted = call(
client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
assert deleted.status_code == 200
assert deleted.json() == {"deleted": True, "cache_cleanup_pending": True}
assert call(client, "GET", "/internal/v1/user-isos", USER).json()["isos"] == []
tombstone_key = f"{USER['user_id']}:{iso_id}"
for _ in range(100):
if any(row[0] == "delete_user_iso" for row in services["hk"].calls):
break
time.sleep(0.01)
assert store.get("user_iso_tombstone", tombstone_key) is not None
scoped = HostStore(store, "hk")
assert scoped.get("user_iso_replica", tombstone_key) is not None
services["hk"].user_iso_delete_error = VMError(
"用户 ISO 在宿主机上不存在", 404,
error_code="user_iso_volume_not_found",
)
calls_before = len([
row for row in services["hk"].calls if row[0] == "delete_user_iso"
])
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
assert len([
row for row in services["hk"].calls if row[0] == "delete_user_iso"
]) == calls_before
engine.cache_cleanup_retry_seconds = 0.02
engine._wake_cache_cleanup()
for _ in range(100):
if store.get("user_iso_tombstone", tombstone_key) is None:
break
time.sleep(0.01)
assert store.get("user_iso_tombstone", tombstone_key) is None
assert scoped.get("user_iso_replica", tombstone_key) is None
def test_concurrent_global_deletes_preserve_committed_tombstone(boundary, tmp_path):
client, engine, store, _logs, services = boundary
iso_id = import_user_iso_for_test(client, tmp_path, "Concurrent Delete Installer")
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
record = engine.registry.user_isos.get_for(USER["user_id"], iso_id)
marker_key = f"{USER['user_id']}:{iso_id}"
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
"status": "ready",
"volume_name": record["volume_name"],
"size_bytes": record["size_bytes"],
"sha256": record["sha256"],
}))
from virsh_panel.libvirt_service import VMError
services["hk"].user_iso_volumes.add((USER["user_id"], record["volume_name"]))
services["hk"].user_iso_delete_error = VMError(
"用户 ISO 仍被虚拟机安装过程引用,不能删除", 409)
original_delete = engine.registry.user_isos.delete_for
started = threading.Event()
release = threading.Event()
def blocking_delete(user_id, selected_iso_id):
started.set()
assert release.wait(timeout=5)
return original_delete(user_id, selected_iso_id)
try:
with patch.object(engine.registry.user_isos, "delete_for",
side_effect=blocking_delete) as delete_for:
with ThreadPoolExecutor(max_workers=2) as executor:
first = executor.submit(
call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
assert started.wait(timeout=2)
second = executor.submit(
call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
time.sleep(0.05)
release.set()
responses = [first.result(timeout=5), second.result(timeout=5)]
assert delete_for.call_count == 1
finally:
release.set()
assert [response.status_code for response in responses] == [200, 200]
tombstone = json.loads(store.get("user_iso_tombstone", marker_key))
assert tombstone["source_deleted"] is True
assert tombstone["pending_hosts"] == ["hk"]
def test_user_iso_cache_cleanup_blocks_online_update(boundary, tmp_path):
client, engine, store, _logs, services = boundary
iso_id = import_user_iso_for_test(client, tmp_path, "Update Gate Installer")
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
record = engine.registry.user_isos.get_for(USER["user_id"], iso_id)
marker_key = f"{USER['user_id']}:{iso_id}"
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
"status": "ready",
"volume_name": record["volume_name"],
"size_bytes": record["size_bytes"],
"sha256": record["sha256"],
}))
service = services["hk"]
service.user_iso_volumes.add((USER["user_id"], record["volume_name"]))
release = threading.Event()
service.user_iso_delete_release = release
updater = FakeUpdater()
updater.token = "valid-token-123456"
engine.updater = updater
try:
deleted = call(
client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
assert deleted.status_code == 200
assert service.user_iso_delete_started.wait(timeout=2)
blocked = call(client, "POST", "/internal/v1/update/apply", SUPER, {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert blocked.status_code == 409
assert updater.active is None
finally:
release.set()
for _ in range(100):
if not engine.maintenance_tasks:
break
time.sleep(0.01)
assert not engine.maintenance_tasks
def test_unrelated_host_requests_never_run_cache_cleanup(boundary):
client, engine, store, _logs, services = boundary
iso_id = "f" * 32
volume_name = f"panel-user-iso-{iso_id}.iso"
marker_key = f"{USER['user_id']}:{iso_id}"
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
"status": "ready", "volume_name": volume_name,
"size_bytes": 1024 * 1024, "sha256": "f" * 64,
}))
store.set("user_iso_tombstone", marker_key, json.dumps({
"user_id": USER["user_id"], "iso_id": iso_id,
"volume_name": volume_name, "pending_hosts": ["hk"],
"source_deleted": True, "created_at": 1,
}))
with patch.object(
engine.updater, "active_job",
side_effect=AssertionError("host reads must not repair updater state")), \
patch.object(
engine, "_cleanup_host_user_iso_tombstones",
side_effect=AssertionError("host reads must not run ISO GC")), \
patch.object(
engine, "_cleanup_host_cloud_template_tombstones",
side_effect=AssertionError("host reads must not run template GC")):
response = call(client, "GET", "/internal/v1/hosts/hk/resources")
assert response.status_code == 200
assert not any(row[0] == "delete_user_iso" for row in services["hk"].calls)
assert store.get("user_iso_tombstone", marker_key) is not None
with patch.object(
engine, "_cleanup_host_user_iso_tombstones",
side_effect=AssertionError("host writes must not run ISO GC")), \
patch.object(
engine, "_cleanup_host_cloud_template_tombstones",
side_effect=AssertionError("host writes must not run template GC")):
invalid_write = call(
client, "PUT",
f"/internal/v1/hosts/hk/host/network-pools/{'a' * 32}",
SUPER, {},
)
assert invalid_write.status_code == 422
def test_removed_inventory_routes_do_not_enter_dispatch(boundary):
client, engine, *_ = boundary
with patch.object(
engine, "_update_lock",
side_effect=AssertionError("removed inventory route reached mutation dispatch")), \
patch.object(
engine.updater, "active_job",
side_effect=AssertionError("removed inventory route reached updater")):
capacity = call(
client, "GET", "/internal/v1/hosts/hk/host/capacity", ADMIN)
network = call(
client, "GET", "/internal/v1/hosts/hk/host/network-options", ADMIN)
assert capacity.status_code == 404
assert network.status_code == 404
def test_slow_tombstone_cleanup_does_not_block_engine_startup(tmp_path):
class StartupRegistry:
def __init__(self):
self.components = {"offline": object()}
self.user_isos = SimpleNamespace(has_active_jobs=lambda: False)
self.started = False
self.closed = False
async def start(self):
self.started = True
async def close(self):
self.closed = True
async def scenario():
registry = StartupRegistry()
updater = FakeUpdater()
store = MemoryStore()
iso_id = "f" * 32
tombstone_key = f"1:{iso_id}"
store.set("user_iso_tombstone", tombstone_key, json.dumps({
"user_id": 1,
"iso_id": iso_id,
"volume_name": f"panel-user-iso-{iso_id}.iso",
"pending_hosts": ["offline"],
"source_deleted": True,
"created_at": 1,
}))
engine = EngineApplication(
{"service_secret": base64.b64encode(SECRET).decode(),
"data_dir": str(tmp_path)},
store, MemoryStore(), registry=registry, updater=updater,
)
cleanup_started = asyncio.Event()
cleanup_release = asyncio.Event()
async def slow_cleanup(host_id):
assert host_id == "offline"
cleanup_started.set()
await cleanup_release.wait()
engine._cleanup_host_user_iso_tombstones = slow_cleanup
incoming = asyncio.Queue()
outgoing = asyncio.Queue()
async def receive():
return await incoming.get()
async def send(message):
await outgoing.put(message)
await incoming.put({"type": "lifespan.startup"})
lifespan = asyncio.create_task(engine._lifespan(receive, send))
startup = await asyncio.wait_for(outgoing.get(), timeout=0.5)
assert startup == {"type": "lifespan.startup.complete"}
await asyncio.wait_for(cleanup_started.wait(), timeout=0.5)
assert registry.started is True
assert engine.maintenance_tasks
assert engine._long_running_host_job() is True
cleanup_release.set()
await incoming.put({"type": "lifespan.shutdown"})
shutdown = await asyncio.wait_for(outgoing.get(), timeout=0.5)
assert shutdown == {"type": "lifespan.shutdown.complete"}
await asyncio.wait_for(lifespan, timeout=0.5)
assert registry.closed is True
asyncio.run(scenario())
def test_tombstone_cleanup_pauses_for_update_then_resumes(tmp_path):
class StartupRegistry:
def __init__(self):
self.components = {"offline": object()}
self.user_isos = SimpleNamespace(has_active_jobs=lambda: False)
async def start(self):
return None
async def close(self):
return None
async def scenario():
store = MemoryStore()
iso_id = "e" * 32
tombstone_key = f"1:{iso_id}"
store.set("user_iso_tombstone", tombstone_key, json.dumps({
"user_id": 1,
"iso_id": iso_id,
"volume_name": f"panel-user-iso-{iso_id}.iso",
"pending_hosts": ["offline"],
"source_deleted": True,
"created_at": 1,
}))
updater = FakeUpdater()
updater.active = {"job_id": "update-job-running"}
engine = EngineApplication(
{"service_secret": base64.b64encode(SECRET).decode(),
"data_dir": str(tmp_path)},
store, MemoryStore(), registry=StartupRegistry(), updater=updater,
)
engine.cache_cleanup_retry_seconds = 0.02
cleanup_started = asyncio.Event()
async def cleanup(host_id):
assert host_id == "offline"
cleanup_started.set()
store.delete("user_iso_tombstone", tombstone_key)
engine._cleanup_host_user_iso_tombstones = cleanup
incoming = asyncio.Queue()
outgoing = asyncio.Queue()
async def receive():
return await incoming.get()
async def send(message):
await outgoing.put(message)
await incoming.put({"type": "lifespan.startup"})
lifespan = asyncio.create_task(engine._lifespan(receive, send))
assert await asyncio.wait_for(outgoing.get(), timeout=0.5) == {
"type": "lifespan.startup.complete",
}
await asyncio.sleep(0.06)
assert not cleanup_started.is_set()
updater.active = None
await asyncio.wait_for(cleanup_started.wait(), timeout=0.5)
assert store.get("user_iso_tombstone", tombstone_key) is None
await incoming.put({"type": "lifespan.shutdown"})
assert await asyncio.wait_for(outgoing.get(), timeout=0.5) == {
"type": "lifespan.shutdown.complete",
}
await asyncio.wait_for(lifespan, timeout=0.5)
asyncio.run(scenario())
def test_user_iso_global_import_does_not_contact_a_host(boundary, tmp_path):
client, _engine, _store, _logs, services = boundary
download = tmp_path / "global.iso"
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, "f" * 64)):
response = call(client, "POST", "/internal/v1/user-isos/import",
USER, {"url": "https://downloads.example.test/os.iso",
"name": "Installer", "sha256": ""})
assert response.status_code == 202
job_id = response.json()["job_id"]
for _ in range(100):
job = call(
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
assert services == {}
def test_user_iso_download_is_bounded_by_panel_storage(boundary, tmp_path):
client, engine, _store, _logs, _services = boundary
download = tmp_path / "bounded.iso"
download.write_bytes(b"i" * (1024 * 1024))
def bounded_download(_url, _sha256, _directory, *, max_bytes):
assert max_bytes == 2 * 1024 * 1024
return download, download.stat().st_size, "d" * 64
with patch.object(engine.registry.user_isos.files, "available_bytes",
return_value=2 * 1024 * 1024), patch(
"virsh_panel.user_isos.download_user_iso", side_effect=bounded_download):
response = call(client, "POST", "/internal/v1/user-isos/import",
USER, {"url": "https://downloads.example.test/os.iso",
"name": "Bounded Installer", "sha256": ""})
assert response.status_code == 202
job_id = response.json()["job_id"]
for _ in range(100):
job = call(client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
def test_user_iso_job_hides_internal_paths_in_failures(boundary):
client, _engine, _store, _logs, _services = boundary
with patch("virsh_panel.user_isos.download_user_iso",
side_effect=TemplateError("目录 /private/data/user-isos/41 损坏", 500)):
response = call(client, "POST", "/internal/v1/user-isos/import",
USER, {"url": "https://downloads.example.test/os.iso",
"name": "Error Installer", "sha256": ""})
assert response.status_code == 202
job_id = response.json()["job_id"]
for _ in range(100):
job = call(client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "failed"
assert job["error"] == "ISO 下载或保存失败,请联系管理员查看日志"
assert "/private/data" not in json.dumps(job, ensure_ascii=False)
def test_user_iso_import_persistence_failure_does_not_stick_pending(boundary, tmp_path):
client, _engine, store, _logs, _services = boundary
assert call(client, "GET", "/internal/v1/user-isos").status_code == 200
original_set = store.set
def fail_user_iso_job(kind, key, value):
if kind == "user_iso_import_job":
raise DatabaseError()
return original_set(kind, key, value)
body = {"url": "https://downloads.example.test/os.iso",
"name": "Installer", "sha256": ""}
with patch.object(store, "set", side_effect=fail_user_iso_job):
failed = call(client, "POST", "/internal/v1/user-isos/import",
USER, body)
assert failed.status_code == 503
download = tmp_path / "retry.iso"
download.write_bytes(b"i" * (1024 * 1024))
with patch("virsh_panel.user_isos.download_user_iso",
return_value=(download, download.stat().st_size, "b" * 64)):
retried = call(client, "POST", "/internal/v1/user-isos/import",
USER, body)
assert retried.status_code == 202, retried.text
for _ in range(100):
job = call(
client, "GET",
f"/internal/v1/user-isos/import-jobs/{retried.json()['job_id']}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
assert job["status"] == "completed", job
def test_interrupted_user_iso_import_is_restored_as_failed(boundary):
client, _engine, store, _logs, _services = boundary
job_id = "interrupted-user-iso"
scope = user_iso_scope(USER["user_id"])
store.set("user_iso_import_job", job_id, json.dumps({
"job_id": job_id, "scope": scope, "owner_user_id": USER["user_id"],
"status": "running", "phase": "正在安全下载 ISO", "created_at": 1,
}))
_engine.registry.user_isos._restored = False
_engine.registry.user_isos.restore_jobs()
restored = call(
client, "GET",
f"/internal/v1/user-isos/import-jobs/{job_id}",
)
assert restored.status_code == 200
assert restored.json()["status"] == "failed"
assert "重启" in restored.json()["phase"]
def test_user_iso_list_exposes_only_owners_active_import(boundary, tmp_path):
client, engine, _store, _logs, _services = boundary
download = tmp_path / "active.iso"
download.write_bytes(b"i" * (1024 * 1024))
started = threading.Event()
release = threading.Event()
def blocking_download(_url, _sha256, _directory, *, max_bytes):
assert max_bytes >= download.stat().st_size
started.set()
assert release.wait(timeout=5)
return download, download.stat().st_size, "c" * 64
body = {"url": "https://downloads.example.test/os.iso",
"name": "Active Installer", "sha256": ""}
try:
with patch("virsh_panel.user_isos.download_user_iso", side_effect=blocking_download):
queued = call(client, "POST", "/internal/v1/user-isos/import",
USER, body)
assert queued.status_code == 202
assert started.wait(timeout=2)
assert engine._long_running_host_job() is True
own = call(client, "GET", "/internal/v1/user-isos", USER).json()
active = own["active_import_job"]
assert active["job_id"] == queued.json()["job_id"]
assert active["status"] == "running"
assert set(active) <= {"job_id", "status", "phase", "created_at",
"started_at", "finished_at", "iso", "error"}
assert "scope" not in active and "owner_user_id" not in active
other = call(client, "GET", "/internal/v1/user-isos",
{**USER, "user_id": 42}).json()
assert "active_import_job" not in other
release.set()
for _ in range(100):
job = call(
client, "GET",
f"/internal/v1/user-isos/import-jobs/{active['job_id']}",
).json()
if job["status"] in {"completed", "failed"}:
break
time.sleep(0.01)
finally:
release.set()
assert job["status"] == "completed", job
assert "active_import_job" not in call(
client, "GET", "/internal/v1/user-isos", USER).json()
def test_hmac_binds_actor_payload_target_and_rejects_replay():
store = MemoryStore()
body = b"{}"
target = "/internal/v1/hosts/hk/vms?range=1h"
headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER,
timestamp=1000, nonce="n" * 24).items()}
assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
changed = {**headers, "x-cloud-actor": encode_actor(SUPER)}
with pytest.raises(EngineAuthError):
authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000)
with pytest.raises(EngineAuthError):
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061)
invalid_cleanup = {**USER, "system_cleanup": "true"}
invalid_headers = {key.lower(): value for key, value in signed_headers(
SECRET, "POST", target, body, invalid_cleanup,
timestamp=1000, nonce="c" * 24,
).items()}
with pytest.raises(EngineAuthError):
authenticate(
invalid_headers, SECRET, "POST", target,
hashlib.sha256(body).hexdigest(), store, now=1000,
)
def test_private_route_allowlist_and_no_browser_admin_cookie(boundary):
client, *_ = boundary
for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin",
"/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"):
assert call(client, "POST", path, SUPER, {}).status_code == 404
assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401
assert TRUSTED_ACTOR.get() is None
def test_internal_update_routes_are_super_admin_only_and_never_echo_token(boundary):
client, engine, *_ = boundary
updater = FakeUpdater()
triggered = []
engine.updater = updater
def trigger():
triggered.append(True)
updater.active["status"] = "running"
engine.update_trigger = trigger
settings_path = "/internal/v1/update/settings"
ready = call(client, "GET", "/internal/v1/health", SUPER)
assert ready.status_code == 200
assert ready.json() == {"status": "ok", "accepting_mutations": True}
assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401
assert call(client, "GET", settings_path, USER).status_code == 403
assert call(client, "GET", settings_path, ADMIN).status_code == 403
assert call(client, "GET", settings_path, {**SUPER, "can_manage": False}).status_code == 403
token = "gitea-private-token-123456"
saved = call(client, "PUT", settings_path, SUPER, {"access_token": token})
assert saved.status_code == 200
assert saved.json()["token_configured"] is True
assert saved.json()["repository"] == "https://g.luoli.live/root/virt.git"
assert saved.json()["branch"] == "venix"
assert token not in saved.text
assert token not in call(client, "GET", settings_path, SUPER).text
checked = call(client, "POST", "/internal/v1/update/check", SUPER, {})
assert checked.status_code == 200
assert checked.json()["status"] == "update_available"
applied = call(client, "POST", "/internal/v1/update/apply", SUPER, {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert applied.status_code == 202
job_id = applied.json()["job_id"]
assert job_id in updater.jobs
assert "private" not in applied.json() and token not in applied.text
assert triggered == [True]
job = call(client, "GET", f"/internal/v1/update/jobs/{job_id}", SUPER)
assert job.status_code == 200 and token not in job.text and "private" not in job.json()
# While the persisted update job is active, reads remain available but no
# host/VM mutation may begin and overlap the code/service restart.
maintenance = call(client, "GET", "/internal/v1/health", SUPER)
assert maintenance.status_code == 200
assert maintenance.json() == {"status": "ok", "accepting_mutations": False}
assert call(client, "POST", "/internal/v1/hosts/scan", SUPER,
{"host": "host.example.test", "port": 22}).status_code == 409
def test_internal_update_validation_clear_and_trigger_failure(boundary):
client, engine, *_ = boundary
updater = FakeUpdater()
engine.updater = updater
settings_path = "/internal/v1/update/settings"
assert call(client, "PUT", settings_path, SUPER, {}).status_code == 422
assert call(client, "PUT", settings_path, SUPER,
{"access_token": "x", "unexpected": True}).status_code == 422
assert call(client, "POST", "/internal/v1/update/check", SUPER,
{"repository": "https://evil.example/repo.git"}).status_code == 422
assert call(client, "POST", "/internal/v1/update/apply", SUPER,
{"expected_current_commit": "a" * 40}).status_code == 422
assert call(client, "GET", "/internal/v1/update/jobs/missing-job-123456", SUPER).status_code == 404
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
cleared = call(client, "PUT", settings_path, SUPER, {"clear_token": True})
assert cleared.status_code == 200 and cleared.json()["token_configured"] is False
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
def fail_trigger():
from virsh_panel.updater import UpdateError
raise UpdateError("独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503)
engine.update_trigger = fail_trigger
failed = call(client, "POST", "/internal/v1/update/apply", SUPER, {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert failed.status_code == 503
assert len(updater.jobs) == 1
job = next(iter(updater.jobs.values()))
assert job["status"] == "failed"
assert "令牌" not in job["error"]
@pytest.mark.parametrize(("state", "expected"), [
("activating", True),
("active", True),
("deactivating", True),
("inactive", False),
("failed", False),
("maintenance", None),
])
def test_update_oneshot_transitional_states_keep_maintenance_gate(state, expected):
with patch("virsh_panel.internal.subprocess.run",
return_value=SimpleNamespace(returncode=0, stdout=state + "\n")):
assert EngineApplication._update_service_active() is expected
def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary):
client, engine, *_ = boundary
path = "/internal/v1/hosts/hk"
assert call(client, "GET", path + "/host").status_code == 403
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200
assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200
assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404
assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == []
assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False},
{"action": "reboot"}).status_code == 403
def test_stable_host_identity_and_namespaces():
store = MemoryStore()
first, second = HostStore(store, "a"), HostStore(store, "b")
first.set("host", "current", "one")
second.set("host", "current", "two")
assert first.get("host", "current") == "one"
assert second.get("host", "current") == "two"
a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"}
assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"})
assert host_scope(a) != host_scope({**a, "host_id": "b"})
assert MainStore.RECORD_TABLE == "engine_records"
assert LogStore.METRIC_TABLE == "engine_vm_metric_samples"
def test_lightweight_vm_state_reads_are_allowlisted(boundary):
client, _engine, _store, _logs, services = boundary
listed = call(client, "GET", "/internal/v1/hosts/hk/vm-states", SUPER)
assert listed.status_code == 200, listed.text
assert listed.json()["items"] == [{
"uuid": VM_ID, "name": "alma-test", "state": "running",
}]
single = call(
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/state", SUPER,
)
assert single.status_code == 200, single.text
assert single.json() == {
"uuid": VM_ID, "name": "alma-test", "state": "running",
}
assert ("list_vm_states",) in services["hk"].calls
assert ("get_vm_state", VM_ID) in services["hk"].calls
def test_selected_vm_metadata_is_allowlisted_and_owner_scoped(boundary):
client, engine, _store, _logs, services = boundary
path = f"/internal/v1/hosts/hk/vm-metadata?ids={VM_ID}"
admin = call(client, "GET", path, SUPER)
assert admin.status_code == 200, admin.text
assert admin.json()["items"][0]["uuid"] == VM_ID
assert services["hk"].calls[-1] == ("list_vm_metadata", (VM_ID,))
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
component.state.users.assign_vm(scope, VM_ID, USER["user_id"])
owner = call(client, "GET", path, USER)
assert owner.status_code == 200, owner.text
calls_after_owner = services["hk"].calls.count(("list_vm_metadata", (VM_ID,)))
assert calls_after_owner == 2
stranger = call(client, "GET", path, {**USER, "user_id": 42})
assert stranger.status_code == 404
assert stranger.json()["error"] == "虚拟机不存在"
assert services["hk"].calls.count(("list_vm_metadata", (VM_ID,))) == calls_after_owner
def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403
provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"}
created = call(client, "POST", path, provisioning, CREATE_BODY)
assert created.status_code == 201, created.text
vm_id = created.json()["uuid"]
assert created.json()["result"]["uuid"] == vm_id
assert created.json()["status"] == "succeeded"
component = engine.registry.components["hk"]
assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41)
again = call(client, "POST", path, provisioning, CREATE_BODY)
assert again.status_code == 201 and again.json()["uuid"] == vm_id
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409
operation = call(client, "GET", "/internal/v1/operations/order-000001").json()
assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id
assert operation["safe_to_refund"] is False
assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404
def test_uncertain_creation_never_refunds_or_recreates(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
call(client, "GET", path)
services["hk"].create_failure = True
actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"}
failed = call(client, "POST", path, actor, CREATE_BODY)
assert failed.status_code == 502
assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"]
record = json.loads(store.get("operation", "order-uncertain"))
assert record["resource_naming"] == "uuid-v1"
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
assert component.state.users.owns_vm(scope, record["vm_uuid"], 41)
component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password")
component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]}))
again = call(client, "POST", path, actor, CREATE_BODY)
assert again.status_code == 202 and again.json()["status"] == "uncertain"
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
services["hk"].absent = True
absence_checks = list(services["hk"].absent_checks)
operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json()
# A momentary absence cannot prove the original unknown writer has stopped.
assert operation["status"] == "uncertain" and operation["safe_to_refund"] is False
assert services["hk"].absent_checks == absence_checks
assert component.state.users.owns_vm(scope, record["vm_uuid"], 41)
assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") == "saved-password"
assert component.store.get("ip_claim", f"{scope}.192.0.2.1") is not None
@pytest.mark.parametrize("error_code", [
"libvirt_execution_uncertain", "libvirt_execution_blocked",
])
def test_isolated_creation_keeps_claims_and_never_refunds_on_temporary_absence(
boundary, monkeypatch, error_code):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
call(client, "GET", path)
service = services["hk"]
service.absent = True
attempts = []
def uncertain_create(spec):
attempts.append(spec)
raise VMError("远端执行状态未确认", 504, error_code=error_code)
monkeypatch.setattr(service, "create_vm", uncertain_create)
actor = {**USER, "can_provision": True, "operation_id": "isolated-create-unknown"}
response = call(client, "POST", path, actor, CREATE_BODY)
assert response.status_code == 504, response.text
assert response.json()["status"] == "uncertain"
assert response.json()["error_code"] == error_code
assert response.json()["safe_to_refund"] is False
assert service.absent_checks == []
record = json.loads(store.get("operation", actor["operation_id"]))
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
credential_key = f"{scope}.{record['vm_uuid']}"
component.store.set("vm_credential", credential_key, "saved-test-credential")
claim_key = f"{scope}.192.0.2.1"
component.store.set("ip_claim", claim_key, json.dumps({"vm_uuid": record["vm_uuid"]}))
engine.active_operations.clear()
polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}")
repeated = call(client, "POST", path, actor, CREATE_BODY)
assert polled.status_code == 200
assert repeated.status_code == 202
for result in (polled.json(), repeated.json()):
assert result["status"] == "uncertain"
assert result["error_code"] == error_code
assert result["safe_to_refund"] is False
assert "finished_at" not in result
assert len(attempts) == 1
assert service.absent_checks == []
assert component.state.users.owns_vm(scope, record["vm_uuid"], USER["user_id"])
assert component.store.get("vm_credential", credential_key) == "saved-test-credential"
assert component.store.get("ip_claim", claim_key) is not None
def test_legacy_name_collision_operation_uses_legacy_absent_reconciliation(boundary):
client, _engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
services["hk"].absent = True
legacy_vm_id = "00000000-0000-4000-8000-000000000002"
store.set("operation", "legacy-name-collision", json.dumps({
"operation_id": "legacy-name-collision",
"host_id": "hk",
"user_id": USER["user_id"],
"request_hash": "legacy",
"vm_uuid": legacy_vm_id,
"vm_name": CREATE_BODY["name"],
"kind": "create",
"resource": "vms",
"method": "POST",
"status": "uncertain",
"result": None,
"error": "虚拟机名称已存在",
"safe_to_refund": False,
"created_at": 1,
}))
operation = call(
client, "GET", "/internal/v1/operations/legacy-name-collision", USER,
).json()
assert operation["status"] == "failed"
assert operation["safe_to_refund"] is True
assert services["hk"].absent_checks[-1] == (
legacy_vm_id, CREATE_BODY["name"], False, True)
@pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"],
{**CREATE_BODY, "guest_ssh_key_id": 3}])
def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload):
client, engine, _, _, services = boundary
path = "/internal/v1/hosts/hk/vms"
actor = {**USER, "can_provision": True, "operation_id": "rejected-create"}
response = call(client, "POST", path, actor, payload)
assert response.status_code == 422
# Simulate a dispatcher losing the POST response and querying after restart.
engine.active_operations.clear()
saved = call(client, "GET", "/internal/v1/operations/rejected-create").json()
assert saved["status"] == "failed" and saved["safe_to_refund"] is True
assert call(client, "POST", path, actor, payload).status_code == 422
assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409
assert not any(row[0] == "create_vm" for row in services["hk"].calls)
def test_admin_created_vm_also_has_external_owner(boundary):
client, engine, *_ = boundary
actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"}
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
assert response.status_code == 201
components = engine.registry.components["hk"]
assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1)
def test_power_action_has_persistent_idempotency(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403
actor = {**USER, "operation_id": "reboot-operation"}
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1
assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409
operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json()
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
saved = json.loads(store.get("operation", "reboot-operation"))
assert saved["target_action"] == "reboot"
assert saved["mutation_started"] is True
assert operation["mutation_started"] is True
def test_acknowledged_shutdown_releases_target_before_guest_stops(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
service = services["hk"]
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
receipt = {"uuid": VM_ID, "action": "shutdown", "command_accepted": True}
actor = {**USER, "operation_id": "accepted-shutdown"}
response = call(client, "POST", path, actor, {"action": "shutdown"})
assert response.status_code == 200, response.text
assert response.json()["status"] == "succeeded"
assert response.json()["result"] == receipt
assert service.vm["state"] == "running" # This guest ignores shutdown.
assert engine.coordinator.operation_claim(actor["operation_id"]) is None
# Poll/retry returns the durable receipt even when live observation fails.
with patch.object(service, "get_vm", side_effect=VMError("observation unavailable", 502)), \
patch.object(service, "get_vm_state", side_effect=VMError("observation unavailable", 502)):
operation = call(client, "GET", "/internal/v1/operations/accepted-shutdown", USER)
assert operation.status_code == 200
assert operation.json()["result"] == receipt
retried = call(client, "POST", path, actor, {"action": "shutdown"})
assert retried.status_code == 200
assert len([row for row in service.calls if row[0] == "action"]) == 1
forced = call(client, "POST", path, {**USER, "operation_id": "next-poweroff"},
{"action": "poweroff"})
assert forced.status_code == 200, forced.text
assert forced.json()["status"] == "succeeded"
assert ("action", VM_ID, "poweroff") in service.calls
assert engine.coordinator.operation_claim("next-poweroff") is None
def test_orphaned_power_action_reconciles_only_the_final_target_state(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
services["hk"].vm["state"] = "stopped"
operation = {
"operation_id": "orphan-power-state", "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": f"vms/{VM_ID}/actions",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"safe_to_refund": False, "created_at": 1,
"actor_can_manage": True, "actor_is_admin": False,
"target_action": "shutdown",
}
store.set("operation", operation["operation_id"], json.dumps(operation))
recovered = call(
client, "GET", "/internal/v1/operations/orphan-power-state", USER,
).json()
assert recovered["status"] == "succeeded"
assert recovered["result"] == {
"uuid": VM_ID, "name": "alma-test", "state": "stopped",
}
assert ("get_vm_state", VM_ID) in services["hk"].calls
assert not any(row[0] == "action" for row in services["hk"].calls)
services["hk"].vm["state"] = "running"
mismatched = {**operation, "operation_id": "orphan-power-mismatch",
"target_action": "poweroff"}
store.set("operation", mismatched["operation_id"], json.dumps(mismatched))
failed = call(
client, "GET", "/internal/v1/operations/orphan-power-mismatch", USER,
).json()
assert failed["status"] == "failed"
assert "不会自动重放" in failed["error"]
assert not any(row[0] == "action" for row in services["hk"].calls)
for action in ("reboot", "reset"):
ambiguous = {
**operation,
"operation_id": f"orphan-power-{action}",
"target_action": action,
}
store.set("operation", ambiguous["operation_id"], json.dumps(ambiguous))
unresolved = call(
client, "GET",
f"/internal/v1/operations/{ambiguous['operation_id']}", USER,
).json()
assert unresolved["status"] == "uncertain"
assert unresolved["result"] is None
assert "不会自动重放" in unresolved["error"]
assert not any(row[0] == "action" for row in services["hk"].calls)
services["hk"].vm["state"] = "stopped"
reboot_off = {
**operation,
"operation_id": "orphan-power-reboot-off",
"target_action": "reboot",
}
store.set("operation", reboot_off["operation_id"], json.dumps(reboot_off))
unresolved_off = call(
client, "GET",
"/internal/v1/operations/orphan-power-reboot-off", USER,
).json()
assert unresolved_off["status"] == "uncertain"
assert unresolved_off["result"] is None
assert "不会自动重放" in unresolved_off["error"]
assert not any(row[0] == "action" for row in services["hk"].calls)
def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary):
client, engine, _, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
rescue_key = f"{scope}.{VM_ID}"
components.store.set("vm_rescue", rescue_key, json.dumps({
"original_xml": "<domain><name>alma-test</name></domain>",
"template_id": "0123456789abcdef0123456789abcdef",
"volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso",
"status": "active",
}))
services["hk"].vm.update(
state="shutoff", rescue_mode=True, rescue_staged=True,
rescue_volume_name="panel-rescue-0123456789abcdef0123456789abcdef.iso",
)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
ordinary = call(
client, "DELETE", path,
{**USER, "operation_id": "ordinary-rescue-delete", "system_cleanup": False},
)
assert ordinary.status_code == 409
assert components.store.get("vm_rescue", rescue_key) is not None
assert not any(item[0] == "delete_vm" for item in services["hk"].calls)
cleanup = call(
client, "DELETE", path,
{**USER, "operation_id": "expiry-rescue-delete", "system_cleanup": True},
)
assert cleanup.status_code == 200, cleanup.text
assert cleanup.json()["status"] == "succeeded"
assert ("delete_vm", VM_ID, True) in services["hk"].calls
assert services["hk"].calls[-1] == ("delete_firewall_filter", VM_ID)
assert components.store.get("vm_rescue", rescue_key) is None
assert components.state.users.owner_of_vm(scope, VM_ID) is None
def test_deleted_vm_reconciles_exact_snapshot_and_control_plane_cleanup(
boundary):
client, engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
profile = components.manager.profile
scope = host_scope(profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
identity_key = f"{scope}.{VM_ID}"
components.store.set("vm_credential", identity_key, json.dumps({
"auth_mode": "ssh_key", "password": None, "generation": "g1",
}))
components.store.set("vm_rescue", identity_key, json.dumps({
"original_xml": "<domain><name>alma-test</name></domain>",
"template_id": "0123456789abcdef0123456789abcdef",
"volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso",
"status": "active",
}))
claim_key = f"{scope}.8.8.8.4"
components.store.set("ip_claim", claim_key, json.dumps({
"vm_uuid": VM_ID,
}))
historical_failed_key = "d" * 32
ready_snapshot_keys = ["e" * 32, "f" * 32]
snapshot_keys = [historical_failed_key, *ready_snapshot_keys]
volume_names = {
key: snapshot_volume_name(VM_ID, key) for key in snapshot_keys
}
catalog_keys = {key: f"{VM_ID}:{key}" for key in snapshot_keys}
for created_at, key in enumerate(snapshot_keys, start=1):
components.store.set("vm_snapshot", catalog_keys[key], json.dumps({
"snapshot_key": key,
"vm_id": VM_ID,
"name": f"清退恢复点-{created_at}",
"state": ("failed" if key == historical_failed_key
else "ready"),
"consistency": "crash",
"volume_name": volume_names[key],
"created_at": created_at,
}))
service = services["hk"]
service.vm.update(
state="shutoff", rescue_mode=True, rescue_staged=True,
)
failed_once = False
def flaky_snapshot_delete(vm_id, snapshot_key, volume_name, *,
allow_missing_vm=False):
nonlocal failed_once
service.calls.append((
"delete_snapshot_volume", vm_id, snapshot_key, volume_name,
allow_missing_vm,
))
if snapshot_key == historical_failed_key and not failed_once:
failed_once = True
raise VMError("快照卷删除响应丢失", 502)
return {"snapshot_key": snapshot_key, "deleted": True}
service.delete_snapshot_volume = flaky_snapshot_delete
operation_id = "delete-snapshot-cleanup"
actor = {
**USER,
"operation_id": operation_id,
"system_cleanup": True,
}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
first = call(client, "DELETE", path, actor)
assert first.status_code == 503, first.text
assert first.json()["status"] == "uncertain"
assert first.json()["error_code"] == (
"vm_delete_snapshot_cleanup_pending")
assert all(volume_name not in first.text
for volume_name in volume_names.values())
saved = json.loads(store.get("operation", operation_id))
assert saved["cleanup_pending"] == "vm_delete"
assert saved["vm_delete_committed"] is True
assert saved["delete_cleanup_checkpoint"] == {
"version": 1,
"vm_uuid": VM_ID,
"snapshots": [
{
"snapshot_key": key,
"volume_name": volume_names[key],
}
for key in snapshot_keys
],
"system_cleanup": True,
}
assert len([item for item in service.calls
if item[0] == "delete_vm"]) == 1
assert all(components.store.get(
"vm_snapshot", catalog_keys[key]) is None
for key in ready_snapshot_keys)
assert components.store.get(
"vm_snapshot", catalog_keys[historical_failed_key]) is not None
assert components.state.users.owner_of_vm(scope, VM_ID) == USER["user_id"]
assert components.store.get("vm_credential", identity_key) is not None
assert components.store.get("vm_rescue", identity_key) is not None
assert components.store.get("ip_claim", claim_key) is not None
recovered = call(
client, "GET", f"/internal/v1/operations/{operation_id}", actor)
assert recovered.status_code == 200, recovered.text
assert recovered.json()["status"] == "succeeded"
assert recovered.json()["result"] == {
"uuid": VM_ID,
"deleted": True,
"deleted_snapshots": snapshot_keys,
"public_ip_claims_released": True,
"firewall_cleaned": True,
}
assert all(volume_name not in recovered.text
for volume_name in volume_names.values())
assert len([item for item in service.calls
if item[0] == "delete_vm"]) == 1
assert all(components.store.get("vm_snapshot", catalog_keys[key]) is None
for key in snapshot_keys)
assert components.state.users.owner_of_vm(scope, VM_ID) is None
assert components.store.get("vm_credential", identity_key) is None
assert components.store.get("vm_rescue", identity_key) is None
assert components.store.get("ip_claim", claim_key) is None
def test_deleted_vm_reconciles_postcommit_control_plane_failure(boundary):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
identity_key = f"{scope}.{VM_ID}"
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
components.store.set("vm_credential", identity_key, json.dumps({
"auth_mode": "ssh_key", "password": None, "generation": "g1",
}))
claim_key = f"{scope}.8.8.8.4"
components.store.set("ip_claim", claim_key, json.dumps({
"vm_uuid": VM_ID,
}))
service = services["hk"]
service.vm["state"] = "shutoff"
original_delete = components.store.delete
failed_once = False
def flaky_delete(kind, key):
nonlocal failed_once
if kind == "vm_credential" and not failed_once:
failed_once = True
raise DatabaseError("credential delete interrupted")
return original_delete(kind, key)
components.store.delete = flaky_delete
operation_id = "delete-control-cleanup"
actor = {
**USER, "operation_id": operation_id, "system_cleanup": True,
}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
first = call(client, "DELETE", path, actor)
assert first.status_code == 503, first.text
assert first.json()["error_code"] == (
"vm_delete_snapshot_cleanup_pending")
assert len([item for item in service.calls
if item[0] == "delete_vm"]) == 1
assert components.store.get("vm_credential", identity_key) is not None
assert components.store.get("ip_claim", claim_key) is not None
components.store.delete = original_delete
recovered = call(
client, "GET", f"/internal/v1/operations/{operation_id}", actor)
assert recovered.status_code == 200
assert recovered.json()["status"] == "succeeded"
assert len([item for item in service.calls
if item[0] == "delete_vm"]) == 1
assert components.state.users.owner_of_vm(scope, VM_ID) is None
assert components.store.get("vm_credential", identity_key) is None
assert components.store.get("ip_claim", claim_key) is None
def test_deleted_vm_volume_cleanup_reconciles_v2_manifest(boundary):
client, engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
service = services["hk"]
service.vm["state"] = "shutoff"
service.delete_vm_error = VMError(
"虚拟机定义已删除,但磁盘删除失败", 503,
error_code="vm_delete_volume_cleanup_pending",
)
manifest = {
"vm_uuid": VM_ID,
# Reinstall intentionally creates a fresh UUID seed unrelated to the
# VM UUID. The persisted delete checkpoint must accept that identity.
"seed_volume": "panel-seed-11111111-2222-4333-8444-555555555555.iso",
"primary_volume": "panel-restore-0123456789abcdef0123456789abcdef0123456789abcdef.qcow2",
}
service.delete_vm_cleanup_manifest = lambda vm_id: manifest.copy()
cleanup_calls = []
def cleanup_deleted_vm_volumes(vm_id, resources):
cleanup_calls.append((vm_id, resources))
return {"deleted": [resources["primary_volume"]]}
service.cleanup_deleted_vm_volumes = cleanup_deleted_vm_volumes
operation_id = "delete-volume-cleanup-v2"
actor = {**USER, "operation_id": operation_id}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
first = call(client, "DELETE", path, actor)
assert first.status_code == 503, first.text
assert first.json()["error_code"] == "vm_delete_volume_cleanup_pending"
saved = json.loads(store.get("operation", operation_id))
assert saved["cleanup_pending"] == "vm_delete"
assert saved["vm_delete_committed"] is True
assert saved["delete_cleanup_checkpoint"]["version"] == 2
assert saved["delete_cleanup_checkpoint"]["resources"] == {
"seed_volume": manifest["seed_volume"],
"primary_volume": manifest["primary_volume"],
}
recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}", actor)
assert recovered.status_code == 200, recovered.text
assert recovered.json()["status"] == "succeeded"
assert cleanup_calls == [(
VM_ID, saved["delete_cleanup_checkpoint"]["resources"])]
assert len([item for item in service.calls if item[0] == "delete_vm"]) == 1
def test_isolated_delete_cleanup_keeps_claims_and_stops_automatic_cleanup(
boundary, monkeypatch):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
component.state.users.assign_vm(scope, VM_ID, USER["user_id"])
claim_key = f"{scope}.192.0.2.1"
component.store.set("ip_claim", claim_key, json.dumps({"vm_uuid": VM_ID}))
operation_id = "isolated-delete-cleanup"
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": f"vms/{VM_ID}", "method": "DELETE",
"vm_uuid": VM_ID, "status": "uncertain", "safe_to_refund": False,
"created_at": 1, "cleanup_pending": "vm_delete",
"vm_delete_committed": True, "delete_ip_claim_keys": [claim_key],
"delete_cleanup_checkpoint": {
"version": 1, "vm_uuid": VM_ID, "snapshots": [],
"system_cleanup": False,
},
}))
attempts = []
def uncertain_cleanup(vm_id):
attempts.append(vm_id)
raise VMError("过滤器删除结果未确认", 504,
error_code="libvirt_execution_uncertain")
monkeypatch.setattr(services["hk"], "delete_firewall_filter", uncertain_cleanup)
for _ in range(2):
response = call(client, "GET", f"/internal/v1/operations/{operation_id}")
assert response.status_code == 200, response.text
assert response.json()["status"] == "uncertain"
assert response.json()["error_code"] == "libvirt_execution_uncertain"
assert attempts == [VM_ID]
assert component.state.users.owns_vm(scope, VM_ID, USER["user_id"])
assert component.store.get("ip_claim", claim_key) is not None
saved = json.loads(store.get("operation", operation_id))
assert saved["cleanup_pending"] == "vm_delete"
assert saved["delete_cleanup_checkpoint"]["vm_uuid"] == VM_ID
@pytest.mark.parametrize(("committed", "checkpoint_volume"), [
(False, None),
(True, "panel-snapshot-tampered.qcow2"),
])
def test_vm_delete_cleanup_requires_durable_commit_and_exact_identity(
boundary, committed, checkpoint_volume):
client, _engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
snapshot_key = "c" * 32
volume_name = snapshot_volume_name(VM_ID, snapshot_key)
operation_id = (
"delete-cleanup-tampered" if checkpoint_volume
else "delete-cleanup-not-committed"
)
operation = {
"operation_id": operation_id,
"host_id": "hk",
"user_id": USER["user_id"],
"vm_uuid": VM_ID,
"kind": "mutation",
"resource": f"vms/{VM_ID}",
"method": "DELETE",
"status": "uncertain",
"result": None,
"safe_to_refund": False,
"cleanup_pending": "vm_delete",
"delete_ip_claim_keys": [],
"delete_cleanup_checkpoint": {
"version": 1,
"vm_uuid": VM_ID,
"snapshots": [{
"snapshot_key": snapshot_key,
"volume_name": checkpoint_volume or volume_name,
}],
"system_cleanup": True,
},
"created_at": 1,
}
if committed:
operation["vm_delete_committed"] = True
store.set("operation", operation_id, json.dumps(operation))
before = list(services["hk"].calls)
response = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
)
assert response.status_code == 200
assert response.json()["status"] == "uncertain"
assert "delete_cleanup_checkpoint" not in response.text
assert volume_name not in response.text
if checkpoint_volume is not None:
assert checkpoint_volume not in response.text
assert services["hk"].calls == before
assert not any(item[0] in {"delete_vm", "delete_snapshot_volume"}
for item in services["hk"].calls[len(before):])
@pytest.mark.parametrize(("route", "method", "target"), [
("service-suspend", "service_suspend", {"state": "shutoff", "autostart": False}),
("service-resume", "service_resume", {"autostart": True}),
("network-suspend", "network_suspend", {"network_suspended": True}),
("network-resume", "network_resume", {"network_suspended": False}),
])
def test_private_target_state_operations_are_owned_idempotent_mutations(
boundary, route, method, target):
client, engine, _, _, services = boundary
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
assert call(client, "POST", base, USER, {}).status_code == 403
assert call(client, "POST", base, {**USER, "can_manage": False,
"operation_id": f"blocked-{route}"}, {}).status_code == 403
assert call(client, "POST", base, {**USER, "user_id": 42,
"operation_id": f"foreign-{route}"}, {}).status_code == 404
actor = {**USER, "operation_id": f"target-{route}"}
first = call(client, "POST", base, actor, {})
assert first.status_code == 200, first.text
assert first.json()["status"] == "succeeded"
assert all(first.json()[key] == value for key, value in target.items())
repeated = call(client, "POST", base, actor, {})
assert repeated.status_code == 200
assert len([item for item in services["hk"].calls if item[0] == method]) == 1
@pytest.mark.parametrize(("route", "method"), [
("service-suspend", "service_suspend"),
("service-resume", "service_resume"),
("network-suspend", "network_suspend"),
("network-resume", "network_resume"),
])
@pytest.mark.parametrize("rescue_marker", ["rescue_mode", "rescue_staged"])
def test_private_target_state_rejects_rescue_marker_before_mutation(
boundary, route, method, rescue_marker):
client, engine, _store, _logs, services = boundary
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
services["hk"].vm[rescue_marker] = True
actor = {**USER, "operation_id": f"rescue-blocked-{route}-{rescue_marker}"}
body = {"start_if_suspended": False} if route == "service-resume" else {}
response = call(client, "POST", base, actor, body)
assert response.status_code == 409, response.text
assert response.json()["status"] == "failed"
assert not any(item[0] == method for item in services["hk"].calls)
@pytest.mark.parametrize(("route", "method", "drift"), [
("service-suspend", "service_suspend", {"state": "running", "autostart": True}),
("service-resume", "service_resume", {"autostart": False}),
("network-suspend", "network_suspend", {"network_suspended": False}),
("network-resume", "network_resume", {"network_suspended": True}),
])
def test_target_state_operation_replays_after_lost_response(
boundary, route, method, drift):
client, engine, store, _, services = boundary
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
actor = {**USER, "operation_id": f"replay-{route}"}
assert call(client, "POST", base, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(drift)
engine.active_operations.clear()
replayed = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
actor)
assert replayed.status_code == 200, replayed.text
assert replayed.json()["status"] == "succeeded"
assert len([item for item in services["hk"].calls if item[0] == method]) == 2
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost again")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(drift)
replayed_post = call(client, "POST", base, actor, {})
assert replayed_post.status_code == 200, replayed_post.text
assert replayed_post.json()["status"] == "succeeded"
assert len([item for item in services["hk"].calls if item[0] == method]) == 3
@pytest.mark.parametrize(("route", "method"), [
("service-suspend", "service_suspend"),
("service-resume", "service_resume"),
("network-suspend", "network_suspend"),
("network-resume", "network_resume"),
])
@pytest.mark.parametrize("error_code", [
"libvirt_execution_uncertain", "libvirt_execution_blocked",
])
def test_isolated_target_failure_is_durable_and_never_replayed(
boundary, monkeypatch, route, method, error_code):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
attempts = []
def uncertain_target(vm_id, **options):
attempts.append((vm_id, options))
raise VMError("远端执行状态未确认", 504, error_code=error_code)
monkeypatch.setattr(services["hk"], method, uncertain_target)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
actor = {**USER, "operation_id": f"isolated-{route}"}
first = call(client, "POST", path, actor, {})
assert first.status_code == 504, first.text
assert first.json()["error_code"] == error_code
engine.active_operations.clear()
before = list(services["hk"].calls)
polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}")
repeated = call(client, "POST", path, actor, {})
for response in (first, polled, repeated):
assert response.json()["status"] == "uncertain"
assert response.json()["safe_to_refund"] is False
assert response.json()["error_code"] == error_code
saved = json.loads(store.get("operation", actor["operation_id"]))
assert saved["error_code"] == error_code
assert saved["mutation_started"] is True
assert "finished_at" not in saved
assert len(attempts) == 1
assert services["hk"].calls == before
def test_target_recovery_persists_new_execution_uncertainty_before_next_poll(
boundary, monkeypatch):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
actor = {**USER, "operation_id": "isolated-recovery-target"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume"
assert call(client, "POST", path, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None)
store.set("operation", actor["operation_id"], json.dumps(saved))
attempts = []
def uncertain_resume(vm_id, **options):
attempts.append(vm_id)
raise VMError("远端执行状态未确认", 504,
error_code="libvirt_execution_uncertain")
monkeypatch.setattr(services["hk"], "service_resume", uncertain_resume)
for _ in range(2):
recovered = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}")
assert recovered.json()["status"] == "uncertain"
assert recovered.json()["error_code"] == "libvirt_execution_uncertain"
assert attempts == [VM_ID]
saved = json.loads(store.get("operation", actor["operation_id"]))
assert saved["error_code"] == "libvirt_execution_uncertain"
assert "finished_at" not in saved
def test_execution_failure_during_network_preflight_is_not_a_safe_terminal_failure(
boundary, monkeypatch):
client, engine, store, _, _ = boundary
async def blocked_preflight(*args):
raise VMError("宿主机有未决远端操作", 503,
error_code="libvirt_execution_blocked")
monkeypatch.setattr(engine, "_capture_network_transition", blocked_preflight)
actor = {**USER, "operation_id": "isolated-network-preflight"}
response = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", actor, {},
)
assert response.status_code == 503
assert response.json()["error_code"] == "libvirt_execution_blocked"
saved = json.loads(store.get("operation", actor["operation_id"]))
assert saved["network_mutation_started"] is False
assert saved["status"] == "uncertain"
assert saved["safe_to_refund"] is False
polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}")
assert polled.json()["status"] == "uncertain"
assert polled.json()["error_code"] == "libvirt_execution_blocked"
@pytest.mark.parametrize("error_code", [
"libvirt_execution_timeout", "libvirt_execution_stopped",
])
def test_read_execution_error_is_preserved_in_operation_response(
boundary, monkeypatch, error_code):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
def unavailable_action(*args):
raise VMError("执行器暂时不可用", 503, error_code=error_code)
monkeypatch.setattr(services["hk"], "action", unavailable_action)
actor = {**USER, "operation_id": "isolated-power-read-error"}
response = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions",
actor, {"action": "start"})
assert response.status_code == 503
assert response.json()["error_code"] == error_code
saved = json.loads(store.get("operation", actor["operation_id"]))
assert saved["error_code"] == error_code
@pytest.mark.parametrize(("resource", "kind"), [
(f"vms/{VM_ID}/reinstall", "reinstall_job"),
(f"vms/{VM_ID}/rescue", "rescue_job"),
(f"vms/{VM_ID}/snapshots", "snapshot_job"),
])
def test_isolated_background_job_does_not_become_terminal_after_restart(
boundary, resource, kind):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
operation_id = f"isolated-job-{kind}"
job_id = "isolated-background-job"
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": resource, "vm_uuid": VM_ID,
"status": "pending", "result": None, "safe_to_refund": False,
"created_at": 1, "job_id": job_id,
}))
component.store.set(kind, job_id, json.dumps({
"operation_id": operation_id, "owner": "php:41", "vm_id": VM_ID,
"status": "uncertain", "phase": "面板重启后等待人工核查",
"error_code": "libvirt_execution_uncertain",
"error": "远端执行状态未确认", "credential_generation": "preserved",
}))
before = list(services["hk"].calls)
response = call(client, "GET", f"/internal/v1/operations/{operation_id}")
assert response.status_code == 200, response.text
assert response.json()["status"] == "uncertain"
assert response.json()["error_code"] == "libvirt_execution_uncertain"
assert services["hk"].calls == before
saved = json.loads(store.get("operation", operation_id))
assert saved["status"] == "uncertain"
assert saved["error_code"] == "libvirt_execution_uncertain"
assert "finished_at" not in saved
@pytest.mark.parametrize(("marker", "expected_status"), [
("live", "failed"),
("active", "failed"),
("queued", "uncertain"),
("failed", "uncertain"),
("corrupt", "uncertain"),
])
def test_target_state_replay_fails_closed_during_rescue(
boundary, marker, expected_status):
client, engine, store, _, services = boundary
route, method = "network-resume", "network_resume"
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
actor = {**USER, "operation_id": f"replay-rescue-{marker}"}
assert call(client, "POST", base, actor, {}).status_code == 200
before = len([item for item in services["hk"].calls if item[0] == method])
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
if marker == "live":
services["hk"].vm["rescue_mode"] = True
elif marker == "corrupt":
component.store.set(
"vm_rescue", f"{host_scope(component.manager.profile)}.{VM_ID}",
"not-json",
)
elif marker in {"active", "queued", "failed"}:
component.store.set(
"vm_rescue", f"{host_scope(component.manager.profile)}.{VM_ID}",
json.dumps({"vm_id": VM_ID, "status": marker}),
)
engine.active_operations.clear()
replayed = call(
client, "GET", f"/internal/v1/operations/{actor['operation_id']}", actor,
)
assert replayed.status_code == 200, replayed.text
assert replayed.json()["status"] == expected_status
assert len([item for item in services["hk"].calls if item[0] == method]) == before
def test_non_reconciling_operation_lookup_never_replays_target_state(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend"
actor = {**USER, "operation_id": "peek-target-state"}
assert call(client, "POST", path, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm["network_suspended"] = False
engine.active_operations.clear()
before = len(services["hk"].calls)
peeked = call(client, "GET",
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
assert peeked.status_code == 200
assert peeked.json()["status"] == "uncertain"
assert peeked.json()["executor_active"] is False
assert len(services["hk"].calls) == before
assert services["hk"].vm["network_suspended"] is False
claim = engine.coordinator.claim(
operation_id=actor["operation_id"], resources=(f"vm:hk:{VM_ID}",))
active = call(client, "GET",
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
assert active.status_code == 200
assert active.json()["executor_active"] is True
assert len(services["hk"].calls) == before
assert engine.coordinator.release(claim)
def test_target_state_lock_serializes_same_vm_and_not_other_vms(boundary):
_, engine, *_ = boundary
async def exercise():
first = engine._target_lock("hk", VM_ID)
same = engine._target_lock("hk", VM_ID)
other = engine._target_lock("hk", "22222222-2222-4222-8222-222222222222")
assert first is same
assert first is not other
entered = []
owner_entered = asyncio.Event()
release = asyncio.Event()
async def owner():
async with first:
entered.append("owner")
owner_entered.set()
await release.wait()
entered.append("owner-finished")
async def waiter():
async with same:
entered.append("waiter")
owner_task = asyncio.create_task(owner())
await asyncio.wait_for(owner_entered.wait(), timeout=1)
waiter_task = asyncio.create_task(waiter())
await asyncio.sleep(0)
assert entered == ["owner"]
release.set()
await asyncio.gather(owner_task, waiter_task)
assert entered == ["owner", "owner-finished", "waiter"]
errors = []
def run_in_isolated_loop():
try:
asyncio.run(exercise())
except BaseException as exc: # propagate assertion failures to pytest
errors.append(exc)
thread = threading.Thread(target=run_in_isolated_loop)
thread.start()
thread.join(timeout=5)
assert not thread.is_alive()
if errors:
raise errors[0]
def test_expiry_target_retries_after_inflight_user_mutation_on_same_vm(boundary):
client, engine, _, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
service = services["hk"]
action_entered = threading.Event()
release_action = threading.Event()
network_entered = threading.Event()
original_action = service.action
original_network_suspend = service.network_suspend
def slow_action(vm_id, action):
action_entered.set()
assert release_action.wait(timeout=5), "test did not release user mutation"
return original_action(vm_id, action)
def observed_network_suspend(vm_id):
network_entered.set()
return original_network_suspend(vm_id)
service.action = slow_action
service.network_suspend = observed_network_suspend
responses = {}
def user_request():
responses["action"] = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions",
{**USER, "operation_id": "concurrent-user-action"}, {"action": "reset"})
def policy_request():
responses["policy"] = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
{**USER, "operation_id": "concurrent-expiry-policy"}, {})
first = threading.Thread(target=user_request)
first.start()
assert action_entered.wait(timeout=5)
second = threading.Thread(target=policy_request)
second.start()
try:
second.join(timeout=2)
assert not second.is_alive()
assert responses["policy"].status_code == 409
assert responses["policy"].json()["error_code"] == "coordination_busy"
assert not network_entered.is_set()
finally:
release_action.set()
first.join(timeout=5)
assert not first.is_alive() and not second.is_alive()
assert responses["action"].status_code == 200
policy_request()
assert responses["policy"].status_code == 200
assert network_entered.is_set()
mutations = [item[0] for item in service.calls
if item[0] in {"action", "network_suspend"}]
assert mutations == ["action", "network_suspend"]
assert service.vm["network_suspended"] is True
def test_target_state_reconciliation_does_not_bypass_original_owner(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
store.set("operation", "unsafe-target", json.dumps({
"operation_id": "unsafe-target", "host_id": "hk", "user_id": 42,
"kind": "mutation", "resource": f"vms/{VM_ID}/network-suspend",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"actor_can_manage": True, "actor_is_admin": False,
"safe_to_refund": False,
}))
response = call(client, "GET", "/internal/v1/operations/unsafe-target",
{**USER, "user_id": 42})
assert response.status_code == 200
assert response.json()["status"] == "failed"
assert not any(item[0] == "network_suspend" for item in services["hk"].calls)
def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID,
"status": "running", "phase": "正在救援"}
components.store.set("rescue_job", "job-abc", json.dumps(job))
operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID,
"status": "uncertain", "result": None, "safe_to_refund": False}
store.set("operation", "lost-rescue-job", json.dumps(operation))
recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID
job.update(status="completed", phase="救援系统启动")
components.store.set("rescue_job", "job-abc", json.dumps(job))
completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
assert completed["status"] == "succeeded"
assert not completed["safe_to_refund"]
def test_snapshot_mutation_operation_returns_only_safe_result(boundary):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
services["hk"].vm.update(
managed=True, os_source="cloud_template",
rescue_mode=False, rescue_staged=False,
)
actor = {**USER, "operation_id": "snapshot-create-0001"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots"
queued = call(client, "POST", path, actor, {
"snapshot_key": "a" * 32,
"name": "发布前恢复点",
"consistency_mode": "crash",
})
assert queued.status_code == 202, queued.text
for _ in range(100):
operation = call(
client, "GET", "/internal/v1/operations/snapshot-create-0001",
actor,
).json()
if operation["status"] != "running" and operation["status"] != "pending":
break
time.sleep(0.01)
assert operation["status"] == "succeeded"
assert operation["result"]["snapshot_key"] == "a" * 32
assert operation["result"]["created"] is True
assert operation["result"]["consistency"] == "crash"
assert "volume_name" not in operation["result"]
def test_snapshot_qga_failure_code_survives_operation_poll(boundary):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
services["hk"].vm.update(
managed=True, os_source="cloud_template",
rescue_mode=False, rescue_staged=False,
)
services["hk"].snapshot_create_error = VMError(
"QEMU Guest Agent 无法冻结文件系统,快照未创建", 409,
error_code="snapshot_guest_agent_unavailable",
)
actor = {**USER, "operation_id": "snapshot-create-qga1"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots"
queued = call(client, "POST", path, actor, {
"snapshot_key": "b" * 32,
"name": "一致性恢复点",
"consistency_mode": "filesystem",
})
assert queued.status_code == 202, queued.text
for _ in range(100):
operation = call(
client, "GET", "/internal/v1/operations/snapshot-create-qga1",
actor,
).json()
if operation["status"] not in {"running", "pending"}:
break
time.sleep(0.01)
assert operation["status"] == "failed"
assert operation["error_code"] == "snapshot_guest_agent_unavailable"
def test_snapshot_restart_recovers_freeze_and_orphan_transition(boundary):
client, _engine, store, _logs, services = boundary
scoped = HostStore(store, "hk")
freeze_key = "c" * 32
orphan_key = "d" * 32
for snapshot_key in (freeze_key, orphan_key):
scoped.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
"snapshot_key": snapshot_key,
"vm_id": VM_ID,
"name": "中断恢复点",
"state": "creating",
"consistency": "filesystem",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
scoped.set("snapshot_job", "snapshot-interrupted", json.dumps({
"operation_id": "snapshot-interrupted",
"owner": "php:41",
"vm_id": VM_ID,
"snapshot_key": freeze_key,
"action": "create",
"consistency": "filesystem",
"guest_freeze_state": "attempted",
"status": "uncertain",
"phase": "正在通过 QEMU Guest Agent 冻结访客文件系统",
}))
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
interrupted = json.loads(scoped.get(
"snapshot_job", "snapshot-interrupted"))
assert interrupted["status"] == "uncertain"
assert interrupted["guest_freeze_state"] == "recovered"
assert interrupted["freeze_recovery"] == "confirmed"
assert ("thaw_guest_filesystems", VM_ID) in services["hk"].calls
record = json.loads(scoped.get(
"vm_snapshot", f"{VM_ID}:{freeze_key}"))
assert record["state"] == "uncertain"
assert scoped.get(
"vm_snapshot", f"{VM_ID}:{orphan_key}") is None
@pytest.mark.parametrize(("initial_state", "expected_state"), [
("creating", None),
("restoring", "ready"),
("deleting", "ready"),
])
def test_snapshot_registry_without_durable_job_rolls_back_before_mutation(
boundary, initial_state, expected_state):
client, _engine, store, _logs, services = boundary
scoped = HostStore(store, "hk")
snapshot_key = {
"creating": "b",
"restoring": "c",
"deleting": "d",
}[initial_state] * 32
scoped.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
"snapshot_key": snapshot_key,
"vm_id": VM_ID,
"name": "尚未入队恢复点",
"state": initial_state,
"consistency": "crash",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
raw = scoped.get("vm_snapshot", f"{VM_ID}:{snapshot_key}")
if expected_state is None:
assert raw is None
else:
assert json.loads(raw)["state"] == expected_state
assert not any(call_[0] in {
"create_live_snapshot", "restore_live_snapshot",
"delete_snapshot_volume",
} for call_ in services["hk"].calls)
@pytest.mark.parametrize(("action", "initial_state", "expected_state"), [
("create", "creating", None),
("restore", "restoring", "ready"),
("delete", "deleting", "ready"),
])
def test_snapshot_cancel_tombstone_finishes_catalog_rollback_after_restart(
boundary, action, initial_state, expected_state):
client, _engine, store, _logs, services = boundary
scoped = HostStore(store, "hk")
snapshot_key = {
"create": "e", "restore": "f", "delete": "a",
}[action] * 32
catalog_key = f"{VM_ID}:{snapshot_key}"
scoped.set("vm_snapshot", catalog_key, json.dumps({
"snapshot_key": snapshot_key,
"vm_id": VM_ID,
"name": "取消中的排队恢复点",
"state": initial_state,
"consistency": "crash",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
job_id = f"cancelled-before-start-{action}"
operation_id = f"snapshot-cancel-restart-{action}"
scoped.set("snapshot_job", job_id, json.dumps({
"job_id": job_id,
"operation_id": operation_id,
"scope": host_scope({"host_id": "hk"}),
"owner": "php:41",
"vm_id": VM_ID,
"snapshot_key": snapshot_key,
"action": action,
"consistency": "crash",
"status": "queued",
"phase": "正在取消尚未开始的排队快照",
"created_at": 1,
"cancelled_before_start": True,
"cancel_reason": "network_suspend",
"cancel_requested_at": 1,
}))
resource = f"vms/{VM_ID}/snapshots"
method = "POST"
if action != "create":
resource += f"/{snapshot_key}"
if action == "restore":
resource += "/restore"
else:
method = "DELETE"
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id,
"host_id": "hk",
"user_id": USER["user_id"],
"kind": "mutation",
"resource": resource,
"method": method,
"vm_uuid": VM_ID,
"status": "pending",
"job_id": job_id,
"result": {"job_id": job_id, "status": "queued"},
"safe_to_refund": False,
"created_at": 1,
}))
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
saved_job = json.loads(scoped.get("snapshot_job", job_id))
assert saved_job["status"] == "failed"
assert saved_job["cancelled_before_start"] is True
raw = scoped.get("vm_snapshot", catalog_key)
if expected_state is None:
assert raw is None
else:
assert json.loads(raw)["state"] == expected_state
operation = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
)
assert operation.status_code == 200
assert operation.json()["status"] == "failed"
assert not any(call_[0] in {
"create_live_snapshot", "restore_live_snapshot",
"delete_snapshot_volume",
} for call_ in services["hk"].calls)
def test_snapshot_uncertain_restore_hides_private_volume_evidence(boundary):
client, engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
snapshot_key = "4" * 32
operation_id = "snapshot-restore-private"
job_id = "snapshot-job-private"
snapshot_volume = snapshot_volume_name(VM_ID, snapshot_key)
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "恢复点",
"state": "uncertain", "consistency": "crash",
"volume_name": snapshot_volume, "created_at": 1,
}))
private_old = f"panel-{VM_ID}.qcow2"
private_replacement = "panel-restore-private.qcow2"
components.store.set("snapshot_job", job_id, json.dumps({
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "restore",
"consistency": "crash", "status": "uncertain", "phase": "结果待核查",
"restore_old_volume": private_old,
"restore_replacement_volume": private_replacement,
"restore_was_running": True,
}))
services["hk"].snapshot_reconcile_restore = {"outcome": "uncertain"}
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation",
"resource": f"vms/{VM_ID}/snapshots/{snapshot_key}/restore",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"safe_to_refund": False,
}))
operation = call(
client, "GET", f"/internal/v1/operations/{operation_id}").json()
assert operation["status"] == "uncertain"
serialized = json.dumps(operation, ensure_ascii=False)
assert private_old not in serialized
assert private_replacement not in serialized
assert snapshot_volume not in serialized
def test_snapshot_restore_without_persisted_plan_converges_failed_without_replay(boundary):
client, engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
snapshot_key = "e" * 32
operation_id = "snapshot-restore-no-plan"
job_id = "snapshot-job-no-plan"
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "恢复点",
"state": "uncertain", "consistency": "crash",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
components.store.set("snapshot_job", job_id, json.dumps({
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "restore",
"consistency": "crash", "status": "uncertain", "phase": "面板重启",
}))
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation",
"resource": f"vms/{VM_ID}/snapshots/{snapshot_key}/restore",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"safe_to_refund": False,
}))
operation = call(
client, "GET", f"/internal/v1/operations/{operation_id}").json()
assert operation["status"] == "failed"
record = json.loads(components.store.get(
"vm_snapshot", f"{VM_ID}:{snapshot_key}"))
assert record["state"] == "ready"
assert not any(call_[0] in {
"restore_live_snapshot", "reconcile_live_snapshot_restore",
} for call_ in services["hk"].calls)
def test_snapshot_uncertain_delete_without_catalog_converges_by_original_operation(boundary):
client, engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
snapshot_key = "5" * 32
operation_id = "snapshot-delete-reconcile"
job_id = "snapshot-job-delete"
components.store.set("snapshot_job", job_id, json.dumps({
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "delete",
"consistency": "filesystem", "status": "uncertain",
"phase": "结果待核查",
}))
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation",
"resource": f"vms/{VM_ID}/snapshots/{snapshot_key}",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"safe_to_refund": False,
}))
operation = call(
client, "GET", f"/internal/v1/operations/{operation_id}").json()
assert operation["status"] == "succeeded"
assert operation["result"] == {
"snapshot_key": snapshot_key,
"deleted": True,
"consistency": "filesystem",
}
saved_job = json.loads(components.store.get("snapshot_job", job_id))
assert saved_job["status"] == "succeeded"
assert not any(call_[0] == "delete_snapshot_volume"
for call_ in services["hk"].calls)
assert len([call_ for call_ in services["hk"].calls
if call_[0] == "reconcile_live_snapshot_delete"]) == 1
def test_snapshot_uncertain_create_reconciles_with_consistency(boundary):
client, engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
snapshot_key = "6" * 32
operation_id = "snapshot-create-reconcile"
job_id = "snapshot-job-create"
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "恢复点",
"state": "uncertain", "consistency": "crash",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
components.store.set("snapshot_job", job_id, json.dumps({
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "create",
"consistency": "crash", "status": "uncertain", "phase": "结果待核查",
"backup_verified_capacity": 20 * 1024**3,
"backup_verified_identity": "a" * 64,
}))
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation",
"resource": f"vms/{VM_ID}/snapshots",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"safe_to_refund": False,
}))
operation = call(
client, "GET", f"/internal/v1/operations/{operation_id}").json()
assert operation["status"] == "succeeded"
assert operation["result"]["created"] is True
assert operation["result"]["consistency"] == "crash"
assert "volume_name" not in operation["result"]
assert not any(call_[0] == "create_live_snapshot"
for call_ in services["hk"].calls)
assert len([call_ for call_ in services["hk"].calls
if call_[0] == "reconcile_live_snapshot_create"]) == 1
def test_snapshot_confirmed_failed_create_removes_hidden_catalog(boundary):
client, engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
snapshot_key = "b" * 32
operation_id = "snapshot-create-confirmed-failed"
job_id = "snapshot-job-create-failed"
catalog_key = f"{VM_ID}:{snapshot_key}"
components.store.set("vm_snapshot", catalog_key, json.dumps({
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "失败恢复点",
"state": "uncertain", "consistency": "crash",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
components.store.set("snapshot_job", job_id, json.dumps({
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "create",
"consistency": "crash", "status": "uncertain", "phase": "结果待核查",
}))
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": f"vms/{VM_ID}/snapshots",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"safe_to_refund": False,
}))
services["hk"].snapshot_reconcile_create = {
"outcome": "failed", "snapshot_key": snapshot_key,
}
operation = call(
client, "GET", f"/internal/v1/operations/{operation_id}").json()
assert operation["status"] == "failed"
assert components.store.get("vm_snapshot", catalog_key) is None
saved_job = json.loads(components.store.get("snapshot_job", job_id))
assert saved_job["status"] == "failed"
@pytest.mark.parametrize(
"snapshot_state", ["creating", "restoring", "deleting", "uncertain"])
def test_transitional_snapshot_catalog_blocks_power_mutation_without_pending_job(
boundary, snapshot_state):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
snapshot_key = "7" * 32
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "中断恢复点",
"state": snapshot_state, "consistency": "crash",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
actor = {**USER, "operation_id": "power-blocked-by-snapshot"}
response = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions",
actor, {"action": "poweroff"})
assert response.status_code == 409
assert not any(call_[0] == "action" for call_ in services["hk"].calls)
def test_network_suspend_bypasses_durable_snapshot_uncertain_when_no_backup(
boundary):
route, method = "network-suspend", "network_suspend"
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
snapshot_key = "8" * 32
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
"snapshot_key": snapshot_key, "vm_id": VM_ID,
"name": "待核查恢复点", "state": "uncertain",
"consistency": "crash",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
actor = {**USER, "operation_id": f"expiry-uncertain-{route}"}
response = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}",
actor, {})
assert response.status_code == 200, response.text
assert response.json()["status"] == "succeeded"
assert len([item for item in services["hk"].calls
if item[0] == method]) == 1
def test_network_suspend_waits_while_snapshot_backup_is_still_active(
boundary):
route, method = "network-suspend", "network_suspend"
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
snapshot_key = "9" * 32
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
"snapshot_key": snapshot_key, "vm_id": VM_ID,
"name": "复制中的恢复点", "state": "uncertain",
"consistency": "filesystem",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
services["hk"].snapshot_backup_active = True
actor = {**USER, "operation_id": f"expiry-active-{route}"}
response = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}",
actor, {})
assert response.status_code == 409, response.text
assert response.json()["status"] == "failed"
assert len([item for item in services["hk"].calls
if item[0] == method]) == 1
@pytest.mark.parametrize(("action", "method"), [
("create", "create_live_snapshot"),
("restore", "restore_live_snapshot"),
("delete", "delete_snapshot_volume"),
])
def test_network_suspend_cancels_same_vm_snapshot_still_queued_behind_host_job(
boundary, action, method):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
profile = components.manager.profile
scope = host_scope(profile)
service = services["hk"]
queued_vm = "22222222-2222-4222-8222-222222222222"
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
components.state.users.assign_vm(scope, queued_vm, USER["user_id"])
service.vm.update(
managed=True, os_source="cloud_template",
rescue_mode=False, rescue_staged=False,
)
queued_vm_record = {
**service.vm,
"uuid": queued_vm,
"name": "queued-vm",
"network_suspended": False,
}
original_get_vm = service.get_vm
original_create = service.create_live_snapshot
original_network_suspend = service.network_suspend
def get_vm(vm_id):
if vm_id == queued_vm:
service.calls.append(("get_vm", vm_id))
return dict(queued_vm_record)
return original_get_vm(vm_id)
blocker_started = threading.Event()
blocker_release = threading.Event()
def create_live_snapshot(vm_id, snapshot_key, *, consistency,
progress=None, recovery_job_id=None):
if vm_id == VM_ID:
blocker_started.set()
assert blocker_release.wait(timeout=5)
return original_create(
vm_id, snapshot_key, consistency=consistency,
progress=progress)
def network_suspend(vm_id):
if vm_id == queued_vm:
service.calls.append(("network_suspend", vm_id))
queued_vm_record["network_suspended"] = True
return dict(queued_vm_record)
return original_network_suspend(vm_id)
service.get_vm = get_vm
service.create_live_snapshot = create_live_snapshot
service.network_suspend = network_suspend
blocker_actor = {**USER, "operation_id": f"snapshot-blocker-{action}"}
blocker_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots"
blocker = call(client, "POST", blocker_path, blocker_actor, {
"snapshot_key": "a" * 32,
"name": "占用宿主快照执行器",
"consistency_mode": "crash",
})
assert blocker.status_code == 202, blocker.text
assert blocker_started.wait(timeout=2)
snapshot_key = "b" * 32
catalog_key = f"{queued_vm}:{snapshot_key}"
if action != "create":
components.store.set("vm_snapshot", catalog_key, json.dumps({
"snapshot_key": snapshot_key,
"vm_id": queued_vm,
"name": "待操作恢复点",
"state": "ready",
"consistency": "crash",
"volume_name": snapshot_volume_name(queued_vm, snapshot_key),
"created_at": 1,
}))
snapshot_actor = {
**USER, "operation_id": f"queued-{action}-cancel",
}
if action == "create":
queued_path = f"/internal/v1/hosts/hk/vms/{queued_vm}/snapshots"
queued_body = {
"snapshot_key": snapshot_key,
"name": "应被到期策略取消",
"consistency_mode": "crash",
}
queued_method = "POST"
else:
queued_path = (
f"/internal/v1/hosts/hk/vms/{queued_vm}/snapshots/"
f"{snapshot_key}"
+ ("/restore" if action == "restore" else "")
)
queued_body = {}
queued_method = "POST" if action == "restore" else "DELETE"
try:
queued = call(
client, queued_method, queued_path, snapshot_actor, queued_body)
assert queued.status_code == 202, queued.text
suspend_actor = {
**USER, "operation_id": f"expiry-network-{action}",
}
suspended = call(
client, "POST",
f"/internal/v1/hosts/hk/vms/{queued_vm}/network-suspend",
suspend_actor, {},
)
assert suspended.status_code == 200, suspended.text
assert suspended.json()["status"] == "succeeded"
snapshot_operation = call(
client, "GET",
f"/internal/v1/operations/{snapshot_actor['operation_id']}",
snapshot_actor,
)
assert snapshot_operation.status_code == 200
assert snapshot_operation.json()["status"] == "failed"
assert "未调用虚拟化操作" in snapshot_operation.json()["error"]
assert not any(
item[0] == method and len(item) > 1 and item[1] == queued_vm
for item in service.calls
)
raw = components.store.get("vm_snapshot", catalog_key)
if action == "create":
assert raw is None
else:
assert json.loads(raw)["state"] == "ready"
finally:
blocker_release.set()
def test_network_suspend_rejects_snapshot_future_already_running(boundary):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
service = services["hk"]
service.vm.update(
managed=True, os_source="cloud_template",
rescue_mode=False, rescue_staged=False,
)
original_create = service.create_live_snapshot
started = threading.Event()
release = threading.Event()
def running_snapshot(vm_id, snapshot_key, *, consistency, progress=None,
recovery_job_id=None):
started.set()
assert release.wait(timeout=5)
return original_create(
vm_id, snapshot_key, consistency=consistency,
progress=progress, recovery_job_id=recovery_job_id)
service.create_live_snapshot = running_snapshot
snapshot_actor = {**USER, "operation_id": "running-snapshot-guard"}
queued = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots",
snapshot_actor, {
"snapshot_key": "c" * 32,
"name": "已开始的快照",
"consistency_mode": "crash",
},
)
assert queued.status_code == 202
assert started.wait(timeout=2)
try:
before = len([item for item in service.calls
if item[0] == "network_suspend"])
suspended = call(
client, "POST",
f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
{**USER, "operation_id": "running-snapshot-network"}, {},
)
assert suspended.status_code == 409
assert suspended.json()["error_code"] == "coordination_busy"
assert _store.get("operation", "running-snapshot-network") is None
assert len([item for item in service.calls
if item[0] == "network_suspend"]) == before
finally:
release.set()
def test_cancelled_snapshot_phase_one_save_failure_converges_on_poll(boundary):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
service = services["hk"]
queued_vm = "33333333-3333-4333-8333-333333333333"
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
components.state.users.assign_vm(scope, queued_vm, USER["user_id"])
service.vm.update(
managed=True, os_source="cloud_template",
rescue_mode=False, rescue_staged=False,
)
queued_vm_record = {
**service.vm, "uuid": queued_vm, "name": "queued-save-failure",
}
original_get_vm = service.get_vm
original_create = service.create_live_snapshot
original_network_suspend = service.network_suspend
def get_vm(vm_id):
return (dict(queued_vm_record) if vm_id == queued_vm
else original_get_vm(vm_id))
blocker_started = threading.Event()
blocker_release = threading.Event()
def create_live_snapshot(vm_id, snapshot_key, *, consistency,
progress=None, recovery_job_id=None):
if vm_id == VM_ID:
blocker_started.set()
assert blocker_release.wait(timeout=5)
return original_create(
vm_id, snapshot_key, consistency=consistency,
progress=progress)
def network_suspend(vm_id):
if vm_id == queued_vm:
service.calls.append(("network_suspend", vm_id))
return {**queued_vm_record, "network_suspended": True}
return original_network_suspend(vm_id)
service.get_vm = get_vm
service.create_live_snapshot = create_live_snapshot
service.network_suspend = network_suspend
blocker = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots",
{**USER, "operation_id": "phase-one-blocker"}, {
"snapshot_key": "d" * 32,
"name": "占用执行器",
"consistency_mode": "crash",
},
)
assert blocker.status_code == 202
assert blocker_started.wait(timeout=2)
snapshot_actor = {**USER, "operation_id": "phase-one-cancelled"}
snapshot_key = "e" * 32
queued = call(
client, "POST",
f"/internal/v1/hosts/hk/vms/{queued_vm}/snapshots",
snapshot_actor, {
"snapshot_key": snapshot_key,
"name": "取消写入故障",
"consistency_mode": "crash",
},
)
assert queued.status_code == 202
original_set = components.store.set
failed_once = False
def flaky_set(kind, key, value):
nonlocal failed_once
if kind == "snapshot_job" and not failed_once:
payload = json.loads(value)
if (payload.get("cancelled_before_start") is True
and payload.get("status") == "queued"):
failed_once = True
raise DatabaseError("phase-one tombstone write failed")
return original_set(kind, key, value)
components.store.set = flaky_set
try:
first_suspend = call(
client, "POST",
f"/internal/v1/hosts/hk/vms/{queued_vm}/network-suspend",
{**USER, "operation_id": "phase-one-network-first"}, {},
)
assert first_suspend.status_code == 503
assert failed_once is True
snapshot_operation = call(
client, "GET",
f"/internal/v1/operations/{snapshot_actor['operation_id']}",
snapshot_actor,
)
assert snapshot_operation.status_code == 200
assert snapshot_operation.json()["status"] == "failed"
assert components.store.get(
"vm_snapshot", f"{queued_vm}:{snapshot_key}") is None
assert not any(
item[0] == "create_live_snapshot" and item[1] == queued_vm
for item in service.calls
)
second_suspend = call(
client, "POST",
f"/internal/v1/hosts/hk/vms/{queued_vm}/network-suspend",
{**USER, "operation_id": "phase-one-network-retry"}, {},
)
assert second_suspend.status_code == 200, second_suspend.text
finally:
components.store.set = original_set
blocker_release.set()
def test_network_suspend_barrier_rejects_snapshot_admission_until_target_lands(
boundary):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
service = services["hk"]
vm_id = "44444444-4444-4444-8444-444444444444"
components.state.users.assign_vm(scope, vm_id, USER["user_id"])
vm = {
**service.vm,
"uuid": vm_id,
"name": "barrier-vm",
"managed": True,
"os_source": "cloud_template",
"rescue_mode": False,
"rescue_staged": False,
}
original_get_vm = service.get_vm
original_network_suspend = service.network_suspend
def get_vm(target):
return dict(vm) if target == vm_id else original_get_vm(target)
suspend_started = threading.Event()
suspend_release = threading.Event()
def network_suspend(target):
if target == vm_id:
service.calls.append(("network_suspend", target))
suspend_started.set()
assert suspend_release.wait(timeout=5)
return {**vm, "network_suspended": True}
return original_network_suspend(target)
service.get_vm = get_vm
service.network_suspend = network_suspend
suspend_actor = {**USER, "operation_id": "barrier-network-suspend"}
direct = TestClient(
engine.registry.apps["hk"], base_url="http://127.0.0.1")
snapshot_key = "f" * 32
try:
with ThreadPoolExecutor(max_workers=1) as executor:
pending_suspend = executor.submit(
call, client, "POST",
f"/internal/v1/hosts/hk/vms/{vm_id}/network-suspend",
suspend_actor, {},
)
assert suspend_started.wait(timeout=2)
token = TRUSTED_ACTOR.set({
**USER,
"operation_id": "barrier-snapshot-rejected",
})
try:
rejected = direct.post(
f"/api/v1/vms/{vm_id}/snapshots",
json={
"snapshot_key": snapshot_key,
"name": "不应越过暂停屏障",
"consistency_mode": "crash",
},
)
finally:
TRUSTED_ACTOR.reset(token)
assert rejected.status_code == 409, rejected.text
assert components.store.get(
"vm_snapshot", f"{vm_id}:{snapshot_key}") is None
assert not any(
json.loads(raw).get("snapshot_key") == snapshot_key
for raw in components.store.list("snapshot_job").values()
)
suspend_release.set()
suspended = pending_suspend.result(timeout=5)
assert suspended.status_code == 200, suspended.text
finally:
suspend_release.set()
direct.close()
@pytest.mark.parametrize("action", ["create", "restore", "delete"])
def test_initial_snapshot_job_save_failure_leaves_no_network_suspend_ghost(
boundary, action):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
service = services["hk"]
service.vm.update(
managed=True, os_source="cloud_template",
rescue_mode=False, rescue_staged=False,
)
snapshot_key = {
"create": "7", "restore": "8", "delete": "9",
}[action] * 32
catalog_key = f"{VM_ID}:{snapshot_key}"
if action != "create":
components.store.set("vm_snapshot", catalog_key, json.dumps({
"snapshot_key": snapshot_key,
"vm_id": VM_ID,
"name": "初始写入故障恢复点",
"state": "ready",
"consistency": "crash",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
original_set = components.store.set
failed_once = False
def flaky_set(kind, key, value):
nonlocal failed_once
if kind == "snapshot_job" and not failed_once:
failed_once = True
raise DatabaseError("initial snapshot job write failed")
return original_set(kind, key, value)
components.store.set = flaky_set
actor = {**USER, "operation_id": f"initial-save-failure-{action}"}
if action == "create":
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots"
body = {
"snapshot_key": snapshot_key,
"name": "初始写入故障",
"consistency_mode": "crash",
}
method = "POST"
else:
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots/{snapshot_key}"
if action == "restore":
path += "/restore"
method = "POST"
else:
method = "DELETE"
body = {}
try:
rejected = call(client, method, path, actor, body)
assert rejected.status_code == 503
assert failed_once is True
finally:
components.store.set = original_set
raw = components.store.get("vm_snapshot", catalog_key)
if action == "create":
assert raw is None
else:
assert json.loads(raw)["state"] == "ready"
assert not any(call_[0] in {
"create_live_snapshot", "restore_live_snapshot",
"delete_snapshot_volume",
} for call_ in service.calls)
suspended = call(
client, "POST",
f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
{**USER, "operation_id": f"network-after-save-failure-{action}"}, {},
)
assert suspended.status_code == 200, suspended.text
def test_snapshot_executor_submit_failure_is_terminal_and_does_not_block_suspend(
boundary):
client, engine, _store, _logs, services = boundary
class FailingSnapshotExecutor:
def submit(self, *_args, **_kwargs):
raise RuntimeError("executor stopped")
def shutdown(self, **_kwargs):
return None
def live_workers(self):
return ()
real_executor = ManagedThreadPoolExecutor
def executor_factory(*args, **kwargs):
if kwargs.get("thread_name_prefix") == "vm-snapshot":
return FailingSnapshotExecutor()
return real_executor(*args, **kwargs)
with patch("virsh_panel.app.ManagedThreadPoolExecutor",
side_effect=executor_factory):
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
services["hk"].vm.update(
managed=True, os_source="cloud_template",
rescue_mode=False, rescue_staged=False,
)
actor = {**USER, "operation_id": "snapshot-submit-terminal"}
snapshot_key = "6" * 32
rejected = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots",
actor, {
"snapshot_key": snapshot_key,
"name": "执行器停止",
"consistency_mode": "crash",
},
)
assert rejected.status_code == 503
assert components.store.get(
"vm_snapshot", f"{VM_ID}:{snapshot_key}") is None
operation = call(
client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
actor,
)
assert operation.status_code == 200
assert operation.json()["status"] == "failed"
assert "未调用虚拟化操作" in operation.json()["error"]
assert not any(item[0] == "create_live_snapshot"
for item in services["hk"].calls)
suspended = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
{**USER, "operation_id": "network-after-submit-failure"}, {},
)
assert suspended.status_code == 200, suspended.text
@pytest.mark.parametrize(("route", "method"), [
("service-suspend", "service_suspend"),
("service-resume", "service_resume"),
("network-resume", "network_resume"),
])
def test_non_fail_safe_targets_remain_blocked_by_snapshot_uncertain(
boundary, route, method):
client, engine, _store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
snapshot_key = "a" * 32
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
"snapshot_key": snapshot_key, "vm_id": VM_ID,
"name": "待核查恢复点", "state": "uncertain",
"consistency": "crash",
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
"created_at": 1,
}))
actor = {**USER, "operation_id": f"resume-blocked-{route}"}
body = {"start_if_suspended": False} if route == "service-resume" else {}
response = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}",
actor, body)
assert response.status_code == 409, response.text
assert not any(item[0] == method for item in services["hk"].calls)
def test_interrupted_async_job_becomes_terminal_without_replay(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.store.set("reinstall_job", "job-abcd", json.dumps({
"operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID,
"status": "failed", "phase": "面板重启,任务已中断"}))
store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job",
"host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall",
"vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False}))
recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json()
assert recovered["status"] == "failed"
assert "中断" in recovered["error"]
def test_reinstall_cleanup_pending_stays_uncertain_without_replay(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.store.set("reinstall_job", "job-cleanup-pending", json.dumps({
"operation_id": "reinstall-cleanup-pending",
"owner": "php:41", "vm_id": VM_ID,
"status": "uncertain", "phase": "重装已完成,但旧系统盘清理待核对",
"error": "重装已完成,但旧系统盘清理待核对",
"error_code": "reinstall_cleanup_pending",
"reinstall_committed": True, "cleanup_pending": True,
"vm": {"uuid": VM_ID, "name": "cloud-vm", "state": "running"},
"warning": "旧存储卷 panel-old.qcow2 删除失败,请检查宿主机",
}))
store.set("operation", "reinstall-cleanup-pending", json.dumps({
"operation_id": "reinstall-cleanup-pending", "host_id": "hk",
"user_id": 41, "kind": "mutation",
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID,
"status": "pending", "result": None, "safe_to_refund": False,
}))
recovered = call(
client, "GET", "/internal/v1/operations/reinstall-cleanup-pending",
USER,
)
payload = recovered.json()
assert recovered.status_code == 200
assert payload["status"] == "uncertain"
assert payload["error_code"] == "reinstall_cleanup_pending"
assert payload["result"]["reinstall_committed"] is True
assert payload["result"]["cleanup_pending"] is True
assert payload["result"]["vm"]["state"] == "running"
def test_committed_reinstall_restart_marker_is_not_downgraded(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.store.set("reinstall_job", "job-restart-committed", json.dumps({
"operation_id": "reinstall-restart-committed",
"owner": "php:41", "vm_id": VM_ID,
"status": "failed", "phase": "面板重启,任务已中断",
"error": "面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交",
"reinstall_committed": True,
}))
store.set("operation", "reinstall-restart-committed", json.dumps({
"operation_id": "reinstall-restart-committed", "host_id": "hk",
"user_id": 41, "kind": "mutation",
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID,
"status": "pending", "result": None, "safe_to_refund": False,
}))
recovered = call(
client, "GET", "/internal/v1/operations/reinstall-restart-committed",
USER,
)
payload = recovered.json()
assert recovered.status_code == 200
assert payload["status"] == "uncertain"
assert payload["error_code"] == "reinstall_result_uncertain"
def test_committed_running_reinstall_marker_is_uncertain(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.store.set("reinstall_job", "job-running-committed", json.dumps({
"operation_id": "reinstall-running-committed", "owner": "php:41",
"vm_id": VM_ID, "status": "running", "phase": "cleaning_up",
"reinstall_committed": True,
}))
store.set("operation", "reinstall-running-committed", json.dumps({
"operation_id": "reinstall-running-committed", "host_id": "hk",
"user_id": 41, "kind": "mutation",
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID,
"status": "running", "result": None, "safe_to_refund": False,
}))
recovered = call(
client, "GET", "/internal/v1/operations/reinstall-running-committed",
USER,
)
payload = recovered.json()
assert recovered.status_code == 200
assert payload["status"] == "uncertain"
assert payload["error_code"] == "reinstall_result_uncertain"
def test_legacy_reinstall_post_commit_phase_is_uncertain(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.store.set("reinstall_job", "job-legacy-phase", json.dumps({
"operation_id": "reinstall-legacy-phase", "owner": "php:41",
"vm_id": VM_ID, "status": "succeeded", "phase": "cleaning_up",
"generated_password": "should-remain-private",
}))
store.set("operation", "reinstall-legacy-phase", json.dumps({
"operation_id": "reinstall-legacy-phase", "host_id": "hk",
"user_id": 41, "kind": "mutation",
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID,
"status": "pending", "result": None, "safe_to_refund": False,
}))
recovered = call(
client, "GET", "/internal/v1/operations/reinstall-legacy-phase",
USER,
)
payload = recovered.json()
assert recovered.status_code == 200
assert payload["status"] == "uncertain"
assert payload["error_code"] == "reinstall_result_uncertain"
assert payload["result"]["phase"] == "cleaning_up"
@pytest.mark.parametrize("resource", [
f"vms/{VM_ID}/actions",
f"vms/{VM_ID}/reinstall",
f"vms/{VM_ID}/rescue",
])
def test_orphaned_mutation_becomes_terminal_without_replay(boundary, resource):
client, _, store, *_ = boundary
operation_id = "orphan-" + resource.rsplit("/", 1)[-1].replace("?", "-")
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": resource, "vm_uuid": VM_ID,
"status": "running", "result": None, "safe_to_refund": False,
"created_at": 1,
}))
recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}").json()
assert recovered["status"] == "failed"
assert "不会自动重放" in recovered["error"]
def test_orphaned_delete_remains_uncertain_without_replay(boundary):
client, _engine, store, _logs, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
operation_id = "orphan-destructive-delete"
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id,
"host_id": "hk",
"user_id": 41,
"kind": "mutation",
"resource": f"vms/{VM_ID}",
"method": "DELETE",
"vm_uuid": VM_ID,
"status": "running",
"result": None,
"safe_to_refund": False,
"delete_ip_claim_keys": [],
"created_at": 1,
}))
before = list(services["hk"].calls)
first = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
).json()
second = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
).json()
assert first["status"] == second["status"] == "uncertain"
assert "原操作" in second["error"]
assert services["hk"].calls == before
def test_network_change_lost_response_keeps_both_addresses_uncertain(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
profile = components.manager.profile
profile.update({
"host": "8.8.8.3",
"default_network": "bridge:br0",
"public_pool_cidr": "8.8.8.0/28",
"public_gateway": "8.8.8.1",
"public_dns_servers": ["1.1.1.1"],
"public_reserved_ips": ["8.8.8.2"],
"public_vm_prefix": 28,
})
service = services["hk"]
service.vm.update(
managed=True,
os_source="cloud_template",
public_ip_cidr="8.8.8.4/28",
)
scope = host_scope(profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
actor = {**USER, "operation_id": "network-change-lost-response"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network"
changed = call(client, "POST", path, actor, {
"public_ip_cidr": "8.8.8.6/28",
"confirm_restart": True,
})
assert changed.status_code == 200, changed.text
saved = json.loads(store.get("operation", actor["operation_id"]))
assert saved["network_source_cidr"] == "8.8.8.4/28"
assert saved["network_target_cidr"] == "8.8.8.6/28"
assert saved["network_mutation_started"] is True
# Model a lost response after libvirt may already have switched. A single
# metadata read cannot prove that XML, cloud-init and nwfilter all agree,
# so automatic reconciliation must retain both address claims.
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
components.store.set("ip_claim", f"{scope}.8.8.8.4", json.dumps({
"vm_uuid": VM_ID,
}))
engine.active_operations.clear()
reconciled = call(
client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
actor,
)
assert reconciled.status_code == 200
assert reconciled.json()["status"] == "uncertain"
assert "人工核查" in reconciled.json()["error"]
assert components.store.get("ip_claim", f"{scope}.8.8.8.4") is not None
source_record = {
**saved,
"operation_id": "network-change-still-source",
"network_source_cidr": "8.8.8.6/28",
"network_target_cidr": "8.8.8.7/28",
"status": "uncertain",
}
store.set("operation", source_record["operation_id"],
json.dumps(source_record))
components.store.set("ip_claim", f"{scope}.8.8.8.7", json.dumps({
"vm_uuid": VM_ID,
}))
source = call(
client, "GET",
f"/internal/v1/operations/{source_record['operation_id']}", actor,
)
assert source.json()["status"] == "uncertain"
assert components.store.get("ip_claim", f"{scope}.8.8.8.7") is not None
service.vm["public_ip_cidr"] = "8.8.8.8/28"
drift_record = {
**source_record,
"operation_id": "network-change-third-address",
"status": "uncertain",
}
store.set("operation", drift_record["operation_id"],
json.dumps(drift_record))
drift = call(
client, "GET",
f"/internal/v1/operations/{drift_record['operation_id']}", actor,
)
assert drift.json()["status"] == "uncertain"
assert "人工核查" in drift.json()["error"]
def test_network_change_can_move_between_cidrs_on_the_same_selector(boundary):
client, engine, store, _, services = boundary
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
components = engine.registry.components["hk"]
profile = components.manager.profile
profile["default_network"] = "bridge:br0"
first_key = "3" * 32
second_key = "4" * 32
common = {
"revision": 1,
"state": "active",
"network_selector": "bridge:br0",
"dns_servers": ["1.1.1.1"],
"reserved_ips": [],
"guest_prefix_length": 29,
}
assert call(
client, "PUT",
f"/internal/v1/hosts/hk/host/network-pools/{first_key}", SUPER,
{**common, "cidr": "8.8.8.0/29", "gateway": "8.8.8.1"},
).status_code == 200
assert call(
client, "PUT",
f"/internal/v1/hosts/hk/host/network-pools/{second_key}", SUPER,
{**common, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1"},
).status_code == 200
service = services["hk"]
service.vm.update(
managed=True,
os_source="cloud_template",
public_ip_cidr="8.8.8.3/29",
public_pool_key=first_key,
)
scope = host_scope(profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
actor = {**USER, "operation_id": "network-cross-cidr-0001"}
changed = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
actor, {
"public_pool_key": second_key,
"public_ip_cidr": "9.9.9.3/29",
"confirm_restart": True,
},
)
assert changed.status_code == 200, changed.text
assert changed.json()["public_pool_key"] == second_key
operation = json.loads(store.get("operation", actor["operation_id"]))
assert operation["network_source_pool_key"] == first_key
assert operation["network_target_pool_key"] == second_key
claim = json.loads(components.store.get(
"ip_claim", f"{scope}.9.9.9.3"))
assert claim["pool_key"] == second_key
def test_network_change_upgrades_legacy_firewall_before_switching_ip(boundary):
client, engine, _store, _, services = boundary
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
components = engine.registry.components["hk"]
profile = components.manager.profile
profile.update({
"host": "8.8.8.3",
"default_network": "bridge:br0",
"public_pool_cidr": "8.8.8.0/28",
"public_gateway": "8.8.8.1",
"public_dns_servers": ["1.1.1.1"],
"public_reserved_ips": ["8.8.8.2"],
"public_vm_prefix": 28,
})
service = services["hk"]
service.vm.update(
managed=True,
os_source="cloud_template",
public_ip_cidr="8.8.8.4/28",
)
scope = host_scope(profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
firewall_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
body = firewall_body()
applied = call(
client, "PUT", firewall_path,
{**USER, "operation_id": "network-firewall-upgrade-0001"}, body,
)
assert applied.status_code == 200, applied.text
record_key = f"{scope}.{VM_ID}"
legacy_digest = build_firewall_filter_xml(
VM_ID, body["enabled"], body["rules"], compiler_version=1,
)[3]
legacy_record = json.loads(
components.store.get("vm_firewall", record_key))
legacy_record.pop("filter_version")
legacy_record["filter_digest"] = legacy_digest
components.store.set(
"vm_firewall", record_key,
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
)
service.firewall_filter_digest = legacy_digest
applies = sum(row[0] == "apply_firewall_policy" for row in service.calls)
changed = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
{**USER, "operation_id": "network-firewall-upgrade-0002"}, {
"public_ip_cidr": "8.8.8.6/28",
"confirm_restart": True,
},
)
assert changed.status_code == 200, changed.text
assert changed.json()["network_changed"] is True
assert sum(row[0] == "apply_firewall_policy"
for row in service.calls) == applies + 1
call_names = [row[0] for row in service.calls]
assert call_names.index(
"apply_firewall_policy", call_names.index("apply_firewall_policy") + 1,
) < call_names.index("change_cloud_vm_network")
upgraded = json.loads(components.store.get("vm_firewall", record_key))
assert upgraded["filter_version"] == FIREWALL_FILTER_COMPILER_VERSION
assert upgraded["filter_digest"] == build_firewall_filter_xml(
VM_ID, body["enabled"], body["rules"])[3]
def test_network_change_rejects_drifted_legacy_firewall(boundary):
client, engine, _store, _, services = boundary
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
components = engine.registry.components["hk"]
profile = components.manager.profile
profile.update({
"host": "8.8.8.3",
"default_network": "bridge:br0",
"public_pool_cidr": "8.8.8.0/28",
"public_gateway": "8.8.8.1",
"public_dns_servers": ["1.1.1.1"],
"public_reserved_ips": ["8.8.8.2"],
"public_vm_prefix": 28,
})
service = services["hk"]
service.vm.update(
managed=True,
os_source="cloud_template",
public_ip_cidr="8.8.8.4/28",
)
scope = host_scope(profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
firewall_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
body = firewall_body()
applied = call(
client, "PUT", firewall_path,
{**USER, "operation_id": "network-firewall-drift-0001"}, body,
)
assert applied.status_code == 200, applied.text
record_key = f"{scope}.{VM_ID}"
legacy_digest = build_firewall_filter_xml(
VM_ID, body["enabled"], body["rules"], compiler_version=1,
)[3]
legacy_record = json.loads(
components.store.get("vm_firewall", record_key))
legacy_record.pop("filter_version")
legacy_record["filter_digest"] = legacy_digest
components.store.set(
"vm_firewall", record_key,
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
)
service.firewall_filter_digest = "e" * 64
applies = sum(row[0] == "apply_firewall_policy" for row in service.calls)
changed = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
{**USER, "operation_id": "network-firewall-drift-0002"}, {
"public_ip_cidr": "8.8.8.6/28",
"confirm_restart": True,
},
)
assert changed.status_code == 409, changed.text
assert changed.json()["error_code"] == "firewall_drift_detected"
assert sum(row[0] == "apply_firewall_policy"
for row in service.calls) == applies
assert not any(row[0] == "change_cloud_vm_network"
for row in service.calls)
@pytest.mark.parametrize("tracked", [False, True])
def test_network_change_rejects_untracked_or_current_firewall_drift(
boundary, tracked,
):
client, engine, _store, _, services = boundary
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
components = engine.registry.components["hk"]
profile = components.manager.profile
profile.update({
"host": "8.8.8.3",
"default_network": "bridge:br0",
"public_pool_cidr": "8.8.8.0/28",
"public_gateway": "8.8.8.1",
"public_dns_servers": ["1.1.1.1"],
"public_reserved_ips": ["8.8.8.2"],
"public_vm_prefix": 28,
})
service = services["hk"]
service.vm.update(
managed=True,
os_source="cloud_template",
public_ip_cidr="8.8.8.4/28",
)
scope = host_scope(profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
if tracked:
body = firewall_body()
applied = call(
client, "PUT", f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall",
{**USER, "operation_id": "network-current-drift-0001"}, body,
)
assert applied.status_code == 200, applied.text
service.firewall_filter_digest = "d" * 64
applies = sum(row[0] == "apply_firewall_policy" for row in service.calls)
changed = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
{**USER, "operation_id": "network-current-drift-0002"}, {
"public_ip_cidr": "8.8.8.6/28",
"confirm_restart": True,
},
)
assert changed.status_code == 409, changed.text
assert changed.json()["error_code"] == "firewall_drift_detected"
assert sum(row[0] == "apply_firewall_policy"
for row in service.calls) == applies
assert not any(row[0] == "change_cloud_vm_network"
for row in service.calls)
@pytest.mark.parametrize("error_code,status", [
("firewall_apply_failed", 502),
("firewall_result_uncertain", 503),
])
def test_network_change_firewall_preflight_failure_is_not_ip_uncertain(
boundary, error_code, status,
):
client, engine, store, _, services = boundary
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
components = engine.registry.components["hk"]
profile = components.manager.profile
profile.update({
"host": "8.8.8.3",
"default_network": "bridge:br0",
"public_pool_cidr": "8.8.8.0/28",
"public_gateway": "8.8.8.1",
"public_dns_servers": ["1.1.1.1"],
"public_reserved_ips": ["8.8.8.2"],
"public_vm_prefix": 28,
})
service = services["hk"]
service.vm.update(
managed=True,
os_source="cloud_template",
public_ip_cidr="8.8.8.4/28",
)
scope = host_scope(profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
body = firewall_body()
applied = call(
client, "PUT", f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall",
{**USER, "operation_id": f"network-preflight-{status}-0001"}, body,
)
assert applied.status_code == 200, applied.text
record_key = f"{scope}.{VM_ID}"
legacy_digest = build_firewall_filter_xml(
VM_ID, body["enabled"], body["rules"], compiler_version=1,
)[3]
legacy_record = json.loads(
components.store.get("vm_firewall", record_key))
legacy_record.pop("filter_version")
legacy_record["filter_digest"] = legacy_digest
components.store.set(
"vm_firewall", record_key,
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
)
service.firewall_filter_digest = legacy_digest
service.firewall_apply_error = VMError(
"firewall preflight failed", status, error_code=error_code)
operation_id = f"network-preflight-{status}-0002"
changed = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
{**USER, "operation_id": operation_id}, {
"public_ip_cidr": "8.8.8.6/28",
"confirm_restart": True,
},
)
assert changed.status_code == status, changed.text
assert changed.json()["status"] == "failed"
assert changed.json()["error_code"] == error_code
operation = json.loads(store.get("operation", operation_id))
assert operation["status"] == "failed"
assert operation["network_mutation_started"] is False
polled = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
assert polled.status_code == 200, polled.text
assert polled.json()["status"] == "failed"
assert polled.json()["error_code"] == error_code
assert components.store.get("ip_claim", f"{scope}.8.8.8.6") is None
assert not any(row[0] == "change_cloud_vm_network"
for row in service.calls)
def test_network_change_capture_failure_is_terminal_before_ip_claim(boundary):
client, engine, store, _, services = boundary
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
components = engine.registry.components["hk"]
profile = components.manager.profile
profile.update({
"host": "8.8.8.3",
"default_network": "bridge:br0",
"public_pool_cidr": "8.8.8.0/28",
"public_gateway": "8.8.8.1",
"public_dns_servers": ["1.1.1.1"],
"public_reserved_ips": ["8.8.8.2"],
"public_vm_prefix": 28,
})
service = services["hk"]
service.vm.update(
managed=True,
os_source="cloud_template",
public_ip_cidr="8.8.8.4/28",
)
scope = host_scope(profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
service.get_vm = MagicMock(side_effect=RuntimeError("capture failed"))
operation_id = "network-capture-failed-0001"
changed = call(
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
{**USER, "operation_id": operation_id}, {
"public_ip_cidr": "8.8.8.6/28",
"confirm_restart": True,
},
)
assert changed.status_code == 503, changed.text
operation = json.loads(store.get("operation", operation_id))
assert operation["status"] == "failed"
assert operation["network_mutation_started"] is False
polled = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
assert polled.status_code == 200, polled.text
assert polled.json()["status"] == "failed"
assert "error_code" not in polled.json()
assert components.store.get("ip_claim", f"{scope}.8.8.8.6") is None
assert not any(row[0] == "change_cloud_vm_network"
for row in service.calls)
def test_deleted_vm_retries_only_public_ip_claim_cleanup(boundary):
client, engine, store, _, _services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
claim_key = f"{scope}.8.8.8.4"
components.store.set("ip_claim", claim_key, json.dumps({
"vm_uuid": VM_ID,
}))
operation = {
"operation_id": "delete-claim-cleanup",
"host_id": "hk",
"user_id": USER["user_id"],
"vm_uuid": VM_ID,
"kind": "mutation",
"resource": f"vms/{VM_ID}",
"method": "DELETE",
"status": "uncertain",
"result": None,
"safe_to_refund": False,
"cleanup_pending": "public_ip_claim",
"delete_ip_claim_keys": [claim_key],
"created_at": 1,
}
store.set("operation", operation["operation_id"], json.dumps(operation))
recovered = call(
client, "GET",
f"/internal/v1/operations/{operation['operation_id']}", USER,
)
assert recovered.status_code == 200
assert recovered.json()["status"] == "succeeded"
assert recovered.json()["result"]["public_ip_claims_released"] is True
assert components.store.get("ip_claim", claim_key) is None
foreign_key = f"{scope}.8.8.8.5"
components.store.set("ip_claim", foreign_key, json.dumps({
"vm_uuid": "22222222-2222-4222-8222-222222222222",
}))
foreign = {
**operation,
"operation_id": "delete-foreign-claim",
"status": "uncertain",
"cleanup_pending": "public_ip_claim",
"delete_ip_claim_keys": [foreign_key],
}
store.set("operation", foreign["operation_id"], json.dumps(foreign))
blocked = call(
client, "GET",
f"/internal/v1/operations/{foreign['operation_id']}", USER,
)
assert blocked.json()["status"] == "uncertain"
assert components.store.get("ip_claim", foreign_key) is not None
def test_service_resume_replays_exact_start_target_after_lost_response(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
services["hk"].vm.update(state="shutoff", autostart=False)
actor = {**USER, "operation_id": "resume-start-target"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume"
first = call(client, "POST", path, actor, {"start_if_suspended": True})
assert first.status_code == 200, first.text
assert services["hk"].vm["state"] == "running"
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
saved = json.loads(store.get("operation", actor["operation_id"]))
saved.update(status="uncertain", result=None, error="response lost")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(state="shutoff", autostart=False)
engine.active_operations.clear()
replayed = call(client, "GET", "/internal/v1/operations/resume-start-target", actor)
assert replayed.status_code == 200
assert replayed.json()["status"] == "succeeded"
assert services["hk"].vm["state"] == "running"
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
def test_service_suspend_replay_preserves_original_running_state(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
actor = {**USER, "operation_id": "suspend-running-hint"}
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend"
assert call(client, "POST", path, actor, {}).status_code == 200
saved = json.loads(store.get("operation", actor["operation_id"]))
assert saved["service_was_running_hint"] is True
# Model a hard crash after libvirt stopped the guest but before the
# successful result was durable. Reconciliation now observes shutoff, yet
# must retain the original running state for the billing resume policy.
saved.update(status="uncertain", result=None, error="engine crashed")
store.set("operation", actor["operation_id"], json.dumps(saved))
services["hk"].vm.update(state="shutoff", autostart=False)
engine.active_operations.clear()
reconciled = call(client, "GET",
"/internal/v1/operations/suspend-running-hint", actor)
assert reconciled.status_code == 200
assert reconciled.json()["status"] == "succeeded"
assert reconciled.json()["result"]["service_was_running"] is True
def test_service_suspend_replay_without_running_hint_fails_closed(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
operation_id = "suspend-missing-running-hint"
store.set("operation", operation_id, json.dumps({
"operation_id": operation_id,
"host_id": "hk",
"user_id": USER["user_id"],
"vm_uuid": VM_ID,
"kind": "mutation",
"resource": f"vms/{VM_ID}/service-suspend",
"method": "POST",
"status": "uncertain",
"actor_can_manage": True,
"actor_is_admin": False,
"created_at": 1,
}))
before_calls = list(services["hk"].calls)
response = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
assert response.status_code == 200
assert response.json()["status"] == "failed"
assert "已取消服务暂停" in response.json()["error"]
assert services["hk"].calls == before_calls
def test_service_suspend_does_not_stop_vm_when_running_hint_cannot_persist(
boundary, monkeypatch):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
operation_id = "suspend-hint-write-failed"
original_set = store.set
def fail_operation_write(kind, key, value):
if kind == "operation" and key == operation_id:
raise RuntimeError("database unavailable")
return original_set(kind, key, value)
monkeypatch.setattr(store, "set", fail_operation_write)
failed = call(
client, "POST",
f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend",
{**USER, "operation_id": operation_id}, {})
assert failed.status_code == 503
assert services["hk"].vm["state"] == "running"
assert not any(
item[0] == "service_suspend" for item in services["hk"].calls)
monkeypatch.setattr(store, "set", original_set)
recovered = call(
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
assert recovered.status_code == 200
assert recovered.json()["status"] == "failed"
assert services["hk"].vm["state"] == "running"
assert not any(
item[0] == "service_suspend" for item in services["hk"].calls)
def test_metrics_query_uses_log_store_and_owner_permission(boundary):
client, engine, _, logs, _ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
scope = host_scope(component.manager.profile)
component.state.users.assign_vm(scope, VM_ID, 41)
response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d")
assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20
with patch("virsh_panel.app.time.time", return_value=3_000_000):
traffic = call(
client, "GET",
f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily?days=2",
)
assert traffic.status_code == 200
assert traffic.json()["points"][-1] == {
"date": "1970-02-04", "day_start": 2_937_600,
"download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120,
}
assert logs.daily_traffic_calls == [(scope, VM_ID, {
"host_id": "hk", "user_id": 41,
"since": 2_851_200, "until": 3_000_000, "limit": 2,
})]
denied = call(
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily",
{**USER, "user_id": 42},
)
assert denied.status_code == 404
assert len(logs.daily_traffic_calls) == 1
def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
assert issued.status_code == 200, issued.text
assert issued.json()["websocket_path"] == "/engine-console"
ticket = issued.json()["ticket"]
key = hashlib.sha256(ticket.encode()).hexdigest()
record = json.loads(store.get("console_ticket", key))
assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session"
assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket",
{**USER, "user_id": 42}, {}).status_code == 404
from starlette.websockets import WebSocketDisconnect
with pytest.raises(WebSocketDisconnect):
with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"},
subprotocols=["cloud-ticket." + ticket]):
pass
assert store.get("console_ticket", key) is not None
def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary):
client, engine, store, *_ = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
checks = 0
async def active(*args):
nonlocal checks
checks += 1
return checks <= 2 # Ticket issuance + handshake; then PHP revokes.
engine.introspection = active
engine.console_check_interval = 0.01
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
ticket = issued.json()["ticket"]
protocol = "cloud-ticket." + ticket
async def relay(socket, **kwargs):
await socket.send_bytes(b"vnc")
await asyncio.Event().wait()
from starlette.websockets import WebSocketDisconnect
with patch("virsh_panel.console.relay_console", relay):
with client.websocket_connect("wss://panel.example.test/engine-console",
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket:
assert socket.accepted_subprotocol == protocol
assert socket.receive_bytes() == b"vnc"
with pytest.raises(WebSocketDisconnect):
socket.receive_bytes()
key = hashlib.sha256(ticket.encode()).hexdigest()
assert store.get("console_ticket", key) is None
assert store.get("ticket_used", key) == "1"
with pytest.raises(WebSocketDisconnect):
with client.websocket_connect("wss://panel.example.test/engine-console",
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]):
pass
def test_streamed_iso_digest_must_match_before_import_is_queued(boundary):
client, engine, store, _, services = boundary
path = "/internal/v1/hosts/hk/rescue-templates?name=rescue"
signed_body = b"A" * (1024 * 1024)
wrong_body = b"B" * len(signed_body)
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest()
signed["Content-Type"] = "application/octet-stream"
response = client.post(path, content=wrong_body, headers=signed)
assert response.status_code == 401, response.text
assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls)
assert engine.registry.components["hk"].store.list("rescue_import_job") == {}
def cloud_upload_fixture(size=1024 * 1024 + 257):
header = bytearray(104)
header[:4] = b"QFI\xfb"
header[4:8] = struct.pack(">I", 3)
header[20:24] = struct.pack(">I", 16)
header[24:32] = struct.pack(">Q", 20 * 1024**3)
header[100:104] = struct.pack(">I", 104)
return bytes(header) + b"\0" * (size - len(header))
def wait_for_cloud_upload_job(client, job_id):
path = f"/internal/v1/vm-templates/import-jobs/{job_id}"
for _ in range(200):
response = call(client, "GET", path, SUPER)
assert response.status_code == 200, response.text
job = response.json()
if job["status"] in {"completed", "failed"}:
return job
time.sleep(0.01)
raise AssertionError("uploaded cloud template import did not finish")
def cloud_engine_scope(method, target, body, actor, *, headers=None):
path, separator, query = target.partition("?")
request_headers = signed_headers(SECRET, method, target, body, actor)
request_headers.update(headers or {})
return {
"type": "http",
"asgi": {"version": "3.0"},
"http_version": "1.1",
"method": method,
"scheme": "http",
"path": path,
"raw_path": path.encode("ascii"),
"query_string": query.encode("ascii") if separator else b"",
"headers": [
(name.lower().encode("ascii"), value.encode("latin-1"))
for name, value in request_headers.items()
],
"client": ("testclient", 50000),
"server": ("127.0.0.1", 80),
}
async def cloud_engine_request(engine, method, target, body, actor, *,
headers=None, receive=None):
scope = cloud_engine_scope(
method, target, body, actor, headers=headers)
delivered = False
async def receive_body():
nonlocal delivered
if delivered:
return {"type": "http.disconnect"}
delivered = True
return {"type": "http.request", "body": body, "more_body": False}
messages = []
async def send(message):
messages.append(message)
await engine(scope, receive or receive_body, send)
status = next(message["status"] for message in messages
if message["type"] == "http.response.start")
raw = b"".join(message.get("body", b"") for message in messages
if message["type"] == "http.response.body")
return status, json.loads(raw)
def test_global_cloud_template_upload_streams_past_json_limit(boundary):
client, engine, *_ = boundary
query = urlencode({
"name": "Debian 12 fixed",
"os_family": "Debian",
"os_version": "12",
})
path = "/internal/v1/vm-templates/upload?" + query
payload = cloud_upload_fixture()
with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"):
response = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
})
assert response.status_code == 202, response.text
job = wait_for_cloud_upload_job(client, response.json()["job_id"])
assert job["status"] == "completed", job
assert job["template"]["name"] == "Debian 12 fixed"
assert job["template"]["os_family"] == "Debian"
assert job["template"]["os_version"] == "12"
assert job["template"]["virtual_size"] == 20 * 1024**3
with engine.registry.cloud_templates.source(
job["template"]["id"]) as (_record, source):
assert source.stat().st_size == len(payload)
assert hashlib.sha256(source.read_bytes()).digest() == hashlib.sha256(
payload).digest()
def test_global_cloud_template_upload_digest_must_match_before_queue(boundary):
client, engine, *_ = boundary
path = ("/internal/v1/vm-templates/upload?"
"name=Debian+12&os_family=Debian&os_version=12")
signed_body = cloud_upload_fixture()
wrong_body = signed_body[:-1] + b"x"
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(
signed_body).hexdigest()
signed["Content-Type"] = "application/octet-stream"
response = client.post(path, content=wrong_body, headers=signed)
assert response.status_code == 401, response.text
assert engine.registry.cloud_templates.jobs == {}
assert engine.registry.cloud_templates.pending == set()
root = engine.registry.cloud_templates.files.directory()
assert not list(root.glob(".cloud-image-upload-*.qcow2"))
@pytest.mark.parametrize(
("family", "version"),
[
("debian", "12"),
("Debian", "14"),
("CentOS", "10"),
("Ubuntu", "24"),
],
)
def test_global_cloud_template_upload_rejects_noncanonical_metadata(
boundary, family, version):
client, engine, *_ = boundary
path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": "Invalid metadata",
"os_family": family,
"os_version": version,
})
payload = cloud_upload_fixture()
response = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
})
assert response.status_code == 422, response.text
assert engine.registry.cloud_templates.jobs == {}
def test_global_cloud_template_upload_requires_super_admin_and_octet_stream(
boundary):
client, engine, *_ = boundary
path = ("/internal/v1/vm-templates/upload?"
"name=Debian+12&os_family=Debian&os_version=12")
payload = cloud_upload_fixture()
digest = hashlib.sha256(payload).hexdigest()
forbidden = call(client, "POST", path, ADMIN, raw=payload, headers={
"X-Cloud-Content-SHA256": digest,
"Content-Type": "application/octet-stream",
})
assert forbidden.status_code == 403, forbidden.text
wrong_type = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": digest,
"Content-Type": "multipart/form-data",
})
assert wrong_type.status_code == 415, wrong_type.text
assert engine.registry.cloud_templates.jobs == {}
def test_global_cloud_template_upload_enforces_minimum_size(boundary):
client, engine, *_ = boundary
path = ("/internal/v1/vm-templates/upload?"
"name=Too+Small&os_family=Debian&os_version=12")
payload = cloud_upload_fixture(104)
response = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
})
assert response.status_code == 422, response.text
assert engine.registry.cloud_templates.jobs == {}
assert engine.registry.cloud_templates.pending == set()
def test_global_cloud_template_upload_timeout_releases_reservation(
boundary, monkeypatch):
_client, engine, *_ = boundary
query = b"name=Timeout&os_family=Debian&os_version=12"
scope = {
"query_string": query,
"headers": [
(b"content-type", b"application/octet-stream"),
(b"content-length", str(1024 * 1024).encode()),
],
"engine_upload_validation": {"complete": False, "valid": False},
"engine_upload_digest": "0" * 64,
}
async def stalled_receive():
await asyncio.sleep(1)
return {"type": "http.request", "body": b"", "more_body": False}
async def send(_message):
raise AssertionError("timeout must not start a response")
monkeypatch.setattr("virsh_panel.internal.CLOUD_UPLOAD_IDLE_SECONDS", 0.01)
with pytest.raises(TemplateError, match="上传超时") as timeout:
asyncio.run(engine._upload_cloud_template(scope, stalled_receive, send))
assert timeout.value.status == 408
assert engine.registry.cloud_templates.pending == set()
assert engine.registry.cloud_templates.upload_reservations == {}
root = engine.registry.cloud_templates.files.directory()
assert not list(root.glob(".cloud-image-upload-*.qcow2"))
def test_stalled_cloud_upload_does_not_block_other_engine_writes(
boundary, monkeypatch):
_client, engine, *_ = boundary
payload = cloud_upload_fixture()
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": "Stalled upload",
"os_family": "Debian",
"os_version": "12",
})
upload_started = asyncio.Event()
release_upload = asyncio.Event()
class FastRemoteHostManager:
def __init__(self, *_args, **_kwargs):
pass
@staticmethod
def scan(host, port):
return {"host": host, "port": port, "reachable": True}
@staticmethod
def close():
pass
monkeypatch.setattr(
"virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager)
async def stalled_receive():
upload_started.set()
await release_upload.wait()
return {"type": "http.disconnect"}
async def scenario():
upload = asyncio.create_task(cloud_engine_request(
engine,
"POST",
upload_path,
payload,
SUPER,
headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
"Content-Length": str(len(payload)),
},
receive=stalled_receive,
))
await asyncio.wait_for(upload_started.wait(), timeout=1)
assert engine.registry.cloud_templates.pending
scan_body = json.dumps({
"host": "host.example.test", "port": 22,
}, separators=(",", ":")).encode()
status, result = await asyncio.wait_for(cloud_engine_request(
engine,
"POST",
"/internal/v1/hosts/scan",
scan_body,
SUPER,
headers={"Content-Type": "application/json"},
), timeout=0.5)
assert status == 200
assert result["host"] == "host.example.test"
release_upload.set()
upload_status, _result = await asyncio.wait_for(upload, timeout=1)
assert upload_status == 400
asyncio.run(scenario())
assert engine.registry.cloud_templates.pending == set()
assert engine.registry.cloud_templates.upload_reservations == {}
def test_cloud_upload_fsync_does_not_block_event_loop_or_engine_writes(
boundary, monkeypatch):
_client, engine, *_ = boundary
payload = cloud_upload_fixture()
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": "Slow fsync upload",
"os_family": "Debian",
"os_version": "12",
})
fsync_started = threading.Event()
release_fsync = threading.Event()
original_fsync = os.fsync
def slow_fsync(descriptor):
fsync_started.set()
assert release_fsync.wait(timeout=2), "test did not release fsync"
return original_fsync(descriptor)
class FastRemoteHostManager:
def __init__(self, *_args, **_kwargs):
pass
@staticmethod
def scan(host, port):
return {"host": host, "port": port, "reachable": True}
@staticmethod
def close():
pass
monkeypatch.setattr("virsh_panel.internal.os.fsync", slow_fsync)
monkeypatch.setattr(
"virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager)
monkeypatch.setattr(
"virsh_panel.cloud_templates._verify_qcow2_with_qemu",
lambda *_args: None)
async def scenario():
safety_release = threading.Timer(1.5, release_fsync.set)
safety_release.start()
started_at = time.monotonic()
try:
upload = asyncio.create_task(cloud_engine_request(
engine,
"POST",
upload_path,
payload,
SUPER,
headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
"Content-Length": str(len(payload)),
},
))
while not fsync_started.is_set():
await asyncio.sleep(0.001)
scan_body = json.dumps({
"host": "host.example.test", "port": 22,
}, separators=(",", ":")).encode()
status, _result = await asyncio.wait_for(cloud_engine_request(
engine,
"POST",
"/internal/v1/hosts/scan",
scan_body,
SUPER,
headers={"Content-Type": "application/json"},
), timeout=0.5)
assert status == 200
assert time.monotonic() - started_at < 0.5
assert not release_fsync.is_set()
release_fsync.set()
upload_status, _result = await asyncio.wait_for(upload, timeout=2)
assert upload_status == 202
finally:
release_fsync.set()
safety_release.cancel()
asyncio.run(scenario())
def test_cloud_upload_rechecks_update_state_before_queue(
boundary, monkeypatch):
_client, engine, *_ = boundary
updater = FakeUpdater()
engine.updater = updater
payload = cloud_upload_fixture()
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": "Update race upload",
"os_family": "Debian",
"os_version": "12",
})
upload_started = asyncio.Event()
release_upload = asyncio.Event()
delivered = False
monkeypatch.setattr(
"virsh_panel.cloud_templates._verify_qcow2_with_qemu",
lambda *_args: None)
async def delayed_receive():
nonlocal delivered
if delivered:
return {"type": "http.disconnect"}
delivered = True
upload_started.set()
await release_upload.wait()
return {"type": "http.request", "body": payload, "more_body": False}
async def scenario():
upload = asyncio.create_task(cloud_engine_request(
engine,
"POST",
upload_path,
payload,
SUPER,
headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
"Content-Length": str(len(payload)),
},
receive=delayed_receive,
))
await asyncio.wait_for(upload_started.wait(), timeout=1)
assert engine.registry.cloud_templates.pending
updater.active = {
"job_id": "update-job-1234567890", "status": "running",
}
release_upload.set()
status, result = await asyncio.wait_for(upload, timeout=2)
assert status == 409
assert result["error"] == "面板正在更新,暂时不能执行虚拟化写操作"
try:
asyncio.run(scenario())
finally:
updater.active = None
assert engine.registry.cloud_templates.pending == set()
assert engine.registry.cloud_templates.upload_reservations == {}
assert engine.registry.cloud_templates.jobs == {}
def test_upload_same_os_version_keeps_existing_template(boundary):
client, engine, *_ = boundary
def upload(name, payload):
path = "/internal/v1/vm-templates/upload?" + urlencode({
"name": name,
"os_family": "Debian",
"os_version": "12",
})
response = call(client, "POST", path, SUPER, raw=payload, headers={
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
"Content-Type": "application/octet-stream",
})
assert response.status_code == 202, response.text
job = wait_for_cloud_upload_job(client, response.json()["job_id"])
assert job["status"] == "completed", job
return job["template"]
official_payload = cloud_upload_fixture()[:-1] + b"o"
fixed_payload = cloud_upload_fixture()[:-1] + b"f"
with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"):
official = upload("Debian 12 official", official_payload)
fixed = upload("Debian 12 fixed", fixed_payload)
assert official["id"] != fixed["id"]
templates = call(
client, "GET", "/internal/v1/vm-templates", SUPER).json()["templates"]
assert {row["id"] for row in templates} == {official["id"], fixed["id"]}
with engine.registry.cloud_templates.source(
official["id"]) as (_record, source):
assert hashlib.sha256(source.read_bytes()).hexdigest() == official["sha256"]
def test_firewall_contract_persists_round_trips_and_is_revision_idempotent(
boundary):
client, engine, _store, _logs, services = boundary
assert call(
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
).status_code == 200
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
baseline = call(client, "GET", path, USER)
assert baseline.status_code == 200, baseline.text
assert baseline.json() == {
"enabled": False,
"revision": 0,
"policy_digest": firewall_policy_digest(False, []),
"filter_digest": None,
"rules": [],
"sync_state": "active",
}
rejected = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-bad-digest"},
{**firewall_body(), "policy_digest": "0" * 64},
)
assert rejected.status_code == 422, rejected.text
assert rejected.json()["error_code"] == "firewall_rule_invalid"
assert not any(row[0] == "apply_firewall_policy"
for row in services["hk"].calls)
class ForbiddenUpdateLock:
async def __aenter__(self):
raise AssertionError("firewall acquired the engine-wide update lock")
async def __aexit__(self, *_args):
return False
original_target_lock = engine._target_lock
target_entries = []
@asynccontextmanager
async def observed_target_lock(host_id, vm_id):
target_entries.append((host_id, vm_id))
async with original_target_lock(host_id, vm_id):
yield
engine._update_lock = lambda: ForbiddenUpdateLock()
engine._target_lock = observed_target_lock
body = firewall_body()
assert body["policy_digest"] == (
"46c4d40eef4410edaf998e17981ae91d161348e2fc6572b2bd96c8b5cbf9bfc3"
)
applied = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-apply-0001"}, body,
)
assert applied.status_code == 200, applied.text
assert applied.json()["revision"] == 1
assert applied.json()["policy_digest"] == body["policy_digest"]
assert target_entries == [("hk", VM_ID)]
record_key = f"{scope}.{VM_ID}"
saved = json.loads(components.store.get("vm_firewall", record_key))
assert saved["revision"] == 1
assert saved["policy_digest"] == body["policy_digest"]
fetched = call(client, "GET", path, USER)
assert fetched.status_code == 200, fetched.text
assert fetched.json()["revision"] == 1
assert fetched.json()["rules"] == FIREWALL_RULES
applies = sum(row[0] == "apply_firewall_policy"
for row in services["hk"].calls)
repeated = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-apply-0002"}, body,
)
assert repeated.status_code == 200, repeated.text
assert repeated.json()["revision"] == 1
assert sum(row[0] == "apply_firewall_policy"
for row in services["hk"].calls) == applies
jumped = firewall_body(revision=3)
advanced = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-apply-0003"}, jumped,
)
assert advanced.status_code == 200, advanced.text
assert advanced.json()["revision"] == 3
conflicting = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-apply-conflict"},
firewall_body(revision=3, enabled=False, rules=[]),
)
assert conflicting.status_code == 409
rollback = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-apply-0004"},
firewall_body(revision=2),
)
assert rollback.status_code == 409
def test_firewall_get_detects_drift_and_tampered_local_evidence(boundary):
client, engine, _store, _logs, services = boundary
assert call(
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
).status_code == 200
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
body = firewall_body()
response = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-drift-0001"}, body,
)
assert response.status_code == 200, response.text
services["hk"].firewall_filter_digest = "f" * 64
drifted = call(client, "GET", path, USER)
assert drifted.status_code == 200, drifted.text
assert drifted.json()["sync_state"] == "drifted"
assert drifted.json()["revision"] is None
same_revision = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-drift-0002"}, body,
)
assert same_revision.status_code == 409
assert same_revision.json()["error_code"] == "firewall_drift_detected"
key = f"{scope}.{VM_ID}"
tampered = json.loads(components.store.get("vm_firewall", key))
tampered["rules"][0]["description"] = "Tampered"
components.store.set(
"vm_firewall", key,
json.dumps(tampered, ensure_ascii=False, separators=(",", ":")),
)
corrupt = call(client, "GET", path, USER)
assert corrupt.status_code == 503
assert corrupt.json()["error"] == "虚拟机防火墙记录损坏"
def test_firewall_same_revision_upgrades_legacy_filter_once(boundary):
client, engine, _store, _logs, services = boundary
assert call(
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
).status_code == 200
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
body = firewall_body()
applied = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-upgrade-0001"}, body,
)
assert applied.status_code == 200, applied.text
record_key = f"{scope}.{VM_ID}"
legacy_digest = build_firewall_filter_xml(
VM_ID, body["enabled"], body["rules"], compiler_version=1,
)[3]
legacy_record = json.loads(
components.store.get("vm_firewall", record_key))
legacy_record.pop("filter_version")
legacy_record["filter_digest"] = legacy_digest
components.store.set(
"vm_firewall", record_key,
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
)
services["hk"].firewall_filter_digest = legacy_digest
applies = sum(row[0] == "apply_firewall_policy"
for row in services["hk"].calls)
upgraded = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-upgrade-0002"}, body,
)
assert upgraded.status_code == 200, upgraded.text
assert sum(row[0] == "apply_firewall_policy"
for row in services["hk"].calls) == applies + 1
upgraded_record = json.loads(
components.store.get("vm_firewall", record_key))
current_digest = build_firewall_filter_xml(
VM_ID, body["enabled"], body["rules"])[3]
assert upgraded_record["filter_version"] == FIREWALL_FILTER_COMPILER_VERSION
assert upgraded_record["filter_digest"] == current_digest
repeated = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-upgrade-0003"}, body,
)
assert repeated.status_code == 200, repeated.text
assert sum(row[0] == "apply_firewall_policy"
for row in services["hk"].calls) == applies + 1
def test_firewall_uncertain_operation_reconciles_exact_host_evidence(boundary):
client, engine, store, _logs, services = boundary
assert call(
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
).status_code == 200
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
actor = {**USER, "operation_id": "firewall-lost-0001"}
body = firewall_body()
first = call(client, "PUT", path, actor, body)
assert first.status_code == 200, first.text
apply_count = sum(row[0] == "apply_firewall_policy"
for row in services["hk"].calls)
operation = json.loads(store.get("operation", actor["operation_id"]))
operation.update(status="uncertain", result=None)
operation.pop("finished_at", None)
store.set("operation", actor["operation_id"], json.dumps(operation))
record_key = f"{scope}.{VM_ID}"
components.store.delete("vm_firewall", record_key)
class ForbiddenUpdateLock:
async def __aenter__(self):
raise AssertionError("firewall reconciliation acquired global lock")
async def __aexit__(self, *_args):
return False
engine._update_lock = lambda: ForbiddenUpdateLock()
recovered = call(
client, "GET",
f"/internal/v1/operations/{actor['operation_id']}", USER,
)
assert recovered.status_code == 200, recovered.text
assert recovered.json()["status"] == "succeeded"
assert recovered.json()["result"]["revision"] == 1
assert recovered.json()["result"]["policy_digest"] == body["policy_digest"]
assert components.store.get("vm_firewall", record_key) is not None
assert sum(row[0] == "apply_firewall_policy"
for row in services["hk"].calls) == apply_count
old_body = body
target = firewall_body(revision=2, enabled=False, rules=[])
old_operation = {
**operation,
"operation_id": "firewall-old-0002",
"status": "uncertain",
"result": None,
"firewall_target": target,
}
store.set(
"operation", old_operation["operation_id"], json.dumps(old_operation))
old_state = call(
client, "GET",
f"/internal/v1/operations/{old_operation['operation_id']}", USER,
)
assert old_state.status_code == 200
assert old_state.json()["status"] == "failed"
mixed_operation = {
**old_operation,
"operation_id": "firewall-mixed-0003",
"status": "uncertain",
}
services["hk"].firewall_filter_digest = "e" * 64
store.set(
"operation", mixed_operation["operation_id"],
json.dumps(mixed_operation),
)
mixed = call(
client, "GET",
f"/internal/v1/operations/{mixed_operation['operation_id']}", USER,
)
assert mixed.status_code == 200
assert mixed.json()["status"] == "uncertain"
assert mixed.json()["error_code"] == "firewall_result_uncertain"
def test_firewall_uncertain_legacy_operation_survives_compiler_upgrade(boundary):
client, engine, store, _logs, services = boundary
assert call(
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
).status_code == 200
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
actor = {**USER, "operation_id": "firewall-legacy-lost-0001"}
body = firewall_body()
applied = call(client, "PUT", path, actor, body)
assert applied.status_code == 200, applied.text
legacy_digest = build_firewall_filter_xml(
VM_ID, body["enabled"], body["rules"], compiler_version=1,
)[3]
record_key = f"{scope}.{VM_ID}"
legacy_record = json.loads(
components.store.get("vm_firewall", record_key))
legacy_record.pop("filter_version")
legacy_record["filter_digest"] = legacy_digest
components.store.set(
"vm_firewall", record_key,
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
)
services["hk"].firewall_filter_digest = legacy_digest
operation = json.loads(store.get("operation", actor["operation_id"]))
operation["firewall_target"].pop("filter_version")
operation.update(status="uncertain", result=None)
operation.pop("finished_at", None)
store.set("operation", actor["operation_id"], json.dumps(operation))
apply_count = sum(
row[0] == "apply_firewall_policy" for row in services["hk"].calls)
recovered = call(
client, "GET",
f"/internal/v1/operations/{actor['operation_id']}", USER,
)
assert recovered.status_code == 200, recovered.text
assert recovered.json()["status"] == "succeeded"
assert recovered.json()["result"]["filter_version"] == 1
recovered_record = json.loads(
components.store.get("vm_firewall", record_key))
assert recovered_record["filter_version"] == 1
assert recovered_record["filter_digest"] == legacy_digest
assert sum(row[0] == "apply_firewall_policy"
for row in services["hk"].calls) == apply_count
def test_firewall_old_uncertain_operation_cannot_downgrade_new_record(boundary):
client, engine, store, _logs, services = boundary
assert call(
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
).status_code == 200
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
body_v1 = firewall_body(revision=1)
actor_v1 = {**USER, "operation_id": "firewall-old-lost-0001"}
first = call(client, "PUT", path, actor_v1, body_v1)
assert first.status_code == 200, first.text
old_operation = json.loads(
store.get("operation", actor_v1["operation_id"]))
body_v2 = firewall_body(revision=2)
second = call(
client, "PUT", path,
{**USER, "operation_id": "firewall-newer-0002"}, body_v2,
)
assert second.status_code == 200, second.text
record_key = f"{scope}.{VM_ID}"
newer_record = components.store.get("vm_firewall", record_key)
apply_count = sum(
row[0] == "apply_firewall_policy" for row in services["hk"].calls)
old_operation.update(status="uncertain", result=None)
old_operation.pop("finished_at", None)
store.set(
"operation", actor_v1["operation_id"], json.dumps(old_operation))
reconciled = call(
client, "GET",
f"/internal/v1/operations/{actor_v1['operation_id']}", USER,
)
assert reconciled.status_code == 200, reconciled.text
assert reconciled.json()["status"] == "uncertain"
assert reconciled.json()["error_code"] == "firewall_result_uncertain"
assert components.store.get("vm_firewall", record_key) == newer_record
assert json.loads(newer_record)["revision"] == 2
assert sum(row[0] == "apply_firewall_policy"
for row in services["hk"].calls) == apply_count
def test_deleted_vm_retries_firewall_cleanup_without_replaying_delete(boundary):
client, engine, _store, _logs, services = boundary
assert call(
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
).status_code == 200
components = engine.registry.components["hk"]
scope = host_scope(components.manager.profile)
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
firewall_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
applied = call(
client, "PUT", firewall_path,
{**USER, "operation_id": "firewall-before-delete"},
firewall_body(),
)
assert applied.status_code == 200, applied.text
record_key = f"{scope}.{VM_ID}"
assert components.store.get("vm_firewall", record_key) is not None
services["hk"].firewall_cleanup_error = VMError(
"cleanup unavailable", 502, error_code="firewall_cleanup_pending")
actor = {**USER, "operation_id": "delete-firewall-pending"}
deleted = call(
client, "DELETE",
f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true",
actor,
)
assert deleted.status_code == 503, deleted.text
assert deleted.json()["error_code"] == "vm_delete_firewall_cleanup_pending"
assert sum(row[0] == "delete_vm" for row in services["hk"].calls) == 1
services["hk"].firewall_cleanup_error = None
recovered = call(
client, "GET",
f"/internal/v1/operations/{actor['operation_id']}", USER,
)
assert recovered.status_code == 200, recovered.text
assert recovered.json()["status"] == "succeeded"
assert recovered.json()["result"]["firewall_cleaned"] is True
assert components.store.get("vm_firewall", record_key) is None
assert sum(row[0] == "delete_vm" for row in services["hk"].calls) == 1
def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables():
connection = MagicMock()
cursor = connection.cursor.return_value.__enter__.return_value
config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test",
"ssl_ca": "/private/test/ca.pem"}
with patch("virsh_panel.database.pymysql.connect", return_value=connection):
main = MainStore(config, b"m" * 32)
statements = [call.args[0] for call in cursor.execute.call_args_list]
assert any("engine_records" in statement for statement in statements)
assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements)
assert not any("vm_metric_samples" in statement for statement in statements)
assert main._config["ssl_verify_identity"] is True
cursor.reset_mock()
LogStore({**config, "database": "logs"}, b"m" * 32)
statements = [call.args[0] for call in cursor.execute.call_args_list]
assert "engine_vm_metric_samples" in statements[0]
assert "engine_vm_traffic_totals" in statements[1]
assert "engine_vm_traffic_checkpoints" in statements[2]
assert all("ALTER TABLE engine_vm_metric_samples" in statement
for statement in statements[3:])
assert LogStore.TRAFFIC_TABLE == "engine_vm_traffic_totals"
assert LogStore.TRAFFIC_CHECKPOINT_TABLE == "engine_vm_traffic_checkpoints"