You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
5675 lines
229 KiB
Python
5675 lines
229 KiB
Python
"""Private engine tests: no PHP cookies, no real DB and no real hypervisor."""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import struct
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import asynccontextmanager, contextmanager
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
from urllib.parse import urlencode
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from test_app import (CLOUD_CREATE_BODY, CREATE_BODY, OFFICIAL_CLOUD_URL,
|
|
FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID)
|
|
from virsh_panel.app import create_app
|
|
from virsh_panel.cloud_templates import (TemplateError, template_id_for,
|
|
template_scope, template_volume_name)
|
|
from virsh_panel.config import Settings
|
|
from virsh_panel.database import DatabaseError, DatabaseTemplateStore
|
|
from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore,
|
|
LogStore, MainStore, TRUSTED_ACTOR)
|
|
from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor,
|
|
signed_headers)
|
|
from virsh_panel.internal import EngineApplication, HostRegistry
|
|
from virsh_panel.libvirt_service import (
|
|
FIREWALL_FILTER_COMPILER_VERSION, VMError, build_firewall_filter_xml,
|
|
firewall_policy_digest,
|
|
snapshot_volume_name,
|
|
)
|
|
from virsh_panel.metrics import host_scope
|
|
from virsh_panel.user_isos import user_iso_scope
|
|
|
|
|
|
SECRET = b"s" * 32
|
|
USER = {"user_id": 41, "is_admin": False, "is_super_admin": False,
|
|
"can_manage": True, "session_id": "test-session"}
|
|
SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True}
|
|
ADMIN = {**SUPER, "is_super_admin": False}
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self):
|
|
self.records = {}
|
|
self.nonces = set()
|
|
self.lock = threading.RLock()
|
|
self.daily_traffic_calls = []
|
|
|
|
def get(self, kind, key):
|
|
with self.lock:
|
|
return self.records.get((kind, key))
|
|
|
|
def set(self, kind, key, value):
|
|
with self.lock:
|
|
self.records[kind, key] = value
|
|
|
|
def insert(self, kind, key, value):
|
|
with self.lock:
|
|
if (kind, key) in self.records:
|
|
return False
|
|
self.records[kind, key] = value
|
|
return True
|
|
|
|
def delete(self, kind, key):
|
|
with self.lock:
|
|
self.records.pop((kind, key), None)
|
|
|
|
def list(self, kind):
|
|
with self.lock:
|
|
return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind}
|
|
|
|
def consume_nonce(self, nonce, expires_at, now):
|
|
with self.lock:
|
|
if nonce in self.nonces:
|
|
return False
|
|
self.nonces.add(nonce)
|
|
return True
|
|
|
|
def query_metric_samples(self, *args, **kwargs):
|
|
return [{"timestamp": 123, "cpu_percent": 20.0}]
|
|
|
|
def query_daily_traffic(self, scope, vm_id, **kwargs):
|
|
self.daily_traffic_calls.append((scope, vm_id, kwargs))
|
|
day_start = kwargs["until"] - kwargs["until"] % 86400
|
|
return [{"day_start": day_start, "download_bytes": 4096,
|
|
"upload_bytes": 1024, "total_bytes": 5120}]
|
|
|
|
def get_value(self, key):
|
|
return self.get("value", key)
|
|
|
|
def set_value(self, key, value):
|
|
return self.set("value", key, value)
|
|
|
|
def delete_value(self, key):
|
|
return self.delete("value", key)
|
|
|
|
def get_secret(self, key):
|
|
return self.get("secret", key)
|
|
|
|
def set_secret(self, key, value):
|
|
return self.set("secret", key, value)
|
|
|
|
def delete_secret(self, key):
|
|
return self.delete("secret", key)
|
|
|
|
|
|
class FakeUpdater:
|
|
def __init__(self):
|
|
self.token = None
|
|
self.active = None
|
|
self.jobs = {}
|
|
|
|
def settings(self):
|
|
return {
|
|
"repository": "https://g.luoli.live/root/virt.git",
|
|
"branch": "venix",
|
|
"token_configured": self.token is not None,
|
|
"checkout_ready": True,
|
|
"current_commit": "a" * 40,
|
|
"last_job_id": None,
|
|
"active_job_id": self.active["job_id"] if self.active else None,
|
|
}
|
|
|
|
def set_token(self, token):
|
|
self.token = token
|
|
|
|
def clear_token(self):
|
|
self.token = None
|
|
|
|
def active_job(self):
|
|
return self.active
|
|
|
|
def check(self):
|
|
return {"repository": "https://g.luoli.live/root/virt.git", "branch": "venix",
|
|
"current_commit": "a" * 40, "remote_commit": "b" * 40,
|
|
"status": "update_available"}
|
|
|
|
def create_job(self, *, expected_current_commit=None, expected_remote_commit=None):
|
|
if self.token is None:
|
|
from virsh_panel.updater import UpdateError
|
|
raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409)
|
|
if expected_current_commit != "a" * 40 or expected_remote_commit != "b" * 40:
|
|
from virsh_panel.updater import UpdateError
|
|
raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409)
|
|
job = {"job_id": "update-job-1234567890", "status": "queued",
|
|
"phase": "等待独立更新服务启动", "created_at": 1,
|
|
"private": self.token}
|
|
self.jobs[job["job_id"]] = job
|
|
self.active = job
|
|
return job.copy()
|
|
|
|
def get_job(self, job_id):
|
|
return self.jobs.get(job_id)
|
|
|
|
def fail_job(self, job_id, message):
|
|
self.jobs[job_id].update(status="failed", error=message)
|
|
self.active = None
|
|
return self.jobs[job_id].copy()
|
|
|
|
|
|
class Service(FakeVMService):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.create_failure = False
|
|
self.absent = False
|
|
self.absent_checks = []
|
|
self.cloud_template_delete_error = None
|
|
self.vm.update(managed=True, os_source="iso")
|
|
|
|
def create_vm(self, spec):
|
|
self.calls.append(("create_vm", spec))
|
|
if self.create_failure:
|
|
from virsh_panel.libvirt_service import VMError
|
|
raise VMError("宿主机失联,回滚状态不明", 502)
|
|
self.vm.update(uuid=spec["vm_uuid"], name=spec["name"])
|
|
return self.vm.copy()
|
|
|
|
def provisioning_absent(self, vm_uuid, name, *, uuid_resources=False,
|
|
legacy_name_collision=False):
|
|
self.absent_checks.append(
|
|
(vm_uuid, name, uuid_resources, legacy_name_collision))
|
|
return self.absent
|
|
|
|
def change_cloud_vm_network(self, vm_id, network, *, confirm_restart=False):
|
|
self.calls.append((
|
|
"change_cloud_vm_network", vm_id,
|
|
network["public_ip_cidr"], confirm_restart,
|
|
))
|
|
if vm_id != self.vm["uuid"]:
|
|
raise VMError("虚拟机不存在", 404)
|
|
self.vm["public_ip_cidr"] = network["public_ip_cidr"]
|
|
self.vm["public_pool_key"] = network.get("public_pool_key")
|
|
return {
|
|
"vm": self.vm.copy(),
|
|
"public_pool_key": network.get("public_pool_key"),
|
|
"public_ip_cidr": network["public_ip_cidr"],
|
|
"network_changed": True,
|
|
}
|
|
|
|
def delete_cloud_template_replicas(self, source_volume_name):
|
|
self.calls.append(("delete_cloud_template_replicas", source_volume_name))
|
|
if self.cloud_template_delete_error is not None:
|
|
raise self.cloud_template_delete_error
|
|
return {"source_volume_name": source_volume_name, "deleted": True}
|
|
|
|
|
|
@pytest.fixture
|
|
def boundary(tmp_path):
|
|
store = MemoryStore()
|
|
logs = MemoryStore()
|
|
config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}
|
|
services = {}
|
|
def factory(host_id, user_isos, cloud_templates):
|
|
(tmp_path / host_id).mkdir(mode=0o700, exist_ok=True)
|
|
scoped = HostStore(store, host_id)
|
|
manager = FakeRemoteManager()
|
|
manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22,
|
|
"fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images",
|
|
"iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default",
|
|
"libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config",
|
|
"ssh_known_hosts_file": "/fake/known_hosts"}
|
|
service = Service()
|
|
services[host_id] = service
|
|
components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager,
|
|
DatabaseTemplateStore(scoped), scoped, logs, user_isos,
|
|
cloud_templates)
|
|
app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components,
|
|
service_factory=lambda profile: service)
|
|
return components, app
|
|
registry = HostRegistry(config, store, logs, factory=factory)
|
|
async def active(*args):
|
|
return True
|
|
engine = EngineApplication(config, store, logs, registry=registry, introspection=active)
|
|
engine._service = lambda profile: services[profile["host_id"]]
|
|
with TestClient(engine, base_url="http://127.0.0.1") as client:
|
|
yield client, engine, store, logs, services
|
|
|
|
|
|
def call(client, method, path, actor=USER, value=None, raw=None, headers=None):
|
|
body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode()
|
|
signed = signed_headers(SECRET, method, path, body, actor)
|
|
if value is not None:
|
|
signed["Content-Type"] = "application/json"
|
|
signed.update(headers or {})
|
|
return client.request(method, path, content=body, headers=signed)
|
|
|
|
|
|
FIREWALL_RULES = [
|
|
{
|
|
"protocol": "icmp",
|
|
"source_cidr": "0.0.0.0/0",
|
|
"port_from": None,
|
|
"port_to": None,
|
|
"description": "Ping",
|
|
},
|
|
{
|
|
"protocol": "tcp",
|
|
"source_cidr": "203.0.113.55/32",
|
|
"port_from": 22,
|
|
"port_to": 22,
|
|
"description": "SSH",
|
|
},
|
|
]
|
|
|
|
|
|
def firewall_body(*, revision=1, enabled=True, rules=None):
|
|
rules = FIREWALL_RULES if rules is None else rules
|
|
return {
|
|
"enabled": enabled,
|
|
"revision": revision,
|
|
"policy_digest": firewall_policy_digest(enabled, rules),
|
|
"rules": rules,
|
|
}
|
|
|
|
|
|
def test_legacy_firewall_compiler_digest_fixture_is_stable():
|
|
body = firewall_body()
|
|
assert build_firewall_filter_xml(
|
|
VM_ID, body["enabled"], body["rules"], compiler_version=1,
|
|
)[3] == "42e1cd7c3e410dc6a0d80740753b19b6b4a67ca00c4b8265b2f0bb201f9af793"
|
|
|
|
|
|
def test_host_capacity_endpoint_is_removed(boundary):
|
|
client, _engine, _store, _logs, _services = boundary
|
|
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/host/capacity", USER).status_code == 404
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER).status_code == 404
|
|
|
|
|
|
def import_user_iso_for_test(client, tmp_path, name, *, digest="e" * 64):
|
|
download = tmp_path / (hashlib.sha256(name.encode()).hexdigest() + ".iso")
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, digest)):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
|
|
"url": "https://downloads.example.test/os.iso",
|
|
"name": name,
|
|
"sha256": "",
|
|
})
|
|
assert queued.status_code == 202, queued.text
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
return job["iso"]["id"]
|
|
|
|
|
|
def test_host_network_options_endpoint_is_removed(boundary):
|
|
client, _engine, _store, _logs, _services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options", USER).status_code == 404
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER).status_code == 404
|
|
|
|
|
|
def test_host_public_network_pools_are_revisioned_idempotent_and_aggregated(
|
|
boundary):
|
|
client, _engine, _store, _logs, services = boundary
|
|
# Materialize the host child app before configuring its host-scoped
|
|
# encrypted pool catalog.
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER,
|
|
).json() == {
|
|
"items": [], "count": 0, "maximum": 32,
|
|
"compatibility_mode": False,
|
|
}
|
|
|
|
first_key = "a" * 32
|
|
second_key = "b" * 32
|
|
first_path = f"/internal/v1/hosts/hk/host/network-pools/{first_key}"
|
|
first = {
|
|
"revision": 1,
|
|
"state": "active",
|
|
"network_selector": "default",
|
|
"cidr": "8.8.8.0/29",
|
|
"gateway": "8.8.8.1",
|
|
"dns_servers": ["1.1.1.1"],
|
|
"reserved_ips": ["8.8.8.2"],
|
|
"guest_prefix_length": 29,
|
|
}
|
|
assert call(client, "PUT", first_path, USER, first).status_code == 403
|
|
created = call(client, "PUT", first_path, SUPER, first)
|
|
assert created.status_code == 200, created.text
|
|
assert created.json()["applied"] is True
|
|
assert created.json()["pool"]["pool_key"] == first_key
|
|
|
|
repeated = call(client, "PUT", first_path, SUPER, first)
|
|
assert repeated.status_code == 200
|
|
assert repeated.json()["idempotent"] is True
|
|
conflict = call(client, "PUT", first_path, SUPER, {
|
|
**first, "gateway": "8.8.8.2",
|
|
})
|
|
assert conflict.status_code == 409
|
|
|
|
second_path = f"/internal/v1/hosts/hk/host/network-pools/{second_key}"
|
|
second = {
|
|
**first,
|
|
"cidr": "9.9.9.0/29",
|
|
"gateway": "9.9.9.1",
|
|
"reserved_ips": ["9.9.9.2"],
|
|
}
|
|
assert call(client, "PUT", second_path, SUPER, second).status_code == 200
|
|
advanced = call(client, "PUT", first_path, SUPER, {
|
|
**first, "revision": 2, "state": "draining",
|
|
})
|
|
assert advanced.status_code == 200
|
|
assert advanced.json()["pool"]["state"] == "draining"
|
|
assert call(client, "PUT", first_path, SUPER, first).status_code == 409
|
|
overlapping = call(client, "PUT", f"/internal/v1/hosts/hk/host/network-pools/{'c' * 32}",
|
|
SUPER, {**first, "cidr": "8.8.8.0/30"})
|
|
assert overlapping.status_code == 400
|
|
|
|
catalog = call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER,
|
|
).json()
|
|
assert catalog["count"] == 2
|
|
assert [item["pool_key"] for item in catalog["items"]] == [
|
|
first_key, second_key,
|
|
]
|
|
assert catalog["compatibility_mode"] is False
|
|
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER,
|
|
).status_code == 404
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER,
|
|
).status_code == 404
|
|
|
|
disabled = call(client, "PUT", second_path, SUPER, {
|
|
**second, "revision": 2, "state": "disabled",
|
|
})
|
|
assert disabled.status_code == 200
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER,
|
|
).status_code == 404
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER,
|
|
).status_code == 404
|
|
|
|
|
|
def test_first_native_pool_cannot_orphan_occupied_legacy_network(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.4.4",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/29",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 29,
|
|
})
|
|
scope = host_scope(profile)
|
|
components.store.set("ip_claim", f"{scope}.8.8.8.3", json.dumps({
|
|
"vm_uuid": "00000000-0000-4000-8000-000000000003",
|
|
}))
|
|
pool_key = "d" * 32
|
|
path = f"/internal/v1/hosts/hk/host/network-pools/{pool_key}"
|
|
base = {
|
|
"revision": 1,
|
|
"state": "active",
|
|
"network_selector": "bridge:br0",
|
|
"dns_servers": ["1.1.1.1"],
|
|
"reserved_ips": ["9.9.9.2"],
|
|
"guest_prefix_length": 29,
|
|
}
|
|
rejected = call(client, "PUT", path, SUPER, {
|
|
**base, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1",
|
|
})
|
|
assert rejected.status_code == 409
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER,
|
|
).json()["compatibility_mode"] is True
|
|
|
|
adopted = call(client, "PUT", path, SUPER, {
|
|
**base,
|
|
"cidr": "8.8.8.0/29",
|
|
"gateway": "8.8.8.1",
|
|
"reserved_ips": ["8.8.8.2"],
|
|
})
|
|
assert adopted.status_code == 200, adopted.text
|
|
assert adopted.json()["compatibility_mode"] is False
|
|
|
|
|
|
def import_cloud_template_for_test(client, tmp_path, name="Delete Cloud Image"):
|
|
payload = name.encode("utf-8")
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
|
|
def fake_download(_url, _sha256, directory, **_kwargs):
|
|
source = directory / ".cloud-image-delete-boundary.qcow2"
|
|
source.write_bytes(payload)
|
|
source.chmod(0o600)
|
|
return source, 2 * 1024**3, digest
|
|
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=fake_download):
|
|
queued = call(client, "POST", "/internal/v1/vm-templates/import",
|
|
SUPER, {"url": OFFICIAL_CLOUD_URL,
|
|
"name": name, "sha256": ""})
|
|
assert queued.status_code == 202, queued.text
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET",
|
|
f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER,
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
return job["template"], payload
|
|
|
|
|
|
def test_paid_scheduler_can_claim_exact_host_pool_ip(boundary, tmp_path):
|
|
client, engine, _store, _logs, services = boundary
|
|
template, _payload = import_cloud_template_for_test(
|
|
client, tmp_path, "Scheduled Address Image",
|
|
)
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/resources", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
service = services["hk"]
|
|
scope = host_scope(profile)
|
|
components.store.set("ip_claim", f"{scope}.8.8.8.4", json.dumps({
|
|
"vm_uuid": "00000000-0000-4000-8000-000000000004",
|
|
}))
|
|
occupied_claim = f"{scope}.8.8.8.5"
|
|
components.store.set("ip_claim", occupied_claim, json.dumps({
|
|
"vm_uuid": "00000000-0000-4000-8000-000000000005",
|
|
}))
|
|
base = {
|
|
key: value for key, value in CLOUD_CREATE_BODY.items()
|
|
if key not in {"gateway", "dns_servers"}
|
|
}
|
|
base["template_id"] = template["id"]
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
|
|
unauthorized = call(client, "POST", path, {
|
|
**USER, "operation_id": "scheduled-ip-unpaid-0001",
|
|
}, {**base, "name": "unpaid-exact-ip", "public_ip_cidr": "8.8.8.8/28"})
|
|
assert unauthorized.status_code == 403
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.8") is None
|
|
|
|
service.absent = True
|
|
outside = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-outside-0001",
|
|
}, {**base, "name": "outside-exact-ip", "public_ip_cidr": "8.8.8.20/28"})
|
|
assert outside.status_code == 422
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.20") is None
|
|
|
|
occupied_vm = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-vm-used-0001",
|
|
}, {**base, "name": "vm-used-exact-ip", "public_ip_cidr": "8.8.8.4/28"})
|
|
assert occupied_vm.status_code == 409
|
|
vm_claim = json.loads(components.store.get(
|
|
"ip_claim", f"{scope}.8.8.8.4",
|
|
))
|
|
assert vm_claim == {
|
|
"vm_uuid": "00000000-0000-4000-8000-000000000004",
|
|
}
|
|
|
|
occupied_task = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-claim-used-0001",
|
|
}, {**base, "name": "claim-used-exact-ip", "public_ip_cidr": "8.8.8.5/28"})
|
|
assert occupied_task.status_code == 409
|
|
assert components.store.get("ip_claim", occupied_claim) is not None
|
|
|
|
created = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-success-0001",
|
|
}, {**base, "name": "scheduled-exact-ip", "public_ip_cidr": "8.8.8.6/28"})
|
|
assert created.status_code == 201, created.text
|
|
created_id = created.json()["uuid"]
|
|
created_call = [
|
|
row for row in service.calls if row[0] == "create_cloud_vm"
|
|
][-1]
|
|
assert created_call[1]["public_ip_cidr"] == "8.8.8.6/28"
|
|
assert created_call[1]["public_pool_key"] == "legacy-default"
|
|
assert created_call[1]["gateway"] == "8.8.8.1"
|
|
assert created_call[1]["dns_servers"] == ["1.1.1.1"]
|
|
assert not any(row[0] == "list_vms" for row in service.calls)
|
|
created_claim = json.loads(components.store.get(
|
|
"ip_claim", f"{scope}.8.8.8.6",
|
|
))
|
|
assert created_claim["vm_uuid"] == created_id
|
|
assert created_claim["pool_key"] == "legacy-default"
|
|
|
|
def rolled_back_cloud_create(_spec, _volume_name):
|
|
raise VMError("模拟创建失败,宿主机已完成回滚", 502)
|
|
|
|
service.create_cloud_vm = rolled_back_cloud_create
|
|
failed = call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": "scheduled-ip-rollback-0001",
|
|
}, {**base, "name": "rolled-back-exact-ip", "public_ip_cidr": "8.8.8.7/28"})
|
|
assert failed.status_code == 502
|
|
assert failed.json()["status"] == "failed"
|
|
assert failed.json()["safe_to_refund"] is True
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.7") is None
|
|
|
|
|
|
def test_exact_public_ip_claim_is_atomic_without_inventory_scan(
|
|
boundary, tmp_path):
|
|
client, engine, _store, _logs, services = boundary
|
|
template, _payload = import_cloud_template_for_test(
|
|
client, tmp_path, "Concurrent Scheduled Address Image",
|
|
)
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/resources", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
service = services["hk"]
|
|
# The exact-claim path must use only the panel-owned claim ledger. Any
|
|
# fallback to a domain inventory scan turns this guard into a test failure.
|
|
def forbidden_inventory_scan():
|
|
raise AssertionError("exact IP claim must not enumerate libvirt VMs")
|
|
|
|
service.list_vms = forbidden_inventory_scan
|
|
service.calls.clear()
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
base = {
|
|
key: value for key, value in CLOUD_CREATE_BODY.items()
|
|
if key not in {"gateway", "dns_servers"}
|
|
}
|
|
base["template_id"] = template["id"]
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
barrier = threading.Barrier(2)
|
|
|
|
def create(number):
|
|
barrier.wait(timeout=2)
|
|
return call(client, "POST", path, {
|
|
**USER, "can_provision": True,
|
|
"operation_id": f"scheduled-ip-concurrent-{number:04d}",
|
|
}, {
|
|
**base, "name": f"scheduled-concurrent-{number}",
|
|
"public_ip_cidr": "8.8.8.6/28",
|
|
})
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
responses = [
|
|
future.result(timeout=5)
|
|
for future in (
|
|
executor.submit(create, 1),
|
|
executor.submit(create, 2),
|
|
)
|
|
]
|
|
|
|
assert sorted(response.status_code for response in responses) == [201, 409]
|
|
successful = next(response for response in responses
|
|
if response.status_code == 201)
|
|
scope = host_scope(profile)
|
|
claim = json.loads(components.store.get(
|
|
"ip_claim", f"{scope}.8.8.8.6",
|
|
))
|
|
assert claim["vm_uuid"] == successful.json()["uuid"]
|
|
assert not any(row[0] == "list_vms" for row in service.calls)
|
|
|
|
|
|
def test_paid_scheduler_claims_the_requested_native_public_pool(
|
|
boundary, tmp_path):
|
|
client, engine, _store, _logs, services = boundary
|
|
template, _payload = import_cloud_template_for_test(
|
|
client, tmp_path, "Native Pool Address Image",
|
|
)
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources", SUPER).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile["default_network"] = "bridge:br0"
|
|
first_key = "1" * 32
|
|
second_key = "2" * 32
|
|
common = {
|
|
"revision": 1,
|
|
"state": "active",
|
|
"network_selector": "bridge:br0",
|
|
"dns_servers": ["1.1.1.1"],
|
|
"reserved_ips": [],
|
|
"guest_prefix_length": 29,
|
|
}
|
|
assert call(
|
|
client, "PUT",
|
|
f"/internal/v1/hosts/hk/host/network-pools/{first_key}", SUPER,
|
|
{**common, "cidr": "8.8.8.0/29", "gateway": "8.8.8.1"},
|
|
).status_code == 200
|
|
assert call(
|
|
client, "PUT",
|
|
f"/internal/v1/hosts/hk/host/network-pools/{second_key}", SUPER,
|
|
{**common, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1"},
|
|
).status_code == 200
|
|
base = {
|
|
key: value for key, value in CLOUD_CREATE_BODY.items()
|
|
if key not in {"gateway", "dns_servers"}
|
|
}
|
|
base["template_id"] = template["id"]
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
provisioning = {**USER, "can_provision": True}
|
|
|
|
wrong = call(client, "POST", path, {
|
|
**provisioning, "operation_id": "native-pool-mismatch-0001",
|
|
}, {
|
|
**base, "name": "wrong-native-pool",
|
|
"public_pool_key": first_key,
|
|
"public_ip_cidr": "9.9.9.3/29",
|
|
})
|
|
assert wrong.status_code == 422
|
|
|
|
created = call(client, "POST", path, {
|
|
**provisioning, "operation_id": "native-pool-success-0001",
|
|
}, {
|
|
**base, "name": "right-native-pool",
|
|
"public_pool_key": second_key,
|
|
"public_ip_cidr": "9.9.9.3/29",
|
|
})
|
|
assert created.status_code == 201, created.text
|
|
create_call = [
|
|
row for row in services["hk"].calls if row[0] == "create_cloud_vm"
|
|
][-1]
|
|
assert create_call[1]["public_pool_key"] == second_key
|
|
assert create_call[1]["gateway"] == "9.9.9.1"
|
|
scope = host_scope(profile)
|
|
claim = json.loads(components.store.get("ip_claim", f"{scope}.9.9.9.3"))
|
|
assert claim["pool_key"] == second_key
|
|
|
|
|
|
def test_global_cloud_template_is_visible_and_cached_once_per_host(
|
|
boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
payload = b"verified-global-cloud-template"
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
|
|
def fake_download(_url, _sha256, directory, **_kwargs):
|
|
source = directory / ".cloud-image-boundary.qcow2"
|
|
source.write_bytes(payload)
|
|
source.chmod(0o600)
|
|
return source, 2 * 1024**3, digest
|
|
|
|
denied = call(client, "POST", "/internal/v1/vm-templates/import",
|
|
USER, {"url": OFFICIAL_CLOUD_URL,
|
|
"name": "Not Allowed", "sha256": ""})
|
|
assert denied.status_code == 403
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=fake_download):
|
|
queued = call(client, "POST", "/internal/v1/vm-templates/import",
|
|
SUPER, {"url": OFFICIAL_CLOUD_URL,
|
|
"name": "AlmaLinux Global", "sha256": ""})
|
|
assert queued.status_code == 202, queued.text
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET",
|
|
f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER,
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
template_id = job["template"]["id"]
|
|
assert services == {}
|
|
assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == {
|
|
"templates": [job["template"]],
|
|
}
|
|
|
|
provisioning = {**USER, "can_provision": True}
|
|
body = {**CLOUD_CREATE_BODY, "template_id": template_id}
|
|
hk = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0001"}, body,
|
|
)
|
|
assert hk.status_code == 201, hk.text
|
|
jp = call(
|
|
client, "POST", "/internal/v1/hosts/jp/vms",
|
|
{**provisioning, "operation_id": "global-template-jp-0001"},
|
|
{**body, "name": "cloud-vm-jp", "public_ip_cidr": "8.8.8.19/24"},
|
|
)
|
|
assert jp.status_code == 201, jp.text
|
|
first_markers = {
|
|
host_id: json.loads(HostStore(store, host_id).get(
|
|
"cloud_template_replica", template_id))
|
|
for host_id in ("hk", "jp")
|
|
}
|
|
# The public catalog intentionally omits the private backing filename.
|
|
source_volume_name = template_volume_name(template_id)
|
|
for host_id, marker in first_markers.items():
|
|
assert marker["status"] == "ready"
|
|
assert marker["sha256"] == digest
|
|
assert len(marker["replica_identity"]) == 64
|
|
assert marker["replica_volume_name"] != source_volume_name
|
|
assert marker["replica_volume_name"].startswith(
|
|
source_volume_name.removesuffix(".qcow2") + "-")
|
|
created = [row for row in services[host_id].calls
|
|
if row[0] == "create_cloud_vm"]
|
|
assert created[-1][2] == marker["replica_volume_name"]
|
|
|
|
observed_markers = []
|
|
original_hk_cache = services["hk"].cache_template
|
|
|
|
def inspect_hk_marker(*args, **kwargs):
|
|
marker = json.loads(HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id))
|
|
observed_markers.append(marker)
|
|
return original_hk_cache(*args, **kwargs)
|
|
|
|
services["hk"].cache_template = inspect_hk_marker
|
|
hk_again = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0002"},
|
|
{**body, "name": "cloud-vm-hk-2", "public_ip_cidr": "8.8.8.20/24"},
|
|
)
|
|
assert hk_again.status_code == 201, hk_again.text
|
|
services["hk"].cache_template = original_hk_cache
|
|
assert [marker["status"] for marker in observed_markers] == ["syncing"]
|
|
fallback_volume_name = observed_markers[0]["replica_volume_name"]
|
|
assert fallback_volume_name != first_markers["hk"]["replica_volume_name"]
|
|
|
|
hk_cache = [row for row in services["hk"].calls
|
|
if row[0] == "cache_template"]
|
|
jp_cache = [row for row in services["jp"].calls
|
|
if row[0] == "cache_template"]
|
|
assert hk_cache[0][4:] == (None, None)
|
|
assert hk_cache[1][1] == fallback_volume_name
|
|
assert hk_cache[1][4] == first_markers["hk"]["replica_volume_name"]
|
|
assert hk_cache[1][5] == first_markers["hk"]["replica_identity"]
|
|
assert jp_cache[0][4:] == (None, None)
|
|
for host_id in ("hk", "jp"):
|
|
marker = json.loads(HostStore(store, host_id).get(
|
|
"cloud_template_replica", template_id))
|
|
assert marker["status"] == "ready"
|
|
assert marker["sha256"] == digest
|
|
assert len(marker["replica_identity"]) == 64
|
|
# A successful trusted reuse keeps the proven prior generation even
|
|
# though the durable syncing marker contained a fresh fallback name.
|
|
assert marker["replica_volume_name"] == (
|
|
first_markers[host_id]["replica_volume_name"])
|
|
|
|
# Rebinding the logical host record to a different SSH host identity must
|
|
# invalidate its ready proof even when the pool name is unchanged.
|
|
engine.registry.components["hk"].manager.profile["fingerprint"] = (
|
|
"SHA256:replacement-host")
|
|
rebound = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0003"},
|
|
{**body, "name": "cloud-vm-hk-3", "public_ip_cidr": "8.8.8.21/24"},
|
|
)
|
|
assert rebound.status_code == 201, rebound.text
|
|
hk_cache = [row for row in services["hk"].calls
|
|
if row[0] == "cache_template"]
|
|
assert hk_cache[-1][4:] == (None, None)
|
|
rebound_marker = json.loads(HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id))
|
|
assert rebound_marker["status"] == "ready"
|
|
assert rebound_marker["replica_volume_name"] == hk_cache[-1][1]
|
|
assert (rebound_marker["replica_volume_name"]
|
|
!= first_markers["hk"]["replica_volume_name"])
|
|
|
|
# A crash/failure after the syncing marker was stored must retry that
|
|
# exact planned generation. Allocating yet another name would leak the
|
|
# possibly partial remote volume and make recovery non-deterministic.
|
|
failed_calls = []
|
|
|
|
def fail_hk_cache(image_path, volume_name, virtual_size, **kwargs):
|
|
failed_calls.append((volume_name, kwargs, json.loads(
|
|
HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id))))
|
|
raise VMError("云镜像缓存同步中断", 502)
|
|
|
|
services["hk"].cache_template = fail_hk_cache
|
|
interrupted = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0004"},
|
|
{**body, "name": "cloud-vm-hk-4", "public_ip_cidr": "8.8.8.22/24"},
|
|
)
|
|
assert interrupted.status_code == 502, interrupted.text
|
|
assert len(failed_calls) == 1
|
|
planned_volume_name, failed_kwargs, failed_marker = failed_calls[0]
|
|
assert failed_marker["status"] == "syncing"
|
|
assert failed_marker["replica_volume_name"] == planned_volume_name
|
|
assert failed_kwargs["trusted_replica_volume_name"] == (
|
|
rebound_marker["replica_volume_name"])
|
|
assert failed_kwargs["trusted_replica_identity"] == (
|
|
rebound_marker["replica_identity"])
|
|
|
|
services["hk"].cache_template = original_hk_cache
|
|
retried = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**provisioning, "operation_id": "global-template-hk-0005"},
|
|
{**body, "name": "cloud-vm-hk-5", "public_ip_cidr": "8.8.8.23/24"},
|
|
)
|
|
assert retried.status_code == 201, retried.text
|
|
hk_cache = [row for row in services["hk"].calls
|
|
if row[0] == "cache_template"]
|
|
assert hk_cache[-1][1] == planned_volume_name
|
|
assert hk_cache[-1][4:] == (None, None)
|
|
recovered_marker = json.loads(HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id))
|
|
assert recovered_marker["status"] == "ready"
|
|
assert recovered_marker["replica_volume_name"] == planned_volume_name
|
|
assert [row for row in services["hk"].calls
|
|
if row[0] == "create_cloud_vm"][-1][2] == planned_volume_name
|
|
assert (tmp_path / "cloud-templates"
|
|
/ engine.registry.cloud_templates.get(template_id)["volume_name"]).is_file()
|
|
|
|
|
|
def test_legacy_host_template_id_remains_usable_for_an_already_queued_order(
|
|
boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
# Initialize the original target host, then seed the same host-scoped
|
|
# catalog record an order created before the global-catalog upgrade holds.
|
|
assert call(client, "GET", "/internal/v1/hosts/legacy/vms").status_code == 200
|
|
profile = engine.registry.components["legacy"].manager.profile
|
|
scope = template_scope(profile)
|
|
digest = "d" * 64
|
|
template_id = template_id_for(scope, digest, "10")
|
|
DatabaseTemplateStore(HostStore(store, "legacy")).add(scope, {
|
|
"id": template_id,
|
|
"name": "Legacy AlmaLinux",
|
|
"os_family": "AlmaLinux",
|
|
"os_version": "10",
|
|
"sha256": digest,
|
|
"volume_name": template_volume_name(template_id),
|
|
"virtual_size": 2 * 1024**3,
|
|
"image_bytes": 1024,
|
|
"created_at": 1,
|
|
})
|
|
|
|
response = call(
|
|
client, "POST", "/internal/v1/hosts/legacy/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "legacy-template-order-0001"},
|
|
{**CLOUD_CREATE_BODY, "template_id": template_id,
|
|
"name": "legacy-paid-order", "public_ip_cidr": "8.8.8.22/24"},
|
|
)
|
|
|
|
assert response.status_code == 201, response.text
|
|
assert not any(row[0] == "cache_template"
|
|
for row in services["legacy"].calls)
|
|
create = next(row for row in services["legacy"].calls
|
|
if row[0] == "create_cloud_vm")
|
|
assert create[2] == template_volume_name(template_id)
|
|
|
|
|
|
def test_global_cloud_template_delete_is_idempotent_and_retries_host_gc(
|
|
boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
imported, payload = import_cloud_template_for_test(client, tmp_path)
|
|
template_id = imported["id"]
|
|
template_created_at = imported["created_at"]
|
|
source_volume_name = template_volume_name(template_id)
|
|
source_path = tmp_path / "cloud-templates" / source_volume_name
|
|
assert source_path.is_file()
|
|
|
|
for host_id in ("hk", "jp"):
|
|
assert call(
|
|
client, "GET", f"/internal/v1/hosts/{host_id}/resources"
|
|
).status_code == 200
|
|
HostStore(store, host_id).set(
|
|
"cloud_template_replica", template_id, "ready-marker")
|
|
|
|
assert call(
|
|
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
|
|
ADMIN, {"created_at": template_created_at},
|
|
).status_code == 403
|
|
services["hk"].cloud_template_delete_error = VMError(
|
|
"云镜像仍被虚拟机引用,不能删除", 409)
|
|
original_store_delete = store.delete
|
|
failed_catalog_delete = False
|
|
|
|
def fail_first_catalog_delete(kind, key):
|
|
nonlocal failed_catalog_delete
|
|
if (kind == "cloud_template" and key == template_id
|
|
and not failed_catalog_delete):
|
|
failed_catalog_delete = True
|
|
raise RuntimeError("catalog delete failed before commit")
|
|
return original_store_delete(kind, key)
|
|
|
|
store.delete = fail_first_catalog_delete
|
|
try:
|
|
deleted = call(
|
|
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
|
|
SUPER, {"created_at": template_created_at},
|
|
)
|
|
finally:
|
|
store.delete = original_store_delete
|
|
assert failed_catalog_delete is True
|
|
assert deleted.status_code == 200, deleted.text
|
|
assert deleted.json() == {
|
|
"deleted": True, "cache_cleanup_pending": True,
|
|
}
|
|
assert not source_path.exists()
|
|
assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == {
|
|
"templates": [],
|
|
}
|
|
|
|
for _ in range(100):
|
|
tombstone = json.loads(store.get(
|
|
"cloud_template_tombstone", template_id))
|
|
if tombstone["pending_hosts"] == ["hk"]:
|
|
break
|
|
time.sleep(0.01)
|
|
assert tombstone["source_deleted"] is True
|
|
assert tombstone["source_volume_name"] == source_volume_name
|
|
assert tombstone["pending_hosts"] == ["hk"]
|
|
assert HostStore(store, "jp").get(
|
|
"cloud_template_replica", template_id) is None
|
|
assert HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id) is not None
|
|
|
|
# Catalog and host reads are side-effect free. The durable delete event's
|
|
# maintenance supervisor retries independently of inventory/read traffic.
|
|
calls_before = len([
|
|
row for row in services["hk"].calls
|
|
if row[0] == "delete_cloud_template_replicas"
|
|
])
|
|
services["hk"].cloud_template_delete_error = VMError(
|
|
"存储池 images 不存在", 404)
|
|
with patch.object(
|
|
engine, "_background",
|
|
side_effect=AssertionError("catalog reads must not schedule cache GC")):
|
|
assert call(
|
|
client, "GET", "/internal/v1/vm-templates", USER
|
|
).status_code == 200
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
assert len([
|
|
row for row in services["hk"].calls
|
|
if row[0] == "delete_cloud_template_replicas"
|
|
]) == calls_before
|
|
still_pending = json.loads(store.get(
|
|
"cloud_template_tombstone", template_id))
|
|
assert still_pending["pending_hosts"] == ["hk"]
|
|
assert HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id) is not None
|
|
|
|
engine.cache_cleanup_retry_seconds = 0.02
|
|
engine._wake_cache_cleanup()
|
|
for _ in range(100):
|
|
if len([
|
|
row for row in services["hk"].calls
|
|
if row[0] == "delete_cloud_template_replicas"
|
|
]) > calls_before:
|
|
break
|
|
time.sleep(0.01)
|
|
still_pending = json.loads(store.get(
|
|
"cloud_template_tombstone", template_id))
|
|
assert still_pending["pending_hosts"] == ["hk"]
|
|
|
|
services["hk"].cloud_template_delete_error = VMError(
|
|
"云镜像宿主机缓存不存在", 404,
|
|
error_code="cloud_template_cache_not_found",
|
|
)
|
|
engine._wake_cache_cleanup()
|
|
for _ in range(100):
|
|
terminal = json.loads(store.get(
|
|
"cloud_template_tombstone", template_id))
|
|
if terminal["pending_hosts"] == []:
|
|
break
|
|
time.sleep(0.01)
|
|
terminal = json.loads(store.get("cloud_template_tombstone", template_id))
|
|
assert terminal["pending_hosts"] == []
|
|
assert HostStore(store, "hk").get(
|
|
"cloud_template_replica", template_id) is None
|
|
|
|
repeated = call(
|
|
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
|
|
SUPER, {"created_at": template_created_at})
|
|
assert repeated.status_code == 200
|
|
assert repeated.json() == {
|
|
"deleted": True, "cache_cleanup_pending": False,
|
|
}
|
|
|
|
# A duplicate task from the old deletion must not touch a later deletion
|
|
# generation (or the same-ID image/replica that may exist between them).
|
|
old_deletion_id = terminal["deletion_id"]
|
|
engine.registry.cloud_templates._prepare_reimport(template_id)
|
|
reimported = {
|
|
**imported,
|
|
"volume_name": source_volume_name,
|
|
"created_at": template_created_at + 1,
|
|
}
|
|
temporary = engine.registry.cloud_templates.files.directory() \
|
|
/ ".cloud-image-reimport-boundary.qcow2"
|
|
temporary.write_bytes(payload)
|
|
temporary.chmod(0o600)
|
|
engine.registry.cloud_templates.files.publish(
|
|
temporary, template_id, expected_size=len(payload))
|
|
engine.registry.cloud_templates.catalog.add(reimported)
|
|
stale_delete = call(
|
|
client, "DELETE", f"/internal/v1/vm-templates/{template_id}",
|
|
SUPER, {"created_at": template_created_at})
|
|
assert stale_delete.status_code == 409
|
|
assert engine.registry.cloud_templates.get(
|
|
template_id)["created_at"] == template_created_at + 1
|
|
|
|
next_tombstone = {
|
|
**terminal,
|
|
"deletion_id": "b" * 32,
|
|
"template_created_at": template_created_at + 1,
|
|
"pending_hosts": ["hk"],
|
|
}
|
|
store.set("cloud_template_tombstone", template_id,
|
|
json.dumps(next_tombstone))
|
|
calls_before = list(services["hk"].calls)
|
|
engine._cleanup_cloud_template_tombstone_on_host_sync(
|
|
template_id, old_deletion_id, "hk", services["hk"],
|
|
engine.registry.components["hk"],
|
|
)
|
|
assert services["hk"].calls == calls_before
|
|
assert json.loads(store.get(
|
|
"cloud_template_tombstone", template_id))["pending_hosts"] == ["hk"]
|
|
|
|
|
|
def test_user_iso_routes_are_owner_scoped_and_create_uses_opaque_id(boundary, tmp_path):
|
|
client, _engine, _store, _logs, services = boundary
|
|
download = tmp_path / "download.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
|
|
def fake_download(_url, _sha256, _directory, *, max_bytes):
|
|
assert max_bytes >= download.stat().st_size
|
|
return download, download.stat().st_size, "a" * 64
|
|
|
|
with patch("virsh_panel.user_isos.download_user_iso", side_effect=fake_download):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "My Installer", "sha256": ""})
|
|
assert queued.status_code == 202
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
iso_id = job["iso"]["id"]
|
|
assert "volume_name" not in job["iso"]
|
|
listed = call(client, "GET", "/internal/v1/user-isos").json()
|
|
assert [row["id"] for row in listed["isos"]] == [iso_id]
|
|
assert "volume_name" not in listed["isos"][0]
|
|
global_source = (tmp_path / "user-isos" / str(USER["user_id"])
|
|
/ f"panel-user-iso-{iso_id}.iso")
|
|
assert global_source.is_file()
|
|
assert services == {}
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/user-isos", USER,
|
|
).status_code == 404
|
|
assert call(
|
|
client, "POST", "/internal/v1/hosts/hk/user-isos/import", USER, {
|
|
"url": "https://downloads.example.test/alias.iso",
|
|
"name": "Host Alias Must Fail", "sha256": "",
|
|
},
|
|
).status_code == 404
|
|
assert call(
|
|
client, "DELETE", f"/internal/v1/hosts/hk/user-isos/{iso_id}", USER,
|
|
).status_code == 404
|
|
assert global_source.is_file()
|
|
|
|
other = {**USER, "user_id": 42}
|
|
assert call(client, "GET", f"/internal/v1/user-isos/{iso_id}",
|
|
other).status_code == 404
|
|
assert call(client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
other).status_code == 404
|
|
assert call(client, "DELETE", f"/internal/v1/user-isos/{iso_id}",
|
|
other).status_code == 404
|
|
|
|
provisioning = {**USER, "can_provision": True,
|
|
"operation_id": "create-user-iso-0001"}
|
|
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
body["user_iso_id"] = iso_id
|
|
created = call(client, "POST", "/internal/v1/hosts/hk/vms",
|
|
provisioning, body)
|
|
assert created.status_code == 201, created.text
|
|
create_calls = [row for row in services["hk"].calls
|
|
if row[0] == "create_user_iso_vm"]
|
|
assert len(create_calls) == 1
|
|
assert create_calls[0][2] == USER["user_id"]
|
|
assert create_calls[0][4] == iso_id
|
|
cache_calls = [row for row in services["hk"].calls
|
|
if row[0] == "cache_user_iso"]
|
|
assert len(cache_calls) == 1
|
|
assert cache_calls[0][4] is False
|
|
|
|
other_host_actor = {
|
|
**provisioning, "operation_id": "create-user-iso-0003",
|
|
}
|
|
other_host_body = {**body, "name": "other-host-installer"}
|
|
other_host = call(client, "POST", "/internal/v1/hosts/jp/vms",
|
|
other_host_actor, other_host_body)
|
|
assert other_host.status_code == 201, other_host.text
|
|
assert len([row for row in services["jp"].calls
|
|
if row[0] == "cache_user_iso"]) == 1
|
|
assert global_source.is_file()
|
|
|
|
ambiguous_actor = {**provisioning, "operation_id": "create-user-iso-0002"}
|
|
ambiguous = call(client, "POST", "/internal/v1/hosts/hk/vms",
|
|
ambiguous_actor, {**body, "iso": "legacy.iso"})
|
|
assert ambiguous.status_code == 422
|
|
|
|
|
|
def test_user_iso_writes_require_manage_capability(boundary):
|
|
client, *_rest = boundary
|
|
actor = {**USER, "can_manage": False}
|
|
response = call(client, "POST", "/internal/v1/user-isos/import",
|
|
actor, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Installer", "sha256": ""})
|
|
assert response.status_code == 403
|
|
|
|
|
|
def test_concurrent_first_use_serializes_replica_marker_and_cache(boundary, tmp_path):
|
|
client, _engine, _store, _logs, services = boundary
|
|
download = tmp_path / "concurrent.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, "1" * 64)):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
|
|
"url": "https://downloads.example.test/concurrent.iso",
|
|
"name": "Concurrent Installer", "sha256": "",
|
|
})
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
iso_id = job["iso"]["id"]
|
|
|
|
# Materialize the target child app before starting concurrent requests.
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
service = services["hk"]
|
|
release = threading.Event()
|
|
service.user_iso_cache_release = release
|
|
base = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
base["user_iso_id"] = iso_id
|
|
|
|
def create(number):
|
|
actor = {**USER, "can_provision": True,
|
|
"operation_id": f"concurrent-user-iso-{number:04d}"}
|
|
return call(client, "POST", "/internal/v1/hosts/hk/vms", actor,
|
|
{**base, "name": f"concurrent-vm-{number}"})
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
first = executor.submit(create, 1)
|
|
assert service.user_iso_cache_started.wait(timeout=2)
|
|
second = executor.submit(create, 2)
|
|
time.sleep(0.05)
|
|
release.set()
|
|
responses = [first.result(timeout=5), second.result(timeout=5)]
|
|
|
|
assert [response.status_code for response in responses] == [201, 201]
|
|
cache_calls = [row for row in service.calls if row[0] == "cache_user_iso"]
|
|
assert [row[4] for row in cache_calls] == [False, True]
|
|
|
|
|
|
def test_user_iso_source_stays_in_use_through_vm_create(boundary, tmp_path):
|
|
client, engine, _store, _logs, services = boundary
|
|
iso_id = import_user_iso_for_test(client, tmp_path, "Create Lease Installer")
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
service = services["hk"]
|
|
release = threading.Event()
|
|
service.user_iso_create_release = release
|
|
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
body["user_iso_id"] = iso_id
|
|
actor = {**USER, "can_provision": True,
|
|
"operation_id": "user-iso-source-lease-0001"}
|
|
|
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
creating = executor.submit(
|
|
call, client, "POST", "/internal/v1/hosts/hk/vms", actor, body)
|
|
assert service.user_iso_create_started.wait(timeout=2)
|
|
with pytest.raises(TemplateError, match="正在同步") as busy:
|
|
engine.registry.user_isos.delete_for(USER["user_id"], iso_id)
|
|
assert busy.value.status == 409
|
|
release.set()
|
|
created = creating.result(timeout=5)
|
|
|
|
assert created.status_code == 201, created.text
|
|
assert engine.registry.user_isos.get_for(USER["user_id"], iso_id)["id"] == iso_id
|
|
|
|
|
|
def test_ready_replica_marker_survives_failed_reuse(boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
iso_id = import_user_iso_for_test(client, tmp_path, "Ready Marker Installer")
|
|
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
body["user_iso_id"] = iso_id
|
|
first = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "ready-marker-create-0001"}, body)
|
|
assert first.status_code == 201, first.text
|
|
marker_key = f"{USER['user_id']}:{iso_id}"
|
|
scoped = HostStore(store, "hk")
|
|
assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready"
|
|
|
|
with patch.object(
|
|
engine.registry.user_isos, "_verify_source",
|
|
side_effect=TemplateError("ISO 源文件的 SHA-256 校验值不匹配", 500)):
|
|
failed = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "ready-marker-create-0002"},
|
|
{**body, "name": "ready-marker-failed"})
|
|
assert failed.status_code == 500
|
|
assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready"
|
|
|
|
recovered = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "ready-marker-create-0003"},
|
|
{**body, "name": "ready-marker-recovered"})
|
|
assert recovered.status_code == 201, recovered.text
|
|
cache_calls = [row for row in services["hk"].calls
|
|
if row[0] == "cache_user_iso"]
|
|
assert cache_calls[-1][4] is True
|
|
|
|
|
|
def test_global_delete_retries_tombstone_until_host_cache_is_reclaimed(
|
|
boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
download = tmp_path / "delete.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, "2" * 64)):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {
|
|
"url": "https://downloads.example.test/delete.iso",
|
|
"name": "Delete Installer", "sha256": "",
|
|
})
|
|
job_id = queued.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
iso_id = job["iso"]["id"]
|
|
body = {key: value for key, value in CREATE_BODY.items() if key != "iso"}
|
|
body["user_iso_id"] = iso_id
|
|
created = call(
|
|
client, "POST", "/internal/v1/hosts/hk/vms",
|
|
{**USER, "can_provision": True,
|
|
"operation_id": "delete-user-iso-cache-0001"}, body)
|
|
assert created.status_code == 201, created.text
|
|
|
|
from virsh_panel.libvirt_service import VMError
|
|
services["hk"].user_iso_delete_error = VMError(
|
|
"存储池 iso 不存在", 404)
|
|
deleted = call(
|
|
client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
|
|
assert deleted.status_code == 200
|
|
assert deleted.json() == {"deleted": True, "cache_cleanup_pending": True}
|
|
assert call(client, "GET", "/internal/v1/user-isos", USER).json()["isos"] == []
|
|
|
|
tombstone_key = f"{USER['user_id']}:{iso_id}"
|
|
for _ in range(100):
|
|
if any(row[0] == "delete_user_iso" for row in services["hk"].calls):
|
|
break
|
|
time.sleep(0.01)
|
|
assert store.get("user_iso_tombstone", tombstone_key) is not None
|
|
scoped = HostStore(store, "hk")
|
|
assert scoped.get("user_iso_replica", tombstone_key) is not None
|
|
|
|
services["hk"].user_iso_delete_error = VMError(
|
|
"用户 ISO 在宿主机上不存在", 404,
|
|
error_code="user_iso_volume_not_found",
|
|
)
|
|
calls_before = len([
|
|
row for row in services["hk"].calls if row[0] == "delete_user_iso"
|
|
])
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
assert len([
|
|
row for row in services["hk"].calls if row[0] == "delete_user_iso"
|
|
]) == calls_before
|
|
engine.cache_cleanup_retry_seconds = 0.02
|
|
engine._wake_cache_cleanup()
|
|
for _ in range(100):
|
|
if store.get("user_iso_tombstone", tombstone_key) is None:
|
|
break
|
|
time.sleep(0.01)
|
|
assert store.get("user_iso_tombstone", tombstone_key) is None
|
|
assert scoped.get("user_iso_replica", tombstone_key) is None
|
|
|
|
|
|
def test_concurrent_global_deletes_preserve_committed_tombstone(boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
iso_id = import_user_iso_for_test(client, tmp_path, "Concurrent Delete Installer")
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
record = engine.registry.user_isos.get_for(USER["user_id"], iso_id)
|
|
marker_key = f"{USER['user_id']}:{iso_id}"
|
|
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
|
|
"status": "ready",
|
|
"volume_name": record["volume_name"],
|
|
"size_bytes": record["size_bytes"],
|
|
"sha256": record["sha256"],
|
|
}))
|
|
from virsh_panel.libvirt_service import VMError
|
|
services["hk"].user_iso_volumes.add((USER["user_id"], record["volume_name"]))
|
|
services["hk"].user_iso_delete_error = VMError(
|
|
"用户 ISO 仍被虚拟机安装过程引用,不能删除", 409)
|
|
|
|
original_delete = engine.registry.user_isos.delete_for
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
|
|
def blocking_delete(user_id, selected_iso_id):
|
|
started.set()
|
|
assert release.wait(timeout=5)
|
|
return original_delete(user_id, selected_iso_id)
|
|
|
|
try:
|
|
with patch.object(engine.registry.user_isos, "delete_for",
|
|
side_effect=blocking_delete) as delete_for:
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
first = executor.submit(
|
|
call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
|
|
assert started.wait(timeout=2)
|
|
second = executor.submit(
|
|
call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
|
|
time.sleep(0.05)
|
|
release.set()
|
|
responses = [first.result(timeout=5), second.result(timeout=5)]
|
|
assert delete_for.call_count == 1
|
|
finally:
|
|
release.set()
|
|
|
|
assert [response.status_code for response in responses] == [200, 200]
|
|
tombstone = json.loads(store.get("user_iso_tombstone", marker_key))
|
|
assert tombstone["source_deleted"] is True
|
|
assert tombstone["pending_hosts"] == ["hk"]
|
|
|
|
|
|
def test_user_iso_cache_cleanup_blocks_online_update(boundary, tmp_path):
|
|
client, engine, store, _logs, services = boundary
|
|
iso_id = import_user_iso_for_test(client, tmp_path, "Update Gate Installer")
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200
|
|
record = engine.registry.user_isos.get_for(USER["user_id"], iso_id)
|
|
marker_key = f"{USER['user_id']}:{iso_id}"
|
|
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
|
|
"status": "ready",
|
|
"volume_name": record["volume_name"],
|
|
"size_bytes": record["size_bytes"],
|
|
"sha256": record["sha256"],
|
|
}))
|
|
service = services["hk"]
|
|
service.user_iso_volumes.add((USER["user_id"], record["volume_name"]))
|
|
release = threading.Event()
|
|
service.user_iso_delete_release = release
|
|
updater = FakeUpdater()
|
|
updater.token = "valid-token-123456"
|
|
engine.updater = updater
|
|
try:
|
|
deleted = call(
|
|
client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER)
|
|
assert deleted.status_code == 200
|
|
assert service.user_iso_delete_started.wait(timeout=2)
|
|
blocked = call(client, "POST", "/internal/v1/update/apply", SUPER, {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
})
|
|
assert blocked.status_code == 409
|
|
assert updater.active is None
|
|
finally:
|
|
release.set()
|
|
for _ in range(100):
|
|
if not engine.maintenance_tasks:
|
|
break
|
|
time.sleep(0.01)
|
|
assert not engine.maintenance_tasks
|
|
|
|
|
|
def test_unrelated_host_requests_never_run_cache_cleanup(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
iso_id = "f" * 32
|
|
volume_name = f"panel-user-iso-{iso_id}.iso"
|
|
marker_key = f"{USER['user_id']}:{iso_id}"
|
|
HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({
|
|
"status": "ready", "volume_name": volume_name,
|
|
"size_bytes": 1024 * 1024, "sha256": "f" * 64,
|
|
}))
|
|
store.set("user_iso_tombstone", marker_key, json.dumps({
|
|
"user_id": USER["user_id"], "iso_id": iso_id,
|
|
"volume_name": volume_name, "pending_hosts": ["hk"],
|
|
"source_deleted": True, "created_at": 1,
|
|
}))
|
|
with patch.object(
|
|
engine.updater, "active_job",
|
|
side_effect=AssertionError("host reads must not repair updater state")), \
|
|
patch.object(
|
|
engine, "_cleanup_host_user_iso_tombstones",
|
|
side_effect=AssertionError("host reads must not run ISO GC")), \
|
|
patch.object(
|
|
engine, "_cleanup_host_cloud_template_tombstones",
|
|
side_effect=AssertionError("host reads must not run template GC")):
|
|
response = call(client, "GET", "/internal/v1/hosts/hk/resources")
|
|
|
|
assert response.status_code == 200
|
|
assert not any(row[0] == "delete_user_iso" for row in services["hk"].calls)
|
|
assert store.get("user_iso_tombstone", marker_key) is not None
|
|
|
|
with patch.object(
|
|
engine, "_cleanup_host_user_iso_tombstones",
|
|
side_effect=AssertionError("host writes must not run ISO GC")), \
|
|
patch.object(
|
|
engine, "_cleanup_host_cloud_template_tombstones",
|
|
side_effect=AssertionError("host writes must not run template GC")):
|
|
invalid_write = call(
|
|
client, "PUT",
|
|
f"/internal/v1/hosts/hk/host/network-pools/{'a' * 32}",
|
|
SUPER, {},
|
|
)
|
|
assert invalid_write.status_code == 422
|
|
|
|
|
|
def test_removed_inventory_routes_do_not_enter_dispatch(boundary):
|
|
client, engine, *_ = boundary
|
|
with patch.object(
|
|
engine, "_update_lock",
|
|
side_effect=AssertionError("removed inventory route reached mutation dispatch")), \
|
|
patch.object(
|
|
engine.updater, "active_job",
|
|
side_effect=AssertionError("removed inventory route reached updater")):
|
|
capacity = call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/capacity", ADMIN)
|
|
network = call(
|
|
client, "GET", "/internal/v1/hosts/hk/host/network-options", ADMIN)
|
|
|
|
assert capacity.status_code == 404
|
|
assert network.status_code == 404
|
|
|
|
|
|
def test_slow_tombstone_cleanup_does_not_block_engine_startup(tmp_path):
|
|
class StartupRegistry:
|
|
def __init__(self):
|
|
self.components = {"offline": object()}
|
|
self.user_isos = SimpleNamespace(has_active_jobs=lambda: False)
|
|
self.started = False
|
|
self.closed = False
|
|
|
|
async def start(self):
|
|
self.started = True
|
|
|
|
async def close(self):
|
|
self.closed = True
|
|
|
|
async def scenario():
|
|
registry = StartupRegistry()
|
|
updater = FakeUpdater()
|
|
store = MemoryStore()
|
|
iso_id = "f" * 32
|
|
tombstone_key = f"1:{iso_id}"
|
|
store.set("user_iso_tombstone", tombstone_key, json.dumps({
|
|
"user_id": 1,
|
|
"iso_id": iso_id,
|
|
"volume_name": f"panel-user-iso-{iso_id}.iso",
|
|
"pending_hosts": ["offline"],
|
|
"source_deleted": True,
|
|
"created_at": 1,
|
|
}))
|
|
engine = EngineApplication(
|
|
{"service_secret": base64.b64encode(SECRET).decode(),
|
|
"data_dir": str(tmp_path)},
|
|
store, MemoryStore(), registry=registry, updater=updater,
|
|
)
|
|
cleanup_started = asyncio.Event()
|
|
cleanup_release = asyncio.Event()
|
|
|
|
async def slow_cleanup(host_id):
|
|
assert host_id == "offline"
|
|
cleanup_started.set()
|
|
await cleanup_release.wait()
|
|
|
|
engine._cleanup_host_user_iso_tombstones = slow_cleanup
|
|
incoming = asyncio.Queue()
|
|
outgoing = asyncio.Queue()
|
|
|
|
async def receive():
|
|
return await incoming.get()
|
|
|
|
async def send(message):
|
|
await outgoing.put(message)
|
|
|
|
await incoming.put({"type": "lifespan.startup"})
|
|
lifespan = asyncio.create_task(engine._lifespan(receive, send))
|
|
startup = await asyncio.wait_for(outgoing.get(), timeout=0.5)
|
|
assert startup == {"type": "lifespan.startup.complete"}
|
|
await asyncio.wait_for(cleanup_started.wait(), timeout=0.5)
|
|
assert registry.started is True
|
|
assert engine.maintenance_tasks
|
|
assert engine._long_running_host_job() is True
|
|
|
|
cleanup_release.set()
|
|
await incoming.put({"type": "lifespan.shutdown"})
|
|
shutdown = await asyncio.wait_for(outgoing.get(), timeout=0.5)
|
|
assert shutdown == {"type": "lifespan.shutdown.complete"}
|
|
await asyncio.wait_for(lifespan, timeout=0.5)
|
|
assert registry.closed is True
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_tombstone_cleanup_pauses_for_update_then_resumes(tmp_path):
|
|
class StartupRegistry:
|
|
def __init__(self):
|
|
self.components = {"offline": object()}
|
|
self.user_isos = SimpleNamespace(has_active_jobs=lambda: False)
|
|
|
|
async def start(self):
|
|
return None
|
|
|
|
async def close(self):
|
|
return None
|
|
|
|
async def scenario():
|
|
store = MemoryStore()
|
|
iso_id = "e" * 32
|
|
tombstone_key = f"1:{iso_id}"
|
|
store.set("user_iso_tombstone", tombstone_key, json.dumps({
|
|
"user_id": 1,
|
|
"iso_id": iso_id,
|
|
"volume_name": f"panel-user-iso-{iso_id}.iso",
|
|
"pending_hosts": ["offline"],
|
|
"source_deleted": True,
|
|
"created_at": 1,
|
|
}))
|
|
updater = FakeUpdater()
|
|
updater.active = {"job_id": "update-job-running"}
|
|
engine = EngineApplication(
|
|
{"service_secret": base64.b64encode(SECRET).decode(),
|
|
"data_dir": str(tmp_path)},
|
|
store, MemoryStore(), registry=StartupRegistry(), updater=updater,
|
|
)
|
|
engine.cache_cleanup_retry_seconds = 0.02
|
|
cleanup_started = asyncio.Event()
|
|
|
|
async def cleanup(host_id):
|
|
assert host_id == "offline"
|
|
cleanup_started.set()
|
|
store.delete("user_iso_tombstone", tombstone_key)
|
|
|
|
engine._cleanup_host_user_iso_tombstones = cleanup
|
|
incoming = asyncio.Queue()
|
|
outgoing = asyncio.Queue()
|
|
|
|
async def receive():
|
|
return await incoming.get()
|
|
|
|
async def send(message):
|
|
await outgoing.put(message)
|
|
|
|
await incoming.put({"type": "lifespan.startup"})
|
|
lifespan = asyncio.create_task(engine._lifespan(receive, send))
|
|
assert await asyncio.wait_for(outgoing.get(), timeout=0.5) == {
|
|
"type": "lifespan.startup.complete",
|
|
}
|
|
await asyncio.sleep(0.06)
|
|
assert not cleanup_started.is_set()
|
|
|
|
updater.active = None
|
|
await asyncio.wait_for(cleanup_started.wait(), timeout=0.5)
|
|
assert store.get("user_iso_tombstone", tombstone_key) is None
|
|
|
|
await incoming.put({"type": "lifespan.shutdown"})
|
|
assert await asyncio.wait_for(outgoing.get(), timeout=0.5) == {
|
|
"type": "lifespan.shutdown.complete",
|
|
}
|
|
await asyncio.wait_for(lifespan, timeout=0.5)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_user_iso_global_import_does_not_contact_a_host(boundary, tmp_path):
|
|
client, _engine, _store, _logs, services = boundary
|
|
download = tmp_path / "global.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, "f" * 64)):
|
|
response = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Installer", "sha256": ""})
|
|
assert response.status_code == 202
|
|
job_id = response.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
assert services == {}
|
|
|
|
|
|
def test_user_iso_download_is_bounded_by_panel_storage(boundary, tmp_path):
|
|
client, engine, _store, _logs, _services = boundary
|
|
download = tmp_path / "bounded.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
|
|
def bounded_download(_url, _sha256, _directory, *, max_bytes):
|
|
assert max_bytes == 2 * 1024 * 1024
|
|
return download, download.stat().st_size, "d" * 64
|
|
|
|
with patch.object(engine.registry.user_isos.files, "available_bytes",
|
|
return_value=2 * 1024 * 1024), patch(
|
|
"virsh_panel.user_isos.download_user_iso", side_effect=bounded_download):
|
|
response = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Bounded Installer", "sha256": ""})
|
|
assert response.status_code == 202
|
|
job_id = response.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
|
|
|
|
def test_user_iso_job_hides_internal_paths_in_failures(boundary):
|
|
client, _engine, _store, _logs, _services = boundary
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
side_effect=TemplateError("目录 /private/data/user-isos/41 损坏", 500)):
|
|
response = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Error Installer", "sha256": ""})
|
|
assert response.status_code == 202
|
|
job_id = response.json()["job_id"]
|
|
for _ in range(100):
|
|
job = call(client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}").json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "failed"
|
|
assert job["error"] == "ISO 下载或保存失败,请联系管理员查看日志"
|
|
assert "/private/data" not in json.dumps(job, ensure_ascii=False)
|
|
|
|
|
|
def test_user_iso_import_persistence_failure_does_not_stick_pending(boundary, tmp_path):
|
|
client, _engine, store, _logs, _services = boundary
|
|
assert call(client, "GET", "/internal/v1/user-isos").status_code == 200
|
|
original_set = store.set
|
|
|
|
def fail_user_iso_job(kind, key, value):
|
|
if kind == "user_iso_import_job":
|
|
raise DatabaseError()
|
|
return original_set(kind, key, value)
|
|
|
|
body = {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Installer", "sha256": ""}
|
|
with patch.object(store, "set", side_effect=fail_user_iso_job):
|
|
failed = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, body)
|
|
assert failed.status_code == 503
|
|
|
|
download = tmp_path / "retry.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
with patch("virsh_panel.user_isos.download_user_iso",
|
|
return_value=(download, download.stat().st_size, "b" * 64)):
|
|
retried = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, body)
|
|
assert retried.status_code == 202, retried.text
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{retried.json()['job_id']}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
assert job["status"] == "completed", job
|
|
|
|
|
|
def test_interrupted_user_iso_import_is_restored_as_failed(boundary):
|
|
client, _engine, store, _logs, _services = boundary
|
|
job_id = "interrupted-user-iso"
|
|
scope = user_iso_scope(USER["user_id"])
|
|
store.set("user_iso_import_job", job_id, json.dumps({
|
|
"job_id": job_id, "scope": scope, "owner_user_id": USER["user_id"],
|
|
"status": "running", "phase": "正在安全下载 ISO", "created_at": 1,
|
|
}))
|
|
_engine.registry.user_isos._restored = False
|
|
_engine.registry.user_isos.restore_jobs()
|
|
restored = call(
|
|
client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{job_id}",
|
|
)
|
|
assert restored.status_code == 200
|
|
assert restored.json()["status"] == "failed"
|
|
assert "重启" in restored.json()["phase"]
|
|
|
|
|
|
def test_user_iso_list_exposes_only_owners_active_import(boundary, tmp_path):
|
|
client, engine, _store, _logs, _services = boundary
|
|
download = tmp_path / "active.iso"
|
|
download.write_bytes(b"i" * (1024 * 1024))
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
|
|
def blocking_download(_url, _sha256, _directory, *, max_bytes):
|
|
assert max_bytes >= download.stat().st_size
|
|
started.set()
|
|
assert release.wait(timeout=5)
|
|
return download, download.stat().st_size, "c" * 64
|
|
|
|
body = {"url": "https://downloads.example.test/os.iso",
|
|
"name": "Active Installer", "sha256": ""}
|
|
try:
|
|
with patch("virsh_panel.user_isos.download_user_iso", side_effect=blocking_download):
|
|
queued = call(client, "POST", "/internal/v1/user-isos/import",
|
|
USER, body)
|
|
assert queued.status_code == 202
|
|
assert started.wait(timeout=2)
|
|
assert engine._long_running_host_job() is True
|
|
own = call(client, "GET", "/internal/v1/user-isos", USER).json()
|
|
active = own["active_import_job"]
|
|
assert active["job_id"] == queued.json()["job_id"]
|
|
assert active["status"] == "running"
|
|
assert set(active) <= {"job_id", "status", "phase", "created_at",
|
|
"started_at", "finished_at", "iso", "error"}
|
|
assert "scope" not in active and "owner_user_id" not in active
|
|
other = call(client, "GET", "/internal/v1/user-isos",
|
|
{**USER, "user_id": 42}).json()
|
|
assert "active_import_job" not in other
|
|
release.set()
|
|
for _ in range(100):
|
|
job = call(
|
|
client, "GET",
|
|
f"/internal/v1/user-isos/import-jobs/{active['job_id']}",
|
|
).json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
break
|
|
time.sleep(0.01)
|
|
finally:
|
|
release.set()
|
|
assert job["status"] == "completed", job
|
|
assert "active_import_job" not in call(
|
|
client, "GET", "/internal/v1/user-isos", USER).json()
|
|
|
|
|
|
def test_hmac_binds_actor_payload_target_and_rejects_replay():
|
|
store = MemoryStore()
|
|
body = b"{}"
|
|
target = "/internal/v1/hosts/hk/vms?range=1h"
|
|
headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER,
|
|
timestamp=1000, nonce="n" * 24).items()}
|
|
assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
changed = {**headers, "x-cloud-actor": encode_actor(SUPER)}
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000)
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061)
|
|
|
|
invalid_cleanup = {**USER, "system_cleanup": "true"}
|
|
invalid_headers = {key.lower(): value for key, value in signed_headers(
|
|
SECRET, "POST", target, body, invalid_cleanup,
|
|
timestamp=1000, nonce="c" * 24,
|
|
).items()}
|
|
with pytest.raises(EngineAuthError):
|
|
authenticate(
|
|
invalid_headers, SECRET, "POST", target,
|
|
hashlib.sha256(body).hexdigest(), store, now=1000,
|
|
)
|
|
|
|
|
|
def test_private_route_allowlist_and_no_browser_admin_cookie(boundary):
|
|
client, *_ = boundary
|
|
for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin",
|
|
"/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"):
|
|
assert call(client, "POST", path, SUPER, {}).status_code == 404
|
|
assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401
|
|
assert TRUSTED_ACTOR.get() is None
|
|
|
|
|
|
def test_internal_update_routes_are_super_admin_only_and_never_echo_token(boundary):
|
|
client, engine, *_ = boundary
|
|
updater = FakeUpdater()
|
|
triggered = []
|
|
engine.updater = updater
|
|
def trigger():
|
|
triggered.append(True)
|
|
updater.active["status"] = "running"
|
|
engine.update_trigger = trigger
|
|
settings_path = "/internal/v1/update/settings"
|
|
|
|
ready = call(client, "GET", "/internal/v1/health", SUPER)
|
|
assert ready.status_code == 200
|
|
assert ready.json() == {"status": "ok", "accepting_mutations": True}
|
|
|
|
assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401
|
|
assert call(client, "GET", settings_path, USER).status_code == 403
|
|
assert call(client, "GET", settings_path, ADMIN).status_code == 403
|
|
assert call(client, "GET", settings_path, {**SUPER, "can_manage": False}).status_code == 403
|
|
|
|
token = "gitea-private-token-123456"
|
|
saved = call(client, "PUT", settings_path, SUPER, {"access_token": token})
|
|
assert saved.status_code == 200
|
|
assert saved.json()["token_configured"] is True
|
|
assert saved.json()["repository"] == "https://g.luoli.live/root/virt.git"
|
|
assert saved.json()["branch"] == "venix"
|
|
assert token not in saved.text
|
|
assert token not in call(client, "GET", settings_path, SUPER).text
|
|
|
|
checked = call(client, "POST", "/internal/v1/update/check", SUPER, {})
|
|
assert checked.status_code == 200
|
|
assert checked.json()["status"] == "update_available"
|
|
applied = call(client, "POST", "/internal/v1/update/apply", SUPER, {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
})
|
|
assert applied.status_code == 202
|
|
assert applied.json()["job_id"] == "update-job-1234567890"
|
|
assert "private" not in applied.json() and token not in applied.text
|
|
assert triggered == [True]
|
|
job = call(client, "GET", "/internal/v1/update/jobs/update-job-1234567890", SUPER)
|
|
assert job.status_code == 200 and token not in job.text and "private" not in job.json()
|
|
|
|
# While the persisted update job is active, reads remain available but no
|
|
# host/VM mutation may begin and overlap the code/service restart.
|
|
maintenance = call(client, "GET", "/internal/v1/health", SUPER)
|
|
assert maintenance.status_code == 200
|
|
assert maintenance.json() == {"status": "ok", "accepting_mutations": False}
|
|
assert call(client, "POST", "/internal/v1/hosts/scan", SUPER,
|
|
{"host": "host.example.test", "port": 22}).status_code == 409
|
|
|
|
|
|
def test_internal_update_validation_clear_and_trigger_failure(boundary):
|
|
client, engine, *_ = boundary
|
|
updater = FakeUpdater()
|
|
engine.updater = updater
|
|
settings_path = "/internal/v1/update/settings"
|
|
assert call(client, "PUT", settings_path, SUPER, {}).status_code == 422
|
|
assert call(client, "PUT", settings_path, SUPER,
|
|
{"access_token": "x", "unexpected": True}).status_code == 422
|
|
assert call(client, "POST", "/internal/v1/update/check", SUPER,
|
|
{"repository": "https://evil.example/repo.git"}).status_code == 422
|
|
assert call(client, "POST", "/internal/v1/update/apply", SUPER,
|
|
{"expected_current_commit": "a" * 40}).status_code == 422
|
|
assert call(client, "GET", "/internal/v1/update/jobs/missing-job-123456", SUPER).status_code == 404
|
|
|
|
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
|
|
cleared = call(client, "PUT", settings_path, SUPER, {"clear_token": True})
|
|
assert cleared.status_code == 200 and cleared.json()["token_configured"] is False
|
|
call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"})
|
|
|
|
def fail_trigger():
|
|
from virsh_panel.updater import UpdateError
|
|
raise UpdateError("独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503)
|
|
|
|
engine.update_trigger = fail_trigger
|
|
failed = call(client, "POST", "/internal/v1/update/apply", SUPER, {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
})
|
|
assert failed.status_code == 503
|
|
job = updater.jobs["update-job-1234567890"]
|
|
assert job["status"] == "failed"
|
|
assert "令牌" not in job["error"]
|
|
|
|
|
|
@pytest.mark.parametrize(("state", "expected"), [
|
|
("activating", True),
|
|
("active", True),
|
|
("deactivating", True),
|
|
("inactive", False),
|
|
("failed", False),
|
|
("maintenance", None),
|
|
])
|
|
def test_update_oneshot_transitional_states_keep_maintenance_gate(state, expected):
|
|
with patch("virsh_panel.internal.subprocess.run",
|
|
return_value=SimpleNamespace(returncode=0, stdout=state + "\n")):
|
|
assert EngineApplication._update_service_active() is expected
|
|
|
|
|
|
def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary):
|
|
client, engine, *_ = boundary
|
|
path = "/internal/v1/hosts/hk"
|
|
assert call(client, "GET", path + "/host").status_code == 403
|
|
assert call(client, "GET", path + "/host", ADMIN).status_code == 200
|
|
assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
|
|
assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200
|
|
assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200
|
|
assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404
|
|
assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == []
|
|
assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False},
|
|
{"action": "reboot"}).status_code == 403
|
|
|
|
|
|
def test_stable_host_identity_and_namespaces():
|
|
store = MemoryStore()
|
|
first, second = HostStore(store, "a"), HostStore(store, "b")
|
|
first.set("host", "current", "one")
|
|
second.set("host", "current", "two")
|
|
assert first.get("host", "current") == "one"
|
|
assert second.get("host", "current") == "two"
|
|
a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"}
|
|
assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"})
|
|
assert host_scope(a) != host_scope({**a, "host_id": "b"})
|
|
assert MainStore.RECORD_TABLE == "engine_records"
|
|
assert LogStore.METRIC_TABLE == "engine_vm_metric_samples"
|
|
|
|
|
|
def test_lightweight_vm_state_reads_are_allowlisted(boundary):
|
|
client, _engine, _store, _logs, services = boundary
|
|
listed = call(client, "GET", "/internal/v1/hosts/hk/vm-states", SUPER)
|
|
assert listed.status_code == 200, listed.text
|
|
assert listed.json()["items"] == [{
|
|
"uuid": VM_ID, "name": "alma-test", "state": "running",
|
|
}]
|
|
single = call(
|
|
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/state", SUPER,
|
|
)
|
|
assert single.status_code == 200, single.text
|
|
assert single.json() == {
|
|
"uuid": VM_ID, "name": "alma-test", "state": "running",
|
|
}
|
|
assert ("list_vm_states",) in services["hk"].calls
|
|
assert ("get_vm_state", VM_ID) in services["hk"].calls
|
|
|
|
|
|
def test_selected_vm_metadata_is_allowlisted_and_owner_scoped(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
path = f"/internal/v1/hosts/hk/vm-metadata?ids={VM_ID}"
|
|
|
|
admin = call(client, "GET", path, SUPER)
|
|
assert admin.status_code == 200, admin.text
|
|
assert admin.json()["items"][0]["uuid"] == VM_ID
|
|
assert services["hk"].calls[-1] == ("list_vm_metadata", (VM_ID,))
|
|
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
component.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
owner = call(client, "GET", path, USER)
|
|
assert owner.status_code == 200, owner.text
|
|
calls_after_owner = services["hk"].calls.count(("list_vm_metadata", (VM_ID,)))
|
|
assert calls_after_owner == 2
|
|
|
|
stranger = call(client, "GET", path, {**USER, "user_id": 42})
|
|
assert stranger.status_code == 404
|
|
assert stranger.json()["error"] == "虚拟机不存在"
|
|
assert services["hk"].calls.count(("list_vm_metadata", (VM_ID,))) == calls_after_owner
|
|
|
|
|
|
def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403
|
|
provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"}
|
|
created = call(client, "POST", path, provisioning, CREATE_BODY)
|
|
assert created.status_code == 201, created.text
|
|
vm_id = created.json()["uuid"]
|
|
assert created.json()["result"]["uuid"] == vm_id
|
|
assert created.json()["status"] == "succeeded"
|
|
component = engine.registry.components["hk"]
|
|
assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41)
|
|
again = call(client, "POST", path, provisioning, CREATE_BODY)
|
|
assert again.status_code == 201 and again.json()["uuid"] == vm_id
|
|
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
|
|
assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409
|
|
operation = call(client, "GET", "/internal/v1/operations/order-000001").json()
|
|
assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id
|
|
assert operation["safe_to_refund"] is False
|
|
assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404
|
|
|
|
|
|
def test_uncertain_creation_never_refunds_or_recreates(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
call(client, "GET", path)
|
|
services["hk"].create_failure = True
|
|
actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"}
|
|
failed = call(client, "POST", path, actor, CREATE_BODY)
|
|
assert failed.status_code == 502
|
|
assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"]
|
|
record = json.loads(store.get("operation", "order-uncertain"))
|
|
assert record["resource_naming"] == "uuid-v1"
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
assert component.state.users.owns_vm(scope, record["vm_uuid"], 41)
|
|
component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password")
|
|
component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]}))
|
|
again = call(client, "POST", path, actor, CREATE_BODY)
|
|
assert again.status_code == 202 and again.json()["status"] == "uncertain"
|
|
assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1
|
|
services["hk"].absent = True
|
|
operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json()
|
|
assert operation["status"] == "failed" and operation["safe_to_refund"] is True
|
|
assert services["hk"].absent_checks[-1] == (
|
|
record["vm_uuid"], CREATE_BODY["name"], True, False)
|
|
assert not component.state.users.owns_vm(scope, record["vm_uuid"], 41)
|
|
assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") is None
|
|
assert component.store.list("ip_claim") == {}
|
|
|
|
|
|
def test_legacy_name_collision_operation_uses_legacy_absent_reconciliation(boundary):
|
|
client, _engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
services["hk"].absent = True
|
|
legacy_vm_id = "00000000-0000-4000-8000-000000000002"
|
|
store.set("operation", "legacy-name-collision", json.dumps({
|
|
"operation_id": "legacy-name-collision",
|
|
"host_id": "hk",
|
|
"user_id": USER["user_id"],
|
|
"request_hash": "legacy",
|
|
"vm_uuid": legacy_vm_id,
|
|
"vm_name": CREATE_BODY["name"],
|
|
"kind": "create",
|
|
"resource": "vms",
|
|
"method": "POST",
|
|
"status": "uncertain",
|
|
"result": None,
|
|
"error": "虚拟机名称已存在",
|
|
"safe_to_refund": False,
|
|
"created_at": 1,
|
|
}))
|
|
|
|
operation = call(
|
|
client, "GET", "/internal/v1/operations/legacy-name-collision", USER,
|
|
).json()
|
|
|
|
assert operation["status"] == "failed"
|
|
assert operation["safe_to_refund"] is True
|
|
assert services["hk"].absent_checks[-1] == (
|
|
legacy_vm_id, CREATE_BODY["name"], False, True)
|
|
|
|
|
|
@pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"],
|
|
{**CREATE_BODY, "guest_ssh_key_id": 3}])
|
|
def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload):
|
|
client, engine, _, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/vms"
|
|
actor = {**USER, "can_provision": True, "operation_id": "rejected-create"}
|
|
response = call(client, "POST", path, actor, payload)
|
|
assert response.status_code == 422
|
|
# Simulate a dispatcher losing the POST response and querying after restart.
|
|
engine.active_operations.clear()
|
|
saved = call(client, "GET", "/internal/v1/operations/rejected-create").json()
|
|
assert saved["status"] == "failed" and saved["safe_to_refund"] is True
|
|
assert call(client, "POST", path, actor, payload).status_code == 422
|
|
assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409
|
|
assert not any(row[0] == "create_vm" for row in services["hk"].calls)
|
|
|
|
|
|
def test_admin_created_vm_also_has_external_owner(boundary):
|
|
client, engine, *_ = boundary
|
|
actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"}
|
|
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
|
|
assert response.status_code == 201
|
|
components = engine.registry.components["hk"]
|
|
assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1)
|
|
|
|
|
|
def test_power_action_has_persistent_idempotency(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
|
assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403
|
|
actor = {**USER, "operation_id": "reboot-operation"}
|
|
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
|
|
assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200
|
|
assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1
|
|
assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409
|
|
operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json()
|
|
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
|
|
saved = json.loads(store.get("operation", "reboot-operation"))
|
|
assert saved["target_action"] == "reboot"
|
|
|
|
|
|
def test_orphaned_power_action_reconciles_only_the_final_target_state(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
services["hk"].vm["state"] = "stopped"
|
|
operation = {
|
|
"operation_id": "orphan-power-state", "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation", "resource": f"vms/{VM_ID}/actions",
|
|
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
|
|
"safe_to_refund": False, "created_at": 1,
|
|
"actor_can_manage": True, "actor_is_admin": False,
|
|
"target_action": "shutdown",
|
|
}
|
|
store.set("operation", operation["operation_id"], json.dumps(operation))
|
|
recovered = call(
|
|
client, "GET", "/internal/v1/operations/orphan-power-state", USER,
|
|
).json()
|
|
assert recovered["status"] == "succeeded"
|
|
assert recovered["result"] == {
|
|
"uuid": VM_ID, "name": "alma-test", "state": "stopped",
|
|
}
|
|
assert ("get_vm_state", VM_ID) in services["hk"].calls
|
|
assert not any(row[0] == "action" for row in services["hk"].calls)
|
|
|
|
services["hk"].vm["state"] = "running"
|
|
mismatched = {**operation, "operation_id": "orphan-power-mismatch",
|
|
"target_action": "poweroff"}
|
|
store.set("operation", mismatched["operation_id"], json.dumps(mismatched))
|
|
failed = call(
|
|
client, "GET", "/internal/v1/operations/orphan-power-mismatch", USER,
|
|
).json()
|
|
assert failed["status"] == "failed"
|
|
assert "不会自动重放" in failed["error"]
|
|
assert not any(row[0] == "action" for row in services["hk"].calls)
|
|
|
|
for action in ("reboot", "reset"):
|
|
ambiguous = {
|
|
**operation,
|
|
"operation_id": f"orphan-power-{action}",
|
|
"target_action": action,
|
|
}
|
|
store.set("operation", ambiguous["operation_id"], json.dumps(ambiguous))
|
|
unresolved = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{ambiguous['operation_id']}", USER,
|
|
).json()
|
|
assert unresolved["status"] == "uncertain"
|
|
assert unresolved["result"] is None
|
|
assert "不会自动重放" in unresolved["error"]
|
|
assert not any(row[0] == "action" for row in services["hk"].calls)
|
|
|
|
services["hk"].vm["state"] = "stopped"
|
|
reboot_off = {
|
|
**operation,
|
|
"operation_id": "orphan-power-reboot-off",
|
|
"target_action": "reboot",
|
|
}
|
|
store.set("operation", reboot_off["operation_id"], json.dumps(reboot_off))
|
|
unresolved_off = call(
|
|
client, "GET",
|
|
"/internal/v1/operations/orphan-power-reboot-off", USER,
|
|
).json()
|
|
assert unresolved_off["status"] == "uncertain"
|
|
assert unresolved_off["result"] is None
|
|
assert "不会自动重放" in unresolved_off["error"]
|
|
assert not any(row[0] == "action" for row in services["hk"].calls)
|
|
|
|
|
|
def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary):
|
|
client, engine, _, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
rescue_key = f"{scope}.{VM_ID}"
|
|
components.store.set("vm_rescue", rescue_key, json.dumps({
|
|
"original_xml": "<domain><name>alma-test</name></domain>",
|
|
"template_id": "0123456789abcdef0123456789abcdef",
|
|
"volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso",
|
|
"status": "active",
|
|
}))
|
|
services["hk"].vm.update(
|
|
state="shutoff", rescue_mode=True, rescue_staged=True,
|
|
rescue_volume_name="panel-rescue-0123456789abcdef0123456789abcdef.iso",
|
|
)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
|
|
|
|
ordinary = call(
|
|
client, "DELETE", path,
|
|
{**USER, "operation_id": "ordinary-rescue-delete", "system_cleanup": False},
|
|
)
|
|
assert ordinary.status_code == 409
|
|
assert components.store.get("vm_rescue", rescue_key) is not None
|
|
assert not any(item[0] == "delete_vm" for item in services["hk"].calls)
|
|
|
|
cleanup = call(
|
|
client, "DELETE", path,
|
|
{**USER, "operation_id": "expiry-rescue-delete", "system_cleanup": True},
|
|
)
|
|
assert cleanup.status_code == 200, cleanup.text
|
|
assert cleanup.json()["status"] == "succeeded"
|
|
assert ("delete_vm", VM_ID, True) in services["hk"].calls
|
|
assert services["hk"].calls[-1] == ("delete_firewall_filter", VM_ID)
|
|
assert components.store.get("vm_rescue", rescue_key) is None
|
|
assert components.state.users.owner_of_vm(scope, VM_ID) is None
|
|
|
|
|
|
def test_deleted_vm_reconciles_exact_snapshot_and_control_plane_cleanup(
|
|
boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
identity_key = f"{scope}.{VM_ID}"
|
|
components.store.set("vm_credential", identity_key, json.dumps({
|
|
"auth_mode": "ssh_key", "password": None, "generation": "g1",
|
|
}))
|
|
components.store.set("vm_rescue", identity_key, json.dumps({
|
|
"original_xml": "<domain><name>alma-test</name></domain>",
|
|
"template_id": "0123456789abcdef0123456789abcdef",
|
|
"volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso",
|
|
"status": "active",
|
|
}))
|
|
claim_key = f"{scope}.8.8.8.4"
|
|
components.store.set("ip_claim", claim_key, json.dumps({
|
|
"vm_uuid": VM_ID,
|
|
}))
|
|
historical_failed_key = "d" * 32
|
|
ready_snapshot_keys = ["e" * 32, "f" * 32]
|
|
snapshot_keys = [historical_failed_key, *ready_snapshot_keys]
|
|
volume_names = {
|
|
key: snapshot_volume_name(VM_ID, key) for key in snapshot_keys
|
|
}
|
|
catalog_keys = {key: f"{VM_ID}:{key}" for key in snapshot_keys}
|
|
for created_at, key in enumerate(snapshot_keys, start=1):
|
|
components.store.set("vm_snapshot", catalog_keys[key], json.dumps({
|
|
"snapshot_key": key,
|
|
"vm_id": VM_ID,
|
|
"name": f"清退恢复点-{created_at}",
|
|
"state": ("failed" if key == historical_failed_key
|
|
else "ready"),
|
|
"consistency": "crash",
|
|
"volume_name": volume_names[key],
|
|
"created_at": created_at,
|
|
}))
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
state="shutoff", rescue_mode=True, rescue_staged=True,
|
|
)
|
|
failed_once = False
|
|
|
|
def flaky_snapshot_delete(vm_id, snapshot_key, volume_name, *,
|
|
allow_missing_vm=False):
|
|
nonlocal failed_once
|
|
service.calls.append((
|
|
"delete_snapshot_volume", vm_id, snapshot_key, volume_name,
|
|
allow_missing_vm,
|
|
))
|
|
if snapshot_key == historical_failed_key and not failed_once:
|
|
failed_once = True
|
|
raise VMError("快照卷删除响应丢失", 502)
|
|
return {"snapshot_key": snapshot_key, "deleted": True}
|
|
|
|
service.delete_snapshot_volume = flaky_snapshot_delete
|
|
operation_id = "delete-snapshot-cleanup"
|
|
actor = {
|
|
**USER,
|
|
"operation_id": operation_id,
|
|
"system_cleanup": True,
|
|
}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
|
|
|
|
first = call(client, "DELETE", path, actor)
|
|
|
|
assert first.status_code == 503, first.text
|
|
assert first.json()["status"] == "uncertain"
|
|
assert first.json()["error_code"] == (
|
|
"vm_delete_snapshot_cleanup_pending")
|
|
assert all(volume_name not in first.text
|
|
for volume_name in volume_names.values())
|
|
saved = json.loads(store.get("operation", operation_id))
|
|
assert saved["cleanup_pending"] == "vm_delete"
|
|
assert saved["vm_delete_committed"] is True
|
|
assert saved["delete_cleanup_checkpoint"] == {
|
|
"version": 1,
|
|
"vm_uuid": VM_ID,
|
|
"snapshots": [
|
|
{
|
|
"snapshot_key": key,
|
|
"volume_name": volume_names[key],
|
|
}
|
|
for key in snapshot_keys
|
|
],
|
|
"system_cleanup": True,
|
|
}
|
|
assert len([item for item in service.calls
|
|
if item[0] == "delete_vm"]) == 1
|
|
assert all(components.store.get(
|
|
"vm_snapshot", catalog_keys[key]) is None
|
|
for key in ready_snapshot_keys)
|
|
assert components.store.get(
|
|
"vm_snapshot", catalog_keys[historical_failed_key]) is not None
|
|
assert components.state.users.owner_of_vm(scope, VM_ID) == USER["user_id"]
|
|
assert components.store.get("vm_credential", identity_key) is not None
|
|
assert components.store.get("vm_rescue", identity_key) is not None
|
|
assert components.store.get("ip_claim", claim_key) is not None
|
|
|
|
recovered = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", actor)
|
|
|
|
assert recovered.status_code == 200, recovered.text
|
|
assert recovered.json()["status"] == "succeeded"
|
|
assert recovered.json()["result"] == {
|
|
"uuid": VM_ID,
|
|
"deleted": True,
|
|
"deleted_snapshots": snapshot_keys,
|
|
"public_ip_claims_released": True,
|
|
"firewall_cleaned": True,
|
|
}
|
|
assert all(volume_name not in recovered.text
|
|
for volume_name in volume_names.values())
|
|
assert len([item for item in service.calls
|
|
if item[0] == "delete_vm"]) == 1
|
|
assert all(components.store.get("vm_snapshot", catalog_keys[key]) is None
|
|
for key in snapshot_keys)
|
|
assert components.state.users.owner_of_vm(scope, VM_ID) is None
|
|
assert components.store.get("vm_credential", identity_key) is None
|
|
assert components.store.get("vm_rescue", identity_key) is None
|
|
assert components.store.get("ip_claim", claim_key) is None
|
|
|
|
|
|
def test_deleted_vm_reconciles_postcommit_control_plane_failure(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
identity_key = f"{scope}.{VM_ID}"
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
components.store.set("vm_credential", identity_key, json.dumps({
|
|
"auth_mode": "ssh_key", "password": None, "generation": "g1",
|
|
}))
|
|
claim_key = f"{scope}.8.8.8.4"
|
|
components.store.set("ip_claim", claim_key, json.dumps({
|
|
"vm_uuid": VM_ID,
|
|
}))
|
|
service = services["hk"]
|
|
service.vm["state"] = "shutoff"
|
|
original_delete = components.store.delete
|
|
failed_once = False
|
|
|
|
def flaky_delete(kind, key):
|
|
nonlocal failed_once
|
|
if kind == "vm_credential" and not failed_once:
|
|
failed_once = True
|
|
raise DatabaseError("credential delete interrupted")
|
|
return original_delete(kind, key)
|
|
|
|
components.store.delete = flaky_delete
|
|
operation_id = "delete-control-cleanup"
|
|
actor = {
|
|
**USER, "operation_id": operation_id, "system_cleanup": True,
|
|
}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
|
|
|
|
first = call(client, "DELETE", path, actor)
|
|
|
|
assert first.status_code == 503, first.text
|
|
assert first.json()["error_code"] == (
|
|
"vm_delete_snapshot_cleanup_pending")
|
|
assert len([item for item in service.calls
|
|
if item[0] == "delete_vm"]) == 1
|
|
assert components.store.get("vm_credential", identity_key) is not None
|
|
assert components.store.get("ip_claim", claim_key) is not None
|
|
|
|
components.store.delete = original_delete
|
|
recovered = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", actor)
|
|
|
|
assert recovered.status_code == 200
|
|
assert recovered.json()["status"] == "succeeded"
|
|
assert len([item for item in service.calls
|
|
if item[0] == "delete_vm"]) == 1
|
|
assert components.state.users.owner_of_vm(scope, VM_ID) is None
|
|
assert components.store.get("vm_credential", identity_key) is None
|
|
assert components.store.get("ip_claim", claim_key) is None
|
|
|
|
|
|
def test_deleted_vm_volume_cleanup_reconciles_v2_manifest(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
service = services["hk"]
|
|
service.vm["state"] = "shutoff"
|
|
service.delete_vm_error = VMError(
|
|
"虚拟机定义已删除,但磁盘删除失败", 503,
|
|
error_code="vm_delete_volume_cleanup_pending",
|
|
)
|
|
manifest = {
|
|
"vm_uuid": VM_ID,
|
|
# Reinstall intentionally creates a fresh UUID seed unrelated to the
|
|
# VM UUID. The persisted delete checkpoint must accept that identity.
|
|
"seed_volume": "panel-seed-11111111-2222-4333-8444-555555555555.iso",
|
|
"primary_volume": "panel-restore-0123456789abcdef0123456789abcdef0123456789abcdef.qcow2",
|
|
}
|
|
service.delete_vm_cleanup_manifest = lambda vm_id: manifest.copy()
|
|
cleanup_calls = []
|
|
|
|
def cleanup_deleted_vm_volumes(vm_id, resources):
|
|
cleanup_calls.append((vm_id, resources))
|
|
return {"deleted": [resources["primary_volume"]]}
|
|
|
|
service.cleanup_deleted_vm_volumes = cleanup_deleted_vm_volumes
|
|
operation_id = "delete-volume-cleanup-v2"
|
|
actor = {**USER, "operation_id": operation_id}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true"
|
|
|
|
first = call(client, "DELETE", path, actor)
|
|
|
|
assert first.status_code == 503, first.text
|
|
assert first.json()["error_code"] == "vm_delete_volume_cleanup_pending"
|
|
saved = json.loads(store.get("operation", operation_id))
|
|
assert saved["cleanup_pending"] == "vm_delete"
|
|
assert saved["vm_delete_committed"] is True
|
|
assert saved["delete_cleanup_checkpoint"]["version"] == 2
|
|
assert saved["delete_cleanup_checkpoint"]["resources"] == {
|
|
"seed_volume": manifest["seed_volume"],
|
|
"primary_volume": manifest["primary_volume"],
|
|
}
|
|
|
|
recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}", actor)
|
|
|
|
assert recovered.status_code == 200, recovered.text
|
|
assert recovered.json()["status"] == "succeeded"
|
|
assert cleanup_calls == [(
|
|
VM_ID, saved["delete_cleanup_checkpoint"]["resources"])]
|
|
assert len([item for item in service.calls if item[0] == "delete_vm"]) == 1
|
|
|
|
|
|
@pytest.mark.parametrize(("committed", "checkpoint_volume"), [
|
|
(False, None),
|
|
(True, "panel-snapshot-tampered.qcow2"),
|
|
])
|
|
def test_vm_delete_cleanup_requires_durable_commit_and_exact_identity(
|
|
boundary, committed, checkpoint_volume):
|
|
client, _engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
snapshot_key = "c" * 32
|
|
volume_name = snapshot_volume_name(VM_ID, snapshot_key)
|
|
operation_id = (
|
|
"delete-cleanup-tampered" if checkpoint_volume
|
|
else "delete-cleanup-not-committed"
|
|
)
|
|
operation = {
|
|
"operation_id": operation_id,
|
|
"host_id": "hk",
|
|
"user_id": USER["user_id"],
|
|
"vm_uuid": VM_ID,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}",
|
|
"method": "DELETE",
|
|
"status": "uncertain",
|
|
"result": None,
|
|
"safe_to_refund": False,
|
|
"cleanup_pending": "vm_delete",
|
|
"delete_ip_claim_keys": [],
|
|
"delete_cleanup_checkpoint": {
|
|
"version": 1,
|
|
"vm_uuid": VM_ID,
|
|
"snapshots": [{
|
|
"snapshot_key": snapshot_key,
|
|
"volume_name": checkpoint_volume or volume_name,
|
|
}],
|
|
"system_cleanup": True,
|
|
},
|
|
"created_at": 1,
|
|
}
|
|
if committed:
|
|
operation["vm_delete_committed"] = True
|
|
store.set("operation", operation_id, json.dumps(operation))
|
|
before = list(services["hk"].calls)
|
|
|
|
response = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "uncertain"
|
|
assert "delete_cleanup_checkpoint" not in response.text
|
|
assert volume_name not in response.text
|
|
if checkpoint_volume is not None:
|
|
assert checkpoint_volume not in response.text
|
|
assert services["hk"].calls == before
|
|
assert not any(item[0] in {"delete_vm", "delete_snapshot_volume"}
|
|
for item in services["hk"].calls[len(before):])
|
|
|
|
|
|
@pytest.mark.parametrize(("route", "method", "target"), [
|
|
("service-suspend", "service_suspend", {"state": "shutoff", "autostart": False}),
|
|
("service-resume", "service_resume", {"autostart": True}),
|
|
("network-suspend", "network_suspend", {"network_suspended": True}),
|
|
("network-resume", "network_resume", {"network_suspended": False}),
|
|
])
|
|
def test_private_target_state_operations_are_owned_idempotent_mutations(
|
|
boundary, route, method, target):
|
|
client, engine, _, _, services = boundary
|
|
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
|
|
assert call(client, "POST", base, USER, {}).status_code == 403
|
|
assert call(client, "POST", base, {**USER, "can_manage": False,
|
|
"operation_id": f"blocked-{route}"}, {}).status_code == 403
|
|
assert call(client, "POST", base, {**USER, "user_id": 42,
|
|
"operation_id": f"foreign-{route}"}, {}).status_code == 404
|
|
|
|
actor = {**USER, "operation_id": f"target-{route}"}
|
|
first = call(client, "POST", base, actor, {})
|
|
assert first.status_code == 200, first.text
|
|
assert first.json()["status"] == "succeeded"
|
|
assert all(first.json()[key] == value for key, value in target.items())
|
|
repeated = call(client, "POST", base, actor, {})
|
|
assert repeated.status_code == 200
|
|
assert len([item for item in services["hk"].calls if item[0] == method]) == 1
|
|
|
|
|
|
@pytest.mark.parametrize(("route", "method", "drift"), [
|
|
("service-suspend", "service_suspend", {"state": "running", "autostart": True}),
|
|
("service-resume", "service_resume", {"autostart": False}),
|
|
("network-suspend", "network_suspend", {"network_suspended": False}),
|
|
("network-resume", "network_resume", {"network_suspended": True}),
|
|
])
|
|
def test_target_state_operation_replays_after_lost_response(
|
|
boundary, route, method, drift):
|
|
client, engine, store, _, services = boundary
|
|
base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}"
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
actor = {**USER, "operation_id": f"replay-{route}"}
|
|
assert call(client, "POST", base, actor, {}).status_code == 200
|
|
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(drift)
|
|
engine.active_operations.clear()
|
|
|
|
replayed = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
|
|
actor)
|
|
assert replayed.status_code == 200, replayed.text
|
|
assert replayed.json()["status"] == "succeeded"
|
|
assert len([item for item in services["hk"].calls if item[0] == method]) == 2
|
|
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost again")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(drift)
|
|
replayed_post = call(client, "POST", base, actor, {})
|
|
assert replayed_post.status_code == 200, replayed_post.text
|
|
assert replayed_post.json()["status"] == "succeeded"
|
|
assert len([item for item in services["hk"].calls if item[0] == method]) == 3
|
|
|
|
|
|
def test_non_reconciling_operation_lookup_never_replays_target_state(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend"
|
|
actor = {**USER, "operation_id": "peek-target-state"}
|
|
assert call(client, "POST", path, actor, {}).status_code == 200
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm["network_suspended"] = False
|
|
engine.active_operations.clear()
|
|
before = len(services["hk"].calls)
|
|
|
|
peeked = call(client, "GET",
|
|
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
|
|
assert peeked.status_code == 200
|
|
assert peeked.json()["status"] == "uncertain"
|
|
assert peeked.json()["executor_active"] is False
|
|
assert len(services["hk"].calls) == before
|
|
assert services["hk"].vm["network_suspended"] is False
|
|
|
|
engine.active_operations.add(actor["operation_id"])
|
|
active = call(client, "GET",
|
|
"/internal/v1/operations/peek-target-state?reconcile=false", actor)
|
|
assert active.status_code == 200
|
|
assert active.json()["executor_active"] is True
|
|
assert len(services["hk"].calls) == before
|
|
engine.active_operations.discard(actor["operation_id"])
|
|
|
|
|
|
def test_target_state_lock_serializes_same_vm_and_not_other_vms(boundary):
|
|
_, engine, *_ = boundary
|
|
|
|
async def exercise():
|
|
first = engine._target_lock("hk", VM_ID)
|
|
same = engine._target_lock("hk", VM_ID)
|
|
other = engine._target_lock("hk", "22222222-2222-4222-8222-222222222222")
|
|
assert first is same
|
|
assert first is not other
|
|
entered = []
|
|
release = asyncio.Event()
|
|
|
|
async def owner():
|
|
async with first:
|
|
entered.append("owner")
|
|
await release.wait()
|
|
entered.append("owner-finished")
|
|
|
|
async def waiter():
|
|
async with same:
|
|
entered.append("waiter")
|
|
|
|
owner_task = asyncio.create_task(owner())
|
|
await asyncio.sleep(0)
|
|
waiter_task = asyncio.create_task(waiter())
|
|
await asyncio.sleep(0)
|
|
assert entered == ["owner"]
|
|
release.set()
|
|
await asyncio.gather(owner_task, waiter_task)
|
|
assert entered == ["owner", "owner-finished", "waiter"]
|
|
|
|
errors = []
|
|
def run_in_isolated_loop():
|
|
try:
|
|
asyncio.run(exercise())
|
|
except BaseException as exc: # propagate assertion failures to pytest
|
|
errors.append(exc)
|
|
|
|
thread = threading.Thread(target=run_in_isolated_loop)
|
|
thread.start()
|
|
thread.join(timeout=5)
|
|
assert not thread.is_alive()
|
|
if errors:
|
|
raise errors[0]
|
|
|
|
|
|
def test_expiry_target_waits_for_inflight_user_mutation_on_same_vm(boundary):
|
|
client, engine, _, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
service = services["hk"]
|
|
action_entered = threading.Event()
|
|
release_action = threading.Event()
|
|
network_entered = threading.Event()
|
|
original_action = service.action
|
|
original_network_suspend = service.network_suspend
|
|
|
|
def slow_action(vm_id, action):
|
|
action_entered.set()
|
|
assert release_action.wait(timeout=5), "test did not release user mutation"
|
|
return original_action(vm_id, action)
|
|
|
|
def observed_network_suspend(vm_id):
|
|
network_entered.set()
|
|
return original_network_suspend(vm_id)
|
|
|
|
service.action = slow_action
|
|
service.network_suspend = observed_network_suspend
|
|
responses = {}
|
|
|
|
def user_request():
|
|
responses["action"] = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions",
|
|
{**USER, "operation_id": "concurrent-user-action"}, {"action": "reset"})
|
|
|
|
def policy_request():
|
|
responses["policy"] = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
|
|
{**USER, "operation_id": "concurrent-expiry-policy"}, {})
|
|
|
|
first = threading.Thread(target=user_request)
|
|
first.start()
|
|
assert action_entered.wait(timeout=5)
|
|
second = threading.Thread(target=policy_request)
|
|
second.start()
|
|
assert not network_entered.wait(timeout=0.2)
|
|
release_action.set()
|
|
first.join(timeout=5)
|
|
second.join(timeout=5)
|
|
assert not first.is_alive() and not second.is_alive()
|
|
assert responses["action"].status_code == 200
|
|
assert responses["policy"].status_code == 200
|
|
assert network_entered.is_set()
|
|
mutations = [item[0] for item in service.calls
|
|
if item[0] in {"action", "network_suspend"}]
|
|
assert mutations == ["action", "network_suspend"]
|
|
assert service.vm["network_suspended"] is True
|
|
|
|
|
|
def test_target_state_reconciliation_does_not_bypass_original_owner(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
store.set("operation", "unsafe-target", json.dumps({
|
|
"operation_id": "unsafe-target", "host_id": "hk", "user_id": 42,
|
|
"kind": "mutation", "resource": f"vms/{VM_ID}/network-suspend",
|
|
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
|
|
"actor_can_manage": True, "actor_is_admin": False,
|
|
"safe_to_refund": False,
|
|
}))
|
|
response = call(client, "GET", "/internal/v1/operations/unsafe-target",
|
|
{**USER, "user_id": 42})
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "failed"
|
|
assert not any(item[0] == "network_suspend" for item in services["hk"].calls)
|
|
|
|
|
|
def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID,
|
|
"status": "running", "phase": "正在救援"}
|
|
components.store.set("rescue_job", "job-abc", json.dumps(job))
|
|
operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID,
|
|
"status": "uncertain", "result": None, "safe_to_refund": False}
|
|
store.set("operation", "lost-rescue-job", json.dumps(operation))
|
|
recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
|
|
assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID
|
|
job.update(status="completed", phase="救援系统启动")
|
|
components.store.set("rescue_job", "job-abc", json.dumps(job))
|
|
completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json()
|
|
assert completed["status"] == "succeeded"
|
|
assert not completed["safe_to_refund"]
|
|
|
|
|
|
def test_snapshot_mutation_operation_returns_only_safe_result(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
services["hk"].vm.update(
|
|
managed=True, os_source="cloud_template",
|
|
rescue_mode=False, rescue_staged=False,
|
|
)
|
|
actor = {**USER, "operation_id": "snapshot-create-0001"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots"
|
|
|
|
queued = call(client, "POST", path, actor, {
|
|
"snapshot_key": "a" * 32,
|
|
"name": "发布前恢复点",
|
|
"consistency_mode": "crash",
|
|
})
|
|
assert queued.status_code == 202, queued.text
|
|
for _ in range(100):
|
|
operation = call(
|
|
client, "GET", "/internal/v1/operations/snapshot-create-0001",
|
|
actor,
|
|
).json()
|
|
if operation["status"] != "running" and operation["status"] != "pending":
|
|
break
|
|
time.sleep(0.01)
|
|
|
|
assert operation["status"] == "succeeded"
|
|
assert operation["result"]["snapshot_key"] == "a" * 32
|
|
assert operation["result"]["created"] is True
|
|
assert operation["result"]["consistency"] == "crash"
|
|
assert "volume_name" not in operation["result"]
|
|
|
|
|
|
def test_snapshot_qga_failure_code_survives_operation_poll(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
services["hk"].vm.update(
|
|
managed=True, os_source="cloud_template",
|
|
rescue_mode=False, rescue_staged=False,
|
|
)
|
|
services["hk"].snapshot_create_error = VMError(
|
|
"QEMU Guest Agent 无法冻结文件系统,快照未创建", 409,
|
|
error_code="snapshot_guest_agent_unavailable",
|
|
)
|
|
actor = {**USER, "operation_id": "snapshot-create-qga1"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots"
|
|
|
|
queued = call(client, "POST", path, actor, {
|
|
"snapshot_key": "b" * 32,
|
|
"name": "一致性恢复点",
|
|
"consistency_mode": "filesystem",
|
|
})
|
|
assert queued.status_code == 202, queued.text
|
|
for _ in range(100):
|
|
operation = call(
|
|
client, "GET", "/internal/v1/operations/snapshot-create-qga1",
|
|
actor,
|
|
).json()
|
|
if operation["status"] not in {"running", "pending"}:
|
|
break
|
|
time.sleep(0.01)
|
|
|
|
assert operation["status"] == "failed"
|
|
assert operation["error_code"] == "snapshot_guest_agent_unavailable"
|
|
|
|
|
|
def test_snapshot_restart_recovers_freeze_and_orphan_transition(boundary):
|
|
client, _engine, store, _logs, services = boundary
|
|
scoped = HostStore(store, "hk")
|
|
freeze_key = "c" * 32
|
|
orphan_key = "d" * 32
|
|
for snapshot_key in (freeze_key, orphan_key):
|
|
scoped.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
|
|
"snapshot_key": snapshot_key,
|
|
"vm_id": VM_ID,
|
|
"name": "中断恢复点",
|
|
"state": "creating",
|
|
"consistency": "filesystem",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
scoped.set("snapshot_job", "snapshot-interrupted", json.dumps({
|
|
"operation_id": "snapshot-interrupted",
|
|
"owner": "php:41",
|
|
"vm_id": VM_ID,
|
|
"snapshot_key": freeze_key,
|
|
"action": "create",
|
|
"consistency": "filesystem",
|
|
"guest_freeze_state": "attempted",
|
|
"status": "uncertain",
|
|
"phase": "正在通过 QEMU Guest Agent 冻结访客文件系统",
|
|
}))
|
|
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
|
|
interrupted = json.loads(scoped.get(
|
|
"snapshot_job", "snapshot-interrupted"))
|
|
assert interrupted["status"] == "uncertain"
|
|
assert interrupted["guest_freeze_state"] == "recovered"
|
|
assert interrupted["freeze_recovery"] == "confirmed"
|
|
assert ("thaw_guest_filesystems", VM_ID) in services["hk"].calls
|
|
record = json.loads(scoped.get(
|
|
"vm_snapshot", f"{VM_ID}:{freeze_key}"))
|
|
assert record["state"] == "uncertain"
|
|
assert scoped.get(
|
|
"vm_snapshot", f"{VM_ID}:{orphan_key}") is None
|
|
|
|
|
|
@pytest.mark.parametrize(("initial_state", "expected_state"), [
|
|
("creating", None),
|
|
("restoring", "ready"),
|
|
("deleting", "ready"),
|
|
])
|
|
def test_snapshot_registry_without_durable_job_rolls_back_before_mutation(
|
|
boundary, initial_state, expected_state):
|
|
client, _engine, store, _logs, services = boundary
|
|
scoped = HostStore(store, "hk")
|
|
snapshot_key = {
|
|
"creating": "b",
|
|
"restoring": "c",
|
|
"deleting": "d",
|
|
}[initial_state] * 32
|
|
scoped.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
|
|
"snapshot_key": snapshot_key,
|
|
"vm_id": VM_ID,
|
|
"name": "尚未入队恢复点",
|
|
"state": initial_state,
|
|
"consistency": "crash",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
|
|
raw = scoped.get("vm_snapshot", f"{VM_ID}:{snapshot_key}")
|
|
if expected_state is None:
|
|
assert raw is None
|
|
else:
|
|
assert json.loads(raw)["state"] == expected_state
|
|
assert not any(call_[0] in {
|
|
"create_live_snapshot", "restore_live_snapshot",
|
|
"delete_snapshot_volume",
|
|
} for call_ in services["hk"].calls)
|
|
|
|
|
|
@pytest.mark.parametrize(("action", "initial_state", "expected_state"), [
|
|
("create", "creating", None),
|
|
("restore", "restoring", "ready"),
|
|
("delete", "deleting", "ready"),
|
|
])
|
|
def test_snapshot_cancel_tombstone_finishes_catalog_rollback_after_restart(
|
|
boundary, action, initial_state, expected_state):
|
|
client, _engine, store, _logs, services = boundary
|
|
scoped = HostStore(store, "hk")
|
|
snapshot_key = {
|
|
"create": "e", "restore": "f", "delete": "a",
|
|
}[action] * 32
|
|
catalog_key = f"{VM_ID}:{snapshot_key}"
|
|
scoped.set("vm_snapshot", catalog_key, json.dumps({
|
|
"snapshot_key": snapshot_key,
|
|
"vm_id": VM_ID,
|
|
"name": "取消中的排队恢复点",
|
|
"state": initial_state,
|
|
"consistency": "crash",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
job_id = f"cancelled-before-start-{action}"
|
|
operation_id = f"snapshot-cancel-restart-{action}"
|
|
scoped.set("snapshot_job", job_id, json.dumps({
|
|
"job_id": job_id,
|
|
"operation_id": operation_id,
|
|
"scope": host_scope({"host_id": "hk"}),
|
|
"owner": "php:41",
|
|
"vm_id": VM_ID,
|
|
"snapshot_key": snapshot_key,
|
|
"action": action,
|
|
"consistency": "crash",
|
|
"status": "queued",
|
|
"phase": "正在取消尚未开始的排队快照",
|
|
"created_at": 1,
|
|
"cancelled_before_start": True,
|
|
"cancel_reason": "network_suspend",
|
|
"cancel_requested_at": 1,
|
|
}))
|
|
resource = f"vms/{VM_ID}/snapshots"
|
|
method = "POST"
|
|
if action != "create":
|
|
resource += f"/{snapshot_key}"
|
|
if action == "restore":
|
|
resource += "/restore"
|
|
else:
|
|
method = "DELETE"
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id,
|
|
"host_id": "hk",
|
|
"user_id": USER["user_id"],
|
|
"kind": "mutation",
|
|
"resource": resource,
|
|
"method": method,
|
|
"vm_uuid": VM_ID,
|
|
"status": "pending",
|
|
"job_id": job_id,
|
|
"result": {"job_id": job_id, "status": "queued"},
|
|
"safe_to_refund": False,
|
|
"created_at": 1,
|
|
}))
|
|
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
|
|
saved_job = json.loads(scoped.get("snapshot_job", job_id))
|
|
assert saved_job["status"] == "failed"
|
|
assert saved_job["cancelled_before_start"] is True
|
|
raw = scoped.get("vm_snapshot", catalog_key)
|
|
if expected_state is None:
|
|
assert raw is None
|
|
else:
|
|
assert json.loads(raw)["state"] == expected_state
|
|
operation = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
|
|
)
|
|
assert operation.status_code == 200
|
|
assert operation.json()["status"] == "failed"
|
|
assert not any(call_[0] in {
|
|
"create_live_snapshot", "restore_live_snapshot",
|
|
"delete_snapshot_volume",
|
|
} for call_ in services["hk"].calls)
|
|
|
|
|
|
def test_snapshot_uncertain_restore_hides_private_volume_evidence(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
snapshot_key = "4" * 32
|
|
operation_id = "snapshot-restore-private"
|
|
job_id = "snapshot-job-private"
|
|
snapshot_volume = snapshot_volume_name(VM_ID, snapshot_key)
|
|
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
|
|
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "恢复点",
|
|
"state": "uncertain", "consistency": "crash",
|
|
"volume_name": snapshot_volume, "created_at": 1,
|
|
}))
|
|
private_old = f"panel-{VM_ID}.qcow2"
|
|
private_replacement = "panel-restore-private.qcow2"
|
|
components.store.set("snapshot_job", job_id, json.dumps({
|
|
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
|
|
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "restore",
|
|
"consistency": "crash", "status": "uncertain", "phase": "结果待核查",
|
|
"restore_old_volume": private_old,
|
|
"restore_replacement_volume": private_replacement,
|
|
"restore_was_running": True,
|
|
}))
|
|
services["hk"].snapshot_reconcile_restore = {"outcome": "uncertain"}
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/snapshots/{snapshot_key}/restore",
|
|
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
|
|
"safe_to_refund": False,
|
|
}))
|
|
|
|
operation = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}").json()
|
|
|
|
assert operation["status"] == "uncertain"
|
|
serialized = json.dumps(operation, ensure_ascii=False)
|
|
assert private_old not in serialized
|
|
assert private_replacement not in serialized
|
|
assert snapshot_volume not in serialized
|
|
|
|
|
|
def test_snapshot_restore_without_persisted_plan_converges_failed_without_replay(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
snapshot_key = "e" * 32
|
|
operation_id = "snapshot-restore-no-plan"
|
|
job_id = "snapshot-job-no-plan"
|
|
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
|
|
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "恢复点",
|
|
"state": "uncertain", "consistency": "crash",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
components.store.set("snapshot_job", job_id, json.dumps({
|
|
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
|
|
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "restore",
|
|
"consistency": "crash", "status": "uncertain", "phase": "面板重启",
|
|
}))
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/snapshots/{snapshot_key}/restore",
|
|
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
|
|
"safe_to_refund": False,
|
|
}))
|
|
|
|
operation = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}").json()
|
|
|
|
assert operation["status"] == "failed"
|
|
record = json.loads(components.store.get(
|
|
"vm_snapshot", f"{VM_ID}:{snapshot_key}"))
|
|
assert record["state"] == "ready"
|
|
assert not any(call_[0] in {
|
|
"restore_live_snapshot", "reconcile_live_snapshot_restore",
|
|
} for call_ in services["hk"].calls)
|
|
|
|
|
|
def test_snapshot_uncertain_delete_without_catalog_converges_by_original_operation(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
snapshot_key = "5" * 32
|
|
operation_id = "snapshot-delete-reconcile"
|
|
job_id = "snapshot-job-delete"
|
|
components.store.set("snapshot_job", job_id, json.dumps({
|
|
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
|
|
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "delete",
|
|
"consistency": "filesystem", "status": "uncertain",
|
|
"phase": "结果待核查",
|
|
}))
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/snapshots/{snapshot_key}",
|
|
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
|
|
"safe_to_refund": False,
|
|
}))
|
|
|
|
operation = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}").json()
|
|
|
|
assert operation["status"] == "succeeded"
|
|
assert operation["result"] == {
|
|
"snapshot_key": snapshot_key,
|
|
"deleted": True,
|
|
"consistency": "filesystem",
|
|
}
|
|
saved_job = json.loads(components.store.get("snapshot_job", job_id))
|
|
assert saved_job["status"] == "succeeded"
|
|
assert not any(call_[0] == "delete_snapshot_volume"
|
|
for call_ in services["hk"].calls)
|
|
assert len([call_ for call_ in services["hk"].calls
|
|
if call_[0] == "reconcile_live_snapshot_delete"]) == 1
|
|
|
|
|
|
def test_snapshot_uncertain_create_reconciles_with_consistency(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
snapshot_key = "6" * 32
|
|
operation_id = "snapshot-create-reconcile"
|
|
job_id = "snapshot-job-create"
|
|
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
|
|
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "恢复点",
|
|
"state": "uncertain", "consistency": "crash",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
components.store.set("snapshot_job", job_id, json.dumps({
|
|
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
|
|
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "create",
|
|
"consistency": "crash", "status": "uncertain", "phase": "结果待核查",
|
|
"backup_verified_capacity": 20 * 1024**3,
|
|
"backup_verified_identity": "a" * 64,
|
|
}))
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/snapshots",
|
|
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
|
|
"safe_to_refund": False,
|
|
}))
|
|
|
|
operation = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}").json()
|
|
|
|
assert operation["status"] == "succeeded"
|
|
assert operation["result"]["created"] is True
|
|
assert operation["result"]["consistency"] == "crash"
|
|
assert "volume_name" not in operation["result"]
|
|
assert not any(call_[0] == "create_live_snapshot"
|
|
for call_ in services["hk"].calls)
|
|
assert len([call_ for call_ in services["hk"].calls
|
|
if call_[0] == "reconcile_live_snapshot_create"]) == 1
|
|
|
|
|
|
def test_snapshot_confirmed_failed_create_removes_hidden_catalog(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
snapshot_key = "b" * 32
|
|
operation_id = "snapshot-create-confirmed-failed"
|
|
job_id = "snapshot-job-create-failed"
|
|
catalog_key = f"{VM_ID}:{snapshot_key}"
|
|
components.store.set("vm_snapshot", catalog_key, json.dumps({
|
|
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "失败恢复点",
|
|
"state": "uncertain", "consistency": "crash",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
components.store.set("snapshot_job", job_id, json.dumps({
|
|
"job_id": job_id, "operation_id": operation_id, "owner": "php:41",
|
|
"vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "create",
|
|
"consistency": "crash", "status": "uncertain", "phase": "结果待核查",
|
|
}))
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation", "resource": f"vms/{VM_ID}/snapshots",
|
|
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
|
|
"safe_to_refund": False,
|
|
}))
|
|
services["hk"].snapshot_reconcile_create = {
|
|
"outcome": "failed", "snapshot_key": snapshot_key,
|
|
}
|
|
|
|
operation = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}").json()
|
|
|
|
assert operation["status"] == "failed"
|
|
assert components.store.get("vm_snapshot", catalog_key) is None
|
|
saved_job = json.loads(components.store.get("snapshot_job", job_id))
|
|
assert saved_job["status"] == "failed"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"snapshot_state", ["creating", "restoring", "deleting", "uncertain"])
|
|
def test_transitional_snapshot_catalog_blocks_power_mutation_without_pending_job(
|
|
boundary, snapshot_state):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
snapshot_key = "7" * 32
|
|
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
|
|
"snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "中断恢复点",
|
|
"state": snapshot_state, "consistency": "crash",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
actor = {**USER, "operation_id": "power-blocked-by-snapshot"}
|
|
|
|
response = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions",
|
|
actor, {"action": "poweroff"})
|
|
|
|
assert response.status_code == 409
|
|
assert not any(call_[0] == "action" for call_ in services["hk"].calls)
|
|
|
|
|
|
def test_network_suspend_bypasses_durable_snapshot_uncertain_when_no_backup(
|
|
boundary):
|
|
route, method = "network-suspend", "network_suspend"
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
snapshot_key = "8" * 32
|
|
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
|
|
"snapshot_key": snapshot_key, "vm_id": VM_ID,
|
|
"name": "待核查恢复点", "state": "uncertain",
|
|
"consistency": "crash",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
actor = {**USER, "operation_id": f"expiry-uncertain-{route}"}
|
|
|
|
response = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}",
|
|
actor, {})
|
|
|
|
assert response.status_code == 200, response.text
|
|
assert response.json()["status"] == "succeeded"
|
|
assert len([item for item in services["hk"].calls
|
|
if item[0] == method]) == 1
|
|
|
|
|
|
def test_network_suspend_waits_while_snapshot_backup_is_still_active(
|
|
boundary):
|
|
route, method = "network-suspend", "network_suspend"
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
snapshot_key = "9" * 32
|
|
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
|
|
"snapshot_key": snapshot_key, "vm_id": VM_ID,
|
|
"name": "复制中的恢复点", "state": "uncertain",
|
|
"consistency": "filesystem",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
services["hk"].snapshot_backup_active = True
|
|
actor = {**USER, "operation_id": f"expiry-active-{route}"}
|
|
|
|
response = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}",
|
|
actor, {})
|
|
|
|
assert response.status_code == 409, response.text
|
|
assert response.json()["status"] == "failed"
|
|
assert len([item for item in services["hk"].calls
|
|
if item[0] == method]) == 1
|
|
|
|
|
|
@pytest.mark.parametrize(("action", "method"), [
|
|
("create", "create_live_snapshot"),
|
|
("restore", "restore_live_snapshot"),
|
|
("delete", "delete_snapshot_volume"),
|
|
])
|
|
def test_network_suspend_cancels_same_vm_snapshot_still_queued_behind_host_job(
|
|
boundary, action, method):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
scope = host_scope(profile)
|
|
service = services["hk"]
|
|
queued_vm = "22222222-2222-4222-8222-222222222222"
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
components.state.users.assign_vm(scope, queued_vm, USER["user_id"])
|
|
service.vm.update(
|
|
managed=True, os_source="cloud_template",
|
|
rescue_mode=False, rescue_staged=False,
|
|
)
|
|
queued_vm_record = {
|
|
**service.vm,
|
|
"uuid": queued_vm,
|
|
"name": "queued-vm",
|
|
"network_suspended": False,
|
|
}
|
|
original_get_vm = service.get_vm
|
|
original_create = service.create_live_snapshot
|
|
original_network_suspend = service.network_suspend
|
|
|
|
def get_vm(vm_id):
|
|
if vm_id == queued_vm:
|
|
service.calls.append(("get_vm", vm_id))
|
|
return dict(queued_vm_record)
|
|
return original_get_vm(vm_id)
|
|
|
|
blocker_started = threading.Event()
|
|
blocker_release = threading.Event()
|
|
|
|
def create_live_snapshot(vm_id, snapshot_key, *, consistency,
|
|
progress=None):
|
|
if vm_id == VM_ID:
|
|
blocker_started.set()
|
|
assert blocker_release.wait(timeout=5)
|
|
return original_create(
|
|
vm_id, snapshot_key, consistency=consistency,
|
|
progress=progress)
|
|
|
|
def network_suspend(vm_id):
|
|
if vm_id == queued_vm:
|
|
service.calls.append(("network_suspend", vm_id))
|
|
queued_vm_record["network_suspended"] = True
|
|
return dict(queued_vm_record)
|
|
return original_network_suspend(vm_id)
|
|
|
|
service.get_vm = get_vm
|
|
service.create_live_snapshot = create_live_snapshot
|
|
service.network_suspend = network_suspend
|
|
blocker_actor = {**USER, "operation_id": f"snapshot-blocker-{action}"}
|
|
blocker_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots"
|
|
blocker = call(client, "POST", blocker_path, blocker_actor, {
|
|
"snapshot_key": "a" * 32,
|
|
"name": "占用宿主快照执行器",
|
|
"consistency_mode": "crash",
|
|
})
|
|
assert blocker.status_code == 202, blocker.text
|
|
assert blocker_started.wait(timeout=2)
|
|
|
|
snapshot_key = "b" * 32
|
|
catalog_key = f"{queued_vm}:{snapshot_key}"
|
|
if action != "create":
|
|
components.store.set("vm_snapshot", catalog_key, json.dumps({
|
|
"snapshot_key": snapshot_key,
|
|
"vm_id": queued_vm,
|
|
"name": "待操作恢复点",
|
|
"state": "ready",
|
|
"consistency": "crash",
|
|
"volume_name": snapshot_volume_name(queued_vm, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
snapshot_actor = {
|
|
**USER, "operation_id": f"queued-{action}-cancel",
|
|
}
|
|
if action == "create":
|
|
queued_path = f"/internal/v1/hosts/hk/vms/{queued_vm}/snapshots"
|
|
queued_body = {
|
|
"snapshot_key": snapshot_key,
|
|
"name": "应被到期策略取消",
|
|
"consistency_mode": "crash",
|
|
}
|
|
queued_method = "POST"
|
|
else:
|
|
queued_path = (
|
|
f"/internal/v1/hosts/hk/vms/{queued_vm}/snapshots/"
|
|
f"{snapshot_key}"
|
|
+ ("/restore" if action == "restore" else "")
|
|
)
|
|
queued_body = {}
|
|
queued_method = "POST" if action == "restore" else "DELETE"
|
|
try:
|
|
queued = call(
|
|
client, queued_method, queued_path, snapshot_actor, queued_body)
|
|
assert queued.status_code == 202, queued.text
|
|
|
|
suspend_actor = {
|
|
**USER, "operation_id": f"expiry-network-{action}",
|
|
}
|
|
suspended = call(
|
|
client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{queued_vm}/network-suspend",
|
|
suspend_actor, {},
|
|
)
|
|
|
|
assert suspended.status_code == 200, suspended.text
|
|
assert suspended.json()["status"] == "succeeded"
|
|
snapshot_operation = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{snapshot_actor['operation_id']}",
|
|
snapshot_actor,
|
|
)
|
|
assert snapshot_operation.status_code == 200
|
|
assert snapshot_operation.json()["status"] == "failed"
|
|
assert "未调用虚拟化操作" in snapshot_operation.json()["error"]
|
|
assert not any(
|
|
item[0] == method and len(item) > 1 and item[1] == queued_vm
|
|
for item in service.calls
|
|
)
|
|
raw = components.store.get("vm_snapshot", catalog_key)
|
|
if action == "create":
|
|
assert raw is None
|
|
else:
|
|
assert json.loads(raw)["state"] == "ready"
|
|
finally:
|
|
blocker_release.set()
|
|
|
|
|
|
def test_network_suspend_rejects_snapshot_future_already_running(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True, os_source="cloud_template",
|
|
rescue_mode=False, rescue_staged=False,
|
|
)
|
|
original_create = service.create_live_snapshot
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
|
|
def running_snapshot(vm_id, snapshot_key, *, consistency, progress=None):
|
|
started.set()
|
|
assert release.wait(timeout=5)
|
|
return original_create(
|
|
vm_id, snapshot_key, consistency=consistency,
|
|
progress=progress)
|
|
|
|
service.create_live_snapshot = running_snapshot
|
|
snapshot_actor = {**USER, "operation_id": "running-snapshot-guard"}
|
|
queued = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots",
|
|
snapshot_actor, {
|
|
"snapshot_key": "c" * 32,
|
|
"name": "已开始的快照",
|
|
"consistency_mode": "crash",
|
|
},
|
|
)
|
|
assert queued.status_code == 202
|
|
assert started.wait(timeout=2)
|
|
try:
|
|
before = len([item for item in service.calls
|
|
if item[0] == "network_suspend"])
|
|
suspended = call(
|
|
client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
|
|
{**USER, "operation_id": "running-snapshot-network"}, {},
|
|
)
|
|
assert suspended.status_code == 409
|
|
assert suspended.json()["status"] == "failed"
|
|
assert len([item for item in service.calls
|
|
if item[0] == "network_suspend"]) == before
|
|
finally:
|
|
release.set()
|
|
|
|
|
|
def test_cancelled_snapshot_phase_one_save_failure_converges_on_poll(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
service = services["hk"]
|
|
queued_vm = "33333333-3333-4333-8333-333333333333"
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
components.state.users.assign_vm(scope, queued_vm, USER["user_id"])
|
|
service.vm.update(
|
|
managed=True, os_source="cloud_template",
|
|
rescue_mode=False, rescue_staged=False,
|
|
)
|
|
queued_vm_record = {
|
|
**service.vm, "uuid": queued_vm, "name": "queued-save-failure",
|
|
}
|
|
original_get_vm = service.get_vm
|
|
original_create = service.create_live_snapshot
|
|
original_network_suspend = service.network_suspend
|
|
|
|
def get_vm(vm_id):
|
|
return (dict(queued_vm_record) if vm_id == queued_vm
|
|
else original_get_vm(vm_id))
|
|
|
|
blocker_started = threading.Event()
|
|
blocker_release = threading.Event()
|
|
|
|
def create_live_snapshot(vm_id, snapshot_key, *, consistency,
|
|
progress=None):
|
|
if vm_id == VM_ID:
|
|
blocker_started.set()
|
|
assert blocker_release.wait(timeout=5)
|
|
return original_create(
|
|
vm_id, snapshot_key, consistency=consistency,
|
|
progress=progress)
|
|
|
|
def network_suspend(vm_id):
|
|
if vm_id == queued_vm:
|
|
service.calls.append(("network_suspend", vm_id))
|
|
return {**queued_vm_record, "network_suspended": True}
|
|
return original_network_suspend(vm_id)
|
|
|
|
service.get_vm = get_vm
|
|
service.create_live_snapshot = create_live_snapshot
|
|
service.network_suspend = network_suspend
|
|
blocker = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots",
|
|
{**USER, "operation_id": "phase-one-blocker"}, {
|
|
"snapshot_key": "d" * 32,
|
|
"name": "占用执行器",
|
|
"consistency_mode": "crash",
|
|
},
|
|
)
|
|
assert blocker.status_code == 202
|
|
assert blocker_started.wait(timeout=2)
|
|
snapshot_actor = {**USER, "operation_id": "phase-one-cancelled"}
|
|
snapshot_key = "e" * 32
|
|
queued = call(
|
|
client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{queued_vm}/snapshots",
|
|
snapshot_actor, {
|
|
"snapshot_key": snapshot_key,
|
|
"name": "取消写入故障",
|
|
"consistency_mode": "crash",
|
|
},
|
|
)
|
|
assert queued.status_code == 202
|
|
original_set = components.store.set
|
|
failed_once = False
|
|
|
|
def flaky_set(kind, key, value):
|
|
nonlocal failed_once
|
|
if kind == "snapshot_job" and not failed_once:
|
|
payload = json.loads(value)
|
|
if (payload.get("cancelled_before_start") is True
|
|
and payload.get("status") == "queued"):
|
|
failed_once = True
|
|
raise DatabaseError("phase-one tombstone write failed")
|
|
return original_set(kind, key, value)
|
|
|
|
components.store.set = flaky_set
|
|
try:
|
|
first_suspend = call(
|
|
client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{queued_vm}/network-suspend",
|
|
{**USER, "operation_id": "phase-one-network-first"}, {},
|
|
)
|
|
assert first_suspend.status_code == 503
|
|
assert failed_once is True
|
|
|
|
snapshot_operation = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{snapshot_actor['operation_id']}",
|
|
snapshot_actor,
|
|
)
|
|
assert snapshot_operation.status_code == 200
|
|
assert snapshot_operation.json()["status"] == "failed"
|
|
assert components.store.get(
|
|
"vm_snapshot", f"{queued_vm}:{snapshot_key}") is None
|
|
assert not any(
|
|
item[0] == "create_live_snapshot" and item[1] == queued_vm
|
|
for item in service.calls
|
|
)
|
|
|
|
second_suspend = call(
|
|
client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{queued_vm}/network-suspend",
|
|
{**USER, "operation_id": "phase-one-network-retry"}, {},
|
|
)
|
|
assert second_suspend.status_code == 200, second_suspend.text
|
|
finally:
|
|
components.store.set = original_set
|
|
blocker_release.set()
|
|
|
|
|
|
def test_network_suspend_barrier_rejects_snapshot_admission_until_target_lands(
|
|
boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
service = services["hk"]
|
|
vm_id = "44444444-4444-4444-8444-444444444444"
|
|
components.state.users.assign_vm(scope, vm_id, USER["user_id"])
|
|
vm = {
|
|
**service.vm,
|
|
"uuid": vm_id,
|
|
"name": "barrier-vm",
|
|
"managed": True,
|
|
"os_source": "cloud_template",
|
|
"rescue_mode": False,
|
|
"rescue_staged": False,
|
|
}
|
|
original_get_vm = service.get_vm
|
|
original_network_suspend = service.network_suspend
|
|
|
|
def get_vm(target):
|
|
return dict(vm) if target == vm_id else original_get_vm(target)
|
|
|
|
suspend_started = threading.Event()
|
|
suspend_release = threading.Event()
|
|
|
|
def network_suspend(target):
|
|
if target == vm_id:
|
|
service.calls.append(("network_suspend", target))
|
|
suspend_started.set()
|
|
assert suspend_release.wait(timeout=5)
|
|
return {**vm, "network_suspended": True}
|
|
return original_network_suspend(target)
|
|
|
|
service.get_vm = get_vm
|
|
service.network_suspend = network_suspend
|
|
suspend_actor = {**USER, "operation_id": "barrier-network-suspend"}
|
|
direct = TestClient(
|
|
engine.registry.apps["hk"], base_url="http://127.0.0.1")
|
|
snapshot_key = "f" * 32
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
pending_suspend = executor.submit(
|
|
call, client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{vm_id}/network-suspend",
|
|
suspend_actor, {},
|
|
)
|
|
assert suspend_started.wait(timeout=2)
|
|
token = TRUSTED_ACTOR.set({
|
|
**USER,
|
|
"operation_id": "barrier-snapshot-rejected",
|
|
})
|
|
try:
|
|
rejected = direct.post(
|
|
f"/api/v1/vms/{vm_id}/snapshots",
|
|
json={
|
|
"snapshot_key": snapshot_key,
|
|
"name": "不应越过暂停屏障",
|
|
"consistency_mode": "crash",
|
|
},
|
|
)
|
|
finally:
|
|
TRUSTED_ACTOR.reset(token)
|
|
assert rejected.status_code == 409, rejected.text
|
|
assert components.store.get(
|
|
"vm_snapshot", f"{vm_id}:{snapshot_key}") is None
|
|
assert not any(
|
|
json.loads(raw).get("snapshot_key") == snapshot_key
|
|
for raw in components.store.list("snapshot_job").values()
|
|
)
|
|
suspend_release.set()
|
|
suspended = pending_suspend.result(timeout=5)
|
|
assert suspended.status_code == 200, suspended.text
|
|
finally:
|
|
suspend_release.set()
|
|
direct.close()
|
|
|
|
|
|
@pytest.mark.parametrize("action", ["create", "restore", "delete"])
|
|
def test_initial_snapshot_job_save_failure_leaves_no_network_suspend_ghost(
|
|
boundary, action):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True, os_source="cloud_template",
|
|
rescue_mode=False, rescue_staged=False,
|
|
)
|
|
snapshot_key = {
|
|
"create": "7", "restore": "8", "delete": "9",
|
|
}[action] * 32
|
|
catalog_key = f"{VM_ID}:{snapshot_key}"
|
|
if action != "create":
|
|
components.store.set("vm_snapshot", catalog_key, json.dumps({
|
|
"snapshot_key": snapshot_key,
|
|
"vm_id": VM_ID,
|
|
"name": "初始写入故障恢复点",
|
|
"state": "ready",
|
|
"consistency": "crash",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
original_set = components.store.set
|
|
failed_once = False
|
|
|
|
def flaky_set(kind, key, value):
|
|
nonlocal failed_once
|
|
if kind == "snapshot_job" and not failed_once:
|
|
failed_once = True
|
|
raise DatabaseError("initial snapshot job write failed")
|
|
return original_set(kind, key, value)
|
|
|
|
components.store.set = flaky_set
|
|
actor = {**USER, "operation_id": f"initial-save-failure-{action}"}
|
|
if action == "create":
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots"
|
|
body = {
|
|
"snapshot_key": snapshot_key,
|
|
"name": "初始写入故障",
|
|
"consistency_mode": "crash",
|
|
}
|
|
method = "POST"
|
|
else:
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots/{snapshot_key}"
|
|
if action == "restore":
|
|
path += "/restore"
|
|
method = "POST"
|
|
else:
|
|
method = "DELETE"
|
|
body = {}
|
|
try:
|
|
rejected = call(client, method, path, actor, body)
|
|
assert rejected.status_code == 503
|
|
assert failed_once is True
|
|
finally:
|
|
components.store.set = original_set
|
|
|
|
raw = components.store.get("vm_snapshot", catalog_key)
|
|
if action == "create":
|
|
assert raw is None
|
|
else:
|
|
assert json.loads(raw)["state"] == "ready"
|
|
assert not any(call_[0] in {
|
|
"create_live_snapshot", "restore_live_snapshot",
|
|
"delete_snapshot_volume",
|
|
} for call_ in service.calls)
|
|
|
|
suspended = call(
|
|
client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
|
|
{**USER, "operation_id": f"network-after-save-failure-{action}"}, {},
|
|
)
|
|
assert suspended.status_code == 200, suspended.text
|
|
|
|
|
|
def test_snapshot_executor_submit_failure_is_terminal_and_does_not_block_suspend(
|
|
boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
|
|
class FailingSnapshotExecutor:
|
|
def submit(self, *_args, **_kwargs):
|
|
raise RuntimeError("executor stopped")
|
|
|
|
def shutdown(self, **_kwargs):
|
|
return None
|
|
|
|
real_executor = ThreadPoolExecutor
|
|
|
|
def executor_factory(*args, **kwargs):
|
|
if kwargs.get("thread_name_prefix") == "vm-snapshot":
|
|
return FailingSnapshotExecutor()
|
|
return real_executor(*args, **kwargs)
|
|
|
|
with patch("virsh_panel.app.ThreadPoolExecutor",
|
|
side_effect=executor_factory):
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
services["hk"].vm.update(
|
|
managed=True, os_source="cloud_template",
|
|
rescue_mode=False, rescue_staged=False,
|
|
)
|
|
actor = {**USER, "operation_id": "snapshot-submit-terminal"}
|
|
snapshot_key = "6" * 32
|
|
|
|
rejected = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots",
|
|
actor, {
|
|
"snapshot_key": snapshot_key,
|
|
"name": "执行器停止",
|
|
"consistency_mode": "crash",
|
|
},
|
|
)
|
|
|
|
assert rejected.status_code == 503
|
|
assert components.store.get(
|
|
"vm_snapshot", f"{VM_ID}:{snapshot_key}") is None
|
|
operation = call(
|
|
client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
|
|
actor,
|
|
)
|
|
assert operation.status_code == 200
|
|
assert operation.json()["status"] == "failed"
|
|
assert "未调用虚拟化操作" in operation.json()["error"]
|
|
assert not any(item[0] == "create_live_snapshot"
|
|
for item in services["hk"].calls)
|
|
|
|
suspended = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend",
|
|
{**USER, "operation_id": "network-after-submit-failure"}, {},
|
|
)
|
|
assert suspended.status_code == 200, suspended.text
|
|
|
|
|
|
@pytest.mark.parametrize(("route", "method"), [
|
|
("service-suspend", "service_suspend"),
|
|
("service-resume", "service_resume"),
|
|
("network-resume", "network_resume"),
|
|
])
|
|
def test_non_fail_safe_targets_remain_blocked_by_snapshot_uncertain(
|
|
boundary, route, method):
|
|
client, engine, _store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
snapshot_key = "a" * 32
|
|
components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({
|
|
"snapshot_key": snapshot_key, "vm_id": VM_ID,
|
|
"name": "待核查恢复点", "state": "uncertain",
|
|
"consistency": "crash",
|
|
"volume_name": snapshot_volume_name(VM_ID, snapshot_key),
|
|
"created_at": 1,
|
|
}))
|
|
actor = {**USER, "operation_id": f"resume-blocked-{route}"}
|
|
body = {"start_if_suspended": False} if route == "service-resume" else {}
|
|
|
|
response = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}",
|
|
actor, body)
|
|
|
|
assert response.status_code == 409, response.text
|
|
assert not any(item[0] == method for item in services["hk"].calls)
|
|
|
|
|
|
def test_interrupted_async_job_becomes_terminal_without_replay(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.store.set("reinstall_job", "job-abcd", json.dumps({
|
|
"operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID,
|
|
"status": "failed", "phase": "面板重启,任务已中断"}))
|
|
store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job",
|
|
"host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall",
|
|
"vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False}))
|
|
recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json()
|
|
assert recovered["status"] == "failed"
|
|
assert "中断" in recovered["error"]
|
|
|
|
|
|
def test_reinstall_cleanup_pending_stays_uncertain_without_replay(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.store.set("reinstall_job", "job-cleanup-pending", json.dumps({
|
|
"operation_id": "reinstall-cleanup-pending",
|
|
"owner": "php:41", "vm_id": VM_ID,
|
|
"status": "uncertain", "phase": "重装已完成,但旧系统盘清理待核对",
|
|
"error": "重装已完成,但旧系统盘清理待核对",
|
|
"error_code": "reinstall_cleanup_pending",
|
|
"reinstall_committed": True, "cleanup_pending": True,
|
|
"vm": {"uuid": VM_ID, "name": "cloud-vm", "state": "running"},
|
|
"warning": "旧存储卷 panel-old.qcow2 删除失败,请检查宿主机",
|
|
}))
|
|
store.set("operation", "reinstall-cleanup-pending", json.dumps({
|
|
"operation_id": "reinstall-cleanup-pending", "host_id": "hk",
|
|
"user_id": 41, "kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID,
|
|
"status": "pending", "result": None, "safe_to_refund": False,
|
|
}))
|
|
recovered = call(
|
|
client, "GET", "/internal/v1/operations/reinstall-cleanup-pending",
|
|
USER,
|
|
)
|
|
payload = recovered.json()
|
|
assert recovered.status_code == 200
|
|
assert payload["status"] == "uncertain"
|
|
assert payload["error_code"] == "reinstall_cleanup_pending"
|
|
assert payload["result"]["reinstall_committed"] is True
|
|
assert payload["result"]["cleanup_pending"] is True
|
|
assert payload["result"]["vm"]["state"] == "running"
|
|
|
|
|
|
def test_committed_reinstall_restart_marker_is_not_downgraded(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.store.set("reinstall_job", "job-restart-committed", json.dumps({
|
|
"operation_id": "reinstall-restart-committed",
|
|
"owner": "php:41", "vm_id": VM_ID,
|
|
"status": "failed", "phase": "面板重启,任务已中断",
|
|
"error": "面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交",
|
|
"reinstall_committed": True,
|
|
}))
|
|
store.set("operation", "reinstall-restart-committed", json.dumps({
|
|
"operation_id": "reinstall-restart-committed", "host_id": "hk",
|
|
"user_id": 41, "kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID,
|
|
"status": "pending", "result": None, "safe_to_refund": False,
|
|
}))
|
|
recovered = call(
|
|
client, "GET", "/internal/v1/operations/reinstall-restart-committed",
|
|
USER,
|
|
)
|
|
payload = recovered.json()
|
|
assert recovered.status_code == 200
|
|
assert payload["status"] == "uncertain"
|
|
assert payload["error_code"] == "reinstall_result_uncertain"
|
|
|
|
|
|
def test_committed_running_reinstall_marker_is_uncertain(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.store.set("reinstall_job", "job-running-committed", json.dumps({
|
|
"operation_id": "reinstall-running-committed", "owner": "php:41",
|
|
"vm_id": VM_ID, "status": "running", "phase": "cleaning_up",
|
|
"reinstall_committed": True,
|
|
}))
|
|
store.set("operation", "reinstall-running-committed", json.dumps({
|
|
"operation_id": "reinstall-running-committed", "host_id": "hk",
|
|
"user_id": 41, "kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID,
|
|
"status": "running", "result": None, "safe_to_refund": False,
|
|
}))
|
|
recovered = call(
|
|
client, "GET", "/internal/v1/operations/reinstall-running-committed",
|
|
USER,
|
|
)
|
|
payload = recovered.json()
|
|
assert recovered.status_code == 200
|
|
assert payload["status"] == "uncertain"
|
|
assert payload["error_code"] == "reinstall_result_uncertain"
|
|
|
|
|
|
def test_legacy_reinstall_post_commit_phase_is_uncertain(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.store.set("reinstall_job", "job-legacy-phase", json.dumps({
|
|
"operation_id": "reinstall-legacy-phase", "owner": "php:41",
|
|
"vm_id": VM_ID, "status": "succeeded", "phase": "cleaning_up",
|
|
"generated_password": "should-remain-private",
|
|
}))
|
|
store.set("operation", "reinstall-legacy-phase", json.dumps({
|
|
"operation_id": "reinstall-legacy-phase", "host_id": "hk",
|
|
"user_id": 41, "kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID,
|
|
"status": "pending", "result": None, "safe_to_refund": False,
|
|
}))
|
|
recovered = call(
|
|
client, "GET", "/internal/v1/operations/reinstall-legacy-phase",
|
|
USER,
|
|
)
|
|
payload = recovered.json()
|
|
assert recovered.status_code == 200
|
|
assert payload["status"] == "uncertain"
|
|
assert payload["error_code"] == "reinstall_result_uncertain"
|
|
assert payload["result"]["phase"] == "cleaning_up"
|
|
|
|
|
|
@pytest.mark.parametrize("resource", [
|
|
f"vms/{VM_ID}/actions",
|
|
f"vms/{VM_ID}/reinstall",
|
|
f"vms/{VM_ID}/rescue",
|
|
])
|
|
def test_orphaned_mutation_becomes_terminal_without_replay(boundary, resource):
|
|
client, _, store, *_ = boundary
|
|
operation_id = "orphan-" + resource.rsplit("/", 1)[-1].replace("?", "-")
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id, "host_id": "hk", "user_id": 41,
|
|
"kind": "mutation", "resource": resource, "vm_uuid": VM_ID,
|
|
"status": "running", "result": None, "safe_to_refund": False,
|
|
"created_at": 1,
|
|
}))
|
|
recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}").json()
|
|
assert recovered["status"] == "failed"
|
|
assert "不会自动重放" in recovered["error"]
|
|
|
|
|
|
def test_orphaned_delete_remains_uncertain_without_replay(boundary):
|
|
client, _engine, store, _logs, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
operation_id = "orphan-destructive-delete"
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id,
|
|
"host_id": "hk",
|
|
"user_id": 41,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}",
|
|
"method": "DELETE",
|
|
"vm_uuid": VM_ID,
|
|
"status": "running",
|
|
"result": None,
|
|
"safe_to_refund": False,
|
|
"delete_ip_claim_keys": [],
|
|
"created_at": 1,
|
|
}))
|
|
|
|
before = list(services["hk"].calls)
|
|
first = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
|
|
).json()
|
|
second = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER,
|
|
).json()
|
|
|
|
assert first["status"] == second["status"] == "uncertain"
|
|
assert "原操作" in second["error"]
|
|
assert services["hk"].calls == before
|
|
|
|
|
|
def test_network_change_lost_response_keeps_both_addresses_uncertain(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True,
|
|
os_source="cloud_template",
|
|
public_ip_cidr="8.8.8.4/28",
|
|
)
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
actor = {**USER, "operation_id": "network-change-lost-response"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network"
|
|
|
|
changed = call(client, "POST", path, actor, {
|
|
"public_ip_cidr": "8.8.8.6/28",
|
|
"confirm_restart": True,
|
|
})
|
|
assert changed.status_code == 200, changed.text
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
assert saved["network_source_cidr"] == "8.8.8.4/28"
|
|
assert saved["network_target_cidr"] == "8.8.8.6/28"
|
|
assert saved["network_mutation_started"] is True
|
|
|
|
# Model a lost response after libvirt may already have switched. A single
|
|
# metadata read cannot prove that XML, cloud-init and nwfilter all agree,
|
|
# so automatic reconciliation must retain both address claims.
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
components.store.set("ip_claim", f"{scope}.8.8.8.4", json.dumps({
|
|
"vm_uuid": VM_ID,
|
|
}))
|
|
engine.active_operations.clear()
|
|
reconciled = call(
|
|
client, "GET", f"/internal/v1/operations/{actor['operation_id']}",
|
|
actor,
|
|
)
|
|
assert reconciled.status_code == 200
|
|
assert reconciled.json()["status"] == "uncertain"
|
|
assert "人工核查" in reconciled.json()["error"]
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.4") is not None
|
|
|
|
source_record = {
|
|
**saved,
|
|
"operation_id": "network-change-still-source",
|
|
"network_source_cidr": "8.8.8.6/28",
|
|
"network_target_cidr": "8.8.8.7/28",
|
|
"status": "uncertain",
|
|
}
|
|
store.set("operation", source_record["operation_id"],
|
|
json.dumps(source_record))
|
|
components.store.set("ip_claim", f"{scope}.8.8.8.7", json.dumps({
|
|
"vm_uuid": VM_ID,
|
|
}))
|
|
source = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{source_record['operation_id']}", actor,
|
|
)
|
|
assert source.json()["status"] == "uncertain"
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.7") is not None
|
|
|
|
service.vm["public_ip_cidr"] = "8.8.8.8/28"
|
|
drift_record = {
|
|
**source_record,
|
|
"operation_id": "network-change-third-address",
|
|
"status": "uncertain",
|
|
}
|
|
store.set("operation", drift_record["operation_id"],
|
|
json.dumps(drift_record))
|
|
drift = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{drift_record['operation_id']}", actor,
|
|
)
|
|
assert drift.json()["status"] == "uncertain"
|
|
assert "人工核查" in drift.json()["error"]
|
|
|
|
|
|
def test_network_change_can_move_between_cidrs_on_the_same_selector(boundary):
|
|
client, engine, store, _, services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile["default_network"] = "bridge:br0"
|
|
first_key = "3" * 32
|
|
second_key = "4" * 32
|
|
common = {
|
|
"revision": 1,
|
|
"state": "active",
|
|
"network_selector": "bridge:br0",
|
|
"dns_servers": ["1.1.1.1"],
|
|
"reserved_ips": [],
|
|
"guest_prefix_length": 29,
|
|
}
|
|
assert call(
|
|
client, "PUT",
|
|
f"/internal/v1/hosts/hk/host/network-pools/{first_key}", SUPER,
|
|
{**common, "cidr": "8.8.8.0/29", "gateway": "8.8.8.1"},
|
|
).status_code == 200
|
|
assert call(
|
|
client, "PUT",
|
|
f"/internal/v1/hosts/hk/host/network-pools/{second_key}", SUPER,
|
|
{**common, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1"},
|
|
).status_code == 200
|
|
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True,
|
|
os_source="cloud_template",
|
|
public_ip_cidr="8.8.8.3/29",
|
|
public_pool_key=first_key,
|
|
)
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
actor = {**USER, "operation_id": "network-cross-cidr-0001"}
|
|
changed = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
|
|
actor, {
|
|
"public_pool_key": second_key,
|
|
"public_ip_cidr": "9.9.9.3/29",
|
|
"confirm_restart": True,
|
|
},
|
|
)
|
|
assert changed.status_code == 200, changed.text
|
|
assert changed.json()["public_pool_key"] == second_key
|
|
operation = json.loads(store.get("operation", actor["operation_id"]))
|
|
assert operation["network_source_pool_key"] == first_key
|
|
assert operation["network_target_pool_key"] == second_key
|
|
claim = json.loads(components.store.get(
|
|
"ip_claim", f"{scope}.9.9.9.3"))
|
|
assert claim["pool_key"] == second_key
|
|
|
|
|
|
def test_network_change_upgrades_legacy_firewall_before_switching_ip(boundary):
|
|
client, engine, _store, _, services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True,
|
|
os_source="cloud_template",
|
|
public_ip_cidr="8.8.8.4/28",
|
|
)
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
firewall_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
|
|
body = firewall_body()
|
|
applied = call(
|
|
client, "PUT", firewall_path,
|
|
{**USER, "operation_id": "network-firewall-upgrade-0001"}, body,
|
|
)
|
|
assert applied.status_code == 200, applied.text
|
|
|
|
record_key = f"{scope}.{VM_ID}"
|
|
legacy_digest = build_firewall_filter_xml(
|
|
VM_ID, body["enabled"], body["rules"], compiler_version=1,
|
|
)[3]
|
|
legacy_record = json.loads(
|
|
components.store.get("vm_firewall", record_key))
|
|
legacy_record.pop("filter_version")
|
|
legacy_record["filter_digest"] = legacy_digest
|
|
components.store.set(
|
|
"vm_firewall", record_key,
|
|
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
|
|
)
|
|
service.firewall_filter_digest = legacy_digest
|
|
applies = sum(row[0] == "apply_firewall_policy" for row in service.calls)
|
|
|
|
changed = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
|
|
{**USER, "operation_id": "network-firewall-upgrade-0002"}, {
|
|
"public_ip_cidr": "8.8.8.6/28",
|
|
"confirm_restart": True,
|
|
},
|
|
)
|
|
|
|
assert changed.status_code == 200, changed.text
|
|
assert changed.json()["network_changed"] is True
|
|
assert sum(row[0] == "apply_firewall_policy"
|
|
for row in service.calls) == applies + 1
|
|
call_names = [row[0] for row in service.calls]
|
|
assert call_names.index(
|
|
"apply_firewall_policy", call_names.index("apply_firewall_policy") + 1,
|
|
) < call_names.index("change_cloud_vm_network")
|
|
upgraded = json.loads(components.store.get("vm_firewall", record_key))
|
|
assert upgraded["filter_version"] == FIREWALL_FILTER_COMPILER_VERSION
|
|
assert upgraded["filter_digest"] == build_firewall_filter_xml(
|
|
VM_ID, body["enabled"], body["rules"])[3]
|
|
|
|
|
|
def test_network_change_rejects_drifted_legacy_firewall(boundary):
|
|
client, engine, _store, _, services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True,
|
|
os_source="cloud_template",
|
|
public_ip_cidr="8.8.8.4/28",
|
|
)
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
firewall_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
|
|
body = firewall_body()
|
|
applied = call(
|
|
client, "PUT", firewall_path,
|
|
{**USER, "operation_id": "network-firewall-drift-0001"}, body,
|
|
)
|
|
assert applied.status_code == 200, applied.text
|
|
|
|
record_key = f"{scope}.{VM_ID}"
|
|
legacy_digest = build_firewall_filter_xml(
|
|
VM_ID, body["enabled"], body["rules"], compiler_version=1,
|
|
)[3]
|
|
legacy_record = json.loads(
|
|
components.store.get("vm_firewall", record_key))
|
|
legacy_record.pop("filter_version")
|
|
legacy_record["filter_digest"] = legacy_digest
|
|
components.store.set(
|
|
"vm_firewall", record_key,
|
|
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
|
|
)
|
|
service.firewall_filter_digest = "e" * 64
|
|
applies = sum(row[0] == "apply_firewall_policy" for row in service.calls)
|
|
|
|
changed = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
|
|
{**USER, "operation_id": "network-firewall-drift-0002"}, {
|
|
"public_ip_cidr": "8.8.8.6/28",
|
|
"confirm_restart": True,
|
|
},
|
|
)
|
|
|
|
assert changed.status_code == 409, changed.text
|
|
assert changed.json()["error_code"] == "firewall_drift_detected"
|
|
assert sum(row[0] == "apply_firewall_policy"
|
|
for row in service.calls) == applies
|
|
assert not any(row[0] == "change_cloud_vm_network"
|
|
for row in service.calls)
|
|
|
|
|
|
@pytest.mark.parametrize("tracked", [False, True])
|
|
def test_network_change_rejects_untracked_or_current_firewall_drift(
|
|
boundary, tracked,
|
|
):
|
|
client, engine, _store, _, services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True,
|
|
os_source="cloud_template",
|
|
public_ip_cidr="8.8.8.4/28",
|
|
)
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
if tracked:
|
|
body = firewall_body()
|
|
applied = call(
|
|
client, "PUT", f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall",
|
|
{**USER, "operation_id": "network-current-drift-0001"}, body,
|
|
)
|
|
assert applied.status_code == 200, applied.text
|
|
service.firewall_filter_digest = "d" * 64
|
|
applies = sum(row[0] == "apply_firewall_policy" for row in service.calls)
|
|
|
|
changed = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
|
|
{**USER, "operation_id": "network-current-drift-0002"}, {
|
|
"public_ip_cidr": "8.8.8.6/28",
|
|
"confirm_restart": True,
|
|
},
|
|
)
|
|
|
|
assert changed.status_code == 409, changed.text
|
|
assert changed.json()["error_code"] == "firewall_drift_detected"
|
|
assert sum(row[0] == "apply_firewall_policy"
|
|
for row in service.calls) == applies
|
|
assert not any(row[0] == "change_cloud_vm_network"
|
|
for row in service.calls)
|
|
|
|
|
|
@pytest.mark.parametrize("error_code,status", [
|
|
("firewall_apply_failed", 502),
|
|
("firewall_result_uncertain", 503),
|
|
])
|
|
def test_network_change_firewall_preflight_failure_is_not_ip_uncertain(
|
|
boundary, error_code, status,
|
|
):
|
|
client, engine, store, _, services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True,
|
|
os_source="cloud_template",
|
|
public_ip_cidr="8.8.8.4/28",
|
|
)
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
body = firewall_body()
|
|
applied = call(
|
|
client, "PUT", f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall",
|
|
{**USER, "operation_id": f"network-preflight-{status}-0001"}, body,
|
|
)
|
|
assert applied.status_code == 200, applied.text
|
|
record_key = f"{scope}.{VM_ID}"
|
|
legacy_digest = build_firewall_filter_xml(
|
|
VM_ID, body["enabled"], body["rules"], compiler_version=1,
|
|
)[3]
|
|
legacy_record = json.loads(
|
|
components.store.get("vm_firewall", record_key))
|
|
legacy_record.pop("filter_version")
|
|
legacy_record["filter_digest"] = legacy_digest
|
|
components.store.set(
|
|
"vm_firewall", record_key,
|
|
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
|
|
)
|
|
service.firewall_filter_digest = legacy_digest
|
|
service.firewall_apply_error = VMError(
|
|
"firewall preflight failed", status, error_code=error_code)
|
|
operation_id = f"network-preflight-{status}-0002"
|
|
|
|
changed = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
|
|
{**USER, "operation_id": operation_id}, {
|
|
"public_ip_cidr": "8.8.8.6/28",
|
|
"confirm_restart": True,
|
|
},
|
|
)
|
|
|
|
assert changed.status_code == status, changed.text
|
|
assert changed.json()["status"] == "failed"
|
|
assert changed.json()["error_code"] == error_code
|
|
operation = json.loads(store.get("operation", operation_id))
|
|
assert operation["status"] == "failed"
|
|
assert operation["network_mutation_started"] is False
|
|
polled = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
|
|
assert polled.status_code == 200, polled.text
|
|
assert polled.json()["status"] == "failed"
|
|
assert polled.json()["error_code"] == error_code
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.6") is None
|
|
assert not any(row[0] == "change_cloud_vm_network"
|
|
for row in service.calls)
|
|
|
|
|
|
def test_network_change_capture_failure_is_terminal_before_ip_claim(boundary):
|
|
client, engine, store, _, services = boundary
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
profile = components.manager.profile
|
|
profile.update({
|
|
"host": "8.8.8.3",
|
|
"default_network": "bridge:br0",
|
|
"public_pool_cidr": "8.8.8.0/28",
|
|
"public_gateway": "8.8.8.1",
|
|
"public_dns_servers": ["1.1.1.1"],
|
|
"public_reserved_ips": ["8.8.8.2"],
|
|
"public_vm_prefix": 28,
|
|
})
|
|
service = services["hk"]
|
|
service.vm.update(
|
|
managed=True,
|
|
os_source="cloud_template",
|
|
public_ip_cidr="8.8.8.4/28",
|
|
)
|
|
scope = host_scope(profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
service.get_vm = MagicMock(side_effect=RuntimeError("capture failed"))
|
|
operation_id = "network-capture-failed-0001"
|
|
|
|
changed = call(
|
|
client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network",
|
|
{**USER, "operation_id": operation_id}, {
|
|
"public_ip_cidr": "8.8.8.6/28",
|
|
"confirm_restart": True,
|
|
},
|
|
)
|
|
|
|
assert changed.status_code == 503, changed.text
|
|
operation = json.loads(store.get("operation", operation_id))
|
|
assert operation["status"] == "failed"
|
|
assert operation["network_mutation_started"] is False
|
|
polled = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
|
|
assert polled.status_code == 200, polled.text
|
|
assert polled.json()["status"] == "failed"
|
|
assert "error_code" not in polled.json()
|
|
assert components.store.get("ip_claim", f"{scope}.8.8.8.6") is None
|
|
assert not any(row[0] == "change_cloud_vm_network"
|
|
for row in service.calls)
|
|
|
|
|
|
def test_deleted_vm_retries_only_public_ip_claim_cleanup(boundary):
|
|
client, engine, store, _, _services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
claim_key = f"{scope}.8.8.8.4"
|
|
components.store.set("ip_claim", claim_key, json.dumps({
|
|
"vm_uuid": VM_ID,
|
|
}))
|
|
operation = {
|
|
"operation_id": "delete-claim-cleanup",
|
|
"host_id": "hk",
|
|
"user_id": USER["user_id"],
|
|
"vm_uuid": VM_ID,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}",
|
|
"method": "DELETE",
|
|
"status": "uncertain",
|
|
"result": None,
|
|
"safe_to_refund": False,
|
|
"cleanup_pending": "public_ip_claim",
|
|
"delete_ip_claim_keys": [claim_key],
|
|
"created_at": 1,
|
|
}
|
|
store.set("operation", operation["operation_id"], json.dumps(operation))
|
|
|
|
recovered = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{operation['operation_id']}", USER,
|
|
)
|
|
assert recovered.status_code == 200
|
|
assert recovered.json()["status"] == "succeeded"
|
|
assert recovered.json()["result"]["public_ip_claims_released"] is True
|
|
assert components.store.get("ip_claim", claim_key) is None
|
|
|
|
foreign_key = f"{scope}.8.8.8.5"
|
|
components.store.set("ip_claim", foreign_key, json.dumps({
|
|
"vm_uuid": "22222222-2222-4222-8222-222222222222",
|
|
}))
|
|
foreign = {
|
|
**operation,
|
|
"operation_id": "delete-foreign-claim",
|
|
"status": "uncertain",
|
|
"cleanup_pending": "public_ip_claim",
|
|
"delete_ip_claim_keys": [foreign_key],
|
|
}
|
|
store.set("operation", foreign["operation_id"], json.dumps(foreign))
|
|
blocked = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{foreign['operation_id']}", USER,
|
|
)
|
|
assert blocked.json()["status"] == "uncertain"
|
|
assert components.store.get("ip_claim", foreign_key) is not None
|
|
|
|
|
|
def test_service_resume_replays_exact_start_target_after_lost_response(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
services["hk"].vm.update(state="shutoff", autostart=False)
|
|
actor = {**USER, "operation_id": "resume-start-target"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume"
|
|
|
|
first = call(client, "POST", path, actor, {"start_if_suspended": True})
|
|
assert first.status_code == 200, first.text
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
|
|
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
saved.update(status="uncertain", result=None, error="response lost")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(state="shutoff", autostart=False)
|
|
engine.active_operations.clear()
|
|
replayed = call(client, "GET", "/internal/v1/operations/resume-start-target", actor)
|
|
assert replayed.status_code == 200
|
|
assert replayed.json()["status"] == "succeeded"
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert services["hk"].calls[-1] == ("service_resume", VM_ID, True)
|
|
|
|
|
|
def test_service_suspend_replay_preserves_original_running_state(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
actor = {**USER, "operation_id": "suspend-running-hint"}
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend"
|
|
assert call(client, "POST", path, actor, {}).status_code == 200
|
|
saved = json.loads(store.get("operation", actor["operation_id"]))
|
|
assert saved["service_was_running_hint"] is True
|
|
|
|
# Model a hard crash after libvirt stopped the guest but before the
|
|
# successful result was durable. Reconciliation now observes shutoff, yet
|
|
# must retain the original running state for the billing resume policy.
|
|
saved.update(status="uncertain", result=None, error="engine crashed")
|
|
store.set("operation", actor["operation_id"], json.dumps(saved))
|
|
services["hk"].vm.update(state="shutoff", autostart=False)
|
|
engine.active_operations.clear()
|
|
reconciled = call(client, "GET",
|
|
"/internal/v1/operations/suspend-running-hint", actor)
|
|
assert reconciled.status_code == 200
|
|
assert reconciled.json()["status"] == "succeeded"
|
|
assert reconciled.json()["result"]["service_was_running"] is True
|
|
|
|
|
|
def test_service_suspend_replay_without_running_hint_fails_closed(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
operation_id = "suspend-missing-running-hint"
|
|
store.set("operation", operation_id, json.dumps({
|
|
"operation_id": operation_id,
|
|
"host_id": "hk",
|
|
"user_id": USER["user_id"],
|
|
"vm_uuid": VM_ID,
|
|
"kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/service-suspend",
|
|
"method": "POST",
|
|
"status": "uncertain",
|
|
"actor_can_manage": True,
|
|
"actor_is_admin": False,
|
|
"created_at": 1,
|
|
}))
|
|
before_calls = list(services["hk"].calls)
|
|
|
|
response = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "failed"
|
|
assert "已取消服务暂停" in response.json()["error"]
|
|
assert services["hk"].calls == before_calls
|
|
|
|
|
|
def test_service_suspend_does_not_stop_vm_when_running_hint_cannot_persist(
|
|
boundary, monkeypatch):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
components = engine.registry.components["hk"]
|
|
components.state.users.assign_vm(
|
|
host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
|
operation_id = "suspend-hint-write-failed"
|
|
original_set = store.set
|
|
|
|
def fail_operation_write(kind, key, value):
|
|
if kind == "operation" and key == operation_id:
|
|
raise RuntimeError("database unavailable")
|
|
return original_set(kind, key, value)
|
|
|
|
monkeypatch.setattr(store, "set", fail_operation_write)
|
|
failed = call(
|
|
client, "POST",
|
|
f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend",
|
|
{**USER, "operation_id": operation_id}, {})
|
|
|
|
assert failed.status_code == 503
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert not any(
|
|
item[0] == "service_suspend" for item in services["hk"].calls)
|
|
|
|
monkeypatch.setattr(store, "set", original_set)
|
|
recovered = call(
|
|
client, "GET", f"/internal/v1/operations/{operation_id}", USER)
|
|
assert recovered.status_code == 200
|
|
assert recovered.json()["status"] == "failed"
|
|
assert services["hk"].vm["state"] == "running"
|
|
assert not any(
|
|
item[0] == "service_suspend" for item in services["hk"].calls)
|
|
|
|
def test_metrics_query_uses_log_store_and_owner_permission(boundary):
|
|
client, engine, _, logs, _ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
scope = host_scope(component.manager.profile)
|
|
component.state.users.assign_vm(scope, VM_ID, 41)
|
|
response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d")
|
|
assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20
|
|
|
|
with patch("virsh_panel.app.time.time", return_value=3_000_000):
|
|
traffic = call(
|
|
client, "GET",
|
|
f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily?days=2",
|
|
)
|
|
assert traffic.status_code == 200
|
|
assert traffic.json()["points"][-1] == {
|
|
"date": "1970-02-04", "day_start": 2_937_600,
|
|
"download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120,
|
|
}
|
|
assert logs.daily_traffic_calls == [(scope, VM_ID, {
|
|
"host_id": "hk", "user_id": 41,
|
|
"since": 2_851_200, "until": 3_000_000, "limit": 2,
|
|
})]
|
|
denied = call(
|
|
client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily",
|
|
{**USER, "user_id": 42},
|
|
)
|
|
assert denied.status_code == 404
|
|
assert len(logs.daily_traffic_calls) == 1
|
|
|
|
|
|
def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
|
|
assert issued.status_code == 200, issued.text
|
|
assert issued.json()["websocket_path"] == "/engine-console"
|
|
ticket = issued.json()["ticket"]
|
|
key = hashlib.sha256(ticket.encode()).hexdigest()
|
|
record = json.loads(store.get("console_ticket", key))
|
|
assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session"
|
|
assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket",
|
|
{**USER, "user_id": 42}, {}).status_code == 404
|
|
from starlette.websockets import WebSocketDisconnect
|
|
with pytest.raises(WebSocketDisconnect):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"},
|
|
subprotocols=["cloud-ticket." + ticket]):
|
|
pass
|
|
assert store.get("console_ticket", key) is not None
|
|
|
|
|
|
def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary):
|
|
client, engine, store, *_ = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
checks = 0
|
|
async def active(*args):
|
|
nonlocal checks
|
|
checks += 1
|
|
return checks <= 2 # Ticket issuance + handshake; then PHP revokes.
|
|
engine.introspection = active
|
|
engine.console_check_interval = 0.01
|
|
issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {})
|
|
ticket = issued.json()["ticket"]
|
|
protocol = "cloud-ticket." + ticket
|
|
async def relay(socket, **kwargs):
|
|
await socket.send_bytes(b"vnc")
|
|
await asyncio.Event().wait()
|
|
from starlette.websockets import WebSocketDisconnect
|
|
with patch("virsh_panel.console.relay_console", relay):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console",
|
|
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket:
|
|
assert socket.accepted_subprotocol == protocol
|
|
assert socket.receive_bytes() == b"vnc"
|
|
with pytest.raises(WebSocketDisconnect):
|
|
socket.receive_bytes()
|
|
key = hashlib.sha256(ticket.encode()).hexdigest()
|
|
assert store.get("console_ticket", key) is None
|
|
assert store.get("ticket_used", key) == "1"
|
|
with pytest.raises(WebSocketDisconnect):
|
|
with client.websocket_connect("wss://panel.example.test/engine-console",
|
|
headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]):
|
|
pass
|
|
|
|
|
|
def test_streamed_iso_digest_must_match_before_import_is_queued(boundary):
|
|
client, engine, store, _, services = boundary
|
|
path = "/internal/v1/hosts/hk/rescue-templates?name=rescue"
|
|
signed_body = b"A" * (1024 * 1024)
|
|
wrong_body = b"B" * len(signed_body)
|
|
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
|
|
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest()
|
|
signed["Content-Type"] = "application/octet-stream"
|
|
response = client.post(path, content=wrong_body, headers=signed)
|
|
assert response.status_code == 401, response.text
|
|
assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls)
|
|
assert engine.registry.components["hk"].store.list("rescue_import_job") == {}
|
|
|
|
|
|
def cloud_upload_fixture(size=1024 * 1024 + 257):
|
|
header = bytearray(104)
|
|
header[:4] = b"QFI\xfb"
|
|
header[4:8] = struct.pack(">I", 3)
|
|
header[20:24] = struct.pack(">I", 16)
|
|
header[24:32] = struct.pack(">Q", 20 * 1024**3)
|
|
header[100:104] = struct.pack(">I", 104)
|
|
return bytes(header) + b"\0" * (size - len(header))
|
|
|
|
|
|
def wait_for_cloud_upload_job(client, job_id):
|
|
path = f"/internal/v1/vm-templates/import-jobs/{job_id}"
|
|
for _ in range(200):
|
|
response = call(client, "GET", path, SUPER)
|
|
assert response.status_code == 200, response.text
|
|
job = response.json()
|
|
if job["status"] in {"completed", "failed"}:
|
|
return job
|
|
time.sleep(0.01)
|
|
raise AssertionError("uploaded cloud template import did not finish")
|
|
|
|
|
|
def cloud_engine_scope(method, target, body, actor, *, headers=None):
|
|
path, separator, query = target.partition("?")
|
|
request_headers = signed_headers(SECRET, method, target, body, actor)
|
|
request_headers.update(headers or {})
|
|
return {
|
|
"type": "http",
|
|
"asgi": {"version": "3.0"},
|
|
"http_version": "1.1",
|
|
"method": method,
|
|
"scheme": "http",
|
|
"path": path,
|
|
"raw_path": path.encode("ascii"),
|
|
"query_string": query.encode("ascii") if separator else b"",
|
|
"headers": [
|
|
(name.lower().encode("ascii"), value.encode("latin-1"))
|
|
for name, value in request_headers.items()
|
|
],
|
|
"client": ("testclient", 50000),
|
|
"server": ("127.0.0.1", 80),
|
|
}
|
|
|
|
|
|
async def cloud_engine_request(engine, method, target, body, actor, *,
|
|
headers=None, receive=None):
|
|
scope = cloud_engine_scope(
|
|
method, target, body, actor, headers=headers)
|
|
delivered = False
|
|
|
|
async def receive_body():
|
|
nonlocal delivered
|
|
if delivered:
|
|
return {"type": "http.disconnect"}
|
|
delivered = True
|
|
return {"type": "http.request", "body": body, "more_body": False}
|
|
|
|
messages = []
|
|
|
|
async def send(message):
|
|
messages.append(message)
|
|
|
|
await engine(scope, receive or receive_body, send)
|
|
status = next(message["status"] for message in messages
|
|
if message["type"] == "http.response.start")
|
|
raw = b"".join(message.get("body", b"") for message in messages
|
|
if message["type"] == "http.response.body")
|
|
return status, json.loads(raw)
|
|
|
|
|
|
def test_global_cloud_template_upload_streams_past_json_limit(boundary):
|
|
client, engine, *_ = boundary
|
|
query = urlencode({
|
|
"name": "Debian 12 fixed",
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
path = "/internal/v1/vm-templates/upload?" + query
|
|
payload = cloud_upload_fixture()
|
|
with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"):
|
|
response = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert response.status_code == 202, response.text
|
|
job = wait_for_cloud_upload_job(client, response.json()["job_id"])
|
|
assert job["status"] == "completed", job
|
|
assert job["template"]["name"] == "Debian 12 fixed"
|
|
assert job["template"]["os_family"] == "Debian"
|
|
assert job["template"]["os_version"] == "12"
|
|
assert job["template"]["virtual_size"] == 20 * 1024**3
|
|
with engine.registry.cloud_templates.source(
|
|
job["template"]["id"]) as (_record, source):
|
|
assert source.stat().st_size == len(payload)
|
|
assert hashlib.sha256(source.read_bytes()).digest() == hashlib.sha256(
|
|
payload).digest()
|
|
|
|
|
|
def test_global_cloud_template_upload_digest_must_match_before_queue(boundary):
|
|
client, engine, *_ = boundary
|
|
path = ("/internal/v1/vm-templates/upload?"
|
|
"name=Debian+12&os_family=Debian&os_version=12")
|
|
signed_body = cloud_upload_fixture()
|
|
wrong_body = signed_body[:-1] + b"x"
|
|
signed = signed_headers(SECRET, "POST", path, signed_body, SUPER)
|
|
signed["X-Cloud-Content-SHA256"] = hashlib.sha256(
|
|
signed_body).hexdigest()
|
|
signed["Content-Type"] = "application/octet-stream"
|
|
response = client.post(path, content=wrong_body, headers=signed)
|
|
assert response.status_code == 401, response.text
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
root = engine.registry.cloud_templates.files.directory()
|
|
assert not list(root.glob(".cloud-image-upload-*.qcow2"))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("family", "version"),
|
|
[
|
|
("debian", "12"),
|
|
("Debian", "14"),
|
|
("CentOS", "10"),
|
|
("Ubuntu", "24"),
|
|
],
|
|
)
|
|
def test_global_cloud_template_upload_rejects_noncanonical_metadata(
|
|
boundary, family, version):
|
|
client, engine, *_ = boundary
|
|
path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": "Invalid metadata",
|
|
"os_family": family,
|
|
"os_version": version,
|
|
})
|
|
payload = cloud_upload_fixture()
|
|
response = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert response.status_code == 422, response.text
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
|
|
|
|
def test_global_cloud_template_upload_requires_super_admin_and_octet_stream(
|
|
boundary):
|
|
client, engine, *_ = boundary
|
|
path = ("/internal/v1/vm-templates/upload?"
|
|
"name=Debian+12&os_family=Debian&os_version=12")
|
|
payload = cloud_upload_fixture()
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
forbidden = call(client, "POST", path, ADMIN, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": digest,
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert forbidden.status_code == 403, forbidden.text
|
|
wrong_type = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": digest,
|
|
"Content-Type": "multipart/form-data",
|
|
})
|
|
assert wrong_type.status_code == 415, wrong_type.text
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
|
|
|
|
def test_global_cloud_template_upload_enforces_minimum_size(boundary):
|
|
client, engine, *_ = boundary
|
|
path = ("/internal/v1/vm-templates/upload?"
|
|
"name=Too+Small&os_family=Debian&os_version=12")
|
|
payload = cloud_upload_fixture(104)
|
|
response = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert response.status_code == 422, response.text
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
|
|
|
|
def test_global_cloud_template_upload_timeout_releases_reservation(
|
|
boundary, monkeypatch):
|
|
_client, engine, *_ = boundary
|
|
query = b"name=Timeout&os_family=Debian&os_version=12"
|
|
scope = {
|
|
"query_string": query,
|
|
"headers": [
|
|
(b"content-type", b"application/octet-stream"),
|
|
(b"content-length", str(1024 * 1024).encode()),
|
|
],
|
|
"engine_upload_validation": {"complete": False, "valid": False},
|
|
"engine_upload_digest": "0" * 64,
|
|
}
|
|
|
|
async def stalled_receive():
|
|
await asyncio.sleep(1)
|
|
return {"type": "http.request", "body": b"", "more_body": False}
|
|
|
|
async def send(_message):
|
|
raise AssertionError("timeout must not start a response")
|
|
|
|
monkeypatch.setattr("virsh_panel.internal.CLOUD_UPLOAD_IDLE_SECONDS", 0.01)
|
|
with pytest.raises(TemplateError, match="上传超时") as timeout:
|
|
asyncio.run(engine._upload_cloud_template(scope, stalled_receive, send))
|
|
assert timeout.value.status == 408
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
assert engine.registry.cloud_templates.upload_reservations == {}
|
|
root = engine.registry.cloud_templates.files.directory()
|
|
assert not list(root.glob(".cloud-image-upload-*.qcow2"))
|
|
|
|
|
|
def test_stalled_cloud_upload_does_not_block_other_engine_writes(
|
|
boundary, monkeypatch):
|
|
_client, engine, *_ = boundary
|
|
payload = cloud_upload_fixture()
|
|
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": "Stalled upload",
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
upload_started = asyncio.Event()
|
|
release_upload = asyncio.Event()
|
|
|
|
class FastRemoteHostManager:
|
|
def __init__(self, *_args, **_kwargs):
|
|
pass
|
|
|
|
@staticmethod
|
|
def scan(host, port):
|
|
return {"host": host, "port": port, "reachable": True}
|
|
|
|
@staticmethod
|
|
def close():
|
|
pass
|
|
|
|
monkeypatch.setattr(
|
|
"virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager)
|
|
|
|
async def stalled_receive():
|
|
upload_started.set()
|
|
await release_upload.wait()
|
|
return {"type": "http.disconnect"}
|
|
|
|
async def scenario():
|
|
upload = asyncio.create_task(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
upload_path,
|
|
payload,
|
|
SUPER,
|
|
headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
"Content-Length": str(len(payload)),
|
|
},
|
|
receive=stalled_receive,
|
|
))
|
|
await asyncio.wait_for(upload_started.wait(), timeout=1)
|
|
assert engine.registry.cloud_templates.pending
|
|
|
|
scan_body = json.dumps({
|
|
"host": "host.example.test", "port": 22,
|
|
}, separators=(",", ":")).encode()
|
|
status, result = await asyncio.wait_for(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
"/internal/v1/hosts/scan",
|
|
scan_body,
|
|
SUPER,
|
|
headers={"Content-Type": "application/json"},
|
|
), timeout=0.5)
|
|
assert status == 200
|
|
assert result["host"] == "host.example.test"
|
|
|
|
release_upload.set()
|
|
upload_status, _result = await asyncio.wait_for(upload, timeout=1)
|
|
assert upload_status == 400
|
|
|
|
asyncio.run(scenario())
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
assert engine.registry.cloud_templates.upload_reservations == {}
|
|
|
|
|
|
def test_cloud_upload_fsync_does_not_block_event_loop_or_engine_writes(
|
|
boundary, monkeypatch):
|
|
_client, engine, *_ = boundary
|
|
payload = cloud_upload_fixture()
|
|
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": "Slow fsync upload",
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
fsync_started = threading.Event()
|
|
release_fsync = threading.Event()
|
|
original_fsync = os.fsync
|
|
|
|
def slow_fsync(descriptor):
|
|
fsync_started.set()
|
|
assert release_fsync.wait(timeout=2), "test did not release fsync"
|
|
return original_fsync(descriptor)
|
|
|
|
class FastRemoteHostManager:
|
|
def __init__(self, *_args, **_kwargs):
|
|
pass
|
|
|
|
@staticmethod
|
|
def scan(host, port):
|
|
return {"host": host, "port": port, "reachable": True}
|
|
|
|
@staticmethod
|
|
def close():
|
|
pass
|
|
|
|
monkeypatch.setattr("virsh_panel.internal.os.fsync", slow_fsync)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.cloud_templates._verify_qcow2_with_qemu",
|
|
lambda *_args: None)
|
|
|
|
async def scenario():
|
|
safety_release = threading.Timer(1.5, release_fsync.set)
|
|
safety_release.start()
|
|
started_at = time.monotonic()
|
|
try:
|
|
upload = asyncio.create_task(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
upload_path,
|
|
payload,
|
|
SUPER,
|
|
headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
"Content-Length": str(len(payload)),
|
|
},
|
|
))
|
|
while not fsync_started.is_set():
|
|
await asyncio.sleep(0.001)
|
|
|
|
scan_body = json.dumps({
|
|
"host": "host.example.test", "port": 22,
|
|
}, separators=(",", ":")).encode()
|
|
status, _result = await asyncio.wait_for(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
"/internal/v1/hosts/scan",
|
|
scan_body,
|
|
SUPER,
|
|
headers={"Content-Type": "application/json"},
|
|
), timeout=0.5)
|
|
assert status == 200
|
|
assert time.monotonic() - started_at < 0.5
|
|
assert not release_fsync.is_set()
|
|
release_fsync.set()
|
|
upload_status, _result = await asyncio.wait_for(upload, timeout=2)
|
|
assert upload_status == 202
|
|
finally:
|
|
release_fsync.set()
|
|
safety_release.cancel()
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_cloud_upload_rechecks_update_state_before_queue(
|
|
boundary, monkeypatch):
|
|
_client, engine, *_ = boundary
|
|
updater = FakeUpdater()
|
|
engine.updater = updater
|
|
payload = cloud_upload_fixture()
|
|
upload_path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": "Update race upload",
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
upload_started = asyncio.Event()
|
|
release_upload = asyncio.Event()
|
|
delivered = False
|
|
monkeypatch.setattr(
|
|
"virsh_panel.cloud_templates._verify_qcow2_with_qemu",
|
|
lambda *_args: None)
|
|
|
|
async def delayed_receive():
|
|
nonlocal delivered
|
|
if delivered:
|
|
return {"type": "http.disconnect"}
|
|
delivered = True
|
|
upload_started.set()
|
|
await release_upload.wait()
|
|
return {"type": "http.request", "body": payload, "more_body": False}
|
|
|
|
async def scenario():
|
|
upload = asyncio.create_task(cloud_engine_request(
|
|
engine,
|
|
"POST",
|
|
upload_path,
|
|
payload,
|
|
SUPER,
|
|
headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
"Content-Length": str(len(payload)),
|
|
},
|
|
receive=delayed_receive,
|
|
))
|
|
await asyncio.wait_for(upload_started.wait(), timeout=1)
|
|
assert engine.registry.cloud_templates.pending
|
|
updater.active = {
|
|
"job_id": "update-job-1234567890", "status": "running",
|
|
}
|
|
release_upload.set()
|
|
status, result = await asyncio.wait_for(upload, timeout=2)
|
|
assert status == 409
|
|
assert result["error"] == "面板正在更新,暂时不能执行虚拟化写操作"
|
|
|
|
try:
|
|
asyncio.run(scenario())
|
|
finally:
|
|
updater.active = None
|
|
assert engine.registry.cloud_templates.pending == set()
|
|
assert engine.registry.cloud_templates.upload_reservations == {}
|
|
assert engine.registry.cloud_templates.jobs == {}
|
|
|
|
|
|
def test_upload_same_os_version_keeps_existing_template(boundary):
|
|
client, engine, *_ = boundary
|
|
|
|
def upload(name, payload):
|
|
path = "/internal/v1/vm-templates/upload?" + urlencode({
|
|
"name": name,
|
|
"os_family": "Debian",
|
|
"os_version": "12",
|
|
})
|
|
response = call(client, "POST", path, SUPER, raw=payload, headers={
|
|
"X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(),
|
|
"Content-Type": "application/octet-stream",
|
|
})
|
|
assert response.status_code == 202, response.text
|
|
job = wait_for_cloud_upload_job(client, response.json()["job_id"])
|
|
assert job["status"] == "completed", job
|
|
return job["template"]
|
|
|
|
official_payload = cloud_upload_fixture()[:-1] + b"o"
|
|
fixed_payload = cloud_upload_fixture()[:-1] + b"f"
|
|
with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"):
|
|
official = upload("Debian 12 official", official_payload)
|
|
fixed = upload("Debian 12 fixed", fixed_payload)
|
|
|
|
assert official["id"] != fixed["id"]
|
|
templates = call(
|
|
client, "GET", "/internal/v1/vm-templates", SUPER).json()["templates"]
|
|
assert {row["id"] for row in templates} == {official["id"], fixed["id"]}
|
|
with engine.registry.cloud_templates.source(
|
|
official["id"]) as (_record, source):
|
|
assert hashlib.sha256(source.read_bytes()).hexdigest() == official["sha256"]
|
|
|
|
|
|
def test_firewall_contract_persists_round_trips_and_is_revision_idempotent(
|
|
boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
|
|
|
|
baseline = call(client, "GET", path, USER)
|
|
assert baseline.status_code == 200, baseline.text
|
|
assert baseline.json() == {
|
|
"enabled": False,
|
|
"revision": 0,
|
|
"policy_digest": firewall_policy_digest(False, []),
|
|
"filter_digest": None,
|
|
"rules": [],
|
|
"sync_state": "active",
|
|
}
|
|
|
|
rejected = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-bad-digest"},
|
|
{**firewall_body(), "policy_digest": "0" * 64},
|
|
)
|
|
assert rejected.status_code == 422, rejected.text
|
|
assert rejected.json()["error_code"] == "firewall_rule_invalid"
|
|
assert not any(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls)
|
|
|
|
class ForbiddenUpdateLock:
|
|
async def __aenter__(self):
|
|
raise AssertionError("firewall acquired the engine-wide update lock")
|
|
|
|
async def __aexit__(self, *_args):
|
|
return False
|
|
|
|
original_target_lock = engine._target_lock
|
|
target_entries = []
|
|
|
|
@asynccontextmanager
|
|
async def observed_target_lock(host_id, vm_id):
|
|
target_entries.append((host_id, vm_id))
|
|
async with original_target_lock(host_id, vm_id):
|
|
yield
|
|
|
|
engine._update_lock = lambda: ForbiddenUpdateLock()
|
|
engine._target_lock = observed_target_lock
|
|
|
|
body = firewall_body()
|
|
assert body["policy_digest"] == (
|
|
"46c4d40eef4410edaf998e17981ae91d161348e2fc6572b2bd96c8b5cbf9bfc3"
|
|
)
|
|
applied = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-apply-0001"}, body,
|
|
)
|
|
assert applied.status_code == 200, applied.text
|
|
assert applied.json()["revision"] == 1
|
|
assert applied.json()["policy_digest"] == body["policy_digest"]
|
|
assert target_entries == [("hk", VM_ID)]
|
|
record_key = f"{scope}.{VM_ID}"
|
|
saved = json.loads(components.store.get("vm_firewall", record_key))
|
|
assert saved["revision"] == 1
|
|
assert saved["policy_digest"] == body["policy_digest"]
|
|
|
|
fetched = call(client, "GET", path, USER)
|
|
assert fetched.status_code == 200, fetched.text
|
|
assert fetched.json()["revision"] == 1
|
|
assert fetched.json()["rules"] == FIREWALL_RULES
|
|
|
|
applies = sum(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls)
|
|
repeated = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-apply-0002"}, body,
|
|
)
|
|
assert repeated.status_code == 200, repeated.text
|
|
assert repeated.json()["revision"] == 1
|
|
assert sum(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls) == applies
|
|
|
|
jumped = firewall_body(revision=3)
|
|
advanced = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-apply-0003"}, jumped,
|
|
)
|
|
assert advanced.status_code == 200, advanced.text
|
|
assert advanced.json()["revision"] == 3
|
|
conflicting = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-apply-conflict"},
|
|
firewall_body(revision=3, enabled=False, rules=[]),
|
|
)
|
|
assert conflicting.status_code == 409
|
|
rollback = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-apply-0004"},
|
|
firewall_body(revision=2),
|
|
)
|
|
assert rollback.status_code == 409
|
|
|
|
|
|
def test_firewall_get_detects_drift_and_tampered_local_evidence(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
|
|
body = firewall_body()
|
|
response = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-drift-0001"}, body,
|
|
)
|
|
assert response.status_code == 200, response.text
|
|
|
|
services["hk"].firewall_filter_digest = "f" * 64
|
|
drifted = call(client, "GET", path, USER)
|
|
assert drifted.status_code == 200, drifted.text
|
|
assert drifted.json()["sync_state"] == "drifted"
|
|
assert drifted.json()["revision"] is None
|
|
same_revision = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-drift-0002"}, body,
|
|
)
|
|
assert same_revision.status_code == 409
|
|
assert same_revision.json()["error_code"] == "firewall_drift_detected"
|
|
|
|
key = f"{scope}.{VM_ID}"
|
|
tampered = json.loads(components.store.get("vm_firewall", key))
|
|
tampered["rules"][0]["description"] = "Tampered"
|
|
components.store.set(
|
|
"vm_firewall", key,
|
|
json.dumps(tampered, ensure_ascii=False, separators=(",", ":")),
|
|
)
|
|
corrupt = call(client, "GET", path, USER)
|
|
assert corrupt.status_code == 503
|
|
assert corrupt.json()["error"] == "虚拟机防火墙记录损坏"
|
|
|
|
|
|
def test_firewall_same_revision_upgrades_legacy_filter_once(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
|
|
body = firewall_body()
|
|
applied = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-upgrade-0001"}, body,
|
|
)
|
|
assert applied.status_code == 200, applied.text
|
|
|
|
record_key = f"{scope}.{VM_ID}"
|
|
legacy_digest = build_firewall_filter_xml(
|
|
VM_ID, body["enabled"], body["rules"], compiler_version=1,
|
|
)[3]
|
|
legacy_record = json.loads(
|
|
components.store.get("vm_firewall", record_key))
|
|
legacy_record.pop("filter_version")
|
|
legacy_record["filter_digest"] = legacy_digest
|
|
components.store.set(
|
|
"vm_firewall", record_key,
|
|
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
|
|
)
|
|
services["hk"].firewall_filter_digest = legacy_digest
|
|
applies = sum(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls)
|
|
|
|
upgraded = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-upgrade-0002"}, body,
|
|
)
|
|
|
|
assert upgraded.status_code == 200, upgraded.text
|
|
assert sum(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls) == applies + 1
|
|
upgraded_record = json.loads(
|
|
components.store.get("vm_firewall", record_key))
|
|
current_digest = build_firewall_filter_xml(
|
|
VM_ID, body["enabled"], body["rules"])[3]
|
|
assert upgraded_record["filter_version"] == FIREWALL_FILTER_COMPILER_VERSION
|
|
assert upgraded_record["filter_digest"] == current_digest
|
|
|
|
repeated = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-upgrade-0003"}, body,
|
|
)
|
|
assert repeated.status_code == 200, repeated.text
|
|
assert sum(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls) == applies + 1
|
|
|
|
|
|
def test_firewall_uncertain_operation_reconciles_exact_host_evidence(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
|
|
actor = {**USER, "operation_id": "firewall-lost-0001"}
|
|
body = firewall_body()
|
|
first = call(client, "PUT", path, actor, body)
|
|
assert first.status_code == 200, first.text
|
|
apply_count = sum(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls)
|
|
|
|
operation = json.loads(store.get("operation", actor["operation_id"]))
|
|
operation.update(status="uncertain", result=None)
|
|
operation.pop("finished_at", None)
|
|
store.set("operation", actor["operation_id"], json.dumps(operation))
|
|
record_key = f"{scope}.{VM_ID}"
|
|
components.store.delete("vm_firewall", record_key)
|
|
|
|
class ForbiddenUpdateLock:
|
|
async def __aenter__(self):
|
|
raise AssertionError("firewall reconciliation acquired global lock")
|
|
|
|
async def __aexit__(self, *_args):
|
|
return False
|
|
|
|
engine._update_lock = lambda: ForbiddenUpdateLock()
|
|
recovered = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{actor['operation_id']}", USER,
|
|
)
|
|
assert recovered.status_code == 200, recovered.text
|
|
assert recovered.json()["status"] == "succeeded"
|
|
assert recovered.json()["result"]["revision"] == 1
|
|
assert recovered.json()["result"]["policy_digest"] == body["policy_digest"]
|
|
assert components.store.get("vm_firewall", record_key) is not None
|
|
assert sum(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls) == apply_count
|
|
|
|
old_body = body
|
|
target = firewall_body(revision=2, enabled=False, rules=[])
|
|
old_operation = {
|
|
**operation,
|
|
"operation_id": "firewall-old-0002",
|
|
"status": "uncertain",
|
|
"result": None,
|
|
"firewall_target": target,
|
|
}
|
|
store.set(
|
|
"operation", old_operation["operation_id"], json.dumps(old_operation))
|
|
old_state = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{old_operation['operation_id']}", USER,
|
|
)
|
|
assert old_state.status_code == 200
|
|
assert old_state.json()["status"] == "failed"
|
|
|
|
mixed_operation = {
|
|
**old_operation,
|
|
"operation_id": "firewall-mixed-0003",
|
|
"status": "uncertain",
|
|
}
|
|
services["hk"].firewall_filter_digest = "e" * 64
|
|
store.set(
|
|
"operation", mixed_operation["operation_id"],
|
|
json.dumps(mixed_operation),
|
|
)
|
|
mixed = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{mixed_operation['operation_id']}", USER,
|
|
)
|
|
assert mixed.status_code == 200
|
|
assert mixed.json()["status"] == "uncertain"
|
|
assert mixed.json()["error_code"] == "firewall_result_uncertain"
|
|
|
|
|
|
def test_firewall_uncertain_legacy_operation_survives_compiler_upgrade(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
|
|
actor = {**USER, "operation_id": "firewall-legacy-lost-0001"}
|
|
body = firewall_body()
|
|
applied = call(client, "PUT", path, actor, body)
|
|
assert applied.status_code == 200, applied.text
|
|
|
|
legacy_digest = build_firewall_filter_xml(
|
|
VM_ID, body["enabled"], body["rules"], compiler_version=1,
|
|
)[3]
|
|
record_key = f"{scope}.{VM_ID}"
|
|
legacy_record = json.loads(
|
|
components.store.get("vm_firewall", record_key))
|
|
legacy_record.pop("filter_version")
|
|
legacy_record["filter_digest"] = legacy_digest
|
|
components.store.set(
|
|
"vm_firewall", record_key,
|
|
json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")),
|
|
)
|
|
services["hk"].firewall_filter_digest = legacy_digest
|
|
operation = json.loads(store.get("operation", actor["operation_id"]))
|
|
operation["firewall_target"].pop("filter_version")
|
|
operation.update(status="uncertain", result=None)
|
|
operation.pop("finished_at", None)
|
|
store.set("operation", actor["operation_id"], json.dumps(operation))
|
|
apply_count = sum(
|
|
row[0] == "apply_firewall_policy" for row in services["hk"].calls)
|
|
|
|
recovered = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{actor['operation_id']}", USER,
|
|
)
|
|
|
|
assert recovered.status_code == 200, recovered.text
|
|
assert recovered.json()["status"] == "succeeded"
|
|
assert recovered.json()["result"]["filter_version"] == 1
|
|
recovered_record = json.loads(
|
|
components.store.get("vm_firewall", record_key))
|
|
assert recovered_record["filter_version"] == 1
|
|
assert recovered_record["filter_digest"] == legacy_digest
|
|
assert sum(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls) == apply_count
|
|
|
|
|
|
def test_firewall_old_uncertain_operation_cannot_downgrade_new_record(boundary):
|
|
client, engine, store, _logs, services = boundary
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
|
|
body_v1 = firewall_body(revision=1)
|
|
actor_v1 = {**USER, "operation_id": "firewall-old-lost-0001"}
|
|
first = call(client, "PUT", path, actor_v1, body_v1)
|
|
assert first.status_code == 200, first.text
|
|
old_operation = json.loads(
|
|
store.get("operation", actor_v1["operation_id"]))
|
|
|
|
body_v2 = firewall_body(revision=2)
|
|
second = call(
|
|
client, "PUT", path,
|
|
{**USER, "operation_id": "firewall-newer-0002"}, body_v2,
|
|
)
|
|
assert second.status_code == 200, second.text
|
|
record_key = f"{scope}.{VM_ID}"
|
|
newer_record = components.store.get("vm_firewall", record_key)
|
|
apply_count = sum(
|
|
row[0] == "apply_firewall_policy" for row in services["hk"].calls)
|
|
|
|
old_operation.update(status="uncertain", result=None)
|
|
old_operation.pop("finished_at", None)
|
|
store.set(
|
|
"operation", actor_v1["operation_id"], json.dumps(old_operation))
|
|
reconciled = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{actor_v1['operation_id']}", USER,
|
|
)
|
|
|
|
assert reconciled.status_code == 200, reconciled.text
|
|
assert reconciled.json()["status"] == "uncertain"
|
|
assert reconciled.json()["error_code"] == "firewall_result_uncertain"
|
|
assert components.store.get("vm_firewall", record_key) == newer_record
|
|
assert json.loads(newer_record)["revision"] == 2
|
|
assert sum(row[0] == "apply_firewall_policy"
|
|
for row in services["hk"].calls) == apply_count
|
|
|
|
|
|
def test_deleted_vm_retries_firewall_cleanup_without_replaying_delete(boundary):
|
|
client, engine, _store, _logs, services = boundary
|
|
assert call(
|
|
client, "GET", "/internal/v1/hosts/hk/vms", SUPER,
|
|
).status_code == 200
|
|
components = engine.registry.components["hk"]
|
|
scope = host_scope(components.manager.profile)
|
|
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
|
firewall_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall"
|
|
applied = call(
|
|
client, "PUT", firewall_path,
|
|
{**USER, "operation_id": "firewall-before-delete"},
|
|
firewall_body(),
|
|
)
|
|
assert applied.status_code == 200, applied.text
|
|
record_key = f"{scope}.{VM_ID}"
|
|
assert components.store.get("vm_firewall", record_key) is not None
|
|
|
|
services["hk"].firewall_cleanup_error = VMError(
|
|
"cleanup unavailable", 502, error_code="firewall_cleanup_pending")
|
|
actor = {**USER, "operation_id": "delete-firewall-pending"}
|
|
deleted = call(
|
|
client, "DELETE",
|
|
f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true",
|
|
actor,
|
|
)
|
|
assert deleted.status_code == 503, deleted.text
|
|
assert deleted.json()["error_code"] == "vm_delete_firewall_cleanup_pending"
|
|
assert sum(row[0] == "delete_vm" for row in services["hk"].calls) == 1
|
|
|
|
services["hk"].firewall_cleanup_error = None
|
|
recovered = call(
|
|
client, "GET",
|
|
f"/internal/v1/operations/{actor['operation_id']}", USER,
|
|
)
|
|
assert recovered.status_code == 200, recovered.text
|
|
assert recovered.json()["status"] == "succeeded"
|
|
assert recovered.json()["result"]["firewall_cleaned"] is True
|
|
assert components.store.get("vm_firewall", record_key) is None
|
|
assert sum(row[0] == "delete_vm" for row in services["hk"].calls) == 1
|
|
|
|
|
|
def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables():
|
|
connection = MagicMock()
|
|
cursor = connection.cursor.return_value.__enter__.return_value
|
|
config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test",
|
|
"ssl_ca": "/private/test/ca.pem"}
|
|
with patch("virsh_panel.database.pymysql.connect", return_value=connection):
|
|
main = MainStore(config, b"m" * 32)
|
|
statements = [call.args[0] for call in cursor.execute.call_args_list]
|
|
assert any("engine_records" in statement for statement in statements)
|
|
assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements)
|
|
assert not any("vm_metric_samples" in statement for statement in statements)
|
|
assert main._config["ssl_verify_identity"] is True
|
|
cursor.reset_mock()
|
|
LogStore({**config, "database": "logs"}, b"m" * 32)
|
|
statements = [call.args[0] for call in cursor.execute.call_args_list]
|
|
assert "engine_vm_metric_samples" in statements[0]
|
|
assert "engine_vm_traffic_totals" in statements[1]
|
|
assert "engine_vm_traffic_checkpoints" in statements[2]
|
|
assert all("ALTER TABLE engine_vm_metric_samples" in statement
|
|
for statement in statements[3:])
|
|
assert LogStore.TRAFFIC_TABLE == "engine_vm_traffic_totals"
|
|
assert LogStore.TRAFFIC_CHECKPOINT_TABLE == "engine_vm_traffic_checkpoints"
|