Compare commits
No commits in common. 'dev' and 'main' have entirely different histories.
@ -1,37 +0,0 @@
|
|||||||
# Virt Cloud development rules
|
|
||||||
|
|
||||||
- This repository is the independent Virt Cloud project. Work only inside this
|
|
||||||
checkout unless the user explicitly authorizes changes to another project.
|
|
||||||
- When a task depends on historical decisions, test results, or project
|
|
||||||
context, and the knowledge vault is available, first read
|
|
||||||
`/Users/admin/Documents/www/codex-knowledge/AGENTS.md`, then
|
|
||||||
`Memory/INDEX.md` and `Memory/CURRENT.md`; follow only the linked notes
|
|
||||||
relevant to the task. If the vault is unavailable, say so explicitly and do
|
|
||||||
not guess the old project state.
|
|
||||||
- The Git remote is `https://g.luoli.live/root/virt.git`.
|
|
||||||
- All subsequent changes, commits and pushes for this project must use the
|
|
||||||
`venix` branch. Do not commit or push them to `main` or force-push.
|
|
||||||
- Before editing or committing, check the current branch and working tree.
|
|
||||||
Preserve unrelated user changes. If this checkout is not on `venix`, stop
|
|
||||||
before changing files and resolve the branch safely.
|
|
||||||
- Never commit real configuration, credentials, private keys, database dumps,
|
|
||||||
installed dependencies, runtime logs or caches. Keep source configuration
|
|
||||||
examples and source schema files versioned.
|
|
||||||
- The PHP application lives in `web/`; its eventual HTTP document root is
|
|
||||||
`web/public/`, not the repository root.
|
|
||||||
- Only the venix theme is retained. Ordinary users must not receive host
|
|
||||||
administration or retired node-management routes.
|
|
||||||
- PHP owns accounts, wallet purchases, orders, public-key ownership and public
|
|
||||||
HTTP authorization. The Python engine owns libvirt, host credentials and VM
|
|
||||||
operations. Keep the engine private and authenticate every internal request.
|
|
||||||
- Purchases provision one VM from the plan snapshot. Each purchase or renewal
|
|
||||||
creates an independent prepaid service period with its own traffic quota.
|
|
||||||
An exhausted quota disconnects VM networking until the next paid period;
|
|
||||||
final expiry disconnects networking without powering off the VM. An unpaid
|
|
||||||
renewal invoice enters a three-day grace period and then permanently removes
|
|
||||||
the VM definition and its panel-managed disks. Never replay an ambiguous
|
|
||||||
destructive operation or refund it without confirmed cleanup; reconcile its
|
|
||||||
operation identifier.
|
|
||||||
- Users share the original VM-panel styling; administrator pages retain venix.
|
|
||||||
- See `docs/PRUNING_REPORT.md` for retained/removed feature boundaries and
|
|
||||||
`docs/INTEGRATION_PLAN.md` for the planned Python engine integration.
|
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
# Third-party code
|
||||||
|
|
||||||
|
`static/vendor/novnc/` contains the `core/` and `vendor/` directories from
|
||||||
|
[noVNC v1.7.0](https://github.com/novnc/noVNC/releases/tag/v1.7.0), plus its
|
||||||
|
`LICENSE.txt`. The upstream source archive was downloaded from the noVNC GitHub
|
||||||
|
release tag without modifications. The bundled `vendor/pako/` dependency retains
|
||||||
|
its own license file.
|
||||||
|
|
||||||
@ -1,106 +0,0 @@
|
|||||||
{
|
|
||||||
"schema_version": 1,
|
|
||||||
"policy_version": 1,
|
|
||||||
"scope": "public_ipv4_interface",
|
|
||||||
"applies_to": "new_public_cloud_image_instances_only",
|
|
||||||
"migrate_existing": false,
|
|
||||||
"rules": [
|
|
||||||
{
|
|
||||||
"id": "qotd",
|
|
||||||
"label": "QOTD",
|
|
||||||
"category": "permanent",
|
|
||||||
"protocols": ["tcp", "udp"],
|
|
||||||
"port_from": 17,
|
|
||||||
"port_to": 17,
|
|
||||||
"directions": ["ingress", "egress"],
|
|
||||||
"reason": "降低反射放大与滥用风险。"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "chargen",
|
|
||||||
"label": "Chargen",
|
|
||||||
"category": "permanent",
|
|
||||||
"protocols": ["tcp", "udp"],
|
|
||||||
"port_from": 19,
|
|
||||||
"port_to": 19,
|
|
||||||
"directions": ["ingress", "egress"],
|
|
||||||
"reason": "降低反射放大与滥用风险。"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "ssdp",
|
|
||||||
"label": "SSDP",
|
|
||||||
"category": "permanent",
|
|
||||||
"protocols": ["tcp", "udp"],
|
|
||||||
"port_from": 1900,
|
|
||||||
"port_to": 1900,
|
|
||||||
"directions": ["ingress", "egress"],
|
|
||||||
"reason": "降低公网发现服务与反射放大滥用风险。"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "memcached",
|
|
||||||
"label": "Memcached UDP",
|
|
||||||
"category": "permanent",
|
|
||||||
"protocols": ["udp"],
|
|
||||||
"port_from": 11211,
|
|
||||||
"port_to": 11211,
|
|
||||||
"directions": ["ingress", "egress"],
|
|
||||||
"reason": "降低 Memcached UDP 反射放大风险。"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "brand-rex-53413",
|
|
||||||
"label": "Netis / Netcore 后门端口",
|
|
||||||
"category": "permanent",
|
|
||||||
"protocols": ["udp"],
|
|
||||||
"port_from": 53413,
|
|
||||||
"port_to": 53413,
|
|
||||||
"directions": ["ingress", "egress"],
|
|
||||||
"reason": "阻断已知路由器后门相关公网流量。"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "smtp-25",
|
|
||||||
"label": "SMTP 直投",
|
|
||||||
"category": "default",
|
|
||||||
"protocols": ["tcp"],
|
|
||||||
"port_from": 25,
|
|
||||||
"port_to": 25,
|
|
||||||
"directions": ["egress"],
|
|
||||||
"reason": "降低垃圾邮件滥用风险;例外申请流程尚未实现。"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "netbios",
|
|
||||||
"label": "NetBIOS",
|
|
||||||
"category": "default",
|
|
||||||
"protocols": ["tcp", "udp"],
|
|
||||||
"port_from": 137,
|
|
||||||
"port_to": 139,
|
|
||||||
"directions": ["ingress", "egress"],
|
|
||||||
"reason": "默认阻断 NetBIOS 服务暴露与公网访问。"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "smb",
|
|
||||||
"label": "SMB",
|
|
||||||
"category": "default",
|
|
||||||
"protocols": ["tcp", "udp"],
|
|
||||||
"port_from": 445,
|
|
||||||
"port_to": 445,
|
|
||||||
"directions": ["ingress", "egress"],
|
|
||||||
"reason": "默认阻断 SMB 文件共享服务的公网暴露与访问。"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "kms-1688",
|
|
||||||
"label": "KMS 服务端",
|
|
||||||
"category": "default",
|
|
||||||
"protocols": ["tcp"],
|
|
||||||
"port_from": 1688,
|
|
||||||
"port_to": 1688,
|
|
||||||
"directions": ["ingress"],
|
|
||||||
"reason": "默认阻断 KMS 服务端口直接暴露到公网。"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"explicitly_unrestricted": [
|
|
||||||
{
|
|
||||||
"protocol": "tcp",
|
|
||||||
"port": 587,
|
|
||||||
"reason": "SMTP 提交端口当前不受平台默认策略限制。"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@ -0,0 +1,22 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Virsh Panel one-shot update from the pinned private Gitea repository
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
WorkingDirectory=/home/virsh-panel
|
||||||
|
Environment=PANEL_DATA_DIR=/home/virsh-panel/data
|
||||||
|
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||||
|
ExecStart=/home/virsh-panel/.venv/bin/python -m virsh_panel.update_worker
|
||||||
|
ExecStopPost=/home/virsh-panel/.venv/bin/python -m virsh_panel.update_worker --finalize-incomplete
|
||||||
|
TimeoutStartSec=30min
|
||||||
|
UMask=0077
|
||||||
|
LimitCORE=0
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=false
|
||||||
|
ReadWritePaths=/home/virsh-panel
|
||||||
@ -0,0 +1,26 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Virsh Panel (local HTTP upstream and remote libvirt)
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=exec
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
WorkingDirectory=/home/virsh-panel
|
||||||
|
Environment=PANEL_DATA_DIR=/home/virsh-panel/data
|
||||||
|
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||||
|
ExecStartPre=+/usr/bin/install -d -o root -g root -m 0700 /home/virsh-panel/data
|
||||||
|
ExecStart=/home/virsh-panel/.venv/bin/python -m uvicorn virsh_panel.app:create_app --factory --host 127.0.0.1 --port 8000 --workers 1 --proxy-headers --forwarded-allow-ips 127.0.0.1
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
UMask=0077
|
||||||
|
NoNewPrivileges=true
|
||||||
|
CapabilityBoundingSet=CAP_SETPCAP
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=false
|
||||||
|
ReadWritePaths=-/home/virsh-panel/data
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@ -1,25 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Virt Cloud private libvirt engine
|
|
||||||
After=network-online.target
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=exec
|
|
||||||
User=root
|
|
||||||
Group=root
|
|
||||||
WorkingDirectory=/home/virt-cloud/engine
|
|
||||||
Environment=CLOUD_ENGINE_CONFIG=/home/virt-cloud/runtime/engine.json
|
|
||||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
|
||||||
ExecStart=/home/virt-cloud/engine/.venv/bin/python -m uvicorn virsh_panel.internal:create_engine --factory --host 127.0.0.1 --port 8000 --workers 1 --proxy-headers --forwarded-allow-ips 127.0.0.1 --no-access-log
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=5
|
|
||||||
UMask=0077
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=false
|
|
||||||
ReadWritePaths=/home/virt-cloud/runtime
|
|
||||||
LimitCORE=0
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@ -1,24 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Virt Cloud service-period and traffic lifecycle worker
|
|
||||||
After=network-online.target virt-cloud-engine.service
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=apache
|
|
||||||
Group=apache
|
|
||||||
WorkingDirectory=/home/virt-cloud/web
|
|
||||||
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudLifecycle
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=5
|
|
||||||
UMask=0077
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectHome=false
|
|
||||||
ProtectSystem=strict
|
|
||||||
ReadWritePaths=-/home/virt-cloud/runtime/php
|
|
||||||
ReadWritePaths=-/home/virt-cloud/web/storage
|
|
||||||
TimeoutStopSec=30
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@ -1,24 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Virt Cloud enforcement policy worker %i
|
|
||||||
After=network-online.target virt-cloud-engine.service
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=apache
|
|
||||||
Group=apache
|
|
||||||
WorkingDirectory=/home/virt-cloud/web
|
|
||||||
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudWorker --policy-only
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=3
|
|
||||||
UMask=0077
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectHome=false
|
|
||||||
ProtectSystem=strict
|
|
||||||
ReadWritePaths=-/home/virt-cloud/runtime/php
|
|
||||||
ReadWritePaths=-/home/virt-cloud/web/storage
|
|
||||||
TimeoutStopSec=30
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@ -1,23 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Virt Cloud privileged online update worker
|
|
||||||
After=network-online.target mariadb.service
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=oneshot
|
|
||||||
User=root
|
|
||||||
Group=root
|
|
||||||
WorkingDirectory=/home/virt-cloud/engine
|
|
||||||
Environment=CLOUD_ENGINE_CONFIG=/home/virt-cloud/runtime/engine.json
|
|
||||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
|
||||||
ExecStart=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker
|
|
||||||
ExecStopPost=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker --finalize-incomplete
|
|
||||||
TimeoutStartSec=45min
|
|
||||||
TimeoutStopSec=10min
|
|
||||||
UMask=0077
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=false
|
|
||||||
ReadWritePaths=/home/virt-cloud
|
|
||||||
LimitCORE=0
|
|
||||||
@ -1,25 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Virt Cloud purchased VM provisioning worker
|
|
||||||
After=network-online.target virt-cloud-engine.service
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
# AlmaLinux PHP-FPM normally runs as apache; use the same user as your PHP pool.
|
|
||||||
User=apache
|
|
||||||
Group=apache
|
|
||||||
WorkingDirectory=/home/virt-cloud/web
|
|
||||||
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudWorker
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=5
|
|
||||||
UMask=0077
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectHome=false
|
|
||||||
ProtectSystem=strict
|
|
||||||
ReadWritePaths=-/home/virt-cloud/runtime/php
|
|
||||||
ReadWritePaths=-/home/virt-cloud/web/storage
|
|
||||||
TimeoutStopSec=30
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@ -1,5 +0,0 @@
|
|||||||
.venv/
|
|
||||||
data/
|
|
||||||
__pycache__/
|
|
||||||
*.py[cod]
|
|
||||||
.pytest_cache/
|
|
||||||
@ -1,496 +0,0 @@
|
|||||||
"""Cloud-init public networking and NoCloud seed behavior."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import importlib.util
|
|
||||||
import json
|
|
||||||
import uuid
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from virsh_panel.cloud_init import (
|
|
||||||
ROOT_PASSWORD_ALPHABET, ROOT_PASSWORD_DIGITS, ROOT_PASSWORD_LENGTH,
|
|
||||||
ROOT_PASSWORD_LOWERCASE, ROOT_PASSWORD_SYMBOLS, ROOT_PASSWORD_UPPERCASE,
|
|
||||||
_build_seed_iso, _enable_network_updates, build_seed_iso,
|
|
||||||
generate_root_password,
|
|
||||||
hash_root_password,
|
|
||||||
read_seed_network, replace_seed_network, replace_seed_settings,
|
|
||||||
validate_cloud_network, validate_guest_hostname,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
PUBLIC_KEY = (
|
|
||||||
"ssh-ed25519 "
|
|
||||||
"AAAAC3NzaC1lZDI1NTE5AAAAIF7mFh4aYz3lRzNqjoFQasAQLpm7BdSULhe1iWuvj3y5 "
|
|
||||||
"panel-test"
|
|
||||||
)
|
|
||||||
GUEST_PASSWORD = "A-strong-guest-password-2026"
|
|
||||||
|
|
||||||
|
|
||||||
def _user_data(seed: bytes) -> dict:
|
|
||||||
"""Read the single-line JSON cloud-config embedded in the generated ISO."""
|
|
||||||
prefix = b"#cloud-config\n"
|
|
||||||
start = seed.index(prefix) + len(prefix)
|
|
||||||
return json.loads(seed[start:seed.index(b"\n", start)])
|
|
||||||
|
|
||||||
|
|
||||||
def _meta_data(seed: bytes) -> dict:
|
|
||||||
"""Read the single-line JSON meta-data embedded in the generated ISO."""
|
|
||||||
prefix = b'{"instance-id":'
|
|
||||||
start = seed.index(prefix)
|
|
||||||
return json.loads(seed[start:seed.index(b"\n", start)])
|
|
||||||
|
|
||||||
|
|
||||||
def _network_data(seed: bytes) -> dict:
|
|
||||||
"""Read the single-line JSON network-config embedded in the ISO."""
|
|
||||||
prefix = b'{"version": 2, "ethernets":'
|
|
||||||
start = seed.index(prefix)
|
|
||||||
return json.loads(seed[start:seed.index(b"\n", start)])
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("address", "gateway"),
|
|
||||||
[
|
|
||||||
("8.8.8.18/24", "8.8.8.1"),
|
|
||||||
("8.8.8.18/28", "8.8.8.17"),
|
|
||||||
("123.123.123.114/24", "123.123.123.113"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_public_network_accepts_24_and_28(address, gateway):
|
|
||||||
validated = validate_cloud_network(
|
|
||||||
address, gateway, ["1.1.1.1", "8.8.8.8"], PUBLIC_KEY,
|
|
||||||
)
|
|
||||||
assert validated == {
|
|
||||||
"public_ip_cidr": address,
|
|
||||||
"gateway": gateway,
|
|
||||||
"dns_servers": ["1.1.1.1", "8.8.8.8"],
|
|
||||||
"guest_ssh_public_key": PUBLIC_KEY,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_cloud_login_accepts_key_and_root_password_together():
|
|
||||||
network = ("8.8.8.18/24", "8.8.8.1", ["1.1.1.1"])
|
|
||||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
|
||||||
assert password_hash.startswith("$6$")
|
|
||||||
assert GUEST_PASSWORD not in password_hash
|
|
||||||
assert hash_root_password(GUEST_PASSWORD) != password_hash
|
|
||||||
|
|
||||||
validated = validate_cloud_network(
|
|
||||||
*network, "", root_password_hash=password_hash,
|
|
||||||
)
|
|
||||||
assert validated["root_password_hash"] == password_hash
|
|
||||||
assert not validated["guest_ssh_public_key"]
|
|
||||||
|
|
||||||
together = validate_cloud_network(
|
|
||||||
*network, PUBLIC_KEY, root_password_hash=password_hash,
|
|
||||||
)
|
|
||||||
assert together["guest_ssh_public_key"] == PUBLIC_KEY
|
|
||||||
assert together["root_password_hash"] == password_hash
|
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
validate_cloud_network(*network, "", root_password_hash=None)
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
validate_cloud_network(*network, "", root_password_hash=GUEST_PASSWORD)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("invalid", [
|
|
||||||
"Short1!", "alllowercase12!", "ALLUPPERCASE12!", "NoSymbolPassword12",
|
|
||||||
"NoDigitsLong!Aa", "A-lowercase password12!", "A-lowercase\npassword12!",
|
|
||||||
"A-lowercase-password-1" + "x" * 110, "ValidAscii1!含中文",
|
|
||||||
])
|
|
||||||
def test_manual_root_password_requires_ascii_and_all_character_classes(invalid):
|
|
||||||
with pytest.raises(ValueError, match="初始 root 密码"):
|
|
||||||
hash_root_password(invalid)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("value", [
|
|
||||||
"node-1", "node.example.com", "A-Node.Example",
|
|
||||||
])
|
|
||||||
def test_guest_hostname_is_canonical_and_bounded(value):
|
|
||||||
assert validate_guest_hostname(value) == value.lower()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("value", [
|
|
||||||
"", ".node", "node.", "node..example", "-node", "node-",
|
|
||||||
"node_name", "node with spaces", "节点", "a" * 64,
|
|
||||||
])
|
|
||||||
def test_guest_hostname_rejects_unsafe_values(value):
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
validate_guest_hostname(value)
|
|
||||||
|
|
||||||
|
|
||||||
def test_generated_root_password_always_meets_policy():
|
|
||||||
assert ROOT_PASSWORD_ALPHABET == (
|
|
||||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
|
||||||
"0123456789!@#$%*-_=+?"
|
|
||||||
)
|
|
||||||
samples = set()
|
|
||||||
for _ in range(50):
|
|
||||||
password = generate_root_password()
|
|
||||||
samples.add(password)
|
|
||||||
assert len(password) == ROOT_PASSWORD_LENGTH
|
|
||||||
assert set(password) <= set(ROOT_PASSWORD_ALPHABET)
|
|
||||||
assert any(char in ROOT_PASSWORD_UPPERCASE for char in password)
|
|
||||||
assert any(char in ROOT_PASSWORD_LOWERCASE for char in password)
|
|
||||||
assert any(char in ROOT_PASSWORD_DIGITS for char in password)
|
|
||||||
assert any(char in ROOT_PASSWORD_SYMBOLS for char in password)
|
|
||||||
assert len(samples) == 50
|
|
||||||
assert len({password[:4] for password in samples}) > 1
|
|
||||||
assert hash_root_password(password).startswith("$6$")
|
|
||||||
|
|
||||||
|
|
||||||
def test_generated_root_password_rejects_whole_noncompliant_candidate(monkeypatch):
|
|
||||||
# First candidate is 32 uppercase A characters and must be discarded.
|
|
||||||
# The second one is a known compliant candidate with no post-processing.
|
|
||||||
indexes = iter([0] * 32 + [0, 26, 52, 62] * 8)
|
|
||||||
bounds = []
|
|
||||||
|
|
||||||
def deterministic_randbelow(bound):
|
|
||||||
bounds.append(bound)
|
|
||||||
return next(indexes)
|
|
||||||
|
|
||||||
monkeypatch.setattr("virsh_panel.cloud_init.secrets.randbelow",
|
|
||||||
deterministic_randbelow)
|
|
||||||
assert generate_root_password() == "Aa0!" * 8
|
|
||||||
assert bounds == [len(ROOT_PASSWORD_ALPHABET)] * 64
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("address", "gateway"),
|
|
||||||
[
|
|
||||||
("192.168.1.18/24", "192.168.1.1"),
|
|
||||||
("10.0.0.18/24", "10.0.0.1"),
|
|
||||||
("172.16.0.18/24", "172.16.0.1"),
|
|
||||||
("127.0.0.18/24", "127.0.0.1"),
|
|
||||||
("169.254.0.18/24", "169.254.0.1"),
|
|
||||||
("224.0.0.18/24", "224.0.0.1"),
|
|
||||||
("2001:4860:4860::8888/64", "2001:4860:4860::1"),
|
|
||||||
("8.8.8.0/24", "8.8.8.1"),
|
|
||||||
("8.8.8.255/24", "8.8.8.1"),
|
|
||||||
("8.8.8.18/28", "8.8.8.1"),
|
|
||||||
("8.8.8.18", "8.8.8.1"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_public_network_rejects_nonpublic_or_unroutable_pairs(address, gateway):
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
validate_cloud_network(address, gateway, ["1.1.1.1"], PUBLIC_KEY)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("dns", "key"),
|
|
||||||
[
|
|
||||||
(["not-an-ip"], PUBLIC_KEY),
|
|
||||||
(["192.168.1.1"], PUBLIC_KEY),
|
|
||||||
([], PUBLIC_KEY),
|
|
||||||
(["1.1.1.1"], "not-an-ssh-key"),
|
|
||||||
(["1.1.1.1"], PUBLIC_KEY + "\nruncmd: [evil]"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_public_network_rejects_bad_dns_and_ssh_key(dns, key):
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
validate_cloud_network("8.8.8.18/24", "8.8.8.1", dns, key)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
|
||||||
reason="pycdlib is not installed")
|
|
||||||
def test_seed_iso_has_nocloud_files_and_mac_specific_static_network():
|
|
||||||
vm_uuid = str(uuid.uuid4())
|
|
||||||
mac = "52:54:00:12:34:56"
|
|
||||||
validated = validate_cloud_network(
|
|
||||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
|
||||||
)
|
|
||||||
seed = build_seed_iso(vm_uuid, "cloud-vm", mac, validated)
|
|
||||||
|
|
||||||
assert isinstance(seed, bytes)
|
|
||||||
assert seed[32769:32774] == b"CD001" # ISO-9660 primary volume descriptor
|
|
||||||
assert b"CIDATA" in seed
|
|
||||||
for marker in (
|
|
||||||
b"user-data", b"meta-data", b"network-config", b"8.8.8.18/28",
|
|
||||||
b"8.8.8.17", b"1.1.1.1", mac.encode(), PUBLIC_KEY.encode(),
|
|
||||||
):
|
|
||||||
assert marker in seed
|
|
||||||
assert b'"routes": [{"to": "0.0.0.0/0", "via": "8.8.8.17"}]' in seed
|
|
||||||
assert b'"to": "default"' not in seed
|
|
||||||
assert b'"renderer": "networkd"' not in seed
|
|
||||||
assert b'"set-name"' not in seed
|
|
||||||
assert b"IPv6AcceptRA=no" not in seed
|
|
||||||
assert b"systemd-machine-id-setup" not in seed
|
|
||||||
user_data = _user_data(seed)
|
|
||||||
metadata = _meta_data(seed)
|
|
||||||
assert user_data["disable_root"] is False
|
|
||||||
assert user_data["ssh_pwauth"] is False
|
|
||||||
assert user_data["updates"] == {"network": {"when": ["boot"]}}
|
|
||||||
assert metadata["virt-cloud-network-policy"] == "boot-v1"
|
|
||||||
assert user_data["users"][0]["name"] == "root"
|
|
||||||
assert user_data["users"][0]["ssh_authorized_keys"] == [PUBLIC_KEY]
|
|
||||||
assert "hashed_passwd" not in user_data["users"][0]
|
|
||||||
assert "passwd" not in user_data["users"][0]
|
|
||||||
sshd_dropin = user_data["write_files"][0]
|
|
||||||
assert sshd_dropin["path"] == "/etc/ssh/sshd_config.d/00-virsh-panel-root.conf"
|
|
||||||
assert "PermitRootLogin prohibit-password" in sshd_dropin["content"]
|
|
||||||
assert "PasswordAuthentication no" in sshd_dropin["content"]
|
|
||||||
assert "runcmd" not in user_data
|
|
||||||
assert "bootcmd" not in user_data
|
|
||||||
assert "dsmode" not in metadata
|
|
||||||
assert all(
|
|
||||||
item["path"] != "/etc/systemd/network/05-virt-cloud-public.network"
|
|
||||||
for item in user_data["write_files"]
|
|
||||||
)
|
|
||||||
assert b"almalinux" not in seed
|
|
||||||
assert read_seed_network(seed) == {
|
|
||||||
"public_ip_cidr": "8.8.8.18/28",
|
|
||||||
"gateway": "8.8.8.17",
|
|
||||||
"dns_servers": ["1.1.1.1"],
|
|
||||||
"mac": mac,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_enable_network_updates_upgrades_legacy_panel_user_data():
|
|
||||||
legacy = b'#cloud-config\n{"users": [{"name": "root"}]}\n'
|
|
||||||
|
|
||||||
upgraded = _enable_network_updates(legacy)
|
|
||||||
|
|
||||||
assert upgraded.startswith(b"#cloud-config\n")
|
|
||||||
payload = json.loads(upgraded.split(b"\n", 1)[1])
|
|
||||||
assert payload["users"] == [{"name": "root"}]
|
|
||||||
assert payload["updates"] == {"network": {"when": ["boot"]}}
|
|
||||||
assert _enable_network_updates(upgraded) == upgraded
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
|
||||||
reason="pycdlib is not installed")
|
|
||||||
def test_debian12_network_fix_uses_standard_nocloud_network_config():
|
|
||||||
mac = "52:54:00:12:34:56"
|
|
||||||
validated = validate_cloud_network(
|
|
||||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1", "8.8.8.8"], PUBLIC_KEY,
|
|
||||||
)
|
|
||||||
seed = build_seed_iso(
|
|
||||||
str(uuid.uuid4()), "debian-12-vm", mac, validated,
|
|
||||||
debian12_network_fix=True,
|
|
||||||
)
|
|
||||||
user_data = _user_data(seed)
|
|
||||||
metadata = _meta_data(seed)
|
|
||||||
network_data = _network_data(seed)
|
|
||||||
|
|
||||||
assert network_data == {
|
|
||||||
"version": 2,
|
|
||||||
"ethernets": {
|
|
||||||
"public0": {
|
|
||||||
"match": {"macaddress": mac},
|
|
||||||
"dhcp4": False,
|
|
||||||
"dhcp6": False,
|
|
||||||
"addresses": ["8.8.8.18/28"],
|
|
||||||
"routes": [{"to": "0.0.0.0/0", "via": "8.8.8.17"}],
|
|
||||||
"nameservers": {"addresses": ["1.1.1.1", "8.8.8.8"]},
|
|
||||||
"accept-ra": False,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"renderer": "networkd",
|
|
||||||
}
|
|
||||||
assert metadata["dsmode"] == "local"
|
|
||||||
bootcmd = user_data["bootcmd"]
|
|
||||||
assert bootcmd[0][:2] == ["/bin/sh", "-ec"]
|
|
||||||
queue_networkd = bootcmd[0][2]
|
|
||||||
assert queue_networkd.splitlines() == [
|
|
||||||
"systemd-machine-id-setup",
|
|
||||||
"netplan generate",
|
|
||||||
"systemctl unmask systemd-networkd.service",
|
|
||||||
"systemctl enable systemd-networkd.service",
|
|
||||||
"systemctl start --no-block systemd-networkd.service",
|
|
||||||
]
|
|
||||||
assert "ip address" not in queue_networkd
|
|
||||||
assert "ip route" not in queue_networkd
|
|
||||||
assert "restart" not in json.dumps(bootcmd)
|
|
||||||
assert "--now" not in json.dumps(bootcmd)
|
|
||||||
assert "runcmd" not in user_data
|
|
||||||
assert b'"set-name"' not in seed
|
|
||||||
assert b"enp1s0" not in seed
|
|
||||||
assert all(
|
|
||||||
not item["path"].startswith("/etc/systemd/network/")
|
|
||||||
for item in user_data["write_files"]
|
|
||||||
)
|
|
||||||
assert read_seed_network(seed) == {
|
|
||||||
"public_ip_cidr": "8.8.8.18/28",
|
|
||||||
"gateway": "8.8.8.17",
|
|
||||||
"dns_servers": ["1.1.1.1", "8.8.8.8"],
|
|
||||||
"mac": mac,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("invalid", [None, 0, 1, "networkd"])
|
|
||||||
def test_seed_rejects_invalid_debian12_network_fix_option(invalid):
|
|
||||||
validated = validate_cloud_network(
|
|
||||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
|
||||||
)
|
|
||||||
with pytest.raises(ValueError, match="Debian 12 网络修复选项"):
|
|
||||||
build_seed_iso(
|
|
||||||
str(uuid.uuid4()), "cloud-vm", "52:54:00:12:34:56", validated,
|
|
||||||
debian12_network_fix=invalid,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
|
||||||
reason="pycdlib is not installed")
|
|
||||||
def test_read_seed_network_rejects_invalid_iso_without_exposing_credentials():
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
read_seed_network(b"not an ISO")
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
read_seed_network(b"x" * (1024 * 1024 + 1))
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
|
||||||
reason="pycdlib is not installed")
|
|
||||||
def test_password_seed_sets_root_hash_and_enables_ssh_password_login():
|
|
||||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
|
||||||
validated = validate_cloud_network(
|
|
||||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], "",
|
|
||||||
root_password_hash=password_hash,
|
|
||||||
)
|
|
||||||
seed = build_seed_iso(
|
|
||||||
str(uuid.uuid4()), "password-vm", "52:54:00:12:34:56", validated,
|
|
||||||
)
|
|
||||||
user_data = _user_data(seed)
|
|
||||||
|
|
||||||
assert user_data["disable_root"] is False
|
|
||||||
assert user_data["ssh_pwauth"] is True
|
|
||||||
assert user_data["users"][0]["name"] == "root"
|
|
||||||
assert user_data["users"][0]["lock_passwd"] is False
|
|
||||||
assert user_data["users"][0]["hashed_passwd"] == password_hash
|
|
||||||
assert "ssh_authorized_keys" not in user_data["users"][0]
|
|
||||||
assert user_data["chpasswd"] == {
|
|
||||||
"expire": False,
|
|
||||||
"users": [{"name": "root", "password": password_hash, "type": "hash"}],
|
|
||||||
}
|
|
||||||
sshd_dropin = user_data["write_files"][0]
|
|
||||||
assert sshd_dropin["path"] == "/etc/ssh/sshd_config.d/00-virsh-panel-root.conf"
|
|
||||||
assert "PermitRootLogin yes" in sshd_dropin["content"]
|
|
||||||
assert "PasswordAuthentication yes" in sshd_dropin["content"]
|
|
||||||
assert GUEST_PASSWORD.encode() not in seed
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
|
||||||
reason="pycdlib is not installed")
|
|
||||||
def test_seed_allows_root_password_and_ssh_key_together():
|
|
||||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
|
||||||
validated = validate_cloud_network(
|
|
||||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
|
||||||
root_password_hash=password_hash,
|
|
||||||
)
|
|
||||||
seed = build_seed_iso(
|
|
||||||
str(uuid.uuid4()), "both-vm", "52:54:00:12:34:56", validated,
|
|
||||||
)
|
|
||||||
user_data = _user_data(seed)
|
|
||||||
root = user_data["users"][0]
|
|
||||||
assert root["name"] == "root"
|
|
||||||
assert root["lock_passwd"] is False
|
|
||||||
assert root["ssh_authorized_keys"] == [PUBLIC_KEY]
|
|
||||||
assert root["hashed_passwd"] == password_hash
|
|
||||||
assert user_data["ssh_pwauth"] is True
|
|
||||||
assert user_data["chpasswd"]["users"][0]["password"] == password_hash
|
|
||||||
assert "PermitRootLogin yes" in user_data["write_files"][0]["content"]
|
|
||||||
assert "PasswordAuthentication yes" in user_data["write_files"][0]["content"]
|
|
||||||
assert GUEST_PASSWORD.encode() not in seed
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
|
||||||
reason="pycdlib is not installed")
|
|
||||||
def test_replace_seed_network_preserves_user_data_and_metadata():
|
|
||||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
|
||||||
validated = validate_cloud_network(
|
|
||||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
|
||||||
root_password_hash=password_hash,
|
|
||||||
)
|
|
||||||
seed = build_seed_iso(
|
|
||||||
str(uuid.uuid4()), "network-change-vm", "52:54:00:12:34:56", validated,
|
|
||||||
)
|
|
||||||
import pycdlib
|
|
||||||
from io import BytesIO
|
|
||||||
|
|
||||||
def file_bytes(image: bytes, path: str) -> bytes:
|
|
||||||
iso = pycdlib.PyCdlib()
|
|
||||||
iso.open_fp(BytesIO(image))
|
|
||||||
output = BytesIO()
|
|
||||||
iso.get_file_from_iso_fp(output, rr_path=path)
|
|
||||||
iso.close()
|
|
||||||
return output.getvalue()
|
|
||||||
|
|
||||||
user_data = file_bytes(seed, "/user-data")
|
|
||||||
metadata = file_bytes(seed, "/meta-data")
|
|
||||||
original_metadata = json.loads(metadata)
|
|
||||||
changed = replace_seed_network(
|
|
||||||
seed, "8.8.8.19/28", "8.8.8.17", ["8.8.8.8"],
|
|
||||||
)
|
|
||||||
assert file_bytes(changed, "/user-data") == user_data
|
|
||||||
changed_metadata = json.loads(file_bytes(changed, "/meta-data"))
|
|
||||||
assert changed_metadata["instance-id"] == original_metadata["instance-id"]
|
|
||||||
assert changed_metadata["local-hostname"] == original_metadata["local-hostname"]
|
|
||||||
assert changed_metadata["virt-cloud-network-policy"] == "boot-v1"
|
|
||||||
assert read_seed_network(changed) == {
|
|
||||||
"public_ip_cidr": "8.8.8.19/28",
|
|
||||||
"gateway": "8.8.8.17",
|
|
||||||
"dns_servers": ["8.8.8.8"],
|
|
||||||
"mac": "52:54:00:12:34:56",
|
|
||||||
}
|
|
||||||
|
|
||||||
# A seed created before the network-refresh policy has no marker and must
|
|
||||||
# rotate its NoCloud generation once so cloud-init reloads the new files.
|
|
||||||
legacy_user_data = b'#cloud-config\n{"users": [{"name": "root"}]}\n'
|
|
||||||
legacy_metadata = json.dumps({
|
|
||||||
"instance-id": original_metadata["instance-id"],
|
|
||||||
"local-hostname": original_metadata["local-hostname"],
|
|
||||||
}).encode("ascii") + b"\n"
|
|
||||||
legacy_network = file_bytes(seed, "/network-config")
|
|
||||||
legacy_seed = _build_seed_iso((
|
|
||||||
("/USERDATA.;1", "user-data", "/user-data", legacy_user_data),
|
|
||||||
("/METADATA.;1", "meta-data", "/meta-data", legacy_metadata),
|
|
||||||
("/NETWORK.;1", "network-config", "/network-config", legacy_network),
|
|
||||||
))
|
|
||||||
migrated = replace_seed_network(
|
|
||||||
legacy_seed, "8.8.8.19/28", "8.8.8.17", ["8.8.8.8"],
|
|
||||||
)
|
|
||||||
migrated_metadata = json.loads(file_bytes(migrated, "/meta-data"))
|
|
||||||
assert migrated_metadata["instance-id"] != original_metadata["instance-id"]
|
|
||||||
assert migrated_metadata["instance-id"].startswith("virt-cloud-network-")
|
|
||||||
migrated_user_data = file_bytes(migrated, "/user-data")
|
|
||||||
assert json.loads(migrated_user_data.split(b"\n", 1)[1])["updates"] == {
|
|
||||||
"network": {"when": ["boot"]}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
|
||||||
reason="pycdlib is not installed")
|
|
||||||
def test_replace_seed_network_rejects_non_panel_seed():
|
|
||||||
with pytest.raises(ValueError, match="无法读取原 cloud-init 配置盘"):
|
|
||||||
replace_seed_network(
|
|
||||||
b"not-an-iso", "8.8.8.19/28", "8.8.8.17", ["8.8.8.8"],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
|
||||||
reason="pycdlib is not installed")
|
|
||||||
def test_replace_seed_settings_rotates_generation_without_exposing_password():
|
|
||||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
|
||||||
validated = validate_cloud_network(
|
|
||||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
|
||||||
root_password_hash=password_hash,
|
|
||||||
)
|
|
||||||
seed = build_seed_iso(
|
|
||||||
str(uuid.uuid4()), "settings-vm", "52:54:00:12:34:56", validated,
|
|
||||||
)
|
|
||||||
changed = replace_seed_settings(
|
|
||||||
seed, hostname="Node.Example", root_password_hash=hash_root_password(
|
|
||||||
"Another-strong-password-2026!"
|
|
||||||
), ssh_public_key=PUBLIC_KEY,
|
|
||||||
)
|
|
||||||
user_data = _user_data(changed)
|
|
||||||
root = user_data["users"][0]
|
|
||||||
assert root["ssh_authorized_keys"] == [PUBLIC_KEY]
|
|
||||||
assert root["lock_passwd"] is False
|
|
||||||
assert root["hashed_passwd"].startswith("$6$")
|
|
||||||
assert GUEST_PASSWORD.encode() not in changed
|
|
||||||
assert b"Another-strong-password-2026!" not in changed
|
|
||||||
assert user_data["ssh_pwauth"] is True
|
|
||||||
assert user_data["chpasswd"]["users"][0]["type"] == "hash"
|
|
||||||
assert _meta_data(changed)["local-hostname"] == "node.example"
|
|
||||||
assert _meta_data(changed)["instance-id"].startswith("virt-cloud-settings-")
|
|
||||||
assert read_seed_network(changed) == read_seed_network(seed)
|
|
||||||
@ -1,750 +0,0 @@
|
|||||||
"""Global cloud-template catalog, source file and import job tests."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import stat
|
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
from types import SimpleNamespace
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from virsh_panel.cloud_template_runtime import (
|
|
||||||
CACHE_TOMBSTONE_KIND,
|
|
||||||
CATALOG_KIND,
|
|
||||||
IMPORT_JOB_KIND,
|
|
||||||
CloudTemplateRuntime,
|
|
||||||
GlobalCloudTemplateFiles,
|
|
||||||
GlobalCloudTemplateStore,
|
|
||||||
PERSISTENT_FREE_RESERVE_BYTES,
|
|
||||||
cloud_template_id_for,
|
|
||||||
)
|
|
||||||
from virsh_panel.cloud_templates import GIB, TemplateError, template_volume_name
|
|
||||||
|
|
||||||
|
|
||||||
ALMA_10_URL = (
|
|
||||||
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/"
|
|
||||||
"AlmaLinux-10-GenericCloud-latest.x86_64.qcow2"
|
|
||||||
)
|
|
||||||
ALMA_8_URL = (
|
|
||||||
"https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/"
|
|
||||||
"AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
|
|
||||||
)
|
|
||||||
DEBIAN_12_URL = (
|
|
||||||
"https://cloud.debian.org/images/cloud/bookworm/latest/"
|
|
||||||
"debian-12-generic-amd64.qcow2"
|
|
||||||
)
|
|
||||||
UBUNTU_24_URL = (
|
|
||||||
"https://cloud-images.ubuntu.com/releases/24.04/release/"
|
|
||||||
"ubuntu-24.04-server-cloudimg-amd64.img"
|
|
||||||
)
|
|
||||||
CENTOS_STREAM_10_URL = (
|
|
||||||
"https://cloud.centos.org/centos/10-stream/x86_64/images/"
|
|
||||||
"CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class MemoryStore:
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.values: dict[tuple[str, str], str] = {}
|
|
||||||
self.lock = threading.RLock()
|
|
||||||
|
|
||||||
def list(self, kind: str) -> dict[str, str]:
|
|
||||||
with self.lock:
|
|
||||||
return {
|
|
||||||
key: value for (record_kind, key), value in self.values.items()
|
|
||||||
if record_kind == kind
|
|
||||||
}
|
|
||||||
|
|
||||||
def get(self, kind: str, key: str) -> str | None:
|
|
||||||
with self.lock:
|
|
||||||
return self.values.get((kind, key))
|
|
||||||
|
|
||||||
def set(self, kind: str, key: str, value: str) -> None:
|
|
||||||
with self.lock:
|
|
||||||
self.values[(kind, key)] = value
|
|
||||||
|
|
||||||
def insert(self, kind: str, key: str, value: str) -> bool:
|
|
||||||
with self.lock:
|
|
||||||
identity = (kind, key)
|
|
||||||
if identity in self.values:
|
|
||||||
return False
|
|
||||||
self.values[identity] = value
|
|
||||||
return True
|
|
||||||
|
|
||||||
def delete(self, kind: str, key: str) -> None:
|
|
||||||
with self.lock:
|
|
||||||
self.values.pop((kind, key), None)
|
|
||||||
|
|
||||||
|
|
||||||
class CommitThenRaiseCatalogStore(MemoryStore):
|
|
||||||
"""Simulate a committed INSERT whose acknowledgement is lost."""
|
|
||||||
|
|
||||||
def insert(self, kind: str, key: str, value: str) -> bool:
|
|
||||||
inserted = super().insert(kind, key, value)
|
|
||||||
if kind == CATALOG_KIND and inserted:
|
|
||||||
raise RuntimeError("catalog insert acknowledgement lost")
|
|
||||||
return inserted
|
|
||||||
|
|
||||||
|
|
||||||
class AmbiguousCatalogCommitStore(CommitThenRaiseCatalogStore):
|
|
||||||
"""Also fail the first catalog read after the ambiguous INSERT."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
super().__init__()
|
|
||||||
self.fail_next_catalog_read = False
|
|
||||||
|
|
||||||
def list(self, kind: str) -> dict[str, str]:
|
|
||||||
if kind == CATALOG_KIND and self.fail_next_catalog_read:
|
|
||||||
self.fail_next_catalog_read = False
|
|
||||||
raise RuntimeError("catalog read unavailable")
|
|
||||||
return super().list(kind)
|
|
||||||
|
|
||||||
def insert(self, kind: str, key: str, value: str) -> bool:
|
|
||||||
try:
|
|
||||||
return super().insert(kind, key, value)
|
|
||||||
except RuntimeError:
|
|
||||||
self.fail_next_catalog_read = True
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
class LoseFirstCompletedJobWriteStore(MemoryStore):
|
|
||||||
"""Drop one terminal job update while retaining its running record."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
super().__init__()
|
|
||||||
self.lost_completed_write = False
|
|
||||||
|
|
||||||
def set(self, kind: str, key: str, value: str) -> None:
|
|
||||||
if kind == IMPORT_JOB_KIND:
|
|
||||||
job = json.loads(value)
|
|
||||||
if (job.get("status") == "completed"
|
|
||||||
and not self.lost_completed_write):
|
|
||||||
self.lost_completed_write = True
|
|
||||||
raise RuntimeError("completed job write acknowledgement lost")
|
|
||||||
super().set(kind, key, value)
|
|
||||||
|
|
||||||
|
|
||||||
def wait_for_job(runtime: CloudTemplateRuntime, job_id: str) -> dict:
|
|
||||||
for _ in range(200):
|
|
||||||
job = runtime.get_job(job_id)
|
|
||||||
if job["status"] in {"completed", "failed"}:
|
|
||||||
return job
|
|
||||||
time.sleep(0.01)
|
|
||||||
raise AssertionError("cloud template import did not finish")
|
|
||||||
|
|
||||||
|
|
||||||
def fake_download_bytes(payload: bytes):
|
|
||||||
digest = hashlib.sha256(payload).hexdigest()
|
|
||||||
|
|
||||||
def download(_url: str, expected_sha256: str, directory: Path, **_kwargs):
|
|
||||||
assert expected_sha256 in {"", digest}
|
|
||||||
source = Path(directory) / ".cloud-image-test.qcow2"
|
|
||||||
source.write_bytes(payload)
|
|
||||||
os.chmod(source, 0o600)
|
|
||||||
return source, 2 * GIB, digest
|
|
||||||
|
|
||||||
return digest, download
|
|
||||||
|
|
||||||
|
|
||||||
def record_for(payload: bytes, *, name: str = "AlmaLinux 10") -> dict:
|
|
||||||
digest = hashlib.sha256(payload).hexdigest()
|
|
||||||
template_id = cloud_template_id_for(digest, "10")
|
|
||||||
return {
|
|
||||||
"id": template_id,
|
|
||||||
"name": name,
|
|
||||||
"os_family": "AlmaLinux",
|
|
||||||
"os_version": "10",
|
|
||||||
"sha256": digest,
|
|
||||||
"volume_name": template_volume_name(template_id),
|
|
||||||
"virtual_size": 2 * GIB,
|
|
||||||
"image_bytes": len(payload),
|
|
||||||
"created_at": 1,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_stable_global_id_and_catalog_use_an_independent_record_kind():
|
|
||||||
digest = "a" * 64
|
|
||||||
first = cloud_template_id_for(digest, "10")
|
|
||||||
second = cloud_template_id_for(digest.upper(), "10")
|
|
||||||
assert first == second
|
|
||||||
assert first.startswith("alma10-")
|
|
||||||
assert template_volume_name(first).endswith(".qcow2")
|
|
||||||
|
|
||||||
store = MemoryStore()
|
|
||||||
catalog = GlobalCloudTemplateStore(store)
|
|
||||||
record = record_for(b"catalog-image")
|
|
||||||
catalog.add(record)
|
|
||||||
|
|
||||||
assert catalog.get(record["id"]) == record
|
|
||||||
assert list(store.list(CATALOG_KIND)) == [record["id"]]
|
|
||||||
assert store.list("template") == {}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"message",
|
|
||||||
[
|
|
||||||
"官方云镜像地址无法解析",
|
|
||||||
"官方云镜像地址解析结果无效",
|
|
||||||
"官方云镜像地址解析到非公网地址",
|
|
||||||
"无法连接官方云镜像下载地址",
|
|
||||||
"官方云镜像重定向目标不安全",
|
|
||||||
"官方云镜像重定向次数过多",
|
|
||||||
"官方云镜像下载服务器返回异常状态",
|
|
||||||
"官方云镜像下载响应格式无效",
|
|
||||||
"云镜像下载内容不完整",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_public_job_preserves_fixed_safe_download_errors(message):
|
|
||||||
job = {
|
|
||||||
"job_id": "safe-error",
|
|
||||||
"status": "failed",
|
|
||||||
"phase": "导入失败",
|
|
||||||
"created_at": 1,
|
|
||||||
"error": message,
|
|
||||||
}
|
|
||||||
assert CloudTemplateRuntime.public_job(job)["error"] == message
|
|
||||||
|
|
||||||
|
|
||||||
def test_import_publishes_private_source_and_restores_global_catalog(tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
payload = b"verified-global-cloud-image"
|
|
||||||
digest, downloader = fake_download_bytes(payload)
|
|
||||||
try:
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
||||||
side_effect=downloader,
|
|
||||||
):
|
|
||||||
queued = runtime.queue_import(
|
|
||||||
ALMA_10_URL, "AlmaLinux 10", digest
|
|
||||||
)
|
|
||||||
job = wait_for_job(runtime, queued["job_id"])
|
|
||||||
|
|
||||||
assert job["status"] == "completed", job
|
|
||||||
template_id = job["template"]["id"]
|
|
||||||
assert template_id == cloud_template_id_for(digest, "10")
|
|
||||||
assert "volume_name" not in job["template"]
|
|
||||||
listed = runtime.list()
|
|
||||||
assert listed == [job["template"]]
|
|
||||||
assert "active_import_job" not in runtime.status()
|
|
||||||
|
|
||||||
root = tmp_path / "cloud-templates"
|
|
||||||
source = root / template_volume_name(template_id)
|
|
||||||
assert source.read_bytes() == payload
|
|
||||||
assert stat.S_IMODE(root.stat().st_mode) == 0o700
|
|
||||||
assert stat.S_IMODE(source.stat().st_mode) == 0o600
|
|
||||||
assert store.get(IMPORT_JOB_KIND, queued["job_id"]) is not None
|
|
||||||
|
|
||||||
# A new runtime reads the same host-independent catalog and verifies
|
|
||||||
# the persisted source from disk instead of trusting process memory.
|
|
||||||
restored = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
try:
|
|
||||||
restored.restore_jobs()
|
|
||||||
assert restored.list() == listed
|
|
||||||
with restored.source(template_id) as (record, path):
|
|
||||||
assert record["sha256"] == digest
|
|
||||||
assert path == source
|
|
||||||
assert restored.in_use[template_id] == 1
|
|
||||||
assert template_id not in restored.in_use
|
|
||||||
finally:
|
|
||||||
restored.close()
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("url", "family", "version", "prefix"),
|
|
||||||
[
|
|
||||||
(ALMA_8_URL, "AlmaLinux", "8", "alma8-"),
|
|
||||||
(DEBIAN_12_URL, "Debian", "12", "debian12-"),
|
|
||||||
(UBUNTU_24_URL, "Ubuntu", "24.04", "ubuntu2404-"),
|
|
||||||
(CENTOS_STREAM_10_URL, "CentOS Stream", "10", "centosstream10-"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_import_derives_family_version_and_safe_source_name(
|
|
||||||
tmp_path, url, family, version, prefix):
|
|
||||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
||||||
payload = f"{family}-{version}-cloud-image".encode()
|
|
||||||
digest, downloader = fake_download_bytes(payload)
|
|
||||||
try:
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
||||||
side_effect=downloader,
|
|
||||||
):
|
|
||||||
queued = runtime.queue_import(url, f"{family} {version}", digest)
|
|
||||||
job = wait_for_job(runtime, queued["job_id"])
|
|
||||||
|
|
||||||
assert job["status"] == "completed", job
|
|
||||||
template = job["template"]
|
|
||||||
assert template["id"].startswith(prefix)
|
|
||||||
assert template["os_family"] == family
|
|
||||||
assert template["os_version"] == version
|
|
||||||
internal = runtime.get(template["id"])
|
|
||||||
assert internal["volume_name"] == template_volume_name(template["id"])
|
|
||||||
assert (
|
|
||||||
tmp_path / "cloud-templates" / internal["volume_name"]
|
|
||||||
).read_bytes() == payload
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_available_bytes_preserves_panel_disk_reserve(tmp_path):
|
|
||||||
files = GlobalCloudTemplateFiles(tmp_path)
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
|
|
||||||
return_value=SimpleNamespace(
|
|
||||||
free=PERSISTENT_FREE_RESERVE_BYTES + 3 * 1024 * 1024
|
|
||||||
),
|
|
||||||
):
|
|
||||||
assert files.available_bytes() == 3 * 1024 * 1024
|
|
||||||
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
|
|
||||||
return_value=SimpleNamespace(free=PERSISTENT_FREE_RESERVE_BYTES - 1),
|
|
||||||
):
|
|
||||||
assert files.available_bytes() == 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_import_rejects_before_queue_when_only_reserved_space_remains(tmp_path):
|
|
||||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
||||||
try:
|
|
||||||
with patch.object(
|
|
||||||
runtime.files, "available_bytes", return_value=1024 * 1024 - 1
|
|
||||||
):
|
|
||||||
with pytest.raises(TemplateError, match="空间不足") as insufficient:
|
|
||||||
runtime.queue_import(ALMA_10_URL, "No Disk Space", "")
|
|
||||||
assert insufficient.value.status == 507
|
|
||||||
assert runtime.jobs == {}
|
|
||||||
assert runtime.pending == set()
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_import_passes_only_space_above_reserve_to_downloader(tmp_path):
|
|
||||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
||||||
payload = b"bounded-global-cloud-image"
|
|
||||||
digest = hashlib.sha256(payload).hexdigest()
|
|
||||||
limits: list[int] = []
|
|
||||||
|
|
||||||
def bounded_download(_url: str, _expected: str, directory: Path, *,
|
|
||||||
max_bytes: int):
|
|
||||||
limits.append(max_bytes)
|
|
||||||
source = Path(directory) / ".cloud-image-bounded.qcow2"
|
|
||||||
source.write_bytes(payload)
|
|
||||||
os.chmod(source, 0o600)
|
|
||||||
return source, 2 * GIB, digest
|
|
||||||
|
|
||||||
try:
|
|
||||||
with patch.object(
|
|
||||||
runtime.files, "available_bytes",
|
|
||||||
side_effect=[3 * 1024 * 1024, 2 * 1024 * 1024],
|
|
||||||
), patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
||||||
side_effect=bounded_download,
|
|
||||||
):
|
|
||||||
queued = runtime.queue_import(ALMA_10_URL, "Bounded", "")
|
|
||||||
job = wait_for_job(runtime, queued["job_id"])
|
|
||||||
assert job["status"] == "completed", job
|
|
||||||
assert limits == [2 * 1024 * 1024]
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_duplicate_digest_and_casefold_name_are_rejected_globally():
|
|
||||||
store = MemoryStore()
|
|
||||||
catalog = GlobalCloudTemplateStore(store)
|
|
||||||
first = record_for(b"first-image", name="AlmaLinux Gold")
|
|
||||||
catalog.add(first)
|
|
||||||
|
|
||||||
with pytest.raises(TemplateError, match="已经导入") as duplicate_digest:
|
|
||||||
catalog.add(first)
|
|
||||||
assert duplicate_digest.value.status == 409
|
|
||||||
|
|
||||||
second = record_for(b"second-image", name="almalinux gold")
|
|
||||||
with pytest.raises(TemplateError, match="同名") as duplicate_name:
|
|
||||||
catalog.add(second)
|
|
||||||
assert duplicate_name.value.status == 409
|
|
||||||
|
|
||||||
|
|
||||||
def test_rename_changes_only_display_name_and_keeps_source_identity(tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
catalog = GlobalCloudTemplateStore(store)
|
|
||||||
record = record_for(b"rename-image")
|
|
||||||
catalog.add(record)
|
|
||||||
updated = catalog.rename(record["id"], "AlmaLinux 10 Production")
|
|
||||||
|
|
||||||
assert updated["name"] == "AlmaLinux 10 Production"
|
|
||||||
for key in (
|
|
||||||
"id", "sha256", "volume_name", "virtual_size", "image_bytes",
|
|
||||||
"os_family", "os_version", "created_at",
|
|
||||||
):
|
|
||||||
assert updated[key] == record[key]
|
|
||||||
|
|
||||||
files = GlobalCloudTemplateFiles(tmp_path)
|
|
||||||
temporary = files.directory() / ".cloud-image-rename.qcow2"
|
|
||||||
temporary.write_bytes(b"rename-image")
|
|
||||||
os.chmod(temporary, 0o600)
|
|
||||||
source = files.publish(
|
|
||||||
temporary, record["id"], expected_size=len(b"rename-image")
|
|
||||||
)
|
|
||||||
assert source.name == record["volume_name"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_delete_removes_global_catalog_source_and_verification_cache(tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
payload = b"delete-global-cloud-image"
|
|
||||||
record = record_for(payload)
|
|
||||||
runtime.catalog.add(record)
|
|
||||||
temporary = runtime.files.directory() / ".cloud-image-delete.qcow2"
|
|
||||||
temporary.write_bytes(payload)
|
|
||||||
os.chmod(temporary, 0o600)
|
|
||||||
source = runtime.files.publish(
|
|
||||||
temporary, record["id"], expected_size=len(payload)
|
|
||||||
)
|
|
||||||
runtime.verified_sources[record["id"]] = (1, 2, 3, 4, record["sha256"])
|
|
||||||
try:
|
|
||||||
runtime.delete(record["id"])
|
|
||||||
|
|
||||||
assert not source.exists()
|
|
||||||
assert store.get(CATALOG_KIND, record["id"]) is None
|
|
||||||
assert record["id"] not in runtime.verified_sources
|
|
||||||
with pytest.raises(TemplateError) as missing:
|
|
||||||
runtime.get(record["id"])
|
|
||||||
assert missing.value.status == 404
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_delete_rejects_import_or_active_source_lease(tmp_path):
|
|
||||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
||||||
payload = b"leased-global-cloud-image"
|
|
||||||
record = record_for(payload)
|
|
||||||
runtime.catalog.add(record)
|
|
||||||
temporary = runtime.files.directory() / ".cloud-image-leased.qcow2"
|
|
||||||
temporary.write_bytes(payload)
|
|
||||||
os.chmod(temporary, 0o600)
|
|
||||||
source = runtime.files.publish(
|
|
||||||
temporary, record["id"], expected_size=len(payload)
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
runtime.pending.add("active-import")
|
|
||||||
with pytest.raises(TemplateError, match="正在导入") as importing:
|
|
||||||
runtime.delete(record["id"])
|
|
||||||
assert importing.value.status == 409
|
|
||||||
runtime.pending.clear()
|
|
||||||
|
|
||||||
with runtime.source(record["id"]):
|
|
||||||
with pytest.raises(TemplateError, match="正在同步") as leased:
|
|
||||||
runtime.delete(record["id"])
|
|
||||||
assert leased.value.status == 409
|
|
||||||
assert source.exists()
|
|
||||||
assert runtime.get(record["id"])["id"] == record["id"]
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_reimport_waits_for_old_host_cache_gc_then_reuses_stable_id(tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
record = record_for(b"reimport-identical-image")
|
|
||||||
tombstone = {
|
|
||||||
"deletion_id": "a" * 32,
|
|
||||||
"template_id": record["id"],
|
|
||||||
"template_created_at": record["created_at"],
|
|
||||||
"source_volume_name": record["volume_name"],
|
|
||||||
"pending_hosts": ["offline-host"],
|
|
||||||
"source_deleted": True,
|
|
||||||
"created_at": 1,
|
|
||||||
}
|
|
||||||
store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone))
|
|
||||||
try:
|
|
||||||
with pytest.raises(TemplateError, match="缓存仍在清理") as pending:
|
|
||||||
runtime._prepare_reimport(record["id"])
|
|
||||||
assert pending.value.status == 409
|
|
||||||
assert store.get(CACHE_TOMBSTONE_KIND, record["id"]) is not None
|
|
||||||
|
|
||||||
tombstone["pending_hosts"] = []
|
|
||||||
store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone))
|
|
||||||
previous_created_at = runtime._prepare_reimport(record["id"])
|
|
||||||
assert store.get(CACHE_TOMBSTONE_KIND, record["id"]) is None
|
|
||||||
assert previous_created_at == record["created_at"]
|
|
||||||
first_generation = runtime._next_generation(
|
|
||||||
record["id"], previous_created_at)
|
|
||||||
second_generation = runtime._next_generation(record["id"])
|
|
||||||
assert second_generation > first_generation > previous_created_at
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_restore_cancels_provisional_delete_when_source_and_catalog_survive(
|
|
||||||
tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
payload = b"delete-never-committed"
|
|
||||||
record = record_for(payload)
|
|
||||||
runtime.catalog.add(record)
|
|
||||||
temporary = runtime.files.directory() / ".cloud-image-delete-intact.qcow2"
|
|
||||||
temporary.write_bytes(payload)
|
|
||||||
os.chmod(temporary, 0o600)
|
|
||||||
source = runtime.files.publish(
|
|
||||||
temporary, record["id"], expected_size=len(payload)
|
|
||||||
)
|
|
||||||
tombstone = {
|
|
||||||
"deletion_id": "c" * 32,
|
|
||||||
"template_id": record["id"],
|
|
||||||
"template_created_at": record["created_at"],
|
|
||||||
"source_volume_name": record["volume_name"],
|
|
||||||
"pending_hosts": ["host-one"],
|
|
||||||
"source_deleted": False,
|
|
||||||
"created_at": 1,
|
|
||||||
}
|
|
||||||
store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone))
|
|
||||||
try:
|
|
||||||
runtime.restore_jobs()
|
|
||||||
|
|
||||||
assert runtime.get(record["id"])["id"] == record["id"]
|
|
||||||
assert source.is_file()
|
|
||||||
assert store.get(CACHE_TOMBSTONE_KIND, record["id"]) is None
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_restore_finishes_delete_after_source_unlink_crash_window(tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
payload = b"delete-unlink-committed"
|
|
||||||
record = record_for(payload)
|
|
||||||
runtime.catalog.add(record)
|
|
||||||
tombstone = {
|
|
||||||
"deletion_id": "d" * 32,
|
|
||||||
"template_id": record["id"],
|
|
||||||
"template_created_at": record["created_at"],
|
|
||||||
"source_volume_name": record["volume_name"],
|
|
||||||
"pending_hosts": ["host-one"],
|
|
||||||
"source_deleted": False,
|
|
||||||
"created_at": 1,
|
|
||||||
}
|
|
||||||
store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone))
|
|
||||||
try:
|
|
||||||
# Listing is observational. Only delete recovery/maintenance may
|
|
||||||
# advance the durable tombstone and remove local source/catalog data.
|
|
||||||
assert [row["id"] for row in runtime.list()] == [record["id"]]
|
|
||||||
before = json.loads(store.get(CACHE_TOMBSTONE_KIND, record["id"]))
|
|
||||||
assert before["source_deleted"] is False
|
|
||||||
assert runtime.get(record["id"])["id"] == record["id"]
|
|
||||||
runtime.restore_jobs()
|
|
||||||
|
|
||||||
with pytest.raises(TemplateError) as missing:
|
|
||||||
runtime.get(record["id"])
|
|
||||||
assert missing.value.status == 404
|
|
||||||
restored = json.loads(store.get(
|
|
||||||
CACHE_TOMBSTONE_KIND, record["id"]
|
|
||||||
))
|
|
||||||
assert restored["source_deleted"] is True
|
|
||||||
assert restored["pending_hosts"] == ["host-one"]
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_source_rejects_permission_damage_and_content_tampering(tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
payload = b"source-integrity"
|
|
||||||
record = record_for(payload)
|
|
||||||
runtime.catalog.add(record)
|
|
||||||
temporary = runtime.files.directory() / ".cloud-image-integrity.qcow2"
|
|
||||||
temporary.write_bytes(payload)
|
|
||||||
os.chmod(temporary, 0o600)
|
|
||||||
source = runtime.files.publish(
|
|
||||||
temporary, record["id"], expected_size=len(payload)
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
os.chmod(source, 0o640)
|
|
||||||
with pytest.raises(TemplateError, match="已损坏"):
|
|
||||||
with runtime.source(record["id"]):
|
|
||||||
pass
|
|
||||||
|
|
||||||
os.chmod(source, 0o600)
|
|
||||||
source.write_bytes(b"x" * len(payload))
|
|
||||||
with pytest.raises(TemplateError, match="SHA-256"):
|
|
||||||
with runtime.source(record["id"]):
|
|
||||||
pass
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_concurrent_import_is_serialized_and_second_request_is_rejected(tmp_path):
|
|
||||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
||||||
started = threading.Event()
|
|
||||||
release = threading.Event()
|
|
||||||
payload = b"blocking-image"
|
|
||||||
digest = hashlib.sha256(payload).hexdigest()
|
|
||||||
|
|
||||||
def blocking_download(_url: str, _expected: str, directory: Path,
|
|
||||||
**_kwargs):
|
|
||||||
started.set()
|
|
||||||
assert release.wait(timeout=5)
|
|
||||||
source = Path(directory) / ".cloud-image-blocking.qcow2"
|
|
||||||
source.write_bytes(payload)
|
|
||||||
os.chmod(source, 0o600)
|
|
||||||
return source, 2 * GIB, digest
|
|
||||||
|
|
||||||
try:
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
||||||
side_effect=blocking_download,
|
|
||||||
):
|
|
||||||
first = runtime.queue_import(ALMA_10_URL, "First", "")
|
|
||||||
assert started.wait(timeout=2)
|
|
||||||
assert runtime.has_active_jobs() is True
|
|
||||||
with pytest.raises(TemplateError, match="正在导入") as busy:
|
|
||||||
runtime.queue_import(ALMA_10_URL, "Second", "")
|
|
||||||
assert busy.value.status == 409
|
|
||||||
release.set()
|
|
||||||
assert wait_for_job(runtime, first["job_id"])["status"] == "completed"
|
|
||||||
assert runtime.has_active_jobs() is False
|
|
||||||
finally:
|
|
||||||
release.set()
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_restore_marks_interrupted_job_failed_and_removes_partial_files(tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
job_id = "interrupted-cloud-template"
|
|
||||||
store.set(IMPORT_JOB_KIND, job_id, json.dumps({
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "running",
|
|
||||||
"phase": "正在下载并校验云镜像",
|
|
||||||
"created_at": 1,
|
|
||||||
}))
|
|
||||||
files = GlobalCloudTemplateFiles(tmp_path)
|
|
||||||
partial = files.directory() / ".cloud-image-abandoned.qcow2"
|
|
||||||
partial.write_bytes(b"partial")
|
|
||||||
os.chmod(partial, 0o600)
|
|
||||||
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
try:
|
|
||||||
runtime.restore_jobs()
|
|
||||||
job = runtime.get_job(job_id)
|
|
||||||
assert job["status"] == "failed"
|
|
||||||
assert "重启" in job["phase"]
|
|
||||||
assert "重启" in job["error"]
|
|
||||||
assert not partial.exists()
|
|
||||||
assert runtime.has_active_jobs() is False
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_import_failure_redacts_internal_paths_and_releases_pending(tmp_path):
|
|
||||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
||||||
try:
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
||||||
side_effect=TemplateError(
|
|
||||||
"私有目录 /home/virt-cloud/data/cloud-templates 损坏", 500
|
|
||||||
),
|
|
||||||
):
|
|
||||||
queued = runtime.queue_import(ALMA_10_URL, "Broken", "")
|
|
||||||
failed = wait_for_job(runtime, queued["job_id"])
|
|
||||||
assert failed["status"] == "failed"
|
|
||||||
assert failed["error"] == "云镜像下载或保存失败,请联系管理员查看日志"
|
|
||||||
assert "/home/" not in json.dumps(failed, ensure_ascii=False)
|
|
||||||
assert runtime.pending == set()
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_catalog_insert_commit_with_lost_ack_is_completed_not_rolled_back(
|
|
||||||
tmp_path):
|
|
||||||
store = CommitThenRaiseCatalogStore()
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
payload = b"catalog-commit-ack-lost"
|
|
||||||
digest, downloader = fake_download_bytes(payload)
|
|
||||||
try:
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
||||||
side_effect=downloader,
|
|
||||||
):
|
|
||||||
queued = runtime.queue_import(ALMA_10_URL, "Committed", digest)
|
|
||||||
job = wait_for_job(runtime, queued["job_id"])
|
|
||||||
|
|
||||||
assert job["status"] == "completed"
|
|
||||||
template_id = cloud_template_id_for(digest, "10")
|
|
||||||
assert runtime.get(template_id)["sha256"] == digest
|
|
||||||
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
|
|
||||||
assert source.read_bytes() == payload
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_ambiguous_catalog_commit_preserves_source_for_startup_recovery(
|
|
||||||
tmp_path):
|
|
||||||
store = AmbiguousCatalogCommitStore()
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
payload = b"ambiguous-catalog-commit"
|
|
||||||
digest, downloader = fake_download_bytes(payload)
|
|
||||||
try:
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
||||||
side_effect=downloader,
|
|
||||||
):
|
|
||||||
queued = runtime.queue_import(ALMA_10_URL, "Ambiguous", digest)
|
|
||||||
failed = wait_for_job(runtime, queued["job_id"])
|
|
||||||
|
|
||||||
assert failed["status"] == "failed"
|
|
||||||
template_id = cloud_template_id_for(digest, "10")
|
|
||||||
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
|
|
||||||
assert source.read_bytes() == payload
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
restored = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
try:
|
|
||||||
restored.restore_jobs()
|
|
||||||
recovered = restored.get_job(queued["job_id"])
|
|
||||||
assert recovered["status"] == "completed"
|
|
||||||
assert recovered["template"]["id"] == template_id
|
|
||||||
assert source.read_bytes() == payload
|
|
||||||
finally:
|
|
||||||
restored.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_lost_completed_job_write_is_success_in_memory_and_repaired_on_restart(
|
|
||||||
tmp_path):
|
|
||||||
store = LoseFirstCompletedJobWriteStore()
|
|
||||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
payload = b"lost-completed-job-write"
|
|
||||||
digest, downloader = fake_download_bytes(payload)
|
|
||||||
try:
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
||||||
side_effect=downloader,
|
|
||||||
):
|
|
||||||
queued = runtime.queue_import(ALMA_10_URL, "Recoverable", digest)
|
|
||||||
completed = wait_for_job(runtime, queued["job_id"])
|
|
||||||
|
|
||||||
assert completed["status"] == "completed"
|
|
||||||
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
|
|
||||||
assert persisted["status"] == "running"
|
|
||||||
assert persisted["candidate_template_id"] == cloud_template_id_for(
|
|
||||||
digest, "10"
|
|
||||||
)
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
restored = CloudTemplateRuntime(store, tmp_path)
|
|
||||||
try:
|
|
||||||
restored.restore_jobs()
|
|
||||||
repaired = restored.get_job(queued["job_id"])
|
|
||||||
assert repaired["status"] == "completed"
|
|
||||||
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
|
|
||||||
assert persisted["status"] == "completed"
|
|
||||||
finally:
|
|
||||||
restored.close()
|
|
||||||
@ -1,502 +0,0 @@
|
|||||||
"""Official-image import restrictions and private template catalog tests."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import io
|
|
||||||
import socket
|
|
||||||
import stat
|
|
||||||
import struct
|
|
||||||
from types import SimpleNamespace
|
|
||||||
from unittest.mock import patch
|
|
||||||
from urllib.parse import urlsplit
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from virsh_panel.cloud_templates import (
|
|
||||||
TemplateError,
|
|
||||||
TemplateStore,
|
|
||||||
_RejectRedirects,
|
|
||||||
_open_image_response,
|
|
||||||
_redirect_target,
|
|
||||||
_resolve_global_addresses,
|
|
||||||
download_verified_image,
|
|
||||||
image_metadata_for,
|
|
||||||
image_version_for,
|
|
||||||
template_id_for,
|
|
||||||
template_scope,
|
|
||||||
template_volume_name,
|
|
||||||
validate_image_url,
|
|
||||||
verify_qcow2_image,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
OFFICIAL = (
|
|
||||||
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/"
|
|
||||||
"AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2"
|
|
||||||
)
|
|
||||||
OFFICIAL_9 = (
|
|
||||||
"https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/"
|
|
||||||
"AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
|
|
||||||
)
|
|
||||||
OFFICIAL_8 = (
|
|
||||||
"https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/"
|
|
||||||
"AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
|
|
||||||
)
|
|
||||||
DEBIAN_12 = (
|
|
||||||
"https://cloud.debian.org/images/cloud/bookworm/latest/"
|
|
||||||
"debian-12-generic-amd64.qcow2"
|
|
||||||
)
|
|
||||||
DEBIAN_13_BUILD = (
|
|
||||||
"https://cloud.debian.org/images/cloud/trixie/20260801-2204/"
|
|
||||||
"debian-13-generic-amd64-20260801-2204.qcow2"
|
|
||||||
)
|
|
||||||
DEBIAN_12_GENERICCLOUD = (
|
|
||||||
"https://cloud.debian.org/images/cloud/bookworm/latest/"
|
|
||||||
"debian-12-genericcloud-amd64.qcow2"
|
|
||||||
)
|
|
||||||
UBUNTU_24 = (
|
|
||||||
"https://cloud-images.ubuntu.com/releases/noble/release/"
|
|
||||||
"ubuntu-24.04-server-cloudimg-amd64.img"
|
|
||||||
)
|
|
||||||
UBUNTU_26_CURRENT = (
|
|
||||||
"https://cloud-images.ubuntu.com/resolute/current/"
|
|
||||||
"resolute-server-cloudimg-amd64.img"
|
|
||||||
)
|
|
||||||
CENTOS_STREAM_9 = (
|
|
||||||
"https://cloud.centos.org/centos/9-stream/x86_64/images/"
|
|
||||||
"CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
|
|
||||||
)
|
|
||||||
CENTOS_STREAM_10 = (
|
|
||||||
"https://cloud.centos.org/centos/10-stream/x86_64/images/"
|
|
||||||
"CentOS-Stream-GenericCloud-x86_64-10-20260810.0.x86_64.qcow2"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class FakeHeaders:
|
|
||||||
def __init__(self, values=None):
|
|
||||||
self.values = values or {}
|
|
||||||
|
|
||||||
def get_all(self, name, default=None):
|
|
||||||
return self.values.get(name, default)
|
|
||||||
|
|
||||||
|
|
||||||
class FakeImageResponse(io.BytesIO):
|
|
||||||
def __init__(self, body=b"", *, status=200, headers=None):
|
|
||||||
super().__init__(body)
|
|
||||||
self.status = status
|
|
||||||
self.header_values = headers or {}
|
|
||||||
self.headers = FakeHeaders(self.header_values)
|
|
||||||
|
|
||||||
def getheader(self, name, default=None):
|
|
||||||
values = self.header_values.get(name)
|
|
||||||
if not values:
|
|
||||||
return default
|
|
||||||
return values[0]
|
|
||||||
|
|
||||||
|
|
||||||
def fake_image_response(body: bytes):
|
|
||||||
return (
|
|
||||||
SimpleNamespace(close=lambda: None),
|
|
||||||
FakeImageResponse(
|
|
||||||
body,
|
|
||||||
headers={"Content-Length": [str(len(body))]},
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_official_almalinux_8_9_and_10_are_distinct_supported_versions():
|
|
||||||
assert validate_image_url(OFFICIAL_8) == OFFICIAL_8
|
|
||||||
assert validate_image_url(OFFICIAL_9) == OFFICIAL_9
|
|
||||||
assert image_metadata_for(OFFICIAL_8) == ("AlmaLinux", "8")
|
|
||||||
assert image_version_for(OFFICIAL_9) == "9"
|
|
||||||
assert image_version_for(OFFICIAL) == "10"
|
|
||||||
scope = "a" * 64
|
|
||||||
digest = "b" * 64
|
|
||||||
assert template_id_for(scope, digest, "8").startswith("alma8-")
|
|
||||||
assert template_id_for(scope, digest, "9").startswith("alma9-")
|
|
||||||
assert template_id_for(scope, digest, "10").startswith("alma10-")
|
|
||||||
assert template_volume_name(template_id_for(scope, digest, "9")).startswith(
|
|
||||||
"panel-template-alma9-"
|
|
||||||
)
|
|
||||||
with pytest.raises(TemplateError):
|
|
||||||
validate_image_url(OFFICIAL_9.replace("/almalinux/9/", "/almalinux/10/"))
|
|
||||||
with pytest.raises(TemplateError):
|
|
||||||
validate_image_url(OFFICIAL_9.replace("AlmaLinux-9-", "AlmaLinux-8-"))
|
|
||||||
with pytest.raises(TemplateError):
|
|
||||||
template_id_for(scope, digest, "7")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("url", "family", "version", "prefix"),
|
|
||||||
[
|
|
||||||
(DEBIAN_12, "Debian", "12", "debian12-"),
|
|
||||||
(DEBIAN_12_GENERICCLOUD, "Debian", "12", "debian12-"),
|
|
||||||
(DEBIAN_13_BUILD, "Debian", "13", "debian13-"),
|
|
||||||
(UBUNTU_24, "Ubuntu", "24.04", "ubuntu2404-"),
|
|
||||||
(UBUNTU_26_CURRENT, "Ubuntu", "26.04", "ubuntu2604-"),
|
|
||||||
(CENTOS_STREAM_9, "CentOS Stream", "9", "centosstream9-"),
|
|
||||||
(CENTOS_STREAM_10, "CentOS Stream", "10", "centosstream10-"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_supported_official_cloud_images_have_distinct_safe_identifiers(
|
|
||||||
url, family, version, prefix):
|
|
||||||
assert validate_image_url(url) == url
|
|
||||||
assert image_metadata_for(url) == (family, version)
|
|
||||||
template_id = template_id_for("a" * 64, "b" * 64, version, family)
|
|
||||||
assert template_id.startswith(prefix)
|
|
||||||
assert template_volume_name(template_id) == (
|
|
||||||
f"panel-template-{template_id}.qcow2"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_template_identifier_rejects_invalid_digest_precisely():
|
|
||||||
with pytest.raises(TemplateError, match="SHA-256 格式"):
|
|
||||||
template_id_for("a" * 64, "not-a-digest", "12", "Debian")
|
|
||||||
with pytest.raises(TemplateError, match="系统版本无效"):
|
|
||||||
template_id_for("a" * 64, "b" * 64, ["12"], "Debian")
|
|
||||||
|
|
||||||
|
|
||||||
def qcow2_fixture(virtual_size=2 * 1024**3) -> bytes:
|
|
||||||
header = bytearray(104)
|
|
||||||
header[:4] = b"QFI\xfb"
|
|
||||||
header[4:8] = struct.pack(">I", 3)
|
|
||||||
header[20:24] = struct.pack(">I", 16)
|
|
||||||
header[24:32] = struct.pack(">Q", virtual_size)
|
|
||||||
header[100:104] = struct.pack(">I", 104)
|
|
||||||
return bytes(header) + b"fake qcow2 content for controlled import test"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"url",
|
|
||||||
[
|
|
||||||
"http://repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2",
|
|
||||||
"https://evil.example/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2",
|
|
||||||
"https://repo.almalinux.org.evil.example/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2",
|
|
||||||
"https://user:pass@repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2",
|
|
||||||
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/../../etc/passwd",
|
|
||||||
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2?token=secret",
|
|
||||||
"https://cloud.debian.org/images/cloud/bookworm/latest/debian-13-generic-amd64.qcow2",
|
|
||||||
"https://cloud.debian.org/images/cloud/bookworm/20260801-2204/debian-12-generic-amd64-20260802-2205.qcow2",
|
|
||||||
"https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-arm64.qcow2",
|
|
||||||
"https://cloud-images.ubuntu.com/releases/noble/release/ubuntu-22.04-server-cloudimg-amd64.img",
|
|
||||||
"https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img",
|
|
||||||
"https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2",
|
|
||||||
"https://cloud.centos.org/centos/9-stream/aarch64/images/CentOS-Stream-GenericCloud-9-latest.aarch64.qcow2",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_import_url_rejects_nonofficial_and_ambiguous_targets(url):
|
|
||||||
with pytest.raises(TemplateError):
|
|
||||||
validate_image_url(url)
|
|
||||||
|
|
||||||
|
|
||||||
def test_verified_image_is_private_and_has_expected_virtual_size(tmp_path):
|
|
||||||
body = qcow2_fixture()
|
|
||||||
digest = hashlib.sha256(body).hexdigest()
|
|
||||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
|
||||||
return_value=fake_image_response(body)), patch(
|
|
||||||
"virsh_panel.cloud_templates._verify_qcow2_with_qemu"
|
|
||||||
) as checked:
|
|
||||||
downloaded, virtual_size, actual_sha256 = download_verified_image(
|
|
||||||
OFFICIAL, digest, tmp_path
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
assert downloaded.read_bytes() == body
|
|
||||||
assert virtual_size == 2 * 1024**3
|
|
||||||
assert actual_sha256 == digest
|
|
||||||
assert stat.S_IMODE(downloaded.stat().st_mode) == 0o600
|
|
||||||
checked.assert_called_once_with(downloaded, 2 * 1024**3)
|
|
||||||
finally:
|
|
||||||
downloaded.unlink()
|
|
||||||
|
|
||||||
|
|
||||||
def test_image_without_expected_hash_records_computed_digest(tmp_path):
|
|
||||||
body = qcow2_fixture()
|
|
||||||
digest = hashlib.sha256(body).hexdigest()
|
|
||||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
|
||||||
return_value=fake_image_response(body)), patch(
|
|
||||||
"virsh_panel.cloud_templates._verify_qcow2_with_qemu"
|
|
||||||
):
|
|
||||||
downloaded, virtual_size, actual_sha256 = download_verified_image(
|
|
||||||
OFFICIAL, "", tmp_path
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
assert virtual_size == 2 * 1024**3
|
|
||||||
assert actual_sha256 == digest
|
|
||||||
finally:
|
|
||||||
downloaded.unlink()
|
|
||||||
|
|
||||||
|
|
||||||
def test_failed_hash_never_leaves_unverified_template_file(tmp_path):
|
|
||||||
body = qcow2_fixture()
|
|
||||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
|
||||||
return_value=fake_image_response(body)):
|
|
||||||
with pytest.raises(TemplateError, match="SHA-256"):
|
|
||||||
download_verified_image(OFFICIAL, "0" * 64, tmp_path)
|
|
||||||
assert list(tmp_path.iterdir()) == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_limit_preserves_persistent_disk_reserve(tmp_path):
|
|
||||||
body = qcow2_fixture()
|
|
||||||
digest = hashlib.sha256(body).hexdigest()
|
|
||||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
|
||||||
return_value=fake_image_response(body)):
|
|
||||||
with pytest.raises(TemplateError, match="持久目录空间不足") as error:
|
|
||||||
download_verified_image(
|
|
||||||
OFFICIAL, digest, tmp_path, max_bytes=len(body) - 1
|
|
||||||
)
|
|
||||||
assert error.value.status == 507
|
|
||||||
assert list(tmp_path.iterdir()) == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_redirect_to_nonofficial_target_is_rejected_before_following():
|
|
||||||
handler = _RejectRedirects()
|
|
||||||
with pytest.raises(TemplateError, match="重定向"):
|
|
||||||
handler.redirect_request(None, None, 302, "Found", {},
|
|
||||||
"http://127.0.0.1/private")
|
|
||||||
|
|
||||||
|
|
||||||
def test_debian_official_redirect_downloads_same_object_from_public_mirror(
|
|
||||||
tmp_path):
|
|
||||||
mirror = DEBIAN_12.replace(
|
|
||||||
"cloud.debian.org", "chuangtzu.ftp.acc.umu.se"
|
|
||||||
)
|
|
||||||
body = qcow2_fixture()
|
|
||||||
first = FakeImageResponse(
|
|
||||||
status=302, headers={"Location": [mirror]}
|
|
||||||
)
|
|
||||||
final = FakeImageResponse(
|
|
||||||
body,
|
|
||||||
status=200,
|
|
||||||
headers={"Content-Length": [str(len(body))]},
|
|
||||||
)
|
|
||||||
connections = [SimpleNamespace(close=lambda: None) for _ in range(2)]
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_templates._resolve_global_addresses",
|
|
||||||
return_value=["2001:4860:4860::8888"],
|
|
||||||
), patch(
|
|
||||||
"virsh_panel.cloud_templates._open_pinned_response",
|
|
||||||
side_effect=[(connections[0], first), (connections[1], final)],
|
|
||||||
) as opened, patch(
|
|
||||||
"virsh_panel.cloud_templates._verify_qcow2_with_qemu"
|
|
||||||
):
|
|
||||||
downloaded, virtual_size, digest = download_verified_image(
|
|
||||||
DEBIAN_12, hashlib.sha256(body).hexdigest(), tmp_path
|
|
||||||
)
|
|
||||||
|
|
||||||
assert downloaded.read_bytes() == body
|
|
||||||
assert virtual_size == 2 * 1024**3
|
|
||||||
assert digest == hashlib.sha256(body).hexdigest()
|
|
||||||
assert opened.call_args_list[0].args[0].hostname == "cloud.debian.org"
|
|
||||||
assert opened.call_args_list[1].args[0].hostname == "chuangtzu.ftp.acc.umu.se"
|
|
||||||
assert opened.call_args_list[1].args[0].path == urlsplit(DEBIAN_12).path
|
|
||||||
downloaded.unlink()
|
|
||||||
|
|
||||||
|
|
||||||
def test_ubuntu_redirect_stays_on_official_host_and_same_release():
|
|
||||||
target = UBUNTU_24.replace(
|
|
||||||
"/releases/noble/release/", "/releases/24.04/release/"
|
|
||||||
)
|
|
||||||
first = FakeImageResponse(
|
|
||||||
status=302, headers={"Location": [UBUNTU_24]}
|
|
||||||
)
|
|
||||||
final = FakeImageResponse(b"image", status=200)
|
|
||||||
connections = [SimpleNamespace(close=lambda: None) for _ in range(2)]
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_templates._resolve_global_addresses",
|
|
||||||
return_value=["8.8.8.8"],
|
|
||||||
), patch(
|
|
||||||
"virsh_panel.cloud_templates._open_pinned_response",
|
|
||||||
side_effect=[(connections[0], first), (connections[1], final)],
|
|
||||||
):
|
|
||||||
connection, response = _open_image_response(target)
|
|
||||||
|
|
||||||
assert connection is connections[1]
|
|
||||||
assert response is final
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"target",
|
|
||||||
[
|
|
||||||
"http://127.0.0.1/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2",
|
|
||||||
"https://mirror.example/images/cloud/bookworm/latest/debian-13-generic-amd64.qcow2",
|
|
||||||
"https://user:password@mirror.example/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_debian_redirect_rejects_unsafe_or_different_object(target):
|
|
||||||
with pytest.raises(TemplateError, match="重定向目标不安全"):
|
|
||||||
_redirect_target(
|
|
||||||
DEBIAN_12, DEBIAN_12, target, ("Debian", "12")
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_ubuntu_redirect_rejects_other_host_or_release():
|
|
||||||
for target in (
|
|
||||||
UBUNTU_24.replace("cloud-images.ubuntu.com", "mirror.example"),
|
|
||||||
UBUNTU_24.replace("24.04", "22.04"),
|
|
||||||
):
|
|
||||||
with pytest.raises(TemplateError, match="重定向目标不安全"):
|
|
||||||
_redirect_target(
|
|
||||||
UBUNTU_24, UBUNTU_24, target, ("Ubuntu", "24.04")
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_image_download_dns_rejects_any_nonpublic_answer():
|
|
||||||
rows = [
|
|
||||||
(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "",
|
|
||||||
("8.8.8.8", 443)),
|
|
||||||
(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "",
|
|
||||||
("127.0.0.1", 443)),
|
|
||||||
]
|
|
||||||
with patch("virsh_panel.cloud_templates.socket.getaddrinfo", return_value=rows):
|
|
||||||
with pytest.raises(TemplateError, match="非公网"):
|
|
||||||
_resolve_global_addresses("mirror.example")
|
|
||||||
|
|
||||||
|
|
||||||
def test_image_redirect_count_is_bounded():
|
|
||||||
def redirect_response(*_args):
|
|
||||||
connection = SimpleNamespace(close=lambda: None)
|
|
||||||
response = FakeImageResponse(
|
|
||||||
status=302, headers={"Location": [UBUNTU_24]}
|
|
||||||
)
|
|
||||||
return connection, response
|
|
||||||
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_templates._resolve_global_addresses",
|
|
||||||
return_value=["8.8.8.8"],
|
|
||||||
), patch(
|
|
||||||
"virsh_panel.cloud_templates._open_pinned_response",
|
|
||||||
side_effect=redirect_response,
|
|
||||||
):
|
|
||||||
with pytest.raises(TemplateError, match="次数过多"):
|
|
||||||
_open_image_response(UBUNTU_24)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("response", "message"),
|
|
||||||
[
|
|
||||||
(FakeImageResponse(status=404), "异常状态"),
|
|
||||||
(FakeImageResponse(
|
|
||||||
qcow2_fixture(), headers={"Content-Encoding": ["gzip"]}
|
|
||||||
), "响应格式无效"),
|
|
||||||
(FakeImageResponse(
|
|
||||||
qcow2_fixture(),
|
|
||||||
headers={"Content-Length": [str(len(qcow2_fixture()) + 1)]},
|
|
||||||
), "内容不完整"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_image_download_rejects_unsafe_or_incomplete_http_response(
|
|
||||||
tmp_path, response, message):
|
|
||||||
connection = SimpleNamespace(close=lambda: None)
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_templates._open_image_response",
|
|
||||||
return_value=(connection, response),
|
|
||||||
):
|
|
||||||
with pytest.raises(TemplateError, match=message):
|
|
||||||
download_verified_image(OFFICIAL, "", tmp_path)
|
|
||||||
assert list(tmp_path.iterdir()) == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_external_data_file_feature_is_rejected(tmp_path):
|
|
||||||
body = bytearray(qcow2_fixture())
|
|
||||||
body[72:80] = struct.pack(">Q", 4)
|
|
||||||
digest = hashlib.sha256(body).hexdigest()
|
|
||||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
|
||||||
return_value=fake_image_response(body)):
|
|
||||||
with pytest.raises(TemplateError, match="外部依赖"):
|
|
||||||
download_verified_image(OFFICIAL, digest, tmp_path)
|
|
||||||
assert list(tmp_path.iterdir()) == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_qcow2_full_integrity_check_rejects_corrupt_image(tmp_path):
|
|
||||||
image = tmp_path / "corrupt.qcow2"
|
|
||||||
image.write_bytes(qcow2_fixture())
|
|
||||||
info = [{"format": "qcow2", "virtual-size": 2 * 1024**3}]
|
|
||||||
with patch(
|
|
||||||
"virsh_panel.cloud_templates._qemu_img_json",
|
|
||||||
side_effect=[info, TemplateError("云镜像完整性检查失败", 422)],
|
|
||||||
) as inspect:
|
|
||||||
with pytest.raises(TemplateError, match="完整性检查"):
|
|
||||||
verify_qcow2_image(image)
|
|
||||||
assert inspect.call_count == 2
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"info",
|
|
||||||
[
|
|
||||||
[{"format": "raw", "virtual-size": 2 * 1024**3}],
|
|
||||||
[{"format": "qcow2", "virtual-size": 2 * 1024**3,
|
|
||||||
"backing-filename": "/private/base.qcow2"}],
|
|
||||||
[{"format": "qcow2", "virtual-size": 2 * 1024**3,
|
|
||||||
"format-specific": {"data": {"data-file": "/private/data"}}}],
|
|
||||||
[{"format": "qcow2", "virtual-size": 2 * 1024**3},
|
|
||||||
{"format": "qcow2", "virtual-size": 2 * 1024**3}],
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_qcow2_full_integrity_check_rejects_dependencies_and_wrong_format(
|
|
||||||
tmp_path, info):
|
|
||||||
image = tmp_path / "unsafe.qcow2"
|
|
||||||
image.write_bytes(qcow2_fixture())
|
|
||||||
with patch("virsh_panel.cloud_templates._qemu_img_json",
|
|
||||||
return_value=info) as inspect:
|
|
||||||
with pytest.raises(TemplateError, match="外部依赖"):
|
|
||||||
verify_qcow2_image(image)
|
|
||||||
inspect.assert_called_once()
|
|
||||||
|
|
||||||
|
|
||||||
def test_template_catalog_is_private_and_scoped_to_host_and_disk_pool(tmp_path):
|
|
||||||
data_dir = tmp_path / "data"
|
|
||||||
data_dir.mkdir(mode=0o700)
|
|
||||||
store = TemplateStore(data_dir)
|
|
||||||
profile = {
|
|
||||||
"host": "hypervisor.example", "port": 2222,
|
|
||||||
"fingerprint": "SHA256:fixture", "storage_pool": "/home/kvm",
|
|
||||||
}
|
|
||||||
scope = template_scope(profile)
|
|
||||||
other_scope = template_scope({**profile, "storage_pool": "/other/pool"})
|
|
||||||
record = {
|
|
||||||
"id": "alma10-example", "name": "AlmaLinux 10", "sha256": "a" * 64,
|
|
||||||
"volume_name": "alma10-cloud.qcow2", "virtual_size": 2 * 1024**3,
|
|
||||||
}
|
|
||||||
assert store.add(scope, record) == record
|
|
||||||
assert store.list(scope) == [record]
|
|
||||||
assert store.list(other_scope) == []
|
|
||||||
assert TemplateStore(data_dir).get(scope, record["id"]) == record
|
|
||||||
assert stat.S_IMODE(store.file.stat().st_mode) == 0o600
|
|
||||||
with pytest.raises(TemplateError):
|
|
||||||
store.get(other_scope, record["id"])
|
|
||||||
with pytest.raises(TemplateError):
|
|
||||||
store.add(scope, record)
|
|
||||||
|
|
||||||
|
|
||||||
def test_template_rename_preserves_image_record_and_host_scope(tmp_path):
|
|
||||||
data_dir = tmp_path / "data"
|
|
||||||
data_dir.mkdir(mode=0o700)
|
|
||||||
store = TemplateStore(data_dir)
|
|
||||||
scope = "a" * 64
|
|
||||||
other_scope = "b" * 64
|
|
||||||
template_id = template_id_for(scope, "c" * 64, "9")
|
|
||||||
record = {
|
|
||||||
"id": template_id, "name": "AlmaLinux 9", "os_family": "AlmaLinux",
|
|
||||||
"os_version": "9", "sha256": "c" * 64,
|
|
||||||
"volume_name": template_volume_name(template_id), "virtual_size": 2 * 1024**3,
|
|
||||||
}
|
|
||||||
store.add(scope, record)
|
|
||||||
store.add(scope, {**record, "id": "other", "name": "Another name"})
|
|
||||||
store.add(other_scope, {**record, "name": "Other host"})
|
|
||||||
|
|
||||||
with pytest.raises(TemplateError) as inaccessible:
|
|
||||||
store.rename(other_scope, "other", "Missing")
|
|
||||||
assert inaccessible.value.status == 404
|
|
||||||
with pytest.raises(TemplateError) as duplicate:
|
|
||||||
store.rename(scope, template_id, "another NAME")
|
|
||||||
assert duplicate.value.status == 409
|
|
||||||
for invalid in ("", "x" * 65, "bad\x7fentry"):
|
|
||||||
with pytest.raises(TemplateError):
|
|
||||||
store.rename(scope, template_id, invalid)
|
|
||||||
|
|
||||||
updated = store.rename(scope, template_id, " 新名称 ")
|
|
||||||
assert updated == {**record, "name": "新名称"}
|
|
||||||
assert TemplateStore(data_dir).get(scope, template_id) == updated
|
|
||||||
assert store.get(other_scope, template_id)["name"] == "Other host"
|
|
||||||
assert stat.S_IMODE(store.file.stat().st_mode) == 0o600
|
|
||||||
@ -1,356 +0,0 @@
|
|||||||
"""Offline contracts for the code-only privileged update worker."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import unittest
|
|
||||||
from contextlib import nullcontext
|
|
||||||
from pathlib import Path
|
|
||||||
from types import SimpleNamespace
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from virsh_panel import update_worker
|
|
||||||
|
|
||||||
|
|
||||||
class FakeStore:
|
|
||||||
def __init__(self, job_id: str | None):
|
|
||||||
self.job_id = job_id
|
|
||||||
|
|
||||||
def get_value(self, name: str) -> str | None:
|
|
||||||
if name == "update.active_job":
|
|
||||||
return self.job_id
|
|
||||||
raise AssertionError(name)
|
|
||||||
|
|
||||||
|
|
||||||
class FakeUpdater:
|
|
||||||
def __init__(self, job: dict | None):
|
|
||||||
self.job = job
|
|
||||||
self.repo_dir = Path("/tmp")
|
|
||||||
self.failed: list[tuple[str, str]] = []
|
|
||||||
|
|
||||||
def get_job(self, _job_id: str) -> dict | None:
|
|
||||||
return self.job
|
|
||||||
|
|
||||||
def _checkout_head(self, *, require_clean: bool):
|
|
||||||
if not require_clean:
|
|
||||||
raise AssertionError("recovery must require a clean checkout")
|
|
||||||
return "a" * 40
|
|
||||||
|
|
||||||
def fail_job(self, job_id: str, message: str) -> None:
|
|
||||||
self.failed.append((job_id, message))
|
|
||||||
if self.job is not None:
|
|
||||||
self.job["status"] = "failed"
|
|
||||||
|
|
||||||
|
|
||||||
class UpdateWorkerTests(unittest.TestCase):
|
|
||||||
def test_service_consumers_start_only_after_engine_is_healthy(self):
|
|
||||||
calls = []
|
|
||||||
deployment = object.__new__(update_worker.DeploymentTransaction)
|
|
||||||
deployment.repo_dir = Path("/tmp")
|
|
||||||
deployment.config = {"service_secret": "unused"}
|
|
||||||
deployment.units = (
|
|
||||||
"virt-cloud-engine.service",
|
|
||||||
"virt-cloud-worker.service",
|
|
||||||
"virt-cloud-lifecycle.service",
|
|
||||||
"virt-cloud-policy-worker@1.service",
|
|
||||||
"virt-cloud-policy-worker@2.service",
|
|
||||||
"php-fpm.service",
|
|
||||||
)
|
|
||||||
|
|
||||||
def start(units, *, message):
|
|
||||||
calls.append(("start", units, message))
|
|
||||||
|
|
||||||
def wait(_repo_dir, units):
|
|
||||||
calls.append(("active", units))
|
|
||||||
|
|
||||||
with patch.object(deployment, "_start_services", side_effect=start), \
|
|
||||||
patch.object(update_worker, "_wait_units", side_effect=wait), \
|
|
||||||
patch.object(update_worker, "_wait_engine_health",
|
|
||||||
side_effect=lambda config: calls.append(("health", config))):
|
|
||||||
deployment.start_and_check()
|
|
||||||
|
|
||||||
engine = ("virt-cloud-engine.service",)
|
|
||||||
consumers = (
|
|
||||||
"virt-cloud-worker.service",
|
|
||||||
"virt-cloud-lifecycle.service",
|
|
||||||
"virt-cloud-policy-worker@1.service",
|
|
||||||
"virt-cloud-policy-worker@2.service",
|
|
||||||
)
|
|
||||||
web = ("php-fpm.service",)
|
|
||||||
self.assertEqual(calls[0][:2], ("start", web))
|
|
||||||
self.assertEqual(calls[1], ("active", web))
|
|
||||||
self.assertEqual(calls[2][:2], ("start", engine))
|
|
||||||
self.assertEqual(calls[3], ("active", engine))
|
|
||||||
self.assertEqual(calls[4], ("health", deployment.config))
|
|
||||||
self.assertEqual(calls[5][:2], ("start", consumers))
|
|
||||||
self.assertEqual(calls[6], ("active", consumers))
|
|
||||||
self.assertIn("PHP-FPM", calls[0][2])
|
|
||||||
self.assertIn("私有引擎", calls[2][2])
|
|
||||||
self.assertIn("后台服务", calls[5][2])
|
|
||||||
|
|
||||||
def test_engine_health_failure_restores_gated_consumers_before_raising(self):
|
|
||||||
calls = []
|
|
||||||
deployment = object.__new__(update_worker.DeploymentTransaction)
|
|
||||||
deployment.repo_dir = Path("/tmp")
|
|
||||||
deployment.config = {"service_secret": "unused"}
|
|
||||||
deployment.units = (
|
|
||||||
"virt-cloud-engine.service",
|
|
||||||
"virt-cloud-worker.service",
|
|
||||||
"virt-cloud-lifecycle.service",
|
|
||||||
"virt-cloud-policy-worker@1.service",
|
|
||||||
"virt-cloud-policy-worker@2.service",
|
|
||||||
"php-fpm.service",
|
|
||||||
)
|
|
||||||
|
|
||||||
def start(units, *, message):
|
|
||||||
calls.append(("start", units, message))
|
|
||||||
|
|
||||||
def unhealthy(config):
|
|
||||||
calls.append(("health", config))
|
|
||||||
raise update_worker.UpdateError("引擎未就绪", 500)
|
|
||||||
|
|
||||||
with patch.object(deployment, "_start_services", side_effect=start), \
|
|
||||||
patch.object(update_worker, "_wait_units",
|
|
||||||
side_effect=lambda _repo, units: calls.append(("active", units))), \
|
|
||||||
patch.object(update_worker, "_wait_engine_health",
|
|
||||||
side_effect=unhealthy):
|
|
||||||
with self.assertRaisesRegex(update_worker.UpdateError, "引擎未就绪"):
|
|
||||||
deployment.start_and_check()
|
|
||||||
|
|
||||||
self.assertEqual(calls[0][:2], ("start", ("php-fpm.service",)))
|
|
||||||
self.assertEqual(calls[2][:2], ("start", ("virt-cloud-engine.service",)))
|
|
||||||
self.assertEqual(calls[4], ("health", deployment.config))
|
|
||||||
self.assertEqual(calls[5][:2], ("start", (
|
|
||||||
"virt-cloud-worker.service",
|
|
||||||
"virt-cloud-lifecycle.service",
|
|
||||||
"virt-cloud-policy-worker@1.service",
|
|
||||||
"virt-cloud-policy-worker@2.service",
|
|
||||||
)))
|
|
||||||
|
|
||||||
def test_start_services_uses_only_the_explicit_unit_group(self):
|
|
||||||
deployment = object.__new__(update_worker.DeploymentTransaction)
|
|
||||||
deployment.repo_dir = Path("/tmp")
|
|
||||||
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
|
|
||||||
patch.object(update_worker, "_run") as run:
|
|
||||||
deployment._start_services(
|
|
||||||
("virt-cloud-engine.service",), message="engine start failed"
|
|
||||||
)
|
|
||||||
|
|
||||||
run.assert_called_once_with(
|
|
||||||
["/usr/bin/systemctl", "start", "virt-cloud-engine.service"],
|
|
||||||
cwd=Path("/tmp"),
|
|
||||||
timeout=180,
|
|
||||||
message="engine start failed",
|
|
||||||
environment={"PATH": update_worker._EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_running_job_is_marked_failed_once(self):
|
|
||||||
store = FakeStore("job-1234567890123456")
|
|
||||||
updater = FakeUpdater({"status": "running"})
|
|
||||||
self.assertTrue(update_worker.finalize_incomplete_job(updater, store))
|
|
||||||
self.assertEqual(updater.job["status"], "failed")
|
|
||||||
self.assertEqual(len(updater.failed), 1)
|
|
||||||
self.assertIn("日志", updater.failed[0][1])
|
|
||||||
self.assertFalse(update_worker.finalize_incomplete_job(updater, store))
|
|
||||||
self.assertEqual(len(updater.failed), 1)
|
|
||||||
|
|
||||||
def test_queued_job_is_marked_failed(self):
|
|
||||||
store = FakeStore("job-1234567890123456")
|
|
||||||
updater = FakeUpdater({"status": "queued"})
|
|
||||||
self.assertTrue(update_worker.finalize_incomplete_job(updater, store))
|
|
||||||
self.assertEqual(updater.job["status"], "failed")
|
|
||||||
|
|
||||||
def test_terminal_or_missing_job_is_not_modified(self):
|
|
||||||
for status in ("succeeded", "failed", "blocked"):
|
|
||||||
with self.subTest(status=status):
|
|
||||||
updater = FakeUpdater({"status": status})
|
|
||||||
self.assertFalse(update_worker.finalize_incomplete_job(
|
|
||||||
updater, FakeStore("job-1234567890123456")))
|
|
||||||
self.assertEqual(updater.failed, [])
|
|
||||||
self.assertFalse(update_worker.finalize_incomplete_job(
|
|
||||||
FakeUpdater(None), FakeStore("job-1234567890123456")))
|
|
||||||
self.assertFalse(update_worker.finalize_incomplete_job(
|
|
||||||
FakeUpdater({"status": "running"}), FakeStore(None)))
|
|
||||||
|
|
||||||
def test_cli_finalize_mode_recovers_services_only_after_schema_checkpoint(self):
|
|
||||||
store = FakeStore("job-1234567890123456")
|
|
||||||
updater = FakeUpdater({
|
|
||||||
"status": "running",
|
|
||||||
"deployment_state": "schema_ready",
|
|
||||||
"new_commit": "a" * 40,
|
|
||||||
})
|
|
||||||
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
|
|
||||||
patch.object(update_worker, "_managed_service_units", return_value=(
|
|
||||||
"virt-cloud-engine.service", "php-fpm.service")), \
|
|
||||||
patch.object(update_worker.DeploymentTransaction, "start_and_check") as start:
|
|
||||||
self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0)
|
|
||||||
self.assertEqual(updater.job["status"], "failed")
|
|
||||||
start.assert_called_once_with()
|
|
||||||
|
|
||||||
def test_cli_finalize_mode_leaves_services_stopped_without_safe_checkpoint(self):
|
|
||||||
for state in ("code_switched", "schema_migrating"):
|
|
||||||
with self.subTest(state=state):
|
|
||||||
store = FakeStore("job-1234567890123456")
|
|
||||||
updater = FakeUpdater({"status": "running", "deployment_state": state})
|
|
||||||
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
|
|
||||||
patch.object(update_worker.DeploymentTransaction, "start_and_check") as start:
|
|
||||||
self.assertEqual(update_worker.main(["--finalize-incomplete"]), 0)
|
|
||||||
start.assert_not_called()
|
|
||||||
|
|
||||||
def test_cli_recover_blocked_runs_forward_migration_and_health_sequence(self):
|
|
||||||
calls = []
|
|
||||||
|
|
||||||
class RecoveryUpdater(FakeUpdater):
|
|
||||||
def _apply_lock(self):
|
|
||||||
calls.append("lock")
|
|
||||||
return nullcontext()
|
|
||||||
|
|
||||||
def begin_blocked_recovery(self):
|
|
||||||
calls.append("begin")
|
|
||||||
return {"job_id": "job-1234567890123456"}
|
|
||||||
|
|
||||||
def recovery_checkpoint(self, job_id, state, phase):
|
|
||||||
calls.append(("checkpoint", job_id, state, phase))
|
|
||||||
|
|
||||||
def complete_blocked_recovery(self, job_id):
|
|
||||||
calls.append(("complete", job_id))
|
|
||||||
|
|
||||||
class RecoveryDeployment:
|
|
||||||
def __init__(self, repo_dir, config, *, recovery=False):
|
|
||||||
calls.append(("deployment", repo_dir, config, recovery))
|
|
||||||
|
|
||||||
def quiesce(self):
|
|
||||||
calls.append("quiesce")
|
|
||||||
|
|
||||||
def activate(self, mark_schema_ready):
|
|
||||||
calls.append("cloudInit")
|
|
||||||
mark_schema_ready()
|
|
||||||
calls.append("services-healthy")
|
|
||||||
|
|
||||||
store = FakeStore("job-1234567890123456")
|
|
||||||
updater = RecoveryUpdater({"status": "blocked"})
|
|
||||||
with patch.object(update_worker, "_load_updater", return_value=(updater, store, {})), \
|
|
||||||
patch.object(update_worker, "DeploymentTransaction", RecoveryDeployment):
|
|
||||||
self.assertEqual(update_worker.main(["--recover-blocked"]), 0)
|
|
||||||
|
|
||||||
self.assertEqual(calls[0][0], "deployment")
|
|
||||||
self.assertTrue(calls[0][3])
|
|
||||||
self.assertEqual(calls[1:4], ["lock", "begin", "quiesce"])
|
|
||||||
self.assertEqual(calls[4][:3], (
|
|
||||||
"checkpoint", "job-1234567890123456", "schema_migrating"))
|
|
||||||
self.assertEqual(calls[5], "cloudInit")
|
|
||||||
self.assertEqual(calls[6][:3], (
|
|
||||||
"checkpoint", "job-1234567890123456", "schema_ready"))
|
|
||||||
self.assertEqual(calls[7:], [
|
|
||||||
"services-healthy", ("complete", "job-1234567890123456")])
|
|
||||||
|
|
||||||
def test_update_worker_is_a_separate_hardened_oneshot(self):
|
|
||||||
deploy = Path(__file__).resolve().parents[2] / "deploy"
|
|
||||||
engine_unit = (deploy / "virt-cloud-engine.service").read_text(encoding="utf-8")
|
|
||||||
update_unit = (deploy / "virt-cloud-update.service").read_text(encoding="utf-8")
|
|
||||||
self.assertIn("virsh_panel.internal:create_engine", engine_unit)
|
|
||||||
self.assertNotIn("update_worker", engine_unit)
|
|
||||||
for value in (
|
|
||||||
"Type=oneshot",
|
|
||||||
"User=root",
|
|
||||||
"WorkingDirectory=/home/virt-cloud/engine",
|
|
||||||
"CLOUD_ENGINE_CONFIG=/home/virt-cloud/runtime/engine.json",
|
|
||||||
"ExecStart=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker",
|
|
||||||
"ExecStopPost=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker --finalize-incomplete",
|
|
||||||
"TimeoutStartSec=45min",
|
|
||||||
"UMask=0077",
|
|
||||||
"NoNewPrivileges=true",
|
|
||||||
"PrivateTmp=true",
|
|
||||||
"ProtectSystem=strict",
|
|
||||||
"ProtectHome=false",
|
|
||||||
"ReadWritePaths=/home/virt-cloud",
|
|
||||||
"LimitCORE=0",
|
|
||||||
):
|
|
||||||
self.assertIn(value, update_unit)
|
|
||||||
self.assertNotIn("[Install]", update_unit)
|
|
||||||
self.assertNotIn("/home/virsh-panel", update_unit)
|
|
||||||
self.assertNotIn("virsh-panel.service", update_unit)
|
|
||||||
self.assertNotIn("/etc/systemd/system", update_unit)
|
|
||||||
self.assertNotIn("ExecStopPost=-/usr/bin/systemctl start", update_unit)
|
|
||||||
|
|
||||||
def test_online_worker_is_code_only_and_uses_configured_fpm(self):
|
|
||||||
source = Path(update_worker.__file__).read_text(encoding="utf-8")
|
|
||||||
for forbidden in (
|
|
||||||
"daemon-reload",
|
|
||||||
"/etc/systemd/system",
|
|
||||||
'"pip", "install"',
|
|
||||||
'_executable("composer")',
|
|
||||||
'[composer, "install"',
|
|
||||||
"update-staging",
|
|
||||||
"web/vendor",
|
|
||||||
"tempfile.",
|
|
||||||
"def _git_file(",
|
|
||||||
"prepare=",
|
|
||||||
):
|
|
||||||
self.assertNotIn(forbidden, source)
|
|
||||||
self.assertIn("mark_schema_ready()", source)
|
|
||||||
units = update_worker._service_units({"update_php_fpm_unit": "php85-php-fpm.service"})
|
|
||||||
self.assertIn("php85-php-fpm.service", units)
|
|
||||||
self.assertNotIn("php-fpm.service", units)
|
|
||||||
with self.assertRaisesRegex(Exception, "服务名"):
|
|
||||||
update_worker._service_units({"update_php_fpm_unit": "bad;service"})
|
|
||||||
|
|
||||||
def test_normal_update_preserves_running_installed_units_and_skips_missing_optional(self):
|
|
||||||
states = {
|
|
||||||
"virt-cloud-engine.service": ("loaded", "active"),
|
|
||||||
"virt-cloud-worker.service": ("loaded", "active"),
|
|
||||||
"virt-cloud-lifecycle.service": ("not-found", "inactive"),
|
|
||||||
"virt-cloud-policy-worker@1.service": ("not-found", "inactive"),
|
|
||||||
"virt-cloud-policy-worker@2.service": ("not-found", "inactive"),
|
|
||||||
"php-fpm.service": ("loaded", "active"),
|
|
||||||
}
|
|
||||||
|
|
||||||
def systemctl(argv, **_kwargs):
|
|
||||||
load, active = states[argv[-1]]
|
|
||||||
return SimpleNamespace(
|
|
||||||
returncode=0,
|
|
||||||
stdout=f"LoadState={load}\nActiveState={active}\n",
|
|
||||||
)
|
|
||||||
|
|
||||||
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
|
|
||||||
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
|
|
||||||
units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=False)
|
|
||||||
self.assertEqual(units, (
|
|
||||||
"virt-cloud-engine.service",
|
|
||||||
"virt-cloud-worker.service",
|
|
||||||
"php-fpm.service",
|
|
||||||
))
|
|
||||||
|
|
||||||
def test_recovery_starts_installed_units_even_when_they_are_stopped(self):
|
|
||||||
states = {
|
|
||||||
unit: ("loaded", "inactive") for unit in update_worker._service_units({})
|
|
||||||
}
|
|
||||||
|
|
||||||
def systemctl(argv, **_kwargs):
|
|
||||||
load, active = states[argv[-1]]
|
|
||||||
return SimpleNamespace(
|
|
||||||
returncode=0,
|
|
||||||
stdout=f"LoadState={load}\nActiveState={active}\n",
|
|
||||||
)
|
|
||||||
|
|
||||||
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
|
|
||||||
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
|
|
||||||
units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=True)
|
|
||||||
self.assertEqual(units, update_worker._service_units({}))
|
|
||||||
|
|
||||||
def test_update_refuses_before_stopping_when_required_service_is_missing(self):
|
|
||||||
def systemctl(argv, **_kwargs):
|
|
||||||
unit = argv[-1]
|
|
||||||
load = "not-found" if unit == "php-fpm.service" else "loaded"
|
|
||||||
return SimpleNamespace(
|
|
||||||
returncode=0,
|
|
||||||
stdout=f"LoadState={load}\nActiveState=active\n",
|
|
||||||
)
|
|
||||||
|
|
||||||
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
|
|
||||||
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
|
|
||||||
with self.assertRaisesRegex(Exception, "php-fpm.service 未安装"):
|
|
||||||
update_worker._managed_service_units(Path("/tmp"), {}, recovery=False)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@ -1,614 +0,0 @@
|
|||||||
"""Offline checks for the private-repository update safety boundary."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import stat
|
|
||||||
import subprocess
|
|
||||||
import tempfile
|
|
||||||
import time
|
|
||||||
import unittest
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from virsh_panel.updater import BRANCH, REPOSITORY_URL, UpdateError, Updater
|
|
||||||
|
|
||||||
|
|
||||||
class MemoryStore:
|
|
||||||
def __init__(self):
|
|
||||||
self.secrets: dict[str, str] = {}
|
|
||||||
self.values: dict[str, str] = {}
|
|
||||||
self.deployment_states: list[str] = []
|
|
||||||
|
|
||||||
def get_secret(self, name):
|
|
||||||
return self.secrets.get(name)
|
|
||||||
|
|
||||||
def set_secret(self, name, value):
|
|
||||||
self.secrets[name] = value
|
|
||||||
|
|
||||||
def delete_secret(self, name):
|
|
||||||
self.secrets.pop(name, None)
|
|
||||||
|
|
||||||
def get_value(self, name):
|
|
||||||
return self.values.get(name)
|
|
||||||
|
|
||||||
def set_value(self, name, value):
|
|
||||||
self.values[name] = value
|
|
||||||
if name.startswith("update.job."):
|
|
||||||
payload = json.loads(value)
|
|
||||||
state = payload.get("deployment_state")
|
|
||||||
if state and (not self.deployment_states or self.deployment_states[-1] != state):
|
|
||||||
self.deployment_states.append(state)
|
|
||||||
|
|
||||||
|
|
||||||
def git(path: Path, *args: str) -> str:
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", *args], cwd=path, capture_output=True, text=True, check=True,
|
|
||||||
)
|
|
||||||
return result.stdout.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def make_repo(tmp_path: Path) -> Path:
|
|
||||||
source = tmp_path / "source"
|
|
||||||
source.mkdir()
|
|
||||||
git(source, "init", "-b", BRANCH)
|
|
||||||
git(source, "config", "user.name", "Updater Test")
|
|
||||||
git(source, "config", "user.email", "updater@example.test")
|
|
||||||
(source / "README.md").write_text("old\n", encoding="utf-8")
|
|
||||||
(source / "engine").mkdir()
|
|
||||||
(source / "engine" / "requirements.txt").write_text("example-package==1.0\n", encoding="utf-8")
|
|
||||||
(source / ".gitignore").write_text(
|
|
||||||
"engine/.venv/\nruntime/\nweb/vendor/\nweb/storage/ticket-attachments/\n",
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
git(source, "add", "README.md", "engine/requirements.txt", ".gitignore")
|
|
||||||
git(source, "commit", "-m", "initial")
|
|
||||||
checkout = tmp_path / "panel"
|
|
||||||
subprocess.run(
|
|
||||||
["git", "clone", "--branch", BRANCH, str(source), str(checkout)],
|
|
||||||
capture_output=True, text=True, check=True,
|
|
||||||
)
|
|
||||||
git(checkout, "remote", "set-url", "origin", REPOSITORY_URL)
|
|
||||||
return checkout
|
|
||||||
|
|
||||||
|
|
||||||
def advance_source(source: Path, *, filename: str = "README.md") -> str:
|
|
||||||
(source / filename).parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
(source / filename).write_text("new\n", encoding="utf-8")
|
|
||||||
# Protected runtime paths are normally ignored; forcing them lets the
|
|
||||||
# updater test prove that a malicious target commit is still rejected.
|
|
||||||
git(source, "add", "-f", filename)
|
|
||||||
git(source, "commit", "-m", "next")
|
|
||||||
return git(source, "rev-parse", "HEAD")
|
|
||||||
|
|
||||||
|
|
||||||
def local_fetch(updater: Updater, source: Path):
|
|
||||||
def fetch() -> str:
|
|
||||||
git(updater.repo_dir, "fetch", "--no-tags", str(source), f"refs/heads/{BRANCH}")
|
|
||||||
return git(updater.repo_dir, "rev-parse", "FETCH_HEAD")
|
|
||||||
|
|
||||||
updater._fetch_remote = fetch
|
|
||||||
|
|
||||||
|
|
||||||
class UpdaterTests(unittest.TestCase):
|
|
||||||
def setUp(self):
|
|
||||||
self.temp = tempfile.TemporaryDirectory(prefix="virsh-panel-updater-test-")
|
|
||||||
self.addCleanup(self.temp.cleanup)
|
|
||||||
self.tmp_path = Path(self.temp.name)
|
|
||||||
|
|
||||||
def test_settings_and_token_never_reveal_secret(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
store = MemoryStore()
|
|
||||||
updater = Updater(checkout, store)
|
|
||||||
updater.set_token("a-test-access-token-123")
|
|
||||||
settings = updater.settings()
|
|
||||||
self.assertEqual(settings["repository"], REPOSITORY_URL)
|
|
||||||
self.assertTrue(settings["token_configured"])
|
|
||||||
self.assertTrue(settings["checkout_ready"])
|
|
||||||
self.assertIsNone(settings["active_job_id"])
|
|
||||||
self.assertIsNone(settings["last_job_id"])
|
|
||||||
self.assertNotIn("a-test-access-token-123", repr(settings))
|
|
||||||
with updater._git_auth() as env:
|
|
||||||
self.assertEqual(env["VIRT_CLOUD_GIT_TOKEN"], "a-test-access-token-123")
|
|
||||||
self.assertNotIn("a-test-access-token-123", Path(env["GIT_ASKPASS"]).read_text())
|
|
||||||
updater.clear_token()
|
|
||||||
self.assertFalse(updater.settings()["token_configured"])
|
|
||||||
|
|
||||||
def test_check_is_read_only_and_reports_new_version(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
current = git(checkout, "rev-parse", "HEAD")
|
|
||||||
updater._remote_head = lambda: current
|
|
||||||
self.assertEqual(updater.check()["status"], "current")
|
|
||||||
updater._remote_head = lambda: "a" * 40
|
|
||||||
self.assertEqual(updater.check()["status"], "update_available")
|
|
||||||
self.assertEqual(git(checkout, "rev-parse", "HEAD"), current)
|
|
||||||
|
|
||||||
def test_zip_or_dirty_checkout_is_rejected(self):
|
|
||||||
plain = self.tmp_path / "plain"
|
|
||||||
plain.mkdir()
|
|
||||||
with self.assertRaisesRegex(UpdateError, "Git"):
|
|
||||||
Updater(plain, MemoryStore()).check()
|
|
||||||
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
(checkout / "README.md").write_text("user changes\n", encoding="utf-8")
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
with self.assertRaisesRegex(UpdateError, "未提交"):
|
|
||||||
updater.create_job()
|
|
||||||
with self.assertRaisesRegex(UpdateError, "未提交"):
|
|
||||||
updater.apply()
|
|
||||||
|
|
||||||
def test_project_ignores_ticket_attachment_runtime_data(self):
|
|
||||||
project_root = Path(__file__).resolve().parents[2]
|
|
||||||
self.assertIn(
|
|
||||||
"/web/storage/ticket-attachments/",
|
|
||||||
(project_root / ".gitignore").read_text(encoding="utf-8"),
|
|
||||||
)
|
|
||||||
self.assertIn(
|
|
||||||
"storage/ticket-attachments/",
|
|
||||||
(project_root / "web" / ".gitignore").read_text(encoding="utf-8"),
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_ignored_ticket_attachments_do_not_make_checkout_dirty(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
attachment = checkout / "web" / "storage" / "ticket-attachments" / "ticket-image.png"
|
|
||||||
attachment.parent.mkdir(parents=True)
|
|
||||||
attachment.write_bytes(b"private runtime image")
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
updater._remote_head = lambda: git(checkout, "rev-parse", "HEAD")
|
|
||||||
|
|
||||||
self.assertTrue(updater.settings()["checkout_ready"])
|
|
||||||
self.assertEqual(updater.check()["status"], "current")
|
|
||||||
self.assertEqual(git(checkout, "status", "--porcelain=v1", "--untracked-files=all"), "")
|
|
||||||
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
target = advance_source(source)
|
|
||||||
local_fetch(updater, source)
|
|
||||||
applied = updater.apply()
|
|
||||||
self.assertEqual(applied["new_commit"], target)
|
|
||||||
self.assertEqual(attachment.read_bytes(), b"private runtime image")
|
|
||||||
|
|
||||||
def test_target_commit_cannot_vendor_ticket_attachments(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
target = source / "web" / "storage" / "ticket-attachments" / "bundled.png"
|
|
||||||
target.parent.mkdir(parents=True)
|
|
||||||
target.write_bytes(b"must remain runtime-only")
|
|
||||||
git(source, "add", "-f", str(target.relative_to(source)))
|
|
||||||
git(source, "commit", "-m", "attempt to vendor ticket image")
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
local_fetch(updater, source)
|
|
||||||
|
|
||||||
with self.assertRaisesRegex(UpdateError, "运行数据"):
|
|
||||||
updater.apply()
|
|
||||||
|
|
||||||
def test_fast_forward_job_is_persisted_and_calls_restart(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
old = git(checkout, "rev-parse", "HEAD")
|
|
||||||
target = advance_source(source)
|
|
||||||
store = MemoryStore()
|
|
||||||
updater = Updater(checkout, store)
|
|
||||||
updater.set_token("a-test-access-token-123")
|
|
||||||
local_fetch(updater, source)
|
|
||||||
queued = updater.create_job()
|
|
||||||
self.assertEqual(updater.settings()["active_job_id"], queued["job_id"])
|
|
||||||
restarted = []
|
|
||||||
|
|
||||||
def restart(mark_schema_ready):
|
|
||||||
restarted.append(True)
|
|
||||||
mark_schema_ready()
|
|
||||||
|
|
||||||
result = updater.run_pending_job(
|
|
||||||
before_switch=lambda: None,
|
|
||||||
restart=restart,
|
|
||||||
)
|
|
||||||
self.assertEqual(result["status"], "succeeded")
|
|
||||||
self.assertEqual(result["old_commit"], old)
|
|
||||||
self.assertEqual(result["new_commit"], target)
|
|
||||||
self.assertTrue(result["rollback_ref"].startswith("refs/virt-cloud/rollback/"))
|
|
||||||
self.assertEqual(updater.get_job(queued["job_id"]), result)
|
|
||||||
self.assertEqual(updater.settings()["last_job_id"], queued["job_id"])
|
|
||||||
self.assertIsNone(updater.settings()["active_job_id"])
|
|
||||||
self.assertEqual(restarted, [True])
|
|
||||||
self.assertEqual(store.deployment_states, [
|
|
||||||
"preparing",
|
|
||||||
"quiescing",
|
|
||||||
"services_stopped",
|
|
||||||
"code_switched",
|
|
||||||
"schema_migrating",
|
|
||||||
"schema_ready",
|
|
||||||
"services_ready",
|
|
||||||
])
|
|
||||||
self.assertEqual(result["deployment_state"], "services_ready")
|
|
||||||
self.assertEqual(git(checkout, "rev-parse", "HEAD"), target)
|
|
||||||
self.assertEqual(git(checkout, "rev-parse", result["rollback_ref"]), old)
|
|
||||||
|
|
||||||
def test_apply_restores_tracked_modes_without_exposing_ignored_secrets(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
nested = source / "web" / "app" / "NewFeature.php"
|
|
||||||
nested.parent.mkdir(parents=True)
|
|
||||||
nested.write_text("<?php\n", encoding="utf-8")
|
|
||||||
executable = source / "tools" / "task"
|
|
||||||
executable.parent.mkdir()
|
|
||||||
executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
|
||||||
executable.chmod(0o755)
|
|
||||||
git(source, "add", "web/app/NewFeature.php", "tools/task")
|
|
||||||
git(source, "commit", "-m", "add application files")
|
|
||||||
|
|
||||||
private = checkout / "runtime" / "engine" / "master.key"
|
|
||||||
private.parent.mkdir(parents=True)
|
|
||||||
private.write_text("secret\n", encoding="utf-8")
|
|
||||||
private.parent.chmod(0o700)
|
|
||||||
private.chmod(0o600)
|
|
||||||
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
local_fetch(updater, source)
|
|
||||||
previous_umask = os.umask(0o077)
|
|
||||||
try:
|
|
||||||
updater.apply()
|
|
||||||
finally:
|
|
||||||
os.umask(previous_umask)
|
|
||||||
|
|
||||||
self.assertEqual(stat.S_IMODE((checkout / "web/app/NewFeature.php").stat().st_mode), 0o644)
|
|
||||||
self.assertEqual(stat.S_IMODE((checkout / "tools/task").stat().st_mode), 0o755)
|
|
||||||
self.assertEqual(stat.S_IMODE((checkout / "web/app").stat().st_mode), 0o755)
|
|
||||||
self.assertEqual(stat.S_IMODE(private.stat().st_mode), 0o600)
|
|
||||||
self.assertEqual(stat.S_IMODE(private.parent.stat().st_mode), 0o700)
|
|
||||||
|
|
||||||
def test_target_tree_with_symlink_is_rejected_before_quiesce(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
(source / "linked-config").symlink_to("/etc/passwd")
|
|
||||||
git(source, "add", "linked-config")
|
|
||||||
git(source, "commit", "-m", "add forbidden symlink")
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
local_fetch(updater, source)
|
|
||||||
quiesced = []
|
|
||||||
with self.assertRaisesRegex(UpdateError, "符号链接"):
|
|
||||||
updater.apply(before_switch=lambda: quiesced.append(True))
|
|
||||||
self.assertEqual(quiesced, [])
|
|
||||||
|
|
||||||
def test_remote_history_rewrite_never_overwrites_local_code(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
target = advance_source(source)
|
|
||||||
git(checkout, "config", "user.name", "Updater Test")
|
|
||||||
git(checkout, "config", "user.email", "updater@example.test")
|
|
||||||
(checkout / "README.md").write_text("local fork\n", encoding="utf-8")
|
|
||||||
git(checkout, "add", "README.md")
|
|
||||||
git(checkout, "commit", "-m", "local fork")
|
|
||||||
local = git(checkout, "rev-parse", "HEAD")
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
local_fetch(updater, source)
|
|
||||||
with self.assertRaisesRegex(UpdateError, "分叉"):
|
|
||||||
updater.apply()
|
|
||||||
self.assertEqual(git(checkout, "rev-parse", "HEAD"), local)
|
|
||||||
self.assertNotEqual(local, target)
|
|
||||||
|
|
||||||
def test_dependency_and_deployment_changes_require_manual_upgrade_before_quiesce(self):
|
|
||||||
for filename, message in (
|
|
||||||
("engine/requirements.txt", "Python 依赖"),
|
|
||||||
("web/composer.json", "PHP 依赖"),
|
|
||||||
("web/composer.lock", "PHP 依赖"),
|
|
||||||
("deploy/virt-cloud-engine.service", "systemd 服务"),
|
|
||||||
):
|
|
||||||
with self.subTest(filename=filename):
|
|
||||||
case = self.tmp_path / filename.replace("/", "-")
|
|
||||||
case.mkdir()
|
|
||||||
checkout = make_repo(case)
|
|
||||||
source = case / "source"
|
|
||||||
advance_source(source, filename=filename)
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
local_fetch(updater, source)
|
|
||||||
old = git(checkout, "rev-parse", "HEAD")
|
|
||||||
quiesced = []
|
|
||||||
with self.assertRaisesRegex(UpdateError, message):
|
|
||||||
updater.apply(before_switch=lambda: quiesced.append(True))
|
|
||||||
self.assertEqual(quiesced, [])
|
|
||||||
self.assertEqual(git(checkout, "rev-parse", "HEAD"), old)
|
|
||||||
|
|
||||||
def test_stale_unclaimed_job_is_failed_and_no_longer_blocks(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
store = MemoryStore()
|
|
||||||
job_id = "stale-job-1234567890"
|
|
||||||
store.set_value("update.active_job", job_id)
|
|
||||||
store.set_value("update.job." + job_id, json.dumps({
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "queued",
|
|
||||||
"phase": "等待独立更新服务启动",
|
|
||||||
"created_at": int(time.time()) - 120,
|
|
||||||
}))
|
|
||||||
updater = Updater(checkout, store)
|
|
||||||
self.assertIsNone(updater.active_job())
|
|
||||||
self.assertEqual(updater.get_job(job_id)["status"], "failed")
|
|
||||||
self.assertEqual(updater.settings()["last_job_id"], job_id)
|
|
||||||
|
|
||||||
def test_running_job_stays_active_while_worker_is_running(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
store = MemoryStore()
|
|
||||||
job_id = "running-job-123456789"
|
|
||||||
current = git(checkout, "rev-parse", "HEAD")
|
|
||||||
store.set_value("update.active_job", job_id)
|
|
||||||
store.set_value("update.job." + job_id, json.dumps({
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "running",
|
|
||||||
"phase": "正在准备",
|
|
||||||
"deployment_state": "preparing",
|
|
||||||
"expected_current_commit": current,
|
|
||||||
"created_at": int(time.time()) - 5000,
|
|
||||||
"started_at": int(time.time()) - 5000,
|
|
||||||
}))
|
|
||||||
running = Updater(checkout, store, worker_active=lambda: True)
|
|
||||||
self.assertIsNotNone(running.active_job())
|
|
||||||
self.assertEqual(running.get_job(job_id)["status"], "running")
|
|
||||||
|
|
||||||
def test_inactive_worker_keeps_maintenance_lock_for_ambiguous_deployment(self):
|
|
||||||
for index, state in enumerate((None, "unknown", "code_switched")):
|
|
||||||
with self.subTest(state=state):
|
|
||||||
case = self.tmp_path / f"ambiguous-{index}-{state or 'missing'}"
|
|
||||||
case.mkdir()
|
|
||||||
checkout = make_repo(case)
|
|
||||||
store = MemoryStore()
|
|
||||||
job_id = "running-job-123456789"
|
|
||||||
job = {
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "running",
|
|
||||||
"phase": "正在更新",
|
|
||||||
"started_at": int(time.time()) - 5000,
|
|
||||||
}
|
|
||||||
if state is not None:
|
|
||||||
job["deployment_state"] = state
|
|
||||||
if state == "unknown":
|
|
||||||
job["old_commit"] = git(checkout, "rev-parse", "HEAD")
|
|
||||||
store.set_value("update.active_job", job_id)
|
|
||||||
store.set_value("update.job." + job_id, json.dumps(job))
|
|
||||||
stopped = Updater(checkout, store, worker_active=lambda: False)
|
|
||||||
active = stopped.active_job()
|
|
||||||
self.assertIsNotNone(active)
|
|
||||||
self.assertEqual(active["status"], "running")
|
|
||||||
self.assertIn("维护锁已保留", active["phase"])
|
|
||||||
self.assertIn("维护锁已保留", active["error"])
|
|
||||||
|
|
||||||
def test_inactive_worker_releases_only_known_safe_deployment_states(self):
|
|
||||||
for state in (
|
|
||||||
"preparing", "quiescing", "services_stopped", "code_switched",
|
|
||||||
"schema_ready", "services_ready",
|
|
||||||
):
|
|
||||||
with self.subTest(state=state):
|
|
||||||
case = self.tmp_path / state
|
|
||||||
case.mkdir()
|
|
||||||
checkout = make_repo(case)
|
|
||||||
store = MemoryStore()
|
|
||||||
job_id = "running-job-123456789"
|
|
||||||
current = git(checkout, "rev-parse", "HEAD")
|
|
||||||
store.set_value("update.active_job", job_id)
|
|
||||||
job = {
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "running",
|
|
||||||
"phase": "正在更新",
|
|
||||||
"deployment_state": state,
|
|
||||||
"old_commit": current,
|
|
||||||
"started_at": int(time.time()) - 5000,
|
|
||||||
}
|
|
||||||
if state in {"schema_ready", "services_ready"}:
|
|
||||||
job["new_commit"] = current
|
|
||||||
store.set_value("update.job." + job_id, json.dumps(job))
|
|
||||||
stopped = Updater(checkout, store, worker_active=lambda: False)
|
|
||||||
self.assertIsNone(stopped.active_job())
|
|
||||||
self.assertEqual(stopped.get_job(job_id)["status"], "failed")
|
|
||||||
|
|
||||||
def test_inactive_worker_keeps_lock_when_early_state_checkout_is_not_clean_old_commit(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
store = MemoryStore()
|
|
||||||
job_id = "running-job-123456789"
|
|
||||||
current = git(checkout, "rev-parse", "HEAD")
|
|
||||||
store.set_value("update.active_job", job_id)
|
|
||||||
store.set_value("update.job." + job_id, json.dumps({
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "running",
|
|
||||||
"phase": "正在暂停服务",
|
|
||||||
"deployment_state": "services_stopped",
|
|
||||||
"old_commit": current,
|
|
||||||
"started_at": int(time.time()) - 5000,
|
|
||||||
}))
|
|
||||||
(checkout / "README.md").write_text("dirty\n", encoding="utf-8")
|
|
||||||
stopped = Updater(checkout, store, worker_active=lambda: False)
|
|
||||||
active = stopped.active_job()
|
|
||||||
self.assertIsNotNone(active)
|
|
||||||
self.assertEqual(active["status"], "running")
|
|
||||||
self.assertIn("维护锁已保留", active["phase"])
|
|
||||||
|
|
||||||
def test_inactive_worker_keeps_lock_during_schema_migration_after_source_rollback(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
store = MemoryStore()
|
|
||||||
job_id = "running-job-123456789"
|
|
||||||
current = git(checkout, "rev-parse", "HEAD")
|
|
||||||
store.set_value("update.active_job", job_id)
|
|
||||||
store.set_value("update.job." + job_id, json.dumps({
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "running",
|
|
||||||
"phase": "正在执行数据库迁移",
|
|
||||||
"deployment_state": "schema_migrating",
|
|
||||||
"old_commit": current,
|
|
||||||
"started_at": int(time.time()) - 5000,
|
|
||||||
}))
|
|
||||||
stopped = Updater(checkout, store, worker_active=lambda: False)
|
|
||||||
active = stopped.active_job()
|
|
||||||
self.assertIsNotNone(active)
|
|
||||||
self.assertEqual(active["status"], "running")
|
|
||||||
self.assertIn("维护锁已保留", active["phase"])
|
|
||||||
|
|
||||||
def test_git_url_rewrite_is_rejected_before_sending_token(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
git(checkout, "config", "url.https://evil.example/.insteadOf", "https://g.luoli.live/")
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
updater.set_token("a-test-access-token-123")
|
|
||||||
with self.assertRaisesRegex(UpdateError, "URL 重写"):
|
|
||||||
updater.check()
|
|
||||||
|
|
||||||
def test_new_code_cannot_track_private_data_or_submodules(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
target = advance_source(source, filename="runtime/engine/master.key")
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
local_fetch(updater, source)
|
|
||||||
old = git(checkout, "rev-parse", "HEAD")
|
|
||||||
with self.assertRaisesRegex(UpdateError, "运行数据"):
|
|
||||||
updater.apply()
|
|
||||||
self.assertEqual(git(checkout, "rev-parse", "HEAD"), old)
|
|
||||||
self.assertNotEqual(old, target)
|
|
||||||
|
|
||||||
def test_failed_restart_is_persisted_without_hiding_applied_commit(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
target = advance_source(source)
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
updater.set_token("a-test-access-token-123")
|
|
||||||
local_fetch(updater, source)
|
|
||||||
job = updater.create_job()
|
|
||||||
|
|
||||||
def fail_restart(_mark_schema_ready):
|
|
||||||
raise UpdateError("重启失败", 500)
|
|
||||||
|
|
||||||
with self.assertRaisesRegex(UpdateError, "重启失败"):
|
|
||||||
updater.run_pending_job(restart=fail_restart)
|
|
||||||
stored = updater.get_job(job["job_id"])
|
|
||||||
self.assertEqual(stored["status"], "blocked")
|
|
||||||
self.assertEqual(stored["deployment_state"], "schema_migrating")
|
|
||||||
self.assertEqual(stored["new_commit"], target)
|
|
||||||
self.assertTrue(stored["rollback_ref"])
|
|
||||||
self.assertIn("代码可能已切换", stored["error"])
|
|
||||||
self.assertIn("维护锁已保留", stored["error"])
|
|
||||||
self.assertEqual(updater.settings()["active_job_id"], job["job_id"])
|
|
||||||
|
|
||||||
# Once migration has started, a source rollback cannot prove that
|
|
||||||
# partially committed DDL is compatible with the old application.
|
|
||||||
git(checkout, "reset", "--hard", stored["old_commit"])
|
|
||||||
active = updater.active_job()
|
|
||||||
self.assertIsNotNone(active)
|
|
||||||
self.assertEqual(active["status"], "blocked")
|
|
||||||
self.assertEqual(updater.settings()["active_job_id"], job["job_id"])
|
|
||||||
|
|
||||||
def test_explicit_forward_recovery_clears_old_error_only_after_safe_completion(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
old = git(checkout, "rev-parse", "HEAD")
|
|
||||||
target = advance_source(source)
|
|
||||||
store = MemoryStore()
|
|
||||||
updater = Updater(checkout, store)
|
|
||||||
local_fetch(updater, source)
|
|
||||||
applied = updater.apply()
|
|
||||||
self.assertEqual(applied["new_commit"], target)
|
|
||||||
|
|
||||||
job_id = "blocked-job-123456789"
|
|
||||||
store.set_value("update.active_job", job_id)
|
|
||||||
store.set_value("update.job." + job_id, json.dumps({
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "blocked",
|
|
||||||
"phase": "更新中断",
|
|
||||||
"deployment_state": "schema_migrating",
|
|
||||||
"old_commit": old,
|
|
||||||
"new_commit": target,
|
|
||||||
"rollback_ref": applied["rollback_ref"],
|
|
||||||
"error": "旧失败原因",
|
|
||||||
"finished_at": 123,
|
|
||||||
}))
|
|
||||||
|
|
||||||
claimed = updater.begin_blocked_recovery()
|
|
||||||
self.assertEqual(claimed["status"], "running")
|
|
||||||
self.assertTrue(claimed["recovery_requested"])
|
|
||||||
self.assertNotIn("error", claimed)
|
|
||||||
self.assertNotIn("finished_at", claimed)
|
|
||||||
migrating = updater.recovery_checkpoint(
|
|
||||||
job_id, "schema_migrating", "正在重新执行数据库初始化")
|
|
||||||
self.assertEqual(migrating["deployment_state"], "schema_migrating")
|
|
||||||
ready = updater.recovery_checkpoint(
|
|
||||||
job_id, "schema_ready", "数据库初始化完成")
|
|
||||||
self.assertEqual(ready["deployment_state"], "schema_ready")
|
|
||||||
# SSH recovery is not the systemd update unit. Starting the new engine
|
|
||||||
# for its health check must not let that engine release maintenance
|
|
||||||
# before the recovery CLI records successful completion.
|
|
||||||
observing_engine = Updater(checkout, store, worker_active=lambda: False)
|
|
||||||
observed = observing_engine.active_job()
|
|
||||||
self.assertIsNotNone(observed)
|
|
||||||
self.assertEqual(observed["status"], "running")
|
|
||||||
self.assertTrue(observed["recovery_requested"])
|
|
||||||
completed = updater.complete_blocked_recovery(job_id)
|
|
||||||
self.assertEqual(completed["status"], "succeeded")
|
|
||||||
self.assertEqual(completed["deployment_state"], "services_ready")
|
|
||||||
self.assertEqual(completed["phase"], "SSH 人工恢复完成")
|
|
||||||
self.assertNotIn("error", completed)
|
|
||||||
self.assertIsNone(updater.active_job())
|
|
||||||
|
|
||||||
def test_forward_recovery_rejects_old_checkout_and_failed_migration_keeps_lock(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
old = git(checkout, "rev-parse", "HEAD")
|
|
||||||
target = advance_source(source)
|
|
||||||
store = MemoryStore()
|
|
||||||
updater = Updater(checkout, store)
|
|
||||||
local_fetch(updater, source)
|
|
||||||
applied = updater.apply()
|
|
||||||
job_id = "blocked-job-123456789"
|
|
||||||
store.set_value("update.active_job", job_id)
|
|
||||||
store.set_value("update.job." + job_id, json.dumps({
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "blocked",
|
|
||||||
"phase": "更新中断",
|
|
||||||
"deployment_state": "schema_migrating",
|
|
||||||
"old_commit": old,
|
|
||||||
"new_commit": target,
|
|
||||||
"rollback_ref": applied["rollback_ref"],
|
|
||||||
"error": "迁移中断",
|
|
||||||
}))
|
|
||||||
|
|
||||||
git(checkout, "reset", "--hard", old)
|
|
||||||
with self.assertRaisesRegex(UpdateError, "新提交"):
|
|
||||||
updater.begin_blocked_recovery()
|
|
||||||
self.assertEqual(updater.get_job(job_id)["status"], "blocked")
|
|
||||||
|
|
||||||
git(checkout, "reset", "--hard", target)
|
|
||||||
updater.begin_blocked_recovery()
|
|
||||||
updater.recovery_checkpoint(job_id, "schema_migrating", "正在迁移")
|
|
||||||
failed = updater.fail_job(job_id, "人工恢复失败")
|
|
||||||
self.assertEqual(failed["status"], "blocked")
|
|
||||||
self.assertEqual(failed["deployment_state"], "schema_migrating")
|
|
||||||
self.assertIn("维护锁已保留", failed["error"])
|
|
||||||
self.assertIsNotNone(updater.active_job())
|
|
||||||
|
|
||||||
def test_checked_remote_commit_is_pinned_through_worker_execution(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
source = self.tmp_path / "source"
|
|
||||||
old = git(checkout, "rev-parse", "HEAD")
|
|
||||||
checked = advance_source(source)
|
|
||||||
store = MemoryStore()
|
|
||||||
updater = Updater(checkout, store)
|
|
||||||
updater.set_token("a-test-access-token-123")
|
|
||||||
updater._remote_head = lambda: checked
|
|
||||||
local_fetch(updater, source)
|
|
||||||
queued = updater.create_job(
|
|
||||||
expected_current_commit=old,
|
|
||||||
expected_remote_commit=checked,
|
|
||||||
)
|
|
||||||
advance_source(source, filename="VERSION")
|
|
||||||
with self.assertRaisesRegex(UpdateError, f"{BRANCH} 分支已变化"):
|
|
||||||
updater.run_pending_job(restart=lambda mark_schema_ready: mark_schema_ready())
|
|
||||||
self.assertEqual(git(checkout, "rev-parse", "HEAD"), old)
|
|
||||||
self.assertEqual(updater.get_job(queued["job_id"])["status"], "failed")
|
|
||||||
|
|
||||||
def test_create_job_rejects_changed_remote_since_check(self):
|
|
||||||
checkout = make_repo(self.tmp_path)
|
|
||||||
updater = Updater(checkout, MemoryStore())
|
|
||||||
updater.set_token("a-test-access-token-123")
|
|
||||||
updater._remote_head = lambda: "b" * 40
|
|
||||||
with self.assertRaisesRegex(UpdateError, f"{BRANCH} 分支已变化"):
|
|
||||||
updater.create_job(
|
|
||||||
expected_current_commit=git(checkout, "rev-parse", "HEAD"),
|
|
||||||
expected_remote_commit="a" * 40,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@ -1,331 +0,0 @@
|
|||||||
"""User ISO catalog and remote-download security tests."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import socket
|
|
||||||
from pathlib import Path
|
|
||||||
from types import SimpleNamespace
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from virsh_panel.cloud_templates import TemplateError
|
|
||||||
from virsh_panel.user_isos import (
|
|
||||||
MAX_USER_ISO_BYTES, MAX_USER_ISOS, MAX_USER_ISO_TOTAL_BYTES,
|
|
||||||
PERSISTENT_FREE_RESERVE_BYTES, UserISOFiles, UserISORuntime, UserISOStore,
|
|
||||||
_resolve_global_addresses,
|
|
||||||
download_user_iso, user_iso_scope, user_iso_volume_name,
|
|
||||||
validate_user_iso_url, verify_user_iso,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class MemoryStore:
|
|
||||||
def __init__(self):
|
|
||||||
self.records = {}
|
|
||||||
|
|
||||||
def get(self, kind, key):
|
|
||||||
return self.records.get((kind, key))
|
|
||||||
|
|
||||||
def set(self, kind, key, value):
|
|
||||||
self.records[kind, key] = value
|
|
||||||
|
|
||||||
def insert(self, kind, key, value):
|
|
||||||
if (kind, key) in self.records:
|
|
||||||
return False
|
|
||||||
self.records[kind, key] = value
|
|
||||||
return True
|
|
||||||
|
|
||||||
def delete(self, kind, key):
|
|
||||||
self.records.pop((kind, key), None)
|
|
||||||
|
|
||||||
def list(self, kind):
|
|
||||||
return {key: value for (row_kind, key), value in self.records.items()
|
|
||||||
if row_kind == kind}
|
|
||||||
|
|
||||||
|
|
||||||
def bootable_iso() -> bytes:
|
|
||||||
image = bytearray(1024 * 1024)
|
|
||||||
boot = memoryview(image)[16 * 2048:17 * 2048]
|
|
||||||
boot[0] = 0
|
|
||||||
boot[1:6] = b"CD001"
|
|
||||||
boot[6] = 1
|
|
||||||
boot[7:30] = b"EL TORITO SPECIFICATION"
|
|
||||||
boot[71:75] = (20).to_bytes(4, "little")
|
|
||||||
end = memoryview(image)[17 * 2048:18 * 2048]
|
|
||||||
end[0] = 255
|
|
||||||
end[1:6] = b"CD001"
|
|
||||||
end[6] = 1
|
|
||||||
return bytes(image)
|
|
||||||
|
|
||||||
|
|
||||||
def record(number: int, *, size: int = 1024 * 1024) -> dict:
|
|
||||||
iso_id = f"{number:032x}"
|
|
||||||
return {
|
|
||||||
"id": iso_id,
|
|
||||||
"name": f"Installer {number}",
|
|
||||||
"volume_name": user_iso_volume_name(iso_id),
|
|
||||||
"size_bytes": size,
|
|
||||||
"sha256": f"{number:064x}",
|
|
||||||
"created_at": number,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_catalog_is_owner_scoped_and_enforces_count_quota():
|
|
||||||
store = UserISOStore(None)
|
|
||||||
first = user_iso_scope(41)
|
|
||||||
second = user_iso_scope(42)
|
|
||||||
store.add(first, record(1))
|
|
||||||
assert store.get(first, f"{1:032x}")["name"] == "Installer 1"
|
|
||||||
assert store.list(second) == []
|
|
||||||
with pytest.raises(TemplateError):
|
|
||||||
store.get(second, f"{1:032x}")
|
|
||||||
for number in range(2, MAX_USER_ISOS + 1):
|
|
||||||
store.add(first, record(number))
|
|
||||||
with pytest.raises(TemplateError, match="最多保存 5 个"):
|
|
||||||
store.remaining_bytes(first)
|
|
||||||
|
|
||||||
|
|
||||||
def test_catalog_enforces_10_gib_total_quota():
|
|
||||||
store = UserISOStore(None)
|
|
||||||
scope = user_iso_scope(41)
|
|
||||||
store.add(scope, record(1, size=MAX_USER_ISO_TOTAL_BYTES))
|
|
||||||
with pytest.raises(TemplateError, match="10 GiB"):
|
|
||||||
store.remaining_bytes(scope)
|
|
||||||
|
|
||||||
|
|
||||||
def test_panel_storage_reserves_256_mib_for_engine_health(tmp_path):
|
|
||||||
files = UserISOFiles(tmp_path)
|
|
||||||
with patch("virsh_panel.user_isos.shutil.disk_usage", return_value=SimpleNamespace(
|
|
||||||
free=PERSISTENT_FREE_RESERVE_BYTES + 12345)):
|
|
||||||
assert files.available_bytes(41) == 12345
|
|
||||||
|
|
||||||
|
|
||||||
def test_reconcile_removes_only_safe_temporary_and_orphan_files(tmp_path):
|
|
||||||
files = UserISOFiles(tmp_path)
|
|
||||||
catalog = UserISOStore(None)
|
|
||||||
owner = files.owner_directory(41)
|
|
||||||
active = record(1)
|
|
||||||
catalog.add(user_iso_scope(41), active)
|
|
||||||
active_path = owner / active["volume_name"]
|
|
||||||
active_path.write_bytes(b"active")
|
|
||||||
temporary = owner / ".user-iso-abcdef.iso"
|
|
||||||
temporary.write_bytes(b"partial")
|
|
||||||
orphan = owner / user_iso_volume_name(f"{2:032x}")
|
|
||||||
orphan.write_bytes(b"orphan")
|
|
||||||
symlink = owner / ".user-iso-symlink.iso"
|
|
||||||
symlink.symlink_to(active_path)
|
|
||||||
|
|
||||||
files.reconcile(catalog)
|
|
||||||
|
|
||||||
assert active_path.exists()
|
|
||||||
assert not temporary.exists()
|
|
||||||
assert not orphan.exists()
|
|
||||||
assert symlink.is_symlink()
|
|
||||||
|
|
||||||
|
|
||||||
def test_reconcile_does_not_delete_from_insecure_owner_directory(tmp_path):
|
|
||||||
files = UserISOFiles(tmp_path)
|
|
||||||
owner = files.owner_directory(41)
|
|
||||||
temporary = owner / ".user-iso-abcdef.iso"
|
|
||||||
temporary.write_bytes(b"partial")
|
|
||||||
os.chmod(owner, 0o755)
|
|
||||||
|
|
||||||
files.reconcile(UserISOStore(None))
|
|
||||||
|
|
||||||
assert temporary.exists()
|
|
||||||
|
|
||||||
|
|
||||||
def test_restore_finishes_catalog_delete_after_source_unlink_crash(tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
runtime = UserISORuntime(store, tmp_path)
|
|
||||||
item = record(3)
|
|
||||||
scope = user_iso_scope(41)
|
|
||||||
owner = runtime.files.owner_directory(41)
|
|
||||||
source = owner / item["volume_name"]
|
|
||||||
source.write_bytes(b"x" * item["size_bytes"])
|
|
||||||
os.chmod(source, 0o600)
|
|
||||||
runtime.catalog.add(scope, item)
|
|
||||||
key = f"41:{item['id']}"
|
|
||||||
store.set("user_iso_tombstone", key, json.dumps({
|
|
||||||
"user_id": 41,
|
|
||||||
"iso_id": item["id"],
|
|
||||||
"volume_name": item["volume_name"],
|
|
||||||
"pending_hosts": ["hk"],
|
|
||||||
"source_deleted": False,
|
|
||||||
"created_at": 1,
|
|
||||||
}))
|
|
||||||
source.unlink()
|
|
||||||
try:
|
|
||||||
runtime.restore_jobs()
|
|
||||||
with pytest.raises(TemplateError) as missing:
|
|
||||||
runtime.catalog.get(scope, item["id"])
|
|
||||||
assert missing.value.status == 404
|
|
||||||
tombstone = json.loads(store.get("user_iso_tombstone", key))
|
|
||||||
assert tombstone["source_deleted"] is True
|
|
||||||
assert tombstone["pending_hosts"] == ["hk"]
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_restore_rejects_tombstone_key_mismatch_without_catalog(tmp_path):
|
|
||||||
store = MemoryStore()
|
|
||||||
runtime = UserISORuntime(store, tmp_path)
|
|
||||||
first_id = f"{4:032x}"
|
|
||||||
second_id = f"{5:032x}"
|
|
||||||
store.set("user_iso_tombstone", f"41:{first_id}", json.dumps({
|
|
||||||
"user_id": 41,
|
|
||||||
"iso_id": second_id,
|
|
||||||
"volume_name": user_iso_volume_name(second_id),
|
|
||||||
"pending_hosts": ["hk"],
|
|
||||||
"source_deleted": True,
|
|
||||||
"created_at": 1,
|
|
||||||
}))
|
|
||||||
try:
|
|
||||||
with pytest.raises(TemplateError, match="回收记录损坏"):
|
|
||||||
runtime.restore_jobs()
|
|
||||||
finally:
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("url", [
|
|
||||||
"http://downloads.example.test/os.iso",
|
|
||||||
"https://user:pass@downloads.example.test/os.iso",
|
|
||||||
"https://downloads.example.test:444/os.iso",
|
|
||||||
"https://downloads.example.test/os.iso#fragment",
|
|
||||||
])
|
|
||||||
def test_remote_url_requires_plain_https_443(url):
|
|
||||||
with pytest.raises(TemplateError):
|
|
||||||
validate_user_iso_url(url)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("url", [
|
|
||||||
"https://downloads.example.test/os.iso",
|
|
||||||
"https://downloads.example.test/releases/OS.ISO?token=abc123",
|
|
||||||
])
|
|
||||||
def test_remote_url_accepts_iso_path_case_insensitively_with_query(url):
|
|
||||||
assert validate_user_iso_url(url).path.lower().endswith(".iso")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("url", [
|
|
||||||
"https://downloads.example.test/os.img",
|
|
||||||
"https://downloads.example.test/download",
|
|
||||||
"https://downloads.example.test/os.iso.img",
|
|
||||||
"https://downloads.example.test/os.iso/extra",
|
|
||||||
"https://downloads.example.test/download?file=os.iso",
|
|
||||||
])
|
|
||||||
def test_remote_url_rejects_path_without_real_iso_suffix(url):
|
|
||||||
with pytest.raises(TemplateError, match=r"\.iso 结尾") as invalid:
|
|
||||||
validate_user_iso_url(url)
|
|
||||||
assert invalid.value.status == 422
|
|
||||||
assert UserISORuntime.public_error(str(invalid.value)) == str(invalid.value)
|
|
||||||
|
|
||||||
|
|
||||||
def test_dns_rejects_private_or_mixed_answers():
|
|
||||||
answers = [
|
|
||||||
(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("8.8.8.8", 443)),
|
|
||||||
(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", 443)),
|
|
||||||
]
|
|
||||||
with patch("virsh_panel.user_isos.socket.getaddrinfo", return_value=answers):
|
|
||||||
with pytest.raises(TemplateError, match="内网、回环或保留"):
|
|
||||||
_resolve_global_addresses("downloads.example.test")
|
|
||||||
|
|
||||||
|
|
||||||
def test_bootable_iso_validation(tmp_path):
|
|
||||||
valid = tmp_path / "valid.iso"
|
|
||||||
valid.write_bytes(bootable_iso())
|
|
||||||
verify_user_iso(valid)
|
|
||||||
invalid = tmp_path / "invalid.iso"
|
|
||||||
invalid.write_bytes(b"x" * (1024 * 1024))
|
|
||||||
with pytest.raises(TemplateError, match="ISO 9660"):
|
|
||||||
verify_user_iso(invalid)
|
|
||||||
|
|
||||||
|
|
||||||
class FakeHeaders:
|
|
||||||
def __init__(self, size: int):
|
|
||||||
self.size = size
|
|
||||||
|
|
||||||
def get_all(self, name: str, default):
|
|
||||||
return [str(self.size)] if name == "Content-Length" else default
|
|
||||||
|
|
||||||
|
|
||||||
class FakeResponse:
|
|
||||||
def __init__(self, content: bytes, *, status: int = 200):
|
|
||||||
self.content = content
|
|
||||||
self.status = status
|
|
||||||
self.offset = 0
|
|
||||||
self.headers = FakeHeaders(len(content))
|
|
||||||
self.closed = False
|
|
||||||
|
|
||||||
def getheader(self, _name: str, default=None):
|
|
||||||
return default
|
|
||||||
|
|
||||||
def read(self, size: int):
|
|
||||||
chunk = self.content[self.offset:self.offset + size]
|
|
||||||
self.offset += len(chunk)
|
|
||||||
return chunk
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
self.closed = True
|
|
||||||
|
|
||||||
|
|
||||||
class FakeConnection:
|
|
||||||
def __init__(self):
|
|
||||||
self.closed = False
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
self.closed = True
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_uses_resolved_pinned_addresses_and_private_temp_file(tmp_path):
|
|
||||||
content = bootable_iso()
|
|
||||||
response = FakeResponse(content)
|
|
||||||
connection = FakeConnection()
|
|
||||||
with patch("virsh_panel.user_isos._resolve_global_addresses",
|
|
||||||
return_value=["8.8.8.8"]) as resolve, patch(
|
|
||||||
"virsh_panel.user_isos._open_pinned_response",
|
|
||||||
return_value=(connection, response)) as opened:
|
|
||||||
path, size, digest = download_user_iso(
|
|
||||||
"https://downloads.example.test/os.iso?signature=secret",
|
|
||||||
"", tmp_path,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
assert size == len(content)
|
|
||||||
assert digest == hashlib.sha256(content).hexdigest()
|
|
||||||
assert path.stat().st_mode & 0o077 == 0
|
|
||||||
resolve.assert_called_once_with("downloads.example.test")
|
|
||||||
assert opened.call_args.args[1] == ["8.8.8.8"]
|
|
||||||
assert response.closed and connection.closed
|
|
||||||
finally:
|
|
||||||
Path(path).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_rejects_redirect_without_following_or_leaving_temp_file(tmp_path):
|
|
||||||
response = FakeResponse(b"", status=302)
|
|
||||||
connection = FakeConnection()
|
|
||||||
with patch("virsh_panel.user_isos._resolve_global_addresses",
|
|
||||||
return_value=["8.8.8.8"]), patch(
|
|
||||||
"virsh_panel.user_isos._open_pinned_response",
|
|
||||||
return_value=(connection, response)):
|
|
||||||
with pytest.raises(TemplateError, match="重定向"):
|
|
||||||
download_user_iso(
|
|
||||||
"https://downloads.example.test/redirect.iso", "", tmp_path)
|
|
||||||
assert list(tmp_path.iterdir()) == []
|
|
||||||
assert response.closed and connection.closed
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_rejects_declared_file_over_global_10_gib_quota(tmp_path):
|
|
||||||
response = FakeResponse(b"")
|
|
||||||
response.headers = FakeHeaders(MAX_USER_ISO_BYTES + 1)
|
|
||||||
connection = FakeConnection()
|
|
||||||
with patch("virsh_panel.user_isos._resolve_global_addresses",
|
|
||||||
return_value=["8.8.8.8"]), patch(
|
|
||||||
"virsh_panel.user_isos._open_pinned_response",
|
|
||||||
return_value=(connection, response)):
|
|
||||||
with pytest.raises(TemplateError, match="10 GiB") as oversized:
|
|
||||||
download_user_iso(
|
|
||||||
"https://downloads.example.test/oversized.iso", "", tmp_path)
|
|
||||||
assert oversized.value.status == 413
|
|
||||||
assert list(tmp_path.iterdir()) == []
|
|
||||||
@ -1,973 +0,0 @@
|
|||||||
"""Validate public IPv4 guest settings and build a NoCloud seed ISO.
|
|
||||||
|
|
||||||
The seed may contain a guest SSH public key and a salted root password hash,
|
|
||||||
but never a plaintext password, private key, or panel credential. JSON is
|
|
||||||
used as a YAML subset so untrusted fields cannot inject directives.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import base64
|
|
||||||
import binascii
|
|
||||||
import ipaddress
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import secrets
|
|
||||||
import string
|
|
||||||
import uuid
|
|
||||||
from io import BytesIO
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
_VM_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,62}\Z")
|
|
||||||
# Linux host names are case-insensitive DNS labels. Keep the panel contract
|
|
||||||
# deliberately narrower than the kernel's permissive hostname(5) parser so a
|
|
||||||
# pasted value can never become a cloud-init directive or an ambiguous FQDN.
|
|
||||||
_HOSTNAME_LABEL = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\Z")
|
|
||||||
_MAC = re.compile(r"(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\Z")
|
|
||||||
_NETWORK_POLICY_KEY = "virt-cloud-network-policy"
|
|
||||||
_NETWORK_POLICY_VALUE = "boot-v1"
|
|
||||||
_SSH_TYPES = {
|
|
||||||
"ssh-ed25519",
|
|
||||||
"ssh-rsa",
|
|
||||||
"ecdsa-sha2-nistp256",
|
|
||||||
"ecdsa-sha2-nistp384",
|
|
||||||
"ecdsa-sha2-nistp521",
|
|
||||||
}
|
|
||||||
_SHA512_CRYPT = re.compile(
|
|
||||||
r"\$6\$rounds=([1-9][0-9]{3,8})\$[./0-9A-Za-z]{1,16}\$[./0-9A-Za-z]{86}\Z"
|
|
||||||
)
|
|
||||||
ROOT_PASSWORD_LENGTH = 32
|
|
||||||
ROOT_PASSWORD_MIN_LENGTH = 12
|
|
||||||
ROOT_PASSWORD_MAX_LENGTH = 128
|
|
||||||
ROOT_PASSWORD_UPPERCASE = string.ascii_uppercase
|
|
||||||
ROOT_PASSWORD_LOWERCASE = string.ascii_lowercase
|
|
||||||
ROOT_PASSWORD_DIGITS = string.digits
|
|
||||||
ROOT_PASSWORD_SYMBOLS = "!@#$%*-_=+?"
|
|
||||||
ROOT_PASSWORD_ALPHABET = (
|
|
||||||
ROOT_PASSWORD_UPPERCASE + ROOT_PASSWORD_LOWERCASE
|
|
||||||
+ ROOT_PASSWORD_DIGITS + ROOT_PASSWORD_SYMBOLS
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_guest_hostname(value: Any) -> str:
|
|
||||||
"""Return a canonical guest hostname suitable for cloud-init metadata.
|
|
||||||
|
|
||||||
A single-label hostname and a DNS-style FQDN are both accepted. Leading
|
|
||||||
or trailing dots, empty labels, underscores, whitespace, Unicode and
|
|
||||||
control characters are rejected; the returned value is lower-case so a
|
|
||||||
retry of the same settings request is deterministic.
|
|
||||||
"""
|
|
||||||
if not isinstance(value, str):
|
|
||||||
raise ValueError("主机名格式无效")
|
|
||||||
hostname = value.strip().lower()
|
|
||||||
if not 1 <= len(hostname) <= 253 or hostname.endswith("."):
|
|
||||||
raise ValueError("主机名长度必须为 1 到 253 个字符")
|
|
||||||
labels = hostname.split(".")
|
|
||||||
if any(not _HOSTNAME_LABEL.fullmatch(label) for label in labels):
|
|
||||||
raise ValueError("主机名只能包含字母、数字、连字符和点,且每段不能以连字符开头或结尾")
|
|
||||||
return hostname
|
|
||||||
|
|
||||||
|
|
||||||
def default_guest_hostname(name: str, vm_uuid: str | None = None) -> str:
|
|
||||||
"""Derive the initial guest hostname using the same rule as seed creation."""
|
|
||||||
if not isinstance(name, str) or not _VM_NAME.fullmatch(name):
|
|
||||||
raise ValueError("虚拟机名称无效")
|
|
||||||
hostname = re.sub(r"[^a-z0-9-]", "-", name.lower()).strip("-")[:63]
|
|
||||||
if hostname:
|
|
||||||
return hostname
|
|
||||||
try:
|
|
||||||
suffix = str(uuid.UUID(vm_uuid)).replace("-", "")[:8] if vm_uuid else "guest"
|
|
||||||
except (ValueError, TypeError, AttributeError):
|
|
||||||
suffix = "guest"
|
|
||||||
return f"vm-{suffix}"
|
|
||||||
|
|
||||||
|
|
||||||
def _root_password_has_required_classes(password: str) -> bool:
|
|
||||||
return (
|
|
||||||
any(char in ROOT_PASSWORD_UPPERCASE for char in password)
|
|
||||||
and any(char in ROOT_PASSWORD_LOWERCASE for char in password)
|
|
||||||
and any(char in ROOT_PASSWORD_DIGITS for char in password)
|
|
||||||
and any(not char.isalnum() for char in password)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def hash_root_password(password: str) -> str:
|
|
||||||
"""Hash a guest root password for /etc/shadow before writing the seed."""
|
|
||||||
if (not isinstance(password, str)
|
|
||||||
or not ROOT_PASSWORD_MIN_LENGTH <= len(password)
|
|
||||||
<= ROOT_PASSWORD_MAX_LENGTH
|
|
||||||
or any(not 0x21 <= ord(char) <= 0x7e for char in password)):
|
|
||||||
raise ValueError("初始 root 密码须为 12 至 128 位可打印 ASCII 字符且不能包含空格")
|
|
||||||
if not _root_password_has_required_classes(password):
|
|
||||||
raise ValueError("初始 root 密码须包含大写字母、小写字母、数字和符号")
|
|
||||||
from passlib.hash import sha512_crypt
|
|
||||||
|
|
||||||
return sha512_crypt.using(rounds=100_000).hash(password)
|
|
||||||
|
|
||||||
|
|
||||||
def generate_root_password() -> str:
|
|
||||||
"""Return a uniform random password from all compliant 32-char strings.
|
|
||||||
|
|
||||||
Generating every position from the complete alphabet and rejecting the
|
|
||||||
rare candidate missing a required class avoids fixed category positions
|
|
||||||
and the distribution bias introduced by "one of each, then shuffle".
|
|
||||||
``secrets.randbelow`` performs an unbiased CSPRNG selection.
|
|
||||||
"""
|
|
||||||
while True:
|
|
||||||
password = "".join(
|
|
||||||
ROOT_PASSWORD_ALPHABET[
|
|
||||||
secrets.randbelow(len(ROOT_PASSWORD_ALPHABET))
|
|
||||||
]
|
|
||||||
for _ in range(ROOT_PASSWORD_LENGTH)
|
|
||||||
)
|
|
||||||
if _root_password_has_required_classes(password):
|
|
||||||
return password
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_guest_auth(guest_ssh_public_key: Any,
|
|
||||||
root_password_hash: Any) -> tuple[str, str | None]:
|
|
||||||
if not isinstance(guest_ssh_public_key, str):
|
|
||||||
raise ValueError("虚拟机 SSH 公钥格式无效")
|
|
||||||
public_key = guest_ssh_public_key.strip()
|
|
||||||
if not public_key and not root_password_hash:
|
|
||||||
raise ValueError("请填写虚拟机 SSH 公钥或初始 root 密码")
|
|
||||||
if public_key:
|
|
||||||
public_key = _validate_ssh_public_key(public_key)
|
|
||||||
if root_password_hash is not None:
|
|
||||||
if not isinstance(root_password_hash, str):
|
|
||||||
raise ValueError("初始 root 密码哈希格式无效")
|
|
||||||
match = _SHA512_CRYPT.fullmatch(root_password_hash)
|
|
||||||
if match is None or not 100_000 <= int(match.group(1)) <= 1_000_000:
|
|
||||||
raise ValueError("初始 root 密码哈希格式无效")
|
|
||||||
return public_key, root_password_hash
|
|
||||||
|
|
||||||
|
|
||||||
def _ssh_string(blob: bytes, offset: int) -> tuple[bytes, int]:
|
|
||||||
if offset + 4 > len(blob):
|
|
||||||
raise ValueError("SSH 公钥内容无效")
|
|
||||||
size = int.from_bytes(blob[offset:offset + 4], "big")
|
|
||||||
start = offset + 4
|
|
||||||
end = start + size
|
|
||||||
if size > 8192 or end > len(blob):
|
|
||||||
raise ValueError("SSH 公钥内容无效")
|
|
||||||
return blob[start:end], end
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_ssh_public_key(value: Any) -> str:
|
|
||||||
if not isinstance(value, str) or not value or len(value) > 16384:
|
|
||||||
raise ValueError("请填写虚拟机登录用的 SSH 公钥")
|
|
||||||
if any(char in value for char in ("\n", "\r", "\x00")):
|
|
||||||
raise ValueError("SSH 公钥必须是单行 OpenSSH 公钥")
|
|
||||||
fields = value.strip().split(maxsplit=2)
|
|
||||||
if len(fields) < 2 or fields[0] not in _SSH_TYPES:
|
|
||||||
raise ValueError("SSH 公钥格式无效,仅支持 Ed25519、RSA 或 ECDSA")
|
|
||||||
key_type, encoded = fields[:2]
|
|
||||||
try:
|
|
||||||
blob = base64.b64decode(encoded, validate=True)
|
|
||||||
embedded_type, offset = _ssh_string(blob, 0)
|
|
||||||
except (ValueError, binascii.Error) as exc:
|
|
||||||
raise ValueError("SSH 公钥内容无效") from exc
|
|
||||||
if embedded_type != key_type.encode("ascii"):
|
|
||||||
raise ValueError("SSH 公钥类型与内容不匹配")
|
|
||||||
|
|
||||||
# An OpenSSH key is a series of length-prefixed binary strings. Verify
|
|
||||||
# its structure rather than accepting arbitrary base64 after the prefix.
|
|
||||||
if key_type == "ssh-ed25519":
|
|
||||||
key, offset = _ssh_string(blob, offset)
|
|
||||||
if len(key) != 32:
|
|
||||||
raise ValueError("Ed25519 公钥长度无效")
|
|
||||||
elif key_type == "ssh-rsa":
|
|
||||||
exponent, offset = _ssh_string(blob, offset)
|
|
||||||
modulus, offset = _ssh_string(blob, offset)
|
|
||||||
if (not exponent or int.from_bytes(exponent, "big") < 3
|
|
||||||
or int.from_bytes(exponent, "big") % 2 == 0
|
|
||||||
or int.from_bytes(modulus, "big").bit_length() < 2048):
|
|
||||||
raise ValueError("RSA 公钥无效或不足 2048 位")
|
|
||||||
else:
|
|
||||||
curve, offset = _ssh_string(blob, offset)
|
|
||||||
point, offset = _ssh_string(blob, offset)
|
|
||||||
suffix = key_type.removeprefix("ecdsa-sha2-")
|
|
||||||
point_lengths = {"nistp256": 65, "nistp384": 97, "nistp521": 133}
|
|
||||||
if (curve != suffix.encode("ascii") or len(point) != point_lengths[suffix]
|
|
||||||
or point[0] != 4):
|
|
||||||
raise ValueError("ECDSA 公钥内容无效")
|
|
||||||
if offset != len(blob):
|
|
||||||
raise ValueError("SSH 公钥存在多余数据")
|
|
||||||
# Keep a harmless OpenSSH comment for operator identification. The
|
|
||||||
# generated user-data is JSON-escaped, not assembled as raw YAML.
|
|
||||||
return value.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def _ipv4(value: Any, label: str) -> ipaddress.IPv4Address:
|
|
||||||
if not isinstance(value, str) or not value or value != value.strip():
|
|
||||||
raise ValueError(f"{label}必须是 IPv4 地址")
|
|
||||||
try:
|
|
||||||
address = ipaddress.ip_address(value)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise ValueError(f"{label}必须是 IPv4 地址") from exc
|
|
||||||
if not isinstance(address, ipaddress.IPv4Address):
|
|
||||||
raise ValueError(f"{label}必须是 IPv4 地址")
|
|
||||||
return address
|
|
||||||
|
|
||||||
|
|
||||||
def validate_cloud_network(
|
|
||||||
public_ip_cidr: str,
|
|
||||||
gateway: str,
|
|
||||||
dns_servers: list[str] | tuple[str, ...],
|
|
||||||
guest_ssh_public_key: str = "",
|
|
||||||
root_password_hash: str | None = None,
|
|
||||||
*,
|
|
||||||
require_auth: bool = True,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Return canonical guest network fields or raise ``ValueError``.
|
|
||||||
|
|
||||||
This first version supports an on-link gateway on a public IPv4 subnet,
|
|
||||||
including /24 and /28. Routed /32/off-link gateways need a separate
|
|
||||||
explicit model; silently treating them as on-link would strand the guest.
|
|
||||||
"""
|
|
||||||
if (not isinstance(public_ip_cidr, str) or not public_ip_cidr
|
|
||||||
or public_ip_cidr != public_ip_cidr.strip()):
|
|
||||||
raise ValueError("公网 IP 必须采用 IPv4 地址/前缀长度格式,例如实际分配的地址加 /28")
|
|
||||||
try:
|
|
||||||
interface = ipaddress.ip_interface(public_ip_cidr)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise ValueError("公网 IP/CIDR 格式无效") from exc
|
|
||||||
if not isinstance(interface, ipaddress.IPv4Interface):
|
|
||||||
raise ValueError("暂时只支持公网 IPv4")
|
|
||||||
network = interface.network
|
|
||||||
if not 1 <= network.prefixlen <= 30:
|
|
||||||
raise ValueError("当前网桥方案仅支持 /1 到 /30 的 IPv4 前缀")
|
|
||||||
if (not interface.ip.is_global or interface.ip.is_multicast
|
|
||||||
or interface.ip.is_reserved):
|
|
||||||
raise ValueError("IP 不是可用的公网 IPv4 地址")
|
|
||||||
if interface.ip in (network.network_address, network.broadcast_address):
|
|
||||||
raise ValueError("不能使用子网地址或广播地址作为虚拟机 IP")
|
|
||||||
|
|
||||||
gateway_ip = _ipv4(gateway, "网关")
|
|
||||||
if (not gateway_ip.is_global or gateway_ip.is_multicast or gateway_ip.is_reserved
|
|
||||||
or gateway_ip not in network or gateway_ip == interface.ip
|
|
||||||
or gateway_ip in (network.network_address, network.broadcast_address)):
|
|
||||||
raise ValueError("网关必须是同一网段内另一可用 IPv4 地址")
|
|
||||||
|
|
||||||
if (not isinstance(dns_servers, (list, tuple)) or not 1 <= len(dns_servers) <= 4):
|
|
||||||
raise ValueError("请填写 1 到 4 个 DNS IPv4 地址")
|
|
||||||
dns_ips = [_ipv4(item, "DNS") for item in dns_servers]
|
|
||||||
if any(not address.is_global or address.is_multicast or address.is_reserved
|
|
||||||
for address in dns_ips):
|
|
||||||
raise ValueError("当前公网模板只支持可公开路由的 DNS IPv4 地址")
|
|
||||||
dns = [str(address) for address in dns_ips]
|
|
||||||
if len(set(dns)) != len(dns):
|
|
||||||
raise ValueError("DNS 地址不能重复")
|
|
||||||
if not isinstance(require_auth, bool):
|
|
||||||
raise ValueError("网络认证校验选项无效")
|
|
||||||
if require_auth:
|
|
||||||
public_key, password_hash = _validate_guest_auth(
|
|
||||||
guest_ssh_public_key, root_password_hash)
|
|
||||||
else:
|
|
||||||
# A network-only update must not require, receive, or regenerate the
|
|
||||||
# guest credential. Credentials are not accepted or regenerated by
|
|
||||||
# ``replace_seed_network``.
|
|
||||||
public_key, password_hash = "", None
|
|
||||||
result = {
|
|
||||||
"public_ip_cidr": str(interface),
|
|
||||||
"gateway": str(gateway_ip),
|
|
||||||
"dns_servers": dns,
|
|
||||||
"guest_ssh_public_key": public_key,
|
|
||||||
}
|
|
||||||
if password_hash is not None:
|
|
||||||
result["root_password_hash"] = password_hash
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def build_seed_iso(
|
|
||||||
vm_uuid: str,
|
|
||||||
name: str,
|
|
||||||
mac: str,
|
|
||||||
validated_network: dict[str, Any],
|
|
||||||
*,
|
|
||||||
debian12_network_fix: bool = False,
|
|
||||||
) -> bytes:
|
|
||||||
"""Build a CIDATA ISO with user-data, meta-data, and network-config."""
|
|
||||||
try:
|
|
||||||
instance_id = str(uuid.UUID(vm_uuid))
|
|
||||||
except (ValueError, TypeError, AttributeError) as exc:
|
|
||||||
raise ValueError("虚拟机 UUID 无效") from exc
|
|
||||||
if not isinstance(name, str) or not _VM_NAME.fullmatch(name):
|
|
||||||
raise ValueError("虚拟机名称无效")
|
|
||||||
if not isinstance(mac, str) or not _MAC.fullmatch(mac):
|
|
||||||
raise ValueError("虚拟机 MAC 地址无效")
|
|
||||||
normalized_mac = mac.lower()
|
|
||||||
mac_octets = [int(part, 16) for part in normalized_mac.split(":")]
|
|
||||||
if mac_octets[0] & 1 or not any(mac_octets):
|
|
||||||
raise ValueError("虚拟机 MAC 地址不能是组播或全零地址")
|
|
||||||
if not isinstance(validated_network, dict):
|
|
||||||
raise ValueError("网络配置无效")
|
|
||||||
if not isinstance(debian12_network_fix, bool):
|
|
||||||
raise ValueError("Debian 12 网络修复选项无效")
|
|
||||||
network = validate_cloud_network(
|
|
||||||
validated_network.get("public_ip_cidr"),
|
|
||||||
validated_network.get("gateway"),
|
|
||||||
validated_network.get("dns_servers"),
|
|
||||||
validated_network.get("guest_ssh_public_key"),
|
|
||||||
validated_network.get("root_password_hash"),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Keep Linux hostname valid even when the libvirt VM name contains dots or
|
|
||||||
# underscores. The libvirt name itself remains unchanged.
|
|
||||||
hostname = default_guest_hostname(name, instance_id)
|
|
||||||
use_public_key = bool(network["guest_ssh_public_key"])
|
|
||||||
use_password = bool(network.get("root_password_hash"))
|
|
||||||
root_user: dict[str, Any] = {"name": "root", "lock_passwd": not use_password}
|
|
||||||
if use_public_key:
|
|
||||||
root_user["ssh_authorized_keys"] = [network["guest_ssh_public_key"]]
|
|
||||||
if use_password:
|
|
||||||
root_user["hashed_passwd"] = network["root_password_hash"]
|
|
||||||
# RHEL/AlmaLinux 10 defaults to prohibit-password for root. An early
|
|
||||||
# drop-in makes the two requested authentication modes explicit; cloud-init
|
|
||||||
# itself writes a later 50-cloud-init.conf for PasswordAuthentication.
|
|
||||||
sshd_config = (
|
|
||||||
("PermitRootLogin yes\nPasswordAuthentication yes\n" if use_password else
|
|
||||||
"PermitRootLogin prohibit-password\nPasswordAuthentication no\n")
|
|
||||||
+ "KbdInteractiveAuthentication no\n"
|
|
||||||
)
|
|
||||||
write_files = [{
|
|
||||||
"path": "/etc/ssh/sshd_config.d/00-virsh-panel-root.conf",
|
|
||||||
"content": sshd_config,
|
|
||||||
"permissions": "0644",
|
|
||||||
}]
|
|
||||||
user_data: dict[str, Any] = {
|
|
||||||
"users": [root_user],
|
|
||||||
"ssh_pwauth": use_password,
|
|
||||||
"disable_root": False,
|
|
||||||
"write_files": write_files,
|
|
||||||
# Cloud-init renders a datasource network configuration only on the
|
|
||||||
# first boot by default. Public-IP replacement updates the NoCloud
|
|
||||||
# seed and reboots the guest, so explicitly opt in to applying the
|
|
||||||
# current network-config on every boot.
|
|
||||||
"updates": {"network": {"when": ["boot"]}},
|
|
||||||
}
|
|
||||||
if use_password:
|
|
||||||
_set_root_chpasswd(user_data, network["root_password_hash"])
|
|
||||||
if debian12_network_fix:
|
|
||||||
# NoCloud normally consumes user-data in its network stage. Debian 12
|
|
||||||
# can wait for a still-disabled systemd-networkd before reaching that
|
|
||||||
# stage, so consume this local seed during init-local instead. The
|
|
||||||
# network itself remains entirely in the standard network-config file;
|
|
||||||
# bootcmd only prepares the renderer and starts its backend.
|
|
||||||
queue_networkd = "\n".join([
|
|
||||||
# cloud-init clean --machine-id can leave a custom Debian image
|
|
||||||
# without this file. The systemd helper initializes a valid ID
|
|
||||||
# from the guest's system sources and is a no-op when one already
|
|
||||||
# exists.
|
|
||||||
"systemd-machine-id-setup",
|
|
||||||
# cloud-init 22.4 on Debian 12 can write the Netplan YAML during
|
|
||||||
# init-local without generating networkd's runtime configuration.
|
|
||||||
"netplan generate",
|
|
||||||
"systemctl unmask systemd-networkd.service",
|
|
||||||
"systemctl enable systemd-networkd.service",
|
|
||||||
"systemctl start --no-block systemd-networkd.service",
|
|
||||||
])
|
|
||||||
user_data["bootcmd"] = [
|
|
||||||
["/bin/sh", "-ec", queue_networkd],
|
|
||||||
]
|
|
||||||
metadata = {
|
|
||||||
"instance-id": instance_id,
|
|
||||||
"local-hostname": hostname,
|
|
||||||
# This marker lets a later IP change distinguish a new panel seed
|
|
||||||
# (already carrying the network refresh policy) from a legacy seed
|
|
||||||
# that needs one cache-invalidating instance generation.
|
|
||||||
_NETWORK_POLICY_KEY: _NETWORK_POLICY_VALUE,
|
|
||||||
}
|
|
||||||
if debian12_network_fix:
|
|
||||||
# Consume bootcmd in cloud-init-local, before systemd waits for
|
|
||||||
# network-online. This stays scoped to the Debian 12 compatibility
|
|
||||||
# path because remote user-data cannot be fetched in local mode.
|
|
||||||
metadata["dsmode"] = "local"
|
|
||||||
network_config = {
|
|
||||||
"version": 2,
|
|
||||||
"ethernets": {
|
|
||||||
# ``public0`` is only a stable Netplan/cloud-init configuration
|
|
||||||
# identifier. Do not rename the guest interface: Debian 12 can
|
|
||||||
# apply a first-boot rename after network activation has already
|
|
||||||
# run, leaving the renamed device up but without its static IP.
|
|
||||||
# Matching the libvirt MAC applies this configuration directly to
|
|
||||||
# the kernel-provided interface name (for example ``enp1s0``).
|
|
||||||
"public0": {
|
|
||||||
"match": {"macaddress": normalized_mac},
|
|
||||||
"dhcp4": False,
|
|
||||||
"dhcp6": False,
|
|
||||||
"addresses": [network["public_ip_cidr"]],
|
|
||||||
"routes": [{"to": "0.0.0.0/0", "via": network["gateway"]}],
|
|
||||||
"nameservers": {"addresses": network["dns_servers"]},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if debian12_network_fix:
|
|
||||||
# dhcp6=false does not disable DHCPv6 started in response to a router
|
|
||||||
# advertisement. This product currently provisions IPv4 only, so
|
|
||||||
# express that intent through Netplan instead of writing a second,
|
|
||||||
# provider-specific .network file.
|
|
||||||
network_config["ethernets"]["public0"]["accept-ra"] = False
|
|
||||||
network_config["renderer"] = "networkd"
|
|
||||||
|
|
||||||
files = (
|
|
||||||
("/USERDATA.;1", "user-data", "/user-data",
|
|
||||||
b"#cloud-config\n" + json.dumps(user_data, ensure_ascii=True).encode("ascii") + b"\n"),
|
|
||||||
("/METADATA.;1", "meta-data", "/meta-data",
|
|
||||||
json.dumps(metadata, ensure_ascii=True).encode("ascii") + b"\n"),
|
|
||||||
("/NETWORK.;1", "network-config", "/network-config",
|
|
||||||
json.dumps(network_config, ensure_ascii=True).encode("ascii") + b"\n"),
|
|
||||||
)
|
|
||||||
return _build_seed_iso(files)
|
|
||||||
|
|
||||||
|
|
||||||
def _build_seed_iso(files: tuple[tuple[str, str, str, bytes], ...]) -> bytes:
|
|
||||||
"""Build a bounded CIDATA image from trusted, already-encoded files."""
|
|
||||||
import pycdlib
|
|
||||||
|
|
||||||
iso = pycdlib.PyCdlib()
|
|
||||||
payloads: list[BytesIO] = []
|
|
||||||
try:
|
|
||||||
iso.new(vol_ident="CIDATA", rock_ridge="1.09", joliet=3)
|
|
||||||
for iso_path, rock_ridge_name, joliet_path, contents in files:
|
|
||||||
if not isinstance(contents, bytes) or len(contents) > 256 * 1024:
|
|
||||||
raise ValueError("cloud-init 配置文件过大")
|
|
||||||
payload = BytesIO(contents)
|
|
||||||
payloads.append(payload) # pycdlib reads these during write_fp.
|
|
||||||
iso.add_fp(payload, len(contents), iso_path=iso_path,
|
|
||||||
rr_name=rock_ridge_name, joliet_path=joliet_path)
|
|
||||||
output = BytesIO()
|
|
||||||
iso.write_fp(output)
|
|
||||||
result = output.getvalue()
|
|
||||||
if not result or len(result) > 1024 * 1024:
|
|
||||||
raise ValueError("cloud-init 配置盘大小无效")
|
|
||||||
return result
|
|
||||||
finally:
|
|
||||||
iso.close()
|
|
||||||
|
|
||||||
|
|
||||||
def _enable_network_updates(user_data: bytes) -> bytes:
|
|
||||||
"""Ensure a panel cloud-config reapplies datasource networking on boot.
|
|
||||||
|
|
||||||
Panel-created user-data is a bounded JSON document prefixed with
|
|
||||||
``#cloud-config``. Keeping this transformation limited to that format
|
|
||||||
avoids guessing how an arbitrary user-data document should be merged while
|
|
||||||
still allowing old panel instances (created before IP replacement was
|
|
||||||
introduced) to opt in when their seed is updated.
|
|
||||||
"""
|
|
||||||
if (not isinstance(user_data, bytes) or not user_data
|
|
||||||
or len(user_data) > 256 * 1024):
|
|
||||||
raise ValueError("原 cloud-init 用户数据大小无效")
|
|
||||||
prefix = b"#cloud-config\n"
|
|
||||||
if not user_data.startswith(prefix):
|
|
||||||
raise ValueError("原 cloud-init 用户数据格式无效")
|
|
||||||
try:
|
|
||||||
config = json.loads(user_data[len(prefix):])
|
|
||||||
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
||||||
raise ValueError("原 cloud-init 用户数据格式无效") from exc
|
|
||||||
if not isinstance(config, dict) or not config:
|
|
||||||
raise ValueError("原 cloud-init 用户数据格式无效")
|
|
||||||
updates = config.get("updates")
|
|
||||||
if updates is None:
|
|
||||||
updates = {}
|
|
||||||
elif not isinstance(updates, dict):
|
|
||||||
raise ValueError("原 cloud-init 更新策略格式无效")
|
|
||||||
network = updates.get("network")
|
|
||||||
if network is None:
|
|
||||||
network = {}
|
|
||||||
elif not isinstance(network, dict):
|
|
||||||
raise ValueError("原 cloud-init 网络更新策略格式无效")
|
|
||||||
when = network.get("when")
|
|
||||||
if when is None:
|
|
||||||
when = []
|
|
||||||
elif (not isinstance(when, list)
|
|
||||||
or any(not isinstance(event, str) for event in when)):
|
|
||||||
raise ValueError("原 cloud-init 网络更新事件格式无效")
|
|
||||||
if "boot" not in when:
|
|
||||||
when.append("boot")
|
|
||||||
network["when"] = when
|
|
||||||
updates["network"] = network
|
|
||||||
config["updates"] = updates
|
|
||||||
# Avoid changing the seed (and its digest) when the policy is already
|
|
||||||
# present. This also preserves the original opaque user-data bytes for
|
|
||||||
# current instances while upgrading older panel-generated seeds.
|
|
||||||
if user_data[len(prefix):].rstrip(b"\n") == (
|
|
||||||
json.dumps(config, ensure_ascii=True).encode("ascii")):
|
|
||||||
return user_data
|
|
||||||
return (prefix + json.dumps(config, ensure_ascii=True).encode("ascii")
|
|
||||||
+ b"\n")
|
|
||||||
|
|
||||||
|
|
||||||
def _decode_seed_metadata(meta_data: bytes) -> dict[str, Any]:
|
|
||||||
"""Decode and validate the JSON metadata emitted by this panel."""
|
|
||||||
|
|
||||||
if (not isinstance(meta_data, bytes) or not meta_data
|
|
||||||
or len(meta_data) > 256 * 1024):
|
|
||||||
raise ValueError("原 cloud-init 元数据大小无效")
|
|
||||||
try:
|
|
||||||
metadata = json.loads(meta_data)
|
|
||||||
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
||||||
raise ValueError("原 cloud-init 元数据格式无效") from exc
|
|
||||||
if (not isinstance(metadata, dict) or not metadata
|
|
||||||
or not isinstance(metadata.get("instance-id"), str)
|
|
||||||
or not metadata["instance-id"].strip()):
|
|
||||||
raise ValueError("原 cloud-init 元数据缺少有效实例标识")
|
|
||||||
return metadata
|
|
||||||
|
|
||||||
|
|
||||||
def _encode_seed_metadata(metadata: dict[str, Any]) -> bytes:
|
|
||||||
"""Encode bounded panel metadata without exposing credentials."""
|
|
||||||
|
|
||||||
encoded = json.dumps(metadata, ensure_ascii=True).encode("ascii") + b"\n"
|
|
||||||
if len(encoded) > 256 * 1024:
|
|
||||||
raise ValueError("原 cloud-init 元数据过大")
|
|
||||||
return encoded
|
|
||||||
|
|
||||||
|
|
||||||
def _prepare_seed_metadata(meta_data: bytes, *, rotate: bool) -> bytes:
|
|
||||||
"""Mark a seed as network-refresh capable, rotating its ID when needed."""
|
|
||||||
|
|
||||||
metadata = _decode_seed_metadata(meta_data)
|
|
||||||
if rotate:
|
|
||||||
metadata["instance-id"] = f"virt-cloud-network-{uuid.uuid4().hex}"
|
|
||||||
metadata[_NETWORK_POLICY_KEY] = _NETWORK_POLICY_VALUE
|
|
||||||
return _encode_seed_metadata(metadata)
|
|
||||||
|
|
||||||
|
|
||||||
def replace_seed_network(
|
|
||||||
seed: bytes,
|
|
||||||
public_ip_cidr: str,
|
|
||||||
gateway: str,
|
|
||||||
dns_servers: list[str] | tuple[str, ...],
|
|
||||||
) -> bytes:
|
|
||||||
"""Replace network settings in a panel-created NoCloud seed.
|
|
||||||
|
|
||||||
The guest credentials and hostname fields are retained. The generated
|
|
||||||
user-data is upgraded with the network-on-boot policy. A legacy seed gets
|
|
||||||
one fresh NoCloud instance generation so its cached datasource is
|
|
||||||
invalidated; already-migrated seeds keep their generation to avoid
|
|
||||||
re-running unrelated per-instance user-data modules on every IP change.
|
|
||||||
Reject non-panel seeds instead of trying to infer their format.
|
|
||||||
"""
|
|
||||||
if not isinstance(seed, bytes) or not seed or len(seed) > 1024 * 1024:
|
|
||||||
raise ValueError("原 cloud-init 配置盘大小无效")
|
|
||||||
import pycdlib
|
|
||||||
|
|
||||||
iso = pycdlib.PyCdlib()
|
|
||||||
opened = False
|
|
||||||
files: dict[str, bytes] = {}
|
|
||||||
try:
|
|
||||||
iso.open_fp(BytesIO(seed))
|
|
||||||
opened = True
|
|
||||||
for rr_path, key in (("/user-data", "user-data"),
|
|
||||||
("/meta-data", "meta-data"),
|
|
||||||
("/network-config", "network-config")):
|
|
||||||
output = BytesIO()
|
|
||||||
iso.get_file_from_iso_fp(output, rr_path=rr_path)
|
|
||||||
value = output.getvalue()
|
|
||||||
if not value or len(value) > 256 * 1024:
|
|
||||||
raise ValueError("原 cloud-init 配置文件大小无效")
|
|
||||||
files[key] = value
|
|
||||||
except (OSError, KeyError, TypeError, ValueError,
|
|
||||||
pycdlib.pycdlibexception.PyCdlibException) as exc:
|
|
||||||
raise ValueError("无法读取原 cloud-init 配置盘") from exc
|
|
||||||
finally:
|
|
||||||
if opened:
|
|
||||||
iso.close()
|
|
||||||
|
|
||||||
try:
|
|
||||||
user_data = _enable_network_updates(files["user-data"])
|
|
||||||
original_metadata = _decode_seed_metadata(files["meta-data"])
|
|
||||||
# A legacy seed has neither the explicit marker nor the boot policy in
|
|
||||||
# its cached user-data. Rotate its NoCloud generation once so the
|
|
||||||
# guest consumes this migration. New seeds and already-migrated
|
|
||||||
# seeds can refresh network-config through the supported boot event
|
|
||||||
# without re-running all per-instance user-data modules.
|
|
||||||
needs_generation = (
|
|
||||||
user_data != files["user-data"]
|
|
||||||
or original_metadata.get(_NETWORK_POLICY_KEY)
|
|
||||||
!= _NETWORK_POLICY_VALUE
|
|
||||||
)
|
|
||||||
meta_data = _prepare_seed_metadata(
|
|
||||||
files["meta-data"], rotate=needs_generation)
|
|
||||||
config = json.loads(files["network-config"])
|
|
||||||
if (config.get("version") != 2
|
|
||||||
or not isinstance(config.get("ethernets"), dict)
|
|
||||||
or set(config["ethernets"]) != {"public0"}):
|
|
||||||
raise ValueError
|
|
||||||
public = config["ethernets"]["public0"]
|
|
||||||
match = public.get("match")
|
|
||||||
mac = match.get("macaddress") if isinstance(match, dict) else None
|
|
||||||
if (public.get("dhcp4") is not False or public.get("dhcp6") is not False
|
|
||||||
or not isinstance(mac, str) or not _MAC.fullmatch(mac)
|
|
||||||
or not isinstance(public.get("addresses"), list)
|
|
||||||
or len(public["addresses"]) != 1
|
|
||||||
or not isinstance(public.get("routes"), list)
|
|
||||||
or len(public["routes"]) != 1):
|
|
||||||
raise ValueError
|
|
||||||
route = public["routes"][0]
|
|
||||||
if route.get("to") != "0.0.0.0/0":
|
|
||||||
raise ValueError
|
|
||||||
# Validate only fields that are changing. No credential is accepted
|
|
||||||
# by this operation; only the network-on-boot policy is merged into
|
|
||||||
# the panel-generated user-data above.
|
|
||||||
network = validate_cloud_network(
|
|
||||||
public_ip_cidr, gateway, dns_servers, require_auth=False)
|
|
||||||
public["addresses"] = [network["public_ip_cidr"]]
|
|
||||||
route["via"] = network["gateway"]
|
|
||||||
nameservers = public.get("nameservers")
|
|
||||||
if not isinstance(nameservers, dict):
|
|
||||||
raise ValueError
|
|
||||||
nameservers["addresses"] = list(network["dns_servers"])
|
|
||||||
network_data = (json.dumps(config, ensure_ascii=True).encode("ascii")
|
|
||||||
+ b"\n")
|
|
||||||
rebuilt = (
|
|
||||||
("/USERDATA.;1", "user-data", "/user-data", user_data),
|
|
||||||
("/METADATA.;1", "meta-data", "/meta-data", meta_data),
|
|
||||||
("/NETWORK.;1", "network-config", "/network-config", network_data),
|
|
||||||
)
|
|
||||||
return _build_seed_iso(rebuilt)
|
|
||||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
||||||
raise ValueError("原 cloud-init 网络配置格式无效") from exc
|
|
||||||
|
|
||||||
|
|
||||||
def _read_seed_files(seed: bytes) -> dict[str, bytes]:
|
|
||||||
"""Read the three bounded files from a panel-generated NoCloud seed."""
|
|
||||||
if not isinstance(seed, bytes) or not seed or len(seed) > 1024 * 1024:
|
|
||||||
raise ValueError("原 cloud-init 配置盘大小无效")
|
|
||||||
import pycdlib
|
|
||||||
|
|
||||||
iso = pycdlib.PyCdlib()
|
|
||||||
opened = False
|
|
||||||
files: dict[str, bytes] = {}
|
|
||||||
try:
|
|
||||||
iso.open_fp(BytesIO(seed))
|
|
||||||
opened = True
|
|
||||||
for rr_path, key in (("/user-data", "user-data"),
|
|
||||||
("/meta-data", "meta-data"),
|
|
||||||
("/network-config", "network-config")):
|
|
||||||
output = BytesIO()
|
|
||||||
iso.get_file_from_iso_fp(output, rr_path=rr_path)
|
|
||||||
value = output.getvalue()
|
|
||||||
if not value or len(value) > 256 * 1024:
|
|
||||||
raise ValueError("原 cloud-init 配置文件大小无效")
|
|
||||||
files[key] = value
|
|
||||||
except (OSError, KeyError, TypeError, ValueError,
|
|
||||||
pycdlib.pycdlibexception.PyCdlibException) as exc:
|
|
||||||
raise ValueError("无法读取原 cloud-init 配置盘") from exc
|
|
||||||
finally:
|
|
||||||
if opened:
|
|
||||||
iso.close()
|
|
||||||
return files
|
|
||||||
|
|
||||||
|
|
||||||
def _decode_seed_user_data(user_data: bytes) -> dict[str, Any]:
|
|
||||||
"""Decode the bounded JSON cloud-config emitted by this panel."""
|
|
||||||
prefix = b"#cloud-config\n"
|
|
||||||
if (not isinstance(user_data, bytes) or not user_data
|
|
||||||
or len(user_data) > 256 * 1024
|
|
||||||
or not user_data.startswith(prefix)):
|
|
||||||
raise ValueError("原 cloud-init 用户数据格式无效")
|
|
||||||
try:
|
|
||||||
value = json.loads(user_data[len(prefix):])
|
|
||||||
except (TypeError, ValueError, UnicodeError, json.JSONDecodeError) as exc:
|
|
||||||
raise ValueError("原 cloud-init 用户数据格式无效") from exc
|
|
||||||
if not isinstance(value, dict) or not value:
|
|
||||||
raise ValueError("原 cloud-init 用户数据格式无效")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _root_user(config: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
users = config.get("users")
|
|
||||||
if not isinstance(users, list) or len(users) > 32:
|
|
||||||
raise ValueError("原 cloud-init 用户配置格式无效")
|
|
||||||
matches = [item for item in users
|
|
||||||
if isinstance(item, dict) and item.get("name") == "root"]
|
|
||||||
if len(matches) != 1:
|
|
||||||
raise ValueError("原 cloud-init root 用户配置格式无效")
|
|
||||||
root = matches[0]
|
|
||||||
lock_passwd = root.get("lock_passwd")
|
|
||||||
if lock_passwd is not None and not isinstance(lock_passwd, bool):
|
|
||||||
raise ValueError("原 cloud-init root 用户配置格式无效")
|
|
||||||
existing_hash = root.get("hashed_passwd")
|
|
||||||
if existing_hash is not None:
|
|
||||||
try:
|
|
||||||
_validate_guest_auth("", existing_hash)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise ValueError("原 cloud-init root 密码配置格式无效") from exc
|
|
||||||
if lock_passwd is True:
|
|
||||||
raise ValueError("原 cloud-init root 密码配置无效")
|
|
||||||
keys = root.get("ssh_authorized_keys")
|
|
||||||
if keys is not None:
|
|
||||||
if (not isinstance(keys, list) or len(keys) > 64
|
|
||||||
or any(not isinstance(item, str) for item in keys)):
|
|
||||||
raise ValueError("原 cloud-init SSH 公钥配置格式无效")
|
|
||||||
for key in keys:
|
|
||||||
try:
|
|
||||||
_validate_ssh_public_key(key)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise ValueError("原 cloud-init SSH 公钥配置格式无效") from exc
|
|
||||||
return root
|
|
||||||
|
|
||||||
|
|
||||||
def _update_root_sshd_config(config: dict[str, Any], *, use_password: bool) -> None:
|
|
||||||
"""Keep the panel-managed root sshd drop-in aligned with credentials."""
|
|
||||||
if not isinstance(use_password, bool):
|
|
||||||
raise ValueError("root 密码配置无效")
|
|
||||||
write_files = config.get("write_files")
|
|
||||||
if write_files is None:
|
|
||||||
write_files = []
|
|
||||||
config["write_files"] = write_files
|
|
||||||
if not isinstance(write_files, list) or len(write_files) > 128:
|
|
||||||
raise ValueError("原 cloud-init 写入文件配置格式无效")
|
|
||||||
path = "/etc/ssh/sshd_config.d/00-virsh-panel-root.conf"
|
|
||||||
matches = [item for item in write_files
|
|
||||||
if isinstance(item, dict) and item.get("path") == path]
|
|
||||||
if len(matches) > 1:
|
|
||||||
raise ValueError("原 cloud-init SSH 配置重复")
|
|
||||||
content = (
|
|
||||||
("PermitRootLogin yes\nPasswordAuthentication yes\n" if use_password
|
|
||||||
else "PermitRootLogin prohibit-password\nPasswordAuthentication no\n")
|
|
||||||
+ "KbdInteractiveAuthentication no\n"
|
|
||||||
)
|
|
||||||
if matches:
|
|
||||||
item = matches[0]
|
|
||||||
if (item.get("permissions", "0644") != "0644"
|
|
||||||
or any(key not in {"path", "content", "permissions"}
|
|
||||||
for key in item)):
|
|
||||||
raise ValueError("原 cloud-init SSH 配置格式无效")
|
|
||||||
item["content"] = content
|
|
||||||
item["permissions"] = "0644"
|
|
||||||
else:
|
|
||||||
write_files.append({"path": path, "content": content,
|
|
||||||
"permissions": "0644"})
|
|
||||||
config["ssh_pwauth"] = use_password
|
|
||||||
config["disable_root"] = False
|
|
||||||
|
|
||||||
|
|
||||||
def _replace_root_authorized_keys(config: dict[str, Any], public_key: str) -> None:
|
|
||||||
"""Make the requested SSH key replacement explicit and deterministic.
|
|
||||||
|
|
||||||
``users[].ssh_authorized_keys`` is retained for distro compatibility, but
|
|
||||||
a panel-managed write_files entry guarantees that an old authorized_keys
|
|
||||||
line is replaced rather than merely appended by a distro cloud-init
|
|
||||||
implementation. The key has already passed strict OpenSSH validation and
|
|
||||||
is JSON-encoded, so no shell interpolation is involved.
|
|
||||||
"""
|
|
||||||
public_key = _validate_ssh_public_key(public_key)
|
|
||||||
write_files = config.get("write_files")
|
|
||||||
if write_files is None:
|
|
||||||
write_files = []
|
|
||||||
config["write_files"] = write_files
|
|
||||||
if (not isinstance(write_files, list) or len(write_files) > 128
|
|
||||||
or any(not isinstance(item, dict) for item in write_files)):
|
|
||||||
raise ValueError("原 cloud-init 写入文件配置格式无效")
|
|
||||||
path = "/root/.ssh/authorized_keys"
|
|
||||||
matches = [item for item in write_files if item.get("path") == path]
|
|
||||||
if len(matches) > 1:
|
|
||||||
raise ValueError("原 cloud-init SSH 公钥文件配置重复")
|
|
||||||
replacement = {
|
|
||||||
"path": path,
|
|
||||||
"content": public_key + "\n",
|
|
||||||
"owner": "root:root",
|
|
||||||
"permissions": "0600",
|
|
||||||
}
|
|
||||||
if matches:
|
|
||||||
item = matches[0]
|
|
||||||
if any(key not in {"path", "content", "owner", "permissions"}
|
|
||||||
for key in item):
|
|
||||||
raise ValueError("原 cloud-init SSH 公钥文件配置格式无效")
|
|
||||||
item.clear()
|
|
||||||
item.update(replacement)
|
|
||||||
else:
|
|
||||||
write_files.append(replacement)
|
|
||||||
|
|
||||||
|
|
||||||
def _set_root_chpasswd(config: dict[str, Any], root_password_hash: str) -> None:
|
|
||||||
"""Set a hashed root password through cloud-init's password module.
|
|
||||||
|
|
||||||
``users[].hashed_passwd`` is retained for older cloud-init releases, but
|
|
||||||
recent releases have reported existing users as locked even when that
|
|
||||||
field is paired with ``lock_passwd: false``. The ``chpasswd`` module runs
|
|
||||||
after user creation and applies the hash to an already-existing root
|
|
||||||
account, which makes password replacement consistent across supported
|
|
||||||
AlmaLinux, Debian and Ubuntu images. The hash is never converted back to
|
|
||||||
plaintext.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
_validate_guest_auth("", root_password_hash)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise ValueError("root 密码哈希格式无效") from exc
|
|
||||||
config["chpasswd"] = {
|
|
||||||
"expire": False,
|
|
||||||
"users": [{
|
|
||||||
"name": "root",
|
|
||||||
"password": root_password_hash,
|
|
||||||
"type": "hash",
|
|
||||||
}],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _remove_root_chpasswd(config: dict[str, Any]) -> None:
|
|
||||||
"""Remove a panel-managed root chpasswd entry when password auth is off."""
|
|
||||||
chpasswd = config.get("chpasswd")
|
|
||||||
if chpasswd is None:
|
|
||||||
return
|
|
||||||
if not isinstance(chpasswd, dict):
|
|
||||||
raise ValueError("原 cloud-init 密码配置格式无效")
|
|
||||||
users = chpasswd.get("users")
|
|
||||||
if users is None:
|
|
||||||
config.pop("chpasswd", None)
|
|
||||||
return
|
|
||||||
if not isinstance(users, list) or len(users) > 32:
|
|
||||||
raise ValueError("原 cloud-init 密码配置格式无效")
|
|
||||||
root_count = sum(
|
|
||||||
1 for item in users
|
|
||||||
if isinstance(item, dict) and item.get("name") == "root"
|
|
||||||
)
|
|
||||||
if root_count > 1:
|
|
||||||
raise ValueError("原 cloud-init root 密码配置重复")
|
|
||||||
remaining = [item for item in users
|
|
||||||
if not (isinstance(item, dict) and item.get("name") == "root")]
|
|
||||||
if remaining:
|
|
||||||
chpasswd["users"] = remaining
|
|
||||||
else:
|
|
||||||
config.pop("chpasswd", None)
|
|
||||||
|
|
||||||
|
|
||||||
def replace_seed_settings(
|
|
||||||
seed: bytes,
|
|
||||||
*,
|
|
||||||
hostname: str | None = None,
|
|
||||||
root_password_hash: str | None = None,
|
|
||||||
ssh_public_key: str | None = None,
|
|
||||||
) -> bytes:
|
|
||||||
"""Replace panel-managed guest hostname/password/key in a NoCloud seed.
|
|
||||||
|
|
||||||
``None`` means "leave this setting unchanged"; an empty value is never a
|
|
||||||
request to clear credentials. The metadata instance ID is rotated so
|
|
||||||
cloud-init treats the next boot as a new panel configuration and applies
|
|
||||||
the changed users/hostname modules. Network data is copied unchanged and
|
|
||||||
is validated through :func:`read_seed_network` before any output is built.
|
|
||||||
"""
|
|
||||||
if hostname is None and root_password_hash is None and ssh_public_key is None:
|
|
||||||
raise ValueError("至少需要修改一项虚拟机设置")
|
|
||||||
if hostname is not None:
|
|
||||||
hostname = validate_guest_hostname(hostname)
|
|
||||||
if root_password_hash is not None:
|
|
||||||
try:
|
|
||||||
_validate_guest_auth("", root_password_hash)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise ValueError("root 密码哈希格式无效") from exc
|
|
||||||
if ssh_public_key is not None:
|
|
||||||
ssh_public_key = _validate_ssh_public_key(ssh_public_key)
|
|
||||||
|
|
||||||
files = _read_seed_files(seed)
|
|
||||||
try:
|
|
||||||
metadata = _decode_seed_metadata(files["meta-data"])
|
|
||||||
# Also verify that this is a panel network seed before retaining it.
|
|
||||||
read_seed_network(seed)
|
|
||||||
# Upgrade legacy panel seeds to the network-on-boot policy while the
|
|
||||||
# settings generation is already being rotated. This keeps a later
|
|
||||||
# IP replacement from silently losing its cloud-init refresh hook.
|
|
||||||
user_data = _decode_seed_user_data(
|
|
||||||
_enable_network_updates(files["user-data"])
|
|
||||||
)
|
|
||||||
root = _root_user(user_data)
|
|
||||||
except (KeyError, TypeError, ValueError) as exc:
|
|
||||||
raise ValueError("原 cloud-init 配置格式无效") from exc
|
|
||||||
|
|
||||||
if hostname is not None:
|
|
||||||
metadata["local-hostname"] = hostname
|
|
||||||
if not isinstance(metadata.get("local-hostname"), str):
|
|
||||||
raise ValueError("原 cloud-init 主机名配置格式无效")
|
|
||||||
|
|
||||||
if root_password_hash is not None:
|
|
||||||
root["hashed_passwd"] = root_password_hash
|
|
||||||
root["lock_passwd"] = False
|
|
||||||
if ssh_public_key is not None:
|
|
||||||
root["ssh_authorized_keys"] = [ssh_public_key]
|
|
||||||
_replace_root_authorized_keys(user_data, ssh_public_key)
|
|
||||||
|
|
||||||
# A panel-generated seed either has a validated hash or explicitly locks
|
|
||||||
# root. Preserve the existing state for key-only changes; changing a
|
|
||||||
# password always enables password authentication above.
|
|
||||||
use_password = bool(root.get("hashed_passwd"))
|
|
||||||
if "hashed_passwd" not in root and root.get("lock_passwd") is False:
|
|
||||||
# Do not silently preserve a malformed unlocked root record.
|
|
||||||
raise ValueError("原 cloud-init root 密码配置格式无效")
|
|
||||||
root["lock_passwd"] = not use_password
|
|
||||||
if use_password:
|
|
||||||
_set_root_chpasswd(user_data, root["hashed_passwd"])
|
|
||||||
else:
|
|
||||||
_remove_root_chpasswd(user_data)
|
|
||||||
_update_root_sshd_config(user_data, use_password=use_password)
|
|
||||||
user_data = json.loads(json.dumps(user_data, ensure_ascii=True))
|
|
||||||
metadata["instance-id"] = f"virt-cloud-settings-{uuid.uuid4().hex}"
|
|
||||||
metadata[_NETWORK_POLICY_KEY] = _NETWORK_POLICY_VALUE
|
|
||||||
|
|
||||||
rebuilt = (
|
|
||||||
("/USERDATA.;1", "user-data", "/user-data",
|
|
||||||
b"#cloud-config\n" + json.dumps(user_data, ensure_ascii=True).encode("ascii") + b"\n"),
|
|
||||||
("/METADATA.;1", "meta-data", "/meta-data",
|
|
||||||
json.dumps(metadata, ensure_ascii=True).encode("ascii") + b"\n"),
|
|
||||||
("/NETWORK.;1", "network-config", "/network-config",
|
|
||||||
files["network-config"]),
|
|
||||||
)
|
|
||||||
return _build_seed_iso(rebuilt)
|
|
||||||
|
|
||||||
|
|
||||||
def read_seed_network(seed: bytes) -> dict[str, Any]:
|
|
||||||
"""Read the network settings from a panel-created NoCloud ISO.
|
|
||||||
|
|
||||||
Only the network-config file is parsed: user-data can contain a password
|
|
||||||
hash or an SSH key and must never be returned to the web API.
|
|
||||||
"""
|
|
||||||
if not isinstance(seed, bytes) or not seed or len(seed) > 1024 * 1024:
|
|
||||||
raise ValueError("原 cloud-init 配置盘大小无效")
|
|
||||||
import pycdlib
|
|
||||||
|
|
||||||
iso = pycdlib.PyCdlib()
|
|
||||||
opened = False
|
|
||||||
try:
|
|
||||||
iso.open_fp(BytesIO(seed))
|
|
||||||
opened = True
|
|
||||||
output = BytesIO()
|
|
||||||
iso.get_file_from_iso_fp(output, rr_path="/network-config")
|
|
||||||
raw = output.getvalue()
|
|
||||||
if len(raw) > 65536:
|
|
||||||
raise ValueError("原网络配置过大")
|
|
||||||
config = json.loads(raw)
|
|
||||||
except (OSError, KeyError, TypeError, ValueError, pycdlib.pycdlibexception.PyCdlibException) as exc:
|
|
||||||
raise ValueError("无法读取原 cloud-init 网络配置") from exc
|
|
||||||
finally:
|
|
||||||
if opened:
|
|
||||||
iso.close()
|
|
||||||
|
|
||||||
try:
|
|
||||||
if config["version"] != 2:
|
|
||||||
raise ValueError
|
|
||||||
ethernets = config["ethernets"]
|
|
||||||
if not isinstance(ethernets, dict) or set(ethernets) != {"public0"}:
|
|
||||||
raise ValueError
|
|
||||||
public = ethernets["public0"]
|
|
||||||
if public["dhcp4"] is not False or public["dhcp6"] is not False:
|
|
||||||
raise ValueError
|
|
||||||
addresses = public["addresses"]
|
|
||||||
routes = public["routes"]
|
|
||||||
dns = public["nameservers"]["addresses"]
|
|
||||||
mac = public["match"]["macaddress"]
|
|
||||||
if (not isinstance(addresses, list) or len(addresses) != 1
|
|
||||||
or not isinstance(routes, list) or len(routes) != 1
|
|
||||||
or routes[0]["to"] != "0.0.0.0/0"
|
|
||||||
or not isinstance(mac, str) or not _MAC.fullmatch(mac)):
|
|
||||||
raise ValueError
|
|
||||||
return {
|
|
||||||
"public_ip_cidr": addresses[0],
|
|
||||||
"gateway": routes[0]["via"],
|
|
||||||
"dns_servers": dns,
|
|
||||||
"mac": mac.lower(),
|
|
||||||
}
|
|
||||||
except (KeyError, TypeError, ValueError) as exc:
|
|
||||||
raise ValueError("原 cloud-init 网络配置格式无效") from exc
|
|
||||||
@ -1,247 +0,0 @@
|
|||||||
"""PHP-owned identities, stable host namespaces, and separate engine databases."""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import threading
|
|
||||||
from contextvars import ContextVar
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from types import SimpleNamespace
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from .auth import Session
|
|
||||||
from .database import (DatabaseError, DatabaseRemoteHostManager, DatabaseStore,
|
|
||||||
DatabaseTemplateStore, _METRIC_MIGRATIONS, _METRIC_SCHEMA,
|
|
||||||
_SCHEMA, _TRAFFIC_CHECKPOINT_SCHEMA, _TRAFFIC_SCHEMA, pymysql)
|
|
||||||
|
|
||||||
|
|
||||||
TRUSTED_ACTOR: ContextVar[dict | None] = ContextVar("cloud_trusted_actor", default=None)
|
|
||||||
|
|
||||||
|
|
||||||
def actor() -> dict:
|
|
||||||
value = TRUSTED_ACTOR.get()
|
|
||||||
if value is None:
|
|
||||||
raise PermissionError("Missing authenticated PHP actor")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def actor_session() -> Session:
|
|
||||||
value = actor()
|
|
||||||
return Session("internal", f"php:{value['user_id']}", "", float("inf"), value["user_id"])
|
|
||||||
|
|
||||||
|
|
||||||
def actor_account(session: Session) -> dict:
|
|
||||||
value = actor()
|
|
||||||
if session.user_id != value["user_id"]:
|
|
||||||
raise PermissionError("Actor does not match request")
|
|
||||||
return {"id": value["user_id"], "username": f"php:{value['user_id']}",
|
|
||||||
"admin": value["is_admin"], "approved": True, "email_verified": True}
|
|
||||||
|
|
||||||
|
|
||||||
class MainStore(DatabaseStore):
|
|
||||||
RECORD_TABLE = "engine_records"
|
|
||||||
|
|
||||||
def _initialize_schema(self) -> None:
|
|
||||||
connection = self._connect()
|
|
||||||
try:
|
|
||||||
with connection.cursor() as cursor:
|
|
||||||
cursor.execute(_SCHEMA.replace("panel_records", self.RECORD_TABLE))
|
|
||||||
cursor.execute("""CREATE TABLE IF NOT EXISTS engine_request_nonces (
|
|
||||||
nonce VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
|
||||||
expires_at BIGINT UNSIGNED NOT NULL,
|
|
||||||
PRIMARY KEY(nonce), KEY nonce_expiry(expires_at)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4""")
|
|
||||||
connection.commit()
|
|
||||||
except pymysql.MySQLError as exc:
|
|
||||||
connection.rollback()
|
|
||||||
raise DatabaseError("无法初始化引擎数据库") from exc
|
|
||||||
finally:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
def consume_nonce(self, nonce: str, expires_at: int, now: int) -> bool:
|
|
||||||
connection = self._connect()
|
|
||||||
try:
|
|
||||||
with connection.cursor() as cursor:
|
|
||||||
cursor.execute("DELETE FROM engine_request_nonces WHERE expires_at < %s LIMIT 1000", (now,))
|
|
||||||
cursor.execute("INSERT INTO engine_request_nonces(nonce,expires_at) VALUES(%s,%s)",
|
|
||||||
(nonce, expires_at))
|
|
||||||
connection.commit()
|
|
||||||
return True
|
|
||||||
except pymysql.err.IntegrityError as exc:
|
|
||||||
connection.rollback()
|
|
||||||
if exc.args and exc.args[0] == 1062:
|
|
||||||
return False
|
|
||||||
raise DatabaseError() from exc
|
|
||||||
except pymysql.MySQLError as exc:
|
|
||||||
connection.rollback()
|
|
||||||
raise DatabaseError() from exc
|
|
||||||
finally:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
class LogStore(DatabaseStore):
|
|
||||||
METRIC_TABLE = "engine_vm_metric_samples"
|
|
||||||
TRAFFIC_TABLE = "engine_vm_traffic_totals"
|
|
||||||
TRAFFIC_CHECKPOINT_TABLE = "engine_vm_traffic_checkpoints"
|
|
||||||
|
|
||||||
def _initialize_schema(self) -> None:
|
|
||||||
connection = self._connect()
|
|
||||||
try:
|
|
||||||
with connection.cursor() as cursor:
|
|
||||||
cursor.execute(_METRIC_SCHEMA.replace("vm_metric_samples", self.METRIC_TABLE))
|
|
||||||
cursor.execute(_TRAFFIC_SCHEMA.replace("vm_traffic_totals", self.TRAFFIC_TABLE))
|
|
||||||
cursor.execute(_TRAFFIC_CHECKPOINT_SCHEMA.replace(
|
|
||||||
"vm_traffic_checkpoints", self.TRAFFIC_CHECKPOINT_TABLE))
|
|
||||||
for statement in _METRIC_MIGRATIONS:
|
|
||||||
cursor.execute(statement.format(metric_table=self.METRIC_TABLE))
|
|
||||||
connection.commit()
|
|
||||||
except pymysql.MySQLError as exc:
|
|
||||||
connection.rollback()
|
|
||||||
raise DatabaseError("无法初始化日志库的虚拟机指标表") from exc
|
|
||||||
finally:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
class HostStore:
|
|
||||||
"""One current profile per stable host ID, never one shared current host."""
|
|
||||||
def __init__(self, store: Any, host_id: str):
|
|
||||||
self.store = store
|
|
||||||
self.prefix = hashlib.sha256(host_id.encode("ascii")).hexdigest() + ":"
|
|
||||||
self._compare_lock = threading.RLock()
|
|
||||||
|
|
||||||
def _key(self, key: str) -> str:
|
|
||||||
return self.prefix + key
|
|
||||||
|
|
||||||
def get(self, kind: str, key: str):
|
|
||||||
return self.store.get(kind, self._key(key))
|
|
||||||
|
|
||||||
def set(self, kind: str, key: str, value: str):
|
|
||||||
return self.store.set(kind, self._key(key), value)
|
|
||||||
|
|
||||||
def insert(self, kind: str, key: str, value: str):
|
|
||||||
return self.store.insert(kind, self._key(key), value)
|
|
||||||
|
|
||||||
def compare_and_set(
|
|
||||||
self, kind: str, key: str, expected: str | None, value: str,
|
|
||||||
) -> bool:
|
|
||||||
compare = getattr(self.store, "compare_and_set", None)
|
|
||||||
if callable(compare):
|
|
||||||
return compare(kind, self._key(key), expected, value)
|
|
||||||
# In-memory test stores do not need a database transaction, but they do
|
|
||||||
# need the same process-local semantics as production.
|
|
||||||
with self._compare_lock:
|
|
||||||
current = self.get(kind, key)
|
|
||||||
if current != expected:
|
|
||||||
return False
|
|
||||||
self.set(kind, key, value)
|
|
||||||
return True
|
|
||||||
|
|
||||||
def delete(self, kind: str, key: str):
|
|
||||||
return self.store.delete(kind, self._key(key))
|
|
||||||
|
|
||||||
def list(self, kind: str):
|
|
||||||
return {key[len(self.prefix):]: value for key, value in self.store.list(kind).items()
|
|
||||||
if key.startswith(self.prefix)}
|
|
||||||
|
|
||||||
def get_value(self, key: str):
|
|
||||||
return self.get("value", key)
|
|
||||||
|
|
||||||
def set_value(self, key: str, value: str):
|
|
||||||
return self.set("value", key, value)
|
|
||||||
|
|
||||||
def delete_value(self, key: str):
|
|
||||||
return self.delete("value", key)
|
|
||||||
|
|
||||||
def get_secret(self, key: str):
|
|
||||||
return self.get("secret", key)
|
|
||||||
|
|
||||||
def set_secret(self, key: str, value: str):
|
|
||||||
return self.set("secret", key, value)
|
|
||||||
|
|
||||||
|
|
||||||
class ExternalOwnership:
|
|
||||||
"""Immutable PHP subject -> host/UUID owner mapping, with no users FK."""
|
|
||||||
def __init__(self, store: HostStore):
|
|
||||||
self.store = store
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _key(scope: str, vm_id: str) -> str:
|
|
||||||
return f"{scope}.{vm_id}"
|
|
||||||
|
|
||||||
def get_by_id(self, user_id: int):
|
|
||||||
value = actor()
|
|
||||||
return actor_account(actor_session()) if value["user_id"] == user_id else None
|
|
||||||
|
|
||||||
def owns_vm(self, scope: str, vm_id: str, user_id: int) -> bool:
|
|
||||||
return self.owner_of_vm(scope, vm_id) == user_id
|
|
||||||
|
|
||||||
def owner_of_vm(self, scope: str, vm_id: str) -> int | None:
|
|
||||||
raw = self.store.get("vm_owner", self._key(scope, vm_id))
|
|
||||||
return None if raw is None else int(json.loads(raw)["user_id"])
|
|
||||||
|
|
||||||
def assign_vm(self, scope: str, vm_id: str, user_id: int) -> None:
|
|
||||||
if isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1:
|
|
||||||
raise ValueError("External owner must be a positive PHP user ID")
|
|
||||||
key = self._key(scope, vm_id)
|
|
||||||
payload = json.dumps({"user_id": user_id}, separators=(",", ":"))
|
|
||||||
if not self.store.insert("vm_owner", key, payload):
|
|
||||||
if not self.owns_vm(scope, vm_id, user_id):
|
|
||||||
raise DatabaseError("虚拟机已有其他归属,不能重复分配")
|
|
||||||
|
|
||||||
def list_owned_vm_ids(self, scope: str, user_id: int) -> set[str]:
|
|
||||||
prefix = scope + "."
|
|
||||||
return {key[len(prefix):] for key, raw in self.store.list("vm_owner").items()
|
|
||||||
if key.startswith(prefix) and json.loads(raw)["user_id"] == user_id}
|
|
||||||
|
|
||||||
def delete_vm_owner(self, scope: str, vm_id: str) -> None:
|
|
||||||
self.store.delete("vm_owner", self._key(scope, vm_id))
|
|
||||||
|
|
||||||
|
|
||||||
class StableHostManager(DatabaseRemoteHostManager):
|
|
||||||
def __init__(self, store, data_dir, *, host_id: str, **kwargs):
|
|
||||||
self.host_id = host_id
|
|
||||||
super().__init__(store, data_dir, **kwargs)
|
|
||||||
|
|
||||||
def current(self):
|
|
||||||
profile = super().current()
|
|
||||||
if profile is not None:
|
|
||||||
profile["host_id"] = self.host_id
|
|
||||||
return profile
|
|
||||||
|
|
||||||
def commit(self, staged):
|
|
||||||
profile = super().commit(staged)
|
|
||||||
profile["host_id"] = self.host_id
|
|
||||||
return profile
|
|
||||||
|
|
||||||
def unlock_current(self, passphrase):
|
|
||||||
profile = super().unlock_current(passphrase)
|
|
||||||
profile["host_id"] = self.host_id
|
|
||||||
return profile
|
|
||||||
|
|
||||||
def current_for_background(self):
|
|
||||||
profile = super().current_for_background()
|
|
||||||
if profile is not None:
|
|
||||||
profile["host_id"] = self.host_id
|
|
||||||
return profile
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class EngineComponents:
|
|
||||||
state: Any
|
|
||||||
manager: Any
|
|
||||||
templates: Any
|
|
||||||
store: Any
|
|
||||||
metric_store: Any
|
|
||||||
user_isos: Any | None = None
|
|
||||||
cloud_templates: Any | None = None
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def for_host(cls, store, log_store, host_id, data_dir, user_isos=None,
|
|
||||||
cloud_templates=None):
|
|
||||||
scoped = HostStore(store, host_id)
|
|
||||||
state = SimpleNamespace(users=ExternalOwnership(scoped))
|
|
||||||
manager = StableHostManager(scoped, data_dir, host_id=host_id)
|
|
||||||
return cls(
|
|
||||||
state, manager, DatabaseTemplateStore(scoped), scoped, log_store,
|
|
||||||
user_isos, cloud_templates,
|
|
||||||
)
|
|
||||||
@ -1,100 +0,0 @@
|
|||||||
"""Non-browser HMAC transport: signed actors, payloads, time and replay defense."""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import base64
|
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import secrets
|
|
||||||
import time
|
|
||||||
|
|
||||||
|
|
||||||
class EngineAuthError(Exception):
|
|
||||||
def __init__(self, message="内部服务认证失败", status=401):
|
|
||||||
super().__init__(message)
|
|
||||||
self.status = status
|
|
||||||
|
|
||||||
|
|
||||||
def canonical(method: str, target: str, digest: str, timestamp: str,
|
|
||||||
nonce: str, encoded_actor: str) -> bytes:
|
|
||||||
return "\n".join((method.upper(), target, digest, timestamp, nonce, encoded_actor)).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def decode_secret(encoded: str) -> bytes:
|
|
||||||
try:
|
|
||||||
value = base64.b64decode(encoded, validate=True)
|
|
||||||
except (ValueError, TypeError) as exc:
|
|
||||||
raise ValueError("Engine service secret must be base64-encoded 32 bytes") from exc
|
|
||||||
if len(value) != 32:
|
|
||||||
raise ValueError("Engine service secret must be base64-encoded 32 bytes")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def encode_actor(value: dict) -> str:
|
|
||||||
return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":"),
|
|
||||||
ensure_ascii=True).encode()).decode().rstrip("=")
|
|
||||||
|
|
||||||
|
|
||||||
def signed_headers(secret: bytes, method: str, target: str, body: bytes, actor: dict,
|
|
||||||
*, timestamp: int | None = None, nonce: str | None = None) -> dict:
|
|
||||||
timestamp_raw = str(int(time.time()) if timestamp is None else timestamp)
|
|
||||||
nonce = nonce or secrets.token_urlsafe(24)
|
|
||||||
encoded = encode_actor(actor)
|
|
||||||
digest = hashlib.sha256(body).hexdigest()
|
|
||||||
signature = hmac.new(secret, canonical(method, target, digest, timestamp_raw, nonce, encoded),
|
|
||||||
hashlib.sha256).hexdigest()
|
|
||||||
return {"X-Cloud-Timestamp": timestamp_raw, "X-Cloud-Nonce": nonce,
|
|
||||||
"X-Cloud-Actor": encoded, "X-Cloud-Signature": signature}
|
|
||||||
|
|
||||||
|
|
||||||
def parse_actor(encoded: str) -> dict:
|
|
||||||
if not encoded or len(encoded) > 8192 or not re.fullmatch(r"[A-Za-z0-9_-]+", encoded):
|
|
||||||
raise EngineAuthError()
|
|
||||||
try:
|
|
||||||
raw = base64.b64decode(encoded + "=" * (-len(encoded) % 4), altchars=b"-_", validate=True)
|
|
||||||
value = json.loads(raw)
|
|
||||||
except (ValueError, UnicodeError) as exc:
|
|
||||||
raise EngineAuthError() from exc
|
|
||||||
if not isinstance(value, dict):
|
|
||||||
raise EngineAuthError()
|
|
||||||
user_id = value.get("user_id")
|
|
||||||
if isinstance(user_id, bool) or not isinstance(user_id, int) or not 1 <= user_id < 2**63:
|
|
||||||
raise EngineAuthError()
|
|
||||||
for key in ("is_admin", "is_super_admin", "can_manage"):
|
|
||||||
if not isinstance(value.get(key), bool):
|
|
||||||
raise EngineAuthError()
|
|
||||||
if value["is_super_admin"] and not value["is_admin"]:
|
|
||||||
raise EngineAuthError()
|
|
||||||
for key in ("can_provision", "system_cleanup"):
|
|
||||||
if key in value and not isinstance(value[key], bool):
|
|
||||||
raise EngineAuthError()
|
|
||||||
if "operation_id" in value and not re.fullmatch(r"[A-Za-z0-9_-]{8,80}", str(value["operation_id"])):
|
|
||||||
raise EngineAuthError()
|
|
||||||
session_id = value.get("session_id", "")
|
|
||||||
if not isinstance(session_id, str) or len(session_id) > 256 or any(ord(c) < 33 for c in session_id):
|
|
||||||
raise EngineAuthError()
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def authenticate(headers: dict, secret: bytes, method: str, target: str,
|
|
||||||
digest: str, nonce_store, *, now: int | None = None) -> dict:
|
|
||||||
now = int(time.time()) if now is None else now
|
|
||||||
timestamp_raw = headers.get("x-cloud-timestamp", "")
|
|
||||||
nonce = headers.get("x-cloud-nonce", "")
|
|
||||||
encoded_actor = headers.get("x-cloud-actor", "")
|
|
||||||
signature = headers.get("x-cloud-signature", "")
|
|
||||||
if (not re.fullmatch(r"[0-9]{1,12}", timestamp_raw)
|
|
||||||
or abs(now - int(timestamp_raw)) > 60
|
|
||||||
or not re.fullmatch(r"[A-Za-z0-9_-]{16,128}", nonce)
|
|
||||||
or not re.fullmatch(r"[0-9a-f]{64}", signature)
|
|
||||||
or not re.fullmatch(r"[0-9a-f]{64}", digest)):
|
|
||||||
raise EngineAuthError()
|
|
||||||
expected = hmac.new(secret, canonical(method, target, digest, timestamp_raw, nonce, encoded_actor),
|
|
||||||
hashlib.sha256).hexdigest()
|
|
||||||
if not hmac.compare_digest(expected, signature):
|
|
||||||
raise EngineAuthError()
|
|
||||||
value = parse_actor(encoded_actor)
|
|
||||||
if not nonce_store.consume_nonce(nonce, now + 121, now):
|
|
||||||
raise EngineAuthError("内部服务请求已使用,已拒绝重放")
|
|
||||||
return value
|
|
||||||
@ -1,404 +0,0 @@
|
|||||||
"""Load the source-controlled platform policy shared with the admin panel."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
import re
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
POLICY_MANIFEST_PATH = (
|
|
||||||
Path(__file__).resolve().parents[2]
|
|
||||||
/ "config"
|
|
||||||
/ "platform_default_network_policy.json"
|
|
||||||
)
|
|
||||||
MAX_MANIFEST_BYTES = 65536
|
|
||||||
MAX_CATALOG_RULES = 100
|
|
||||||
MAX_PLATFORM_FILTER_RULES = 70
|
|
||||||
MAX_PLATFORM_POLICY_SNAPSHOT_BYTES = 32768
|
|
||||||
PLATFORM_POLICY_SNAPSHOT_SCHEMA_VERSION = 1
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class PlatformNetworkRule:
|
|
||||||
rule_id: str
|
|
||||||
label: str
|
|
||||||
category: str
|
|
||||||
protocols: tuple[str, ...]
|
|
||||||
port_from: int
|
|
||||||
port_to: int
|
|
||||||
directions: tuple[str, ...]
|
|
||||||
reason: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class UnrestrictedPort:
|
|
||||||
protocol: str
|
|
||||||
port: int
|
|
||||||
reason: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class PlatformDefaultNetworkPolicy:
|
|
||||||
schema_version: int
|
|
||||||
policy_version: int
|
|
||||||
scope: str
|
|
||||||
applies_to: str
|
|
||||||
migrate_existing: bool
|
|
||||||
rules: tuple[PlatformNetworkRule, ...]
|
|
||||||
explicitly_unrestricted: tuple[UnrestrictedPort, ...]
|
|
||||||
|
|
||||||
|
|
||||||
def _exact_keys(value: dict[str, Any], expected: set[str], context: str) -> None:
|
|
||||||
if set(value) != expected:
|
|
||||||
raise RuntimeError(f"{context} fields are invalid")
|
|
||||||
|
|
||||||
|
|
||||||
def _text(
|
|
||||||
value: Any, context: str, *, max_bytes: int,
|
|
||||||
identifier: bool = False,
|
|
||||||
) -> str:
|
|
||||||
if (not isinstance(value, str) or not value or value != value.strip()
|
|
||||||
or len(value.encode("utf-8")) > max_bytes
|
|
||||||
or any(ord(char) < 32 or ord(char) == 127 for char in value)):
|
|
||||||
raise RuntimeError(f"{context} is invalid")
|
|
||||||
if identifier and not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", value):
|
|
||||||
raise RuntimeError(f"{context} is invalid")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _string_tuple(
|
|
||||||
value: Any, allowed: set[str], context: str,
|
|
||||||
) -> tuple[str, ...]:
|
|
||||||
if (not isinstance(value, list) or not value
|
|
||||||
or any(not isinstance(item, str) or item not in allowed for item in value)
|
|
||||||
or len(set(value)) != len(value)):
|
|
||||||
raise RuntimeError(f"{context} is invalid")
|
|
||||||
return tuple(value)
|
|
||||||
|
|
||||||
|
|
||||||
def load_platform_default_network_policy(
|
|
||||||
path: Path = POLICY_MANIFEST_PATH,
|
|
||||||
) -> PlatformDefaultNetworkPolicy:
|
|
||||||
"""Return a strictly validated immutable policy or fail engine startup."""
|
|
||||||
try:
|
|
||||||
encoded = path.read_bytes()
|
|
||||||
if not encoded or len(encoded) > MAX_MANIFEST_BYTES:
|
|
||||||
raise RuntimeError("platform default network policy size is invalid")
|
|
||||||
raw = json.loads(encoded.decode("utf-8"))
|
|
||||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
||||||
raise RuntimeError("platform default network policy cannot be loaded") from exc
|
|
||||||
if not isinstance(raw, dict):
|
|
||||||
raise RuntimeError("platform default network policy must be an object")
|
|
||||||
_exact_keys(raw, {
|
|
||||||
"schema_version", "policy_version", "scope", "applies_to",
|
|
||||||
"migrate_existing", "rules", "explicitly_unrestricted",
|
|
||||||
}, "platform default network policy")
|
|
||||||
if raw["schema_version"] != 1 or raw["policy_version"] != 1:
|
|
||||||
raise RuntimeError("platform default network policy version is unsupported")
|
|
||||||
if (raw["scope"] != "public_ipv4_interface"
|
|
||||||
or raw["applies_to"] != "new_public_cloud_image_instances_only"
|
|
||||||
or raw["migrate_existing"] is not False):
|
|
||||||
raise RuntimeError("platform default network policy scope is invalid")
|
|
||||||
if (not isinstance(raw["rules"], list) or not raw["rules"]
|
|
||||||
or len(raw["rules"]) > MAX_CATALOG_RULES):
|
|
||||||
raise RuntimeError("platform default network policy rules are invalid")
|
|
||||||
|
|
||||||
rules: list[PlatformNetworkRule] = []
|
|
||||||
rule_ids: set[str] = set()
|
|
||||||
expanded: set[tuple[str, int, int, str]] = set()
|
|
||||||
categories = {"permanent": 0, "default": 0}
|
|
||||||
for index, item in enumerate(raw["rules"]):
|
|
||||||
if not isinstance(item, dict):
|
|
||||||
raise RuntimeError(f"platform rule {index} is invalid")
|
|
||||||
_exact_keys(item, {
|
|
||||||
"id", "label", "category", "protocols", "port_from",
|
|
||||||
"port_to", "directions", "reason",
|
|
||||||
}, f"platform rule {index}")
|
|
||||||
rule_id = _text(
|
|
||||||
item["id"], f"platform rule {index} id",
|
|
||||||
max_bytes=64, identifier=True)
|
|
||||||
if rule_id in rule_ids:
|
|
||||||
raise RuntimeError("platform rule ids must be unique")
|
|
||||||
rule_ids.add(rule_id)
|
|
||||||
category = item["category"]
|
|
||||||
if category not in {"permanent", "default"}:
|
|
||||||
raise RuntimeError(f"platform rule {index} category is invalid")
|
|
||||||
categories[category] += 1
|
|
||||||
protocols = _string_tuple(
|
|
||||||
item["protocols"], {"tcp", "udp"},
|
|
||||||
f"platform rule {index} protocols",
|
|
||||||
)
|
|
||||||
directions = _string_tuple(
|
|
||||||
item["directions"], {"ingress", "egress"},
|
|
||||||
f"platform rule {index} directions",
|
|
||||||
)
|
|
||||||
port_from = item["port_from"]
|
|
||||||
port_to = item["port_to"]
|
|
||||||
if (isinstance(port_from, bool) or not isinstance(port_from, int)
|
|
||||||
or isinstance(port_to, bool) or not isinstance(port_to, int)
|
|
||||||
or not 1 <= port_from <= port_to <= 65535):
|
|
||||||
raise RuntimeError(f"platform rule {index} port range is invalid")
|
|
||||||
for protocol in protocols:
|
|
||||||
for direction in directions:
|
|
||||||
identity = (protocol, port_from, port_to, direction)
|
|
||||||
if identity in expanded:
|
|
||||||
raise RuntimeError("platform rules must not contain duplicates")
|
|
||||||
expanded.add(identity)
|
|
||||||
if len(expanded) > MAX_PLATFORM_FILTER_RULES:
|
|
||||||
raise RuntimeError("platform rules exceed the filter priority budget")
|
|
||||||
rules.append(PlatformNetworkRule(
|
|
||||||
rule_id=rule_id,
|
|
||||||
label=_text(
|
|
||||||
item["label"], f"platform rule {index} label",
|
|
||||||
max_bytes=120),
|
|
||||||
category=category,
|
|
||||||
protocols=protocols,
|
|
||||||
port_from=port_from,
|
|
||||||
port_to=port_to,
|
|
||||||
directions=directions,
|
|
||||||
reason=_text(
|
|
||||||
item["reason"], f"platform rule {index} reason",
|
|
||||||
max_bytes=500),
|
|
||||||
))
|
|
||||||
if any(count < 1 for count in categories.values()):
|
|
||||||
raise RuntimeError("platform default network policy categories are invalid")
|
|
||||||
|
|
||||||
unrestricted_raw = raw["explicitly_unrestricted"]
|
|
||||||
if (not isinstance(unrestricted_raw, list)
|
|
||||||
or len(unrestricted_raw) > MAX_CATALOG_RULES):
|
|
||||||
raise RuntimeError("explicitly unrestricted ports are invalid")
|
|
||||||
unrestricted: list[UnrestrictedPort] = []
|
|
||||||
unrestricted_ids: set[tuple[str, int]] = set()
|
|
||||||
for index, item in enumerate(unrestricted_raw):
|
|
||||||
if not isinstance(item, dict):
|
|
||||||
raise RuntimeError(f"unrestricted port {index} is invalid")
|
|
||||||
_exact_keys(item, {"protocol", "port", "reason"},
|
|
||||||
f"unrestricted port {index}")
|
|
||||||
protocol = item["protocol"]
|
|
||||||
port = item["port"]
|
|
||||||
if (protocol not in {"tcp", "udp"}
|
|
||||||
or isinstance(port, bool) or not isinstance(port, int)
|
|
||||||
or not 1 <= port <= 65535
|
|
||||||
or (protocol, port) in unrestricted_ids):
|
|
||||||
raise RuntimeError(f"unrestricted port {index} is invalid")
|
|
||||||
if any(
|
|
||||||
protocol == rule_protocol and port_from <= port <= port_to
|
|
||||||
for rule_protocol, port_from, port_to, _direction in expanded
|
|
||||||
):
|
|
||||||
raise RuntimeError("a port cannot be both restricted and unrestricted")
|
|
||||||
unrestricted_ids.add((protocol, port))
|
|
||||||
unrestricted.append(UnrestrictedPort(
|
|
||||||
protocol=protocol,
|
|
||||||
port=port,
|
|
||||||
reason=_text(
|
|
||||||
item["reason"], f"unrestricted port {index} reason",
|
|
||||||
max_bytes=500),
|
|
||||||
))
|
|
||||||
|
|
||||||
return PlatformDefaultNetworkPolicy(
|
|
||||||
schema_version=raw["schema_version"],
|
|
||||||
policy_version=raw["policy_version"],
|
|
||||||
scope=raw["scope"],
|
|
||||||
applies_to=raw["applies_to"],
|
|
||||||
migrate_existing=raw["migrate_existing"],
|
|
||||||
rules=tuple(rules),
|
|
||||||
explicitly_unrestricted=tuple(unrestricted),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
PLATFORM_DEFAULT_NETWORK_POLICY = load_platform_default_network_policy()
|
|
||||||
|
|
||||||
|
|
||||||
def _rule_document(rule: PlatformNetworkRule | dict[str, Any]) -> dict[str, Any]:
|
|
||||||
if isinstance(rule, PlatformNetworkRule):
|
|
||||||
return {
|
|
||||||
"id": rule.rule_id,
|
|
||||||
"label": rule.label,
|
|
||||||
"category": rule.category,
|
|
||||||
"protocols": list(rule.protocols),
|
|
||||||
"port_from": rule.port_from,
|
|
||||||
"port_to": rule.port_to,
|
|
||||||
"directions": list(rule.directions),
|
|
||||||
"reason": rule.reason,
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
"id": rule["id"],
|
|
||||||
"label": rule["label"],
|
|
||||||
"category": rule["category"],
|
|
||||||
"protocols": list(rule["protocols"]),
|
|
||||||
"port_from": rule["port_from"],
|
|
||||||
"port_to": rule["port_to"],
|
|
||||||
"directions": list(rule["directions"]),
|
|
||||||
"reason": rule["reason"],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _snapshot_document(
|
|
||||||
catalog_revision: int, rules: list[dict[str, Any]],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"schema_version": PLATFORM_POLICY_SNAPSHOT_SCHEMA_VERSION,
|
|
||||||
"base_policy_version": PLATFORM_DEFAULT_NETWORK_POLICY.policy_version,
|
|
||||||
"catalog_revision": catalog_revision,
|
|
||||||
"rules": rules,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def platform_policy_snapshot_digest(document: dict[str, Any]) -> str:
|
|
||||||
encoded = json.dumps(
|
|
||||||
document, ensure_ascii=False, separators=(",", ":"),
|
|
||||||
).encode("utf-8")
|
|
||||||
return hashlib.sha256(encoded).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def default_platform_policy_snapshot() -> dict[str, Any]:
|
|
||||||
"""Return an explicit immutable baseline for legacy/no-test creates."""
|
|
||||||
document = _snapshot_document(
|
|
||||||
0, [_rule_document(rule) for rule in PLATFORM_DEFAULT_NETWORK_POLICY.rules],
|
|
||||||
)
|
|
||||||
return {**document, "digest": platform_policy_snapshot_digest(document)}
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_platform_policy_snapshot(value: Any) -> dict[str, Any]:
|
|
||||||
"""Validate one complete PHP-sealed static+test policy snapshot."""
|
|
||||||
expected_keys = {
|
|
||||||
"schema_version", "base_policy_version", "catalog_revision",
|
|
||||||
"rules", "digest",
|
|
||||||
}
|
|
||||||
if not isinstance(value, dict) or set(value) != expected_keys:
|
|
||||||
raise ValueError("platform policy snapshot fields are invalid")
|
|
||||||
revision = value.get("catalog_revision")
|
|
||||||
digest = value.get("digest")
|
|
||||||
if (value.get("schema_version") != PLATFORM_POLICY_SNAPSHOT_SCHEMA_VERSION
|
|
||||||
or value.get("base_policy_version")
|
|
||||||
!= PLATFORM_DEFAULT_NETWORK_POLICY.policy_version
|
|
||||||
or isinstance(revision, bool) or not isinstance(revision, int)
|
|
||||||
or not 0 <= revision <= 2**63 - 1
|
|
||||||
or not isinstance(digest, str)
|
|
||||||
or re.fullmatch(r"[0-9a-f]{64}", digest) is None
|
|
||||||
or not isinstance(value.get("rules"), list)):
|
|
||||||
raise ValueError("platform policy snapshot is invalid")
|
|
||||||
|
|
||||||
base = [
|
|
||||||
_rule_document(rule) for rule in PLATFORM_DEFAULT_NETWORK_POLICY.rules
|
|
||||||
]
|
|
||||||
raw_rules = value["rules"]
|
|
||||||
if len(raw_rules) < len(base) or raw_rules[:len(base)] != base:
|
|
||||||
raise ValueError("platform policy baseline does not match this engine")
|
|
||||||
|
|
||||||
expanded: list[tuple[str, int, int, str]] = [
|
|
||||||
(protocol, rule.port_from, rule.port_to, direction)
|
|
||||||
for rule in PLATFORM_DEFAULT_NETWORK_POLICY.rules
|
|
||||||
for direction in rule.directions
|
|
||||||
for protocol in rule.protocols
|
|
||||||
]
|
|
||||||
unrestricted = {
|
|
||||||
(item.protocol, item.port)
|
|
||||||
for item in PLATFORM_DEFAULT_NETWORK_POLICY.explicitly_unrestricted
|
|
||||||
}
|
|
||||||
dynamic: list[dict[str, Any]] = []
|
|
||||||
seen_ids: set[str] = set()
|
|
||||||
rule_keys = {
|
|
||||||
"id", "label", "category", "protocols", "port_from", "port_to",
|
|
||||||
"directions", "reason",
|
|
||||||
}
|
|
||||||
for index, raw in enumerate(raw_rules[len(base):]):
|
|
||||||
if (not isinstance(raw, dict) or set(raw) != rule_keys
|
|
||||||
or raw.get("category") != "test"):
|
|
||||||
raise ValueError(f"platform test rule {index} is invalid")
|
|
||||||
rule_id = raw.get("id")
|
|
||||||
if (not isinstance(rule_id, str)
|
|
||||||
or re.fullmatch(r"test-[0-9a-f]{32}", rule_id) is None
|
|
||||||
or rule_id in seen_ids):
|
|
||||||
raise ValueError(f"platform test rule {index} id is invalid")
|
|
||||||
seen_ids.add(rule_id)
|
|
||||||
try:
|
|
||||||
protocols = _string_tuple(
|
|
||||||
raw.get("protocols"), {"tcp", "udp"},
|
|
||||||
f"platform test rule {index} protocols",
|
|
||||||
)
|
|
||||||
except RuntimeError as exc:
|
|
||||||
raise ValueError(
|
|
||||||
f"platform test rule {index} protocols are invalid",
|
|
||||||
) from exc
|
|
||||||
protocols = tuple(
|
|
||||||
item for item in ("tcp", "udp") if item in protocols
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
directions = _string_tuple(
|
|
||||||
raw.get("directions"), {"ingress", "egress"},
|
|
||||||
f"platform test rule {index} directions",
|
|
||||||
)
|
|
||||||
except RuntimeError as exc:
|
|
||||||
raise ValueError(
|
|
||||||
f"platform test rule {index} directions are invalid",
|
|
||||||
) from exc
|
|
||||||
directions = tuple(
|
|
||||||
item for item in ("ingress", "egress") if item in directions
|
|
||||||
)
|
|
||||||
port_from = raw.get("port_from")
|
|
||||||
port_to = raw.get("port_to")
|
|
||||||
if (isinstance(port_from, bool) or not isinstance(port_from, int)
|
|
||||||
or isinstance(port_to, bool) or not isinstance(port_to, int)
|
|
||||||
or not 1 <= port_from <= port_to <= 65535):
|
|
||||||
raise ValueError(f"platform test rule {index} ports are invalid")
|
|
||||||
for protocol in protocols:
|
|
||||||
if any(
|
|
||||||
blocked_protocol == protocol
|
|
||||||
and port_from <= blocked_port <= port_to
|
|
||||||
for blocked_protocol, blocked_port in unrestricted
|
|
||||||
):
|
|
||||||
raise ValueError("explicitly unrestricted port cannot be tested")
|
|
||||||
for direction in directions:
|
|
||||||
if any(
|
|
||||||
old_protocol == protocol and old_direction == direction
|
|
||||||
and port_from <= old_to and port_to >= old_from
|
|
||||||
for old_protocol, old_from, old_to, old_direction in expanded
|
|
||||||
):
|
|
||||||
raise ValueError("platform test rules overlap")
|
|
||||||
expanded.append((protocol, port_from, port_to, direction))
|
|
||||||
if len(expanded) > MAX_PLATFORM_FILTER_RULES:
|
|
||||||
raise ValueError("platform policy exceeds the filter priority budget")
|
|
||||||
try:
|
|
||||||
label = _text(
|
|
||||||
raw.get("label"), f"platform test rule {index} label",
|
|
||||||
max_bytes=120,
|
|
||||||
)
|
|
||||||
reason = _text(
|
|
||||||
raw.get("reason"), f"platform test rule {index} reason",
|
|
||||||
max_bytes=500,
|
|
||||||
)
|
|
||||||
except RuntimeError as exc:
|
|
||||||
raise ValueError(
|
|
||||||
f"platform test rule {index} text is invalid",
|
|
||||||
) from exc
|
|
||||||
dynamic.append(_rule_document({
|
|
||||||
"id": rule_id,
|
|
||||||
"label": label,
|
|
||||||
"category": "test",
|
|
||||||
"protocols": protocols,
|
|
||||||
"port_from": port_from,
|
|
||||||
"port_to": port_to,
|
|
||||||
"directions": directions,
|
|
||||||
"reason": reason,
|
|
||||||
}))
|
|
||||||
dynamic.sort(key=lambda item: item["id"])
|
|
||||||
rules = base + dynamic
|
|
||||||
if raw_rules != rules:
|
|
||||||
raise ValueError("platform policy rules are not canonical")
|
|
||||||
document = _snapshot_document(revision, rules)
|
|
||||||
expected_digest = platform_policy_snapshot_digest(document)
|
|
||||||
normalized = {**document, "digest": expected_digest}
|
|
||||||
encoded = json.dumps(
|
|
||||||
normalized, ensure_ascii=False, separators=(",", ":"),
|
|
||||||
).encode("utf-8")
|
|
||||||
if len(encoded) > MAX_PLATFORM_POLICY_SNAPSHOT_BYTES:
|
|
||||||
raise ValueError("platform policy snapshot is too large")
|
|
||||||
if not hmac.compare_digest(expected_digest, digest):
|
|
||||||
raise ValueError("platform policy snapshot digest is invalid")
|
|
||||||
return normalized
|
|
||||||
@ -1,864 +0,0 @@
|
|||||||
"""Owner-scoped catalogs and hardened remote downloads for user ISO images."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import http.client
|
|
||||||
import ipaddress
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import socket
|
|
||||||
import ssl
|
|
||||||
import stat
|
|
||||||
import tempfile
|
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
import uuid
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
from urllib.parse import SplitResult, urlsplit
|
|
||||||
|
|
||||||
from .cloud_templates import TemplateError
|
|
||||||
|
|
||||||
|
|
||||||
USER_ISO_ID_RE = re.compile(r"[0-9a-f]{32}\Z")
|
|
||||||
USER_ISO_VOLUME_RE = re.compile(r"panel-user-iso-[0-9a-f]{32}\.iso\Z")
|
|
||||||
HOST_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
|
|
||||||
SHA256_RE = re.compile(r"[0-9a-fA-F]{64}\Z")
|
|
||||||
MAX_USER_ISO_BYTES = 10 * 1024**3
|
|
||||||
MAX_USER_ISOS = 5
|
|
||||||
MAX_USER_ISO_TOTAL_BYTES = 10 * 1024**3
|
|
||||||
MAX_DOWNLOAD_SECONDS = 3600
|
|
||||||
PERSISTENT_FREE_RESERVE_BYTES = 256 * 1024**2
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def _positive_user_id(user_id: Any) -> int:
|
|
||||||
if isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1:
|
|
||||||
raise TemplateError("用户身份无效", 403)
|
|
||||||
return user_id
|
|
||||||
|
|
||||||
|
|
||||||
def user_iso_scope(user_id: int) -> str:
|
|
||||||
"""Return the engine-global catalog scope for one PHP owner."""
|
|
||||||
user_id = _positive_user_id(user_id)
|
|
||||||
identity = ["virt-cloud-user-iso", user_id]
|
|
||||||
raw = json.dumps(identity, ensure_ascii=False, separators=(",", ":"))
|
|
||||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def user_iso_volume_name(iso_id: str) -> str:
|
|
||||||
if not isinstance(iso_id, str) or not USER_ISO_ID_RE.fullmatch(iso_id):
|
|
||||||
raise TemplateError("ISO 标识无效")
|
|
||||||
return f"panel-user-iso-{iso_id}.iso"
|
|
||||||
|
|
||||||
|
|
||||||
def validate_user_iso_name(name: Any) -> str:
|
|
||||||
if not isinstance(name, str) or not 1 <= len(name.strip()) <= 64:
|
|
||||||
raise TemplateError("ISO 名称应为 1 至 64 个字符")
|
|
||||||
if any(ord(char) < 32 or ord(char) == 127 for char in name):
|
|
||||||
raise TemplateError("ISO 名称不能包含控制字符")
|
|
||||||
return name.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def validate_user_iso_url(url: Any) -> SplitResult:
|
|
||||||
"""Validate syntax before DNS; the downloader pins a separately checked IP."""
|
|
||||||
if not isinstance(url, str) or not 1 <= len(url) <= 2048:
|
|
||||||
raise TemplateError("ISO 下载地址无效")
|
|
||||||
try:
|
|
||||||
parsed = urlsplit(url)
|
|
||||||
port = parsed.port
|
|
||||||
hostname = parsed.hostname
|
|
||||||
except (TypeError, ValueError) as exc:
|
|
||||||
raise TemplateError("ISO 下载地址无效") from exc
|
|
||||||
if (parsed.scheme != "https" or not hostname or port not in (None, 443)
|
|
||||||
or parsed.username or parsed.password or parsed.fragment):
|
|
||||||
raise TemplateError("ISO 仅支持无账号、无跳转的 HTTPS 443 直链")
|
|
||||||
if not parsed.path.lower().endswith(".iso"):
|
|
||||||
raise TemplateError("ISO 下载地址的文件名必须以 .iso 结尾", 422)
|
|
||||||
try:
|
|
||||||
url.encode("ascii")
|
|
||||||
hostname.encode("idna")
|
|
||||||
except UnicodeError as exc:
|
|
||||||
raise TemplateError("ISO 下载地址必须使用 ASCII/百分号编码") from exc
|
|
||||||
if any(ord(char) < 33 or ord(char) == 127 for char in url):
|
|
||||||
raise TemplateError("ISO 下载地址包含无效字符")
|
|
||||||
return parsed
|
|
||||||
|
|
||||||
|
|
||||||
def _resolve_global_addresses(hostname: str) -> list[str]:
|
|
||||||
try:
|
|
||||||
rows = socket.getaddrinfo(
|
|
||||||
hostname, 443, family=socket.AF_UNSPEC,
|
|
||||||
type=socket.SOCK_STREAM, proto=socket.IPPROTO_TCP,
|
|
||||||
)
|
|
||||||
except socket.gaierror as exc:
|
|
||||||
raise TemplateError("ISO 下载地址无法解析", 422) from exc
|
|
||||||
addresses: list[str] = []
|
|
||||||
for _family, _socktype, _proto, _canonname, sockaddr in rows:
|
|
||||||
try:
|
|
||||||
address = str(ipaddress.ip_address(sockaddr[0]))
|
|
||||||
except (IndexError, TypeError, ValueError) as exc:
|
|
||||||
raise TemplateError("ISO 下载地址解析结果无效", 422) from exc
|
|
||||||
# Reject the whole hostname when even one answer is unsafe. Otherwise
|
|
||||||
# a mixed public/private answer can become an SSRF failover primitive.
|
|
||||||
if not ipaddress.ip_address(address).is_global:
|
|
||||||
raise TemplateError("ISO 下载地址不能指向内网、回环或保留地址", 422)
|
|
||||||
if address not in addresses:
|
|
||||||
addresses.append(address)
|
|
||||||
if not addresses:
|
|
||||||
raise TemplateError("ISO 下载地址无法解析", 422)
|
|
||||||
return addresses
|
|
||||||
|
|
||||||
|
|
||||||
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
|
||||||
"""TLS-connect to a validated address while verifying the original host."""
|
|
||||||
|
|
||||||
def __init__(self, hostname: str, address: str, *, timeout: float) -> None:
|
|
||||||
super().__init__(hostname, 443, timeout=timeout,
|
|
||||||
context=ssl.create_default_context())
|
|
||||||
self._pinned_address = address
|
|
||||||
|
|
||||||
def connect(self) -> None:
|
|
||||||
# There is deliberately no proxy/tunnel path here. Connecting to the
|
|
||||||
# already validated numeric address prevents a second DNS lookup and
|
|
||||||
# therefore closes the usual DNS-rebinding gap.
|
|
||||||
raw = socket.create_connection(
|
|
||||||
(self._pinned_address, 443), self.timeout, self.source_address,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
self.sock = self._context.wrap_socket(raw, server_hostname=self.host)
|
|
||||||
except Exception:
|
|
||||||
raw.close()
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
def _open_pinned_response(parsed: SplitResult, addresses: list[str]):
|
|
||||||
hostname = parsed.hostname
|
|
||||||
assert hostname is not None
|
|
||||||
target = parsed.path or "/"
|
|
||||||
if parsed.query:
|
|
||||||
target += "?" + parsed.query
|
|
||||||
host_header = f"[{hostname}]" if ":" in hostname else hostname
|
|
||||||
last_error: Exception | None = None
|
|
||||||
for address in addresses:
|
|
||||||
connection = _PinnedHTTPSConnection(hostname, address, timeout=30)
|
|
||||||
try:
|
|
||||||
connection.request("GET", target, headers={
|
|
||||||
"Host": host_header,
|
|
||||||
"User-Agent": "virt-cloud-user-iso/1",
|
|
||||||
"Accept": "application/octet-stream,*/*;q=0.1",
|
|
||||||
"Accept-Encoding": "identity",
|
|
||||||
"Connection": "close",
|
|
||||||
})
|
|
||||||
return connection, connection.getresponse()
|
|
||||||
except (OSError, ssl.SSLError, http.client.HTTPException) as exc:
|
|
||||||
last_error = exc
|
|
||||||
connection.close()
|
|
||||||
raise TemplateError("无法连接 ISO 下载地址", 502) from last_error
|
|
||||||
|
|
||||||
|
|
||||||
def _declared_size(response: http.client.HTTPResponse) -> int | None:
|
|
||||||
values = response.headers.get_all("Content-Length", [])
|
|
||||||
if not values:
|
|
||||||
return None
|
|
||||||
if len(values) != 1 or not re.fullmatch(r"[0-9]+", values[0].strip()):
|
|
||||||
raise TemplateError("ISO 下载响应长度无效", 422)
|
|
||||||
return int(values[0])
|
|
||||||
|
|
||||||
|
|
||||||
def verify_user_iso(path: Path, *, max_bytes: int = MAX_USER_ISO_BYTES) -> None:
|
|
||||||
"""Require a bounded ISO9660 image with an El Torito boot catalog."""
|
|
||||||
size = path.stat().st_size
|
|
||||||
if not 1024 * 1024 <= size <= min(MAX_USER_ISO_BYTES, max_bytes):
|
|
||||||
raise TemplateError("ISO 大小必须在 1 MiB 至可用配额之间", 413)
|
|
||||||
bootable = False
|
|
||||||
with path.open("rb") as image:
|
|
||||||
for sector in range(16, 80):
|
|
||||||
image.seek(sector * 2048)
|
|
||||||
descriptor = image.read(2048)
|
|
||||||
if len(descriptor) != 2048 or descriptor[1:6] != b"CD001":
|
|
||||||
raise TemplateError("下载文件不是有效的 ISO 9660 镜像", 422)
|
|
||||||
if (descriptor[0] == 0
|
|
||||||
and descriptor[7:30].rstrip(b" ") == b"EL TORITO SPECIFICATION"
|
|
||||||
and int.from_bytes(descriptor[71:75], "little") > 0):
|
|
||||||
bootable = True
|
|
||||||
if descriptor[0] == 255:
|
|
||||||
break
|
|
||||||
if not bootable:
|
|
||||||
raise TemplateError("ISO 未包含可识别的 El Torito 启动记录", 422)
|
|
||||||
|
|
||||||
|
|
||||||
def download_user_iso(
|
|
||||||
url: str, expected_sha256: str, directory: Path, *,
|
|
||||||
max_bytes: int = MAX_USER_ISO_BYTES,
|
|
||||||
) -> tuple[Path, int, str]:
|
|
||||||
"""Download without proxies/redirects and pin TLS to validated public IPs."""
|
|
||||||
parsed = validate_user_iso_url(url)
|
|
||||||
if not isinstance(expected_sha256, str) or (
|
|
||||||
expected_sha256 and not SHA256_RE.fullmatch(expected_sha256)):
|
|
||||||
raise TemplateError("SHA-256 可留空;填写时必须为 64 位十六进制校验值")
|
|
||||||
if (isinstance(max_bytes, bool) or not isinstance(max_bytes, int)
|
|
||||||
or not 1024 * 1024 <= max_bytes <= MAX_USER_ISO_BYTES):
|
|
||||||
raise TemplateError("用户 ISO 可用配额不足", 409)
|
|
||||||
hostname = parsed.hostname
|
|
||||||
assert hostname is not None
|
|
||||||
addresses = _resolve_global_addresses(hostname)
|
|
||||||
fd, raw_path = tempfile.mkstemp(prefix=".user-iso-", suffix=".iso", dir=directory)
|
|
||||||
path = Path(raw_path)
|
|
||||||
connection = None
|
|
||||||
response = None
|
|
||||||
try:
|
|
||||||
os.fchmod(fd, 0o600)
|
|
||||||
connection, response = _open_pinned_response(parsed, addresses)
|
|
||||||
if 300 <= response.status < 400:
|
|
||||||
raise TemplateError("ISO 下载地址发生重定向,请使用最终 HTTPS 直链", 422)
|
|
||||||
if response.status != 200:
|
|
||||||
raise TemplateError(f"ISO 下载服务器返回 HTTP {response.status}", 422)
|
|
||||||
encoding = response.getheader("Content-Encoding", "identity").strip().lower()
|
|
||||||
if encoding not in {"", "identity"}:
|
|
||||||
raise TemplateError("ISO 下载响应不能使用内容压缩", 422)
|
|
||||||
declared = _declared_size(response)
|
|
||||||
if declared is not None and declared > max_bytes:
|
|
||||||
raise TemplateError("ISO 超过用户全局目录的 10 GiB 配额或剩余容量", 413)
|
|
||||||
checksum = hashlib.sha256()
|
|
||||||
size = 0
|
|
||||||
deadline = time.monotonic() + MAX_DOWNLOAD_SECONDS
|
|
||||||
with os.fdopen(fd, "wb") as output:
|
|
||||||
fd = -1
|
|
||||||
while True:
|
|
||||||
if time.monotonic() > deadline:
|
|
||||||
raise TemplateError("ISO 下载超时", 504)
|
|
||||||
chunk = response.read(1024 * 1024)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
size += len(chunk)
|
|
||||||
if size > max_bytes:
|
|
||||||
raise TemplateError("ISO 超过用户全局目录的 10 GiB 配额或剩余容量", 413)
|
|
||||||
output.write(chunk)
|
|
||||||
checksum.update(chunk)
|
|
||||||
output.flush()
|
|
||||||
os.fsync(output.fileno())
|
|
||||||
if declared is not None and size != declared:
|
|
||||||
raise TemplateError("ISO 下载内容不完整", 502)
|
|
||||||
digest = checksum.hexdigest()
|
|
||||||
if expected_sha256 and digest.lower() != expected_sha256.lower():
|
|
||||||
raise TemplateError("ISO 的 SHA-256 校验值不匹配", 422)
|
|
||||||
verify_user_iso(path, max_bytes=max_bytes)
|
|
||||||
return path, size, digest
|
|
||||||
except Exception:
|
|
||||||
path.unlink(missing_ok=True)
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
if fd >= 0:
|
|
||||||
os.close(fd)
|
|
||||||
if response is not None:
|
|
||||||
response.close()
|
|
||||||
if connection is not None:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
class UserISOFiles:
|
|
||||||
"""Private engine-local source files, separate from every host cache."""
|
|
||||||
|
|
||||||
def __init__(self, data_dir: Path) -> None:
|
|
||||||
data_dir = Path(data_dir)
|
|
||||||
if not data_dir.is_absolute() or data_dir == Path("/"):
|
|
||||||
raise ValueError("user ISO data directory must be absolute and dedicated")
|
|
||||||
self.data_dir = data_dir
|
|
||||||
self.root = data_dir / "user-isos"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _ensure_directory(path: Path) -> None:
|
|
||||||
try:
|
|
||||||
info = path.lstat()
|
|
||||||
except FileNotFoundError:
|
|
||||||
path.mkdir(mode=0o700)
|
|
||||||
info = path.lstat()
|
|
||||||
if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
||||||
or info.st_uid != os.geteuid()):
|
|
||||||
raise TemplateError("用户 ISO 持久目录不安全,请联系管理员", 500)
|
|
||||||
if stat.S_IMODE(info.st_mode) != 0o700:
|
|
||||||
os.chmod(path, 0o700)
|
|
||||||
|
|
||||||
def owner_directory(self, user_id: int) -> Path:
|
|
||||||
user_id = _positive_user_id(user_id)
|
|
||||||
self.data_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
||||||
self._ensure_directory(self.data_dir)
|
|
||||||
self._ensure_directory(self.root)
|
|
||||||
owner = self.root / str(user_id)
|
|
||||||
self._ensure_directory(owner)
|
|
||||||
return owner
|
|
||||||
|
|
||||||
def available_bytes(self, user_id: int) -> int:
|
|
||||||
free = int(shutil.disk_usage(self.owner_directory(user_id)).free)
|
|
||||||
return max(0, free - PERSISTENT_FREE_RESERVE_BYTES)
|
|
||||||
|
|
||||||
def path(self, user_id: int, iso_id: str, *, expected_size: int | None = None) -> Path:
|
|
||||||
path = self.owner_directory(user_id) / user_iso_volume_name(iso_id)
|
|
||||||
try:
|
|
||||||
info = path.lstat()
|
|
||||||
except FileNotFoundError as exc:
|
|
||||||
raise TemplateError("ISO 源文件不存在,请删除后重新导入", 409) from exc
|
|
||||||
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
||||||
or info.st_uid != os.geteuid() or info.st_mode & 0o077
|
|
||||||
or (expected_size is not None and info.st_size != expected_size)):
|
|
||||||
raise TemplateError("ISO 源文件已损坏,请联系管理员", 500)
|
|
||||||
return path
|
|
||||||
|
|
||||||
def publish(self, temporary: Path, user_id: int, iso_id: str,
|
|
||||||
*, expected_size: int) -> Path:
|
|
||||||
temporary = Path(temporary)
|
|
||||||
try:
|
|
||||||
info = temporary.lstat()
|
|
||||||
except FileNotFoundError as exc:
|
|
||||||
raise TemplateError("ISO 下载临时文件丢失", 500) from exc
|
|
||||||
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
||||||
or info.st_uid != os.geteuid() or info.st_size != expected_size):
|
|
||||||
raise TemplateError("ISO 下载临时文件无效", 500)
|
|
||||||
destination = self.owner_directory(user_id) / user_iso_volume_name(iso_id)
|
|
||||||
try:
|
|
||||||
# The download is created in this same private directory. Linking
|
|
||||||
# publishes it atomically and refuses to replace an existing ID.
|
|
||||||
os.link(temporary, destination)
|
|
||||||
os.chmod(destination, 0o600)
|
|
||||||
temporary.unlink()
|
|
||||||
descriptor = os.open(destination.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
||||||
try:
|
|
||||||
os.fsync(descriptor)
|
|
||||||
finally:
|
|
||||||
os.close(descriptor)
|
|
||||||
except FileExistsError as exc:
|
|
||||||
raise TemplateError("此 ISO 已存在", 409) from exc
|
|
||||||
except OSError as exc:
|
|
||||||
destination.unlink(missing_ok=True)
|
|
||||||
raise TemplateError("ISO 无法保存到面板持久目录", 507) from exc
|
|
||||||
return destination
|
|
||||||
|
|
||||||
def delete(self, user_id: int, iso_id: str) -> None:
|
|
||||||
path = self.owner_directory(user_id) / user_iso_volume_name(iso_id)
|
|
||||||
try:
|
|
||||||
info = path.lstat()
|
|
||||||
except FileNotFoundError:
|
|
||||||
return
|
|
||||||
if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode):
|
|
||||||
raise TemplateError("ISO 源文件无法安全删除", 500)
|
|
||||||
try:
|
|
||||||
path.unlink()
|
|
||||||
except OSError as exc:
|
|
||||||
raise TemplateError("ISO 源文件删除失败,请联系管理员", 500) from exc
|
|
||||||
|
|
||||||
def reconcile(self, catalog: UserISOStore) -> None:
|
|
||||||
"""Remove only provable interrupted-download and uncataloged files."""
|
|
||||||
try:
|
|
||||||
self.root.lstat()
|
|
||||||
except FileNotFoundError:
|
|
||||||
return
|
|
||||||
self._ensure_directory(self.root)
|
|
||||||
temporary_name = re.compile(r"\.user-iso-[A-Za-z0-9_-]{6,64}\.iso\Z")
|
|
||||||
for owner in self.root.iterdir():
|
|
||||||
try:
|
|
||||||
owner_id = int(owner.name)
|
|
||||||
info = owner.lstat()
|
|
||||||
except (OSError, ValueError):
|
|
||||||
continue
|
|
||||||
if (owner_id < 1 or str(owner_id) != owner.name
|
|
||||||
or not stat.S_ISDIR(info.st_mode)
|
|
||||||
or stat.S_ISLNK(info.st_mode)
|
|
||||||
or info.st_uid != os.geteuid()
|
|
||||||
or stat.S_IMODE(info.st_mode) != 0o700):
|
|
||||||
continue
|
|
||||||
active = {
|
|
||||||
row["volume_name"]
|
|
||||||
for row in catalog.list(user_iso_scope(owner_id))
|
|
||||||
}
|
|
||||||
for candidate in owner.iterdir():
|
|
||||||
try:
|
|
||||||
candidate_info = candidate.lstat()
|
|
||||||
except OSError:
|
|
||||||
continue
|
|
||||||
if (not stat.S_ISREG(candidate_info.st_mode)
|
|
||||||
or stat.S_ISLNK(candidate_info.st_mode)
|
|
||||||
or candidate_info.st_uid != os.geteuid()):
|
|
||||||
continue
|
|
||||||
interrupted = temporary_name.fullmatch(candidate.name) is not None
|
|
||||||
orphan = (USER_ISO_VOLUME_RE.fullmatch(candidate.name) is not None
|
|
||||||
and candidate.name not in active)
|
|
||||||
if interrupted or orphan:
|
|
||||||
candidate.unlink(missing_ok=True)
|
|
||||||
|
|
||||||
|
|
||||||
class UserISOStore:
|
|
||||||
"""Engine-global catalog whose API always requires an owner scope."""
|
|
||||||
|
|
||||||
def __init__(self, store: Any | None) -> None:
|
|
||||||
self.store = store
|
|
||||||
self._memory: dict[str, str] = {}
|
|
||||||
self._lock = threading.RLock()
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _validate(record: Any) -> dict[str, Any]:
|
|
||||||
if (not isinstance(record, dict)
|
|
||||||
or not USER_ISO_ID_RE.fullmatch(str(record.get("id", "")))
|
|
||||||
or record.get("volume_name") != user_iso_volume_name(record["id"])
|
|
||||||
or not isinstance(record.get("name"), str)
|
|
||||||
or validate_user_iso_name(record["name"]) != record["name"]
|
|
||||||
or isinstance(record.get("size_bytes"), bool)
|
|
||||||
or not isinstance(record.get("size_bytes"), int)
|
|
||||||
or not 1024 * 1024 <= record["size_bytes"] <= MAX_USER_ISO_BYTES
|
|
||||||
or not SHA256_RE.fullmatch(str(record.get("sha256", "")))
|
|
||||||
or isinstance(record.get("created_at"), bool)
|
|
||||||
or not isinstance(record.get("created_at"), int)
|
|
||||||
or record["created_at"] < 1):
|
|
||||||
raise TemplateError("用户 ISO 数据库记录损坏", 500)
|
|
||||||
return record
|
|
||||||
|
|
||||||
def _list(self, scope: str) -> list[dict[str, Any]]:
|
|
||||||
source = (self.store.list("user_iso") if self.store is not None
|
|
||||||
else self._memory.copy())
|
|
||||||
records = []
|
|
||||||
for key, payload in source.items():
|
|
||||||
if not key.startswith(f"{scope}:"):
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
record = json.loads(payload)
|
|
||||||
except (TypeError, ValueError) as exc:
|
|
||||||
raise TemplateError("用户 ISO 数据库记录损坏", 500) from exc
|
|
||||||
records.append(self._validate(record))
|
|
||||||
return sorted(records, key=lambda item: (item["created_at"], item["id"]), reverse=True)
|
|
||||||
|
|
||||||
def list(self, scope: str) -> list[dict[str, Any]]:
|
|
||||||
with self._lock:
|
|
||||||
return self._list(scope)
|
|
||||||
|
|
||||||
def get(self, scope: str, iso_id: str) -> dict[str, Any]:
|
|
||||||
user_iso_volume_name(iso_id)
|
|
||||||
with self._lock:
|
|
||||||
for record in self._list(scope):
|
|
||||||
if record["id"] == iso_id:
|
|
||||||
return record
|
|
||||||
raise TemplateError("ISO 不存在或不属于当前用户", 404)
|
|
||||||
|
|
||||||
def remaining_bytes(self, scope: str) -> int:
|
|
||||||
with self._lock:
|
|
||||||
records = self._list(scope)
|
|
||||||
if len(records) >= MAX_USER_ISOS:
|
|
||||||
raise TemplateError("每个用户最多保存 5 个 ISO", 409)
|
|
||||||
remaining = MAX_USER_ISO_TOTAL_BYTES - sum(row["size_bytes"] for row in records)
|
|
||||||
if remaining < 1024 * 1024:
|
|
||||||
raise TemplateError("用户 ISO 全局目录的 10 GiB 存储配额已用完", 409)
|
|
||||||
return min(MAX_USER_ISO_BYTES, remaining)
|
|
||||||
|
|
||||||
def add(self, scope: str, record: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
record = self._validate(record.copy())
|
|
||||||
key = f"{scope}:{record['id']}"
|
|
||||||
value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
|
||||||
with self._lock:
|
|
||||||
records = self._list(scope)
|
|
||||||
if len(records) >= MAX_USER_ISOS:
|
|
||||||
raise TemplateError("每个用户最多保存 5 个 ISO", 409)
|
|
||||||
if sum(row["size_bytes"] for row in records) + record["size_bytes"] > MAX_USER_ISO_TOTAL_BYTES:
|
|
||||||
raise TemplateError("用户 ISO 全局目录总容量不能超过 10 GiB", 409)
|
|
||||||
if any(row["id"] == record["id"] for row in records):
|
|
||||||
raise TemplateError("此 ISO 已存在", 409)
|
|
||||||
if any(row["name"].casefold() == record["name"].casefold() for row in records):
|
|
||||||
raise TemplateError("当前用户已有同名 ISO", 409)
|
|
||||||
if self.store is not None:
|
|
||||||
if not self.store.insert("user_iso", key, value):
|
|
||||||
raise TemplateError("此 ISO 已存在", 409)
|
|
||||||
else:
|
|
||||||
self._memory[key] = value
|
|
||||||
return record
|
|
||||||
|
|
||||||
def delete(self, scope: str, iso_id: str) -> None:
|
|
||||||
self.get(scope, iso_id)
|
|
||||||
key = f"{scope}:{iso_id}"
|
|
||||||
with self._lock:
|
|
||||||
if self.store is not None:
|
|
||||||
self.store.delete("user_iso", key)
|
|
||||||
else:
|
|
||||||
self._memory.pop(key, None)
|
|
||||||
|
|
||||||
|
|
||||||
class UserISORuntime:
|
|
||||||
"""One engine-process runtime shared by every per-host child app."""
|
|
||||||
|
|
||||||
def __init__(self, store: Any | None, data_dir: Path) -> None:
|
|
||||||
self.store = store
|
|
||||||
self.catalog = UserISOStore(store)
|
|
||||||
self.files = UserISOFiles(data_dir)
|
|
||||||
# Serializing global downloads makes the filesystem-free-space reserve
|
|
||||||
# meaningful across different owners: every job rechecks free space
|
|
||||||
# after the previous job has published its bytes.
|
|
||||||
self.executor = ThreadPoolExecutor(
|
|
||||||
max_workers=1, thread_name_prefix="user-iso-import")
|
|
||||||
self.lock = threading.RLock()
|
|
||||||
self.jobs: dict[str, dict[str, Any]] = {}
|
|
||||||
self.pending: set[str] = set()
|
|
||||||
self.in_use: dict[tuple[int, str], int] = {}
|
|
||||||
self.verified_sources: dict[tuple[int, str], tuple[int, int, int, int, str]] = {}
|
|
||||||
self._restored = False
|
|
||||||
|
|
||||||
def _mark_verified(self, user_id: int, iso_id: str, path: Path,
|
|
||||||
digest: str) -> None:
|
|
||||||
info = path.stat()
|
|
||||||
self.verified_sources[(user_id, iso_id)] = (
|
|
||||||
info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, digest)
|
|
||||||
|
|
||||||
def _verify_source(self, user_id: int, iso_id: str, path: Path,
|
|
||||||
digest: str) -> None:
|
|
||||||
info = path.stat()
|
|
||||||
identity = (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns)
|
|
||||||
cached = self.verified_sources.get((user_id, iso_id))
|
|
||||||
if cached is not None and cached[:4] == identity and cached[4] == digest:
|
|
||||||
return
|
|
||||||
checksum = hashlib.sha256()
|
|
||||||
with path.open("rb") as source:
|
|
||||||
while chunk := source.read(1024 * 1024):
|
|
||||||
checksum.update(chunk)
|
|
||||||
if checksum.hexdigest() != digest:
|
|
||||||
raise TemplateError("ISO 源文件的 SHA-256 校验值不匹配", 500)
|
|
||||||
self._mark_verified(user_id, iso_id, path, digest)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def public_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
return {key: value for key, value in record.items()
|
|
||||||
if key in {"id", "name", "size_bytes", "sha256", "created_at"}}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def public_error(value: Any) -> str:
|
|
||||||
safe = {
|
|
||||||
"面板在 ISO 下载期间重启;请重新提交导入",
|
|
||||||
"ISO 下载地址无效", "ISO 下载地址无法解析",
|
|
||||||
"ISO 下载地址的文件名必须以 .iso 结尾",
|
|
||||||
"ISO 下载地址解析结果无效",
|
|
||||||
"ISO 下载地址不能指向内网、回环或保留地址",
|
|
||||||
"ISO 下载地址发生重定向,请使用最终 HTTPS 直链",
|
|
||||||
"无法连接 ISO 下载地址", "ISO 下载响应长度无效",
|
|
||||||
"ISO 下载响应不能使用内容压缩", "ISO 下载超时",
|
|
||||||
"ISO 下载内容不完整", "ISO 的 SHA-256 校验值不匹配",
|
|
||||||
"ISO 大小必须在 1 MiB 至可用配额之间",
|
|
||||||
"下载文件不是有效的 ISO 9660 镜像",
|
|
||||||
"ISO 未包含可识别的 El Torito 启动记录",
|
|
||||||
"ISO 超过用户全局目录的 10 GiB 配额或剩余容量",
|
|
||||||
"用户 ISO 可用配额不足",
|
|
||||||
"每个用户最多保存 5 个 ISO",
|
|
||||||
"用户 ISO 全局目录的 10 GiB 存储配额已用完",
|
|
||||||
"用户 ISO 全局目录总容量不能超过 10 GiB",
|
|
||||||
"面板用户 ISO 持久目录空间不足",
|
|
||||||
"ISO 无法保存到面板持久目录",
|
|
||||||
"用户 ISO 导入失败,请查看引擎日志",
|
|
||||||
}
|
|
||||||
if isinstance(value, str) and value in safe:
|
|
||||||
return value
|
|
||||||
if isinstance(value, str) and re.fullmatch(
|
|
||||||
r"ISO 下载服务器返回 HTTP [1-5][0-9]{2}", value):
|
|
||||||
return value
|
|
||||||
return "ISO 下载或保存失败,请联系管理员查看日志"
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def public_job(cls, job: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
allowed = {"job_id", "status", "phase", "created_at", "started_at",
|
|
||||||
"finished_at", "iso", "error"}
|
|
||||||
result = {key: value for key, value in job.items() if key in allowed}
|
|
||||||
if isinstance(result.get("iso"), dict):
|
|
||||||
result["iso"] = cls.public_record(result["iso"])
|
|
||||||
if "error" in result:
|
|
||||||
result["error"] = cls.public_error(result["error"])
|
|
||||||
return result
|
|
||||||
|
|
||||||
def list_for(self, user_id: int) -> dict[str, Any]:
|
|
||||||
user_id = _positive_user_id(user_id)
|
|
||||||
scope = user_iso_scope(user_id)
|
|
||||||
result = {
|
|
||||||
"isos": [self.public_record(row) for row in self.catalog.list(scope)],
|
|
||||||
}
|
|
||||||
with self.lock:
|
|
||||||
active = [job for job in self.jobs.values()
|
|
||||||
if job.get("scope") == scope
|
|
||||||
and job.get("owner_user_id") == user_id
|
|
||||||
and job.get("status") in {"queued", "running"}]
|
|
||||||
if active:
|
|
||||||
current = max(active, key=lambda job: (
|
|
||||||
int(job.get("created_at", 0)), str(job.get("job_id", ""))))
|
|
||||||
result["active_import_job"] = self.public_job(current)
|
|
||||||
return result
|
|
||||||
|
|
||||||
def get_for(self, user_id: int, iso_id: str) -> dict[str, Any]:
|
|
||||||
user_id = _positive_user_id(user_id)
|
|
||||||
return self.catalog.get(user_iso_scope(user_id), iso_id)
|
|
||||||
|
|
||||||
def get_job_for(self, user_id: int, job_id: str) -> dict[str, Any]:
|
|
||||||
user_id = _positive_user_id(user_id)
|
|
||||||
scope = user_iso_scope(user_id)
|
|
||||||
with self.lock:
|
|
||||||
job = self.jobs.get(job_id)
|
|
||||||
if (job is None or job.get("scope") != scope
|
|
||||||
or job.get("owner_user_id") != user_id):
|
|
||||||
raise TemplateError("ISO 导入任务不存在", 404)
|
|
||||||
return self.public_job(job)
|
|
||||||
|
|
||||||
def _remaining_download_bytes(self, user_id: int, scope: str) -> int:
|
|
||||||
remaining = self.catalog.remaining_bytes(scope)
|
|
||||||
available = self.files.available_bytes(user_id)
|
|
||||||
maximum = min(remaining, available)
|
|
||||||
if maximum < 1024 * 1024:
|
|
||||||
raise TemplateError("面板用户 ISO 持久目录空间不足", 409)
|
|
||||||
return maximum
|
|
||||||
|
|
||||||
def queue_import(self, user_id: int, url: str, name: str,
|
|
||||||
expected_sha256: str) -> dict[str, str]:
|
|
||||||
user_id = _positive_user_id(user_id)
|
|
||||||
validate_user_iso_url(url)
|
|
||||||
name = validate_user_iso_name(name)
|
|
||||||
if (not isinstance(expected_sha256, str)
|
|
||||||
or (expected_sha256 and not SHA256_RE.fullmatch(expected_sha256))):
|
|
||||||
raise TemplateError(
|
|
||||||
"SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422)
|
|
||||||
expected_sha256 = expected_sha256.lower()
|
|
||||||
scope = user_iso_scope(user_id)
|
|
||||||
with self.lock:
|
|
||||||
existing = self.catalog.list(scope)
|
|
||||||
self._remaining_download_bytes(user_id, scope)
|
|
||||||
if any(row["name"].casefold() == name.casefold() for row in existing):
|
|
||||||
raise TemplateError("当前用户已有同名 ISO", 409)
|
|
||||||
if scope in self.pending:
|
|
||||||
raise TemplateError("已有用户 ISO 正在导入,请等待完成", 409)
|
|
||||||
job_id = str(uuid.uuid4())
|
|
||||||
self.pending.add(scope)
|
|
||||||
job = {
|
|
||||||
"job_id": job_id, "scope": scope, "owner_user_id": user_id,
|
|
||||||
"status": "queued", "phase": "等待下载", "created_at": int(time.time()),
|
|
||||||
}
|
|
||||||
try:
|
|
||||||
self.save_job(job_id, job)
|
|
||||||
self.executor.submit(
|
|
||||||
self._run_import, job_id, user_id, scope, url, name,
|
|
||||||
expected_sha256)
|
|
||||||
except Exception as exc:
|
|
||||||
self.pending.discard(scope)
|
|
||||||
self.delete_job(job_id)
|
|
||||||
if isinstance(exc, RuntimeError):
|
|
||||||
raise TemplateError("引擎正在停止,无法开始 ISO 导入", 503) from exc
|
|
||||||
raise
|
|
||||||
return {"job_id": job_id, "status": "queued"}
|
|
||||||
|
|
||||||
def _run_import(self, job_id: str, user_id: int, scope: str, url: str,
|
|
||||||
name: str, expected_sha256: str) -> None:
|
|
||||||
temporary: Path | None = None
|
|
||||||
published = False
|
|
||||||
cataloged = False
|
|
||||||
iso_id = ""
|
|
||||||
try:
|
|
||||||
with self.lock:
|
|
||||||
job = self.jobs[job_id]
|
|
||||||
job.update(status="running", phase="正在安全下载 ISO",
|
|
||||||
started_at=int(time.time()))
|
|
||||||
self.save_job(job_id, job)
|
|
||||||
maximum = self._remaining_download_bytes(user_id, scope)
|
|
||||||
directory = self.files.owner_directory(user_id)
|
|
||||||
temporary, size, digest = download_user_iso(
|
|
||||||
url, expected_sha256, directory, max_bytes=maximum)
|
|
||||||
iso_id = uuid.uuid4().hex
|
|
||||||
record = {
|
|
||||||
"id": iso_id, "name": name,
|
|
||||||
"volume_name": user_iso_volume_name(iso_id),
|
|
||||||
"size_bytes": size, "sha256": digest,
|
|
||||||
"created_at": int(time.time()),
|
|
||||||
}
|
|
||||||
with self.lock:
|
|
||||||
job = self.jobs[job_id]
|
|
||||||
job["phase"] = "正在保存到面板 ISO 目录"
|
|
||||||
self.save_job(job_id, job)
|
|
||||||
self.files.publish(temporary, user_id, iso_id, expected_size=size)
|
|
||||||
temporary = None
|
|
||||||
published = True
|
|
||||||
source_path = self.files.path(
|
|
||||||
user_id, iso_id, expected_size=size)
|
|
||||||
self._mark_verified(user_id, iso_id, source_path, digest)
|
|
||||||
self.catalog.add(scope, record)
|
|
||||||
cataloged = True
|
|
||||||
completed = {
|
|
||||||
"job_id": job_id, "scope": scope, "owner_user_id": user_id,
|
|
||||||
"status": "completed", "phase": "ISO 已保存,可用于所有宿主机",
|
|
||||||
"created_at": job["created_at"],
|
|
||||||
"started_at": job.get("started_at"),
|
|
||||||
"finished_at": int(time.time()), "iso": record,
|
|
||||||
}
|
|
||||||
self.save_job(job_id, completed)
|
|
||||||
except Exception as exc:
|
|
||||||
if isinstance(exc, TemplateError):
|
|
||||||
error = str(exc)
|
|
||||||
else:
|
|
||||||
LOG.exception("User ISO import failed: job=%s owner=%s", job_id, user_id)
|
|
||||||
error = "用户 ISO 导入失败,请查看引擎日志"
|
|
||||||
with self.lock:
|
|
||||||
if cataloged:
|
|
||||||
try:
|
|
||||||
self.catalog.delete(scope, iso_id)
|
|
||||||
except Exception:
|
|
||||||
LOG.exception("Could not clean failed user ISO catalog: job=%s", job_id)
|
|
||||||
if published:
|
|
||||||
try:
|
|
||||||
self.files.delete(user_id, iso_id)
|
|
||||||
except Exception:
|
|
||||||
LOG.exception("Could not clean failed user ISO source: job=%s", job_id)
|
|
||||||
previous = self.jobs.get(job_id, {})
|
|
||||||
failed = {
|
|
||||||
"job_id": job_id, "scope": scope, "owner_user_id": user_id,
|
|
||||||
"status": "failed", "phase": "导入失败", "error": error,
|
|
||||||
"created_at": previous.get("created_at", int(time.time())),
|
|
||||||
"started_at": previous.get("started_at"),
|
|
||||||
"finished_at": int(time.time()),
|
|
||||||
}
|
|
||||||
try:
|
|
||||||
self.save_job(job_id, failed)
|
|
||||||
except Exception:
|
|
||||||
LOG.exception("Could not persist user ISO import failure: job=%s", job_id)
|
|
||||||
finally:
|
|
||||||
if temporary is not None:
|
|
||||||
temporary.unlink(missing_ok=True)
|
|
||||||
with self.lock:
|
|
||||||
self.pending.discard(scope)
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def source(self, user_id: int, iso_id: str):
|
|
||||||
user_id = _positive_user_id(user_id)
|
|
||||||
key = (user_id, iso_id)
|
|
||||||
with self.lock:
|
|
||||||
record = self.get_for(user_id, iso_id)
|
|
||||||
path = self.files.path(
|
|
||||||
user_id, iso_id, expected_size=record["size_bytes"])
|
|
||||||
self._verify_source(
|
|
||||||
user_id, iso_id, path, record["sha256"])
|
|
||||||
self.in_use[key] = self.in_use.get(key, 0) + 1
|
|
||||||
try:
|
|
||||||
yield record, path
|
|
||||||
finally:
|
|
||||||
with self.lock:
|
|
||||||
count = self.in_use.get(key, 0)
|
|
||||||
if count <= 1:
|
|
||||||
self.in_use.pop(key, None)
|
|
||||||
else:
|
|
||||||
self.in_use[key] = count - 1
|
|
||||||
|
|
||||||
def delete_for(self, user_id: int, iso_id: str) -> None:
|
|
||||||
user_id = _positive_user_id(user_id)
|
|
||||||
scope = user_iso_scope(user_id)
|
|
||||||
key = (user_id, iso_id)
|
|
||||||
with self.lock:
|
|
||||||
if scope in self.pending:
|
|
||||||
raise TemplateError("用户 ISO 正在导入,请稍后删除", 409)
|
|
||||||
if self.in_use.get(key, 0):
|
|
||||||
raise TemplateError("ISO 正在同步到宿主机,请稍后删除", 409)
|
|
||||||
self.catalog.get(scope, iso_id)
|
|
||||||
# Delete bytes first. If the database write then fails, the
|
|
||||||
# catalog continues counting the quota and a retry can finish the
|
|
||||||
# deletion; users cannot fill the disk with unaccounted orphans.
|
|
||||||
self.files.delete(user_id, iso_id)
|
|
||||||
self.catalog.delete(scope, iso_id)
|
|
||||||
self.verified_sources.pop(key, None)
|
|
||||||
|
|
||||||
def save_job(self, job_id: str, job: dict[str, Any]) -> None:
|
|
||||||
self.jobs[job_id] = job
|
|
||||||
if self.store is not None:
|
|
||||||
self.store.set("user_iso_import_job", job_id, json.dumps(
|
|
||||||
job, ensure_ascii=False, separators=(",", ":")))
|
|
||||||
|
|
||||||
def delete_job(self, job_id: str) -> None:
|
|
||||||
self.jobs.pop(job_id, None)
|
|
||||||
if self.store is not None:
|
|
||||||
self.store.delete("user_iso_import_job", job_id)
|
|
||||||
|
|
||||||
def restore_jobs(self) -> None:
|
|
||||||
"""Mark work interrupted by an engine restart as failed exactly once."""
|
|
||||||
with self.lock:
|
|
||||||
if self._restored:
|
|
||||||
return
|
|
||||||
self.files.reconcile(self.catalog)
|
|
||||||
if self.store is not None:
|
|
||||||
for key, raw in self.store.list("user_iso_tombstone").items():
|
|
||||||
try:
|
|
||||||
tombstone = json.loads(raw)
|
|
||||||
owner_id = _positive_user_id(tombstone["user_id"])
|
|
||||||
iso_id = tombstone["iso_id"]
|
|
||||||
volume_name = user_iso_volume_name(iso_id)
|
|
||||||
pending_hosts = tombstone["pending_hosts"]
|
|
||||||
source_deleted = tombstone["source_deleted"]
|
|
||||||
except (KeyError, TypeError, ValueError) as exc:
|
|
||||||
raise TemplateError(
|
|
||||||
"用户 ISO 删除回收记录损坏", 500) from exc
|
|
||||||
if (key != f"{owner_id}:{iso_id}"
|
|
||||||
or tombstone.get("volume_name") != volume_name
|
|
||||||
or not isinstance(pending_hosts, list)
|
|
||||||
or any(not isinstance(host_id, str)
|
|
||||||
or not HOST_ID_RE.fullmatch(host_id)
|
|
||||||
for host_id in pending_hosts)
|
|
||||||
or len(set(pending_hosts)) != len(pending_hosts)
|
|
||||||
or not isinstance(source_deleted, bool)
|
|
||||||
or isinstance(tombstone.get("created_at"), bool)
|
|
||||||
or not isinstance(tombstone.get("created_at"), int)
|
|
||||||
or tombstone["created_at"] < 1):
|
|
||||||
raise TemplateError("用户 ISO 删除回收记录损坏", 500)
|
|
||||||
try:
|
|
||||||
catalog_scope = user_iso_scope(owner_id)
|
|
||||||
record = self.catalog.get(catalog_scope, iso_id)
|
|
||||||
except TemplateError as exc:
|
|
||||||
if exc.status != 404:
|
|
||||||
raise
|
|
||||||
exists = False
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
self.files.path(
|
|
||||||
owner_id, iso_id,
|
|
||||||
expected_size=record["size_bytes"])
|
|
||||||
except TemplateError:
|
|
||||||
# Crash window: source unlink committed but catalog
|
|
||||||
# deletion did not. Complete the user's deletion;
|
|
||||||
# retaining a visible record with no bytes would
|
|
||||||
# make every future deployment fail.
|
|
||||||
self.catalog.delete(catalog_scope, iso_id)
|
|
||||||
exists = False
|
|
||||||
else:
|
|
||||||
exists = True
|
|
||||||
if exists:
|
|
||||||
# The source/catalog deletion did not commit, so the
|
|
||||||
# user still owns the ISO and remote replicas stay.
|
|
||||||
self.store.delete("user_iso_tombstone", key)
|
|
||||||
elif not tombstone.get("source_deleted"):
|
|
||||||
tombstone["source_deleted"] = True
|
|
||||||
self.store.set("user_iso_tombstone", key, json.dumps(
|
|
||||||
tombstone, ensure_ascii=False, separators=(",", ":")))
|
|
||||||
source = self.store.list("user_iso_import_job") if self.store is not None else {}
|
|
||||||
for job_id, raw in source.items():
|
|
||||||
try:
|
|
||||||
job = json.loads(raw)
|
|
||||||
except (TypeError, ValueError) as exc:
|
|
||||||
raise TemplateError("用户 ISO 任务记录损坏", 500) from exc
|
|
||||||
if not isinstance(job, dict):
|
|
||||||
raise TemplateError("用户 ISO 任务记录损坏", 500)
|
|
||||||
if job.get("status") in {"queued", "running"}:
|
|
||||||
job.update(
|
|
||||||
status="failed",
|
|
||||||
phase="引擎重启,任务已中断",
|
|
||||||
error="面板在 ISO 下载期间重启;请重新提交导入",
|
|
||||||
finished_at=int(time.time()),
|
|
||||||
)
|
|
||||||
self.save_job(job_id, job)
|
|
||||||
self.jobs[job_id] = job
|
|
||||||
self._restored = True
|
|
||||||
|
|
||||||
def close(self) -> None:
|
|
||||||
self.executor.shutdown(wait=True, cancel_futures=False)
|
|
||||||
|
|
||||||
def has_active_jobs(self) -> bool:
|
|
||||||
with self.lock:
|
|
||||||
return bool(self.pending) or any(
|
|
||||||
job.get("status") in {"queued", "running"}
|
|
||||||
for job in self.jobs.values()
|
|
||||||
)
|
|
||||||
|
Before Width: | Height: | Size: 3.2 KiB After Width: | Height: | Size: 3.2 KiB |
|
Before Width: | Height: | Size: 1.7 KiB After Width: | Height: | Size: 1.7 KiB |
|
Before Width: | Height: | Size: 4.9 KiB After Width: | Height: | Size: 4.9 KiB |
|
Before Width: | Height: | Size: 1.3 KiB After Width: | Height: | Size: 1.3 KiB |