Compare commits
No commits in common. 'venix' and 'main' have entirely different histories.
@ -1,37 +0,0 @@
|
||||
# Virt Cloud development rules
|
||||
|
||||
- This repository is the independent Virt Cloud project. Work only inside this
|
||||
checkout unless the user explicitly authorizes changes to another project.
|
||||
- When a task depends on historical decisions, test results, or project
|
||||
context, and the knowledge vault is available, first read
|
||||
`/Users/admin/Documents/www/codex-knowledge/AGENTS.md`, then
|
||||
`Memory/INDEX.md` and `Memory/CURRENT.md`; follow only the linked notes
|
||||
relevant to the task. If the vault is unavailable, say so explicitly and do
|
||||
not guess the old project state.
|
||||
- The Git remote is `https://g.luoli.live/root/virt.git`.
|
||||
- All subsequent changes, commits and pushes for this project must use the
|
||||
`venix` branch. Do not commit or push them to `main` or force-push.
|
||||
- Before editing or committing, check the current branch and working tree.
|
||||
Preserve unrelated user changes. If this checkout is not on `venix`, stop
|
||||
before changing files and resolve the branch safely.
|
||||
- Never commit real configuration, credentials, private keys, database dumps,
|
||||
installed dependencies, runtime logs or caches. Keep source configuration
|
||||
examples and source schema files versioned.
|
||||
- The PHP application lives in `web/`; its eventual HTTP document root is
|
||||
`web/public/`, not the repository root.
|
||||
- Only the venix theme is retained. Ordinary users must not receive host
|
||||
administration or retired node-management routes.
|
||||
- PHP owns accounts, wallet purchases, orders, public-key ownership and public
|
||||
HTTP authorization. The Python engine owns libvirt, host credentials and VM
|
||||
operations. Keep the engine private and authenticate every internal request.
|
||||
- Purchases provision one VM from the plan snapshot. Each purchase or renewal
|
||||
creates an independent prepaid service period with its own traffic quota.
|
||||
An exhausted quota disconnects VM networking until the next paid period;
|
||||
final expiry disconnects networking without powering off the VM. An unpaid
|
||||
renewal invoice enters a three-day grace period and then permanently removes
|
||||
the VM definition and its panel-managed disks. Never replay an ambiguous
|
||||
destructive operation or refund it without confirmed cleanup; reconcile its
|
||||
operation identifier.
|
||||
- Users share the original VM-panel styling; administrator pages retain venix.
|
||||
- See `docs/PRUNING_REPORT.md` for retained/removed feature boundaries and
|
||||
`docs/INTEGRATION_PLAN.md` for the planned Python engine integration.
|
||||
@ -0,0 +1,8 @@
|
||||
# Third-party code
|
||||
|
||||
`static/vendor/novnc/` contains the `core/` and `vendor/` directories from
|
||||
[noVNC v1.7.0](https://github.com/novnc/noVNC/releases/tag/v1.7.0), plus its
|
||||
`LICENSE.txt`. The upstream source archive was downloaded from the noVNC GitHub
|
||||
release tag without modifications. The bundled `vendor/pako/` dependency retains
|
||||
its own license file.
|
||||
|
||||
@ -1,106 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"policy_version": 1,
|
||||
"scope": "public_ipv4_interface",
|
||||
"applies_to": "new_public_cloud_image_instances_only",
|
||||
"migrate_existing": false,
|
||||
"rules": [
|
||||
{
|
||||
"id": "qotd",
|
||||
"label": "QOTD",
|
||||
"category": "permanent",
|
||||
"protocols": ["tcp", "udp"],
|
||||
"port_from": 17,
|
||||
"port_to": 17,
|
||||
"directions": ["ingress", "egress"],
|
||||
"reason": "降低反射放大与滥用风险。"
|
||||
},
|
||||
{
|
||||
"id": "chargen",
|
||||
"label": "Chargen",
|
||||
"category": "permanent",
|
||||
"protocols": ["tcp", "udp"],
|
||||
"port_from": 19,
|
||||
"port_to": 19,
|
||||
"directions": ["ingress", "egress"],
|
||||
"reason": "降低反射放大与滥用风险。"
|
||||
},
|
||||
{
|
||||
"id": "ssdp",
|
||||
"label": "SSDP",
|
||||
"category": "permanent",
|
||||
"protocols": ["tcp", "udp"],
|
||||
"port_from": 1900,
|
||||
"port_to": 1900,
|
||||
"directions": ["ingress", "egress"],
|
||||
"reason": "降低公网发现服务与反射放大滥用风险。"
|
||||
},
|
||||
{
|
||||
"id": "memcached",
|
||||
"label": "Memcached UDP",
|
||||
"category": "permanent",
|
||||
"protocols": ["udp"],
|
||||
"port_from": 11211,
|
||||
"port_to": 11211,
|
||||
"directions": ["ingress", "egress"],
|
||||
"reason": "降低 Memcached UDP 反射放大风险。"
|
||||
},
|
||||
{
|
||||
"id": "brand-rex-53413",
|
||||
"label": "Netis / Netcore 后门端口",
|
||||
"category": "permanent",
|
||||
"protocols": ["udp"],
|
||||
"port_from": 53413,
|
||||
"port_to": 53413,
|
||||
"directions": ["ingress", "egress"],
|
||||
"reason": "阻断已知路由器后门相关公网流量。"
|
||||
},
|
||||
{
|
||||
"id": "smtp-25",
|
||||
"label": "SMTP 直投",
|
||||
"category": "default",
|
||||
"protocols": ["tcp"],
|
||||
"port_from": 25,
|
||||
"port_to": 25,
|
||||
"directions": ["egress"],
|
||||
"reason": "降低垃圾邮件滥用风险;例外申请流程尚未实现。"
|
||||
},
|
||||
{
|
||||
"id": "netbios",
|
||||
"label": "NetBIOS",
|
||||
"category": "default",
|
||||
"protocols": ["tcp", "udp"],
|
||||
"port_from": 137,
|
||||
"port_to": 139,
|
||||
"directions": ["ingress", "egress"],
|
||||
"reason": "默认阻断 NetBIOS 服务暴露与公网访问。"
|
||||
},
|
||||
{
|
||||
"id": "smb",
|
||||
"label": "SMB",
|
||||
"category": "default",
|
||||
"protocols": ["tcp", "udp"],
|
||||
"port_from": 445,
|
||||
"port_to": 445,
|
||||
"directions": ["ingress", "egress"],
|
||||
"reason": "默认阻断 SMB 文件共享服务的公网暴露与访问。"
|
||||
},
|
||||
{
|
||||
"id": "kms-1688",
|
||||
"label": "KMS 服务端",
|
||||
"category": "default",
|
||||
"protocols": ["tcp"],
|
||||
"port_from": 1688,
|
||||
"port_to": 1688,
|
||||
"directions": ["ingress"],
|
||||
"reason": "默认阻断 KMS 服务端口直接暴露到公网。"
|
||||
}
|
||||
],
|
||||
"explicitly_unrestricted": [
|
||||
{
|
||||
"protocol": "tcp",
|
||||
"port": 587,
|
||||
"reason": "SMTP 提交端口当前不受平台默认策略限制。"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
[Unit]
|
||||
Description=Virsh Panel one-shot update from the pinned private Gitea repository
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
Group=root
|
||||
WorkingDirectory=/home/virsh-panel
|
||||
Environment=PANEL_DATA_DIR=/home/virsh-panel/data
|
||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||
ExecStart=/home/virsh-panel/.venv/bin/python -m virsh_panel.update_worker
|
||||
ExecStopPost=/home/virsh-panel/.venv/bin/python -m virsh_panel.update_worker --finalize-incomplete
|
||||
TimeoutStartSec=30min
|
||||
UMask=0077
|
||||
LimitCORE=0
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=false
|
||||
ReadWritePaths=/home/virsh-panel
|
||||
@ -0,0 +1,26 @@
|
||||
[Unit]
|
||||
Description=Virsh Panel (local HTTP upstream and remote libvirt)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
User=root
|
||||
Group=root
|
||||
WorkingDirectory=/home/virsh-panel
|
||||
Environment=PANEL_DATA_DIR=/home/virsh-panel/data
|
||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||
ExecStartPre=+/usr/bin/install -d -o root -g root -m 0700 /home/virsh-panel/data
|
||||
ExecStart=/home/virsh-panel/.venv/bin/python -m uvicorn virsh_panel.app:create_app --factory --host 127.0.0.1 --port 8000 --workers 1 --proxy-headers --forwarded-allow-ips 127.0.0.1
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
CapabilityBoundingSet=CAP_SETPCAP
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=false
|
||||
ReadWritePaths=-/home/virsh-panel/data
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -1,27 +0,0 @@
|
||||
[Unit]
|
||||
Description=Virt Cloud private libvirt engine
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
User=root
|
||||
Group=root
|
||||
WorkingDirectory=/home/virt-cloud/engine
|
||||
Environment=CLOUD_ENGINE_CONFIG=/home/virt-cloud/runtime/engine.json
|
||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||
ExecStart=/home/virt-cloud/engine/.venv/bin/python -m uvicorn virsh_panel.internal:create_engine --factory --host 127.0.0.1 --port 8000 --workers 1 --timeout-graceful-shutdown 10 --proxy-headers --forwarded-allow-ips 127.0.0.1 --no-access-log
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=45s
|
||||
KillMode=control-group
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=false
|
||||
ReadWritePaths=/home/virt-cloud/runtime
|
||||
LimitCORE=0
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -1,24 +0,0 @@
|
||||
[Unit]
|
||||
Description=Virt Cloud service-period and traffic lifecycle worker
|
||||
After=network-online.target virt-cloud-engine.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=apache
|
||||
Group=apache
|
||||
WorkingDirectory=/home/virt-cloud/web
|
||||
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudLifecycle
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=false
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=-/home/virt-cloud/runtime/php
|
||||
ReadWritePaths=-/home/virt-cloud/web/storage
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -1,24 +0,0 @@
|
||||
[Unit]
|
||||
Description=Virt Cloud enforcement policy worker %i
|
||||
After=network-online.target virt-cloud-engine.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=apache
|
||||
Group=apache
|
||||
WorkingDirectory=/home/virt-cloud/web
|
||||
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudWorker --policy-only
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=false
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=-/home/virt-cloud/runtime/php
|
||||
ReadWritePaths=-/home/virt-cloud/web/storage
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -1,23 +0,0 @@
|
||||
[Unit]
|
||||
Description=Virt Cloud privileged online update worker
|
||||
After=network-online.target mariadb.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
Group=root
|
||||
WorkingDirectory=/home/virt-cloud/engine
|
||||
Environment=CLOUD_ENGINE_CONFIG=/home/virt-cloud/runtime/engine.json
|
||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||
ExecStart=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker
|
||||
ExecStopPost=/home/virt-cloud/engine/.venv/bin/python -m virsh_panel.update_worker --finalize-incomplete
|
||||
TimeoutStartSec=45min
|
||||
TimeoutStopSec=10min
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=false
|
||||
ReadWritePaths=/home/virt-cloud
|
||||
LimitCORE=0
|
||||
@ -1,24 +0,0 @@
|
||||
[Unit]
|
||||
Description=Virt Cloud VPC control-plane worker
|
||||
After=network-online.target virt-cloud-engine.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=apache
|
||||
Group=apache
|
||||
WorkingDirectory=/home/virt-cloud/web
|
||||
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudVpcWorker
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=false
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=-/home/virt-cloud/runtime/php
|
||||
ReadWritePaths=-/home/virt-cloud/web/storage
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -1,25 +0,0 @@
|
||||
[Unit]
|
||||
Description=Virt Cloud purchased VM provisioning worker
|
||||
After=network-online.target virt-cloud-engine.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
# AlmaLinux PHP-FPM normally runs as apache; use the same user as your PHP pool.
|
||||
User=apache
|
||||
Group=apache
|
||||
WorkingDirectory=/home/virt-cloud/web
|
||||
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudWorker
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=false
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=-/home/virt-cloud/runtime/php
|
||||
ReadWritePaths=-/home/virt-cloud/web/storage
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,6 +0,0 @@
|
||||
.DS_Store
|
||||
.trash/
|
||||
.obsidian/cache/
|
||||
.obsidian/workspace.json
|
||||
.obsidian/workspaces.json
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
---
|
||||
type: hub
|
||||
status: active
|
||||
created: 2026-09-29
|
||||
updated: 2026-09-29
|
||||
tags:
|
||||
- knowledge-base
|
||||
---
|
||||
|
||||
# 个人知识库
|
||||
|
||||
这是个人知识的入口页。知识以本地 Markdown 保存,由
|
||||
[[Concepts/Obsidian|Obsidian]] 编辑和可视化,由
|
||||
[[Concepts/Codex|Codex]] 检索、总结和维护。
|
||||
|
||||
## 继续工作
|
||||
|
||||
- [[Memory/CURRENT|当前状态]]
|
||||
- [[Memory/INDEX|知识索引]]
|
||||
- [[Memory/PROFILE|个人偏好]]
|
||||
- [[Memory/DECISIONS|决策索引]]
|
||||
|
||||
## 当前项目
|
||||
|
||||
- [[Projects/个人知识库系统|个人知识库系统]]
|
||||
|
||||
## 核心概念
|
||||
|
||||
- [[Concepts/Obsidian|Obsidian]]
|
||||
- [[Concepts/Codex|Codex]]
|
||||
- [[Concepts/Codex集成|Codex 集成]]
|
||||
- [[Concepts/原子笔记|原子笔记]]
|
||||
- [[Concepts/内部链接|内部链接]]
|
||||
- [[Concepts/知识图谱|知识图谱]]
|
||||
|
||||
## 快速入口
|
||||
|
||||
- [[01-Inbox|收件箱]]
|
||||
- [[Canvas/知识库总览.canvas|知识库总览 Canvas]]
|
||||
- [[README|使用说明]]
|
||||
@ -1,17 +0,0 @@
|
||||
---
|
||||
type: inbox
|
||||
status: active
|
||||
updated: 2026-09-29
|
||||
tags:
|
||||
- inbox
|
||||
---
|
||||
|
||||
# 收件箱
|
||||
|
||||
临时想法先写在这里,整理时再拆成独立的
|
||||
[[Concepts/原子笔记|原子笔记]],并补充有意义的内部链接。
|
||||
|
||||
## 待整理
|
||||
|
||||
- 暂无。
|
||||
|
||||
@ -1,134 +0,0 @@
|
||||
{
|
||||
"nodes": [
|
||||
{
|
||||
"id": "home",
|
||||
"type": "file",
|
||||
"file": "00-Home.md",
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"width": 360,
|
||||
"height": 220
|
||||
},
|
||||
{
|
||||
"id": "index",
|
||||
"type": "file",
|
||||
"file": "Memory/INDEX.md",
|
||||
"x": -500,
|
||||
"y": 0,
|
||||
"width": 340,
|
||||
"height": 220
|
||||
},
|
||||
{
|
||||
"id": "current",
|
||||
"type": "file",
|
||||
"file": "Memory/CURRENT.md",
|
||||
"x": -500,
|
||||
"y": 300,
|
||||
"width": 340,
|
||||
"height": 260
|
||||
},
|
||||
{
|
||||
"id": "project",
|
||||
"type": "file",
|
||||
"file": "Projects/个人知识库系统.md",
|
||||
"x": 0,
|
||||
"y": 300,
|
||||
"width": 360,
|
||||
"height": 260
|
||||
},
|
||||
{
|
||||
"id": "codex",
|
||||
"type": "file",
|
||||
"file": "Concepts/Codex集成.md",
|
||||
"x": 520,
|
||||
"y": -120,
|
||||
"width": 340,
|
||||
"height": 240
|
||||
},
|
||||
{
|
||||
"id": "obsidian",
|
||||
"type": "file",
|
||||
"file": "Concepts/Obsidian.md",
|
||||
"x": 520,
|
||||
"y": 180,
|
||||
"width": 340,
|
||||
"height": 240
|
||||
},
|
||||
{
|
||||
"id": "graph",
|
||||
"type": "file",
|
||||
"file": "Concepts/知识图谱.md",
|
||||
"x": 520,
|
||||
"y": 480,
|
||||
"width": 340,
|
||||
"height": 240
|
||||
},
|
||||
{
|
||||
"id": "decision",
|
||||
"type": "file",
|
||||
"file": "Decisions/2026-09-29-采用本地Markdown与双向链接.md",
|
||||
"x": 0,
|
||||
"y": 640,
|
||||
"width": 360,
|
||||
"height": 240
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"id": "e1",
|
||||
"fromNode": "home",
|
||||
"fromSide": "left",
|
||||
"toNode": "index",
|
||||
"toSide": "right",
|
||||
"label": "导航"
|
||||
},
|
||||
{
|
||||
"id": "e2",
|
||||
"fromNode": "index",
|
||||
"fromSide": "bottom",
|
||||
"toNode": "current",
|
||||
"toSide": "top",
|
||||
"label": "恢复状态"
|
||||
},
|
||||
{
|
||||
"id": "e3",
|
||||
"fromNode": "home",
|
||||
"fromSide": "bottom",
|
||||
"toNode": "project",
|
||||
"toSide": "top",
|
||||
"label": "当前项目"
|
||||
},
|
||||
{
|
||||
"id": "e4",
|
||||
"fromNode": "project",
|
||||
"fromSide": "right",
|
||||
"toNode": "codex",
|
||||
"toSide": "left",
|
||||
"label": "调用"
|
||||
},
|
||||
{
|
||||
"id": "e5",
|
||||
"fromNode": "project",
|
||||
"fromSide": "right",
|
||||
"toNode": "obsidian",
|
||||
"toSide": "left",
|
||||
"label": "管理"
|
||||
},
|
||||
{
|
||||
"id": "e6",
|
||||
"fromNode": "obsidian",
|
||||
"fromSide": "bottom",
|
||||
"toNode": "graph",
|
||||
"toSide": "top",
|
||||
"label": "可视化"
|
||||
},
|
||||
{
|
||||
"id": "e7",
|
||||
"fromNode": "project",
|
||||
"fromSide": "bottom",
|
||||
"toNode": "decision",
|
||||
"toSide": "top",
|
||||
"label": "依据"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
---
|
||||
type: decision
|
||||
status: accepted
|
||||
date: 2026-09-29
|
||||
updated: 2026-09-29
|
||||
tags:
|
||||
- decision/virt-cloud
|
||||
- billing
|
||||
---
|
||||
|
||||
# Virt Cloud 统一账单
|
||||
|
||||
## 背景
|
||||
|
||||
旧 `bought` 购买记录已经废弃且确认无数据,继续保留会与云账单形成两个事实源。
|
||||
|
||||
## 决定
|
||||
|
||||
- `cloud_invoices` 是账单唯一事实源,不再读写或恢复 `bought`。
|
||||
- 管理后台通过一个账单系统分别展示账单、开通任务和支付尝试,不把财务状态与履约状态合并成一个状态。
|
||||
- 旧表清理必须先验证为空;存在数据时停止升级,不做静默丢弃。
|
||||
- 支付尝试保留为独立审计流水,不能替代账单最终支付渠道。
|
||||
|
||||
## 影响
|
||||
|
||||
- 新报表、筛选和运营页面都应基于云账单表及其关联表。
|
||||
- 不再为旧购买记录维护兼容路由、模型、模板或双写逻辑。
|
||||
- 后续扩展退款、续费和对账时,继续保持“财务、履约、支付尝试”三类状态边界。
|
||||
|
||||
## 相关
|
||||
|
||||
- [[Projects/Virt Cloud|Virt Cloud]]
|
||||
- [[Memory/DECISIONS|决策索引]]
|
||||
- [[Concepts/Codex集成|Codex 集成]]
|
||||
@ -1,34 +0,0 @@
|
||||
---
|
||||
type: decision
|
||||
status: accepted
|
||||
date: 2026-09-29
|
||||
updated: 2026-09-29
|
||||
tags:
|
||||
- decision
|
||||
---
|
||||
|
||||
# 采用本地 Markdown 与双向链接
|
||||
|
||||
## 背景
|
||||
|
||||
需要让个人知识长期可迁移,同时能被 Obsidian 和 Codex 使用。
|
||||
|
||||
## 决定
|
||||
|
||||
以本地 Markdown 为事实源,用 `[[内部链接]]` 表达关系;
|
||||
[[Concepts/Obsidian|Obsidian]] 负责编辑和可视化,
|
||||
[[Concepts/Codex|Codex]] 负责检索、总结和维护。
|
||||
|
||||
## 理由
|
||||
|
||||
- 不依赖专有数据库。
|
||||
- Codex 可直接读取。
|
||||
- Git 可以追踪和恢复改动。
|
||||
- [[Concepts/知识图谱|知识图谱]] 可由链接自动生成。
|
||||
|
||||
## 影响
|
||||
|
||||
- Canvas 只作为人工整理的视图,不作为唯一事实源。
|
||||
- 重要知识需要主动拆成 [[Concepts/原子笔记|原子笔记]]。
|
||||
- 新 Codex 对话通过 [[Memory/CURRENT|当前状态]] 恢复,而不是读取聊天全文。
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
---
|
||||
type: decision
|
||||
status: accepted
|
||||
date: 2026-09-30
|
||||
updated: 2026-09-30
|
||||
tags:
|
||||
- decision/virt-cloud
|
||||
- monitoring
|
||||
---
|
||||
|
||||
# Virt Cloud 监控独立页面
|
||||
|
||||
## 背景
|
||||
|
||||
讨论过在管理员虚拟机管理页增加实时资源与连接统计,用户目前不需要实现。
|
||||
|
||||
## 决定与理由
|
||||
|
||||
- 暂不在虚拟机管理页增加任何监控功能或按钮,保持该页现有职责。
|
||||
- 后续如需虚拟机指标与网络恶意使用监测,另建独立的“虚拟机指标”页面,避免管理页继续扩张。
|
||||
|
||||
## 影响
|
||||
|
||||
- 这是页面边界决定,不是立即开发授权。
|
||||
- 采集方式、告警和处置规则尚未确定;不可据此自动安装 Agent、改防火墙或封禁实例。
|
||||
|
||||
## 相关
|
||||
|
||||
- [[Projects/Virt Cloud|Virt Cloud]]
|
||||
- [[Memory/CURRENT|当前状态]]
|
||||
- [[Memory/DECISIONS|决策索引]]
|
||||
@ -1,30 +0,0 @@
|
||||
---
|
||||
type: index
|
||||
status: active
|
||||
updated: 2026-10-09
|
||||
tags:
|
||||
- memory/decisions
|
||||
---
|
||||
|
||||
# 决策索引
|
||||
|
||||
- [[Decisions/2026-10-09-Virt Cloud VPC自动子网与地区Fabric向导|Virt Cloud VPC 自动子网与地区 Fabric 向导]]
|
||||
- [[Decisions/2026-10-09-Virt Cloud Fabric underlay与firewalld所有权边界|Virt Cloud Fabric underlay 与 firewalld 所有权边界]]
|
||||
- [[Decisions/2026-10-07-Virt Cloud账户实例数量上限移除|Virt Cloud 账户实例数量上限移除]]
|
||||
- [[Decisions/2026-10-07-Virt Cloud部署页SQL回归修复|Virt Cloud 部署页 SQL 回归修复]]
|
||||
- [[Decisions/2026-10-07-Virt Cloud面板资源账本取代宿主机盘点|Virt Cloud 面板资源账本取代宿主机盘点]]
|
||||
- [[Decisions/2026-10-08-Virt Cloud VPC与Fabric静态实现及验收边界|Virt Cloud VPC 与 Fabric 静态实现及验收边界]]
|
||||
- [[Decisions/2026-09-29-采用本地Markdown与双向链接|采用本地 Markdown 与双向链接]]
|
||||
- [[Decisions/2026-09-29-Virt Cloud统一账单|Virt Cloud 统一账单]]
|
||||
- [[Decisions/2026-09-30-Virt Cloud监控独立页面|Virt Cloud 监控独立页面]]
|
||||
- [[Decisions/2026-10-01-Virt Cloud资源池与线路销售方案|Virt Cloud 资源池与线路销售方案]]
|
||||
- [[Decisions/2026-10-01-Virt Cloud跨宿主机L3私网|Virt Cloud 跨宿主机 L3 私网]]
|
||||
- [[Decisions/2026-10-02-Virt Cloud管理员更换公网IP第一阶段|Virt Cloud 管理员更换公网 IP 第一阶段]]
|
||||
- [[Decisions/2026-10-03-Virt Cloud单宿主机多公网CIDR|Virt Cloud 单宿主机多公网 CIDR]]
|
||||
- [[Decisions/2026-10-05-Virt Cloud本地在线快照|Virt Cloud 本地在线快照]]
|
||||
- [[Decisions/2026-10-05-Virt Cloud实例防火墙与换IP一致性|Virt Cloud 实例防火墙与换 IP 一致性]]
|
||||
- [[Decisions/2026-10-06-Virt Cloud平台默认与测试端口策略|Virt Cloud 平台默认与测试端口策略]]
|
||||
|
||||
## 记录规则
|
||||
|
||||
每项长期决定独立成文,至少记录日期、状态、背景、决定、理由和影响。
|
||||
@ -1,50 +0,0 @@
|
||||
---
|
||||
type: index
|
||||
status: active
|
||||
updated: 2026-10-12
|
||||
tags:
|
||||
- memory/index
|
||||
---
|
||||
|
||||
# 知识索引
|
||||
|
||||
Codex 从这里判断该读取哪些笔记,而不是扫描整个 Vault。
|
||||
|
||||
## 状态与偏好
|
||||
|
||||
- [[Memory/CURRENT|当前状态]]:进行中的工作与明确下一步。
|
||||
- [[Memory/PROFILE|个人偏好]]:会影响未来协作的稳定偏好。
|
||||
- [[Memory/DECISIONS|决策索引]]:长期有效决定的入口。
|
||||
|
||||
## 项目
|
||||
|
||||
- [[Projects/个人知识库系统|个人知识库系统]]
|
||||
- [[Projects/Virt Cloud|Virt Cloud]]
|
||||
|
||||
## 方法
|
||||
|
||||
- [[Concepts/Virt Cloud VPC永久邻居与重复协调|VPC 永久邻居与健康协调]]
|
||||
|
||||
- [[Concepts/原子笔记|原子笔记]]
|
||||
- [[Concepts/内部链接|内部链接]]
|
||||
- [[Concepts/知识图谱|知识图谱]]
|
||||
- [[Concepts/Codex集成|Codex 集成]]
|
||||
- [[Concepts/Virt Cloud WireGuard Fabric与VPC L3私网|WireGuard Fabric 与 VPC L3 私网]]
|
||||
|
||||
## 工具
|
||||
|
||||
- [[Concepts/Obsidian|Obsidian]]
|
||||
- [[Concepts/Codex|Codex]]
|
||||
|
||||
## 最近交接
|
||||
|
||||
- [[Sessions/2026-10-12-Virt Cloud VPC邻居持久化与双宿主互通|VPC 邻居持久化、双宿主实测与部署边界]]
|
||||
|
||||
- [[Sessions/2026-10-09-Virt Cloud Fabric公网优先与firewalld交付|Virt Cloud Fabric 公网优先与 firewalld 交付]]
|
||||
- [[Sessions/2026-10-08-Virt Cloud VPC用户控制台与数据面交付|Virt Cloud VPC 用户控制台与数据面交付]]
|
||||
- [[Sessions/2026-10-07-Virt Cloud实例部署、资源账本与账户上限清理|Virt Cloud 实例部署、资源账本与账户上限清理]]
|
||||
- [[Sessions/2026-10-06-Virt Cloud快照删除修复与libvirt排错|Virt Cloud 快照删除修复与 libvirt 排错]]
|
||||
- [[Sessions/2026-10-06-Virt Cloud平台端口策略与测试验证|Virt Cloud 平台端口策略与测试验证]]
|
||||
- [[Sessions/2026-10-05-Virt Cloud盘点快照与防火墙|Virt Cloud 盘点、在线快照与实例防火墙]]
|
||||
- [[Sessions/2026-09-29-知识库初始化|知识库初始化]]
|
||||
- [[Sessions/2026-10-01-Virt Cloud资源调度线路备份与私网讨论|Virt Cloud 资源调度线路备份与私网讨论]]
|
||||
@ -1,25 +0,0 @@
|
||||
---
|
||||
type: profile
|
||||
status: active
|
||||
updated: 2026-09-29
|
||||
tags:
|
||||
- memory/profile
|
||||
---
|
||||
|
||||
# 个人偏好
|
||||
|
||||
只记录会影响未来协作的稳定偏好,不记录秘密或敏感信息。
|
||||
|
||||
## 工作方式
|
||||
|
||||
- 希望新 Codex 对话可以快速恢复之前的工作背景。
|
||||
- 偏好本地、可迁移、可被普通文本工具读取的知识库。
|
||||
- 希望知识之间可以通过动态图谱观察关联。
|
||||
- 精确项目状态应写入文件,而不是只依赖自动记忆。
|
||||
|
||||
## 相关
|
||||
|
||||
- [[Concepts/Codex集成|Codex 集成]]
|
||||
- [[Concepts/Obsidian|Obsidian]]
|
||||
- [[Projects/个人知识库系统|个人知识库系统]]
|
||||
|
||||
@ -1,36 +0,0 @@
|
||||
---
|
||||
type: project
|
||||
status: active
|
||||
created: 2026-09-29
|
||||
updated: 2026-09-29
|
||||
tags:
|
||||
- project
|
||||
---
|
||||
|
||||
# 个人知识库系统
|
||||
|
||||
## 目标
|
||||
|
||||
让知识由 [[Concepts/Obsidian|Obsidian]] 管理和可视化,并让
|
||||
[[Concepts/Codex|Codex]] 在新对话中恢复必要上下文。
|
||||
|
||||
## 组成
|
||||
|
||||
- [[Concepts/原子笔记|原子笔记]]:控制每个图谱节点的粒度。
|
||||
- [[Concepts/知识图谱|知识图谱]]:由 Markdown 内部链接自动生成。
|
||||
- [[Concepts/Codex集成|Codex 集成]]:通过本地文件和 `AGENTS.md` 路由。
|
||||
- [[Memory/CURRENT|当前状态]]:承担跨对话交接。
|
||||
- [[Memory/DECISIONS|决策索引]]:保存长期决定。
|
||||
|
||||
## 成功标准
|
||||
|
||||
- 新 Codex 对话能概括当前目标、进展和下一步。
|
||||
- Obsidian Graph 能显示项目、概念、状态和决定之间的连线。
|
||||
- 知识仍可被普通文本工具读取,不依赖封闭平台。
|
||||
- 重要变更可以通过 Git 查看和恢复。
|
||||
|
||||
## 当前阶段
|
||||
|
||||
参见 [[Memory/CURRENT|当前状态]] 和
|
||||
[[Sessions/2026-09-29-知识库初始化|初始化交接]]。
|
||||
|
||||
@ -1,25 +0,0 @@
|
||||
---
|
||||
type: decision
|
||||
status: proposed
|
||||
date: "{{date:YYYY-MM-DD}}"
|
||||
updated: "{{date:YYYY-MM-DD}}"
|
||||
tags:
|
||||
- decision
|
||||
---
|
||||
|
||||
# {{title}}
|
||||
|
||||
## 背景
|
||||
|
||||
## 决定
|
||||
|
||||
## 理由
|
||||
|
||||
## 备选方案
|
||||
|
||||
## 影响
|
||||
|
||||
## 关联
|
||||
|
||||
-
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
status: seed
|
||||
created: "{{date:YYYY-MM-DD}}"
|
||||
updated: "{{date:YYYY-MM-DD}}"
|
||||
aliases: []
|
||||
tags:
|
||||
- concept
|
||||
---
|
||||
|
||||
# {{title}}
|
||||
|
||||
一句话说明:
|
||||
|
||||
## 要点
|
||||
|
||||
-
|
||||
|
||||
## 关联
|
||||
|
||||
- 上位概念:
|
||||
- 相邻概念:
|
||||
- 应用于:
|
||||
|
||||
## 来源或验证
|
||||
|
||||
-
|
||||
|
||||
@ -1,25 +0,0 @@
|
||||
---
|
||||
type: project
|
||||
status: active
|
||||
created: "{{date:YYYY-MM-DD}}"
|
||||
updated: "{{date:YYYY-MM-DD}}"
|
||||
tags:
|
||||
- project
|
||||
---
|
||||
|
||||
# {{title}}
|
||||
|
||||
## 目标
|
||||
|
||||
## 非目标
|
||||
|
||||
## 当前状态
|
||||
|
||||
## 下一步
|
||||
|
||||
- [ ]
|
||||
|
||||
## 决策与概念
|
||||
|
||||
-
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
.venv/
|
||||
data/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
@ -1,303 +0,0 @@
|
||||
"""Importable libvirt stand-in for real RPC subprocess boundary tests.
|
||||
|
||||
This module deliberately lives outside ``virsh_panel``. Production requests
|
||||
cannot choose it; tests explicitly inject its module name into the supervisor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
VIR_ERR_NO_DOMAIN = 42
|
||||
VIR_ERR_OPERATION_FAILED = 9
|
||||
VIR_ERR_NO_STORAGE_VOL = 50
|
||||
|
||||
|
||||
class libvirtError(Exception):
|
||||
def __init__(self, message, err=None):
|
||||
super().__init__(message)
|
||||
self.err = err or (
|
||||
VIR_ERR_NO_DOMAIN, 10, message, 2,
|
||||
"domain", "detail", None, 17, 23,
|
||||
)
|
||||
|
||||
def get_error_code(self):
|
||||
return self.err[0]
|
||||
|
||||
def get_error_domain(self):
|
||||
return self.err[1]
|
||||
|
||||
def get_error_message(self):
|
||||
return self.err[2]
|
||||
|
||||
def get_error_level(self):
|
||||
return self.err[3]
|
||||
|
||||
def get_str1(self):
|
||||
return self.err[4]
|
||||
|
||||
def get_str2(self):
|
||||
return self.err[5]
|
||||
|
||||
def get_str3(self):
|
||||
return self.err[6]
|
||||
|
||||
def get_int1(self):
|
||||
return self.err[7]
|
||||
|
||||
def get_int2(self):
|
||||
return self.err[8]
|
||||
|
||||
|
||||
class virDomain:
|
||||
def __init__(self, connection, uuid="00000000-0000-4000-8000-000000000001"):
|
||||
self.connection = connection
|
||||
self.uuid = uuid
|
||||
|
||||
def UUIDString(self):
|
||||
return self.uuid
|
||||
|
||||
def name(self):
|
||||
return "rpc-guest"
|
||||
|
||||
def info(self):
|
||||
return [1, 2097152, 1048576, 2, 12345]
|
||||
|
||||
def state(self, flags=0):
|
||||
return (1, 0)
|
||||
|
||||
def isActive(self):
|
||||
return 1
|
||||
|
||||
def XMLDesc(self, flags=0):
|
||||
return f"<domain><name>rpc-guest</name><uuid>{self.uuid}</uuid></domain>"
|
||||
|
||||
def snapshotLookupByName(self, name, flags=0):
|
||||
return virDomainSnapshot(self, name)
|
||||
|
||||
def destroy(self):
|
||||
if self.connection.uri == "test:///write-hangs":
|
||||
self.connection.hang()
|
||||
return 0
|
||||
|
||||
|
||||
class virDomainSnapshot:
|
||||
def __init__(self, domain, name):
|
||||
self.domain = domain
|
||||
self.snapshot_name = name
|
||||
|
||||
def getName(self):
|
||||
return self.snapshot_name
|
||||
|
||||
def getDomain(self):
|
||||
return self.domain
|
||||
|
||||
|
||||
class virStorageVol:
|
||||
def __init__(self, pool, name="disk.qcow2"):
|
||||
self.pool = pool
|
||||
self.volume_name = name
|
||||
self.data = b"initial-volume-bytes\x00\xff"
|
||||
|
||||
def name(self):
|
||||
return self.volume_name
|
||||
|
||||
def info(self):
|
||||
return [0, 4096, len(self.data)]
|
||||
|
||||
def upload(self, stream, offset, length, flags=0):
|
||||
stream.volume = self
|
||||
stream.writing = True
|
||||
stream.offset = offset
|
||||
stream.expected = length
|
||||
stream.data = b""
|
||||
return 0
|
||||
|
||||
def download(self, stream, offset, length, flags=0):
|
||||
stream.volume = self
|
||||
stream.writing = False
|
||||
stream.offset = 0
|
||||
stream.data = self.data[offset:offset + length]
|
||||
return 0
|
||||
|
||||
|
||||
class virStoragePool:
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
self.volume = virStorageVol(self)
|
||||
|
||||
def name(self):
|
||||
return "images"
|
||||
|
||||
def info(self):
|
||||
return [2, 1000000, 4096, 995904]
|
||||
|
||||
def listAllVolumes(self, flags=0):
|
||||
return [self.volume]
|
||||
|
||||
def storageVolLookupByName(self, name):
|
||||
if name != self.volume.name():
|
||||
raise libvirtError("storage volume is absent", (
|
||||
VIR_ERR_NO_STORAGE_VOL, 18, "storage volume is absent", 2,
|
||||
name, None, None, 0, 0,
|
||||
))
|
||||
return self.volume
|
||||
|
||||
|
||||
class virStream:
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
self.volume = None
|
||||
self.writing = False
|
||||
self.offset = 0
|
||||
self.data = b""
|
||||
|
||||
def send(self, data):
|
||||
# A short write catches callers that incorrectly assume send-all.
|
||||
count = min(len(data), 3)
|
||||
self.data += data[:count]
|
||||
return count
|
||||
|
||||
def recv(self, length):
|
||||
chunk = self.data[self.offset:self.offset + min(length, 4)]
|
||||
self.offset += len(chunk)
|
||||
return chunk
|
||||
|
||||
def finish(self):
|
||||
if self.writing:
|
||||
self.volume.data = self.data
|
||||
return 0
|
||||
|
||||
def abort(self):
|
||||
return 0
|
||||
|
||||
|
||||
class virNWFilter:
|
||||
def name(self):
|
||||
return "clean-traffic"
|
||||
|
||||
def XMLDesc(self, flags=0):
|
||||
return "<filter name='clean-traffic'/>"
|
||||
|
||||
|
||||
class virConnect:
|
||||
def __init__(self, uri):
|
||||
self.uri = uri
|
||||
self.domain = virDomain(self)
|
||||
self.pool = virStoragePool(self)
|
||||
|
||||
def getURI(self):
|
||||
return self.uri
|
||||
|
||||
def getHostname(self):
|
||||
if self.uri == "test:///read-hangs":
|
||||
self.hang()
|
||||
return "rpc-test-host"
|
||||
|
||||
def getInfo(self):
|
||||
return ["x86_64", 8192, 4, 3000, 1, 1, 4, 1]
|
||||
|
||||
def getVersion(self):
|
||||
return 10000000
|
||||
|
||||
def lookupByUUIDString(self, uuid):
|
||||
if uuid != self.domain.UUIDString():
|
||||
raise libvirtError("domain is absent")
|
||||
return self.domain
|
||||
|
||||
def listAllDomains(self, flags=0):
|
||||
return [self.domain]
|
||||
|
||||
def getAllDomainStats(self, stats=0, flags=0):
|
||||
return [(self.domain, {"cpu.time": 12345, "nested": [None, True, 1.25]})]
|
||||
|
||||
def domainListGetStats(self, domains, stats=0, flags=0):
|
||||
return [(domain, {"cpu.time": 12345}) for domain in domains]
|
||||
|
||||
def storagePoolLookupByName(self, name):
|
||||
assert name == "images"
|
||||
return self.pool
|
||||
|
||||
def newStream(self, flags=0):
|
||||
return virStream(self)
|
||||
|
||||
def nwfilterLookupByName(self, name):
|
||||
assert name == "clean-traffic"
|
||||
return virNWFilter()
|
||||
|
||||
def echo(self, value):
|
||||
return value
|
||||
|
||||
def process_id(self):
|
||||
return os.getpid()
|
||||
|
||||
def sleep(self, seconds):
|
||||
time.sleep(seconds)
|
||||
return "completed"
|
||||
|
||||
def hang(self, marker=None, ignore_term=False):
|
||||
if ignore_term:
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
if marker is not None:
|
||||
Path(marker).write_text(str(os.getpid()), encoding="ascii")
|
||||
while True:
|
||||
time.sleep(60)
|
||||
|
||||
def crash(self, code=17):
|
||||
os._exit(code)
|
||||
|
||||
def start_stubborn_child(self, heartbeat):
|
||||
# Model an SSH descendant which ignores TERM after its RPC parent exits.
|
||||
program = (
|
||||
"import os,signal,sys,time; from pathlib import Path; "
|
||||
"signal.signal(signal.SIGTERM, signal.SIG_IGN); "
|
||||
"p=Path(sys.argv[1]); "
|
||||
"exec('while True:\\n p.write_text(str(time.monotonic_ns()))\\n time.sleep(0.01)')"
|
||||
)
|
||||
child = subprocess.Popen(
|
||||
[sys.executable, "-c", program, heartbeat],
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
deadline = time.monotonic() + 2.0
|
||||
while not Path(heartbeat).exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
if not Path(heartbeat).exists():
|
||||
child.kill()
|
||||
child.wait(timeout=2.0)
|
||||
raise RuntimeError("test descendant did not start")
|
||||
return child.pid
|
||||
|
||||
def ordinary_error(self):
|
||||
raise ValueError("backend rejected input")
|
||||
|
||||
def close(self):
|
||||
if self.uri == "test:///close-hangs":
|
||||
self.hang()
|
||||
return 0
|
||||
|
||||
|
||||
def open(uri):
|
||||
if uri == "test:///open-hangs":
|
||||
while True:
|
||||
time.sleep(60)
|
||||
if uri == "test:///open-error":
|
||||
raise libvirtError("connection failed", (
|
||||
VIR_ERR_OPERATION_FAILED, 10, "connection failed", 2,
|
||||
None, None, None, 0, 0,
|
||||
))
|
||||
return virConnect(uri)
|
||||
|
||||
|
||||
def qemuAgentCommand(domain, command, timeout, flags):
|
||||
payload = json.loads(command)
|
||||
return json.dumps({"return": {"uuid": domain.UUIDString(),
|
||||
"execute": payload["execute"],
|
||||
"timeout": timeout, "flags": flags}})
|
||||
@ -1,124 +0,0 @@
|
||||
"""Persistent fake remote state for the real emergency-thaw RPC boundary."""
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
VIR_DOMAIN_XML_INACTIVE = 2
|
||||
VIR_DOMAIN_JOB_NONE = 0
|
||||
VIR_DOMAIN_JOB_UNBOUNDED = 2
|
||||
VIR_DOMAIN_JOB_OPERATION_BACKUP = 12
|
||||
|
||||
|
||||
class libvirtError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class virDomain:
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
|
||||
def UUIDString(self):
|
||||
return self.connection.read()['vm_id']
|
||||
|
||||
def ID(self):
|
||||
return self.connection.read().get('runtime_id', 7)
|
||||
|
||||
def isActive(self):
|
||||
return self.connection.read().get('running', True)
|
||||
|
||||
def XMLDesc(self, flags=0):
|
||||
state = self.connection.read()
|
||||
token = '' if flags else (
|
||||
'<qga-runtime xmlns="urn:virsh-panel:qga-runtime:1" '
|
||||
f'domain-uuid="{state["vm_id"]}" token="{state["token"]}"/>')
|
||||
return f'<domain><metadata>{token}</metadata></domain>'
|
||||
|
||||
def fsFreeze(self, mounts, flags):
|
||||
self.connection.update(frozen=True)
|
||||
if self.connection.read().get('freeze_hangs'):
|
||||
time.sleep(10)
|
||||
return 1
|
||||
|
||||
def backupBegin(self, xml, checkpoint, flags):
|
||||
state = self.connection.read()
|
||||
if not state.get('backup_missing'):
|
||||
self.connection.update(backup_active=True)
|
||||
if state.get('backup_hangs'):
|
||||
time.sleep(10)
|
||||
return 0
|
||||
|
||||
def fsThaw(self, mounts, flags):
|
||||
state = self.connection.read()
|
||||
self.connection.update(thaw_calls=state.get('thaw_calls', 0) + 1)
|
||||
if state.get('thaw_hangs'):
|
||||
time.sleep(10)
|
||||
self.connection.update(frozen=False)
|
||||
if state.get('thaw_invalid'):
|
||||
return -1
|
||||
if state.get('thaw_type_error'):
|
||||
raise TypeError('error after mutation')
|
||||
return 1
|
||||
|
||||
def jobStats(self, flags):
|
||||
state = self.connection.read()
|
||||
return {'type': 2 if state.get('backup_active') else 0,
|
||||
'operation': state.get('job_operation', 12)}
|
||||
|
||||
def backupGetXMLDesc(self, flags=0):
|
||||
state = self.connection.read()
|
||||
target = state.get('backup_path', state['target_path'])
|
||||
return ('<domainbackup mode="push"><disks>'
|
||||
f'<disk name="vda" backup="yes" type="file"><target file="{target}"/>'
|
||||
'</disk><disk name="vdb" backup="no"/></disks></domainbackup>')
|
||||
|
||||
def destroy(self):
|
||||
self.connection.update(destroyed=True)
|
||||
return 0
|
||||
|
||||
|
||||
class virStorageVol:
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
|
||||
def path(self):
|
||||
return self.connection.read()['target_path']
|
||||
|
||||
|
||||
class virStoragePool:
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
|
||||
def storageVolLookupByName(self, name):
|
||||
if name != self.connection.read()['volume_name']:
|
||||
raise libvirtError('wrong volume')
|
||||
return virStorageVol(self.connection)
|
||||
|
||||
|
||||
class virConnect:
|
||||
def __init__(self, uri):
|
||||
self.path = Path(uri)
|
||||
|
||||
def read(self):
|
||||
return json.loads(self.path.read_text())
|
||||
|
||||
def update(self, **values):
|
||||
state = self.read()
|
||||
state.update(values)
|
||||
self.path.write_text(json.dumps(state))
|
||||
|
||||
def lookupByUUIDString(self, vm_id):
|
||||
if vm_id != self.read()['vm_id']:
|
||||
raise libvirtError('wrong domain')
|
||||
return virDomain(self)
|
||||
|
||||
def storagePoolLookupByName(self, name):
|
||||
if name != 'images':
|
||||
raise libvirtError('wrong pool')
|
||||
return virStoragePool(self)
|
||||
|
||||
def close(self):
|
||||
return 0
|
||||
|
||||
|
||||
def open(uri):
|
||||
return virConnect(uri)
|
||||
@ -1,36 +0,0 @@
|
||||
{
|
||||
"document": {
|
||||
"schema_version": 1,
|
||||
"vpc_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"region": "test-region",
|
||||
"fabric_id": "test-fabric",
|
||||
"fabric_revision": 3,
|
||||
"cidr": "10.20.0.0/24",
|
||||
"desired_revision": 10,
|
||||
"operation_id": "vpc-delete-digest-fixture-op",
|
||||
"subnets": [],
|
||||
"hosts": [
|
||||
{
|
||||
"host_id": "11",
|
||||
"members": []
|
||||
},
|
||||
{
|
||||
"host_id": "2",
|
||||
"members": []
|
||||
}
|
||||
],
|
||||
"members": [],
|
||||
"routes": [],
|
||||
"acls": [],
|
||||
"retiring_members": [
|
||||
{
|
||||
"member_id": "11111111111111111111111111111111",
|
||||
"instance_uuid": "11111111-2222-4333-8444-555555555555",
|
||||
"host_id": "2",
|
||||
"private_ip_cidr": "10.20.0.10/32",
|
||||
"private_mac": "12:e8:3c:ae:d0:18"
|
||||
}
|
||||
]
|
||||
},
|
||||
"expected_digest": "2f83e2fe639c82702a21c200461cd4c59a57030ec7ba9603e949c5175de73314"
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
"""An unstarted local admission failure must not leave durable uncertainty."""
|
||||
|
||||
import asyncio
|
||||
from contextlib import contextmanager
|
||||
|
||||
from test_engine_coordination import local_backends, prepare_hosts, request, running_engine # noqa: F401
|
||||
from virsh_panel.coordination import current_execution
|
||||
from virsh_panel.resource_locks import ResourceLockRegistry, ResourceLockTimeout
|
||||
|
||||
|
||||
def test_rescue_template_local_lock_timeout_releases_durable_claim(tmp_path, monkeypatch):
|
||||
original_hold = ResourceLockRegistry.hold
|
||||
observed = []
|
||||
|
||||
@contextmanager
|
||||
def injected_timeout(self, resources, *, timeout=5.0):
|
||||
keys = tuple(resources)
|
||||
if any(key.startswith("rescue-template:") for key in keys):
|
||||
execution = current_execution()
|
||||
assert execution is not None
|
||||
assert set(keys).issubset(execution.claim.resources)
|
||||
row = execution.coordinator.snapshot()[execution.claim.token]
|
||||
assert row["state"] == "active"
|
||||
observed.append(execution.claim.token)
|
||||
raise ResourceLockTimeout("injected before template work")
|
||||
with original_hold(self, keys, timeout=timeout):
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(ResourceLockRegistry, "hold", injected_timeout)
|
||||
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, _store, services):
|
||||
await prepare_hosts(engine, "host-a")
|
||||
services["host-a"].calls.clear()
|
||||
baseline = engine.coordinator.snapshot()
|
||||
status, body = await request(
|
||||
engine, "DELETE",
|
||||
"/internal/v1/hosts/host-a/rescue-templates/" + "a" * 32,
|
||||
operation_id="template-local-admission-timeout",
|
||||
)
|
||||
assert status == 409, body
|
||||
assert observed, "the failure must occur inside a durable resource claim"
|
||||
assert engine.coordinator.snapshot() == baseline
|
||||
assert services["host-a"].calls == []
|
||||
assert engine.coordinator.operation_claim("template-local-admission-timeout") is None
|
||||
|
||||
asyncio.run(run())
|
||||
@ -1,114 +0,0 @@
|
||||
"""Lost isolated execution must retain job evidence and mutation guards."""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from test_app import (
|
||||
VM_ID, configure_host, configure_reinstall_case, configure_snapshot_case,
|
||||
setup_admin, wait_for_reinstall_job, wait_for_snapshot_state, web,
|
||||
)
|
||||
from virsh_panel.app import reinstall_post_commit
|
||||
from virsh_panel.libvirt_rpc import EXECUTION_ERROR_CODES
|
||||
from virsh_panel.libvirt_service import VMError
|
||||
|
||||
|
||||
def test_shutdown_rejects_new_http_requests(web):
|
||||
client, *_ = web
|
||||
client.app.state.begin_shutdown()
|
||||
response = client.get("/api/v1/health")
|
||||
assert response.status_code == 503
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_code", sorted(EXECUTION_ERROR_CODES))
|
||||
def test_reinstall_interruption_preserves_password_and_guard(web, error_code):
|
||||
client, _state, _token, service, _manager, _ = web
|
||||
csrf, template_id, _volume = configure_reinstall_case(web)
|
||||
|
||||
def interrupted(*args, **kwargs):
|
||||
raise VMError("isolated execution interrupted", 503, error_code=error_code)
|
||||
|
||||
service.reinstall_cloud_vm = interrupted
|
||||
body = {"template_id": template_id, "confirmation_name": "cloud-vm"}
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/reinstall", json=body, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_for_reinstall_job(client, response.json()["job_id"], csrf)
|
||||
assert result["status"] == "uncertain"
|
||||
assert result["error_code"] == error_code
|
||||
assert result["generated_password"]
|
||||
assert not reinstall_post_commit(result)
|
||||
assert "finished_at" not in result
|
||||
duplicate = client.post(f"/api/v1/vms/{VM_ID}/reinstall", json=body, headers=headers)
|
||||
assert duplicate.status_code == 409
|
||||
power = client.post(f"/api/v1/vms/{VM_ID}/actions",
|
||||
json={"action": "poweroff"}, headers=headers)
|
||||
assert power.status_code == 409
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_code", sorted(EXECUTION_ERROR_CODES))
|
||||
def test_snapshot_interruption_preserves_catalog_and_guard(web, error_code):
|
||||
client, service, csrf = configure_snapshot_case(web)
|
||||
service.snapshot_create_error = VMError(
|
||||
"isolated execution interrupted", 503, error_code=error_code)
|
||||
key = "e" * 32
|
||||
response = client.post(
|
||||
f"/api/v1/vms/{VM_ID}/snapshots",
|
||||
json={"snapshot_key": key, "name": "snapshot", "consistency_mode": "crash"},
|
||||
headers={"X-CSRF-Token": csrf},
|
||||
)
|
||||
assert response.status_code == 202, response.text
|
||||
assert wait_for_snapshot_state(client, key, "uncertain")["state"] == "uncertain"
|
||||
saved = client.app.state.snapshot_job_state(response.json()["job_id"])
|
||||
assert saved["status"] == "uncertain"
|
||||
assert saved["error_code"] == error_code
|
||||
deleted = client.delete(f"/api/v1/vms/{VM_ID}", headers={"X-CSRF-Token": csrf})
|
||||
assert deleted.status_code == 409
|
||||
assert not any(call[0] == "delete_vm" for call in service.calls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_code", sorted(EXECUTION_ERROR_CODES))
|
||||
def test_rescue_interruption_preserves_original_xml_and_guard(web, error_code):
|
||||
client, _state, token, service, _manager, _ = web
|
||||
csrf = setup_admin(client, token)
|
||||
configure_host(client, csrf)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
iso = bytearray(1024 * 1024)
|
||||
for sector, kind in ((16, 1), (17, 0), (18, 255)):
|
||||
offset = sector * 2048
|
||||
iso[offset] = kind
|
||||
iso[offset + 1:offset + 6] = b"CD001"
|
||||
iso[offset + 6] = 1
|
||||
iso[17 * 2048 + 7:17 * 2048 + 30] = b"EL TORITO SPECIFICATION"
|
||||
iso[17 * 2048 + 71:17 * 2048 + 75] = (20).to_bytes(4, "little")
|
||||
response = client.post("/api/v1/rescue-templates?name=Rescue",
|
||||
content=bytes(iso), headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
for _ in range(100):
|
||||
imported = client.get(
|
||||
f"/api/v1/rescue-templates/import-jobs/{response.json()['job_id']}").json()
|
||||
if imported["status"] not in {"queued", "running"}:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert imported["status"] == "completed"
|
||||
template = client.get("/api/v1/rescue-templates").json()["templates"][0]
|
||||
service.rescue_enter_error = VMError(
|
||||
"isolated execution interrupted", 503, error_code=error_code)
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/rescue", json={
|
||||
"template_id": template["id"], "confirmation_name": "alma-test",
|
||||
}, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
for _ in range(100):
|
||||
result = client.get(f"/api/v1/rescue-jobs/{response.json()['job_id']}").json()
|
||||
if result["status"] not in {"queued", "running"}:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert result["status"] == "uncertain"
|
||||
assert result["error_code"] == error_code
|
||||
assert client.get(f"/api/v1/vms/{VM_ID}/rescue").json()["status"] == "uncertain"
|
||||
# The exception path must not run the ordinary failure rollback, which
|
||||
# compares prepare_rescue_vm XML and can delete the durable original XML.
|
||||
assert sum(call[0] == "prepare_rescue_vm" for call in service.calls) == 1
|
||||
power = client.post(f"/api/v1/vms/{VM_ID}/actions",
|
||||
json={"action": "poweroff"}, headers=headers)
|
||||
assert power.status_code == 409
|
||||
@ -1,182 +0,0 @@
|
||||
"""Standalone admission stays per VM when no Engine coordinator is installed."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from test_app import CREATE_BODY, HOST_BODY, VM_ID, configure_host, setup_admin, web # noqa: F401
|
||||
|
||||
|
||||
OTHER_VM_ID = "00000000-0000-4000-8000-000000000002"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("second_vm", [VM_ID, OTHER_VM_ID])
|
||||
def test_stalled_action_serializes_only_the_same_vm(web, second_vm):
|
||||
client, _, token, service, _, _ = web
|
||||
csrf = setup_admin(client, token)
|
||||
configure_host(client, csrf)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
second_entered = threading.Event()
|
||||
calls = []
|
||||
service.get_vm = lambda vm_id: {**service.vm, "uuid": vm_id}
|
||||
|
||||
def action(vm_id, _action):
|
||||
calls.append(vm_id)
|
||||
if len(calls) == 1:
|
||||
started.set()
|
||||
assert release.wait(5)
|
||||
else:
|
||||
second_entered.set()
|
||||
return {**service.vm, "uuid": vm_id}
|
||||
|
||||
service.action = action
|
||||
|
||||
def send(vm_id):
|
||||
return client.post(f"/api/v1/vms/{vm_id}/actions",
|
||||
json={"action": "reset"}, headers=headers)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
first = executor.submit(send, VM_ID)
|
||||
try:
|
||||
assert started.wait(2)
|
||||
second = executor.submit(send, second_vm)
|
||||
if second_vm == VM_ID:
|
||||
assert not second_entered.wait(0.15)
|
||||
assert not second.done()
|
||||
else:
|
||||
result = second.result(timeout=0.75)
|
||||
assert result.status_code == 200, result.text
|
||||
assert result.json()["uuid"] == OTHER_VM_ID
|
||||
finally:
|
||||
release.set()
|
||||
assert first.result(timeout=2).status_code == 200
|
||||
assert second.result(timeout=2).status_code == 200
|
||||
assert calls == [VM_ID, second_vm]
|
||||
|
||||
|
||||
def test_stalled_request_blocks_legacy_host_replacement_and_update(web):
|
||||
client, _, token, service, manager, _ = web
|
||||
csrf = setup_admin(client, token)
|
||||
configure_host(client, csrf)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
manager.calls.clear()
|
||||
|
||||
def action(_vm_id, _action):
|
||||
started.set()
|
||||
assert release.wait(5)
|
||||
return service.vm
|
||||
|
||||
service.action = action
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
active = executor.submit(
|
||||
client.post, f"/api/v1/vms/{VM_ID}/actions",
|
||||
json={"action": "reset"}, headers=headers)
|
||||
try:
|
||||
assert started.wait(2)
|
||||
host = executor.submit(client.put, "/api/v1/host",
|
||||
json=HOST_BODY, headers=headers).result(timeout=0.75)
|
||||
assert host.status_code == 409, host.text
|
||||
update = executor.submit(
|
||||
client.post, "/api/v1/update/apply",
|
||||
json={"expected_current_commit": "a" * 40,
|
||||
"expected_remote_commit": "b" * 40},
|
||||
headers=headers).result(timeout=0.75)
|
||||
assert update.status_code == 409, update.text
|
||||
assert not manager.calls
|
||||
finally:
|
||||
release.set()
|
||||
assert active.result(timeout=2).status_code == 200
|
||||
# The active counter is diagnostic admission, not a permanent lock.
|
||||
assert client.put("/api/v1/host", json=HOST_BODY, headers=headers).status_code == 200
|
||||
|
||||
|
||||
def test_stalled_create_does_not_block_an_existing_vm(web):
|
||||
client, _, token, service, _, _ = web
|
||||
csrf = setup_admin(client, token)
|
||||
configure_host(client, csrf)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def create(_spec):
|
||||
started.set()
|
||||
assert release.wait(5)
|
||||
return {**service.vm, "uuid": OTHER_VM_ID}
|
||||
|
||||
service.create_vm = create
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
creating = executor.submit(client.post, "/api/v1/vms",
|
||||
json=CREATE_BODY, headers=headers)
|
||||
try:
|
||||
assert started.wait(2)
|
||||
action = executor.submit(
|
||||
client.post, f"/api/v1/vms/{VM_ID}/actions",
|
||||
json={"action": "reset"}, headers=headers).result(timeout=0.75)
|
||||
assert action.status_code == 200, action.text
|
||||
assert action.json()["uuid"] == VM_ID
|
||||
finally:
|
||||
release.set()
|
||||
assert creating.result(timeout=2).status_code == 201
|
||||
|
||||
|
||||
def test_template_delete_blocks_rescue_admission_but_not_another_vm(web):
|
||||
client, _, token, service, _, _ = web
|
||||
csrf = setup_admin(client, token)
|
||||
configure_host(client, csrf)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
iso = bytearray(1024 * 1024)
|
||||
for sector, kind in ((16, 1), (17, 0), (18, 255)):
|
||||
offset = sector * 2048
|
||||
iso[offset] = kind
|
||||
iso[offset + 1:offset + 6] = b"CD001"
|
||||
iso[offset + 6] = 1
|
||||
iso[17 * 2048 + 7:17 * 2048 + 30] = b"EL TORITO SPECIFICATION"
|
||||
iso[17 * 2048 + 71:17 * 2048 + 75] = (20).to_bytes(4, "little")
|
||||
uploaded = client.post("/api/v1/rescue-templates?name=Rescue",
|
||||
content=bytes(iso), headers=headers)
|
||||
assert uploaded.status_code == 202, uploaded.text
|
||||
for _ in range(100):
|
||||
job = client.get(
|
||||
f"/api/v1/rescue-templates/import-jobs/{uploaded.json()['job_id']}"
|
||||
).json()
|
||||
if job["status"] in {"completed", "failed"}:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert job["status"] == "completed"
|
||||
template = client.get("/api/v1/rescue-templates").json()["templates"][0]
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def delete(_volume):
|
||||
started.set()
|
||||
assert release.wait(5)
|
||||
|
||||
service.delete_rescue_iso = delete
|
||||
service.get_vm = lambda vm_id: {**service.vm, "uuid": vm_id}
|
||||
service.action = lambda vm_id, _action: {**service.vm, "uuid": vm_id}
|
||||
with ThreadPoolExecutor(max_workers=3) as executor:
|
||||
deleting = executor.submit(client.delete,
|
||||
f"/api/v1/rescue-templates/{template['id']}", headers=headers)
|
||||
try:
|
||||
assert started.wait(2)
|
||||
entering = executor.submit(
|
||||
client.post, f"/api/v1/vms/{VM_ID}/rescue",
|
||||
json={"template_id": template["id"], "confirmation_name": service.vm["name"]},
|
||||
headers=headers)
|
||||
action = executor.submit(
|
||||
client.post, f"/api/v1/vms/{OTHER_VM_ID}/actions",
|
||||
json={"action": "reset"}, headers=headers).result(timeout=0.75)
|
||||
assert action.status_code == 200, action.text
|
||||
assert action.json()["uuid"] == OTHER_VM_ID
|
||||
assert not entering.done()
|
||||
finally:
|
||||
release.set()
|
||||
assert deleting.result(timeout=2).status_code == 200
|
||||
assert entering.result(timeout=2).status_code == 404
|
||||
assert not any(call[0] == "enter_rescue_vm" for call in service.calls)
|
||||
@ -1,315 +0,0 @@
|
||||
"""Live prewrite receipts settle only the worker that produced them."""
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from test_app import (
|
||||
VM_ID, configure_host, configure_reinstall_case, configure_snapshot_case,
|
||||
setup_admin, web, # noqa: F401
|
||||
)
|
||||
from test_durable_jobs import MemoryCAS, jobs # noqa: F401
|
||||
from virsh_panel import app as app_module
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationError, DurableCoordinator, execution_context,
|
||||
current_execution, execution_failure_before_write, mark_admission_rejected,
|
||||
mark_execution_unknown, mark_remote_write_started,
|
||||
)
|
||||
from virsh_panel.database import DatabaseError
|
||||
from virsh_panel.durable_jobs import read_job_outcome, submit_durable
|
||||
from virsh_panel.libvirt_service import VMError
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_code", [
|
||||
None, "libvirt_execution_timeout", "libvirt_execution_stopped",
|
||||
"libvirt_connection_failed", "coordination_uncertain",
|
||||
])
|
||||
def test_live_failed_worker_receipt_releases_claim(jobs, error_code):
|
||||
store, coordinator, parent, executor = jobs
|
||||
|
||||
def worker():
|
||||
if error_code == "coordination_uncertain":
|
||||
mark_admission_rejected()
|
||||
assert execution_failure_before_write(error_code)
|
||||
store.set("snapshot_job", "job-one", json.dumps({
|
||||
"status": "failed", "error_code": error_code,
|
||||
"mutation_started": False, "execution_outcome": "not_dispatched",
|
||||
}))
|
||||
|
||||
with execution_context(coordinator, parent, tracked_mutations=True):
|
||||
future = submit_durable(
|
||||
executor, worker,
|
||||
outcome=lambda: read_job_outcome(store, "snapshot_job", "job-one"),
|
||||
job_kind="snapshot_job", job_id="job-one")
|
||||
future.result(timeout=2)
|
||||
assert list(coordinator.snapshot()) == [parent.token]
|
||||
coordinator.release(parent)
|
||||
replacement = coordinator.claim(resources=parent.resources)
|
||||
assert coordinator.release(replacement)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("unsafe", ["untracked", "prior_write", "unknown", "old_record", "database"])
|
||||
def test_stored_prewrite_flag_does_not_override_missing_or_contrary_evidence(jobs, unsafe):
|
||||
store, coordinator, parent, executor = jobs
|
||||
record = {
|
||||
"status": "failed", "error_code": "libvirt_execution_timeout",
|
||||
"mutation_started": False, "execution_outcome": "not_dispatched",
|
||||
}
|
||||
if unsafe == "unknown":
|
||||
record["error_code"] = "libvirt_execution_uncertain"
|
||||
if unsafe == "old_record":
|
||||
record.pop("execution_outcome")
|
||||
|
||||
def worker():
|
||||
if unsafe == "prior_write":
|
||||
mark_remote_write_started()
|
||||
if unsafe == "database":
|
||||
mark_execution_unknown()
|
||||
store.set("snapshot_job", "job-one", json.dumps(record))
|
||||
|
||||
with execution_context(coordinator, parent, tracked_mutations=unsafe != "untracked"):
|
||||
future = submit_durable(
|
||||
executor, worker,
|
||||
outcome=lambda: read_job_outcome(store, "snapshot_job", "job-one"),
|
||||
job_kind="snapshot_job", job_id="job-one")
|
||||
future.result(timeout=2)
|
||||
child = next(row for row in coordinator.snapshot().values() if row["kind"] == "background")
|
||||
assert child["state"] == "uncertain"
|
||||
|
||||
|
||||
def test_tracked_unwritten_queued_and_running_worker_still_blocks_competitors(jobs):
|
||||
store, coordinator, parent, executor = jobs
|
||||
occupied, worker_started = threading.Event(), threading.Event()
|
||||
release_queue, release_worker = threading.Event(), threading.Event()
|
||||
|
||||
def occupy():
|
||||
occupied.set()
|
||||
assert release_queue.wait(2)
|
||||
|
||||
def worker():
|
||||
worker_started.set()
|
||||
assert execution_failure_before_write("libvirt_execution_timeout")
|
||||
assert release_worker.wait(2)
|
||||
store.set("snapshot_job", "job-one", json.dumps({"status": "completed"}))
|
||||
|
||||
blocker = executor.submit(occupy)
|
||||
assert occupied.wait(2)
|
||||
try:
|
||||
with execution_context(coordinator, parent, tracked_mutations=True):
|
||||
future = submit_durable(
|
||||
executor, worker,
|
||||
outcome=lambda: read_job_outcome(store, "snapshot_job", "job-one"),
|
||||
job_kind="snapshot_job", job_id="job-one")
|
||||
coordinator.release(parent)
|
||||
with pytest.raises(CoordinationError):
|
||||
coordinator.claim(resources=parent.resources)
|
||||
release_queue.set()
|
||||
blocker.result(timeout=2)
|
||||
assert worker_started.wait(2)
|
||||
with pytest.raises(CoordinationError):
|
||||
coordinator.claim(resources=parent.resources)
|
||||
finally:
|
||||
release_queue.set()
|
||||
release_worker.set()
|
||||
future.result(timeout=2)
|
||||
assert coordinator.snapshot() == {}
|
||||
|
||||
|
||||
def instrument_background(monkeypatch, *, write=False):
|
||||
"""Supply the live RPC evidence context to the app's fake host worker."""
|
||||
coordinator = DurableCoordinator(MemoryCAS())
|
||||
parent = coordinator.claim(resources=("vm:host-a:" + VM_ID,))
|
||||
original = app_module.submit_durable
|
||||
seen = {}
|
||||
|
||||
def submit(executor, fn, *args, **kwargs):
|
||||
kind = kwargs["job_kind"]
|
||||
seen[kind] = inspect.getclosurevars(fn).nonlocals
|
||||
|
||||
def run(*worker_args, **worker_kwargs):
|
||||
with execution_context(coordinator, parent, tracked_mutations=True):
|
||||
seen["execution"] = current_execution()
|
||||
if write:
|
||||
mark_remote_write_started()
|
||||
return fn(*worker_args, **worker_kwargs)
|
||||
|
||||
return original(executor, run, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(app_module, "submit_durable", submit)
|
||||
return seen
|
||||
|
||||
|
||||
def wait_job(seen, kind, job_id):
|
||||
jobs_by_kind = {
|
||||
"reinstall_job": "reinstall_jobs", "rescue_job": "rescue_jobs",
|
||||
"snapshot_job": "snapshot_jobs",
|
||||
}
|
||||
for _ in range(200):
|
||||
saved = dict(seen[kind][jobs_by_kind[kind]][job_id])
|
||||
if saved["status"] not in {"queued", "running"}:
|
||||
return saved
|
||||
time.sleep(0.01)
|
||||
pytest.fail("background worker did not settle")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("write", [False, True])
|
||||
def test_reinstall_preflight_timeout_does_not_create_new_pending_guard(web, monkeypatch, write):
|
||||
client, _state, _token, service, _manager, _ = web
|
||||
csrf, template_id, _ = configure_reinstall_case(web)
|
||||
seen = instrument_background(monkeypatch, write=write)
|
||||
|
||||
def fail(*_args, **_kwargs):
|
||||
raise VMError("preflight read timed out", 503, error_code="libvirt_execution_timeout")
|
||||
|
||||
service.reinstall_cloud_vm = fail
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/reinstall", json={
|
||||
"template_id": template_id, "confirmation_name": "cloud-vm",
|
||||
}, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, "reinstall_job", response.json()["job_id"])
|
||||
assert result["status"] == ("uncertain" if write else "failed")
|
||||
assert (result.get("execution_outcome") == "not_dispatched") is (not write)
|
||||
power = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=headers)
|
||||
assert power.status_code == (409 if write else 200), power.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("write", [False, True])
|
||||
def test_snapshot_preflight_timeout_rolls_back_catalog_before_releasing_guard(web, monkeypatch, write):
|
||||
client, service, csrf = configure_snapshot_case(web)
|
||||
seen = instrument_background(monkeypatch, write=write)
|
||||
service.snapshot_create_error = VMError(
|
||||
"preflight read timed out", 503, error_code="libvirt_execution_timeout")
|
||||
key = "d" * 32
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/snapshots", json={
|
||||
"snapshot_key": key, "name": "preflight", "consistency_mode": "crash",
|
||||
}, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, "snapshot_job", response.json()["job_id"])
|
||||
assert result["status"] == ("uncertain" if write else "failed")
|
||||
assert (result.get("execution_outcome") == "not_dispatched") is (not write)
|
||||
snapshot = client.get(f"/api/v1/vms/{VM_ID}/snapshots/{key}")
|
||||
assert snapshot.status_code == (200 if write else 404), snapshot.text
|
||||
power = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=headers)
|
||||
assert power.status_code == (409 if write else 200), power.text
|
||||
|
||||
|
||||
def configure_rescue(web):
|
||||
client, _state, token, service, _manager, _ = web
|
||||
csrf = setup_admin(client, token)
|
||||
configure_host(client, csrf)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
iso = bytearray(1024 * 1024)
|
||||
for sector, kind in ((16, 1), (17, 0), (18, 255)):
|
||||
offset = sector * 2048
|
||||
iso[offset] = kind
|
||||
iso[offset + 1:offset + 6] = b"CD001"
|
||||
iso[offset + 6] = 1
|
||||
iso[17 * 2048 + 7:17 * 2048 + 30] = b"EL TORITO SPECIFICATION"
|
||||
iso[17 * 2048 + 71:17 * 2048 + 75] = (20).to_bytes(4, "little")
|
||||
response = client.post("/api/v1/rescue-templates?name=Rescue", content=bytes(iso), headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
for _ in range(200):
|
||||
imported = client.get(
|
||||
f"/api/v1/rescue-templates/import-jobs/{response.json()['job_id']}").json()
|
||||
if imported["status"] not in {"queued", "running"}:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert imported["status"] == "completed"
|
||||
template = client.get("/api/v1/rescue-templates").json()["templates"][0]
|
||||
return client, service, headers, template
|
||||
|
||||
|
||||
@pytest.mark.parametrize("write", [False, True])
|
||||
def test_rescue_preflight_timeout_clears_only_unstarted_entry(web, monkeypatch, write):
|
||||
client, service, headers, template = configure_rescue(web)
|
||||
seen = instrument_background(monkeypatch, write=write)
|
||||
service.rescue_enter_error = VMError(
|
||||
"preflight read timed out", 503, error_code="libvirt_execution_timeout")
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/rescue", json={
|
||||
"template_id": template["id"], "confirmation_name": "alma-test",
|
||||
}, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, "rescue_job", response.json()["job_id"])
|
||||
assert result["status"] == ("uncertain" if write else "failed")
|
||||
assert (result.get("execution_outcome") == "not_dispatched") is (not write)
|
||||
assert sum(call[0] == "prepare_rescue_vm" for call in service.calls) == 1
|
||||
power = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=headers)
|
||||
assert power.status_code == (409 if write else 200), power.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["reinstall", "snapshot", "rescue"])
|
||||
def test_database_failure_never_produces_worker_no_dispatch_receipt(web, monkeypatch, kind):
|
||||
error = DatabaseError("lost database commit response")
|
||||
if kind == "reinstall":
|
||||
client, _state, _token, service, _manager, _ = web
|
||||
csrf, template_id, _ = configure_reinstall_case(web)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
body = {"template_id": template_id, "confirmation_name": "cloud-vm"}
|
||||
resource = "reinstall"
|
||||
|
||||
def fail(*_args, **_kwargs):
|
||||
raise error
|
||||
|
||||
service.reinstall_cloud_vm = fail
|
||||
elif kind == "snapshot":
|
||||
client, service, csrf = configure_snapshot_case(web)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
body = {"snapshot_key": "a" * 32, "name": "test", "consistency_mode": "crash"}
|
||||
resource = "snapshots"
|
||||
service.snapshot_create_error = error
|
||||
else:
|
||||
client, service, headers, template = configure_rescue(web)
|
||||
body = {"template_id": template["id"], "confirmation_name": "alma-test"}
|
||||
resource = "rescue"
|
||||
service.rescue_enter_error = error
|
||||
seen = instrument_background(monkeypatch)
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/{resource}", json=body, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, kind + "_job", response.json()["job_id"])
|
||||
assert result.get("execution_outcome") != "not_dispatched"
|
||||
assert result.get("mutation_started") is not False
|
||||
assert seen["execution"].execution_unknown.is_set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["force", "exit"])
|
||||
def test_unstarted_rescue_transition_restores_previous_recovery_row(web, monkeypatch, action):
|
||||
client, service, headers, template = configure_rescue(web)
|
||||
service.rescue_pending_manual_restart = action == "force"
|
||||
entered = client.post(f"/api/v1/vms/{VM_ID}/rescue", json={
|
||||
"template_id": template["id"], "confirmation_name": "alma-test",
|
||||
}, headers=headers)
|
||||
assert entered.status_code == 202, entered.text
|
||||
for _ in range(200):
|
||||
previous = client.get(f"/api/v1/vms/{VM_ID}/rescue").json()
|
||||
if previous["status"] in {"active", "staged"}:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert previous["status"] == ("staged" if action == "force" else "active")
|
||||
seen = instrument_background(monkeypatch)
|
||||
|
||||
def fail(*_args, **_kwargs):
|
||||
raise VMError("read timeout", 503, error_code="libvirt_execution_timeout")
|
||||
|
||||
monkeypatch.setattr(service, "force_start_staged_rescue_vm" if action == "force"
|
||||
else "exit_rescue_vm", fail)
|
||||
resource = "force-start" if action == "force" else "exit"
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/rescue/{resource}", json={
|
||||
"confirmation_name": "alma-test",
|
||||
**({"ack_data_loss": True} if action == "force" else {}),
|
||||
}, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, "rescue_job", response.json()["job_id"])
|
||||
assert result["status"] == "failed"
|
||||
assert result["execution_outcome"] == "not_dispatched"
|
||||
restored = client.get(f"/api/v1/vms/{VM_ID}/rescue").json()
|
||||
assert restored["status"] == previous["status"]
|
||||
# The new failed transition must not replace the original recovery owner.
|
||||
namespace = seen["rescue_job"]
|
||||
profile = web[4].current()
|
||||
saved = namespace["load_rescue_state"](profile, VM_ID)
|
||||
assert saved["job_id"] == entered.json()["job_id"]
|
||||
@ -1,745 +0,0 @@
|
||||
"""Cloud-init public networking and NoCloud seed behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.cloud_init import (
|
||||
ROOT_PASSWORD_ALPHABET, ROOT_PASSWORD_DIGITS, ROOT_PASSWORD_LENGTH,
|
||||
ROOT_PASSWORD_LOWERCASE, ROOT_PASSWORD_SYMBOLS, ROOT_PASSWORD_UPPERCASE,
|
||||
_build_seed_iso, _enable_network_updates, build_seed_iso,
|
||||
generate_root_password,
|
||||
hash_root_password,
|
||||
read_seed_network, replace_seed_network, replace_seed_private_network,
|
||||
replace_seed_settings,
|
||||
validate_cloud_network, validate_guest_hostname, validate_private_network,
|
||||
_parse_network_config,
|
||||
SeedNetworkPolicyMigrationRequired,
|
||||
)
|
||||
|
||||
|
||||
PUBLIC_KEY = (
|
||||
"ssh-ed25519 "
|
||||
"AAAAC3NzaC1lZDI1NTE5AAAAIF7mFh4aYz3lRzNqjoFQasAQLpm7BdSULhe1iWuvj3y5 "
|
||||
"panel-test"
|
||||
)
|
||||
GUEST_PASSWORD = "A-strong-guest-password-2026"
|
||||
|
||||
|
||||
def _user_data(seed: bytes) -> dict:
|
||||
"""Read the single-line JSON cloud-config embedded in the generated ISO."""
|
||||
prefix = b"#cloud-config\n"
|
||||
start = seed.index(prefix) + len(prefix)
|
||||
return json.loads(seed[start:seed.index(b"\n", start)])
|
||||
|
||||
|
||||
def _meta_data(seed: bytes) -> dict:
|
||||
"""Read the single-line JSON meta-data embedded in the generated ISO."""
|
||||
prefix = b'{"instance-id":'
|
||||
start = seed.index(prefix)
|
||||
return json.loads(seed[start:seed.index(b"\n", start)])
|
||||
|
||||
|
||||
def _network_data(seed: bytes) -> dict:
|
||||
"""Read the single-line JSON network-config embedded in the ISO."""
|
||||
prefix = b'{"version": 2, "ethernets":'
|
||||
start = seed.index(prefix)
|
||||
return json.loads(seed[start:seed.index(b"\n", start)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("address", "gateway"),
|
||||
[
|
||||
("8.8.8.18/24", "8.8.8.1"),
|
||||
("8.8.8.18/28", "8.8.8.17"),
|
||||
("123.123.123.114/24", "123.123.123.113"),
|
||||
],
|
||||
)
|
||||
def test_public_network_accepts_24_and_28(address, gateway):
|
||||
validated = validate_cloud_network(
|
||||
address, gateway, ["1.1.1.1", "8.8.8.8"], PUBLIC_KEY,
|
||||
)
|
||||
assert validated == {
|
||||
"public_ip_cidr": address,
|
||||
"gateway": gateway,
|
||||
"dns_servers": ["1.1.1.1", "8.8.8.8"],
|
||||
"guest_ssh_public_key": PUBLIC_KEY,
|
||||
}
|
||||
|
||||
|
||||
def test_cloud_login_accepts_key_and_root_password_together():
|
||||
network = ("8.8.8.18/24", "8.8.8.1", ["1.1.1.1"])
|
||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
||||
assert password_hash.startswith("$6$")
|
||||
assert GUEST_PASSWORD not in password_hash
|
||||
assert hash_root_password(GUEST_PASSWORD) != password_hash
|
||||
|
||||
validated = validate_cloud_network(
|
||||
*network, "", root_password_hash=password_hash,
|
||||
)
|
||||
assert validated["root_password_hash"] == password_hash
|
||||
assert not validated["guest_ssh_public_key"]
|
||||
|
||||
together = validate_cloud_network(
|
||||
*network, PUBLIC_KEY, root_password_hash=password_hash,
|
||||
)
|
||||
assert together["guest_ssh_public_key"] == PUBLIC_KEY
|
||||
assert together["root_password_hash"] == password_hash
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
validate_cloud_network(*network, "", root_password_hash=None)
|
||||
with pytest.raises(ValueError):
|
||||
validate_cloud_network(*network, "", root_password_hash=GUEST_PASSWORD)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid", [
|
||||
"Short1!", "alllowercase12!", "ALLUPPERCASE12!", "NoSymbolPassword12",
|
||||
"NoDigitsLong!Aa", "A-lowercase password12!", "A-lowercase\npassword12!",
|
||||
"A-lowercase-password-1" + "x" * 110, "ValidAscii1!含中文",
|
||||
])
|
||||
def test_manual_root_password_requires_ascii_and_all_character_classes(invalid):
|
||||
with pytest.raises(ValueError, match="初始 root 密码"):
|
||||
hash_root_password(invalid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [
|
||||
"node-1", "node.example.com", "A-Node.Example",
|
||||
])
|
||||
def test_guest_hostname_is_canonical_and_bounded(value):
|
||||
assert validate_guest_hostname(value) == value.lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [
|
||||
"", ".node", "node.", "node..example", "-node", "node-",
|
||||
"node_name", "node with spaces", "节点", "a" * 64,
|
||||
])
|
||||
def test_guest_hostname_rejects_unsafe_values(value):
|
||||
with pytest.raises(ValueError):
|
||||
validate_guest_hostname(value)
|
||||
|
||||
|
||||
def test_generated_root_password_always_meets_policy():
|
||||
assert ROOT_PASSWORD_ALPHABET == (
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
"0123456789!@#$%*-_=+?"
|
||||
)
|
||||
samples = set()
|
||||
for _ in range(50):
|
||||
password = generate_root_password()
|
||||
samples.add(password)
|
||||
assert len(password) == ROOT_PASSWORD_LENGTH
|
||||
assert set(password) <= set(ROOT_PASSWORD_ALPHABET)
|
||||
assert any(char in ROOT_PASSWORD_UPPERCASE for char in password)
|
||||
assert any(char in ROOT_PASSWORD_LOWERCASE for char in password)
|
||||
assert any(char in ROOT_PASSWORD_DIGITS for char in password)
|
||||
assert any(char in ROOT_PASSWORD_SYMBOLS for char in password)
|
||||
assert len(samples) == 50
|
||||
assert len({password[:4] for password in samples}) > 1
|
||||
assert hash_root_password(password).startswith("$6$")
|
||||
|
||||
|
||||
def test_generated_root_password_rejects_whole_noncompliant_candidate(monkeypatch):
|
||||
# First candidate is 32 uppercase A characters and must be discarded.
|
||||
# The second one is a known compliant candidate with no post-processing.
|
||||
indexes = iter([0] * 32 + [0, 26, 52, 62] * 8)
|
||||
bounds = []
|
||||
|
||||
def deterministic_randbelow(bound):
|
||||
bounds.append(bound)
|
||||
return next(indexes)
|
||||
|
||||
monkeypatch.setattr("virsh_panel.cloud_init.secrets.randbelow",
|
||||
deterministic_randbelow)
|
||||
assert generate_root_password() == "Aa0!" * 8
|
||||
assert bounds == [len(ROOT_PASSWORD_ALPHABET)] * 64
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("address", "gateway"),
|
||||
[
|
||||
("192.168.1.18/24", "192.168.1.1"),
|
||||
("10.0.0.18/24", "10.0.0.1"),
|
||||
("172.16.0.18/24", "172.16.0.1"),
|
||||
("127.0.0.18/24", "127.0.0.1"),
|
||||
("169.254.0.18/24", "169.254.0.1"),
|
||||
("224.0.0.18/24", "224.0.0.1"),
|
||||
("2001:4860:4860::8888/64", "2001:4860:4860::1"),
|
||||
("8.8.8.0/24", "8.8.8.1"),
|
||||
("8.8.8.255/24", "8.8.8.1"),
|
||||
("8.8.8.18/28", "8.8.8.1"),
|
||||
("8.8.8.18", "8.8.8.1"),
|
||||
],
|
||||
)
|
||||
def test_public_network_rejects_nonpublic_or_unroutable_pairs(address, gateway):
|
||||
with pytest.raises(ValueError):
|
||||
validate_cloud_network(address, gateway, ["1.1.1.1"], PUBLIC_KEY)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dns", "key"),
|
||||
[
|
||||
(["not-an-ip"], PUBLIC_KEY),
|
||||
(["192.168.1.1"], PUBLIC_KEY),
|
||||
([], PUBLIC_KEY),
|
||||
(["1.1.1.1"], "not-an-ssh-key"),
|
||||
(["1.1.1.1"], PUBLIC_KEY + "\nruncmd: [evil]"),
|
||||
],
|
||||
)
|
||||
def test_public_network_rejects_bad_dns_and_ssh_key(dns, key):
|
||||
with pytest.raises(ValueError):
|
||||
validate_cloud_network("8.8.8.18/24", "8.8.8.1", dns, key)
|
||||
|
||||
|
||||
def test_private_network_is_an_explicit_static_role_without_a_default_route():
|
||||
assert validate_private_network("10.24.3.7/32") == {
|
||||
"private_ip_cidr": "10.24.3.7/32",
|
||||
}
|
||||
config = {
|
||||
"version": 2,
|
||||
"ethernets": {
|
||||
"public0": {
|
||||
"match": {"macaddress": "52:54:00:12:34:56"},
|
||||
"dhcp4": False,
|
||||
"dhcp6": False,
|
||||
"addresses": ["8.8.8.18/28"],
|
||||
"routes": [{"to": "0.0.0.0/0", "via": "8.8.8.17"}],
|
||||
"nameservers": {"addresses": ["1.1.1.1"]},
|
||||
},
|
||||
"private0": {
|
||||
"match": {"macaddress": "52:54:00:12:34:57"},
|
||||
"dhcp4": False,
|
||||
"dhcp6": False,
|
||||
"accept-ra": False,
|
||||
"addresses": ["10.24.3.7/32"],
|
||||
},
|
||||
},
|
||||
}
|
||||
parsed = _parse_network_config(config)
|
||||
assert parsed["roles"] == ["public0", "private0"]
|
||||
assert parsed["private_ip_cidr"] == "10.24.3.7/32"
|
||||
with pytest.raises(ValueError, match="无默认路由"):
|
||||
_parse_network_config({
|
||||
**config,
|
||||
"ethernets": {
|
||||
**config["ethernets"],
|
||||
"private0": {
|
||||
**config["ethernets"]["private0"],
|
||||
"routes": [{"to": "0.0.0.0/0", "via": "10.24.3.1"}],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [
|
||||
"10.24.3.7/31", "127.0.0.1/32", "169.254.1.1/32",
|
||||
"192.0.2.1/32", "2001:db8::1/128",
|
||||
])
|
||||
def test_private_network_rejects_non_rfc1918_guest_endpoints(value):
|
||||
with pytest.raises(ValueError):
|
||||
validate_private_network(value)
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_seed_iso_can_add_private_role_without_public_route_or_dns():
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
||||
)
|
||||
seed = build_seed_iso(
|
||||
str(uuid.uuid4()), "multi-nic-vm", "52:54:00:12:34:56", validated,
|
||||
private_mac="52:54:00:12:34:57",
|
||||
private_network={
|
||||
"private_ip_cidr": "10.24.3.7/32",
|
||||
"routes": ["10.24.0.0/16"],
|
||||
},
|
||||
)
|
||||
network = _network_data(seed)
|
||||
assert set(network["ethernets"]) == {"public0", "private0"}
|
||||
assert network["ethernets"]["private0"] == {
|
||||
"match": {"macaddress": "52:54:00:12:34:57"},
|
||||
"dhcp4": False,
|
||||
"dhcp6": False,
|
||||
"accept-ra": False,
|
||||
"addresses": ["10.24.3.7/32"],
|
||||
"routes": [{"to": "10.24.0.0/16", "scope": "link"}],
|
||||
}
|
||||
assert read_seed_network(seed) == {
|
||||
"public_ip_cidr": "8.8.8.18/28",
|
||||
"gateway": "8.8.8.17",
|
||||
"dns_servers": ["1.1.1.1"],
|
||||
"mac": "52:54:00:12:34:56",
|
||||
"private_ip_cidr": "10.24.3.7/32",
|
||||
"private_mac": "52:54:00:12:34:57",
|
||||
"private_routes": ["10.24.0.0/16"],
|
||||
"roles": ["public0", "private0"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_private_seed_migrates_legacy_routes_to_stable_vpc_prefix():
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
||||
)
|
||||
seed = build_seed_iso(
|
||||
str(uuid.uuid4()), "private-route-vm", "52:54:00:12:34:56", validated,
|
||||
private_mac="52:54:00:12:34:57",
|
||||
private_network={
|
||||
"private_ip_cidr": "10.24.3.7/32",
|
||||
"routes": ["10.24.0.0/16"],
|
||||
},
|
||||
)
|
||||
legacy_network = _network_data(seed)
|
||||
legacy_network["ethernets"]["private0"]["routes"] = [
|
||||
{"to": "10.24.3.8/32", "scope": "link"},
|
||||
{"to": "10.24.3.10/32", "scope": "link"},
|
||||
]
|
||||
seed = _build_seed_iso((
|
||||
("/USERDATA.;1", "user-data", "/user-data",
|
||||
b"#cloud-config\n" + json.dumps(
|
||||
_user_data(seed), ensure_ascii=True).encode("ascii") + b"\n"),
|
||||
("/METADATA.;1", "meta-data", "/meta-data",
|
||||
json.dumps(_meta_data(seed), ensure_ascii=True).encode("ascii") + b"\n"),
|
||||
("/NETWORK.;1", "network-config", "/network-config",
|
||||
json.dumps(legacy_network, ensure_ascii=True).encode("ascii") + b"\n"),
|
||||
))
|
||||
assert read_seed_network(seed)["private_routes"] == [
|
||||
"10.24.3.10/32", "10.24.3.8/32",
|
||||
]
|
||||
|
||||
changed = replace_seed_private_network(
|
||||
seed,
|
||||
private_mac="52:54:00:12:34:58",
|
||||
private_ip_cidr="10.24.3.9/32",
|
||||
private_routes=["10.24.0.0/16"],
|
||||
)
|
||||
changed_network = read_seed_network(changed)
|
||||
assert _meta_data(changed)["instance-id"] == _meta_data(seed)["instance-id"]
|
||||
assert _user_data(changed) == _user_data(seed)
|
||||
assert changed_network["private_ip_cidr"] == "10.24.3.9/32"
|
||||
assert changed_network["private_mac"] == "52:54:00:12:34:58"
|
||||
assert changed_network["private_routes"] == ["10.24.0.0/16"]
|
||||
unchanged = replace_seed_private_network(
|
||||
changed,
|
||||
private_mac="52:54:00:12:34:58",
|
||||
private_ip_cidr="10.24.3.9/32",
|
||||
private_routes=["10.24.0.0/16"],
|
||||
)
|
||||
assert unchanged == changed
|
||||
|
||||
removed = replace_seed_private_network(
|
||||
changed, private_mac=None, private_ip_cidr=None,
|
||||
)
|
||||
removed_network = read_seed_network(removed)
|
||||
assert _meta_data(removed)["instance-id"] == _meta_data(seed)["instance-id"]
|
||||
assert _user_data(removed) == _user_data(seed)
|
||||
assert "private_ip_cidr" not in removed_network
|
||||
assert "private_routes" not in removed_network
|
||||
|
||||
for invalid_routes in (
|
||||
None,
|
||||
[],
|
||||
["10.24.0.0/16", "10.24.3.10/32"],
|
||||
["10.24.0.0/16", "10.24.0.0/16"],
|
||||
["10.25.0.0/16"],
|
||||
["10.24.3.0/25"],
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
replace_seed_private_network(
|
||||
seed,
|
||||
private_mac="52:54:00:12:34:58",
|
||||
private_ip_cidr="10.24.3.9/32",
|
||||
private_routes=invalid_routes,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_private_seed_rejects_public_connected_prefix_overlap():
|
||||
# The public address itself is global, but this deliberately broad
|
||||
# connected prefix includes the RFC1918 VPC. private0 must not silently
|
||||
# steal that connected public route.
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/5", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
||||
)
|
||||
seed = build_seed_iso(
|
||||
str(uuid.uuid4()), "overlap-vm", "52:54:00:12:34:56", validated,
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
replace_seed_private_network(
|
||||
seed,
|
||||
private_mac="52:54:00:12:34:58",
|
||||
private_ip_cidr="10.24.3.9/32",
|
||||
private_routes=["10.24.0.0/16"],
|
||||
)
|
||||
|
||||
|
||||
def test_private_route_parser_rejects_mixed_legacy_and_stable_routes():
|
||||
config = {
|
||||
"version": 2,
|
||||
"ethernets": {
|
||||
"public0": {
|
||||
"match": {"macaddress": "52:54:00:12:34:56"},
|
||||
"dhcp4": False, "dhcp6": False,
|
||||
"addresses": ["8.8.8.18/28"],
|
||||
"routes": [{"to": "0.0.0.0/0", "via": "8.8.8.17"}],
|
||||
"nameservers": {"addresses": ["1.1.1.1"]},
|
||||
},
|
||||
"private0": {
|
||||
"match": {"macaddress": "52:54:00:12:34:57"},
|
||||
"dhcp4": False, "dhcp6": False, "accept-ra": False,
|
||||
"addresses": ["10.24.3.7/32"],
|
||||
"routes": [
|
||||
{"to": "10.24.0.0/16", "scope": "link"},
|
||||
{"to": "10.24.3.8/32", "scope": "link"},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
with pytest.raises(ValueError, match="唯一 VPC"):
|
||||
_parse_network_config(config)
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_private_seed_rejects_legacy_policy_without_rotating_instance_id():
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
||||
)
|
||||
seed = build_seed_iso(
|
||||
str(uuid.uuid4()), "legacy-private-vm", "52:54:00:12:34:56",
|
||||
validated,
|
||||
)
|
||||
legacy_user_data = _user_data(seed)
|
||||
legacy_user_data.pop("updates")
|
||||
legacy_metadata = _meta_data(seed)
|
||||
legacy_metadata.pop("virt-cloud-network-policy")
|
||||
legacy_seed = _build_seed_iso((
|
||||
("/USERDATA.;1", "user-data", "/user-data",
|
||||
b"#cloud-config\n" + json.dumps(
|
||||
legacy_user_data, ensure_ascii=True).encode("ascii") + b"\n"),
|
||||
("/METADATA.;1", "meta-data", "/meta-data",
|
||||
json.dumps(legacy_metadata, ensure_ascii=True).encode("ascii")
|
||||
+ b"\n"),
|
||||
("/NETWORK.;1", "network-config", "/network-config",
|
||||
json.dumps(_network_data(seed), ensure_ascii=True).encode("ascii")
|
||||
+ b"\n"),
|
||||
))
|
||||
|
||||
with pytest.raises(SeedNetworkPolicyMigrationRequired):
|
||||
replace_seed_private_network(
|
||||
legacy_seed,
|
||||
private_mac="52:54:00:12:34:57",
|
||||
private_ip_cidr="10.24.3.7/32",
|
||||
)
|
||||
assert _meta_data(legacy_seed)["instance-id"] == (
|
||||
legacy_metadata["instance-id"])
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_seed_iso_has_nocloud_files_and_mac_specific_static_network():
|
||||
vm_uuid = str(uuid.uuid4())
|
||||
mac = "52:54:00:12:34:56"
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
||||
)
|
||||
seed = build_seed_iso(vm_uuid, "cloud-vm", mac, validated)
|
||||
|
||||
assert isinstance(seed, bytes)
|
||||
assert seed[32769:32774] == b"CD001" # ISO-9660 primary volume descriptor
|
||||
assert b"CIDATA" in seed
|
||||
for marker in (
|
||||
b"user-data", b"meta-data", b"network-config", b"8.8.8.18/28",
|
||||
b"8.8.8.17", b"1.1.1.1", mac.encode(), PUBLIC_KEY.encode(),
|
||||
):
|
||||
assert marker in seed
|
||||
assert b'"routes": [{"to": "0.0.0.0/0", "via": "8.8.8.17"}]' in seed
|
||||
assert b'"to": "default"' not in seed
|
||||
assert b'"renderer": "networkd"' not in seed
|
||||
assert b'"set-name"' not in seed
|
||||
assert b"IPv6AcceptRA=no" not in seed
|
||||
assert b"systemd-machine-id-setup" not in seed
|
||||
user_data = _user_data(seed)
|
||||
metadata = _meta_data(seed)
|
||||
assert user_data["disable_root"] is False
|
||||
assert user_data["ssh_pwauth"] is False
|
||||
assert user_data["updates"] == {"network": {"when": ["boot"]}}
|
||||
assert metadata["virt-cloud-network-policy"] == "boot-v1"
|
||||
assert user_data["users"][0]["name"] == "root"
|
||||
assert user_data["users"][0]["ssh_authorized_keys"] == [PUBLIC_KEY]
|
||||
assert "hashed_passwd" not in user_data["users"][0]
|
||||
assert "passwd" not in user_data["users"][0]
|
||||
sshd_dropin = user_data["write_files"][0]
|
||||
assert sshd_dropin["path"] == "/etc/ssh/sshd_config.d/00-virsh-panel-root.conf"
|
||||
assert "PermitRootLogin prohibit-password" in sshd_dropin["content"]
|
||||
assert "PasswordAuthentication no" in sshd_dropin["content"]
|
||||
assert "runcmd" not in user_data
|
||||
assert "bootcmd" not in user_data
|
||||
assert "dsmode" not in metadata
|
||||
assert all(
|
||||
item["path"] != "/etc/systemd/network/05-virt-cloud-public.network"
|
||||
for item in user_data["write_files"]
|
||||
)
|
||||
assert b"almalinux" not in seed
|
||||
assert read_seed_network(seed) == {
|
||||
"public_ip_cidr": "8.8.8.18/28",
|
||||
"gateway": "8.8.8.17",
|
||||
"dns_servers": ["1.1.1.1"],
|
||||
"mac": mac,
|
||||
}
|
||||
|
||||
|
||||
def test_enable_network_updates_upgrades_legacy_panel_user_data():
|
||||
legacy = b'#cloud-config\n{"users": [{"name": "root"}]}\n'
|
||||
|
||||
upgraded = _enable_network_updates(legacy)
|
||||
|
||||
assert upgraded.startswith(b"#cloud-config\n")
|
||||
payload = json.loads(upgraded.split(b"\n", 1)[1])
|
||||
assert payload["users"] == [{"name": "root"}]
|
||||
assert payload["updates"] == {"network": {"when": ["boot"]}}
|
||||
assert _enable_network_updates(upgraded) == upgraded
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_debian12_network_fix_uses_standard_nocloud_network_config():
|
||||
mac = "52:54:00:12:34:56"
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1", "8.8.8.8"], PUBLIC_KEY,
|
||||
)
|
||||
seed = build_seed_iso(
|
||||
str(uuid.uuid4()), "debian-12-vm", mac, validated,
|
||||
debian12_network_fix=True,
|
||||
)
|
||||
user_data = _user_data(seed)
|
||||
metadata = _meta_data(seed)
|
||||
network_data = _network_data(seed)
|
||||
|
||||
assert network_data == {
|
||||
"version": 2,
|
||||
"ethernets": {
|
||||
"public0": {
|
||||
"match": {"macaddress": mac},
|
||||
"dhcp4": False,
|
||||
"dhcp6": False,
|
||||
"addresses": ["8.8.8.18/28"],
|
||||
"routes": [{"to": "0.0.0.0/0", "via": "8.8.8.17"}],
|
||||
"nameservers": {"addresses": ["1.1.1.1", "8.8.8.8"]},
|
||||
"accept-ra": False,
|
||||
},
|
||||
},
|
||||
"renderer": "networkd",
|
||||
}
|
||||
assert metadata["dsmode"] == "local"
|
||||
bootcmd = user_data["bootcmd"]
|
||||
assert bootcmd[0][:2] == ["/bin/sh", "-ec"]
|
||||
queue_networkd = bootcmd[0][2]
|
||||
assert queue_networkd.splitlines() == [
|
||||
"systemd-machine-id-setup",
|
||||
"netplan generate",
|
||||
"systemctl unmask systemd-networkd.service",
|
||||
"systemctl enable systemd-networkd.service",
|
||||
"systemctl start --no-block systemd-networkd.service",
|
||||
]
|
||||
assert "ip address" not in queue_networkd
|
||||
assert "ip route" not in queue_networkd
|
||||
assert "restart" not in json.dumps(bootcmd)
|
||||
assert "--now" not in json.dumps(bootcmd)
|
||||
assert "runcmd" not in user_data
|
||||
assert b'"set-name"' not in seed
|
||||
assert b"enp1s0" not in seed
|
||||
assert all(
|
||||
not item["path"].startswith("/etc/systemd/network/")
|
||||
for item in user_data["write_files"]
|
||||
)
|
||||
assert read_seed_network(seed) == {
|
||||
"public_ip_cidr": "8.8.8.18/28",
|
||||
"gateway": "8.8.8.17",
|
||||
"dns_servers": ["1.1.1.1", "8.8.8.8"],
|
||||
"mac": mac,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid", [None, 0, 1, "networkd"])
|
||||
def test_seed_rejects_invalid_debian12_network_fix_option(invalid):
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
||||
)
|
||||
with pytest.raises(ValueError, match="Debian 12 网络修复选项"):
|
||||
build_seed_iso(
|
||||
str(uuid.uuid4()), "cloud-vm", "52:54:00:12:34:56", validated,
|
||||
debian12_network_fix=invalid,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_read_seed_network_rejects_invalid_iso_without_exposing_credentials():
|
||||
with pytest.raises(ValueError):
|
||||
read_seed_network(b"not an ISO")
|
||||
with pytest.raises(ValueError):
|
||||
read_seed_network(b"x" * (1024 * 1024 + 1))
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_password_seed_sets_root_hash_and_enables_ssh_password_login():
|
||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], "",
|
||||
root_password_hash=password_hash,
|
||||
)
|
||||
seed = build_seed_iso(
|
||||
str(uuid.uuid4()), "password-vm", "52:54:00:12:34:56", validated,
|
||||
)
|
||||
user_data = _user_data(seed)
|
||||
|
||||
assert user_data["disable_root"] is False
|
||||
assert user_data["ssh_pwauth"] is True
|
||||
assert user_data["users"][0]["name"] == "root"
|
||||
assert user_data["users"][0]["lock_passwd"] is False
|
||||
assert user_data["users"][0]["hashed_passwd"] == password_hash
|
||||
assert "ssh_authorized_keys" not in user_data["users"][0]
|
||||
assert user_data["chpasswd"] == {
|
||||
"expire": False,
|
||||
"users": [{"name": "root", "password": password_hash, "type": "hash"}],
|
||||
}
|
||||
sshd_dropin = user_data["write_files"][0]
|
||||
assert sshd_dropin["path"] == "/etc/ssh/sshd_config.d/00-virsh-panel-root.conf"
|
||||
assert "PermitRootLogin yes" in sshd_dropin["content"]
|
||||
assert "PasswordAuthentication yes" in sshd_dropin["content"]
|
||||
assert GUEST_PASSWORD.encode() not in seed
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_seed_allows_root_password_and_ssh_key_together():
|
||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
||||
root_password_hash=password_hash,
|
||||
)
|
||||
seed = build_seed_iso(
|
||||
str(uuid.uuid4()), "both-vm", "52:54:00:12:34:56", validated,
|
||||
)
|
||||
user_data = _user_data(seed)
|
||||
root = user_data["users"][0]
|
||||
assert root["name"] == "root"
|
||||
assert root["lock_passwd"] is False
|
||||
assert root["ssh_authorized_keys"] == [PUBLIC_KEY]
|
||||
assert root["hashed_passwd"] == password_hash
|
||||
assert user_data["ssh_pwauth"] is True
|
||||
assert user_data["chpasswd"]["users"][0]["password"] == password_hash
|
||||
assert "PermitRootLogin yes" in user_data["write_files"][0]["content"]
|
||||
assert "PasswordAuthentication yes" in user_data["write_files"][0]["content"]
|
||||
assert GUEST_PASSWORD.encode() not in seed
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_replace_seed_network_preserves_user_data_and_metadata():
|
||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
||||
root_password_hash=password_hash,
|
||||
)
|
||||
seed = build_seed_iso(
|
||||
str(uuid.uuid4()), "network-change-vm", "52:54:00:12:34:56", validated,
|
||||
)
|
||||
import pycdlib
|
||||
from io import BytesIO
|
||||
|
||||
def file_bytes(image: bytes, path: str) -> bytes:
|
||||
iso = pycdlib.PyCdlib()
|
||||
iso.open_fp(BytesIO(image))
|
||||
output = BytesIO()
|
||||
iso.get_file_from_iso_fp(output, rr_path=path)
|
||||
iso.close()
|
||||
return output.getvalue()
|
||||
|
||||
user_data = file_bytes(seed, "/user-data")
|
||||
metadata = file_bytes(seed, "/meta-data")
|
||||
original_metadata = json.loads(metadata)
|
||||
changed = replace_seed_network(
|
||||
seed, "8.8.8.19/28", "8.8.8.17", ["8.8.8.8"],
|
||||
)
|
||||
assert file_bytes(changed, "/user-data") == user_data
|
||||
changed_metadata = json.loads(file_bytes(changed, "/meta-data"))
|
||||
assert changed_metadata["instance-id"] == original_metadata["instance-id"]
|
||||
assert changed_metadata["local-hostname"] == original_metadata["local-hostname"]
|
||||
assert changed_metadata["virt-cloud-network-policy"] == "boot-v1"
|
||||
assert read_seed_network(changed) == {
|
||||
"public_ip_cidr": "8.8.8.19/28",
|
||||
"gateway": "8.8.8.17",
|
||||
"dns_servers": ["8.8.8.8"],
|
||||
"mac": "52:54:00:12:34:56",
|
||||
}
|
||||
|
||||
# A seed created before the network-refresh policy has no marker and must
|
||||
# rotate its NoCloud generation once so cloud-init reloads the new files.
|
||||
legacy_user_data = b'#cloud-config\n{"users": [{"name": "root"}]}\n'
|
||||
legacy_metadata = json.dumps({
|
||||
"instance-id": original_metadata["instance-id"],
|
||||
"local-hostname": original_metadata["local-hostname"],
|
||||
}).encode("ascii") + b"\n"
|
||||
legacy_network = file_bytes(seed, "/network-config")
|
||||
legacy_seed = _build_seed_iso((
|
||||
("/USERDATA.;1", "user-data", "/user-data", legacy_user_data),
|
||||
("/METADATA.;1", "meta-data", "/meta-data", legacy_metadata),
|
||||
("/NETWORK.;1", "network-config", "/network-config", legacy_network),
|
||||
))
|
||||
migrated = replace_seed_network(
|
||||
legacy_seed, "8.8.8.19/28", "8.8.8.17", ["8.8.8.8"],
|
||||
)
|
||||
migrated_metadata = json.loads(file_bytes(migrated, "/meta-data"))
|
||||
assert migrated_metadata["instance-id"] != original_metadata["instance-id"]
|
||||
assert migrated_metadata["instance-id"].startswith("virt-cloud-network-")
|
||||
migrated_user_data = file_bytes(migrated, "/user-data")
|
||||
assert json.loads(migrated_user_data.split(b"\n", 1)[1])["updates"] == {
|
||||
"network": {"when": ["boot"]}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_replace_seed_network_rejects_non_panel_seed():
|
||||
with pytest.raises(ValueError, match="无法读取原 cloud-init 配置盘"):
|
||||
replace_seed_network(
|
||||
b"not-an-iso", "8.8.8.19/28", "8.8.8.17", ["8.8.8.8"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None,
|
||||
reason="pycdlib is not installed")
|
||||
def test_replace_seed_settings_rotates_generation_without_exposing_password():
|
||||
password_hash = hash_root_password(GUEST_PASSWORD)
|
||||
validated = validate_cloud_network(
|
||||
"8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY,
|
||||
root_password_hash=password_hash,
|
||||
)
|
||||
seed = build_seed_iso(
|
||||
str(uuid.uuid4()), "settings-vm", "52:54:00:12:34:56", validated,
|
||||
)
|
||||
changed = replace_seed_settings(
|
||||
seed, hostname="Node.Example", root_password_hash=hash_root_password(
|
||||
"Another-strong-password-2026!"
|
||||
), ssh_public_key=PUBLIC_KEY,
|
||||
)
|
||||
user_data = _user_data(changed)
|
||||
root = user_data["users"][0]
|
||||
assert root["ssh_authorized_keys"] == [PUBLIC_KEY]
|
||||
assert root["lock_passwd"] is False
|
||||
assert root["hashed_passwd"].startswith("$6$")
|
||||
assert GUEST_PASSWORD.encode() not in changed
|
||||
assert b"Another-strong-password-2026!" not in changed
|
||||
assert user_data["ssh_pwauth"] is True
|
||||
assert user_data["chpasswd"]["users"][0]["type"] == "hash"
|
||||
assert _meta_data(changed)["local-hostname"] == "node.example"
|
||||
assert _meta_data(changed)["instance-id"].startswith("virt-cloud-settings-")
|
||||
assert read_seed_network(changed) == read_seed_network(seed)
|
||||
@ -1,750 +0,0 @@
|
||||
"""Global cloud-template catalog, source file and import job tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.cloud_template_runtime import (
|
||||
CACHE_TOMBSTONE_KIND,
|
||||
CATALOG_KIND,
|
||||
IMPORT_JOB_KIND,
|
||||
CloudTemplateRuntime,
|
||||
GlobalCloudTemplateFiles,
|
||||
GlobalCloudTemplateStore,
|
||||
PERSISTENT_FREE_RESERVE_BYTES,
|
||||
cloud_template_id_for,
|
||||
)
|
||||
from virsh_panel.cloud_templates import GIB, TemplateError, template_volume_name
|
||||
|
||||
|
||||
ALMA_10_URL = (
|
||||
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/"
|
||||
"AlmaLinux-10-GenericCloud-latest.x86_64.qcow2"
|
||||
)
|
||||
ALMA_8_URL = (
|
||||
"https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/"
|
||||
"AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
|
||||
)
|
||||
DEBIAN_12_URL = (
|
||||
"https://cloud.debian.org/images/cloud/bookworm/latest/"
|
||||
"debian-12-generic-amd64.qcow2"
|
||||
)
|
||||
UBUNTU_24_URL = (
|
||||
"https://cloud-images.ubuntu.com/releases/24.04/release/"
|
||||
"ubuntu-24.04-server-cloudimg-amd64.img"
|
||||
)
|
||||
CENTOS_STREAM_10_URL = (
|
||||
"https://cloud.centos.org/centos/10-stream/x86_64/images/"
|
||||
"CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2"
|
||||
)
|
||||
|
||||
|
||||
class MemoryStore:
|
||||
def __init__(self) -> None:
|
||||
self.values: dict[tuple[str, str], str] = {}
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def list(self, kind: str) -> dict[str, str]:
|
||||
with self.lock:
|
||||
return {
|
||||
key: value for (record_kind, key), value in self.values.items()
|
||||
if record_kind == kind
|
||||
}
|
||||
|
||||
def get(self, kind: str, key: str) -> str | None:
|
||||
with self.lock:
|
||||
return self.values.get((kind, key))
|
||||
|
||||
def set(self, kind: str, key: str, value: str) -> None:
|
||||
with self.lock:
|
||||
self.values[(kind, key)] = value
|
||||
|
||||
def insert(self, kind: str, key: str, value: str) -> bool:
|
||||
with self.lock:
|
||||
identity = (kind, key)
|
||||
if identity in self.values:
|
||||
return False
|
||||
self.values[identity] = value
|
||||
return True
|
||||
|
||||
def delete(self, kind: str, key: str) -> None:
|
||||
with self.lock:
|
||||
self.values.pop((kind, key), None)
|
||||
|
||||
|
||||
class CommitThenRaiseCatalogStore(MemoryStore):
|
||||
"""Simulate a committed INSERT whose acknowledgement is lost."""
|
||||
|
||||
def insert(self, kind: str, key: str, value: str) -> bool:
|
||||
inserted = super().insert(kind, key, value)
|
||||
if kind == CATALOG_KIND and inserted:
|
||||
raise RuntimeError("catalog insert acknowledgement lost")
|
||||
return inserted
|
||||
|
||||
|
||||
class AmbiguousCatalogCommitStore(CommitThenRaiseCatalogStore):
|
||||
"""Also fail the first catalog read after the ambiguous INSERT."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.fail_next_catalog_read = False
|
||||
|
||||
def list(self, kind: str) -> dict[str, str]:
|
||||
if kind == CATALOG_KIND and self.fail_next_catalog_read:
|
||||
self.fail_next_catalog_read = False
|
||||
raise RuntimeError("catalog read unavailable")
|
||||
return super().list(kind)
|
||||
|
||||
def insert(self, kind: str, key: str, value: str) -> bool:
|
||||
try:
|
||||
return super().insert(kind, key, value)
|
||||
except RuntimeError:
|
||||
self.fail_next_catalog_read = True
|
||||
raise
|
||||
|
||||
|
||||
class LoseFirstCompletedJobWriteStore(MemoryStore):
|
||||
"""Drop one terminal job update while retaining its running record."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.lost_completed_write = False
|
||||
|
||||
def set(self, kind: str, key: str, value: str) -> None:
|
||||
if kind == IMPORT_JOB_KIND:
|
||||
job = json.loads(value)
|
||||
if (job.get("status") == "completed"
|
||||
and not self.lost_completed_write):
|
||||
self.lost_completed_write = True
|
||||
raise RuntimeError("completed job write acknowledgement lost")
|
||||
super().set(kind, key, value)
|
||||
|
||||
|
||||
def wait_for_job(runtime: CloudTemplateRuntime, job_id: str) -> dict:
|
||||
for _ in range(200):
|
||||
job = runtime.get_job(job_id)
|
||||
if job["status"] in {"completed", "failed"}:
|
||||
return job
|
||||
time.sleep(0.01)
|
||||
raise AssertionError("cloud template import did not finish")
|
||||
|
||||
|
||||
def fake_download_bytes(payload: bytes):
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
|
||||
def download(_url: str, expected_sha256: str, directory: Path, **_kwargs):
|
||||
assert expected_sha256 in {"", digest}
|
||||
source = Path(directory) / ".cloud-image-test.qcow2"
|
||||
source.write_bytes(payload)
|
||||
os.chmod(source, 0o600)
|
||||
return source, 2 * GIB, digest
|
||||
|
||||
return digest, download
|
||||
|
||||
|
||||
def record_for(payload: bytes, *, name: str = "AlmaLinux 10") -> dict:
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
template_id = cloud_template_id_for(digest, "10")
|
||||
return {
|
||||
"id": template_id,
|
||||
"name": name,
|
||||
"os_family": "AlmaLinux",
|
||||
"os_version": "10",
|
||||
"sha256": digest,
|
||||
"volume_name": template_volume_name(template_id),
|
||||
"virtual_size": 2 * GIB,
|
||||
"image_bytes": len(payload),
|
||||
"created_at": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_stable_global_id_and_catalog_use_an_independent_record_kind():
|
||||
digest = "a" * 64
|
||||
first = cloud_template_id_for(digest, "10")
|
||||
second = cloud_template_id_for(digest.upper(), "10")
|
||||
assert first == second
|
||||
assert first.startswith("alma10-")
|
||||
assert template_volume_name(first).endswith(".qcow2")
|
||||
|
||||
store = MemoryStore()
|
||||
catalog = GlobalCloudTemplateStore(store)
|
||||
record = record_for(b"catalog-image")
|
||||
catalog.add(record)
|
||||
|
||||
assert catalog.get(record["id"]) == record
|
||||
assert list(store.list(CATALOG_KIND)) == [record["id"]]
|
||||
assert store.list("template") == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
"官方云镜像地址无法解析",
|
||||
"官方云镜像地址解析结果无效",
|
||||
"官方云镜像地址解析到非公网地址",
|
||||
"无法连接官方云镜像下载地址",
|
||||
"官方云镜像重定向目标不安全",
|
||||
"官方云镜像重定向次数过多",
|
||||
"官方云镜像下载服务器返回异常状态",
|
||||
"官方云镜像下载响应格式无效",
|
||||
"云镜像下载内容不完整",
|
||||
],
|
||||
)
|
||||
def test_public_job_preserves_fixed_safe_download_errors(message):
|
||||
job = {
|
||||
"job_id": "safe-error",
|
||||
"status": "failed",
|
||||
"phase": "导入失败",
|
||||
"created_at": 1,
|
||||
"error": message,
|
||||
}
|
||||
assert CloudTemplateRuntime.public_job(job)["error"] == message
|
||||
|
||||
|
||||
def test_import_publishes_private_source_and_restores_global_catalog(tmp_path):
|
||||
store = MemoryStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"verified-global-cloud-image"
|
||||
digest, downloader = fake_download_bytes(payload)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=downloader,
|
||||
):
|
||||
queued = runtime.queue_import(
|
||||
ALMA_10_URL, "AlmaLinux 10", digest
|
||||
)
|
||||
job = wait_for_job(runtime, queued["job_id"])
|
||||
|
||||
assert job["status"] == "completed", job
|
||||
template_id = job["template"]["id"]
|
||||
assert template_id == cloud_template_id_for(digest, "10")
|
||||
assert "volume_name" not in job["template"]
|
||||
listed = runtime.list()
|
||||
assert listed == [job["template"]]
|
||||
assert "active_import_job" not in runtime.status()
|
||||
|
||||
root = tmp_path / "cloud-templates"
|
||||
source = root / template_volume_name(template_id)
|
||||
assert source.read_bytes() == payload
|
||||
assert stat.S_IMODE(root.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE(source.stat().st_mode) == 0o600
|
||||
assert store.get(IMPORT_JOB_KIND, queued["job_id"]) is not None
|
||||
|
||||
# A new runtime reads the same host-independent catalog and verifies
|
||||
# the persisted source from disk instead of trusting process memory.
|
||||
restored = CloudTemplateRuntime(store, tmp_path)
|
||||
try:
|
||||
restored.restore_jobs()
|
||||
assert restored.list() == listed
|
||||
with restored.source(template_id) as (record, path):
|
||||
assert record["sha256"] == digest
|
||||
assert path == source
|
||||
assert restored.in_use[template_id] == 1
|
||||
assert template_id not in restored.in_use
|
||||
finally:
|
||||
restored.close()
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "family", "version", "prefix"),
|
||||
[
|
||||
(ALMA_8_URL, "AlmaLinux", "8", "alma8-"),
|
||||
(DEBIAN_12_URL, "Debian", "12", "debian12-"),
|
||||
(UBUNTU_24_URL, "Ubuntu", "24.04", "ubuntu2404-"),
|
||||
(CENTOS_STREAM_10_URL, "CentOS Stream", "10", "centosstream10-"),
|
||||
],
|
||||
)
|
||||
def test_import_derives_family_version_and_safe_source_name(
|
||||
tmp_path, url, family, version, prefix):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
payload = f"{family}-{version}-cloud-image".encode()
|
||||
digest, downloader = fake_download_bytes(payload)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=downloader,
|
||||
):
|
||||
queued = runtime.queue_import(url, f"{family} {version}", digest)
|
||||
job = wait_for_job(runtime, queued["job_id"])
|
||||
|
||||
assert job["status"] == "completed", job
|
||||
template = job["template"]
|
||||
assert template["id"].startswith(prefix)
|
||||
assert template["os_family"] == family
|
||||
assert template["os_version"] == version
|
||||
internal = runtime.get(template["id"])
|
||||
assert internal["volume_name"] == template_volume_name(template["id"])
|
||||
assert (
|
||||
tmp_path / "cloud-templates" / internal["volume_name"]
|
||||
).read_bytes() == payload
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_available_bytes_preserves_panel_disk_reserve(tmp_path):
|
||||
files = GlobalCloudTemplateFiles(tmp_path)
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
|
||||
return_value=SimpleNamespace(
|
||||
free=PERSISTENT_FREE_RESERVE_BYTES + 3 * 1024 * 1024
|
||||
),
|
||||
):
|
||||
assert files.available_bytes() == 3 * 1024 * 1024
|
||||
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
|
||||
return_value=SimpleNamespace(free=PERSISTENT_FREE_RESERVE_BYTES - 1),
|
||||
):
|
||||
assert files.available_bytes() == 0
|
||||
|
||||
|
||||
def test_import_rejects_before_queue_when_only_reserved_space_remains(tmp_path):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
try:
|
||||
with patch.object(
|
||||
runtime.files, "available_bytes", return_value=1024 * 1024 - 1
|
||||
):
|
||||
with pytest.raises(TemplateError, match="空间不足") as insufficient:
|
||||
runtime.queue_import(ALMA_10_URL, "No Disk Space", "")
|
||||
assert insufficient.value.status == 507
|
||||
assert runtime.jobs == {}
|
||||
assert runtime.pending == set()
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_import_passes_only_space_above_reserve_to_downloader(tmp_path):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
payload = b"bounded-global-cloud-image"
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
limits: list[int] = []
|
||||
|
||||
def bounded_download(_url: str, _expected: str, directory: Path, *,
|
||||
max_bytes: int):
|
||||
limits.append(max_bytes)
|
||||
source = Path(directory) / ".cloud-image-bounded.qcow2"
|
||||
source.write_bytes(payload)
|
||||
os.chmod(source, 0o600)
|
||||
return source, 2 * GIB, digest
|
||||
|
||||
try:
|
||||
with patch.object(
|
||||
runtime.files, "available_bytes",
|
||||
side_effect=[3 * 1024 * 1024, 2 * 1024 * 1024],
|
||||
), patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=bounded_download,
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Bounded", "")
|
||||
job = wait_for_job(runtime, queued["job_id"])
|
||||
assert job["status"] == "completed", job
|
||||
assert limits == [2 * 1024 * 1024]
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_duplicate_digest_and_casefold_name_are_rejected_globally():
|
||||
store = MemoryStore()
|
||||
catalog = GlobalCloudTemplateStore(store)
|
||||
first = record_for(b"first-image", name="AlmaLinux Gold")
|
||||
catalog.add(first)
|
||||
|
||||
with pytest.raises(TemplateError, match="已经导入") as duplicate_digest:
|
||||
catalog.add(first)
|
||||
assert duplicate_digest.value.status == 409
|
||||
|
||||
second = record_for(b"second-image", name="almalinux gold")
|
||||
with pytest.raises(TemplateError, match="同名") as duplicate_name:
|
||||
catalog.add(second)
|
||||
assert duplicate_name.value.status == 409
|
||||
|
||||
|
||||
def test_rename_changes_only_display_name_and_keeps_source_identity(tmp_path):
|
||||
store = MemoryStore()
|
||||
catalog = GlobalCloudTemplateStore(store)
|
||||
record = record_for(b"rename-image")
|
||||
catalog.add(record)
|
||||
updated = catalog.rename(record["id"], "AlmaLinux 10 Production")
|
||||
|
||||
assert updated["name"] == "AlmaLinux 10 Production"
|
||||
for key in (
|
||||
"id", "sha256", "volume_name", "virtual_size", "image_bytes",
|
||||
"os_family", "os_version", "created_at",
|
||||
):
|
||||
assert updated[key] == record[key]
|
||||
|
||||
files = GlobalCloudTemplateFiles(tmp_path)
|
||||
temporary = files.directory() / ".cloud-image-rename.qcow2"
|
||||
temporary.write_bytes(b"rename-image")
|
||||
os.chmod(temporary, 0o600)
|
||||
source = files.publish(
|
||||
temporary, record["id"], expected_size=len(b"rename-image")
|
||||
)
|
||||
assert source.name == record["volume_name"]
|
||||
|
||||
|
||||
def test_delete_removes_global_catalog_source_and_verification_cache(tmp_path):
|
||||
store = MemoryStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"delete-global-cloud-image"
|
||||
record = record_for(payload)
|
||||
runtime.catalog.add(record)
|
||||
temporary = runtime.files.directory() / ".cloud-image-delete.qcow2"
|
||||
temporary.write_bytes(payload)
|
||||
os.chmod(temporary, 0o600)
|
||||
source = runtime.files.publish(
|
||||
temporary, record["id"], expected_size=len(payload)
|
||||
)
|
||||
runtime.verified_sources[record["id"]] = (1, 2, 3, 4, record["sha256"])
|
||||
try:
|
||||
runtime.delete(record["id"])
|
||||
|
||||
assert not source.exists()
|
||||
assert store.get(CATALOG_KIND, record["id"]) is None
|
||||
assert record["id"] not in runtime.verified_sources
|
||||
with pytest.raises(TemplateError) as missing:
|
||||
runtime.get(record["id"])
|
||||
assert missing.value.status == 404
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_delete_rejects_import_or_active_source_lease(tmp_path):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
payload = b"leased-global-cloud-image"
|
||||
record = record_for(payload)
|
||||
runtime.catalog.add(record)
|
||||
temporary = runtime.files.directory() / ".cloud-image-leased.qcow2"
|
||||
temporary.write_bytes(payload)
|
||||
os.chmod(temporary, 0o600)
|
||||
source = runtime.files.publish(
|
||||
temporary, record["id"], expected_size=len(payload)
|
||||
)
|
||||
try:
|
||||
runtime.pending.add("active-import")
|
||||
with pytest.raises(TemplateError, match="正在导入") as importing:
|
||||
runtime.delete(record["id"])
|
||||
assert importing.value.status == 409
|
||||
runtime.pending.clear()
|
||||
|
||||
with runtime.source(record["id"]):
|
||||
with pytest.raises(TemplateError, match="正在同步") as leased:
|
||||
runtime.delete(record["id"])
|
||||
assert leased.value.status == 409
|
||||
assert source.exists()
|
||||
assert runtime.get(record["id"])["id"] == record["id"]
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_reimport_waits_for_old_host_cache_gc_then_reuses_stable_id(tmp_path):
|
||||
store = MemoryStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
record = record_for(b"reimport-identical-image")
|
||||
tombstone = {
|
||||
"deletion_id": "a" * 32,
|
||||
"template_id": record["id"],
|
||||
"template_created_at": record["created_at"],
|
||||
"source_volume_name": record["volume_name"],
|
||||
"pending_hosts": ["offline-host"],
|
||||
"source_deleted": True,
|
||||
"created_at": 1,
|
||||
}
|
||||
store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone))
|
||||
try:
|
||||
with pytest.raises(TemplateError, match="缓存仍在清理") as pending:
|
||||
runtime._prepare_reimport(record["id"])
|
||||
assert pending.value.status == 409
|
||||
assert store.get(CACHE_TOMBSTONE_KIND, record["id"]) is not None
|
||||
|
||||
tombstone["pending_hosts"] = []
|
||||
store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone))
|
||||
previous_created_at = runtime._prepare_reimport(record["id"])
|
||||
assert store.get(CACHE_TOMBSTONE_KIND, record["id"]) is None
|
||||
assert previous_created_at == record["created_at"]
|
||||
first_generation = runtime._next_generation(
|
||||
record["id"], previous_created_at)
|
||||
second_generation = runtime._next_generation(record["id"])
|
||||
assert second_generation > first_generation > previous_created_at
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_restore_cancels_provisional_delete_when_source_and_catalog_survive(
|
||||
tmp_path):
|
||||
store = MemoryStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"delete-never-committed"
|
||||
record = record_for(payload)
|
||||
runtime.catalog.add(record)
|
||||
temporary = runtime.files.directory() / ".cloud-image-delete-intact.qcow2"
|
||||
temporary.write_bytes(payload)
|
||||
os.chmod(temporary, 0o600)
|
||||
source = runtime.files.publish(
|
||||
temporary, record["id"], expected_size=len(payload)
|
||||
)
|
||||
tombstone = {
|
||||
"deletion_id": "c" * 32,
|
||||
"template_id": record["id"],
|
||||
"template_created_at": record["created_at"],
|
||||
"source_volume_name": record["volume_name"],
|
||||
"pending_hosts": ["host-one"],
|
||||
"source_deleted": False,
|
||||
"created_at": 1,
|
||||
}
|
||||
store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone))
|
||||
try:
|
||||
runtime.restore_jobs()
|
||||
|
||||
assert runtime.get(record["id"])["id"] == record["id"]
|
||||
assert source.is_file()
|
||||
assert store.get(CACHE_TOMBSTONE_KIND, record["id"]) is None
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_restore_finishes_delete_after_source_unlink_crash_window(tmp_path):
|
||||
store = MemoryStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"delete-unlink-committed"
|
||||
record = record_for(payload)
|
||||
runtime.catalog.add(record)
|
||||
tombstone = {
|
||||
"deletion_id": "d" * 32,
|
||||
"template_id": record["id"],
|
||||
"template_created_at": record["created_at"],
|
||||
"source_volume_name": record["volume_name"],
|
||||
"pending_hosts": ["host-one"],
|
||||
"source_deleted": False,
|
||||
"created_at": 1,
|
||||
}
|
||||
store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone))
|
||||
try:
|
||||
# Listing is observational. Only delete recovery/maintenance may
|
||||
# advance the durable tombstone and remove local source/catalog data.
|
||||
assert [row["id"] for row in runtime.list()] == [record["id"]]
|
||||
before = json.loads(store.get(CACHE_TOMBSTONE_KIND, record["id"]))
|
||||
assert before["source_deleted"] is False
|
||||
assert runtime.get(record["id"])["id"] == record["id"]
|
||||
runtime.restore_jobs()
|
||||
|
||||
with pytest.raises(TemplateError) as missing:
|
||||
runtime.get(record["id"])
|
||||
assert missing.value.status == 404
|
||||
restored = json.loads(store.get(
|
||||
CACHE_TOMBSTONE_KIND, record["id"]
|
||||
))
|
||||
assert restored["source_deleted"] is True
|
||||
assert restored["pending_hosts"] == ["host-one"]
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_source_rejects_permission_damage_and_content_tampering(tmp_path):
|
||||
store = MemoryStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"source-integrity"
|
||||
record = record_for(payload)
|
||||
runtime.catalog.add(record)
|
||||
temporary = runtime.files.directory() / ".cloud-image-integrity.qcow2"
|
||||
temporary.write_bytes(payload)
|
||||
os.chmod(temporary, 0o600)
|
||||
source = runtime.files.publish(
|
||||
temporary, record["id"], expected_size=len(payload)
|
||||
)
|
||||
try:
|
||||
os.chmod(source, 0o640)
|
||||
with pytest.raises(TemplateError, match="已损坏"):
|
||||
with runtime.source(record["id"]):
|
||||
pass
|
||||
|
||||
os.chmod(source, 0o600)
|
||||
source.write_bytes(b"x" * len(payload))
|
||||
with pytest.raises(TemplateError, match="SHA-256"):
|
||||
with runtime.source(record["id"]):
|
||||
pass
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_concurrent_import_is_serialized_and_second_request_is_rejected(tmp_path):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
payload = b"blocking-image"
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
|
||||
def blocking_download(_url: str, _expected: str, directory: Path,
|
||||
**_kwargs):
|
||||
started.set()
|
||||
assert release.wait(timeout=5)
|
||||
source = Path(directory) / ".cloud-image-blocking.qcow2"
|
||||
source.write_bytes(payload)
|
||||
os.chmod(source, 0o600)
|
||||
return source, 2 * GIB, digest
|
||||
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=blocking_download,
|
||||
):
|
||||
first = runtime.queue_import(ALMA_10_URL, "First", "")
|
||||
assert started.wait(timeout=2)
|
||||
assert runtime.has_active_jobs() is True
|
||||
with pytest.raises(TemplateError, match="正在导入") as busy:
|
||||
runtime.queue_import(ALMA_10_URL, "Second", "")
|
||||
assert busy.value.status == 409
|
||||
release.set()
|
||||
assert wait_for_job(runtime, first["job_id"])["status"] == "completed"
|
||||
assert runtime.has_active_jobs() is False
|
||||
finally:
|
||||
release.set()
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_restore_marks_interrupted_job_failed_and_removes_partial_files(tmp_path):
|
||||
store = MemoryStore()
|
||||
job_id = "interrupted-cloud-template"
|
||||
store.set(IMPORT_JOB_KIND, job_id, json.dumps({
|
||||
"job_id": job_id,
|
||||
"status": "running",
|
||||
"phase": "正在下载并校验云镜像",
|
||||
"created_at": 1,
|
||||
}))
|
||||
files = GlobalCloudTemplateFiles(tmp_path)
|
||||
partial = files.directory() / ".cloud-image-abandoned.qcow2"
|
||||
partial.write_bytes(b"partial")
|
||||
os.chmod(partial, 0o600)
|
||||
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
try:
|
||||
runtime.restore_jobs()
|
||||
job = runtime.get_job(job_id)
|
||||
assert job["status"] == "failed"
|
||||
assert "重启" in job["phase"]
|
||||
assert "重启" in job["error"]
|
||||
assert not partial.exists()
|
||||
assert runtime.has_active_jobs() is False
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_import_failure_redacts_internal_paths_and_releases_pending(tmp_path):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=TemplateError(
|
||||
"私有目录 /home/virt-cloud/data/cloud-templates 损坏", 500
|
||||
),
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Broken", "")
|
||||
failed = wait_for_job(runtime, queued["job_id"])
|
||||
assert failed["status"] == "failed"
|
||||
assert failed["error"] == "云镜像下载或保存失败,请联系管理员查看日志"
|
||||
assert "/home/" not in json.dumps(failed, ensure_ascii=False)
|
||||
assert runtime.pending == set()
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_catalog_insert_commit_with_lost_ack_is_completed_not_rolled_back(
|
||||
tmp_path):
|
||||
store = CommitThenRaiseCatalogStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"catalog-commit-ack-lost"
|
||||
digest, downloader = fake_download_bytes(payload)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=downloader,
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Committed", digest)
|
||||
job = wait_for_job(runtime, queued["job_id"])
|
||||
|
||||
assert job["status"] == "completed"
|
||||
template_id = cloud_template_id_for(digest, "10")
|
||||
assert runtime.get(template_id)["sha256"] == digest
|
||||
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
|
||||
assert source.read_bytes() == payload
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_ambiguous_catalog_commit_preserves_source_for_startup_recovery(
|
||||
tmp_path):
|
||||
store = AmbiguousCatalogCommitStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"ambiguous-catalog-commit"
|
||||
digest, downloader = fake_download_bytes(payload)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=downloader,
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Ambiguous", digest)
|
||||
failed = wait_for_job(runtime, queued["job_id"])
|
||||
|
||||
assert failed["status"] == "failed"
|
||||
template_id = cloud_template_id_for(digest, "10")
|
||||
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
|
||||
assert source.read_bytes() == payload
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
restored = CloudTemplateRuntime(store, tmp_path)
|
||||
try:
|
||||
restored.restore_jobs()
|
||||
recovered = restored.get_job(queued["job_id"])
|
||||
assert recovered["status"] == "completed"
|
||||
assert recovered["template"]["id"] == template_id
|
||||
assert source.read_bytes() == payload
|
||||
finally:
|
||||
restored.close()
|
||||
|
||||
|
||||
def test_lost_completed_job_write_is_success_in_memory_and_repaired_on_restart(
|
||||
tmp_path):
|
||||
store = LoseFirstCompletedJobWriteStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"lost-completed-job-write"
|
||||
digest, downloader = fake_download_bytes(payload)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=downloader,
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Recoverable", digest)
|
||||
completed = wait_for_job(runtime, queued["job_id"])
|
||||
|
||||
assert completed["status"] == "completed"
|
||||
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
|
||||
assert persisted["status"] == "running"
|
||||
assert persisted["candidate_template_id"] == cloud_template_id_for(
|
||||
digest, "10"
|
||||
)
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
restored = CloudTemplateRuntime(store, tmp_path)
|
||||
try:
|
||||
restored.restore_jobs()
|
||||
repaired = restored.get_job(queued["job_id"])
|
||||
assert repaired["status"] == "completed"
|
||||
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
|
||||
assert persisted["status"] == "completed"
|
||||
finally:
|
||||
restored.close()
|
||||
@ -1,164 +0,0 @@
|
||||
"""Private per-image scripts: persistence, edit conflicts and API isolation."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
|
||||
from test_cloud_template_runtime import MemoryStore, record_for
|
||||
from test_engine_boundary import ADMIN, SUPER, USER, boundary, call
|
||||
from virsh_panel.cloud_template_runtime import CATALOG_KIND, CloudTemplateRuntime
|
||||
from virsh_panel.cloud_templates import TemplateError
|
||||
from virsh_panel.startup_script import MAX_STARTUP_SCRIPT_BYTES, normalize_startup_script
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def catalog(tmp_path):
|
||||
store = MemoryStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
record = record_for(b"per-image-startup")
|
||||
runtime.catalog.add(record)
|
||||
try:
|
||||
yield runtime, store, record
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_script_survives_runtime_restart_and_rename_but_is_not_public(catalog, tmp_path):
|
||||
runtime, store, record = catalog
|
||||
template_id = record["id"]
|
||||
assert runtime.get_startup_script(template_id) == {
|
||||
"startup_script": "", "revision": 0, "created_at": 1,
|
||||
}
|
||||
assert runtime.get(template_id)["startup_script"] == ""
|
||||
script = "#!/bin/bash\necho 'private startup content'\n"
|
||||
assert runtime.set_startup_script(
|
||||
template_id, script.replace("\n", "\r\n"),
|
||||
expected_revision=0, created_at=1,
|
||||
) == {"startup_script": script, "revision": 1, "created_at": 1}
|
||||
|
||||
# Independent rename/script submissions must preserve both settings.
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
rename = executor.submit(runtime.rename, template_id, "Renamed image")
|
||||
write = executor.submit(
|
||||
runtime.set_startup_script, template_id, script + "true\n",
|
||||
expected_revision=1, created_at=1)
|
||||
public = rename.result()
|
||||
write.result()
|
||||
assert "startup_script" not in public
|
||||
restarted = CloudTemplateRuntime(store, tmp_path)
|
||||
try:
|
||||
restored = restarted.get(template_id)
|
||||
assert restored["name"] == "Renamed image"
|
||||
assert restored["startup_script"] == script + "true\n"
|
||||
assert restored["startup_script_revision"] == 2
|
||||
public = restarted.list()[0]
|
||||
assert "startup_script" not in public
|
||||
assert "startup_script_revision" not in public
|
||||
job = restarted.public_job({"template": restored, "status": "completed"})
|
||||
assert job["template"] == public
|
||||
finally:
|
||||
restarted.close()
|
||||
|
||||
|
||||
def test_conflicting_editor_cannot_replace_or_clear_newer_script(catalog):
|
||||
runtime, _store, record = catalog
|
||||
template_id = record["id"]
|
||||
runtime.set_startup_script(template_id, "echo first\n", expected_revision=0, created_at=1)
|
||||
for script in ("echo stale\n", ""):
|
||||
with pytest.raises(TemplateError) as error:
|
||||
runtime.set_startup_script(template_id, script, expected_revision=0, created_at=1)
|
||||
assert error.value.status == 409
|
||||
assert runtime.get_startup_script(template_id)["startup_script"] == "echo first\n"
|
||||
unchanged = runtime.set_startup_script(
|
||||
template_id, "echo first\n", expected_revision=1, created_at=1)
|
||||
assert unchanged["revision"] == 1
|
||||
cleared = runtime.set_startup_script(template_id, "", expected_revision=1, created_at=1)
|
||||
assert cleared == {"startup_script": "", "revision": 2, "created_at": 1}
|
||||
|
||||
|
||||
def test_delete_reimport_never_restores_previous_generation_script(catalog):
|
||||
runtime, store, record = catalog
|
||||
template_id = record["id"]
|
||||
runtime.set_startup_script(template_id, "echo old\n", expected_revision=0, created_at=1)
|
||||
runtime.delete(template_id)
|
||||
assert store.get(CATALOG_KIND, template_id) is None
|
||||
runtime.catalog.add({**record, "created_at": 2})
|
||||
assert runtime.get_startup_script(template_id) == {
|
||||
"startup_script": "", "revision": 0, "created_at": 2,
|
||||
}
|
||||
with pytest.raises(TemplateError) as error:
|
||||
runtime.set_startup_script(template_id, "echo stale\n", expected_revision=0, created_at=1)
|
||||
assert error.value.status == 409
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, 0, [], {}, "echo\0bad", "\ud800",
|
||||
"a" * (MAX_STARTUP_SCRIPT_BYTES + 1)])
|
||||
def test_invalid_script_is_rejected_without_mutating_catalog(catalog, value):
|
||||
runtime, store, record = catalog
|
||||
before = store.get(CATALOG_KIND, record["id"])
|
||||
with pytest.raises(TemplateError) as error:
|
||||
runtime.set_startup_script(record["id"], value, expected_revision=0, created_at=1)
|
||||
assert error.value.status == 422
|
||||
assert store.get(CATALOG_KIND, record["id"]) == before
|
||||
|
||||
|
||||
def test_script_limit_is_utf8_bytes_and_crlf_normalization_precedes_limit():
|
||||
exact = "中" * 10922 + "ab"
|
||||
assert len(exact.encode("utf-8")) == MAX_STARTUP_SCRIPT_BYTES
|
||||
assert normalize_startup_script(exact) == exact
|
||||
with pytest.raises(ValueError):
|
||||
normalize_startup_script(exact + "c")
|
||||
assert normalize_startup_script("\r\n" * MAX_STARTUP_SCRIPT_BYTES) == "\n" * MAX_STARTUP_SCRIPT_BYTES
|
||||
|
||||
|
||||
def test_private_script_api_permissions_validation_and_no_host_dispatch(boundary):
|
||||
client, engine, _store, _logs, services = boundary
|
||||
record = record_for(b"startup-api")
|
||||
engine.registry.cloud_templates.catalog.add(record)
|
||||
path = f"/internal/v1/vm-templates/{record['id']}/startup-script"
|
||||
body = {"script": "echo configured\r\n", "expected_revision": 0, "created_at": 1}
|
||||
|
||||
assert call(client, "GET", path, USER).status_code == 403
|
||||
assert call(client, "PATCH", path, USER, body).status_code == 403
|
||||
assert call(client, "PATCH", path, ADMIN, body).status_code == 403
|
||||
assert call(client, "PATCH", path, {**SUPER, "can_manage": False}, body).status_code == 403
|
||||
assert call(client, "GET", path, ADMIN).json() == {
|
||||
"startup_script": "", "revision": 0, "created_at": 1,
|
||||
}
|
||||
|
||||
updated = call(client, "PATCH", path, SUPER, body)
|
||||
assert updated.status_code == 200, updated.text
|
||||
assert updated.json() == {
|
||||
"startup_script": "echo configured\n", "revision": 1, "created_at": 1,
|
||||
}
|
||||
assert call(client, "GET", path, SUPER).json() == updated.json()
|
||||
assert call(client, "PATCH", path, SUPER, body).status_code == 409
|
||||
|
||||
listed = call(client, "GET", "/internal/v1/vm-templates", USER).json()
|
||||
assert "startup_script" not in listed["templates"][0]
|
||||
assert "startup_script_revision" not in listed["templates"][0]
|
||||
assert services == {}
|
||||
assert engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [
|
||||
{}, {"script": "a"}, [], None,
|
||||
{"script": [], "expected_revision": 0, "created_at": 1},
|
||||
{"script": "a", "expected_revision": False, "created_at": 1},
|
||||
{"script": "a", "expected_revision": "0", "created_at": 1},
|
||||
{"script": "a", "expected_revision": -1, "created_at": 1},
|
||||
{"script": "a", "expected_revision": 0, "created_at": True},
|
||||
{"script": "a", "expected_revision": 0, "created_at": "1"},
|
||||
{"script": "a", "expected_revision": 0, "created_at": 0},
|
||||
{"script": "a", "expected_revision": 0, "created_at": 1, "host_id": "injected"},
|
||||
])
|
||||
def test_script_api_rejects_malformed_bodies(boundary, body):
|
||||
client, engine, _store, _logs, _services = boundary
|
||||
record = record_for(b"startup-invalid-api")
|
||||
runtime = engine.registry.cloud_templates
|
||||
runtime.catalog.add(record)
|
||||
path = f"/internal/v1/vm-templates/{record['id']}/startup-script"
|
||||
response = call(client, "PATCH", path, SUPER, body)
|
||||
assert response.status_code == 422, response.text
|
||||
assert runtime.get_startup_script(record["id"])["revision"] == 0
|
||||
assert engine.coordinator.snapshot() == {}
|
||||
@ -1,502 +0,0 @@
|
||||
"""Official-image import restrictions and private template catalog tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import socket
|
||||
import stat
|
||||
import struct
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.cloud_templates import (
|
||||
TemplateError,
|
||||
TemplateStore,
|
||||
_RejectRedirects,
|
||||
_open_image_response,
|
||||
_redirect_target,
|
||||
_resolve_global_addresses,
|
||||
download_verified_image,
|
||||
image_metadata_for,
|
||||
image_version_for,
|
||||
template_id_for,
|
||||
template_scope,
|
||||
template_volume_name,
|
||||
validate_image_url,
|
||||
verify_qcow2_image,
|
||||
)
|
||||
|
||||
|
||||
OFFICIAL = (
|
||||
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/"
|
||||
"AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2"
|
||||
)
|
||||
OFFICIAL_9 = (
|
||||
"https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/"
|
||||
"AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
|
||||
)
|
||||
OFFICIAL_8 = (
|
||||
"https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/"
|
||||
"AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
|
||||
)
|
||||
DEBIAN_12 = (
|
||||
"https://cloud.debian.org/images/cloud/bookworm/latest/"
|
||||
"debian-12-generic-amd64.qcow2"
|
||||
)
|
||||
DEBIAN_13_BUILD = (
|
||||
"https://cloud.debian.org/images/cloud/trixie/20260801-2204/"
|
||||
"debian-13-generic-amd64-20260801-2204.qcow2"
|
||||
)
|
||||
DEBIAN_12_GENERICCLOUD = (
|
||||
"https://cloud.debian.org/images/cloud/bookworm/latest/"
|
||||
"debian-12-genericcloud-amd64.qcow2"
|
||||
)
|
||||
UBUNTU_24 = (
|
||||
"https://cloud-images.ubuntu.com/releases/noble/release/"
|
||||
"ubuntu-24.04-server-cloudimg-amd64.img"
|
||||
)
|
||||
UBUNTU_26_CURRENT = (
|
||||
"https://cloud-images.ubuntu.com/resolute/current/"
|
||||
"resolute-server-cloudimg-amd64.img"
|
||||
)
|
||||
CENTOS_STREAM_9 = (
|
||||
"https://cloud.centos.org/centos/9-stream/x86_64/images/"
|
||||
"CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
|
||||
)
|
||||
CENTOS_STREAM_10 = (
|
||||
"https://cloud.centos.org/centos/10-stream/x86_64/images/"
|
||||
"CentOS-Stream-GenericCloud-x86_64-10-20260810.0.x86_64.qcow2"
|
||||
)
|
||||
|
||||
|
||||
class FakeHeaders:
|
||||
def __init__(self, values=None):
|
||||
self.values = values or {}
|
||||
|
||||
def get_all(self, name, default=None):
|
||||
return self.values.get(name, default)
|
||||
|
||||
|
||||
class FakeImageResponse(io.BytesIO):
|
||||
def __init__(self, body=b"", *, status=200, headers=None):
|
||||
super().__init__(body)
|
||||
self.status = status
|
||||
self.header_values = headers or {}
|
||||
self.headers = FakeHeaders(self.header_values)
|
||||
|
||||
def getheader(self, name, default=None):
|
||||
values = self.header_values.get(name)
|
||||
if not values:
|
||||
return default
|
||||
return values[0]
|
||||
|
||||
|
||||
def fake_image_response(body: bytes):
|
||||
return (
|
||||
SimpleNamespace(close=lambda: None),
|
||||
FakeImageResponse(
|
||||
body,
|
||||
headers={"Content-Length": [str(len(body))]},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_official_almalinux_8_9_and_10_are_distinct_supported_versions():
|
||||
assert validate_image_url(OFFICIAL_8) == OFFICIAL_8
|
||||
assert validate_image_url(OFFICIAL_9) == OFFICIAL_9
|
||||
assert image_metadata_for(OFFICIAL_8) == ("AlmaLinux", "8")
|
||||
assert image_version_for(OFFICIAL_9) == "9"
|
||||
assert image_version_for(OFFICIAL) == "10"
|
||||
scope = "a" * 64
|
||||
digest = "b" * 64
|
||||
assert template_id_for(scope, digest, "8").startswith("alma8-")
|
||||
assert template_id_for(scope, digest, "9").startswith("alma9-")
|
||||
assert template_id_for(scope, digest, "10").startswith("alma10-")
|
||||
assert template_volume_name(template_id_for(scope, digest, "9")).startswith(
|
||||
"panel-template-alma9-"
|
||||
)
|
||||
with pytest.raises(TemplateError):
|
||||
validate_image_url(OFFICIAL_9.replace("/almalinux/9/", "/almalinux/10/"))
|
||||
with pytest.raises(TemplateError):
|
||||
validate_image_url(OFFICIAL_9.replace("AlmaLinux-9-", "AlmaLinux-8-"))
|
||||
with pytest.raises(TemplateError):
|
||||
template_id_for(scope, digest, "7")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "family", "version", "prefix"),
|
||||
[
|
||||
(DEBIAN_12, "Debian", "12", "debian12-"),
|
||||
(DEBIAN_12_GENERICCLOUD, "Debian", "12", "debian12-"),
|
||||
(DEBIAN_13_BUILD, "Debian", "13", "debian13-"),
|
||||
(UBUNTU_24, "Ubuntu", "24.04", "ubuntu2404-"),
|
||||
(UBUNTU_26_CURRENT, "Ubuntu", "26.04", "ubuntu2604-"),
|
||||
(CENTOS_STREAM_9, "CentOS Stream", "9", "centosstream9-"),
|
||||
(CENTOS_STREAM_10, "CentOS Stream", "10", "centosstream10-"),
|
||||
],
|
||||
)
|
||||
def test_supported_official_cloud_images_have_distinct_safe_identifiers(
|
||||
url, family, version, prefix):
|
||||
assert validate_image_url(url) == url
|
||||
assert image_metadata_for(url) == (family, version)
|
||||
template_id = template_id_for("a" * 64, "b" * 64, version, family)
|
||||
assert template_id.startswith(prefix)
|
||||
assert template_volume_name(template_id) == (
|
||||
f"panel-template-{template_id}.qcow2"
|
||||
)
|
||||
|
||||
|
||||
def test_template_identifier_rejects_invalid_digest_precisely():
|
||||
with pytest.raises(TemplateError, match="SHA-256 格式"):
|
||||
template_id_for("a" * 64, "not-a-digest", "12", "Debian")
|
||||
with pytest.raises(TemplateError, match="系统版本无效"):
|
||||
template_id_for("a" * 64, "b" * 64, ["12"], "Debian")
|
||||
|
||||
|
||||
def qcow2_fixture(virtual_size=2 * 1024**3) -> bytes:
|
||||
header = bytearray(104)
|
||||
header[:4] = b"QFI\xfb"
|
||||
header[4:8] = struct.pack(">I", 3)
|
||||
header[20:24] = struct.pack(">I", 16)
|
||||
header[24:32] = struct.pack(">Q", virtual_size)
|
||||
header[100:104] = struct.pack(">I", 104)
|
||||
return bytes(header) + b"fake qcow2 content for controlled import test"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
[
|
||||
"http://repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2",
|
||||
"https://evil.example/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2",
|
||||
"https://repo.almalinux.org.evil.example/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2",
|
||||
"https://user:pass@repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2",
|
||||
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/../../etc/passwd",
|
||||
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.1-20260101.x86_64.qcow2?token=secret",
|
||||
"https://cloud.debian.org/images/cloud/bookworm/latest/debian-13-generic-amd64.qcow2",
|
||||
"https://cloud.debian.org/images/cloud/bookworm/20260801-2204/debian-12-generic-amd64-20260802-2205.qcow2",
|
||||
"https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-arm64.qcow2",
|
||||
"https://cloud-images.ubuntu.com/releases/noble/release/ubuntu-22.04-server-cloudimg-amd64.img",
|
||||
"https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img",
|
||||
"https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2",
|
||||
"https://cloud.centos.org/centos/9-stream/aarch64/images/CentOS-Stream-GenericCloud-9-latest.aarch64.qcow2",
|
||||
],
|
||||
)
|
||||
def test_import_url_rejects_nonofficial_and_ambiguous_targets(url):
|
||||
with pytest.raises(TemplateError):
|
||||
validate_image_url(url)
|
||||
|
||||
|
||||
def test_verified_image_is_private_and_has_expected_virtual_size(tmp_path):
|
||||
body = qcow2_fixture()
|
||||
digest = hashlib.sha256(body).hexdigest()
|
||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
||||
return_value=fake_image_response(body)), patch(
|
||||
"virsh_panel.cloud_templates._verify_qcow2_with_qemu"
|
||||
) as checked:
|
||||
downloaded, virtual_size, actual_sha256 = download_verified_image(
|
||||
OFFICIAL, digest, tmp_path
|
||||
)
|
||||
try:
|
||||
assert downloaded.read_bytes() == body
|
||||
assert virtual_size == 2 * 1024**3
|
||||
assert actual_sha256 == digest
|
||||
assert stat.S_IMODE(downloaded.stat().st_mode) == 0o600
|
||||
checked.assert_called_once_with(downloaded, 2 * 1024**3)
|
||||
finally:
|
||||
downloaded.unlink()
|
||||
|
||||
|
||||
def test_image_without_expected_hash_records_computed_digest(tmp_path):
|
||||
body = qcow2_fixture()
|
||||
digest = hashlib.sha256(body).hexdigest()
|
||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
||||
return_value=fake_image_response(body)), patch(
|
||||
"virsh_panel.cloud_templates._verify_qcow2_with_qemu"
|
||||
):
|
||||
downloaded, virtual_size, actual_sha256 = download_verified_image(
|
||||
OFFICIAL, "", tmp_path
|
||||
)
|
||||
try:
|
||||
assert virtual_size == 2 * 1024**3
|
||||
assert actual_sha256 == digest
|
||||
finally:
|
||||
downloaded.unlink()
|
||||
|
||||
|
||||
def test_failed_hash_never_leaves_unverified_template_file(tmp_path):
|
||||
body = qcow2_fixture()
|
||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
||||
return_value=fake_image_response(body)):
|
||||
with pytest.raises(TemplateError, match="SHA-256"):
|
||||
download_verified_image(OFFICIAL, "0" * 64, tmp_path)
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_download_limit_preserves_persistent_disk_reserve(tmp_path):
|
||||
body = qcow2_fixture()
|
||||
digest = hashlib.sha256(body).hexdigest()
|
||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
||||
return_value=fake_image_response(body)):
|
||||
with pytest.raises(TemplateError, match="持久目录空间不足") as error:
|
||||
download_verified_image(
|
||||
OFFICIAL, digest, tmp_path, max_bytes=len(body) - 1
|
||||
)
|
||||
assert error.value.status == 507
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_redirect_to_nonofficial_target_is_rejected_before_following():
|
||||
handler = _RejectRedirects()
|
||||
with pytest.raises(TemplateError, match="重定向"):
|
||||
handler.redirect_request(None, None, 302, "Found", {},
|
||||
"http://127.0.0.1/private")
|
||||
|
||||
|
||||
def test_debian_official_redirect_downloads_same_object_from_public_mirror(
|
||||
tmp_path):
|
||||
mirror = DEBIAN_12.replace(
|
||||
"cloud.debian.org", "chuangtzu.ftp.acc.umu.se"
|
||||
)
|
||||
body = qcow2_fixture()
|
||||
first = FakeImageResponse(
|
||||
status=302, headers={"Location": [mirror]}
|
||||
)
|
||||
final = FakeImageResponse(
|
||||
body,
|
||||
status=200,
|
||||
headers={"Content-Length": [str(len(body))]},
|
||||
)
|
||||
connections = [SimpleNamespace(close=lambda: None) for _ in range(2)]
|
||||
with patch(
|
||||
"virsh_panel.cloud_templates._resolve_global_addresses",
|
||||
return_value=["2001:4860:4860::8888"],
|
||||
), patch(
|
||||
"virsh_panel.cloud_templates._open_pinned_response",
|
||||
side_effect=[(connections[0], first), (connections[1], final)],
|
||||
) as opened, patch(
|
||||
"virsh_panel.cloud_templates._verify_qcow2_with_qemu"
|
||||
):
|
||||
downloaded, virtual_size, digest = download_verified_image(
|
||||
DEBIAN_12, hashlib.sha256(body).hexdigest(), tmp_path
|
||||
)
|
||||
|
||||
assert downloaded.read_bytes() == body
|
||||
assert virtual_size == 2 * 1024**3
|
||||
assert digest == hashlib.sha256(body).hexdigest()
|
||||
assert opened.call_args_list[0].args[0].hostname == "cloud.debian.org"
|
||||
assert opened.call_args_list[1].args[0].hostname == "chuangtzu.ftp.acc.umu.se"
|
||||
assert opened.call_args_list[1].args[0].path == urlsplit(DEBIAN_12).path
|
||||
downloaded.unlink()
|
||||
|
||||
|
||||
def test_ubuntu_redirect_stays_on_official_host_and_same_release():
|
||||
target = UBUNTU_24.replace(
|
||||
"/releases/noble/release/", "/releases/24.04/release/"
|
||||
)
|
||||
first = FakeImageResponse(
|
||||
status=302, headers={"Location": [UBUNTU_24]}
|
||||
)
|
||||
final = FakeImageResponse(b"image", status=200)
|
||||
connections = [SimpleNamespace(close=lambda: None) for _ in range(2)]
|
||||
with patch(
|
||||
"virsh_panel.cloud_templates._resolve_global_addresses",
|
||||
return_value=["8.8.8.8"],
|
||||
), patch(
|
||||
"virsh_panel.cloud_templates._open_pinned_response",
|
||||
side_effect=[(connections[0], first), (connections[1], final)],
|
||||
):
|
||||
connection, response = _open_image_response(target)
|
||||
|
||||
assert connection is connections[1]
|
||||
assert response is final
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
"http://127.0.0.1/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2",
|
||||
"https://mirror.example/images/cloud/bookworm/latest/debian-13-generic-amd64.qcow2",
|
||||
"https://user:password@mirror.example/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2",
|
||||
],
|
||||
)
|
||||
def test_debian_redirect_rejects_unsafe_or_different_object(target):
|
||||
with pytest.raises(TemplateError, match="重定向目标不安全"):
|
||||
_redirect_target(
|
||||
DEBIAN_12, DEBIAN_12, target, ("Debian", "12")
|
||||
)
|
||||
|
||||
|
||||
def test_ubuntu_redirect_rejects_other_host_or_release():
|
||||
for target in (
|
||||
UBUNTU_24.replace("cloud-images.ubuntu.com", "mirror.example"),
|
||||
UBUNTU_24.replace("24.04", "22.04"),
|
||||
):
|
||||
with pytest.raises(TemplateError, match="重定向目标不安全"):
|
||||
_redirect_target(
|
||||
UBUNTU_24, UBUNTU_24, target, ("Ubuntu", "24.04")
|
||||
)
|
||||
|
||||
|
||||
def test_image_download_dns_rejects_any_nonpublic_answer():
|
||||
rows = [
|
||||
(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "",
|
||||
("8.8.8.8", 443)),
|
||||
(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "",
|
||||
("127.0.0.1", 443)),
|
||||
]
|
||||
with patch("virsh_panel.cloud_templates.socket.getaddrinfo", return_value=rows):
|
||||
with pytest.raises(TemplateError, match="非公网"):
|
||||
_resolve_global_addresses("mirror.example")
|
||||
|
||||
|
||||
def test_image_redirect_count_is_bounded():
|
||||
def redirect_response(*_args):
|
||||
connection = SimpleNamespace(close=lambda: None)
|
||||
response = FakeImageResponse(
|
||||
status=302, headers={"Location": [UBUNTU_24]}
|
||||
)
|
||||
return connection, response
|
||||
|
||||
with patch(
|
||||
"virsh_panel.cloud_templates._resolve_global_addresses",
|
||||
return_value=["8.8.8.8"],
|
||||
), patch(
|
||||
"virsh_panel.cloud_templates._open_pinned_response",
|
||||
side_effect=redirect_response,
|
||||
):
|
||||
with pytest.raises(TemplateError, match="次数过多"):
|
||||
_open_image_response(UBUNTU_24)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("response", "message"),
|
||||
[
|
||||
(FakeImageResponse(status=404), "异常状态"),
|
||||
(FakeImageResponse(
|
||||
qcow2_fixture(), headers={"Content-Encoding": ["gzip"]}
|
||||
), "响应格式无效"),
|
||||
(FakeImageResponse(
|
||||
qcow2_fixture(),
|
||||
headers={"Content-Length": [str(len(qcow2_fixture()) + 1)]},
|
||||
), "内容不完整"),
|
||||
],
|
||||
)
|
||||
def test_image_download_rejects_unsafe_or_incomplete_http_response(
|
||||
tmp_path, response, message):
|
||||
connection = SimpleNamespace(close=lambda: None)
|
||||
with patch(
|
||||
"virsh_panel.cloud_templates._open_image_response",
|
||||
return_value=(connection, response),
|
||||
):
|
||||
with pytest.raises(TemplateError, match=message):
|
||||
download_verified_image(OFFICIAL, "", tmp_path)
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_external_data_file_feature_is_rejected(tmp_path):
|
||||
body = bytearray(qcow2_fixture())
|
||||
body[72:80] = struct.pack(">Q", 4)
|
||||
digest = hashlib.sha256(body).hexdigest()
|
||||
with patch("virsh_panel.cloud_templates._open_image_response",
|
||||
return_value=fake_image_response(body)):
|
||||
with pytest.raises(TemplateError, match="外部依赖"):
|
||||
download_verified_image(OFFICIAL, digest, tmp_path)
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_qcow2_full_integrity_check_rejects_corrupt_image(tmp_path):
|
||||
image = tmp_path / "corrupt.qcow2"
|
||||
image.write_bytes(qcow2_fixture())
|
||||
info = [{"format": "qcow2", "virtual-size": 2 * 1024**3}]
|
||||
with patch(
|
||||
"virsh_panel.cloud_templates._qemu_img_json",
|
||||
side_effect=[info, TemplateError("云镜像完整性检查失败", 422)],
|
||||
) as inspect:
|
||||
with pytest.raises(TemplateError, match="完整性检查"):
|
||||
verify_qcow2_image(image)
|
||||
assert inspect.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"info",
|
||||
[
|
||||
[{"format": "raw", "virtual-size": 2 * 1024**3}],
|
||||
[{"format": "qcow2", "virtual-size": 2 * 1024**3,
|
||||
"backing-filename": "/private/base.qcow2"}],
|
||||
[{"format": "qcow2", "virtual-size": 2 * 1024**3,
|
||||
"format-specific": {"data": {"data-file": "/private/data"}}}],
|
||||
[{"format": "qcow2", "virtual-size": 2 * 1024**3},
|
||||
{"format": "qcow2", "virtual-size": 2 * 1024**3}],
|
||||
],
|
||||
)
|
||||
def test_qcow2_full_integrity_check_rejects_dependencies_and_wrong_format(
|
||||
tmp_path, info):
|
||||
image = tmp_path / "unsafe.qcow2"
|
||||
image.write_bytes(qcow2_fixture())
|
||||
with patch("virsh_panel.cloud_templates._qemu_img_json",
|
||||
return_value=info) as inspect:
|
||||
with pytest.raises(TemplateError, match="外部依赖"):
|
||||
verify_qcow2_image(image)
|
||||
inspect.assert_called_once()
|
||||
|
||||
|
||||
def test_template_catalog_is_private_and_scoped_to_host_and_disk_pool(tmp_path):
|
||||
data_dir = tmp_path / "data"
|
||||
data_dir.mkdir(mode=0o700)
|
||||
store = TemplateStore(data_dir)
|
||||
profile = {
|
||||
"host": "hypervisor.example", "port": 2222,
|
||||
"fingerprint": "SHA256:fixture", "storage_pool": "/home/kvm",
|
||||
}
|
||||
scope = template_scope(profile)
|
||||
other_scope = template_scope({**profile, "storage_pool": "/other/pool"})
|
||||
record = {
|
||||
"id": "alma10-example", "name": "AlmaLinux 10", "sha256": "a" * 64,
|
||||
"volume_name": "alma10-cloud.qcow2", "virtual_size": 2 * 1024**3,
|
||||
}
|
||||
assert store.add(scope, record) == record
|
||||
assert store.list(scope) == [record]
|
||||
assert store.list(other_scope) == []
|
||||
assert TemplateStore(data_dir).get(scope, record["id"]) == record
|
||||
assert stat.S_IMODE(store.file.stat().st_mode) == 0o600
|
||||
with pytest.raises(TemplateError):
|
||||
store.get(other_scope, record["id"])
|
||||
with pytest.raises(TemplateError):
|
||||
store.add(scope, record)
|
||||
|
||||
|
||||
def test_template_rename_preserves_image_record_and_host_scope(tmp_path):
|
||||
data_dir = tmp_path / "data"
|
||||
data_dir.mkdir(mode=0o700)
|
||||
store = TemplateStore(data_dir)
|
||||
scope = "a" * 64
|
||||
other_scope = "b" * 64
|
||||
template_id = template_id_for(scope, "c" * 64, "9")
|
||||
record = {
|
||||
"id": template_id, "name": "AlmaLinux 9", "os_family": "AlmaLinux",
|
||||
"os_version": "9", "sha256": "c" * 64,
|
||||
"volume_name": template_volume_name(template_id), "virtual_size": 2 * 1024**3,
|
||||
}
|
||||
store.add(scope, record)
|
||||
store.add(scope, {**record, "id": "other", "name": "Another name"})
|
||||
store.add(other_scope, {**record, "name": "Other host"})
|
||||
|
||||
with pytest.raises(TemplateError) as inaccessible:
|
||||
store.rename(other_scope, "other", "Missing")
|
||||
assert inaccessible.value.status == 404
|
||||
with pytest.raises(TemplateError) as duplicate:
|
||||
store.rename(scope, template_id, "another NAME")
|
||||
assert duplicate.value.status == 409
|
||||
for invalid in ("", "x" * 65, "bad\x7fentry"):
|
||||
with pytest.raises(TemplateError):
|
||||
store.rename(scope, template_id, invalid)
|
||||
|
||||
updated = store.rename(scope, template_id, " 新名称 ")
|
||||
assert updated == {**record, "name": "新名称"}
|
||||
assert TemplateStore(data_dir).get(scope, template_id) == updated
|
||||
assert store.get(other_scope, template_id)["name"] == "Other host"
|
||||
assert stat.S_IMODE(store.file.stat().st_mode) == 0o600
|
||||
@ -1,378 +0,0 @@
|
||||
"""Connection admission failures must not become permanent mutation fences.
|
||||
|
||||
Exercise the signed engine and real VMService/coordination boundaries. Only
|
||||
SSH staging, remote objects and the RPC connection transport are simulated.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from test_app import FakeRemoteManager, HOST_BODY, VM_ID
|
||||
from test_engine_boundary import MemoryStore, SECRET, SUPER, call
|
||||
from test_host_service_scope import StagedManager
|
||||
from virsh_panel.app import create_app
|
||||
from virsh_panel.config import Settings
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationError, DurableCoordinator, connection_failure_before_write,
|
||||
current_execution, execution_context, mark_remote_write_started,
|
||||
resource_guard, update_current_claim,
|
||||
)
|
||||
from virsh_panel.database import DatabaseTemplateStore
|
||||
from virsh_panel.durable_jobs import submit_durable
|
||||
from virsh_panel.engine_context import EngineComponents, ExternalOwnership, HostStore
|
||||
from virsh_panel.internal import EngineApplication, HostRegistry, json_response
|
||||
from virsh_panel.libvirt_rpc import LibvirtRuntime, RPCSession
|
||||
from virsh_panel.libvirt_service import GIB, VMError, VMService
|
||||
from virsh_panel.metrics import MetricsCollector
|
||||
from virsh_panel.remote_host import RemoteHostError
|
||||
import virsh_panel.libvirt_service as service_module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def signed_host(tmp_path, monkeypatch):
|
||||
opened, managers = [], {}
|
||||
behavior = SimpleNamespace(fail_open=True, fail_after_write=False)
|
||||
|
||||
def open_session(uri, scope, *, timeout):
|
||||
opened.append(scope)
|
||||
if behavior.fail_open:
|
||||
raise service_module.libvirt.libvirtError("connection refused before open")
|
||||
connection = SimpleNamespace(networkLookupByName=lambda _name:
|
||||
SimpleNamespace(isActive=lambda: True))
|
||||
return SimpleNamespace(connection=connection, failure=None, close=lambda: None)
|
||||
|
||||
runtime = SimpleNamespace(open=open_session, queue_timeout=0.02)
|
||||
monkeypatch.setattr("virsh_panel.app.get_runtime", lambda: runtime)
|
||||
monkeypatch.setattr(service_module, "get_runtime", lambda: runtime)
|
||||
monkeypatch.setattr(MetricsCollector, "start", lambda _self: None)
|
||||
|
||||
def pool(service, conn, name, *_args):
|
||||
if behavior.fail_after_write:
|
||||
mark_remote_write_started()
|
||||
# A completed first session's write is still part of this request.
|
||||
behavior.fail_open = True
|
||||
with service._connect():
|
||||
pytest.fail("second connection should fail")
|
||||
return SimpleNamespace(name=lambda: name, listVolumes=lambda: [],
|
||||
info=lambda: (0, 100 * GIB, 0, 100 * GIB))
|
||||
|
||||
monkeypatch.setattr(VMService, "_active_pool", pool)
|
||||
monkeypatch.setattr(VMService, "_ensure_pool", pool)
|
||||
monkeypatch.setattr(VMService, "_refresh_directory_pool", lambda *_args: None)
|
||||
monkeypatch.setattr(VMService, "_same_pool_location", lambda *_args: False)
|
||||
store, logs = MemoryStore(), MemoryStore()
|
||||
config = {"service_secret": base64.b64encode(SECRET).decode(),
|
||||
"data_dir": str(tmp_path)}
|
||||
|
||||
def factory(host_id, user_isos, cloud_templates):
|
||||
directory = tmp_path / host_id
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
scoped = HostStore(store, host_id)
|
||||
manager = managers[host_id] = StagedManager()
|
||||
manager.host_id = host_id
|
||||
components = EngineComponents(
|
||||
SimpleNamespace(users=ExternalOwnership(scoped)), manager,
|
||||
DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates)
|
||||
return components, create_app(Settings(data_dir=directory),
|
||||
engine_components=components)
|
||||
|
||||
registry = HostRegistry(config, store, logs, factory=factory)
|
||||
engine = EngineApplication(config, store, logs, registry=registry)
|
||||
with TestClient(engine, base_url="http://127.0.0.1") as client:
|
||||
yield SimpleNamespace(client=client, engine=engine, behavior=behavior,
|
||||
opened=opened, managers=managers)
|
||||
|
||||
|
||||
def test_signed_host_save_connection_failure_releases_claims_and_allows_retry(signed_host):
|
||||
host = signed_host
|
||||
path = "/internal/v1/hosts/7/host"
|
||||
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
|
||||
assert response.status_code == 502, response.text
|
||||
assert response.json()["error_code"] == "libvirt_connection_failed"
|
||||
assert host.managers["7"].current() is None
|
||||
assert [row[0] for row in host.managers["7"].calls] == ["stage", "test", "abort"]
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
# No orphan request/resource claim may block maintenance either.
|
||||
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
assert host.engine.coordinator.release(maintenance)
|
||||
|
||||
host.behavior.fail_open = False
|
||||
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
|
||||
assert response.status_code == 200, response.text
|
||||
assert host.managers["7"].current()["host_id"] == "7"
|
||||
assert host.opened == ["7", "7", "7"]
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("receipt_lost", [False, True])
|
||||
def test_signed_ssh_probe_failure_requires_durable_prestart_proof(
|
||||
signed_host, monkeypatch, receipt_lost):
|
||||
host = signed_host
|
||||
path = "/internal/v1/hosts/7/host"
|
||||
assert call(host.client, "GET", path, SUPER).status_code == 200
|
||||
manager = host.managers["7"]
|
||||
original_probe = manager.test
|
||||
|
||||
def refused(_staged):
|
||||
raise RemoteHostError("SSH probe connection refused", 502)
|
||||
|
||||
monkeypatch.setattr(manager, "test", refused)
|
||||
if receipt_lost:
|
||||
def cannot_record(*_args, **_kwargs):
|
||||
raise CoordinationError("coordination_unavailable", 503)
|
||||
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
|
||||
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
|
||||
assert response.status_code == (503 if receipt_lost else 502), response.text
|
||||
assert host.opened == []
|
||||
assert manager.current() is None
|
||||
if receipt_lost:
|
||||
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
||||
with pytest.raises(CoordinationError):
|
||||
host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
return
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
assert host.engine.coordinator.release(maintenance)
|
||||
monkeypatch.setattr(manager, "test", original_probe)
|
||||
host.behavior.fail_open = False
|
||||
assert call(host.client, "PUT", path, SUPER, HOST_BODY).status_code == 200
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method,suffix", [("PUT", "host"), ("POST", "host/test")])
|
||||
@pytest.mark.parametrize("receipt_lost", [False, True])
|
||||
def test_signed_stage_scan_failure_requires_durable_prestart_proof(
|
||||
signed_host, monkeypatch, method, suffix, receipt_lost):
|
||||
host = signed_host
|
||||
assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200
|
||||
manager = host.managers["7"]
|
||||
original_stage = manager.stage
|
||||
|
||||
def refused(*_args, **_kwargs):
|
||||
raise RemoteHostError("keyscan failed during staging", 502)
|
||||
|
||||
monkeypatch.setattr(manager, "stage", refused)
|
||||
if receipt_lost:
|
||||
def cannot_record(*_args, **_kwargs):
|
||||
raise CoordinationError("coordination_unavailable", 503)
|
||||
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
|
||||
path = f"/internal/v1/hosts/7/{suffix}"
|
||||
response = call(host.client, method, path, SUPER, HOST_BODY)
|
||||
assert response.status_code == (503 if receipt_lost else 502), response.text
|
||||
assert host.opened == []
|
||||
assert manager.current() is None
|
||||
if receipt_lost:
|
||||
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
||||
with pytest.raises(CoordinationError):
|
||||
host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
return
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
assert host.engine.coordinator.release(maintenance)
|
||||
monkeypatch.setattr(manager, "stage", original_stage)
|
||||
host.behavior.fail_open = False
|
||||
assert call(host.client, method, path, SUPER, HOST_BODY).status_code == 200
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("global_route", [False, True], ids=["host-scan", "discovery-scan"])
|
||||
@pytest.mark.parametrize("receipt_lost", [False, True])
|
||||
def test_signed_keyscan_failure_requires_durable_prestart_proof(
|
||||
signed_host, monkeypatch, global_route, receipt_lost):
|
||||
host = signed_host
|
||||
if global_route:
|
||||
manager = FakeRemoteManager()
|
||||
manager.close = lambda: None
|
||||
monkeypatch.setattr("virsh_panel.remote_host.RemoteHostManager", lambda *_args: manager)
|
||||
path = "/internal/v1/hosts/scan"
|
||||
else:
|
||||
assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200
|
||||
manager = host.managers["7"]
|
||||
path = "/internal/v1/hosts/7/host/scan"
|
||||
original_scan = manager.scan
|
||||
|
||||
def refused(*_args, **_kwargs):
|
||||
raise RemoteHostError("keyscan timed out", 502)
|
||||
|
||||
monkeypatch.setattr(manager, "scan", refused)
|
||||
if receipt_lost:
|
||||
def cannot_record(*_args, **_kwargs):
|
||||
raise CoordinationError("coordination_unavailable", 503)
|
||||
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
|
||||
body = {"host": HOST_BODY["host"], "port": HOST_BODY["port"]}
|
||||
response = call(host.client, "POST", path, SUPER, body)
|
||||
# The global discovery route preserves its generic remote-error 503.
|
||||
assert response.status_code == (503 if receipt_lost or global_route else 502), response.text
|
||||
assert host.opened == []
|
||||
if receipt_lost:
|
||||
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
||||
with pytest.raises(CoordinationError):
|
||||
host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
return
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
assert host.engine.coordinator.release(maintenance)
|
||||
monkeypatch.setattr(manager, "scan", original_scan)
|
||||
assert call(host.client, "POST", path, SUPER, body).status_code == 200
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
def test_signed_host_save_later_connection_failure_retains_prior_writes(signed_host):
|
||||
host = signed_host
|
||||
host.behavior.fail_open = False
|
||||
host.behavior.fail_after_write = True
|
||||
response = call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER, HOST_BODY)
|
||||
assert response.status_code == 502, response.text
|
||||
claims = host.engine.coordinator.snapshot().values()
|
||||
assert {row["kind"] for row in claims if row["state"] == "uncertain"} == {
|
||||
"request", "resource"}
|
||||
assert any("storage:7" in row["resources"] for row in claims)
|
||||
with pytest.raises(CoordinationError):
|
||||
host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
|
||||
|
||||
def test_signed_vm_connection_failure_does_not_keep_the_vm_locked(signed_host):
|
||||
host = signed_host
|
||||
host.behavior.fail_open = False
|
||||
assert call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER,
|
||||
HOST_BODY).status_code == 200
|
||||
host.behavior.fail_open = True
|
||||
response = call(host.client, "POST", f"/internal/v1/hosts/7/vms/{VM_ID}/actions",
|
||||
{**SUPER, "operation_id": "start-connection-one"}, {"action": "start"})
|
||||
assert response.status_code == 502, response.text
|
||||
assert response.json()["status"] == "failed"
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
next_operation = host.engine.coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
assert host.engine.coordinator.release(next_operation)
|
||||
|
||||
|
||||
def test_mixed_ssh_request_does_not_use_missing_rpc_marker_as_proof(signed_host, monkeypatch):
|
||||
host = signed_host
|
||||
calls = []
|
||||
|
||||
async def mixed_handler(scope, receive, send, body, delegated, **_kwargs):
|
||||
calls.append("SSH helper changed host")
|
||||
assert not current_execution().connection_failure_safe
|
||||
await json_response(send, 502, {"error_code": "libvirt_connection_failed"})
|
||||
|
||||
monkeypatch.setattr(host.engine, "_dispatch_regular", mixed_handler)
|
||||
response = call(host.client, "POST", "/internal/v1/vpcs/apply", SUPER, {})
|
||||
assert response.status_code == 502
|
||||
assert calls == ["SSH helper changed host"]
|
||||
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
||||
|
||||
|
||||
def test_service_guard_cannot_relax_a_mixed_workflows_unknown_side_effects(monkeypatch):
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
parent = coordinator.claim()
|
||||
|
||||
def refused(*_args, **_kwargs):
|
||||
raise service_module.libvirt.libvirtError("connection refused")
|
||||
|
||||
monkeypatch.setattr(service_module, "get_runtime", lambda:
|
||||
SimpleNamespace(open=refused, queue_timeout=0.02))
|
||||
service = VMService("test", "images", "iso", "default", execution_scope="7")
|
||||
with execution_context(coordinator, parent):
|
||||
with pytest.raises(VMError):
|
||||
service.ensure_storage_pools()
|
||||
assert any(row["state"] == "uncertain" and row["resources"] == ["storage:7"]
|
||||
for row in coordinator.snapshot().values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", ["libvirt_connection_failed", "libvirt_execution_timeout",
|
||||
"libvirt_execution_stopped", "libvirt_execution_uncertain"])
|
||||
def test_nested_connection_failures_never_release_earlier_session_writes(code):
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
with execution_context(coordinator, parent, connection_failure_safe=True):
|
||||
with resource_guard(("storage:7",)):
|
||||
mark_remote_write_started()
|
||||
updated = coordinator.add_resources(parent, ("allocation:7",))
|
||||
update_current_claim(updated)
|
||||
assert not connection_failure_before_write("libvirt_connection_failed")
|
||||
with pytest.raises(VMError):
|
||||
with resource_guard(("storage:7",), connection_failure_safe=True):
|
||||
raise VMError("later session failed", 502, error_code=code)
|
||||
assert any(row["kind"] == "resource" and row["state"] == "uncertain"
|
||||
for row in coordinator.snapshot().values())
|
||||
|
||||
|
||||
def test_background_write_evidence_returns_to_parent_thread_context():
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
with execution_context(coordinator, parent, connection_failure_safe=True):
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = submit_durable(executor, mark_remote_write_started,
|
||||
outcome=lambda: {"status": "completed"},
|
||||
job_kind="snapshot_job", job_id="one")
|
||||
future.result(timeout=2)
|
||||
assert not connection_failure_before_write("libvirt_connection_failed")
|
||||
assert current_execution().remote_write_started.is_set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("earlier_write", [False, True])
|
||||
def test_background_terminal_connection_failure_preserves_actual_execution_evidence(
|
||||
monkeypatch, earlier_write):
|
||||
store = MemoryStore()
|
||||
coordinator = DurableCoordinator(store)
|
||||
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
|
||||
def refused(*_args, **_kwargs):
|
||||
raise service_module.libvirt.libvirtError("connection refused")
|
||||
|
||||
monkeypatch.setattr(service_module, "get_runtime", lambda:
|
||||
SimpleNamespace(open=refused, queue_timeout=0.02))
|
||||
service = VMService("test", "images", "iso", "default", execution_scope="7")
|
||||
|
||||
def job():
|
||||
if earlier_write:
|
||||
mark_remote_write_started()
|
||||
try:
|
||||
service.ensure_storage_pools()
|
||||
except VMError as exc:
|
||||
store.set("reinstall_job", "one", json.dumps({
|
||||
"status": "failed", "error_code": exc.error_code}))
|
||||
|
||||
with execution_context(coordinator, parent, connection_failure_safe=True):
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
submit_durable(executor, job,
|
||||
outcome=lambda: json.loads(store.get("reinstall_job", "one")),
|
||||
job_kind="reinstall_job", job_id="one").result(timeout=2)
|
||||
coordinator.release(parent)
|
||||
claims = list(coordinator.snapshot().values())
|
||||
if earlier_write:
|
||||
assert {row["kind"] for row in claims if row["state"] == "uncertain"} == {
|
||||
"background", "resource"}
|
||||
else:
|
||||
assert claims == []
|
||||
|
||||
|
||||
def test_new_execution_does_not_inherit_another_requests_write_evidence():
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
first, second = coordinator.claim(), coordinator.claim()
|
||||
with execution_context(coordinator, first, connection_failure_safe=True):
|
||||
mark_remote_write_started()
|
||||
with execution_context(coordinator, second, connection_failure_safe=True):
|
||||
assert connection_failure_before_write("libvirt_connection_failed")
|
||||
assert not connection_failure_before_write("libvirt_connection_failed")
|
||||
|
||||
|
||||
def test_rpc_intent_hook_marks_parent_before_a_write(tmp_path):
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
parent = coordinator.claim()
|
||||
runtime = LibvirtRuntime(tmp_path / "intents")
|
||||
session = RPCSession(runtime, "test:///default", "7", 5)
|
||||
with execution_context(coordinator, parent, connection_failure_safe=True):
|
||||
sequence = runtime._before_write(session, "create")
|
||||
assert session.intent.is_file()
|
||||
assert current_execution().remote_write_started.is_set()
|
||||
runtime._ack_write(session, sequence)
|
||||
assert session.intent is None
|
||||
# The RPC reply clears only its pending receipt, never the enclosing
|
||||
# workflow's evidence that a remote write has already happened.
|
||||
assert current_execution().remote_write_started.is_set()
|
||||
@ -1,16 +0,0 @@
|
||||
"""A coordinator CAS failure must preserve the same VM recovery evidence as RPC loss."""
|
||||
|
||||
import pytest
|
||||
|
||||
from test_app import web # noqa: F401
|
||||
from test_app_shutdown import (
|
||||
test_reinstall_interruption_preserves_password_and_guard as _check_reinstall,
|
||||
test_rescue_interruption_preserves_original_xml_and_guard as _check_rescue,
|
||||
test_snapshot_interruption_preserves_catalog_and_guard as _check_snapshot,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_code", ["coordination_uncertain", "coordination_unavailable"])
|
||||
@pytest.mark.parametrize("check", [_check_reinstall, _check_rescue, _check_snapshot])
|
||||
def test_coordination_failure_preserves_durable_job_evidence(web, error_code, check):
|
||||
check(web, error_code)
|
||||
@ -1,427 +0,0 @@
|
||||
"""Cross-process admission using the production encrypted CAS implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import replace
|
||||
import json
|
||||
import multiprocessing
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
import pytest
|
||||
|
||||
from virsh_panel import coordination
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationError, DurableCoordinator, RECORD_KEY, RECORD_KIND,
|
||||
configure_default_coordinator, current_execution, execution_context,
|
||||
get_default_coordinator, resource_guard, update_current_claim,
|
||||
)
|
||||
from virsh_panel.database import DatabaseStore, pymysql
|
||||
|
||||
|
||||
class SQLiteCursor:
|
||||
def __init__(self, raw):
|
||||
self.raw = raw.cursor()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
self.raw.close()
|
||||
|
||||
def execute(self, statement, values=()):
|
||||
try:
|
||||
return self.raw.execute(statement.replace("%s", "?").replace(" FOR UPDATE", ""), values)
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise pymysql.err.IntegrityError(1062, "duplicate test row") from exc
|
||||
|
||||
def fetchone(self):
|
||||
row = self.raw.fetchone()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
def fetchall(self):
|
||||
return [dict(row) for row in self.raw.fetchall()]
|
||||
|
||||
|
||||
class SQLiteConnection:
|
||||
def __init__(self, path):
|
||||
self.raw = sqlite3.connect(path, timeout=5.0, isolation_level=None)
|
||||
self.raw.row_factory = sqlite3.Row
|
||||
# The SQLite writer lock substitutes for MariaDB's row lock. Production
|
||||
# get/insert/CAS/encryption methods execute unchanged.
|
||||
self.raw.execute("BEGIN IMMEDIATE")
|
||||
|
||||
def cursor(self):
|
||||
return SQLiteCursor(self.raw)
|
||||
|
||||
def commit(self):
|
||||
self.raw.commit()
|
||||
|
||||
def rollback(self):
|
||||
self.raw.rollback()
|
||||
|
||||
def close(self):
|
||||
self.raw.close()
|
||||
|
||||
|
||||
class EncryptedSQLiteStore(DatabaseStore):
|
||||
RECORD_TABLE = "engine_records"
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = str(path)
|
||||
self._cipher = AESGCM(b"a" * 32)
|
||||
with sqlite3.connect(self.path, timeout=5.0) as connection:
|
||||
connection.execute("CREATE TABLE IF NOT EXISTS engine_records ("
|
||||
"record_type TEXT, record_key TEXT, encrypted_value BLOB, "
|
||||
"PRIMARY KEY(record_type,record_key))")
|
||||
|
||||
def _connect(self):
|
||||
return SQLiteConnection(self.path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def coordinator(tmp_path):
|
||||
return DurableCoordinator(EncryptedSQLiteStore(tmp_path / "state.sqlite"))
|
||||
|
||||
|
||||
def _claim_in_process(path, options, ready, release, output):
|
||||
instance = DurableCoordinator(EncryptedSQLiteStore(path))
|
||||
ready.wait(timeout=10)
|
||||
try:
|
||||
claim = instance.claim(**options)
|
||||
except CoordinationError as exc:
|
||||
output.put(("rejected", exc.error_code))
|
||||
else:
|
||||
output.put(("admitted", claim.token))
|
||||
release.wait(timeout=10)
|
||||
instance.release(claim)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first,second,admitted", [
|
||||
({"resources": ("vm:a:one",)}, {"resources": ("vm:a:one",)}, 1),
|
||||
({"resources": ("vm:a:one",)}, {"resources": ("vm:b:one",)}, 2),
|
||||
({"operation_id": "operation-a", "resources": ("vm:a:one",)},
|
||||
{"operation_id": "operation-a", "resources": ("vm:b:one",)}, 1),
|
||||
({"resources": ("vm:a:one",)}, {"maintenance": True}, 1),
|
||||
])
|
||||
def test_independent_processes_share_atomic_admission(tmp_path, first, second, admitted):
|
||||
path = tmp_path / "state.sqlite"
|
||||
EncryptedSQLiteStore(path)
|
||||
context = multiprocessing.get_context("spawn")
|
||||
ready, release, output = context.Barrier(2), context.Event(), context.Queue()
|
||||
processes = [context.Process(target=_claim_in_process,
|
||||
args=(str(path), options, ready, release, output))
|
||||
for options in (first, second)]
|
||||
try:
|
||||
for process in processes:
|
||||
process.start()
|
||||
results = [output.get(timeout=10) for _ in processes]
|
||||
assert sum(result[0] == "admitted" for result in results) == admitted
|
||||
observer = DurableCoordinator(EncryptedSQLiteStore(path))
|
||||
assert len(observer.snapshot()) == admitted
|
||||
finally:
|
||||
release.set()
|
||||
for process in processes:
|
||||
process.join(timeout=5)
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join(timeout=2)
|
||||
output.close()
|
||||
output.join_thread()
|
||||
assert all(process.exitcode == 0 for process in processes)
|
||||
assert observer.snapshot() == {}
|
||||
|
||||
|
||||
def test_same_execution_handoff_preserves_operation_until_last_claim(coordinator):
|
||||
request = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",))
|
||||
job = coordinator.claim(operation_id="operation-one", resources=request.resources,
|
||||
execution_id=request.execution_id, kind="background")
|
||||
assert coordinator.release(request)
|
||||
assert coordinator.operation_claim("operation-one")["token"] == job.token
|
||||
assert coordinator.retain(job, "libvirt_execution_uncertain")
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
coordinator.claim(resources=job.resources, execution_id=job.execution_id)
|
||||
assert raised.value.error_code == "coordination_uncertain"
|
||||
assert coordinator.release(job) is False
|
||||
|
||||
|
||||
def test_operation_lookup_prefers_unknown_sibling(coordinator):
|
||||
request = coordinator.claim(operation_id="operation-one")
|
||||
job = coordinator.claim(operation_id="operation-one", execution_id=request.execution_id)
|
||||
coordinator.retain(job)
|
||||
assert coordinator.operation_claim("operation-one")["state"] == "uncertain"
|
||||
assert coordinator.release(request)
|
||||
assert coordinator.operation_claim("operation-one")["token"] == job.token
|
||||
|
||||
|
||||
def test_maintenance_never_ignores_its_own_execution(coordinator):
|
||||
request = coordinator.claim(resources=("vm:a:one",))
|
||||
with pytest.raises(CoordinationError):
|
||||
coordinator.claim(maintenance=True, execution_id=request.execution_id)
|
||||
coordinator.release(request)
|
||||
maintenance = coordinator.claim(maintenance=True)
|
||||
with pytest.raises(CoordinationError):
|
||||
coordinator.claim(resources=("vm:a:two",), execution_id=maintenance.execution_id)
|
||||
assert coordinator.release(maintenance)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("uncertain", [False, True])
|
||||
def test_restart_and_old_timestamps_never_grant_execution(coordinator, monkeypatch, uncertain):
|
||||
claim = coordinator.claim(operation_id="old-operation", resources=("vm:a:one",))
|
||||
if uncertain:
|
||||
coordinator.retain(claim)
|
||||
monkeypatch.setattr(coordination.time, "time", lambda: 99999999999)
|
||||
restarted = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path))
|
||||
assert restarted.has_work()
|
||||
with pytest.raises(CoordinationError):
|
||||
restarted.claim(resources=claim.resources)
|
||||
with pytest.raises(CoordinationError):
|
||||
restarted.claim(maintenance=True)
|
||||
assert restarted.snapshot()[claim.token]["owner_id"] == claim.owner_id
|
||||
|
||||
|
||||
def test_identity_and_resolution_prevent_stale_release(coordinator):
|
||||
first = coordinator.claim(resources=("vm:a:one",))
|
||||
assert coordinator.release(replace(first, owner_id="z" * 32)) is False
|
||||
coordinator.retain(first)
|
||||
assert coordinator.release(first) is False
|
||||
with pytest.raises(ValueError):
|
||||
coordinator.resolve(first, " ")
|
||||
assert coordinator.resolve(first, "review/host-a/verified-completed")
|
||||
second = coordinator.claim(resources=first.resources)
|
||||
assert coordinator.release(first) is False
|
||||
assert coordinator.resolve(first, "old review") is False
|
||||
assert second.token in coordinator.snapshot()
|
||||
document = json.loads(coordinator.store.get(RECORD_KIND, RECORD_KEY))
|
||||
assert document["resolutions"][-1]["token"] == first.token
|
||||
|
||||
|
||||
def test_refresh_metadata_and_resource_expansion_preserve_identity(coordinator, monkeypatch):
|
||||
claim = coordinator.claim(operation_id="operation-one", metadata={"phase": "queued"})
|
||||
original = coordinator.snapshot()[claim.token]
|
||||
monkeypatch.setattr(coordination.time, "time", lambda: original["updated_at"] + 60)
|
||||
expanded = coordinator.add_resources(claim, ("vm:a:one",))
|
||||
assert expanded.token == claim.token and expanded.execution_id == claim.execution_id
|
||||
assert expanded.resources == ("vm:a:one",)
|
||||
assert coordinator.refresh(expanded)
|
||||
assert coordinator.update_metadata(expanded, {"job_id": "maintenance-job"})
|
||||
coordinator.retain(expanded)
|
||||
assert coordinator.refresh(expanded)
|
||||
current = coordinator.snapshot()[claim.token]
|
||||
assert current["metadata"] == {"phase": "queued", "job_id": "maintenance-job"}
|
||||
assert current["updated_at"] == original["updated_at"] + 60
|
||||
assert current["state"] == "uncertain"
|
||||
assert coordinator.release(expanded) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("uncertain", [False, True])
|
||||
def test_resource_expansion_cannot_overlap_another_execution(coordinator, uncertain):
|
||||
blocker = coordinator.claim(resources=("vm:a:one",))
|
||||
if uncertain:
|
||||
coordinator.retain(blocker)
|
||||
candidate = coordinator.claim(metadata={"keep": True})
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
coordinator.add_resources(candidate, blocker.resources)
|
||||
assert raised.value.status == (503 if uncertain else 409)
|
||||
assert coordinator.snapshot()[candidate.token]["resources"] == []
|
||||
assert coordinator.snapshot()[candidate.token]["metadata"] == {"keep": True}
|
||||
|
||||
|
||||
def test_cas_retry_preserves_competing_claim(coordinator, monkeypatch):
|
||||
compare = coordinator.store.compare_and_set
|
||||
injected = []
|
||||
other = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path))
|
||||
|
||||
def competing_claim(kind, key, expected, value):
|
||||
if not injected:
|
||||
injected.append(other.claim(resources=("vm:a:other",)))
|
||||
return compare(kind, key, expected, value)
|
||||
|
||||
monkeypatch.setattr(coordinator.store, "compare_and_set", competing_claim)
|
||||
claim = coordinator.claim(resources=("vm:a:one",))
|
||||
assert set(coordinator.snapshot()) == {claim.token, injected[0].token}
|
||||
|
||||
|
||||
def test_missing_cas_and_exhausted_contention_fail_closed(coordinator, monkeypatch):
|
||||
with pytest.raises(ValueError, match="atomic database CAS"):
|
||||
DurableCoordinator(object())
|
||||
monkeypatch.setattr(coordinator.store, "compare_and_set", lambda *_args: False)
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
coordinator.claim(resources=("vm:a:one",))
|
||||
assert raised.value.error_code == "coordination_unavailable"
|
||||
assert raised.value.status == 503
|
||||
assert coordinator.snapshot() == {}
|
||||
|
||||
|
||||
def test_corrupt_record_is_not_reinitialized(coordinator):
|
||||
coordinator.store.insert(RECORD_KIND, RECORD_KEY, '{"version":1,"claims":null}')
|
||||
with pytest.raises(CoordinationError):
|
||||
coordinator.claim(resources=("vm:a:one",))
|
||||
assert coordinator.store.get(RECORD_KIND, RECORD_KEY) == '{"version":1,"claims":null}'
|
||||
|
||||
|
||||
def test_capacity_and_normal_release_do_not_accumulate_entries(coordinator):
|
||||
limited = DurableCoordinator(coordinator.store, max_claims=1)
|
||||
for _index in range(40):
|
||||
claim = limited.claim(resources=("vm:a:one",))
|
||||
with pytest.raises(CoordinationError):
|
||||
limited.claim(resources=("vm:b:one",))
|
||||
assert limited.release(claim)
|
||||
assert limited.snapshot() == {}
|
||||
assert len(coordinator.store.get(RECORD_KIND, RECORD_KEY)) < 100
|
||||
|
||||
|
||||
def test_context_flows_to_thread_and_nested_guards_release_individually(coordinator):
|
||||
outer = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",),
|
||||
metadata={"job_id": "durable-job"})
|
||||
|
||||
def guarded_thread():
|
||||
assert current_execution().claim.token == outer.token
|
||||
with resource_guard(("storage:a", "vm:a:one", "storage:a")) as inner:
|
||||
assert inner.execution_id == outer.execution_id
|
||||
assert inner.operation_id == outer.operation_id
|
||||
assert inner.metadata == {"job_id": "durable-job"}
|
||||
assert inner.resources == ("storage:a", "vm:a:one")
|
||||
assert current_execution().claim.token == inner.token
|
||||
with resource_guard(("vm:a:one",)):
|
||||
assert len(coordinator.snapshot()) == 3
|
||||
assert current_execution().claim.token == outer.token
|
||||
|
||||
async def run():
|
||||
with execution_context(coordinator, outer):
|
||||
await asyncio.to_thread(guarded_thread)
|
||||
|
||||
asyncio.run(run())
|
||||
assert current_execution() is None
|
||||
assert set(coordinator.snapshot()) == {outer.token}
|
||||
assert coordinator.release(outer)
|
||||
|
||||
|
||||
def test_default_guard_is_reentrant_and_missing_default_fails_closed(coordinator):
|
||||
original = get_default_coordinator()
|
||||
try:
|
||||
configure_default_coordinator(None)
|
||||
with pytest.raises(CoordinationError):
|
||||
with resource_guard(("vm:a:one",)):
|
||||
pytest.fail("No durable coordinator")
|
||||
configure_default_coordinator(coordinator)
|
||||
with resource_guard(("vm:a:one",)) as first:
|
||||
with resource_guard(("vm:a:one",)) as second:
|
||||
assert first.execution_id == second.execution_id
|
||||
assert not coordinator.has_work()
|
||||
finally:
|
||||
configure_default_coordinator(original)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status,error_code,uncertain", [
|
||||
(404, None, False), (500, None, True),
|
||||
(504, "libvirt_execution_timeout", False),
|
||||
(503, "libvirt_execution_stopped", False),
|
||||
(504, "libvirt_execution_uncertain", True),
|
||||
(503, "libvirt_execution_blocked", True),
|
||||
])
|
||||
def test_guard_retains_only_unconfirmed_execution(coordinator, status, error_code, uncertain):
|
||||
class ServiceError(Exception):
|
||||
pass
|
||||
|
||||
failure = ServiceError("failure")
|
||||
failure.status, failure.error_code = status, error_code
|
||||
outer = coordinator.claim()
|
||||
with execution_context(coordinator, outer):
|
||||
with pytest.raises(ServiceError):
|
||||
with resource_guard(("vm:a:one",)):
|
||||
raise failure
|
||||
coordinator.release(outer)
|
||||
remaining = coordinator.snapshot()
|
||||
assert bool(remaining) is uncertain
|
||||
if uncertain:
|
||||
assert next(iter(remaining.values()))["state"] == "uncertain"
|
||||
|
||||
|
||||
def test_cancelled_guard_keeps_remote_execution_ownership(coordinator):
|
||||
outer = coordinator.claim()
|
||||
with execution_context(coordinator, outer):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
with resource_guard(("vm:a:one",)):
|
||||
raise asyncio.CancelledError
|
||||
coordinator.release(outer)
|
||||
assert next(iter(coordinator.snapshot().values()))["state"] == "uncertain"
|
||||
|
||||
|
||||
def test_guard_contention_has_bounded_wait(coordinator):
|
||||
blocker = coordinator.claim(resources=("vm:a:one",))
|
||||
other = coordinator.claim()
|
||||
started = time.monotonic()
|
||||
with execution_context(coordinator, other):
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
with resource_guard((item for item in blocker.resources), timeout=0.05):
|
||||
pytest.fail("Conflicting execution admitted")
|
||||
assert 0.04 <= time.monotonic() - started < 0.5
|
||||
assert raised.value.error_code == "coordination_busy"
|
||||
assert set(coordinator.snapshot()) == {blocker.token, other.token}
|
||||
|
||||
|
||||
def test_lost_claim_commit_receipt_keeps_durable_admission(coordinator, monkeypatch):
|
||||
compare = coordinator.store.compare_and_set
|
||||
|
||||
def commit_then_disconnect(*args):
|
||||
assert compare(*args)
|
||||
raise ConnectionError("injected lost commit receipt")
|
||||
|
||||
monkeypatch.setattr(coordinator.store, "compare_and_set", commit_then_disconnect)
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
coordinator.claim(operation_id="lost-receipt", resources=("vm:a:one",))
|
||||
assert raised.value.error_code == "coordination_unavailable"
|
||||
restarted = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path))
|
||||
assert restarted.operation_claim("lost-receipt") is not None
|
||||
with pytest.raises(CoordinationError):
|
||||
restarted.claim(resources=("vm:a:one",))
|
||||
|
||||
|
||||
def test_guard_release_database_error_cannot_erase_ownership(coordinator, monkeypatch):
|
||||
outer = coordinator.claim()
|
||||
with execution_context(coordinator, outer):
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
with resource_guard(("vm:a:one",)):
|
||||
def unavailable(*_args):
|
||||
raise ConnectionError("injected database outage")
|
||||
monkeypatch.setattr(coordinator.store, "compare_and_set", unavailable)
|
||||
assert raised.value.error_code == "coordination_unavailable"
|
||||
assert len(coordinator.snapshot()) == 2
|
||||
|
||||
|
||||
def test_unknown_conflict_takes_precedence_over_an_active_sibling(coordinator):
|
||||
active = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",))
|
||||
uncertain = coordinator.claim(operation_id="operation-one", resources=active.resources,
|
||||
execution_id=active.execution_id)
|
||||
coordinator.retain(uncertain)
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
coordinator.claim(resources=active.resources)
|
||||
assert raised.value.error_code == "coordination_uncertain"
|
||||
|
||||
|
||||
def test_current_claim_can_only_be_replaced_by_same_identity(coordinator):
|
||||
original = coordinator.claim(operation_id="create-one")
|
||||
expanded = coordinator.add_resources(original, ("vm:a:created",))
|
||||
with execution_context(coordinator, original):
|
||||
update_current_claim(expanded)
|
||||
assert current_execution().claim.resources == ("vm:a:created",)
|
||||
with pytest.raises(ValueError):
|
||||
update_current_claim(replace(expanded, token="other" * 8))
|
||||
assert current_execution() is None
|
||||
|
||||
|
||||
def test_safe_error_cannot_hide_a_concurrently_retained_guard(coordinator):
|
||||
class NotFound(Exception):
|
||||
status = 404
|
||||
|
||||
outer = coordinator.claim()
|
||||
with execution_context(coordinator, outer):
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
with resource_guard(("vm:a:one",)) as claim:
|
||||
coordinator.retain(claim)
|
||||
raise NotFound()
|
||||
assert raised.value.error_code == "coordination_uncertain"
|
||||
assert coordinator.snapshot()[claim.token]["state"] == "uncertain"
|
||||
@ -1,213 +0,0 @@
|
||||
"""Fabric's queued HTTP response transfers ownership through remote completion."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationError, DurableCoordinator, current_execution, execution_context,
|
||||
)
|
||||
from virsh_panel.fabric import FabricCoordinator
|
||||
from virsh_panel.internal import _DurableAsyncLock
|
||||
from tests.test_durable_jobs import MemoryCAS
|
||||
from test_fabric import (
|
||||
AsyncCoordinatorAdapter, Registry, _async_components, plan, plan_payload,
|
||||
)
|
||||
|
||||
|
||||
class FabricStore(MemoryCAS):
|
||||
def list(self, kind):
|
||||
with self.lock:
|
||||
return {key: value for (row_kind, key), value in self.rows.items()
|
||||
if row_kind == kind}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("terminal", ["succeeded", "failed", "uncertain"])
|
||||
def test_real_enqueue_pipeline_settles_only_confirmed_host_receipts(monkeypatch, terminal):
|
||||
async def run():
|
||||
current = plan()
|
||||
store = FabricStore()
|
||||
coordinator = DurableCoordinator(store)
|
||||
catalog = _DurableAsyncLock(coordinator, ("catalog:network",))
|
||||
components = _async_components(current)
|
||||
for component in components.values():
|
||||
component.manager.async_statuses = [terminal]
|
||||
fabric = FabricCoordinator(store, Registry(components), catalog_lock=lambda: catalog)
|
||||
monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
||||
monkeypatch.setattr("virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
|
||||
parent = coordinator.claim(kind="request")
|
||||
try:
|
||||
with execution_context(coordinator, parent):
|
||||
admitted = await fabric.enqueue_apply(plan_payload(current))
|
||||
task = fabric._operation_tasks_by_fabric[current.fabric_id]
|
||||
assert coordinator.release(parent)
|
||||
await asyncio.wait_for(task, 2)
|
||||
desired = fabric._load("fabric_desired", current.fabric_id)
|
||||
assert desired["operation"]["id"] == admitted["operation"]["id"]
|
||||
assert desired["operation"]["status"] == terminal
|
||||
assert all(node["operation"]["status"] == terminal for node in desired["nodes"])
|
||||
assert all(component.manager.async_dispatch_calls == 1
|
||||
for component in components.values())
|
||||
if terminal in {"succeeded", "failed"}:
|
||||
assert coordinator.snapshot() == {}
|
||||
update = coordinator.claim(maintenance=True, kind="update")
|
||||
assert coordinator.release(update)
|
||||
else:
|
||||
claims = list(coordinator.snapshot().values())
|
||||
assert len(claims) == 1 and claims[0]["state"] == "uncertain"
|
||||
with pytest.raises(CoordinationError):
|
||||
coordinator.claim(maintenance=True, kind="update")
|
||||
finally:
|
||||
await fabric.shutdown_operations()
|
||||
fabric.close()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def fabric_worker(monkeypatch):
|
||||
store = MemoryCAS()
|
||||
coordinator = DurableCoordinator(store)
|
||||
catalog = _DurableAsyncLock(coordinator, ("catalog:network",))
|
||||
fabric = FabricCoordinator(store, SimpleNamespace(), catalog_lock=lambda: catalog)
|
||||
monkeypatch.setattr(fabric, "_validate_resource_exclusivity", lambda _plan: None)
|
||||
return store, coordinator, fabric
|
||||
|
||||
|
||||
def finish_job(fabric, fabric_id, operation_id, *, root="succeeded",
|
||||
node="succeeded", checkpoint="complete"):
|
||||
desired = fabric._load("fabric_desired", fabric_id)
|
||||
desired["operation"].update(id=operation_id, status=root, finished_at=1)
|
||||
for item in desired["nodes"]:
|
||||
item["operation"].update(
|
||||
id=operation_id, status=node, checkpoint=checkpoint, finished_at=1)
|
||||
fabric._save("fabric_desired", fabric_id, desired)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("terminal", ["succeeded", "failed"])
|
||||
def test_enqueue_handoff_blocks_update_until_all_persisted_receipts_finish(monkeypatch, terminal):
|
||||
async def run():
|
||||
store, coordinator, fabric = fabric_worker(monkeypatch)
|
||||
entered, release = asyncio.Event(), asyncio.Event()
|
||||
seen = []
|
||||
|
||||
async def worker(fabric_id, operation_id, _kind):
|
||||
seen.append(current_execution().claim.execution_id)
|
||||
entered.set()
|
||||
await release.wait()
|
||||
# The final catalog publication must reenter its own durable
|
||||
# background claim instead of conflicting with itself.
|
||||
async with fabric.catalog_mutation_lock():
|
||||
finish_job(fabric, fabric_id, operation_id, root=terminal, node=terminal)
|
||||
|
||||
monkeypatch.setattr(fabric, "_run_enqueued_operation", worker)
|
||||
parent = coordinator.claim(kind="request")
|
||||
with execution_context(coordinator, parent):
|
||||
await fabric.enqueue_apply(plan_payload())
|
||||
task = next(iter(fabric._operation_tasks))
|
||||
child = next(row for row in coordinator.snapshot().values()
|
||||
if row["kind"] == "background")
|
||||
assert child["metadata"] == {
|
||||
"job_kind": "fabric_operation", "job_id": child["operation_id"]}
|
||||
assert set(child["resources"]) == {"catalog:network", "fabric:hk-fabric"}
|
||||
assert coordinator.release(parent)
|
||||
other_worker = DurableCoordinator(store)
|
||||
with pytest.raises(CoordinationError, match="资源正由其他任务"):
|
||||
other_worker.claim(maintenance=True, kind="update")
|
||||
unrelated = other_worker.claim(resources=("vm:host-c:unrelated",))
|
||||
assert other_worker.release(unrelated)
|
||||
await entered.wait()
|
||||
release.set()
|
||||
await task
|
||||
assert seen == [parent.execution_id]
|
||||
assert coordinator.snapshot() == {}
|
||||
update = other_worker.claim(maintenance=True, kind="update")
|
||||
assert other_worker.release(update)
|
||||
fabric.close()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("root,node,checkpoint", [
|
||||
("uncertain", "uncertain", "dispatch"),
|
||||
("interrupted", "failed", "identity"),
|
||||
("failed", "running", "dispatch"),
|
||||
("failed", "pending", "dispatch"),
|
||||
("failed", "uncertain", "dispatch"),
|
||||
("succeeded", "succeeded", "final_readback"),
|
||||
("succeeded", "failed", "complete"),
|
||||
])
|
||||
def test_overall_result_does_not_override_unsettled_node_evidence(
|
||||
monkeypatch, root, node, checkpoint):
|
||||
async def run():
|
||||
_store, coordinator, fabric = fabric_worker(monkeypatch)
|
||||
async def worker(fabric_id, operation_id, _kind):
|
||||
finish_job(fabric, fabric_id, operation_id,
|
||||
root=root, node=node, checkpoint=checkpoint)
|
||||
monkeypatch.setattr(fabric, "_run_enqueued_operation", worker)
|
||||
parent = coordinator.claim(kind="request")
|
||||
with execution_context(coordinator, parent):
|
||||
await fabric.enqueue_apply(plan_payload())
|
||||
tasks = tuple(fabric._operation_tasks)
|
||||
coordinator.release(parent)
|
||||
await asyncio.gather(*tasks)
|
||||
claims = list(coordinator.snapshot().values())
|
||||
assert len(claims) == 1 and claims[0]["state"] == "uncertain"
|
||||
with pytest.raises(CoordinationError) as blocked:
|
||||
coordinator.claim(maintenance=True, kind="update")
|
||||
assert blocked.value.error_code == "coordination_uncertain"
|
||||
fabric.close()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("started", [False, True])
|
||||
def test_cancelled_async_task_keeps_durable_ownership(monkeypatch, started):
|
||||
async def run():
|
||||
_store, coordinator, fabric = fabric_worker(monkeypatch)
|
||||
entered = asyncio.Event()
|
||||
async def worker():
|
||||
entered.set()
|
||||
await asyncio.Event().wait()
|
||||
parent = coordinator.claim(kind="request")
|
||||
with execution_context(coordinator, parent):
|
||||
await fabric._track_operation_task("hk-fabric", "a" * 32, worker())
|
||||
task = next(iter(fabric._operation_tasks))
|
||||
coordinator.release(parent)
|
||||
if started:
|
||||
await entered.wait()
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
for _ in range(100):
|
||||
claims = list(coordinator.snapshot().values())
|
||||
if claims and claims[0]["state"] == "uncertain":
|
||||
break
|
||||
await asyncio.sleep(0.001)
|
||||
assert len(claims) == 1 and claims[0]["state"] == "uncertain"
|
||||
with pytest.raises(CoordinationError):
|
||||
coordinator.claim(maintenance=True, kind="update")
|
||||
fabric.close()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_failed_outcome_read_does_not_release_completed_coroutine(monkeypatch):
|
||||
async def run():
|
||||
_store, coordinator, fabric = fabric_worker(monkeypatch)
|
||||
async def worker():
|
||||
return None
|
||||
def unavailable(*_args):
|
||||
raise RuntimeError("database response lost")
|
||||
monkeypatch.setattr(fabric, "_durable_operation_outcome", unavailable)
|
||||
parent = coordinator.claim(kind="request")
|
||||
with execution_context(coordinator, parent):
|
||||
await fabric._track_operation_task("hk-fabric", "b" * 32, worker())
|
||||
task = next(iter(fabric._operation_tasks))
|
||||
coordinator.release(parent)
|
||||
with pytest.raises(RuntimeError):
|
||||
await task
|
||||
child = next(iter(coordinator.snapshot().values()))
|
||||
assert child["state"] == "uncertain"
|
||||
fabric.close()
|
||||
|
||||
asyncio.run(run())
|
||||
@ -1,222 +0,0 @@
|
||||
"""A 202 requester cannot release ownership of its running background job."""
|
||||
|
||||
from contextvars import ContextVar
|
||||
import json
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationError, DurableCoordinator, current_execution, execution_context,
|
||||
resource_guard,
|
||||
)
|
||||
from virsh_panel.durable_jobs import mark_not_started, read_job_outcome, submit_durable
|
||||
from virsh_panel.shutdown import ManagedThreadPoolExecutor, close_executors
|
||||
|
||||
|
||||
class MemoryCAS:
|
||||
def __init__(self):
|
||||
self.rows = {}
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def get(self, kind, key):
|
||||
with self.lock:
|
||||
return self.rows.get((kind, key))
|
||||
|
||||
def set(self, kind, key, value):
|
||||
with self.lock:
|
||||
self.rows[kind, key] = value
|
||||
|
||||
def insert(self, kind, key, value):
|
||||
with self.lock:
|
||||
if (kind, key) in self.rows:
|
||||
return False
|
||||
self.rows[kind, key] = value
|
||||
return True
|
||||
|
||||
def compare_and_set(self, kind, key, expected, desired):
|
||||
with self.lock:
|
||||
if self.rows.get((kind, key)) != expected:
|
||||
return False
|
||||
self.rows[kind, key] = desired
|
||||
return True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def jobs():
|
||||
store = MemoryCAS()
|
||||
coordinator = DurableCoordinator(store)
|
||||
parent = coordinator.claim(operation_id="request-one", resources=("vm:host-a:one",))
|
||||
executor = ManagedThreadPoolExecutor(1, "durable-job-test")
|
||||
store.set("snapshot_job", "job-one", json.dumps({"status": "queued"}))
|
||||
try:
|
||||
yield store, coordinator, parent, executor
|
||||
finally:
|
||||
assert close_executors((executor,))
|
||||
|
||||
|
||||
def submit(jobs, fn, *, outcome=None):
|
||||
store, coordinator, parent, executor = jobs
|
||||
with execution_context(coordinator, parent):
|
||||
return submit_durable(
|
||||
executor, fn,
|
||||
outcome=outcome or (lambda: read_job_outcome(store, "snapshot_job", "job-one")),
|
||||
job_kind="snapshot_job", job_id="job-one",
|
||||
)
|
||||
|
||||
|
||||
def test_child_exists_before_202_handoff_and_outlives_parent_claim(jobs):
|
||||
store, coordinator, parent, executor = jobs
|
||||
started, release = threading.Event(), threading.Event()
|
||||
marker = ContextVar("job-delegated-actor", default=None)
|
||||
marker.set("tenant-one")
|
||||
seen = []
|
||||
|
||||
def worker():
|
||||
seen.append((current_execution().claim.execution_id, marker.get()))
|
||||
started.set()
|
||||
assert release.wait(1)
|
||||
store.set("snapshot_job", "job-one", json.dumps({"status": "completed"}))
|
||||
|
||||
future = submit(jobs, worker)
|
||||
assert started.wait(1)
|
||||
try:
|
||||
claims = coordinator.snapshot()
|
||||
child = next(row for row in claims.values() if row["kind"] == "background")
|
||||
assert child["operation_id"] == parent.operation_id
|
||||
assert child["resources"] == list(parent.resources)
|
||||
assert child["metadata"] == {"job_kind": "snapshot_job", "job_id": "job-one"}
|
||||
assert coordinator.release(parent)
|
||||
other_process = DurableCoordinator(store)
|
||||
with pytest.raises(CoordinationError) as blocked:
|
||||
other_process.claim(resources=parent.resources)
|
||||
assert blocked.value.error_code == "coordination_busy"
|
||||
finally:
|
||||
release.set()
|
||||
future.result(timeout=1)
|
||||
assert seen == [(parent.execution_id, "tenant-one")]
|
||||
assert coordinator.snapshot() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["completed", "succeeded", "failed"])
|
||||
def test_committed_terminal_job_releases_child(jobs, status):
|
||||
store, coordinator, parent, _executor = jobs
|
||||
future = submit(jobs, lambda: store.set(
|
||||
"snapshot_job", "job-one", json.dumps({"status": status})))
|
||||
future.result(timeout=1)
|
||||
assert list(coordinator.snapshot()) == [parent.token]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("record", [None, {"status": "running"}, {"status": "uncertain"},
|
||||
{"status": "failed", "cleanup_pending": True}])
|
||||
def test_missing_or_unresolved_persisted_outcome_retains_child(jobs, record):
|
||||
_store, coordinator, parent, _executor = jobs
|
||||
future = submit(jobs, lambda: None, outcome=lambda: record)
|
||||
future.result(timeout=1)
|
||||
child = next(row for row in coordinator.snapshot().values() if row["kind"] == "background")
|
||||
assert child["state"] == "uncertain"
|
||||
coordinator.release(parent)
|
||||
with pytest.raises(CoordinationError) as blocked:
|
||||
coordinator.claim(resources=parent.resources)
|
||||
assert blocked.value.error_code == "coordination_uncertain"
|
||||
|
||||
|
||||
def test_swallowed_remote_failure_cannot_make_failed_job_release_ownership(jobs):
|
||||
store, coordinator, _parent, _executor = jobs
|
||||
|
||||
def worker():
|
||||
try:
|
||||
with resource_guard(["storage:host-a"]):
|
||||
raise RuntimeError("remote response lost")
|
||||
except RuntimeError:
|
||||
store.set("snapshot_job", "job-one", json.dumps({"status": "failed"}))
|
||||
|
||||
submit(jobs, worker).result(timeout=1)
|
||||
claims = coordinator.snapshot().values()
|
||||
assert {row["kind"] for row in claims if row["state"] == "uncertain"} == {
|
||||
"background", "resource",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["worker", "readback", "release"])
|
||||
def test_worker_or_final_store_failure_retains_child(jobs, monkeypatch, failure):
|
||||
store, coordinator, _parent, _executor = jobs
|
||||
|
||||
def fail():
|
||||
raise RuntimeError("unconfirmed")
|
||||
|
||||
def worker():
|
||||
if failure == "worker":
|
||||
fail()
|
||||
store.set("snapshot_job", "job-one", json.dumps({"status": "completed"}))
|
||||
|
||||
if failure == "release":
|
||||
monkeypatch.setattr(coordinator, "release", lambda _claim: fail())
|
||||
result = submit(jobs, worker, outcome=fail if failure == "readback" else None)
|
||||
with pytest.raises(RuntimeError, match="unconfirmed"):
|
||||
result.result(timeout=1)
|
||||
child = next(row for row in coordinator.snapshot().values() if row["kind"] == "background")
|
||||
assert child["state"] == "uncertain"
|
||||
|
||||
|
||||
def test_cancelling_queued_job_releases_child_but_does_not_claim_success(jobs):
|
||||
store, coordinator, parent, executor = jobs
|
||||
started, release, ran = threading.Event(), threading.Event(), threading.Event()
|
||||
|
||||
def occupied():
|
||||
started.set()
|
||||
assert release.wait(1)
|
||||
|
||||
active = executor.submit(occupied)
|
||||
assert started.wait(1)
|
||||
try:
|
||||
queued = submit(jobs, ran.set)
|
||||
assert queued.cancel()
|
||||
assert list(coordinator.snapshot()) == [parent.token]
|
||||
assert read_job_outcome(store, "snapshot_job", "job-one")["status"] == "queued"
|
||||
finally:
|
||||
release.set()
|
||||
active.result(timeout=1)
|
||||
assert not ran.is_set()
|
||||
|
||||
|
||||
def test_executor_rejection_does_not_leave_a_child_claim(jobs):
|
||||
_store, coordinator, parent, executor = jobs
|
||||
assert close_executors((executor,))
|
||||
with pytest.raises(RuntimeError, match="after shutdown"):
|
||||
submit(jobs, lambda: None)
|
||||
assert list(coordinator.snapshot()) == [parent.token]
|
||||
|
||||
|
||||
def test_unscoped_native_adapter_keeps_original_executor_contract():
|
||||
executor = ManagedThreadPoolExecutor(1)
|
||||
try:
|
||||
future = submit_durable(
|
||||
executor, lambda: 42,
|
||||
outcome=lambda: pytest.fail("no durable execution context"),
|
||||
job_kind="test", job_id="test",
|
||||
)
|
||||
assert future.result(timeout=1) == 42
|
||||
finally:
|
||||
assert close_executors((executor,))
|
||||
|
||||
|
||||
def test_prestart_proof_survives_release_of_inner_target_claim(jobs):
|
||||
_store, coordinator, parent, _executor = jobs
|
||||
target = coordinator.claim(
|
||||
resources=parent.resources, operation_id=parent.operation_id,
|
||||
execution_id=parent.execution_id, kind="target")
|
||||
with execution_context(coordinator, target):
|
||||
mark_not_started()
|
||||
assert coordinator.release(target)
|
||||
assert coordinator.snapshot()[parent.token]["metadata"]["failed_before_start"] is True
|
||||
|
||||
|
||||
def test_prestart_proof_store_failure_does_not_become_local_success(jobs, monkeypatch):
|
||||
_store, coordinator, parent, _executor = jobs
|
||||
monkeypatch.setattr(coordinator, "update_metadata", lambda *_args: False)
|
||||
with execution_context(coordinator, parent):
|
||||
with pytest.raises(CoordinationError) as failed:
|
||||
mark_not_started()
|
||||
assert failed.value.error_code == "coordination_unavailable"
|
||||
assert not coordinator.snapshot()[parent.token]["metadata"].get("failed_before_start")
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,552 +0,0 @@
|
||||
"""Real ASGI cancellation and durable admission, without a remote host."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import gc
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from test_engine_boundary import (
|
||||
CREATE_BODY, SECRET, SUPER, VM_ID, DatabaseTemplateStore, EngineApplication,
|
||||
EngineComponents, ExternalOwnership, FakeRemoteManager, FakeUpdater,
|
||||
HostRegistry, HostStore, MemoryStore, Service, Settings, cloud_engine_request,
|
||||
create_app,
|
||||
)
|
||||
from virsh_panel.database import DatabaseError
|
||||
from virsh_panel.metrics import host_scope
|
||||
|
||||
|
||||
class _NoMetrics:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
async def aclose(self):
|
||||
return True
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def local_backends(monkeypatch):
|
||||
monkeypatch.setattr("virsh_panel.app.MetricsCollector", _NoMetrics)
|
||||
# These services are injected fakes. Their lifecycle must not close an
|
||||
# unrelated process-wide RPC runtime created by another test fixture.
|
||||
monkeypatch.setattr("virsh_panel.libvirt_rpc.get_runtime", lambda: None)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def running_engine(directory, store=None):
|
||||
"""Exercise startup, requests and shutdown on the same owning loop."""
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
store = MemoryStore() if store is None else store
|
||||
logs = MemoryStore()
|
||||
config = {
|
||||
"service_secret": base64.b64encode(SECRET).decode(),
|
||||
"data_dir": str(directory),
|
||||
"shutdown_timeout_seconds": 0.5,
|
||||
}
|
||||
services = {}
|
||||
|
||||
def factory(host_id, user_isos, cloud_templates):
|
||||
host_directory = directory / host_id
|
||||
host_directory.mkdir(exist_ok=True)
|
||||
scoped = HostStore(store, host_id)
|
||||
manager = FakeRemoteManager()
|
||||
manager.host_id = host_id
|
||||
manager.profile = {
|
||||
"host_id": host_id, "host": host_id + ".example.test", "port": 22,
|
||||
"fingerprint": "SHA256:example", "key_locked": False,
|
||||
"storage_pool": "images", "iso_pool": "iso",
|
||||
"rescue_iso_pool": "rescue", "default_network": "default",
|
||||
"libvirt_uri": "fake", "ssh_target": "fake",
|
||||
"ssh_config_file": "/fake/config", "ssh_known_hosts_file": "/fake/known_hosts",
|
||||
}
|
||||
service = services.setdefault(host_id, Service())
|
||||
components = EngineComponents(
|
||||
SimpleNamespace(users=ExternalOwnership(scoped)), manager,
|
||||
DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates,
|
||||
)
|
||||
app = create_app(
|
||||
Settings(data_dir=host_directory), engine_components=components,
|
||||
service_factory=lambda _profile: service,
|
||||
)
|
||||
return components, app
|
||||
|
||||
registry = HostRegistry(config, store, logs, factory=factory)
|
||||
updater = FakeUpdater()
|
||||
updater.token = "test-only-token"
|
||||
|
||||
async def active(*_args):
|
||||
return True
|
||||
|
||||
engine = EngineApplication(
|
||||
config, store, logs, registry=registry, introspection=active, updater=updater,
|
||||
)
|
||||
engine._service = lambda profile: services[profile["host_id"]]
|
||||
incoming, outgoing = asyncio.Queue(), asyncio.Queue()
|
||||
lifecycle = asyncio.create_task(engine._lifespan(incoming.get, outgoing.put))
|
||||
await incoming.put({"type": "lifespan.startup"})
|
||||
started = await asyncio.wait_for(outgoing.get(), 3)
|
||||
assert started["type"] == "lifespan.startup.complete", started
|
||||
try:
|
||||
yield engine, store, services
|
||||
finally:
|
||||
await incoming.put({"type": "lifespan.shutdown"})
|
||||
stopped = await asyncio.wait_for(outgoing.get(), 3)
|
||||
assert stopped["type"] == "lifespan.shutdown.complete", stopped
|
||||
await lifecycle
|
||||
|
||||
|
||||
async def request(engine, method, target, value=None, *, operation_id=None):
|
||||
actor = dict(SUPER)
|
||||
if operation_id is not None:
|
||||
actor.update(operation_id=operation_id, can_provision=True)
|
||||
body = b"" if value is None else json.dumps(value).encode()
|
||||
return await cloud_engine_request(
|
||||
engine, method, target, body, actor,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
|
||||
async def wait_until(predicate, *, timeout=2.0):
|
||||
deadline = time.monotonic() + timeout
|
||||
while not predicate():
|
||||
if time.monotonic() >= deadline:
|
||||
pytest.fail("test condition did not become true before its deadline")
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
|
||||
async def prepare_hosts(engine, *hosts):
|
||||
for host in hosts:
|
||||
status, payload = await request(engine, "GET", f"/internal/v1/hosts/{host}/vms")
|
||||
assert status == 200, payload
|
||||
|
||||
|
||||
def vm_path(host, resource="actions"):
|
||||
return f"/internal/v1/hosts/{host}/vms/{VM_ID}/{resource}"
|
||||
|
||||
|
||||
def test_stalled_host_mutation_does_not_serialize_other_hosts(tmp_path):
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, _store, services):
|
||||
await prepare_hosts(engine, "host-a", "host-b")
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
action = services["host-a"].action
|
||||
|
||||
def stalled_action(*args):
|
||||
entered.set()
|
||||
assert release.wait(3), "test did not release host A"
|
||||
return action(*args)
|
||||
|
||||
services["host-a"].action = stalled_action
|
||||
first = asyncio.create_task(request(
|
||||
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
||||
operation_id="coord-host-a-reset",
|
||||
))
|
||||
try:
|
||||
await wait_until(entered.is_set)
|
||||
read_status, _ = await asyncio.wait_for(request(
|
||||
engine, "GET", f"/internal/v1/hosts/host-a/vms"), 0.75)
|
||||
assert read_status == 200
|
||||
other_status, other = await asyncio.wait_for(request(
|
||||
engine, "POST", vm_path("host-b"), {"action": "reset"},
|
||||
operation_id="coord-host-b-reset",
|
||||
), 0.75)
|
||||
assert other_status == 200, other
|
||||
assert not release.is_set()
|
||||
assert not first.done()
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.wait_for(first, 3)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_stalled_vm_action_does_not_serialize_other_vms_on_same_host(tmp_path):
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, _store, services):
|
||||
await prepare_hosts(engine, "host-a")
|
||||
service = services["host-a"]
|
||||
other_id = "22222222-2222-4222-8222-222222222222"
|
||||
other_vm = {**service.vm, "uuid": other_id, "name": "second-vm"}
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
get_vm, action = service.get_vm, service.action
|
||||
|
||||
def get_either_vm(vm_id):
|
||||
if vm_id == other_id:
|
||||
service.calls.append(("get_vm", vm_id))
|
||||
return other_vm.copy()
|
||||
return get_vm(vm_id)
|
||||
|
||||
def stalled_action(vm_id, target):
|
||||
if vm_id == VM_ID:
|
||||
entered.set()
|
||||
assert release.wait(3), "test did not release the first VM"
|
||||
return action(vm_id, target)
|
||||
assert vm_id == other_id
|
||||
service.calls.append(("action", vm_id, target))
|
||||
return other_vm.copy()
|
||||
|
||||
service.get_vm = get_either_vm
|
||||
service.action = stalled_action
|
||||
first = asyncio.create_task(request(
|
||||
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
||||
operation_id="coord-same-host-first-vm",
|
||||
))
|
||||
try:
|
||||
await wait_until(entered.is_set)
|
||||
other_status, other = await asyncio.wait_for(request(
|
||||
engine, "POST", f"/internal/v1/hosts/host-a/vms/{other_id}/actions",
|
||||
{"action": "reset"}, operation_id="coord-same-host-second-vm",
|
||||
), 0.75)
|
||||
assert other_status == 200, other
|
||||
assert other["uuid"] == other_id
|
||||
assert ("action", other_id, "reset") in service.calls
|
||||
assert not first.done()
|
||||
assert not release.is_set()
|
||||
finally:
|
||||
release.set()
|
||||
first_status, first_result = await asyncio.wait_for(first, 3)
|
||||
assert first_status == 200, first_result
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_http_cancellation_keeps_executor_vm_claim_and_update_gate(tmp_path):
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, store, services):
|
||||
await prepare_hosts(engine, "host-a")
|
||||
service = services["host-a"]
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
finished = threading.Event()
|
||||
suspend = service.network_suspend
|
||||
calls = []
|
||||
|
||||
def stalled_suspend(*args):
|
||||
calls.append(args)
|
||||
entered.set()
|
||||
assert release.wait(3), "test did not release suspended worker"
|
||||
try:
|
||||
return suspend(*args)
|
||||
finally:
|
||||
finished.set()
|
||||
|
||||
service.network_suspend = stalled_suspend
|
||||
op_id = "coord-cancelled-suspend"
|
||||
path = vm_path("host-a", "network-suspend")
|
||||
first = asyncio.create_task(request(
|
||||
engine, "POST", path, {}, operation_id=op_id))
|
||||
try:
|
||||
await wait_until(entered.is_set)
|
||||
first.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await first
|
||||
peek_status, peek = await request(
|
||||
engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false")
|
||||
assert peek_status == 200
|
||||
assert peek["executor_active"] is True
|
||||
|
||||
retry_status, retry = await request(
|
||||
engine, "POST", path, {}, operation_id=op_id)
|
||||
assert retry_status == 202, retry
|
||||
assert len(calls) == 1
|
||||
conflict_status, conflict = await request(
|
||||
engine, "POST", vm_path("host-a", "network-resume"), {},
|
||||
operation_id="coord-opposite-after-cancel",
|
||||
)
|
||||
assert conflict_status == 409, conflict
|
||||
assert not any(call[0] == "network_resume" for call in service.calls)
|
||||
update_status, update = await request(
|
||||
engine, "POST", "/internal/v1/update/apply", {
|
||||
"expected_current_commit": "a" * 40,
|
||||
"expected_remote_commit": "b" * 40,
|
||||
})
|
||||
assert update_status == 409, update
|
||||
finally:
|
||||
release.set()
|
||||
await wait_until(finished.is_set)
|
||||
await wait_until(lambda: json.loads(
|
||||
store.get("operation", op_id))["status"] == "succeeded")
|
||||
status, operation = await request(
|
||||
engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false")
|
||||
assert status == 200
|
||||
assert operation["executor_active"] is False
|
||||
assert len(calls) == 1
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_same_operation_retry_cannot_execute_before_first_record_insert(tmp_path):
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, store, services):
|
||||
await prepare_hosts(engine, "host-a")
|
||||
op_id = "coord-before-operation-insert"
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original_insert = store.insert
|
||||
first_insert = True
|
||||
|
||||
def gated_insert(kind, key, value):
|
||||
nonlocal first_insert
|
||||
if kind == "operation" and key == op_id and first_insert:
|
||||
first_insert = False
|
||||
entered.set()
|
||||
assert release.wait(3), "test did not release operation insert"
|
||||
return original_insert(kind, key, value)
|
||||
|
||||
store.insert = gated_insert
|
||||
body = {"action": "reset"}
|
||||
first = asyncio.create_task(request(
|
||||
engine, "POST", vm_path("host-a"), body, operation_id=op_id))
|
||||
try:
|
||||
await wait_until(entered.is_set)
|
||||
assert store.get("operation", op_id) is None
|
||||
assert engine.coordinator.operation_claim(op_id) is not None
|
||||
status, retry = await asyncio.wait_for(request(
|
||||
engine, "POST", vm_path("host-a"), body, operation_id=op_id), 0.75)
|
||||
assert status == 409, retry
|
||||
assert retry["error_code"] == "coordination_busy"
|
||||
assert not any(call[0] == "action" for call in services["host-a"].calls)
|
||||
finally:
|
||||
release.set()
|
||||
first_status, first_result = await asyncio.wait_for(first, 3)
|
||||
assert first_status == 200, first_result
|
||||
assert sum(call[0] == "action" for call in services["host-a"].calls) == 1
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_pending_snapshot_keeps_vm_claim_until_background_receipt(tmp_path):
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, _store, services):
|
||||
await prepare_hosts(engine, "host-a")
|
||||
service = services["host-a"]
|
||||
service.vm.update(managed=True, os_source="cloud_template",
|
||||
rescue_mode=False, rescue_staged=False)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original = service.create_live_snapshot
|
||||
|
||||
def stalled_snapshot(*args, **kwargs):
|
||||
entered.set()
|
||||
assert release.wait(3), "test did not release snapshot worker"
|
||||
return original(*args, **kwargs)
|
||||
|
||||
service.create_live_snapshot = stalled_snapshot
|
||||
op_id = "coord-background-snapshot"
|
||||
status, queued = await request(
|
||||
engine, "POST", vm_path("host-a", "snapshots"), {
|
||||
"snapshot_key": "a" * 32, "name": "coordination recovery point",
|
||||
"consistency_mode": "crash",
|
||||
}, operation_id=op_id,
|
||||
)
|
||||
assert status == 202, queued
|
||||
try:
|
||||
await wait_until(entered.is_set)
|
||||
assert engine.coordinator.operation_claim(op_id) is not None
|
||||
peek_status, peek = await request(
|
||||
engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false")
|
||||
assert peek_status == 200, peek
|
||||
assert peek["executor_active"] is True
|
||||
conflict_status, conflict = await request(
|
||||
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
||||
operation_id="coord-conflict-with-snapshot",
|
||||
)
|
||||
assert conflict_status == 409, conflict
|
||||
assert not any(call[0] == "action" for call in service.calls)
|
||||
finally:
|
||||
release.set()
|
||||
await wait_until(lambda: engine.coordinator.operation_claim(op_id) is None)
|
||||
status, completed = await request(
|
||||
engine, "GET", f"/internal/v1/operations/{op_id}")
|
||||
assert status == 200, completed
|
||||
assert completed["status"] == "succeeded"
|
||||
assert completed["result"]["created"] is True
|
||||
assert sum(call[0] == "create_live_snapshot" for call in service.calls) == 1
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
class _FailedResultStore(MemoryStore):
|
||||
failed_operation = "coord-result-save-failure"
|
||||
reject_result = True
|
||||
|
||||
def set(self, kind, key, value):
|
||||
if (self.reject_result and kind == "operation"
|
||||
and key == self.failed_operation
|
||||
and json.loads(value).get("status") == "succeeded"):
|
||||
raise DatabaseError("injected final operation persistence failure")
|
||||
super().set(kind, key, value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("restart", [False, True])
|
||||
def test_result_save_failure_retains_unknown_claim_and_never_replays(tmp_path, restart):
|
||||
async def check_blocked(engine, services, op_id):
|
||||
await prepare_hosts(engine, "host-a")
|
||||
before = list(services["host-a"].calls)
|
||||
for method, path, value in (
|
||||
("GET", f"/internal/v1/operations/{op_id}", None),
|
||||
("POST", vm_path("host-a"), {"action": "reset"}),
|
||||
):
|
||||
status, outcome = await request(
|
||||
engine, method, path, value,
|
||||
operation_id=op_id if method == "POST" else None,
|
||||
)
|
||||
assert status == (200 if method == "GET" else 202), outcome
|
||||
assert outcome["status"] == "uncertain"
|
||||
assert outcome["safe_to_refund"] is False
|
||||
assert services["host-a"].calls == before
|
||||
changed_status, changed = await request(
|
||||
engine, "POST", vm_path("host-a"), {"action": "poweroff"},
|
||||
operation_id=op_id,
|
||||
)
|
||||
assert changed_status == 409, changed
|
||||
assert services["host-a"].calls == before
|
||||
status, blocked = await request(
|
||||
engine, "POST", vm_path("host-a"), {"action": "poweroff"},
|
||||
operation_id="coord-new-conflicting-operation",
|
||||
)
|
||||
assert status == 503, blocked
|
||||
assert services["host-a"].calls == before
|
||||
|
||||
async def run():
|
||||
store = _FailedResultStore()
|
||||
op_id = store.failed_operation
|
||||
async with running_engine(tmp_path, store) as (engine, _store, services):
|
||||
await prepare_hosts(engine, "host-a")
|
||||
status, outcome = await request(
|
||||
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
||||
operation_id=op_id,
|
||||
)
|
||||
assert status >= 500, outcome
|
||||
assert sum(call[0] == "action" for call in services["host-a"].calls) == 1
|
||||
store.reject_result = False
|
||||
if not restart:
|
||||
await check_blocked(engine, services, op_id)
|
||||
if restart:
|
||||
async with running_engine(tmp_path, store) as (engine, _store, services):
|
||||
await check_blocked(engine, services, op_id)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", [False, True])
|
||||
def test_restarted_running_create_without_error_code_cannot_refund_or_release_ip(tmp_path, legacy):
|
||||
"""A crash can leave only the old owner claim and mutation checkpoint."""
|
||||
async def run():
|
||||
store = MemoryStore()
|
||||
op_id = "coord-crashed-create"
|
||||
async with running_engine(tmp_path, store) as (engine, _store, _services):
|
||||
status, created = await request(
|
||||
engine, "POST", "/internal/v1/hosts/host-a/vms", CREATE_BODY,
|
||||
operation_id=op_id,
|
||||
)
|
||||
assert status == 201, created
|
||||
row = json.loads(store.get("operation", op_id))
|
||||
vm_id = row["vm_uuid"]
|
||||
component = engine.registry.components["host-a"]
|
||||
scope = host_scope(component.manager.profile)
|
||||
credential_key = f"{scope}.{vm_id}"
|
||||
ip_key = f"{scope}.192.0.2.42"
|
||||
component.store.set("vm_credential", credential_key, "test-credential")
|
||||
component.store.set("ip_claim", ip_key, json.dumps({"vm_uuid": vm_id}))
|
||||
|
||||
# Reproduce the durable state immediately after a write began but
|
||||
# before its receipt/error was committed. There is deliberately
|
||||
# no libvirt_execution_* error code to trigger the old shortcut.
|
||||
row.update(status="running", result=None, created_at=1,
|
||||
mutation_started=True, safe_to_refund=False)
|
||||
row.pop("error_code", None)
|
||||
row.pop("finished_at", None)
|
||||
if legacy:
|
||||
row.pop("execution_id", None)
|
||||
store.set("operation", op_id, json.dumps(row))
|
||||
owner_before_restart = engine.coordinator.owner_id
|
||||
if not legacy:
|
||||
engine.coordinator.claim(operation_id=op_id)
|
||||
|
||||
async with running_engine(tmp_path, store) as (engine, _store, services):
|
||||
assert engine.coordinator.owner_id != owner_before_restart
|
||||
await prepare_hosts(engine, "host-a")
|
||||
service = services["host-a"]
|
||||
service.absent = True
|
||||
before = list(service.calls)
|
||||
status, outcome = await request(
|
||||
engine, "GET", f"/internal/v1/operations/{op_id}")
|
||||
assert status == 200, outcome
|
||||
assert outcome["status"] == "uncertain"
|
||||
assert outcome["safe_to_refund"] is False
|
||||
assert outcome["executor_active"] is False
|
||||
assert "finished_at" not in outcome
|
||||
repeated_status, repeated = await request(
|
||||
engine, "POST", "/internal/v1/hosts/host-a/vms", CREATE_BODY,
|
||||
operation_id=op_id,
|
||||
)
|
||||
assert repeated_status == 202, repeated
|
||||
assert repeated["status"] == "uncertain"
|
||||
assert service.calls == before
|
||||
assert service.absent_checks == []
|
||||
component = engine.registry.components["host-a"]
|
||||
assert component.state.users.owns_vm(scope, vm_id, SUPER["user_id"])
|
||||
assert component.store.get("vm_credential", credential_key) == "test-credential"
|
||||
assert component.store.get("ip_claim", ip_key) is not None
|
||||
if legacy:
|
||||
conflict_status, conflict = await request(
|
||||
engine, "POST", f"/internal/v1/hosts/host-a/vms/{vm_id}/actions",
|
||||
{"action": "poweroff"}, operation_id="coord-new-after-legacy-crash",
|
||||
)
|
||||
assert conflict_status == 503, conflict
|
||||
assert service.calls == before
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_vm_lock_registry_reclaims_idle_locks_without_splitting_waiters(tmp_path):
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, _store, _services):
|
||||
for number in range(1000):
|
||||
async with engine._target_lock("host-a", str(uuid.UUID(int=number))):
|
||||
pass
|
||||
gc.collect()
|
||||
assert len(engine.target_locks) == 0
|
||||
|
||||
entered = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
|
||||
async def holder():
|
||||
async with engine._target_lock("host-a", VM_ID):
|
||||
entered.set()
|
||||
await release.wait()
|
||||
|
||||
owned = asyncio.create_task(holder())
|
||||
await entered.wait()
|
||||
lock = engine._target_lock("host-a", VM_ID)
|
||||
waiter = asyncio.create_task(lock.acquire())
|
||||
await asyncio.sleep(0)
|
||||
try:
|
||||
gc.collect()
|
||||
assert engine._target_lock("host-a", VM_ID) is lock
|
||||
assert lock.locked()
|
||||
assert not waiter.done()
|
||||
waiter.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await waiter
|
||||
assert engine._target_lock("host-a", VM_ID) is lock
|
||||
finally:
|
||||
release.set()
|
||||
await owned
|
||||
del lock, waiter, owned
|
||||
gc.collect()
|
||||
assert len(engine.target_locks) == 0
|
||||
|
||||
asyncio.run(run())
|
||||
@ -1,237 +0,0 @@
|
||||
"""Maintenance admission shared by independent engine owners."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
import json
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from test_engine_boundary import FakeUpdater, MemoryStore
|
||||
from test_engine_coordination import (
|
||||
local_backends, prepare_hosts, request, running_engine, vm_path, wait_until,
|
||||
)
|
||||
|
||||
|
||||
UPDATE = "/internal/v1/update/apply"
|
||||
CONFIRMED_UPDATE = {
|
||||
"expected_current_commit": "a" * 40,
|
||||
"expected_remote_commit": "b" * 40,
|
||||
}
|
||||
JOB_KIND = "test_update_job"
|
||||
|
||||
|
||||
def save_update_job(store, job):
|
||||
store.set(JOB_KIND, job["job_id"], json.dumps(job))
|
||||
|
||||
|
||||
def read_update_job(store, job_id):
|
||||
raw = store.get(JOB_KIND, job_id)
|
||||
return json.loads(raw) if raw else None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def two_engines(directory, store):
|
||||
# Separate owner identities model independent workers. Both applications
|
||||
# intentionally share durable storage and no active_operations/task sets.
|
||||
async with running_engine(directory / "a", store) as first:
|
||||
try:
|
||||
async with running_engine(directory / "b", store) as second:
|
||||
assert first[0].coordinator.owner_id != second[0].coordinator.owner_id
|
||||
yield first, second
|
||||
finally:
|
||||
# Production workers have different loops. Restore A's still-live
|
||||
# executor after the nested B harness closes its own executor.
|
||||
asyncio.get_running_loop().set_default_executor(first[0]._shutdown_executor)
|
||||
|
||||
|
||||
def test_remote_work_in_one_engine_rejects_another_engines_update(tmp_path):
|
||||
async def run():
|
||||
store = MemoryStore()
|
||||
async with two_engines(tmp_path, store) as (first, second):
|
||||
engine_a, _, services_a = first
|
||||
engine_b, _, _services_b = second
|
||||
await prepare_hosts(engine_a, "host-a")
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original_action = services_a["host-a"].action
|
||||
|
||||
def stalled_action(*args):
|
||||
entered.set()
|
||||
assert release.wait(5), "test did not release host work"
|
||||
return original_action(*args)
|
||||
|
||||
services_a["host-a"].action = stalled_action
|
||||
operation = asyncio.create_task(request(
|
||||
engine_a, "POST", vm_path("host-a"), {"action": "reset"},
|
||||
operation_id="maintenance-other-worker-live",
|
||||
))
|
||||
try:
|
||||
await wait_until(entered.is_set)
|
||||
assert not engine_b.active_operations
|
||||
assert not engine_b.maintenance_tasks
|
||||
status, payload = await request(engine_b, "POST", UPDATE, CONFIRMED_UPDATE)
|
||||
assert status == 409, payload
|
||||
assert payload["error_code"] == "coordination_busy"
|
||||
assert engine_b.updater.jobs == {}
|
||||
assert not operation.done()
|
||||
finally:
|
||||
release.set()
|
||||
status, payload = await asyncio.wait_for(operation, 3)
|
||||
assert status == 200, payload
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("terminal", ["failed", "succeeded"])
|
||||
def test_maintenance_claim_precedes_job_creation_and_terminal_receipt_releases_it(
|
||||
tmp_path, monkeypatch, terminal):
|
||||
store = MemoryStore()
|
||||
monkeypatch.setattr(FakeUpdater, "get_job",
|
||||
lambda self, job_id: read_update_job(store, job_id))
|
||||
|
||||
async def run():
|
||||
async with two_engines(tmp_path, store) as (first, second):
|
||||
engine_a, _, _services_a = first
|
||||
engine_b, _, services_b = second
|
||||
await prepare_hosts(engine_b, "host-b")
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original_create = engine_a.updater.create_job
|
||||
|
||||
def stalled_create(**kwargs):
|
||||
entered.set()
|
||||
assert release.wait(5), "test did not release job creation"
|
||||
job = original_create(**kwargs)
|
||||
save_update_job(store, job)
|
||||
return job
|
||||
|
||||
def start_update():
|
||||
job = engine_a.updater.active
|
||||
job["status"] = "running"
|
||||
save_update_job(store, job)
|
||||
|
||||
engine_a.updater.create_job = stalled_create
|
||||
engine_a.update_trigger = start_update
|
||||
update = asyncio.create_task(request(engine_a, "POST", UPDATE, CONFIRMED_UPDATE))
|
||||
try:
|
||||
await wait_until(entered.is_set)
|
||||
claims = engine_b.coordinator.snapshot()
|
||||
assert len(claims) == 1
|
||||
maintenance = next(iter(claims.values()))
|
||||
assert maintenance["maintenance"] is True
|
||||
assert maintenance["owner_id"] == engine_a.coordinator.owner_id
|
||||
reserved_job_id = maintenance["metadata"]["update_job_id"]
|
||||
assert read_update_job(store, reserved_job_id) is None
|
||||
assert engine_b.updater.active_job() is None
|
||||
status, payload = await request(
|
||||
engine_b, "POST", vm_path("host-b"), {"action": "reset"},
|
||||
operation_id="maintenance-before-job-blocks-write",
|
||||
)
|
||||
assert status == 409, payload
|
||||
assert payload["error_code"] == "coordination_busy"
|
||||
assert not any(call[0] == "action" for call in services_b["host-b"].calls)
|
||||
read_status, _ = await request(engine_b, "GET", "/internal/v1/hosts/host-b/vms")
|
||||
assert read_status == 200
|
||||
finally:
|
||||
release.set()
|
||||
status, payload = await asyncio.wait_for(update, 3)
|
||||
assert status == 202, payload
|
||||
|
||||
claims = engine_b.coordinator.snapshot()
|
||||
assert len(claims) == 1
|
||||
maintenance = next(iter(claims.values()))
|
||||
job_id = maintenance["metadata"]["update_job_id"]
|
||||
assert job_id == reserved_job_id
|
||||
assert read_update_job(store, job_id)["status"] == "running"
|
||||
status, payload = await request(
|
||||
engine_b, "POST", vm_path("host-b"), {"action": "reset"},
|
||||
operation_id="maintenance-bound-job-blocks-write",
|
||||
)
|
||||
assert status == 409, payload
|
||||
|
||||
job = read_update_job(store, job_id)
|
||||
job["status"] = terminal
|
||||
save_update_job(store, job)
|
||||
status, payload = await request(
|
||||
engine_b, "POST", vm_path("host-b"), {"action": "reset"},
|
||||
operation_id="maintenance-terminal-allows-write",
|
||||
)
|
||||
assert status == 200, payload
|
||||
assert maintenance["token"] not in engine_b.coordinator.snapshot()
|
||||
assert any(call[0] == "action" for call in services_b["host-b"].calls)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bound,uncertain,job_status", [
|
||||
(False, False, None),
|
||||
(False, True, None),
|
||||
(True, False, None),
|
||||
(True, False, "running"),
|
||||
(True, False, "blocked"),
|
||||
# A completed updater receipt cannot clear independently unknown work.
|
||||
(True, True, "succeeded"),
|
||||
(True, True, "failed"),
|
||||
])
|
||||
def test_unbound_or_unknown_maintenance_survives_restart(
|
||||
tmp_path, monkeypatch, bound, uncertain, job_status):
|
||||
store = MemoryStore()
|
||||
monkeypatch.setattr(FakeUpdater, "get_job",
|
||||
lambda self, job_id: read_update_job(store, job_id))
|
||||
|
||||
async def assert_blocked(engine, expected_token, op_id):
|
||||
await prepare_hosts(engine, "host-a")
|
||||
await engine._settle_update_claims()
|
||||
assert expected_token in engine.coordinator.snapshot()
|
||||
status, payload = await request(
|
||||
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
||||
operation_id=op_id,
|
||||
)
|
||||
assert status == (503 if uncertain else 409), payload
|
||||
assert payload["error_code"] == (
|
||||
"coordination_uncertain" if uncertain else "coordination_busy")
|
||||
status, health = await request(engine, "GET", "/internal/v1/health")
|
||||
assert status == 200
|
||||
assert health["accepting_mutations"] is False
|
||||
|
||||
async def run():
|
||||
async with running_engine(tmp_path / "original", store) as (engine, _, services):
|
||||
claim = engine.coordinator.claim(maintenance=True, kind="update")
|
||||
if bound:
|
||||
engine.coordinator.update_metadata(claim, {"update_job_id": "persisted-update-job"})
|
||||
if job_status is not None:
|
||||
save_update_job(store, {"job_id": "persisted-update-job", "status": job_status})
|
||||
if uncertain:
|
||||
engine.coordinator.retain(claim)
|
||||
# Claim age is diagnostic; old timestamps are never a takeover.
|
||||
await assert_blocked(engine, claim.token, "maintenance-before-restart")
|
||||
original_owner = engine.coordinator.owner_id
|
||||
assert not any(call[0] == "action" for call in services["host-a"].calls)
|
||||
|
||||
async with running_engine(tmp_path / "restarted", store) as (engine, _, services):
|
||||
assert engine.coordinator.owner_id != original_owner
|
||||
await assert_blocked(engine, claim.token, "maintenance-after-restart")
|
||||
assert not any(call[0] == "action" for call in services["host-a"].calls)
|
||||
assert engine.coordinator.snapshot()[claim.token]["owner_id"] == original_owner
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["running", "interrupted", "uncertain"])
|
||||
def test_legacy_fabric_attempt_without_child_claim_blocks_update(tmp_path, status):
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, store, _services):
|
||||
store.set("fabric_desired", "legacy-fabric", json.dumps({
|
||||
"operation": {"id": "a" * 32, "status": status},
|
||||
"nodes": [],
|
||||
}))
|
||||
assert not engine.coordinator.has_work()
|
||||
response, _body = await request(engine, "POST", "/internal/v1/update/apply", {
|
||||
"expected_current_commit": "a" * 40,
|
||||
"expected_remote_commit": "b" * 40,
|
||||
})
|
||||
assert response == 409
|
||||
assert not engine.coordinator.has_work()
|
||||
asyncio.run(run())
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue