@ -2,8 +2,10 @@
from __future__ import annotations
import asyncio
from concurrent . futures import ThreadPoolExecutor
import json
import threading
import time
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
@ -13,6 +15,7 @@ import pytest
from virsh_panel . fabric import FabricError , FabricNode , FabricPlan
from virsh_panel . vpc import (
VPC_DELETE_RECEIPT_KIND ,
VPC_GUEST_EXEC_KIND ,
VPC_SCHEMA_VERSION ,
VpcCoordinator ,
VpcError ,
@ -304,6 +307,77 @@ def test_coordinator_persists_desired_children_and_blocks_without_fabric():
assert set ( store . list ( " vpc_host " ) ) == { " vpc-hk-1.host-a " }
def test_guest_exec_ledger_is_immediate_bounded_private_and_restart_safe ( ) :
store = MemoryStore ( )
plan = parse_vpc_plan ( payload ( ) )
member = plan . members [ 0 ]
coordinator = VpcCoordinator ( store , FakeFabric ( ) )
state , persist , clear = coordinator . _guest_exec_context ( plan , member )
assert state == { }
core = {
" version " : 1 ,
" phase " : " pending " ,
" domain_uuid " : member . instance_uuid ,
" domain_runtime_id " : 7 ,
" qemu_runtime_token " : " a " * 32 ,
" guest_network_generation " : " generation-1 " ,
" probe_kind " : " link " ,
" pid " : 41 ,
}
persist ( core )
key = f " { plan . vpc_id } . { member . instance_uuid } "
encoded = store . get ( VPC_GUEST_EXEC_KIND , key )
assert encoded is not None
assert len ( encoded . encode ( " utf-8 " ) ) < = 1024
assert json . loads ( encoded ) [ " pid " ] == 41
restarted = VpcCoordinator ( store , FakeFabric ( ) )
loaded , _persist_again , clear_again = restarted . _guest_exec_context (
plan , member )
assert loaded == core
asyncio . run ( coordinator . apply ( payload ( ) ) )
public = json . dumps ( coordinator . get_status ( plan . vpc_id ) , sort_keys = True )
assert ' " pid " ' not in public
assert " qemu_runtime_token " not in public
assert " a " * 32 not in public
clear_again ( )
assert store . get ( VPC_GUEST_EXEC_KIND , key ) is None
persist ( core )
coordinator . _purge_deleted_vpc ( plan . vpc_id )
assert store . get ( VPC_GUEST_EXEC_KIND , key ) is None
def test_guest_exec_ledger_rejects_oversize_or_unknown_fields ( ) :
store = MemoryStore ( )
plan = parse_vpc_plan ( payload ( ) )
member = plan . members [ 0 ]
key = f " { plan . vpc_id } . { member . instance_uuid } "
store . set ( VPC_GUEST_EXEC_KIND , key , " x " * 1025 )
with pytest . raises ( VpcError , match = " 账本损坏 " ) :
VpcCoordinator ( store , FakeFabric ( ) ) . _guest_exec_context ( plan , member )
bad = {
" vpc_id " : plan . vpc_id ,
" host_id " : member . host_id ,
" member_id " : member . member_id ,
" instance_uuid " : member . instance_uuid ,
" version " : 1 ,
" phase " : " launch_intent " ,
" domain_uuid " : member . instance_uuid ,
" domain_runtime_id " : 7 ,
" qemu_runtime_token " : " a " * 32 ,
" guest_network_generation " : " generation-1 " ,
" probe_kind " : " link " ,
" unexpected " : True ,
}
store . set ( VPC_GUEST_EXEC_KIND , key , json . dumps ( bad ) )
with pytest . raises ( VpcError , match = " 账本损坏 " ) :
VpcCoordinator ( store , FakeFabric ( ) ) . _guest_exec_context ( plan , member )
def test_observed_projection_keeps_fabric_gate_for_data_plane_deferral ( ) :
store = MemoryStore ( )
result = asyncio . run ( VpcCoordinator ( store , FakeFabric ( ) ) . apply ( payload ( ) ) )
@ -942,11 +1016,32 @@ def test_instance_reconcile_returns_host_verified_private_state_and_preserves_li
state = " active " , observed_revision = plan . desired_revision ,
observed_digest = plan . desired_digest , transport_verified = True ,
private_dataplane_verified = True ,
config_verified = True ,
config_observed_revision = plan . desired_revision ,
config_observed_digest = plan . desired_digest ,
member_nics = [ {
" member_id " : " member-a " , " instance_uuid " : INSTANCE_A ,
" interface " : " private0 " , " private_ip_cidr " : " 10.20.0.10/32 " ,
" private_mac " : derive_private_mac ( " vpc-hk-1 " , " member-a " ) ,
" xml_verified " : True ,
" config_verified " : True ,
" seed_verified " : True ,
" restart_required " : False ,
" domain_running " : True ,
" guest_network_verified " : requested != " down " ,
" guest_network_readback " : (
" confirmed " if requested != " down " else " mismatch " ) ,
" guest_reason_code " : (
None if requested != " down "
else " vpc_private_guest_link_mismatch " ) ,
" guest_private_mac " : ( derive_private_mac (
" vpc-hk-1 " , " member-a " )
if requested != " down " else None ) ,
" guest_private_ip_cidr " : (
" 10.20.0.10/32 " if requested != " down " else None ) ,
" guest_link_state " : (
" up " if requested != " down " else None ) ,
" guest_routes " : [ ] ,
" link_state " : " down " if requested == " down " else " up " ,
} ] ,
)
@ -970,6 +1065,59 @@ def test_instance_reconcile_returns_host_verified_private_state_and_preserves_li
assert state [ " link_state " ] == " up "
def test_instance_reconcile_invalidates_stale_proof_when_fabric_is_inactive ( ) :
store = MemoryStore ( )
coordinator = VpcCoordinator (
store , FakeFabric ( hosts = ( " host-a " , ) ) )
asyncio . run ( coordinator . apply ( payload ( ) ) )
async def confirm ( plan , record , gate , previous_plan = None ,
private_link_states = None ) :
record [ " hosts " ] [ 0 ] . update (
state = " active " , observed_revision = plan . desired_revision ,
observed_digest = plan . desired_digest , transport_verified = True ,
private_dataplane_verified = True , config_verified = True ,
config_observed_revision = plan . desired_revision ,
config_observed_digest = plan . desired_digest ,
member_nics = [ {
" member_id " : " member-a " , " instance_uuid " : INSTANCE_A ,
" interface " : " private0 " ,
" private_ip_cidr " : " 10.20.0.10/32 " ,
" private_mac " : derive_private_mac (
" vpc-hk-1 " , " member-a " ) ,
" xml_verified " : True , " config_verified " : True ,
" seed_verified " : True , " restart_required " : False ,
" domain_running " : True , " guest_network_verified " : True ,
" guest_network_readback " : " confirmed " ,
" guest_reason_code " : None ,
" guest_private_mac " : derive_private_mac (
" vpc-hk-1 " , " member-a " ) ,
" guest_private_ip_cidr " : " 10.20.0.10/32 " ,
" guest_link_state " : " up " ,
" guest_routes " : [ ] , " link_state " : " up " ,
} ] ,
)
coordinator . _reconcile_dataplane = confirm
active = asyncio . run ( coordinator . reconcile_instance (
INSTANCE_A , host_id = " host-a " ,
expected = { " vpc_id " : " vpc-hk-1 " , " member_id " : " member-a " } ,
) )
assert active [ " readback " ] == " confirmed "
coordinator . fabric = FakeFabric (
state = " degraded " , gate = False , hosts = ( " host-a " , ) )
coordinator . _reconcile_dataplane = (
VpcCoordinator . _reconcile_dataplane . __get__ (
coordinator , VpcCoordinator ) )
stale = asyncio . run ( coordinator . reconcile_instance (
INSTANCE_A , host_id = " host-a " ,
expected = { " vpc_id " : " vpc-hk-1 " , " member_id " : " member-a " } ,
) )
assert stale [ " state " ] == " uncertain "
assert stale [ " readback " ] == " missing "
def test_removed_member_reconcile_uses_durable_retiring_tombstone ( ) :
store = MemoryStore ( )
coordinator = VpcCoordinator ( store , FakeFabric ( hosts = ( " host-a " , ) ) )
@ -983,13 +1131,7 @@ def test_removed_member_reconcile_uses_durable_retiring_tombstone():
async def fake_reconcile ( plan , record , gate , previous_plan = None ,
private_link_states = None ) :
host = record [ " hosts " ] [ 0 ]
host . update (
state = " active " , observed_revision = plan . desired_revision ,
observed_digest = plan . desired_digest , transport_verified = True ,
private_dataplane_verified = True , member_nics = [ ] ,
retiring_members = [ ] ,
)
confirm_delete_hosts ( plan , record )
coordinator . _reconcile_dataplane = fake_reconcile
state = asyncio . run ( coordinator . reconcile_instance (
@ -1036,10 +1178,23 @@ def test_instance_reconcile_rejects_host_mac_drift():
state = " active " , observed_revision = plan . desired_revision ,
observed_digest = plan . desired_digest , transport_verified = True ,
private_dataplane_verified = True ,
config_verified = True ,
config_observed_revision = plan . desired_revision ,
config_observed_digest = plan . desired_digest ,
member_nics = [ {
" member_id " : " member-a " , " instance_uuid " : INSTANCE_A ,
" interface " : " private0 " , " private_ip_cidr " : " 10.20.0.10/32 " ,
" private_mac " : " 02:00:00:00:00:01 " , " xml_verified " : True ,
" config_verified " : True , " seed_verified " : True ,
" restart_required " : False ,
" domain_running " : True ,
" guest_network_verified " : True ,
" guest_network_readback " : " confirmed " ,
" guest_reason_code " : None ,
" guest_private_mac " : " 02:00:00:00:00:01 " ,
" guest_private_ip_cidr " : " 10.20.0.10/32 " ,
" guest_link_state " : " up " ,
" guest_routes " : [ ] ,
" link_state " : " up " ,
} ] ,
)
@ -1086,13 +1241,7 @@ def test_removed_member_reconcile_locks_retiring_vm_identity():
async def fake_unlocked ( plan , record , gate , previous_plan = None ,
private_link_states = None ) :
host = record [ " hosts " ] [ 0 ]
host . update (
state = " active " , observed_revision = plan . desired_revision ,
observed_digest = plan . desired_digest , transport_verified = True ,
private_dataplane_verified = True , member_nics = [ ] ,
retiring_members = [ ] ,
)
confirm_delete_hosts ( plan , record )
coordinator . _reconcile_dataplane_unlocked = fake_unlocked
state = asyncio . run ( coordinator . reconcile_instance (
@ -1221,6 +1370,7 @@ def test_root_delete_accepts_revision_jump_and_superseded_unknown_host():
assert result == {
" vpc_id " : " vpc-hk-1 " ,
" operation_id " : " vpc-delete-revision-jump-0001 " ,
" evidence_version " : 2 ,
" desired_revision " : 4 ,
" desired_digest " : target [ " desired_digest " ] ,
" state " : " deleted " ,
@ -1500,6 +1650,643 @@ def test_delete_pending_projection_does_not_probe_confirmed_offline_host():
record [ " hosts " ] [ 0 ] , plan . desired_revision , plan . desired_digest )
def test_member_tombstone_closes_from_config_proof_when_sibling_not_ready ( ) :
target = payload (
desired_revision = 2 , operation_id = " vpc-op-config-proof-0002 " ,
hosts = [ { " host_id " : " host-a " , " members " : [ {
" member_id " : " member-b " , " instance_uuid " : INSTANCE_B ,
" private_ip_cidr " : " 10.20.0.11/32 " , " subnet_id " : " subnet-a " ,
} ] } ] ,
)
plan = parse_vpc_plan ( target )
fabric_plan = FabricPlan (
" fabric-hk " , " hk " , " 100.64.0.0/24 " , 7 , 51820 ,
( FabricNode (
" host-a " , " 100.64.0.1 " , " 192.0.2.1:51820 " , 51820 ) , ) ,
" f " * 64 ,
)
class RuntimeFabric ( FakeFabric ) :
@staticmethod
def _plan_from_record ( _record ) :
return fabric_plan
async def _components ( self , _plan ) :
return {
" host-a " : (
SimpleNamespace ( manager = object ( ) ) ,
{ " host_id " : " host-a " } ,
) ,
}
coordinator = VpcCoordinator (
MemoryStore ( ) , RuntimeFabric ( hosts = ( " host-a " , ) ) ,
registry = object ( ) ,
)
coordinator . _mark_lifecycle_conflicts = lambda * args , * * kwargs : False
coordinator . _validate_dataplane_resources = lambda _plan : None
coordinator . _fabric_record = lambda _plan : {
" public_keys " : { " host-a " : " a " * 44 } ,
}
coordinator . _fabric_peers_for_plan = lambda _plan , _record : [ ]
coordinator . _run_host_dataplane = lambda * args , * * kwargs : {
" state " : " applying " , " transport_verified " : True ,
" private_dataplane_verified " : False ,
" config_verified " : True ,
" config_observed_revision " : plan . desired_revision ,
" config_observed_digest " : plan . desired_digest ,
" observed_revision " : 0 , " observed_digest " : None ,
" member_nics " : [ {
" member_id " : " member-b " , " instance_uuid " : INSTANCE_B ,
" config_verified " : True , " xml_verified " : True ,
" domain_running " : False , " guest_network_verified " : False ,
" restart_required " : True ,
} ] ,
" reason_code " : " vpc_private_guest_restart_required " ,
}
record = {
" vpc_id " : plan . vpc_id ,
" desired_revision " : plan . desired_revision ,
" desired_digest " : plan . desired_digest ,
" hosts " : [ {
" host_id " : " host-a " , " member_ids " : [ " member-b " ] ,
" retiring_members " : [ retiring_member_a ( ) ] ,
" desired_revision " : plan . desired_revision ,
" desired_digest " : plan . desired_digest ,
} ] ,
}
asyncio . run ( coordinator . _reconcile_dataplane_unlocked (
plan , record , { " active " : True , " available " : True } ,
previous_plan = plan ,
) )
assert record [ " hosts " ] [ 0 ] [ " state " ] == " applying "
assert record [ " hosts " ] [ 0 ] [ " retiring_members " ] == [ ]
def test_reconcile_skips_exact_host_so_later_host_gets_next_budget ( ) :
target = payload (
hosts = [
{ " host_id " : " host-a " , " members " : [ {
" member_id " : " member-a " , " instance_uuid " : INSTANCE_A ,
" private_ip_cidr " : " 10.20.0.10/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
{ " host_id " : " host-b " , " members " : [ {
" member_id " : " member-b " , " instance_uuid " : INSTANCE_B ,
" private_ip_cidr " : " 10.20.0.11/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
] ,
)
plan = parse_vpc_plan ( target )
gate = { " active " : True , " available " : True }
coordinator = VpcCoordinator (
MemoryStore ( ) , FakeFabric ( hosts = ( " host-a " , " host-b " ) ) ,
registry = object ( ) ,
)
record = coordinator . _desired_record (
plan , gate , state = " applying " )
calls = [ ]
def exact ( host ) :
member = next (
item for item in plan . members
if item . host_id == host [ " host_id " ] )
private_mac = derive_private_mac ( plan . vpc_id , member . member_id )
host . update ( {
" state " : " active " ,
" transport_verified " : True ,
" private_dataplane_verified " : True ,
" config_verified " : True ,
" observed_revision " : plan . desired_revision ,
" observed_digest " : plan . desired_digest ,
" config_observed_revision " : plan . desired_revision ,
" config_observed_digest " : plan . desired_digest ,
" retiring_members " : [ ] ,
" member_nics " : [ {
" member_id " : member . member_id ,
" instance_uuid " : member . instance_uuid ,
" private_ip_cidr " : f " { member . private_ip } /32 " ,
" private_mac " : private_mac ,
" config_verified " : True ,
" xml_verified " : True ,
" seed_verified " : True ,
" domain_running " : True ,
" guest_network_verified " : True ,
" restart_required " : False ,
} ] ,
} )
async def consume_one_host ( _plan , execution , _gate , _previous , * * kwargs ) :
host_ids = [ host [ " host_id " ] for host in execution [ " hosts " ] ]
calls . append ( host_ids )
if len ( calls ) == 1 :
exact ( execution [ " hosts " ] [ 0 ] )
execution [ " hosts " ] [ 1 ] . update ( {
" state " : " uncertain " ,
" transport_verified " : False ,
" private_dataplane_verified " : False ,
" last_error_code " :
" vpc_private_guest_probe_deadline_exceeded " ,
} )
else :
assert host_ids == [ " host-b " ]
exact ( execution [ " hosts " ] [ 0 ] )
coordinator . _reconcile_dataplane = consume_one_host
asyncio . run ( coordinator . _reconcile_pending_hosts (
plan , record , gate , plan ) )
asyncio . run ( coordinator . _reconcile_pending_hosts (
plan , record , gate , plan ) )
assert calls == [ [ " host-a " , " host-b " ] , [ " host-b " ] ]
assert all ( host [ " state " ] == " active " for host in record [ " hosts " ] )
def test_reconcile_reprobes_oldest_active_host_and_revokes_stale_guest_proof ( ) :
target = payload ( hosts = [
{ " host_id " : " host-a " , " members " : [ {
" member_id " : " member-a " , " instance_uuid " : INSTANCE_A ,
" private_ip_cidr " : " 10.20.0.10/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
{ " host_id " : " host-b " , " members " : [ {
" member_id " : " member-b " , " instance_uuid " : INSTANCE_B ,
" private_ip_cidr " : " 10.20.0.11/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
] )
plan = parse_vpc_plan ( target )
gate = { " active " : True , " available " : True }
coordinator = VpcCoordinator (
MemoryStore ( ) , FakeFabric ( hosts = ( " host-a " , " host-b " ) ) ,
registry = object ( ) ,
)
record = coordinator . _desired_record ( plan , gate , state = " applying " )
attempted_at = { " host-a " : 200 , " host-b " : 100 }
for host in record [ " hosts " ] :
member = next (
item for item in plan . members
if item . host_id == host [ " host_id " ] )
private_mac = derive_private_mac ( plan . vpc_id , member . member_id )
host . update ( {
" state " : " active " ,
" transport_verified " : True ,
" private_dataplane_verified " : True ,
" config_verified " : True ,
" observed_revision " : plan . desired_revision ,
" observed_digest " : plan . desired_digest ,
" config_observed_revision " : plan . desired_revision ,
" config_observed_digest " : plan . desired_digest ,
" retiring_members " : [ ] ,
" probe_attempted_at_ns " : attempted_at [ host [ " host_id " ] ] ,
" member_nics " : [ {
" member_id " : member . member_id ,
" instance_uuid " : member . instance_uuid ,
" private_ip_cidr " : f " { member . private_ip } /32 " ,
" private_mac " : private_mac ,
" config_verified " : True ,
" xml_verified " : True ,
" seed_verified " : True ,
" domain_running " : True ,
" guest_network_verified " : True ,
" restart_required " : False ,
} ] ,
} )
calls = [ ]
async def lose_qga_proof ( _plan , execution , _gate , _previous , * * kwargs ) :
calls . append ( [ host [ " host_id " ] for host in execution [ " hosts " ] ] )
host = execution [ " hosts " ] [ 0 ]
host . update ( {
" state " : " applying " ,
" private_dataplane_verified " : False ,
" observed_revision " : 0 ,
" observed_digest " : None ,
" last_error_code " : " vpc_private_guest_agent_unavailable " ,
} )
host [ " member_nics " ] [ 0 ] . update ( {
" guest_network_verified " : False ,
" guest_network_readback " : " unavailable " ,
" guest_reason_code " : " vpc_private_guest_agent_unavailable " ,
} )
coordinator . _reconcile_dataplane = lose_qga_proof
asyncio . run ( coordinator . _reconcile_pending_hosts (
plan , record , gate , plan ) )
assert calls == [ [ " host-b " ] ]
hosts = { host [ " host_id " ] : host for host in record [ " hosts " ] }
assert hosts [ " host-a " ] [ " state " ] == " active "
assert hosts [ " host-b " ] [ " state " ] == " applying "
assert hosts [ " host-b " ] [ " private_dataplane_verified " ] is False
assert hosts [ " host-b " ] [ " member_nics " ] [ 0 ] [
" guest_network_verified " ] is False
assert record [ " state " ] == " applying "
assert record [ " next_stage_allowed " ] is False
assert record [ " private_dataplane_verified " ] is False
def test_reconcile_rotates_pending_host_after_it_consumes_budget ( ) :
target = payload ( hosts = [
{ " host_id " : " host-a " , " members " : [ {
" member_id " : " member-a " , " instance_uuid " : INSTANCE_A ,
" private_ip_cidr " : " 10.20.0.10/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
{ " host_id " : " host-b " , " members " : [ {
" member_id " : " member-b " , " instance_uuid " : INSTANCE_B ,
" private_ip_cidr " : " 10.20.0.11/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
] )
plan = parse_vpc_plan ( target )
gate = { " active " : True , " available " : True }
coordinator = VpcCoordinator (
MemoryStore ( ) , FakeFabric ( hosts = ( " host-a " , " host-b " ) ) ,
registry = object ( ) ,
)
record = coordinator . _desired_record (
plan , gate , state = " applying " )
calls = [ ]
async def consume_budget ( _plan , execution , _gate , _previous , * * kwargs ) :
calls . append ( [ host [ " host_id " ] for host in execution [ " hosts " ] ] )
consumer = execution [ " hosts " ] [ 0 ]
consumer [ " probe_attempted_at_ns " ] = 100 + len ( calls )
consumer . update ( {
" state " : " uncertain " ,
" transport_verified " : False ,
" private_dataplane_verified " : False ,
" last_error_code " :
" vpc_private_guest_probe_deadline_exceeded " ,
} )
coordinator . _reconcile_dataplane = consume_budget
asyncio . run ( coordinator . _reconcile_pending_hosts (
plan , record , gate , plan ) )
asyncio . run ( coordinator . _reconcile_pending_hosts (
plan , record , gate , plan ) )
assert calls == [ [ " host-a " , " host-b " ] , [ " host-b " , " host-a " ] ]
def test_instance_lifecycle_target_precedes_unrelated_pending_host ( ) :
target = payload ( hosts = [
{ " host_id " : " host-a " , " members " : [ {
" member_id " : " member-a " , " instance_uuid " : INSTANCE_A ,
" private_ip_cidr " : " 10.20.0.10/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
{ " host_id " : " host-b " , " members " : [ {
" member_id " : " member-b " , " instance_uuid " : INSTANCE_B ,
" private_ip_cidr " : " 10.20.0.11/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
] )
plan = parse_vpc_plan ( target )
gate = { " active " : True , " available " : True }
coordinator = VpcCoordinator (
MemoryStore ( ) , FakeFabric ( hosts = ( " host-a " , " host-b " ) ) ,
registry = object ( ) ,
)
record = coordinator . _desired_record ( plan , gate , state = " applying " )
record [ " hosts " ] [ 1 ] [ " probe_attempted_at_ns " ] = 999
calls = [ ]
async def capture_order ( _plan , execution , _gate , _previous , * * kwargs ) :
calls . append ( [ host [ " host_id " ] for host in execution [ " hosts " ] ] )
coordinator . _reconcile_dataplane = capture_order
asyncio . run ( coordinator . _reconcile_pending_hosts (
plan , record , gate , plan ,
private_link_states = { INSTANCE_B : " down " } ,
) )
assert calls == [ [ " host-b " , " host-a " ] ]
def test_guest_probe_pool_is_rolling_bounded_and_deadline_fail_closed (
monkeypatch ) :
import sys
import types
try :
import libvirt # noqa: F401
except ModuleNotFoundError :
stub = types . ModuleType ( " libvirt " )
stub . libvirtError = type ( " libvirtError " , ( Exception , ) , { } )
monkeypatch . setitem ( sys . modules , " libvirt " , stub )
from virsh_panel . libvirt_service import VMService
from virsh_panel . vpc_dataplane import VpcPrivateHostAdapter
members = [ {
" member_id " : f " member- { index } " ,
" instance_uuid " :
f " 00000000-0000-4000-8000- { index + 1 : 012x } " ,
" private_ip_cidr " : f " 10.20.0. { index + 10 } /32 " ,
" subnet_id " : " subnet-a " ,
} for index in range ( 10 ) ]
plan = parse_vpc_plan ( payload ( hosts = [ {
" host_id " : " host-a " , " members " : members ,
} ] ) )
monkeypatch . setattr ( " virsh_panel.vpc.VPC_GUEST_PROBE_WORKERS " , 2 )
monkeypatch . setattr ( VMService , " __init__ " , lambda self , * args , * * kwargs : None )
def configure ( self , instance_uuid , * * kwargs ) :
return {
" xml " : f " inactive: { instance_uuid } " ,
" live " : f " live: { instance_uuid } " ,
" restart_required " : False ,
}
monkeypatch . setattr ( VMService , " ensure_private_interface " , configure )
lock = threading . Lock ( )
active = 0
maximum = 0
started = [ ]
def probe ( self , instance_uuid , * * kwargs ) :
nonlocal active , maximum
assert isinstance ( kwargs . get ( " guest_probe_deadline " ) , float )
with lock :
active + = 1
maximum = max ( maximum , active )
started . append ( instance_uuid )
time . sleep ( 0.1 )
with lock :
active - = 1
return {
" inactive " : f " inactive: { instance_uuid } " ,
" live " : f " live: { instance_uuid } " ,
" guest_network " : {
" state " : " confirmed " , " verified " : True ,
" source " : " qemu_guest_agent " ,
} ,
}
monkeypatch . setattr ( VMService , " read_private_domain_pair " , probe )
monkeypatch . setattr (
VpcPrivateHostAdapter , " __init__ " ,
lambda self , manager , profile , * * kwargs : None ,
)
monkeypatch . setattr (
VpcPrivateHostAdapter , " apply " , lambda self , dataplane : { } )
captured = { }
def observe ( self , dataplane , * , xml_by_instance = None ) :
captured . update ( xml_by_instance or { } )
return {
" state " : " applying " ,
" config_verified " : True ,
" config_observed_revision " : plan . desired_revision ,
" config_observed_digest " : plan . desired_digest ,
" member_nics " : [ ] ,
" reason_code " : " vpc_private_guest_probe_deadline_exceeded " ,
}
monkeypatch . setattr ( VpcPrivateHostAdapter , " observe " , observe )
coordinator = VpcCoordinator ( MemoryStore ( ) , FakeFabric ( ) )
result = coordinator . _run_host_dataplane (
plan , " host-a " , SimpleNamespace ( manager = object ( ) ) ,
{ " libvirt_uri " : " qemu:///system " , " storage_pool " : " default " ,
" iso_pool " : " iso " , " default_network " : " default " } ,
[ ] , None , fabric_public_keys = { } ,
request_deadline = time . monotonic ( ) + 1.05 ,
)
assert maximum == 2
assert len ( started ) == 2
assert active == 0
assert len ( captured ) == 10
deadline_rows = [
pair [ " guest_network " ] for pair in captured . values ( )
if pair [ " guest_network " ] . get ( " reason_code " )
== " vpc_private_guest_probe_deadline_exceeded "
]
assert len ( deadline_rows ) == 8
assert result [ " config_verified " ] is True
assert result [ " private_dataplane_verified " ] is False
def test_guest_probe_default_pool_bounds_libvirt_reply_amplification ( ) :
from virsh_panel . vpc import VPC_GUEST_PROBE_WORKERS
# virDomainQemuAgentCommand materializes libvirt's bounded raw reply before
# Python can enforce its 192 KiB parse limit. Keep the default narrow even
# though individual tests may monkeypatch it lower for deterministic timing.
assert VPC_GUEST_PROBE_WORKERS == 4
def test_guest_probe_limit_is_process_wide_across_vpc_requests ( monkeypatch ) :
import sys
import types
try :
import libvirt # noqa: F401
except ModuleNotFoundError :
stub = types . ModuleType ( " libvirt " )
stub . libvirtError = type ( " libvirtError " , ( Exception , ) , { } )
monkeypatch . setitem ( sys . modules , " libvirt " , stub )
from virsh_panel . libvirt_service import VMService
from virsh_panel . vpc_dataplane import VpcPrivateHostAdapter
def make_plan ( label , second_octet , first_uuid ) :
members = [ {
" member_id " : f " member- { label } - { index } " ,
" instance_uuid " :
f " 00000000-0000-4000-8000- { first_uuid + index : 012x } " ,
" private_ip_cidr " :
f " 10. { second_octet } .0. { index + 10 } /32 " ,
" subnet_id " : f " subnet- { label } " ,
} for index in range ( 4 ) ]
return parse_vpc_plan ( payload (
vpc_id = f " vpc-hk- { label } " ,
cidr = f " 10. { second_octet } .0.0/24 " ,
operation_id = f " vpc-op- { label } -00000001 " ,
subnets = [ {
" subnet_id " : f " subnet- { label } " ,
" cidr " : f " 10. { second_octet } .0.0/25 " ,
} ] ,
hosts = [ { " host_id " : " host-a " , " members " : members } ] ,
) )
plans = ( make_plan ( " a " , 20 , 1 ) , make_plan ( " b " , 21 , 101 ) )
monkeypatch . setattr ( VMService , " __init__ " , lambda self , * args , * * kwargs : None )
monkeypatch . setattr (
VMService , " ensure_private_interface " ,
lambda self , instance_uuid , * * kwargs : {
" xml " : f " inactive: { instance_uuid } " ,
" live " : f " live: { instance_uuid } " ,
" restart_required " : False ,
} ,
)
lock = threading . Lock ( )
release_full_wave = threading . Event ( )
active = 0
maximum = 0
started = 0
def probe ( self , instance_uuid , * * kwargs ) :
nonlocal active , maximum , started
with lock :
active + = 1
started + = 1
maximum = max ( maximum , active )
if active > = 8 :
release_full_wave . set ( )
# Without the process gate, both four-worker request pools enter
# together. With it, a bounded wave of four leaves before the next.
release_full_wave . wait ( 0.2 )
with lock :
active - = 1
return {
" inactive " : f " inactive: { instance_uuid } " ,
" live " : f " live: { instance_uuid } " ,
" guest_network " : {
" state " : " confirmed " , " verified " : True ,
" source " : " qemu_guest_agent " ,
} ,
}
monkeypatch . setattr ( VMService , " read_private_domain_pair " , probe )
monkeypatch . setattr (
VpcPrivateHostAdapter , " __init__ " ,
lambda self , manager , profile , * * kwargs : None ,
)
monkeypatch . setattr (
VpcPrivateHostAdapter , " apply " , lambda self , dataplane : { } )
def observe ( self , dataplane , * , xml_by_instance = None ) :
return {
" state " : " active " ,
" config_verified " : True ,
" config_observed_revision " : dataplane . vpc . desired_revision ,
" config_observed_digest " : dataplane . vpc . desired_digest ,
" member_nics " : [ ] ,
}
monkeypatch . setattr ( VpcPrivateHostAdapter , " observe " , observe )
def run ( plan ) :
return VpcCoordinator (
MemoryStore ( ) , FakeFabric ( ) ,
) . _run_host_dataplane (
plan , " host-a " , SimpleNamespace ( manager = object ( ) ) ,
{ " libvirt_uri " : " qemu:///system " , " storage_pool " : " default " ,
" iso_pool " : " iso " , " default_network " : " default " } ,
[ ] , None , fabric_public_keys = { } ,
request_deadline = time . monotonic ( ) + 3 ,
)
with ThreadPoolExecutor ( max_workers = 2 ) as requests :
results = [ future . result ( ) for future in (
requests . submit ( run , plans [ 0 ] ) ,
requests . submit ( run , plans [ 1 ] ) ,
) ]
assert started == 8
assert maximum == 4
assert all ( result [ " config_verified " ] is True for result in results )
def test_request_probe_deadline_stops_next_host_and_keeps_config_proof (
monkeypatch ) :
target = payload ( hosts = [
{ " host_id " : " host-a " , " members " : [ {
" member_id " : " member-a " , " instance_uuid " : INSTANCE_A ,
" private_ip_cidr " : " 10.20.0.10/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
{ " host_id " : " host-b " , " members " : [ {
" member_id " : " member-b " , " instance_uuid " : INSTANCE_B ,
" private_ip_cidr " : " 10.20.0.11/32 " ,
" subnet_id " : " subnet-a " ,
} ] } ,
] )
plan = parse_vpc_plan ( target )
fabric_plan = FabricPlan (
" fabric-hk " , " hk " , " 100.64.0.0/24 " , 7 , 51820 ,
(
FabricNode (
" host-a " , " 100.64.0.1 " , " 192.0.2.1:51820 " , 51820 ) ,
FabricNode (
" host-b " , " 100.64.0.2 " , " 192.0.2.2:51820 " , 51820 ) ,
) ,
" f " * 64 ,
)
class DeadlineFabric ( FakeFabric ) :
@staticmethod
def _plan_from_record ( _record ) :
return fabric_plan
async def _components ( self , _plan ) :
return {
host_id : (
SimpleNamespace ( manager = object ( ) ) ,
{ " host_id " : host_id } ,
)
for host_id in ( " host-a " , " host-b " )
}
coordinator = VpcCoordinator (
MemoryStore ( ) , DeadlineFabric ( hosts = ( " host-a " , " host-b " ) ) ,
registry = object ( ) ,
)
coordinator . _mark_lifecycle_conflicts = lambda * args , * * kwargs : False
coordinator . _validate_dataplane_resources = lambda _plan : None
coordinator . _fabric_record = lambda _plan : {
" public_keys " : { " host-a " : " a " * 44 , " host-b " : " b " * 44 } ,
}
coordinator . _fabric_peers_for_plan = lambda _plan , _record : [ ]
monkeypatch . setattr ( " virsh_panel.vpc.VPC_REQUEST_DEADLINE_SECONDS " , 0.05 )
calls = [ ]
def slow_host ( * args , * * kwargs ) :
host_id = args [ 1 ]
calls . append ( host_id )
time . sleep ( 0.08 )
return {
" state " : " active " ,
" transport_verified " : True ,
" private_dataplane_verified " : True ,
" config_verified " : True ,
" config_observed_revision " : plan . desired_revision ,
" config_observed_digest " : plan . desired_digest ,
" member_nics " : [ {
" member_id " : " member-a " ,
" guest_network_verified " : True ,
} ] ,
" reason_code " : None ,
}
coordinator . _run_host_dataplane = slow_host
gate = { " active " : True , " available " : True }
record = coordinator . _desired_record (
plan , gate , state = " applying " )
asyncio . run ( coordinator . _reconcile_dataplane_unlocked (
plan , record , gate , previous_plan = plan ) )
assert calls == [ " host-a " ]
first , second = record [ " hosts " ]
assert first [ " transport_verified " ] is True
assert first [ " probe_attempted_at_ns " ] > 0
assert first [ " config_verified " ] is True
assert first [ " config_observed_revision " ] == plan . desired_revision
assert first [ " private_dataplane_verified " ] is False
assert first [ " member_nics " ] [ 0 ] [ " guest_network_verified " ] is False
assert first [ " last_error_code " ] \
== " vpc_private_guest_probe_deadline_exceeded "
assert second [ " transport_verified " ] is False
assert second [ " last_error_code " ] \
== " vpc_private_guest_probe_deadline_exceeded "
def test_root_delete_surfaces_only_allowlisted_deterministic_host_errors ( ) :
deterministic_codes = (
" vpc_private_remove_requires_stopped " ,
@ -1712,7 +2499,7 @@ def test_delete_manifest_removes_unknown_member_nic_before_host_resources(
monkeypatch . setattr ( VMService , " remove_private_interface " , remove_nic )
monkeypatch . setattr (
VpcPrivateHostAdapter , " __init__ " ,
lambda self , manager , profile : None ,
lambda self , manager , profile , * * kwargs : None ,
)
def remove_host ( self , dataplane ) :