You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc_state_overlay.py

1103 lines
45 KiB
Python

"""Check durable VPC peer state and complete Fabric overlay ranges."""
from __future__ import annotations
from dataclasses import replace
import json
import shlex
import subprocess
import sys
import pytest
from test_vpc_nft_dataplane import _plan
import virsh_panel.vpc_dataplane as dataplane_module
from virsh_panel.shell_contracts import WG_ALLOWED_IPS_SH
OTHER_WG_KEY = "AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI="
def _loader_from_script(script, *, observe=False):
start = script.index('VPC_STATE_DIR="/etc/virt-cloud/vpc-dataplane"')
end = 'FABRIC_NFT_SNAPSHOT=' if observe else 'vpc_state_load || exit 74'
return script[start:script.index(end, start)]
def _two_vpc_plans():
first = _plan()
payload = first.vpc.canonical()
payload['vpc_id'] = 'vpc-second-network'
payload['operation_id'] = 'vpc-second-network-op-0001'
payload['cidr'] = '10.45.0.0/24'
for subnet in payload['subnets']:
subnet['cidr'] = subnet['cidr'].replace('10.44.', '10.45.')
for member in payload['members']:
member['private_ip_cidr'] = member['private_ip_cidr'].replace(
'10.44.', '10.45.')
payload['fabric_peers'] = [{
'host_id': first.peers[0].host_id,
'public_key': first.peers[0].public_key,
'overlay_ip': first.peers[0].overlay_cidr,
'tenant_allowed_ips': ['10.45.0.11/32'],
}]
second = dataplane_module.parse_vpc_dataplane_plan(
payload, first.host_id, fabric_plan_digest=first.fabric_plan_digest)
return first, second
def _state_text(plan, *, cidr_after_routes=False):
text = dataplane_module._state(plan, dataplane_module._routes(plan))
if cidr_after_routes:
# State membership must be checked after parsing the complete record.
# A route can precede its CIDR without inheriting a previous file's CIDR.
lines = text.splitlines()
cidr = next(line for line in lines if line.startswith('cidr|'))
lines.remove(cidr)
lines.append(cidr)
text = '\n'.join(lines) + '\n'
return text
def _legacy_aggregate_state_text(plan, *, cidr_after_routes=False):
"""Render the pre-migration state shape with full Fabric peer closure."""
lines = _state_text(
plan, cidr_after_routes=cidr_after_routes).splitlines()
lines = [line for line in lines if not line.startswith('peer|')]
for peer in plan.peers:
if peer.tenant_allowed_ips:
lines.append(
f"peer|{peer.host_id}|{peer.public_key}|"
+ ','.join(peer.tenant_allowed_ips))
return '\n'.join(lines) + '\n'
def _with_global_peer_tenants(plan, *extra_ips):
"""Use the same trusted peer closure that the coordinator supplies."""
payload = plan.vpc.canonical()
tenants = tuple(sorted(set(plan.peers[0].tenant_allowed_ips) | set(extra_ips)))
peer = plan.peers[0]
payload['fabric_peers'] = [{
'host_id': peer.host_id, 'public_key': peer.public_key,
'overlay_ip': peer.overlay_cidr, 'tenant_allowed_ips': list(tenants),
}]
return dataplane_module.parse_vpc_dataplane_plan(
payload, plan.host_id, fabric_plan_digest=plan.fabric_plan_digest,
trusted_tenant_allowed_ips={peer.host_id: tenants})
def _run_multi_vpc_loader(tmp_path, plan, texts, mode, *, extra_script=''):
state_dir = tmp_path / 'states'
state_dir.mkdir(mode=0o700)
# The production loader sorts valid token filenames through the shell glob.
# Fixed token names let callers exchange the two real rendered records and
# prove that acceptance does not depend on which VPC is read first.
for index, text in enumerate(texts, 1):
state_file = state_dir / f'{index:011x}.state'
state_file.write_text(text)
state_file.chmod(0o600)
observe = mode == 'observe'
script = (dataplane_module.render_observe_script(plan) if observe
else dataplane_module.render_apply_script(plan))
harness = '\n'.join((
'set -eu',
_loader_from_script(script, observe=observe),
'VPC_STATE_DIR=' + shlex.quote(str(state_dir)),
'VPC_PEER_FILE=' + shlex.quote(str(tmp_path / 'peers')),
'VPC_ROUTE_FILE=' + shlex.quote(str(tmp_path / 'routes')),
'VPC_TENANT_FILE=' + shlex.quote(str(tmp_path / 'tenants')),
'FIREWALL_MODE=native_nftables',
r'''
vc_fixture_stat() {
[ "$1" = -c ] || return 9
case "$2" in
'%u:%a')
if [ "$3" = "$VPC_STATE_DIR" ]; then printf '0:700\n'
elif [ "${3%/*}" = "$VPC_STATE_DIR" ] && [ -f "$3" ]; then
printf '0:600\n'
else return 9; fi ;;
'%s') [ "${3%/*}" = "$VPC_STATE_DIR" ] && [ -f "$3" ] || return 9
wc -c < "$3" | tr -d ' ' ;;
*) return 9 ;;
esac
}
STAT=vc_fixture_stat
vpc_state_load
printf 'state=%s\n' "$VPC_STATE_STATUS"
'''.strip(),
extra_script,
))
return subprocess.run(['/bin/sh', '-c', harness], text=True,
capture_output=True, check=False)
def _run_observe_wg_contract(
tmp_path, plan, texts, actual_by_key, *, peer_read_status=0,
target_state_present=True):
"""Run the rendered observer's exact state-union/AllowedIPs block."""
state_dir = tmp_path / 'states'
state_dir.mkdir(mode=0o700)
for index, state_text in enumerate(texts, 1):
state_file = state_dir / f'{index:011x}.state'
state_file.write_text(state_text)
state_file.chmod(0o600)
wg_rows = tmp_path / 'wg-rows'
wg_rows.write_text(''.join(
f'{key}\t{allowed}\n' for key, allowed in actual_by_key.items()))
script = dataplane_module.render_observe_script(plan)
block_start = script.index('vpc_state_loaded=no')
block_end = script.index('fabric_nft_closure=no', block_start)
target_state = (
state_dir / '00000000001.state'
if target_state_present and texts
else tmp_path / 'removed-target.state'
)
harness = '\n'.join((
'set -eu',
_loader_from_script(script, observe=True),
WG_ALLOWED_IPS_SH,
'VPC_STATE_DIR=' + shlex.quote(str(state_dir)),
'VPC_PEER_FILE=' + shlex.quote(str(tmp_path / 'peers')),
'VPC_ROUTE_FILE=' + shlex.quote(str(tmp_path / 'routes')),
'VPC_TENANT_FILE=' + shlex.quote(str(tmp_path / 'tenants')),
'STATE=' + shlex.quote(str(target_state)),
'VPC_WG_ROWS=' + shlex.quote(str(wg_rows)),
'VPC_WG_PEER_READ_STATUS=' + str(peer_read_status),
'FIREWALL_MODE=native_nftables',
r'''
vc_fixture_stat() {
[ "$1" = -c ] || return 9
case "$2" in
'%u:%a')
if [ "$3" = "$VPC_STATE_DIR" ]; then printf '0:700\n'
elif [ "${3%/*}" = "$VPC_STATE_DIR" ] && [ -f "$3" ]; then
printf '0:600\n'
else return 9; fi ;;
'%s') [ "${3%/*}" = "$VPC_STATE_DIR" ] && [ -f "$3" ] || return 9
wc -c < "$3" | tr -d ' ' ;;
*) return 9 ;;
esac
}
vc_fixture_wg() {
[ "$1" = show ] && [ "$2" = wg-fabric0 ] && [ "$#" = 3 ] || return 9
case "$3" in
allowed-ips) cat "$VPC_WG_ROWS" ;;
peers)
[ "$VPC_WG_PEER_READ_STATUS" = 0 ] ||
return "$VPC_WG_PEER_READ_STATUS"
cut -f1 "$VPC_WG_ROWS" ;;
*) return 9 ;;
esac
}
STAT=vc_fixture_stat
WG=vc_fixture_wg
WG_IFACE=wg-fabric0
'''.strip(),
script[block_start:block_end],
))
return subprocess.run(['/bin/sh', '-c', harness], text=True,
capture_output=True, check=False)
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('reverse', [False, True])
@pytest.mark.parametrize('cidr_after_routes', [False, True])
def test_multi_vpc_state_load_keeps_each_route_in_its_own_cidr(
tmp_path, mode, reverse, cidr_after_routes):
first, second = _two_vpc_plans()
texts = [_state_text(plan, cidr_after_routes=cidr_after_routes)
for plan in (first, second)]
result = _run_multi_vpc_loader(
tmp_path, first, texts[::-1] if reverse else texts, mode)
assert result.returncode == 0, result.stderr
assert result.stdout == 'state=owned\n'
expected_routes = []
for plan, prefix in ((first, '10.44.0'), (second, '10.45.0')):
table = plan.resources.route_table
expected_routes.extend([
f'{prefix}.10/32|{plan.resources.bridge}|{table}|-|100',
f'{prefix}.11/32|wg-fabric0|{table}|-|100',
])
assert sorted((tmp_path / 'routes').read_text().splitlines()) == sorted(
expected_routes)
assert sorted((tmp_path / 'tenants').read_text().splitlines()) == [
'10.44.0.10/32', '10.44.0.11/32',
'10.45.0.10/32', '10.45.0.11/32',
]
key = first.peers[0].public_key
assert sorted((tmp_path / 'peers').read_text().splitlines()) == [
key + '|10.44.0.11/32', key + '|10.45.0.11/32',
]
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('records', ['single', 'forward', 'reverse'])
@pytest.mark.parametrize('cidr_after_routes', [False, True])
def test_global_peer_tenants_join_routes_once_in_the_fabric_address_universe(
tmp_path, mode, records, cidr_after_routes):
first, second = _two_vpc_plans()
extra = '10.46.0.21/32'
first = _with_global_peer_tenants(first, extra)
plans = [first] if records == 'single' else [first, second]
if records == 'reverse':
plans.reverse()
texts = [_state_text(plan, cidr_after_routes=cidr_after_routes)
for plan in plans]
result = _run_multi_vpc_loader(tmp_path, first, texts, mode)
assert result.returncode == 0, result.stderr
assert result.stdout == 'state=owned\n'
route_ips = {row['destination_cidr'] for plan in plans
for row in dataplane_module._routes(plan)}
expected = sorted(route_ips)
actual = (tmp_path / 'tenants').read_text().splitlines()
assert sorted(actual) == expected
assert len(actual) == len(set(actual))
expected_peers = sorted(
plan.peers[0].public_key + '|' + row['destination_cidr']
for plan in plans for row in dataplane_module._routes(plan)
if row['device'] == 'wg-fabric0')
assert sorted((tmp_path / 'peers').read_text().splitlines()) == expected_peers
# A trusted aggregate entry contributes no durable route, peer, or nft
# authorization. Runtime convergence still checks it through _allowed().
routes = (tmp_path / 'routes').read_text().splitlines()
assert {row.split('|', 1)[0] for row in routes} == route_ips
assert extra not in route_ips
assert extra not in actual
assert all(not row.endswith('|' + extra)
for row in (tmp_path / 'peers').read_text().splitlines())
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('cidr_after_routes', [False, True])
def test_peer_only_state_does_not_authorize_aggregate_without_owned_routes(
tmp_path, mode, cidr_after_routes):
carrier = _with_global_peer_tenants(_plan(), '10.46.0.21/32')
# A retained record may outlive all local routes. Render it with the new
# producer and extract the loader from a participating-host transaction.
peer_only = replace(carrier, members=())
text = _state_text(peer_only, cidr_after_routes=cidr_after_routes)
assert '\nroute|' not in text and '\nlocal_neighbor|' not in text
result = _run_multi_vpc_loader(tmp_path, carrier, [text], mode)
assert result.returncode == 0, result.stderr
assert result.stdout == 'state=owned\n'
assert (tmp_path / 'routes').read_text() == ''
assert (tmp_path / 'peers').read_text() == ''
assert (tmp_path / 'tenants').read_text() == ''
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('reverse', [False, True])
@pytest.mark.parametrize('cidr_after_routes', [False, True])
def test_identical_global_peer_closure_is_deduplicated_across_vpc_states(
tmp_path, mode, reverse, cidr_after_routes):
first, second = _two_vpc_plans()
aggregate = (
'10.44.0.11/32', '10.45.0.11/32', '10.46.0.21/32',
)
first = _with_global_peer_tenants(first, *aggregate)
second = _with_global_peer_tenants(second, *aggregate)
assert first.peers[0].tenant_allowed_ips == aggregate
assert second.peers[0].tenant_allowed_ips == aggregate
texts = [_legacy_aggregate_state_text(
plan, cidr_after_routes=cidr_after_routes)
for plan in (first, second)]
result = _run_multi_vpc_loader(
tmp_path, first, texts[::-1] if reverse else texts, mode)
assert result.returncode == 0, result.stderr
assert result.stdout == 'state=owned\n'
key = first.peers[0].public_key
peer_rows = (tmp_path / 'peers').read_text().splitlines()
route_backed = aggregate[:2]
assert sorted(peer_rows) == [key + '|' + address
for address in route_backed]
assert len(peer_rows) == len(set(peer_rows))
route_ips = {
row['destination_cidr']
for plan in (first, second)
for row in dataplane_module._routes(plan)
}
tenant_ips = (tmp_path / 'tenants').read_text().splitlines()
assert tenant_ips == sorted(route_ips)
assert len(tenant_ips) == len(set(tenant_ips))
assert all(not row.endswith('|' + aggregate[2]) for row in peer_rows)
assert aggregate[2] not in tenant_ips
def test_new_state_persists_member_and_service_routes_not_fabric_aggregate():
base = _plan()
payload = base.vpc.canonical()
payload['routes'] = [{
'route_id': 'service-b',
'destination_cidr': '10.44.0.100/32',
'next_hop_member_id': 'member-b',
'metric': 20,
}]
tenants = ('10.44.0.100/32', '10.44.0.11/32', '10.46.0.21/32')
peer = base.peers[0]
payload['fabric_peers'] = [{
'host_id': peer.host_id,
'public_key': peer.public_key,
'overlay_ip': peer.overlay_cidr,
'tenant_allowed_ips': list(tenants),
}]
plan = dataplane_module.parse_vpc_dataplane_plan(
payload, base.host_id, fabric_plan_digest=base.fabric_plan_digest,
trusted_tenant_allowed_ips={peer.host_id: tenants})
state = _state_text(plan)
key = plan.peers[0].public_key
assert (f'peer|host-b|{key}|'
'10.44.0.100/32,10.44.0.11/32\n') in state
assert '10.46.0.21/32' not in state
assert set(dataplane_module._allowed(plan, plan.peers[0]).split(',')) == {
plan.peers[0].overlay_cidr,
'10.44.0.100/32',
'10.44.0.11/32',
'10.46.0.21/32',
}
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('reverse', [False, True])
@pytest.mark.parametrize('cidr_after_routes', [False, True])
def test_legacy_and_new_state_mix_keeps_unique_route_backed_peer_rows(
tmp_path, mode, reverse, cidr_after_routes):
first, second = _two_vpc_plans()
aggregate = (
'10.44.0.11/32', '10.45.0.11/32', '10.46.0.21/32',
)
first = _with_global_peer_tenants(first, *aggregate)
second = _with_global_peer_tenants(second, *aggregate)
texts = [
_legacy_aggregate_state_text(
first, cidr_after_routes=cidr_after_routes),
_state_text(second, cidr_after_routes=cidr_after_routes),
]
result = _run_multi_vpc_loader(
tmp_path, first, texts[::-1] if reverse else texts, mode)
assert result.returncode == 0, result.stderr
key = first.peers[0].public_key
assert sorted((tmp_path / 'peers').read_text().splitlines()) == [
key + '|10.44.0.11/32', key + '|10.45.0.11/32',
]
assert (tmp_path / 'tenants').read_text().splitlines() == [
'10.44.0.10/32', '10.44.0.11/32',
'10.45.0.10/32', '10.45.0.11/32',
]
@pytest.mark.parametrize('mode', ['apply', 'observe'])
def test_legacy_aggregate_withdraws_deleted_vpc_route_ownership(
tmp_path, mode):
first, second = _two_vpc_plans()
aggregate = ('10.44.0.11/32', '10.45.0.11/32')
first = _with_global_peer_tenants(first, *aggregate)
second = _with_global_peer_tenants(second, *aggregate)
legacy = [_legacy_aggregate_state_text(plan)
for plan in (first, second)]
before = tmp_path / 'before'
before.mkdir()
loaded = _run_multi_vpc_loader(before, first, legacy, mode)
assert loaded.returncode == 0, loaded.stderr
key = first.peers[0].public_key
assert sorted((before / 'peers').read_text().splitlines()) == [
key + '|10.44.0.11/32', key + '|10.45.0.11/32',
]
assert (before / 'tenants').read_text().splitlines() == [
'10.44.0.10/32', '10.44.0.11/32',
'10.45.0.10/32', '10.45.0.11/32',
]
after = tmp_path / 'after'
after.mkdir()
removed = _run_multi_vpc_loader(after, first, [legacy[0]], mode)
assert removed.returncode == 0, removed.stderr
assert (after / 'peers').read_text() == key + '|10.44.0.11/32\n'
assert (after / 'tenants').read_text().splitlines() == [
'10.44.0.10/32', '10.44.0.11/32',
]
def test_observer_uses_only_local_route_backed_state_for_interleaved_topology(
tmp_path):
# Fabric A/B/C has VPC1 on A+B and VPC2 on B+C. The catalog aggregate for
# peer B includes both B addresses, but host A has only VPC1 state and must
# therefore require only overlay(B)+VPC1(B).
first, _second = _two_vpc_plans()
catalog_view = _with_global_peer_tenants(first, '10.45.0.11/32')
peer = catalog_view.peers[0]
result = _run_observe_wg_contract(
tmp_path, catalog_view, [_state_text(first)], {
peer.public_key: peer.overlay_cidr + ',10.44.0.11/32',
})
assert result.returncode == 0, result.stderr
assert (f'__VC_VPC_WG_STATE__={peer.host_id}|owned\n'
in result.stdout)
allowed = next(
line for line in result.stdout.splitlines()
if line.startswith('__VC_VPC_WG_ALLOWED__='))
assert '10.44.0.11/32' in allowed
assert '10.45.0.11/32' not in allowed
def test_observer_exact_union_withdraws_deleted_same_peer_state(tmp_path):
first, second = _two_vpc_plans()
aggregate = ('10.44.0.11/32', '10.45.0.11/32')
observed_plan = _with_global_peer_tenants(first, *aggregate)
peer = observed_plan.peers[0]
union = peer.overlay_cidr + ',' + ','.join(aggregate)
before = tmp_path / 'before'
before.mkdir()
combined = _run_observe_wg_contract(
before, observed_plan, [_state_text(first), _state_text(second)],
{peer.public_key: union})
assert combined.returncode == 0, combined.stderr
assert f'__VC_VPC_WG_STATE__={peer.host_id}|owned' in combined.stdout
stale_dir = tmp_path / 'stale-after-delete'
stale_dir.mkdir()
stale = _run_observe_wg_contract(
stale_dir, observed_plan, [_state_text(first)],
{peer.public_key: union})
assert stale.returncode == 0, stale.stderr
assert f'__VC_VPC_WG_STATE__={peer.host_id}|missing' in stale.stdout
exact_dir = tmp_path / 'exact-after-delete'
exact_dir.mkdir()
exact = _run_observe_wg_contract(
exact_dir, observed_plan, [_state_text(first)], {
peer.public_key: peer.overlay_cidr + ',10.44.0.11/32',
})
assert exact.returncode == 0, exact.stderr
assert f'__VC_VPC_WG_STATE__={peer.host_id}|owned' in exact.stdout
def test_observer_rejects_stale_tenant_on_authoritative_non_vpc_peer(
tmp_path):
first, second = _two_vpc_plans()
first_key = first.peers[0].public_key
second = replace(second, peers=(replace(
second.peers[0], public_key=OTHER_WG_KEY,
overlay_ip='100.64.0.3'),))
observed = replace(
first,
fabric_peer_keys=frozenset({first_key, OTHER_WG_KEY}),
fabric_peer_overlays=frozenset({
(first_key, '100.64.0.2/32'),
(OTHER_WG_KEY, '100.64.0.3/32'),
}),
)
rows = {
first_key: '100.64.0.2/32,10.44.0.11/32',
OTHER_WG_KEY: '100.64.0.3/32,10.45.0.11/32',
}
exact_dir = tmp_path / 'exact-closure'
exact_dir.mkdir()
exact = _run_observe_wg_contract(
exact_dir, observed, [_state_text(first), _state_text(second)], rows)
assert exact.returncode == 0, exact.stderr
assert '__VC_VPC_WG_PEER_SET__=exact\n' in exact.stdout
assert '__VC_VPC_REMOVAL_STATE_CLOSURE__=invalid\n' in exact.stdout
assert '__VC_VPC_REMOVAL_WG_PEER_SET__=exact\n' not in exact.stdout
wrong_overlay_dir = tmp_path / 'wrong-overlay-closure'
wrong_overlay_dir.mkdir()
wrong_overlay = _run_observe_wg_contract(
wrong_overlay_dir, observed,
[_state_text(first), _state_text(second)], {
**rows,
OTHER_WG_KEY: '100.64.0.4/32,10.45.0.11/32',
})
assert wrong_overlay.returncode == 0, wrong_overlay.stderr
assert '__VC_VPC_WG_PEER_SET__=missing\n' in wrong_overlay.stdout
# VPC2's final route/state was withdrawn while its host remains a valid
# Fabric peer. The stale AllowedIP must invalidate VPC1's host readback too;
# otherwise WireGuard can still route VPC2's deleted address to that key.
stale_dir = tmp_path / 'stale-closure'
stale_dir.mkdir()
stale = _run_observe_wg_contract(
stale_dir, observed, [_state_text(first)], rows)
assert stale.returncode == 0, stale.stderr
assert '__VC_VPC_WG_PEER_SET__=missing\n' in stale.stdout
assert '__VC_VPC_WG_PEER_SET__=exact\n' not in stale.stdout
def test_observer_empty_authoritative_peer_set_requires_successful_readback(
tmp_path):
base = _plan()
plan = replace(
base,
members=tuple(member for member in base.members
if member.host_id == base.host_id),
peers=(),
fabric_peer_keys=frozenset(),
fabric_peer_overlays=frozenset(),
)
result = _run_observe_wg_contract(
tmp_path, plan, [_state_text(plan)], {}, peer_read_status=1)
assert result.returncode == 0, result.stderr
assert '__VC_VPC_WG_PEER_SET__=missing\n' in result.stdout
assert '__VC_VPC_WG_PEER_SET__=exact\n' not in result.stdout
def test_removal_wg_closure_accepts_empty_remaining_state(tmp_path):
plan = _plan()
plan = replace(
plan,
fabric_peer_keys=frozenset(peer.public_key for peer in plan.peers),
fabric_peer_overlays=frozenset(
(peer.public_key, peer.overlay_cidr) for peer in plan.peers),
)
peer = plan.peers[0]
result = _run_observe_wg_contract(
tmp_path, plan, [], {peer.public_key: peer.overlay_cidr},
target_state_present=False)
assert result.returncode == 0, result.stderr
assert '__VC_VPC_REMOVAL_STATE_CLOSURE__=empty\n' in result.stdout
assert '__VC_VPC_REMOVAL_WG_PEER_SET__=exact\n' in result.stdout
assert '__VC_VPC_WG_PEER_SET__=exact\n' not in result.stdout
def test_removal_wg_closure_rejects_stale_tenant_after_last_state(tmp_path):
plan = _plan()
plan = replace(
plan,
fabric_peer_keys=frozenset(peer.public_key for peer in plan.peers),
fabric_peer_overlays=frozenset(
(peer.public_key, peer.overlay_cidr) for peer in plan.peers),
)
peer = plan.peers[0]
result = _run_observe_wg_contract(
tmp_path, plan, [], {
peer.public_key: peer.overlay_cidr + ',10.44.0.11/32',
}, target_state_present=False)
assert result.returncode == 0, result.stderr
assert '__VC_VPC_REMOVAL_STATE_CLOSURE__=empty\n' in result.stdout
assert '__VC_VPC_REMOVAL_WG_PEER_SET__=missing\n' in result.stdout
assert '__VC_VPC_REMOVAL_WG_PEER_SET__=exact\n' not in result.stdout
def test_removal_wg_closure_accepts_other_owned_vpc_state(tmp_path):
first, second = _two_vpc_plans()
first = replace(
first,
fabric_peer_keys=frozenset(peer.public_key for peer in first.peers),
fabric_peer_overlays=frozenset(
(peer.public_key, peer.overlay_cidr) for peer in first.peers),
)
peer = first.peers[0]
result = _run_observe_wg_contract(
tmp_path, first, [_state_text(second)], {
peer.public_key: peer.overlay_cidr + ',10.45.0.11/32',
}, target_state_present=False)
assert result.returncode == 0, result.stderr
assert '__VC_VPC_REMOVAL_STATE_CLOSURE__=owned\n' in result.stdout
assert '__VC_VPC_REMOVAL_WG_PEER_SET__=exact\n' in result.stdout
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('cidr_after_routes', [False, True])
def test_global_peer_authorization_cannot_authorize_a_foreign_state_route(
tmp_path, mode, cidr_after_routes):
extra = '10.46.0.21/32'
plan = _with_global_peer_tenants(_plan(), extra)
text = _state_text(plan, cidr_after_routes=cidr_after_routes).replace(
'route|10.44.0.11/32|', 'route|' + extra + '|')
result = _run_multi_vpc_loader(tmp_path, plan, [text], mode)
assert result.returncode == (1 if mode == 'observe' else 71), result.stderr
assert 'state=owned' not in result.stdout
@pytest.mark.parametrize('mode', ['apply', 'observe'])
def test_tenant_union_deduplication_cannot_hide_duplicate_route_ownership(
tmp_path, mode):
plan = _with_global_peer_tenants(_plan(), '10.46.0.21/32')
text = _state_text(plan)
# A remote route bypasses the local-neighbor/direct-route check. Its
# duplicate must still fail before tenant-set de-duplication can hide it.
text += next(line for line in text.splitlines()
if line.startswith('route|') and '|wg-fabric0|' in line) + '\n'
result = _run_multi_vpc_loader(tmp_path, plan, [text], mode)
assert result.returncode == (1 if mode == 'observe' else 71), result.stderr
assert 'state=owned' not in result.stdout
def _stateful_tenant_nft(tmp_path, mutation):
tool = tmp_path / 'nft'
tool.write_text(
'#!' + sys.executable + '\nROOT=' + repr(str(tmp_path))
+ '\nMUTATION=' + repr(mutation) + '\n' + r'''
import ipaddress,json,re,sys
from pathlib import Path
root=Path(ROOT);args=sys.argv[1:]
if args==['-s','list','table','inet','virt_cloud_wg_fabric']:
print('fixture-owned-fabric-table')
elif args[:2]==['-c','-f'] or args[:1]==['-f']:
batch=Path(args[-1]).read_text()
lines=batch.splitlines()
assert lines[0]=='flush set inet virt_cloud_wg_fabric vpc_tenant_ips',lines
match=re.fullmatch(r'add element inet virt_cloud_wg_fabric vpc_tenant_ips \{ (.*) \}',lines[1])
assert len(lines)==2 and match,batch
values=[str(ipaddress.ip_network(value.strip(),strict=True))
for value in match[1].split(',')]
assert all(value.endswith('/32') for value in values),values
assert len(values)==len(set(values)),values
(root/('checked.batch' if args[0]=='-c' else 'applied.batch')).write_text(batch)
if args[0]!='-c':
if MUTATION=='missing':values.remove('10.44.0.11/32')
elif MUTATION=='extra':values.append('10.47.0.99/32')
(root/'live-set.json').write_text(json.dumps(sorted(values)))
elif args==['list','set','inet','virt_cloud_wg_fabric','vpc_tenant_ips']:
values=json.loads((root/'live-set.json').read_text())
print('table inet virt_cloud_wg_fabric {\n set vpc_tenant_ips {\n type ipv4_addr\n flags interval')
print(' elements = { '+', '.join(value.removesuffix('/32') for value in values)+' }')
print(' }\n}')
else:raise SystemExit(90)
''')
tool.chmod(0o700)
return tool
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('mutation', ['none', 'missing', 'extra'])
def test_global_tenant_refresh_and_fabric_observer_share_the_exact_universe(
tmp_path, mode, mutation):
from test_fabric import FakeManager, KEY1, KEY2, plan as fabric_plan
from virsh_panel.fabric import FabricHostAdapter, _VPC_NFT_OBSERVE_FRAGMENT
base = _plan()
carrier = replace(base, peers=(replace(base.peers[0], public_key=KEY2),))
carrier = _with_global_peer_tenants(carrier, '10.46.0.21/32')
nft = _stateful_tenant_nft(tmp_path, mutation)
script = '\n'.join((
'NFT=' + shlex.quote(str(nft)),
'EXPECTED_DIGEST=' + shlex.quote(carrier.fabric_plan_digest),
# Ownership is independent of the tenant-universe contract exercised
# here. Both exact-state reads must see this same owned table.
'nft_plan_owned() { [ "$1" = fixture-owned-fabric-table ] && '
'[ "$2" = "$EXPECTED_DIGEST" ]; }',
'vpc_state_refresh_nft',
'if vpc_state_nft_set_matches; then printf "nft_matches=yes\\n"; '
'else printf "nft_matches=no\\n"; fi',
_VPC_NFT_OBSERVE_FRAGMENT,
))
result = _run_multi_vpc_loader(
tmp_path, carrier, [_state_text(carrier)], mode, extra_script=script)
assert result.returncode == 0, result.stderr
wanted = ['10.44.0.10/32', '10.44.0.11/32']
assert (tmp_path / 'checked.batch').read_text() == (
tmp_path / 'applied.batch').read_text()
matches = mutation == 'none'
assert ('nft_matches=yes\n' in result.stdout) == matches
# Use the real loader output and real nft observation fragment for every
# VPC marker. The existing manager supplies only the unrelated healthy
# Fabric background; the adapter's strict union comparison is unchanged.
peer_rows = (tmp_path / 'peers').read_text().splitlines()
tenant_ips = (tmp_path / 'tenants').read_text().splitlines()
nft_markers = '\n'.join(line for line in result.stdout.splitlines()
if line.startswith('__VC_FABRIC_VPC_NFT_')) + '\n'
current = fabric_plan()
manager = FakeManager(KEY1)
original_exec = manager._exec
def readback(argv, *, timeout, input_text=None):
background = original_exec(argv, timeout=timeout, input_text=input_text)
output = '\n'.join(line for line in background.stdout.splitlines()
if not line.startswith('__VC_FABRIC_VPC_')) + '\n'
peer = current.nodes[1]
marker = f'__VC_FABRIC_PEER__={KEY2}|{peer.endpoint}|{peer.allowed_ip}'
extras = ','.join(row.split('|', 1)[1] for row in peer_rows)
output = output.replace(marker + '\n', marker + ',' + extras + '\n')
output += '__VC_FABRIC_VPC_STATE__=owned\n'
output += ''.join('__VC_FABRIC_VPC_TENANT__=' + row + '\n'
for row in peer_rows)
output += ''.join('__VC_FABRIC_VPC_TENANT_IP__=' + ip + '\n'
for ip in tenant_ips)
output += nft_markers
return background.__class__(background.args, background.returncode,
output, background.stderr)
manager._exec = readback
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {'host-a': KEY1, 'host-b': KEY2})
assert observed['observed']['peer_allowed_exact'] is True
assert observed['observed']['vpc_nft_state'] == 'owned'
assert observed['observed']['vpc_nft_exact'] is matches
assert observed['config_ok'] is matches
assert 'add element inet virt_cloud_wg_fabric vpc_tenant_ips { ' + (
', '.join(wanted)) + ' }' in (tmp_path / 'applied.batch').read_text()
assert observed['observed']['vpc_nft_ips'] == json.loads(
(tmp_path / 'live-set.json').read_text())
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('reverse', [False, True])
@pytest.mark.parametrize('cidr_after_routes', [False, True])
@pytest.mark.parametrize('foreign_prefix', ['10.45.0.99/32', '10.46.0.99/32'])
def test_multi_vpc_state_load_rejects_routes_outside_the_owning_record(
tmp_path, mode, reverse, cidr_after_routes, foreign_prefix):
first, second = _two_vpc_plans()
bad = _state_text(first, cidr_after_routes=cidr_after_routes).replace(
'route|10.44.0.11/32|', 'route|' + foreign_prefix + '|')
texts = [bad, _state_text(second)]
result = _run_multi_vpc_loader(
tmp_path, first, texts[::-1] if reverse else texts, mode)
assert result.returncode == (1 if mode == 'observe' else 71), result.stderr
assert 'state=owned' not in result.stdout
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('reverse', [False, True])
def test_multi_vpc_state_load_rejects_one_peer_ip_with_different_keys(
tmp_path, mode, reverse):
first, second = _two_vpc_plans()
key = first.peers[0].public_key
duplicate = _state_text(second).replace(
'peer|' + second.peers[0].host_id + '|' + key + '|10.45.0.11/32',
'peer|' + second.peers[0].host_id + '|' + OTHER_WG_KEY
+ '|10.44.0.11/32,10.45.0.11/32')
texts = [_state_text(first), duplicate]
result = _run_multi_vpc_loader(
tmp_path, first, texts[::-1] if reverse else texts, mode)
assert result.returncode == (1 if mode == 'observe' else 71), result.stderr
assert 'state=owned' not in result.stdout
@pytest.mark.parametrize('mode', ['apply', 'observe'])
def test_vpc_state_load_rejects_duplicate_peer_ip_within_one_state(
tmp_path, mode):
plan = _plan()
text = _state_text(plan)
text += next(line for line in text.splitlines()
if line.startswith('peer|')) + '\n'
result = _run_multi_vpc_loader(tmp_path, plan, [text], mode)
assert result.returncode == (1 if mode == 'observe' else 71), result.stderr
assert 'state=owned' not in result.stdout
@pytest.mark.parametrize('mode', ['apply', 'observe'])
def test_vpc_state_load_requires_one_key_for_every_wireguard_route(
tmp_path, mode):
plan = _plan()
text = '\n'.join(line for line in _state_text(plan).splitlines()
if not line.startswith('peer|')) + '\n'
result = _run_multi_vpc_loader(tmp_path, plan, [text], mode)
assert result.returncode == (1 if mode == 'observe' else 71), result.stderr
assert 'state=owned' not in result.stdout
@pytest.mark.parametrize('mode', ['apply', 'observe'])
@pytest.mark.parametrize('reverse', [False, True])
def test_legacy_aggregate_cannot_mask_missing_peer_in_route_owner(
tmp_path, mode, reverse):
first, second = _two_vpc_plans()
broken_owner = '\n'.join(
line for line in _state_text(first).splitlines()
if not line.startswith('peer|')) + '\n'
legacy_carrier = _legacy_aggregate_state_text(
_with_global_peer_tenants(second, '10.44.0.11/32'))
texts = [broken_owner, legacy_carrier]
result = _run_multi_vpc_loader(
tmp_path, first, texts[::-1] if reverse else texts, mode)
assert result.returncode == (1 if mode == 'observe' else 71), result.stderr
assert 'state=owned' not in result.stdout
@pytest.mark.parametrize('peer_mode', ['none', 'valid', 'duplicate_ip', 'public_ip'])
def test_full_vpc_state_load_handles_peers_under_nounset(tmp_path, peer_mode):
plan = _plan()
text = dataplane_module._state(plan, dataplane_module._routes(plan))
if peer_mode == 'none':
text = '\n'.join(line for line in text.splitlines()
if not line.startswith('peer|')) + '\n'
elif peer_mode == 'duplicate_ip':
text += next(line for line in text.splitlines()
if line.startswith('peer|')) + '\n'
elif peer_mode == 'public_ip':
text = text.replace('10.44.0.11/32', '203.0.113.11/32')
state_dir = tmp_path / 'states'
state_dir.mkdir(mode=0o700)
state_file = state_dir / (plan.resources.token + '.state')
state_file.write_text(text)
state_file.chmod(0o600)
script = dataplane_module.render_apply_script(plan)
# The parser executes unchanged with set -u. This fixture adapter replaces
# GNU stat/root ownership only, so macOS can execute the same shell test.
harness = '\n'.join((
'set -eu',
_loader_from_script(script),
'VPC_STATE_DIR=' + shlex.quote(str(state_dir)),
'VPC_FIXTURE_STATE=' + shlex.quote(str(state_file)),
'VPC_PEER_FILE=' + shlex.quote(str(tmp_path / 'peers')),
'VPC_ROUTE_FILE=' + shlex.quote(str(tmp_path / 'routes')),
'VPC_TENANT_FILE=' + shlex.quote(str(tmp_path / 'tenants')),
'FIREWALL_MODE=native_nftables',
r'''
vc_fixture_stat() {
[ "$1" = -c ] || return 9
case "$2" in
'%u:%a')
if [ "$3" = "$VPC_STATE_DIR" ]; then printf '0:700\n'
elif [ "$3" = "$VPC_FIXTURE_STATE" ]; then printf '0:600\n'
else return 9; fi ;;
'%s') [ "$3" = "$VPC_FIXTURE_STATE" ] || return 9
wc -c < "$3" | tr -d ' ' ;;
*) return 9 ;;
esac
}
STAT=vc_fixture_stat
vpc_state_load
printf 'state=%s\n' "$VPC_STATE_STATUS"
'''.strip(),
))
result = subprocess.run(['/bin/sh', '-c', harness], text=True,
capture_output=True, check=False)
if peer_mode == 'valid':
assert result.returncode == 0, result.stderr
assert result.stdout == 'state=owned\n'
peers = (tmp_path / 'peers').read_text()
assert peers == plan.peers[0].public_key + '|10.44.0.11/32\n'
else:
assert result.returncode != 0
assert 'state=owned' not in result.stdout
@pytest.mark.parametrize('transaction', ['apply', 'remove'])
@pytest.mark.parametrize(('actual', 'overlay'), [
('100.64.0.2/32,10.44.0.99/32', '100.64.0.2/32'),
('10.44.0.99/32,100.99.106.2/32', '100.99.106.2/32'),
('100.127.255.254/32,10.44.0.99/32', '100.127.255.254/32'),
('100.99.106.2/32', '100.99.106.2/32'),
('100.63.255.254/32,10.44.0.99/32', None),
('100.128.0.2/32,10.44.0.99/32', None),
('100.99.106.2/24,10.44.0.99/32', None),
('100.99.256.2/32,10.44.0.99/32', None),
('100.99.106.2/32,100.64.0.2/32', None),
('100.99.106.2/32,100.99.106.2/32', None),
('100.99.106.2/32,0.0.0.0/0', None),
('100.99.106.2/32,203.0.113.1/32', None),
('100.99.106.2/32,,10.44.0.99/32', None),
('100.99.106.2/32,', None),
('', None),
])
def test_vpc_apply_and_remove_merge_actual_full_fabric_overlay(
tmp_path, transaction, actual, overlay):
plan = _plan()
expected_overlay = overlay or '100.99.106.2/32'
plan = replace(plan, peers=(replace(
plan.peers[0], overlay_ip=expected_overlay.removesuffix('/32')),))
if transaction == 'apply':
script = dataplane_module.render_apply_script(plan)
start = script.index('expected_fabric_peer_keys=')
end = script.index('# The shared Fabric quarantine table', start)
else:
script = dataplane_module.render_remove_script(replace(
plan, members=tuple(m for m in plan.members
if m.host_id != plan.host_id)))
start = script.index('expected_fabric_peer_keys=')
end = script.index('vc_vpc_fabric_nft_live_ready || exit 74', start)
merge = script[start:end]
peer_file = tmp_path / 'peers'
peer_file.write_text(plan.peers[0].public_key + '|10.44.0.11/32\n')
actual_file = tmp_path / 'actual'
actual_file.write_text(actual)
mutations = tmp_path / 'mutations'
mutations.write_text('')
harness = '\n'.join((
'set -eu',
_loader_from_script(script),
WG_ALLOWED_IPS_SH,
'VPC_PEER_FILE=' + shlex.quote(str(peer_file)),
'VC_ACTUAL_FILE=' + shlex.quote(str(actual_file)),
'VC_KEY=' + shlex.quote(plan.peers[0].public_key),
'VC_MUTATIONS=' + shlex.quote(str(mutations)),
'WG_IFACE=wg-fabric0',
r'''
vc_fixture_wg() {
if [ "$*" = 'show wg-fabric0 peers' ]; then
printf '%s\n' "$VC_KEY"
elif [ "$*" = 'show wg-fabric0 allowed-ips' ]; then
printf '%s\t%s\n' "$VC_KEY" "$(cat "$VC_ACTUAL_FILE")"
elif [ "$1" = set ] && [ "$2" = wg-fabric0 ] &&
[ "$3" = peer ] && [ "$4" = "$VC_KEY" ] &&
[ "$5" = allowed-ips ]; then
printf '%s' "$6" > "$VC_ACTUAL_FILE"
printf '%s\n' "$6" >> "$VC_MUTATIONS"
else return 9; fi
}
WG=vc_fixture_wg
'''.strip(),
merge,
))
result = subprocess.run(['/bin/sh', '-c', harness], text=True,
capture_output=True, check=False)
if overlay is not None:
assert result.returncode == 0, result.stderr
assert mutations.read_text() == overlay + ',10.44.0.11/32\n'
else:
assert result.returncode == 74, result.stderr
assert mutations.read_text() == ''
def test_vpc_apply_converges_full_authoritative_fabric_peer_closure(tmp_path):
base = _plan()
first_key = base.peers[0].public_key
plan = replace(
base,
fabric_peer_keys=frozenset({first_key, OTHER_WG_KEY}),
fabric_peer_overlays=frozenset({
(first_key, '100.64.0.2/32'),
(OTHER_WG_KEY, '100.64.0.3/32'),
}),
)
script = dataplane_module.render_apply_script(plan)
block_start = script.index(
'expected_fabric_peer_keys=',
script.index('VC_VPC_APPLY_PHASE=wireguard'))
block_end = script.index(
'# The shared Fabric quarantine table', block_start)
peer_file = tmp_path / 'peers'
peer_file.write_text(first_key + '|10.44.0.11/32\n')
wg_rows = tmp_path / 'wg-rows'
wg_rows.write_text(
first_key + '\t100.64.0.2/32,10.44.0.11/32\n'
+ OTHER_WG_KEY + '\t100.64.0.4/32,10.45.0.11/32\n')
mutations = tmp_path / 'mutations'
mutations.write_text('')
harness = '\n'.join((
'set -eu',
_loader_from_script(script),
WG_ALLOWED_IPS_SH,
'VPC_PEER_FILE=' + shlex.quote(str(peer_file)),
'VC_WG_ROWS=' + shlex.quote(str(wg_rows)),
'VC_WG_MUTATIONS=' + shlex.quote(str(mutations)),
'WG_IFACE=wg-fabric0',
r'''
vc_fixture_wg() {
if [ "$*" = 'show wg-fabric0 peers' ]; then
cut -f1 "$VC_WG_ROWS"
elif [ "$*" = 'show wg-fabric0 allowed-ips' ]; then
cat "$VC_WG_ROWS"
elif [ "$1" = set ] && [ "$2" = wg-fabric0 ] &&
[ "$3" = peer ] && [ "$5" = allowed-ips ] && [ "$#" = 6 ]; then
awk -F '\t' -v OFS='\t' -v wanted="$4" -v allowed="$6" '
$1 == wanted {$2=allowed; found++} {print}
END {exit found == 1 ? 0 : 1}
' "$VC_WG_ROWS" > "$VC_WG_ROWS.next" || return 9
mv -f "$VC_WG_ROWS.next" "$VC_WG_ROWS" || return 9
printf '%s|%s\n' "$4" "$6" >> "$VC_WG_MUTATIONS"
else return 9; fi
}
WG=vc_fixture_wg
'''.strip(),
script[block_start:block_end],
))
result = subprocess.run(['/bin/sh', '-c', harness], text=True,
capture_output=True, check=False)
assert result.returncode == 0, result.stderr
assert wg_rows.read_text().splitlines() == [
first_key + '\t100.64.0.2/32,10.44.0.11/32',
OTHER_WG_KEY + '\t100.64.0.3/32',
]
assert mutations.read_text().splitlines() == [
first_key + '|100.64.0.2/32,10.44.0.11/32',
OTHER_WG_KEY + '|100.64.0.3/32',
]
def test_vpc_remove_converges_authoritative_closure_for_retiring_host(
tmp_path):
base = _plan()
first_key = base.peers[0].public_key
plan = replace(
base,
members=tuple(member for member in base.members
if member.host_id != base.host_id),
fabric_peer_keys=frozenset({first_key, OTHER_WG_KEY}),
fabric_peer_overlays=frozenset({
(first_key, '100.64.0.2/32'),
(OTHER_WG_KEY, '100.64.0.3/32'),
}),
)
script = dataplane_module.render_remove_script(plan)
block_start = script.index('expected_fabric_peer_keys=')
block_end = script.index(
'vc_vpc_fabric_nft_live_ready || exit 74', block_start)
peer_file = tmp_path / 'peers'
peer_file.write_text(OTHER_WG_KEY + '|10.45.0.11/32\n')
wg_rows = tmp_path / 'wg-rows'
wg_rows.write_text(
first_key + '\t100.64.0.8/32,10.44.0.11/32\n'
+ OTHER_WG_KEY + '\t100.64.0.9/32,10.45.0.11/32\n')
mutations = tmp_path / 'mutations'
mutations.write_text('')
harness = '\n'.join((
'set -eu',
_loader_from_script(script),
WG_ALLOWED_IPS_SH,
'VPC_PEER_FILE=' + shlex.quote(str(peer_file)),
'VC_WG_ROWS=' + shlex.quote(str(wg_rows)),
'VC_WG_MUTATIONS=' + shlex.quote(str(mutations)),
'WG_IFACE=wg-fabric0',
r'''
vc_fixture_wg() {
if [ "$*" = 'show wg-fabric0 peers' ]; then
cut -f1 "$VC_WG_ROWS"
elif [ "$*" = 'show wg-fabric0 allowed-ips' ]; then
cat "$VC_WG_ROWS"
elif [ "$1" = set ] && [ "$2" = wg-fabric0 ] &&
[ "$3" = peer ] && [ "$5" = allowed-ips ] && [ "$#" = 6 ]; then
awk -F '\t' -v OFS='\t' -v wanted="$4" -v allowed="$6" '
$1 == wanted {$2=allowed; found++} {print}
END {exit found == 1 ? 0 : 1}
' "$VC_WG_ROWS" > "$VC_WG_ROWS.next" || return 9
mv -f "$VC_WG_ROWS.next" "$VC_WG_ROWS" || return 9
printf '%s|%s\n' "$4" "$6" >> "$VC_WG_MUTATIONS"
else return 9; fi
}
WG=vc_fixture_wg
'''.strip(),
script[block_start:block_end],
))
result = subprocess.run(['/bin/sh', '-c', harness], text=True,
capture_output=True, check=False)
assert result.returncode == 0, result.stderr
assert wg_rows.read_text().splitlines() == [
first_key + '\t100.64.0.2/32',
OTHER_WG_KEY + '\t100.64.0.3/32,10.45.0.11/32',
]
assert mutations.read_text().splitlines() == [
first_key + '|100.64.0.2/32',
OTHER_WG_KEY + '|100.64.0.3/32,10.45.0.11/32',
]
@pytest.mark.parametrize('malformed', [
'100.99.106.2/32\n100.64.0.2/32',
'100.99.106.2/32 10.44.0.99/32',
'100.99.106.2/32;anything',
'100.99.106.2/32,*',
])
def test_overlay_parser_rejects_malformed_raw_input_without_shell_side_effects(
malformed):
script = dataplane_module.render_apply_script(_plan())
harness = '\n'.join((
'set -eu', _loader_from_script(script),
'IFS=:',
'if vpc_state_overlay_from_allowed ' + shlex.quote(malformed)
+ '; then exit 99; fi',
'[ "$IFS" = : ]',
'case "$-" in *f*) exit 98 ;; esac',
))
result = subprocess.run(['/bin/sh', '-c', harness], capture_output=True,
text=True, check=False)
assert result.returncode == 0, result.stderr
assert result.stdout == ''