You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc_apply_receipts.py

982 lines
38 KiB
Python

"""Successful apply receipts avoid disrupting an unchanged VPC bridge."""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
import hashlib
import os
from pathlib import Path
import re
import shlex
import subprocess
import sys
from threading import RLock
import pytest
from test_vpc_dataplane import _active_observation_markers, _plan
from virsh_panel import vpc_dataplane as dp
from virsh_panel.coordination import (
CoordinationError,
DurableCoordinator,
current_execution,
execution_context,
resource_guard,
)
SHELLS = [p for p in ('/bin/sh', '/bin/dash', '/bin/bash') if Path(p).exists()]
RECEIPT = ','.join(c * 64 for c in 'abc')
@pytest.fixture(autouse=True)
def empty_receipts():
with dp._APPLY_RECEIPT_LOCK:
dp._APPLY_RECEIPTS.clear()
yield
with dp._APPLY_RECEIPT_LOCK:
dp._APPLY_RECEIPTS.clear()
@pytest.fixture
def plan():
p = _plan()
return replace(
p,
fabric_peer_keys=frozenset(x.public_key for x in p.peers),
fabric_peer_overlays=frozenset(
(x.public_key, x.overlay_cidr) for x in p.peers),
)
class Adapter(dp.VpcPrivateHostAdapter):
def __init__(self, plan, **profile):
super().__init__(None, {
'ssh_target': 'owned', 'ssh_config_file': '/generation-a/config',
'host': '192.0.2.1', 'port': 22, 'fingerprint': 'host-key-a', **profile,
})
self.plan = plan
self.calls = []
self.observe_error = None
self.apply_error = None
self.remove_error = None
self.observation = '\n'.join(_active_observation_markers(plan) + [
'__VC_VPC_FIREWALL_MODE__=native_nftables',
'__VC_VPC_FIREWALL_STATE__=owned',
'__VC_VPC_STATE_FIREWALL_MODE__=native_nftables',
'__VC_VPC_WG_PEER_SET__=exact',
'__VC_VPC_HELPER_CONTRACT__=owned',
'__VC_VPC_BRIDGE_ADMIN__=up',
'__VC_VPC_NFT_FILE_EXACT__=yes',
'__VC_VPC_STATE_FILE_EXACT__=yes',
'__VC_VPC_PERSISTENCE_EXACT__=yes',
f'__VC_VPC_NFT_RECEIPT__={RECEIPT}',
])
self.applied = '\n'.join([
'__VC_VPC_APPLIED__=yes',
f'__VC_VPC_OWNER__={dp.DATAPLANE_OWNER_MARKER}',
f'__VC_VPC_REVISION__={plan.desired_revision}',
f'__VC_VPC_DIGEST__={plan.desired_digest}',
f'__VC_VPC_NFT_RECEIPT__={RECEIPT}',
])
self.removed = '\n'.join([
'__VC_VPC_REMOVED__=yes',
f'__VC_VPC_OWNER__={dp.DATAPLANE_OWNER_MARKER}',
f'__VC_VPC_REVISION__={plan.desired_revision}',
f'__VC_VPC_DIGEST__={plan.desired_digest}',
'__VC_VPC_ROLLBACK__=none',
])
def _deployment_contract(self):
return ('networkmanager_nmcli', '/usr/sbin/ifup', '/usr/sbin/ifdown',
'/usr/sbin/ifquery', 'native_nftables')
def _run(self, script):
if '__VC_VPC_APPLIED__=yes' in script:
self.calls.append('apply')
if self.apply_error:
raise self.apply_error
return self.applied
if '__VC_VPC_REMOVED__=yes' in script:
self.calls.append('remove')
if self.remove_error:
raise self.remove_error
return self.removed
self.calls.append('observe')
if self.observe_error:
raise self.observe_error
return self.observation
class CoordinationStore:
def __init__(self):
self.records = {}
self.lock = RLock()
def get(self, kind, key):
with self.lock:
return self.records.get((kind, key))
def insert(self, kind, key, value):
with self.lock:
if (kind, key) in self.records:
return False
self.records[kind, key] = value
return True
def compare_and_set(self, kind, key, expected, value):
with self.lock:
if self.records.get((kind, key)) != expected:
return False
self.records[kind, key] = value
return True
def _empty_local(plan):
return replace(
plan,
members=tuple(member for member in plan.members
if member.host_id != plan.host_id),
)
def _strict_removal_observation(*, closure='empty'):
values = {
'BRIDGE': 'missing',
'BRIDGE_OWNER': 'missing',
'NFT_TABLE': 'missing',
'NFT_TABLE_COUNT': '0',
'ROUTE_SCOPE_COUNT': '0',
'RULE_SCOPE_COUNT': '0',
'NM_PROFILE_COUNT': '0',
'NM_PROFILE_QUERY': 'ok',
'IFUP_PROFILE_COUNT': '0',
'IFUP_PROFILE_QUERY': 'ok',
'NFT_MAIN_COUNT': '1',
'NFT_SNAPSHOT': 'yes',
'FIREWALL_MODE': 'native_nftables',
'FIREWALL_STATE': 'owned',
'NFT_FILE_PRESENT': 'no',
'SYSCTL_FILE_PRESENT': 'no',
'PROXY_FILE_PRESENT': 'no',
'IF_FRAGMENT_PRESENT': 'no',
'OWNER_FILE_PRESENT': 'no',
'STATE_FILE_PRESENT': 'no',
'TRANSACTION_PRESENT': 'no',
'NFT_INCLUDE_PRESENT': 'no',
'REMOVAL_BRIDGE_ABSENT': 'yes',
'REMOVAL_ROUTE_ABSENT': 'yes',
'REMOVAL_RULE_ABSENT': 'yes',
'REMOVAL_NFT_ABSENT': 'yes',
'REMOVAL_STATE_CLOSURE': closure,
'REMOVAL_FABRIC_NFT': 'yes',
'REMOVAL_WG_PEER_SET': 'exact',
}
return '\n'.join(
f'__VC_VPC_{key}__={value}' for key, value in values.items())
def _assert_unknown_mutation_is_fenced(adapter, mutation, call_name,
exception_type):
coordinator = DurableCoordinator(CoordinationStore())
first = coordinator.claim(operation_id='vpc-write-first')
with execution_context(coordinator, first):
with pytest.raises(exception_type):
with resource_guard(('catalog:network',)):
mutation()
assert current_execution().execution_unknown.is_set()
assert coordinator.release(first)
claims = list(coordinator.snapshot().values())
assert len(claims) == 1
assert claims[0]['kind'] == 'resource'
assert claims[0]['state'] == 'uncertain'
assert claims[0]['resources'] == ['catalog:network']
assert adapter.calls.count(call_name) == 1
before = list(adapter.calls)
second = coordinator.claim(operation_id='vpc-write-second')
with execution_context(coordinator, second):
with pytest.raises(CoordinationError) as blocked:
with resource_guard(('catalog:network',), timeout=0):
mutation()
assert blocked.value.error_code == 'coordination_uncertain'
assert adapter.calls == before
assert coordinator.release(second)
def test_success_is_reused_across_adapter_instances_without_activating_member(plan):
first = Adapter(plan)
assert first.apply(plan)['state'] == 'applying'
assert first.calls == ['apply']
second = Adapter(plan)
result = second.apply(plan)
assert second.calls == ['observe']
assert result == {'state': 'applying', 'readback': 'pending', 'rollback': 'none',
'owner': dp.DATAPLANE_OWNER_MARKER}
public = second.observe(plan)
assert public['state'] != 'active'
assert public['config_verified'] is False
assert public['member_nics'][0]['xml_verified'] is False
assert second.calls == ['observe', 'observe']
@pytest.mark.parametrize(('old', 'new'), [
('LOCAL_NEIGHBOR_STATE__=owned', 'LOCAL_NEIGHBOR_STATE__=missing'),
('BRIDGE_ADMIN__=up', 'BRIDGE_ADMIN__=down'),
('HELPER_CONTRACT__=owned', 'HELPER_CONTRACT__=missing'),
('NFT_FILE_EXACT__=yes', 'NFT_FILE_EXACT__=no'),
('STATE_FILE_EXACT__=yes', 'STATE_FILE_EXACT__=no'),
('PERSISTENCE_EXACT__=yes', 'PERSISTENCE_EXACT__=no'),
('FIREWALL_STATE__=owned', 'FIREWALL_STATE__=mismatch'),
('PERSISTENCE__=yes', 'PERSISTENCE__=no'),
('BRIDGE_BOUNDARY__=yes', 'BRIDGE_BOUNDARY__=no'),
('FABRIC_NFT__=yes', 'FABRIC_NFT__=no'),
('WG_PEER_SET__=exact', 'WG_PEER_SET__=extra'),
('RP_FILTER__=2', 'RP_FILTER__=1'),
('ROUTE_EXTRA__=no', 'ROUTE_EXTRA__=yes'),
('RULE_EXTRA__=no', 'RULE_EXTRA__=yes'),
('PROXY_STATE__=owned', 'PROXY_STATE__=missing'),
(f'NFT_RECEIPT__={RECEIPT}', 'NFT_RECEIPT__=' + ','.join(c * 64 for c in 'abd')),
('NFT_RECEIPT__=', 'UNKNOWN_RECEIPT__='),
('FIREWALL_MODE__=', 'UNKNOWN_FIREWALL_MODE__='),
])
def test_every_host_gate_or_full_nft_drift_requires_protected_repair(plan, old, new):
adapter = Adapter(plan)
adapter.apply(plan)
assert old in adapter.observation
adapter.observation = adapter.observation.replace(old, new)
adapter.apply(plan)
assert adapter.calls == ['apply', 'observe', 'apply']
def test_explicit_route_linkdown_never_skips(plan):
adapter = Adapter(plan)
adapter.apply(plan)
adapter.observation = re.sub(
r'(__VC_VPC_ROUTE__=[^\n]+)\|owned', r'\1|linkdown|owned',
adapter.observation)
adapter.apply(plan)
assert adapter.calls == ['apply', 'observe', 'apply']
@pytest.mark.parametrize('error', [TimeoutError('read timed out'),
dp.VpcDataplaneError('read failed', 502, 'read')])
def test_read_exception_invalidates_receipt_without_mutation(plan, error):
adapter = Adapter(plan)
adapter.apply(plan)
adapter.observe_error = error
with pytest.raises(type(error)):
adapter.apply(plan)
assert adapter.calls == ['apply', 'observe']
assert not dp._APPLY_RECEIPTS
@pytest.mark.parametrize('change', [
lambda s: s.replace('APPLIED__=yes', 'APPLIED__=no'),
lambda s: s.replace('OWNER__=virt-cloud-vpc-dataplane-v1', 'OWNER__=foreign'),
lambda s: re.sub(r'REVISION__=[^\n]+', 'REVISION__=999', s),
lambda s: re.sub(r'DIGEST__=[^\n]+', 'DIGEST__=' + 'f' * 64, s),
lambda s: s.replace('NFT_RECEIPT__=', 'MISSING_RECEIPT__='),
lambda s: s.replace(RECEIPT, 'malformed'),
lambda s: s + f'\n__VC_VPC_NFT_RECEIPT__={RECEIPT}',
lambda s: s + '\n__VC_VPC_APPLIED__=yes',
lambda s: s + f'\n__VC_VPC_OWNER__={dp.DATAPLANE_OWNER_MARKER}',
lambda s: s + '\n' + next(
line for line in s.splitlines() if 'REVISION__=' in line),
lambda s: s + '\n' + next(
line for line in s.splitlines() if 'DIGEST__=' in line),
lambda s: s.replace('REVISION__=', 'MISSING_REVISION__='),
lambda s: s.replace('DIGEST__=', 'MISSING_DIGEST__='),
])
def test_ambiguous_or_incomplete_apply_never_seeds_skip(plan, change):
adapter = Adapter(plan)
adapter.applied = change(adapter.applied)
with pytest.raises(dp.VpcDataplaneError) as rejected:
adapter.apply(plan)
assert rejected.value.error_code == 'vpc_apply_unconfirmed'
assert not dp._APPLY_RECEIPTS
@pytest.mark.parametrize('failure', ['lost', 'incomplete'])
def test_lost_or_incomplete_apply_response_retains_claim_and_blocks_replay(
plan, failure):
adapter = Adapter(plan)
if failure == 'lost':
adapter.apply_error = TimeoutError('apply response lost')
expected = TimeoutError
else:
adapter.applied = adapter.applied.replace(
'DIGEST__=', 'MISSING_DIGEST__=')
expected = dp.VpcDataplaneError
_assert_unknown_mutation_is_fenced(
adapter, lambda: adapter.apply(plan), 'apply', expected)
@pytest.mark.parametrize(('key', 'value'), [
('ssh_config_file', '/generation-b/config'), ('host', '192.0.2.2'),
('port', 2222), ('fingerprint', 'other-key'), ('ssh_target', 'other-alias'),
])
def test_new_host_identity_requires_successful_apply(plan, key, value):
Adapter(plan).apply(plan)
second = Adapter(plan, **{key: value})
second.apply(plan)
assert second.calls == ['apply']
def test_changed_plan_or_source_contract_does_not_use_old_receipt(plan, monkeypatch):
adapter = Adapter(plan)
adapter.apply(plan)
original = dp.render_apply_script
monkeypatch.setattr(dp, 'render_apply_script', lambda *a, **kw: original(*a, **kw) + '\n# new-source-contract\n')
adapter.apply(plan)
assert adapter.calls == ['apply', 'apply']
def test_repair_failure_discards_old_success(plan):
adapter = Adapter(plan)
adapter.apply(plan)
adapter.observation = adapter.observation.replace('BRIDGE_ADMIN__=up', 'BRIDGE_ADMIN__=down')
adapter.apply_error = TimeoutError('uncertain write')
with pytest.raises(TimeoutError):
adapter.apply(plan)
assert not dp._APPLY_RECEIPTS
def test_remove_discards_success_receipt(plan):
adapter = Adapter(plan)
adapter.apply(plan)
empty = _empty_local(plan)
adapter.remove(empty)
assert not dp._APPLY_RECEIPTS
@pytest.mark.parametrize('change', [
lambda s: s.replace('REMOVED__=yes', 'REMOVED__=no'),
lambda s: s.replace('OWNER__=virt-cloud-vpc-dataplane-v1', 'OWNER__=foreign'),
lambda s: re.sub(r'REVISION__=[^\n]+', 'REVISION__=999', s),
lambda s: re.sub(r'DIGEST__=[^\n]+', 'DIGEST__=' + 'f' * 64, s),
lambda s: s.replace('ROLLBACK__=none', 'ROLLBACK__=partial'),
lambda s: s.replace('REMOVED__=', 'MISSING_REMOVED__='),
lambda s: s.replace('OWNER__=', 'MISSING_OWNER__='),
lambda s: s.replace('REVISION__=', 'MISSING_REVISION__='),
lambda s: s.replace('DIGEST__=', 'MISSING_DIGEST__='),
lambda s: s.replace('ROLLBACK__=', 'MISSING_ROLLBACK__='),
lambda s: s + '\n' + next(
line for line in s.splitlines() if 'REMOVED__=' in line),
lambda s: s + '\n' + next(
line for line in s.splitlines() if 'OWNER__=' in line),
lambda s: s + '\n' + next(
line for line in s.splitlines() if 'REVISION__=' in line),
lambda s: s + '\n' + next(
line for line in s.splitlines() if 'DIGEST__=' in line),
lambda s: s + '\n' + next(
line for line in s.splitlines() if 'ROLLBACK__=' in line),
])
def test_ambiguous_or_incomplete_remove_never_confirms(plan, change):
adapter = Adapter(plan)
adapter.removed = change(adapter.removed)
with pytest.raises(dp.VpcDataplaneError) as rejected:
adapter.remove(_empty_local(plan))
assert rejected.value.error_code == 'vpc_remove_unconfirmed'
@pytest.mark.parametrize('failure', ['lost', 'incomplete'])
def test_lost_or_incomplete_remove_response_retains_claim_and_blocks_replay(
plan, failure):
adapter = Adapter(plan)
empty = _empty_local(plan)
if failure == 'lost':
adapter.remove_error = TimeoutError('remove response lost')
expected = TimeoutError
else:
adapter.removed = adapter.removed.replace(
'ROLLBACK__=', 'MISSING_ROLLBACK__=')
expected = dp.VpcDataplaneError
_assert_unknown_mutation_is_fenced(
adapter, lambda: adapter.remove(empty), 'remove', expected)
@pytest.mark.parametrize('mutation_name', ['apply', 'remove'])
def test_receipt_marker_parser_exception_retains_claim_and_blocks_replay(
plan, monkeypatch, mutation_name):
adapter = Adapter(plan)
mutation = (
(lambda: adapter.apply(plan))
if mutation_name == 'apply'
else (lambda: adapter.remove(_empty_local(plan)))
)
def broken_markers(_output):
raise ValueError('marker parser failed')
monkeypatch.setattr(dp, '_markers', broken_markers)
_assert_unknown_mutation_is_fenced(
adapter, mutation, mutation_name, ValueError)
@pytest.mark.parametrize('closure', ['owned', 'empty'])
def test_removal_observer_accepts_exact_remaining_fabric_closure(plan, closure):
adapter = Adapter(plan)
adapter.observation = _strict_removal_observation(closure=closure)
observed = adapter.observe_removal(_empty_local(plan))
assert observed['removed'] is True
assert observed['readback'] == 'confirmed'
assert adapter.calls == ['observe']
@pytest.mark.parametrize(('query', 'count'), [
('unavailable', '0'),
('failed', '0'),
('invalid', '0'),
('ok', '1'),
])
def test_networkmanager_removal_requires_successful_zero_profile_inventory(
plan, query, count):
adapter = Adapter(plan)
adapter.observation = (_strict_removal_observation()
.replace('NM_PROFILE_QUERY__=ok',
f'NM_PROFILE_QUERY__={query}')
.replace('NM_PROFILE_COUNT__=0',
f'NM_PROFILE_COUNT__={count}'))
with pytest.raises(dp.VpcDataplaneError) as rejected:
adapter.observe_removal(_empty_local(plan))
assert rejected.value.error_code == 'vpc_remove_readback_mismatch'
def test_ifupdown_removal_uses_full_interfaces_inventory_without_nmcli(plan):
class IfupdownAdapter(Adapter):
def _deployment_contract(self):
return ('ifupdown', '/usr/sbin/ifup', '/usr/sbin/ifdown',
'/usr/sbin/ifquery', 'native_nftables')
adapter = IfupdownAdapter(plan)
adapter.observation = _strict_removal_observation().replace(
'NM_PROFILE_QUERY__=ok', 'NM_PROFILE_QUERY__=not_applicable')
observed = adapter.observe_removal(_empty_local(plan))
assert observed['removed'] is True
assert adapter.calls == ['observe']
def test_observer_nmcli_inventory_preserves_query_failure(plan):
rendered = dp.render_observe_script(
_empty_local(plan), deployment_mode='networkmanager_nmcli')
assert 'MODE=networkmanager_nmcli' in rendered
assert '"$NMCLI" -t --escape no -f UUID connection show' in rendered
assert 'vc_nm_inventory_capture scalar connection.id' in rendered
assert 'vc_nm_inventory_capture scalar connection.interface-name' in rendered
assert 'vc_nm_inventory_capture scalar connection.stable-id' in rendered
assert ('connection show --active'
not in dp.NETWORKMANAGER_VPC_TARGET_INVENTORY_SH)
assert '__VC_VPC_NM_PROFILE_QUERY__=$nm_profile_query' in rendered
assert '__VC_VPC_IFUP_PROFILE_QUERY__=$ifup_profile_query' in rendered
NM_INVENTORY_MOCK = r'''
import os
import sys
args = sys.argv[1:]
if '-f' in args and args[args.index('-f') + 1] == 'UUID':
if os.environ.get('NM_LIST_FAIL') == 'yes':
raise SystemExit(9)
rows = os.environ.get('NM_UUID_ROWS', '')
state_path = os.environ.get('NM_LIST_STATE')
if state_path:
try:
seen = int(open(state_path).read())
except FileNotFoundError:
seen = 0
open(state_path, 'w').write(str(seen + 1))
if seen:
rows = os.environ.get('NM_UUID_ROWS_AFTER', rows)
sys.stdout.write(rows)
raise SystemExit(0)
if '-g' not in args:
raise SystemExit(10)
field = args[args.index('-g') + 1]
if field == os.environ.get('NM_FAIL_FIELD'):
raise SystemExit(11)
values = {
'connection.id': os.environ.get('NM_ID', 'unrelated'),
'connection.interface-name': os.environ.get('NM_IFACE', '--'),
'connection.stable-id': os.environ.get('NM_STABLE', '--'),
'connection.type': os.environ.get('NM_TYPE', 'ethernet'),
}
if field not in values:
raise SystemExit(12)
sys.stdout.write(values[field] + '\n')
'''
@pytest.mark.parametrize('shell', SHELLS)
@pytest.mark.parametrize(('values', 'expected'), [
({
'NM_ID': 'renamed-panel-profile',
'NM_IFACE': 'vcv-test',
'NM_STABLE': 'virt-cloud-vpc-dataplane-v1 vpc=vpc-test',
'NM_TYPE': 'bridge',
}, '1|ok|no'),
({
'NM_ID': 'foreign-name',
'NM_IFACE': 'vcv-test',
'NM_STABLE': '--',
'NM_TYPE': 'bridge',
}, '1|ok|no'),
({
'NM_ID': 'foreign-name',
'NM_IFACE': 'other0',
'NM_STABLE': 'virt-cloud-vpc-dataplane-v1 vpc=vpc-test',
'NM_TYPE': 'bridge',
}, '1|ok|no'),
({
'NM_ID': 'unrelated',
'NM_IFACE': 'other0',
'NM_STABLE': '--',
'NM_TYPE': 'ethernet',
}, '0|ok|no'),
])
def test_nm_full_inventory_detects_renamed_interface_and_stable_id_profiles(
tmp_path, shell, values, expected):
nmcli = executable(tmp_path / 'nmcli-inventory', NM_INVENTORY_MOCK)
harness = '\n'.join((
'set -eu',
'NMCLI=$1',
'BRIDGE=vcv-test',
"NM_PROFILE_MARKER='virt-cloud-vpc-dataplane-v1 vpc=vpc-test'",
dp.NETWORKMANAGER_VPC_TARGET_INVENTORY_SH,
'vc_nm_target_inventory',
"printf '%s|%s|%s\\n' \"$NM_TARGET_COUNT\" "
'"$NM_TARGET_QUERY" "$NM_TARGET_EXACT"',
))
result = subprocess.run(
[shell, '-s', nmcli], input=harness, text=True, capture_output=True,
env=os.environ | {
'NM_UUID_ROWS': '11111111-1111-4111-8111-111111111111\n',
**values,
}, check=False)
assert result.returncode == 0, result.stderr
assert result.stdout.strip() == expected
@pytest.mark.parametrize('shell', SHELLS)
@pytest.mark.parametrize(('values', 'query'), [
({'NM_LIST_FAIL': 'yes'}, 'failed'),
({'NM_FAIL_FIELD': 'connection.interface-name'}, 'failed'),
({
'NM_UUID_ROWS': (
'11111111-1111-4111-8111-111111111111\n'
'11111111-1111-4111-8111-111111111111\n'),
}, 'invalid'),
({'NM_ID': 'bad\nsecond-row'}, 'invalid'),
({
'NM_LIST_STATE': 'SET_BY_TEST',
'NM_UUID_ROWS_AFTER': '',
}, 'invalid'),
])
def test_nm_full_inventory_fails_closed_on_query_or_ambiguous_output(
tmp_path, shell, values, query):
nmcli = executable(tmp_path / 'nmcli-failure', NM_INVENTORY_MOCK)
harness = '\n'.join((
'set -eu',
'NMCLI=$1',
'BRIDGE=vcv-test',
"NM_PROFILE_MARKER='virt-cloud-vpc-dataplane-v1 vpc=vpc-test'",
dp.NETWORKMANAGER_VPC_TARGET_INVENTORY_SH,
'if vc_nm_target_inventory; then exit 91; fi',
"printf '%s\\n' \"$NM_TARGET_QUERY\"",
))
env = os.environ | {
'NM_UUID_ROWS': '11111111-1111-4111-8111-111111111111\n',
**values,
}
if env.get('NM_LIST_STATE') == 'SET_BY_TEST':
env['NM_LIST_STATE'] = str(tmp_path / 'list-state')
result = subprocess.run(
[shell, '-s', nmcli], input=harness, text=True, capture_output=True,
env=env, check=False)
assert result.returncode == 0, result.stderr
assert result.stdout.strip() == query
@pytest.mark.parametrize('shell', SHELLS)
@pytest.mark.parametrize(('source', 'content'), [
('main', 'auto lo vcv-test\niface lo inet loopback\n'),
('fragment', 'iface vcv-test inet manual\n bridge_ports none\n'),
('fragment', 'allow-hotplug eth0 vcv-test\n'),
])
def test_ifupdown_full_inventory_finds_main_and_other_fragment_stanzas(
tmp_path, shell, source, content):
root = tmp_path / 'network'
fragments = root / 'interfaces.d'
fragments.mkdir(parents=True)
(root / 'interfaces').write_text(
content if source == 'main' else 'iface lo inet loopback\n')
if source == 'fragment':
(fragments / 'other-config').write_text(content)
harness = '\n'.join((
'set -eu',
'IFUP_ROOT=$1',
'BRIDGE=vcv-test',
dp.IFUPDOWN_VPC_TARGET_INVENTORY_SH,
'vc_ifupdown_target_inventory',
"printf '%s|%s\\n' \"$IFUP_TARGET_COUNT\" \"$IFUP_TARGET_QUERY\"",
))
result = subprocess.run(
[shell, '-s', str(root)], input=harness, text=True,
capture_output=True, check=False)
assert result.returncode == 0, result.stderr
assert result.stdout.strip() == '1|ok'
@pytest.mark.parametrize('shell', SHELLS)
def test_ifupdown_full_inventory_rejects_symlink_and_malformed_file(
tmp_path, shell):
root = tmp_path / 'network'
fragments = root / 'interfaces.d'
fragments.mkdir(parents=True)
(root / 'interfaces').write_text('iface lo inet loopback\n')
target = tmp_path / 'outside'
target.write_text('iface vcv-test inet manual\n')
(fragments / 'linked').symlink_to(target)
harness = '\n'.join((
'set -eu',
'IFUP_ROOT=$1',
'BRIDGE=vcv-test',
dp.IFUPDOWN_VPC_TARGET_INVENTORY_SH,
'if vc_ifupdown_target_inventory; then exit 91; fi',
))
linked = subprocess.run(
[shell, '-s', str(root)], input=harness, text=True,
capture_output=True, check=False)
assert linked.returncode == 0, linked.stderr
(fragments / 'linked').unlink()
(fragments / 'malformed').write_text('iface\n')
malformed = subprocess.run(
[shell, '-s', str(root)], input=harness, text=True,
capture_output=True, check=False)
assert malformed.returncode == 0, malformed.stderr
@pytest.mark.parametrize(('query', 'count'), [
('invalid', '0'),
('ok', '1'),
])
def test_removal_receipt_rejects_stale_or_unreadable_ifupdown_inventory(
plan, query, count):
adapter = Adapter(plan)
adapter.observation = (_strict_removal_observation()
.replace('IFUP_PROFILE_QUERY__=ok',
f'IFUP_PROFILE_QUERY__={query}')
.replace('IFUP_PROFILE_COUNT__=0',
f'IFUP_PROFILE_COUNT__={count}'))
with pytest.raises(dp.VpcDataplaneError) as rejected:
adapter.observe_removal(_empty_local(plan))
assert rejected.value.error_code == 'vpc_remove_readback_mismatch'
@pytest.mark.parametrize('shell', SHELLS)
@pytest.mark.parametrize(('mode', 'changes', 'expected'), [
('networkmanager_nmcli', {}, 'yes'),
('networkmanager_nmcli', {'ifup_profile_count': '1'}, 'no'),
('networkmanager_nmcli', {'ifup_profile_query': 'invalid'}, 'no'),
('networkmanager_nmcli', {'nm_profile_count': '2'}, 'no'),
('ifupdown', {}, 'yes'),
('ifupdown', {'ifup_profile_count': '2'}, 'no'),
('ifupdown', {'ifup_profile_query': 'invalid'}, 'no'),
])
def test_active_persistence_requires_exact_full_manager_inventory(
plan, shell, mode, changes, expected):
rendered = dp.render_observe_script(
plan, deployment_mode=mode)
start = rendered.index('if [ "$firewall_mode_state" = owned ]')
end = rendered.index('for artifact_spec in', start)
gate = rendered[start:end]
values = {
'MODE': mode,
'firewall_mode_state': 'owned',
'NFT_MAIN_COUNT': '1',
'persistence_target': 'yes',
'opposite_clean': 'yes',
'nft_file_ok': 'yes',
'sysctl_file_ok': 'yes',
'nft_snapshot_state': 'yes',
'nm_profile_query': ('ok' if mode == 'networkmanager_nmcli'
else 'not_applicable'),
'nm_profile_count': ('1' if mode == 'networkmanager_nmcli' else '0'),
'nm_owned': ('yes' if mode == 'networkmanager_nmcli' else 'no'),
'nm_present': ('yes' if mode == 'networkmanager_nmcli' else 'no'),
'ifup_profile_query': 'ok',
'ifup_profile_count': ('0' if mode == 'networkmanager_nmcli' else '1'),
'ifup_owned': ('no' if mode == 'networkmanager_nmcli' else 'yes'),
**changes,
}
harness = '\n'.join((
'set -eu',
*(key + '=' + shlex.quote(value) for key, value in values.items()),
'persistence=no',
gate,
"printf '%s\\n' \"$persistence\"",
))
result = subprocess.run(
[shell, '-s'], input=harness, text=True,
capture_output=True, check=False)
assert result.returncode == 0, result.stderr
assert result.stdout.splitlines()[-1] == expected
STRICT_REMOVAL_KEYS = tuple(
line.removeprefix('__VC_VPC_').split('__=', 1)[0]
for line in _strict_removal_observation().splitlines()
)
@pytest.mark.parametrize('key', STRICT_REMOVAL_KEYS)
def test_removal_observer_requires_every_unique_scalar_marker(plan, key):
adapter = Adapter(plan)
lines = _strict_removal_observation().splitlines()
prefix = f'__VC_VPC_{key}__='
adapter.observation = '\n'.join(
line for line in lines if not line.startswith(prefix))
with pytest.raises(dp.VpcDataplaneError) as missing:
adapter.observe_removal(_empty_local(plan))
assert missing.value.error_code == 'vpc_remove_readback_mismatch'
adapter.calls.clear()
matching = next(line for line in lines if line.startswith(prefix))
adapter.observation = '\n'.join(lines + [matching])
with pytest.raises(dp.VpcDataplaneError) as duplicate:
adapter.observe_removal(_empty_local(plan))
assert duplicate.value.error_code == 'vpc_remove_readback_mismatch'
def test_removal_observer_rejects_active_only_fabric_markers(plan):
adapter = Adapter(plan)
adapter.observation = '\n'.join([
_strict_removal_observation()
.replace('__VC_VPC_REMOVAL_FABRIC_NFT__=yes\n', '')
.replace('__VC_VPC_REMOVAL_WG_PEER_SET__=exact', ''),
'__VC_VPC_FABRIC_NFT__=yes',
'__VC_VPC_WG_PEER_SET__=exact',
])
with pytest.raises(dp.VpcDataplaneError) as rejected:
adapter.observe_removal(_empty_local(plan))
assert rejected.value.error_code == 'vpc_remove_readback_mismatch'
def test_cache_is_bounded_under_concurrent_adapters(monkeypatch):
monkeypatch.setattr(dp, '_APPLY_RECEIPT_LIMIT', 8)
with ThreadPoolExecutor(max_workers=8) as executor:
list(executor.map(lambda i: Adapter._save_receipt((str(i),), str(i), RECEIPT), range(100)))
assert len(dp._APPLY_RECEIPTS) == 8
Adapter._save_receipt(('latest',), 'latest', RECEIPT)
assert len(dp._APPLY_RECEIPTS) == 8
assert next(reversed(dp._APPLY_RECEIPTS)) == ('latest',)
NFT_MOCK = r'''
import os, sys
args=sys.argv[1:]
assert args[:3] == ['-s','-n','-y'], args
scenario=os.environ.get('SCENARIO','healthy')
if args[3:5] == ['list','tables']:
if scenario=='inventory_error': sys.exit(96)
if scenario=='malformed_inventory': print('permission denied'); sys.exit()
if scenario=='missing': sys.exit()
print('table '+args[5]+' owned')
if scenario=='duplicate': print('table '+args[5]+' owned')
else:
assert args[3:5]==['list','table'] and args[6]=='owned', args
if scenario=='read_error': sys.exit(97)
if scenario=='empty': sys.exit()
action='accept' if scenario=='verdict_drift' else 'drop'
print('table '+args[5]+' owned { chain forward { '+action+' comment "same-owner-digest"; } }')
'''
def executable(path, body):
path.write_text('#!' + sys.executable + '\n' + body)
path.chmod(0o700)
return str(path)
@pytest.mark.parametrize('shell', SHELLS)
@pytest.mark.parametrize(('scenario', 'absent'), [
('missing', True),
('healthy', False),
('inventory_error', False),
('malformed_inventory', False),
('duplicate', False),
])
def test_removal_nft_absence_requires_complete_strict_family_inventories(
tmp_path, plan, shell, scenario, absent):
nft = executable(tmp_path / 'nft-absence', NFT_MOCK)
rendered = dp.render_observe_script(_empty_local(plan))
start = rendered.index('vc_vpc_target_nft_absent() {')
end = rendered.index('firewall_mode_state=owned', start)
function = rendered[start:end]
script = '\n'.join((
'set -eu',
'NFT=$1',
'TABLE=owned',
function,
'vc_vpc_target_nft_absent',
))
result = subprocess.run(
[shell, '-s', nft], input=script, text=True, capture_output=True,
env=os.environ | {'SCENARIO': scenario}, check=False)
assert (result.returncode == 0) is absent, result.stderr
@pytest.mark.parametrize('shell', SHELLS)
@pytest.mark.parametrize('scenario', ['healthy', 'verdict_drift', 'inventory_error',
'malformed_inventory', 'read_error', 'missing', 'duplicate', 'empty'])
def test_actual_nft_receipt_shell_rejects_read_errors_and_kept_comment_drift(tmp_path, shell, scenario):
nft = executable(tmp_path / 'nft', NFT_MOCK)
sha = executable(tmp_path / 'sha256sum', 'import hashlib,sys\nprint(hashlib.sha256(sys.stdin.buffer.read()).hexdigest()+" -")\n')
script = dp.VPC_APPLY_RECEIPT_SH + '\nNFT=$1; SHA256SUM=$2; TABLE=owned\nvc_vpc_nft_receipt\n'
result = subprocess.run([shell, '-eu', '-s', nft, sha], input=script,
text=True, capture_output=True,
env=os.environ | {'SCENARIO': scenario})
if scenario in {'healthy', 'verdict_drift'}:
expected = ','.join(hashlib.sha256(
('table '+family+' owned { chain forward { '+
('accept' if scenario=='verdict_drift' else 'drop')+
' comment "same-owner-digest"; } }\n').encode()).hexdigest()
for family in ('inet', 'bridge', 'arp'))
assert result.returncode == 0, result.stderr
assert result.stdout.strip() == expected
elif scenario == 'missing':
assert result.returncode == 0 and result.stdout.strip() == 'missing'
else:
assert result.returncode != 0
assert not result.stdout
@pytest.mark.parametrize('shell', SHELLS)
@pytest.mark.parametrize(('line', 'expected'), [
('20: vcv-test: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP', 'up'),
('20: vcv-test: <BROADCAST,MULTICAST,UP> mtu 1500 state DOWN', 'up'),
('20: vcv-test: <BROADCAST,MULTICAST> mtu 1500 state DOWN', 'down'),
('', None), ('malformed', None),
])
def test_bridge_admin_uses_flag_not_carrier_state(tmp_path, shell, line, expected):
ip = executable(tmp_path / 'ip', 'import os\nprint(os.environ["LINK"])\n')
result = subprocess.run([shell, '-eu', '-s', ip],
input=dp.VPC_APPLY_RECEIPT_SH + '\nIP=$1; BRIDGE=vcv-test\nvc_vpc_bridge_admin_state\n',
text=True, capture_output=True, env=os.environ | {'LINK': line})
if expected is None:
assert result.returncode != 0
else:
assert result.returncode == 0, result.stderr
assert result.stdout.strip() == expected
def test_receipt_is_anchored_in_protected_apply_and_persistence_exact(plan):
script = dp.render_apply_script(plan)
receipt = script.index('vc_apply_nft_receipt=$(vc_vpc_nft_receipt)')
assert script.rfind('vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 74', 0, receipt) > 0
assert script.index('vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 74', receipt) < script.index('__VC_VPC_NFT_RECEIPT__=$vc_apply_nft_receipt')
assert script.index('__VC_VPC_NFT_RECEIPT__=$vc_apply_nft_receipt') < script.index('COMMITTED=yes', receipt)
persisted = dp.render_vpc_nft_rules(plan).rstrip() + '\n'
assert hashlib.sha256(persisted.encode()).hexdigest() in dp.render_observe_script(plan)
@pytest.mark.parametrize('key', ['host', 'port', 'ssh_target', 'ssh_config_file', 'fingerprint'])
def test_missing_host_identity_never_enters_shared_cache(plan, key):
adapter = Adapter(plan)
del adapter.profile[key]
adapter.apply(plan)
adapter.apply(plan)
assert adapter.calls == ['apply', 'apply']
assert not dp._APPLY_RECEIPTS
@pytest.mark.parametrize('receipt', ['bad', RECEIPT + '\n__VC_VPC_NFT_RECEIPT__=' + RECEIPT])
def test_malformed_observer_receipt_propagates_without_repair(plan, receipt):
adapter = Adapter(plan)
adapter.apply(plan)
adapter.observation = adapter.observation.replace(RECEIPT, receipt)
with pytest.raises(dp.VpcDataplaneError, match='收据无效'):
adapter.apply(plan)
assert adapter.calls == ['apply', 'observe']
assert not dp._APPLY_RECEIPTS
def test_explicit_nft_absence_repairs_without_losing_public_observation(plan):
adapter = Adapter(plan)
adapter.apply(plan)
adapter.observation = adapter.observation.replace(RECEIPT, 'missing').replace(
'ANTI_SPOOF__=yes', 'ANTI_SPOOF__=no')
assert adapter.observe(plan)['state'] != 'active'
adapter.apply(plan)
assert adapter.calls == ['apply', 'observe', 'observe', 'apply']
@pytest.mark.parametrize('shell', SHELLS)
@pytest.mark.parametrize('mutation', [
'none', 'nft', 'state', 'ifup', 'missing_state', 'helper', 'old_route_scope',
])
def test_rendered_host_receipt_requires_exact_persistent_bytes(tmp_path, shell, mutation, plan):
nft_file = tmp_path / 'vpc.nft'
state = tmp_path / 'vpc.state'
fragment = tmp_path / 'vpc.ifup'
nft_file.write_text(dp.render_vpc_nft_rules(plan).rstrip() + '\n')
state.write_text(dp._state(plan, dp._routes(plan)))
fragment.write_text(
f'# {dp.DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} '
f'revision={plan.desired_revision} digest={plan.desired_digest}\n'
f'iface {plan.resources.bridge} inet manual\n'
' bridge_ports none\n bridge-stp off\n')
if mutation in {'nft', 'state', 'ifup'}:
path = {'nft': nft_file, 'state': state, 'ifup': fragment}[mutation]
path.write_text(path.read_text() + '# retained-owner-but-drift\n')
if mutation == 'missing_state':
state.unlink()
sha = executable(tmp_path / 'sha256sum',
'import hashlib,sys,pathlib\n'
'raw=pathlib.Path(sys.argv[1]).read_bytes() if len(sys.argv)>1 else sys.stdin.buffer.read()\n'
'print(hashlib.sha256(raw).hexdigest()+" -")\n')
helper = tmp_path / 'virt-cloud-wg-fabric'
if mutation != 'helper':
header = (
'#!/bin/sh\n# virt-cloud-wg-fabric-v1\n'
f'# fabric-plan-digest={plan.fabric_plan_digest}\n'
f'PLAN_DIGEST="{plan.fabric_plan_digest}"\n'
'# libvirt-startup-independent-v1\n'
'# vpc-local-neighbor-contract-v1\n')
header += ('# vpc-state-route-scope-contract-v3\n'
if mutation == 'old_route_scope'
else '# vpc-state-route-scope-contract-v4\n')
helper.write_text(header)
helper.chmod(0o700)
stat = executable(tmp_path / 'stat', 'import sys\n'
f'print("0:0:700" if sys.argv[-1] == {str(helper)!r} else "0:0:600")\n')
nft = executable(tmp_path / 'nft', NFT_MOCK)
ip = executable(tmp_path / 'ip', 'print("20: vcv-test: <UP,LOWER_UP> mtu 1500 state UP")\n')
rendered = dp.render_observe_script(plan)
fragment_script = rendered.split('# vc-apply-receipt-observe-begin', 1)[1].split(
'# vc-apply-receipt-observe-end', 1)[0]
settings = {
'IP': ip, 'NFT': nft, 'SHA256SUM': sha, 'STAT': stat,
'BRIDGE': 'vcv-test', 'TABLE': 'owned', 'NFT_FILE': str(nft_file),
'STATE': str(state), 'IF_FRAGMENT': str(fragment),
'EXPECTED_DIGEST': plan.fabric_plan_digest,
'nft_file_ok': 'yes', 'bridge_shape': 'yes', 'nm_owned': 'no',
'ifup_owned': 'yes', 'nm_present': 'no',
}
helper_guard = dp.VPC_LOCAL_NEIGHBOR_TRANSACTION_SH.replace(
'vc_ln_helper=/usr/local/libexec/virt-cloud-wg-fabric',
'vc_ln_helper=' + shlex.quote(str(helper)))
script = dp.VPC_APPLY_RECEIPT_SH + '\n' + helper_guard + '\n' + '\n'.join(
k + '=' + shlex.quote(v) for k, v in settings.items()) + '\n'
result = subprocess.run([shell, '-eu'], input=script + fragment_script,
text=True, capture_output=True)
assert result.returncode == 0, result.stderr
markers = dp._markers(result.stdout)
assert markers['bridge_admin'] == 'up'
assert markers['nft_file_exact'] == ('no' if mutation == 'nft' else 'yes')
assert markers['state_file_exact'] == ('no' if mutation in {'state', 'missing_state'} else 'yes')
assert markers['persistence_exact'] == ('no' if mutation == 'ifup' else 'yes')
assert markers['helper_contract'] == (
'missing' if mutation in {'helper', 'old_route_scope'} else 'owned')