You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc_apply_reentry.py

266 lines
12 KiB
Python

"""Execute the generated apply/rollback fragments across link and NM flushes."""
from __future__ import annotations
import os
from pathlib import Path
import shlex
import subprocess
import pytest
from test_vpc_nft_dataplane import _plan
from virsh_panel.shell_contracts import LOCAL_NEIGHBOR_SH
from virsh_panel.vpc_dataplane import (
VPC_APPLY_RUNTIME_SH, VPC_LOCAL_NEIGHBOR_TRANSACTION_SH,
VPC_ROUTE_RECORDS_SH, render_apply_script,
)
MOCK = r'''
fake_ip() {
case "$*" in
# Ordinary guest-neighbor behavior has its own suite; keep this fixture
# empty while exercising the real neighbor baseline/rollback readers.
'-o -4 neigh show dev vcv-test') return 0 ;;
'-o -4 -N route show table all')
[ "$SCENARIO" != route_read_error ] || return 97
cat "$ROUTE_STATE"; return 0 ;;
'neigh show proxy dev vcv-test')
[ "$SCENARIO" != proxy_read_error ] || return 97
cat "$PROXY_STATE"; return 0 ;;
'link show dev vcv-test') return 0 ;;
'link set dev vcv-test down')
printf '%s\n' down > "$LINK_STATE"
: > "$PROXY_STATE"
awk '$0 !~ /dev vcv-test /' "$ROUTE_STATE" > "$ROUTE_STATE.tmp"
mv "$ROUTE_STATE.tmp" "$ROUTE_STATE"
printf '%s\n' down >> "$ACTION_LOG"
return 0 ;;
'link set dev vcv-test up') printf '%s\n' up > "$LINK_STATE"; return 0 ;;
esac
if [ "$1 $2 $3" = 'neigh del proxy' ]; then
awk -v value="$4" '$1 != value' "$PROXY_STATE" > "$PROXY_STATE.tmp"
mv "$PROXY_STATE.tmp" "$PROXY_STATE"
printf '%s\n' "proxy_del:$4" >> "$ACTION_LOG"
return 0
fi
if [ "$1 $2 $3" = 'neigh replace proxy' ]; then
[ "$SCENARIO" != proxy_write_error ] || return 97
printf '%s\n' "$4" >> "$PROXY_STATE"
sort -u "$PROXY_STATE" -o "$PROXY_STATE"
printf '%s\n' "proxy_add:$4" >> "$ACTION_LOG"
return 0
fi
if [ "$1 $2" = 'route replace' ]; then
[ "$SCENARIO" != route_write_error ] || return 97
dest=$3; shift 3
dev=; table=; via=; metric=
while [ "$#" -gt 0 ]; do
case "$1" in
dev) dev=$2; shift 2 ;;
table) table=$2; shift 2 ;;
via) via=$2; shift 2 ;;
metric) metric=$2; shift 2 ;;
proto) shift 2 ;;
onlink) shift ;;
*) return 99 ;;
esac
done
dest=${dest%/32}
awk -v value="$dest" '$1 != value' "$ROUTE_STATE" > "$ROUTE_STATE.tmp"
if [ -n "$via" ]; then
printf '%s\n' "$dest via $via dev $dev table $table proto 186 metric $metric onlink" >> "$ROUTE_STATE.tmp"
else
printf '%s\n' "$dest dev $dev table $table proto 186 scope link metric $metric" >> "$ROUTE_STATE.tmp"
fi
mv "$ROUTE_STATE.tmp" "$ROUTE_STATE"
printf '%s\n' "route_add:$dest" >> "$ACTION_LOG"
return 0
fi
return 99
}
fake_nm() {
[ "$1 $2" = 'connection up' ] || return 99
: > "$PROXY_STATE"
awk '$0 !~ /dev vcv-test /' "$ROUTE_STATE" > "$ROUTE_STATE.tmp"
mv "$ROUTE_STATE.tmp" "$ROUTE_STATE"
printf '%s\n' up > "$LINK_STATE"
printf '%s\n' nm_up >> "$ACTION_LOG"
return 0
}
vc_vpc_bridge_link_state() { cat "$LINK_STATE"; }
vc_vpc_bridge_topology_matches() { [ "$SCENARIO" != topology_drift ]; }
vc_bridge_port_records() { printf '%s\n' '5: vnet2: <UP> master vcv-test'; }
vc_nm_profile_resolve() { NM_PROFILE_UUID=known-uuid; return 0; }
vc_nm_profile_base_validate() { return 0; }
vc_nm_device_expect_managed() { return 0; }
vc_nm_profile_active() { return 0; }
vc_nm_device_quarantine() { fake_ip link set dev "$BRIDGE" down; }
vc_nm_device_release() { return 0; }
vc_vpc_nft_matches_snapshot() { [ "$SCENARIO" != nft_drift ]; }
vc_nft_table_matches_snapshot() { [ "$SCENARIO" != nft_drift ]; }
cp() {
if [ "$SCENARIO" = state_copy_error ] && [ "${3-}" = "$STATE_FILE" ]; then return 97; fi
command cp "$@"
}
IP=fake_ip; NMCLI=fake_nm; WG=true; NFT=true; CMP=cmp
BRIDGE=vcv-test; ROUTE_TABLE=16201; TABLE=virt_cloud_vpc_test
RULE_PRIORITY=26201; WG_RULE_PRIORITY=26202; WG_IFACE=wg-fabric0
'''
def env_and_files(tmp_path, scenario):
txn = tmp_path/'txn'; txn.mkdir()
proc = tmp_path/'proc'; proc.mkdir()
for key,value in (('rp_filter','2'),('proxy_arp','0'),('proxy_arp_pvlan','1')):
(proc/key).write_text(value+'\n')
state = (
'route|10.44.0.10/32|vcv-test|16201|-|100\n'
'route|10.44.0.11/32|wg-fabric0|16201|-|100\n'
)
(txn/'state.before').write_text(state)
(txn/'neighbors.desired').write_text('')
(txn/'owner.before').write_text('owner\n')
(tmp_path/'state').write_text(state)
(tmp_path/'owner').write_text('owner\n')
(tmp_path/'proxy.file').write_text('# virt-cloud-vpc-proxy-v1\n10.44.0.10\n10.44.0.11\n')
routes = (
'10.44.0.10 dev vcv-test table 16201 proto 186 scope link metric 100\n'
'10.44.0.11 dev wg-fabric0 table 16201 proto 186 scope link metric 100\n'
)
proxies = '10.44.0.10\n10.44.0.11\n'
if scenario == 'foreign_route': routes += '10.44.0.99 dev vcv-test table 16201 proto 186 metric 100\n'
if scenario == 'foreign_proxy': proxies += '10.44.0.99\n'
if scenario == 'malformed_route': routes += 'unreachable 10.44.0.99 table 16201\n'
if scenario == 'malformed_proxy': proxies += '10.44.0.99 dev wrong proxy\n'
if scenario == 'missing_runtime': routes=''; proxies=''
(tmp_path/'routes').write_text(routes)
(tmp_path/'proxies').write_text(proxies)
(tmp_path/'link').write_text('up\n'); (tmp_path/'actions').write_text('')
env=dict(os.environ, SCENARIO=scenario, TXN=str(txn), PROC_ROOT=str(proc),
ROUTE_STATE=str(tmp_path/'routes'), PROXY_STATE=str(tmp_path/'proxies'),
LINK_STATE=str(tmp_path/'link'), ACTION_LOG=str(tmp_path/'actions'),
PROXY_FILE=str(tmp_path/'proxy.file'), OWNER_FILE=str(tmp_path/'owner'),
STATE_FILE=str(tmp_path/'state'), SYSCTL_FILE=str(tmp_path/'sysctl'),
NFT_FILE=str(tmp_path/'nft'), NFT_MAIN=str(tmp_path/'nftmain'))
return env,routes,proxies
def source_helpers():
return ('\n'.join((VPC_ROUTE_RECORDS_SH, VPC_APPLY_RUNTIME_SH,
LOCAL_NEIGHBOR_SH, VPC_LOCAL_NEIGHBOR_TRANSACTION_SH))).replace(
'/proc/sys/net/ipv4/conf/$BRIDGE/', '$PROC_ROOT/')
@pytest.mark.parametrize('shell',['/bin/sh','/bin/bash'])
@pytest.mark.parametrize('scenario',['healthy','missing_runtime','foreign_route','foreign_proxy','malformed_route','malformed_proxy','route_read_error','proxy_read_error'])
def test_baseline_is_read_only_and_rejects_foreign_runtime(tmp_path,shell,scenario):
env,routes,proxies=env_and_files(tmp_path,scenario)
body='set -eu\n'+MOCK+source_helpers()+'''\nBRIDGE_TOPOLOGY_CAPTURED=yes
vc_vpc_runtime_baseline
'''
result=subprocess.run([shell,'-s'],input=body,text=True,capture_output=True,env=env)
assert (result.returncode==0)==(scenario in ('healthy','missing_runtime')),result.stderr
assert (tmp_path/'routes').read_text()==routes
assert (tmp_path/'proxies').read_text()==proxies
assert (tmp_path/'link').read_text()=='up\n'
assert (tmp_path/'actions').read_text()==''
if result.returncode==0:
assert (tmp_path/'txn/proxy.before').read_text()==proxies
assert (tmp_path/'txn/routes.before').is_file()
@pytest.mark.parametrize('shell',['/bin/sh','/bin/bash'])
@pytest.mark.parametrize('mode',['ifupdown','networkmanager_nmcli'])
@pytest.mark.parametrize('scenario',['healthy','missing_runtime','nft_drift','topology_drift','route_write_error','proxy_write_error','state_copy_error'])
def test_real_rollback_restores_prequarantine_runtime_after_manager_activation(tmp_path,shell,mode,scenario):
env,routes,proxies=env_and_files(tmp_path,scenario)
script=render_apply_script(_plan(),deployment_mode=mode)
init=next(line for line in script.splitlines() if line.startswith('COMMITTED=no;'))
rollback=script[script.index('rollback() {'):script.index('trap rollback EXIT HUP INT TERM')]
helper_start=script.index('vc_vpc_native_ifupdown_rollback_reopen() {')
# Include exactly the function, not the later helper-version admission
# gate. That independent pre-transaction gate has its own shell coverage.
helper=script[helper_start:script.index('\n}\n',helper_start)+3]
body='set -eu\n'+MOCK+source_helpers()+'\n'+helper+'\n'+init+'\nMODE='+shlex.quote(mode)+r'''
BRIDGE_TOPOLOGY_CAPTURED=yes; BRIDGE_WAS_UP=yes
NM_PROFILE_WAS_PRESENT=yes; NM_WAS_ACTIVE=yes
vc_vpc_runtime_baseline
vc_vpc_neighbors_baseline
MUTATION_STARTED=yes; RUNTIME_QUARANTINED=yes
if [ "$MODE" = networkmanager_nmcli ]; then NM_DEVICE_QUARANTINED=yes; fi
fake_ip link set dev "$BRIDGE" down
'''+rollback.replace('/proc/sys/net/ipv4/conf/$BRIDGE/','$PROC_ROOT/')+'''
trap rollback EXIT
exit 71
'''
result=subprocess.run([shell,'-s'],input=body,text=True,capture_output=True,env=env)
success=scenario in ('healthy','missing_runtime')
assert result.returncode==(71 if success else 79),result.stderr
if success:
assert (tmp_path/'proxies').read_text()==proxies
assert sorted((tmp_path/'routes').read_text().splitlines())==sorted(routes.splitlines())
assert (tmp_path/'link').read_text()=='up\n'
assert not (tmp_path/'txn').exists()
if mode=='networkmanager_nmcli' and scenario=='healthy':
actions=(tmp_path/'actions').read_text().splitlines()
assert actions.index('nm_up') < actions.index('proxy_add:10.44.0.10')
assert actions.index('nm_up') < actions.index('route_add:10.44.0.10')
else:
assert (tmp_path/'txn/state').read_text()=='rollback_failed\n'
assert (tmp_path/'link').read_text()=='down\n'
assert 'link del' not in (tmp_path/'actions').read_text()
@pytest.mark.parametrize('mode',['ifupdown','networkmanager_nmcli'])
def test_generated_script_snapshots_before_quarantine_and_never_overwrites_baseline(mode):
script=render_apply_script(_plan(),deployment_mode=mode)
for shell in ('/bin/sh','/bin/bash'):
checked=subprocess.run([shell,'-n'],input=script,text=True,capture_output=True)
assert checked.returncode==0,checked.stderr
baseline=script.index('vc_vpc_runtime_baseline || exit 71')
quarantine=script.index('RUNTIME_QUARANTINED=yes',baseline)
nft_write=script.index('\nvc_vpc_install_rules\n',quarantine)
assert script.index('trap rollback EXIT') < baseline < quarantine < nft_write
assert ': > "$TXN/routes.before"' not in script[nft_write:]
assert ': > "$TXN/proxy.before"' not in script[nft_write:]
assert '>> "$TXN/routes.before"' not in script[nft_write:]
assert 'done < "$TXN/proxy.activation.current"' in script[nft_write:]
@pytest.mark.parametrize('shell',['/bin/sh','/bin/bash'])
@pytest.mark.parametrize('scenario',['foreign_route','foreign_proxy','route_read_error','proxy_read_error'])
def test_snapshot_failure_trap_does_not_down_or_remove_existing_runtime(tmp_path,shell,scenario):
env,routes,proxies=env_and_files(tmp_path,scenario)
script=render_apply_script(_plan(),deployment_mode='ifupdown')
init=next(line for line in script.splitlines() if line.startswith('COMMITTED=no;'))
rollback=script[script.index('rollback() {'):script.index('trap rollback EXIT HUP INT TERM')]
body='set -eu\n'+MOCK+source_helpers()+'\n'+init+'\n'+rollback+r'''
BRIDGE_TOPOLOGY_CAPTURED=yes
trap rollback EXIT
vc_vpc_runtime_baseline || exit 71
exit 99
'''
result=subprocess.run([shell,'-s'],input=body,text=True,capture_output=True,env=env)
assert result.returncode==71,result.stderr
assert (tmp_path/'routes').read_text()==routes
assert (tmp_path/'proxies').read_text()==proxies
assert (tmp_path/'link').read_text()=='up\n'
assert (tmp_path/'actions').read_text()==''
assert (tmp_path/'owner').read_text()=='owner\n'
assert not (tmp_path/'txn').exists()
@pytest.mark.parametrize('shell',['/bin/sh','/bin/bash'])
def test_proxy_convergence_does_not_delete_entries_already_flushed_by_nm(tmp_path,shell):
env,_,_=env_and_files(tmp_path,'healthy')
script=render_apply_script(_plan(),deployment_mode='networkmanager_nmcli')
start=script.index('vc_vpc_proxy_snapshot "$TXN/proxy.activation.current" || exit 72')
end=script.index('PROXY_TMP=',start)
body='set -eu\n'+MOCK+source_helpers()+r'''
BRIDGE_TOPOLOGY_CAPTURED=yes
vc_vpc_runtime_baseline
fake_ip link set dev "$BRIDGE" down
fake_nm connection up uuid known-uuid
'''+script[start:end]
result=subprocess.run([shell,'-s'],input=body,text=True,capture_output=True,env=env)
assert result.returncode==0,result.stderr
assert (tmp_path/'proxies').read_text()=='10.44.0.10\n10.44.0.11\n'
assert 'proxy_del:' not in (tmp_path/'actions').read_text()