fix(vpc): reconcile absent route tables during delete

venix
admin 1 month ago
parent 61726f87e4
commit d990d386e1

@ -3140,6 +3140,7 @@ def test_root_delete_surfaces_only_allowlisted_deterministic_host_errors():
"vpc_private_interface_conflict",
"vpc_seed_missing",
"vpc_seed_readback_invalid",
"vpc_seed_network_policy_migration_required",
"vpc_private_xml_invalid",
)
for index, error_code in enumerate(deterministic_codes, start=1):

@ -747,8 +747,12 @@ def test_remove_script_checks_ownership_before_destructive_cleanup():
assert 'files_removed|wg_merged)' in script
assert script.index("exec 8>>") < script.index('exec 9>>"$LOCK"')
assert '"$IP" -o -4 -N rule show pref' in script
assert 'route show table "$ROUTE_TABLE" 2>/dev/null) || return 1' in script
assert 'route show table "$ROUTE_TABLE" 2>/dev/null) || exit 72' in script
assert '"$IP" -o -4 -N route show table all 2>/dev/null' in script
assert 'verify_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE")' in script
assert 'orphan_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE")' in script
assert 'existing=$(vc_vpc_route_table_read "$table" "$destination")' in script
assert 'route show table "$ROUTE_TABLE" 2>/dev/null' not in script
assert 'route show table "$table" "$destination" 2>/dev/null' not in script
assert "vc_ip_rule_scope_check" in script
assert "managed 71 72" in script
assert "empty 1 1" in script
@ -764,6 +768,89 @@ def test_remove_script_checks_ownership_before_destructive_cleanup():
assert '"$NFT" delete table "$family" "$name"' not in script
def test_remove_route_inventory_accepts_absent_custom_table_and_fails_closed(
tmp_path):
plan = _plan()
delete_plan = replace(
plan,
members=tuple(member for member in plan.members
if member.host_id != plan.host_id),
)
script = render_remove_script(delete_plan)
function = re.search(
r"(?ms)^vc_vpc_route_table_read\(\) \{.*?^\}\n",
script,
)
assert function is not None
fake_ip = tmp_path / "ip"
fake_ip.write_text(r'''#!/bin/sh
[ "$*" = "-o -4 -N route show table all" ] || exit 96
[ "${VC_ROUTE_STATUS:-0}" = 0 ] || exit "$VC_ROUTE_STATUS"
printf '%s\n' "${VC_ROUTE_INVENTORY:-}"
''', encoding="utf-8")
fake_ip.chmod(0o700)
base_env = os.environ.copy()
def run_contract(inventory="", status="0", destination=""):
contract = function.group(0) + r'''
IP=$1
ROUTE_TABLE=12102
rows=$(vc_vpc_route_table_read "$ROUTE_TABLE" "$2") || exit 81
printf '%s\n' "$rows"
'''
return subprocess.run(
["/bin/sh", "-s", str(fake_ip), destination],
input=contract, text=True, capture_output=True, check=False,
env=base_env | {
"VC_ROUTE_INVENTORY": inventory,
"VC_ROUTE_STATUS": status,
},
)
absent = run_contract(
"default via 192.0.2.1 dev eth0 proto 16\n"
"local 192.0.2.10 dev eth0 table 255 proto 2 scope host"
)
assert absent.returncode == 0, absent.stderr
assert absent.stdout == "\n"
expected = "10.0.0.2 dev vcv-test proto 186 table 12102 metric 100"
present = run_contract(
expected + "\n"
"10.0.0.3 dev vcv-test proto 186 table 12103 metric 100"
)
assert present.returncode == 0, present.stderr
assert present.stdout == expected + "\n"
selected = run_contract(
expected + "\n"
"10.0.0.3 dev vcv-test proto 186 table 12102 metric 100",
destination="10.0.0.3/32",
)
assert selected.returncode == 0, selected.stderr
assert selected.stdout.startswith("10.0.0.3 ")
assert "10.0.0.2" not in selected.stdout
typed = run_contract(
"6 10.0.0.4 table 12102 proto 186",
destination="10.0.0.4/32",
)
assert typed.returncode == 0, typed.stderr
assert typed.stdout == "6 10.0.0.4 table 12102 proto 186\n"
failed = run_contract(status="2")
assert failed.returncode == 81, failed.stderr
for malformed in (
"10.0.0.2 dev vcv-test table",
"10.0.0.2 dev vcv-test table custom",
"10.0.0.2 dev vcv-test table 12102 table 12102",
):
result = run_contract(malformed)
assert result.returncode == 81, (malformed, result.stderr)
def test_remove_artifact_identity_allows_older_revision_only():
plan = _plan()
delete_plan = replace(

@ -66,6 +66,7 @@ VPC_DELETE_DETERMINISTIC_HOST_ERRORS = frozenset({
"vpc_private_interface_conflict",
"vpc_seed_missing",
"vpc_seed_readback_invalid",
"vpc_seed_network_policy_migration_required",
"vpc_private_xml_invalid",
})
VPC_FABRIC_ADMISSION_ERRORS = frozenset({

@ -2904,8 +2904,42 @@ set_phase() {{
mv -f "$phase_tmp" "$TXN/state"
PHASE="$1"
}}
vc_vpc_route_table_read() {{
vc_vpc_route_wanted_table=$1
vc_vpc_route_wanted_destination=${{2-}}
case "$vc_vpc_route_wanted_table" in ''|*[!0-9]*) return 1 ;; esac
vc_vpc_route_inventory=$("$IP" -o -4 -N route show table all 2>/dev/null) || return 1
printf '%s\\n' "$vc_vpc_route_inventory" | awk \
-v wanted_table="$vc_vpc_route_wanted_table" \
-v wanted_destination="$vc_vpc_route_wanted_destination" '
NF == 0 {{next}}
{{
table_count=0
table_value="254"
row_bad=0
for (i=1; i<=NF; i++) {{
if ($i == "table") {{
table_count++
if (i == NF || $(i+1) !~ /^[0-9]+$/) row_bad=1
else table_value=$(i+1)
}}
}}
if (table_count > 1 || row_bad) bad=1
route_destination=$1
if ($1 ~ /^[0-9]+$/ && NF > 1) route_destination=$2
wanted_host=wanted_destination
if (length(wanted_host) > 3 &&
substr(wanted_host, length(wanted_host) - 2) == "/32")
wanted_host=substr(wanted_host, 1, length(wanted_host) - 3)
if (!row_bad && table_count <= 1 && table_value == wanted_table &&
(wanted_destination == "" || route_destination == wanted_destination ||
route_destination == wanted_host)) print
}}
END {{exit bad ? 1 : 0}}
'
}}
verify_routes_removed() {{
verify_routes=$("$IP" -o -4 route show table "$ROUTE_TABLE" 2>/dev/null) || return 1
verify_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE") || return 1
[ -z "$verify_routes" ] || return 1
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "{r.rule_priority}" "$WG_IFACE" empty 1 1 || return 1
if $IP link show dev "$BRIDGE" >/dev/null 2>&1; then
@ -3006,7 +3040,7 @@ else
# Without the state file there is no safe owner proof for deterministic
# policy routes/rules. Refuse to claim a clean removal while any of those
# resources remain; an operator can restore the state and retry.
orphan_routes=$("$IP" -o -4 route show table "$ROUTE_TABLE" 2>/dev/null) || exit 72
orphan_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE") || exit 72
[ -z "$orphan_routes" ] || exit 71
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "{r.rule_priority}" "$WG_IFACE" empty 71 72 || exit $?
fi
@ -3197,7 +3231,7 @@ if [ -f "$TXN/state.before" ]; then
*) printf '%s\\n' "$via" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$' || exit 71 ;;
esac
case "$metric" in ''|*[!0-9]*) exit 71;; esac
existing=$("$IP" -o -4 route show table "$table" "$destination" 2>/dev/null) || exit 72
existing=$(vc_vpc_route_table_read "$table" "$destination") || exit 72
[ -n "$existing" ] || continue
existing_count=$(printf '%s\\n' "$existing" | awk 'NF {{ count++ }} END {{ print count + 0 }}')
[ "$existing_count" = 1 ] || exit 71

@ -93,6 +93,16 @@ final class CloudEngineClient
$result['_engine_http_status'] = $response->getStatusCode();
return $result;
}
$safeVpcServerErrorCode = self::safeVpcServerErrorCode(
$method, $signedPath, $result, $response->getStatusCode()
);
if ($safeVpcServerErrorCode === 'vpc_delete_incomplete') {
throw new CloudError(
'VPC 宿主机清理尚未全部确认,请稍后重新核对',
503,
$safeVpcServerErrorCode
);
}
// Only explicitly reviewed endpoint-specific policies return
// short, credential-free guidance useful to the caller. Do not
// turn this into a general engine-error passthrough: arbitrary
@ -110,6 +120,19 @@ final class CloudEngineClient
return $result;
}
private static function safeVpcServerErrorCode(
string $method, ?string $path, array $result, ?int $status = null
): ?string
{
if (strtoupper($method) !== 'POST' || $status !== 503 || $path === null
|| ($result['error_code'] ?? null) !== 'vpc_delete_incomplete') {
return null;
}
return preg_match(
'#^/internal/v1/vpcs/[0-9a-f]{32}/delete$#D', $path
) === 1 ? 'vpc_delete_incomplete' : null;
}
private static function safeAllowlistedError(?string $path, array $result, ?int $status = null): ?string
{
if ($path === null) {
@ -271,6 +294,7 @@ final class CloudEngineClient
'vpc_private_interface_conflict' => 'VPC 私网网卡身份不一致,请联系管理员核对',
'vpc_seed_missing' => 'VPC 成员 cloud-init 配置盘缺失,无法确认私网清理',
'vpc_seed_readback_invalid' => 'VPC 成员 cloud-init 配置盘无法核对,请联系管理员',
'vpc_seed_network_policy_migration_required' => 'VPC 成员仍使用旧版 cloud-init 网络策略,需要迁移后再删除',
'vpc_private_xml_invalid' => 'VPC 成员持久化 XML 无效,无法确认私网清理',
'vpc_deleted' => 'VPC 已删除,不能重新应用旧目标',
'vpc_deleting' => 'VPC 正在删除,不能应用普通变更',

@ -3,7 +3,8 @@ namespace App\Services\Cloud;
final class CloudError extends \RuntimeException
{
public function __construct(string $message, public readonly int $httpStatus = 422)
public function __construct(string $message, public readonly int $httpStatus = 422,
public readonly ?string $errorCode = null)
{
parent::__construct($message);
}

@ -1135,7 +1135,8 @@ final class CloudVpcService
return;
}
$row->status = $status;
$row->error_code = self::safeErrorCode($error->getMessage())
$row->error_code = self::safeErrorCode((string) ($error->errorCode ?? ''))
?? self::safeErrorCode($error->getMessage())
?? ($status === 'uncertain' ? 'vpc_result_uncertain' : 'vpc_apply_failed');
$row->updated_at = $now;
$row->finished_at = $status === 'failed' ? $now : null;

@ -23,6 +23,7 @@ $controller = $read('app/Controllers/CloudVpcController.php');
$routes = $read('config/routes_cloud.php');
$middleware = $read('app/Middleware/CloudApi.php');
$engineClient = $read('app/Services/Cloud/CloudEngineClient.php');
$cloudError = $read('app/Services/Cloud/CloudError.php');
$accountDeletion = $read('app/Services/UserDataDeletionService.php');
$jobService = $read('app/Services/Cloud/CloudJobService.php');
$orderService = $read('app/Services/Cloud/CloudOrderService.php');
@ -210,7 +211,13 @@ $assert(str_contains($service, 'public static function delete(')
&& str_contains($service, 'CloudPrivateCidrAllocator::release')
&& str_contains($service, '$childModel::where(\'vpc_id\', $vpc->id)->delete()')
&& str_contains($service, "\$vpc->state = 'deleted'")
&& str_contains($service, "\$root->state = (string) \$row->kind === 'vpc_delete'"),
&& str_contains($service, "\$root->state = (string) \$row->kind === 'vpc_delete'")
&& str_contains($service, "\$error->errorCode ?? ''")
&& str_contains($engineClient, 'safeVpcServerErrorCode')
&& str_contains($engineClient, "strtoupper(\$method) !== 'POST'")
&& str_contains($engineClient,
'#^/internal/v1/vpcs/[0-9a-f]{32}/delete$#D')
&& str_contains($cloudError, 'public readonly ?string $errorCode = null'),
'VPC root deletion lost confirmation, durable operation, exact receipt or CIDR CAS finalization.');
$assert(!str_contains($routes, "/subnets'")
&& !str_contains($routes, '/subnets/{')

@ -0,0 +1,131 @@
<?php
namespace App\Services {
final class Config
{
public static function get($name, $default = null)
{
return $name === 'cloud_engine_url' ? 'http://127.0.0.1:8000' : $default;
}
}
}
namespace App\Services\Cloud {
final class CloudSecrets
{
public static function key(string $purpose): string
{
return str_repeat('k', 32);
}
}
}
namespace GuzzleHttp {
final class VpcErrorTestResponse
{
public function __construct(private int $status, private string $body)
{
}
public function getStatusCode(): int
{
return $this->status;
}
public function getBody(): string
{
return $this->body;
}
}
class Client
{
public function __construct(private array $responses = [])
{
}
public function request(string $method, string $url, array $options): VpcErrorTestResponse
{
$response = array_shift($this->responses);
if (!$response instanceof VpcErrorTestResponse) {
throw new \RuntimeException('No fake engine response was queued.');
}
return $response;
}
}
}
namespace {
use App\Services\Cloud\CloudEngineClient;
use App\Services\Cloud\CloudError;
use GuzzleHttp\Client;
use GuzzleHttp\VpcErrorTestResponse;
require dirname(__DIR__) . '/app/Services/Cloud/CloudError.php';
require dirname(__DIR__) . '/app/Services/Cloud/CloudEngineClient.php';
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$actor = ['user_id' => 1, 'is_admin' => false, 'can_manage' => false];
$deletePath = '/internal/v1/vpcs/' . str_repeat('a', 32) . '/delete';
$safeMessage = 'VPC 宿主机清理尚未全部确认,请稍后重新核对';
$request = static function (string $path, int $status, string $errorCode,
string $method = 'POST') use ($actor): CloudError {
$client = new CloudEngineClient(new Client([
new VpcErrorTestResponse($status, json_encode([
'error_code' => $errorCode,
'error' => 'Traceback: host=/root/private key=secret',
], JSON_UNESCAPED_UNICODE)),
]));
try {
$client->request($method, $path, $actor, []);
} catch (CloudError $error) {
return $error;
}
throw new RuntimeException('The fake VPC engine failure did not raise CloudError.');
};
$reviewed = $request($deletePath, 503, 'vpc_delete_incomplete');
$assert($reviewed->getMessage() === $safeMessage
&& $reviewed->httpStatus === 503
&& $reviewed->errorCode === 'vpc_delete_incomplete'
&& !str_contains($reviewed->getMessage(), '/root')
&& !str_contains($reviewed->getMessage(), 'private'),
'The reviewed delete-incomplete response lost its fixed message or machine code.');
foreach ([
[$deletePath, 503, 'vpc_unknown_failure'],
['/internal/v1/vpcs/apply', 503, 'vpc_delete_incomplete'],
['/internal/v1/vpcs/' . str_repeat('a', 32) . '/reconcile', 503,
'vpc_delete_incomplete'],
['/internal/v1/vpcs/' . str_repeat('a', 32), 503,
'vpc_delete_incomplete'],
[$deletePath . '?unexpected=true', 503, 'vpc_delete_incomplete'],
[$deletePath, 500, 'vpc_delete_incomplete'],
[$deletePath, 502, 'vpc_delete_incomplete'],
] as [$path, $status, $errorCode]) {
$generic = $request($path, $status, $errorCode);
$assert($generic->getMessage() === '虚拟化操作未成功(引擎状态 ' . $status . ')'
&& $generic->httpStatus === 502
&& $generic->errorCode === null
&& !str_contains($generic->getMessage(), '/root')
&& !str_contains($generic->getMessage(), 'private'),
'A non-allowlisted VPC server error crossed the generic trust boundary.');
}
foreach (['GET', 'PUT'] as $method) {
$generic = $request(
$deletePath, 503, 'vpc_delete_incomplete', $method
);
$assert($generic->getMessage() === '虚拟化操作未成功(引擎状态 503)'
&& $generic->httpStatus === 502
&& $generic->errorCode === null,
'A non-POST VPC delete error crossed the typed 503 trust boundary.');
}
echo "Cloud VPC engine error-code contract passed.\n";
}
Loading…
Cancel
Save