diff --git a/engine/tests/test_vpc.py b/engine/tests/test_vpc.py index 700449f1..0638f85b 100644 --- a/engine/tests/test_vpc.py +++ b/engine/tests/test_vpc.py @@ -3140,6 +3140,7 @@ def test_root_delete_surfaces_only_allowlisted_deterministic_host_errors(): "vpc_private_interface_conflict", "vpc_seed_missing", "vpc_seed_readback_invalid", + "vpc_seed_network_policy_migration_required", "vpc_private_xml_invalid", ) for index, error_code in enumerate(deterministic_codes, start=1): diff --git a/engine/tests/test_vpc_dataplane.py b/engine/tests/test_vpc_dataplane.py index 1fec6e0f..b82a553a 100644 --- a/engine/tests/test_vpc_dataplane.py +++ b/engine/tests/test_vpc_dataplane.py @@ -747,8 +747,12 @@ def test_remove_script_checks_ownership_before_destructive_cleanup(): assert 'files_removed|wg_merged)' in script assert script.index("exec 8>>") < script.index('exec 9>>"$LOCK"') assert '"$IP" -o -4 -N rule show pref' in script - assert 'route show table "$ROUTE_TABLE" 2>/dev/null) || return 1' in script - assert 'route show table "$ROUTE_TABLE" 2>/dev/null) || exit 72' in script + assert '"$IP" -o -4 -N route show table all 2>/dev/null' in script + assert 'verify_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE")' in script + assert 'orphan_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE")' in script + assert 'existing=$(vc_vpc_route_table_read "$table" "$destination")' in script + assert 'route show table "$ROUTE_TABLE" 2>/dev/null' not in script + assert 'route show table "$table" "$destination" 2>/dev/null' not in script assert "vc_ip_rule_scope_check" in script assert "managed 71 72" in script assert "empty 1 1" in script @@ -764,6 +768,89 @@ def test_remove_script_checks_ownership_before_destructive_cleanup(): assert '"$NFT" delete table "$family" "$name"' not in script +def test_remove_route_inventory_accepts_absent_custom_table_and_fails_closed( + tmp_path): + plan = _plan() + delete_plan = replace( + plan, + members=tuple(member for member in plan.members + if member.host_id != plan.host_id), + ) + script = render_remove_script(delete_plan) + function = re.search( + r"(?ms)^vc_vpc_route_table_read\(\) \{.*?^\}\n", + script, + ) + assert function is not None + + fake_ip = tmp_path / "ip" + fake_ip.write_text(r'''#!/bin/sh +[ "$*" = "-o -4 -N route show table all" ] || exit 96 +[ "${VC_ROUTE_STATUS:-0}" = 0 ] || exit "$VC_ROUTE_STATUS" +printf '%s\n' "${VC_ROUTE_INVENTORY:-}" +''', encoding="utf-8") + fake_ip.chmod(0o700) + base_env = os.environ.copy() + + def run_contract(inventory="", status="0", destination=""): + contract = function.group(0) + r''' +IP=$1 +ROUTE_TABLE=12102 +rows=$(vc_vpc_route_table_read "$ROUTE_TABLE" "$2") || exit 81 +printf '%s\n' "$rows" +''' + return subprocess.run( + ["/bin/sh", "-s", str(fake_ip), destination], + input=contract, text=True, capture_output=True, check=False, + env=base_env | { + "VC_ROUTE_INVENTORY": inventory, + "VC_ROUTE_STATUS": status, + }, + ) + + absent = run_contract( + "default via 192.0.2.1 dev eth0 proto 16\n" + "local 192.0.2.10 dev eth0 table 255 proto 2 scope host" + ) + assert absent.returncode == 0, absent.stderr + assert absent.stdout == "\n" + + expected = "10.0.0.2 dev vcv-test proto 186 table 12102 metric 100" + present = run_contract( + expected + "\n" + "10.0.0.3 dev vcv-test proto 186 table 12103 metric 100" + ) + assert present.returncode == 0, present.stderr + assert present.stdout == expected + "\n" + + selected = run_contract( + expected + "\n" + "10.0.0.3 dev vcv-test proto 186 table 12102 metric 100", + destination="10.0.0.3/32", + ) + assert selected.returncode == 0, selected.stderr + assert selected.stdout.startswith("10.0.0.3 ") + assert "10.0.0.2" not in selected.stdout + + typed = run_contract( + "6 10.0.0.4 table 12102 proto 186", + destination="10.0.0.4/32", + ) + assert typed.returncode == 0, typed.stderr + assert typed.stdout == "6 10.0.0.4 table 12102 proto 186\n" + + failed = run_contract(status="2") + assert failed.returncode == 81, failed.stderr + + for malformed in ( + "10.0.0.2 dev vcv-test table", + "10.0.0.2 dev vcv-test table custom", + "10.0.0.2 dev vcv-test table 12102 table 12102", + ): + result = run_contract(malformed) + assert result.returncode == 81, (malformed, result.stderr) + + def test_remove_artifact_identity_allows_older_revision_only(): plan = _plan() delete_plan = replace( diff --git a/engine/virsh_panel/vpc.py b/engine/virsh_panel/vpc.py index b84e5dec..2567809c 100644 --- a/engine/virsh_panel/vpc.py +++ b/engine/virsh_panel/vpc.py @@ -66,6 +66,7 @@ VPC_DELETE_DETERMINISTIC_HOST_ERRORS = frozenset({ "vpc_private_interface_conflict", "vpc_seed_missing", "vpc_seed_readback_invalid", + "vpc_seed_network_policy_migration_required", "vpc_private_xml_invalid", }) VPC_FABRIC_ADMISSION_ERRORS = frozenset({ diff --git a/engine/virsh_panel/vpc_dataplane.py b/engine/virsh_panel/vpc_dataplane.py index 607437db..66642596 100644 --- a/engine/virsh_panel/vpc_dataplane.py +++ b/engine/virsh_panel/vpc_dataplane.py @@ -2904,8 +2904,42 @@ set_phase() {{ mv -f "$phase_tmp" "$TXN/state" PHASE="$1" }} +vc_vpc_route_table_read() {{ + vc_vpc_route_wanted_table=$1 + vc_vpc_route_wanted_destination=${{2-}} + case "$vc_vpc_route_wanted_table" in ''|*[!0-9]*) return 1 ;; esac + vc_vpc_route_inventory=$("$IP" -o -4 -N route show table all 2>/dev/null) || return 1 + printf '%s\\n' "$vc_vpc_route_inventory" | awk \ + -v wanted_table="$vc_vpc_route_wanted_table" \ + -v wanted_destination="$vc_vpc_route_wanted_destination" ' + NF == 0 {{next}} + {{ + table_count=0 + table_value="254" + row_bad=0 + for (i=1; i<=NF; i++) {{ + if ($i == "table") {{ + table_count++ + if (i == NF || $(i+1) !~ /^[0-9]+$/) row_bad=1 + else table_value=$(i+1) + }} + }} + if (table_count > 1 || row_bad) bad=1 + route_destination=$1 + if ($1 ~ /^[0-9]+$/ && NF > 1) route_destination=$2 + wanted_host=wanted_destination + if (length(wanted_host) > 3 && + substr(wanted_host, length(wanted_host) - 2) == "/32") + wanted_host=substr(wanted_host, 1, length(wanted_host) - 3) + if (!row_bad && table_count <= 1 && table_value == wanted_table && + (wanted_destination == "" || route_destination == wanted_destination || + route_destination == wanted_host)) print + }} + END {{exit bad ? 1 : 0}} + ' +}} verify_routes_removed() {{ - verify_routes=$("$IP" -o -4 route show table "$ROUTE_TABLE" 2>/dev/null) || return 1 + verify_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE") || return 1 [ -z "$verify_routes" ] || return 1 vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "{r.rule_priority}" "$WG_IFACE" empty 1 1 || return 1 if $IP link show dev "$BRIDGE" >/dev/null 2>&1; then @@ -3006,7 +3040,7 @@ else # Without the state file there is no safe owner proof for deterministic # policy routes/rules. Refuse to claim a clean removal while any of those # resources remain; an operator can restore the state and retry. - orphan_routes=$("$IP" -o -4 route show table "$ROUTE_TABLE" 2>/dev/null) || exit 72 + orphan_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE") || exit 72 [ -z "$orphan_routes" ] || exit 71 vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "{r.rule_priority}" "$WG_IFACE" empty 71 72 || exit $? fi @@ -3197,7 +3231,7 @@ if [ -f "$TXN/state.before" ]; then *) printf '%s\\n' "$via" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$' || exit 71 ;; esac case "$metric" in ''|*[!0-9]*) exit 71;; esac - existing=$("$IP" -o -4 route show table "$table" "$destination" 2>/dev/null) || exit 72 + existing=$(vc_vpc_route_table_read "$table" "$destination") || exit 72 [ -n "$existing" ] || continue existing_count=$(printf '%s\\n' "$existing" | awk 'NF {{ count++ }} END {{ print count + 0 }}') [ "$existing_count" = 1 ] || exit 71 diff --git a/web/app/Services/Cloud/CloudEngineClient.php b/web/app/Services/Cloud/CloudEngineClient.php index 37e8ea20..ae18cfa8 100644 --- a/web/app/Services/Cloud/CloudEngineClient.php +++ b/web/app/Services/Cloud/CloudEngineClient.php @@ -93,6 +93,16 @@ final class CloudEngineClient $result['_engine_http_status'] = $response->getStatusCode(); return $result; } + $safeVpcServerErrorCode = self::safeVpcServerErrorCode( + $method, $signedPath, $result, $response->getStatusCode() + ); + if ($safeVpcServerErrorCode === 'vpc_delete_incomplete') { + throw new CloudError( + 'VPC 宿主机清理尚未全部确认,请稍后重新核对', + 503, + $safeVpcServerErrorCode + ); + } // Only explicitly reviewed endpoint-specific policies return // short, credential-free guidance useful to the caller. Do not // turn this into a general engine-error passthrough: arbitrary @@ -110,6 +120,19 @@ final class CloudEngineClient return $result; } + private static function safeVpcServerErrorCode( + string $method, ?string $path, array $result, ?int $status = null + ): ?string + { + if (strtoupper($method) !== 'POST' || $status !== 503 || $path === null + || ($result['error_code'] ?? null) !== 'vpc_delete_incomplete') { + return null; + } + return preg_match( + '#^/internal/v1/vpcs/[0-9a-f]{32}/delete$#D', $path + ) === 1 ? 'vpc_delete_incomplete' : null; + } + private static function safeAllowlistedError(?string $path, array $result, ?int $status = null): ?string { if ($path === null) { @@ -271,6 +294,7 @@ final class CloudEngineClient 'vpc_private_interface_conflict' => 'VPC 私网网卡身份不一致,请联系管理员核对', 'vpc_seed_missing' => 'VPC 成员 cloud-init 配置盘缺失,无法确认私网清理', 'vpc_seed_readback_invalid' => 'VPC 成员 cloud-init 配置盘无法核对,请联系管理员', + 'vpc_seed_network_policy_migration_required' => 'VPC 成员仍使用旧版 cloud-init 网络策略,需要迁移后再删除', 'vpc_private_xml_invalid' => 'VPC 成员持久化 XML 无效,无法确认私网清理', 'vpc_deleted' => 'VPC 已删除,不能重新应用旧目标', 'vpc_deleting' => 'VPC 正在删除,不能应用普通变更', diff --git a/web/app/Services/Cloud/CloudError.php b/web/app/Services/Cloud/CloudError.php index 285fd19b..d60f47d0 100644 --- a/web/app/Services/Cloud/CloudError.php +++ b/web/app/Services/Cloud/CloudError.php @@ -3,7 +3,8 @@ namespace App\Services\Cloud; final class CloudError extends \RuntimeException { - public function __construct(string $message, public readonly int $httpStatus = 422) + public function __construct(string $message, public readonly int $httpStatus = 422, + public readonly ?string $errorCode = null) { parent::__construct($message); } diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php index 208bccc1..2340fbf9 100644 --- a/web/app/Services/Cloud/CloudVpcService.php +++ b/web/app/Services/Cloud/CloudVpcService.php @@ -1135,7 +1135,8 @@ final class CloudVpcService return; } $row->status = $status; - $row->error_code = self::safeErrorCode($error->getMessage()) + $row->error_code = self::safeErrorCode((string) ($error->errorCode ?? '')) + ?? self::safeErrorCode($error->getMessage()) ?? ($status === 'uncertain' ? 'vpc_result_uncertain' : 'vpc_apply_failed'); $row->updated_at = $now; $row->finished_at = $status === 'failed' ? $now : null; diff --git a/web/tests/CloudVpcControlPlaneContractTest.php b/web/tests/CloudVpcControlPlaneContractTest.php index 4918f740..421264c4 100644 --- a/web/tests/CloudVpcControlPlaneContractTest.php +++ b/web/tests/CloudVpcControlPlaneContractTest.php @@ -23,6 +23,7 @@ $controller = $read('app/Controllers/CloudVpcController.php'); $routes = $read('config/routes_cloud.php'); $middleware = $read('app/Middleware/CloudApi.php'); $engineClient = $read('app/Services/Cloud/CloudEngineClient.php'); +$cloudError = $read('app/Services/Cloud/CloudError.php'); $accountDeletion = $read('app/Services/UserDataDeletionService.php'); $jobService = $read('app/Services/Cloud/CloudJobService.php'); $orderService = $read('app/Services/Cloud/CloudOrderService.php'); @@ -210,7 +211,13 @@ $assert(str_contains($service, 'public static function delete(') && str_contains($service, 'CloudPrivateCidrAllocator::release') && str_contains($service, '$childModel::where(\'vpc_id\', $vpc->id)->delete()') && str_contains($service, "\$vpc->state = 'deleted'") - && str_contains($service, "\$root->state = (string) \$row->kind === 'vpc_delete'"), + && str_contains($service, "\$root->state = (string) \$row->kind === 'vpc_delete'") + && str_contains($service, "\$error->errorCode ?? ''") + && str_contains($engineClient, 'safeVpcServerErrorCode') + && str_contains($engineClient, "strtoupper(\$method) !== 'POST'") + && str_contains($engineClient, + '#^/internal/v1/vpcs/[0-9a-f]{32}/delete$#D') + && str_contains($cloudError, 'public readonly ?string $errorCode = null'), 'VPC root deletion lost confirmation, durable operation, exact receipt or CIDR CAS finalization.'); $assert(!str_contains($routes, "/subnets'") && !str_contains($routes, '/subnets/{') diff --git a/web/tests/CloudVpcEngineErrorContractTest.php b/web/tests/CloudVpcEngineErrorContractTest.php new file mode 100644 index 00000000..dc1f53c4 --- /dev/null +++ b/web/tests/CloudVpcEngineErrorContractTest.php @@ -0,0 +1,131 @@ +status; + } + + public function getBody(): string + { + return $this->body; + } + } + + class Client + { + public function __construct(private array $responses = []) + { + } + + public function request(string $method, string $url, array $options): VpcErrorTestResponse + { + $response = array_shift($this->responses); + if (!$response instanceof VpcErrorTestResponse) { + throw new \RuntimeException('No fake engine response was queued.'); + } + return $response; + } + } +} + +namespace { +use App\Services\Cloud\CloudEngineClient; +use App\Services\Cloud\CloudError; +use GuzzleHttp\Client; +use GuzzleHttp\VpcErrorTestResponse; + +require dirname(__DIR__) . '/app/Services/Cloud/CloudError.php'; +require dirname(__DIR__) . '/app/Services/Cloud/CloudEngineClient.php'; + +$assert = static function ($condition, string $message): void { + if (!$condition) { + throw new RuntimeException($message); + } +}; +$actor = ['user_id' => 1, 'is_admin' => false, 'can_manage' => false]; +$deletePath = '/internal/v1/vpcs/' . str_repeat('a', 32) . '/delete'; +$safeMessage = 'VPC 宿主机清理尚未全部确认,请稍后重新核对'; + +$request = static function (string $path, int $status, string $errorCode, + string $method = 'POST') use ($actor): CloudError { + $client = new CloudEngineClient(new Client([ + new VpcErrorTestResponse($status, json_encode([ + 'error_code' => $errorCode, + 'error' => 'Traceback: host=/root/private key=secret', + ], JSON_UNESCAPED_UNICODE)), + ])); + try { + $client->request($method, $path, $actor, []); + } catch (CloudError $error) { + return $error; + } + throw new RuntimeException('The fake VPC engine failure did not raise CloudError.'); +}; + +$reviewed = $request($deletePath, 503, 'vpc_delete_incomplete'); +$assert($reviewed->getMessage() === $safeMessage + && $reviewed->httpStatus === 503 + && $reviewed->errorCode === 'vpc_delete_incomplete' + && !str_contains($reviewed->getMessage(), '/root') + && !str_contains($reviewed->getMessage(), 'private'), + 'The reviewed delete-incomplete response lost its fixed message or machine code.'); + +foreach ([ + [$deletePath, 503, 'vpc_unknown_failure'], + ['/internal/v1/vpcs/apply', 503, 'vpc_delete_incomplete'], + ['/internal/v1/vpcs/' . str_repeat('a', 32) . '/reconcile', 503, + 'vpc_delete_incomplete'], + ['/internal/v1/vpcs/' . str_repeat('a', 32), 503, + 'vpc_delete_incomplete'], + [$deletePath . '?unexpected=true', 503, 'vpc_delete_incomplete'], + [$deletePath, 500, 'vpc_delete_incomplete'], + [$deletePath, 502, 'vpc_delete_incomplete'], +] as [$path, $status, $errorCode]) { + $generic = $request($path, $status, $errorCode); + $assert($generic->getMessage() === '虚拟化操作未成功(引擎状态 ' . $status . ')' + && $generic->httpStatus === 502 + && $generic->errorCode === null + && !str_contains($generic->getMessage(), '/root') + && !str_contains($generic->getMessage(), 'private'), + 'A non-allowlisted VPC server error crossed the generic trust boundary.'); +} + +foreach (['GET', 'PUT'] as $method) { + $generic = $request( + $deletePath, 503, 'vpc_delete_incomplete', $method + ); + $assert($generic->getMessage() === '虚拟化操作未成功(引擎状态 503)' + && $generic->httpStatus === 502 + && $generic->errorCode === null, + 'A non-POST VPC delete error crossed the typed 503 trust boundary.'); +} + +echo "Cloud VPC engine error-code contract passed.\n"; +}