github.com/cilium/cilium@v1.16.2/test/nat46x64/test.sh (about) 1 #!/usr/bin/env bash 2 3 PS4='+[\t] ' 4 set -eu 5 6 IMG_OWNER=${1:-cilium} 7 IMG_TAG=${2:-latest} 8 V=${V:-"0"} # Verbosity. 0 = quiet, 1 = loud 9 if [ "$V" != "0" ]; then 10 set -x 11 fi 12 13 CILIUM_EXEC="docker exec -t lb-node docker exec -t cilium-lb" 14 15 CFG_COMMON=("--enable-ipv4=true" "--enable-ipv6=true" "--devices=eth0" \ 16 "--datapath-mode=lb-only" "--bpf-lb-dsr-dispatch=ipip" \ 17 "--bpf-lb-mode=snat" "--enable-nat46x64-gateway=true") 18 19 TXT_XDP_MAGLEV="Mode:XDP\tAlgorithm:Maglev\tRecorder:Disabled" 20 CFG_XDP_MAGLEV=("--bpf-lb-acceleration=native" "--bpf-lb-algorithm=maglev") 21 22 TXT_TC__MAGLEV="Mode:TC \tAlgorithm:Maglev\tRecorder:Disabled" 23 CFG_TC__MAGLEV=("--bpf-lb-acceleration=disabled" "--bpf-lb-algorithm=maglev") 24 25 TXT_TC__RANDOM="Mode:TC \tAlgorithm:Random\tRecorder:Disabled" 26 CFG_TC__RANDOM=("--bpf-lb-acceleration=disabled" "--bpf-lb-algorithm=random") 27 28 TXT_XDP_MAGLEV_RECORDER="Mode:XDP\tAlgorithm:Maglev\tRecorder:Enabled" 29 30 CMD="$0" 31 32 function trace_offset { 33 local line_no=$1 34 shift 35 >&2 echo -e "\e[92m[${CMD}:${line_no}]\t$*\e[0m" 36 } 37 38 function trace_exec { 39 out=$($@ | nl -bn) 40 if [ "$V" != "0" ]; then 41 trace_offset "${BASH_LINENO[0]}" "Executing $*:\n$out" 42 fi 43 } 44 45 function info { 46 trace_offset "${BASH_LINENO[0]}" "$@" 47 } 48 49 function fatal_offset { 50 local line_no=$1 51 shift 52 >&2 echo -e "\e[31m[${CMD}:${line_no}]\t$*\e[0m" 53 exit 1 54 } 55 56 function fatal { 57 fatal_offset "${BASH_LINENO[0]}" "$@" 58 } 59 60 # $1 - Text to represent the install, used for logging 61 # $2+ - configuration options to pass to Cilium on startup 62 function cilium_install { 63 local cfg_text=$1 64 shift 65 66 trace_offset "${BASH_LINENO[0]}" "Installing Cilium with $cfg_text" 67 docker exec -t lb-node docker rm -f cilium-lb || true 68 docker exec -t lb-node \ 69 docker run --name cilium-lb -td \ 70 -v /sys/fs/bpf:/sys/fs/bpf \ 71 -v /lib/modules:/lib/modules \ 72 --privileged=true \ 73 --network=host \ 74 "quay.io/${IMG_OWNER}/cilium-ci:${IMG_TAG}" \ 75 cilium-agent "${CFG_COMMON[@]}" "$@" 76 result=1 77 for i in $(seq 1 10); do 78 if ${CILIUM_EXEC} cilium-dbg status --brief; then 79 result=0 80 break; 81 fi 82 if [ -z "$(docker exec lb-node docker ps -qf 'name=cilium-lb')" ]; then 83 # Early exit if cilium-agent is really just in trouble 84 result=1 85 break; 86 fi 87 sleep 3 88 done 89 if [ $result -ne 0 ]; then 90 ${CILIUM_EXEC} cilium-dbg status 91 containerID=$(docker exec lb-node docker inspect cilium-lb --format="{{ .Id }}") 92 docker exec lb-node docker logs "${containerID}" 93 fatal_offset "${BASH_LINENO[0]}" "Failed to install Cilium with $cfg_text" 94 fi 95 sleep 1 96 } 97 98 function initialize_docker_env { 99 # With Docker-in-Docker we create two nodes: 100 # 101 # * "lb-node" runs cilium in the LB-only mode. 102 # * "nginx" runs the nginx server. 103 104 trace_offset "${BASH_LINENO[0]}" "Initializing docker environment..." 105 106 docker network create --subnet="172.12.42.0/24,2001:db8:1::/64" --ipv6 cilium-l4lb 107 docker run --privileged --name lb-node -d --restart=on-failure:10 \ 108 --network cilium-l4lb -v /lib/modules:/lib/modules \ 109 docker:dind 110 docker exec -t lb-node mount bpffs /sys/fs/bpf -t bpf 111 docker run --name nginx -d --network cilium-l4lb nginx 112 113 # Wait until Docker is ready in the lb-node node 114 while ! docker exec -t lb-node docker ps >/dev/null; do sleep 1; done 115 116 # Disable TX and RX csum offloading, as veth does not support it. Otherwise, 117 # the forwarded packets by the LB to the worker node will have invalid csums. 118 IFIDX=$(docker exec -i lb-node \ 119 /bin/sh -c 'echo $(( $(ip -o l show eth0 | awk "{print $1}" | cut -d: -f1) ))') 120 LB_VETH_HOST=$(ip -o l | grep "if$IFIDX" | awk '{print $2}' | cut -d@ -f1) 121 ethtool -K "$LB_VETH_HOST" rx off tx off 122 } 123 124 function force_cleanup { 125 ${CILIUM_EXEC} cilium-dbg service delete 1 || true 126 ${CILIUM_EXEC} cilium-dbg service delete 2 || true 127 ip -4 r d "10.0.0.4/32" || true 128 ip -6 r d "fd00:cafe::1" || true 129 docker rm -f lb-node || true 130 docker rm -f nginx || true 131 docker network rm cilium-l4lb || true 132 } 133 134 function cleanup { 135 if tty -s; then 136 read -p "Hold the environment for debugging? [y/n]" -n 1 -r 137 echo 138 if [[ $REPLY =~ ^[Nn]$ ]]; then 139 force_cleanup 140 fi 141 fi 142 } 143 144 # $1 - target service, for example "[fd00:dead:beef:15:bad::1]:80" 145 function wait_service_ready { 146 # Try and sleep until the LB2 comes up, seems to be no other way to detect when the service is ready. 147 set +e 148 for i in $(seq 1 10); do 149 curl -s -o /dev/null "${1}" && break 150 sleep 1 151 done 152 set -e 153 } 154 155 function configure_local_route { 156 LB_VIP_RT="$1" 157 LB_VIP_FAM="$2" 158 159 LB_NODE_IP=$(docker exec -t lb-node \ 160 ip -o "${LB_VIP_FAM}" a s eth0 \ 161 | awk '{print $4}' \ 162 | cut -d/ -f1 \ 163 | head -n1) 164 trace_offset "${BASH_LINENO[0]}" "Installing route: 'ip ${LB_VIP_FAM} r a ${LB_VIP_RT} via $LB_NODE_IP'" 165 ip "${LB_VIP_FAM}" r a "${LB_VIP_RT}" via "$LB_NODE_IP" \ 166 || fatal "Failed to inject route into the localhost" 167 } 168 169 function assert_maglev_maps_sane { 170 MAG_V4=$(${CILIUM_EXEC} cilium-dbg bpf lb maglev list -o=jsonpath='{.\[1\]/v4}' | tr -d '\r') 171 MAG_V6=$(${CILIUM_EXEC} cilium-dbg bpf lb maglev list -o=jsonpath='{.\[1\]/v6}' | tr -d '\r') 172 if [ -n "$MAG_V4" ] || [ -z "$MAG_V6" ]; then 173 ${CILIUM_EXEC} cilium-dbg bpf lb maglev list 174 fatal_offset "${BASH_LINENO[0]}" "Invalid content of Maglev table!" 175 fi 176 } 177 178 function assert_connectivity_ok { 179 for i in $(seq 1 10); do 180 curl -s -o /dev/null "${1}" \ 181 || fatal_offset "${BASH_LINENO[0]}" "Failed connection from localhost to $1 (attempt $i/10)" 182 done 183 } 184 185 function test_services { 186 LB_VIP="$1" 187 LB_VIP_SVC="$2" 188 LB_VIP_FAM="$3" 189 LB_VIP_SUBNET="$4" 190 BACKEND_SVC="$5" 191 LB_ALT="$6" 192 LB_ALT_SVC="$7" 193 LB_ALT_FAM="$8" 194 LB_ALT_SUBNET="$9" 195 196 cilium_install "$TXT_TC__MAGLEV" ${CFG_TC__MAGLEV[@]} 197 198 info "Configuring service ${LB_VIP_SVC} -> ${BACKEND_SVC}" 199 ${CILIUM_EXEC} \ 200 cilium-dbg service update --id 1 --frontend "${LB_VIP_SVC}" --backends "${BACKEND_SVC}" --k8s-load-balancer \ 201 || fatal "Unable to configure service" 202 203 SVC_BEFORE=$(${CILIUM_EXEC} cilium-dbg service list | nl -bn) 204 205 trace_exec "${CILIUM_EXEC} cilium-dbg bpf lb list" 206 assert_maglev_maps_sane 207 208 info "Testing service ${LB_VIP_SVC} -> ${BACKEND_SVC} via TC + Maglev" 209 configure_local_route "${LB_VIP}/${LB_VIP_SUBNET}" "${LB_VIP_FAM}" 210 assert_connectivity_ok "${LB_VIP_SVC}" 211 212 cilium_install "$TXT_XDP_MAGLEV" ${CFG_XDP_MAGLEV[@]} 213 214 # Check that restoration went fine. Note that we currently cannot do runtime test 215 # as veth + XDP is broken when switching protocols. Needs something bare metal. 216 SVC_AFTER=$(${CILIUM_EXEC} cilium-dbg service list | nl -bn) 217 trace_exec "${CILIUM_EXEC} cilium-dbg bpf lb list" 218 219 info "Validating service restore after restart for service ${LB_VIP_SVC} -> ${BACKEND_SVC}" 220 if [ "$SVC_BEFORE" != "$SVC_AFTER" ]; then 221 fatal "Service ${LB_VIP_SVC} was not restored correctly\n" \ 222 "Before:\n" \ 223 "$SVC_BEFORE\n" \ 224 "After:\n" \ 225 "$SVC_AFTER\n" 226 fi 227 228 cilium_install "$TXT_TC__MAGLEV" ${CFG_TC__MAGLEV[@]} 229 info "Testing service ${LB_VIP_SVC} -> ${BACKEND_SVC} via TC + Maglev" 230 assert_connectivity_ok "${LB_VIP_SVC}" 231 232 # Check that curl also works for random selection 233 cilium_install "$TXT_TC__RANDOM" ${CFG_TC__RANDOM[@]} 234 info "Testing service ${LB_VIP_SVC} -> ${BACKEND_SVC} via TC + Random" 235 assert_connectivity_ok "${LB_VIP_SVC}" 236 237 # Add another same-protocol service and reuse backend (using $LB_ALT_FAM) 238 info "Configuring service ${LB_ALT_SVC} -> ${BACKEND_SVC}" 239 ${CILIUM_EXEC} \ 240 cilium-dbg service update --id 2 --frontend "${LB_ALT_SVC}" --backends "${BACKEND_SVC}" --k8s-load-balancer \ 241 || fatal "Unable to configure service" 242 trace_exec "${CILIUM_EXEC} cilium-dbg service list" 243 trace_exec "${CILIUM_EXEC} cilium-dbg bpf lb list" 244 configure_local_route "${LB_ALT}/${LB_ALT_SUBNET}" "${LB_ALT_FAM}" 245 246 info "Checking connectivity via ${LB_VIP_SVC} -> ${BACKEND_SVC}" 247 assert_connectivity_ok "${LB_VIP_SVC}" 248 wait_service_ready "${LB_ALT_SVC}" 249 info "Checking connectivity via ${LB_ALT_SVC} -> ${BACKEND_SVC}" 250 assert_connectivity_ok "${LB_ALT_SVC}" 251 252 cilium_install "$TXT_TC__MAGLEV" ${CFG_TC__MAGLEV[@]} 253 info "Testing service ${LB_VIP_SVC} -> ${BACKEND_SVC} via TC + Maglev" 254 assert_connectivity_ok "${LB_VIP_SVC}" 255 assert_connectivity_ok "${LB_ALT_SVC}" 256 257 ${CILIUM_EXEC} cilium-dbg service delete 1 258 ${CILIUM_EXEC} cilium-dbg service delete 2 259 } 260 261 # Check whether the list of filters matches the expected list. 262 # 263 # $1 - ID 264 # $@ - Filters 265 function check_recorder_list { 266 ID=$1 267 shift 268 num_filters=$# 269 if [ $(${CILIUM_EXEC} cilium-dbg bpf recorder list | grep "ID:${ID}" | wc -l) -ne "$num_filters" ]; then 270 echo "Expected filters:" 271 echo "$@" | nl -bn 272 echo "Found filters:" 273 ${CILIUM_EXEC} cilium-dbg bpf recorder list | nl -bn 274 fatal "Recorder filters did not match expected list" 275 fi 276 } 277 278 force_cleanup 2>&1 >/dev/null 279 initialize_docker_env 280 trap cleanup EXIT 281 282 NGINX_PID=$(docker inspect nginx -f '{{ .State.Pid }}') 283 WORKER_IP4=$(nsenter -t "$NGINX_PID" -n ip -o -4 a s eth0 | awk '{print $4}' | cut -d/ -f1 | head -n1) 284 WORKER_IP6=$(nsenter -t "$NGINX_PID" -n ip -o -6 a s eth0 | awk '{print $4}' | cut -d/ -f1 | head -n1) 285 286 # NAT 4->6 test suite (services) 287 ################################ 288 289 LB_VIP="10.0.0.4" 290 LB_VIP_SVC="$LB_VIP:80" 291 LB_VIP_FAM="-4" 292 BACKEND_SVC="[${WORKER_IP6}]:80" 293 LB_ALT="fd00:dead:beef:15:bad::1" 294 LB_ALT_SVC="[${LB_ALT}]:80" 295 LB_ALT_FAM="-6" 296 297 test_services "$LB_VIP" "$LB_VIP_SVC" "$LB_VIP_FAM" "32" "$BACKEND_SVC" \ 298 "$LB_ALT" "$LB_ALT_SVC" "$LB_ALT_FAM" "128" 299 300 # NAT 6->4 test suite (services) 301 ################################ 302 303 LB_VIP="fd00:cafe::1" 304 LB_VIP_SVC="[$LB_VIP]:80" 305 LB_VIP_FAM="-6" 306 BACKEND_SVC="${WORKER_IP4}:80" 307 LB_ALT="10.0.0.8" 308 LB_ALT_SVC="${LB_ALT}:80" 309 LB_ALT_FAM="-4" 310 311 test_services "$LB_VIP" "$LB_VIP_SVC" "$LB_VIP_FAM" "128" "$BACKEND_SVC" \ 312 "$LB_ALT" "$LB_ALT_SVC" "$LB_ALT_FAM" "32" 313 314 # NAT test suite & PCAP recorder 315 ################################ 316 317 RECORDER_FILTERS_IPV4=("2.2.2.2/0 0 1.1.1.1/32 80 TCP" \ 318 "2.2.2.2/1 0 1.1.1.1/32 80 TCP" \ 319 "2.2.2.2/2 0 1.1.1.1/31 80 TCP" \ 320 "2.2.2.2/3 0 1.1.1.1/30 80 TCP" \ 321 "2.2.2.2/4 0 1.1.1.1/29 80 TCP" \ 322 "2.2.2.2/5 0 1.1.1.1/28 80 TCP" \ 323 "2.2.2.2/6 0 1.1.1.1/27 80 TCP" \ 324 "2.2.2.2/7 0 1.1.1.1/26 80 TCP" \ 325 "2.2.2.2/8 0 1.1.1.1/25 80 TCP" \ 326 "2.2.2.2/9 0 1.1.1.1/24 80 TCP" \ 327 "2.2.2.2/10 0 1.1.1.1/23 80 TCP" \ 328 "2.2.2.2/11 0 1.1.1.1/22 80 TCP" \ 329 "2.2.2.2/12 0 1.1.1.1/21 80 TCP" \ 330 "2.2.2.2/13 0 1.1.1.1/20 80 TCP" \ 331 "2.2.2.2/14 0 1.1.1.1/19 80 TCP" \ 332 "2.2.2.2/15 0 1.1.1.1/18 80 TCP" \ 333 "2.2.2.2/16 0 1.1.1.1/17 80 TCP" \ 334 "2.2.2.2/17 0 1.1.1.1/16 80 TCP" \ 335 "2.2.2.2/18 0 1.1.1.1/15 80 TCP" \ 336 "2.2.2.2/19 0 1.1.1.1/14 80 TCP" \ 337 "2.2.2.2/20 0 1.1.1.1/13 80 TCP" \ 338 "2.2.2.2/21 0 1.1.1.1/12 80 TCP" \ 339 "2.2.2.2/22 0 1.1.1.1/11 80 TCP" \ 340 "2.2.2.2/23 0 1.1.1.1/10 80 TCP" \ 341 "2.2.2.2/24 0 1.1.1.1/9 80 TCP" \ 342 "2.2.2.2/25 0 1.1.1.1/8 80 TCP" \ 343 "2.2.2.2/26 0 1.1.1.1/7 80 TCP" \ 344 "2.2.2.2/27 0 1.1.1.1/6 80 TCP" \ 345 "2.2.2.2/28 0 1.1.1.1/5 80 TCP" \ 346 "2.2.2.2/29 0 1.1.1.1/4 80 TCP" \ 347 "2.2.2.2/30 0 1.1.1.1/3 80 TCP" \ 348 "2.2.2.2/31 0 1.1.1.1/2 80 TCP" \ 349 "2.2.2.2/32 0 1.1.1.1/1 80 TCP" \ 350 "2.2.2.2/32 0 1.1.1.1/0 80 TCP") 351 352 RECORDER_FILTERS_IPV6=("f00d::1/0 80 cafe::/128 0 UDP" \ 353 "f00d::1/1 80 cafe::/127 0 UDP" \ 354 "f00d::1/2 80 cafe::/126 0 UDP" \ 355 "f00d::1/3 80 cafe::/125 0 UDP" \ 356 "f00d::1/4 80 cafe::/124 0 UDP" \ 357 "f00d::1/5 80 cafe::/123 0 UDP" \ 358 "f00d::1/6 80 cafe::/122 0 UDP" \ 359 "f00d::1/7 80 cafe::/121 0 UDP" \ 360 "f00d::1/8 80 cafe::/120 0 UDP" \ 361 "f00d::1/9 80 cafe::/119 0 UDP" \ 362 "f00d::1/10 80 cafe::/118 0 UDP" \ 363 "f00d::1/11 80 cafe::/117 0 UDP" \ 364 "f00d::1/12 80 cafe::/116 0 UDP" \ 365 "f00d::1/13 80 cafe::/115 0 UDP" \ 366 "f00d::1/14 80 cafe::/114 0 UDP" \ 367 "f00d::1/15 80 cafe::/113 0 UDP" \ 368 "f00d::1/16 80 cafe::/112 0 UDP" \ 369 "f00d::1/17 80 cafe::/111 0 UDP" \ 370 "f00d::1/18 80 cafe::/110 0 UDP" \ 371 "f00d::1/19 80 cafe::/109 0 UDP" \ 372 "f00d::1/20 80 cafe::/108 0 UDP" \ 373 "f00d::1/21 80 cafe::/107 0 UDP" \ 374 "f00d::1/22 80 cafe::/106 0 UDP" \ 375 "f00d::1/23 80 cafe::/105 0 UDP" \ 376 "f00d::1/24 80 cafe::/104 0 UDP" \ 377 "f00d::1/25 80 cafe::/103 0 UDP" \ 378 "f00d::1/26 80 cafe::/102 0 UDP" \ 379 "f00d::1/27 80 cafe::/101 0 UDP" \ 380 "f00d::1/28 80 cafe::/100 0 UDP" \ 381 "f00d::1/29 80 cafe::/99 0 UDP" \ 382 "f00d::1/30 80 cafe::/98 0 UDP" \ 383 "f00d::1/31 80 cafe::/97 0 UDP" \ 384 "f00d::1/32 80 cafe::/96 0 UDP" \ 385 "f00d::1/32 80 cafe::/0 0 UDP") 386 387 # Install Cilium as standalone L4LB: XDP/Maglev/SNAT/Recorder 388 cilium_install "$TXT_XDP_MAGLEV_RECORDER" \ 389 --bpf-lb-algorithm=maglev \ 390 --bpf-lb-acceleration=native \ 391 --enable-recorder=true 392 393 # Trigger recompilation with 32 IPv4 filter masks 394 ${CILIUM_EXEC} \ 395 cilium-dbg recorder update --id 1 --caplen 100 \ 396 --filters="$(printf '%s,' "${RECORDER_FILTERS_IPV4[@]}" | sed 's/,*$//')" 397 398 # Trigger recompilation with 32 IPv6 filter masks 399 ${CILIUM_EXEC} \ 400 cilium-dbg recorder update --id 2 --caplen 100 \ 401 --filters="$(printf '%s,' "${RECORDER_FILTERS_IPV6[@]}" | sed 's/,*$//')" 402 403 check_recorder_list 1 "${RECORDER_FILTERS_IPV4[@]}" 404 check_recorder_list 2 "${RECORDER_FILTERS_IPV6[@]}" 405 406 trace_exec ${CILIUM_EXEC} cilium-dbg recorder list 407 trace_exec ${CILIUM_EXEC} cilium-dbg bpf recorder list 408 ${CILIUM_EXEC} cilium-dbg recorder delete 1 409 ${CILIUM_EXEC} cilium-dbg recorder delete 2 410 trace_exec ${CILIUM_EXEC} cilium-dbg recorder list 411 412 force_cleanup 413 echo "YAY!"