github.com/cilium/cilium@v1.16.2/test/l4lb/test.sh (about) 1 #!/usr/bin/env bash 2 3 PS4='+[\t] ' 4 set -eux 5 6 export LC_NUMERIC=C 7 8 IMG_OWNER=${1:-cilium} 9 IMG_TAG=${2:-latest} 10 11 ########### 12 # SETUP # 13 ########### 14 15 # bpf_xdp_veth_host is a dummy XDP program which is going to be attached to LB 16 # node's veth pair end in the host netns. When bpf_xdp, which is attached in 17 # the container netns, forwards a LB request with XDP_TX, the request needs to 18 # be picked in the host netns by a NAPI handler. To register the handler, we 19 # attach the dummy program. 20 apt-get update 21 apt-get install -y gcc-multilib libbpf-dev 22 clang -O2 -Wall --target=bpf -c bpf_xdp_veth_host.c -o bpf_xdp_veth_host.o 23 24 # The worker (aka backend node) will receive IPIP packets from the LB node. 25 # To decapsulate the packets instead of creating an ipip dev which would 26 # complicate network setup, we will attach the following program which 27 # terminates the tunnel. 28 # The program is taken from the Linux kernel selftests. 29 clang -O2 -Wall --target=bpf -c test_tc_tunnel.c -o test_tc_tunnel.o 30 31 # With Docker-in-Docker we create two nodes: 32 # 33 # * "lb-node" runs cilium in the LB-only mode. 34 # * "nginx" runs the nginx server. 35 36 docker network create cilium-l4lb 37 docker run --privileged --name lb-node -d --restart=on-failure:10 \ 38 --network cilium-l4lb -v /lib/modules:/lib/modules \ 39 docker:dind 40 docker run --name nginx -d --network cilium-l4lb nginx 41 42 # Create additional veth pair which is going to be used to test XDP_REDIRECT. 43 ip l a l4lb-veth0 type veth peer l4lb-veth1 44 SECOND_LB_NODE_IP=3.3.3.2 45 ip a a "3.3.3.1/24" dev l4lb-veth0 46 CONTROL_PLANE_PID=$(docker inspect lb-node -f '{{ .State.Pid }}') 47 ip l s dev l4lb-veth1 netns $CONTROL_PLANE_PID 48 ip l s dev l4lb-veth0 up 49 nsenter -t $CONTROL_PLANE_PID -n /bin/sh -c "\ 50 ip a a "${SECOND_LB_NODE_IP}/24" dev l4lb-veth1 && \ 51 ip l s dev l4lb-veth1 up" 52 53 # Wait until Docker is ready in the lb-node node 54 while ! docker exec -t lb-node docker ps >/dev/null; do sleep 1; done 55 56 # Install Cilium as standalone L4LB 57 docker exec -t lb-node mount bpffs /sys/fs/bpf -t bpf 58 docker exec -t lb-node \ 59 docker run --name cilium-lb -td \ 60 -v /sys/fs/bpf:/sys/fs/bpf \ 61 -v /lib/modules:/lib/modules \ 62 --privileged=true \ 63 --network=host \ 64 quay.io/${IMG_OWNER}/cilium-ci:${IMG_TAG} \ 65 cilium-agent \ 66 --enable-ipv4=true \ 67 --enable-ipv6=false \ 68 --datapath-mode=lb-only \ 69 --bpf-lb-algorithm=maglev \ 70 --bpf-lb-dsr-dispatch=ipip \ 71 --bpf-lb-acceleration=native \ 72 --bpf-lb-mode=dsr \ 73 --devices="eth0,l4lb-veth1" \ 74 --direct-routing-device=eth0 75 76 IFIDX=$(docker exec -i lb-node \ 77 /bin/sh -c 'echo $(( $(ip -o l show eth0 | awk "{print $1}" | cut -d: -f1) ))') 78 LB_VETH_HOST=$(ip -o l | grep "if$IFIDX" | awk '{print $2}' | cut -d@ -f1) 79 ip l set dev $LB_VETH_HOST xdp obj bpf_xdp_veth_host.o 80 ip l set dev l4lb-veth0 xdp obj bpf_xdp_veth_host.o 81 82 # Disable TX and RX csum offloading, as veth does not support it. Otherwise, 83 # the forwarded packets by the LB to the worker node will have invalid csums. 84 ethtool -K $LB_VETH_HOST rx off tx off 85 ethtool -K l4lb-veth0 rx off tx off 86 87 NGINX_PID=$(docker inspect nginx -f '{{ .State.Pid }}') 88 WORKER_IP=$(nsenter -t $NGINX_PID -n ip -o -4 a s eth0 | awk '{print $4}' | cut -d/ -f1) 89 WORKER_MAC=$(nsenter -t $NGINX_PID -n ip -o l show dev eth0 | grep -oP '(?<=link/ether )[^ ]+') 90 91 # Install BPF program to terminate encapsulated packets coming from the LB 92 nsenter -t $NGINX_PID -n /bin/sh -c \ 93 'tc qdisc add dev eth0 clsact && tc filter add dev eth0 ingress bpf direct-action object-file ./test_tc_tunnel.o section decap' 94 95 # Wait until Cilium is ready 96 while ! docker exec -t lb-node docker exec -t cilium-lb cilium-dbg status; do sleep 1; done 97 98 ########## 99 # TEST # 100 ########## 101 102 LB_VIP="10.0.0.2" 103 104 nsenter -t $(docker inspect nginx -f '{{ .State.Pid }}') -n /bin/sh -c \ 105 "ip a a dev eth0 ${LB_VIP}/32" 106 107 docker exec -t lb-node docker exec -t cilium-lb \ 108 cilium-dbg service update --id 1 --frontend "${LB_VIP}:80" --backends "${WORKER_IP}:80" --k8s-load-balancer 109 110 LB_NODE_IP=$(docker exec lb-node ip -o -4 a s eth0 | awk '{print $4}' | cut -d/ -f1) 111 ip r a "${LB_VIP}/32" via "$LB_NODE_IP" 112 113 # Issue 10 requests to LB 114 for i in $(seq 1 10); do 115 curl -o /dev/null "${LB_VIP}:80" || (echo "Failed $i"; exit -1) 116 done 117 118 # Now steer the traffic to LB_VIP via the secondary device so that XDP_REDIRECT 119 # can be tested on the L4LB node 120 ip r replace "${LB_VIP}/32" via "$SECOND_LB_NODE_IP" 121 122 # Issue 10 requests to LB 123 for i in $(seq 1 10); do 124 curl -o /dev/null "${LB_VIP}:80" || (echo "Failed $i"; exit -1) 125 done 126 127 # Set nginx to maintenance 128 docker exec -t lb-node docker exec -t cilium-lb \ 129 cilium-dbg service update --id 1 --frontend "${LB_VIP}:80" --backends "${WORKER_IP}:80" --backend-weights "0" --k8s-load-balancer 130 131 # Do not stop on error 132 set +e 133 # Issue 10 requests to LB, we expect all to fail due to a Failed connection 134 for i in $(seq 1 10); do 135 curl -o /dev/null -m 0.5 "${LB_VIP}:80" 136 # code 7 - Failed to connect ... : No route to host 137 if [ ! "$?" -eq 7 ]; then 138 exit -1; 139 fi 140 done 141 142 # Stop on error 143 set -e 144 docker exec -t lb-node docker exec -t cilium-lb \ 145 cilium-dbg service update --id 1 --frontend "${LB_VIP}:80" --backends "${WORKER_IP}:80" --backend-weights "1" --k8s-load-balancer 146 147 curl -o /dev/null "${LB_VIP}:80" -m1 || (echo "Failed"; exit -1) 148 149 # Restart cilium-agent and issue 50 requests to LB 150 docker exec -d lb-node docker restart cilium-lb 151 152 # Requests should not timeout when agent is starting up 153 for i in $(seq 1 50); do 154 curl -o /dev/null "${LB_VIP}:80" -m1 || (echo "Failed"; exit -1) 155 sleep 0.2 156 done 157 158 # Cleanup 159 docker rm -f lb-node 160 docker rm -f nginx 161 docker network rm cilium-l4lb