github.com/cilium/cilium@v1.16.2/test/l4lb/test.sh (about)

     1  #!/usr/bin/env bash
     2  
     3  PS4='+[\t] '
     4  set -eux
     5  
     6  export LC_NUMERIC=C
     7  
     8  IMG_OWNER=${1:-cilium}
     9  IMG_TAG=${2:-latest}
    10  
    11  ###########
    12  #  SETUP  #
    13  ###########
    14  
    15  # bpf_xdp_veth_host is a dummy XDP program which is going to be attached to LB
    16  # node's veth pair end in the host netns. When bpf_xdp, which is attached in
    17  # the container netns, forwards a LB request with XDP_TX, the request needs to
    18  # be picked in the host netns by a NAPI handler. To register the handler, we
    19  # attach the dummy program.
    20  apt-get update
    21  apt-get install -y gcc-multilib libbpf-dev
    22  clang -O2 -Wall --target=bpf -c bpf_xdp_veth_host.c -o bpf_xdp_veth_host.o
    23  
    24  # The worker (aka backend node) will receive IPIP packets from the LB node.
    25  # To decapsulate the packets instead of creating an ipip dev which would
    26  # complicate network setup, we will attach the following program which
    27  # terminates the tunnel.
    28  # The program is taken from the Linux kernel selftests.
    29  clang -O2 -Wall --target=bpf -c test_tc_tunnel.c -o test_tc_tunnel.o
    30  
    31  # With Docker-in-Docker we create two nodes:
    32  #
    33  # * "lb-node" runs cilium in the LB-only mode.
    34  # * "nginx" runs the nginx server.
    35  
    36  docker network create cilium-l4lb
    37  docker run --privileged --name lb-node -d --restart=on-failure:10 \
    38      --network cilium-l4lb -v /lib/modules:/lib/modules \
    39      docker:dind
    40  docker run --name nginx -d --network cilium-l4lb nginx
    41  
    42  # Create additional veth pair which is going to be used to test XDP_REDIRECT.
    43  ip l a l4lb-veth0 type veth peer l4lb-veth1
    44  SECOND_LB_NODE_IP=3.3.3.2
    45  ip a a "3.3.3.1/24" dev l4lb-veth0
    46  CONTROL_PLANE_PID=$(docker inspect lb-node -f '{{ .State.Pid }}')
    47  ip l s dev l4lb-veth1 netns $CONTROL_PLANE_PID
    48  ip l s dev l4lb-veth0 up
    49  nsenter -t $CONTROL_PLANE_PID -n /bin/sh -c "\
    50      ip a a "${SECOND_LB_NODE_IP}/24" dev l4lb-veth1 && \
    51      ip l s dev l4lb-veth1 up"
    52  
    53  # Wait until Docker is ready in the lb-node node
    54  while ! docker exec -t lb-node docker ps >/dev/null; do sleep 1; done
    55  
    56  # Install Cilium as standalone L4LB
    57  docker exec -t lb-node mount bpffs /sys/fs/bpf -t bpf
    58  docker exec -t lb-node \
    59    docker run --name cilium-lb -td \
    60      -v /sys/fs/bpf:/sys/fs/bpf \
    61      -v /lib/modules:/lib/modules \
    62      --privileged=true \
    63      --network=host \
    64      quay.io/${IMG_OWNER}/cilium-ci:${IMG_TAG} \
    65      cilium-agent \
    66      --enable-ipv4=true \
    67      --enable-ipv6=false \
    68      --datapath-mode=lb-only \
    69      --bpf-lb-algorithm=maglev \
    70      --bpf-lb-dsr-dispatch=ipip \
    71      --bpf-lb-acceleration=native \
    72      --bpf-lb-mode=dsr \
    73      --devices="eth0,l4lb-veth1" \
    74      --direct-routing-device=eth0
    75  
    76  IFIDX=$(docker exec -i lb-node \
    77      /bin/sh -c 'echo $(( $(ip -o l show eth0 | awk "{print $1}" | cut -d: -f1) ))')
    78  LB_VETH_HOST=$(ip -o l | grep "if$IFIDX" | awk '{print $2}' | cut -d@ -f1)
    79  ip l set dev $LB_VETH_HOST xdp obj bpf_xdp_veth_host.o
    80  ip l set dev l4lb-veth0 xdp obj bpf_xdp_veth_host.o
    81  
    82  # Disable TX and RX csum offloading, as veth does not support it. Otherwise,
    83  # the forwarded packets by the LB to the worker node will have invalid csums.
    84  ethtool -K $LB_VETH_HOST rx off tx off
    85  ethtool -K l4lb-veth0 rx off tx off
    86  
    87  NGINX_PID=$(docker inspect nginx -f '{{ .State.Pid }}')
    88  WORKER_IP=$(nsenter -t $NGINX_PID -n ip -o -4 a s eth0 | awk '{print $4}' | cut -d/ -f1)
    89  WORKER_MAC=$(nsenter -t $NGINX_PID -n ip -o l show dev eth0 | grep -oP '(?<=link/ether )[^ ]+')
    90  
    91  # Install BPF program to terminate encapsulated packets coming from the LB
    92  nsenter -t $NGINX_PID -n /bin/sh -c \
    93      'tc qdisc add dev eth0 clsact && tc filter add dev eth0 ingress bpf direct-action object-file ./test_tc_tunnel.o section decap'
    94  
    95  # Wait until Cilium is ready
    96  while ! docker exec -t lb-node docker exec -t cilium-lb cilium-dbg status; do sleep 1; done
    97  
    98  ##########
    99  #  TEST  #
   100  ##########
   101  
   102  LB_VIP="10.0.0.2"
   103  
   104  nsenter -t $(docker inspect nginx -f '{{ .State.Pid }}') -n /bin/sh -c \
   105      "ip a a dev eth0 ${LB_VIP}/32"
   106  
   107  docker exec -t lb-node docker exec -t cilium-lb \
   108      cilium-dbg service update --id 1 --frontend "${LB_VIP}:80" --backends "${WORKER_IP}:80" --k8s-load-balancer
   109  
   110  LB_NODE_IP=$(docker exec lb-node ip -o -4 a s eth0 | awk '{print $4}' | cut -d/ -f1)
   111  ip r a "${LB_VIP}/32" via "$LB_NODE_IP"
   112  
   113  # Issue 10 requests to LB
   114  for i in $(seq 1 10); do
   115      curl -o /dev/null "${LB_VIP}:80" || (echo "Failed $i"; exit -1)
   116  done
   117  
   118  # Now steer the traffic to LB_VIP via the secondary device so that XDP_REDIRECT
   119  # can be tested on the L4LB node
   120  ip r replace "${LB_VIP}/32" via "$SECOND_LB_NODE_IP"
   121  
   122  # Issue 10 requests to LB
   123  for i in $(seq 1 10); do
   124      curl -o /dev/null "${LB_VIP}:80" || (echo "Failed $i"; exit -1)
   125  done
   126  
   127  # Set nginx to maintenance
   128  docker exec -t lb-node docker exec -t cilium-lb \
   129      cilium-dbg service update --id 1 --frontend "${LB_VIP}:80" --backends "${WORKER_IP}:80" --backend-weights "0" --k8s-load-balancer
   130  
   131  # Do not stop on error
   132  set +e
   133  # Issue 10 requests to LB, we expect all to fail due to a Failed connection
   134  for i in $(seq 1 10); do
   135      curl -o /dev/null -m 0.5 "${LB_VIP}:80"
   136      # code 7 - Failed to connect ... : No route to host
   137      if [ ! "$?" -eq 7 ]; then
   138          exit -1;
   139      fi
   140  done
   141  
   142  # Stop on error
   143  set -e
   144  docker exec -t lb-node docker exec -t cilium-lb \
   145      cilium-dbg service update --id 1 --frontend "${LB_VIP}:80" --backends "${WORKER_IP}:80" --backend-weights "1" --k8s-load-balancer
   146  
   147  curl -o /dev/null "${LB_VIP}:80" -m1 || (echo "Failed"; exit -1)
   148  
   149  # Restart cilium-agent and issue 50 requests to LB
   150  docker exec -d lb-node docker restart cilium-lb
   151  
   152  # Requests should not timeout when agent is starting up
   153  for i in $(seq 1 50); do
   154      curl -o /dev/null "${LB_VIP}:80" -m1 || (echo "Failed"; exit -1)
   155      sleep 0.2
   156  done
   157  
   158  # Cleanup
   159  docker rm -f lb-node
   160  docker rm -f nginx
   161  docker network rm cilium-l4lb