github.com/cilium/cilium@v1.16.2/test/envoy/helpers.bash (about)

     1  #!/usr/bin/env bash
     2  
     3  CILIUM_FILES="cilium-files"
     4  DUMP_FILE=$(mktemp)
     5  MONITOR_PID=""
     6  LAST_LOG_DATE=""
     7  TEST_NET=cilium
     8  GOPS="/home/vagrant/go/bin/gops"
     9  
    10  # Variables used during Jenkins builds.
    11  BUILD_NUM="${BUILD_NUMBER:-0}"
    12  JOB_BASE="${JOB_BASE_NAME:-local}"
    13  BUILD_ID="${JOB_BASE}-${BUILD_NUM}"
    14  
    15  AGENT_SOCK_PATH=/var/run/cilium/cilium.sock
    16  
    17  # Prefer local build if binary file detected.
    18  for bin in "../cilium/cilium" \
    19    "../daemon/cilium-agent" \
    20    "../plugins/cilium-docker/cilium-docker"; do
    21          if [ -f $bin ]; then
    22            export PATH=$PWD/`dirname $bin`:$PATH
    23          fi
    24  done
    25  
    26  # Prevent Fedora rules in raw table from affecting the test.
    27  ip6tables -t raw -F 2> /dev/null || true
    28  
    29  function log {
    30    local save=$-
    31    set +u
    32    check_num_params "$#" "1"
    33    message=$1
    34    local stack
    35    for (( i=${#FUNCNAME[@]}-1 ; i>0 ; i-- )) ; do
    36      if [[ "${stack}" == "" ]]; then
    37        stack="$(basename $0): ${FUNCNAME[i]}"
    38      else
    39        stack="$stack/${FUNCNAME[i]}"
    40      fi
    41    done
    42    echo "----- ${stack}: $message"
    43    restore_flag $save "u"
    44  }
    45  
    46  # Usage: overwrite $iter 'commands --option --foo bar "quoted args" '
    47  # Executes the commands provided as parameters, moves the cursor back by the
    48  # number of lines output by the command, then prints the output of the command.
    49  # If $iter is zero, then the cursor is not moved; this is equivalent to
    50  # 'shift; eval "$@"'.
    51  function overwrite {
    52    local iter=$1
    53    shift
    54  
    55    local output=$(eval "$@")
    56    if [ ! -z $TERM ] && [ $iter -ne 0 ]; then
    57      local ERASER=$(tput cuu1 ; tput el)
    58      local n_lines=$(echo "$output" | wc -l)
    59      for i in $(seq 1 $n_lines); do
    60        echo -ne "$ERASER"
    61      done
    62    fi
    63    echo "$output"
    64  }
    65  
    66  function get_filename_without_extension {
    67    check_num_params "$#" "1"
    68    local file=$(basename $1)
    69    local filename="${file%.*}"
    70    echo $filename
    71  }
    72  # Note: if you call this, do not change the value of the debug flag - you will make the shell segmentation fault :) 
    73  function redirect_debug_logs {
    74    check_num_params "$#" "1"
    75    local LOGS_DIR=$1
    76    mkdir -p ${LOGS_DIR}
    77    exec {BASH_XTRACEFD}>>${LOGS_DIR}/debug.txt
    78  }
    79  
    80  function monitor_start {
    81    local save=$-
    82    set +e
    83    log "starting monitor and dumping contents to $DUMP_FILE"
    84    cilium-dbg monitor -v $@ > $DUMP_FILE &
    85    MONITOR_PID=$!
    86    restore_flag $save "e"
    87  }
    88  
    89  function monitor_resume {
    90    local save=$-
    91    set +e
    92    log "resuming monitor and dumping contents to $DUMP_FILE"
    93    cilium-dbg monitor -v $@ >> $DUMP_FILE &
    94    MONITOR_PID=$!
    95    restore_flag $save "e"
    96  }
    97  
    98  function monitor_clear {
    99    local save=$-
   100    set +e
   101    log "clearing monitor"
   102    cp /dev/null $DUMP_FILE
   103    nstat > /dev/null
   104    restore_flag $save "e"
   105  }
   106  
   107  function monitor_dump {
   108    local save=$-
   109    set +e
   110    nstat
   111    cat $DUMP_FILE
   112    restore_flag $save "e"
   113  }
   114  
   115  function monitor_stop {
   116    local save=$-
   117    set +e
   118    if [ ! -z "$MONITOR_PID" ]; then
   119      kill $MONITOR_PID || true > /dev/null 2>&1
   120    fi
   121    restore_flag $save "e"
   122  }
   123  
   124  function logs_clear {
   125    LAST_LOG_DATE="$(date +'%F %T')"
   126  }
   127  
   128  function abort {
   129    set +e
   130    echo "------------------------------------------------------------------------"
   131    echo "                            Test Failed"
   132    echo "$*"
   133    echo ""
   134    echo "------------------------------------------------------------------------"
   135  
   136    if [ ! -z "$DEBUG" ]; then
   137      cilium-dbg status
   138      cilium-dbg endpoint list
   139      cilium-dbg policy get
   140      read -n 1 -p "Press any key to continue..."
   141    fi
   142  
   143    monitor_dump
   144    monitor_stop
   145  
   146    echo "------------------------------------------------------------------------"
   147    echo "                            Cilium logs (last 200 lines)"
   148    journalctl --no-pager --since "${LAST_LOG_DATE}" -u cilium | tail -n 200
   149    echo ""
   150    echo "------------------------------------------------------------------------"
   151  
   152    exit 1
   153  }
   154  
   155  function micro_sleep {
   156    sleep 0.5
   157  }
   158  
   159  function kafka_consumer_delay {
   160    # wait for kafka consumer to come up
   161    sleep 5
   162  }
   163  
   164  function to_services_delay {
   165    sleep 5
   166  }
   167  
   168  function restore_flag {
   169    check_num_params "$#" "2"
   170    local save=$1
   171    local flag=$2
   172    if [[ $save =~ $2 ]]; then
   173      set -$2
   174    fi
   175  }
   176  
   177  function check_num_params {
   178    local NUM_PARAMS=$1
   179    local NUM_EXPECTED_PARAMS=$2
   180    if [ "$NUM_PARAMS" -ne "$NUM_EXPECTED_PARAMS" ]; then
   181      echo "${FUNCNAME[ 1 ]}: invalid number of parameters, expected $NUM_EXPECTED_PARAMS parameter(s)"
   182      exit 1
   183    fi
   184  }
   185  
   186  function wait_for_endpoints {
   187    local save=$-
   188    set +e
   189    check_num_params "$#" "1"
   190    local NUM_DESIRED="$1"
   191    local CMD="cilium-dbg endpoint list | grep -v -e \"not-ready\" -e \"reserved\" | grep ready -c || true"
   192    local INFO_CMD="cilium-dbg endpoint list"
   193    local MAX_MINS="2"
   194    local ERROR_OUTPUT="Timeout while waiting for $NUM_DESIRED endpoints"
   195    log "waiting for up to ${MAX_MINS} mins for ${NUM_DESIRED} endpoints to be in \"ready\" state"
   196    wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT"
   197    log "done waiting for up to ${MAX_MINS} mins for ${NUM_DESIRED} endpoints to be in \"ready\" state"
   198    restore_flag $save "e"
   199  }
   200  
   201  function wait_for_endpoints_deletion {
   202    local save=$-
   203    set +e
   204    local NUM_DESIRED="2" # When no endpoints are present there should be two lines only.
   205    local CMD="cilium-dbg endpoint list | grep -v \"reserved\" | wc -l || true"
   206    local INFO_CMD="cilium-dbg endpoint list"
   207    local MAX_MINS="2"
   208    local ERROR_OUTPUT="Timeout while waiting for endpoint removal"
   209    log "waiting for up to ${MAX_MINS} mins for all endpoints to be removed"
   210    wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT"
   211    log "done waiting"
   212    restore_flag $save "e"
   213  }
   214  
   215  function k8s_num_ready {
   216    local save=$-
   217    set +e
   218    local NAMESPACE=$1
   219    local CILIUM_POD=$2
   220    local FILTER=$3
   221    kubectl -n ${NAMESPACE} exec ${CILIUM_POD} -- cilium-dbg endpoint list | grep $FILTER | grep -v -e 'not-ready' -e 'reserved' | grep -c 'ready' || true
   222    restore_flag $save "e"
   223  }
   224  
   225  function wait_for_k8s_endpoints {
   226    local save=$-
   227    set +e
   228    check_num_params "$#" "4"
   229    local NAMESPACE=$1
   230    local CILIUM_POD=$2
   231    local NUM=$3
   232    local FILTER=$4
   233    log "Waiting for $NUM endpoints in namespace $NAMESPACE managed by $CILIUM_POD"
   234  
   235    # Wait some time for at least one endpoint to get into regenerating state
   236    # FIXME: Remove when this is reliable
   237    sleep 5
   238  
   239    local sleep_time=1
   240    local iter=0
   241    local found
   242    found=$(k8s_num_ready "${NAMESPACE}" "${CILIUM_POD}" "${FILTER}")
   243    log "found: $found"
   244    while [[ "$found" -ne "$NUM" ]]; do
   245      if [[ $iter -gt $((5*60/$sleep_time)) ]]; then
   246        echo ""
   247        log "Timeout while waiting for $NUM endpoints"
   248        restore_flag $save "e"
   249        exit 1
   250      else
   251        overwrite $iter '
   252          kubectl -n ${NAMESPACE} exec -- ${CILIUM_POD} cilium-dbg endpoint list
   253          echo -n " [${found}/${NUM}]"
   254        '
   255        sleep $sleep_time
   256      fi
   257      found=$(k8s_num_ready "${NAMESPACE}" "${CILIUM_POD}" "${FILTER}")
   258      log "found: $found"
   259      ((iter++))
   260    done
   261  
   262    overwrite $iter 'kubectl -n ${NAMESPACE} exec ${CILIUM_POD} -- cilium-dbg endpoint list'
   263    restore_flag $save "e"
   264  }
   265  
   266  function wait_for_cilium_status {
   267    local NUM_DESIRED="1"
   268    local CMD="cilium-dbg status | grep 'Cilium:' | grep -c OK || true"
   269    local INFO_CMD="true"
   270    local MAX_MINS="1"
   271    local ERROR_OUTPUT="Timeout while waiting for Cilium to be ready"
   272    wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT"
   273  }
   274  
   275  function wait_for_kubectl_cilium_status {
   276    check_num_params "$#" "2"
   277    namespace=$1
   278    pod=$2
   279    local NUM_DESIRED="1"
   280    local CMD="kubectl -n ${namespace} exec ${pod} -- cilium-dbg status | grep "Cilium:" | grep -c 'OK' || true"
   281    local INFO_CMD="true"
   282    local MAX_MINS="2"
   283    local ERROR_OUTPUT="Timeout while waiting for Cilium to be ready"
   284    wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT"
   285  }
   286  
   287  function wait_for_cilium_ep_gen {
   288    local save=$-
   289    set +e
   290    local MODE=$1
   291  
   292    local NAMESPACE
   293    local POD
   294    local CMD
   295    local INFO_CMD
   296  
   297    if [[ "$MODE" == "k8s" ]]; then
   298      # Only care about provided params if mode is K8s.
   299      check_num_params "$#" "3"
   300      log "mode is K8s"
   301      NAMESPACE=$2
   302      POD=$3
   303      CMD="kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list | grep -c regenerat"
   304      INFO_CMD="kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list"
   305    else
   306      CMD="cilium-dbg endpoint list | grep -c regenerat"
   307      INFO_CMD="cilium-dbg endpoint list"
   308    fi
   309  
   310    local NUM_DESIRED="0"
   311    local MAX_MINS="2"
   312    local ERROR_OUTPUT="Timeout while waiting for endpoints to regenerate"
   313    local sleep_time=1
   314  
   315    local iter=0
   316    local found
   317    found=$(eval "$CMD")
   318  
   319    while [[ "$found" -ne "$NUM_DESIRED" ]]; do
   320      log "$found endpoints are still regenerating; want $NUM_DESIRED"
   321      if [[ $((iter++)) -gt $((${MAX_MINS}*60/$sleep_time)) ]]; then
   322        echo ""
   323        log "${ERROR_OUTPUT}"
   324        exit 1
   325      else
   326        overwrite $iter '
   327          log "still within time limit for waiting for endpoints to be in 'ready' state; sleeping and checking again"
   328          log "output of ${INFO_CMD}"
   329          eval "$INFO_CMD"
   330          echo -n " [$found/$NUM_DESIRED]"
   331          log "sleeping for $sleep_time"
   332        '
   333        sleep $sleep_time
   334      fi
   335      log "evaluating $CMD"
   336      found=$(eval "${CMD}")
   337      log "found: $found"
   338    done
   339    set -e
   340    restore_flag $save "e"
   341  }
   342  
   343  function wait_for_daemon_set_not_ready {
   344    local save=$-
   345    set +e
   346    check_num_params "$#" "2"
   347  
   348    local namespace="${1}"
   349    local name="${2}"
   350  
   351    log "Waiting for instances of Cilium daemon $name in namespace $namespace to be clean up"
   352  
   353    local sleep_time=2
   354    local iter=0
   355    local found="0"
   356    until [[ "$found" -eq "1" ]]; do
   357      if [[ $iter -gt $((5*60/$sleep_time)) ]]; then
   358        echo ""
   359        log "Timeout while waiting for cilium agent to be clean up by kubernetes"
   360        print_k8s_cilium_logs
   361        exit 1
   362      else
   363        overwrite $iter 'kubectl -n ${namespace} get pods -o wide'
   364        sleep $sleep_time
   365      fi
   366      kubectl get pods -n ${namespace} | grep ${name} -q
   367      found=$?
   368      ((iter++))
   369    done
   370  
   371    overwrite $iter 'kubectl -n kube-system get pods -o wide'
   372    restore_flag $save "e"
   373  }
   374  
   375  function wait_for_policy_enforcement {
   376    check_num_params "$#" "1"
   377    local NUM_DESIRED="$1"
   378    local CMD="cilium-dbg endpoint list | grep -c Disabled"
   379    local INFO_CMD="cilium-dbg endpoint list"
   380    local MAX_MINS="2"
   381    local ERROR_OUTPUT="Timeout while waiting for policy to be enabled for all endpoints"
   382    wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT"
   383  }
   384  
   385  function count_lines_in_log {
   386      echo `wc -l $DUMP_FILE | awk '{ print $1 }'`
   387  }
   388  
   389  function wait_for_log_entries {
   390    check_num_params "$#" "1"
   391    local expected=$(($1 + $(count_lines_in_log)))
   392    wait_specified_time_test "test \"\$(count_lines_in_log)\" -ge \"$expected\"" "2"
   393  }
   394  
   395  function wait_for_docker_ipv6_addr {
   396    check_num_params "$#" "1"
   397    name=$1
   398    wait_specified_time_test "test \"\$(docker inspect --format '{{ .NetworkSettings.Networks.cilium.GlobalIPv6Address }}' $name)\" != \"\"" "2"
   399  }
   400  
   401  function wait_for_running_pod {
   402    pod=$1
   403    namespace=${2:-default}
   404    log "Waiting for ${pod} pod to be Running..."
   405    wait_specified_time_test "test \"\$(kubectl get pods -n ${namespace} -o wide | grep ${pod} | grep -c Running)\" -eq \"1\"" "10"
   406  }
   407  
   408  function wait_for_no_pods {
   409    namespace=${1:-default}
   410    log "Waiting for no pods to be Running in namespace ${namespace}"
   411    wait_specified_time_test "test \"\$(kubectl get pods -n ${namespace} -o wide 2>&1 | grep -c 'No resources found')\" -eq \"1\"" "5"
   412  }
   413  
   414  function wait_for_n_running_pods {
   415    local save=$-
   416    set +e
   417    check_num_params "$#" "1"
   418    local NPODS=$1
   419    log "Waiting for $NPODS running pods"
   420  
   421    local sleep_time=1
   422    local iter=0
   423    local found
   424    found=$(kubectl get pod | grep Running -c || true)
   425    until [[ "$found" -eq "$NPODS" ]]; do
   426      if [[ $iter -gt $((5*60/$sleep_time)) ]]; then
   427        echo ""
   428        log "Timeout while waiting for $NPODS running pods"
   429        exit 1
   430      else
   431        overwrite $iter '
   432          kubectl get pod -o wide
   433          echo -n " [${found}/${NPODS}]"
   434        '
   435        sleep $sleep_time
   436      fi
   437      found=$(kubectl get pod | grep Running -c || true)
   438      ((iter++))
   439    done
   440  
   441    overwrite $iter 'kubectl get pod -o wide'
   442    restore_flag $save "e"
   443  }
   444  
   445  # Wait for healthy k8s cluster on $1 nodes
   446  function wait_for_healthy_k8s_cluster {
   447    local save=$-
   448    set +e
   449    local NNODES=$1
   450    log "Waiting for healthy k8s cluster with $NNODES nodes"
   451  
   452    local sleep_time=2
   453    local iter=0
   454    local found
   455    found=$(kubectl get cs | grep -v "STATUS" | grep -c "Healthy")
   456    until [[ "$found" -eq "3" ]]; do
   457      if [[ $iter -gt $((1*60/$sleep_time)) ]]; then
   458        echo ""
   459        log "Timeout while waiting for healthy kubernetes cluster"
   460        exit 1
   461      else
   462        overwrite $iter '
   463          kubectl get cs
   464          log "K8S Components ready: [${found}/3]"
   465        '
   466        sleep $sleep_time
   467      fi
   468      found=$(kubectl get cs | grep -v "STATUS" | grep -c "Healthy")
   469      ((iter++))
   470    done
   471    overwrite $iter 'kubectl get cs'
   472    local iter=0
   473    local found
   474    found=$(kubectl get nodes | grep Ready -c)
   475    until [[ "$found" -eq "$NNODES" ]]; do
   476      if [[ $iter -gt $((1*60/$sleep_time)) ]]; then
   477        echo ""
   478        log "Timeout while waiting for all nodes to be Ready"
   479        exit 1
   480      else
   481        overwrite $iter '
   482          kubectl get nodes
   483          log "Nodes ready [${found}/${NNODES}]"
   484        '
   485        sleep $sleep_time
   486      fi
   487      found=$(kubectl get nodes | grep Ready -c)
   488      ((iter++))
   489    done
   490    restore_flag $save "e"
   491  }
   492  
   493  function k8s_nodes_policy_status {
   494    local save=$-
   495    set +e
   496    local sleep_time=2
   497    local NNODES=$1
   498    local policy_ns=$2
   499    local policy_name=$3
   500    local iter=0
   501    local nodes=$(kubectl get ciliumnetworkpolicies -n "${policy_ns}" "${policy_name}" -o go-template --template='{{len .status.nodes}}')
   502    until [[ "${nodes}" -eq "${NNODES}" ]]; do
   503      if [[ $iter -gt $((1*60/$sleep_time)) ]]; then
   504        echo ""
   505        log "Timeout while waiting for $NNODES to have policy ${policy_ns}/${policy_name} installed"
   506        exit 1
   507      else
   508        overwrite $iter '
   509          kubectl get nodes
   510          log "Nodes with policy accepted [${found}/${NNODES}]"
   511        '
   512        sleep $sleep_time
   513      fi
   514      found=$(kubectl get nodes | grep Ready -c)
   515      ((iter++))
   516    done
   517  
   518    kubectl get ciliumnetworkpolicies -n "${policy_ns}" "${policy_name}" -o go-template --template='{{.status.nodes}}'
   519    restore_flag $save "e"
   520  }
   521  
   522  function gather_files {
   523    local TEST_NAME=$1
   524    local TEST_SUITE=$2
   525    log "gathering files for test $TEST_NAME in test suite $TEST_SUITE"
   526    local CILIUM_ROOT="src/github.com/cilium/cilium"
   527    if [ -z "${TEST_SUITE}" ]; then
   528      TEST_SUITE="runtime-tests"
   529    fi
   530    if [ -z "${GOPATH}" ]; then
   531      local GOPATH="/home/vagrant/go"
   532    fi
   533    CILIUM_DIR="${GOPATH}/${CILIUM_ROOT}/test/envoy/cilium-files/${TEST_NAME}"
   534    local RUN="/var/run/cilium"
   535    local LIB="/var/lib/cilium"
   536    local RUN_DIR="${CILIUM_DIR}${RUN}"
   537    local LIB_DIR="${CILIUM_DIR}${LIB}"
   538    mkdir -p "${CILIUM_DIR}"
   539    mkdir -p "${RUN_DIR}"
   540    mkdir -p "${LIB_DIR}"
   541    if [[ "${TEST_SUITE}" == "runtime-tests" ]]; then
   542      local CLI_OUT_DIR="${CILIUM_DIR}/cli"
   543      local PROF_OUT_DIR="${CILIUM_DIR}/profiling"
   544      mkdir -p "${CLI_OUT_DIR}"
   545      dump_cli_output "${CLI_OUT_DIR}" || true
   546      dump_gops_output "${PROF_OUT_DIR}" "cilium-agent" || true
   547      # Get logs from Consul container.
   548      mkdir -p "${CILIUM_DIR}/consul"
   549      docker logs cilium-consul > "${CILIUM_DIR}/consul/consul-logs.txt" 2>/dev/null
   550    else
   551      # Get logs from each Cilium pod.
   552      local NAMESPACE="kube-system"
   553      local CILIUM_POD_1=$(kubectl -n ${NAMESPACE} get pods -l k8s-app=cilium | awk 'NR==2{ print $1 }')
   554      local CILIUM_POD_2=$(kubectl -n ${NAMESPACE} get pods -l k8s-app=cilium | awk 'NR==3{ print $1 }')
   555      local CLI_OUT_DIR=${CILIUM_DIR}/cli
   556      mkdir -p "${CLI_OUT_DIR}"
   557      log "gathering Cilium logs from pod ${CILIUM_POD_1}"
   558      dump_cli_output_k8s "${CLI_OUT_DIR}" "${NAMESPACE}" "${CILIUM_POD_1}" || true
   559      log "gathering Cilium logs from pod ${CILIUM_POD_2}"
   560      dump_cli_output_k8s "${CLI_OUT_DIR}" "${NAMESPACE}" "${CILIUM_POD_2}" || true
   561    fi
   562    sudo cp -r ${RUN}/state "${RUN_DIR}" || true
   563    sudo cp ${RUN}/*.log "${RUN_DIR}" || true
   564    sudo cp -r ${LIB}/* "${LIB_DIR}" || true
   565    find "${CILIUM_DIR}" -type d -exec sudo chmod 777 {} \;
   566    find "${CILIUM_DIR}" -exec sudo chmod a+r {} \;
   567    log "finished gathering files for test $TEST_NAME in test suite $TEST_SUITE"
   568  }
   569  
   570  function dump_cli_output {
   571    check_num_params "$#" "1"
   572    local DIR=$1
   573    cilium-dbg endpoint list > ${DIR}/endpoint_list.txt
   574    local EPS=$(cilium-dbg endpoint list | tail -n+3 | grep '^[0-9]' | awk '{print $1}')
   575    for ep in ${EPS} ; do
   576      cilium-dbg endpoint get ${ep} > ${DIR}/endpoint_get_${ep}.txt
   577      cilium-dbg bpf policy get ${ep} > ${DIR}/bpf_policy_list_${ep}.txt
   578    done
   579    cilium-dbg service list > ${DIR}/service_list.txt
   580    local SVCS=$(cilium-dbg service list | tail -n+2 | awk '{print $1}')
   581    for svc in ${SVCS} ; do
   582      cilium-dbg service get ${svc} > ${DIR}/service_get_${svc}.txt
   583    done
   584    local IDS=$(cilium-dbg endpoint list | tail -n+3 | awk '{print $4}' | grep -o '[0-9]*')
   585    for id in ${IDS} ; do
   586      cilium-dbg identity get ${id} > ${DIR}/identity_get_${id}.txt
   587    done
   588    cilium-dbg config > ${DIR}/config.txt
   589    cilium-dbg bpf lb list > ${DIR}/bpf_lb_list.txt
   590    cilium-dbg bpf ct list global > ${DIR}/bpf_ct_list_global.txt
   591    cilium-dbg bpf tunnel list > ${DIR}/bpf_tunnel_list.txt
   592    cilium-dbg policy get > ${DIR}/policy_get.txt
   593    cilium-dbg status > ${DIR}/status.txt
   594    cilium-dbg debuginfo -f ${DIR}/debuginfo.txt
   595    cilium-bugtool -t ${DIR}
   596  }
   597  
   598  function dump_cli_output_k8s {
   599    check_num_params "$#" "3"
   600    local DIR=$1
   601    local NAMESPACE=$2
   602    local POD=$3
   603    kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list > ${DIR}/${POD}_endpoint_list.txt
   604    local EPS=$(kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list | tail -n+3 | grep '^[0-9]' | awk '{print $1}')
   605    for ep in ${EPS} ; do
   606      kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint get ${ep} > ${DIR}/${POD}_endpoint_get_${ep}.txt
   607      kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg bpf policy get ${ep} > ${DIR}/${POD}_bpf_policy_list_${ep}.txt
   608    done
   609    kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg service list > ${DIR}/${POD}_service_list.txt
   610    local SVCS=$(kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg service list | tail -n+2 | awk '{print $1}')
   611    for svc in ${SVCS} ; do
   612      kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg service get ${svc} > ${DIR}/${POD}_service_get_${svc}.txt
   613    done
   614    local IDS=$(kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list | tail -n+3 | awk '{print $4}' | grep -o '[0-9]*')
   615    for id in ${IDS} ; do
   616      kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg identity get ${id} > ${DIR}/${POD}_identity_get_${id}.txt
   617    done
   618    kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg config > ${DIR}/${POD}_config.txt
   619    kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg bpf lb list > ${DIR}/${POD}_bpf_lb_list.txt
   620    kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg bpf ct list global > ${DIR}/${POD}_bpf_ct_list_global.txt
   621    kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg bpf tunnel list > ${DIR}/${POD}_bpf_tunnel_list.txt
   622    kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg policy get > ${DIR}/${POD}_policy_get.txt
   623    kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg status > ${DIR}/${POD}_status.txt
   624    kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg debuginfo > ${DIR}/${POD}_debuginfo.txt
   625    local DEBUGTOOL_ARCHIVE=`kubectl exec -n ${NAMESPACE} ${POD} -- cilium-bugtool | grep ARCHIVE | awk '{ print $3}'`
   626    kubectl cp ${NAMESPACE}/${POD}:${DEBUGTOOL_ARCHIVE} ${DIR}/${POD}_bugtool.tar
   627  }
   628  
   629  function dump_gops_output {
   630    check_num_params "$#" "2"
   631    local DIR="$1"
   632    local PROG="$2"
   633    local PROG_PROFILING_DIR="${DIR}/${PROG}"
   634    mkdir -p "${PROG_PROFILING_DIR}"
   635    log "getting gops output for ${PROG} and dumping to dir ${PROG_PROFILING_DIR}"
   636    local PROG_PID=$(sudo ${GOPS} | grep "${PROG}" | awk '{print $1}')
   637    log "running \"gops stack\" for ${PROG}"
   638    sudo ${GOPS} stack ${PROG_PID} > "${PROG_PROFILING_DIR}/${PROG}_stack.txt"
   639    log "running \"gops memstats\" for ${PROG}"
   640    sudo ${GOPS} memstats ${PROG_PID} > "${PROG_PROFILING_DIR}/${PROG}_memstats.txt"
   641    log "running \"gops stats\" for ${PROG}"
   642    sudo ${GOPS} stats ${PROG_PID} > "${PROG_PROFILING_DIR}/${PROG}_stats.txt"
   643    log "done getting gops output for ${PROG}"
   644  }
   645  
   646  function print_k8s_cilium_logs {
   647    for pod in $(kubectl -n kube-system get pods -o wide| grep cilium | awk '{print $1}'); do
   648      kubectl -n kube-system logs $pod
   649      if [ $? -ne 0 ]; then
   650        kubectl -n kube-system logs $pod --previous
   651      fi
   652    done
   653  }
   654  
   655  function wait_for_daemon_set_ready {
   656    local save=$-
   657    set +e
   658    check_num_params "$#" "3"
   659  
   660    local namespace="${1}"
   661    local name="${2}"
   662    local n_ds_expected="${3}"
   663  
   664    log "Waiting for $n_ds_expected instances of Cilium daemon $name in namespace $namespace to become ready"
   665  
   666    local sleep_time=2
   667    local iter=0
   668    local found="0"
   669    until [[ "$found" -eq "$n_ds_expected" ]]; do
   670      if [[ $iter -gt $((5*60/$sleep_time)) ]]; then
   671        echo ""
   672        log "Timeout while waiting for cilium-agent"
   673        print_k8s_cilium_logs
   674        exit 1
   675      else
   676        overwrite $iter '
   677          kubectl -n kube-system get ds
   678          kubectl -n kube-system get pods -o wide
   679          echo -n " [${found}/${n_ds_expected}]"
   680        '
   681        sleep $sleep_time
   682      fi
   683      found=$(kubectl get ds -n ${namespace} ${name} 2>&1 | awk 'NR==2{ print $4 }')
   684      ((iter++))
   685    done
   686    overwrite $iter 'kubectl -n kube-system get pods -o wide'
   687    restore_flag $save "e"
   688  }
   689  
   690  function k8s_wait_for_cilium_status_ready {
   691    local save=$-
   692    set +e
   693    local pod
   694    check_num_params "$#" "1"
   695    local namespace=$1
   696    local pods=$(kubectl -n $namespace get pods -l k8s-app=cilium | grep cilium- | awk '{print $1}')
   697  
   698    for pod in $pods; do
   699      wait_for_kubectl_cilium_status $namespace $pod
   700    done
   701    restore_flag $save "e"
   702  }
   703  
   704  function k8s_count_all_cluster_cilium_eps {
   705    local save=$-
   706    set +e
   707    local total=0
   708    check_num_params "$#" "1"
   709    local pod
   710    local namespace=$1
   711    local pods=$(kubectl -n $namespace get pods -l k8s-app=cilium | grep cilium- | awk '{print $1}')
   712  
   713    for pod in $pods; do
   714      local n_eps=$(kubectl -n $namespace exec $pod -- cilium-dbg endpoint list --no-headers | wc -l)
   715      total=$(( $total + $n_eps ))
   716    done
   717  
   718    echo "$total"
   719    restore_flag $save "e"
   720  }
   721  
   722  function wait_for_api_server_ready {
   723    log "Waiting for kube-apiserver to spin up"
   724    wait_specified_time_test "test \$(kubectl get cs)" "10"
   725  }
   726  
   727  function wait_for_service_endpoints_ready {
   728    check_num_params "$#" "3"
   729    local namespace="${1}"
   730    local name="${2}"
   731    local port="${3}"
   732  
   733    log "Waiting for ${name} service endpoints to be ready"
   734    wait_specified_time_test "test \"\$(kubectl get endpoints -n ${namespace} ${name} | grep -c \":${port}\")\" -eq \"1\"" "10"
   735    log "Done waiting for ${name} service endpoints to be ready"
   736    kubectl get endpoints -n ${namespace} ${name}
   737  }
   738  
   739  function wait_for_service_ready_cilium_pod {
   740    check_num_params "$#" "4"
   741    local namespace="${1}"
   742    local pod="${2}"
   743    local fe_port="${3}"
   744    # TODO: only works for one backend right now.
   745    local be_port="${4}"
   746  
   747    log "Waiting for Cilium pod ${pod} to have services ready with frontend port: ${fe_port} and backend port: ${be_port}"
   748  
   749    wait_specified_time_test "test \"\$(kubectl -n ${namespace} exec ${pod} -- cilium-dbg service list | awk '{ print \$2 }' | grep -c \":${fe_port}\")\" -ge \"1\"" "10"
   750    wait_specified_time_test "test \"\$(kubectl -n ${namespace} exec ${pod} -- cilium-dbg service list | awk '{ print \$5 }' | grep -c \":${be_port}\")\" -ge \"1\"" "10"
   751  
   752    log "Done waiting for Cilium pod ${pod} to have services ready with frontend port: ${fe_port} and backend port: ${be_port}"
   753  
   754    log "Listing all services:"
   755    kubectl -n ${namespace} exec ${pod} -- cilium-dbg service list
   756  }
   757  
   758  function k8s_apply_policy {
   759    declare -A currentRevison
   760    local i
   761    local pod
   762    check_num_params "$#" "3"
   763    local namespace=$1
   764    local action=$2
   765    local policy=$3
   766    local pods=$(kubectl -n $namespace get pods -l k8s-app=cilium | grep cilium- | awk '{print $1}')
   767  
   768    for pod in $pods; do
   769      local rev=$(kubectl -n $namespace exec $pod -- cilium-dbg policy get | grep Revision: | awk '{print $2}')
   770      currentRevison[$pod]=$rev
   771    done
   772  
   773    log "Current policy revisions:"
   774    for i in "${!currentRevison[@]}"
   775    do
   776      echo "  $i: ${currentRevison[$i]}"
   777    done
   778  
   779    kubectl $action -f $policy
   780  
   781    for pod in $pods; do
   782      local nextRev=$(expr ${currentRevison[$pod]} + 1)
   783      log "Waiting for agent $pod endpoints to get to revision $nextRev"
   784      timeout 180s kubectl -n $namespace exec $pod -- cilium-dbg policy wait $nextRev
   785    done
   786  
   787    # Adding sleep as workaround for l7 stresstests
   788    sleep 10s
   789  }
   790  
   791  function policy_delete_and_wait {
   792    log "deleting policy $* and waiting up to 120 seconds to complete"
   793    rev=$(cilium-dbg policy delete $* | grep Revision: | awk '{print $2}')
   794    timeout 120s cilium-dbg policy wait $rev
   795  }
   796  
   797  function policy_import_and_wait {
   798    log "importing policy $* and waiting up to 120 seconds to complete"
   799    rev=$(cilium-dbg policy import $* | grep Revision: | awk '{print $2}')
   800    timeout 120s cilium-dbg policy wait $rev
   801  }
   802  
   803  function get_vm_identity_file {
   804    check_num_params "$#" "1"
   805    local VM_NAME=$1
   806    vagrant ssh-config ${VM_NAME} | grep IdentityFile | awk '{print $2}'
   807  }
   808  
   809  function get_vm_ssh_port {
   810    check_num_params "$#" "1"
   811    local VM_NAME=$1
   812    vagrant ssh-config ${VM_NAME} | grep Port | awk '{ print $2 }'
   813  }
   814  
   815  function copy_files_vm {
   816    check_num_params "$#" "2"
   817    local VM_NAME=$1
   818    local FILES_DIR=$2
   819    local ID_FILE
   820    local PORT
   821  
   822    # Check that the VM is running before we try to gather logs from it.
   823    check_vm_running $VM_NAME
   824  
   825    log "getting the VM identity file for $VM_NAME"
   826    ID_FILE=$(get_vm_identity_file $VM_NAME)
   827    log "getting the port for $VM_NAME to SSH"
   828    PORT=$(get_vm_ssh_port $VM_NAME)
   829  
   830    log "getting cilium logs from $VM_NAME"
   831    vagrant ssh $VM_NAME -c 'sudo -E bash -c "journalctl --no-pager -u cilium > /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files/cilium-logs && chmod a+r /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files/cilium-logs"'
   832    vagrant ssh $VM_NAME -c 'sudo -E bash -c "journalctl --no-pager -u cilium-docker > /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files/cilium-docker-logs && chmod a+r /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files/cilium-docker-logs"'
   833  
   834    log "listing all logs that will be gathered from $VM_NAME"
   835    vagrant ssh $VM_NAME -c 'ls -altr /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files'
   836  
   837    log "copying logs from $VM_NAME onto VM host for accessibility after VM is destroyed"
   838    scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r -P ${PORT} -i ${ID_FILE} vagrant@127.0.0.1:/home/vagrant/go/src/github.com/cilium/cilium/${FILES_DIR} ${WORKSPACE}/cilium-files-${VM_NAME}
   839  }
   840  
   841  function get_k8s_vm_name {
   842    check_num_params "$#" "1"
   843    local VM_PREFIX=$1
   844  
   845    if [ ! -z ${BUILD_NUMBER} ] ; then
   846      local BUILD_ID_NAME="-build-${BUILD_ID}"
   847    fi
   848    echo "${VM_PREFIX}${BUILD_ID_NAME}"
   849  }
   850  
   851  function get_cilium_master_vm_name {
   852    if [ ! -z "${K8STAG}" ] ; then
   853      local K8S_TAG="${K8STAG:-k8s}"
   854    fi
   855  
   856    if [ ! -z "${BUILD_NUMBER}" ] ; then
   857      local BUILD_ID_NAME="-build-${BUILD_ID}"
   858    fi
   859  
   860    echo "cilium${K8S_TAG}-master${BUILD_ID_NAME}"
   861  }
   862  
   863  function check_vm_running {
   864    check_num_params "$#" "1"
   865    local VM=$1
   866    log "getting status of VM ${VM}"
   867    vagrant status ${VM}
   868    log "done getting status of VM ${VM}"
   869  
   870    local VM_STATUS
   871    VM_STATUS=`vagrant status ${VM} | grep ${VM} | awk '{print $2}'`
   872    if [[ "${VM_STATUS}" != "running" ]]; then
   873      log "$VM is not in \"running\" state; exiting"
   874    else
   875      log "$VM is \"running\" continuing"
   876    fi
   877  }
   878  
   879  function wait_for_agent_socket {
   880    check_num_params "$#" "1"
   881    MAX_WAIT=$1
   882  
   883    log "waiting at most ${MAX_WAIT} iterations for cilium agent socket"
   884    local i=0
   885  
   886    while [ "$i" -lt "$MAX_WAIT" ]; do
   887      micro_sleep
   888      i=$[$i+1]
   889      if [ -S $AGENT_SOCK_PATH ]; then
   890        return
   891      fi
   892    done
   893    abort "Waiting for agent socket, timed out"
   894  }
   895  
   896  function wait_for_kill {
   897    check_num_params "$#" "2"
   898    TARGET_PID=$1
   899    MAX_WAIT=$2
   900  
   901    log "waiting at most ${MAX_WAIT} iterations for PID ${TARGET_PID} to be killed"
   902    local i=0
   903  
   904    while [ $i -lt "${MAX_WAIT}" ]; do
   905      micro_sleep
   906      i=$[$i+1]
   907      if ! ps -p $TARGET_PID > /dev/null; then
   908        return
   909      fi
   910    done
   911    abort "Waiting for agent process to be killed, timed out"
   912  }
   913  
   914  # diff_timeout waits for the output of the commands specified via $1 and $2 to
   915  # be identical by comparing the output with `diff -Nru`. The commands are
   916  # executed consecutively with a 2 second pause until the output matches or the
   917  # timeout of 1 minute is reached.
   918  function diff_timeout() {
   919    local save=$-
   920    set +e
   921    local arg1="$1"
   922    local arg2="$2"
   923    local sleep_time=2
   924    local iter=0
   925    local found="0"
   926  
   927    until [[ "$found" -eq "1" ]]; do
   928      if [[ $((iter++)) -gt $((30)) ]]; then
   929        log "Timeout waiting for diff to be empty"
   930        abort "$DIFF"
   931      fi
   932  
   933      DIFF=$(diff -Nru <(eval "$arg1") <(eval "$arg2") || true)
   934      if [[ "$DIFF" == "" ]]; then
   935        found="1"
   936      else
   937        sleep $sleep_time
   938      fi
   939    done
   940    restore_flag $save "e"
   941  }
   942  
   943  #######################################
   944  # Waits for MAX_MINS until the output of CMD
   945  # reaches NUM_DESIRED. While the state is not
   946  # realized, INFO_CMD is emitted. If the state
   947  # is not realized after MAX_MINS, ERROR_OUTPUT
   948  # is emitted.
   949  # Globals: 
   950  # Arguments:
   951  #   NUM_DESIRED: desired number output by CMD.
   952  #   CMD: command to run.
   953  #   INFO_CMD: command to run while state is not
   954  #             realized
   955  #   MAX_MINS: maximum minutes to wait for desired
   956  #             state
   957  #   ERROR_OUTPUT: message that is emitted if desired
   958  #                 state is not realized in MAX_MINS.
   959  # Returns:
   960  #   None
   961  #######################################
   962  function wait_for_desired_state {
   963  
   964    local save=$-
   965    set +e
   966    check_num_params "$#" "5"
   967    local NUM_DESIRED="$1"
   968    local CMD="$2"
   969    local INFO_CMD="$3"
   970    local MAX_MINS="$4"
   971    local ERROR_OUTPUT="$5"
   972    local sleep_time=1
   973    local iter=0
   974    local found
   975    found=$(eval "$CMD")
   976    log "waiting for at most ${MAX_MINS} minutes for command ${CMD} to succeed"
   977    log "found: $found"
   978  
   979    while [[ "$found" -ne "$NUM_DESIRED" ]]; do
   980      if [[ $iter -gt $((${MAX_MINS}*60/$sleep_time)) ]]; then
   981        echo ""
   982        log "$ERROR_OUTPUT"
   983        exit 1
   984      else
   985        overwrite $iter '
   986          log "desired state not realized; will sleep and try again"
   987          eval "$INFO_CMD"
   988          echo -n " [$found/$NUM_DESIRED]"
   989        '
   990        sleep $sleep_time
   991      fi
   992      found=$(eval "${CMD}")
   993      log "found: $found"
   994      ((iter++))
   995    done
   996    log "desired state realized for command ${CMD}"
   997    eval "${INFO_CMD}"
   998    restore_flag $save "e"
   999  }
  1000  
  1001  #######################################
  1002  # Waits for MAX_MINS until CMD returns with
  1003  # return code 0. If the desired state is
  1004  # not realized after MAX_MINS, exits with
  1005  # failure.
  1006  # Globals:
  1007  # Arguments:
  1008  #   CMD: command to run.
  1009  #   MAX_MINS: maximum minutes to wait for desired
  1010  #             state
  1011  # Returns:
  1012  #   None
  1013  #######################################
  1014  function wait_specified_time_test {
  1015    local save=$-
  1016    set +e
  1017    local CMD="$1"
  1018    local MAX_MINS="$2"
  1019  
  1020    local sleep_time=1
  1021    local iter=0
  1022  
  1023    log "waiting for at most ${MAX_MINS} minutes for command ${CMD} to succeed"
  1024    while [[ "${iter}" -lt $((${MAX_MINS}*60/$sleep_time)) ]]; do
  1025      if eval "${CMD}" ; then
  1026        log "${CMD} succeeded"
  1027        break
  1028      fi
  1029      overwrite $iter '
  1030        log "${iter} < $((${MAX_MINS}*60/$sleep_time)) "
  1031        log "${CMD} did not succeed; sleeping and testing the command again"
  1032      '
  1033      sleep ${sleep_time}
  1034      iter=$((iter+1))
  1035    done
  1036    if [[ "${iter}" -ge $((${MAX_MINS}*60/$sleep_time)) ]]; then
  1037      log "Timeout ${MAX_MINS} minutes exceeded for command \"$CMD\""
  1038      log "Exiting with failure."
  1039      exit 1
  1040    fi
  1041    log "${CMD} succeeded"
  1042    restore_flag $save "e"
  1043  }
  1044  
  1045  function create_cilium_docker_network {
  1046    log "creating Docker network of type Cilium"
  1047    docker network inspect $TEST_NET 2> /dev/null || {
  1048      docker network create --ipv6 --subnet ::1/112 --ipam-driver cilium --driver cilium $TEST_NET
  1049    }
  1050  }
  1051  
  1052  function remove_cilium_docker_network {
  1053    local save=$-
  1054    set +e
  1055    log "removing Docker network of type cilium"
  1056    docker network rm $TEST_NET > /dev/null 2>&1 
  1057    restore_flag $save "e"
  1058  }
  1059  
  1060  function remove_all_containers {
  1061    docker rm -f $(docker ps --format '{{.Names}}' | grep -v cilium-consul) > /dev/null 2>&1 || true
  1062  }
  1063  
  1064  function test_succeeded {
  1065    check_num_params "$#" "1"
  1066    local TEST_NAME="$1"
  1067    echo "============================================================"
  1068    echo "==                                                        =="
  1069    echo "==                                                        =="
  1070    echo "==                                                        =="
  1071    echo "    ${TEST_NAME} succeeded!"
  1072    echo "==                                                        =="
  1073    echo "==                                                        =="
  1074    echo "==                                                        =="
  1075    echo "============================================================"
  1076  }
  1077  
  1078  function ping_fail {
  1079    check_num_params "$#" "2"
  1080    C1=$1
  1081    C2=$2
  1082    log "pinging $C2 from $C1 (expecting failure) "
  1083    docker exec -i  ${C1} bash -c "ping -c 5 ${C2}" && {
  1084        abort "Error: Unexpected success pinging ${C2} from ${C1}"
  1085    }
  1086  }
  1087  
  1088  function ping_success {
  1089    check_num_params "$#" "2"
  1090    C1=$1
  1091    C2=$2
  1092    log "pinging $C2 from $C1 (expecting success) "
  1093    docker exec -i ${C1} bash -c "ping -c 5 ${C2}" || {
  1094      abort "Error: Could not ping ${C2} from ${C1}"
  1095    }
  1096  }
  1097  
  1098  function wait_for_cilium_shutdown {
  1099    local save=$-
  1100    set +e
  1101    log "waiting for cilium to shutdown"
  1102    i=0
  1103    while pgrep cilium-agent; do
  1104      micro_sleep
  1105      if [[ ${i} -ge 240 ]]; then
  1106        log "Timeout while waiting for Cilium to shutdown"
  1107        exit 1
  1108      fi
  1109      ((i++))
  1110    done
  1111    log "finished waiting for cilium to shutdown"
  1112    restore_flag $save "e"
  1113  }