github.com/cilium/cilium@v1.16.2/test/envoy/helpers.bash (about) 1 #!/usr/bin/env bash 2 3 CILIUM_FILES="cilium-files" 4 DUMP_FILE=$(mktemp) 5 MONITOR_PID="" 6 LAST_LOG_DATE="" 7 TEST_NET=cilium 8 GOPS="/home/vagrant/go/bin/gops" 9 10 # Variables used during Jenkins builds. 11 BUILD_NUM="${BUILD_NUMBER:-0}" 12 JOB_BASE="${JOB_BASE_NAME:-local}" 13 BUILD_ID="${JOB_BASE}-${BUILD_NUM}" 14 15 AGENT_SOCK_PATH=/var/run/cilium/cilium.sock 16 17 # Prefer local build if binary file detected. 18 for bin in "../cilium/cilium" \ 19 "../daemon/cilium-agent" \ 20 "../plugins/cilium-docker/cilium-docker"; do 21 if [ -f $bin ]; then 22 export PATH=$PWD/`dirname $bin`:$PATH 23 fi 24 done 25 26 # Prevent Fedora rules in raw table from affecting the test. 27 ip6tables -t raw -F 2> /dev/null || true 28 29 function log { 30 local save=$- 31 set +u 32 check_num_params "$#" "1" 33 message=$1 34 local stack 35 for (( i=${#FUNCNAME[@]}-1 ; i>0 ; i-- )) ; do 36 if [[ "${stack}" == "" ]]; then 37 stack="$(basename $0): ${FUNCNAME[i]}" 38 else 39 stack="$stack/${FUNCNAME[i]}" 40 fi 41 done 42 echo "----- ${stack}: $message" 43 restore_flag $save "u" 44 } 45 46 # Usage: overwrite $iter 'commands --option --foo bar "quoted args" ' 47 # Executes the commands provided as parameters, moves the cursor back by the 48 # number of lines output by the command, then prints the output of the command. 49 # If $iter is zero, then the cursor is not moved; this is equivalent to 50 # 'shift; eval "$@"'. 51 function overwrite { 52 local iter=$1 53 shift 54 55 local output=$(eval "$@") 56 if [ ! -z $TERM ] && [ $iter -ne 0 ]; then 57 local ERASER=$(tput cuu1 ; tput el) 58 local n_lines=$(echo "$output" | wc -l) 59 for i in $(seq 1 $n_lines); do 60 echo -ne "$ERASER" 61 done 62 fi 63 echo "$output" 64 } 65 66 function get_filename_without_extension { 67 check_num_params "$#" "1" 68 local file=$(basename $1) 69 local filename="${file%.*}" 70 echo $filename 71 } 72 # Note: if you call this, do not change the value of the debug flag - you will make the shell segmentation fault :) 73 function redirect_debug_logs { 74 check_num_params "$#" "1" 75 local LOGS_DIR=$1 76 mkdir -p ${LOGS_DIR} 77 exec {BASH_XTRACEFD}>>${LOGS_DIR}/debug.txt 78 } 79 80 function monitor_start { 81 local save=$- 82 set +e 83 log "starting monitor and dumping contents to $DUMP_FILE" 84 cilium-dbg monitor -v $@ > $DUMP_FILE & 85 MONITOR_PID=$! 86 restore_flag $save "e" 87 } 88 89 function monitor_resume { 90 local save=$- 91 set +e 92 log "resuming monitor and dumping contents to $DUMP_FILE" 93 cilium-dbg monitor -v $@ >> $DUMP_FILE & 94 MONITOR_PID=$! 95 restore_flag $save "e" 96 } 97 98 function monitor_clear { 99 local save=$- 100 set +e 101 log "clearing monitor" 102 cp /dev/null $DUMP_FILE 103 nstat > /dev/null 104 restore_flag $save "e" 105 } 106 107 function monitor_dump { 108 local save=$- 109 set +e 110 nstat 111 cat $DUMP_FILE 112 restore_flag $save "e" 113 } 114 115 function monitor_stop { 116 local save=$- 117 set +e 118 if [ ! -z "$MONITOR_PID" ]; then 119 kill $MONITOR_PID || true > /dev/null 2>&1 120 fi 121 restore_flag $save "e" 122 } 123 124 function logs_clear { 125 LAST_LOG_DATE="$(date +'%F %T')" 126 } 127 128 function abort { 129 set +e 130 echo "------------------------------------------------------------------------" 131 echo " Test Failed" 132 echo "$*" 133 echo "" 134 echo "------------------------------------------------------------------------" 135 136 if [ ! -z "$DEBUG" ]; then 137 cilium-dbg status 138 cilium-dbg endpoint list 139 cilium-dbg policy get 140 read -n 1 -p "Press any key to continue..." 141 fi 142 143 monitor_dump 144 monitor_stop 145 146 echo "------------------------------------------------------------------------" 147 echo " Cilium logs (last 200 lines)" 148 journalctl --no-pager --since "${LAST_LOG_DATE}" -u cilium | tail -n 200 149 echo "" 150 echo "------------------------------------------------------------------------" 151 152 exit 1 153 } 154 155 function micro_sleep { 156 sleep 0.5 157 } 158 159 function kafka_consumer_delay { 160 # wait for kafka consumer to come up 161 sleep 5 162 } 163 164 function to_services_delay { 165 sleep 5 166 } 167 168 function restore_flag { 169 check_num_params "$#" "2" 170 local save=$1 171 local flag=$2 172 if [[ $save =~ $2 ]]; then 173 set -$2 174 fi 175 } 176 177 function check_num_params { 178 local NUM_PARAMS=$1 179 local NUM_EXPECTED_PARAMS=$2 180 if [ "$NUM_PARAMS" -ne "$NUM_EXPECTED_PARAMS" ]; then 181 echo "${FUNCNAME[ 1 ]}: invalid number of parameters, expected $NUM_EXPECTED_PARAMS parameter(s)" 182 exit 1 183 fi 184 } 185 186 function wait_for_endpoints { 187 local save=$- 188 set +e 189 check_num_params "$#" "1" 190 local NUM_DESIRED="$1" 191 local CMD="cilium-dbg endpoint list | grep -v -e \"not-ready\" -e \"reserved\" | grep ready -c || true" 192 local INFO_CMD="cilium-dbg endpoint list" 193 local MAX_MINS="2" 194 local ERROR_OUTPUT="Timeout while waiting for $NUM_DESIRED endpoints" 195 log "waiting for up to ${MAX_MINS} mins for ${NUM_DESIRED} endpoints to be in \"ready\" state" 196 wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT" 197 log "done waiting for up to ${MAX_MINS} mins for ${NUM_DESIRED} endpoints to be in \"ready\" state" 198 restore_flag $save "e" 199 } 200 201 function wait_for_endpoints_deletion { 202 local save=$- 203 set +e 204 local NUM_DESIRED="2" # When no endpoints are present there should be two lines only. 205 local CMD="cilium-dbg endpoint list | grep -v \"reserved\" | wc -l || true" 206 local INFO_CMD="cilium-dbg endpoint list" 207 local MAX_MINS="2" 208 local ERROR_OUTPUT="Timeout while waiting for endpoint removal" 209 log "waiting for up to ${MAX_MINS} mins for all endpoints to be removed" 210 wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT" 211 log "done waiting" 212 restore_flag $save "e" 213 } 214 215 function k8s_num_ready { 216 local save=$- 217 set +e 218 local NAMESPACE=$1 219 local CILIUM_POD=$2 220 local FILTER=$3 221 kubectl -n ${NAMESPACE} exec ${CILIUM_POD} -- cilium-dbg endpoint list | grep $FILTER | grep -v -e 'not-ready' -e 'reserved' | grep -c 'ready' || true 222 restore_flag $save "e" 223 } 224 225 function wait_for_k8s_endpoints { 226 local save=$- 227 set +e 228 check_num_params "$#" "4" 229 local NAMESPACE=$1 230 local CILIUM_POD=$2 231 local NUM=$3 232 local FILTER=$4 233 log "Waiting for $NUM endpoints in namespace $NAMESPACE managed by $CILIUM_POD" 234 235 # Wait some time for at least one endpoint to get into regenerating state 236 # FIXME: Remove when this is reliable 237 sleep 5 238 239 local sleep_time=1 240 local iter=0 241 local found 242 found=$(k8s_num_ready "${NAMESPACE}" "${CILIUM_POD}" "${FILTER}") 243 log "found: $found" 244 while [[ "$found" -ne "$NUM" ]]; do 245 if [[ $iter -gt $((5*60/$sleep_time)) ]]; then 246 echo "" 247 log "Timeout while waiting for $NUM endpoints" 248 restore_flag $save "e" 249 exit 1 250 else 251 overwrite $iter ' 252 kubectl -n ${NAMESPACE} exec -- ${CILIUM_POD} cilium-dbg endpoint list 253 echo -n " [${found}/${NUM}]" 254 ' 255 sleep $sleep_time 256 fi 257 found=$(k8s_num_ready "${NAMESPACE}" "${CILIUM_POD}" "${FILTER}") 258 log "found: $found" 259 ((iter++)) 260 done 261 262 overwrite $iter 'kubectl -n ${NAMESPACE} exec ${CILIUM_POD} -- cilium-dbg endpoint list' 263 restore_flag $save "e" 264 } 265 266 function wait_for_cilium_status { 267 local NUM_DESIRED="1" 268 local CMD="cilium-dbg status | grep 'Cilium:' | grep -c OK || true" 269 local INFO_CMD="true" 270 local MAX_MINS="1" 271 local ERROR_OUTPUT="Timeout while waiting for Cilium to be ready" 272 wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT" 273 } 274 275 function wait_for_kubectl_cilium_status { 276 check_num_params "$#" "2" 277 namespace=$1 278 pod=$2 279 local NUM_DESIRED="1" 280 local CMD="kubectl -n ${namespace} exec ${pod} -- cilium-dbg status | grep "Cilium:" | grep -c 'OK' || true" 281 local INFO_CMD="true" 282 local MAX_MINS="2" 283 local ERROR_OUTPUT="Timeout while waiting for Cilium to be ready" 284 wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT" 285 } 286 287 function wait_for_cilium_ep_gen { 288 local save=$- 289 set +e 290 local MODE=$1 291 292 local NAMESPACE 293 local POD 294 local CMD 295 local INFO_CMD 296 297 if [[ "$MODE" == "k8s" ]]; then 298 # Only care about provided params if mode is K8s. 299 check_num_params "$#" "3" 300 log "mode is K8s" 301 NAMESPACE=$2 302 POD=$3 303 CMD="kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list | grep -c regenerat" 304 INFO_CMD="kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list" 305 else 306 CMD="cilium-dbg endpoint list | grep -c regenerat" 307 INFO_CMD="cilium-dbg endpoint list" 308 fi 309 310 local NUM_DESIRED="0" 311 local MAX_MINS="2" 312 local ERROR_OUTPUT="Timeout while waiting for endpoints to regenerate" 313 local sleep_time=1 314 315 local iter=0 316 local found 317 found=$(eval "$CMD") 318 319 while [[ "$found" -ne "$NUM_DESIRED" ]]; do 320 log "$found endpoints are still regenerating; want $NUM_DESIRED" 321 if [[ $((iter++)) -gt $((${MAX_MINS}*60/$sleep_time)) ]]; then 322 echo "" 323 log "${ERROR_OUTPUT}" 324 exit 1 325 else 326 overwrite $iter ' 327 log "still within time limit for waiting for endpoints to be in 'ready' state; sleeping and checking again" 328 log "output of ${INFO_CMD}" 329 eval "$INFO_CMD" 330 echo -n " [$found/$NUM_DESIRED]" 331 log "sleeping for $sleep_time" 332 ' 333 sleep $sleep_time 334 fi 335 log "evaluating $CMD" 336 found=$(eval "${CMD}") 337 log "found: $found" 338 done 339 set -e 340 restore_flag $save "e" 341 } 342 343 function wait_for_daemon_set_not_ready { 344 local save=$- 345 set +e 346 check_num_params "$#" "2" 347 348 local namespace="${1}" 349 local name="${2}" 350 351 log "Waiting for instances of Cilium daemon $name in namespace $namespace to be clean up" 352 353 local sleep_time=2 354 local iter=0 355 local found="0" 356 until [[ "$found" -eq "1" ]]; do 357 if [[ $iter -gt $((5*60/$sleep_time)) ]]; then 358 echo "" 359 log "Timeout while waiting for cilium agent to be clean up by kubernetes" 360 print_k8s_cilium_logs 361 exit 1 362 else 363 overwrite $iter 'kubectl -n ${namespace} get pods -o wide' 364 sleep $sleep_time 365 fi 366 kubectl get pods -n ${namespace} | grep ${name} -q 367 found=$? 368 ((iter++)) 369 done 370 371 overwrite $iter 'kubectl -n kube-system get pods -o wide' 372 restore_flag $save "e" 373 } 374 375 function wait_for_policy_enforcement { 376 check_num_params "$#" "1" 377 local NUM_DESIRED="$1" 378 local CMD="cilium-dbg endpoint list | grep -c Disabled" 379 local INFO_CMD="cilium-dbg endpoint list" 380 local MAX_MINS="2" 381 local ERROR_OUTPUT="Timeout while waiting for policy to be enabled for all endpoints" 382 wait_for_desired_state "$NUM_DESIRED" "$CMD" "$INFO_CMD" "$MAX_MINS" "$ERROR_OUTPUT" 383 } 384 385 function count_lines_in_log { 386 echo `wc -l $DUMP_FILE | awk '{ print $1 }'` 387 } 388 389 function wait_for_log_entries { 390 check_num_params "$#" "1" 391 local expected=$(($1 + $(count_lines_in_log))) 392 wait_specified_time_test "test \"\$(count_lines_in_log)\" -ge \"$expected\"" "2" 393 } 394 395 function wait_for_docker_ipv6_addr { 396 check_num_params "$#" "1" 397 name=$1 398 wait_specified_time_test "test \"\$(docker inspect --format '{{ .NetworkSettings.Networks.cilium.GlobalIPv6Address }}' $name)\" != \"\"" "2" 399 } 400 401 function wait_for_running_pod { 402 pod=$1 403 namespace=${2:-default} 404 log "Waiting for ${pod} pod to be Running..." 405 wait_specified_time_test "test \"\$(kubectl get pods -n ${namespace} -o wide | grep ${pod} | grep -c Running)\" -eq \"1\"" "10" 406 } 407 408 function wait_for_no_pods { 409 namespace=${1:-default} 410 log "Waiting for no pods to be Running in namespace ${namespace}" 411 wait_specified_time_test "test \"\$(kubectl get pods -n ${namespace} -o wide 2>&1 | grep -c 'No resources found')\" -eq \"1\"" "5" 412 } 413 414 function wait_for_n_running_pods { 415 local save=$- 416 set +e 417 check_num_params "$#" "1" 418 local NPODS=$1 419 log "Waiting for $NPODS running pods" 420 421 local sleep_time=1 422 local iter=0 423 local found 424 found=$(kubectl get pod | grep Running -c || true) 425 until [[ "$found" -eq "$NPODS" ]]; do 426 if [[ $iter -gt $((5*60/$sleep_time)) ]]; then 427 echo "" 428 log "Timeout while waiting for $NPODS running pods" 429 exit 1 430 else 431 overwrite $iter ' 432 kubectl get pod -o wide 433 echo -n " [${found}/${NPODS}]" 434 ' 435 sleep $sleep_time 436 fi 437 found=$(kubectl get pod | grep Running -c || true) 438 ((iter++)) 439 done 440 441 overwrite $iter 'kubectl get pod -o wide' 442 restore_flag $save "e" 443 } 444 445 # Wait for healthy k8s cluster on $1 nodes 446 function wait_for_healthy_k8s_cluster { 447 local save=$- 448 set +e 449 local NNODES=$1 450 log "Waiting for healthy k8s cluster with $NNODES nodes" 451 452 local sleep_time=2 453 local iter=0 454 local found 455 found=$(kubectl get cs | grep -v "STATUS" | grep -c "Healthy") 456 until [[ "$found" -eq "3" ]]; do 457 if [[ $iter -gt $((1*60/$sleep_time)) ]]; then 458 echo "" 459 log "Timeout while waiting for healthy kubernetes cluster" 460 exit 1 461 else 462 overwrite $iter ' 463 kubectl get cs 464 log "K8S Components ready: [${found}/3]" 465 ' 466 sleep $sleep_time 467 fi 468 found=$(kubectl get cs | grep -v "STATUS" | grep -c "Healthy") 469 ((iter++)) 470 done 471 overwrite $iter 'kubectl get cs' 472 local iter=0 473 local found 474 found=$(kubectl get nodes | grep Ready -c) 475 until [[ "$found" -eq "$NNODES" ]]; do 476 if [[ $iter -gt $((1*60/$sleep_time)) ]]; then 477 echo "" 478 log "Timeout while waiting for all nodes to be Ready" 479 exit 1 480 else 481 overwrite $iter ' 482 kubectl get nodes 483 log "Nodes ready [${found}/${NNODES}]" 484 ' 485 sleep $sleep_time 486 fi 487 found=$(kubectl get nodes | grep Ready -c) 488 ((iter++)) 489 done 490 restore_flag $save "e" 491 } 492 493 function k8s_nodes_policy_status { 494 local save=$- 495 set +e 496 local sleep_time=2 497 local NNODES=$1 498 local policy_ns=$2 499 local policy_name=$3 500 local iter=0 501 local nodes=$(kubectl get ciliumnetworkpolicies -n "${policy_ns}" "${policy_name}" -o go-template --template='{{len .status.nodes}}') 502 until [[ "${nodes}" -eq "${NNODES}" ]]; do 503 if [[ $iter -gt $((1*60/$sleep_time)) ]]; then 504 echo "" 505 log "Timeout while waiting for $NNODES to have policy ${policy_ns}/${policy_name} installed" 506 exit 1 507 else 508 overwrite $iter ' 509 kubectl get nodes 510 log "Nodes with policy accepted [${found}/${NNODES}]" 511 ' 512 sleep $sleep_time 513 fi 514 found=$(kubectl get nodes | grep Ready -c) 515 ((iter++)) 516 done 517 518 kubectl get ciliumnetworkpolicies -n "${policy_ns}" "${policy_name}" -o go-template --template='{{.status.nodes}}' 519 restore_flag $save "e" 520 } 521 522 function gather_files { 523 local TEST_NAME=$1 524 local TEST_SUITE=$2 525 log "gathering files for test $TEST_NAME in test suite $TEST_SUITE" 526 local CILIUM_ROOT="src/github.com/cilium/cilium" 527 if [ -z "${TEST_SUITE}" ]; then 528 TEST_SUITE="runtime-tests" 529 fi 530 if [ -z "${GOPATH}" ]; then 531 local GOPATH="/home/vagrant/go" 532 fi 533 CILIUM_DIR="${GOPATH}/${CILIUM_ROOT}/test/envoy/cilium-files/${TEST_NAME}" 534 local RUN="/var/run/cilium" 535 local LIB="/var/lib/cilium" 536 local RUN_DIR="${CILIUM_DIR}${RUN}" 537 local LIB_DIR="${CILIUM_DIR}${LIB}" 538 mkdir -p "${CILIUM_DIR}" 539 mkdir -p "${RUN_DIR}" 540 mkdir -p "${LIB_DIR}" 541 if [[ "${TEST_SUITE}" == "runtime-tests" ]]; then 542 local CLI_OUT_DIR="${CILIUM_DIR}/cli" 543 local PROF_OUT_DIR="${CILIUM_DIR}/profiling" 544 mkdir -p "${CLI_OUT_DIR}" 545 dump_cli_output "${CLI_OUT_DIR}" || true 546 dump_gops_output "${PROF_OUT_DIR}" "cilium-agent" || true 547 # Get logs from Consul container. 548 mkdir -p "${CILIUM_DIR}/consul" 549 docker logs cilium-consul > "${CILIUM_DIR}/consul/consul-logs.txt" 2>/dev/null 550 else 551 # Get logs from each Cilium pod. 552 local NAMESPACE="kube-system" 553 local CILIUM_POD_1=$(kubectl -n ${NAMESPACE} get pods -l k8s-app=cilium | awk 'NR==2{ print $1 }') 554 local CILIUM_POD_2=$(kubectl -n ${NAMESPACE} get pods -l k8s-app=cilium | awk 'NR==3{ print $1 }') 555 local CLI_OUT_DIR=${CILIUM_DIR}/cli 556 mkdir -p "${CLI_OUT_DIR}" 557 log "gathering Cilium logs from pod ${CILIUM_POD_1}" 558 dump_cli_output_k8s "${CLI_OUT_DIR}" "${NAMESPACE}" "${CILIUM_POD_1}" || true 559 log "gathering Cilium logs from pod ${CILIUM_POD_2}" 560 dump_cli_output_k8s "${CLI_OUT_DIR}" "${NAMESPACE}" "${CILIUM_POD_2}" || true 561 fi 562 sudo cp -r ${RUN}/state "${RUN_DIR}" || true 563 sudo cp ${RUN}/*.log "${RUN_DIR}" || true 564 sudo cp -r ${LIB}/* "${LIB_DIR}" || true 565 find "${CILIUM_DIR}" -type d -exec sudo chmod 777 {} \; 566 find "${CILIUM_DIR}" -exec sudo chmod a+r {} \; 567 log "finished gathering files for test $TEST_NAME in test suite $TEST_SUITE" 568 } 569 570 function dump_cli_output { 571 check_num_params "$#" "1" 572 local DIR=$1 573 cilium-dbg endpoint list > ${DIR}/endpoint_list.txt 574 local EPS=$(cilium-dbg endpoint list | tail -n+3 | grep '^[0-9]' | awk '{print $1}') 575 for ep in ${EPS} ; do 576 cilium-dbg endpoint get ${ep} > ${DIR}/endpoint_get_${ep}.txt 577 cilium-dbg bpf policy get ${ep} > ${DIR}/bpf_policy_list_${ep}.txt 578 done 579 cilium-dbg service list > ${DIR}/service_list.txt 580 local SVCS=$(cilium-dbg service list | tail -n+2 | awk '{print $1}') 581 for svc in ${SVCS} ; do 582 cilium-dbg service get ${svc} > ${DIR}/service_get_${svc}.txt 583 done 584 local IDS=$(cilium-dbg endpoint list | tail -n+3 | awk '{print $4}' | grep -o '[0-9]*') 585 for id in ${IDS} ; do 586 cilium-dbg identity get ${id} > ${DIR}/identity_get_${id}.txt 587 done 588 cilium-dbg config > ${DIR}/config.txt 589 cilium-dbg bpf lb list > ${DIR}/bpf_lb_list.txt 590 cilium-dbg bpf ct list global > ${DIR}/bpf_ct_list_global.txt 591 cilium-dbg bpf tunnel list > ${DIR}/bpf_tunnel_list.txt 592 cilium-dbg policy get > ${DIR}/policy_get.txt 593 cilium-dbg status > ${DIR}/status.txt 594 cilium-dbg debuginfo -f ${DIR}/debuginfo.txt 595 cilium-bugtool -t ${DIR} 596 } 597 598 function dump_cli_output_k8s { 599 check_num_params "$#" "3" 600 local DIR=$1 601 local NAMESPACE=$2 602 local POD=$3 603 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list > ${DIR}/${POD}_endpoint_list.txt 604 local EPS=$(kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list | tail -n+3 | grep '^[0-9]' | awk '{print $1}') 605 for ep in ${EPS} ; do 606 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint get ${ep} > ${DIR}/${POD}_endpoint_get_${ep}.txt 607 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg bpf policy get ${ep} > ${DIR}/${POD}_bpf_policy_list_${ep}.txt 608 done 609 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg service list > ${DIR}/${POD}_service_list.txt 610 local SVCS=$(kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg service list | tail -n+2 | awk '{print $1}') 611 for svc in ${SVCS} ; do 612 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg service get ${svc} > ${DIR}/${POD}_service_get_${svc}.txt 613 done 614 local IDS=$(kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg endpoint list | tail -n+3 | awk '{print $4}' | grep -o '[0-9]*') 615 for id in ${IDS} ; do 616 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg identity get ${id} > ${DIR}/${POD}_identity_get_${id}.txt 617 done 618 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg config > ${DIR}/${POD}_config.txt 619 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg bpf lb list > ${DIR}/${POD}_bpf_lb_list.txt 620 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg bpf ct list global > ${DIR}/${POD}_bpf_ct_list_global.txt 621 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg bpf tunnel list > ${DIR}/${POD}_bpf_tunnel_list.txt 622 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg policy get > ${DIR}/${POD}_policy_get.txt 623 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg status > ${DIR}/${POD}_status.txt 624 kubectl exec -n ${NAMESPACE} ${POD} -- cilium-dbg debuginfo > ${DIR}/${POD}_debuginfo.txt 625 local DEBUGTOOL_ARCHIVE=`kubectl exec -n ${NAMESPACE} ${POD} -- cilium-bugtool | grep ARCHIVE | awk '{ print $3}'` 626 kubectl cp ${NAMESPACE}/${POD}:${DEBUGTOOL_ARCHIVE} ${DIR}/${POD}_bugtool.tar 627 } 628 629 function dump_gops_output { 630 check_num_params "$#" "2" 631 local DIR="$1" 632 local PROG="$2" 633 local PROG_PROFILING_DIR="${DIR}/${PROG}" 634 mkdir -p "${PROG_PROFILING_DIR}" 635 log "getting gops output for ${PROG} and dumping to dir ${PROG_PROFILING_DIR}" 636 local PROG_PID=$(sudo ${GOPS} | grep "${PROG}" | awk '{print $1}') 637 log "running \"gops stack\" for ${PROG}" 638 sudo ${GOPS} stack ${PROG_PID} > "${PROG_PROFILING_DIR}/${PROG}_stack.txt" 639 log "running \"gops memstats\" for ${PROG}" 640 sudo ${GOPS} memstats ${PROG_PID} > "${PROG_PROFILING_DIR}/${PROG}_memstats.txt" 641 log "running \"gops stats\" for ${PROG}" 642 sudo ${GOPS} stats ${PROG_PID} > "${PROG_PROFILING_DIR}/${PROG}_stats.txt" 643 log "done getting gops output for ${PROG}" 644 } 645 646 function print_k8s_cilium_logs { 647 for pod in $(kubectl -n kube-system get pods -o wide| grep cilium | awk '{print $1}'); do 648 kubectl -n kube-system logs $pod 649 if [ $? -ne 0 ]; then 650 kubectl -n kube-system logs $pod --previous 651 fi 652 done 653 } 654 655 function wait_for_daemon_set_ready { 656 local save=$- 657 set +e 658 check_num_params "$#" "3" 659 660 local namespace="${1}" 661 local name="${2}" 662 local n_ds_expected="${3}" 663 664 log "Waiting for $n_ds_expected instances of Cilium daemon $name in namespace $namespace to become ready" 665 666 local sleep_time=2 667 local iter=0 668 local found="0" 669 until [[ "$found" -eq "$n_ds_expected" ]]; do 670 if [[ $iter -gt $((5*60/$sleep_time)) ]]; then 671 echo "" 672 log "Timeout while waiting for cilium-agent" 673 print_k8s_cilium_logs 674 exit 1 675 else 676 overwrite $iter ' 677 kubectl -n kube-system get ds 678 kubectl -n kube-system get pods -o wide 679 echo -n " [${found}/${n_ds_expected}]" 680 ' 681 sleep $sleep_time 682 fi 683 found=$(kubectl get ds -n ${namespace} ${name} 2>&1 | awk 'NR==2{ print $4 }') 684 ((iter++)) 685 done 686 overwrite $iter 'kubectl -n kube-system get pods -o wide' 687 restore_flag $save "e" 688 } 689 690 function k8s_wait_for_cilium_status_ready { 691 local save=$- 692 set +e 693 local pod 694 check_num_params "$#" "1" 695 local namespace=$1 696 local pods=$(kubectl -n $namespace get pods -l k8s-app=cilium | grep cilium- | awk '{print $1}') 697 698 for pod in $pods; do 699 wait_for_kubectl_cilium_status $namespace $pod 700 done 701 restore_flag $save "e" 702 } 703 704 function k8s_count_all_cluster_cilium_eps { 705 local save=$- 706 set +e 707 local total=0 708 check_num_params "$#" "1" 709 local pod 710 local namespace=$1 711 local pods=$(kubectl -n $namespace get pods -l k8s-app=cilium | grep cilium- | awk '{print $1}') 712 713 for pod in $pods; do 714 local n_eps=$(kubectl -n $namespace exec $pod -- cilium-dbg endpoint list --no-headers | wc -l) 715 total=$(( $total + $n_eps )) 716 done 717 718 echo "$total" 719 restore_flag $save "e" 720 } 721 722 function wait_for_api_server_ready { 723 log "Waiting for kube-apiserver to spin up" 724 wait_specified_time_test "test \$(kubectl get cs)" "10" 725 } 726 727 function wait_for_service_endpoints_ready { 728 check_num_params "$#" "3" 729 local namespace="${1}" 730 local name="${2}" 731 local port="${3}" 732 733 log "Waiting for ${name} service endpoints to be ready" 734 wait_specified_time_test "test \"\$(kubectl get endpoints -n ${namespace} ${name} | grep -c \":${port}\")\" -eq \"1\"" "10" 735 log "Done waiting for ${name} service endpoints to be ready" 736 kubectl get endpoints -n ${namespace} ${name} 737 } 738 739 function wait_for_service_ready_cilium_pod { 740 check_num_params "$#" "4" 741 local namespace="${1}" 742 local pod="${2}" 743 local fe_port="${3}" 744 # TODO: only works for one backend right now. 745 local be_port="${4}" 746 747 log "Waiting for Cilium pod ${pod} to have services ready with frontend port: ${fe_port} and backend port: ${be_port}" 748 749 wait_specified_time_test "test \"\$(kubectl -n ${namespace} exec ${pod} -- cilium-dbg service list | awk '{ print \$2 }' | grep -c \":${fe_port}\")\" -ge \"1\"" "10" 750 wait_specified_time_test "test \"\$(kubectl -n ${namespace} exec ${pod} -- cilium-dbg service list | awk '{ print \$5 }' | grep -c \":${be_port}\")\" -ge \"1\"" "10" 751 752 log "Done waiting for Cilium pod ${pod} to have services ready with frontend port: ${fe_port} and backend port: ${be_port}" 753 754 log "Listing all services:" 755 kubectl -n ${namespace} exec ${pod} -- cilium-dbg service list 756 } 757 758 function k8s_apply_policy { 759 declare -A currentRevison 760 local i 761 local pod 762 check_num_params "$#" "3" 763 local namespace=$1 764 local action=$2 765 local policy=$3 766 local pods=$(kubectl -n $namespace get pods -l k8s-app=cilium | grep cilium- | awk '{print $1}') 767 768 for pod in $pods; do 769 local rev=$(kubectl -n $namespace exec $pod -- cilium-dbg policy get | grep Revision: | awk '{print $2}') 770 currentRevison[$pod]=$rev 771 done 772 773 log "Current policy revisions:" 774 for i in "${!currentRevison[@]}" 775 do 776 echo " $i: ${currentRevison[$i]}" 777 done 778 779 kubectl $action -f $policy 780 781 for pod in $pods; do 782 local nextRev=$(expr ${currentRevison[$pod]} + 1) 783 log "Waiting for agent $pod endpoints to get to revision $nextRev" 784 timeout 180s kubectl -n $namespace exec $pod -- cilium-dbg policy wait $nextRev 785 done 786 787 # Adding sleep as workaround for l7 stresstests 788 sleep 10s 789 } 790 791 function policy_delete_and_wait { 792 log "deleting policy $* and waiting up to 120 seconds to complete" 793 rev=$(cilium-dbg policy delete $* | grep Revision: | awk '{print $2}') 794 timeout 120s cilium-dbg policy wait $rev 795 } 796 797 function policy_import_and_wait { 798 log "importing policy $* and waiting up to 120 seconds to complete" 799 rev=$(cilium-dbg policy import $* | grep Revision: | awk '{print $2}') 800 timeout 120s cilium-dbg policy wait $rev 801 } 802 803 function get_vm_identity_file { 804 check_num_params "$#" "1" 805 local VM_NAME=$1 806 vagrant ssh-config ${VM_NAME} | grep IdentityFile | awk '{print $2}' 807 } 808 809 function get_vm_ssh_port { 810 check_num_params "$#" "1" 811 local VM_NAME=$1 812 vagrant ssh-config ${VM_NAME} | grep Port | awk '{ print $2 }' 813 } 814 815 function copy_files_vm { 816 check_num_params "$#" "2" 817 local VM_NAME=$1 818 local FILES_DIR=$2 819 local ID_FILE 820 local PORT 821 822 # Check that the VM is running before we try to gather logs from it. 823 check_vm_running $VM_NAME 824 825 log "getting the VM identity file for $VM_NAME" 826 ID_FILE=$(get_vm_identity_file $VM_NAME) 827 log "getting the port for $VM_NAME to SSH" 828 PORT=$(get_vm_ssh_port $VM_NAME) 829 830 log "getting cilium logs from $VM_NAME" 831 vagrant ssh $VM_NAME -c 'sudo -E bash -c "journalctl --no-pager -u cilium > /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files/cilium-logs && chmod a+r /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files/cilium-logs"' 832 vagrant ssh $VM_NAME -c 'sudo -E bash -c "journalctl --no-pager -u cilium-docker > /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files/cilium-docker-logs && chmod a+r /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files/cilium-docker-logs"' 833 834 log "listing all logs that will be gathered from $VM_NAME" 835 vagrant ssh $VM_NAME -c 'ls -altr /home/vagrant/go/src/github.com/cilium/cilium/test/envoy/cilium-files' 836 837 log "copying logs from $VM_NAME onto VM host for accessibility after VM is destroyed" 838 scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r -P ${PORT} -i ${ID_FILE} vagrant@127.0.0.1:/home/vagrant/go/src/github.com/cilium/cilium/${FILES_DIR} ${WORKSPACE}/cilium-files-${VM_NAME} 839 } 840 841 function get_k8s_vm_name { 842 check_num_params "$#" "1" 843 local VM_PREFIX=$1 844 845 if [ ! -z ${BUILD_NUMBER} ] ; then 846 local BUILD_ID_NAME="-build-${BUILD_ID}" 847 fi 848 echo "${VM_PREFIX}${BUILD_ID_NAME}" 849 } 850 851 function get_cilium_master_vm_name { 852 if [ ! -z "${K8STAG}" ] ; then 853 local K8S_TAG="${K8STAG:-k8s}" 854 fi 855 856 if [ ! -z "${BUILD_NUMBER}" ] ; then 857 local BUILD_ID_NAME="-build-${BUILD_ID}" 858 fi 859 860 echo "cilium${K8S_TAG}-master${BUILD_ID_NAME}" 861 } 862 863 function check_vm_running { 864 check_num_params "$#" "1" 865 local VM=$1 866 log "getting status of VM ${VM}" 867 vagrant status ${VM} 868 log "done getting status of VM ${VM}" 869 870 local VM_STATUS 871 VM_STATUS=`vagrant status ${VM} | grep ${VM} | awk '{print $2}'` 872 if [[ "${VM_STATUS}" != "running" ]]; then 873 log "$VM is not in \"running\" state; exiting" 874 else 875 log "$VM is \"running\" continuing" 876 fi 877 } 878 879 function wait_for_agent_socket { 880 check_num_params "$#" "1" 881 MAX_WAIT=$1 882 883 log "waiting at most ${MAX_WAIT} iterations for cilium agent socket" 884 local i=0 885 886 while [ "$i" -lt "$MAX_WAIT" ]; do 887 micro_sleep 888 i=$[$i+1] 889 if [ -S $AGENT_SOCK_PATH ]; then 890 return 891 fi 892 done 893 abort "Waiting for agent socket, timed out" 894 } 895 896 function wait_for_kill { 897 check_num_params "$#" "2" 898 TARGET_PID=$1 899 MAX_WAIT=$2 900 901 log "waiting at most ${MAX_WAIT} iterations for PID ${TARGET_PID} to be killed" 902 local i=0 903 904 while [ $i -lt "${MAX_WAIT}" ]; do 905 micro_sleep 906 i=$[$i+1] 907 if ! ps -p $TARGET_PID > /dev/null; then 908 return 909 fi 910 done 911 abort "Waiting for agent process to be killed, timed out" 912 } 913 914 # diff_timeout waits for the output of the commands specified via $1 and $2 to 915 # be identical by comparing the output with `diff -Nru`. The commands are 916 # executed consecutively with a 2 second pause until the output matches or the 917 # timeout of 1 minute is reached. 918 function diff_timeout() { 919 local save=$- 920 set +e 921 local arg1="$1" 922 local arg2="$2" 923 local sleep_time=2 924 local iter=0 925 local found="0" 926 927 until [[ "$found" -eq "1" ]]; do 928 if [[ $((iter++)) -gt $((30)) ]]; then 929 log "Timeout waiting for diff to be empty" 930 abort "$DIFF" 931 fi 932 933 DIFF=$(diff -Nru <(eval "$arg1") <(eval "$arg2") || true) 934 if [[ "$DIFF" == "" ]]; then 935 found="1" 936 else 937 sleep $sleep_time 938 fi 939 done 940 restore_flag $save "e" 941 } 942 943 ####################################### 944 # Waits for MAX_MINS until the output of CMD 945 # reaches NUM_DESIRED. While the state is not 946 # realized, INFO_CMD is emitted. If the state 947 # is not realized after MAX_MINS, ERROR_OUTPUT 948 # is emitted. 949 # Globals: 950 # Arguments: 951 # NUM_DESIRED: desired number output by CMD. 952 # CMD: command to run. 953 # INFO_CMD: command to run while state is not 954 # realized 955 # MAX_MINS: maximum minutes to wait for desired 956 # state 957 # ERROR_OUTPUT: message that is emitted if desired 958 # state is not realized in MAX_MINS. 959 # Returns: 960 # None 961 ####################################### 962 function wait_for_desired_state { 963 964 local save=$- 965 set +e 966 check_num_params "$#" "5" 967 local NUM_DESIRED="$1" 968 local CMD="$2" 969 local INFO_CMD="$3" 970 local MAX_MINS="$4" 971 local ERROR_OUTPUT="$5" 972 local sleep_time=1 973 local iter=0 974 local found 975 found=$(eval "$CMD") 976 log "waiting for at most ${MAX_MINS} minutes for command ${CMD} to succeed" 977 log "found: $found" 978 979 while [[ "$found" -ne "$NUM_DESIRED" ]]; do 980 if [[ $iter -gt $((${MAX_MINS}*60/$sleep_time)) ]]; then 981 echo "" 982 log "$ERROR_OUTPUT" 983 exit 1 984 else 985 overwrite $iter ' 986 log "desired state not realized; will sleep and try again" 987 eval "$INFO_CMD" 988 echo -n " [$found/$NUM_DESIRED]" 989 ' 990 sleep $sleep_time 991 fi 992 found=$(eval "${CMD}") 993 log "found: $found" 994 ((iter++)) 995 done 996 log "desired state realized for command ${CMD}" 997 eval "${INFO_CMD}" 998 restore_flag $save "e" 999 } 1000 1001 ####################################### 1002 # Waits for MAX_MINS until CMD returns with 1003 # return code 0. If the desired state is 1004 # not realized after MAX_MINS, exits with 1005 # failure. 1006 # Globals: 1007 # Arguments: 1008 # CMD: command to run. 1009 # MAX_MINS: maximum minutes to wait for desired 1010 # state 1011 # Returns: 1012 # None 1013 ####################################### 1014 function wait_specified_time_test { 1015 local save=$- 1016 set +e 1017 local CMD="$1" 1018 local MAX_MINS="$2" 1019 1020 local sleep_time=1 1021 local iter=0 1022 1023 log "waiting for at most ${MAX_MINS} minutes for command ${CMD} to succeed" 1024 while [[ "${iter}" -lt $((${MAX_MINS}*60/$sleep_time)) ]]; do 1025 if eval "${CMD}" ; then 1026 log "${CMD} succeeded" 1027 break 1028 fi 1029 overwrite $iter ' 1030 log "${iter} < $((${MAX_MINS}*60/$sleep_time)) " 1031 log "${CMD} did not succeed; sleeping and testing the command again" 1032 ' 1033 sleep ${sleep_time} 1034 iter=$((iter+1)) 1035 done 1036 if [[ "${iter}" -ge $((${MAX_MINS}*60/$sleep_time)) ]]; then 1037 log "Timeout ${MAX_MINS} minutes exceeded for command \"$CMD\"" 1038 log "Exiting with failure." 1039 exit 1 1040 fi 1041 log "${CMD} succeeded" 1042 restore_flag $save "e" 1043 } 1044 1045 function create_cilium_docker_network { 1046 log "creating Docker network of type Cilium" 1047 docker network inspect $TEST_NET 2> /dev/null || { 1048 docker network create --ipv6 --subnet ::1/112 --ipam-driver cilium --driver cilium $TEST_NET 1049 } 1050 } 1051 1052 function remove_cilium_docker_network { 1053 local save=$- 1054 set +e 1055 log "removing Docker network of type cilium" 1056 docker network rm $TEST_NET > /dev/null 2>&1 1057 restore_flag $save "e" 1058 } 1059 1060 function remove_all_containers { 1061 docker rm -f $(docker ps --format '{{.Names}}' | grep -v cilium-consul) > /dev/null 2>&1 || true 1062 } 1063 1064 function test_succeeded { 1065 check_num_params "$#" "1" 1066 local TEST_NAME="$1" 1067 echo "============================================================" 1068 echo "== ==" 1069 echo "== ==" 1070 echo "== ==" 1071 echo " ${TEST_NAME} succeeded!" 1072 echo "== ==" 1073 echo "== ==" 1074 echo "== ==" 1075 echo "============================================================" 1076 } 1077 1078 function ping_fail { 1079 check_num_params "$#" "2" 1080 C1=$1 1081 C2=$2 1082 log "pinging $C2 from $C1 (expecting failure) " 1083 docker exec -i ${C1} bash -c "ping -c 5 ${C2}" && { 1084 abort "Error: Unexpected success pinging ${C2} from ${C1}" 1085 } 1086 } 1087 1088 function ping_success { 1089 check_num_params "$#" "2" 1090 C1=$1 1091 C2=$2 1092 log "pinging $C2 from $C1 (expecting success) " 1093 docker exec -i ${C1} bash -c "ping -c 5 ${C2}" || { 1094 abort "Error: Could not ping ${C2} from ${C1}" 1095 } 1096 } 1097 1098 function wait_for_cilium_shutdown { 1099 local save=$- 1100 set +e 1101 log "waiting for cilium to shutdown" 1102 i=0 1103 while pgrep cilium-agent; do 1104 micro_sleep 1105 if [[ ${i} -ge 240 ]]; then 1106 log "Timeout while waiting for Cilium to shutdown" 1107 exit 1 1108 fi 1109 ((i++)) 1110 done 1111 log "finished waiting for cilium to shutdown" 1112 restore_flag $save "e" 1113 }