k8s.io/kubernetes@v1.29.3/test/e2e_node/conformance/run_test.sh (about)

     1  #!/usr/bin/env bash
     2  
     3  # Copyright 2016 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  # This script is only for demonstrating how to use the node test container. In
    18  # production environment, kubelet bootstrap will be more complicated, user
    19  # should configure the node test container accordingly.
    20  # In addition, this script will also be used in the node e2e test to let it use
    21  # the containerized test suite.
    22  
    23  # TODO(random-liu): Use standard installer to install kubelet.
    24  # TODO(random-liu): Use standard tool to start kubelet in production way (such
    25  # as systemd, supervisord etc.)
    26  
    27  # Refresh sudo credentials if needed
    28  if ping -c 1 -q metadata.google.internal &> /dev/null; then
    29    echo 'Running on GCE, not asking for sudo credentials'
    30  elif sudo --non-interactive "$(which bash)" -c true 2> /dev/null; then
    31    # if we can run bash without a password, it's a pretty safe bet that either
    32    # we can run any command without a password, or that sudo credentials
    33    # are already cached - and they've just been re-cached
    34    echo 'No need to refresh sudo credentials'
    35  else
    36    echo 'Updating sudo credentials'
    37    sudo -v || exit 1
    38  fi
    39  
    40  # FOCUS is ginkgo focus to select which tests to run. By default, FOCUS is
    41  # initialized as "\[Conformance\]" in the test container to run all conformance
    42  # test.
    43  FOCUS=${FOCUS:-""}
    44  
    45  # SKIP is ginkgo skip to select which tests to skip. By default, SKIP is
    46  # initialized as "\[Flaky\]|\[Serial\]" in the test container skipping all
    47  # flaky and serial test.
    48  SKIP=${SKIP:-""}
    49  
    50  # TEST_ARGS is the test arguments. It could be used to override default test
    51  # arguments in the container.
    52  TEST_ARGS=${TEST_ARGS:-""}
    53  
    54  # REGISTRY is the image registry for node test image.
    55  REGISTRY=${REGISTRY:-"registry.k8s.io"}
    56  
    57  # ARCH is the architecture of current machine, the script will use this to
    58  # select corresponding test container image.
    59  ARCH=${ARCH:-"amd64"}
    60  
    61  # VERSION is the version of the test container image.
    62  VERSION=${VERSION:-"0.2"}
    63  
    64  # KUBELET_BIN is the kubelet binary name. If it is not specified, use the
    65  # default binary name "kubelet".
    66  KUBELET_BIN=${KUBELET_BIN:-"kubelet"}
    67  
    68  # KUBELET is the kubelet binary path. If it is not specified, assume kubelet is
    69  # in PATH.
    70  KUBELET=${KUBELET:-"$(which "$KUBELET_BIN")"}
    71  
    72  # LOG_DIR is the absolute path of the directory where the test will collect all
    73  # logs to. By default, use the current directory.
    74  LOG_DIR=${LOG_DIR:-$(pwd)}
    75  mkdir -p "$LOG_DIR"
    76  
    77  # NETWORK_PLUGIN is the network plugin used by kubelet. Do not use network
    78  # plugin by default.
    79  NETWORK_PLUGIN=${NETWORK_PLUGIN:-""}
    80  
    81  # CNI_CONF_DIR is the path to network plugin binaries.
    82  CNI_CONF_DIR=${CNI_CONF_DIR:-""}
    83  
    84  # CNI_BIN_DIR is the path to network plugin config files.
    85  CNI_BIN_DIR=${CNI_BIN_DIR:-""}
    86  
    87  # KUBELET_KUBECONFIG is the path to a kubeconfig file, specifying how to connect to the API server.
    88  KUBELET_KUBECONFIG=${KUBELET_KUBECONFIG:-"/var/lib/kubelet/kubeconfig"}
    89  
    90  # Creates a kubeconfig file for the kubelet.
    91  # Args: address (e.g. "http://localhost:8080"), destination file path
    92  function create-kubelet-kubeconfig() {
    93    local api_addr="${1}"
    94    local dest="${2}"
    95    local dest_dir
    96    dest_dir="$(dirname "${dest}")"
    97    mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}"
    98    sudo=$(test -w "${dest_dir}" || echo "sudo -E")
    99    cat <<EOF | ${sudo} tee "${dest}" > /dev/null
   100  apiVersion: v1
   101  kind: Config
   102  clusters:
   103    - cluster:
   104        server: ${api_addr}
   105      name: local
   106  contexts:
   107    - context:
   108        cluster: local
   109      name: local
   110  current-context: local
   111  EOF
   112  }
   113  
   114  # start_kubelet starts kubelet and redirect kubelet log to $LOG_DIR/kubelet.log.
   115  kubelet_log=kubelet.log
   116  start_kubelet() {
   117    echo "Creating kubelet.kubeconfig"
   118    create-kubelet-kubeconfig "http://localhost:8080" "${KUBELET_KUBECONFIG}"
   119    echo "Starting kubelet..."
   120    # we want to run this command as root but log the file to a normal user file
   121    # (so disable SC2024)
   122    # shellcheck disable=SC2024
   123    if ! sudo -b "${KUBELET}" "$@" &>"${LOG_DIR}/${kubelet_log}"; then
   124      echo "Failed to start kubelet"
   125      exit 1
   126    fi
   127  }
   128  
   129  # wait_kubelet retries for 10 times for kubelet to be ready by checking http://127.0.0.1:10248/healthz.
   130  wait_kubelet() {
   131    echo "Health checking kubelet..."
   132    healthCheckURL=http://127.0.0.1:10248/healthz
   133    local maxRetry=10
   134    local cur=1
   135    while [ $cur -le $maxRetry ]; do
   136      if curl -s $healthCheckURL > /dev/null; then
   137        echo "Kubelet is ready"
   138        break
   139      fi
   140      if [ $cur -eq $maxRetry ]; then
   141        echo "Health check exceeds max retry"
   142        exit 1
   143      fi
   144      echo "Kubelet is not ready"
   145      sleep 1
   146      ((cur++))
   147    done
   148  }
   149  
   150  # kill_kubelet kills kubelet.
   151  kill_kubelet() {
   152    echo "Stopping kubelet..."
   153    if ! sudo pkill "${KUBELET_BIN}"; then
   154      echo "Failed to stop kubelet."
   155      exit 1
   156    fi
   157  }
   158  
   159  # run_test runs the node test container.
   160  run_test() {
   161    env=""
   162    if [ -n "$FOCUS" ]; then
   163      env="$env -e FOCUS=\"$FOCUS\""
   164    fi
   165    if [ -n "$SKIP" ]; then
   166      env="$env -e SKIP=\"$SKIP\""
   167    fi
   168    if [ -n "$TEST_ARGS" ]; then
   169      env="$env -e TEST_ARGS=\"$TEST_ARGS\""
   170    fi
   171    # The test assumes that inside the container:
   172    # * kubelet manifest path is mounted to the same path;
   173    # * log collect directory is mounted to /var/result;
   174    # * root file system is mounted to /rootfs.
   175    sudo sh -c "docker run -it --rm --privileged=true --net=host -v /:/rootfs \
   176      -v $config_dir:$config_dir -v $LOG_DIR:/var/result ${env} $REGISTRY/node-test-$ARCH:$VERSION"
   177  }
   178  
   179  # Check whether kubelet is running. If kubelet is running, tell the user to stop
   180  # it before running the test.
   181  pid=$(pidof "${KUBELET_BIN}")
   182  if [ -n "$pid" ]; then
   183    echo "Kubelet is running (pid=$pid), please stop it before running the test."
   184    exit 1
   185  fi
   186  
   187  volume_stats_agg_period=10s
   188  serialize_image_pulls=false
   189  config_dir=$(mktemp -d)
   190  file_check_frequency=10s
   191  pod_cidr=10.100.0.0/24
   192  log_level=4
   193  start_kubelet --kubeconfig "${KUBELET_KUBECONFIG}" \
   194    --volume-stats-agg-period $volume_stats_agg_period \
   195    --serialize-image-pulls=$serialize_image_pulls \
   196    --pod-manifest-path "${config_dir}" \
   197    --file-check-frequency $file_check_frequency \
   198    --pod-cidr=$pod_cidr \
   199    --runtime-cgroups=/docker-daemon \
   200    --kubelet-cgroups=/kubelet \
   201    --system-cgroups=/system \
   202    --cgroup-root=/ \
   203    --v=$log_level \
   204  
   205  wait_kubelet
   206  
   207  run_test
   208  
   209  kill_kubelet
   210  
   211  # Clean up the kubelet config directory
   212  sudo rm -rf "${config_dir}"