k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/cluster/common.sh (about)

     1  #!/usr/bin/env bash
     2  
     3  # Copyright 2017 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  # Common utilities for kube-up/kube-down
    18  
    19  set -o errexit
    20  set -o nounset
    21  set -o pipefail
    22  
    23  KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)
    24  
    25  DEFAULT_KUBECONFIG="${HOME:-.}/.kube/config"
    26  
    27  source "${KUBE_ROOT}/hack/lib/util.sh"
    28  # KUBE_RELEASE_VERSION_REGEX matches things like "v1.2.3" or "v1.2.3-alpha.4"
    29  #
    30  # NOTE This must match the version_regex in build/common.sh
    31  # kube::release::parse_and_validate_release_version()
    32  #
    33  # KUBE_RELEASE_VERSION_REGEX is used in hack/get-build.sh and cluster/gce/util.sh and KUBE_RELEASE_VERSION_DASHED_REGEX is used in cluster/gce/util.sh,
    34  # make sure to remove these vars when not used anymore
    35  export KUBE_RELEASE_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(-([a-zA-Z0-9]+)\\.(0|[1-9][0-9]*))?$"
    36  export KUBE_RELEASE_VERSION_DASHED_REGEX="v(0|[1-9][0-9]*)-(0|[1-9][0-9]*)-(0|[1-9][0-9]*)(-([a-zA-Z0-9]+)-(0|[1-9][0-9]*))?"
    37  
    38  # KUBE_CI_VERSION_REGEX matches things like "v1.2.3-alpha.4.56+abcdefg" and "v1.2.3-56+abcdefg"
    39  #
    40  # NOTE This must match the version_regex in build/common.sh
    41  #
    42  # TODO: KUBE_CI_VERSION_REGEX is used in hack/get-build.sh and KUBE_CI_VERSION_DASHED_REGEX is used in cluster/gce/util.sh,
    43  # make sure to remove these vars when not used anymore
    44  #                              v1                .26               .0              -(rc            .0                .)?1              (  +014f      )?
    45  export KUBE_CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-([a-zA-Z0-9]+\\.(0|[1-9][0-9]*)\\.)?(0|[1-9][0-9]*)(\\+[-0-9a-z]*)?$"
    46  export KUBE_CI_VERSION_DASHED_REGEX="^v(0|[1-9][0-9]*)-(0|[1-9][0-9]*)-(0|[1-9][0-9]*)-([a-zA-Z0-9]+-(0|[1-9][0-9]*)-)?(0|[1-9][0-9]*)(\\+[-0-9a-z]*)?"
    47  
    48  # Generate kubeconfig data for the created cluster.
    49  # Assumed vars:
    50  #   KUBE_USER
    51  #   KUBE_PASSWORD
    52  #   KUBE_MASTER_IP
    53  #   KUBECONFIG
    54  #   CONTEXT
    55  #
    56  # If the apiserver supports bearer auth, also provide:
    57  #   KUBE_BEARER_TOKEN
    58  #
    59  # If the kubeconfig context being created should NOT be set as the current context
    60  # SECONDARY_KUBECONFIG=true
    61  #
    62  # To explicitly name the context being created, use OVERRIDE_CONTEXT
    63  #
    64  # The following can be omitted for --insecure-skip-tls-verify
    65  #   KUBE_CERT
    66  #   KUBE_KEY
    67  #   CA_CERT
    68  function create-kubeconfig() {
    69    KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
    70    local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
    71    SECONDARY_KUBECONFIG=${SECONDARY_KUBECONFIG:-}
    72    OVERRIDE_CONTEXT=${OVERRIDE_CONTEXT:-}
    73  
    74    if [[ "$OVERRIDE_CONTEXT" != "" ]];then
    75        CONTEXT=$OVERRIDE_CONTEXT
    76    fi
    77  
    78    # KUBECONFIG determines the file we write to, but it may not exist yet
    79    OLD_IFS=$IFS
    80    IFS=':'
    81    for cfg in ${KUBECONFIG} ; do
    82      if [[ ! -e "${cfg}" ]]; then
    83        mkdir -p "$(dirname "${cfg}")"
    84        touch "${cfg}"
    85      fi
    86    done
    87    IFS=$OLD_IFS
    88  
    89    local cluster_args=(
    90        "--server=${KUBE_SERVER:-https://${KUBE_MASTER_IP}}"
    91    )
    92    if [[ -z "${CA_CERT:-}" ]]; then
    93      cluster_args+=("--insecure-skip-tls-verify=true")
    94    else
    95      cluster_args+=(
    96        "--certificate-authority=${CA_CERT}"
    97        "--embed-certs=true"
    98      )
    99    fi
   100  
   101    local user_args=()
   102    if [[ -n "${KUBE_BEARER_TOKEN:-}" ]]; then
   103      user_args+=(
   104       "--token=${KUBE_BEARER_TOKEN}"
   105      )
   106    elif [[ -n "${KUBE_USER:-}" && -n "${KUBE_PASSWORD:-}" ]]; then
   107      user_args+=(
   108       "--username=${KUBE_USER}"
   109       "--password=${KUBE_PASSWORD}"
   110      )
   111    fi
   112    if [[ -n "${KUBE_CERT:-}" && -n "${KUBE_KEY:-}" ]]; then
   113      user_args+=(
   114       "--client-certificate=${KUBE_CERT}"
   115       "--client-key=${KUBE_KEY}"
   116       "--embed-certs=true"
   117      )
   118    fi
   119  
   120    KUBECONFIG="${KUBECONFIG}" "${kubectl}" config set-cluster "${CONTEXT}" "${cluster_args[@]}"
   121    if [[ -n "${user_args[*]:-}" ]]; then
   122      KUBECONFIG="${KUBECONFIG}" "${kubectl}" config set-credentials "${CONTEXT}" "${user_args[@]}"
   123    fi
   124    KUBECONFIG="${KUBECONFIG}" "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="${CONTEXT}"
   125  
   126    if [[ "${SECONDARY_KUBECONFIG}" != "true" ]];then
   127        KUBECONFIG="${KUBECONFIG}" "${kubectl}" config use-context "${CONTEXT}"  --cluster="${CONTEXT}"
   128    fi
   129  
   130    # If we have a bearer token, also create a credential entry with basic auth
   131    # so that it is easy to discover the basic auth password for your cluster
   132    # to use in a web browser.
   133    if [[ -n "${KUBE_BEARER_TOKEN:-}" && -n "${KUBE_USER:-}" && -n "${KUBE_PASSWORD:-}" ]]; then
   134      KUBECONFIG="${KUBECONFIG}" "${kubectl}" config set-credentials "${CONTEXT}-basic-auth" "--username=${KUBE_USER}" "--password=${KUBE_PASSWORD}"
   135    fi
   136  
   137     echo "Wrote config for ${CONTEXT} to ${KUBECONFIG}"
   138  }
   139  
   140  # Clear kubeconfig data for a context
   141  # Assumed vars:
   142  #   KUBECONFIG
   143  #   CONTEXT
   144  #
   145  # To explicitly name the context being removed, use OVERRIDE_CONTEXT
   146  function clear-kubeconfig() {
   147    export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
   148    OVERRIDE_CONTEXT=${OVERRIDE_CONTEXT:-}
   149  
   150    if [[ "$OVERRIDE_CONTEXT" != "" ]];then
   151        CONTEXT=$OVERRIDE_CONTEXT
   152    fi
   153  
   154    local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
   155    # Unset the current-context before we delete it, as otherwise kubectl errors.
   156    local cc
   157    cc=$("${kubectl}" config view -o jsonpath='{.current-context}')
   158    if [[ "${cc}" == "${CONTEXT}" ]]; then
   159      "${kubectl}" config unset current-context
   160    fi
   161    "${kubectl}" config unset "clusters.${CONTEXT}"
   162    "${kubectl}" config unset "users.${CONTEXT}"
   163    "${kubectl}" config unset "users.${CONTEXT}-basic-auth"
   164    "${kubectl}" config unset "contexts.${CONTEXT}"
   165  
   166    echo "Cleared config for ${CONTEXT} from ${KUBECONFIG}"
   167  }
   168  
   169  # Gets username, password for the current-context in kubeconfig, if they exist.
   170  # Assumed vars:
   171  #   KUBECONFIG  # if unset, defaults to global
   172  #   KUBE_CONTEXT  # if unset, defaults to current-context
   173  #
   174  # Vars set:
   175  #   KUBE_USER
   176  #   KUBE_PASSWORD
   177  #
   178  # KUBE_USER,KUBE_PASSWORD will be empty if no current-context is set, or
   179  # the current-context user does not exist or contain basicauth entries.
   180  function get-kubeconfig-basicauth() {
   181    export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
   182  
   183    local cc
   184    cc=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.current-context}")
   185    if [[ -n "${KUBE_CONTEXT:-}" ]]; then
   186      cc="${KUBE_CONTEXT}"
   187    fi
   188    local user
   189    user=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.contexts[?(@.name == \"${cc}\")].context.user}")
   190    get-kubeconfig-user-basicauth "${user}"
   191  
   192    if [[ -z "${KUBE_USER:-}" || -z "${KUBE_PASSWORD:-}" ]]; then
   193      # kube-up stores username/password in a an additional kubeconfig section
   194      # suffixed with "-basic-auth". Cloudproviders like GKE store in directly
   195      # in the top level section along with the other credential information.
   196      # TODO: Handle this uniformly, either get rid of "basic-auth" or
   197      # consolidate its usage into a function across scripts in cluster/
   198      get-kubeconfig-user-basicauth "${user}-basic-auth"
   199    fi
   200  }
   201  
   202  # Sets KUBE_USER and KUBE_PASSWORD to the username and password specified in
   203  # the kubeconfig section corresponding to $1.
   204  #
   205  # Args:
   206  #   $1 kubeconfig section to look for basic auth (eg: user or user-basic-auth).
   207  # Assumed vars:
   208  #   KUBE_ROOT
   209  # Vars set:
   210  #   KUBE_USER
   211  #   KUBE_PASSWORD
   212  function get-kubeconfig-user-basicauth() {
   213    KUBE_USER=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.users[?(@.name == \"$1\")].user.username}")
   214    KUBE_PASSWORD=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.users[?(@.name == \"$1\")].user.password}")
   215  }
   216  
   217  # Generate basic auth user and password.
   218  
   219  # Vars set:
   220  #   KUBE_USER
   221  #   KUBE_PASSWORD
   222  function gen-kube-basicauth() {
   223      KUBE_USER='admin'
   224      KUBE_PASSWORD=$(python3 -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))')
   225  }
   226  
   227  # Get the bearer token for the current-context in kubeconfig if one exists.
   228  # Assumed vars:
   229  #   KUBECONFIG  # if unset, defaults to global
   230  #   KUBE_CONTEXT  # if unset, defaults to current-context
   231  #
   232  # Vars set:
   233  #   KUBE_BEARER_TOKEN
   234  #
   235  # KUBE_BEARER_TOKEN will be empty if no current-context is set, or the
   236  # current-context user does not exist or contain a bearer token entry.
   237  function get-kubeconfig-bearertoken() {
   238    export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
   239  
   240    local cc
   241    cc=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.current-context}")
   242    if [[ -n "${KUBE_CONTEXT:-}" ]]; then
   243      cc="${KUBE_CONTEXT}"
   244    fi
   245    local user
   246    user=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.contexts[?(@.name == \"${cc}\")].context.user}")
   247    KUBE_BEARER_TOKEN=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.users[?(@.name == \"${user}\")].user.token}")
   248  }
   249  
   250  # Generate bearer token.
   251  #
   252  # Vars set:
   253  #   KUBE_BEARER_TOKEN
   254  function gen-kube-bearertoken() {
   255      KUBE_BEARER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
   256  }
   257  
   258  function load-or-gen-kube-basicauth() {
   259    if [[ -n "${KUBE_CONTEXT:-}" ]]; then
   260      get-kubeconfig-basicauth
   261    fi
   262  
   263    if [[ -z "${KUBE_USER:-}" || -z "${KUBE_PASSWORD:-}" ]]; then
   264      gen-kube-basicauth
   265    fi
   266  
   267    # Make sure they don't contain any funny characters.
   268    if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
   269      echo "Bad KUBE_USER string."
   270      exit 1
   271    fi
   272    if ! [[ "${KUBE_PASSWORD}" =~ ^[-._@#%/a-zA-Z0-9]+$ ]]; then
   273      echo "Bad KUBE_PASSWORD string."
   274      exit 1
   275    fi
   276  }
   277  
   278  # Sets KUBE_VERSION variable to the proper version number (e.g. "v1.0.6",
   279  # "v1.2.0-alpha.1.881+376438b69c7612") or a version' publication of the form
   280  # <path>/<version> (e.g. "release/stable",' "ci/latest-1").
   281  #
   282  # See the docs on getting builds for more information about version
   283  # publication.
   284  #
   285  # Args:
   286  #   $1 version string from command line
   287  # Vars set and exported for external reference:
   288  #   KUBE_VERSION
   289  function set_binary_version() {
   290    if [[ "${1}" =~ "/" ]]; then
   291      KUBE_VERSION=$(curl -sL "https://dl.k8s.io/${1}.txt")
   292    else
   293      KUBE_VERSION=${1}
   294    fi
   295    export KUBE_VERSION
   296  }
   297  
   298  # Search for the specified tarball in the various known output locations,
   299  # echoing the location if found.
   300  #
   301  # Assumed vars:
   302  #   KUBE_ROOT
   303  #
   304  # Args:
   305  #   $1 name of tarball to search for
   306  function find-tar() {
   307    local -r tarball=$1
   308    locations=(
   309      "${KUBE_ROOT}/node/${tarball}"
   310      "${KUBE_ROOT}/server/${tarball}"
   311      "${KUBE_ROOT}/kubernetes/node/${tarball}"
   312      "${KUBE_ROOT}/kubernetes/server/${tarball}"    
   313      "${KUBE_ROOT}/_output/release-tars/${tarball}"
   314    )
   315    location=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )
   316  
   317    if [[ ! -f "${location}" ]]; then
   318      echo "!!! Cannot find ${tarball}" >&2
   319      exit 1
   320    fi
   321    echo "${location}"
   322  }
   323  
   324  # Verify and find the various tar files that we are going to use on the server.
   325  #
   326  # Assumed vars:
   327  #   KUBE_ROOT
   328  # Vars set and exported:
   329  #   NODE_BINARY_TAR
   330  #   SERVER_BINARY_TAR
   331  #   KUBE_MANIFESTS_TAR
   332  function find-release-tars() {
   333    # Use first item in KUBE_BUILD_PLATFORMS as server platform
   334    KUBE_BUILD_PLATFORMS=${KUBE_BUILD_PLATFORMS:-"linux/amd64"}
   335    SERVER_PLATFORM=$(cut -d' ' -f1 <<< "${KUBE_BUILD_PLATFORMS}")
   336    OS=$(cut -d'/' -f1 <<< "${SERVER_PLATFORM}")
   337    ARCH=$(cut -d'/' -f2 <<< "${SERVER_PLATFORM}")
   338    SERVER_BINARY_TAR=$(find-tar kubernetes-server-"${OS}"-"${ARCH}".tar.gz)
   339    if [[ -z "${SERVER_BINARY_TAR}" ]]; then
   340  	  exit 1
   341    fi
   342    export SERVER_BINARY_TAR
   343  
   344    local find_result
   345    if [[ "${NUM_WINDOWS_NODES}" -gt "0" ]]; then
   346      if NODE_BINARY_TAR=$(find-tar kubernetes-node-windows-"${ARCH}".tar.gz); then
   347        find_result=0
   348      else
   349        find_result=1
   350      fi
   351      export NODE_BINARY_TAR
   352    fi
   353  
   354    # This tarball is used by GCI, Ubuntu Trusty, and Container Linux.
   355    KUBE_MANIFESTS_TAR=
   356    if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "ubuntu" ]] || \
   357       [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "ubuntu" || "${NODE_OS_DISTRIBUTION:-}" == "custom" ]] ; then
   358      if KUBE_MANIFESTS_TAR=$(find-tar kubernetes-manifests.tar.gz); then
   359        find_result=0
   360      else
   361        find_result=1
   362      fi
   363      export KUBE_MANIFESTS_TAR
   364    fi
   365  
   366    # the function result is used in function `verify-release-tars`
   367    if [[ $find_result == 0 ]]; then
   368      return 0
   369    else
   370      return 1
   371    fi
   372  }
   373  
   374  # Run the cfssl command to generates certificate files for etcd service, the
   375  # certificate files will save in $1 directory.
   376  #
   377  # Optional vars:
   378  #   GEN_ETCD_CA_CERT (CA cert encode with base64 and ZIP compression)
   379  #   GEN_ETCD_CA_KEY (CA key encode with base64)
   380  #   ca_cert (require when GEN_ETCD_CA_CERT and GEN_ETCD_CA_KEY is set)
   381  #   ca_key (require when GEN_ETCD_CA_CERT and GEN_ETCD_CA_KEY is set)
   382  # If GEN_ETCD_CA_CERT or GEN_ETCD_CA_KEY is not specified, it will generates certs for CA.
   383  #
   384  # Args:
   385  #   $1 (the directory that certificate files to save)
   386  #   $2 (the ip of etcd member)
   387  #   $3 (the type of etcd certificates, must be one of client, server, peer)
   388  #   $4 (the prefix of the certificate filename, default is $3)
   389  function generate-etcd-cert() {
   390    local cert_dir=${1}
   391    local member_ip=${2}
   392    local type_cert=${3}
   393    local prefix=${4:-"${type_cert}"}
   394  
   395    local GEN_ETCD_CA_CERT=${GEN_ETCD_CA_CERT:-}
   396    local GEN_ETCD_CA_KEY=${GEN_ETCD_CA_KEY:-}
   397  
   398    mkdir -p "${cert_dir}"
   399    pushd "${cert_dir}"
   400  
   401    kube::util::ensure-cfssl .
   402  
   403    if [ ! -r "ca-config.json" ]; then
   404      cat >ca-config.json <<EOF
   405  {
   406      "signing": {
   407          "default": {
   408              "expiry": "43800h"
   409          },
   410          "profiles": {
   411              "server": {
   412                  "expiry": "43800h",
   413                  "usages": [
   414                      "signing",
   415                      "key encipherment",
   416                      "server auth",
   417                      "client auth"
   418                  ]
   419              },
   420              "client": {
   421                  "expiry": "43800h",
   422                  "usages": [
   423                      "signing",
   424                      "key encipherment",
   425                      "client auth"
   426                  ]
   427              },
   428              "peer": {
   429                  "expiry": "43800h",
   430                  "usages": [
   431                      "signing",
   432                      "key encipherment",
   433                      "server auth",
   434                      "client auth"
   435                  ]
   436              }
   437          }
   438      }
   439  }
   440  EOF
   441    fi
   442  
   443    if [ ! -r "ca-csr.json" ]; then
   444      cat >ca-csr.json <<EOF
   445  {
   446      "CN": "Kubernetes",
   447      "key": {
   448          "algo": "ecdsa",
   449          "size": 256
   450      },
   451      "names": [
   452          {
   453              "C": "US",
   454              "L": "CA",
   455              "O": "kubernetes.io"
   456          }
   457      ]
   458  }
   459  EOF
   460    fi
   461  
   462    if [[ -n "${GEN_ETCD_CA_CERT}" && -n "${GEN_ETCD_CA_KEY}" ]]; then
   463      # ca_cert and ca_key are optional external vars supplied in cluster/gce/util.sh,
   464      # so it's ok to disable shellcheck here
   465      # shellcheck disable=SC2154
   466      echo "${ca_cert}" | base64 --decode | gunzip > ca.pem
   467      # shellcheck disable=SC2154
   468      echo "${ca_key}" | base64 --decode > ca-key.pem
   469    fi
   470  
   471    if [[ ! -r "ca.pem" || ! -r "ca-key.pem" ]]; then
   472      ${CFSSL_BIN} gencert -initca ca-csr.json | ${CFSSLJSON_BIN} -bare ca -
   473    fi
   474  
   475    case "${type_cert}" in
   476      client)
   477        echo "Generate client certificates..."
   478        echo '{"CN":"client","hosts":["*"],"key":{"algo":"ecdsa","size":256}}' \
   479         | ${CFSSL_BIN} gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client - \
   480         | ${CFSSLJSON_BIN} -bare "${prefix}"
   481        ;;
   482      server)
   483        echo "Generate server certificates..."
   484        echo '{"CN":"'"${member_ip}"'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \
   485         | ${CFSSL_BIN} gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server -hostname="${member_ip},127.0.0.1" - \
   486         | ${CFSSLJSON_BIN} -bare "${prefix}"
   487        ;;
   488      peer)
   489        echo "Generate peer certificates..."
   490        echo '{"CN":"'"${member_ip}"'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \
   491         | ${CFSSL_BIN} gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer -hostname="${member_ip},127.0.0.1" - \
   492         | ${CFSSLJSON_BIN} -bare "${prefix}"
   493        ;;
   494      *)
   495        echo "Unknow, unsupported etcd certs type: ${type_cert}" >&2
   496        echo "Supported type: client, server, peer" >&2
   497        exit 2
   498    esac
   499  
   500    # the popd will access `directory stack`, no `real` parameters is actually needed
   501    # shellcheck disable=SC2119
   502    popd
   503  }
   504  
   505  # Check whether required binaries exist, prompting to download
   506  # if missing.
   507  # If KUBERNETES_SKIP_CONFIRM is set to y, we'll automatically download binaries
   508  # without prompting.
   509  function verify-kube-binaries() {
   510    if ! "${KUBE_ROOT}/cluster/kubectl.sh" version --client >&/dev/null; then
   511      echo "!!! kubectl appears to be broken or missing"
   512      download-release-binaries
   513    fi
   514  }
   515  
   516  # Check whether required release artifacts exist, prompting to download
   517  # if missing.
   518  # If KUBERNETES_SKIP_CONFIRM is set to y, we'll automatically download binaries
   519  # without prompting.
   520  function verify-release-tars() {
   521    if ! find-release-tars; then
   522      download-release-binaries
   523    fi
   524  }
   525  
   526  # Download release artifacts.
   527  function download-release-binaries() {
   528    get_binaries_script="${KUBE_ROOT}/cluster/get-kube-binaries.sh"
   529    local resp="y"
   530    if [[ ! "${KUBERNETES_SKIP_CONFIRM:-n}" =~ ^[yY]$ ]]; then
   531      echo "Required release artifacts appear to be missing. Do you wish to download them? [Y/n]"
   532      read -r resp
   533    fi
   534    if [[ "${resp}" =~ ^[nN]$ ]]; then
   535      echo "You must download release artifacts to continue. You can use "
   536      echo "  ${get_binaries_script}"
   537      echo "to do this for your automatically."
   538      exit 1
   539    fi
   540    "${get_binaries_script}"
   541  }
   542  
   543  # Run pushd without stack output
   544  function pushd() {
   545    command pushd "$@" > /dev/null
   546  }
   547  
   548  # Run popd without stack output
   549  # the popd will access `directory stack`, no `real` parameters is actually needed
   550  # shellcheck disable=SC2120
   551  function popd() {
   552    command popd "$@" > /dev/null
   553  }