k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/cluster/addons/addon-manager/kube-addons.sh (about)

     1  #!/usr/bin/env bash
     2  
     3  # Copyright 2014 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  # LIMITATIONS
    18  # 1. Exit code is probably not always correct.
    19  # 2. There are no unittests.
    20  # 3. Will not work if the total length of paths to addons is greater than
    21  #    bash can handle. Probably it is not a problem: ARG_MAX=2097152 on GCE.
    22  
    23  # cosmetic improvements to be done
    24  # 1. Improve the log function; add timestamp, file name, etc.
    25  # 2. Logging doesn't work from files that print things out.
    26  # 3. Kubectl prints the output to stderr (the output should be captured and then
    27  #    logged)
    28  
    29  KUBECTL=${KUBECTL_BIN:-/usr/local/bin/kubectl}
    30  KUBECTL_OPTS=${KUBECTL_OPTS:-}
    31  # KUBECTL_PRUNE_WHITELIST is a list of resources whitelisted by default.
    32  # This is currently the same with the default in:
    33  # https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/apply/prune.go.
    34  # To override the default list with other values, set
    35  # KUBECTL_PRUNE_WHITELIST_OVERRIDE environment variable to space-separated
    36  # names of resources to whitelist.
    37  if [ -z "${KUBECTL_PRUNE_WHITELIST_OVERRIDE:-}" ]; then
    38    KUBECTL_PRUNE_WHITELIST=(
    39      core/v1/ConfigMap
    40      core/v1/Endpoints
    41      core/v1/Namespace
    42      core/v1/PersistentVolumeClaim
    43      core/v1/PersistentVolume
    44      core/v1/Pod
    45      core/v1/ReplicationController
    46      core/v1/Secret
    47      core/v1/Service
    48      batch/v1/Job
    49      batch/v1/CronJob
    50      apps/v1/DaemonSet
    51      apps/v1/Deployment
    52      apps/v1/ReplicaSet
    53      apps/v1/StatefulSet
    54      networking.k8s.io/v1/Ingress
    55    )
    56  else
    57    read -ra KUBECTL_PRUNE_WHITELIST <<< "${KUBECTL_PRUNE_WHITELIST_OVERRIDE}"
    58  fi
    59  
    60  # This variable is unused in this file, but not in those that source it.
    61  # shellcheck disable=SC2034
    62  ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-60}
    63  ADDON_PATH=${ADDON_PATH:-/etc/kubernetes/addons}
    64  
    65  # This variable is unused in this file, but not in those that source it.
    66  # shellcheck disable=SC2034
    67  SYSTEM_NAMESPACE=kube-system
    68  
    69  # Addons could use this label with two modes:
    70  # - ADDON_MANAGER_LABEL=Reconcile
    71  # - ADDON_MANAGER_LABEL=EnsureExists
    72  ADDON_MANAGER_LABEL="addonmanager.kubernetes.io/mode"
    73  # This label is deprecated (only for Addon Manager). In future release
    74  # addon-manager may not respect it anymore. Addons with
    75  # CLUSTER_SERVICE_LABEL=true and without ADDON_MANAGER_LABEL=EnsureExists
    76  # will be reconciled for now.
    77  CLUSTER_SERVICE_LABEL="kubernetes.io/cluster-service"
    78  
    79  # Whether only one addon manager should be running in a multi-master setup.
    80  # Disabling this flag will force all addon managers to assume they are the
    81  # leaders.
    82  ADDON_MANAGER_LEADER_ELECTION=${ADDON_MANAGER_LEADER_ELECTION:-true}
    83  
    84  # Remember that you can't log from functions that print some output (because
    85  # logs are also printed on stdout).
    86  # $1 level
    87  # $2 message
    88  function log() {
    89    # manage log levels manually here
    90  
    91    # add the timestamp if you find it useful
    92    case $1 in
    93      DB3 )
    94  #        echo "$1: $2"
    95          ;;
    96      DB2 )
    97  #        echo "$1: $2"
    98          ;;
    99      DBG )
   100  #        echo "$1: $2"
   101          ;;
   102      INFO )
   103          echo "$1: $2"
   104          ;;
   105      WRN )
   106          echo "$1: $2"
   107          ;;
   108      ERR )
   109          echo "$1: $2"
   110          ;;
   111      * )
   112          echo "INVALID_LOG_LEVEL $1: $2"
   113          ;;
   114    esac
   115  }
   116  
   117  # Generate kubectl prune-allowlist flags from provided resource list.
   118  function generate_prune_allowlist_flags() {
   119    local -r resources=( "$@" )
   120    for resource in "${resources[@]}"; do
   121      # Check if $resource isn't composed just of whitespaces by replacing ' '
   122      # with '' and checking whether the resulting string is not empty.
   123      if [[ -n "${resource// /}" ]]; then
   124        printf "%s" "--prune-allowlist ${resource} "
   125      fi
   126    done
   127  }
   128  
   129  # KUBECTL_EXTRA_PRUNE_WHITELIST is a list of extra allowed resources
   130  # besides the default ones.
   131  extra_prune_allowlist=
   132  if [ -n "${KUBECTL_EXTRA_PRUNE_WHITELIST:-}" ]; then
   133    read -ra extra_prune_allowlist <<< "${KUBECTL_EXTRA_PRUNE_WHITELIST}"
   134  fi
   135  prune_allowlist=( "${KUBECTL_PRUNE_WHITELIST[@]}"  "${extra_prune_allowlist[@]}" )
   136  prune_allowlist_flags=$(generate_prune_allowlist_flags "${prune_allowlist[@]}")
   137  
   138  log INFO "== Generated kubectl prune allowlist flags: $prune_allowlist_flags =="
   139  
   140  # $1 filename of addon to start.
   141  # $2 count of tries to start the addon.
   142  # $3 delay in seconds between two consecutive tries
   143  # $4 namespace
   144  function start_addon() {
   145    local -r addon_filename=$1;
   146    local -r tries=$2;
   147    local -r delay=$3;
   148    local -r namespace=$4
   149  
   150    create_resource_from_string "$(cat "${addon_filename}")" "${tries}" "${delay}" "${addon_filename}" "${namespace}"
   151  }
   152  
   153  # $1 string with json or yaml.
   154  # $2 count of tries to start the addon.
   155  # $3 delay in seconds between two consecutive tries
   156  # $4 name of this object to use when logging about it.
   157  # $5 namespace for this object
   158  function create_resource_from_string() {
   159    local -r config_string=$1;
   160    local tries=$2;
   161    local -r delay=$3;
   162    local -r config_name=$4;
   163    local -r namespace=$5;
   164    while [ "${tries}" -gt 0 ]; do
   165      reconcile_resource_from_string "${config_string}" "${config_name}" "${namespace}" && \
   166        ensure_resource_from_string "${config_string}" "${config_name}" "${namespace}" && \
   167        return 0;
   168      (( tries-- ))
   169      log WRN "== Failed to start ${config_name} in namespace ${namespace} at $(date -Is). ${tries} tries remaining. =="
   170      sleep "${delay}";
   171    done
   172    return 1;
   173  }
   174  
   175  # Creates resources with addon mode Reconcile for create_resource_from_string.
   176  # Does not perform pruning.
   177  # $1 string with json or yaml.
   178  # $2 name of this object for logging
   179  # $3 namespace for the object
   180  function reconcile_resource_from_string() {
   181    local -r config_string=$1;
   182    local -r config_name=$2;
   183    local -r namespace=$3;
   184  
   185    # kubectl_output must be declared ahead of time to allow capturing kubectl's exit code and not local's exit code.
   186    local kubectl_output;
   187    # shellcheck disable=SC2086
   188    # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
   189    kubectl_output=$(echo "${config_string}" | ${KUBECTL} ${KUBECTL_OPTS} apply -f - \
   190      --namespace="${namespace}" -l ${ADDON_MANAGER_LABEL}=Reconcile 2>&1) && \
   191        log INFO "== Successfully reconciled ${config_name} in namespace ${namespace} at $(date -Is)" && \
   192        return 0;
   193    if echo "${kubectl_output}" | grep --silent "no objects"; then
   194      # Nothing to do.
   195      return 0;
   196    fi
   197    echo "${kubectl_output}" # for visibility of errors
   198    return 1;
   199  }
   200  
   201  # Creates resources with addon mode EnsureExists for create_resource_from_string.
   202  # Does not perform pruning.
   203  # $1 string with json or yaml.
   204  # $2 name of this object for logging
   205  # $3 namespace for the object
   206  function ensure_resource_from_string() {
   207    local -r config_string=$1;
   208    local -r config_name=$2;
   209    local -r namespace=$3;
   210  
   211    # Resources that are set to the addon mode EnsureExists should not be overwritten if they already exist.
   212    local kubectl_output;
   213    # shellcheck disable=SC2086
   214    # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
   215    kubectl_output=$(echo "${config_string}" | ${KUBECTL} ${KUBECTL_OPTS} create -f - \
   216      --namespace="${namespace}" -l ${ADDON_MANAGER_LABEL}=EnsureExists 2>&1) && \
   217        log INFO "== Successfully started ${config_name} in namespace ${namespace} at $(date -Is)" && \
   218        return 0;
   219    # Detect an already exists failure for creating EnsureExists resources.
   220    # All other errors should result in a retry.
   221    if echo "${kubectl_output}" | grep --silent "AlreadyExists"; then
   222      log INFO "== Skipping start ${config_name} in namespace ${namespace}, already exists at $(date -Is)"
   223      return 0;
   224    elif echo "${kubectl_output}" | grep --silent "no objects"; then
   225      # Nothing to do.
   226      return 0;
   227    fi
   228    echo "${kubectl_output}" # for visibility of errors
   229    return 1;
   230  }
   231  
   232  function reconcile_addons() {
   233    # TODO: Remove the first command in future release.
   234    # Adding this for backward compatibility. Old addons have CLUSTER_SERVICE_LABEL=true and don't have
   235    # ADDON_MANAGER_LABEL=EnsureExists will still be reconciled.
   236    # Filter out `configured` message to not noisily log.
   237    # `created`, `pruned` and errors will be logged.
   238    log INFO "== Reconciling with deprecated label =="
   239    # shellcheck disable=SC2086
   240    # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
   241    ${KUBECTL} ${KUBECTL_OPTS} apply -f ${ADDON_PATH} \
   242      -l ${CLUSTER_SERVICE_LABEL}=true,${ADDON_MANAGER_LABEL}!=EnsureExists \
   243      --prune=true ${prune_allowlist_flags} --recursive | grep -v configured
   244  
   245    log INFO "== Reconciling with addon-manager label =="
   246    # shellcheck disable=SC2086
   247    # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
   248    ${KUBECTL} ${KUBECTL_OPTS} apply -f ${ADDON_PATH} \
   249      -l ${CLUSTER_SERVICE_LABEL}!=true,${ADDON_MANAGER_LABEL}=Reconcile \
   250      --prune=true ${prune_allowlist_flags} --recursive | grep -v configured
   251  
   252    log INFO "== Kubernetes addon reconcile completed at $(date -Is) =="
   253  }
   254  
   255  function ensure_addons() {
   256    # Create objects already exist should fail.
   257    # Filter out `AlreadyExists` message to not noisily log.
   258    # shellcheck disable=SC2086
   259    # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
   260    ${KUBECTL} ${KUBECTL_OPTS} create -f ${ADDON_PATH} \
   261      -l ${ADDON_MANAGER_LABEL}=EnsureExists --recursive 2>&1 | grep -v AlreadyExists
   262  
   263    log INFO "== Kubernetes addon ensure completed at $(date -Is) =="
   264  }
   265  
   266  function is_leader() {
   267    # In multi-master setup, only one addon manager should be running. We use
   268    # existing leader election in kube-controller-manager instead of implementing
   269    # a separate mechanism here.
   270    if ! $ADDON_MANAGER_LEADER_ELECTION; then
   271      log INFO "Leader election disabled."
   272      return 0;
   273    fi
   274    # shellcheck disable=SC2086
   275    # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
   276    KUBE_CONTROLLER_MANAGER_LEADER=$(${KUBECTL} ${KUBECTL_OPTS} -n kube-system get leases.v1.coordination.k8s.io kube-controller-manager -o "jsonpath={.spec.holderIdentity}")
   277  
   278    case "${KUBE_CONTROLLER_MANAGER_LEADER}" in
   279    "")
   280      log ERR "No leader election info found."
   281      return 1
   282      ;;
   283  
   284    "${HOSTNAME}"_*)
   285      log INFO "Leader is $KUBE_CONTROLLER_MANAGER_LEADER"
   286      return 0
   287      ;;
   288  
   289    *)
   290      log INFO "Leader is $KUBE_CONTROLLER_MANAGER_LEADER, not ${HOSTNAME}_*"
   291      return 1
   292      ;;
   293    esac
   294  }