k8s.io/kubernetes@v1.29.3/test/kubemark/start-kubemark.sh (about) 1 #!/usr/bin/env bash 2 3 # Copyright 2015 The Kubernetes Authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 # Script that creates a Kubemark cluster for any given cloud provider. 18 19 set -o errexit 20 set -o nounset 21 set -o pipefail 22 23 TMP_ROOT="$(dirname "${BASH_SOURCE[@]}")/../.." 24 KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}") 25 26 source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh" 27 source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" 28 source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh" 29 source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" 30 31 if [[ -f "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh" ]] ; then 32 source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh" 33 fi 34 35 source "${KUBE_ROOT}/cluster/kubemark/util.sh" 36 37 KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" 38 KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" 39 RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" 40 LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark" 41 INTERNAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark" 42 43 # Generate a random 6-digit alphanumeric tag for the kubemark image. 44 # Used to uniquify image builds across different invocations of this script. 45 KUBEMARK_IMAGE_TAG=$(head /dev/urandom | tr -dc 'a-z0-9' | fold -w 6 | head -n 1) 46 47 # Create a docker image for hollow-node and upload it to the appropriate docker registry. 48 function create-and-upload-hollow-node-image { 49 authenticate-docker 50 KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-${CONTAINER_REGISTRY}/${PROJECT}}" 51 # Build+push the image through makefile. 52 build_cmd=("make" "${KUBEMARK_IMAGE_MAKE_TARGET}") 53 MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark" 54 KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)" 55 if [[ -z "${KUBEMARK_BIN}" ]]; then 56 echo 'Cannot find cmd/kubemark binary' 57 exit 1 58 fi 59 echo "Copying kubemark binary to ${MAKE_DIR}" 60 cp "${KUBEMARK_BIN}" "${MAKE_DIR}" 61 CURR_DIR=$(pwd) 62 cd "${MAKE_DIR}" 63 REGISTRY=${KUBEMARK_IMAGE_REGISTRY} IMAGE_TAG=${KUBEMARK_IMAGE_TAG} run-cmd-with-retries "${build_cmd[@]}" 64 rm kubemark 65 cd "$CURR_DIR" 66 echo "Created and uploaded the kubemark hollow-node image to docker registry." 67 # Cleanup the kubemark image after the script exits. 68 if [[ "${CLEANUP_KUBEMARK_IMAGE:-}" == "true" ]]; then 69 trap delete-kubemark-image EXIT 70 fi 71 } 72 73 function delete-kubemark-image { 74 delete-image "${KUBEMARK_IMAGE_REGISTRY}/kubemark:${KUBEMARK_IMAGE_TAG}" 75 } 76 77 # Generate secret and configMap for the hollow-node pods to work, prepare 78 # manifests of the hollow-node and heapster replication controllers from 79 # templates, and finally create these resources through kubectl. 80 function create-kube-hollow-node-resources { 81 # Create kubemark namespace. 82 "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" 83 84 # Create configmap for configuring hollow- kubelet, proxy and npd. 85 "${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \ 86 --from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json" 87 88 # Create secret for passing kubeconfigs to kubelet, kubeproxy and npd. 89 # It's bad that all component shares the same kubeconfig. 90 # TODO(https://github.com/kubernetes/kubernetes/issues/79883): Migrate all components to separate credentials. 91 "${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \ 92 --from-file=kubelet.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ 93 --from-file=kubeproxy.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ 94 --from-file=npd.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ 95 --from-file=heapster.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ 96 --from-file=cluster_autoscaler.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ 97 --from-file=dns.kubeconfig="${HOLLOWNODE_KUBECONFIG}" 98 99 # Create addon pods. 100 # Heapster. 101 mkdir -p "${RESOURCE_DIRECTORY}/addons" 102 sed "s@{{MASTER_IP}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json" 103 metrics_mem_per_node=4 104 metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES)) 105 sed -i'' -e "s@{{METRICS_MEM}}@${metrics_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json" 106 metrics_cpu_per_node_numerator=${NUM_NODES} 107 metrics_cpu_per_node_denominator=2 108 metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator)) 109 sed -i'' -e "s@{{METRICS_CPU}}@${metrics_cpu}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json" 110 eventer_mem_per_node=500 111 eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES)) 112 sed -i'' -e "s@{{EVENTER_MEM}}@${eventer_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json" 113 114 # Cluster Autoscaler. 115 if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-}" == "true" ]]; then 116 echo "Setting up Cluster Autoscaler" 117 KUBEMARK_AUTOSCALER_MIG_NAME="${KUBEMARK_AUTOSCALER_MIG_NAME:-${NODE_INSTANCE_PREFIX}-group}" 118 KUBEMARK_AUTOSCALER_MIN_NODES="${KUBEMARK_AUTOSCALER_MIN_NODES:-0}" 119 KUBEMARK_AUTOSCALER_MAX_NODES="${KUBEMARK_AUTOSCALER_MAX_NODES:-10}" 120 NUM_NODES=${KUBEMARK_AUTOSCALER_MAX_NODES} 121 echo "Setting maximum cluster size to ${NUM_NODES}." 122 KUBEMARK_MIG_CONFIG="autoscaling.k8s.io/nodegroup: ${KUBEMARK_AUTOSCALER_MIG_NAME}" 123 sed "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/cluster-autoscaler_template.json" > "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" 124 sed -i'' -e "s@{{kubemark_autoscaler_mig_name}}@${KUBEMARK_AUTOSCALER_MIG_NAME}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" 125 sed -i'' -e "s@{{kubemark_autoscaler_min_nodes}}@${KUBEMARK_AUTOSCALER_MIN_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" 126 sed -i'' -e "s@{{kubemark_autoscaler_max_nodes}}@${KUBEMARK_AUTOSCALER_MAX_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" 127 fi 128 129 # Kube DNS. 130 if [[ "${ENABLE_KUBEMARK_KUBE_DNS:-}" == "true" ]]; then 131 echo "Setting up kube-dns" 132 sed "s@{{dns_domain}}@${KUBE_DNS_DOMAIN}@g" "${RESOURCE_DIRECTORY}/kube_dns_template.yaml" > "${RESOURCE_DIRECTORY}/addons/kube_dns.yaml" 133 fi 134 135 "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" 136 137 # Create the replication controller for hollow-nodes. 138 # We allow to override the NUM_REPLICAS when running Cluster Autoscaler. 139 NUM_REPLICAS=${NUM_REPLICAS:-${NUM_NODES}} 140 sed "s@{{numreplicas}}@${NUM_REPLICAS}@g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml" 141 proxy_cpu=20 142 if [ "${NUM_NODES}" -gt 1000 ]; then 143 proxy_cpu=50 144 fi 145 proxy_cpu=${KUBEMARK_HOLLOW_PROXY_MILLICPU:-$proxy_cpu} 146 proxy_mem_per_node=${KUBEMARK_HOLLOW_PROXY_MEM_PER_NODE_KB:-50} 147 proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES)) 148 hollow_node_labels=${HOLLOW_NODE_LABELS:-$(calculate-node-labels)} 149 hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done") 150 hollow_kubelet_params=${hollow_kubelet_params%?} 151 hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done") 152 hollow_proxy_params=${hollow_proxy_params%?} 153 154 sed -i'' -e "s@{{hollow_kubelet_millicpu}}@${KUBEMARK_HOLLOW_KUBELET_MILLICPU:-40}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 155 sed -i'' -e "s@{{hollow_kubelet_mem_Ki}}@${KUBEMARK_HOLLOW_KUBELET_MEM_KB:-$((100*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 156 sed -i'' -e "s@{{hollow_proxy_millicpu}}@${proxy_cpu}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 157 sed -i'' -e "s@{{hollow_proxy_mem_Ki}}@${proxy_mem}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 158 sed -i'' -e "s@{{npd_millicpu}}@${KUBEMARK_NPD_MILLICPU:-20}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 159 sed -i'' -e "s@{{npd_mem_Ki}}@${KUBEMARK_NPD_MEM_KB:-$((20*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 160 sed -i'' -e "s@{{kubemark_image_registry}}@${KUBEMARK_IMAGE_REGISTRY}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 161 sed -i'' -e "s@{{kubemark_image_tag}}@${KUBEMARK_IMAGE_TAG}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 162 sed -i'' -e "s@{{master_ip}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 163 sed -i'' -e "s@{{hollow_node_labels}}@${hollow_node_labels}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 164 sed -i'' -e "s@{{hollow_kubelet_params}}@${hollow_kubelet_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 165 sed -i'' -e "s@{{hollow_proxy_params}}@${hollow_proxy_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 166 sed -i'' -e "s@{{kubemark_mig_config}}@${KUBEMARK_MIG_CONFIG:-}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" 167 "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark" 168 169 echo "Created secrets, configMaps, replication-controllers required for hollow-nodes." 170 } 171 172 # Wait until all hollow-nodes are running or there is a timeout. 173 function wait-for-hollow-nodes-to-run-or-timeout { 174 echo -n "Waiting for all hollow-nodes to become Running" 175 start=$(date +%s) 176 nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true 177 ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) 178 179 until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do 180 echo -n "." 181 sleep 1 182 now=$(date +%s) 183 # Fail it if it already took more than 30 minutes. 184 if [ $((now - start)) -gt 1800 ]; then 185 echo "" 186 "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" describe node 187 # shellcheck disable=SC2154 # Color defined in sourced script 188 echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}" 189 # Try listing nodes again - if it fails it means that API server is not responding 190 if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then 191 echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}." 192 else 193 echo "Got error while trying to list hollow-nodes. Probably API server is down." 194 fi 195 pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true 196 running=$(($(echo "${pods}" | grep -c "Running"))) 197 echo "${running} hollow-nodes are reported as 'Running'" 198 not_running=$(($(echo "${pods}" | grep -vc "Running") - 1)) 199 echo "${not_running} hollow-nodes are reported as NOT 'Running'" 200 echo "${pods}" | grep -v Running 201 exit 1 202 fi 203 nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true 204 ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) 205 done 206 # shellcheck disable=SC2154 # Color defined in sourced script 207 echo -e "${color_green} Done!${color_norm}" 208 } 209 210 ############################### Main Function ######################################## 211 212 # Setup for hollow-nodes. 213 function start-hollow-nodes { 214 # shellcheck disable=SC2154 # Color defined in sourced script 215 echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}" 216 create-and-upload-hollow-node-image 217 create-kube-hollow-node-resources 218 wait-for-hollow-nodes-to-run-or-timeout 219 } 220 221 # Annotates the node objects in the kubemark cluster to make their size 222 # similar to regular nodes. 223 # TODO(#90833): Replace this with image preloading from ClusterLoader to better 224 # reflect the reality in kubemark tests. 225 function resize-node-objects { 226 if [[ -z "$KUBEMARK_NODE_OBJECT_SIZE_BYTES" ]]; then 227 return 0 228 fi 229 230 annotation_size_bytes="${KUBEMARK_NODE_OBJECT_SIZE_BYTES}" 231 echo "Annotating node objects with ${annotation_size_bytes} byte label" 232 label=$( (< /dev/urandom tr -dc 'a-zA-Z0-9' | fold -w "$annotation_size_bytes"; true) | head -n 1) 233 "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get nodes -o name \ 234 | xargs -P50 -r -I% "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" annotate --overwrite % label="$label" 235 echo "Annotating node objects completed" 236 } 237 238 239 detect-project &> /dev/null 240 create-kubemark-master 241 242 if [ -f "${INTERNAL_KUBECONFIG}" ]; then 243 HOLLOWNODE_KUBECONFIG="${INTERNAL_KUBECONFIG}" 244 else 245 HOLLOWNODE_KUBECONFIG="${LOCAL_KUBECONFIG}" 246 fi 247 248 MASTER_IP=$(grep server "${HOLLOWNODE_KUBECONFIG}" | awk -F "/" '{print $3}') 249 250 start-hollow-nodes 251 resize-node-objects 252 253 echo "" 254 echo "Master IP: ${MASTER_IP}" 255 echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}"