k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/cluster/gce/gci/configure-kubeapiserver.sh (about) 1 #!/usr/bin/env bash 2 # Copyright 2016 The Kubernetes Authors. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 17 # Configures etcd related flags of kube-apiserver. 18 function configure-etcd-params { 19 local -n params_ref=$1 20 21 if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then 22 params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://127.0.0.1:2379}" 23 params_ref+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}" 24 params_ref+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}" 25 params_ref+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}" 26 elif [[ -z "${ETCD_APISERVER_CA_KEY:-}" && -z "${ETCD_APISERVER_CA_CERT:-}" && -z "${ETCD_APISERVER_SERVER_KEY:-}" && -z "${ETCD_APISERVER_SERVER_CERT:-}" && -z "${ETCD_APISERVER_CLIENT_KEY:-}" && -z "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then 27 params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}" 28 echo "WARNING: ALL of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver is not enabled." 29 else 30 echo "ERROR: Some of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver cannot be enabled. Please provide all mTLS credential." 31 exit 1 32 fi 33 34 if [[ -z "${ETCD_SERVERS:-}" ]]; then 35 params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://127.0.0.1:4002}" 36 elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then 37 params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}" 38 fi 39 40 if [[ -n "${STORAGE_BACKEND:-}" ]]; then 41 params_ref+=" --storage-backend=${STORAGE_BACKEND}" 42 fi 43 44 if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then 45 params_ref+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" 46 fi 47 48 if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then 49 params_ref+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s" 50 fi 51 } 52 53 # Starts kubernetes apiserver. 54 # It prepares the log file, loads the docker image, calculates variables, sets them 55 # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. 56 # 57 # Assumed vars (which are calculated in function compute-master-manifest-variables) 58 # CLOUD_CONFIG_OPT 59 # CLOUD_CONFIG_VOLUME 60 # CLOUD_CONFIG_MOUNT 61 # DOCKER_REGISTRY 62 # INSECURE_PORT_MAPPING 63 function start-kube-apiserver { 64 echo "Start kubernetes api-server" 65 prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}" "${KUBE_API_SERVER_RUNASUSER:-0}" 66 prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}" "${KUBE_API_SERVER_RUNASUSER:-0}" 67 68 # Calculate variables and assemble the command line. 69 local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" 70 params+=" --allow-privileged=true" 71 params+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-external}" 72 params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}" 73 74 # params is passed by reference, so no "$" 75 configure-etcd-params params 76 77 params+=" --secure-port=443" 78 params+=" --tls-cert-file=${APISERVER_SERVER_CERT_PATH}" 79 params+=" --tls-private-key-file=${APISERVER_SERVER_KEY_PATH}" 80 if [[ -n "${OLD_MASTER_IP:-}" ]]; then 81 local old_ips="${OLD_MASTER_IP}" 82 if [[ -n "${OLD_LOAD_BALANCER_IP:-}" ]]; then 83 old_ips+=",${OLD_LOAD_BALANCER_IP}" 84 fi 85 if [[ -n "${OLD_PRIVATE_VIP:-}" ]]; then 86 old_ips+=",${OLD_PRIVATE_VIP}" 87 fi 88 params+=" --tls-sni-cert-key=${OLD_MASTER_CERT_PATH},${OLD_MASTER_KEY_PATH}:${old_ips}" 89 fi 90 if [[ -n "${TLS_CIPHER_SUITES:-}" ]]; then 91 params+=" --tls-cipher-suites=${TLS_CIPHER_SUITES}" 92 fi 93 if [[ -e "${KUBE_HOME}/bin/gke-internal-configure-helper.sh" ]]; then 94 params+=" $(gke-kube-apiserver-internal-sni-param)" 95 fi 96 params+=" --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname" 97 if [[ -s "${REQUESTHEADER_CA_CERT_PATH:-}" ]]; then 98 params+=" --requestheader-client-ca-file=${REQUESTHEADER_CA_CERT_PATH}" 99 params+=" --requestheader-allowed-names=aggregator" 100 params+=" --requestheader-extra-headers-prefix=X-Remote-Extra-" 101 params+=" --requestheader-group-headers=X-Remote-Group" 102 params+=" --requestheader-username-headers=X-Remote-User" 103 params+=" --proxy-client-cert-file=${PROXY_CLIENT_CERT_PATH}" 104 params+=" --proxy-client-key-file=${PROXY_CLIENT_KEY_PATH}" 105 fi 106 params+=" --enable-aggregator-routing=true" 107 if [[ -e "${APISERVER_CLIENT_CERT_PATH}" ]] && [[ -e "${APISERVER_CLIENT_KEY_PATH}" ]]; then 108 params+=" --kubelet-client-certificate=${APISERVER_CLIENT_CERT_PATH}" 109 params+=" --kubelet-client-key=${APISERVER_CLIENT_KEY_PATH}" 110 fi 111 if [[ -n "${SERVICEACCOUNT_CERT_PATH:-}" ]]; then 112 params+=" --service-account-key-file=${SERVICEACCOUNT_CERT_PATH}" 113 fi 114 local known_tokens_file='/etc/srv/kubernetes/known_tokens.csv' 115 if [[ -f "${known_tokens_file}" ]]; then 116 chown "${KUBE_API_SERVER_RUNASUSER:-0}":"${KUBE_API_SERVER_RUNASGROUP:-0}" "${known_tokens_file}" 117 fi 118 params+=" --token-auth-file=${known_tokens_file}" 119 120 if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]]; then 121 params+=" --request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}s" 122 fi 123 if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then 124 params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" 125 fi 126 if [[ -n "${NUM_NODES:-}" ]]; then 127 # If the cluster is large, increase max-requests-inflight limit in apiserver. 128 if [[ "${NUM_NODES}" -gt 3000 ]]; then 129 params=$(append-param-if-not-present "${params}" "max-requests-inflight" 3000) 130 params=$(append-param-if-not-present "${params}" "max-mutating-requests-inflight" 1000) 131 elif [[ "${NUM_NODES}" -gt 500 ]]; then 132 params=$(append-param-if-not-present "${params}" "max-requests-inflight" 1500) 133 params=$(append-param-if-not-present "${params}" "max-mutating-requests-inflight" 500) 134 fi 135 fi 136 if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then 137 params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" 138 fi 139 params+=" --service-account-issuer=${SERVICEACCOUNT_ISSUER}" 140 params+=" --api-audiences=${SERVICEACCOUNT_ISSUER}" 141 params+=" --service-account-signing-key-file=${SERVICEACCOUNT_KEY_PATH}" 142 143 local audit_policy_config_mount="" 144 local audit_policy_config_volume="" 145 local audit_webhook_config_mount="" 146 local audit_webhook_config_volume="" 147 if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then 148 local -r audit_policy_file="/etc/audit_policy.config" 149 params+=" --audit-policy-file=${audit_policy_file}" 150 # Create the audit policy file, and mount it into the apiserver pod. 151 create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}" 152 audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": true}," 153 audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\", \"type\": \"FileOrCreate\"}}," 154 155 if [[ "${ADVANCED_AUDIT_BACKEND:-log}" == *"log"* ]]; then 156 # The advanced audit log backend config matches the basic audit log config. 157 params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" 158 params+=" --audit-log-maxage=0" 159 params+=" --audit-log-maxbackup=0" 160 # Lumberjack doesn't offer any way to disable size-based rotation. It also 161 # has an in-memory counter that doesn't notice if you truncate the file. 162 # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log 163 # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver 164 # never restarts. Please manually restart apiserver before this time. 165 params+=" --audit-log-maxsize=2000000000" 166 167 # Batching parameters 168 if [[ -n "${ADVANCED_AUDIT_LOG_MODE:-}" ]]; then 169 params+=" --audit-log-mode=${ADVANCED_AUDIT_LOG_MODE}" 170 fi 171 if [[ -n "${ADVANCED_AUDIT_LOG_BUFFER_SIZE:-}" ]]; then 172 params+=" --audit-log-batch-buffer-size=${ADVANCED_AUDIT_LOG_BUFFER_SIZE}" 173 fi 174 if [[ -n "${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE:-}" ]]; then 175 params+=" --audit-log-batch-max-size=${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE}" 176 fi 177 if [[ -n "${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT:-}" ]]; then 178 params+=" --audit-log-batch-max-wait=${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT}" 179 fi 180 if [[ -n "${ADVANCED_AUDIT_LOG_THROTTLE_QPS:-}" ]]; then 181 params+=" --audit-log-batch-throttle-qps=${ADVANCED_AUDIT_LOG_THROTTLE_QPS}" 182 fi 183 if [[ -n "${ADVANCED_AUDIT_LOG_THROTTLE_BURST:-}" ]]; then 184 params+=" --audit-log-batch-throttle-burst=${ADVANCED_AUDIT_LOG_THROTTLE_BURST}" 185 fi 186 if [[ -n "${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF:-}" ]]; then 187 params+=" --audit-log-initial-backoff=${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF}" 188 fi 189 # Truncating backend parameters 190 if [[ -n "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-}" ]]; then 191 params+=" --audit-log-truncate-enabled=${ADVANCED_AUDIT_TRUNCATING_BACKEND}" 192 fi 193 fi 194 if [[ "${ADVANCED_AUDIT_BACKEND:-}" == *"webhook"* ]]; then 195 # Create the audit webhook config file, and mount it into the apiserver pod. 196 local -r audit_webhook_config_file="/etc/audit_webhook.config" 197 params+=" --audit-webhook-config-file=${audit_webhook_config_file}" 198 create-master-audit-webhook-config "${audit_webhook_config_file}" 199 audit_webhook_config_mount="{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"${audit_webhook_config_file}\", \"readOnly\": true}," 200 audit_webhook_config_volume="{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"${audit_webhook_config_file}\", \"type\": \"FileOrCreate\"}}," 201 202 # Batching parameters 203 if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MODE:-}" ]]; then 204 params+=" --audit-webhook-mode=${ADVANCED_AUDIT_WEBHOOK_MODE}" 205 else 206 params+=" --audit-webhook-mode=batch" 207 fi 208 if [[ -n "${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE:-}" ]]; then 209 params+=" --audit-webhook-batch-buffer-size=${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE}" 210 fi 211 if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE:-}" ]]; then 212 params+=" --audit-webhook-batch-max-size=${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE}" 213 fi 214 if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT:-}" ]]; then 215 params+=" --audit-webhook-batch-max-wait=${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT}" 216 fi 217 if [[ -n "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS:-}" ]]; then 218 params+=" --audit-webhook-batch-throttle-qps=${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS}" 219 fi 220 if [[ -n "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST:-}" ]]; then 221 params+=" --audit-webhook-batch-throttle-burst=${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST}" 222 fi 223 if [[ -n "${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF:-}" ]]; then 224 params+=" --audit-webhook-initial-backoff=${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF}" 225 fi 226 # Truncating backend parameters 227 if [[ -n "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-}" ]]; then 228 params+=" --audit-webhook-truncate-enabled=${ADVANCED_AUDIT_TRUNCATING_BACKEND}" 229 fi 230 fi 231 fi 232 233 if [[ "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}" == "true" ]]; then 234 params+=" --audit-dynamic-configuration" 235 RUNTIME_CONFIG="${RUNTIME_CONFIG},auditconfiguration.k8s.io/v1alpha1=true" 236 fi 237 238 if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then 239 params+=" --enable-logs-handler=false" 240 fi 241 if [[ "${APISERVER_SET_KUBELET_CA:-false}" == "true" ]]; then 242 params+=" --kubelet-certificate-authority=${CA_CERT_BUNDLE_PATH}" 243 fi 244 245 if [[ -n "${ADMISSION_CONTROL:-}" ]]; then 246 params+=" --enable-admission-plugins=${ADMISSION_CONTROL}" 247 params+=" --admission-control-config-file=/etc/srv/kubernetes/admission_controller_config.yaml" 248 fi 249 250 if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then 251 params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" 252 fi 253 if [[ -n "${RUNTIME_CONFIG:-}" ]]; then 254 params+=" --runtime-config=${RUNTIME_CONFIG}" 255 fi 256 if [[ -n "${FEATURE_GATES:-}" ]]; then 257 params+=" --feature-gates=${FEATURE_GATES}" 258 fi 259 if [[ -n "${MASTER_ADVERTISE_ADDRESS:-}" ]]; then 260 params+=" --advertise-address=${MASTER_ADVERTISE_ADDRESS}" 261 elif [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then 262 local -r vm_external_ip=$(get-metadata-value "instance/network-interfaces/0/access-configs/0/external-ip") 263 params+=" --advertise-address=${vm_external_ip}" 264 if [[ -n "${KUBE_API_SERVER_RUNASUSER:-}" && -n "${KUBE_API_SERVER_RUNASGROUP:-}" ]]; then 265 chown -R "${KUBE_API_SERVER_RUNASUSER}":"${KUBE_API_SERVER_RUNASGROUP}" /etc/srv/sshproxy/ 266 fi 267 fi 268 269 local webhook_authn_config_mount="" 270 local webhook_authn_config_volume="" 271 if [[ -n "${GCP_AUTHN_URL:-}" ]]; then 272 params+=" --authentication-token-webhook-config-file=/etc/gcp_authn.config" 273 webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": true}," 274 webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\", \"type\": \"File\"}}," 275 if [[ -n "${GCP_AUTHN_CACHE_TTL:-}" ]]; then 276 params+=" --authentication-token-webhook-cache-ttl=${GCP_AUTHN_CACHE_TTL}" 277 fi 278 fi 279 280 local authorization_mode="RBAC" 281 local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" 282 283 # Enable ABAC mode unless the user explicitly opts out with ENABLE_LEGACY_ABAC=false 284 if [[ "${ENABLE_LEGACY_ABAC:-}" != "false" ]]; then 285 echo "Warning: Enabling legacy ABAC policy. All service accounts will have superuser API access. Set ENABLE_LEGACY_ABAC=false to disable this." 286 # Create the ABAC file if it doesn't exist yet, or if we have a KUBE_USER set (to ensure the right user is given permissions) 287 if [[ -n "${KUBE_USER:-}" || ! -e /etc/srv/kubernetes/abac-authz-policy.jsonl ]]; then 288 local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl" 289 if [[ -n "${KUBE_USER:-}" ]]; then 290 sed -i -e "s/{{kube_user}}/${KUBE_USER}/g" "${abac_policy_json}" 291 else 292 sed -i -e "/{{kube_user}}/d" "${abac_policy_json}" 293 fi 294 cp "${abac_policy_json}" /etc/srv/kubernetes/ 295 fi 296 297 params+=" --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl" 298 authorization_mode+=",ABAC" 299 fi 300 301 local webhook_config_mount="" 302 local webhook_config_volume="" 303 if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then 304 authorization_mode="${authorization_mode},Webhook" 305 params+=" --authorization-webhook-config-file=/etc/gcp_authz.config" 306 webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": true}," 307 webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\", \"type\": \"File\"}}," 308 if [[ -n "${GCP_AUTHZ_CACHE_AUTHORIZED_TTL:-}" ]]; then 309 params+=" --authorization-webhook-cache-authorized-ttl=${GCP_AUTHZ_CACHE_AUTHORIZED_TTL}" 310 fi 311 if [[ -n "${GCP_AUTHZ_CACHE_UNAUTHORIZED_TTL:-}" ]]; then 312 params+=" --authorization-webhook-cache-unauthorized-ttl=${GCP_AUTHZ_CACHE_UNAUTHORIZED_TTL}" 313 fi 314 fi 315 authorization_mode="Node,${authorization_mode}" 316 params+=" --authorization-mode=${authorization_mode}" 317 318 local csc_config_mount="" 319 local csc_config_volume="" 320 local default_konnectivity_socket_vol="" 321 local default_konnectivity_socket_mnt="" 322 if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then 323 # Create the EgressSelectorConfiguration yaml file to control the Egress Selector. 324 csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false}," 325 csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}}," 326 327 # UDS socket for communication between apiserver and konnectivity-server 328 local default_konnectivity_socket_path="/etc/srv/kubernetes/konnectivity-server" 329 default_konnectivity_socket_vol="{ \"name\": \"konnectivity-socket\", \"hostPath\": {\"path\": \"${default_konnectivity_socket_path}\", \"type\": \"DirectoryOrCreate\"}}," 330 default_konnectivity_socket_mnt="{ \"name\": \"konnectivity-socket\", \"mountPath\": \"${default_konnectivity_socket_path}\", \"readOnly\": false}," 331 fi 332 if [[ "${EGRESS_VIA_KONNECTIVITY:-false}" == "true" ]]; then 333 params+=" --egress-selector-config-file=/etc/srv/kubernetes/egress_selector_configuration.yaml" 334 fi 335 336 local container_env="" 337 if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then 338 container_env+="{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}" 339 fi 340 if [[ -n "${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR:-}" ]]; then 341 if [[ -n "${container_env}" ]]; then 342 container_env="${container_env}, " 343 fi 344 container_env+="{\"name\": \"KUBE_WATCHLIST_INCONSISTENCY_DETECTOR\", \"value\": \"${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR}\"}" 345 fi 346 if [[ -n "${ENABLE_PATCH_CONVERSION_DETECTOR:-}" ]]; then 347 if [[ -n "${container_env}" ]]; then 348 container_env="${container_env}, " 349 fi 350 container_env+="{\"name\": \"KUBE_PATCH_CONVERSION_DETECTOR\", \"value\": \"${ENABLE_PATCH_CONVERSION_DETECTOR}\"}" 351 fi 352 if [[ -n "${KUBE_APISERVER_GODEBUG:-}" ]]; then 353 if [[ -n "${container_env}" ]]; then 354 container_env="${container_env}, " 355 fi 356 container_env+="{\"name\": \"GODEBUG\", \"value\": \"${KUBE_APISERVER_GODEBUG}\"}" 357 fi 358 if [[ -n "${container_env}" ]]; then 359 container_env="\"env\":[${container_env}]," 360 fi 361 362 local -r src_file="${src_dir}/kube-apiserver.manifest" 363 364 # params is passed by reference, so no "$" 365 setup-etcd-encryption "${src_file}" params 366 367 local healthcheck_ip="127.0.0.1" 368 if [[ ${KUBE_APISERVER_HEALTHCHECK_ON_HOST_IP:-} == "true" ]]; then 369 healthcheck_ip=$(hostname -i) 370 fi 371 372 params="$(convert-manifest-params "${params}")" 373 # Evaluate variables. 374 local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}" 375 sed -i -e "s@{{params}}@${params}@g" "${src_file}" 376 sed -i -e "s@{{container_env}}@${container_env}@g" "${src_file}" 377 sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}" 378 sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" 379 sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" 380 sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" 381 sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}" 382 sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" 383 sed -i -e "s@{{liveness_probe_initial_delay}}@${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${src_file}" 384 sed -i -e "s@{{secure_port}}@443@g" "${src_file}" 385 sed -i -e "s@{{insecure_port_mapping}}@${INSECURE_PORT_MAPPING}@g" "${src_file}" 386 sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" 387 sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" 388 sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}" 389 sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" 390 sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" 391 sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" 392 sed -i -e "s@{{csc_config_mount}}@${csc_config_mount}@g" "${src_file}" 393 sed -i -e "s@{{csc_config_volume}}@${csc_config_volume}@g" "${src_file}" 394 sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" 395 sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" 396 sed -i -e "s@{{audit_webhook_config_mount}}@${audit_webhook_config_mount}@g" "${src_file}" 397 sed -i -e "s@{{audit_webhook_config_volume}}@${audit_webhook_config_volume}@g" "${src_file}" 398 sed -i -e "s@{{konnectivity_socket_mount}}@${default_konnectivity_socket_mnt}@g" "${src_file}" 399 sed -i -e "s@{{konnectivity_socket_volume}}@${default_konnectivity_socket_vol}@g" "${src_file}" 400 sed -i -e "s@{{healthcheck_ip}}@${healthcheck_ip}@g" "${src_file}" 401 402 if [[ -n "${KUBE_API_SERVER_RUNASUSER:-}" && -n "${KUBE_API_SERVER_RUNASGROUP:-}" && -n "${KUBE_PKI_READERS_GROUP:-}" ]]; then 403 sed -i -e "s@{{runAsUser}}@\"runAsUser\": ${KUBE_API_SERVER_RUNASUSER},@g" "${src_file}" 404 sed -i -e "s@{{runAsGroup}}@\"runAsGroup\": ${KUBE_API_SERVER_RUNASGROUP},@g" "${src_file}" 405 sed -i -e "s@{{containerSecurityContext}}@\"securityContext\": { \"capabilities\": { \"drop\": [\"all\"], \"add\": [\"NET_BIND_SERVICE\"] } },@g" "${src_file}" 406 local supplementalGroups="${KUBE_PKI_READERS_GROUP}" 407 if [[ -n "${KMS_PLUGIN_SOCKET_WRITER_GROUP:-}" ]]; then 408 supplementalGroups+=",${KMS_PLUGIN_SOCKET_WRITER_GROUP}" 409 fi 410 if [[ -n "${KONNECTIVITY_SERVER_SOCKET_WRITER_GROUP:-}" ]]; then 411 supplementalGroups+=",${KONNECTIVITY_SERVER_SOCKET_WRITER_GROUP}" 412 fi 413 sed -i -e "s@{{supplementalGroups}}@\"supplementalGroups\": [ ${supplementalGroups} ],@g" "${src_file}" 414 else 415 sed -i -e "s@{{runAsUser}}@@g" "${src_file}" 416 sed -i -e "s@{{runAsGroup}}@@g" "${src_file}" 417 sed -i -e "s@{{containerSecurityContext}}@@g" "${src_file}" 418 sed -i -e "s@{{supplementalGroups}}@@g" "${src_file}" 419 fi 420 421 cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}" 422 } 423 424 425 # Sets-up etcd encryption. 426 # Configuration of etcd level encryption consists of the following steps: 427 # 1. Writing encryption provider config to disk 428 # 2. Adding encryption-provider-config flag to kube-apiserver 429 # 3. Add kms-socket-vol and kms-socket-vol-mnt to enable communication with kms-plugin (if requested) 430 # 431 # Expects parameters: 432 # $1 - path to kube-apiserver template 433 # $2 - kube-apiserver startup flags (must be passed by reference) 434 # 435 # Assumes vars (supplied via kube-env): 436 # ENCRYPTION_PROVIDER_CONFIG 437 # CLOUD_KMS_INTEGRATION 438 # ENCRYPTION_PROVIDER_CONFIG_PATH (will default to /etc/srv/kubernetes/encryption-provider-config.yml) 439 function setup-etcd-encryption { 440 local kube_apiserver_template_path 441 local -n kube_api_server_params 442 local default_encryption_provider_config_vol 443 local default_encryption_provider_config_vol_mnt 444 local encryption_provider_config_vol_mnt 445 local encryption_provider_config_vol 446 local default_kms_socket_dir 447 local default_kms_socket_vol_mnt 448 local default_kms_socket_vol 449 local kms_socket_vol_mnt 450 local kms_socket_vol 451 local encryption_provider_config_path 452 453 kube_apiserver_template_path="$1" 454 if [[ -z "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then 455 sed -i -e " { 456 s@{{encryption_provider_mount}}@@ 457 s@{{encryption_provider_volume}}@@ 458 s@{{kms_socket_mount}}@@ 459 s@{{kms_socket_volume}}@@ 460 } " "${kube_apiserver_template_path}" 461 return 462 fi 463 464 kube_api_server_params="$2" 465 encryption_provider_config_path=${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml} 466 467 echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}" 468 kube_api_server_params+=" --encryption-provider-config=${encryption_provider_config_path}" 469 470 default_encryption_provider_config_vol=$(echo "{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}" | base64 | tr -d '\r\n') 471 default_encryption_provider_config_vol_mnt=$(echo "{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}" | base64 | tr -d '\r\n') 472 473 encryption_provider_config_vol_mnt=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL_MNT:-"${default_encryption_provider_config_vol_mnt}"}" | base64 --decode) 474 encryption_provider_config_vol=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL:-"${default_encryption_provider_config_vol}"}" | base64 --decode) 475 sed -i -e " { 476 s@{{encryption_provider_mount}}@${encryption_provider_config_vol_mnt},@ 477 s@{{encryption_provider_volume}}@${encryption_provider_config_vol},@ 478 } " "${kube_apiserver_template_path}" 479 480 if [[ -n "${CLOUD_KMS_INTEGRATION:-}" ]]; then 481 default_kms_socket_dir="/var/run/kmsplugin" 482 default_kms_socket_vol_mnt=$(echo "{ \"name\": \"kmssocket\", \"mountPath\": \"${default_kms_socket_dir}\", \"readOnly\": false}" | base64 | tr -d '\r\n') 483 default_kms_socket_vol=$(echo "{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${default_kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" | base64 | tr -d '\r\n') 484 485 kms_socket_vol_mnt=$(echo "${KMS_PLUGIN_SOCKET_VOL_MNT:-"${default_kms_socket_vol_mnt}"}" | base64 --decode) 486 kms_socket_vol=$(echo "${KMS_PLUGIN_SOCKET_VOL:-"${default_kms_socket_vol}"}" | base64 --decode) 487 sed -i -e " { 488 s@{{kms_socket_mount}}@${kms_socket_vol_mnt},@ 489 s@{{kms_socket_volume}}@${kms_socket_vol},@ 490 } " "${kube_apiserver_template_path}" 491 else 492 sed -i -e " { 493 s@{{kms_socket_mount}}@@ 494 s@{{kms_socket_volume}}@@ 495 } " "${kube_apiserver_template_path}" 496 fi 497 }