sigs.k8s.io/cluster-api-provider-azure@v1.17.0/scripts/aks-as-mgmt.sh (about) 1 #!/usr/bin/env bash 2 # Copyright 2024 The Kubernetes Authors. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 set -o errexit # exit immediately if a command exits with a non-zero status. 17 set -o nounset # exit when script tries to use undeclared variables. 18 set -o pipefail # make the pipeline fail if any command in it fails. 19 20 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 21 # shellcheck source=hack/ensure-azcli.sh 22 source "${REPO_ROOT}/hack/ensure-azcli.sh" # install az cli and login using WI 23 # shellcheck source=hack/ensure-tags.sh 24 source "${REPO_ROOT}/hack/ensure-tags.sh" # set the right timestamp and job name 25 26 KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl" 27 KIND="${REPO_ROOT}/hack/tools/bin/kind" 28 AZWI="${REPO_ROOT}/hack/tools/bin/azwi" 29 make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}" "${AZWI##*/}" 30 31 export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # management cluster name 32 export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # resource group name 33 export AKS_NODE_RESOURCE_GROUP="node-${AKS_RESOURCE_GROUP}" 34 export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.30.2}" 35 export AZURE_LOCATION="${AZURE_LOCATION:-westus2}" 36 export AKS_NODE_VM_SIZE="${AKS_NODE_VM_SIZE:-"Standard_B2s"}" 37 export AKS_NODE_COUNT="${AKS_NODE_COUNT:-1}" 38 export MGMT_CLUSTER_KUBECONFIG="${MGMT_CLUSTER_KUBECONFIG:-$REPO_ROOT/aks-mgmt.config}" 39 export AZURE_IDENTITY_ID_FILEPATH="${AZURE_IDENTITY_ID_FILEPATH:-$REPO_ROOT/azure_identity_id}" 40 export AZWI_STORAGE_ACCOUNT="capzcioidcissuer${RANDOM_SUFFIX}" 41 export AZWI_STORAGE_CONTAINER="\$web" 42 export SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH="${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH:-}" 43 export SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH="${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH:-}" 44 export REGISTRY="${REGISTRY:-}" 45 46 export AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:-}" 47 export AZURE_CLIENT_ID="${AZURE_CLIENT_ID:-}" 48 export AZURE_TENANT_ID="${AZURE_TENANT_ID:-}" 49 50 main() { 51 52 echo "--------------------------------" 53 echo "MGMT_CLUSTER_NAME: $MGMT_CLUSTER_NAME" 54 echo "AKS_RESOURCE_GROUP: $AKS_RESOURCE_GROUP" 55 echo "AKS_NODE_RESOURCE_GROUP: $AKS_NODE_RESOURCE_GROUP" 56 echo "KUBERNETES_VERSION: $KUBERNETES_VERSION" 57 echo "AZURE_LOCATION: $AZURE_LOCATION" 58 echo "AKS_NODE_VM_SIZE: $AKS_NODE_VM_SIZE" 59 echo "AZURE_NODE_MACHINE_TYPE: $AZURE_NODE_MACHINE_TYPE" 60 echo "AKS_NODE_COUNT: $AKS_NODE_COUNT" 61 echo "MGMT_CLUSTER_KUBECONFIG: $MGMT_CLUSTER_KUBECONFIG" 62 echo "AZURE_IDENTITY_ID_FILEPATH: $AZURE_IDENTITY_ID_FILEPATH" 63 echo "AZWI_STORAGE_ACCOUNT: $AZWI_STORAGE_ACCOUNT" 64 echo "AZWI_STORAGE_CONTAINER: $AZWI_STORAGE_CONTAINER" 65 echo "SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH: $SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH" 66 echo "SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH: $SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH" 67 echo "REGISTRY: $REGISTRY" 68 69 echo "AZURE_SUBSCRIPTION_ID: $AZURE_SUBSCRIPTION_ID" 70 echo "AZURE_CLIENT_ID: $AZURE_CLIENT_ID" 71 echo "AZURE_TENANT_ID: $AZURE_TENANT_ID" 72 echo "--------------------------------" 73 74 create_aks_cluster 75 set_env_varaibles 76 } 77 78 create_aks_cluster() { 79 resource_group_exists=$(az group exists --name "${AKS_RESOURCE_GROUP}" --output tsv) 80 if [ "${resource_group_exists}" == 'true' ]; then 81 echo "resource group \"${AKS_RESOURCE_GROUP}\" already exists, moving on" 82 else 83 echo "creating resource group ${AKS_RESOURCE_GROUP}" 84 az group create --name "${AKS_RESOURCE_GROUP}" \ 85 --location "${AZURE_LOCATION}" \ 86 --output none --only-show-errors \ 87 --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" 88 fi 89 90 aks_exists=$(az aks show --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" 2>&1 || true) # true because we want to continue if the command fails 91 if echo "$aks_exists" | grep -E -q "Resource(NotFound|GroupNotFound)"; then 92 echo "creating aks cluster ${MGMT_CLUSTER_NAME} in the resource group ${AKS_RESOURCE_GROUP}" 93 az aks create --name "${MGMT_CLUSTER_NAME}" \ 94 --resource-group "${AKS_RESOURCE_GROUP}" \ 95 --location "${AZURE_LOCATION}" \ 96 --kubernetes-version "${KUBERNETES_VERSION}" \ 97 --node-count "${AKS_NODE_COUNT}" \ 98 --node-vm-size "${AKS_NODE_VM_SIZE}" \ 99 --node-resource-group "${AKS_NODE_RESOURCE_GROUP}" \ 100 --vm-set-type VirtualMachineScaleSets \ 101 --generate-ssh-keys \ 102 --network-plugin azure \ 103 --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" \ 104 --output none --only-show-errors; 105 elif echo "$aks_exists" | grep -q "${MGMT_CLUSTER_NAME}"; then 106 echo "cluster ${MGMT_CLUSTER_NAME} already exists in RG ${AKS_RESOURCE_GROUP}, moving on" 107 else 108 echo "error : ${aks_exists}" 109 exit 1 110 fi 111 112 # check and save kubeconfig 113 echo "saving credentials of cluster ${MGMT_CLUSTER_NAME} in ${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}" 114 az aks get-credentials --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \ 115 --file "${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}" --only-show-errors 116 117 az aks get-credentials --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \ 118 --overwrite-existing --only-show-errors 119 120 # echo "fetching Client ID for ${MGMT_CLUSTER_NAME}" 121 AKS_MI_CLIENT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \ 122 --only-show-errors | jq -r '.identityProfile.kubeletidentity.clientId') 123 export AKS_MI_CLIENT_ID 124 echo "mgmt client identity: ${AKS_MI_CLIENT_ID}" 125 echo "${AKS_MI_CLIENT_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}" 126 127 # echo "fetching Object ID for ${MGMT_CLUSTER_NAME}" 128 AKS_MI_OBJECT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \ 129 --only-show-errors | jq -r '.identityProfile.kubeletidentity.objectId') 130 export AKS_MI_OBJECT_ID 131 echo "mgmt object identity: ${AKS_MI_OBJECT_ID}" 132 133 # echo "fetching Resource ID for ${MGMT_CLUSTER_NAME}" 134 AKS_MI_RESOURCE_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \ 135 --only-show-errors | jq -r '.identityProfile.kubeletidentity.resourceId') 136 export AKS_MI_RESOURCE_ID 137 echo "mgmt resource identity: ${AKS_MI_RESOURCE_ID}" 138 139 # save resource identity name and resource group 140 MANAGED_IDENTITY_NAME=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" | jq -r '.name') 141 # export MANAGED_IDENTITY_NAME 142 echo "mgmt resource identity name: ${MANAGED_IDENTITY_NAME}" 143 USER_IDENTITY=$MANAGED_IDENTITY_NAME 144 export USER_IDENTITY 145 146 MANAGED_IDENTITY_RG=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" | jq -r '.resourceGroup') 147 export MANAGED_IDENTITY_RG 148 echo "mgmt resource identity resource group: ${MANAGED_IDENTITY_RG}" 149 150 echo "assigning contributor role to the service principal" 151 until az role assignment create --assignee-object-id "${AKS_MI_OBJECT_ID}" --role "Contributor" \ 152 --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal --output none \ 153 --only-show-errors; do 154 echo "retrying to assign role to the service principal" 155 sleep 5 156 done 157 158 echo "using ASO_CREDENTIAL_SECRET_MODE as podidentity" 159 ASO_CREDENTIAL_SECRET_MODE="podidentity" 160 } 161 162 set_env_varaibles(){ 163 cat <<EOF > tilt-settings-temp.yaml 164 kustomize_substitutions: 165 MGMT_CLUSTER_NAME: "${MGMT_CLUSTER_NAME}" 166 AKS_RESOURCE_GROUP: "${AKS_RESOURCE_GROUP}" 167 AKS_NODE_RESOURCE_GROUP: "${AKS_NODE_RESOURCE_GROUP}" 168 MGMT_CLUSTER_KUBECONFIG: "${MGMT_CLUSTER_KUBECONFIG}" 169 AKS_MI_CLIENT_ID: "${AKS_MI_CLIENT_ID}" 170 AKS_MI_OBJECT_ID: "${AKS_MI_OBJECT_ID}" 171 AKS_MI_RESOURCE_ID: "${AKS_MI_RESOURCE_ID}" 172 MANAGED_IDENTITY_NAME: "${MANAGED_IDENTITY_NAME}" 173 MANAGED_IDENTITY_RG: "${MANAGED_IDENTITY_RG}" 174 AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY: "${AKS_MI_CLIENT_ID}" 175 CI_RG: "${MANAGED_IDENTITY_RG}" 176 USER_IDENTITY: "${MANAGED_IDENTITY_NAME}" 177 CLUSTER_IDENTITY_TYPE: "UserAssignedMSI" 178 ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE}" 179 REGISTRY: "${REGISTRY}" 180 allowed_contexts: 181 - "$MGMT_CLUSTER_NAME" 182 - "kind-capz" 183 azure_location: "${AZURE_LOCATION}" 184 EOF 185 186 # create tilt-settings.yaml if it does not exist 187 if [ -f tilt-settings.yaml ]; then 188 echo "tilt-settings.yaml exists" 189 else 190 echo "tilt-settings.yaml does not exist, creating one" 191 touch tilt-settings.yaml 192 fi 193 194 # copy over the existing allowed_contexts to tilt-settings.yaml if it does not exist 195 allowed_contexts_exists=$(yq eval '.allowed_contexts' tilt-settings.yaml) 196 if [ "$allowed_contexts_exists" == "null" ]; then 197 yq eval '.allowed_contexts = load("tilt-settings-temp.yaml") | .allowed_contexts' tilt-settings-temp.yaml > tilt-settings.yaml 198 fi 199 200 # extract allowed_contexts from tilt-settings.yaml 201 current_contexts=$(yq eval '.allowed_contexts' tilt-settings.yaml | sort -u) 202 203 # extract allowed_contexts from tilt-settings-new.yaml 204 new_contexts=$(yq eval '.allowed_contexts' tilt-settings-temp.yaml | sort -u) 205 206 # combine current and new contexts, keeping the union of both 207 combined_contexts=$(echo "$current_contexts"$'\n'"$new_contexts" | sort -u) 208 209 # create a temporary file since env($combined_contexts) is not supported in yq 210 echo "$combined_contexts" > combined_contexts.yaml 211 212 # update allowed_contexts in tilt-settings.yaml with the combined contexts 213 yq eval --inplace ".allowed_contexts = load(\"combined_contexts.yaml\")" tilt-settings.yaml 214 215 # merge the updated kustomize_substitution and azure_location with the existing one in tilt-settings.yaml 216 yq eval-all 'select(fileIndex == 0) *+ {"kustomize_substitutions": select(fileIndex == 1).kustomize_substitutions, "azure_location": select(fileIndex == 1).azure_location}' tilt-settings.yaml tilt-settings-temp.yaml > tilt-settings-new.yaml 217 218 mv tilt-settings-new.yaml tilt-settings.yaml 219 rm -r combined_contexts.yaml 220 rm -f tilt-settings-temp.yaml 221 } 222 223 main