sigs.k8s.io/cluster-api-provider-azure@v1.17.0/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico-dual-stack
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: disabled
     9      csi-proxy: disabled
    10      metrics-server: disabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 10.244.0.0/16
    18        - 2001:1234:5678:9a40::/58
    19      services:
    20        cidrBlocks:
    21        - 10.0.0.0/16
    22        - fd00::/108
    23    controlPlaneRef:
    24      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    25      kind: KubeadmControlPlane
    26      name: ${CLUSTER_NAME}-control-plane
    27    infrastructureRef:
    28      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    29      kind: AzureCluster
    30      name: ${CLUSTER_NAME}
    31  ---
    32  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    33  kind: AzureCluster
    34  metadata:
    35    name: ${CLUSTER_NAME}
    36    namespace: default
    37  spec:
    38    additionalTags:
    39      buildProvenance: ${BUILD_PROVENANCE}
    40      creationTimestamp: ${TIMESTAMP}
    41      jobName: ${JOB_NAME}
    42    identityRef:
    43      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    44      kind: AzureClusterIdentity
    45      name: ${CLUSTER_IDENTITY_NAME}
    46    location: ${AZURE_LOCATION}
    47    networkSpec:
    48      subnets:
    49      - cidrBlocks:
    50        - 10.0.0.0/16
    51        - 2001:1234:5678:9abc::/64
    52        name: control-plane-subnet
    53        role: control-plane
    54      - cidrBlocks:
    55        - 10.1.0.0/16
    56        - 2001:1234:5678:9abd::/64
    57        name: node-subnet
    58        role: node
    59      vnet:
    60        cidrBlocks:
    61        - 10.0.0.0/8
    62        - 2001:1234:5678:9a00::/56
    63        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    64    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    65    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    66  ---
    67  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    68  kind: KubeadmControlPlane
    69  metadata:
    70    name: ${CLUSTER_NAME}-control-plane
    71    namespace: default
    72  spec:
    73    kubeadmConfigSpec:
    74      clusterConfiguration:
    75        apiServer:
    76          extraArgs:
    77            cloud-provider: external
    78            feature-gates: ${K8S_FEATURE_GATES:-""}
    79          timeoutForControlPlane: 20m
    80        controllerManager:
    81          extraArgs:
    82            allocate-node-cidrs: "true"
    83            cloud-provider: external
    84            cluster-cidr: 10.244.0.0/16,2001:1234:5678:9a40::/58
    85            cluster-name: ${CLUSTER_NAME}
    86            configure-cloud-routes: "true"
    87            feature-gates: HPAContainerMetrics=true
    88            v: "4"
    89        etcd:
    90          local:
    91            dataDir: /var/lib/etcddisk/etcd
    92            extraArgs:
    93              quota-backend-bytes: "8589934592"
    94        kubernetesVersion: ci/${CI_VERSION}
    95      diskSetup:
    96        filesystems:
    97        - device: /dev/disk/azure/scsi1/lun0
    98          extraOpts:
    99          - -E
   100          - lazy_itable_init=1,lazy_journal_init=1
   101          filesystem: ext4
   102          label: etcd_disk
   103        - device: ephemeral0.1
   104          filesystem: ext4
   105          label: ephemeral0
   106          replaceFS: ntfs
   107        partitions:
   108        - device: /dev/disk/azure/scsi1/lun0
   109          layout: true
   110          overwrite: false
   111          tableType: gpt
   112      files:
   113      - contentFrom:
   114          secret:
   115            key: control-plane-azure.json
   116            name: ${CLUSTER_NAME}-control-plane-azure-json
   117        owner: root:root
   118        path: /etc/kubernetes/azure.json
   119        permissions: "0644"
   120      - content: |
   121          #!/bin/bash
   122  
   123          set -o nounset
   124          set -o pipefail
   125          set -o errexit
   126          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   127  
   128          echo "Use OOT credential provider"
   129          mkdir -p /var/lib/kubelet/credential-provider
   130          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   131          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   132          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   133          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   134        owner: root:root
   135        path: /tmp/oot-cred-provider.sh
   136        permissions: "0744"
   137      - content: |
   138          #!/bin/bash
   139  
   140          set -o nounset
   141          set -o pipefail
   142          set -o errexit
   143          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   144  
   145          # This test installs release packages or binaries that are a result of the CI and release builds.
   146          # It runs '... --version' commands to verify that the binaries are correctly installed
   147          # and finally uninstalls the packages.
   148          # For the release packages it tests all versions in the support skew.
   149          LINE_SEPARATOR="*************************************************"
   150          echo "$$LINE_SEPARATOR"
   151          CI_VERSION=${CI_VERSION}
   152          if [[ "$${CI_VERSION}" != "" ]]; then
   153            CI_DIR=/tmp/k8s-ci
   154            mkdir -p $$CI_DIR
   155            declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   156            declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   157            CONTAINER_EXT="tar"
   158            echo "* testing CI version $$CI_VERSION"
   159            # Check for semver
   160            if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   161              VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   162              DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   163              curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   164              echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   165              apt-get update
   166              # replace . with \.
   167              VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   168              PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   169              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   170                echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   171                DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   172              done
   173            else
   174              CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   175              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   176                echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   177                wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   178                chmod +x "$$CI_DIR/$$CI_PACKAGE"
   179                mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   180              done
   181              IMAGE_REGISTRY_PREFIX=registry.k8s.io
   182              for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   183                echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   184                wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   185                $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   186                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   187                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   188              done
   189            fi
   190            systemctl restart kubelet
   191          fi
   192          echo "* checking binary versions"
   193          echo "ctr version: " $(ctr version)
   194          echo "kubeadm version: " $(kubeadm version -o=short)
   195          echo "kubectl version: " $(kubectl version --client=true)
   196          echo "kubelet version: " $(kubelet --version)
   197          echo "$$LINE_SEPARATOR"
   198        owner: root:root
   199        path: /tmp/kubeadm-bootstrap.sh
   200        permissions: "0744"
   201      initConfiguration:
   202        localAPIEndpoint:
   203          bindPort: 6443
   204        nodeRegistration:
   205          kubeletExtraArgs:
   206            cloud-provider: external
   207            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   208            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   209          name: '{{ ds.meta_data["local_hostname"] }}'
   210      joinConfiguration:
   211        controlPlane:
   212          localAPIEndpoint:
   213            bindPort: 6443
   214        nodeRegistration:
   215          kubeletExtraArgs:
   216            cloud-provider: external
   217            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   218            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   219          name: '{{ ds.meta_data["local_hostname"] }}'
   220      mounts:
   221      - - LABEL=etcd_disk
   222        - /var/lib/etcddisk
   223      postKubeadmCommands: []
   224      preKubeadmCommands:
   225      - bash -c /tmp/oot-cred-provider.sh
   226      - bash -c /tmp/kubeadm-bootstrap.sh
   227      verbosity: 5
   228    machineTemplate:
   229      infrastructureRef:
   230        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   231        kind: AzureMachineTemplate
   232        name: ${CLUSTER_NAME}-control-plane
   233    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   234    version: ${KUBERNETES_VERSION}
   235  ---
   236  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   237  kind: AzureMachineTemplate
   238  metadata:
   239    name: ${CLUSTER_NAME}-control-plane
   240    namespace: default
   241  spec:
   242    template:
   243      spec:
   244        dataDisks:
   245        - diskSizeGB: 256
   246          lun: 0
   247          nameSuffix: etcddisk
   248        enableIPForwarding: true
   249        identity: UserAssigned
   250        image:
   251          marketplace:
   252            offer: capi
   253            publisher: cncf-upstream
   254            sku: ubuntu-2204-gen1
   255            version: latest
   256        osDisk:
   257          diskSizeGB: 128
   258          osType: Linux
   259        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   260        userAssignedIdentities:
   261        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
   262        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   263  ---
   264  apiVersion: cluster.x-k8s.io/v1beta1
   265  kind: MachineDeployment
   266  metadata:
   267    name: ${CLUSTER_NAME}-md-0
   268    namespace: default
   269  spec:
   270    clusterName: ${CLUSTER_NAME}
   271    replicas: ${WORKER_MACHINE_COUNT:=2}
   272    selector: {}
   273    template:
   274      metadata:
   275        labels:
   276          nodepool: pool1
   277      spec:
   278        bootstrap:
   279          configRef:
   280            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   281            kind: KubeadmConfigTemplate
   282            name: ${CLUSTER_NAME}-md-0
   283        clusterName: ${CLUSTER_NAME}
   284        infrastructureRef:
   285          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   286          kind: AzureMachineTemplate
   287          name: ${CLUSTER_NAME}-md-0
   288        version: ${KUBERNETES_VERSION}
   289  ---
   290  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   291  kind: AzureMachineTemplate
   292  metadata:
   293    name: ${CLUSTER_NAME}-md-0
   294    namespace: default
   295  spec:
   296    template:
   297      spec:
   298        enableIPForwarding: true
   299        identity: UserAssigned
   300        image:
   301          marketplace:
   302            offer: capi
   303            publisher: cncf-upstream
   304            sku: ubuntu-2204-gen1
   305            version: latest
   306        osDisk:
   307          diskSizeGB: 128
   308          osType: Linux
   309        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   310        userAssignedIdentities:
   311        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
   312        vmExtensions:
   313        - name: CustomScript
   314          protectedSettings:
   315            commandToExecute: |
   316              #!/bin/sh
   317              echo "This script is a no-op used for extension testing purposes ..."
   318              touch test_file
   319          publisher: Microsoft.Azure.Extensions
   320          version: "2.1"
   321        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   322  ---
   323  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   324  kind: KubeadmConfigTemplate
   325  metadata:
   326    name: ${CLUSTER_NAME}-md-0
   327    namespace: default
   328  spec:
   329    template:
   330      spec:
   331        files:
   332        - contentFrom:
   333            secret:
   334              key: worker-node-azure.json
   335              name: ${CLUSTER_NAME}-md-0-azure-json
   336          owner: root:root
   337          path: /etc/kubernetes/azure.json
   338          permissions: "0644"
   339        - content: |
   340            #!/bin/bash
   341  
   342            set -o nounset
   343            set -o pipefail
   344            set -o errexit
   345            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   346  
   347            echo "Use OOT credential provider"
   348            mkdir -p /var/lib/kubelet/credential-provider
   349            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   350            chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   351            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   352            chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   353          owner: root:root
   354          path: /tmp/oot-cred-provider.sh
   355          permissions: "0744"
   356        - content: |
   357            #!/bin/bash
   358  
   359            set -o nounset
   360            set -o pipefail
   361            set -o errexit
   362            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   363  
   364            # This test installs release packages or binaries that are a result of the CI and release builds.
   365            # It runs '... --version' commands to verify that the binaries are correctly installed
   366            # and finally uninstalls the packages.
   367            # For the release packages it tests all versions in the support skew.
   368            LINE_SEPARATOR="*************************************************"
   369            echo "$$LINE_SEPARATOR"
   370            CI_VERSION=${CI_VERSION}
   371            if [[ "$${CI_VERSION}" != "" ]]; then
   372              CI_DIR=/tmp/k8s-ci
   373              mkdir -p $$CI_DIR
   374              declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   375              declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   376              CONTAINER_EXT="tar"
   377              echo "* testing CI version $$CI_VERSION"
   378              # Check for semver
   379              if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   380                VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   381                DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   382                curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   383                echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   384                apt-get update
   385                # replace . with \.
   386                VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   387                PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   388                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   389                  echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   390                  DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   391                done
   392              else
   393                CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   394                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   395                  echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   396                  wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   397                  chmod +x "$$CI_DIR/$$CI_PACKAGE"
   398                  mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   399                done
   400                IMAGE_REGISTRY_PREFIX=registry.k8s.io
   401                for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   402                  echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   403                  wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   404                  $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   405                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   406                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   407                done
   408              fi
   409              systemctl restart kubelet
   410            fi
   411            echo "* checking binary versions"
   412            echo "ctr version: " $(ctr version)
   413            echo "kubeadm version: " $(kubeadm version -o=short)
   414            echo "kubectl version: " $(kubectl version --client=true)
   415            echo "kubelet version: " $(kubelet --version)
   416            echo "$$LINE_SEPARATOR"
   417          owner: root:root
   418          path: /tmp/kubeadm-bootstrap.sh
   419          permissions: "0744"
   420        joinConfiguration:
   421          nodeRegistration:
   422            kubeletExtraArgs:
   423              cloud-provider: external
   424              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   425              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   426            name: '{{ ds.meta_data["local_hostname"] }}'
   427        preKubeadmCommands:
   428        - bash -c /tmp/oot-cred-provider.sh
   429        - bash -c /tmp/kubeadm-bootstrap.sh
   430        verbosity: 5
   431  ---
   432  apiVersion: cluster.x-k8s.io/v1beta1
   433  kind: MachineHealthCheck
   434  metadata:
   435    name: ${CLUSTER_NAME}-mhc-0
   436    namespace: default
   437  spec:
   438    clusterName: ${CLUSTER_NAME}
   439    maxUnhealthy: 100%
   440    selector:
   441      matchLabels:
   442        nodepool: pool1
   443    unhealthyConditions:
   444    - status: "True"
   445      timeout: 30s
   446      type: E2ENodeUnhealthy
   447  ---
   448  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   449  kind: AzureClusterIdentity
   450  metadata:
   451    labels:
   452      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   453    name: ${CLUSTER_IDENTITY_NAME}
   454    namespace: default
   455  spec:
   456    allowedNamespaces: {}
   457    clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}
   458    tenantID: ${AZURE_TENANT_ID}
   459    type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity}
   460  ---
   461  apiVersion: addons.cluster.x-k8s.io/v1beta1
   462  kind: ClusterResourceSet
   463  metadata:
   464    name: csi-proxy
   465    namespace: default
   466  spec:
   467    clusterSelector:
   468      matchLabels:
   469        csi-proxy: enabled
   470    resources:
   471    - kind: ConfigMap
   472      name: csi-proxy-addon
   473    strategy: ApplyOnce
   474  ---
   475  apiVersion: addons.cluster.x-k8s.io/v1beta1
   476  kind: ClusterResourceSet
   477  metadata:
   478    name: containerd-logger-${CLUSTER_NAME}
   479    namespace: default
   480  spec:
   481    clusterSelector:
   482      matchLabels:
   483        containerd-logger: enabled
   484    resources:
   485    - kind: ConfigMap
   486      name: containerd-logger-${CLUSTER_NAME}
   487    strategy: ApplyOnce
   488  ---
   489  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   490  kind: HelmChartProxy
   491  metadata:
   492    name: azuredisk-csi-driver-chart
   493    namespace: default
   494  spec:
   495    chartName: azuredisk-csi-driver
   496    clusterSelector:
   497      matchLabels:
   498        azuredisk-csi: "true"
   499    namespace: kube-system
   500    releaseName: azuredisk-csi-driver-oot
   501    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   502    valuesTemplate: |-
   503      controller:
   504        replicas: 1
   505        runOnControlPlane: true
   506      windows:
   507        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   508  ---
   509  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   510  kind: HelmChartProxy
   511  metadata:
   512    name: cloud-provider-azure-chart
   513    namespace: default
   514  spec:
   515    chartName: cloud-provider-azure
   516    clusterSelector:
   517      matchLabels:
   518        cloud-provider: azure
   519    releaseName: cloud-provider-azure-oot
   520    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   521    valuesTemplate: |
   522      infra:
   523        clusterName: {{ .Cluster.metadata.name }}
   524      cloudControllerManager:
   525        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   526        logVerbosity: 4
   527  ---
   528  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   529  kind: HelmChartProxy
   530  metadata:
   531    name: cloud-provider-azure-chart-ci
   532    namespace: default
   533  spec:
   534    chartName: cloud-provider-azure
   535    clusterSelector:
   536      matchLabels:
   537        cloud-provider: azure-ci
   538    releaseName: cloud-provider-azure-oot
   539    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   540    valuesTemplate: |
   541      infra:
   542        clusterName: {{ .Cluster.metadata.name }}
   543      cloudControllerManager:
   544        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   545        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   546        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   547        imageName: "${CCM_IMAGE_NAME:-""}"
   548        imageRepository: "${IMAGE_REGISTRY:-""}"
   549        imageTag: "${IMAGE_TAG_CCM:-""}"
   550        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   551        replicas: ${CCM_COUNT:-1}
   552        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   553      cloudNodeManager:
   554        imageName: "${CNM_IMAGE_NAME:-""}"
   555        imageRepository: "${IMAGE_REGISTRY:-""}"
   556        imageTag: "${IMAGE_TAG_CNM:-""}"
   557  ---
   558  apiVersion: v1
   559  data:
   560    csi-proxy: |
   561      apiVersion: apps/v1
   562      kind: DaemonSet
   563      metadata:
   564        labels:
   565          k8s-app: csi-proxy
   566        name: csi-proxy
   567        namespace: kube-system
   568      spec:
   569        selector:
   570          matchLabels:
   571            k8s-app: csi-proxy
   572        template:
   573          metadata:
   574            labels:
   575              k8s-app: csi-proxy
   576          spec:
   577            nodeSelector:
   578              "kubernetes.io/os": windows
   579            securityContext:
   580              windowsOptions:
   581                hostProcess: true
   582                runAsUserName: "NT AUTHORITY\\SYSTEM"
   583            hostNetwork: true
   584            containers:
   585              - name: csi-proxy
   586                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   587  kind: ConfigMap
   588  metadata:
   589    annotations:
   590      note: generated
   591    labels:
   592      type: generated
   593    name: csi-proxy-addon
   594    namespace: default
   595  ---
   596  apiVersion: v1
   597  data:
   598    containerd-windows-logger: |
   599      apiVersion: apps/v1
   600      kind: DaemonSet
   601      metadata:
   602        labels:
   603          k8s-app: containerd-logger
   604        name: containerd-logger
   605        namespace: kube-system
   606      spec:
   607        selector:
   608          matchLabels:
   609            k8s-app: containerd-logger
   610        template:
   611          metadata:
   612            labels:
   613              k8s-app: containerd-logger
   614          spec:
   615            securityContext:
   616              windowsOptions:
   617                hostProcess: true
   618                runAsUserName: "NT AUTHORITY\\system"
   619            hostNetwork: true
   620            containers:
   621            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
   622              args: [ "config.json" ]
   623              name: containerd-logger
   624              imagePullPolicy: Always
   625              volumeMounts:
   626              - name: containerd-logger-config
   627                mountPath: /config.json
   628                subPath: config.json
   629            nodeSelector:
   630              kubernetes.io/os: windows
   631            tolerations:
   632            - key: CriticalAddonsOnly
   633              operator: Exists
   634            - operator: Exists
   635            volumes:
   636            - configMap:
   637                name: containerd-logger-config
   638              name: containerd-logger-config
   639        updateStrategy:
   640          type: RollingUpdate
   641      ---
   642      kind: ConfigMap
   643      apiVersion: v1
   644      metadata:
   645        name: containerd-logger-config
   646        namespace: kube-system
   647      data:
   648        config.json: |
   649          {
   650            "inputs": [
   651              {
   652                "type": "ETW",
   653                "sessionNamePrefix": "containerd",
   654                "cleanupOldSessions": true,
   655                "reuseExistingSession": true,
   656                "providers": [
   657                  {
   658                    "providerName": "Microsoft.Virtualization.RunHCS",
   659                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
   660                    "level": "Verbose"
   661                  },
   662                  {
   663                    "providerName": "ContainerD",
   664                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
   665                    "level": "Verbose"
   666                  }
   667                ]
   668              }
   669            ],
   670            "filters": [
   671              {
   672                  "type": "drop",
   673                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
   674              },
   675              {
   676                  "type": "drop",
   677                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
   678              },
   679              {
   680                  "type": "drop",
   681                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
   682              },
   683              {
   684                  "type": "drop",
   685                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
   686              },
   687              {
   688                  "type": "drop",
   689                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
   690              },
   691              {
   692                  "type": "drop",
   693                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
   694              },
   695              {
   696                  "type": "drop",
   697                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
   698              }
   699            ],
   700            "outputs": [
   701              {
   702                "type": "StdOutput"
   703              }
   704            ],
   705            "schemaVersion": "2016-08-11"
   706          }
   707  kind: ConfigMap
   708  metadata:
   709    annotations:
   710      note: generated
   711    labels:
   712      type: generated
   713    name: containerd-logger-${CLUSTER_NAME}
   714    namespace: default
   715  ---
   716  apiVersion: addons.cluster.x-k8s.io/v1beta1
   717  kind: ClusterResourceSet
   718  metadata:
   719    name: metrics-server-${CLUSTER_NAME}
   720    namespace: default
   721  spec:
   722    clusterSelector:
   723      matchLabels:
   724        metrics-server: enabled
   725    resources:
   726    - kind: ConfigMap
   727      name: metrics-server-${CLUSTER_NAME}
   728    strategy: ApplyOnce
   729  ---
   730  apiVersion: v1
   731  data:
   732    metrics-server: |
   733      apiVersion: v1
   734      kind: ServiceAccount
   735      metadata:
   736        labels:
   737          k8s-app: metrics-server
   738        name: metrics-server
   739        namespace: kube-system
   740      ---
   741      apiVersion: rbac.authorization.k8s.io/v1
   742      kind: ClusterRole
   743      metadata:
   744        labels:
   745          k8s-app: metrics-server
   746          rbac.authorization.k8s.io/aggregate-to-admin: "true"
   747          rbac.authorization.k8s.io/aggregate-to-edit: "true"
   748          rbac.authorization.k8s.io/aggregate-to-view: "true"
   749        name: system:aggregated-metrics-reader
   750      rules:
   751      - apiGroups:
   752        - metrics.k8s.io
   753        resources:
   754        - pods
   755        - nodes
   756        verbs:
   757        - get
   758        - list
   759        - watch
   760      ---
   761      apiVersion: rbac.authorization.k8s.io/v1
   762      kind: ClusterRole
   763      metadata:
   764        labels:
   765          k8s-app: metrics-server
   766        name: system:metrics-server
   767      rules:
   768      - apiGroups:
   769        - ""
   770        resources:
   771        - nodes/metrics
   772        verbs:
   773        - get
   774      - apiGroups:
   775        - ""
   776        resources:
   777        - pods
   778        - nodes
   779        verbs:
   780        - get
   781        - list
   782        - watch
   783      ---
   784      apiVersion: rbac.authorization.k8s.io/v1
   785      kind: RoleBinding
   786      metadata:
   787        labels:
   788          k8s-app: metrics-server
   789        name: metrics-server-auth-reader
   790        namespace: kube-system
   791      roleRef:
   792        apiGroup: rbac.authorization.k8s.io
   793        kind: Role
   794        name: extension-apiserver-authentication-reader
   795      subjects:
   796      - kind: ServiceAccount
   797        name: metrics-server
   798        namespace: kube-system
   799      ---
   800      apiVersion: rbac.authorization.k8s.io/v1
   801      kind: ClusterRoleBinding
   802      metadata:
   803        labels:
   804          k8s-app: metrics-server
   805        name: metrics-server:system:auth-delegator
   806      roleRef:
   807        apiGroup: rbac.authorization.k8s.io
   808        kind: ClusterRole
   809        name: system:auth-delegator
   810      subjects:
   811      - kind: ServiceAccount
   812        name: metrics-server
   813        namespace: kube-system
   814      ---
   815      apiVersion: rbac.authorization.k8s.io/v1
   816      kind: ClusterRoleBinding
   817      metadata:
   818        labels:
   819          k8s-app: metrics-server
   820        name: system:metrics-server
   821      roleRef:
   822        apiGroup: rbac.authorization.k8s.io
   823        kind: ClusterRole
   824        name: system:metrics-server
   825      subjects:
   826      - kind: ServiceAccount
   827        name: metrics-server
   828        namespace: kube-system
   829      ---
   830      apiVersion: v1
   831      kind: Service
   832      metadata:
   833        labels:
   834          k8s-app: metrics-server
   835        name: metrics-server
   836        namespace: kube-system
   837      spec:
   838        ports:
   839        - name: https
   840          port: 443
   841          protocol: TCP
   842          targetPort: https
   843        selector:
   844          k8s-app: metrics-server
   845      ---
   846      apiVersion: apps/v1
   847      kind: Deployment
   848      metadata:
   849        labels:
   850          k8s-app: metrics-server
   851        name: metrics-server
   852        namespace: kube-system
   853      spec:
   854        selector:
   855          matchLabels:
   856            k8s-app: metrics-server
   857        strategy:
   858          rollingUpdate:
   859            maxUnavailable: 0
   860        template:
   861          metadata:
   862            labels:
   863              k8s-app: metrics-server
   864          spec:
   865            containers:
   866            - args:
   867              - --cert-dir=/tmp
   868              - --secure-port=4443
   869              - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
   870              - --kubelet-use-node-status-port
   871              - --metric-resolution=15s
   872              - --kubelet-insecure-tls
   873              image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
   874              imagePullPolicy: IfNotPresent
   875              livenessProbe:
   876                failureThreshold: 3
   877                httpGet:
   878                  path: /livez
   879                  port: https
   880                  scheme: HTTPS
   881                periodSeconds: 10
   882              name: metrics-server
   883              ports:
   884              - containerPort: 4443
   885                name: https
   886                protocol: TCP
   887              readinessProbe:
   888                failureThreshold: 3
   889                httpGet:
   890                  path: /readyz
   891                  port: https
   892                  scheme: HTTPS
   893                initialDelaySeconds: 20
   894                periodSeconds: 10
   895              resources:
   896                requests:
   897                  cpu: 100m
   898                  memory: 200Mi
   899              securityContext:
   900                allowPrivilegeEscalation: false
   901                readOnlyRootFilesystem: true
   902                runAsNonRoot: true
   903                runAsUser: 1000
   904              volumeMounts:
   905              - mountPath: /tmp
   906                name: tmp-dir
   907            nodeSelector:
   908              kubernetes.io/os: linux
   909            priorityClassName: system-cluster-critical
   910            serviceAccountName: metrics-server
   911            tolerations:
   912            - effect: NoSchedule
   913              key: node-role.kubernetes.io/master
   914              operator: Exists
   915            - effect: NoSchedule
   916              key: node-role.kubernetes.io/control-plane
   917              operator: Exists
   918            volumes:
   919            - emptyDir: {}
   920              name: tmp-dir
   921      ---
   922      apiVersion: apiregistration.k8s.io/v1
   923      kind: APIService
   924      metadata:
   925        labels:
   926          k8s-app: metrics-server
   927        name: v1beta1.metrics.k8s.io
   928      spec:
   929        group: metrics.k8s.io
   930        groupPriorityMinimum: 100
   931        insecureSkipTLSVerify: true
   932        service:
   933          name: metrics-server
   934          namespace: kube-system
   935        version: v1beta1
   936        versionPriority: 100
   937  kind: ConfigMap
   938  metadata:
   939    annotations:
   940      note: generated
   941    labels:
   942      type: generated
   943    name: metrics-server-${CLUSTER_NAME}
   944    namespace: default
   945  ---
   946  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   947  kind: HelmChartProxy
   948  metadata:
   949    name: calico-dual-stack
   950    namespace: default
   951  spec:
   952    chartName: tigera-operator
   953    clusterSelector:
   954      matchLabels:
   955        cni: calico-dual-stack
   956    namespace: tigera-operator
   957    releaseName: projectcalico
   958    repoURL: https://docs.tigera.io/calico/charts
   959    valuesTemplate: |
   960      installation:
   961        cni:
   962          type: Calico
   963          ipam:
   964            type: HostLocal
   965        calicoNetwork:
   966          bgp: Disabled
   967          mtu: 1350
   968          ipPools:
   969          - blockSize: 26
   970            cidr: {{ index .Cluster.spec.clusterNetwork.pods.cidrBlocks 0 }}
   971            encapsulation: None
   972            natOutgoing: Enabled
   973            nodeSelector: all()
   974          - blockSize: 122
   975            cidr: {{ index .Cluster.spec.clusterNetwork.pods.cidrBlocks 1 }}
   976            encapsulation: None
   977            natOutgoing: Enabled
   978            nodeSelector: all()
   979        registry: mcr.microsoft.com/oss
   980      # Image and registry configuration for the tigera/operator pod.
   981      tigeraOperator:
   982        image: tigera/operator
   983        registry: mcr.microsoft.com/oss
   984      calicoctl:
   985        image: mcr.microsoft.com/oss/calico/ctl
   986    version: ${CALICO_VERSION}