sigs.k8s.io/cluster-api-provider-azure@v1.17.0/templates/test/ci/cluster-template-prow-ci-version-dra.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: enabled
     9      csi-proxy: enabled
    10      metrics-server: enabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 192.168.0.0/16
    18    controlPlaneRef:
    19      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    20      kind: KubeadmControlPlane
    21      name: ${CLUSTER_NAME}-control-plane
    22    infrastructureRef:
    23      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    24      kind: AzureCluster
    25      name: ${CLUSTER_NAME}
    26  ---
    27  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    28  kind: AzureCluster
    29  metadata:
    30    name: ${CLUSTER_NAME}
    31    namespace: default
    32  spec:
    33    additionalTags:
    34      buildProvenance: ${BUILD_PROVENANCE}
    35      creationTimestamp: ${TIMESTAMP}
    36      jobName: ${JOB_NAME}
    37    identityRef:
    38      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    39      kind: AzureClusterIdentity
    40      name: ${CLUSTER_IDENTITY_NAME}
    41    location: ${AZURE_LOCATION}
    42    networkSpec:
    43      subnets:
    44      - name: control-plane-subnet
    45        role: control-plane
    46      - name: node-subnet
    47        role: node
    48      vnet:
    49        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    50    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    51    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    52  ---
    53  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    54  kind: KubeadmControlPlane
    55  metadata:
    56    name: ${CLUSTER_NAME}-control-plane
    57    namespace: default
    58  spec:
    59    kubeadmConfigSpec:
    60      clusterConfiguration:
    61        apiServer:
    62          extraArgs:
    63            cloud-provider: external
    64            feature-gates: ${K8S_FEATURE_GATES:-"DynamicResourceAllocation=true"}
    65            runtime-config: resource.k8s.io/v1alpha3=true
    66          timeoutForControlPlane: 20m
    67        controllerManager:
    68          extraArgs:
    69            allocate-node-cidrs: "false"
    70            cloud-provider: external
    71            cluster-name: ${CLUSTER_NAME}
    72            feature-gates: HPAContainerMetrics=true,DynamicResourceAllocation=true
    73            v: "4"
    74        etcd:
    75          local:
    76            dataDir: /var/lib/etcddisk/etcd
    77            extraArgs:
    78              quota-backend-bytes: "8589934592"
    79        kubernetesVersion: ci/${CI_VERSION}
    80        scheduler:
    81          extraArgs:
    82            feature-gates: DynamicResourceAllocation=true
    83      diskSetup:
    84        filesystems:
    85        - device: /dev/disk/azure/scsi1/lun0
    86          extraOpts:
    87          - -E
    88          - lazy_itable_init=1,lazy_journal_init=1
    89          filesystem: ext4
    90          label: etcd_disk
    91        - device: ephemeral0.1
    92          filesystem: ext4
    93          label: ephemeral0
    94          replaceFS: ntfs
    95        partitions:
    96        - device: /dev/disk/azure/scsi1/lun0
    97          layout: true
    98          overwrite: false
    99          tableType: gpt
   100      files:
   101      - contentFrom:
   102          secret:
   103            key: control-plane-azure.json
   104            name: ${CLUSTER_NAME}-control-plane-azure-json
   105        owner: root:root
   106        path: /etc/kubernetes/azure.json
   107        permissions: "0644"
   108      - content: |
   109          #!/bin/bash
   110  
   111          set -o nounset
   112          set -o pipefail
   113          set -o errexit
   114          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   115  
   116          echo "Use OOT credential provider"
   117          mkdir -p /var/lib/kubelet/credential-provider
   118          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   119          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   120          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   121          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   122        owner: root:root
   123        path: /tmp/oot-cred-provider.sh
   124        permissions: "0744"
   125      - content: |
   126          #!/bin/bash
   127  
   128          set -o nounset
   129          set -o pipefail
   130          set -o errexit
   131          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   132  
   133          # This test installs release packages or binaries that are a result of the CI and release builds.
   134          # It runs '... --version' commands to verify that the binaries are correctly installed
   135          # and finally uninstalls the packages.
   136          # For the release packages it tests all versions in the support skew.
   137          LINE_SEPARATOR="*************************************************"
   138          echo "$$LINE_SEPARATOR"
   139          CI_VERSION=${CI_VERSION}
   140          if [[ "$${CI_VERSION}" != "" ]]; then
   141            CI_DIR=/tmp/k8s-ci
   142            mkdir -p $$CI_DIR
   143            declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   144            declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   145            CONTAINER_EXT="tar"
   146            echo "* testing CI version $$CI_VERSION"
   147            # Check for semver
   148            if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   149              VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   150              DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   151              curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   152              echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   153              apt-get update
   154              # replace . with \.
   155              VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   156              PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   157              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   158                echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   159                DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   160              done
   161            else
   162              CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   163              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   164                echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   165                wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   166                chmod +x "$$CI_DIR/$$CI_PACKAGE"
   167                mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   168              done
   169              IMAGE_REGISTRY_PREFIX=registry.k8s.io
   170              for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   171                echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   172                wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   173                $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   174                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   175                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   176              done
   177            fi
   178            systemctl restart kubelet
   179          fi
   180          echo "* checking binary versions"
   181          echo "ctr version: " $(ctr version)
   182          echo "kubeadm version: " $(kubeadm version -o=short)
   183          echo "kubectl version: " $(kubectl version --client=true)
   184          echo "kubelet version: " $(kubelet --version)
   185          echo "$$LINE_SEPARATOR"
   186        owner: root:root
   187        path: /tmp/kubeadm-bootstrap.sh
   188        permissions: "0744"
   189      - content: |
   190          #!/bin/bash
   191  
   192          echo "enabling containerd CDI plugin"
   193          sed -i '/\[plugins."io.containerd.grpc.v1.cri"\]/a\    enable_cdi = true' /etc/containerd/config.toml
   194          systemctl restart containerd
   195        owner: root:root
   196        path: /tmp/containerd-config.sh
   197        permissions: "0744"
   198      initConfiguration:
   199        nodeRegistration:
   200          kubeletExtraArgs:
   201            cloud-provider: external
   202            feature-gates: DynamicResourceAllocation=true
   203            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   204            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   205          name: '{{ ds.meta_data["local_hostname"] }}'
   206      joinConfiguration:
   207        nodeRegistration:
   208          kubeletExtraArgs:
   209            cloud-provider: external
   210            feature-gates: DynamicResourceAllocation=true
   211            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   212            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   213          name: '{{ ds.meta_data["local_hostname"] }}'
   214      mounts:
   215      - - LABEL=etcd_disk
   216        - /var/lib/etcddisk
   217      postKubeadmCommands: []
   218      preKubeadmCommands:
   219      - bash -c /tmp/containerd-config.sh
   220      - bash -c /tmp/oot-cred-provider.sh
   221      - bash -c /tmp/kubeadm-bootstrap.sh
   222      verbosity: 5
   223    machineTemplate:
   224      infrastructureRef:
   225        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   226        kind: AzureMachineTemplate
   227        name: ${CLUSTER_NAME}-control-plane
   228    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   229    version: ${KUBERNETES_VERSION}
   230  ---
   231  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   232  kind: AzureMachineTemplate
   233  metadata:
   234    name: ${CLUSTER_NAME}-control-plane
   235    namespace: default
   236  spec:
   237    template:
   238      spec:
   239        dataDisks:
   240        - diskSizeGB: 256
   241          lun: 0
   242          nameSuffix: etcddisk
   243        identity: UserAssigned
   244        image:
   245          marketplace:
   246            offer: capi
   247            publisher: cncf-upstream
   248            sku: ubuntu-2204-gen1
   249            version: latest
   250        osDisk:
   251          diskSizeGB: 128
   252          osType: Linux
   253        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   254        userAssignedIdentities:
   255        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
   256        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   257  ---
   258  apiVersion: cluster.x-k8s.io/v1beta1
   259  kind: MachineDeployment
   260  metadata:
   261    name: ${CLUSTER_NAME}-md-0
   262    namespace: default
   263  spec:
   264    clusterName: ${CLUSTER_NAME}
   265    replicas: ${WORKER_MACHINE_COUNT:=2}
   266    selector: {}
   267    template:
   268      metadata:
   269        labels:
   270          nodepool: pool1
   271      spec:
   272        bootstrap:
   273          configRef:
   274            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   275            kind: KubeadmConfigTemplate
   276            name: ${CLUSTER_NAME}-md-0
   277        clusterName: ${CLUSTER_NAME}
   278        infrastructureRef:
   279          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   280          kind: AzureMachineTemplate
   281          name: ${CLUSTER_NAME}-md-0
   282        version: ${KUBERNETES_VERSION}
   283  ---
   284  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   285  kind: AzureMachineTemplate
   286  metadata:
   287    name: ${CLUSTER_NAME}-md-0
   288    namespace: default
   289  spec:
   290    template:
   291      spec:
   292        identity: UserAssigned
   293        image:
   294          marketplace:
   295            offer: capi
   296            publisher: cncf-upstream
   297            sku: ubuntu-2204-gen1
   298            version: latest
   299        osDisk:
   300          diskSizeGB: 128
   301          osType: Linux
   302        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   303        userAssignedIdentities:
   304        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
   305        vmExtensions:
   306        - name: CustomScript
   307          protectedSettings:
   308            commandToExecute: |
   309              #!/bin/sh
   310              echo "This script is a no-op used for extension testing purposes ..."
   311              touch test_file
   312          publisher: Microsoft.Azure.Extensions
   313          version: "2.1"
   314        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   315  ---
   316  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   317  kind: KubeadmConfigTemplate
   318  metadata:
   319    name: ${CLUSTER_NAME}-md-0
   320    namespace: default
   321  spec:
   322    template:
   323      spec:
   324        files:
   325        - contentFrom:
   326            secret:
   327              key: worker-node-azure.json
   328              name: ${CLUSTER_NAME}-md-0-azure-json
   329          owner: root:root
   330          path: /etc/kubernetes/azure.json
   331          permissions: "0644"
   332        - content: |
   333            #!/bin/bash
   334  
   335            set -o nounset
   336            set -o pipefail
   337            set -o errexit
   338            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   339  
   340            echo "Use OOT credential provider"
   341            mkdir -p /var/lib/kubelet/credential-provider
   342            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   343            chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   344            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   345            chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   346          owner: root:root
   347          path: /tmp/oot-cred-provider.sh
   348          permissions: "0744"
   349        - content: |
   350            #!/bin/bash
   351  
   352            set -o nounset
   353            set -o pipefail
   354            set -o errexit
   355            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   356  
   357            # This test installs release packages or binaries that are a result of the CI and release builds.
   358            # It runs '... --version' commands to verify that the binaries are correctly installed
   359            # and finally uninstalls the packages.
   360            # For the release packages it tests all versions in the support skew.
   361            LINE_SEPARATOR="*************************************************"
   362            echo "$$LINE_SEPARATOR"
   363            CI_VERSION=${CI_VERSION}
   364            if [[ "$${CI_VERSION}" != "" ]]; then
   365              CI_DIR=/tmp/k8s-ci
   366              mkdir -p $$CI_DIR
   367              declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   368              declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   369              CONTAINER_EXT="tar"
   370              echo "* testing CI version $$CI_VERSION"
   371              # Check for semver
   372              if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   373                VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   374                DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   375                curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   376                echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   377                apt-get update
   378                # replace . with \.
   379                VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   380                PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   381                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   382                  echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   383                  DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   384                done
   385              else
   386                CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   387                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   388                  echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   389                  wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   390                  chmod +x "$$CI_DIR/$$CI_PACKAGE"
   391                  mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   392                done
   393                IMAGE_REGISTRY_PREFIX=registry.k8s.io
   394                for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   395                  echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   396                  wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   397                  $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   398                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   399                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   400                done
   401              fi
   402              systemctl restart kubelet
   403            fi
   404            echo "* checking binary versions"
   405            echo "ctr version: " $(ctr version)
   406            echo "kubeadm version: " $(kubeadm version -o=short)
   407            echo "kubectl version: " $(kubectl version --client=true)
   408            echo "kubelet version: " $(kubelet --version)
   409            echo "$$LINE_SEPARATOR"
   410          owner: root:root
   411          path: /tmp/kubeadm-bootstrap.sh
   412          permissions: "0744"
   413        - content: |
   414            #!/bin/bash
   415  
   416            echo "enabling containerd CDI plugin"
   417            sed -i '/\[plugins."io.containerd.grpc.v1.cri"\]/a\    enable_cdi = true' /etc/containerd/config.toml
   418            systemctl restart containerd
   419          owner: root:root
   420          path: /tmp/containerd-config.sh
   421          permissions: "0744"
   422        joinConfiguration:
   423          nodeRegistration:
   424            kubeletExtraArgs:
   425              cloud-provider: external
   426              feature-gates: ${NODE_FEATURE_GATES:-"DynamicResourceAllocation=true"}
   427              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   428              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   429            name: '{{ ds.meta_data["local_hostname"] }}'
   430        preKubeadmCommands:
   431        - bash -c /tmp/containerd-config.sh
   432        - bash -c /tmp/oot-cred-provider.sh
   433        - bash -c /tmp/kubeadm-bootstrap.sh
   434        verbosity: 5
   435  ---
   436  apiVersion: cluster.x-k8s.io/v1beta1
   437  kind: MachineHealthCheck
   438  metadata:
   439    name: ${CLUSTER_NAME}-mhc-0
   440    namespace: default
   441  spec:
   442    clusterName: ${CLUSTER_NAME}
   443    maxUnhealthy: 100%
   444    selector:
   445      matchLabels:
   446        nodepool: pool1
   447    unhealthyConditions:
   448    - status: "True"
   449      timeout: 30s
   450      type: E2ENodeUnhealthy
   451  ---
   452  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   453  kind: AzureClusterIdentity
   454  metadata:
   455    labels:
   456      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   457    name: ${CLUSTER_IDENTITY_NAME}
   458    namespace: default
   459  spec:
   460    allowedNamespaces: {}
   461    clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}
   462    tenantID: ${AZURE_TENANT_ID}
   463    type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity}
   464  ---
   465  apiVersion: addons.cluster.x-k8s.io/v1beta1
   466  kind: ClusterResourceSet
   467  metadata:
   468    name: csi-proxy
   469    namespace: default
   470  spec:
   471    clusterSelector:
   472      matchLabels:
   473        csi-proxy: enabled
   474    resources:
   475    - kind: ConfigMap
   476      name: csi-proxy-addon
   477    strategy: ApplyOnce
   478  ---
   479  apiVersion: addons.cluster.x-k8s.io/v1beta1
   480  kind: ClusterResourceSet
   481  metadata:
   482    name: containerd-logger-${CLUSTER_NAME}
   483    namespace: default
   484  spec:
   485    clusterSelector:
   486      matchLabels:
   487        containerd-logger: enabled
   488    resources:
   489    - kind: ConfigMap
   490      name: containerd-logger-${CLUSTER_NAME}
   491    strategy: ApplyOnce
   492  ---
   493  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   494  kind: HelmChartProxy
   495  metadata:
   496    name: calico
   497    namespace: default
   498  spec:
   499    chartName: tigera-operator
   500    clusterSelector:
   501      matchLabels:
   502        cni: calico
   503    namespace: tigera-operator
   504    releaseName: projectcalico
   505    repoURL: https://docs.tigera.io/calico/charts
   506    valuesTemplate: |-
   507      installation:
   508        cni:
   509          type: Calico
   510        calicoNetwork:
   511          bgp: Disabled
   512          mtu: 1350
   513          ipPools:
   514          ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
   515          - cidr: {{ $cidr }}
   516            encapsulation: VXLAN{{end}}
   517        registry: mcr.microsoft.com/oss
   518      # Image and registry configuration for the tigera/operator pod.
   519      tigeraOperator:
   520        image: tigera/operator
   521        registry: mcr.microsoft.com/oss
   522      calicoctl:
   523        image: mcr.microsoft.com/oss/calico/ctl
   524    version: ${CALICO_VERSION}
   525  ---
   526  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   527  kind: HelmChartProxy
   528  metadata:
   529    name: azuredisk-csi-driver-chart
   530    namespace: default
   531  spec:
   532    chartName: azuredisk-csi-driver
   533    clusterSelector:
   534      matchLabels:
   535        azuredisk-csi: "true"
   536    namespace: kube-system
   537    releaseName: azuredisk-csi-driver-oot
   538    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   539    valuesTemplate: |-
   540      controller:
   541        replicas: 1
   542        runOnControlPlane: true
   543      windows:
   544        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   545  ---
   546  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   547  kind: HelmChartProxy
   548  metadata:
   549    name: cloud-provider-azure-chart
   550    namespace: default
   551  spec:
   552    chartName: cloud-provider-azure
   553    clusterSelector:
   554      matchLabels:
   555        cloud-provider: azure
   556    releaseName: cloud-provider-azure-oot
   557    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   558    valuesTemplate: |
   559      infra:
   560        clusterName: {{ .Cluster.metadata.name }}
   561      cloudControllerManager:
   562        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   563        logVerbosity: 4
   564  ---
   565  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   566  kind: HelmChartProxy
   567  metadata:
   568    name: cloud-provider-azure-chart-ci
   569    namespace: default
   570  spec:
   571    chartName: cloud-provider-azure
   572    clusterSelector:
   573      matchLabels:
   574        cloud-provider: azure-ci
   575    releaseName: cloud-provider-azure-oot
   576    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   577    valuesTemplate: |
   578      infra:
   579        clusterName: {{ .Cluster.metadata.name }}
   580      cloudControllerManager:
   581        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   582        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   583        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   584        imageName: "${CCM_IMAGE_NAME:-""}"
   585        imageRepository: "${IMAGE_REGISTRY:-""}"
   586        imageTag: "${IMAGE_TAG_CCM:-""}"
   587        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   588        replicas: ${CCM_COUNT:-1}
   589        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   590      cloudNodeManager:
   591        imageName: "${CNM_IMAGE_NAME:-""}"
   592        imageRepository: "${IMAGE_REGISTRY:-""}"
   593        imageTag: "${IMAGE_TAG_CNM:-""}"
   594  ---
   595  apiVersion: v1
   596  data:
   597    csi-proxy: |
   598      apiVersion: apps/v1
   599      kind: DaemonSet
   600      metadata:
   601        labels:
   602          k8s-app: csi-proxy
   603        name: csi-proxy
   604        namespace: kube-system
   605      spec:
   606        selector:
   607          matchLabels:
   608            k8s-app: csi-proxy
   609        template:
   610          metadata:
   611            labels:
   612              k8s-app: csi-proxy
   613          spec:
   614            nodeSelector:
   615              "kubernetes.io/os": windows
   616            securityContext:
   617              windowsOptions:
   618                hostProcess: true
   619                runAsUserName: "NT AUTHORITY\\SYSTEM"
   620            hostNetwork: true
   621            containers:
   622              - name: csi-proxy
   623                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   624  kind: ConfigMap
   625  metadata:
   626    annotations:
   627      note: generated
   628    labels:
   629      type: generated
   630    name: csi-proxy-addon
   631    namespace: default
   632  ---
   633  apiVersion: v1
   634  data:
   635    containerd-windows-logger: |
   636      apiVersion: apps/v1
   637      kind: DaemonSet
   638      metadata:
   639        labels:
   640          k8s-app: containerd-logger
   641        name: containerd-logger
   642        namespace: kube-system
   643      spec:
   644        selector:
   645          matchLabels:
   646            k8s-app: containerd-logger
   647        template:
   648          metadata:
   649            labels:
   650              k8s-app: containerd-logger
   651          spec:
   652            securityContext:
   653              windowsOptions:
   654                hostProcess: true
   655                runAsUserName: "NT AUTHORITY\\system"
   656            hostNetwork: true
   657            containers:
   658            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
   659              args: [ "config.json" ]
   660              name: containerd-logger
   661              imagePullPolicy: Always
   662              volumeMounts:
   663              - name: containerd-logger-config
   664                mountPath: /config.json
   665                subPath: config.json
   666            nodeSelector:
   667              kubernetes.io/os: windows
   668            tolerations:
   669            - key: CriticalAddonsOnly
   670              operator: Exists
   671            - operator: Exists
   672            volumes:
   673            - configMap:
   674                name: containerd-logger-config
   675              name: containerd-logger-config
   676        updateStrategy:
   677          type: RollingUpdate
   678      ---
   679      kind: ConfigMap
   680      apiVersion: v1
   681      metadata:
   682        name: containerd-logger-config
   683        namespace: kube-system
   684      data:
   685        config.json: |
   686          {
   687            "inputs": [
   688              {
   689                "type": "ETW",
   690                "sessionNamePrefix": "containerd",
   691                "cleanupOldSessions": true,
   692                "reuseExistingSession": true,
   693                "providers": [
   694                  {
   695                    "providerName": "Microsoft.Virtualization.RunHCS",
   696                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
   697                    "level": "Verbose"
   698                  },
   699                  {
   700                    "providerName": "ContainerD",
   701                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
   702                    "level": "Verbose"
   703                  }
   704                ]
   705              }
   706            ],
   707            "filters": [
   708              {
   709                  "type": "drop",
   710                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
   711              },
   712              {
   713                  "type": "drop",
   714                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
   715              },
   716              {
   717                  "type": "drop",
   718                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
   719              },
   720              {
   721                  "type": "drop",
   722                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
   723              },
   724              {
   725                  "type": "drop",
   726                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
   727              },
   728              {
   729                  "type": "drop",
   730                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
   731              },
   732              {
   733                  "type": "drop",
   734                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
   735              }
   736            ],
   737            "outputs": [
   738              {
   739                "type": "StdOutput"
   740              }
   741            ],
   742            "schemaVersion": "2016-08-11"
   743          }
   744  kind: ConfigMap
   745  metadata:
   746    annotations:
   747      note: generated
   748    labels:
   749      type: generated
   750    name: containerd-logger-${CLUSTER_NAME}
   751    namespace: default
   752  ---
   753  apiVersion: addons.cluster.x-k8s.io/v1beta1
   754  kind: ClusterResourceSet
   755  metadata:
   756    name: metrics-server-${CLUSTER_NAME}
   757    namespace: default
   758  spec:
   759    clusterSelector:
   760      matchLabels:
   761        metrics-server: enabled
   762    resources:
   763    - kind: ConfigMap
   764      name: metrics-server-${CLUSTER_NAME}
   765    strategy: ApplyOnce
   766  ---
   767  apiVersion: v1
   768  data:
   769    metrics-server: |
   770      apiVersion: v1
   771      kind: ServiceAccount
   772      metadata:
   773        labels:
   774          k8s-app: metrics-server
   775        name: metrics-server
   776        namespace: kube-system
   777      ---
   778      apiVersion: rbac.authorization.k8s.io/v1
   779      kind: ClusterRole
   780      metadata:
   781        labels:
   782          k8s-app: metrics-server
   783          rbac.authorization.k8s.io/aggregate-to-admin: "true"
   784          rbac.authorization.k8s.io/aggregate-to-edit: "true"
   785          rbac.authorization.k8s.io/aggregate-to-view: "true"
   786        name: system:aggregated-metrics-reader
   787      rules:
   788      - apiGroups:
   789        - metrics.k8s.io
   790        resources:
   791        - pods
   792        - nodes
   793        verbs:
   794        - get
   795        - list
   796        - watch
   797      ---
   798      apiVersion: rbac.authorization.k8s.io/v1
   799      kind: ClusterRole
   800      metadata:
   801        labels:
   802          k8s-app: metrics-server
   803        name: system:metrics-server
   804      rules:
   805      - apiGroups:
   806        - ""
   807        resources:
   808        - nodes/metrics
   809        verbs:
   810        - get
   811      - apiGroups:
   812        - ""
   813        resources:
   814        - pods
   815        - nodes
   816        verbs:
   817        - get
   818        - list
   819        - watch
   820      ---
   821      apiVersion: rbac.authorization.k8s.io/v1
   822      kind: RoleBinding
   823      metadata:
   824        labels:
   825          k8s-app: metrics-server
   826        name: metrics-server-auth-reader
   827        namespace: kube-system
   828      roleRef:
   829        apiGroup: rbac.authorization.k8s.io
   830        kind: Role
   831        name: extension-apiserver-authentication-reader
   832      subjects:
   833      - kind: ServiceAccount
   834        name: metrics-server
   835        namespace: kube-system
   836      ---
   837      apiVersion: rbac.authorization.k8s.io/v1
   838      kind: ClusterRoleBinding
   839      metadata:
   840        labels:
   841          k8s-app: metrics-server
   842        name: metrics-server:system:auth-delegator
   843      roleRef:
   844        apiGroup: rbac.authorization.k8s.io
   845        kind: ClusterRole
   846        name: system:auth-delegator
   847      subjects:
   848      - kind: ServiceAccount
   849        name: metrics-server
   850        namespace: kube-system
   851      ---
   852      apiVersion: rbac.authorization.k8s.io/v1
   853      kind: ClusterRoleBinding
   854      metadata:
   855        labels:
   856          k8s-app: metrics-server
   857        name: system:metrics-server
   858      roleRef:
   859        apiGroup: rbac.authorization.k8s.io
   860        kind: ClusterRole
   861        name: system:metrics-server
   862      subjects:
   863      - kind: ServiceAccount
   864        name: metrics-server
   865        namespace: kube-system
   866      ---
   867      apiVersion: v1
   868      kind: Service
   869      metadata:
   870        labels:
   871          k8s-app: metrics-server
   872        name: metrics-server
   873        namespace: kube-system
   874      spec:
   875        ports:
   876        - name: https
   877          port: 443
   878          protocol: TCP
   879          targetPort: https
   880        selector:
   881          k8s-app: metrics-server
   882      ---
   883      apiVersion: apps/v1
   884      kind: Deployment
   885      metadata:
   886        labels:
   887          k8s-app: metrics-server
   888        name: metrics-server
   889        namespace: kube-system
   890      spec:
   891        selector:
   892          matchLabels:
   893            k8s-app: metrics-server
   894        strategy:
   895          rollingUpdate:
   896            maxUnavailable: 0
   897        template:
   898          metadata:
   899            labels:
   900              k8s-app: metrics-server
   901          spec:
   902            containers:
   903            - args:
   904              - --cert-dir=/tmp
   905              - --secure-port=4443
   906              - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
   907              - --kubelet-use-node-status-port
   908              - --metric-resolution=15s
   909              - --kubelet-insecure-tls
   910              image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
   911              imagePullPolicy: IfNotPresent
   912              livenessProbe:
   913                failureThreshold: 3
   914                httpGet:
   915                  path: /livez
   916                  port: https
   917                  scheme: HTTPS
   918                periodSeconds: 10
   919              name: metrics-server
   920              ports:
   921              - containerPort: 4443
   922                name: https
   923                protocol: TCP
   924              readinessProbe:
   925                failureThreshold: 3
   926                httpGet:
   927                  path: /readyz
   928                  port: https
   929                  scheme: HTTPS
   930                initialDelaySeconds: 20
   931                periodSeconds: 10
   932              resources:
   933                requests:
   934                  cpu: 100m
   935                  memory: 200Mi
   936              securityContext:
   937                allowPrivilegeEscalation: false
   938                readOnlyRootFilesystem: true
   939                runAsNonRoot: true
   940                runAsUser: 1000
   941              volumeMounts:
   942              - mountPath: /tmp
   943                name: tmp-dir
   944            nodeSelector:
   945              kubernetes.io/os: linux
   946            priorityClassName: system-cluster-critical
   947            serviceAccountName: metrics-server
   948            tolerations:
   949            - effect: NoSchedule
   950              key: node-role.kubernetes.io/master
   951              operator: Exists
   952            - effect: NoSchedule
   953              key: node-role.kubernetes.io/control-plane
   954              operator: Exists
   955            volumes:
   956            - emptyDir: {}
   957              name: tmp-dir
   958      ---
   959      apiVersion: apiregistration.k8s.io/v1
   960      kind: APIService
   961      metadata:
   962        labels:
   963          k8s-app: metrics-server
   964        name: v1beta1.metrics.k8s.io
   965      spec:
   966        group: metrics.k8s.io
   967        groupPriorityMinimum: 100
   968        insecureSkipTLSVerify: true
   969        service:
   970          name: metrics-server
   971          namespace: kube-system
   972        version: v1beta1
   973        versionPriority: 100
   974  kind: ConfigMap
   975  metadata:
   976    annotations:
   977      note: generated
   978    labels:
   979      type: generated
   980    name: metrics-server-${CLUSTER_NAME}
   981    namespace: default