sigs.k8s.io/cluster-api-provider-azure@v1.17.0/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico-ipv6
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: disabled
     9      csi-proxy: disabled
    10      metrics-server: disabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 2001:1234:5678:9a40::/58
    18      services:
    19        cidrBlocks:
    20        - fd00::/108
    21    controlPlaneRef:
    22      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    23      kind: KubeadmControlPlane
    24      name: ${CLUSTER_NAME}-control-plane
    25    infrastructureRef:
    26      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    27      kind: AzureCluster
    28      name: ${CLUSTER_NAME}
    29  ---
    30  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    31  kind: AzureCluster
    32  metadata:
    33    name: ${CLUSTER_NAME}
    34    namespace: default
    35  spec:
    36    additionalTags:
    37      buildProvenance: ${BUILD_PROVENANCE}
    38      creationTimestamp: ${TIMESTAMP}
    39      jobName: ${JOB_NAME}
    40    identityRef:
    41      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    42      kind: AzureClusterIdentity
    43      name: ${CLUSTER_IDENTITY_NAME}
    44    location: ${AZURE_LOCATION}
    45    networkSpec:
    46      subnets:
    47      - cidrBlocks:
    48        - 10.0.0.0/16
    49        - 2001:1234:5678:9abc::/64
    50        name: control-plane-subnet
    51        role: control-plane
    52      - cidrBlocks:
    53        - 10.1.0.0/16
    54        - 2001:1234:5678:9abd::/64
    55        name: node-subnet
    56        role: node
    57      vnet:
    58        cidrBlocks:
    59        - 10.0.0.0/8
    60        - 2001:1234:5678:9a00::/56
    61        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    62    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    63    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    64  ---
    65  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    66  kind: KubeadmControlPlane
    67  metadata:
    68    name: ${CLUSTER_NAME}-control-plane
    69    namespace: default
    70  spec:
    71    kubeadmConfigSpec:
    72      clusterConfiguration:
    73        apiServer:
    74          extraArgs:
    75            bind-address: '::'
    76            cloud-provider: external
    77            feature-gates: ${K8S_FEATURE_GATES:-""}
    78          timeoutForControlPlane: 20m
    79        controllerManager:
    80          extraArgs:
    81            allocate-node-cidrs: "true"
    82            bind-address: '::'
    83            cloud-provider: external
    84            cluster-cidr: 2001:1234:5678:9a40::/58
    85            cluster-name: ${CLUSTER_NAME}
    86            configure-cloud-routes: "true"
    87            feature-gates: HPAContainerMetrics=true
    88            v: "4"
    89        etcd:
    90          local:
    91            dataDir: /var/lib/etcddisk/etcd
    92            extraArgs:
    93              quota-backend-bytes: "8589934592"
    94        kubernetesVersion: ci/${CI_VERSION}
    95        scheduler:
    96          extraArgs:
    97            bind-address: '::'
    98      diskSetup:
    99        filesystems:
   100        - device: /dev/disk/azure/scsi1/lun0
   101          extraOpts:
   102          - -E
   103          - lazy_itable_init=1,lazy_journal_init=1
   104          filesystem: ext4
   105          label: etcd_disk
   106        - device: ephemeral0.1
   107          filesystem: ext4
   108          label: ephemeral0
   109          replaceFS: ntfs
   110        partitions:
   111        - device: /dev/disk/azure/scsi1/lun0
   112          layout: true
   113          overwrite: false
   114          tableType: gpt
   115      files:
   116      - contentFrom:
   117          secret:
   118            key: control-plane-azure.json
   119            name: ${CLUSTER_NAME}-control-plane-azure-json
   120        owner: root:root
   121        path: /etc/kubernetes/azure.json
   122        permissions: "0644"
   123      - content: |
   124          #!/bin/bash
   125  
   126          set -o nounset
   127          set -o pipefail
   128          set -o errexit
   129          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   130  
   131          echo "Use OOT credential provider"
   132          mkdir -p /var/lib/kubelet/credential-provider
   133          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   134          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   135          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   136          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   137        owner: root:root
   138        path: /tmp/oot-cred-provider.sh
   139        permissions: "0744"
   140      - content: |
   141          #!/bin/bash
   142  
   143          set -o nounset
   144          set -o pipefail
   145          set -o errexit
   146          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   147  
   148          # This test installs release packages or binaries that are a result of the CI and release builds.
   149          # It runs '... --version' commands to verify that the binaries are correctly installed
   150          # and finally uninstalls the packages.
   151          # For the release packages it tests all versions in the support skew.
   152          LINE_SEPARATOR="*************************************************"
   153          echo "$$LINE_SEPARATOR"
   154          CI_VERSION=${CI_VERSION}
   155          if [[ "$${CI_VERSION}" != "" ]]; then
   156            CI_DIR=/tmp/k8s-ci
   157            mkdir -p $$CI_DIR
   158            declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   159            declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   160            CONTAINER_EXT="tar"
   161            echo "* testing CI version $$CI_VERSION"
   162            # Check for semver
   163            if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   164              VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   165              DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   166              curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   167              echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   168              apt-get update
   169              # replace . with \.
   170              VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   171              PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   172              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   173                echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   174                DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   175              done
   176            else
   177              CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   178              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   179                echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   180                wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   181                chmod +x "$$CI_DIR/$$CI_PACKAGE"
   182                mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   183              done
   184              IMAGE_REGISTRY_PREFIX=registry.k8s.io
   185              for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   186                echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   187                wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   188                $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   189                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   190                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   191              done
   192            fi
   193            systemctl restart kubelet
   194          fi
   195          echo "* checking binary versions"
   196          echo "ctr version: " $(ctr version)
   197          echo "kubeadm version: " $(kubeadm version -o=short)
   198          echo "kubectl version: " $(kubectl version --client=true)
   199          echo "kubelet version: " $(kubelet --version)
   200          echo "$$LINE_SEPARATOR"
   201        owner: root:root
   202        path: /tmp/kubeadm-bootstrap.sh
   203        permissions: "0744"
   204      initConfiguration:
   205        localAPIEndpoint:
   206          advertiseAddress: '::'
   207          bindPort: 6443
   208        nodeRegistration:
   209          kubeletExtraArgs:
   210            cloud-provider: external
   211            cluster-dns: fd00::10
   212            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   213            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   214          name: '{{ ds.meta_data["local_hostname"] }}'
   215      joinConfiguration:
   216        controlPlane:
   217          localAPIEndpoint:
   218            advertiseAddress: '::'
   219            bindPort: 6443
   220        nodeRegistration:
   221          kubeletExtraArgs:
   222            cloud-provider: external
   223            cluster-dns: fd00::10
   224            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   225            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   226          name: '{{ ds.meta_data["local_hostname"] }}'
   227      mounts:
   228      - - LABEL=etcd_disk
   229        - /var/lib/etcddisk
   230      postKubeadmCommands: []
   231      preKubeadmCommands:
   232      - bash -c /tmp/oot-cred-provider.sh
   233      - bash -c /tmp/kubeadm-bootstrap.sh
   234      verbosity: 5
   235    machineTemplate:
   236      infrastructureRef:
   237        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   238        kind: AzureMachineTemplate
   239        name: ${CLUSTER_NAME}-control-plane
   240    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   241    version: ${KUBERNETES_VERSION}
   242  ---
   243  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   244  kind: AzureMachineTemplate
   245  metadata:
   246    name: ${CLUSTER_NAME}-control-plane
   247    namespace: default
   248  spec:
   249    template:
   250      spec:
   251        dataDisks:
   252        - diskSizeGB: 256
   253          lun: 0
   254          nameSuffix: etcddisk
   255        enableIPForwarding: true
   256        identity: UserAssigned
   257        image:
   258          marketplace:
   259            offer: capi
   260            publisher: cncf-upstream
   261            sku: ubuntu-2204-gen1
   262            version: latest
   263        osDisk:
   264          diskSizeGB: 128
   265          osType: Linux
   266        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   267        userAssignedIdentities:
   268        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
   269        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   270  ---
   271  apiVersion: cluster.x-k8s.io/v1beta1
   272  kind: MachineDeployment
   273  metadata:
   274    name: ${CLUSTER_NAME}-md-0
   275    namespace: default
   276  spec:
   277    clusterName: ${CLUSTER_NAME}
   278    replicas: ${WORKER_MACHINE_COUNT:=2}
   279    selector: {}
   280    template:
   281      metadata:
   282        labels:
   283          nodepool: pool1
   284      spec:
   285        bootstrap:
   286          configRef:
   287            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   288            kind: KubeadmConfigTemplate
   289            name: ${CLUSTER_NAME}-md-0
   290        clusterName: ${CLUSTER_NAME}
   291        infrastructureRef:
   292          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   293          kind: AzureMachineTemplate
   294          name: ${CLUSTER_NAME}-md-0
   295        version: ${KUBERNETES_VERSION}
   296  ---
   297  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   298  kind: AzureMachineTemplate
   299  metadata:
   300    name: ${CLUSTER_NAME}-md-0
   301    namespace: default
   302  spec:
   303    template:
   304      spec:
   305        enableIPForwarding: true
   306        identity: UserAssigned
   307        image:
   308          marketplace:
   309            offer: capi
   310            publisher: cncf-upstream
   311            sku: ubuntu-2204-gen1
   312            version: latest
   313        osDisk:
   314          diskSizeGB: 128
   315          osType: Linux
   316        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   317        userAssignedIdentities:
   318        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
   319        vmExtensions:
   320        - name: CustomScript
   321          protectedSettings:
   322            commandToExecute: |
   323              #!/bin/sh
   324              echo "This script is a no-op used for extension testing purposes ..."
   325              touch test_file
   326          publisher: Microsoft.Azure.Extensions
   327          version: "2.1"
   328        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   329  ---
   330  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   331  kind: KubeadmConfigTemplate
   332  metadata:
   333    name: ${CLUSTER_NAME}-md-0
   334    namespace: default
   335  spec:
   336    template:
   337      spec:
   338        clusterConfiguration:
   339          apiServer:
   340            extraArgs:
   341              bind-address: '::'
   342          controllerManager:
   343            extraArgs:
   344              bind-address: '::'
   345          scheduler:
   346            extraArgs:
   347              bind-address: '::'
   348        files:
   349        - contentFrom:
   350            secret:
   351              key: worker-node-azure.json
   352              name: ${CLUSTER_NAME}-md-0-azure-json
   353          owner: root:root
   354          path: /etc/kubernetes/azure.json
   355          permissions: "0644"
   356        - content: |
   357            #!/bin/bash
   358  
   359            set -o nounset
   360            set -o pipefail
   361            set -o errexit
   362            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   363  
   364            echo "Use OOT credential provider"
   365            mkdir -p /var/lib/kubelet/credential-provider
   366            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   367            chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   368            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   369            chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   370          owner: root:root
   371          path: /tmp/oot-cred-provider.sh
   372          permissions: "0744"
   373        - content: |
   374            #!/bin/bash
   375  
   376            set -o nounset
   377            set -o pipefail
   378            set -o errexit
   379            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   380  
   381            # This test installs release packages or binaries that are a result of the CI and release builds.
   382            # It runs '... --version' commands to verify that the binaries are correctly installed
   383            # and finally uninstalls the packages.
   384            # For the release packages it tests all versions in the support skew.
   385            LINE_SEPARATOR="*************************************************"
   386            echo "$$LINE_SEPARATOR"
   387            CI_VERSION=${CI_VERSION}
   388            if [[ "$${CI_VERSION}" != "" ]]; then
   389              CI_DIR=/tmp/k8s-ci
   390              mkdir -p $$CI_DIR
   391              declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   392              declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   393              CONTAINER_EXT="tar"
   394              echo "* testing CI version $$CI_VERSION"
   395              # Check for semver
   396              if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   397                VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   398                DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   399                curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   400                echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   401                apt-get update
   402                # replace . with \.
   403                VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   404                PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   405                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   406                  echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   407                  DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   408                done
   409              else
   410                CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   411                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   412                  echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   413                  wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   414                  chmod +x "$$CI_DIR/$$CI_PACKAGE"
   415                  mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   416                done
   417                IMAGE_REGISTRY_PREFIX=registry.k8s.io
   418                for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   419                  echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   420                  wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   421                  $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   422                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   423                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   424                done
   425              fi
   426              systemctl restart kubelet
   427            fi
   428            echo "* checking binary versions"
   429            echo "ctr version: " $(ctr version)
   430            echo "kubeadm version: " $(kubeadm version -o=short)
   431            echo "kubectl version: " $(kubectl version --client=true)
   432            echo "kubelet version: " $(kubelet --version)
   433            echo "$$LINE_SEPARATOR"
   434          owner: root:root
   435          path: /tmp/kubeadm-bootstrap.sh
   436          permissions: "0744"
   437        joinConfiguration:
   438          nodeRegistration:
   439            kubeletExtraArgs:
   440              cloud-provider: external
   441              cluster-dns: '[fd00::10]'
   442              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   443              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   444            name: '{{ ds.meta_data["local_hostname"] }}'
   445        preKubeadmCommands:
   446        - bash -c /tmp/oot-cred-provider.sh
   447        - bash -c /tmp/kubeadm-bootstrap.sh
   448        verbosity: 5
   449  ---
   450  apiVersion: cluster.x-k8s.io/v1beta1
   451  kind: MachineHealthCheck
   452  metadata:
   453    name: ${CLUSTER_NAME}-mhc-0
   454    namespace: default
   455  spec:
   456    clusterName: ${CLUSTER_NAME}
   457    maxUnhealthy: 100%
   458    selector:
   459      matchLabels:
   460        nodepool: pool1
   461    unhealthyConditions:
   462    - status: "True"
   463      timeout: 30s
   464      type: E2ENodeUnhealthy
   465  ---
   466  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   467  kind: AzureClusterIdentity
   468  metadata:
   469    labels:
   470      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   471    name: ${CLUSTER_IDENTITY_NAME}
   472    namespace: default
   473  spec:
   474    allowedNamespaces: {}
   475    clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}
   476    tenantID: ${AZURE_TENANT_ID}
   477    type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity}
   478  ---
   479  apiVersion: addons.cluster.x-k8s.io/v1beta1
   480  kind: ClusterResourceSet
   481  metadata:
   482    name: csi-proxy
   483    namespace: default
   484  spec:
   485    clusterSelector:
   486      matchLabels:
   487        csi-proxy: enabled
   488    resources:
   489    - kind: ConfigMap
   490      name: csi-proxy-addon
   491    strategy: ApplyOnce
   492  ---
   493  apiVersion: addons.cluster.x-k8s.io/v1beta1
   494  kind: ClusterResourceSet
   495  metadata:
   496    name: containerd-logger-${CLUSTER_NAME}
   497    namespace: default
   498  spec:
   499    clusterSelector:
   500      matchLabels:
   501        containerd-logger: enabled
   502    resources:
   503    - kind: ConfigMap
   504      name: containerd-logger-${CLUSTER_NAME}
   505    strategy: ApplyOnce
   506  ---
   507  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   508  kind: HelmChartProxy
   509  metadata:
   510    name: azuredisk-csi-driver-chart
   511    namespace: default
   512  spec:
   513    chartName: azuredisk-csi-driver
   514    clusterSelector:
   515      matchLabels:
   516        azuredisk-csi: "true"
   517    namespace: kube-system
   518    releaseName: azuredisk-csi-driver-oot
   519    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   520    valuesTemplate: |-
   521      controller:
   522        replicas: 1
   523        runOnControlPlane: true
   524      windows:
   525        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   526  ---
   527  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   528  kind: HelmChartProxy
   529  metadata:
   530    name: cloud-provider-azure-chart
   531    namespace: default
   532  spec:
   533    chartName: cloud-provider-azure
   534    clusterSelector:
   535      matchLabels:
   536        cloud-provider: azure
   537    releaseName: cloud-provider-azure-oot
   538    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   539    valuesTemplate: |
   540      infra:
   541        clusterName: {{ .Cluster.metadata.name }}
   542      cloudControllerManager:
   543        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   544        logVerbosity: 4
   545  ---
   546  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   547  kind: HelmChartProxy
   548  metadata:
   549    name: cloud-provider-azure-chart-ci
   550    namespace: default
   551  spec:
   552    chartName: cloud-provider-azure
   553    clusterSelector:
   554      matchLabels:
   555        cloud-provider: azure-ci
   556    releaseName: cloud-provider-azure-oot
   557    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   558    valuesTemplate: |
   559      infra:
   560        clusterName: {{ .Cluster.metadata.name }}
   561      cloudControllerManager:
   562        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   563        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   564        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   565        imageName: "${CCM_IMAGE_NAME:-""}"
   566        imageRepository: "${IMAGE_REGISTRY:-""}"
   567        imageTag: "${IMAGE_TAG_CCM:-""}"
   568        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   569        replicas: ${CCM_COUNT:-1}
   570        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   571      cloudNodeManager:
   572        imageName: "${CNM_IMAGE_NAME:-""}"
   573        imageRepository: "${IMAGE_REGISTRY:-""}"
   574        imageTag: "${IMAGE_TAG_CNM:-""}"
   575  ---
   576  apiVersion: v1
   577  data:
   578    csi-proxy: |
   579      apiVersion: apps/v1
   580      kind: DaemonSet
   581      metadata:
   582        labels:
   583          k8s-app: csi-proxy
   584        name: csi-proxy
   585        namespace: kube-system
   586      spec:
   587        selector:
   588          matchLabels:
   589            k8s-app: csi-proxy
   590        template:
   591          metadata:
   592            labels:
   593              k8s-app: csi-proxy
   594          spec:
   595            nodeSelector:
   596              "kubernetes.io/os": windows
   597            securityContext:
   598              windowsOptions:
   599                hostProcess: true
   600                runAsUserName: "NT AUTHORITY\\SYSTEM"
   601            hostNetwork: true
   602            containers:
   603              - name: csi-proxy
   604                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   605  kind: ConfigMap
   606  metadata:
   607    annotations:
   608      note: generated
   609    labels:
   610      type: generated
   611    name: csi-proxy-addon
   612    namespace: default
   613  ---
   614  apiVersion: v1
   615  data:
   616    containerd-windows-logger: |
   617      apiVersion: apps/v1
   618      kind: DaemonSet
   619      metadata:
   620        labels:
   621          k8s-app: containerd-logger
   622        name: containerd-logger
   623        namespace: kube-system
   624      spec:
   625        selector:
   626          matchLabels:
   627            k8s-app: containerd-logger
   628        template:
   629          metadata:
   630            labels:
   631              k8s-app: containerd-logger
   632          spec:
   633            securityContext:
   634              windowsOptions:
   635                hostProcess: true
   636                runAsUserName: "NT AUTHORITY\\system"
   637            hostNetwork: true
   638            containers:
   639            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
   640              args: [ "config.json" ]
   641              name: containerd-logger
   642              imagePullPolicy: Always
   643              volumeMounts:
   644              - name: containerd-logger-config
   645                mountPath: /config.json
   646                subPath: config.json
   647            nodeSelector:
   648              kubernetes.io/os: windows
   649            tolerations:
   650            - key: CriticalAddonsOnly
   651              operator: Exists
   652            - operator: Exists
   653            volumes:
   654            - configMap:
   655                name: containerd-logger-config
   656              name: containerd-logger-config
   657        updateStrategy:
   658          type: RollingUpdate
   659      ---
   660      kind: ConfigMap
   661      apiVersion: v1
   662      metadata:
   663        name: containerd-logger-config
   664        namespace: kube-system
   665      data:
   666        config.json: |
   667          {
   668            "inputs": [
   669              {
   670                "type": "ETW",
   671                "sessionNamePrefix": "containerd",
   672                "cleanupOldSessions": true,
   673                "reuseExistingSession": true,
   674                "providers": [
   675                  {
   676                    "providerName": "Microsoft.Virtualization.RunHCS",
   677                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
   678                    "level": "Verbose"
   679                  },
   680                  {
   681                    "providerName": "ContainerD",
   682                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
   683                    "level": "Verbose"
   684                  }
   685                ]
   686              }
   687            ],
   688            "filters": [
   689              {
   690                  "type": "drop",
   691                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
   692              },
   693              {
   694                  "type": "drop",
   695                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
   696              },
   697              {
   698                  "type": "drop",
   699                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
   700              },
   701              {
   702                  "type": "drop",
   703                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
   704              },
   705              {
   706                  "type": "drop",
   707                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
   708              },
   709              {
   710                  "type": "drop",
   711                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
   712              },
   713              {
   714                  "type": "drop",
   715                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
   716              }
   717            ],
   718            "outputs": [
   719              {
   720                "type": "StdOutput"
   721              }
   722            ],
   723            "schemaVersion": "2016-08-11"
   724          }
   725  kind: ConfigMap
   726  metadata:
   727    annotations:
   728      note: generated
   729    labels:
   730      type: generated
   731    name: containerd-logger-${CLUSTER_NAME}
   732    namespace: default
   733  ---
   734  apiVersion: addons.cluster.x-k8s.io/v1beta1
   735  kind: ClusterResourceSet
   736  metadata:
   737    name: metrics-server-${CLUSTER_NAME}
   738    namespace: default
   739  spec:
   740    clusterSelector:
   741      matchLabels:
   742        metrics-server: enabled
   743    resources:
   744    - kind: ConfigMap
   745      name: metrics-server-${CLUSTER_NAME}
   746    strategy: ApplyOnce
   747  ---
   748  apiVersion: v1
   749  data:
   750    metrics-server: |
   751      apiVersion: v1
   752      kind: ServiceAccount
   753      metadata:
   754        labels:
   755          k8s-app: metrics-server
   756        name: metrics-server
   757        namespace: kube-system
   758      ---
   759      apiVersion: rbac.authorization.k8s.io/v1
   760      kind: ClusterRole
   761      metadata:
   762        labels:
   763          k8s-app: metrics-server
   764          rbac.authorization.k8s.io/aggregate-to-admin: "true"
   765          rbac.authorization.k8s.io/aggregate-to-edit: "true"
   766          rbac.authorization.k8s.io/aggregate-to-view: "true"
   767        name: system:aggregated-metrics-reader
   768      rules:
   769      - apiGroups:
   770        - metrics.k8s.io
   771        resources:
   772        - pods
   773        - nodes
   774        verbs:
   775        - get
   776        - list
   777        - watch
   778      ---
   779      apiVersion: rbac.authorization.k8s.io/v1
   780      kind: ClusterRole
   781      metadata:
   782        labels:
   783          k8s-app: metrics-server
   784        name: system:metrics-server
   785      rules:
   786      - apiGroups:
   787        - ""
   788        resources:
   789        - nodes/metrics
   790        verbs:
   791        - get
   792      - apiGroups:
   793        - ""
   794        resources:
   795        - pods
   796        - nodes
   797        verbs:
   798        - get
   799        - list
   800        - watch
   801      ---
   802      apiVersion: rbac.authorization.k8s.io/v1
   803      kind: RoleBinding
   804      metadata:
   805        labels:
   806          k8s-app: metrics-server
   807        name: metrics-server-auth-reader
   808        namespace: kube-system
   809      roleRef:
   810        apiGroup: rbac.authorization.k8s.io
   811        kind: Role
   812        name: extension-apiserver-authentication-reader
   813      subjects:
   814      - kind: ServiceAccount
   815        name: metrics-server
   816        namespace: kube-system
   817      ---
   818      apiVersion: rbac.authorization.k8s.io/v1
   819      kind: ClusterRoleBinding
   820      metadata:
   821        labels:
   822          k8s-app: metrics-server
   823        name: metrics-server:system:auth-delegator
   824      roleRef:
   825        apiGroup: rbac.authorization.k8s.io
   826        kind: ClusterRole
   827        name: system:auth-delegator
   828      subjects:
   829      - kind: ServiceAccount
   830        name: metrics-server
   831        namespace: kube-system
   832      ---
   833      apiVersion: rbac.authorization.k8s.io/v1
   834      kind: ClusterRoleBinding
   835      metadata:
   836        labels:
   837          k8s-app: metrics-server
   838        name: system:metrics-server
   839      roleRef:
   840        apiGroup: rbac.authorization.k8s.io
   841        kind: ClusterRole
   842        name: system:metrics-server
   843      subjects:
   844      - kind: ServiceAccount
   845        name: metrics-server
   846        namespace: kube-system
   847      ---
   848      apiVersion: v1
   849      kind: Service
   850      metadata:
   851        labels:
   852          k8s-app: metrics-server
   853        name: metrics-server
   854        namespace: kube-system
   855      spec:
   856        ports:
   857        - name: https
   858          port: 443
   859          protocol: TCP
   860          targetPort: https
   861        selector:
   862          k8s-app: metrics-server
   863      ---
   864      apiVersion: apps/v1
   865      kind: Deployment
   866      metadata:
   867        labels:
   868          k8s-app: metrics-server
   869        name: metrics-server
   870        namespace: kube-system
   871      spec:
   872        selector:
   873          matchLabels:
   874            k8s-app: metrics-server
   875        strategy:
   876          rollingUpdate:
   877            maxUnavailable: 0
   878        template:
   879          metadata:
   880            labels:
   881              k8s-app: metrics-server
   882          spec:
   883            containers:
   884            - args:
   885              - --cert-dir=/tmp
   886              - --secure-port=4443
   887              - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
   888              - --kubelet-use-node-status-port
   889              - --metric-resolution=15s
   890              - --kubelet-insecure-tls
   891              image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
   892              imagePullPolicy: IfNotPresent
   893              livenessProbe:
   894                failureThreshold: 3
   895                httpGet:
   896                  path: /livez
   897                  port: https
   898                  scheme: HTTPS
   899                periodSeconds: 10
   900              name: metrics-server
   901              ports:
   902              - containerPort: 4443
   903                name: https
   904                protocol: TCP
   905              readinessProbe:
   906                failureThreshold: 3
   907                httpGet:
   908                  path: /readyz
   909                  port: https
   910                  scheme: HTTPS
   911                initialDelaySeconds: 20
   912                periodSeconds: 10
   913              resources:
   914                requests:
   915                  cpu: 100m
   916                  memory: 200Mi
   917              securityContext:
   918                allowPrivilegeEscalation: false
   919                readOnlyRootFilesystem: true
   920                runAsNonRoot: true
   921                runAsUser: 1000
   922              volumeMounts:
   923              - mountPath: /tmp
   924                name: tmp-dir
   925            nodeSelector:
   926              kubernetes.io/os: linux
   927            priorityClassName: system-cluster-critical
   928            serviceAccountName: metrics-server
   929            tolerations:
   930            - effect: NoSchedule
   931              key: node-role.kubernetes.io/master
   932              operator: Exists
   933            - effect: NoSchedule
   934              key: node-role.kubernetes.io/control-plane
   935              operator: Exists
   936            volumes:
   937            - emptyDir: {}
   938              name: tmp-dir
   939      ---
   940      apiVersion: apiregistration.k8s.io/v1
   941      kind: APIService
   942      metadata:
   943        labels:
   944          k8s-app: metrics-server
   945        name: v1beta1.metrics.k8s.io
   946      spec:
   947        group: metrics.k8s.io
   948        groupPriorityMinimum: 100
   949        insecureSkipTLSVerify: true
   950        service:
   951          name: metrics-server
   952          namespace: kube-system
   953        version: v1beta1
   954        versionPriority: 100
   955  kind: ConfigMap
   956  metadata:
   957    annotations:
   958      note: generated
   959    labels:
   960      type: generated
   961    name: metrics-server-${CLUSTER_NAME}
   962    namespace: default
   963  ---
   964  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   965  kind: HelmChartProxy
   966  metadata:
   967    name: calico-ipv6
   968    namespace: default
   969  spec:
   970    chartName: tigera-operator
   971    clusterSelector:
   972      matchLabels:
   973        cni: calico-ipv6
   974    namespace: tigera-operator
   975    releaseName: projectcalico
   976    repoURL: https://docs.tigera.io/calico/charts
   977    valuesTemplate: |-
   978      installation:
   979        cni:
   980          type: Calico
   981          ipam:
   982            type: HostLocal
   983        calicoNetwork:
   984          bgp: Disabled
   985          mtu: 1350
   986          ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
   987          - blockSize: 122
   988            cidr: {{ $cidr }}
   989            encapsulation: None
   990            natOutgoing: Enabled
   991            nodeSelector: all(){{end}}
   992        registry: mcr.microsoft.com/oss
   993      # Image and registry configuration for the tigera/operator pod.
   994      tigeraOperator:
   995        image: tigera/operator
   996        registry: mcr.microsoft.com/oss
   997      calicoctl:
   998        image: mcr.microsoft.com/oss/calico/ctl
   999    version: ${CALICO_VERSION}