sigs.k8s.io/cluster-api-provider-azure@v1.14.3/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico-ipv6
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: disabled
     9      csi-proxy: disabled
    10      metrics-server: disabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 2001:1234:5678:9a40::/58
    18      services:
    19        cidrBlocks:
    20        - fd00::/108
    21    controlPlaneRef:
    22      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    23      kind: KubeadmControlPlane
    24      name: ${CLUSTER_NAME}-control-plane
    25    infrastructureRef:
    26      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    27      kind: AzureCluster
    28      name: ${CLUSTER_NAME}
    29  ---
    30  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    31  kind: AzureCluster
    32  metadata:
    33    name: ${CLUSTER_NAME}
    34    namespace: default
    35  spec:
    36    additionalTags:
    37      buildProvenance: ${BUILD_PROVENANCE}
    38      creationTimestamp: ${TIMESTAMP}
    39      jobName: ${JOB_NAME}
    40    identityRef:
    41      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    42      kind: AzureClusterIdentity
    43      name: ${CLUSTER_IDENTITY_NAME}
    44    location: ${AZURE_LOCATION}
    45    networkSpec:
    46      subnets:
    47      - cidrBlocks:
    48        - 10.0.0.0/16
    49        - 2001:1234:5678:9abc::/64
    50        name: control-plane-subnet
    51        role: control-plane
    52      - cidrBlocks:
    53        - 10.1.0.0/16
    54        - 2001:1234:5678:9abd::/64
    55        name: node-subnet
    56        role: node
    57      vnet:
    58        cidrBlocks:
    59        - 10.0.0.0/8
    60        - 2001:1234:5678:9a00::/56
    61        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    62    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    63    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    64  ---
    65  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    66  kind: KubeadmControlPlane
    67  metadata:
    68    name: ${CLUSTER_NAME}-control-plane
    69    namespace: default
    70  spec:
    71    kubeadmConfigSpec:
    72      clusterConfiguration:
    73        apiServer:
    74          extraArgs:
    75            bind-address: '::'
    76            cloud-provider: external
    77            feature-gates: ${K8S_FEATURE_GATES:-""}
    78          timeoutForControlPlane: 20m
    79        controllerManager:
    80          extraArgs:
    81            allocate-node-cidrs: "true"
    82            bind-address: '::'
    83            cloud-provider: external
    84            cluster-cidr: 2001:1234:5678:9a40::/58
    85            cluster-name: ${CLUSTER_NAME}
    86            configure-cloud-routes: "true"
    87            feature-gates: HPAContainerMetrics=true
    88            v: "4"
    89        etcd:
    90          local:
    91            dataDir: /var/lib/etcddisk/etcd
    92            extraArgs:
    93              quota-backend-bytes: "8589934592"
    94        kubernetesVersion: ci/${CI_VERSION}
    95        scheduler:
    96          extraArgs:
    97            bind-address: '::'
    98      diskSetup:
    99        filesystems:
   100        - device: /dev/disk/azure/scsi1/lun0
   101          extraOpts:
   102          - -E
   103          - lazy_itable_init=1,lazy_journal_init=1
   104          filesystem: ext4
   105          label: etcd_disk
   106        - device: ephemeral0.1
   107          filesystem: ext4
   108          label: ephemeral0
   109          replaceFS: ntfs
   110        partitions:
   111        - device: /dev/disk/azure/scsi1/lun0
   112          layout: true
   113          overwrite: false
   114          tableType: gpt
   115      files:
   116      - contentFrom:
   117          secret:
   118            key: control-plane-azure.json
   119            name: ${CLUSTER_NAME}-control-plane-azure-json
   120        owner: root:root
   121        path: /etc/kubernetes/azure.json
   122        permissions: "0644"
   123      - content: |
   124          #!/bin/bash
   125  
   126          set -o nounset
   127          set -o pipefail
   128          set -o errexit
   129          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   130  
   131          az login --identity
   132          echo "Use OOT credential provider"
   133          mkdir -p /var/lib/kubelet/credential-provider
   134          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   135          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   136          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   137          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   138        owner: root:root
   139        path: /tmp/oot-cred-provider.sh
   140        permissions: "0744"
   141      - content: |
   142          #!/bin/bash
   143  
   144          set -o nounset
   145          set -o pipefail
   146          set -o errexit
   147          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   148  
   149          # This test installs release packages or binaries that are a result of the CI and release builds.
   150          # It runs '... --version' commands to verify that the binaries are correctly installed
   151          # and finally uninstalls the packages.
   152          # For the release packages it tests all versions in the support skew.
   153          LINE_SEPARATOR="*************************************************"
   154          echo "$$LINE_SEPARATOR"
   155          CI_VERSION=${CI_VERSION}
   156          if [[ "$${CI_VERSION}" != "" ]]; then
   157            CI_DIR=/tmp/k8s-ci
   158            mkdir -p $$CI_DIR
   159            declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   160            declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   161            CONTAINER_EXT="tar"
   162            echo "* testing CI version $$CI_VERSION"
   163            # Check for semver
   164            if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   165              VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   166              DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   167              curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   168              echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   169              apt-get update
   170              # replace . with \.
   171              VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   172              PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   173              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   174                echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   175                DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   176              done
   177            else
   178              CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   179              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   180                echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   181                wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   182                chmod +x "$$CI_DIR/$$CI_PACKAGE"
   183                mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   184              done
   185              IMAGE_REGISTRY_PREFIX=registry.k8s.io
   186              for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   187                echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   188                wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   189                $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   190                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   191                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   192              done
   193            fi
   194            systemctl restart kubelet
   195          fi
   196          echo "* checking binary versions"
   197          echo "ctr version: " $(ctr version)
   198          echo "kubeadm version: " $(kubeadm version -o=short)
   199          echo "kubectl version: " $(kubectl version --client=true)
   200          echo "kubelet version: " $(kubelet --version)
   201          echo "$$LINE_SEPARATOR"
   202        owner: root:root
   203        path: /tmp/kubeadm-bootstrap.sh
   204        permissions: "0744"
   205      initConfiguration:
   206        localAPIEndpoint:
   207          advertiseAddress: '::'
   208          bindPort: 6443
   209        nodeRegistration:
   210          kubeletExtraArgs:
   211            cloud-provider: external
   212            cluster-dns: fd00::10
   213            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   214            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   215          name: '{{ ds.meta_data["local_hostname"] }}'
   216      joinConfiguration:
   217        controlPlane:
   218          localAPIEndpoint:
   219            advertiseAddress: '::'
   220            bindPort: 6443
   221        nodeRegistration:
   222          kubeletExtraArgs:
   223            cloud-provider: external
   224            cluster-dns: fd00::10
   225            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   226            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   227          name: '{{ ds.meta_data["local_hostname"] }}'
   228      mounts:
   229      - - LABEL=etcd_disk
   230        - /var/lib/etcddisk
   231      postKubeadmCommands: []
   232      preKubeadmCommands:
   233      - bash -c /tmp/oot-cred-provider.sh
   234      - bash -c /tmp/kubeadm-bootstrap.sh
   235      verbosity: 5
   236    machineTemplate:
   237      infrastructureRef:
   238        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   239        kind: AzureMachineTemplate
   240        name: ${CLUSTER_NAME}-control-plane
   241    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   242    version: ${KUBERNETES_VERSION}
   243  ---
   244  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   245  kind: AzureMachineTemplate
   246  metadata:
   247    name: ${CLUSTER_NAME}-control-plane
   248    namespace: default
   249  spec:
   250    template:
   251      spec:
   252        dataDisks:
   253        - diskSizeGB: 256
   254          lun: 0
   255          nameSuffix: etcddisk
   256        enableIPForwarding: true
   257        image:
   258          marketplace:
   259            offer: capi
   260            publisher: cncf-upstream
   261            sku: ubuntu-2204-gen1
   262            version: latest
   263        osDisk:
   264          diskSizeGB: 128
   265          osType: Linux
   266        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   267        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   268  ---
   269  apiVersion: cluster.x-k8s.io/v1beta1
   270  kind: MachineDeployment
   271  metadata:
   272    name: ${CLUSTER_NAME}-md-0
   273    namespace: default
   274  spec:
   275    clusterName: ${CLUSTER_NAME}
   276    replicas: ${WORKER_MACHINE_COUNT:=2}
   277    selector: {}
   278    template:
   279      metadata:
   280        labels:
   281          nodepool: pool1
   282      spec:
   283        bootstrap:
   284          configRef:
   285            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   286            kind: KubeadmConfigTemplate
   287            name: ${CLUSTER_NAME}-md-0
   288        clusterName: ${CLUSTER_NAME}
   289        infrastructureRef:
   290          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   291          kind: AzureMachineTemplate
   292          name: ${CLUSTER_NAME}-md-0
   293        version: ${KUBERNETES_VERSION}
   294  ---
   295  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   296  kind: AzureMachineTemplate
   297  metadata:
   298    name: ${CLUSTER_NAME}-md-0
   299    namespace: default
   300  spec:
   301    template:
   302      spec:
   303        enableIPForwarding: true
   304        image:
   305          marketplace:
   306            offer: capi
   307            publisher: cncf-upstream
   308            sku: ubuntu-2204-gen1
   309            version: latest
   310        osDisk:
   311          diskSizeGB: 128
   312          osType: Linux
   313        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   314        vmExtensions:
   315        - name: CustomScript
   316          protectedSettings:
   317            commandToExecute: |
   318              #!/bin/sh
   319              echo "This script is a no-op used for extension testing purposes ..."
   320              touch test_file
   321          publisher: Microsoft.Azure.Extensions
   322          version: "2.1"
   323        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   324  ---
   325  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   326  kind: KubeadmConfigTemplate
   327  metadata:
   328    name: ${CLUSTER_NAME}-md-0
   329    namespace: default
   330  spec:
   331    template:
   332      spec:
   333        clusterConfiguration:
   334          apiServer:
   335            extraArgs:
   336              bind-address: '::'
   337          controllerManager:
   338            extraArgs:
   339              bind-address: '::'
   340          scheduler:
   341            extraArgs:
   342              bind-address: '::'
   343        files:
   344        - contentFrom:
   345            secret:
   346              key: worker-node-azure.json
   347              name: ${CLUSTER_NAME}-md-0-azure-json
   348          owner: root:root
   349          path: /etc/kubernetes/azure.json
   350          permissions: "0644"
   351        - content: |
   352            #!/bin/bash
   353  
   354            set -o nounset
   355            set -o pipefail
   356            set -o errexit
   357            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   358  
   359            az login --identity
   360            echo "Use OOT credential provider"
   361            mkdir -p /var/lib/kubelet/credential-provider
   362            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   363            chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   364            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   365            chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   366          owner: root:root
   367          path: /tmp/oot-cred-provider.sh
   368          permissions: "0744"
   369        - content: |
   370            #!/bin/bash
   371  
   372            set -o nounset
   373            set -o pipefail
   374            set -o errexit
   375            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   376  
   377            # This test installs release packages or binaries that are a result of the CI and release builds.
   378            # It runs '... --version' commands to verify that the binaries are correctly installed
   379            # and finally uninstalls the packages.
   380            # For the release packages it tests all versions in the support skew.
   381            LINE_SEPARATOR="*************************************************"
   382            echo "$$LINE_SEPARATOR"
   383            CI_VERSION=${CI_VERSION}
   384            if [[ "$${CI_VERSION}" != "" ]]; then
   385              CI_DIR=/tmp/k8s-ci
   386              mkdir -p $$CI_DIR
   387              declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   388              declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   389              CONTAINER_EXT="tar"
   390              echo "* testing CI version $$CI_VERSION"
   391              # Check for semver
   392              if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   393                VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   394                DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   395                curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   396                echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   397                apt-get update
   398                # replace . with \.
   399                VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   400                PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   401                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   402                  echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   403                  DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   404                done
   405              else
   406                CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   407                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   408                  echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   409                  wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   410                  chmod +x "$$CI_DIR/$$CI_PACKAGE"
   411                  mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   412                done
   413                IMAGE_REGISTRY_PREFIX=registry.k8s.io
   414                for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   415                  echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   416                  wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   417                  $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   418                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   419                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   420                done
   421              fi
   422              systemctl restart kubelet
   423            fi
   424            echo "* checking binary versions"
   425            echo "ctr version: " $(ctr version)
   426            echo "kubeadm version: " $(kubeadm version -o=short)
   427            echo "kubectl version: " $(kubectl version --client=true)
   428            echo "kubelet version: " $(kubelet --version)
   429            echo "$$LINE_SEPARATOR"
   430          owner: root:root
   431          path: /tmp/kubeadm-bootstrap.sh
   432          permissions: "0744"
   433        joinConfiguration:
   434          nodeRegistration:
   435            kubeletExtraArgs:
   436              cloud-provider: external
   437              cluster-dns: '[fd00::10]'
   438              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   439              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   440            name: '{{ ds.meta_data["local_hostname"] }}'
   441        preKubeadmCommands:
   442        - bash -c /tmp/oot-cred-provider.sh
   443        - bash -c /tmp/kubeadm-bootstrap.sh
   444        verbosity: 5
   445  ---
   446  apiVersion: cluster.x-k8s.io/v1beta1
   447  kind: MachineHealthCheck
   448  metadata:
   449    name: ${CLUSTER_NAME}-mhc-0
   450    namespace: default
   451  spec:
   452    clusterName: ${CLUSTER_NAME}
   453    maxUnhealthy: 100%
   454    selector:
   455      matchLabels:
   456        nodepool: pool1
   457    unhealthyConditions:
   458    - status: "True"
   459      timeout: 30s
   460      type: E2ENodeUnhealthy
   461  ---
   462  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   463  kind: AzureClusterIdentity
   464  metadata:
   465    labels:
   466      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   467    name: ${CLUSTER_IDENTITY_NAME}
   468    namespace: default
   469  spec:
   470    allowedNamespaces: {}
   471    clientID: ${AZURE_CLIENT_ID}
   472    clientSecret:
   473      name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME}
   474      namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
   475    tenantID: ${AZURE_TENANT_ID}
   476    type: ServicePrincipal
   477  ---
   478  apiVersion: addons.cluster.x-k8s.io/v1beta1
   479  kind: ClusterResourceSet
   480  metadata:
   481    name: csi-proxy
   482    namespace: default
   483  spec:
   484    clusterSelector:
   485      matchLabels:
   486        csi-proxy: enabled
   487    resources:
   488    - kind: ConfigMap
   489      name: csi-proxy-addon
   490    strategy: ApplyOnce
   491  ---
   492  apiVersion: addons.cluster.x-k8s.io/v1beta1
   493  kind: ClusterResourceSet
   494  metadata:
   495    name: containerd-logger-${CLUSTER_NAME}
   496    namespace: default
   497  spec:
   498    clusterSelector:
   499      matchLabels:
   500        containerd-logger: enabled
   501    resources:
   502    - kind: ConfigMap
   503      name: containerd-logger-${CLUSTER_NAME}
   504    strategy: ApplyOnce
   505  ---
   506  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   507  kind: HelmChartProxy
   508  metadata:
   509    name: azuredisk-csi-driver-chart
   510    namespace: default
   511  spec:
   512    chartName: azuredisk-csi-driver
   513    clusterSelector:
   514      matchLabels:
   515        azuredisk-csi: "true"
   516    namespace: kube-system
   517    releaseName: azuredisk-csi-driver-oot
   518    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   519    valuesTemplate: |-
   520      controller:
   521        replicas: 1
   522        runOnControlPlane: true
   523      windows:
   524        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   525  ---
   526  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   527  kind: HelmChartProxy
   528  metadata:
   529    name: cloud-provider-azure-chart
   530    namespace: default
   531  spec:
   532    chartName: cloud-provider-azure
   533    clusterSelector:
   534      matchLabels:
   535        cloud-provider: azure
   536    releaseName: cloud-provider-azure-oot
   537    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   538    valuesTemplate: |
   539      infra:
   540        clusterName: {{ .Cluster.metadata.name }}
   541      cloudControllerManager:
   542        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   543        logVerbosity: 4
   544  ---
   545  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   546  kind: HelmChartProxy
   547  metadata:
   548    name: cloud-provider-azure-chart-ci
   549    namespace: default
   550  spec:
   551    chartName: cloud-provider-azure
   552    clusterSelector:
   553      matchLabels:
   554        cloud-provider: azure-ci
   555    releaseName: cloud-provider-azure-oot
   556    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   557    valuesTemplate: |
   558      infra:
   559        clusterName: {{ .Cluster.metadata.name }}
   560      cloudControllerManager:
   561        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   562        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   563        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   564        imageName: "${CCM_IMAGE_NAME:-""}"
   565        imageRepository: "${IMAGE_REGISTRY:-""}"
   566        imageTag: "${IMAGE_TAG_CCM:-""}"
   567        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   568        replicas: ${CCM_COUNT:-1}
   569        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   570      cloudNodeManager:
   571        imageName: "${CNM_IMAGE_NAME:-""}"
   572        imageRepository: "${IMAGE_REGISTRY:-""}"
   573        imageTag: "${IMAGE_TAG_CNM:-""}"
   574  ---
   575  apiVersion: v1
   576  data:
   577    csi-proxy: |
   578      apiVersion: apps/v1
   579      kind: DaemonSet
   580      metadata:
   581        labels:
   582          k8s-app: csi-proxy
   583        name: csi-proxy
   584        namespace: kube-system
   585      spec:
   586        selector:
   587          matchLabels:
   588            k8s-app: csi-proxy
   589        template:
   590          metadata:
   591            labels:
   592              k8s-app: csi-proxy
   593          spec:
   594            nodeSelector:
   595              "kubernetes.io/os": windows
   596            securityContext:
   597              windowsOptions:
   598                hostProcess: true
   599                runAsUserName: "NT AUTHORITY\\SYSTEM"
   600            hostNetwork: true
   601            containers:
   602              - name: csi-proxy
   603                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   604  kind: ConfigMap
   605  metadata:
   606    annotations:
   607      note: generated
   608    labels:
   609      type: generated
   610    name: csi-proxy-addon
   611    namespace: default
   612  ---
   613  apiVersion: v1
   614  data:
   615    containerd-windows-logger: |
   616      apiVersion: apps/v1
   617      kind: DaemonSet
   618      metadata:
   619        labels:
   620          k8s-app: containerd-logger
   621        name: containerd-logger
   622        namespace: kube-system
   623      spec:
   624        selector:
   625          matchLabels:
   626            k8s-app: containerd-logger
   627        template:
   628          metadata:
   629            labels:
   630              k8s-app: containerd-logger
   631          spec:
   632            securityContext:
   633              windowsOptions:
   634                hostProcess: true
   635                runAsUserName: "NT AUTHORITY\\system"
   636            hostNetwork: true
   637            containers:
   638            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
   639              args: [ "config.json" ]
   640              name: containerd-logger
   641              imagePullPolicy: Always
   642              volumeMounts:
   643              - name: containerd-logger-config
   644                mountPath: /config.json
   645                subPath: config.json
   646            nodeSelector:
   647              kubernetes.io/os: windows
   648            tolerations:
   649            - key: CriticalAddonsOnly
   650              operator: Exists
   651            - operator: Exists
   652            volumes:
   653            - configMap:
   654                name: containerd-logger-config
   655              name: containerd-logger-config
   656        updateStrategy:
   657          type: RollingUpdate
   658      ---
   659      kind: ConfigMap
   660      apiVersion: v1
   661      metadata:
   662        name: containerd-logger-config
   663        namespace: kube-system
   664      data:
   665        config.json: |
   666          {
   667            "inputs": [
   668              {
   669                "type": "ETW",
   670                "sessionNamePrefix": "containerd",
   671                "cleanupOldSessions": true,
   672                "reuseExistingSession": true,
   673                "providers": [
   674                  {
   675                    "providerName": "Microsoft.Virtualization.RunHCS",
   676                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
   677                    "level": "Verbose"
   678                  },
   679                  {
   680                    "providerName": "ContainerD",
   681                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
   682                    "level": "Verbose"
   683                  }
   684                ]
   685              }
   686            ],
   687            "filters": [
   688              {
   689                  "type": "drop",
   690                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
   691              },
   692              {
   693                  "type": "drop",
   694                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
   695              },
   696              {
   697                  "type": "drop",
   698                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
   699              },
   700              {
   701                  "type": "drop",
   702                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
   703              },
   704              {
   705                  "type": "drop",
   706                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
   707              },
   708              {
   709                  "type": "drop",
   710                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
   711              },
   712              {
   713                  "type": "drop",
   714                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
   715              }
   716            ],
   717            "outputs": [
   718              {
   719                "type": "StdOutput"
   720              }
   721            ],
   722            "schemaVersion": "2016-08-11"
   723          }
   724  kind: ConfigMap
   725  metadata:
   726    annotations:
   727      note: generated
   728    labels:
   729      type: generated
   730    name: containerd-logger-${CLUSTER_NAME}
   731    namespace: default
   732  ---
   733  apiVersion: addons.cluster.x-k8s.io/v1beta1
   734  kind: ClusterResourceSet
   735  metadata:
   736    name: metrics-server-${CLUSTER_NAME}
   737    namespace: default
   738  spec:
   739    clusterSelector:
   740      matchLabels:
   741        metrics-server: enabled
   742    resources:
   743    - kind: ConfigMap
   744      name: metrics-server-${CLUSTER_NAME}
   745    strategy: ApplyOnce
   746  ---
   747  apiVersion: v1
   748  data:
   749    metrics-server: |
   750      apiVersion: v1
   751      kind: ServiceAccount
   752      metadata:
   753        labels:
   754          k8s-app: metrics-server
   755        name: metrics-server
   756        namespace: kube-system
   757      ---
   758      apiVersion: rbac.authorization.k8s.io/v1
   759      kind: ClusterRole
   760      metadata:
   761        labels:
   762          k8s-app: metrics-server
   763          rbac.authorization.k8s.io/aggregate-to-admin: "true"
   764          rbac.authorization.k8s.io/aggregate-to-edit: "true"
   765          rbac.authorization.k8s.io/aggregate-to-view: "true"
   766        name: system:aggregated-metrics-reader
   767      rules:
   768      - apiGroups:
   769        - metrics.k8s.io
   770        resources:
   771        - pods
   772        - nodes
   773        verbs:
   774        - get
   775        - list
   776        - watch
   777      ---
   778      apiVersion: rbac.authorization.k8s.io/v1
   779      kind: ClusterRole
   780      metadata:
   781        labels:
   782          k8s-app: metrics-server
   783        name: system:metrics-server
   784      rules:
   785      - apiGroups:
   786        - ""
   787        resources:
   788        - nodes/metrics
   789        verbs:
   790        - get
   791      - apiGroups:
   792        - ""
   793        resources:
   794        - pods
   795        - nodes
   796        verbs:
   797        - get
   798        - list
   799        - watch
   800      ---
   801      apiVersion: rbac.authorization.k8s.io/v1
   802      kind: RoleBinding
   803      metadata:
   804        labels:
   805          k8s-app: metrics-server
   806        name: metrics-server-auth-reader
   807        namespace: kube-system
   808      roleRef:
   809        apiGroup: rbac.authorization.k8s.io
   810        kind: Role
   811        name: extension-apiserver-authentication-reader
   812      subjects:
   813      - kind: ServiceAccount
   814        name: metrics-server
   815        namespace: kube-system
   816      ---
   817      apiVersion: rbac.authorization.k8s.io/v1
   818      kind: ClusterRoleBinding
   819      metadata:
   820        labels:
   821          k8s-app: metrics-server
   822        name: metrics-server:system:auth-delegator
   823      roleRef:
   824        apiGroup: rbac.authorization.k8s.io
   825        kind: ClusterRole
   826        name: system:auth-delegator
   827      subjects:
   828      - kind: ServiceAccount
   829        name: metrics-server
   830        namespace: kube-system
   831      ---
   832      apiVersion: rbac.authorization.k8s.io/v1
   833      kind: ClusterRoleBinding
   834      metadata:
   835        labels:
   836          k8s-app: metrics-server
   837        name: system:metrics-server
   838      roleRef:
   839        apiGroup: rbac.authorization.k8s.io
   840        kind: ClusterRole
   841        name: system:metrics-server
   842      subjects:
   843      - kind: ServiceAccount
   844        name: metrics-server
   845        namespace: kube-system
   846      ---
   847      apiVersion: v1
   848      kind: Service
   849      metadata:
   850        labels:
   851          k8s-app: metrics-server
   852        name: metrics-server
   853        namespace: kube-system
   854      spec:
   855        ports:
   856        - name: https
   857          port: 443
   858          protocol: TCP
   859          targetPort: https
   860        selector:
   861          k8s-app: metrics-server
   862      ---
   863      apiVersion: apps/v1
   864      kind: Deployment
   865      metadata:
   866        labels:
   867          k8s-app: metrics-server
   868        name: metrics-server
   869        namespace: kube-system
   870      spec:
   871        selector:
   872          matchLabels:
   873            k8s-app: metrics-server
   874        strategy:
   875          rollingUpdate:
   876            maxUnavailable: 0
   877        template:
   878          metadata:
   879            labels:
   880              k8s-app: metrics-server
   881          spec:
   882            containers:
   883            - args:
   884              - --cert-dir=/tmp
   885              - --secure-port=4443
   886              - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
   887              - --kubelet-use-node-status-port
   888              - --metric-resolution=15s
   889              - --kubelet-insecure-tls
   890              image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
   891              imagePullPolicy: IfNotPresent
   892              livenessProbe:
   893                failureThreshold: 3
   894                httpGet:
   895                  path: /livez
   896                  port: https
   897                  scheme: HTTPS
   898                periodSeconds: 10
   899              name: metrics-server
   900              ports:
   901              - containerPort: 4443
   902                name: https
   903                protocol: TCP
   904              readinessProbe:
   905                failureThreshold: 3
   906                httpGet:
   907                  path: /readyz
   908                  port: https
   909                  scheme: HTTPS
   910                initialDelaySeconds: 20
   911                periodSeconds: 10
   912              resources:
   913                requests:
   914                  cpu: 100m
   915                  memory: 200Mi
   916              securityContext:
   917                allowPrivilegeEscalation: false
   918                readOnlyRootFilesystem: true
   919                runAsNonRoot: true
   920                runAsUser: 1000
   921              volumeMounts:
   922              - mountPath: /tmp
   923                name: tmp-dir
   924            nodeSelector:
   925              kubernetes.io/os: linux
   926            priorityClassName: system-cluster-critical
   927            serviceAccountName: metrics-server
   928            tolerations:
   929            - effect: NoSchedule
   930              key: node-role.kubernetes.io/master
   931              operator: Exists
   932            - effect: NoSchedule
   933              key: node-role.kubernetes.io/control-plane
   934              operator: Exists
   935            volumes:
   936            - emptyDir: {}
   937              name: tmp-dir
   938      ---
   939      apiVersion: apiregistration.k8s.io/v1
   940      kind: APIService
   941      metadata:
   942        labels:
   943          k8s-app: metrics-server
   944        name: v1beta1.metrics.k8s.io
   945      spec:
   946        group: metrics.k8s.io
   947        groupPriorityMinimum: 100
   948        insecureSkipTLSVerify: true
   949        service:
   950          name: metrics-server
   951          namespace: kube-system
   952        version: v1beta1
   953        versionPriority: 100
   954  kind: ConfigMap
   955  metadata:
   956    annotations:
   957      note: generated
   958    labels:
   959      type: generated
   960    name: metrics-server-${CLUSTER_NAME}
   961    namespace: default
   962  ---
   963  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   964  kind: HelmChartProxy
   965  metadata:
   966    name: calico-ipv6
   967    namespace: default
   968  spec:
   969    chartName: tigera-operator
   970    clusterSelector:
   971      matchLabels:
   972        cni: calico-ipv6
   973    namespace: tigera-operator
   974    releaseName: projectcalico
   975    repoURL: https://docs.tigera.io/calico/charts
   976    valuesTemplate: |-
   977      installation:
   978        cni:
   979          type: Calico
   980          ipam:
   981            type: HostLocal
   982        calicoNetwork:
   983          bgp: Disabled
   984          mtu: 1350
   985          ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
   986          - blockSize: 122
   987            cidr: {{ $cidr }}
   988            encapsulation: None
   989            natOutgoing: Enabled
   990            nodeSelector: all(){{end}}
   991        registry: mcr.microsoft.com/oss
   992      # Image and registry configuration for the tigera/operator pod.
   993      tigeraOperator:
   994        image: tigera/operator
   995        registry: mcr.microsoft.com/oss
   996      calicoctl:
   997        image: mcr.microsoft.com/oss/calico/ctl
   998    version: ${CALICO_VERSION}