sigs.k8s.io/cluster-api-provider-azure@v1.14.3/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico-dual-stack
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: disabled
     9      csi-proxy: disabled
    10      metrics-server: disabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 10.244.0.0/16
    18        - 2001:1234:5678:9a40::/58
    19      services:
    20        cidrBlocks:
    21        - 10.0.0.0/16
    22        - fd00::/108
    23    controlPlaneRef:
    24      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    25      kind: KubeadmControlPlane
    26      name: ${CLUSTER_NAME}-control-plane
    27    infrastructureRef:
    28      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    29      kind: AzureCluster
    30      name: ${CLUSTER_NAME}
    31  ---
    32  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    33  kind: AzureCluster
    34  metadata:
    35    name: ${CLUSTER_NAME}
    36    namespace: default
    37  spec:
    38    additionalTags:
    39      buildProvenance: ${BUILD_PROVENANCE}
    40      creationTimestamp: ${TIMESTAMP}
    41      jobName: ${JOB_NAME}
    42    identityRef:
    43      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    44      kind: AzureClusterIdentity
    45      name: ${CLUSTER_IDENTITY_NAME}
    46    location: ${AZURE_LOCATION}
    47    networkSpec:
    48      subnets:
    49      - cidrBlocks:
    50        - 10.0.0.0/16
    51        - 2001:1234:5678:9abc::/64
    52        name: control-plane-subnet
    53        role: control-plane
    54      - cidrBlocks:
    55        - 10.1.0.0/16
    56        - 2001:1234:5678:9abd::/64
    57        name: node-subnet
    58        role: node
    59      vnet:
    60        cidrBlocks:
    61        - 10.0.0.0/8
    62        - 2001:1234:5678:9a00::/56
    63        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    64    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    65    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    66  ---
    67  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    68  kind: KubeadmControlPlane
    69  metadata:
    70    name: ${CLUSTER_NAME}-control-plane
    71    namespace: default
    72  spec:
    73    kubeadmConfigSpec:
    74      clusterConfiguration:
    75        apiServer:
    76          extraArgs:
    77            cloud-provider: external
    78            feature-gates: ${K8S_FEATURE_GATES:-""}
    79          timeoutForControlPlane: 20m
    80        controllerManager:
    81          extraArgs:
    82            allocate-node-cidrs: "true"
    83            cloud-provider: external
    84            cluster-cidr: 10.244.0.0/16,2001:1234:5678:9a40::/58
    85            cluster-name: ${CLUSTER_NAME}
    86            configure-cloud-routes: "true"
    87            feature-gates: HPAContainerMetrics=true
    88            v: "4"
    89        etcd:
    90          local:
    91            dataDir: /var/lib/etcddisk/etcd
    92            extraArgs:
    93              quota-backend-bytes: "8589934592"
    94        kubernetesVersion: ci/${CI_VERSION}
    95      diskSetup:
    96        filesystems:
    97        - device: /dev/disk/azure/scsi1/lun0
    98          extraOpts:
    99          - -E
   100          - lazy_itable_init=1,lazy_journal_init=1
   101          filesystem: ext4
   102          label: etcd_disk
   103        - device: ephemeral0.1
   104          filesystem: ext4
   105          label: ephemeral0
   106          replaceFS: ntfs
   107        partitions:
   108        - device: /dev/disk/azure/scsi1/lun0
   109          layout: true
   110          overwrite: false
   111          tableType: gpt
   112      files:
   113      - contentFrom:
   114          secret:
   115            key: control-plane-azure.json
   116            name: ${CLUSTER_NAME}-control-plane-azure-json
   117        owner: root:root
   118        path: /etc/kubernetes/azure.json
   119        permissions: "0644"
   120      - content: |
   121          #!/bin/bash
   122  
   123          set -o nounset
   124          set -o pipefail
   125          set -o errexit
   126          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   127  
   128          az login --identity
   129          echo "Use OOT credential provider"
   130          mkdir -p /var/lib/kubelet/credential-provider
   131          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   132          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   133          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   134          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   135        owner: root:root
   136        path: /tmp/oot-cred-provider.sh
   137        permissions: "0744"
   138      - content: |
   139          #!/bin/bash
   140  
   141          set -o nounset
   142          set -o pipefail
   143          set -o errexit
   144          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   145  
   146          # This test installs release packages or binaries that are a result of the CI and release builds.
   147          # It runs '... --version' commands to verify that the binaries are correctly installed
   148          # and finally uninstalls the packages.
   149          # For the release packages it tests all versions in the support skew.
   150          LINE_SEPARATOR="*************************************************"
   151          echo "$$LINE_SEPARATOR"
   152          CI_VERSION=${CI_VERSION}
   153          if [[ "$${CI_VERSION}" != "" ]]; then
   154            CI_DIR=/tmp/k8s-ci
   155            mkdir -p $$CI_DIR
   156            declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   157            declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   158            CONTAINER_EXT="tar"
   159            echo "* testing CI version $$CI_VERSION"
   160            # Check for semver
   161            if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   162              VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   163              DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   164              curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   165              echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   166              apt-get update
   167              # replace . with \.
   168              VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   169              PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   170              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   171                echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   172                DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   173              done
   174            else
   175              CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   176              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   177                echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   178                wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   179                chmod +x "$$CI_DIR/$$CI_PACKAGE"
   180                mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   181              done
   182              IMAGE_REGISTRY_PREFIX=registry.k8s.io
   183              for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   184                echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   185                wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   186                $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   187                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   188                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   189              done
   190            fi
   191            systemctl restart kubelet
   192          fi
   193          echo "* checking binary versions"
   194          echo "ctr version: " $(ctr version)
   195          echo "kubeadm version: " $(kubeadm version -o=short)
   196          echo "kubectl version: " $(kubectl version --client=true)
   197          echo "kubelet version: " $(kubelet --version)
   198          echo "$$LINE_SEPARATOR"
   199        owner: root:root
   200        path: /tmp/kubeadm-bootstrap.sh
   201        permissions: "0744"
   202      initConfiguration:
   203        localAPIEndpoint:
   204          bindPort: 6443
   205        nodeRegistration:
   206          kubeletExtraArgs:
   207            cloud-provider: external
   208            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   209            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   210          name: '{{ ds.meta_data["local_hostname"] }}'
   211      joinConfiguration:
   212        controlPlane:
   213          localAPIEndpoint:
   214            bindPort: 6443
   215        nodeRegistration:
   216          kubeletExtraArgs:
   217            cloud-provider: external
   218            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   219            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   220          name: '{{ ds.meta_data["local_hostname"] }}'
   221      mounts:
   222      - - LABEL=etcd_disk
   223        - /var/lib/etcddisk
   224      postKubeadmCommands: []
   225      preKubeadmCommands:
   226      - bash -c /tmp/oot-cred-provider.sh
   227      - bash -c /tmp/kubeadm-bootstrap.sh
   228      verbosity: 5
   229    machineTemplate:
   230      infrastructureRef:
   231        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   232        kind: AzureMachineTemplate
   233        name: ${CLUSTER_NAME}-control-plane
   234    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   235    version: ${KUBERNETES_VERSION}
   236  ---
   237  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   238  kind: AzureMachineTemplate
   239  metadata:
   240    name: ${CLUSTER_NAME}-control-plane
   241    namespace: default
   242  spec:
   243    template:
   244      spec:
   245        dataDisks:
   246        - diskSizeGB: 256
   247          lun: 0
   248          nameSuffix: etcddisk
   249        enableIPForwarding: true
   250        image:
   251          marketplace:
   252            offer: capi
   253            publisher: cncf-upstream
   254            sku: ubuntu-2204-gen1
   255            version: latest
   256        osDisk:
   257          diskSizeGB: 128
   258          osType: Linux
   259        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   260        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   261  ---
   262  apiVersion: cluster.x-k8s.io/v1beta1
   263  kind: MachineDeployment
   264  metadata:
   265    name: ${CLUSTER_NAME}-md-0
   266    namespace: default
   267  spec:
   268    clusterName: ${CLUSTER_NAME}
   269    replicas: ${WORKER_MACHINE_COUNT:=2}
   270    selector: {}
   271    template:
   272      metadata:
   273        labels:
   274          nodepool: pool1
   275      spec:
   276        bootstrap:
   277          configRef:
   278            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   279            kind: KubeadmConfigTemplate
   280            name: ${CLUSTER_NAME}-md-0
   281        clusterName: ${CLUSTER_NAME}
   282        infrastructureRef:
   283          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   284          kind: AzureMachineTemplate
   285          name: ${CLUSTER_NAME}-md-0
   286        version: ${KUBERNETES_VERSION}
   287  ---
   288  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   289  kind: AzureMachineTemplate
   290  metadata:
   291    name: ${CLUSTER_NAME}-md-0
   292    namespace: default
   293  spec:
   294    template:
   295      spec:
   296        enableIPForwarding: true
   297        image:
   298          marketplace:
   299            offer: capi
   300            publisher: cncf-upstream
   301            sku: ubuntu-2204-gen1
   302            version: latest
   303        osDisk:
   304          diskSizeGB: 128
   305          osType: Linux
   306        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   307        vmExtensions:
   308        - name: CustomScript
   309          protectedSettings:
   310            commandToExecute: |
   311              #!/bin/sh
   312              echo "This script is a no-op used for extension testing purposes ..."
   313              touch test_file
   314          publisher: Microsoft.Azure.Extensions
   315          version: "2.1"
   316        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   317  ---
   318  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   319  kind: KubeadmConfigTemplate
   320  metadata:
   321    name: ${CLUSTER_NAME}-md-0
   322    namespace: default
   323  spec:
   324    template:
   325      spec:
   326        files:
   327        - contentFrom:
   328            secret:
   329              key: worker-node-azure.json
   330              name: ${CLUSTER_NAME}-md-0-azure-json
   331          owner: root:root
   332          path: /etc/kubernetes/azure.json
   333          permissions: "0644"
   334        - content: |
   335            #!/bin/bash
   336  
   337            set -o nounset
   338            set -o pipefail
   339            set -o errexit
   340            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   341  
   342            az login --identity
   343            echo "Use OOT credential provider"
   344            mkdir -p /var/lib/kubelet/credential-provider
   345            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   346            chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   347            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   348            chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   349          owner: root:root
   350          path: /tmp/oot-cred-provider.sh
   351          permissions: "0744"
   352        - content: |
   353            #!/bin/bash
   354  
   355            set -o nounset
   356            set -o pipefail
   357            set -o errexit
   358            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   359  
   360            # This test installs release packages or binaries that are a result of the CI and release builds.
   361            # It runs '... --version' commands to verify that the binaries are correctly installed
   362            # and finally uninstalls the packages.
   363            # For the release packages it tests all versions in the support skew.
   364            LINE_SEPARATOR="*************************************************"
   365            echo "$$LINE_SEPARATOR"
   366            CI_VERSION=${CI_VERSION}
   367            if [[ "$${CI_VERSION}" != "" ]]; then
   368              CI_DIR=/tmp/k8s-ci
   369              mkdir -p $$CI_DIR
   370              declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   371              declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   372              CONTAINER_EXT="tar"
   373              echo "* testing CI version $$CI_VERSION"
   374              # Check for semver
   375              if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   376                VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   377                DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   378                curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   379                echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   380                apt-get update
   381                # replace . with \.
   382                VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   383                PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   384                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   385                  echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   386                  DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   387                done
   388              else
   389                CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   390                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   391                  echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   392                  wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   393                  chmod +x "$$CI_DIR/$$CI_PACKAGE"
   394                  mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   395                done
   396                IMAGE_REGISTRY_PREFIX=registry.k8s.io
   397                for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   398                  echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   399                  wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   400                  $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   401                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   402                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   403                done
   404              fi
   405              systemctl restart kubelet
   406            fi
   407            echo "* checking binary versions"
   408            echo "ctr version: " $(ctr version)
   409            echo "kubeadm version: " $(kubeadm version -o=short)
   410            echo "kubectl version: " $(kubectl version --client=true)
   411            echo "kubelet version: " $(kubelet --version)
   412            echo "$$LINE_SEPARATOR"
   413          owner: root:root
   414          path: /tmp/kubeadm-bootstrap.sh
   415          permissions: "0744"
   416        joinConfiguration:
   417          nodeRegistration:
   418            kubeletExtraArgs:
   419              cloud-provider: external
   420              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   421              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   422            name: '{{ ds.meta_data["local_hostname"] }}'
   423        preKubeadmCommands:
   424        - bash -c /tmp/oot-cred-provider.sh
   425        - bash -c /tmp/kubeadm-bootstrap.sh
   426        verbosity: 5
   427  ---
   428  apiVersion: cluster.x-k8s.io/v1beta1
   429  kind: MachineHealthCheck
   430  metadata:
   431    name: ${CLUSTER_NAME}-mhc-0
   432    namespace: default
   433  spec:
   434    clusterName: ${CLUSTER_NAME}
   435    maxUnhealthy: 100%
   436    selector:
   437      matchLabels:
   438        nodepool: pool1
   439    unhealthyConditions:
   440    - status: "True"
   441      timeout: 30s
   442      type: E2ENodeUnhealthy
   443  ---
   444  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   445  kind: AzureClusterIdentity
   446  metadata:
   447    labels:
   448      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   449    name: ${CLUSTER_IDENTITY_NAME}
   450    namespace: default
   451  spec:
   452    allowedNamespaces: {}
   453    clientID: ${AZURE_CLIENT_ID}
   454    clientSecret:
   455      name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME}
   456      namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
   457    tenantID: ${AZURE_TENANT_ID}
   458    type: ServicePrincipal
   459  ---
   460  apiVersion: addons.cluster.x-k8s.io/v1beta1
   461  kind: ClusterResourceSet
   462  metadata:
   463    name: csi-proxy
   464    namespace: default
   465  spec:
   466    clusterSelector:
   467      matchLabels:
   468        csi-proxy: enabled
   469    resources:
   470    - kind: ConfigMap
   471      name: csi-proxy-addon
   472    strategy: ApplyOnce
   473  ---
   474  apiVersion: addons.cluster.x-k8s.io/v1beta1
   475  kind: ClusterResourceSet
   476  metadata:
   477    name: containerd-logger-${CLUSTER_NAME}
   478    namespace: default
   479  spec:
   480    clusterSelector:
   481      matchLabels:
   482        containerd-logger: enabled
   483    resources:
   484    - kind: ConfigMap
   485      name: containerd-logger-${CLUSTER_NAME}
   486    strategy: ApplyOnce
   487  ---
   488  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   489  kind: HelmChartProxy
   490  metadata:
   491    name: azuredisk-csi-driver-chart
   492    namespace: default
   493  spec:
   494    chartName: azuredisk-csi-driver
   495    clusterSelector:
   496      matchLabels:
   497        azuredisk-csi: "true"
   498    namespace: kube-system
   499    releaseName: azuredisk-csi-driver-oot
   500    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   501    valuesTemplate: |-
   502      controller:
   503        replicas: 1
   504        runOnControlPlane: true
   505      windows:
   506        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   507  ---
   508  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   509  kind: HelmChartProxy
   510  metadata:
   511    name: cloud-provider-azure-chart
   512    namespace: default
   513  spec:
   514    chartName: cloud-provider-azure
   515    clusterSelector:
   516      matchLabels:
   517        cloud-provider: azure
   518    releaseName: cloud-provider-azure-oot
   519    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   520    valuesTemplate: |
   521      infra:
   522        clusterName: {{ .Cluster.metadata.name }}
   523      cloudControllerManager:
   524        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   525        logVerbosity: 4
   526  ---
   527  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   528  kind: HelmChartProxy
   529  metadata:
   530    name: cloud-provider-azure-chart-ci
   531    namespace: default
   532  spec:
   533    chartName: cloud-provider-azure
   534    clusterSelector:
   535      matchLabels:
   536        cloud-provider: azure-ci
   537    releaseName: cloud-provider-azure-oot
   538    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   539    valuesTemplate: |
   540      infra:
   541        clusterName: {{ .Cluster.metadata.name }}
   542      cloudControllerManager:
   543        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   544        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   545        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   546        imageName: "${CCM_IMAGE_NAME:-""}"
   547        imageRepository: "${IMAGE_REGISTRY:-""}"
   548        imageTag: "${IMAGE_TAG_CCM:-""}"
   549        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   550        replicas: ${CCM_COUNT:-1}
   551        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   552      cloudNodeManager:
   553        imageName: "${CNM_IMAGE_NAME:-""}"
   554        imageRepository: "${IMAGE_REGISTRY:-""}"
   555        imageTag: "${IMAGE_TAG_CNM:-""}"
   556  ---
   557  apiVersion: v1
   558  data:
   559    csi-proxy: |
   560      apiVersion: apps/v1
   561      kind: DaemonSet
   562      metadata:
   563        labels:
   564          k8s-app: csi-proxy
   565        name: csi-proxy
   566        namespace: kube-system
   567      spec:
   568        selector:
   569          matchLabels:
   570            k8s-app: csi-proxy
   571        template:
   572          metadata:
   573            labels:
   574              k8s-app: csi-proxy
   575          spec:
   576            nodeSelector:
   577              "kubernetes.io/os": windows
   578            securityContext:
   579              windowsOptions:
   580                hostProcess: true
   581                runAsUserName: "NT AUTHORITY\\SYSTEM"
   582            hostNetwork: true
   583            containers:
   584              - name: csi-proxy
   585                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   586  kind: ConfigMap
   587  metadata:
   588    annotations:
   589      note: generated
   590    labels:
   591      type: generated
   592    name: csi-proxy-addon
   593    namespace: default
   594  ---
   595  apiVersion: v1
   596  data:
   597    containerd-windows-logger: |
   598      apiVersion: apps/v1
   599      kind: DaemonSet
   600      metadata:
   601        labels:
   602          k8s-app: containerd-logger
   603        name: containerd-logger
   604        namespace: kube-system
   605      spec:
   606        selector:
   607          matchLabels:
   608            k8s-app: containerd-logger
   609        template:
   610          metadata:
   611            labels:
   612              k8s-app: containerd-logger
   613          spec:
   614            securityContext:
   615              windowsOptions:
   616                hostProcess: true
   617                runAsUserName: "NT AUTHORITY\\system"
   618            hostNetwork: true
   619            containers:
   620            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
   621              args: [ "config.json" ]
   622              name: containerd-logger
   623              imagePullPolicy: Always
   624              volumeMounts:
   625              - name: containerd-logger-config
   626                mountPath: /config.json
   627                subPath: config.json
   628            nodeSelector:
   629              kubernetes.io/os: windows
   630            tolerations:
   631            - key: CriticalAddonsOnly
   632              operator: Exists
   633            - operator: Exists
   634            volumes:
   635            - configMap:
   636                name: containerd-logger-config
   637              name: containerd-logger-config
   638        updateStrategy:
   639          type: RollingUpdate
   640      ---
   641      kind: ConfigMap
   642      apiVersion: v1
   643      metadata:
   644        name: containerd-logger-config
   645        namespace: kube-system
   646      data:
   647        config.json: |
   648          {
   649            "inputs": [
   650              {
   651                "type": "ETW",
   652                "sessionNamePrefix": "containerd",
   653                "cleanupOldSessions": true,
   654                "reuseExistingSession": true,
   655                "providers": [
   656                  {
   657                    "providerName": "Microsoft.Virtualization.RunHCS",
   658                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
   659                    "level": "Verbose"
   660                  },
   661                  {
   662                    "providerName": "ContainerD",
   663                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
   664                    "level": "Verbose"
   665                  }
   666                ]
   667              }
   668            ],
   669            "filters": [
   670              {
   671                  "type": "drop",
   672                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
   673              },
   674              {
   675                  "type": "drop",
   676                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
   677              },
   678              {
   679                  "type": "drop",
   680                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
   681              },
   682              {
   683                  "type": "drop",
   684                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
   685              },
   686              {
   687                  "type": "drop",
   688                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
   689              },
   690              {
   691                  "type": "drop",
   692                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
   693              },
   694              {
   695                  "type": "drop",
   696                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
   697              }
   698            ],
   699            "outputs": [
   700              {
   701                "type": "StdOutput"
   702              }
   703            ],
   704            "schemaVersion": "2016-08-11"
   705          }
   706  kind: ConfigMap
   707  metadata:
   708    annotations:
   709      note: generated
   710    labels:
   711      type: generated
   712    name: containerd-logger-${CLUSTER_NAME}
   713    namespace: default
   714  ---
   715  apiVersion: addons.cluster.x-k8s.io/v1beta1
   716  kind: ClusterResourceSet
   717  metadata:
   718    name: metrics-server-${CLUSTER_NAME}
   719    namespace: default
   720  spec:
   721    clusterSelector:
   722      matchLabels:
   723        metrics-server: enabled
   724    resources:
   725    - kind: ConfigMap
   726      name: metrics-server-${CLUSTER_NAME}
   727    strategy: ApplyOnce
   728  ---
   729  apiVersion: v1
   730  data:
   731    metrics-server: |
   732      apiVersion: v1
   733      kind: ServiceAccount
   734      metadata:
   735        labels:
   736          k8s-app: metrics-server
   737        name: metrics-server
   738        namespace: kube-system
   739      ---
   740      apiVersion: rbac.authorization.k8s.io/v1
   741      kind: ClusterRole
   742      metadata:
   743        labels:
   744          k8s-app: metrics-server
   745          rbac.authorization.k8s.io/aggregate-to-admin: "true"
   746          rbac.authorization.k8s.io/aggregate-to-edit: "true"
   747          rbac.authorization.k8s.io/aggregate-to-view: "true"
   748        name: system:aggregated-metrics-reader
   749      rules:
   750      - apiGroups:
   751        - metrics.k8s.io
   752        resources:
   753        - pods
   754        - nodes
   755        verbs:
   756        - get
   757        - list
   758        - watch
   759      ---
   760      apiVersion: rbac.authorization.k8s.io/v1
   761      kind: ClusterRole
   762      metadata:
   763        labels:
   764          k8s-app: metrics-server
   765        name: system:metrics-server
   766      rules:
   767      - apiGroups:
   768        - ""
   769        resources:
   770        - nodes/metrics
   771        verbs:
   772        - get
   773      - apiGroups:
   774        - ""
   775        resources:
   776        - pods
   777        - nodes
   778        verbs:
   779        - get
   780        - list
   781        - watch
   782      ---
   783      apiVersion: rbac.authorization.k8s.io/v1
   784      kind: RoleBinding
   785      metadata:
   786        labels:
   787          k8s-app: metrics-server
   788        name: metrics-server-auth-reader
   789        namespace: kube-system
   790      roleRef:
   791        apiGroup: rbac.authorization.k8s.io
   792        kind: Role
   793        name: extension-apiserver-authentication-reader
   794      subjects:
   795      - kind: ServiceAccount
   796        name: metrics-server
   797        namespace: kube-system
   798      ---
   799      apiVersion: rbac.authorization.k8s.io/v1
   800      kind: ClusterRoleBinding
   801      metadata:
   802        labels:
   803          k8s-app: metrics-server
   804        name: metrics-server:system:auth-delegator
   805      roleRef:
   806        apiGroup: rbac.authorization.k8s.io
   807        kind: ClusterRole
   808        name: system:auth-delegator
   809      subjects:
   810      - kind: ServiceAccount
   811        name: metrics-server
   812        namespace: kube-system
   813      ---
   814      apiVersion: rbac.authorization.k8s.io/v1
   815      kind: ClusterRoleBinding
   816      metadata:
   817        labels:
   818          k8s-app: metrics-server
   819        name: system:metrics-server
   820      roleRef:
   821        apiGroup: rbac.authorization.k8s.io
   822        kind: ClusterRole
   823        name: system:metrics-server
   824      subjects:
   825      - kind: ServiceAccount
   826        name: metrics-server
   827        namespace: kube-system
   828      ---
   829      apiVersion: v1
   830      kind: Service
   831      metadata:
   832        labels:
   833          k8s-app: metrics-server
   834        name: metrics-server
   835        namespace: kube-system
   836      spec:
   837        ports:
   838        - name: https
   839          port: 443
   840          protocol: TCP
   841          targetPort: https
   842        selector:
   843          k8s-app: metrics-server
   844      ---
   845      apiVersion: apps/v1
   846      kind: Deployment
   847      metadata:
   848        labels:
   849          k8s-app: metrics-server
   850        name: metrics-server
   851        namespace: kube-system
   852      spec:
   853        selector:
   854          matchLabels:
   855            k8s-app: metrics-server
   856        strategy:
   857          rollingUpdate:
   858            maxUnavailable: 0
   859        template:
   860          metadata:
   861            labels:
   862              k8s-app: metrics-server
   863          spec:
   864            containers:
   865            - args:
   866              - --cert-dir=/tmp
   867              - --secure-port=4443
   868              - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
   869              - --kubelet-use-node-status-port
   870              - --metric-resolution=15s
   871              - --kubelet-insecure-tls
   872              image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
   873              imagePullPolicy: IfNotPresent
   874              livenessProbe:
   875                failureThreshold: 3
   876                httpGet:
   877                  path: /livez
   878                  port: https
   879                  scheme: HTTPS
   880                periodSeconds: 10
   881              name: metrics-server
   882              ports:
   883              - containerPort: 4443
   884                name: https
   885                protocol: TCP
   886              readinessProbe:
   887                failureThreshold: 3
   888                httpGet:
   889                  path: /readyz
   890                  port: https
   891                  scheme: HTTPS
   892                initialDelaySeconds: 20
   893                periodSeconds: 10
   894              resources:
   895                requests:
   896                  cpu: 100m
   897                  memory: 200Mi
   898              securityContext:
   899                allowPrivilegeEscalation: false
   900                readOnlyRootFilesystem: true
   901                runAsNonRoot: true
   902                runAsUser: 1000
   903              volumeMounts:
   904              - mountPath: /tmp
   905                name: tmp-dir
   906            nodeSelector:
   907              kubernetes.io/os: linux
   908            priorityClassName: system-cluster-critical
   909            serviceAccountName: metrics-server
   910            tolerations:
   911            - effect: NoSchedule
   912              key: node-role.kubernetes.io/master
   913              operator: Exists
   914            - effect: NoSchedule
   915              key: node-role.kubernetes.io/control-plane
   916              operator: Exists
   917            volumes:
   918            - emptyDir: {}
   919              name: tmp-dir
   920      ---
   921      apiVersion: apiregistration.k8s.io/v1
   922      kind: APIService
   923      metadata:
   924        labels:
   925          k8s-app: metrics-server
   926        name: v1beta1.metrics.k8s.io
   927      spec:
   928        group: metrics.k8s.io
   929        groupPriorityMinimum: 100
   930        insecureSkipTLSVerify: true
   931        service:
   932          name: metrics-server
   933          namespace: kube-system
   934        version: v1beta1
   935        versionPriority: 100
   936  kind: ConfigMap
   937  metadata:
   938    annotations:
   939      note: generated
   940    labels:
   941      type: generated
   942    name: metrics-server-${CLUSTER_NAME}
   943    namespace: default
   944  ---
   945  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   946  kind: HelmChartProxy
   947  metadata:
   948    name: calico-dual-stack
   949    namespace: default
   950  spec:
   951    chartName: tigera-operator
   952    clusterSelector:
   953      matchLabels:
   954        cni: calico-dual-stack
   955    namespace: tigera-operator
   956    releaseName: projectcalico
   957    repoURL: https://docs.tigera.io/calico/charts
   958    valuesTemplate: |
   959      installation:
   960        cni:
   961          type: Calico
   962          ipam:
   963            type: HostLocal
   964        calicoNetwork:
   965          bgp: Disabled
   966          mtu: 1350
   967          ipPools:
   968          - blockSize: 26
   969            cidr: {{ index .Cluster.spec.clusterNetwork.pods.cidrBlocks 0 }}
   970            encapsulation: None
   971            natOutgoing: Enabled
   972            nodeSelector: all()
   973          - blockSize: 122
   974            cidr: {{ index .Cluster.spec.clusterNetwork.pods.cidrBlocks 1 }}
   975            encapsulation: None
   976            natOutgoing: Enabled
   977            nodeSelector: all()
   978        registry: mcr.microsoft.com/oss
   979      # Image and registry configuration for the tigera/operator pod.
   980      tigeraOperator:
   981        image: tigera/operator
   982        registry: mcr.microsoft.com/oss
   983      calicoctl:
   984        image: mcr.microsoft.com/oss/calico/ctl
   985    version: ${CALICO_VERSION}