sigs.k8s.io/cluster-api-provider-azure@v1.17.0/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: enabled
     9      csi-proxy: enabled
    10      windows: enabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 192.168.0.0/16
    18    controlPlaneRef:
    19      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    20      kind: KubeadmControlPlane
    21      name: ${CLUSTER_NAME}-control-plane
    22    infrastructureRef:
    23      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    24      kind: AzureCluster
    25      name: ${CLUSTER_NAME}
    26  ---
    27  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    28  kind: AzureCluster
    29  metadata:
    30    name: ${CLUSTER_NAME}
    31    namespace: default
    32  spec:
    33    additionalTags:
    34      buildProvenance: ${BUILD_PROVENANCE}
    35      creationTimestamp: ${TIMESTAMP}
    36      jobName: ${JOB_NAME}
    37    identityRef:
    38      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    39      kind: AzureClusterIdentity
    40      name: ${CLUSTER_IDENTITY_NAME}
    41    location: ${AZURE_LOCATION}
    42    networkSpec:
    43      subnets:
    44      - name: control-plane-subnet
    45        role: control-plane
    46      - name: node-subnet
    47        role: node
    48      vnet:
    49        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    50    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    51    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    52  ---
    53  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    54  kind: KubeadmControlPlane
    55  metadata:
    56    annotations:
    57      controlplane.cluster.x-k8s.io/skip-kube-proxy: "true"
    58    name: ${CLUSTER_NAME}-control-plane
    59    namespace: default
    60  spec:
    61    kubeadmConfigSpec:
    62      clusterConfiguration:
    63        apiServer:
    64          extraArgs:
    65            cloud-provider: external
    66          timeoutForControlPlane: 20m
    67        controllerManager:
    68          extraArgs:
    69            allocate-node-cidrs: "false"
    70            cloud-provider: external
    71            cluster-name: ${CLUSTER_NAME}
    72            v: "4"
    73        etcd:
    74          local:
    75            dataDir: /var/lib/etcddisk/etcd
    76            extraArgs:
    77              quota-backend-bytes: "8589934592"
    78        kubernetesVersion: ci/${CI_VERSION}
    79      diskSetup:
    80        filesystems:
    81        - device: /dev/disk/azure/scsi1/lun0
    82          extraOpts:
    83          - -E
    84          - lazy_itable_init=1,lazy_journal_init=1
    85          filesystem: ext4
    86          label: etcd_disk
    87        - device: ephemeral0.1
    88          filesystem: ext4
    89          label: ephemeral0
    90          replaceFS: ntfs
    91        partitions:
    92        - device: /dev/disk/azure/scsi1/lun0
    93          layout: true
    94          overwrite: false
    95          tableType: gpt
    96      files:
    97      - content: |
    98          #!/bin/bash
    99  
   100          set -o nounset
   101          set -o pipefail
   102          set -o errexit
   103  
   104          systemctl stop kubelet
   105          declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
   106          for BINARY in "$${BINARIES[@]}"; do
   107            echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}"
   108            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}"
   109          done
   110          systemctl restart kubelet
   111  
   112          # prepull images from gcr.io/k8s-staging-ci-images and retag it to
   113          # registry.k8s.io so kubeadm can fetch correct images no matter what
   114          declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   115          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   116          IMAGE_REGISTRY_PREFIX=registry.k8s.io
   117          for IMAGE in "$${IMAGES[@]}"; do
   118            $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}"
   119            $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}"
   120          done
   121  
   122          echo "kubeadm version: $(kubeadm version -o=short)"
   123          echo "kubectl version: $(kubectl version --client=true)"
   124          echo "kubelet version: $(kubelet --version)"
   125        owner: root:root
   126        path: /tmp/replace-k8s-binaries.sh
   127        permissions: "0744"
   128      - content: |
   129          #!/bin/bash
   130  
   131          set -o nounset
   132          set -o pipefail
   133          set -o errexit
   134  
   135          curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz
   136          tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq
   137          rm /tmp/yq_linux_amd64.tar.gz
   138  
   139          export KUBECONFIG=/etc/kubernetes/admin.conf
   140          kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}"
   141          systemctl stop kubelet
   142          yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
   143          yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
   144          yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
   145          systemctl restart kubelet
   146        owner: root:root
   147        path: /tmp/replace-k8s-components.sh
   148        permissions: "0744"
   149      - contentFrom:
   150          secret:
   151            key: control-plane-azure.json
   152            name: ${CLUSTER_NAME}-control-plane-azure-json
   153        owner: root:root
   154        path: /etc/kubernetes/azure.json
   155        permissions: "0644"
   156      - content: |
   157          #!/bin/bash
   158  
   159          set -o nounset
   160          set -o pipefail
   161          set -o errexit
   162          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   163  
   164          echo "Use OOT credential provider"
   165          mkdir -p /var/lib/kubelet/credential-provider
   166          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   167          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   168          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   169          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   170        owner: root:root
   171        path: /tmp/oot-cred-provider.sh
   172        permissions: "0744"
   173      initConfiguration:
   174        nodeRegistration:
   175          kubeletExtraArgs:
   176            cloud-provider: external
   177            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   178            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   179          name: '{{ ds.meta_data["local_hostname"] }}'
   180      joinConfiguration:
   181        nodeRegistration:
   182          kubeletExtraArgs:
   183            cloud-provider: external
   184            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   185            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   186          name: '{{ ds.meta_data["local_hostname"] }}'
   187      mounts:
   188      - - LABEL=etcd_disk
   189        - /var/lib/etcddisk
   190      postKubeadmCommands:
   191      - bash -c /tmp/replace-k8s-components.sh
   192      preKubeadmCommands:
   193      - bash -c /tmp/replace-k8s-binaries.sh
   194      - bash -c /tmp/oot-cred-provider.sh
   195      verbosity: 5
   196    machineTemplate:
   197      infrastructureRef:
   198        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   199        kind: AzureMachineTemplate
   200        name: ${CLUSTER_NAME}-control-plane
   201    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   202    version: ${KUBERNETES_VERSION}
   203  ---
   204  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   205  kind: AzureMachineTemplate
   206  metadata:
   207    name: ${CLUSTER_NAME}-control-plane
   208    namespace: default
   209  spec:
   210    template:
   211      spec:
   212        dataDisks:
   213        - diskSizeGB: 256
   214          lun: 0
   215          nameSuffix: etcddisk
   216        identity: UserAssigned
   217        image:
   218          marketplace:
   219            offer: capi
   220            publisher: cncf-upstream
   221            sku: ubuntu-2204-gen1
   222            version: latest
   223        osDisk:
   224          diskSizeGB: 128
   225          osType: Linux
   226        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   227        userAssignedIdentities:
   228        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
   229        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   230  ---
   231  apiVersion: cluster.x-k8s.io/v1beta1
   232  kind: MachinePool
   233  metadata:
   234    name: ${CLUSTER_NAME}-mp-0
   235    namespace: default
   236  spec:
   237    clusterName: ${CLUSTER_NAME}
   238    replicas: ${WORKER_MACHINE_COUNT:=2}
   239    template:
   240      spec:
   241        bootstrap:
   242          configRef:
   243            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   244            kind: KubeadmConfig
   245            name: ${CLUSTER_NAME}-mp-0
   246        clusterName: ${CLUSTER_NAME}
   247        infrastructureRef:
   248          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   249          kind: AzureMachinePool
   250          name: ${CLUSTER_NAME}-mp-0
   251        version: ${KUBERNETES_VERSION}
   252  ---
   253  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   254  kind: AzureMachinePool
   255  metadata:
   256    name: ${CLUSTER_NAME}-mp-0
   257    namespace: default
   258  spec:
   259    location: ${AZURE_LOCATION}
   260    strategy:
   261      rollingUpdate:
   262        deletePolicy: Oldest
   263        maxSurge: 25%
   264        maxUnavailable: 1
   265      type: RollingUpdate
   266    template:
   267      image:
   268        marketplace:
   269          offer: capi
   270          publisher: cncf-upstream
   271          sku: ubuntu-2204-gen1
   272          version: latest
   273      osDisk:
   274        diskSizeGB: 30
   275        managedDisk:
   276          storageAccountType: Premium_LRS
   277        osType: Linux
   278      sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   279      vmExtensions:
   280      - name: CustomScript
   281        protectedSettings:
   282          commandToExecute: |
   283            #!/bin/sh
   284            echo "This script is a no-op used for extension testing purposes ..."
   285            touch test_file
   286        publisher: Microsoft.Azure.Extensions
   287        version: "2.1"
   288      vmSize: ${AZURE_NODE_MACHINE_TYPE}
   289  ---
   290  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   291  kind: KubeadmConfig
   292  metadata:
   293    name: ${CLUSTER_NAME}-mp-0
   294    namespace: default
   295  spec:
   296    files:
   297    - content: |
   298        #!/bin/bash
   299  
   300        set -o nounset
   301        set -o pipefail
   302        set -o errexit
   303        [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   304  
   305        echo "Use OOT credential provider"
   306        mkdir -p /var/lib/kubelet/credential-provider
   307        curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   308        chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   309        curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   310        chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   311      owner: root:root
   312      path: /tmp/oot-cred-provider.sh
   313      permissions: "0744"
   314    - content: |
   315        #!/bin/bash
   316  
   317        set -o nounset
   318        set -o pipefail
   319        set -o errexit
   320  
   321        systemctl stop kubelet
   322        declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
   323        for BINARY in "$${BINARIES[@]}"; do
   324          echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}"
   325          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}"
   326        done
   327        systemctl restart kubelet
   328  
   329        echo "kubeadm version: $(kubeadm version -o=short)"
   330        echo "kubectl version: $(kubectl version --client=true)"
   331        echo "kubelet version: $(kubelet --version)"
   332      owner: root:root
   333      path: /tmp/replace-k8s-binaries.sh
   334      permissions: "0744"
   335    - contentFrom:
   336        secret:
   337          key: control-plane-azure.json
   338          name: ${CLUSTER_NAME}-control-plane-azure-json
   339      owner: root:root
   340      path: /etc/kubernetes/azure.json
   341      permissions: "0644"
   342    joinConfiguration:
   343      nodeRegistration:
   344        kubeletExtraArgs:
   345          cloud-provider: external
   346          image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   347          image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   348        name: '{{ ds.meta_data["local_hostname"] }}'
   349    preKubeadmCommands:
   350    - bash -c /tmp/oot-cred-provider.sh
   351    - bash -c /tmp/replace-k8s-binaries.sh
   352  ---
   353  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   354  kind: AzureClusterIdentity
   355  metadata:
   356    labels:
   357      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   358    name: ${CLUSTER_IDENTITY_NAME}
   359    namespace: default
   360  spec:
   361    allowedNamespaces: {}
   362    clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}
   363    tenantID: ${AZURE_TENANT_ID}
   364    type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity}
   365  ---
   366  apiVersion: cluster.x-k8s.io/v1beta1
   367  kind: MachinePool
   368  metadata:
   369    name: ${CLUSTER_NAME}-mp-win
   370    namespace: default
   371  spec:
   372    clusterName: ${CLUSTER_NAME}
   373    replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0}
   374    template:
   375      spec:
   376        bootstrap:
   377          configRef:
   378            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   379            kind: KubeadmConfig
   380            name: ${CLUSTER_NAME}-mp-win
   381        clusterName: ${CLUSTER_NAME}
   382        infrastructureRef:
   383          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   384          kind: AzureMachinePool
   385          name: ${CLUSTER_NAME}-mp-win
   386        version: ${KUBERNETES_VERSION}
   387  ---
   388  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   389  kind: AzureMachinePool
   390  metadata:
   391    annotations:
   392      runtime: containerd
   393      windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""}
   394    name: ${CLUSTER_NAME}-mp-win
   395    namespace: default
   396  spec:
   397    location: ${AZURE_LOCATION}
   398    template:
   399      image:
   400        marketplace:
   401          offer: capi-windows
   402          publisher: cncf-upstream
   403          sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1
   404          version: latest
   405      osDisk:
   406        diskSizeGB: 128
   407        managedDisk:
   408          storageAccountType: Premium_LRS
   409        osType: Windows
   410      sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   411      vmSize: ${AZURE_NODE_MACHINE_TYPE}
   412  ---
   413  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   414  kind: KubeadmConfig
   415  metadata:
   416    name: ${CLUSTER_NAME}-mp-win
   417    namespace: default
   418  spec:
   419    files:
   420    - contentFrom:
   421        secret:
   422          key: worker-node-azure.json
   423          name: ${CLUSTER_NAME}-mp-win-azure-json
   424      owner: root:root
   425      path: c:/k/azure.json
   426      permissions: "0644"
   427    - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe
   428      path: C:/defender-exclude-calico.ps1
   429      permissions: "0744"
   430    - content: |
   431        $ErrorActionPreference = 'Stop'
   432  
   433        Stop-Service kubelet -Force
   434  
   435        $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}"
   436        if($$KUBE_GIT_VERSION -ne "")
   437        {
   438          $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy")
   439          $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64"
   440          foreach ( $$binary in $$binaries )
   441          {
   442            echo "downloading binary: $$ci_url/$$binary.exe"
   443            curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe"
   444          }
   445        }
   446  
   447        # Tag it to the ci version.  The image knows how to use the copy locally with the configmap
   448        # that is applied at at this stage (windows-kubeproxy-ci.yaml)
   449        ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess
   450        ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess"
   451  
   452        kubeadm.exe version -o=short
   453        kubectl.exe version --client=true
   454        kubelet.exe --version
   455        kube-proxy.exe --version
   456      path: C:/replace-pr-binaries.ps1
   457      permissions: "0744"
   458    - content: |
   459        $ErrorActionPreference = 'Stop'
   460  
   461        echo "Use OOT credential provider"
   462        mkdir C:\var\lib\kubelet\credential-provider
   463        curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe
   464        cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider
   465        curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml
   466      path: C:/oot-cred-provider.ps1
   467      permissions: "0744"
   468    joinConfiguration:
   469      nodeRegistration:
   470        criSocket: npipe:////./pipe/containerd-containerd
   471        kubeletExtraArgs:
   472          cloud-provider: external
   473          image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   474          image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   475          pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9
   476        name: '{{ ds.meta_data["local_hostname"] }}'
   477    postKubeadmCommands:
   478    - nssm set kubelet start SERVICE_AUTO_START
   479    - powershell C:/defender-exclude-calico.ps1
   480    preKubeadmCommands:
   481    - powershell c:/create-external-network.ps1
   482    - powershell C:/replace-pr-binaries.ps1
   483    - powershell C:/oot-cred-provider.ps1
   484    users:
   485    - groups: Administrators
   486      name: capi
   487      sshAuthorizedKeys:
   488      - ${AZURE_SSH_PUBLIC_KEY:=""}
   489  ---
   490  apiVersion: addons.cluster.x-k8s.io/v1beta1
   491  kind: ClusterResourceSet
   492  metadata:
   493    name: ${CLUSTER_NAME}-calico-windows
   494    namespace: default
   495  spec:
   496    clusterSelector:
   497      matchLabels:
   498        cni-windows: ${CLUSTER_NAME}-calico
   499    resources:
   500    - kind: ConfigMap
   501      name: cni-${CLUSTER_NAME}-calico-windows
   502    strategy: ApplyOnce
   503  ---
   504  apiVersion: addons.cluster.x-k8s.io/v1beta1
   505  kind: ClusterResourceSet
   506  metadata:
   507    name: csi-proxy
   508    namespace: default
   509  spec:
   510    clusterSelector:
   511      matchLabels:
   512        csi-proxy: enabled
   513    resources:
   514    - kind: ConfigMap
   515      name: csi-proxy-addon
   516    strategy: ApplyOnce
   517  ---
   518  apiVersion: addons.cluster.x-k8s.io/v1beta1
   519  kind: ClusterResourceSet
   520  metadata:
   521    name: containerd-logger-${CLUSTER_NAME}
   522    namespace: default
   523  spec:
   524    clusterSelector:
   525      matchLabels:
   526        containerd-logger: enabled
   527    resources:
   528    - kind: ConfigMap
   529      name: containerd-logger-${CLUSTER_NAME}
   530    strategy: ApplyOnce
   531  ---
   532  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   533  kind: HelmChartProxy
   534  metadata:
   535    name: calico
   536    namespace: default
   537  spec:
   538    chartName: tigera-operator
   539    clusterSelector:
   540      matchLabels:
   541        cni: calico
   542    namespace: tigera-operator
   543    releaseName: projectcalico
   544    repoURL: https://docs.tigera.io/calico/charts
   545    valuesTemplate: |-
   546      installation:
   547        cni:
   548          type: Calico
   549        calicoNetwork:
   550          bgp: Disabled
   551          mtu: 1350
   552          ipPools:
   553          ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
   554          - cidr: {{ $cidr }}
   555            encapsulation: VXLAN{{end}}
   556        registry: mcr.microsoft.com/oss
   557      # Image and registry configuration for the tigera/operator pod.
   558      tigeraOperator:
   559        image: tigera/operator
   560        registry: mcr.microsoft.com/oss
   561      calicoctl:
   562        image: mcr.microsoft.com/oss/calico/ctl
   563    version: ${CALICO_VERSION}
   564  ---
   565  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   566  kind: HelmChartProxy
   567  metadata:
   568    name: azuredisk-csi-driver-chart
   569    namespace: default
   570  spec:
   571    chartName: azuredisk-csi-driver
   572    clusterSelector:
   573      matchLabels:
   574        azuredisk-csi: "true"
   575    namespace: kube-system
   576    releaseName: azuredisk-csi-driver-oot
   577    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   578    valuesTemplate: |-
   579      controller:
   580        replicas: 1
   581        runOnControlPlane: true
   582      windows:
   583        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   584  ---
   585  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   586  kind: HelmChartProxy
   587  metadata:
   588    name: cloud-provider-azure-chart
   589    namespace: default
   590  spec:
   591    chartName: cloud-provider-azure
   592    clusterSelector:
   593      matchLabels:
   594        cloud-provider: azure
   595    releaseName: cloud-provider-azure-oot
   596    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   597    valuesTemplate: |
   598      infra:
   599        clusterName: {{ .Cluster.metadata.name }}
   600      cloudControllerManager:
   601        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   602        logVerbosity: 4
   603  ---
   604  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   605  kind: HelmChartProxy
   606  metadata:
   607    name: cloud-provider-azure-chart-ci
   608    namespace: default
   609  spec:
   610    chartName: cloud-provider-azure
   611    clusterSelector:
   612      matchLabels:
   613        cloud-provider: azure-ci
   614    releaseName: cloud-provider-azure-oot
   615    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   616    valuesTemplate: |
   617      infra:
   618        clusterName: {{ .Cluster.metadata.name }}
   619      cloudControllerManager:
   620        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   621        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   622        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   623        imageName: "${CCM_IMAGE_NAME:-""}"
   624        imageRepository: "${IMAGE_REGISTRY:-""}"
   625        imageTag: "${IMAGE_TAG_CCM:-""}"
   626        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   627        replicas: ${CCM_COUNT:-1}
   628        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   629      cloudNodeManager:
   630        imageName: "${CNM_IMAGE_NAME:-""}"
   631        imageRepository: "${IMAGE_REGISTRY:-""}"
   632        imageTag: "${IMAGE_TAG_CNM:-""}"
   633  ---
   634  apiVersion: v1
   635  data:
   636    kube-proxy-patch: |-
   637      apiVersion: v1
   638      kind: ConfigMap
   639      metadata:
   640        name: windows-kubeproxy-ci
   641        namespace: kube-system
   642      data:
   643        KUBEPROXY_PATH: "c:/k/kube-proxy.exe"
   644    proxy: |
   645      apiVersion: apps/v1
   646      kind: DaemonSet
   647      metadata:
   648        labels:
   649          k8s-app: kube-proxy
   650        name: kube-proxy-windows
   651        namespace: kube-system
   652      spec:
   653        selector:
   654          matchLabels:
   655            k8s-app: kube-proxy-windows
   656        template:
   657          metadata:
   658            labels:
   659              k8s-app: kube-proxy-windows
   660          spec:
   661            serviceAccountName: kube-proxy
   662            securityContext:
   663              windowsOptions:
   664                hostProcess: true
   665                runAsUserName: "NT AUTHORITY\\system"
   666            hostNetwork: true
   667            containers:
   668            - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess
   669              args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"]
   670              workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/"
   671              name: kube-proxy
   672              env:
   673              - name: NODE_NAME
   674                valueFrom:
   675                  fieldRef:
   676                    apiVersion: v1
   677                    fieldPath: spec.nodeName
   678              - name: POD_IP
   679                valueFrom:
   680                  fieldRef:
   681                    fieldPath: status.podIP
   682              - name: KUBEPROXY_PATH
   683                valueFrom:
   684                  configMapKeyRef:
   685                    name: windows-kubeproxy-ci
   686                    key: KUBEPROXY_PATH
   687                    optional: true
   688              volumeMounts:
   689              - mountPath: /var/lib/kube-proxy
   690                name: kube-proxy
   691            nodeSelector:
   692              kubernetes.io/os: windows
   693            tolerations:
   694            - key: CriticalAddonsOnly
   695              operator: Exists
   696            - operator: Exists
   697            volumes:
   698            - configMap:
   699                name: kube-proxy
   700              name: kube-proxy
   701        updateStrategy:
   702          type: RollingUpdate
   703    windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind:
   704      IPAMConfig\nmetadata:\n  name: default\nspec:\n  autoAllocateBlocks: true\n  strictAffinity:
   705      true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n  name: calico-static-rules\n
   706      \ namespace: calico-system\n  labels:\n    tier: node\n    app: calico\ndata:\n
   707      \ static-rules.json: |\n    {\n      \"Provider\": \"azure\",\n      \"Version\":
   708      \"0.1\",\n      \"Rules\": [\n        {\n          \"Name\": \"EndpointPolicy\",\n
   709      \         \"Rule\": {\n              \"Id\": \"wireserver\",\n              \"Type\":
   710      \"ACL\",\n              \"Protocol\": 6,\n              \"Action\": \"Block\",\n
   711      \             \"Direction\": \"Out\",\n              \"RemoteAddresses\": \"168.63.129.16/32\",\n
   712      \             \"RemotePorts\": \"80\",\n              \"Priority\": 200,\n              \"RuleType\":
   713      \"Switch\"\n            }\n          }\n      ]\n    } \n---\nkind: ConfigMap\napiVersion:
   714      v1\nmetadata:\n  name: calico-config-windows\n  namespace: calico-system\n  labels:\n
   715      \   tier: node\n    app: calico\ndata:\n  veth_mtu: \"1350\"\n  \n  cni_network_config:
   716      |\n    {\n      \"name\": \"Calico\",\n      \"cniVersion\": \"0.3.1\",\n      \"plugins\":
   717      [\n        {\n          \"windows_use_single_network\": true,\n          \"type\":
   718      \"calico\",\n          \"mode\": \"vxlan\",\n          \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n
   719      \         \"nodename_file_optional\": true,\n          \"log_file_path\": \"c:/cni.log\",\n
   720      \         \"log_level\": \"debug\",\n\n          \"vxlan_mac_prefix\": \"0E-2A\",\n
   721      \         \"vxlan_vni\": 4096,\n          \"mtu\": __CNI_MTU__,\n          \"policy\":
   722      {\n            \"type\": \"k8s\"\n          },\n\n          \"log_level\": \"info\",\n\n
   723      \         \"capabilities\": {\"dns\": true},\n          \"DNS\":  {\n            \"Search\":
   724      \ [\n              \"svc.cluster.local\"\n            ]\n          },\n\n          \"datastore_type\":
   725      \"kubernetes\",\n\n          \"kubernetes\": {\n            \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n
   726      \         },\n\n          \"ipam\": {\n            \"type\": \"calico-ipam\",\n
   727      \           \"subnet\": \"usePodCidr\"\n          },\n\n          \"policies\":
   728      \ [\n            {\n              \"Name\":  \"EndpointPolicy\",\n              \"Value\":
   729      \ {\n                \"Type\":  \"OutBoundNAT\",\n                \"ExceptionList\":
   730      \ [\n                  \"__K8S_SERVICE_CIDR__\"\n                ]\n              }\n
   731      \           },\n            {\n              \"Name\":  \"EndpointPolicy\",\n
   732      \             \"Value\":  {\n                \"Type\":  \"SDNROUTE\",\n                \"DestinationPrefix\":
   733      \ \"__K8S_SERVICE_CIDR__\",\n                \"NeedEncap\":  true\n              }\n
   734      \           }\n          ]\n        }\n      ]\n\n    }\n---\napiVersion: apps/v1\nkind:
   735      DaemonSet\nmetadata:\n  name: calico-node-windows\n  labels:\n    tier: node\n
   736      \   app: calico\n  namespace: calico-system\nspec:\n  selector:\n    matchLabels:\n
   737      \     app: calico\n  template:\n    metadata:\n      labels:\n        tier: node\n
   738      \       app: calico\n    spec:\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n
   739      \           nodeSelectorTerms:\n              - matchExpressions:\n                  -
   740      key: kubernetes.io/os\n                    operator: In\n                    values:\n
   741      \                     - windows\n                  - key: kubernetes.io/arch\n
   742      \                   operator: In\n                    values:\n                      -
   743      amd64\n      securityContext:\n        windowsOptions:\n          hostProcess:
   744      true\n          runAsUserName: \"NT AUTHORITY\\\\system\"\n      hostNetwork:
   745      true\n      serviceAccountName: calico-node\n      tolerations:\n      - operator:
   746      Exists\n        effect: NoSchedule\n        # Mark the pod as a critical add-on
   747      for rescheduling.\n      - key: CriticalAddonsOnly\n        operator: Exists\n
   748      \     - effect: NoExecute\n        operator: Exists\n      initContainers:\n        #
   749      This container installs the CNI binaries\n        # and CNI network config file
   750      on each node.\n        - name: install-cni\n          image: sigwindowstools/calico-install:v3.26.1-hostprocess\n
   751      \         args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n
   752      \         imagePullPolicy: Always\n          env:\n            # Name of the CNI
   753      config file to create.\n            - name: CNI_CONF_NAME\n              value:
   754      \"10-calico.conflist\"\n            # The CNI network config to install on each
   755      node.\n            - name: CNI_NETWORK_CONFIG\n              valueFrom:\n                configMapKeyRef:\n
   756      \                 name: calico-config-windows\n                  key: cni_network_config\n
   757      \           # Set the hostname based on the k8s node name.\n            - name:
   758      KUBERNETES_NODE_NAME\n              valueFrom:\n                fieldRef:\n                  fieldPath:
   759      spec.nodeName\n            # CNI MTU Config variable\n            - name: CNI_MTU\n
   760      \             valueFrom:\n                configMapKeyRef:\n                  name:
   761      calico-config-windows\n                  key: veth_mtu\n            # Prevents
   762      the container from sleeping forever.\n            - name: SLEEP\n              value:
   763      \"false\"\n            - name: K8S_SERVICE_CIDR\n              value: \"10.96.0.0/12\"\n
   764      \         volumeMounts:\n            - mountPath: /host/opt/cni/bin\n              name:
   765      cni-bin-dir\n            - mountPath: /host/etc/cni/net.d\n              name:
   766      cni-net-dir\n            - name: kubeadm-config\n              mountPath: /etc/kubeadm-config/\n
   767      \         securityContext:\n            windowsOptions:\n              hostProcess:
   768      true\n              runAsUserName: \"NT AUTHORITY\\\\system\"\n      containers:\n
   769      \     - name: calico-node-startup\n        image: sigwindowstools/calico-node:v3.26.1-hostprocess\n
   770      \       args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n
   771      \       workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        imagePullPolicy:
   772      Always\n        volumeMounts:\n        - name: calico-config-windows\n          mountPath:
   773      /etc/kube-calico-windows/\n        env:\n        - name: POD_NAME\n          valueFrom:\n
   774      \           fieldRef:\n              apiVersion: v1\n              fieldPath:
   775      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   776      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   777      name: CNI_IPAM_TYPE\n          value: \"calico-ipam\"\n        - name: CALICO_NETWORKING_BACKEND\n
   778      \         value: \"vxlan\"\n        - name: KUBECONFIG\n          value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n
   779      \       - name: VXLAN_VNI\n          value: \"4096\"\n      - name: calico-node-felix\n
   780      \       image: sigwindowstools/calico-node:v3.26.1-hostprocess\n        args:
   781      [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n        imagePullPolicy:
   782      Always\n        workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        volumeMounts:\n
   783      \       - name: calico-config-windows\n          mountPath: /etc/kube-calico-windows/\n
   784      \       - name: calico-static-rules\n          mountPath: /calico/static-rules.json\n
   785      \         subPath: static-rules.json\n        env:\n        - name: POD_NAME\n
   786      \         valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath:
   787      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   788      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   789      name: VXLAN_VNI\n          value: \"4096\"\n        - name: KUBECONFIG\n          value:
   790      \"C:/etc/cni/net.d/calico-kubeconfig\"\n      volumes:\n      - name: calico-config-windows\n
   791      \       configMap:\n          name: calico-config-windows\n      - name: calico-static-rules\n
   792      \       configMap:\n          name: calico-static-rules\n      # Used to install
   793      CNI.\n      - name: cni-bin-dir\n        hostPath:\n          path: /opt/cni/bin\n
   794      \     - name: cni-net-dir\n        hostPath:\n          path: /etc/cni/net.d\n
   795      \     - name: kubeadm-config\n        configMap:\n          name: kubeadm-config\n---\napiVersion:
   796      apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: ipamconfigs.crd.projectcalico.org\nspec:\n
   797      \ group: crd.projectcalico.org\n  names:\n    kind: IPAMConfig\n    listKind:
   798      IPAMConfigList\n    plural: ipamconfigs\n    singular: ipamconfig\n  preserveUnknownFields:
   799      false\n  scope: Cluster\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n
   800      \       properties:\n          apiVersion:\n            description: 'APIVersion
   801      defines the versioned schema of this representation\n              of an object.
   802      Servers should convert recognized schemas to the latest\n              internal
   803      value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n
   804      \           type: string\n          kind:\n            description: 'Kind is a
   805      string value representing the REST resource this\n              object represents.
   806      Servers may infer this from the endpoint the client\n              submits requests
   807      to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n
   808      \           type: string\n          metadata:\n            type: object\n          spec:\n
   809      \           description: IPAMConfigSpec contains the specification for an IPAMConfig\n
   810      \             resource.\n            properties:\n              autoAllocateBlocks:\n
   811      \               type: boolean\n              maxBlocksPerHost:\n                description:
   812      MaxBlocksPerHost, if non-zero, is the max number of blocks\n                  that
   813      can be affine to each host.\n                maximum: 2147483647\n                minimum:
   814      0\n                type: integer\n              strictAffinity:\n                type:
   815      boolean\n            required:\n            - autoAllocateBlocks\n            -
   816      strictAffinity\n            type: object\n        type: object\n    served: true\n
   817      \   storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n
   818      \ conditions: []\n  storedVersions: []\n"
   819  kind: ConfigMap
   820  metadata:
   821    annotations:
   822      note: generated
   823    labels:
   824      type: generated
   825    name: cni-${CLUSTER_NAME}-calico-windows
   826    namespace: default
   827  ---
   828  apiVersion: v1
   829  data:
   830    csi-proxy: |
   831      apiVersion: apps/v1
   832      kind: DaemonSet
   833      metadata:
   834        labels:
   835          k8s-app: csi-proxy
   836        name: csi-proxy
   837        namespace: kube-system
   838      spec:
   839        selector:
   840          matchLabels:
   841            k8s-app: csi-proxy
   842        template:
   843          metadata:
   844            labels:
   845              k8s-app: csi-proxy
   846          spec:
   847            nodeSelector:
   848              "kubernetes.io/os": windows
   849            securityContext:
   850              windowsOptions:
   851                hostProcess: true
   852                runAsUserName: "NT AUTHORITY\\SYSTEM"
   853            hostNetwork: true
   854            containers:
   855              - name: csi-proxy
   856                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   857  kind: ConfigMap
   858  metadata:
   859    annotations:
   860      note: generated
   861    labels:
   862      type: generated
   863    name: csi-proxy-addon
   864    namespace: default
   865  ---
   866  apiVersion: v1
   867  data:
   868    containerd-windows-logger: |
   869      apiVersion: apps/v1
   870      kind: DaemonSet
   871      metadata:
   872        labels:
   873          k8s-app: containerd-logger
   874        name: containerd-logger
   875        namespace: kube-system
   876      spec:
   877        selector:
   878          matchLabels:
   879            k8s-app: containerd-logger
   880        template:
   881          metadata:
   882            labels:
   883              k8s-app: containerd-logger
   884          spec:
   885            securityContext:
   886              windowsOptions:
   887                hostProcess: true
   888                runAsUserName: "NT AUTHORITY\\system"
   889            hostNetwork: true
   890            containers:
   891            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
   892              args: [ "config.json" ]
   893              name: containerd-logger
   894              imagePullPolicy: Always
   895              volumeMounts:
   896              - name: containerd-logger-config
   897                mountPath: /config.json
   898                subPath: config.json
   899            nodeSelector:
   900              kubernetes.io/os: windows
   901            tolerations:
   902            - key: CriticalAddonsOnly
   903              operator: Exists
   904            - operator: Exists
   905            volumes:
   906            - configMap:
   907                name: containerd-logger-config
   908              name: containerd-logger-config
   909        updateStrategy:
   910          type: RollingUpdate
   911      ---
   912      kind: ConfigMap
   913      apiVersion: v1
   914      metadata:
   915        name: containerd-logger-config
   916        namespace: kube-system
   917      data:
   918        config.json: |
   919          {
   920            "inputs": [
   921              {
   922                "type": "ETW",
   923                "sessionNamePrefix": "containerd",
   924                "cleanupOldSessions": true,
   925                "reuseExistingSession": true,
   926                "providers": [
   927                  {
   928                    "providerName": "Microsoft.Virtualization.RunHCS",
   929                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
   930                    "level": "Verbose"
   931                  },
   932                  {
   933                    "providerName": "ContainerD",
   934                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
   935                    "level": "Verbose"
   936                  }
   937                ]
   938              }
   939            ],
   940            "filters": [
   941              {
   942                  "type": "drop",
   943                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
   944              },
   945              {
   946                  "type": "drop",
   947                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
   948              },
   949              {
   950                  "type": "drop",
   951                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
   952              },
   953              {
   954                  "type": "drop",
   955                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
   956              },
   957              {
   958                  "type": "drop",
   959                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
   960              },
   961              {
   962                  "type": "drop",
   963                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
   964              },
   965              {
   966                  "type": "drop",
   967                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
   968              }
   969            ],
   970            "outputs": [
   971              {
   972                "type": "StdOutput"
   973              }
   974            ],
   975            "schemaVersion": "2016-08-11"
   976          }
   977  kind: ConfigMap
   978  metadata:
   979    annotations:
   980      note: generated
   981    labels:
   982      type: generated
   983    name: containerd-logger-${CLUSTER_NAME}
   984    namespace: default