sigs.k8s.io/cluster-api-provider-azure@v1.14.3/templates/test/dev/cluster-template-custom-builds.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: enabled
     9      csi-proxy: enabled
    10      metrics-server: enabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 192.168.0.0/16
    18    controlPlaneRef:
    19      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    20      kind: KubeadmControlPlane
    21      name: ${CLUSTER_NAME}-control-plane
    22    infrastructureRef:
    23      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    24      kind: AzureCluster
    25      name: ${CLUSTER_NAME}
    26  ---
    27  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    28  kind: AzureCluster
    29  metadata:
    30    name: ${CLUSTER_NAME}
    31    namespace: default
    32  spec:
    33    additionalTags:
    34      buildProvenance: ${BUILD_PROVENANCE}
    35      creationTimestamp: ${TIMESTAMP}
    36      jobName: ${JOB_NAME}
    37    identityRef:
    38      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    39      kind: AzureClusterIdentity
    40      name: ${CLUSTER_IDENTITY_NAME}
    41    location: ${AZURE_LOCATION}
    42    networkSpec:
    43      subnets:
    44      - name: control-plane-subnet
    45        role: control-plane
    46      - name: node-subnet
    47        role: node
    48      vnet:
    49        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    50    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    51    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    52  ---
    53  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    54  kind: KubeadmControlPlane
    55  metadata:
    56    annotations:
    57      controlplane.cluster.x-k8s.io/skip-kube-proxy: "true"
    58    name: ${CLUSTER_NAME}-control-plane
    59    namespace: default
    60  spec:
    61    kubeadmConfigSpec:
    62      clusterConfiguration:
    63        apiServer:
    64          extraArgs:
    65            cloud-provider: external
    66            feature-gates: ${K8S_FEATURE_GATES:-""}
    67          timeoutForControlPlane: 20m
    68        controllerManager:
    69          extraArgs:
    70            allocate-node-cidrs: "false"
    71            cloud-provider: external
    72            cluster-name: ${CLUSTER_NAME}
    73            feature-gates: HPAContainerMetrics=true
    74            v: "4"
    75        etcd:
    76          local:
    77            dataDir: /var/lib/etcddisk/etcd
    78            extraArgs:
    79              quota-backend-bytes: "8589934592"
    80        kubernetesVersion: ci/${CI_VERSION}
    81      diskSetup:
    82        filesystems:
    83        - device: /dev/disk/azure/scsi1/lun0
    84          extraOpts:
    85          - -E
    86          - lazy_itable_init=1,lazy_journal_init=1
    87          filesystem: ext4
    88          label: etcd_disk
    89        - device: ephemeral0.1
    90          filesystem: ext4
    91          label: ephemeral0
    92          replaceFS: ntfs
    93        partitions:
    94        - device: /dev/disk/azure/scsi1/lun0
    95          layout: true
    96          overwrite: false
    97          tableType: gpt
    98      files:
    99      - contentFrom:
   100          secret:
   101            key: control-plane-azure.json
   102            name: ${CLUSTER_NAME}-control-plane-azure-json
   103        owner: root:root
   104        path: /etc/kubernetes/azure.json
   105        permissions: "0644"
   106      - content: |
   107          #!/bin/bash
   108  
   109          set -o nounset
   110          set -o pipefail
   111          set -o errexit
   112          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   113  
   114          az login --identity
   115          echo "Use OOT credential provider"
   116          mkdir -p /var/lib/kubelet/credential-provider
   117          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   118          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   119          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   120          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   121        owner: root:root
   122        path: /tmp/oot-cred-provider.sh
   123        permissions: "0744"
   124      - content: |
   125          #!/bin/bash
   126  
   127          set -o nounset
   128          set -o pipefail
   129          set -o errexit
   130  
   131          systemctl stop kubelet
   132          declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
   133          az login --identity
   134          for BINARY in "$${BINARIES[@]}"; do
   135            echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}"
   136            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login
   137          done
   138          systemctl restart kubelet
   139  
   140          # prepull images from gcr.io/k8s-staging-ci-images and retag it to
   141          # registry.k8s.io so kubeadm can fetch correct images no matter what
   142          declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   143          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   144          IMAGE_REGISTRY_PREFIX=registry.k8s.io
   145          for IMAGE in "$${IMAGES[@]}"; do
   146            $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}"
   147            $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}"
   148          done
   149  
   150          echo "kubeadm version: $(kubeadm version -o=short)"
   151          echo "kubectl version: $(kubectl version --client=true)"
   152          echo "kubelet version: $(kubelet --version)"
   153        owner: root:root
   154        path: /tmp/replace-k8s-binaries.sh
   155        permissions: "0744"
   156      - content: |
   157          #!/bin/bash
   158  
   159          set -o nounset
   160          set -o pipefail
   161          set -o errexit
   162  
   163          curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz
   164          tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq
   165          rm /tmp/yq_linux_amd64.tar.gz
   166  
   167          export KUBECONFIG=/etc/kubernetes/admin.conf
   168          kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}"
   169          systemctl stop kubelet
   170          yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
   171          yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
   172          yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
   173          systemctl restart kubelet
   174        owner: root:root
   175        path: /tmp/replace-k8s-components.sh
   176        permissions: "0744"
   177      initConfiguration:
   178        nodeRegistration:
   179          kubeletExtraArgs:
   180            cloud-provider: external
   181            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   182            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   183          name: '{{ ds.meta_data["local_hostname"] }}'
   184      joinConfiguration:
   185        nodeRegistration:
   186          kubeletExtraArgs:
   187            cloud-provider: external
   188            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   189            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   190          name: '{{ ds.meta_data["local_hostname"] }}'
   191      mounts:
   192      - - LABEL=etcd_disk
   193        - /var/lib/etcddisk
   194      postKubeadmCommands:
   195      - bash -c /tmp/replace-k8s-components.sh
   196      preKubeadmCommands:
   197      - bash -c /tmp/oot-cred-provider.sh
   198      - bash -c /tmp/replace-k8s-binaries.sh
   199      verbosity: 5
   200    machineTemplate:
   201      infrastructureRef:
   202        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   203        kind: AzureMachineTemplate
   204        name: ${CLUSTER_NAME}-control-plane
   205    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   206    version: ${KUBERNETES_VERSION}
   207  ---
   208  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   209  kind: AzureMachineTemplate
   210  metadata:
   211    name: ${CLUSTER_NAME}-control-plane
   212    namespace: default
   213  spec:
   214    template:
   215      spec:
   216        dataDisks:
   217        - diskSizeGB: 256
   218          lun: 0
   219          nameSuffix: etcddisk
   220        identity: UserAssigned
   221        image:
   222          marketplace:
   223            offer: capi
   224            publisher: cncf-upstream
   225            sku: ubuntu-2204-gen1
   226            version: latest
   227        osDisk:
   228          diskSizeGB: 128
   229          osType: Linux
   230        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   231        userAssignedIdentities:
   232        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
   233        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   234  ---
   235  apiVersion: cluster.x-k8s.io/v1beta1
   236  kind: MachineDeployment
   237  metadata:
   238    name: ${CLUSTER_NAME}-md-0
   239    namespace: default
   240  spec:
   241    clusterName: ${CLUSTER_NAME}
   242    replicas: ${WORKER_MACHINE_COUNT:=2}
   243    selector: {}
   244    template:
   245      metadata:
   246        labels:
   247          nodepool: pool1
   248      spec:
   249        bootstrap:
   250          configRef:
   251            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   252            kind: KubeadmConfigTemplate
   253            name: ${CLUSTER_NAME}-md-0
   254        clusterName: ${CLUSTER_NAME}
   255        infrastructureRef:
   256          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   257          kind: AzureMachineTemplate
   258          name: ${CLUSTER_NAME}-md-0
   259        version: ${KUBERNETES_VERSION}
   260  ---
   261  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   262  kind: AzureMachineTemplate
   263  metadata:
   264    name: ${CLUSTER_NAME}-md-0
   265    namespace: default
   266  spec:
   267    template:
   268      spec:
   269        identity: UserAssigned
   270        image:
   271          marketplace:
   272            offer: capi
   273            publisher: cncf-upstream
   274            sku: ubuntu-2204-gen1
   275            version: latest
   276        osDisk:
   277          diskSizeGB: 128
   278          osType: Linux
   279        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   280        userAssignedIdentities:
   281        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
   282        vmExtensions:
   283        - name: CustomScript
   284          protectedSettings:
   285            commandToExecute: |
   286              #!/bin/sh
   287              echo "This script is a no-op used for extension testing purposes ..."
   288              touch test_file
   289          publisher: Microsoft.Azure.Extensions
   290          version: "2.1"
   291        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   292  ---
   293  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   294  kind: KubeadmConfigTemplate
   295  metadata:
   296    name: ${CLUSTER_NAME}-md-0
   297    namespace: default
   298  spec:
   299    template:
   300      spec:
   301        files:
   302        - contentFrom:
   303            secret:
   304              key: worker-node-azure.json
   305              name: ${CLUSTER_NAME}-md-0-azure-json
   306          owner: root:root
   307          path: /etc/kubernetes/azure.json
   308          permissions: "0644"
   309        - content: |
   310            #!/bin/bash
   311  
   312            set -o nounset
   313            set -o pipefail
   314            set -o errexit
   315            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   316  
   317            az login --identity
   318            echo "Use OOT credential provider"
   319            mkdir -p /var/lib/kubelet/credential-provider
   320            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   321            chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   322            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   323            chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   324          owner: root:root
   325          path: /tmp/oot-cred-provider.sh
   326          permissions: "0744"
   327        - content: |
   328            #!/bin/bash
   329  
   330            set -o nounset
   331            set -o pipefail
   332            set -o errexit
   333  
   334            systemctl stop kubelet
   335            declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
   336            az login --identity
   337            for BINARY in "$${BINARIES[@]}"; do
   338              echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}"
   339              az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login
   340            done
   341            systemctl restart kubelet
   342  
   343            echo "kubeadm version: $(kubeadm version -o=short)"
   344            echo "kubectl version: $(kubectl version --client=true)"
   345            echo "kubelet version: $(kubelet --version)"
   346          owner: root:root
   347          path: /tmp/replace-k8s-binaries.sh
   348          permissions: "0744"
   349        joinConfiguration:
   350          nodeRegistration:
   351            kubeletExtraArgs:
   352              cloud-provider: external
   353              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   354              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   355            name: '{{ ds.meta_data["local_hostname"] }}'
   356        preKubeadmCommands:
   357        - bash -c /tmp/oot-cred-provider.sh
   358        - bash -c /tmp/replace-k8s-binaries.sh
   359  ---
   360  apiVersion: cluster.x-k8s.io/v1beta1
   361  kind: MachineDeployment
   362  metadata:
   363    name: ${CLUSTER_NAME}-md-win
   364    namespace: default
   365  spec:
   366    clusterName: ${CLUSTER_NAME}
   367    replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0}
   368    selector: {}
   369    template:
   370      spec:
   371        bootstrap:
   372          configRef:
   373            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   374            kind: KubeadmConfigTemplate
   375            name: ${CLUSTER_NAME}-md-win
   376        clusterName: ${CLUSTER_NAME}
   377        infrastructureRef:
   378          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   379          kind: AzureMachineTemplate
   380          name: ${CLUSTER_NAME}-md-win
   381        version: ${KUBERNETES_VERSION}
   382  ---
   383  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   384  kind: AzureMachineTemplate
   385  metadata:
   386    annotations:
   387      runtime: containerd
   388    name: ${CLUSTER_NAME}-md-win
   389    namespace: default
   390  spec:
   391    template:
   392      metadata:
   393        annotations:
   394          runtime: containerd
   395          windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""}
   396      spec:
   397        identity: UserAssigned
   398        image:
   399          marketplace:
   400            offer: capi-windows
   401            publisher: cncf-upstream
   402            sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1
   403            version: latest
   404        osDisk:
   405          diskSizeGB: 128
   406          managedDisk:
   407            storageAccountType: Premium_LRS
   408          osType: Windows
   409        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   410        userAssignedIdentities:
   411        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
   412        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   413  ---
   414  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   415  kind: KubeadmConfigTemplate
   416  metadata:
   417    name: ${CLUSTER_NAME}-md-win
   418    namespace: default
   419  spec:
   420    template:
   421      spec:
   422        files:
   423        - contentFrom:
   424            secret:
   425              key: worker-node-azure.json
   426              name: ${CLUSTER_NAME}-md-win-azure-json
   427          owner: root:root
   428          path: c:/k/azure.json
   429          permissions: "0644"
   430        - content: |-
   431            Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe
   432            Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe
   433          path: C:/defender-exclude-calico.ps1
   434          permissions: "0744"
   435        - content: |
   436            # /tmp is assumed created and required for upstream e2e tests to pass
   437            New-Item -ItemType Directory -Force -Path C:\tmp\
   438          path: C:/create-temp-folder.ps1
   439          permissions: "0744"
   440        - content: |
   441            $ErrorActionPreference = 'Stop'
   442  
   443            $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}"
   444            if($$CONTAINERD_URL -ne ""){
   445              # Kubelet service depends on contianerd service so make a best effort attempt to stop it
   446              Stop-Service kubelet -Force -ErrorAction SilentlyContinue
   447              Stop-Service containerd -Force
   448              echo "downloading containerd: $$CONTAINERD_URL"
   449              curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz"
   450              tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1
   451  
   452              Start-Service containerd
   453            }
   454  
   455            containerd.exe --version
   456            containerd-shim-runhcs-v1.exe --version
   457          path: C:/replace-containerd.ps1
   458          permissions: "0744"
   459        - content: |
   460            mkdir -Force c:/localdumps
   461            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f
   462            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f
   463            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f
   464            # Enable sftp so we can copy crash dump files during log collection of stfp
   465            $sshd_config = "$env:ProgramData\ssh\sshd_config"
   466            if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config }
   467            Add-Content -Path $sshd_config "Subsystem sftp              sftp-server.exe"
   468            sc.exe stop sshd
   469            sc.exe start sshd
   470          path: C:/collect-hns-crashes.ps1
   471          permissions: "0744"
   472        - content: |
   473            $ErrorActionPreference = 'Stop'
   474  
   475            az login --identity
   476            echo "Use OOT credential provider"
   477            mkdir C:\var\lib\kubelet\credential-provider
   478            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login
   479            cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider
   480            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login
   481          path: C:/oot-cred-provider.ps1
   482          permissions: "0744"
   483        - content: |
   484            Write-Host "Installing Azure CLI"
   485            $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi
   486            # Need to add manually AZ to PATH as it is not added without a reset
   487            Write-Host "Adding AZ to path:"
   488            $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin"
   489            Write-Host "Version Azure CLI:"
   490            az version
   491          path: C:/install-az-cli.ps1
   492          permissions: "0744"
   493        - content: |
   494            $ErrorActionPreference = 'Stop'
   495  
   496            Stop-Service kubelet -Force
   497  
   498            $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}"
   499            if($$KUBE_GIT_VERSION -ne "")
   500            {
   501              # Need to add manually AZ to PATH as it is not added without a reset
   502              Write-Host "Adding AZ to path:"
   503              $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin"
   504              # Install Binaries
   505              $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy")
   506              $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64"
   507              echo "running az login via identity"
   508              az login --identity
   509              foreach ( $$binary in $$binaries )
   510              {
   511                echo "downloading binary: $$ci_url/$$binary.exe"
   512                az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login
   513              }
   514            }
   515  
   516            # Tag it to the ci version.  The image knows how to use the copy locally with the configmap
   517            # that is applied at at this stage (windows-kubeproxy-ci.yaml)
   518            ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess
   519            ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess"
   520  
   521            kubeadm.exe version -o=short
   522            kubectl.exe version --client=true
   523            kubelet.exe --version
   524            kube-proxy.exe --version
   525          path: C:/replace-pr-binaries.ps1
   526          permissions: "0744"
   527        joinConfiguration:
   528          nodeRegistration:
   529            criSocket: npipe:////./pipe/containerd-containerd
   530            kubeletExtraArgs:
   531              cloud-provider: external
   532              feature-gates: ${NODE_FEATURE_GATES:-""}
   533              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   534              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   535              v: "2"
   536              windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS
   537            name: '{{ ds.meta_data["local_hostname"] }}'
   538        postKubeadmCommands:
   539        - nssm set kubelet start SERVICE_AUTO_START
   540        - powershell C:/defender-exclude-calico.ps1
   541        preKubeadmCommands:
   542        - powershell C:/create-temp-folder.ps1
   543        - powershell C:/replace-containerd.ps1
   544        - powershell C:/collect-hns-crashes.ps1
   545        - powershell C:/oot-cred-provider.ps1
   546        - powershell C:/install-az-cli.ps1
   547        - powershell C:/replace-pr-binaries.ps1
   548        users:
   549        - groups: Administrators
   550          name: capi
   551          sshAuthorizedKeys:
   552          - ${AZURE_SSH_PUBLIC_KEY:=""}
   553  ---
   554  apiVersion: cluster.x-k8s.io/v1beta1
   555  kind: MachineHealthCheck
   556  metadata:
   557    name: ${CLUSTER_NAME}-mhc-0
   558    namespace: default
   559  spec:
   560    clusterName: ${CLUSTER_NAME}
   561    maxUnhealthy: 100%
   562    selector:
   563      matchLabels:
   564        nodepool: pool1
   565    unhealthyConditions:
   566    - status: "True"
   567      timeout: 30s
   568      type: E2ENodeUnhealthy
   569  ---
   570  apiVersion: addons.cluster.x-k8s.io/v1beta1
   571  kind: ClusterResourceSet
   572  metadata:
   573    name: ${CLUSTER_NAME}-calico-windows
   574    namespace: default
   575  spec:
   576    clusterSelector:
   577      matchLabels:
   578        cni-windows: ${CLUSTER_NAME}-calico
   579    resources:
   580    - kind: ConfigMap
   581      name: cni-${CLUSTER_NAME}-calico-windows
   582    strategy: ApplyOnce
   583  ---
   584  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   585  kind: AzureClusterIdentity
   586  metadata:
   587    labels:
   588      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   589    name: ${CLUSTER_IDENTITY_NAME}
   590    namespace: default
   591  spec:
   592    allowedNamespaces: {}
   593    clientID: ${AZURE_CLIENT_ID}
   594    clientSecret:
   595      name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME}
   596      namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
   597    tenantID: ${AZURE_TENANT_ID}
   598    type: ServicePrincipal
   599  ---
   600  apiVersion: addons.cluster.x-k8s.io/v1beta1
   601  kind: ClusterResourceSet
   602  metadata:
   603    name: csi-proxy
   604    namespace: default
   605  spec:
   606    clusterSelector:
   607      matchLabels:
   608        csi-proxy: enabled
   609    resources:
   610    - kind: ConfigMap
   611      name: csi-proxy-addon
   612    strategy: ApplyOnce
   613  ---
   614  apiVersion: addons.cluster.x-k8s.io/v1beta1
   615  kind: ClusterResourceSet
   616  metadata:
   617    name: containerd-logger-${CLUSTER_NAME}
   618    namespace: default
   619  spec:
   620    clusterSelector:
   621      matchLabels:
   622        containerd-logger: enabled
   623    resources:
   624    - kind: ConfigMap
   625      name: containerd-logger-${CLUSTER_NAME}
   626    strategy: ApplyOnce
   627  ---
   628  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   629  kind: HelmChartProxy
   630  metadata:
   631    name: calico
   632    namespace: default
   633  spec:
   634    chartName: tigera-operator
   635    clusterSelector:
   636      matchLabels:
   637        cni: calico
   638    namespace: tigera-operator
   639    releaseName: projectcalico
   640    repoURL: https://docs.tigera.io/calico/charts
   641    valuesTemplate: |-
   642      installation:
   643        cni:
   644          type: Calico
   645        calicoNetwork:
   646          bgp: Disabled
   647          mtu: 1350
   648          ipPools:
   649          ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
   650          - cidr: {{ $cidr }}
   651            encapsulation: VXLAN{{end}}
   652        registry: mcr.microsoft.com/oss
   653      # Image and registry configuration for the tigera/operator pod.
   654      tigeraOperator:
   655        image: tigera/operator
   656        registry: mcr.microsoft.com/oss
   657      calicoctl:
   658        image: mcr.microsoft.com/oss/calico/ctl
   659    version: ${CALICO_VERSION}
   660  ---
   661  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   662  kind: HelmChartProxy
   663  metadata:
   664    name: azuredisk-csi-driver-chart
   665    namespace: default
   666  spec:
   667    chartName: azuredisk-csi-driver
   668    clusterSelector:
   669      matchLabels:
   670        azuredisk-csi: "true"
   671    namespace: kube-system
   672    releaseName: azuredisk-csi-driver-oot
   673    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   674    valuesTemplate: |-
   675      controller:
   676        replicas: 1
   677        runOnControlPlane: true
   678      windows:
   679        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   680  ---
   681  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   682  kind: HelmChartProxy
   683  metadata:
   684    name: cloud-provider-azure-chart
   685    namespace: default
   686  spec:
   687    chartName: cloud-provider-azure
   688    clusterSelector:
   689      matchLabels:
   690        cloud-provider: azure
   691    releaseName: cloud-provider-azure-oot
   692    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   693    valuesTemplate: |
   694      infra:
   695        clusterName: {{ .Cluster.metadata.name }}
   696      cloudControllerManager:
   697        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   698        logVerbosity: 4
   699  ---
   700  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   701  kind: HelmChartProxy
   702  metadata:
   703    name: cloud-provider-azure-chart-ci
   704    namespace: default
   705  spec:
   706    chartName: cloud-provider-azure
   707    clusterSelector:
   708      matchLabels:
   709        cloud-provider: azure-ci
   710    releaseName: cloud-provider-azure-oot
   711    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   712    valuesTemplate: |
   713      infra:
   714        clusterName: {{ .Cluster.metadata.name }}
   715      cloudControllerManager:
   716        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   717        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   718        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   719        imageName: "${CCM_IMAGE_NAME:-""}"
   720        imageRepository: "${IMAGE_REGISTRY:-""}"
   721        imageTag: "${IMAGE_TAG_CCM:-""}"
   722        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   723        replicas: ${CCM_COUNT:-1}
   724        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   725      cloudNodeManager:
   726        imageName: "${CNM_IMAGE_NAME:-""}"
   727        imageRepository: "${IMAGE_REGISTRY:-""}"
   728        imageTag: "${IMAGE_TAG_CNM:-""}"
   729  ---
   730  apiVersion: v1
   731  data:
   732    kube-proxy-patch: |-
   733      apiVersion: v1
   734      kind: ConfigMap
   735      metadata:
   736        name: windows-kubeproxy-ci
   737        namespace: kube-system
   738      data:
   739        KUBEPROXY_PATH: "c:/k/kube-proxy.exe"
   740    proxy: |
   741      apiVersion: apps/v1
   742      kind: DaemonSet
   743      metadata:
   744        labels:
   745          k8s-app: kube-proxy
   746        name: kube-proxy-windows
   747        namespace: kube-system
   748      spec:
   749        selector:
   750          matchLabels:
   751            k8s-app: kube-proxy-windows
   752        template:
   753          metadata:
   754            labels:
   755              k8s-app: kube-proxy-windows
   756          spec:
   757            serviceAccountName: kube-proxy
   758            securityContext:
   759              windowsOptions:
   760                hostProcess: true
   761                runAsUserName: "NT AUTHORITY\\system"
   762            hostNetwork: true
   763            containers:
   764            - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess
   765              args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"]
   766              workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/"
   767              name: kube-proxy
   768              env:
   769              - name: NODE_NAME
   770                valueFrom:
   771                  fieldRef:
   772                    apiVersion: v1
   773                    fieldPath: spec.nodeName
   774              - name: POD_IP
   775                valueFrom:
   776                  fieldRef:
   777                    fieldPath: status.podIP
   778              - name: KUBEPROXY_PATH
   779                valueFrom:
   780                  configMapKeyRef:
   781                    name: windows-kubeproxy-ci
   782                    key: KUBEPROXY_PATH
   783                    optional: true
   784              volumeMounts:
   785              - mountPath: /var/lib/kube-proxy
   786                name: kube-proxy
   787            nodeSelector:
   788              kubernetes.io/os: windows
   789            tolerations:
   790            - key: CriticalAddonsOnly
   791              operator: Exists
   792            - operator: Exists
   793            volumes:
   794            - configMap:
   795                name: kube-proxy
   796              name: kube-proxy
   797        updateStrategy:
   798          type: RollingUpdate
   799    windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind:
   800      IPAMConfig\nmetadata:\n  name: default\nspec:\n  autoAllocateBlocks: true\n  strictAffinity:
   801      true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n  name: calico-static-rules\n
   802      \ namespace: calico-system\n  labels:\n    tier: node\n    app: calico\ndata:\n
   803      \ static-rules.json: |\n    {\n      \"Provider\": \"azure\",\n      \"Version\":
   804      \"0.1\",\n      \"Rules\": [\n        {\n          \"Name\": \"EndpointPolicy\",\n
   805      \         \"Rule\": {\n              \"Id\": \"wireserver\",\n              \"Type\":
   806      \"ACL\",\n              \"Protocol\": 6,\n              \"Action\": \"Block\",\n
   807      \             \"Direction\": \"Out\",\n              \"RemoteAddresses\": \"168.63.129.16/32\",\n
   808      \             \"RemotePorts\": \"80\",\n              \"Priority\": 200,\n              \"RuleType\":
   809      \"Switch\"\n            }\n          }\n      ]\n    } \n---\nkind: ConfigMap\napiVersion:
   810      v1\nmetadata:\n  name: calico-config-windows\n  namespace: calico-system\n  labels:\n
   811      \   tier: node\n    app: calico\ndata:\n  veth_mtu: \"1350\"\n  \n  cni_network_config:
   812      |\n    {\n      \"name\": \"Calico\",\n      \"cniVersion\": \"0.3.1\",\n      \"plugins\":
   813      [\n        {\n          \"windows_use_single_network\": true,\n          \"type\":
   814      \"calico\",\n          \"mode\": \"vxlan\",\n          \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n
   815      \         \"nodename_file_optional\": true,\n          \"log_file_path\": \"c:/cni.log\",\n
   816      \         \"log_level\": \"debug\",\n\n          \"vxlan_mac_prefix\": \"0E-2A\",\n
   817      \         \"vxlan_vni\": 4096,\n          \"mtu\": __CNI_MTU__,\n          \"policy\":
   818      {\n            \"type\": \"k8s\"\n          },\n\n          \"log_level\": \"info\",\n\n
   819      \         \"capabilities\": {\"dns\": true},\n          \"DNS\":  {\n            \"Search\":
   820      \ [\n              \"svc.cluster.local\"\n            ]\n          },\n\n          \"datastore_type\":
   821      \"kubernetes\",\n\n          \"kubernetes\": {\n            \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n
   822      \         },\n\n          \"ipam\": {\n            \"type\": \"calico-ipam\",\n
   823      \           \"subnet\": \"usePodCidr\"\n          },\n\n          \"policies\":
   824      \ [\n            {\n              \"Name\":  \"EndpointPolicy\",\n              \"Value\":
   825      \ {\n                \"Type\":  \"OutBoundNAT\",\n                \"ExceptionList\":
   826      \ [\n                  \"__K8S_SERVICE_CIDR__\"\n                ]\n              }\n
   827      \           },\n            {\n              \"Name\":  \"EndpointPolicy\",\n
   828      \             \"Value\":  {\n                \"Type\":  \"SDNROUTE\",\n                \"DestinationPrefix\":
   829      \ \"__K8S_SERVICE_CIDR__\",\n                \"NeedEncap\":  true\n              }\n
   830      \           }\n          ]\n        }\n      ]\n\n    }\n---\napiVersion: apps/v1\nkind:
   831      DaemonSet\nmetadata:\n  name: calico-node-windows\n  labels:\n    tier: node\n
   832      \   app: calico\n  namespace: calico-system\nspec:\n  selector:\n    matchLabels:\n
   833      \     app: calico\n  template:\n    metadata:\n      labels:\n        tier: node\n
   834      \       app: calico\n    spec:\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n
   835      \           nodeSelectorTerms:\n              - matchExpressions:\n                  -
   836      key: kubernetes.io/os\n                    operator: In\n                    values:\n
   837      \                     - windows\n                  - key: kubernetes.io/arch\n
   838      \                   operator: In\n                    values:\n                      -
   839      amd64\n      securityContext:\n        windowsOptions:\n          hostProcess:
   840      true\n          runAsUserName: \"NT AUTHORITY\\\\system\"\n      hostNetwork:
   841      true\n      serviceAccountName: calico-node\n      tolerations:\n      - operator:
   842      Exists\n        effect: NoSchedule\n        # Mark the pod as a critical add-on
   843      for rescheduling.\n      - key: CriticalAddonsOnly\n        operator: Exists\n
   844      \     - effect: NoExecute\n        operator: Exists\n      initContainers:\n        #
   845      This container installs the CNI binaries\n        # and CNI network config file
   846      on each node.\n        - name: install-cni\n          image: sigwindowstools/calico-install:v3.26.1-hostprocess\n
   847      \         args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n
   848      \         imagePullPolicy: Always\n          env:\n            # Name of the CNI
   849      config file to create.\n            - name: CNI_CONF_NAME\n              value:
   850      \"10-calico.conflist\"\n            # The CNI network config to install on each
   851      node.\n            - name: CNI_NETWORK_CONFIG\n              valueFrom:\n                configMapKeyRef:\n
   852      \                 name: calico-config-windows\n                  key: cni_network_config\n
   853      \           # Set the hostname based on the k8s node name.\n            - name:
   854      KUBERNETES_NODE_NAME\n              valueFrom:\n                fieldRef:\n                  fieldPath:
   855      spec.nodeName\n            # CNI MTU Config variable\n            - name: CNI_MTU\n
   856      \             valueFrom:\n                configMapKeyRef:\n                  name:
   857      calico-config-windows\n                  key: veth_mtu\n            # Prevents
   858      the container from sleeping forever.\n            - name: SLEEP\n              value:
   859      \"false\"\n            - name: K8S_SERVICE_CIDR\n              value: \"10.96.0.0/12\"\n
   860      \         volumeMounts:\n            - mountPath: /host/opt/cni/bin\n              name:
   861      cni-bin-dir\n            - mountPath: /host/etc/cni/net.d\n              name:
   862      cni-net-dir\n            - name: kubeadm-config\n              mountPath: /etc/kubeadm-config/\n
   863      \         securityContext:\n            windowsOptions:\n              hostProcess:
   864      true\n              runAsUserName: \"NT AUTHORITY\\\\system\"\n      containers:\n
   865      \     - name: calico-node-startup\n        image: sigwindowstools/calico-node:v3.26.1-hostprocess\n
   866      \       args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n
   867      \       workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        imagePullPolicy:
   868      Always\n        volumeMounts:\n        - name: calico-config-windows\n          mountPath:
   869      /etc/kube-calico-windows/\n        env:\n        - name: POD_NAME\n          valueFrom:\n
   870      \           fieldRef:\n              apiVersion: v1\n              fieldPath:
   871      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   872      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   873      name: CNI_IPAM_TYPE\n          value: \"calico-ipam\"\n        - name: CALICO_NETWORKING_BACKEND\n
   874      \         value: \"vxlan\"\n        - name: KUBECONFIG\n          value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n
   875      \       - name: VXLAN_VNI\n          value: \"4096\"\n      - name: calico-node-felix\n
   876      \       image: sigwindowstools/calico-node:v3.26.1-hostprocess\n        args:
   877      [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n        imagePullPolicy:
   878      Always\n        workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        volumeMounts:\n
   879      \       - name: calico-config-windows\n          mountPath: /etc/kube-calico-windows/\n
   880      \       - name: calico-static-rules\n          mountPath: /calico/static-rules.json\n
   881      \         subPath: static-rules.json\n        env:\n        - name: POD_NAME\n
   882      \         valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath:
   883      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   884      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   885      name: VXLAN_VNI\n          value: \"4096\"\n        - name: KUBECONFIG\n          value:
   886      \"C:/etc/cni/net.d/calico-kubeconfig\"\n      volumes:\n      - name: calico-config-windows\n
   887      \       configMap:\n          name: calico-config-windows\n      - name: calico-static-rules\n
   888      \       configMap:\n          name: calico-static-rules\n      # Used to install
   889      CNI.\n      - name: cni-bin-dir\n        hostPath:\n          path: /opt/cni/bin\n
   890      \     - name: cni-net-dir\n        hostPath:\n          path: /etc/cni/net.d\n
   891      \     - name: kubeadm-config\n        configMap:\n          name: kubeadm-config\n---\napiVersion:
   892      apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: ipamconfigs.crd.projectcalico.org\nspec:\n
   893      \ group: crd.projectcalico.org\n  names:\n    kind: IPAMConfig\n    listKind:
   894      IPAMConfigList\n    plural: ipamconfigs\n    singular: ipamconfig\n  preserveUnknownFields:
   895      false\n  scope: Cluster\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n
   896      \       properties:\n          apiVersion:\n            description: 'APIVersion
   897      defines the versioned schema of this representation\n              of an object.
   898      Servers should convert recognized schemas to the latest\n              internal
   899      value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n
   900      \           type: string\n          kind:\n            description: 'Kind is a
   901      string value representing the REST resource this\n              object represents.
   902      Servers may infer this from the endpoint the client\n              submits requests
   903      to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n
   904      \           type: string\n          metadata:\n            type: object\n          spec:\n
   905      \           description: IPAMConfigSpec contains the specification for an IPAMConfig\n
   906      \             resource.\n            properties:\n              autoAllocateBlocks:\n
   907      \               type: boolean\n              maxBlocksPerHost:\n                description:
   908      MaxBlocksPerHost, if non-zero, is the max number of blocks\n                  that
   909      can be affine to each host.\n                maximum: 2147483647\n                minimum:
   910      0\n                type: integer\n              strictAffinity:\n                type:
   911      boolean\n            required:\n            - autoAllocateBlocks\n            -
   912      strictAffinity\n            type: object\n        type: object\n    served: true\n
   913      \   storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n
   914      \ conditions: []\n  storedVersions: []\n"
   915  kind: ConfigMap
   916  metadata:
   917    annotations:
   918      note: generated
   919    labels:
   920      type: generated
   921    name: cni-${CLUSTER_NAME}-calico-windows
   922    namespace: default
   923  ---
   924  apiVersion: v1
   925  data:
   926    csi-proxy: |
   927      apiVersion: apps/v1
   928      kind: DaemonSet
   929      metadata:
   930        labels:
   931          k8s-app: csi-proxy
   932        name: csi-proxy
   933        namespace: kube-system
   934      spec:
   935        selector:
   936          matchLabels:
   937            k8s-app: csi-proxy
   938        template:
   939          metadata:
   940            labels:
   941              k8s-app: csi-proxy
   942          spec:
   943            nodeSelector:
   944              "kubernetes.io/os": windows
   945            securityContext:
   946              windowsOptions:
   947                hostProcess: true
   948                runAsUserName: "NT AUTHORITY\\SYSTEM"
   949            hostNetwork: true
   950            containers:
   951              - name: csi-proxy
   952                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   953  kind: ConfigMap
   954  metadata:
   955    annotations:
   956      note: generated
   957    labels:
   958      type: generated
   959    name: csi-proxy-addon
   960    namespace: default
   961  ---
   962  apiVersion: v1
   963  data:
   964    containerd-windows-logger: |
   965      apiVersion: apps/v1
   966      kind: DaemonSet
   967      metadata:
   968        labels:
   969          k8s-app: containerd-logger
   970        name: containerd-logger
   971        namespace: kube-system
   972      spec:
   973        selector:
   974          matchLabels:
   975            k8s-app: containerd-logger
   976        template:
   977          metadata:
   978            labels:
   979              k8s-app: containerd-logger
   980          spec:
   981            securityContext:
   982              windowsOptions:
   983                hostProcess: true
   984                runAsUserName: "NT AUTHORITY\\system"
   985            hostNetwork: true
   986            containers:
   987            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
   988              args: [ "config.json" ]
   989              name: containerd-logger
   990              imagePullPolicy: Always
   991              volumeMounts:
   992              - name: containerd-logger-config
   993                mountPath: /config.json
   994                subPath: config.json
   995            nodeSelector:
   996              kubernetes.io/os: windows
   997            tolerations:
   998            - key: CriticalAddonsOnly
   999              operator: Exists
  1000            - operator: Exists
  1001            volumes:
  1002            - configMap:
  1003                name: containerd-logger-config
  1004              name: containerd-logger-config
  1005        updateStrategy:
  1006          type: RollingUpdate
  1007      ---
  1008      kind: ConfigMap
  1009      apiVersion: v1
  1010      metadata:
  1011        name: containerd-logger-config
  1012        namespace: kube-system
  1013      data:
  1014        config.json: |
  1015          {
  1016            "inputs": [
  1017              {
  1018                "type": "ETW",
  1019                "sessionNamePrefix": "containerd",
  1020                "cleanupOldSessions": true,
  1021                "reuseExistingSession": true,
  1022                "providers": [
  1023                  {
  1024                    "providerName": "Microsoft.Virtualization.RunHCS",
  1025                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
  1026                    "level": "Verbose"
  1027                  },
  1028                  {
  1029                    "providerName": "ContainerD",
  1030                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
  1031                    "level": "Verbose"
  1032                  }
  1033                ]
  1034              }
  1035            ],
  1036            "filters": [
  1037              {
  1038                  "type": "drop",
  1039                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
  1040              },
  1041              {
  1042                  "type": "drop",
  1043                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
  1044              },
  1045              {
  1046                  "type": "drop",
  1047                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
  1048              },
  1049              {
  1050                  "type": "drop",
  1051                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
  1052              },
  1053              {
  1054                  "type": "drop",
  1055                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
  1056              },
  1057              {
  1058                  "type": "drop",
  1059                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
  1060              },
  1061              {
  1062                  "type": "drop",
  1063                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
  1064              }
  1065            ],
  1066            "outputs": [
  1067              {
  1068                "type": "StdOutput"
  1069              }
  1070            ],
  1071            "schemaVersion": "2016-08-11"
  1072          }
  1073  kind: ConfigMap
  1074  metadata:
  1075    annotations:
  1076      note: generated
  1077    labels:
  1078      type: generated
  1079    name: containerd-logger-${CLUSTER_NAME}
  1080    namespace: default
  1081  ---
  1082  apiVersion: addons.cluster.x-k8s.io/v1beta1
  1083  kind: ClusterResourceSet
  1084  metadata:
  1085    name: metrics-server-${CLUSTER_NAME}
  1086    namespace: default
  1087  spec:
  1088    clusterSelector:
  1089      matchLabels:
  1090        metrics-server: enabled
  1091    resources:
  1092    - kind: ConfigMap
  1093      name: metrics-server-${CLUSTER_NAME}
  1094    strategy: ApplyOnce
  1095  ---
  1096  apiVersion: v1
  1097  data:
  1098    metrics-server: |
  1099      apiVersion: v1
  1100      kind: ServiceAccount
  1101      metadata:
  1102        labels:
  1103          k8s-app: metrics-server
  1104        name: metrics-server
  1105        namespace: kube-system
  1106      ---
  1107      apiVersion: rbac.authorization.k8s.io/v1
  1108      kind: ClusterRole
  1109      metadata:
  1110        labels:
  1111          k8s-app: metrics-server
  1112          rbac.authorization.k8s.io/aggregate-to-admin: "true"
  1113          rbac.authorization.k8s.io/aggregate-to-edit: "true"
  1114          rbac.authorization.k8s.io/aggregate-to-view: "true"
  1115        name: system:aggregated-metrics-reader
  1116      rules:
  1117      - apiGroups:
  1118        - metrics.k8s.io
  1119        resources:
  1120        - pods
  1121        - nodes
  1122        verbs:
  1123        - get
  1124        - list
  1125        - watch
  1126      ---
  1127      apiVersion: rbac.authorization.k8s.io/v1
  1128      kind: ClusterRole
  1129      metadata:
  1130        labels:
  1131          k8s-app: metrics-server
  1132        name: system:metrics-server
  1133      rules:
  1134      - apiGroups:
  1135        - ""
  1136        resources:
  1137        - nodes/metrics
  1138        verbs:
  1139        - get
  1140      - apiGroups:
  1141        - ""
  1142        resources:
  1143        - pods
  1144        - nodes
  1145        verbs:
  1146        - get
  1147        - list
  1148        - watch
  1149      ---
  1150      apiVersion: rbac.authorization.k8s.io/v1
  1151      kind: RoleBinding
  1152      metadata:
  1153        labels:
  1154          k8s-app: metrics-server
  1155        name: metrics-server-auth-reader
  1156        namespace: kube-system
  1157      roleRef:
  1158        apiGroup: rbac.authorization.k8s.io
  1159        kind: Role
  1160        name: extension-apiserver-authentication-reader
  1161      subjects:
  1162      - kind: ServiceAccount
  1163        name: metrics-server
  1164        namespace: kube-system
  1165      ---
  1166      apiVersion: rbac.authorization.k8s.io/v1
  1167      kind: ClusterRoleBinding
  1168      metadata:
  1169        labels:
  1170          k8s-app: metrics-server
  1171        name: metrics-server:system:auth-delegator
  1172      roleRef:
  1173        apiGroup: rbac.authorization.k8s.io
  1174        kind: ClusterRole
  1175        name: system:auth-delegator
  1176      subjects:
  1177      - kind: ServiceAccount
  1178        name: metrics-server
  1179        namespace: kube-system
  1180      ---
  1181      apiVersion: rbac.authorization.k8s.io/v1
  1182      kind: ClusterRoleBinding
  1183      metadata:
  1184        labels:
  1185          k8s-app: metrics-server
  1186        name: system:metrics-server
  1187      roleRef:
  1188        apiGroup: rbac.authorization.k8s.io
  1189        kind: ClusterRole
  1190        name: system:metrics-server
  1191      subjects:
  1192      - kind: ServiceAccount
  1193        name: metrics-server
  1194        namespace: kube-system
  1195      ---
  1196      apiVersion: v1
  1197      kind: Service
  1198      metadata:
  1199        labels:
  1200          k8s-app: metrics-server
  1201        name: metrics-server
  1202        namespace: kube-system
  1203      spec:
  1204        ports:
  1205        - name: https
  1206          port: 443
  1207          protocol: TCP
  1208          targetPort: https
  1209        selector:
  1210          k8s-app: metrics-server
  1211      ---
  1212      apiVersion: apps/v1
  1213      kind: Deployment
  1214      metadata:
  1215        labels:
  1216          k8s-app: metrics-server
  1217        name: metrics-server
  1218        namespace: kube-system
  1219      spec:
  1220        selector:
  1221          matchLabels:
  1222            k8s-app: metrics-server
  1223        strategy:
  1224          rollingUpdate:
  1225            maxUnavailable: 0
  1226        template:
  1227          metadata:
  1228            labels:
  1229              k8s-app: metrics-server
  1230          spec:
  1231            containers:
  1232            - args:
  1233              - --cert-dir=/tmp
  1234              - --secure-port=4443
  1235              - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
  1236              - --kubelet-use-node-status-port
  1237              - --metric-resolution=15s
  1238              - --kubelet-insecure-tls
  1239              image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
  1240              imagePullPolicy: IfNotPresent
  1241              livenessProbe:
  1242                failureThreshold: 3
  1243                httpGet:
  1244                  path: /livez
  1245                  port: https
  1246                  scheme: HTTPS
  1247                periodSeconds: 10
  1248              name: metrics-server
  1249              ports:
  1250              - containerPort: 4443
  1251                name: https
  1252                protocol: TCP
  1253              readinessProbe:
  1254                failureThreshold: 3
  1255                httpGet:
  1256                  path: /readyz
  1257                  port: https
  1258                  scheme: HTTPS
  1259                initialDelaySeconds: 20
  1260                periodSeconds: 10
  1261              resources:
  1262                requests:
  1263                  cpu: 100m
  1264                  memory: 200Mi
  1265              securityContext:
  1266                allowPrivilegeEscalation: false
  1267                readOnlyRootFilesystem: true
  1268                runAsNonRoot: true
  1269                runAsUser: 1000
  1270              volumeMounts:
  1271              - mountPath: /tmp
  1272                name: tmp-dir
  1273            nodeSelector:
  1274              kubernetes.io/os: linux
  1275            priorityClassName: system-cluster-critical
  1276            serviceAccountName: metrics-server
  1277            tolerations:
  1278            - effect: NoSchedule
  1279              key: node-role.kubernetes.io/master
  1280              operator: Exists
  1281            - effect: NoSchedule
  1282              key: node-role.kubernetes.io/control-plane
  1283              operator: Exists
  1284            volumes:
  1285            - emptyDir: {}
  1286              name: tmp-dir
  1287      ---
  1288      apiVersion: apiregistration.k8s.io/v1
  1289      kind: APIService
  1290      metadata:
  1291        labels:
  1292          k8s-app: metrics-server
  1293        name: v1beta1.metrics.k8s.io
  1294      spec:
  1295        group: metrics.k8s.io
  1296        groupPriorityMinimum: 100
  1297        insecureSkipTLSVerify: true
  1298        service:
  1299          name: metrics-server
  1300          namespace: kube-system
  1301        version: v1beta1
  1302        versionPriority: 100
  1303  kind: ConfigMap
  1304  metadata:
  1305    annotations:
  1306      note: generated
  1307    labels:
  1308      type: generated
  1309    name: metrics-server-${CLUSTER_NAME}
  1310    namespace: default