sigs.k8s.io/cluster-api-provider-azure@v1.17.0/templates/test/dev/cluster-template-custom-builds.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: enabled
     9      csi-proxy: enabled
    10      metrics-server: enabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 192.168.0.0/16
    18    controlPlaneRef:
    19      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    20      kind: KubeadmControlPlane
    21      name: ${CLUSTER_NAME}-control-plane
    22    infrastructureRef:
    23      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    24      kind: AzureCluster
    25      name: ${CLUSTER_NAME}
    26  ---
    27  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    28  kind: AzureCluster
    29  metadata:
    30    name: ${CLUSTER_NAME}
    31    namespace: default
    32  spec:
    33    additionalTags:
    34      buildProvenance: ${BUILD_PROVENANCE}
    35      creationTimestamp: ${TIMESTAMP}
    36      jobName: ${JOB_NAME}
    37    identityRef:
    38      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    39      kind: AzureClusterIdentity
    40      name: ${CLUSTER_IDENTITY_NAME}
    41    location: ${AZURE_LOCATION}
    42    networkSpec:
    43      subnets:
    44      - name: control-plane-subnet
    45        role: control-plane
    46      - name: node-subnet
    47        role: node
    48      vnet:
    49        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    50    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    51    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    52  ---
    53  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    54  kind: KubeadmControlPlane
    55  metadata:
    56    annotations:
    57      controlplane.cluster.x-k8s.io/skip-kube-proxy: "true"
    58    name: ${CLUSTER_NAME}-control-plane
    59    namespace: default
    60  spec:
    61    kubeadmConfigSpec:
    62      clusterConfiguration:
    63        apiServer:
    64          extraArgs:
    65            cloud-provider: external
    66            feature-gates: ${K8S_FEATURE_GATES:-""}
    67          timeoutForControlPlane: 20m
    68        controllerManager:
    69          extraArgs:
    70            allocate-node-cidrs: "false"
    71            cloud-provider: external
    72            cluster-name: ${CLUSTER_NAME}
    73            feature-gates: HPAContainerMetrics=true
    74            v: "4"
    75        etcd:
    76          local:
    77            dataDir: /var/lib/etcddisk/etcd
    78            extraArgs:
    79              quota-backend-bytes: "8589934592"
    80        kubernetesVersion: ci/${CI_VERSION}
    81      diskSetup:
    82        filesystems:
    83        - device: /dev/disk/azure/scsi1/lun0
    84          extraOpts:
    85          - -E
    86          - lazy_itable_init=1,lazy_journal_init=1
    87          filesystem: ext4
    88          label: etcd_disk
    89        - device: ephemeral0.1
    90          filesystem: ext4
    91          label: ephemeral0
    92          replaceFS: ntfs
    93        partitions:
    94        - device: /dev/disk/azure/scsi1/lun0
    95          layout: true
    96          overwrite: false
    97          tableType: gpt
    98      files:
    99      - contentFrom:
   100          secret:
   101            key: control-plane-azure.json
   102            name: ${CLUSTER_NAME}-control-plane-azure-json
   103        owner: root:root
   104        path: /etc/kubernetes/azure.json
   105        permissions: "0644"
   106      - content: |
   107          #!/bin/bash
   108  
   109          set -o nounset
   110          set -o pipefail
   111          set -o errexit
   112          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   113  
   114          echo "Use OOT credential provider"
   115          mkdir -p /var/lib/kubelet/credential-provider
   116          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   117          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   118          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   119          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   120        owner: root:root
   121        path: /tmp/oot-cred-provider.sh
   122        permissions: "0744"
   123      - content: |
   124          #!/bin/bash
   125  
   126          set -o nounset
   127          set -o pipefail
   128          set -o errexit
   129  
   130          systemctl stop kubelet
   131          declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
   132          az login --identity
   133          for BINARY in "$${BINARIES[@]}"; do
   134            echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}"
   135            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login
   136          done
   137          systemctl restart kubelet
   138  
   139          # prepull images from gcr.io/k8s-staging-ci-images and retag it to
   140          # registry.k8s.io so kubeadm can fetch correct images no matter what
   141          declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   142          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   143          IMAGE_REGISTRY_PREFIX=registry.k8s.io
   144          for IMAGE in "$${IMAGES[@]}"; do
   145            $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}"
   146            $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}"
   147          done
   148  
   149          echo "kubeadm version: $(kubeadm version -o=short)"
   150          echo "kubectl version: $(kubectl version --client=true)"
   151          echo "kubelet version: $(kubelet --version)"
   152        owner: root:root
   153        path: /tmp/replace-k8s-binaries.sh
   154        permissions: "0744"
   155      - content: |
   156          #!/bin/bash
   157  
   158          set -o nounset
   159          set -o pipefail
   160          set -o errexit
   161  
   162          curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz
   163          tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq
   164          rm /tmp/yq_linux_amd64.tar.gz
   165  
   166          export KUBECONFIG=/etc/kubernetes/admin.conf
   167          kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}"
   168          systemctl stop kubelet
   169          yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
   170          yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
   171          yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
   172          systemctl restart kubelet
   173        owner: root:root
   174        path: /tmp/replace-k8s-components.sh
   175        permissions: "0744"
   176      initConfiguration:
   177        nodeRegistration:
   178          kubeletExtraArgs:
   179            cloud-provider: external
   180            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   181            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   182          name: '{{ ds.meta_data["local_hostname"] }}'
   183      joinConfiguration:
   184        nodeRegistration:
   185          kubeletExtraArgs:
   186            cloud-provider: external
   187            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   188            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   189          name: '{{ ds.meta_data["local_hostname"] }}'
   190      mounts:
   191      - - LABEL=etcd_disk
   192        - /var/lib/etcddisk
   193      postKubeadmCommands:
   194      - bash -c /tmp/replace-k8s-components.sh
   195      preKubeadmCommands:
   196      - bash -c /tmp/oot-cred-provider.sh
   197      - bash -c /tmp/replace-k8s-binaries.sh
   198      verbosity: 5
   199    machineTemplate:
   200      infrastructureRef:
   201        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   202        kind: AzureMachineTemplate
   203        name: ${CLUSTER_NAME}-control-plane
   204    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   205    version: ${KUBERNETES_VERSION}
   206  ---
   207  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   208  kind: AzureMachineTemplate
   209  metadata:
   210    name: ${CLUSTER_NAME}-control-plane
   211    namespace: default
   212  spec:
   213    template:
   214      spec:
   215        dataDisks:
   216        - diskSizeGB: 256
   217          lun: 0
   218          nameSuffix: etcddisk
   219        identity: UserAssigned
   220        image:
   221          marketplace:
   222            offer: capi
   223            publisher: cncf-upstream
   224            sku: ubuntu-2204-gen1
   225            version: latest
   226        osDisk:
   227          diskSizeGB: 128
   228          osType: Linux
   229        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   230        userAssignedIdentities:
   231        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
   232        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   233  ---
   234  apiVersion: cluster.x-k8s.io/v1beta1
   235  kind: MachineDeployment
   236  metadata:
   237    name: ${CLUSTER_NAME}-md-0
   238    namespace: default
   239  spec:
   240    clusterName: ${CLUSTER_NAME}
   241    replicas: ${WORKER_MACHINE_COUNT:=2}
   242    selector: {}
   243    template:
   244      metadata:
   245        labels:
   246          nodepool: pool1
   247      spec:
   248        bootstrap:
   249          configRef:
   250            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   251            kind: KubeadmConfigTemplate
   252            name: ${CLUSTER_NAME}-md-0
   253        clusterName: ${CLUSTER_NAME}
   254        infrastructureRef:
   255          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   256          kind: AzureMachineTemplate
   257          name: ${CLUSTER_NAME}-md-0
   258        version: ${KUBERNETES_VERSION}
   259  ---
   260  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   261  kind: AzureMachineTemplate
   262  metadata:
   263    name: ${CLUSTER_NAME}-md-0
   264    namespace: default
   265  spec:
   266    template:
   267      spec:
   268        identity: UserAssigned
   269        image:
   270          marketplace:
   271            offer: capi
   272            publisher: cncf-upstream
   273            sku: ubuntu-2204-gen1
   274            version: latest
   275        osDisk:
   276          diskSizeGB: 128
   277          osType: Linux
   278        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   279        userAssignedIdentities:
   280        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
   281        vmExtensions:
   282        - name: CustomScript
   283          protectedSettings:
   284            commandToExecute: |
   285              #!/bin/sh
   286              echo "This script is a no-op used for extension testing purposes ..."
   287              touch test_file
   288          publisher: Microsoft.Azure.Extensions
   289          version: "2.1"
   290        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   291  ---
   292  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   293  kind: KubeadmConfigTemplate
   294  metadata:
   295    name: ${CLUSTER_NAME}-md-0
   296    namespace: default
   297  spec:
   298    template:
   299      spec:
   300        files:
   301        - contentFrom:
   302            secret:
   303              key: worker-node-azure.json
   304              name: ${CLUSTER_NAME}-md-0-azure-json
   305          owner: root:root
   306          path: /etc/kubernetes/azure.json
   307          permissions: "0644"
   308        - content: |
   309            #!/bin/bash
   310  
   311            set -o nounset
   312            set -o pipefail
   313            set -o errexit
   314            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   315  
   316            echo "Use OOT credential provider"
   317            mkdir -p /var/lib/kubelet/credential-provider
   318            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   319            chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   320            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   321            chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   322          owner: root:root
   323          path: /tmp/oot-cred-provider.sh
   324          permissions: "0744"
   325        - content: |
   326            #!/bin/bash
   327  
   328            set -o nounset
   329            set -o pipefail
   330            set -o errexit
   331  
   332            systemctl stop kubelet
   333            declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
   334            az login --identity
   335            for BINARY in "$${BINARIES[@]}"; do
   336              echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}"
   337              az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login
   338            done
   339            systemctl restart kubelet
   340  
   341            echo "kubeadm version: $(kubeadm version -o=short)"
   342            echo "kubectl version: $(kubectl version --client=true)"
   343            echo "kubelet version: $(kubelet --version)"
   344          owner: root:root
   345          path: /tmp/replace-k8s-binaries.sh
   346          permissions: "0744"
   347        joinConfiguration:
   348          nodeRegistration:
   349            kubeletExtraArgs:
   350              cloud-provider: external
   351              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   352              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   353            name: '{{ ds.meta_data["local_hostname"] }}'
   354        preKubeadmCommands:
   355        - bash -c /tmp/oot-cred-provider.sh
   356        - bash -c /tmp/replace-k8s-binaries.sh
   357  ---
   358  apiVersion: cluster.x-k8s.io/v1beta1
   359  kind: MachineDeployment
   360  metadata:
   361    name: ${CLUSTER_NAME}-md-win
   362    namespace: default
   363  spec:
   364    clusterName: ${CLUSTER_NAME}
   365    replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0}
   366    selector: {}
   367    template:
   368      spec:
   369        bootstrap:
   370          configRef:
   371            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   372            kind: KubeadmConfigTemplate
   373            name: ${CLUSTER_NAME}-md-win
   374        clusterName: ${CLUSTER_NAME}
   375        infrastructureRef:
   376          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   377          kind: AzureMachineTemplate
   378          name: ${CLUSTER_NAME}-md-win
   379        version: ${KUBERNETES_VERSION}
   380  ---
   381  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   382  kind: AzureMachineTemplate
   383  metadata:
   384    annotations:
   385      runtime: containerd
   386    name: ${CLUSTER_NAME}-md-win
   387    namespace: default
   388  spec:
   389    template:
   390      metadata:
   391        annotations:
   392          runtime: containerd
   393          windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""}
   394      spec:
   395        identity: UserAssigned
   396        image:
   397          marketplace:
   398            offer: capi-windows
   399            publisher: cncf-upstream
   400            sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1
   401            version: latest
   402        osDisk:
   403          diskSizeGB: 128
   404          managedDisk:
   405            storageAccountType: Premium_LRS
   406          osType: Windows
   407        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   408        userAssignedIdentities:
   409        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
   410        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   411  ---
   412  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   413  kind: KubeadmConfigTemplate
   414  metadata:
   415    name: ${CLUSTER_NAME}-md-win
   416    namespace: default
   417  spec:
   418    template:
   419      spec:
   420        files:
   421        - contentFrom:
   422            secret:
   423              key: worker-node-azure.json
   424              name: ${CLUSTER_NAME}-md-win-azure-json
   425          owner: root:root
   426          path: c:/k/azure.json
   427          permissions: "0644"
   428        - content: |-
   429            Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe
   430            Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe
   431          path: C:/defender-exclude-calico.ps1
   432          permissions: "0744"
   433        - content: |
   434            # /tmp is assumed created and required for upstream e2e tests to pass
   435            New-Item -ItemType Directory -Force -Path C:\tmp\
   436          path: C:/create-temp-folder.ps1
   437          permissions: "0744"
   438        - content: |
   439            $ErrorActionPreference = 'Stop'
   440  
   441            $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}"
   442            if($$CONTAINERD_URL -ne ""){
   443              # Kubelet service depends on contianerd service so make a best effort attempt to stop it
   444              Stop-Service kubelet -Force -ErrorAction SilentlyContinue
   445              Stop-Service containerd -Force
   446              echo "downloading containerd: $$CONTAINERD_URL"
   447              curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz"
   448              tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1
   449  
   450              Start-Service containerd
   451            }
   452  
   453            containerd.exe --version
   454            containerd-shim-runhcs-v1.exe --version
   455          path: C:/replace-containerd.ps1
   456          permissions: "0744"
   457        - content: |
   458            mkdir -Force c:/localdumps
   459            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f
   460            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f
   461            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f
   462            # Enable sftp so we can copy crash dump files during log collection of stfp
   463            $sshd_config = "$env:ProgramData\ssh\sshd_config"
   464            if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config }
   465            Add-Content -Path $sshd_config "Subsystem sftp              sftp-server.exe"
   466            sc.exe stop sshd
   467            sc.exe start sshd
   468          path: C:/collect-hns-crashes.ps1
   469          permissions: "0744"
   470        - content: |
   471            $ErrorActionPreference = 'Stop'
   472  
   473            echo "Use OOT credential provider"
   474            mkdir C:\var\lib\kubelet\credential-provider
   475            curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe
   476            cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider
   477            curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml
   478          path: C:/oot-cred-provider.ps1
   479          permissions: "0744"
   480        - content: |
   481            Write-Host "Installing Azure CLI"
   482            $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi
   483            # Need to add manually AZ to PATH as it is not added without a reset
   484            Write-Host "Adding AZ to path:"
   485            $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin"
   486            Write-Host "Version Azure CLI:"
   487            az version
   488          path: C:/install-az-cli.ps1
   489          permissions: "0744"
   490        - content: |
   491            $ErrorActionPreference = 'Stop'
   492  
   493            Stop-Service kubelet -Force
   494  
   495            $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}"
   496            if($$KUBE_GIT_VERSION -ne "")
   497            {
   498              # Need to add manually AZ to PATH as it is not added without a reset
   499              Write-Host "Adding AZ to path:"
   500              $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin"
   501              # Install Binaries
   502              $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy")
   503              $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64"
   504              echo "running az login via identity"
   505              az login --identity
   506              foreach ( $$binary in $$binaries )
   507              {
   508                echo "downloading binary: $$ci_url/$$binary.exe"
   509                az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login
   510              }
   511            }
   512  
   513            # Tag it to the ci version.  The image knows how to use the copy locally with the configmap
   514            # that is applied at at this stage (windows-kubeproxy-ci.yaml)
   515            ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess
   516            ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess"
   517  
   518            kubeadm.exe version -o=short
   519            kubectl.exe version --client=true
   520            kubelet.exe --version
   521            kube-proxy.exe --version
   522          path: C:/replace-pr-binaries.ps1
   523          permissions: "0744"
   524        joinConfiguration:
   525          nodeRegistration:
   526            criSocket: npipe:////./pipe/containerd-containerd
   527            kubeletExtraArgs:
   528              cloud-provider: external
   529              feature-gates: ${NODE_FEATURE_GATES:-""}
   530              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   531              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   532              v: "2"
   533              windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS
   534            name: '{{ ds.meta_data["local_hostname"] }}'
   535        postKubeadmCommands:
   536        - nssm set kubelet start SERVICE_AUTO_START
   537        - powershell C:/defender-exclude-calico.ps1
   538        preKubeadmCommands:
   539        - powershell C:/create-temp-folder.ps1
   540        - powershell C:/replace-containerd.ps1
   541        - powershell C:/collect-hns-crashes.ps1
   542        - powershell C:/oot-cred-provider.ps1
   543        - powershell C:/install-az-cli.ps1
   544        - powershell C:/replace-pr-binaries.ps1
   545        users:
   546        - groups: Administrators
   547          name: capi
   548          sshAuthorizedKeys:
   549          - ${AZURE_SSH_PUBLIC_KEY:=""}
   550  ---
   551  apiVersion: cluster.x-k8s.io/v1beta1
   552  kind: MachineHealthCheck
   553  metadata:
   554    name: ${CLUSTER_NAME}-mhc-0
   555    namespace: default
   556  spec:
   557    clusterName: ${CLUSTER_NAME}
   558    maxUnhealthy: 100%
   559    selector:
   560      matchLabels:
   561        nodepool: pool1
   562    unhealthyConditions:
   563    - status: "True"
   564      timeout: 30s
   565      type: E2ENodeUnhealthy
   566  ---
   567  apiVersion: addons.cluster.x-k8s.io/v1beta1
   568  kind: ClusterResourceSet
   569  metadata:
   570    name: ${CLUSTER_NAME}-calico-windows
   571    namespace: default
   572  spec:
   573    clusterSelector:
   574      matchLabels:
   575        cni-windows: ${CLUSTER_NAME}-calico
   576    resources:
   577    - kind: ConfigMap
   578      name: cni-${CLUSTER_NAME}-calico-windows
   579    strategy: ApplyOnce
   580  ---
   581  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   582  kind: AzureClusterIdentity
   583  metadata:
   584    labels:
   585      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   586    name: ${CLUSTER_IDENTITY_NAME}
   587    namespace: default
   588  spec:
   589    allowedNamespaces: {}
   590    clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}
   591    tenantID: ${AZURE_TENANT_ID}
   592    type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity}
   593  ---
   594  apiVersion: addons.cluster.x-k8s.io/v1beta1
   595  kind: ClusterResourceSet
   596  metadata:
   597    name: csi-proxy
   598    namespace: default
   599  spec:
   600    clusterSelector:
   601      matchLabels:
   602        csi-proxy: enabled
   603    resources:
   604    - kind: ConfigMap
   605      name: csi-proxy-addon
   606    strategy: ApplyOnce
   607  ---
   608  apiVersion: addons.cluster.x-k8s.io/v1beta1
   609  kind: ClusterResourceSet
   610  metadata:
   611    name: containerd-logger-${CLUSTER_NAME}
   612    namespace: default
   613  spec:
   614    clusterSelector:
   615      matchLabels:
   616        containerd-logger: enabled
   617    resources:
   618    - kind: ConfigMap
   619      name: containerd-logger-${CLUSTER_NAME}
   620    strategy: ApplyOnce
   621  ---
   622  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   623  kind: HelmChartProxy
   624  metadata:
   625    name: calico
   626    namespace: default
   627  spec:
   628    chartName: tigera-operator
   629    clusterSelector:
   630      matchLabels:
   631        cni: calico
   632    namespace: tigera-operator
   633    releaseName: projectcalico
   634    repoURL: https://docs.tigera.io/calico/charts
   635    valuesTemplate: |-
   636      installation:
   637        cni:
   638          type: Calico
   639        calicoNetwork:
   640          bgp: Disabled
   641          mtu: 1350
   642          ipPools:
   643          ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
   644          - cidr: {{ $cidr }}
   645            encapsulation: VXLAN{{end}}
   646        registry: mcr.microsoft.com/oss
   647      # Image and registry configuration for the tigera/operator pod.
   648      tigeraOperator:
   649        image: tigera/operator
   650        registry: mcr.microsoft.com/oss
   651      calicoctl:
   652        image: mcr.microsoft.com/oss/calico/ctl
   653    version: ${CALICO_VERSION}
   654  ---
   655  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   656  kind: HelmChartProxy
   657  metadata:
   658    name: azuredisk-csi-driver-chart
   659    namespace: default
   660  spec:
   661    chartName: azuredisk-csi-driver
   662    clusterSelector:
   663      matchLabels:
   664        azuredisk-csi: "true"
   665    namespace: kube-system
   666    releaseName: azuredisk-csi-driver-oot
   667    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   668    valuesTemplate: |-
   669      controller:
   670        replicas: 1
   671        runOnControlPlane: true
   672      windows:
   673        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   674  ---
   675  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   676  kind: HelmChartProxy
   677  metadata:
   678    name: cloud-provider-azure-chart
   679    namespace: default
   680  spec:
   681    chartName: cloud-provider-azure
   682    clusterSelector:
   683      matchLabels:
   684        cloud-provider: azure
   685    releaseName: cloud-provider-azure-oot
   686    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   687    valuesTemplate: |
   688      infra:
   689        clusterName: {{ .Cluster.metadata.name }}
   690      cloudControllerManager:
   691        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   692        logVerbosity: 4
   693  ---
   694  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   695  kind: HelmChartProxy
   696  metadata:
   697    name: cloud-provider-azure-chart-ci
   698    namespace: default
   699  spec:
   700    chartName: cloud-provider-azure
   701    clusterSelector:
   702      matchLabels:
   703        cloud-provider: azure-ci
   704    releaseName: cloud-provider-azure-oot
   705    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   706    valuesTemplate: |
   707      infra:
   708        clusterName: {{ .Cluster.metadata.name }}
   709      cloudControllerManager:
   710        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   711        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   712        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   713        imageName: "${CCM_IMAGE_NAME:-""}"
   714        imageRepository: "${IMAGE_REGISTRY:-""}"
   715        imageTag: "${IMAGE_TAG_CCM:-""}"
   716        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   717        replicas: ${CCM_COUNT:-1}
   718        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   719      cloudNodeManager:
   720        imageName: "${CNM_IMAGE_NAME:-""}"
   721        imageRepository: "${IMAGE_REGISTRY:-""}"
   722        imageTag: "${IMAGE_TAG_CNM:-""}"
   723  ---
   724  apiVersion: v1
   725  data:
   726    kube-proxy-patch: |-
   727      apiVersion: v1
   728      kind: ConfigMap
   729      metadata:
   730        name: windows-kubeproxy-ci
   731        namespace: kube-system
   732      data:
   733        KUBEPROXY_PATH: "c:/k/kube-proxy.exe"
   734    proxy: |
   735      apiVersion: apps/v1
   736      kind: DaemonSet
   737      metadata:
   738        labels:
   739          k8s-app: kube-proxy
   740        name: kube-proxy-windows
   741        namespace: kube-system
   742      spec:
   743        selector:
   744          matchLabels:
   745            k8s-app: kube-proxy-windows
   746        template:
   747          metadata:
   748            labels:
   749              k8s-app: kube-proxy-windows
   750          spec:
   751            serviceAccountName: kube-proxy
   752            securityContext:
   753              windowsOptions:
   754                hostProcess: true
   755                runAsUserName: "NT AUTHORITY\\system"
   756            hostNetwork: true
   757            containers:
   758            - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess
   759              args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"]
   760              workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/"
   761              name: kube-proxy
   762              env:
   763              - name: NODE_NAME
   764                valueFrom:
   765                  fieldRef:
   766                    apiVersion: v1
   767                    fieldPath: spec.nodeName
   768              - name: POD_IP
   769                valueFrom:
   770                  fieldRef:
   771                    fieldPath: status.podIP
   772              - name: KUBEPROXY_PATH
   773                valueFrom:
   774                  configMapKeyRef:
   775                    name: windows-kubeproxy-ci
   776                    key: KUBEPROXY_PATH
   777                    optional: true
   778              volumeMounts:
   779              - mountPath: /var/lib/kube-proxy
   780                name: kube-proxy
   781            nodeSelector:
   782              kubernetes.io/os: windows
   783            tolerations:
   784            - key: CriticalAddonsOnly
   785              operator: Exists
   786            - operator: Exists
   787            volumes:
   788            - configMap:
   789                name: kube-proxy
   790              name: kube-proxy
   791        updateStrategy:
   792          type: RollingUpdate
   793    windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind:
   794      IPAMConfig\nmetadata:\n  name: default\nspec:\n  autoAllocateBlocks: true\n  strictAffinity:
   795      true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n  name: calico-static-rules\n
   796      \ namespace: calico-system\n  labels:\n    tier: node\n    app: calico\ndata:\n
   797      \ static-rules.json: |\n    {\n      \"Provider\": \"azure\",\n      \"Version\":
   798      \"0.1\",\n      \"Rules\": [\n        {\n          \"Name\": \"EndpointPolicy\",\n
   799      \         \"Rule\": {\n              \"Id\": \"wireserver\",\n              \"Type\":
   800      \"ACL\",\n              \"Protocol\": 6,\n              \"Action\": \"Block\",\n
   801      \             \"Direction\": \"Out\",\n              \"RemoteAddresses\": \"168.63.129.16/32\",\n
   802      \             \"RemotePorts\": \"80\",\n              \"Priority\": 200,\n              \"RuleType\":
   803      \"Switch\"\n            }\n          }\n      ]\n    } \n---\nkind: ConfigMap\napiVersion:
   804      v1\nmetadata:\n  name: calico-config-windows\n  namespace: calico-system\n  labels:\n
   805      \   tier: node\n    app: calico\ndata:\n  veth_mtu: \"1350\"\n  \n  cni_network_config:
   806      |\n    {\n      \"name\": \"Calico\",\n      \"cniVersion\": \"0.3.1\",\n      \"plugins\":
   807      [\n        {\n          \"windows_use_single_network\": true,\n          \"type\":
   808      \"calico\",\n          \"mode\": \"vxlan\",\n          \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n
   809      \         \"nodename_file_optional\": true,\n          \"log_file_path\": \"c:/cni.log\",\n
   810      \         \"log_level\": \"debug\",\n\n          \"vxlan_mac_prefix\": \"0E-2A\",\n
   811      \         \"vxlan_vni\": 4096,\n          \"mtu\": __CNI_MTU__,\n          \"policy\":
   812      {\n            \"type\": \"k8s\"\n          },\n\n          \"log_level\": \"info\",\n\n
   813      \         \"capabilities\": {\"dns\": true},\n          \"DNS\":  {\n            \"Search\":
   814      \ [\n              \"svc.cluster.local\"\n            ]\n          },\n\n          \"datastore_type\":
   815      \"kubernetes\",\n\n          \"kubernetes\": {\n            \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n
   816      \         },\n\n          \"ipam\": {\n            \"type\": \"calico-ipam\",\n
   817      \           \"subnet\": \"usePodCidr\"\n          },\n\n          \"policies\":
   818      \ [\n            {\n              \"Name\":  \"EndpointPolicy\",\n              \"Value\":
   819      \ {\n                \"Type\":  \"OutBoundNAT\",\n                \"ExceptionList\":
   820      \ [\n                  \"__K8S_SERVICE_CIDR__\"\n                ]\n              }\n
   821      \           },\n            {\n              \"Name\":  \"EndpointPolicy\",\n
   822      \             \"Value\":  {\n                \"Type\":  \"SDNROUTE\",\n                \"DestinationPrefix\":
   823      \ \"__K8S_SERVICE_CIDR__\",\n                \"NeedEncap\":  true\n              }\n
   824      \           }\n          ]\n        }\n      ]\n\n    }\n---\napiVersion: apps/v1\nkind:
   825      DaemonSet\nmetadata:\n  name: calico-node-windows\n  labels:\n    tier: node\n
   826      \   app: calico\n  namespace: calico-system\nspec:\n  selector:\n    matchLabels:\n
   827      \     app: calico\n  template:\n    metadata:\n      labels:\n        tier: node\n
   828      \       app: calico\n    spec:\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n
   829      \           nodeSelectorTerms:\n              - matchExpressions:\n                  -
   830      key: kubernetes.io/os\n                    operator: In\n                    values:\n
   831      \                     - windows\n                  - key: kubernetes.io/arch\n
   832      \                   operator: In\n                    values:\n                      -
   833      amd64\n      securityContext:\n        windowsOptions:\n          hostProcess:
   834      true\n          runAsUserName: \"NT AUTHORITY\\\\system\"\n      hostNetwork:
   835      true\n      serviceAccountName: calico-node\n      tolerations:\n      - operator:
   836      Exists\n        effect: NoSchedule\n        # Mark the pod as a critical add-on
   837      for rescheduling.\n      - key: CriticalAddonsOnly\n        operator: Exists\n
   838      \     - effect: NoExecute\n        operator: Exists\n      initContainers:\n        #
   839      This container installs the CNI binaries\n        # and CNI network config file
   840      on each node.\n        - name: install-cni\n          image: sigwindowstools/calico-install:v3.26.1-hostprocess\n
   841      \         args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n
   842      \         imagePullPolicy: Always\n          env:\n            # Name of the CNI
   843      config file to create.\n            - name: CNI_CONF_NAME\n              value:
   844      \"10-calico.conflist\"\n            # The CNI network config to install on each
   845      node.\n            - name: CNI_NETWORK_CONFIG\n              valueFrom:\n                configMapKeyRef:\n
   846      \                 name: calico-config-windows\n                  key: cni_network_config\n
   847      \           # Set the hostname based on the k8s node name.\n            - name:
   848      KUBERNETES_NODE_NAME\n              valueFrom:\n                fieldRef:\n                  fieldPath:
   849      spec.nodeName\n            # CNI MTU Config variable\n            - name: CNI_MTU\n
   850      \             valueFrom:\n                configMapKeyRef:\n                  name:
   851      calico-config-windows\n                  key: veth_mtu\n            # Prevents
   852      the container from sleeping forever.\n            - name: SLEEP\n              value:
   853      \"false\"\n            - name: K8S_SERVICE_CIDR\n              value: \"10.96.0.0/12\"\n
   854      \         volumeMounts:\n            - mountPath: /host/opt/cni/bin\n              name:
   855      cni-bin-dir\n            - mountPath: /host/etc/cni/net.d\n              name:
   856      cni-net-dir\n            - name: kubeadm-config\n              mountPath: /etc/kubeadm-config/\n
   857      \         securityContext:\n            windowsOptions:\n              hostProcess:
   858      true\n              runAsUserName: \"NT AUTHORITY\\\\system\"\n      containers:\n
   859      \     - name: calico-node-startup\n        image: sigwindowstools/calico-node:v3.26.1-hostprocess\n
   860      \       args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n
   861      \       workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        imagePullPolicy:
   862      Always\n        volumeMounts:\n        - name: calico-config-windows\n          mountPath:
   863      /etc/kube-calico-windows/\n        env:\n        - name: POD_NAME\n          valueFrom:\n
   864      \           fieldRef:\n              apiVersion: v1\n              fieldPath:
   865      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   866      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   867      name: CNI_IPAM_TYPE\n          value: \"calico-ipam\"\n        - name: CALICO_NETWORKING_BACKEND\n
   868      \         value: \"vxlan\"\n        - name: KUBECONFIG\n          value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n
   869      \       - name: VXLAN_VNI\n          value: \"4096\"\n      - name: calico-node-felix\n
   870      \       image: sigwindowstools/calico-node:v3.26.1-hostprocess\n        args:
   871      [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n        imagePullPolicy:
   872      Always\n        workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        volumeMounts:\n
   873      \       - name: calico-config-windows\n          mountPath: /etc/kube-calico-windows/\n
   874      \       - name: calico-static-rules\n          mountPath: /calico/static-rules.json\n
   875      \         subPath: static-rules.json\n        env:\n        - name: POD_NAME\n
   876      \         valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath:
   877      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   878      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   879      name: VXLAN_VNI\n          value: \"4096\"\n        - name: KUBECONFIG\n          value:
   880      \"C:/etc/cni/net.d/calico-kubeconfig\"\n      volumes:\n      - name: calico-config-windows\n
   881      \       configMap:\n          name: calico-config-windows\n      - name: calico-static-rules\n
   882      \       configMap:\n          name: calico-static-rules\n      # Used to install
   883      CNI.\n      - name: cni-bin-dir\n        hostPath:\n          path: /opt/cni/bin\n
   884      \     - name: cni-net-dir\n        hostPath:\n          path: /etc/cni/net.d\n
   885      \     - name: kubeadm-config\n        configMap:\n          name: kubeadm-config\n---\napiVersion:
   886      apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: ipamconfigs.crd.projectcalico.org\nspec:\n
   887      \ group: crd.projectcalico.org\n  names:\n    kind: IPAMConfig\n    listKind:
   888      IPAMConfigList\n    plural: ipamconfigs\n    singular: ipamconfig\n  preserveUnknownFields:
   889      false\n  scope: Cluster\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n
   890      \       properties:\n          apiVersion:\n            description: 'APIVersion
   891      defines the versioned schema of this representation\n              of an object.
   892      Servers should convert recognized schemas to the latest\n              internal
   893      value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n
   894      \           type: string\n          kind:\n            description: 'Kind is a
   895      string value representing the REST resource this\n              object represents.
   896      Servers may infer this from the endpoint the client\n              submits requests
   897      to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n
   898      \           type: string\n          metadata:\n            type: object\n          spec:\n
   899      \           description: IPAMConfigSpec contains the specification for an IPAMConfig\n
   900      \             resource.\n            properties:\n              autoAllocateBlocks:\n
   901      \               type: boolean\n              maxBlocksPerHost:\n                description:
   902      MaxBlocksPerHost, if non-zero, is the max number of blocks\n                  that
   903      can be affine to each host.\n                maximum: 2147483647\n                minimum:
   904      0\n                type: integer\n              strictAffinity:\n                type:
   905      boolean\n            required:\n            - autoAllocateBlocks\n            -
   906      strictAffinity\n            type: object\n        type: object\n    served: true\n
   907      \   storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n
   908      \ conditions: []\n  storedVersions: []\n"
   909  kind: ConfigMap
   910  metadata:
   911    annotations:
   912      note: generated
   913    labels:
   914      type: generated
   915    name: cni-${CLUSTER_NAME}-calico-windows
   916    namespace: default
   917  ---
   918  apiVersion: v1
   919  data:
   920    csi-proxy: |
   921      apiVersion: apps/v1
   922      kind: DaemonSet
   923      metadata:
   924        labels:
   925          k8s-app: csi-proxy
   926        name: csi-proxy
   927        namespace: kube-system
   928      spec:
   929        selector:
   930          matchLabels:
   931            k8s-app: csi-proxy
   932        template:
   933          metadata:
   934            labels:
   935              k8s-app: csi-proxy
   936          spec:
   937            nodeSelector:
   938              "kubernetes.io/os": windows
   939            securityContext:
   940              windowsOptions:
   941                hostProcess: true
   942                runAsUserName: "NT AUTHORITY\\SYSTEM"
   943            hostNetwork: true
   944            containers:
   945              - name: csi-proxy
   946                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   947  kind: ConfigMap
   948  metadata:
   949    annotations:
   950      note: generated
   951    labels:
   952      type: generated
   953    name: csi-proxy-addon
   954    namespace: default
   955  ---
   956  apiVersion: v1
   957  data:
   958    containerd-windows-logger: |
   959      apiVersion: apps/v1
   960      kind: DaemonSet
   961      metadata:
   962        labels:
   963          k8s-app: containerd-logger
   964        name: containerd-logger
   965        namespace: kube-system
   966      spec:
   967        selector:
   968          matchLabels:
   969            k8s-app: containerd-logger
   970        template:
   971          metadata:
   972            labels:
   973              k8s-app: containerd-logger
   974          spec:
   975            securityContext:
   976              windowsOptions:
   977                hostProcess: true
   978                runAsUserName: "NT AUTHORITY\\system"
   979            hostNetwork: true
   980            containers:
   981            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
   982              args: [ "config.json" ]
   983              name: containerd-logger
   984              imagePullPolicy: Always
   985              volumeMounts:
   986              - name: containerd-logger-config
   987                mountPath: /config.json
   988                subPath: config.json
   989            nodeSelector:
   990              kubernetes.io/os: windows
   991            tolerations:
   992            - key: CriticalAddonsOnly
   993              operator: Exists
   994            - operator: Exists
   995            volumes:
   996            - configMap:
   997                name: containerd-logger-config
   998              name: containerd-logger-config
   999        updateStrategy:
  1000          type: RollingUpdate
  1001      ---
  1002      kind: ConfigMap
  1003      apiVersion: v1
  1004      metadata:
  1005        name: containerd-logger-config
  1006        namespace: kube-system
  1007      data:
  1008        config.json: |
  1009          {
  1010            "inputs": [
  1011              {
  1012                "type": "ETW",
  1013                "sessionNamePrefix": "containerd",
  1014                "cleanupOldSessions": true,
  1015                "reuseExistingSession": true,
  1016                "providers": [
  1017                  {
  1018                    "providerName": "Microsoft.Virtualization.RunHCS",
  1019                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
  1020                    "level": "Verbose"
  1021                  },
  1022                  {
  1023                    "providerName": "ContainerD",
  1024                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
  1025                    "level": "Verbose"
  1026                  }
  1027                ]
  1028              }
  1029            ],
  1030            "filters": [
  1031              {
  1032                  "type": "drop",
  1033                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
  1034              },
  1035              {
  1036                  "type": "drop",
  1037                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
  1038              },
  1039              {
  1040                  "type": "drop",
  1041                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
  1042              },
  1043              {
  1044                  "type": "drop",
  1045                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
  1046              },
  1047              {
  1048                  "type": "drop",
  1049                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
  1050              },
  1051              {
  1052                  "type": "drop",
  1053                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
  1054              },
  1055              {
  1056                  "type": "drop",
  1057                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
  1058              }
  1059            ],
  1060            "outputs": [
  1061              {
  1062                "type": "StdOutput"
  1063              }
  1064            ],
  1065            "schemaVersion": "2016-08-11"
  1066          }
  1067  kind: ConfigMap
  1068  metadata:
  1069    annotations:
  1070      note: generated
  1071    labels:
  1072      type: generated
  1073    name: containerd-logger-${CLUSTER_NAME}
  1074    namespace: default
  1075  ---
  1076  apiVersion: addons.cluster.x-k8s.io/v1beta1
  1077  kind: ClusterResourceSet
  1078  metadata:
  1079    name: metrics-server-${CLUSTER_NAME}
  1080    namespace: default
  1081  spec:
  1082    clusterSelector:
  1083      matchLabels:
  1084        metrics-server: enabled
  1085    resources:
  1086    - kind: ConfigMap
  1087      name: metrics-server-${CLUSTER_NAME}
  1088    strategy: ApplyOnce
  1089  ---
  1090  apiVersion: v1
  1091  data:
  1092    metrics-server: |
  1093      apiVersion: v1
  1094      kind: ServiceAccount
  1095      metadata:
  1096        labels:
  1097          k8s-app: metrics-server
  1098        name: metrics-server
  1099        namespace: kube-system
  1100      ---
  1101      apiVersion: rbac.authorization.k8s.io/v1
  1102      kind: ClusterRole
  1103      metadata:
  1104        labels:
  1105          k8s-app: metrics-server
  1106          rbac.authorization.k8s.io/aggregate-to-admin: "true"
  1107          rbac.authorization.k8s.io/aggregate-to-edit: "true"
  1108          rbac.authorization.k8s.io/aggregate-to-view: "true"
  1109        name: system:aggregated-metrics-reader
  1110      rules:
  1111      - apiGroups:
  1112        - metrics.k8s.io
  1113        resources:
  1114        - pods
  1115        - nodes
  1116        verbs:
  1117        - get
  1118        - list
  1119        - watch
  1120      ---
  1121      apiVersion: rbac.authorization.k8s.io/v1
  1122      kind: ClusterRole
  1123      metadata:
  1124        labels:
  1125          k8s-app: metrics-server
  1126        name: system:metrics-server
  1127      rules:
  1128      - apiGroups:
  1129        - ""
  1130        resources:
  1131        - nodes/metrics
  1132        verbs:
  1133        - get
  1134      - apiGroups:
  1135        - ""
  1136        resources:
  1137        - pods
  1138        - nodes
  1139        verbs:
  1140        - get
  1141        - list
  1142        - watch
  1143      ---
  1144      apiVersion: rbac.authorization.k8s.io/v1
  1145      kind: RoleBinding
  1146      metadata:
  1147        labels:
  1148          k8s-app: metrics-server
  1149        name: metrics-server-auth-reader
  1150        namespace: kube-system
  1151      roleRef:
  1152        apiGroup: rbac.authorization.k8s.io
  1153        kind: Role
  1154        name: extension-apiserver-authentication-reader
  1155      subjects:
  1156      - kind: ServiceAccount
  1157        name: metrics-server
  1158        namespace: kube-system
  1159      ---
  1160      apiVersion: rbac.authorization.k8s.io/v1
  1161      kind: ClusterRoleBinding
  1162      metadata:
  1163        labels:
  1164          k8s-app: metrics-server
  1165        name: metrics-server:system:auth-delegator
  1166      roleRef:
  1167        apiGroup: rbac.authorization.k8s.io
  1168        kind: ClusterRole
  1169        name: system:auth-delegator
  1170      subjects:
  1171      - kind: ServiceAccount
  1172        name: metrics-server
  1173        namespace: kube-system
  1174      ---
  1175      apiVersion: rbac.authorization.k8s.io/v1
  1176      kind: ClusterRoleBinding
  1177      metadata:
  1178        labels:
  1179          k8s-app: metrics-server
  1180        name: system:metrics-server
  1181      roleRef:
  1182        apiGroup: rbac.authorization.k8s.io
  1183        kind: ClusterRole
  1184        name: system:metrics-server
  1185      subjects:
  1186      - kind: ServiceAccount
  1187        name: metrics-server
  1188        namespace: kube-system
  1189      ---
  1190      apiVersion: v1
  1191      kind: Service
  1192      metadata:
  1193        labels:
  1194          k8s-app: metrics-server
  1195        name: metrics-server
  1196        namespace: kube-system
  1197      spec:
  1198        ports:
  1199        - name: https
  1200          port: 443
  1201          protocol: TCP
  1202          targetPort: https
  1203        selector:
  1204          k8s-app: metrics-server
  1205      ---
  1206      apiVersion: apps/v1
  1207      kind: Deployment
  1208      metadata:
  1209        labels:
  1210          k8s-app: metrics-server
  1211        name: metrics-server
  1212        namespace: kube-system
  1213      spec:
  1214        selector:
  1215          matchLabels:
  1216            k8s-app: metrics-server
  1217        strategy:
  1218          rollingUpdate:
  1219            maxUnavailable: 0
  1220        template:
  1221          metadata:
  1222            labels:
  1223              k8s-app: metrics-server
  1224          spec:
  1225            containers:
  1226            - args:
  1227              - --cert-dir=/tmp
  1228              - --secure-port=4443
  1229              - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
  1230              - --kubelet-use-node-status-port
  1231              - --metric-resolution=15s
  1232              - --kubelet-insecure-tls
  1233              image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
  1234              imagePullPolicy: IfNotPresent
  1235              livenessProbe:
  1236                failureThreshold: 3
  1237                httpGet:
  1238                  path: /livez
  1239                  port: https
  1240                  scheme: HTTPS
  1241                periodSeconds: 10
  1242              name: metrics-server
  1243              ports:
  1244              - containerPort: 4443
  1245                name: https
  1246                protocol: TCP
  1247              readinessProbe:
  1248                failureThreshold: 3
  1249                httpGet:
  1250                  path: /readyz
  1251                  port: https
  1252                  scheme: HTTPS
  1253                initialDelaySeconds: 20
  1254                periodSeconds: 10
  1255              resources:
  1256                requests:
  1257                  cpu: 100m
  1258                  memory: 200Mi
  1259              securityContext:
  1260                allowPrivilegeEscalation: false
  1261                readOnlyRootFilesystem: true
  1262                runAsNonRoot: true
  1263                runAsUser: 1000
  1264              volumeMounts:
  1265              - mountPath: /tmp
  1266                name: tmp-dir
  1267            nodeSelector:
  1268              kubernetes.io/os: linux
  1269            priorityClassName: system-cluster-critical
  1270            serviceAccountName: metrics-server
  1271            tolerations:
  1272            - effect: NoSchedule
  1273              key: node-role.kubernetes.io/master
  1274              operator: Exists
  1275            - effect: NoSchedule
  1276              key: node-role.kubernetes.io/control-plane
  1277              operator: Exists
  1278            volumes:
  1279            - emptyDir: {}
  1280              name: tmp-dir
  1281      ---
  1282      apiVersion: apiregistration.k8s.io/v1
  1283      kind: APIService
  1284      metadata:
  1285        labels:
  1286          k8s-app: metrics-server
  1287        name: v1beta1.metrics.k8s.io
  1288      spec:
  1289        group: metrics.k8s.io
  1290        groupPriorityMinimum: 100
  1291        insecureSkipTLSVerify: true
  1292        service:
  1293          name: metrics-server
  1294          namespace: kube-system
  1295        version: v1beta1
  1296        versionPriority: 100
  1297  kind: ConfigMap
  1298  metadata:
  1299    annotations:
  1300      note: generated
  1301    labels:
  1302      type: generated
  1303    name: metrics-server-${CLUSTER_NAME}
  1304    namespace: default