sigs.k8s.io/cluster-api-provider-azure@v1.17.0/templates/test/ci/cluster-template-prow-ci-version.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: enabled
     9      csi-proxy: enabled
    10      metrics-server: enabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 192.168.0.0/16
    18    controlPlaneRef:
    19      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    20      kind: KubeadmControlPlane
    21      name: ${CLUSTER_NAME}-control-plane
    22    infrastructureRef:
    23      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    24      kind: AzureCluster
    25      name: ${CLUSTER_NAME}
    26  ---
    27  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    28  kind: AzureCluster
    29  metadata:
    30    name: ${CLUSTER_NAME}
    31    namespace: default
    32  spec:
    33    additionalTags:
    34      buildProvenance: ${BUILD_PROVENANCE}
    35      creationTimestamp: ${TIMESTAMP}
    36      jobName: ${JOB_NAME}
    37    identityRef:
    38      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    39      kind: AzureClusterIdentity
    40      name: ${CLUSTER_IDENTITY_NAME}
    41    location: ${AZURE_LOCATION}
    42    networkSpec:
    43      subnets:
    44      - name: control-plane-subnet
    45        role: control-plane
    46      - name: node-subnet
    47        role: node
    48      vnet:
    49        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    50    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    51    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    52  ---
    53  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    54  kind: KubeadmControlPlane
    55  metadata:
    56    name: ${CLUSTER_NAME}-control-plane
    57    namespace: default
    58  spec:
    59    kubeadmConfigSpec:
    60      clusterConfiguration:
    61        apiServer:
    62          extraArgs:
    63            cloud-provider: external
    64            feature-gates: ${K8S_FEATURE_GATES:-""}
    65          timeoutForControlPlane: 20m
    66        controllerManager:
    67          extraArgs:
    68            allocate-node-cidrs: "false"
    69            cloud-provider: external
    70            cluster-name: ${CLUSTER_NAME}
    71            feature-gates: HPAContainerMetrics=true
    72            v: "4"
    73        etcd:
    74          local:
    75            dataDir: /var/lib/etcddisk/etcd
    76            extraArgs:
    77              quota-backend-bytes: "8589934592"
    78        kubernetesVersion: ci/${CI_VERSION}
    79      diskSetup:
    80        filesystems:
    81        - device: /dev/disk/azure/scsi1/lun0
    82          extraOpts:
    83          - -E
    84          - lazy_itable_init=1,lazy_journal_init=1
    85          filesystem: ext4
    86          label: etcd_disk
    87        - device: ephemeral0.1
    88          filesystem: ext4
    89          label: ephemeral0
    90          replaceFS: ntfs
    91        partitions:
    92        - device: /dev/disk/azure/scsi1/lun0
    93          layout: true
    94          overwrite: false
    95          tableType: gpt
    96      files:
    97      - contentFrom:
    98          secret:
    99            key: control-plane-azure.json
   100            name: ${CLUSTER_NAME}-control-plane-azure-json
   101        owner: root:root
   102        path: /etc/kubernetes/azure.json
   103        permissions: "0644"
   104      - content: |
   105          #!/bin/bash
   106  
   107          set -o nounset
   108          set -o pipefail
   109          set -o errexit
   110          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   111  
   112          echo "Use OOT credential provider"
   113          mkdir -p /var/lib/kubelet/credential-provider
   114          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   115          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   116          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   117          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   118        owner: root:root
   119        path: /tmp/oot-cred-provider.sh
   120        permissions: "0744"
   121      - content: |
   122          #!/bin/bash
   123  
   124          set -o nounset
   125          set -o pipefail
   126          set -o errexit
   127          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   128  
   129          # This test installs release packages or binaries that are a result of the CI and release builds.
   130          # It runs '... --version' commands to verify that the binaries are correctly installed
   131          # and finally uninstalls the packages.
   132          # For the release packages it tests all versions in the support skew.
   133          LINE_SEPARATOR="*************************************************"
   134          echo "$$LINE_SEPARATOR"
   135          CI_VERSION=${CI_VERSION}
   136          if [[ "$${CI_VERSION}" != "" ]]; then
   137            CI_DIR=/tmp/k8s-ci
   138            mkdir -p $$CI_DIR
   139            declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   140            declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   141            CONTAINER_EXT="tar"
   142            echo "* testing CI version $$CI_VERSION"
   143            # Check for semver
   144            if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   145              VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   146              DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   147              curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   148              echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   149              apt-get update
   150              # replace . with \.
   151              VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   152              PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   153              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   154                echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   155                DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   156              done
   157            else
   158              CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   159              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   160                echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   161                wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   162                chmod +x "$$CI_DIR/$$CI_PACKAGE"
   163                mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   164              done
   165              IMAGE_REGISTRY_PREFIX=registry.k8s.io
   166              for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   167                echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   168                wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   169                $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   170                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   171                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   172              done
   173            fi
   174            systemctl restart kubelet
   175          fi
   176          echo "* checking binary versions"
   177          echo "ctr version: " $(ctr version)
   178          echo "kubeadm version: " $(kubeadm version -o=short)
   179          echo "kubectl version: " $(kubectl version --client=true)
   180          echo "kubelet version: " $(kubelet --version)
   181          echo "$$LINE_SEPARATOR"
   182        owner: root:root
   183        path: /tmp/kubeadm-bootstrap.sh
   184        permissions: "0744"
   185      initConfiguration:
   186        nodeRegistration:
   187          kubeletExtraArgs:
   188            cloud-provider: external
   189            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   190            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   191          name: '{{ ds.meta_data["local_hostname"] }}'
   192      joinConfiguration:
   193        nodeRegistration:
   194          kubeletExtraArgs:
   195            cloud-provider: external
   196            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   197            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   198          name: '{{ ds.meta_data["local_hostname"] }}'
   199      mounts:
   200      - - LABEL=etcd_disk
   201        - /var/lib/etcddisk
   202      postKubeadmCommands: []
   203      preKubeadmCommands:
   204      - bash -c /tmp/oot-cred-provider.sh
   205      - bash -c /tmp/kubeadm-bootstrap.sh
   206      verbosity: 5
   207    machineTemplate:
   208      infrastructureRef:
   209        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   210        kind: AzureMachineTemplate
   211        name: ${CLUSTER_NAME}-control-plane
   212    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   213    version: ${KUBERNETES_VERSION}
   214  ---
   215  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   216  kind: AzureMachineTemplate
   217  metadata:
   218    name: ${CLUSTER_NAME}-control-plane
   219    namespace: default
   220  spec:
   221    template:
   222      spec:
   223        dataDisks:
   224        - diskSizeGB: 256
   225          lun: 0
   226          nameSuffix: etcddisk
   227        identity: UserAssigned
   228        image:
   229          marketplace:
   230            offer: capi
   231            publisher: cncf-upstream
   232            sku: ubuntu-2204-gen1
   233            version: latest
   234        osDisk:
   235          diskSizeGB: 128
   236          osType: Linux
   237        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   238        userAssignedIdentities:
   239        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
   240        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   241  ---
   242  apiVersion: cluster.x-k8s.io/v1beta1
   243  kind: MachineDeployment
   244  metadata:
   245    name: ${CLUSTER_NAME}-md-0
   246    namespace: default
   247  spec:
   248    clusterName: ${CLUSTER_NAME}
   249    replicas: ${WORKER_MACHINE_COUNT:=2}
   250    selector: {}
   251    template:
   252      metadata:
   253        labels:
   254          nodepool: pool1
   255      spec:
   256        bootstrap:
   257          configRef:
   258            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   259            kind: KubeadmConfigTemplate
   260            name: ${CLUSTER_NAME}-md-0
   261        clusterName: ${CLUSTER_NAME}
   262        infrastructureRef:
   263          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   264          kind: AzureMachineTemplate
   265          name: ${CLUSTER_NAME}-md-0
   266        version: ${KUBERNETES_VERSION}
   267  ---
   268  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   269  kind: AzureMachineTemplate
   270  metadata:
   271    name: ${CLUSTER_NAME}-md-0
   272    namespace: default
   273  spec:
   274    template:
   275      spec:
   276        identity: UserAssigned
   277        image:
   278          marketplace:
   279            offer: capi
   280            publisher: cncf-upstream
   281            sku: ubuntu-2204-gen1
   282            version: latest
   283        osDisk:
   284          diskSizeGB: 128
   285          osType: Linux
   286        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   287        userAssignedIdentities:
   288        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
   289        vmExtensions:
   290        - name: CustomScript
   291          protectedSettings:
   292            commandToExecute: |
   293              #!/bin/sh
   294              echo "This script is a no-op used for extension testing purposes ..."
   295              touch test_file
   296          publisher: Microsoft.Azure.Extensions
   297          version: "2.1"
   298        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   299  ---
   300  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   301  kind: KubeadmConfigTemplate
   302  metadata:
   303    name: ${CLUSTER_NAME}-md-0
   304    namespace: default
   305  spec:
   306    template:
   307      spec:
   308        files:
   309        - contentFrom:
   310            secret:
   311              key: worker-node-azure.json
   312              name: ${CLUSTER_NAME}-md-0-azure-json
   313          owner: root:root
   314          path: /etc/kubernetes/azure.json
   315          permissions: "0644"
   316        - content: |
   317            #!/bin/bash
   318  
   319            set -o nounset
   320            set -o pipefail
   321            set -o errexit
   322            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   323  
   324            echo "Use OOT credential provider"
   325            mkdir -p /var/lib/kubelet/credential-provider
   326            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
   327            chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   328            curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
   329            chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   330          owner: root:root
   331          path: /tmp/oot-cred-provider.sh
   332          permissions: "0744"
   333        - content: |
   334            #!/bin/bash
   335  
   336            set -o nounset
   337            set -o pipefail
   338            set -o errexit
   339            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   340  
   341            # This test installs release packages or binaries that are a result of the CI and release builds.
   342            # It runs '... --version' commands to verify that the binaries are correctly installed
   343            # and finally uninstalls the packages.
   344            # For the release packages it tests all versions in the support skew.
   345            LINE_SEPARATOR="*************************************************"
   346            echo "$$LINE_SEPARATOR"
   347            CI_VERSION=${CI_VERSION}
   348            if [[ "$${CI_VERSION}" != "" ]]; then
   349              CI_DIR=/tmp/k8s-ci
   350              mkdir -p $$CI_DIR
   351              declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   352              declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   353              CONTAINER_EXT="tar"
   354              echo "* testing CI version $$CI_VERSION"
   355              # Check for semver
   356              if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   357                VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   358                DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   359                curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   360                echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   361                apt-get update
   362                # replace . with \.
   363                VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   364                PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   365                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   366                  echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   367                  DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   368                done
   369              else
   370                CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   371                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   372                  echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   373                  wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   374                  chmod +x "$$CI_DIR/$$CI_PACKAGE"
   375                  mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   376                done
   377                IMAGE_REGISTRY_PREFIX=registry.k8s.io
   378                for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   379                  echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   380                  wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   381                  $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   382                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   383                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   384                done
   385              fi
   386              systemctl restart kubelet
   387            fi
   388            echo "* checking binary versions"
   389            echo "ctr version: " $(ctr version)
   390            echo "kubeadm version: " $(kubeadm version -o=short)
   391            echo "kubectl version: " $(kubectl version --client=true)
   392            echo "kubelet version: " $(kubelet --version)
   393            echo "$$LINE_SEPARATOR"
   394          owner: root:root
   395          path: /tmp/kubeadm-bootstrap.sh
   396          permissions: "0744"
   397        joinConfiguration:
   398          nodeRegistration:
   399            kubeletExtraArgs:
   400              cloud-provider: external
   401              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   402              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   403            name: '{{ ds.meta_data["local_hostname"] }}'
   404        preKubeadmCommands:
   405        - bash -c /tmp/oot-cred-provider.sh
   406        - bash -c /tmp/kubeadm-bootstrap.sh
   407        verbosity: 5
   408  ---
   409  apiVersion: cluster.x-k8s.io/v1beta1
   410  kind: MachineDeployment
   411  metadata:
   412    name: ${CLUSTER_NAME}-md-win
   413    namespace: default
   414  spec:
   415    clusterName: ${CLUSTER_NAME}
   416    replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0}
   417    selector: {}
   418    template:
   419      spec:
   420        bootstrap:
   421          configRef:
   422            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   423            kind: KubeadmConfigTemplate
   424            name: ${CLUSTER_NAME}-md-win
   425        clusterName: ${CLUSTER_NAME}
   426        infrastructureRef:
   427          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   428          kind: AzureMachineTemplate
   429          name: ${CLUSTER_NAME}-md-win
   430        version: ${KUBERNETES_VERSION}
   431  ---
   432  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   433  kind: AzureMachineTemplate
   434  metadata:
   435    annotations:
   436      runtime: containerd
   437    name: ${CLUSTER_NAME}-md-win
   438    namespace: default
   439  spec:
   440    template:
   441      metadata:
   442        annotations:
   443          runtime: containerd
   444          windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""}
   445      spec:
   446        identity: UserAssigned
   447        image:
   448          marketplace:
   449            offer: capi-windows
   450            publisher: cncf-upstream
   451            sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1
   452            version: latest
   453        osDisk:
   454          diskSizeGB: 128
   455          managedDisk:
   456            storageAccountType: Premium_LRS
   457          osType: Windows
   458        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   459        userAssignedIdentities:
   460        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
   461        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   462  ---
   463  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   464  kind: KubeadmConfigTemplate
   465  metadata:
   466    name: ${CLUSTER_NAME}-md-win
   467    namespace: default
   468  spec:
   469    template:
   470      spec:
   471        files:
   472        - contentFrom:
   473            secret:
   474              key: worker-node-azure.json
   475              name: ${CLUSTER_NAME}-md-win-azure-json
   476          owner: root:root
   477          path: c:/k/azure.json
   478          permissions: "0644"
   479        - content: |-
   480            Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe
   481            Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe
   482          path: C:/defender-exclude-calico.ps1
   483          permissions: "0744"
   484        - content: |
   485            # /tmp is assumed created and required for upstream e2e tests to pass
   486            New-Item -ItemType Directory -Force -Path C:\tmp\
   487          path: C:/create-temp-folder.ps1
   488          permissions: "0744"
   489        - content: |
   490            $ErrorActionPreference = 'Stop'
   491  
   492            $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}"
   493            if($$CONTAINERD_URL -ne ""){
   494              # Kubelet service depends on contianerd service so make a best effort attempt to stop it
   495              Stop-Service kubelet -Force -ErrorAction SilentlyContinue
   496              Stop-Service containerd -Force
   497              echo "downloading containerd: $$CONTAINERD_URL"
   498              curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz"
   499              tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1
   500  
   501              Start-Service containerd
   502            }
   503  
   504            containerd.exe --version
   505            containerd-shim-runhcs-v1.exe --version
   506          path: C:/replace-containerd.ps1
   507          permissions: "0744"
   508        - content: |
   509            mkdir -Force c:/localdumps
   510            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f
   511            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f
   512            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f
   513            # Enable sftp so we can copy crash dump files during log collection of stfp
   514            $sshd_config = "$env:ProgramData\ssh\sshd_config"
   515            if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config }
   516            Add-Content -Path $sshd_config "Subsystem sftp              sftp-server.exe"
   517            sc.exe stop sshd
   518            sc.exe start sshd
   519          path: C:/collect-hns-crashes.ps1
   520          permissions: "0744"
   521        - content: |
   522            $ErrorActionPreference = 'Stop'
   523  
   524            echo "Use OOT credential provider"
   525            mkdir C:\var\lib\kubelet\credential-provider
   526            curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe
   527            cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider
   528            curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml
   529          path: C:/oot-cred-provider.ps1
   530          permissions: "0744"
   531        - content: |
   532            $ErrorActionPreference = 'Stop'
   533  
   534            Stop-Service kubelet -Force
   535  
   536            $$CI_VERSION="${CI_VERSION}"
   537            if($$CI_VERSION -ne "")
   538            {
   539              $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy")
   540              $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64"
   541              foreach ( $$binary in $$binaries )
   542              {
   543                echo "downloading binary: $$ci_url/$$binary.exe"
   544                curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe"
   545              }
   546            }
   547  
   548            # Tag it to the ci version.  The image knows how to use the copy locally with the configmap
   549            # that is applied at at this stage (windows-kubeproxy-ci.yaml)
   550            ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess
   551            ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess"
   552  
   553            kubeadm.exe version -o=short
   554            kubectl.exe version --client=true
   555            kubelet.exe --version
   556            kube-proxy.exe --version
   557          path: C:/replace-ci-binaries.ps1
   558          permissions: "0744"
   559        joinConfiguration:
   560          nodeRegistration:
   561            criSocket: npipe:////./pipe/containerd-containerd
   562            kubeletExtraArgs:
   563              cloud-provider: external
   564              feature-gates: ${NODE_FEATURE_GATES:-""}
   565              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   566              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   567              v: "2"
   568              windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS
   569            name: '{{ ds.meta_data["local_hostname"] }}'
   570        postKubeadmCommands:
   571        - nssm set kubelet start SERVICE_AUTO_START
   572        - powershell C:/defender-exclude-calico.ps1
   573        preKubeadmCommands:
   574        - powershell C:/create-temp-folder.ps1
   575        - powershell C:/replace-containerd.ps1
   576        - powershell C:/collect-hns-crashes.ps1
   577        - powershell C:/oot-cred-provider.ps1
   578        - powershell C:/replace-ci-binaries.ps1
   579        users:
   580        - groups: Administrators
   581          name: capi
   582          sshAuthorizedKeys:
   583          - ${AZURE_SSH_PUBLIC_KEY:=""}
   584  ---
   585  apiVersion: cluster.x-k8s.io/v1beta1
   586  kind: MachineHealthCheck
   587  metadata:
   588    name: ${CLUSTER_NAME}-mhc-0
   589    namespace: default
   590  spec:
   591    clusterName: ${CLUSTER_NAME}
   592    maxUnhealthy: 100%
   593    selector:
   594      matchLabels:
   595        nodepool: pool1
   596    unhealthyConditions:
   597    - status: "True"
   598      timeout: 30s
   599      type: E2ENodeUnhealthy
   600  ---
   601  apiVersion: addons.cluster.x-k8s.io/v1beta1
   602  kind: ClusterResourceSet
   603  metadata:
   604    name: ${CLUSTER_NAME}-calico-windows
   605    namespace: default
   606  spec:
   607    clusterSelector:
   608      matchLabels:
   609        cni-windows: ${CLUSTER_NAME}-calico
   610    resources:
   611    - kind: ConfigMap
   612      name: cni-${CLUSTER_NAME}-calico-windows
   613    strategy: ApplyOnce
   614  ---
   615  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   616  kind: AzureClusterIdentity
   617  metadata:
   618    labels:
   619      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   620    name: ${CLUSTER_IDENTITY_NAME}
   621    namespace: default
   622  spec:
   623    allowedNamespaces: {}
   624    clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}
   625    tenantID: ${AZURE_TENANT_ID}
   626    type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity}
   627  ---
   628  apiVersion: addons.cluster.x-k8s.io/v1beta1
   629  kind: ClusterResourceSet
   630  metadata:
   631    name: csi-proxy
   632    namespace: default
   633  spec:
   634    clusterSelector:
   635      matchLabels:
   636        csi-proxy: enabled
   637    resources:
   638    - kind: ConfigMap
   639      name: csi-proxy-addon
   640    strategy: ApplyOnce
   641  ---
   642  apiVersion: addons.cluster.x-k8s.io/v1beta1
   643  kind: ClusterResourceSet
   644  metadata:
   645    name: containerd-logger-${CLUSTER_NAME}
   646    namespace: default
   647  spec:
   648    clusterSelector:
   649      matchLabels:
   650        containerd-logger: enabled
   651    resources:
   652    - kind: ConfigMap
   653      name: containerd-logger-${CLUSTER_NAME}
   654    strategy: ApplyOnce
   655  ---
   656  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   657  kind: HelmChartProxy
   658  metadata:
   659    name: calico
   660    namespace: default
   661  spec:
   662    chartName: tigera-operator
   663    clusterSelector:
   664      matchLabels:
   665        cni: calico
   666    namespace: tigera-operator
   667    releaseName: projectcalico
   668    repoURL: https://docs.tigera.io/calico/charts
   669    valuesTemplate: |-
   670      installation:
   671        cni:
   672          type: Calico
   673        calicoNetwork:
   674          bgp: Disabled
   675          mtu: 1350
   676          ipPools:
   677          ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
   678          - cidr: {{ $cidr }}
   679            encapsulation: VXLAN{{end}}
   680        registry: mcr.microsoft.com/oss
   681      # Image and registry configuration for the tigera/operator pod.
   682      tigeraOperator:
   683        image: tigera/operator
   684        registry: mcr.microsoft.com/oss
   685      calicoctl:
   686        image: mcr.microsoft.com/oss/calico/ctl
   687    version: ${CALICO_VERSION}
   688  ---
   689  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   690  kind: HelmChartProxy
   691  metadata:
   692    name: azuredisk-csi-driver-chart
   693    namespace: default
   694  spec:
   695    chartName: azuredisk-csi-driver
   696    clusterSelector:
   697      matchLabels:
   698        azuredisk-csi: "true"
   699    namespace: kube-system
   700    releaseName: azuredisk-csi-driver-oot
   701    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   702    valuesTemplate: |-
   703      controller:
   704        replicas: 1
   705        runOnControlPlane: true
   706      windows:
   707        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   708  ---
   709  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   710  kind: HelmChartProxy
   711  metadata:
   712    name: cloud-provider-azure-chart
   713    namespace: default
   714  spec:
   715    chartName: cloud-provider-azure
   716    clusterSelector:
   717      matchLabels:
   718        cloud-provider: azure
   719    releaseName: cloud-provider-azure-oot
   720    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   721    valuesTemplate: |
   722      infra:
   723        clusterName: {{ .Cluster.metadata.name }}
   724      cloudControllerManager:
   725        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   726        logVerbosity: 4
   727  ---
   728  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   729  kind: HelmChartProxy
   730  metadata:
   731    name: cloud-provider-azure-chart-ci
   732    namespace: default
   733  spec:
   734    chartName: cloud-provider-azure
   735    clusterSelector:
   736      matchLabels:
   737        cloud-provider: azure-ci
   738    releaseName: cloud-provider-azure-oot
   739    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   740    valuesTemplate: |
   741      infra:
   742        clusterName: {{ .Cluster.metadata.name }}
   743      cloudControllerManager:
   744        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   745        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   746        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   747        imageName: "${CCM_IMAGE_NAME:-""}"
   748        imageRepository: "${IMAGE_REGISTRY:-""}"
   749        imageTag: "${IMAGE_TAG_CCM:-""}"
   750        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   751        replicas: ${CCM_COUNT:-1}
   752        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   753      cloudNodeManager:
   754        imageName: "${CNM_IMAGE_NAME:-""}"
   755        imageRepository: "${IMAGE_REGISTRY:-""}"
   756        imageTag: "${IMAGE_TAG_CNM:-""}"
   757  ---
   758  apiVersion: v1
   759  data:
   760    kube-proxy-patch: |-
   761      apiVersion: v1
   762      kind: ConfigMap
   763      metadata:
   764        name: windows-kubeproxy-ci
   765        namespace: kube-system
   766      data:
   767        KUBEPROXY_PATH: "c:/k/kube-proxy.exe"
   768    proxy: |
   769      apiVersion: apps/v1
   770      kind: DaemonSet
   771      metadata:
   772        labels:
   773          k8s-app: kube-proxy
   774        name: kube-proxy-windows
   775        namespace: kube-system
   776      spec:
   777        selector:
   778          matchLabels:
   779            k8s-app: kube-proxy-windows
   780        template:
   781          metadata:
   782            labels:
   783              k8s-app: kube-proxy-windows
   784          spec:
   785            serviceAccountName: kube-proxy
   786            securityContext:
   787              windowsOptions:
   788                hostProcess: true
   789                runAsUserName: "NT AUTHORITY\\system"
   790            hostNetwork: true
   791            containers:
   792            - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess
   793              args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"]
   794              workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/"
   795              name: kube-proxy
   796              env:
   797              - name: NODE_NAME
   798                valueFrom:
   799                  fieldRef:
   800                    apiVersion: v1
   801                    fieldPath: spec.nodeName
   802              - name: POD_IP
   803                valueFrom:
   804                  fieldRef:
   805                    fieldPath: status.podIP
   806              - name: KUBEPROXY_PATH
   807                valueFrom:
   808                  configMapKeyRef:
   809                    name: windows-kubeproxy-ci
   810                    key: KUBEPROXY_PATH
   811                    optional: true
   812              volumeMounts:
   813              - mountPath: /var/lib/kube-proxy
   814                name: kube-proxy
   815            nodeSelector:
   816              kubernetes.io/os: windows
   817            tolerations:
   818            - key: CriticalAddonsOnly
   819              operator: Exists
   820            - operator: Exists
   821            volumes:
   822            - configMap:
   823                name: kube-proxy
   824              name: kube-proxy
   825        updateStrategy:
   826          type: RollingUpdate
   827    windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind:
   828      IPAMConfig\nmetadata:\n  name: default\nspec:\n  autoAllocateBlocks: true\n  strictAffinity:
   829      true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n  name: calico-static-rules\n
   830      \ namespace: calico-system\n  labels:\n    tier: node\n    app: calico\ndata:\n
   831      \ static-rules.json: |\n    {\n      \"Provider\": \"azure\",\n      \"Version\":
   832      \"0.1\",\n      \"Rules\": [\n        {\n          \"Name\": \"EndpointPolicy\",\n
   833      \         \"Rule\": {\n              \"Id\": \"wireserver\",\n              \"Type\":
   834      \"ACL\",\n              \"Protocol\": 6,\n              \"Action\": \"Block\",\n
   835      \             \"Direction\": \"Out\",\n              \"RemoteAddresses\": \"168.63.129.16/32\",\n
   836      \             \"RemotePorts\": \"80\",\n              \"Priority\": 200,\n              \"RuleType\":
   837      \"Switch\"\n            }\n          }\n      ]\n    } \n---\nkind: ConfigMap\napiVersion:
   838      v1\nmetadata:\n  name: calico-config-windows\n  namespace: calico-system\n  labels:\n
   839      \   tier: node\n    app: calico\ndata:\n  veth_mtu: \"1350\"\n  \n  cni_network_config:
   840      |\n    {\n      \"name\": \"Calico\",\n      \"cniVersion\": \"0.3.1\",\n      \"plugins\":
   841      [\n        {\n          \"windows_use_single_network\": true,\n          \"type\":
   842      \"calico\",\n          \"mode\": \"vxlan\",\n          \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n
   843      \         \"nodename_file_optional\": true,\n          \"log_file_path\": \"c:/cni.log\",\n
   844      \         \"log_level\": \"debug\",\n\n          \"vxlan_mac_prefix\": \"0E-2A\",\n
   845      \         \"vxlan_vni\": 4096,\n          \"mtu\": __CNI_MTU__,\n          \"policy\":
   846      {\n            \"type\": \"k8s\"\n          },\n\n          \"log_level\": \"info\",\n\n
   847      \         \"capabilities\": {\"dns\": true},\n          \"DNS\":  {\n            \"Search\":
   848      \ [\n              \"svc.cluster.local\"\n            ]\n          },\n\n          \"datastore_type\":
   849      \"kubernetes\",\n\n          \"kubernetes\": {\n            \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n
   850      \         },\n\n          \"ipam\": {\n            \"type\": \"calico-ipam\",\n
   851      \           \"subnet\": \"usePodCidr\"\n          },\n\n          \"policies\":
   852      \ [\n            {\n              \"Name\":  \"EndpointPolicy\",\n              \"Value\":
   853      \ {\n                \"Type\":  \"OutBoundNAT\",\n                \"ExceptionList\":
   854      \ [\n                  \"__K8S_SERVICE_CIDR__\"\n                ]\n              }\n
   855      \           },\n            {\n              \"Name\":  \"EndpointPolicy\",\n
   856      \             \"Value\":  {\n                \"Type\":  \"SDNROUTE\",\n                \"DestinationPrefix\":
   857      \ \"__K8S_SERVICE_CIDR__\",\n                \"NeedEncap\":  true\n              }\n
   858      \           }\n          ]\n        }\n      ]\n\n    }\n---\napiVersion: apps/v1\nkind:
   859      DaemonSet\nmetadata:\n  name: calico-node-windows\n  labels:\n    tier: node\n
   860      \   app: calico\n  namespace: calico-system\nspec:\n  selector:\n    matchLabels:\n
   861      \     app: calico\n  template:\n    metadata:\n      labels:\n        tier: node\n
   862      \       app: calico\n    spec:\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n
   863      \           nodeSelectorTerms:\n              - matchExpressions:\n                  -
   864      key: kubernetes.io/os\n                    operator: In\n                    values:\n
   865      \                     - windows\n                  - key: kubernetes.io/arch\n
   866      \                   operator: In\n                    values:\n                      -
   867      amd64\n      securityContext:\n        windowsOptions:\n          hostProcess:
   868      true\n          runAsUserName: \"NT AUTHORITY\\\\system\"\n      hostNetwork:
   869      true\n      serviceAccountName: calico-node\n      tolerations:\n      - operator:
   870      Exists\n        effect: NoSchedule\n        # Mark the pod as a critical add-on
   871      for rescheduling.\n      - key: CriticalAddonsOnly\n        operator: Exists\n
   872      \     - effect: NoExecute\n        operator: Exists\n      initContainers:\n        #
   873      This container installs the CNI binaries\n        # and CNI network config file
   874      on each node.\n        - name: install-cni\n          image: sigwindowstools/calico-install:v3.26.1-hostprocess\n
   875      \         args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n
   876      \         imagePullPolicy: Always\n          env:\n            # Name of the CNI
   877      config file to create.\n            - name: CNI_CONF_NAME\n              value:
   878      \"10-calico.conflist\"\n            # The CNI network config to install on each
   879      node.\n            - name: CNI_NETWORK_CONFIG\n              valueFrom:\n                configMapKeyRef:\n
   880      \                 name: calico-config-windows\n                  key: cni_network_config\n
   881      \           # Set the hostname based on the k8s node name.\n            - name:
   882      KUBERNETES_NODE_NAME\n              valueFrom:\n                fieldRef:\n                  fieldPath:
   883      spec.nodeName\n            # CNI MTU Config variable\n            - name: CNI_MTU\n
   884      \             valueFrom:\n                configMapKeyRef:\n                  name:
   885      calico-config-windows\n                  key: veth_mtu\n            # Prevents
   886      the container from sleeping forever.\n            - name: SLEEP\n              value:
   887      \"false\"\n            - name: K8S_SERVICE_CIDR\n              value: \"10.96.0.0/12\"\n
   888      \         volumeMounts:\n            - mountPath: /host/opt/cni/bin\n              name:
   889      cni-bin-dir\n            - mountPath: /host/etc/cni/net.d\n              name:
   890      cni-net-dir\n            - name: kubeadm-config\n              mountPath: /etc/kubeadm-config/\n
   891      \         securityContext:\n            windowsOptions:\n              hostProcess:
   892      true\n              runAsUserName: \"NT AUTHORITY\\\\system\"\n      containers:\n
   893      \     - name: calico-node-startup\n        image: sigwindowstools/calico-node:v3.26.1-hostprocess\n
   894      \       args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n
   895      \       workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        imagePullPolicy:
   896      Always\n        volumeMounts:\n        - name: calico-config-windows\n          mountPath:
   897      /etc/kube-calico-windows/\n        env:\n        - name: POD_NAME\n          valueFrom:\n
   898      \           fieldRef:\n              apiVersion: v1\n              fieldPath:
   899      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   900      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   901      name: CNI_IPAM_TYPE\n          value: \"calico-ipam\"\n        - name: CALICO_NETWORKING_BACKEND\n
   902      \         value: \"vxlan\"\n        - name: KUBECONFIG\n          value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n
   903      \       - name: VXLAN_VNI\n          value: \"4096\"\n      - name: calico-node-felix\n
   904      \       image: sigwindowstools/calico-node:v3.26.1-hostprocess\n        args:
   905      [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n        imagePullPolicy:
   906      Always\n        workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        volumeMounts:\n
   907      \       - name: calico-config-windows\n          mountPath: /etc/kube-calico-windows/\n
   908      \       - name: calico-static-rules\n          mountPath: /calico/static-rules.json\n
   909      \         subPath: static-rules.json\n        env:\n        - name: POD_NAME\n
   910      \         valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath:
   911      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   912      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   913      name: VXLAN_VNI\n          value: \"4096\"\n        - name: KUBECONFIG\n          value:
   914      \"C:/etc/cni/net.d/calico-kubeconfig\"\n      volumes:\n      - name: calico-config-windows\n
   915      \       configMap:\n          name: calico-config-windows\n      - name: calico-static-rules\n
   916      \       configMap:\n          name: calico-static-rules\n      # Used to install
   917      CNI.\n      - name: cni-bin-dir\n        hostPath:\n          path: /opt/cni/bin\n
   918      \     - name: cni-net-dir\n        hostPath:\n          path: /etc/cni/net.d\n
   919      \     - name: kubeadm-config\n        configMap:\n          name: kubeadm-config\n---\napiVersion:
   920      apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: ipamconfigs.crd.projectcalico.org\nspec:\n
   921      \ group: crd.projectcalico.org\n  names:\n    kind: IPAMConfig\n    listKind:
   922      IPAMConfigList\n    plural: ipamconfigs\n    singular: ipamconfig\n  preserveUnknownFields:
   923      false\n  scope: Cluster\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n
   924      \       properties:\n          apiVersion:\n            description: 'APIVersion
   925      defines the versioned schema of this representation\n              of an object.
   926      Servers should convert recognized schemas to the latest\n              internal
   927      value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n
   928      \           type: string\n          kind:\n            description: 'Kind is a
   929      string value representing the REST resource this\n              object represents.
   930      Servers may infer this from the endpoint the client\n              submits requests
   931      to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n
   932      \           type: string\n          metadata:\n            type: object\n          spec:\n
   933      \           description: IPAMConfigSpec contains the specification for an IPAMConfig\n
   934      \             resource.\n            properties:\n              autoAllocateBlocks:\n
   935      \               type: boolean\n              maxBlocksPerHost:\n                description:
   936      MaxBlocksPerHost, if non-zero, is the max number of blocks\n                  that
   937      can be affine to each host.\n                maximum: 2147483647\n                minimum:
   938      0\n                type: integer\n              strictAffinity:\n                type:
   939      boolean\n            required:\n            - autoAllocateBlocks\n            -
   940      strictAffinity\n            type: object\n        type: object\n    served: true\n
   941      \   storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n
   942      \ conditions: []\n  storedVersions: []\n"
   943  kind: ConfigMap
   944  metadata:
   945    annotations:
   946      note: generated
   947    labels:
   948      type: generated
   949    name: cni-${CLUSTER_NAME}-calico-windows
   950    namespace: default
   951  ---
   952  apiVersion: v1
   953  data:
   954    csi-proxy: |
   955      apiVersion: apps/v1
   956      kind: DaemonSet
   957      metadata:
   958        labels:
   959          k8s-app: csi-proxy
   960        name: csi-proxy
   961        namespace: kube-system
   962      spec:
   963        selector:
   964          matchLabels:
   965            k8s-app: csi-proxy
   966        template:
   967          metadata:
   968            labels:
   969              k8s-app: csi-proxy
   970          spec:
   971            nodeSelector:
   972              "kubernetes.io/os": windows
   973            securityContext:
   974              windowsOptions:
   975                hostProcess: true
   976                runAsUserName: "NT AUTHORITY\\SYSTEM"
   977            hostNetwork: true
   978            containers:
   979              - name: csi-proxy
   980                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   981  kind: ConfigMap
   982  metadata:
   983    annotations:
   984      note: generated
   985    labels:
   986      type: generated
   987    name: csi-proxy-addon
   988    namespace: default
   989  ---
   990  apiVersion: v1
   991  data:
   992    containerd-windows-logger: |
   993      apiVersion: apps/v1
   994      kind: DaemonSet
   995      metadata:
   996        labels:
   997          k8s-app: containerd-logger
   998        name: containerd-logger
   999        namespace: kube-system
  1000      spec:
  1001        selector:
  1002          matchLabels:
  1003            k8s-app: containerd-logger
  1004        template:
  1005          metadata:
  1006            labels:
  1007              k8s-app: containerd-logger
  1008          spec:
  1009            securityContext:
  1010              windowsOptions:
  1011                hostProcess: true
  1012                runAsUserName: "NT AUTHORITY\\system"
  1013            hostNetwork: true
  1014            containers:
  1015            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
  1016              args: [ "config.json" ]
  1017              name: containerd-logger
  1018              imagePullPolicy: Always
  1019              volumeMounts:
  1020              - name: containerd-logger-config
  1021                mountPath: /config.json
  1022                subPath: config.json
  1023            nodeSelector:
  1024              kubernetes.io/os: windows
  1025            tolerations:
  1026            - key: CriticalAddonsOnly
  1027              operator: Exists
  1028            - operator: Exists
  1029            volumes:
  1030            - configMap:
  1031                name: containerd-logger-config
  1032              name: containerd-logger-config
  1033        updateStrategy:
  1034          type: RollingUpdate
  1035      ---
  1036      kind: ConfigMap
  1037      apiVersion: v1
  1038      metadata:
  1039        name: containerd-logger-config
  1040        namespace: kube-system
  1041      data:
  1042        config.json: |
  1043          {
  1044            "inputs": [
  1045              {
  1046                "type": "ETW",
  1047                "sessionNamePrefix": "containerd",
  1048                "cleanupOldSessions": true,
  1049                "reuseExistingSession": true,
  1050                "providers": [
  1051                  {
  1052                    "providerName": "Microsoft.Virtualization.RunHCS",
  1053                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
  1054                    "level": "Verbose"
  1055                  },
  1056                  {
  1057                    "providerName": "ContainerD",
  1058                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
  1059                    "level": "Verbose"
  1060                  }
  1061                ]
  1062              }
  1063            ],
  1064            "filters": [
  1065              {
  1066                  "type": "drop",
  1067                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
  1068              },
  1069              {
  1070                  "type": "drop",
  1071                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
  1072              },
  1073              {
  1074                  "type": "drop",
  1075                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
  1076              },
  1077              {
  1078                  "type": "drop",
  1079                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
  1080              },
  1081              {
  1082                  "type": "drop",
  1083                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
  1084              },
  1085              {
  1086                  "type": "drop",
  1087                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
  1088              },
  1089              {
  1090                  "type": "drop",
  1091                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
  1092              }
  1093            ],
  1094            "outputs": [
  1095              {
  1096                "type": "StdOutput"
  1097              }
  1098            ],
  1099            "schemaVersion": "2016-08-11"
  1100          }
  1101  kind: ConfigMap
  1102  metadata:
  1103    annotations:
  1104      note: generated
  1105    labels:
  1106      type: generated
  1107    name: containerd-logger-${CLUSTER_NAME}
  1108    namespace: default
  1109  ---
  1110  apiVersion: addons.cluster.x-k8s.io/v1beta1
  1111  kind: ClusterResourceSet
  1112  metadata:
  1113    name: metrics-server-${CLUSTER_NAME}
  1114    namespace: default
  1115  spec:
  1116    clusterSelector:
  1117      matchLabels:
  1118        metrics-server: enabled
  1119    resources:
  1120    - kind: ConfigMap
  1121      name: metrics-server-${CLUSTER_NAME}
  1122    strategy: ApplyOnce
  1123  ---
  1124  apiVersion: v1
  1125  data:
  1126    metrics-server: |
  1127      apiVersion: v1
  1128      kind: ServiceAccount
  1129      metadata:
  1130        labels:
  1131          k8s-app: metrics-server
  1132        name: metrics-server
  1133        namespace: kube-system
  1134      ---
  1135      apiVersion: rbac.authorization.k8s.io/v1
  1136      kind: ClusterRole
  1137      metadata:
  1138        labels:
  1139          k8s-app: metrics-server
  1140          rbac.authorization.k8s.io/aggregate-to-admin: "true"
  1141          rbac.authorization.k8s.io/aggregate-to-edit: "true"
  1142          rbac.authorization.k8s.io/aggregate-to-view: "true"
  1143        name: system:aggregated-metrics-reader
  1144      rules:
  1145      - apiGroups:
  1146        - metrics.k8s.io
  1147        resources:
  1148        - pods
  1149        - nodes
  1150        verbs:
  1151        - get
  1152        - list
  1153        - watch
  1154      ---
  1155      apiVersion: rbac.authorization.k8s.io/v1
  1156      kind: ClusterRole
  1157      metadata:
  1158        labels:
  1159          k8s-app: metrics-server
  1160        name: system:metrics-server
  1161      rules:
  1162      - apiGroups:
  1163        - ""
  1164        resources:
  1165        - nodes/metrics
  1166        verbs:
  1167        - get
  1168      - apiGroups:
  1169        - ""
  1170        resources:
  1171        - pods
  1172        - nodes
  1173        verbs:
  1174        - get
  1175        - list
  1176        - watch
  1177      ---
  1178      apiVersion: rbac.authorization.k8s.io/v1
  1179      kind: RoleBinding
  1180      metadata:
  1181        labels:
  1182          k8s-app: metrics-server
  1183        name: metrics-server-auth-reader
  1184        namespace: kube-system
  1185      roleRef:
  1186        apiGroup: rbac.authorization.k8s.io
  1187        kind: Role
  1188        name: extension-apiserver-authentication-reader
  1189      subjects:
  1190      - kind: ServiceAccount
  1191        name: metrics-server
  1192        namespace: kube-system
  1193      ---
  1194      apiVersion: rbac.authorization.k8s.io/v1
  1195      kind: ClusterRoleBinding
  1196      metadata:
  1197        labels:
  1198          k8s-app: metrics-server
  1199        name: metrics-server:system:auth-delegator
  1200      roleRef:
  1201        apiGroup: rbac.authorization.k8s.io
  1202        kind: ClusterRole
  1203        name: system:auth-delegator
  1204      subjects:
  1205      - kind: ServiceAccount
  1206        name: metrics-server
  1207        namespace: kube-system
  1208      ---
  1209      apiVersion: rbac.authorization.k8s.io/v1
  1210      kind: ClusterRoleBinding
  1211      metadata:
  1212        labels:
  1213          k8s-app: metrics-server
  1214        name: system:metrics-server
  1215      roleRef:
  1216        apiGroup: rbac.authorization.k8s.io
  1217        kind: ClusterRole
  1218        name: system:metrics-server
  1219      subjects:
  1220      - kind: ServiceAccount
  1221        name: metrics-server
  1222        namespace: kube-system
  1223      ---
  1224      apiVersion: v1
  1225      kind: Service
  1226      metadata:
  1227        labels:
  1228          k8s-app: metrics-server
  1229        name: metrics-server
  1230        namespace: kube-system
  1231      spec:
  1232        ports:
  1233        - name: https
  1234          port: 443
  1235          protocol: TCP
  1236          targetPort: https
  1237        selector:
  1238          k8s-app: metrics-server
  1239      ---
  1240      apiVersion: apps/v1
  1241      kind: Deployment
  1242      metadata:
  1243        labels:
  1244          k8s-app: metrics-server
  1245        name: metrics-server
  1246        namespace: kube-system
  1247      spec:
  1248        selector:
  1249          matchLabels:
  1250            k8s-app: metrics-server
  1251        strategy:
  1252          rollingUpdate:
  1253            maxUnavailable: 0
  1254        template:
  1255          metadata:
  1256            labels:
  1257              k8s-app: metrics-server
  1258          spec:
  1259            containers:
  1260            - args:
  1261              - --cert-dir=/tmp
  1262              - --secure-port=4443
  1263              - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
  1264              - --kubelet-use-node-status-port
  1265              - --metric-resolution=15s
  1266              - --kubelet-insecure-tls
  1267              image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
  1268              imagePullPolicy: IfNotPresent
  1269              livenessProbe:
  1270                failureThreshold: 3
  1271                httpGet:
  1272                  path: /livez
  1273                  port: https
  1274                  scheme: HTTPS
  1275                periodSeconds: 10
  1276              name: metrics-server
  1277              ports:
  1278              - containerPort: 4443
  1279                name: https
  1280                protocol: TCP
  1281              readinessProbe:
  1282                failureThreshold: 3
  1283                httpGet:
  1284                  path: /readyz
  1285                  port: https
  1286                  scheme: HTTPS
  1287                initialDelaySeconds: 20
  1288                periodSeconds: 10
  1289              resources:
  1290                requests:
  1291                  cpu: 100m
  1292                  memory: 200Mi
  1293              securityContext:
  1294                allowPrivilegeEscalation: false
  1295                readOnlyRootFilesystem: true
  1296                runAsNonRoot: true
  1297                runAsUser: 1000
  1298              volumeMounts:
  1299              - mountPath: /tmp
  1300                name: tmp-dir
  1301            nodeSelector:
  1302              kubernetes.io/os: linux
  1303            priorityClassName: system-cluster-critical
  1304            serviceAccountName: metrics-server
  1305            tolerations:
  1306            - effect: NoSchedule
  1307              key: node-role.kubernetes.io/master
  1308              operator: Exists
  1309            - effect: NoSchedule
  1310              key: node-role.kubernetes.io/control-plane
  1311              operator: Exists
  1312            volumes:
  1313            - emptyDir: {}
  1314              name: tmp-dir
  1315      ---
  1316      apiVersion: apiregistration.k8s.io/v1
  1317      kind: APIService
  1318      metadata:
  1319        labels:
  1320          k8s-app: metrics-server
  1321        name: v1beta1.metrics.k8s.io
  1322      spec:
  1323        group: metrics.k8s.io
  1324        groupPriorityMinimum: 100
  1325        insecureSkipTLSVerify: true
  1326        service:
  1327          name: metrics-server
  1328          namespace: kube-system
  1329        version: v1beta1
  1330        versionPriority: 100
  1331  kind: ConfigMap
  1332  metadata:
  1333    annotations:
  1334      note: generated
  1335    labels:
  1336      type: generated
  1337    name: metrics-server-${CLUSTER_NAME}
  1338    namespace: default