sigs.k8s.io/cluster-api-provider-azure@v1.14.3/templates/test/ci/cluster-template-prow-ci-version.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: enabled
     9      csi-proxy: enabled
    10      metrics-server: enabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 192.168.0.0/16
    18    controlPlaneRef:
    19      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    20      kind: KubeadmControlPlane
    21      name: ${CLUSTER_NAME}-control-plane
    22    infrastructureRef:
    23      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    24      kind: AzureCluster
    25      name: ${CLUSTER_NAME}
    26  ---
    27  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    28  kind: AzureCluster
    29  metadata:
    30    name: ${CLUSTER_NAME}
    31    namespace: default
    32  spec:
    33    additionalTags:
    34      buildProvenance: ${BUILD_PROVENANCE}
    35      creationTimestamp: ${TIMESTAMP}
    36      jobName: ${JOB_NAME}
    37    identityRef:
    38      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    39      kind: AzureClusterIdentity
    40      name: ${CLUSTER_IDENTITY_NAME}
    41    location: ${AZURE_LOCATION}
    42    networkSpec:
    43      subnets:
    44      - name: control-plane-subnet
    45        role: control-plane
    46      - name: node-subnet
    47        role: node
    48      vnet:
    49        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    50    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    51    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    52  ---
    53  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    54  kind: KubeadmControlPlane
    55  metadata:
    56    name: ${CLUSTER_NAME}-control-plane
    57    namespace: default
    58  spec:
    59    kubeadmConfigSpec:
    60      clusterConfiguration:
    61        apiServer:
    62          extraArgs:
    63            cloud-provider: external
    64            feature-gates: ${K8S_FEATURE_GATES:-""}
    65          timeoutForControlPlane: 20m
    66        controllerManager:
    67          extraArgs:
    68            allocate-node-cidrs: "false"
    69            cloud-provider: external
    70            cluster-name: ${CLUSTER_NAME}
    71            feature-gates: HPAContainerMetrics=true
    72            v: "4"
    73        etcd:
    74          local:
    75            dataDir: /var/lib/etcddisk/etcd
    76            extraArgs:
    77              quota-backend-bytes: "8589934592"
    78        kubernetesVersion: ci/${CI_VERSION}
    79      diskSetup:
    80        filesystems:
    81        - device: /dev/disk/azure/scsi1/lun0
    82          extraOpts:
    83          - -E
    84          - lazy_itable_init=1,lazy_journal_init=1
    85          filesystem: ext4
    86          label: etcd_disk
    87        - device: ephemeral0.1
    88          filesystem: ext4
    89          label: ephemeral0
    90          replaceFS: ntfs
    91        partitions:
    92        - device: /dev/disk/azure/scsi1/lun0
    93          layout: true
    94          overwrite: false
    95          tableType: gpt
    96      files:
    97      - contentFrom:
    98          secret:
    99            key: control-plane-azure.json
   100            name: ${CLUSTER_NAME}-control-plane-azure-json
   101        owner: root:root
   102        path: /etc/kubernetes/azure.json
   103        permissions: "0644"
   104      - content: |
   105          #!/bin/bash
   106  
   107          set -o nounset
   108          set -o pipefail
   109          set -o errexit
   110          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   111  
   112          az login --identity
   113          echo "Use OOT credential provider"
   114          mkdir -p /var/lib/kubelet/credential-provider
   115          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   116          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   117          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   118          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   119        owner: root:root
   120        path: /tmp/oot-cred-provider.sh
   121        permissions: "0744"
   122      - content: |
   123          #!/bin/bash
   124  
   125          set -o nounset
   126          set -o pipefail
   127          set -o errexit
   128          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   129  
   130          # This test installs release packages or binaries that are a result of the CI and release builds.
   131          # It runs '... --version' commands to verify that the binaries are correctly installed
   132          # and finally uninstalls the packages.
   133          # For the release packages it tests all versions in the support skew.
   134          LINE_SEPARATOR="*************************************************"
   135          echo "$$LINE_SEPARATOR"
   136          CI_VERSION=${CI_VERSION}
   137          if [[ "$${CI_VERSION}" != "" ]]; then
   138            CI_DIR=/tmp/k8s-ci
   139            mkdir -p $$CI_DIR
   140            declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   141            declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   142            CONTAINER_EXT="tar"
   143            echo "* testing CI version $$CI_VERSION"
   144            # Check for semver
   145            if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   146              VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   147              DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   148              curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   149              echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   150              apt-get update
   151              # replace . with \.
   152              VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   153              PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   154              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   155                echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   156                DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   157              done
   158            else
   159              CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   160              for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   161                echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   162                wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   163                chmod +x "$$CI_DIR/$$CI_PACKAGE"
   164                mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   165              done
   166              IMAGE_REGISTRY_PREFIX=registry.k8s.io
   167              for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   168                echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   169                wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   170                $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   171                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   172                $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   173              done
   174            fi
   175            systemctl restart kubelet
   176          fi
   177          echo "* checking binary versions"
   178          echo "ctr version: " $(ctr version)
   179          echo "kubeadm version: " $(kubeadm version -o=short)
   180          echo "kubectl version: " $(kubectl version --client=true)
   181          echo "kubelet version: " $(kubelet --version)
   182          echo "$$LINE_SEPARATOR"
   183        owner: root:root
   184        path: /tmp/kubeadm-bootstrap.sh
   185        permissions: "0744"
   186      initConfiguration:
   187        nodeRegistration:
   188          kubeletExtraArgs:
   189            cloud-provider: external
   190            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   191            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   192          name: '{{ ds.meta_data["local_hostname"] }}'
   193      joinConfiguration:
   194        nodeRegistration:
   195          kubeletExtraArgs:
   196            cloud-provider: external
   197            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   198            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   199          name: '{{ ds.meta_data["local_hostname"] }}'
   200      mounts:
   201      - - LABEL=etcd_disk
   202        - /var/lib/etcddisk
   203      postKubeadmCommands: []
   204      preKubeadmCommands:
   205      - bash -c /tmp/oot-cred-provider.sh
   206      - bash -c /tmp/kubeadm-bootstrap.sh
   207      verbosity: 5
   208    machineTemplate:
   209      infrastructureRef:
   210        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   211        kind: AzureMachineTemplate
   212        name: ${CLUSTER_NAME}-control-plane
   213    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   214    version: ${KUBERNETES_VERSION}
   215  ---
   216  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   217  kind: AzureMachineTemplate
   218  metadata:
   219    name: ${CLUSTER_NAME}-control-plane
   220    namespace: default
   221  spec:
   222    template:
   223      spec:
   224        dataDisks:
   225        - diskSizeGB: 256
   226          lun: 0
   227          nameSuffix: etcddisk
   228        image:
   229          marketplace:
   230            offer: capi
   231            publisher: cncf-upstream
   232            sku: ubuntu-2204-gen1
   233            version: latest
   234        osDisk:
   235          diskSizeGB: 128
   236          osType: Linux
   237        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   238        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   239  ---
   240  apiVersion: cluster.x-k8s.io/v1beta1
   241  kind: MachineDeployment
   242  metadata:
   243    name: ${CLUSTER_NAME}-md-0
   244    namespace: default
   245  spec:
   246    clusterName: ${CLUSTER_NAME}
   247    replicas: ${WORKER_MACHINE_COUNT:=2}
   248    selector: {}
   249    template:
   250      metadata:
   251        labels:
   252          nodepool: pool1
   253      spec:
   254        bootstrap:
   255          configRef:
   256            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   257            kind: KubeadmConfigTemplate
   258            name: ${CLUSTER_NAME}-md-0
   259        clusterName: ${CLUSTER_NAME}
   260        infrastructureRef:
   261          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   262          kind: AzureMachineTemplate
   263          name: ${CLUSTER_NAME}-md-0
   264        version: ${KUBERNETES_VERSION}
   265  ---
   266  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   267  kind: AzureMachineTemplate
   268  metadata:
   269    name: ${CLUSTER_NAME}-md-0
   270    namespace: default
   271  spec:
   272    template:
   273      spec:
   274        image:
   275          marketplace:
   276            offer: capi
   277            publisher: cncf-upstream
   278            sku: ubuntu-2204-gen1
   279            version: latest
   280        osDisk:
   281          diskSizeGB: 128
   282          osType: Linux
   283        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   284        vmExtensions:
   285        - name: CustomScript
   286          protectedSettings:
   287            commandToExecute: |
   288              #!/bin/sh
   289              echo "This script is a no-op used for extension testing purposes ..."
   290              touch test_file
   291          publisher: Microsoft.Azure.Extensions
   292          version: "2.1"
   293        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   294  ---
   295  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   296  kind: KubeadmConfigTemplate
   297  metadata:
   298    name: ${CLUSTER_NAME}-md-0
   299    namespace: default
   300  spec:
   301    template:
   302      spec:
   303        files:
   304        - contentFrom:
   305            secret:
   306              key: worker-node-azure.json
   307              name: ${CLUSTER_NAME}-md-0-azure-json
   308          owner: root:root
   309          path: /etc/kubernetes/azure.json
   310          permissions: "0644"
   311        - content: |
   312            #!/bin/bash
   313  
   314            set -o nounset
   315            set -o pipefail
   316            set -o errexit
   317            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   318  
   319            az login --identity
   320            echo "Use OOT credential provider"
   321            mkdir -p /var/lib/kubelet/credential-provider
   322            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   323            chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   324            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   325            chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   326          owner: root:root
   327          path: /tmp/oot-cred-provider.sh
   328          permissions: "0744"
   329        - content: |
   330            #!/bin/bash
   331  
   332            set -o nounset
   333            set -o pipefail
   334            set -o errexit
   335            [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   336  
   337            # This test installs release packages or binaries that are a result of the CI and release builds.
   338            # It runs '... --version' commands to verify that the binaries are correctly installed
   339            # and finally uninstalls the packages.
   340            # For the release packages it tests all versions in the support skew.
   341            LINE_SEPARATOR="*************************************************"
   342            echo "$$LINE_SEPARATOR"
   343            CI_VERSION=${CI_VERSION}
   344            if [[ "$${CI_VERSION}" != "" ]]; then
   345              CI_DIR=/tmp/k8s-ci
   346              mkdir -p $$CI_DIR
   347              declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
   348              declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   349              CONTAINER_EXT="tar"
   350              echo "* testing CI version $$CI_VERSION"
   351              # Check for semver
   352              if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
   353                VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
   354                DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
   355                curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
   356                echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
   357                apt-get update
   358                # replace . with \.
   359                VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
   360                PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
   361                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   362                  echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
   363                  DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
   364                done
   365              else
   366                CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
   367                for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
   368                  echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
   369                  wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
   370                  chmod +x "$$CI_DIR/$$CI_PACKAGE"
   371                  mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
   372                done
   373                IMAGE_REGISTRY_PREFIX=registry.k8s.io
   374                for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
   375                  echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
   376                  wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
   377                  $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
   378                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   379                  $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
   380                done
   381              fi
   382              systemctl restart kubelet
   383            fi
   384            echo "* checking binary versions"
   385            echo "ctr version: " $(ctr version)
   386            echo "kubeadm version: " $(kubeadm version -o=short)
   387            echo "kubectl version: " $(kubectl version --client=true)
   388            echo "kubelet version: " $(kubelet --version)
   389            echo "$$LINE_SEPARATOR"
   390          owner: root:root
   391          path: /tmp/kubeadm-bootstrap.sh
   392          permissions: "0744"
   393        joinConfiguration:
   394          nodeRegistration:
   395            kubeletExtraArgs:
   396              cloud-provider: external
   397              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   398              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   399            name: '{{ ds.meta_data["local_hostname"] }}'
   400        preKubeadmCommands:
   401        - bash -c /tmp/oot-cred-provider.sh
   402        - bash -c /tmp/kubeadm-bootstrap.sh
   403        verbosity: 5
   404  ---
   405  apiVersion: cluster.x-k8s.io/v1beta1
   406  kind: MachineDeployment
   407  metadata:
   408    name: ${CLUSTER_NAME}-md-win
   409    namespace: default
   410  spec:
   411    clusterName: ${CLUSTER_NAME}
   412    replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0}
   413    selector: {}
   414    template:
   415      spec:
   416        bootstrap:
   417          configRef:
   418            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   419            kind: KubeadmConfigTemplate
   420            name: ${CLUSTER_NAME}-md-win
   421        clusterName: ${CLUSTER_NAME}
   422        infrastructureRef:
   423          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   424          kind: AzureMachineTemplate
   425          name: ${CLUSTER_NAME}-md-win
   426        version: ${KUBERNETES_VERSION}
   427  ---
   428  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   429  kind: AzureMachineTemplate
   430  metadata:
   431    annotations:
   432      runtime: containerd
   433    name: ${CLUSTER_NAME}-md-win
   434    namespace: default
   435  spec:
   436    template:
   437      metadata:
   438        annotations:
   439          runtime: containerd
   440          windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""}
   441      spec:
   442        image:
   443          marketplace:
   444            offer: capi-windows
   445            publisher: cncf-upstream
   446            sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1
   447            version: latest
   448        osDisk:
   449          diskSizeGB: 128
   450          managedDisk:
   451            storageAccountType: Premium_LRS
   452          osType: Windows
   453        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   454        vmSize: ${AZURE_NODE_MACHINE_TYPE}
   455  ---
   456  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   457  kind: KubeadmConfigTemplate
   458  metadata:
   459    name: ${CLUSTER_NAME}-md-win
   460    namespace: default
   461  spec:
   462    template:
   463      spec:
   464        files:
   465        - contentFrom:
   466            secret:
   467              key: worker-node-azure.json
   468              name: ${CLUSTER_NAME}-md-win-azure-json
   469          owner: root:root
   470          path: c:/k/azure.json
   471          permissions: "0644"
   472        - content: |-
   473            Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe
   474            Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe
   475          path: C:/defender-exclude-calico.ps1
   476          permissions: "0744"
   477        - content: |
   478            # /tmp is assumed created and required for upstream e2e tests to pass
   479            New-Item -ItemType Directory -Force -Path C:\tmp\
   480          path: C:/create-temp-folder.ps1
   481          permissions: "0744"
   482        - content: |
   483            $ErrorActionPreference = 'Stop'
   484  
   485            $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}"
   486            if($$CONTAINERD_URL -ne ""){
   487              # Kubelet service depends on contianerd service so make a best effort attempt to stop it
   488              Stop-Service kubelet -Force -ErrorAction SilentlyContinue
   489              Stop-Service containerd -Force
   490              echo "downloading containerd: $$CONTAINERD_URL"
   491              curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz"
   492              tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1
   493  
   494              Start-Service containerd
   495            }
   496  
   497            containerd.exe --version
   498            containerd-shim-runhcs-v1.exe --version
   499          path: C:/replace-containerd.ps1
   500          permissions: "0744"
   501        - content: |
   502            mkdir -Force c:/localdumps
   503            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f
   504            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f
   505            reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f
   506            # Enable sftp so we can copy crash dump files during log collection of stfp
   507            $sshd_config = "$env:ProgramData\ssh\sshd_config"
   508            if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config }
   509            Add-Content -Path $sshd_config "Subsystem sftp              sftp-server.exe"
   510            sc.exe stop sshd
   511            sc.exe start sshd
   512          path: C:/collect-hns-crashes.ps1
   513          permissions: "0744"
   514        - content: |
   515            $ErrorActionPreference = 'Stop'
   516  
   517            az login --identity
   518            echo "Use OOT credential provider"
   519            mkdir C:\var\lib\kubelet\credential-provider
   520            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login
   521            cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider
   522            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login
   523          path: C:/oot-cred-provider.ps1
   524          permissions: "0744"
   525        - content: |
   526            $ErrorActionPreference = 'Stop'
   527  
   528            Stop-Service kubelet -Force
   529  
   530            $$CI_VERSION="${CI_VERSION}"
   531            if($$CI_VERSION -ne "")
   532            {
   533              $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy")
   534              $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64"
   535              foreach ( $$binary in $$binaries )
   536              {
   537                echo "downloading binary: $$ci_url/$$binary.exe"
   538                curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe"
   539              }
   540            }
   541  
   542            # Tag it to the ci version.  The image knows how to use the copy locally with the configmap
   543            # that is applied at at this stage (windows-kubeproxy-ci.yaml)
   544            ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess
   545            ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess"
   546  
   547            kubeadm.exe version -o=short
   548            kubectl.exe version --client=true
   549            kubelet.exe --version
   550            kube-proxy.exe --version
   551          path: C:/replace-ci-binaries.ps1
   552          permissions: "0744"
   553        joinConfiguration:
   554          nodeRegistration:
   555            criSocket: npipe:////./pipe/containerd-containerd
   556            kubeletExtraArgs:
   557              cloud-provider: external
   558              feature-gates: ${NODE_FEATURE_GATES:-""}
   559              image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   560              image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   561              v: "2"
   562              windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS
   563            name: '{{ ds.meta_data["local_hostname"] }}'
   564        postKubeadmCommands:
   565        - nssm set kubelet start SERVICE_AUTO_START
   566        - powershell C:/defender-exclude-calico.ps1
   567        preKubeadmCommands:
   568        - powershell C:/create-temp-folder.ps1
   569        - powershell C:/replace-containerd.ps1
   570        - powershell C:/collect-hns-crashes.ps1
   571        - powershell C:/oot-cred-provider.ps1
   572        - powershell C:/replace-ci-binaries.ps1
   573        users:
   574        - groups: Administrators
   575          name: capi
   576          sshAuthorizedKeys:
   577          - ${AZURE_SSH_PUBLIC_KEY:=""}
   578  ---
   579  apiVersion: cluster.x-k8s.io/v1beta1
   580  kind: MachineHealthCheck
   581  metadata:
   582    name: ${CLUSTER_NAME}-mhc-0
   583    namespace: default
   584  spec:
   585    clusterName: ${CLUSTER_NAME}
   586    maxUnhealthy: 100%
   587    selector:
   588      matchLabels:
   589        nodepool: pool1
   590    unhealthyConditions:
   591    - status: "True"
   592      timeout: 30s
   593      type: E2ENodeUnhealthy
   594  ---
   595  apiVersion: addons.cluster.x-k8s.io/v1beta1
   596  kind: ClusterResourceSet
   597  metadata:
   598    name: ${CLUSTER_NAME}-calico-windows
   599    namespace: default
   600  spec:
   601    clusterSelector:
   602      matchLabels:
   603        cni-windows: ${CLUSTER_NAME}-calico
   604    resources:
   605    - kind: ConfigMap
   606      name: cni-${CLUSTER_NAME}-calico-windows
   607    strategy: ApplyOnce
   608  ---
   609  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   610  kind: AzureClusterIdentity
   611  metadata:
   612    labels:
   613      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   614    name: ${CLUSTER_IDENTITY_NAME}
   615    namespace: default
   616  spec:
   617    allowedNamespaces: {}
   618    clientID: ${AZURE_CLIENT_ID}
   619    clientSecret:
   620      name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME}
   621      namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
   622    tenantID: ${AZURE_TENANT_ID}
   623    type: ServicePrincipal
   624  ---
   625  apiVersion: addons.cluster.x-k8s.io/v1beta1
   626  kind: ClusterResourceSet
   627  metadata:
   628    name: csi-proxy
   629    namespace: default
   630  spec:
   631    clusterSelector:
   632      matchLabels:
   633        csi-proxy: enabled
   634    resources:
   635    - kind: ConfigMap
   636      name: csi-proxy-addon
   637    strategy: ApplyOnce
   638  ---
   639  apiVersion: addons.cluster.x-k8s.io/v1beta1
   640  kind: ClusterResourceSet
   641  metadata:
   642    name: containerd-logger-${CLUSTER_NAME}
   643    namespace: default
   644  spec:
   645    clusterSelector:
   646      matchLabels:
   647        containerd-logger: enabled
   648    resources:
   649    - kind: ConfigMap
   650      name: containerd-logger-${CLUSTER_NAME}
   651    strategy: ApplyOnce
   652  ---
   653  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   654  kind: HelmChartProxy
   655  metadata:
   656    name: calico
   657    namespace: default
   658  spec:
   659    chartName: tigera-operator
   660    clusterSelector:
   661      matchLabels:
   662        cni: calico
   663    namespace: tigera-operator
   664    releaseName: projectcalico
   665    repoURL: https://docs.tigera.io/calico/charts
   666    valuesTemplate: |-
   667      installation:
   668        cni:
   669          type: Calico
   670        calicoNetwork:
   671          bgp: Disabled
   672          mtu: 1350
   673          ipPools:
   674          ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
   675          - cidr: {{ $cidr }}
   676            encapsulation: VXLAN{{end}}
   677        registry: mcr.microsoft.com/oss
   678      # Image and registry configuration for the tigera/operator pod.
   679      tigeraOperator:
   680        image: tigera/operator
   681        registry: mcr.microsoft.com/oss
   682      calicoctl:
   683        image: mcr.microsoft.com/oss/calico/ctl
   684    version: ${CALICO_VERSION}
   685  ---
   686  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   687  kind: HelmChartProxy
   688  metadata:
   689    name: azuredisk-csi-driver-chart
   690    namespace: default
   691  spec:
   692    chartName: azuredisk-csi-driver
   693    clusterSelector:
   694      matchLabels:
   695        azuredisk-csi: "true"
   696    namespace: kube-system
   697    releaseName: azuredisk-csi-driver-oot
   698    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   699    valuesTemplate: |-
   700      controller:
   701        replicas: 1
   702        runOnControlPlane: true
   703      windows:
   704        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   705  ---
   706  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   707  kind: HelmChartProxy
   708  metadata:
   709    name: cloud-provider-azure-chart
   710    namespace: default
   711  spec:
   712    chartName: cloud-provider-azure
   713    clusterSelector:
   714      matchLabels:
   715        cloud-provider: azure
   716    releaseName: cloud-provider-azure-oot
   717    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   718    valuesTemplate: |
   719      infra:
   720        clusterName: {{ .Cluster.metadata.name }}
   721      cloudControllerManager:
   722        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   723        logVerbosity: 4
   724  ---
   725  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   726  kind: HelmChartProxy
   727  metadata:
   728    name: cloud-provider-azure-chart-ci
   729    namespace: default
   730  spec:
   731    chartName: cloud-provider-azure
   732    clusterSelector:
   733      matchLabels:
   734        cloud-provider: azure-ci
   735    releaseName: cloud-provider-azure-oot
   736    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   737    valuesTemplate: |
   738      infra:
   739        clusterName: {{ .Cluster.metadata.name }}
   740      cloudControllerManager:
   741        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   742        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   743        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   744        imageName: "${CCM_IMAGE_NAME:-""}"
   745        imageRepository: "${IMAGE_REGISTRY:-""}"
   746        imageTag: "${IMAGE_TAG_CCM:-""}"
   747        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   748        replicas: ${CCM_COUNT:-1}
   749        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   750      cloudNodeManager:
   751        imageName: "${CNM_IMAGE_NAME:-""}"
   752        imageRepository: "${IMAGE_REGISTRY:-""}"
   753        imageTag: "${IMAGE_TAG_CNM:-""}"
   754  ---
   755  apiVersion: v1
   756  data:
   757    kube-proxy-patch: |-
   758      apiVersion: v1
   759      kind: ConfigMap
   760      metadata:
   761        name: windows-kubeproxy-ci
   762        namespace: kube-system
   763      data:
   764        KUBEPROXY_PATH: "c:/k/kube-proxy.exe"
   765    proxy: |
   766      apiVersion: apps/v1
   767      kind: DaemonSet
   768      metadata:
   769        labels:
   770          k8s-app: kube-proxy
   771        name: kube-proxy-windows
   772        namespace: kube-system
   773      spec:
   774        selector:
   775          matchLabels:
   776            k8s-app: kube-proxy-windows
   777        template:
   778          metadata:
   779            labels:
   780              k8s-app: kube-proxy-windows
   781          spec:
   782            serviceAccountName: kube-proxy
   783            securityContext:
   784              windowsOptions:
   785                hostProcess: true
   786                runAsUserName: "NT AUTHORITY\\system"
   787            hostNetwork: true
   788            containers:
   789            - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess
   790              args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"]
   791              workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/"
   792              name: kube-proxy
   793              env:
   794              - name: NODE_NAME
   795                valueFrom:
   796                  fieldRef:
   797                    apiVersion: v1
   798                    fieldPath: spec.nodeName
   799              - name: POD_IP
   800                valueFrom:
   801                  fieldRef:
   802                    fieldPath: status.podIP
   803              - name: KUBEPROXY_PATH
   804                valueFrom:
   805                  configMapKeyRef:
   806                    name: windows-kubeproxy-ci
   807                    key: KUBEPROXY_PATH
   808                    optional: true
   809              volumeMounts:
   810              - mountPath: /var/lib/kube-proxy
   811                name: kube-proxy
   812            nodeSelector:
   813              kubernetes.io/os: windows
   814            tolerations:
   815            - key: CriticalAddonsOnly
   816              operator: Exists
   817            - operator: Exists
   818            volumes:
   819            - configMap:
   820                name: kube-proxy
   821              name: kube-proxy
   822        updateStrategy:
   823          type: RollingUpdate
   824    windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind:
   825      IPAMConfig\nmetadata:\n  name: default\nspec:\n  autoAllocateBlocks: true\n  strictAffinity:
   826      true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n  name: calico-static-rules\n
   827      \ namespace: calico-system\n  labels:\n    tier: node\n    app: calico\ndata:\n
   828      \ static-rules.json: |\n    {\n      \"Provider\": \"azure\",\n      \"Version\":
   829      \"0.1\",\n      \"Rules\": [\n        {\n          \"Name\": \"EndpointPolicy\",\n
   830      \         \"Rule\": {\n              \"Id\": \"wireserver\",\n              \"Type\":
   831      \"ACL\",\n              \"Protocol\": 6,\n              \"Action\": \"Block\",\n
   832      \             \"Direction\": \"Out\",\n              \"RemoteAddresses\": \"168.63.129.16/32\",\n
   833      \             \"RemotePorts\": \"80\",\n              \"Priority\": 200,\n              \"RuleType\":
   834      \"Switch\"\n            }\n          }\n      ]\n    } \n---\nkind: ConfigMap\napiVersion:
   835      v1\nmetadata:\n  name: calico-config-windows\n  namespace: calico-system\n  labels:\n
   836      \   tier: node\n    app: calico\ndata:\n  veth_mtu: \"1350\"\n  \n  cni_network_config:
   837      |\n    {\n      \"name\": \"Calico\",\n      \"cniVersion\": \"0.3.1\",\n      \"plugins\":
   838      [\n        {\n          \"windows_use_single_network\": true,\n          \"type\":
   839      \"calico\",\n          \"mode\": \"vxlan\",\n          \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n
   840      \         \"nodename_file_optional\": true,\n          \"log_file_path\": \"c:/cni.log\",\n
   841      \         \"log_level\": \"debug\",\n\n          \"vxlan_mac_prefix\": \"0E-2A\",\n
   842      \         \"vxlan_vni\": 4096,\n          \"mtu\": __CNI_MTU__,\n          \"policy\":
   843      {\n            \"type\": \"k8s\"\n          },\n\n          \"log_level\": \"info\",\n\n
   844      \         \"capabilities\": {\"dns\": true},\n          \"DNS\":  {\n            \"Search\":
   845      \ [\n              \"svc.cluster.local\"\n            ]\n          },\n\n          \"datastore_type\":
   846      \"kubernetes\",\n\n          \"kubernetes\": {\n            \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n
   847      \         },\n\n          \"ipam\": {\n            \"type\": \"calico-ipam\",\n
   848      \           \"subnet\": \"usePodCidr\"\n          },\n\n          \"policies\":
   849      \ [\n            {\n              \"Name\":  \"EndpointPolicy\",\n              \"Value\":
   850      \ {\n                \"Type\":  \"OutBoundNAT\",\n                \"ExceptionList\":
   851      \ [\n                  \"__K8S_SERVICE_CIDR__\"\n                ]\n              }\n
   852      \           },\n            {\n              \"Name\":  \"EndpointPolicy\",\n
   853      \             \"Value\":  {\n                \"Type\":  \"SDNROUTE\",\n                \"DestinationPrefix\":
   854      \ \"__K8S_SERVICE_CIDR__\",\n                \"NeedEncap\":  true\n              }\n
   855      \           }\n          ]\n        }\n      ]\n\n    }\n---\napiVersion: apps/v1\nkind:
   856      DaemonSet\nmetadata:\n  name: calico-node-windows\n  labels:\n    tier: node\n
   857      \   app: calico\n  namespace: calico-system\nspec:\n  selector:\n    matchLabels:\n
   858      \     app: calico\n  template:\n    metadata:\n      labels:\n        tier: node\n
   859      \       app: calico\n    spec:\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n
   860      \           nodeSelectorTerms:\n              - matchExpressions:\n                  -
   861      key: kubernetes.io/os\n                    operator: In\n                    values:\n
   862      \                     - windows\n                  - key: kubernetes.io/arch\n
   863      \                   operator: In\n                    values:\n                      -
   864      amd64\n      securityContext:\n        windowsOptions:\n          hostProcess:
   865      true\n          runAsUserName: \"NT AUTHORITY\\\\system\"\n      hostNetwork:
   866      true\n      serviceAccountName: calico-node\n      tolerations:\n      - operator:
   867      Exists\n        effect: NoSchedule\n        # Mark the pod as a critical add-on
   868      for rescheduling.\n      - key: CriticalAddonsOnly\n        operator: Exists\n
   869      \     - effect: NoExecute\n        operator: Exists\n      initContainers:\n        #
   870      This container installs the CNI binaries\n        # and CNI network config file
   871      on each node.\n        - name: install-cni\n          image: sigwindowstools/calico-install:v3.26.1-hostprocess\n
   872      \         args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n
   873      \         imagePullPolicy: Always\n          env:\n            # Name of the CNI
   874      config file to create.\n            - name: CNI_CONF_NAME\n              value:
   875      \"10-calico.conflist\"\n            # The CNI network config to install on each
   876      node.\n            - name: CNI_NETWORK_CONFIG\n              valueFrom:\n                configMapKeyRef:\n
   877      \                 name: calico-config-windows\n                  key: cni_network_config\n
   878      \           # Set the hostname based on the k8s node name.\n            - name:
   879      KUBERNETES_NODE_NAME\n              valueFrom:\n                fieldRef:\n                  fieldPath:
   880      spec.nodeName\n            # CNI MTU Config variable\n            - name: CNI_MTU\n
   881      \             valueFrom:\n                configMapKeyRef:\n                  name:
   882      calico-config-windows\n                  key: veth_mtu\n            # Prevents
   883      the container from sleeping forever.\n            - name: SLEEP\n              value:
   884      \"false\"\n            - name: K8S_SERVICE_CIDR\n              value: \"10.96.0.0/12\"\n
   885      \         volumeMounts:\n            - mountPath: /host/opt/cni/bin\n              name:
   886      cni-bin-dir\n            - mountPath: /host/etc/cni/net.d\n              name:
   887      cni-net-dir\n            - name: kubeadm-config\n              mountPath: /etc/kubeadm-config/\n
   888      \         securityContext:\n            windowsOptions:\n              hostProcess:
   889      true\n              runAsUserName: \"NT AUTHORITY\\\\system\"\n      containers:\n
   890      \     - name: calico-node-startup\n        image: sigwindowstools/calico-node:v3.26.1-hostprocess\n
   891      \       args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n
   892      \       workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        imagePullPolicy:
   893      Always\n        volumeMounts:\n        - name: calico-config-windows\n          mountPath:
   894      /etc/kube-calico-windows/\n        env:\n        - name: POD_NAME\n          valueFrom:\n
   895      \           fieldRef:\n              apiVersion: v1\n              fieldPath:
   896      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   897      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   898      name: CNI_IPAM_TYPE\n          value: \"calico-ipam\"\n        - name: CALICO_NETWORKING_BACKEND\n
   899      \         value: \"vxlan\"\n        - name: KUBECONFIG\n          value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n
   900      \       - name: VXLAN_VNI\n          value: \"4096\"\n      - name: calico-node-felix\n
   901      \       image: sigwindowstools/calico-node:v3.26.1-hostprocess\n        args:
   902      [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n        imagePullPolicy:
   903      Always\n        workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        volumeMounts:\n
   904      \       - name: calico-config-windows\n          mountPath: /etc/kube-calico-windows/\n
   905      \       - name: calico-static-rules\n          mountPath: /calico/static-rules.json\n
   906      \         subPath: static-rules.json\n        env:\n        - name: POD_NAME\n
   907      \         valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath:
   908      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   909      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   910      name: VXLAN_VNI\n          value: \"4096\"\n        - name: KUBECONFIG\n          value:
   911      \"C:/etc/cni/net.d/calico-kubeconfig\"\n      volumes:\n      - name: calico-config-windows\n
   912      \       configMap:\n          name: calico-config-windows\n      - name: calico-static-rules\n
   913      \       configMap:\n          name: calico-static-rules\n      # Used to install
   914      CNI.\n      - name: cni-bin-dir\n        hostPath:\n          path: /opt/cni/bin\n
   915      \     - name: cni-net-dir\n        hostPath:\n          path: /etc/cni/net.d\n
   916      \     - name: kubeadm-config\n        configMap:\n          name: kubeadm-config\n---\napiVersion:
   917      apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: ipamconfigs.crd.projectcalico.org\nspec:\n
   918      \ group: crd.projectcalico.org\n  names:\n    kind: IPAMConfig\n    listKind:
   919      IPAMConfigList\n    plural: ipamconfigs\n    singular: ipamconfig\n  preserveUnknownFields:
   920      false\n  scope: Cluster\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n
   921      \       properties:\n          apiVersion:\n            description: 'APIVersion
   922      defines the versioned schema of this representation\n              of an object.
   923      Servers should convert recognized schemas to the latest\n              internal
   924      value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n
   925      \           type: string\n          kind:\n            description: 'Kind is a
   926      string value representing the REST resource this\n              object represents.
   927      Servers may infer this from the endpoint the client\n              submits requests
   928      to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n
   929      \           type: string\n          metadata:\n            type: object\n          spec:\n
   930      \           description: IPAMConfigSpec contains the specification for an IPAMConfig\n
   931      \             resource.\n            properties:\n              autoAllocateBlocks:\n
   932      \               type: boolean\n              maxBlocksPerHost:\n                description:
   933      MaxBlocksPerHost, if non-zero, is the max number of blocks\n                  that
   934      can be affine to each host.\n                maximum: 2147483647\n                minimum:
   935      0\n                type: integer\n              strictAffinity:\n                type:
   936      boolean\n            required:\n            - autoAllocateBlocks\n            -
   937      strictAffinity\n            type: object\n        type: object\n    served: true\n
   938      \   storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n
   939      \ conditions: []\n  storedVersions: []\n"
   940  kind: ConfigMap
   941  metadata:
   942    annotations:
   943      note: generated
   944    labels:
   945      type: generated
   946    name: cni-${CLUSTER_NAME}-calico-windows
   947    namespace: default
   948  ---
   949  apiVersion: v1
   950  data:
   951    csi-proxy: |
   952      apiVersion: apps/v1
   953      kind: DaemonSet
   954      metadata:
   955        labels:
   956          k8s-app: csi-proxy
   957        name: csi-proxy
   958        namespace: kube-system
   959      spec:
   960        selector:
   961          matchLabels:
   962            k8s-app: csi-proxy
   963        template:
   964          metadata:
   965            labels:
   966              k8s-app: csi-proxy
   967          spec:
   968            nodeSelector:
   969              "kubernetes.io/os": windows
   970            securityContext:
   971              windowsOptions:
   972                hostProcess: true
   973                runAsUserName: "NT AUTHORITY\\SYSTEM"
   974            hostNetwork: true
   975            containers:
   976              - name: csi-proxy
   977                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   978  kind: ConfigMap
   979  metadata:
   980    annotations:
   981      note: generated
   982    labels:
   983      type: generated
   984    name: csi-proxy-addon
   985    namespace: default
   986  ---
   987  apiVersion: v1
   988  data:
   989    containerd-windows-logger: |
   990      apiVersion: apps/v1
   991      kind: DaemonSet
   992      metadata:
   993        labels:
   994          k8s-app: containerd-logger
   995        name: containerd-logger
   996        namespace: kube-system
   997      spec:
   998        selector:
   999          matchLabels:
  1000            k8s-app: containerd-logger
  1001        template:
  1002          metadata:
  1003            labels:
  1004              k8s-app: containerd-logger
  1005          spec:
  1006            securityContext:
  1007              windowsOptions:
  1008                hostProcess: true
  1009                runAsUserName: "NT AUTHORITY\\system"
  1010            hostNetwork: true
  1011            containers:
  1012            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
  1013              args: [ "config.json" ]
  1014              name: containerd-logger
  1015              imagePullPolicy: Always
  1016              volumeMounts:
  1017              - name: containerd-logger-config
  1018                mountPath: /config.json
  1019                subPath: config.json
  1020            nodeSelector:
  1021              kubernetes.io/os: windows
  1022            tolerations:
  1023            - key: CriticalAddonsOnly
  1024              operator: Exists
  1025            - operator: Exists
  1026            volumes:
  1027            - configMap:
  1028                name: containerd-logger-config
  1029              name: containerd-logger-config
  1030        updateStrategy:
  1031          type: RollingUpdate
  1032      ---
  1033      kind: ConfigMap
  1034      apiVersion: v1
  1035      metadata:
  1036        name: containerd-logger-config
  1037        namespace: kube-system
  1038      data:
  1039        config.json: |
  1040          {
  1041            "inputs": [
  1042              {
  1043                "type": "ETW",
  1044                "sessionNamePrefix": "containerd",
  1045                "cleanupOldSessions": true,
  1046                "reuseExistingSession": true,
  1047                "providers": [
  1048                  {
  1049                    "providerName": "Microsoft.Virtualization.RunHCS",
  1050                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
  1051                    "level": "Verbose"
  1052                  },
  1053                  {
  1054                    "providerName": "ContainerD",
  1055                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
  1056                    "level": "Verbose"
  1057                  }
  1058                ]
  1059              }
  1060            ],
  1061            "filters": [
  1062              {
  1063                  "type": "drop",
  1064                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
  1065              },
  1066              {
  1067                  "type": "drop",
  1068                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
  1069              },
  1070              {
  1071                  "type": "drop",
  1072                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
  1073              },
  1074              {
  1075                  "type": "drop",
  1076                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
  1077              },
  1078              {
  1079                  "type": "drop",
  1080                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
  1081              },
  1082              {
  1083                  "type": "drop",
  1084                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
  1085              },
  1086              {
  1087                  "type": "drop",
  1088                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
  1089              }
  1090            ],
  1091            "outputs": [
  1092              {
  1093                "type": "StdOutput"
  1094              }
  1095            ],
  1096            "schemaVersion": "2016-08-11"
  1097          }
  1098  kind: ConfigMap
  1099  metadata:
  1100    annotations:
  1101      note: generated
  1102    labels:
  1103      type: generated
  1104    name: containerd-logger-${CLUSTER_NAME}
  1105    namespace: default
  1106  ---
  1107  apiVersion: addons.cluster.x-k8s.io/v1beta1
  1108  kind: ClusterResourceSet
  1109  metadata:
  1110    name: metrics-server-${CLUSTER_NAME}
  1111    namespace: default
  1112  spec:
  1113    clusterSelector:
  1114      matchLabels:
  1115        metrics-server: enabled
  1116    resources:
  1117    - kind: ConfigMap
  1118      name: metrics-server-${CLUSTER_NAME}
  1119    strategy: ApplyOnce
  1120  ---
  1121  apiVersion: v1
  1122  data:
  1123    metrics-server: |
  1124      apiVersion: v1
  1125      kind: ServiceAccount
  1126      metadata:
  1127        labels:
  1128          k8s-app: metrics-server
  1129        name: metrics-server
  1130        namespace: kube-system
  1131      ---
  1132      apiVersion: rbac.authorization.k8s.io/v1
  1133      kind: ClusterRole
  1134      metadata:
  1135        labels:
  1136          k8s-app: metrics-server
  1137          rbac.authorization.k8s.io/aggregate-to-admin: "true"
  1138          rbac.authorization.k8s.io/aggregate-to-edit: "true"
  1139          rbac.authorization.k8s.io/aggregate-to-view: "true"
  1140        name: system:aggregated-metrics-reader
  1141      rules:
  1142      - apiGroups:
  1143        - metrics.k8s.io
  1144        resources:
  1145        - pods
  1146        - nodes
  1147        verbs:
  1148        - get
  1149        - list
  1150        - watch
  1151      ---
  1152      apiVersion: rbac.authorization.k8s.io/v1
  1153      kind: ClusterRole
  1154      metadata:
  1155        labels:
  1156          k8s-app: metrics-server
  1157        name: system:metrics-server
  1158      rules:
  1159      - apiGroups:
  1160        - ""
  1161        resources:
  1162        - nodes/metrics
  1163        verbs:
  1164        - get
  1165      - apiGroups:
  1166        - ""
  1167        resources:
  1168        - pods
  1169        - nodes
  1170        verbs:
  1171        - get
  1172        - list
  1173        - watch
  1174      ---
  1175      apiVersion: rbac.authorization.k8s.io/v1
  1176      kind: RoleBinding
  1177      metadata:
  1178        labels:
  1179          k8s-app: metrics-server
  1180        name: metrics-server-auth-reader
  1181        namespace: kube-system
  1182      roleRef:
  1183        apiGroup: rbac.authorization.k8s.io
  1184        kind: Role
  1185        name: extension-apiserver-authentication-reader
  1186      subjects:
  1187      - kind: ServiceAccount
  1188        name: metrics-server
  1189        namespace: kube-system
  1190      ---
  1191      apiVersion: rbac.authorization.k8s.io/v1
  1192      kind: ClusterRoleBinding
  1193      metadata:
  1194        labels:
  1195          k8s-app: metrics-server
  1196        name: metrics-server:system:auth-delegator
  1197      roleRef:
  1198        apiGroup: rbac.authorization.k8s.io
  1199        kind: ClusterRole
  1200        name: system:auth-delegator
  1201      subjects:
  1202      - kind: ServiceAccount
  1203        name: metrics-server
  1204        namespace: kube-system
  1205      ---
  1206      apiVersion: rbac.authorization.k8s.io/v1
  1207      kind: ClusterRoleBinding
  1208      metadata:
  1209        labels:
  1210          k8s-app: metrics-server
  1211        name: system:metrics-server
  1212      roleRef:
  1213        apiGroup: rbac.authorization.k8s.io
  1214        kind: ClusterRole
  1215        name: system:metrics-server
  1216      subjects:
  1217      - kind: ServiceAccount
  1218        name: metrics-server
  1219        namespace: kube-system
  1220      ---
  1221      apiVersion: v1
  1222      kind: Service
  1223      metadata:
  1224        labels:
  1225          k8s-app: metrics-server
  1226        name: metrics-server
  1227        namespace: kube-system
  1228      spec:
  1229        ports:
  1230        - name: https
  1231          port: 443
  1232          protocol: TCP
  1233          targetPort: https
  1234        selector:
  1235          k8s-app: metrics-server
  1236      ---
  1237      apiVersion: apps/v1
  1238      kind: Deployment
  1239      metadata:
  1240        labels:
  1241          k8s-app: metrics-server
  1242        name: metrics-server
  1243        namespace: kube-system
  1244      spec:
  1245        selector:
  1246          matchLabels:
  1247            k8s-app: metrics-server
  1248        strategy:
  1249          rollingUpdate:
  1250            maxUnavailable: 0
  1251        template:
  1252          metadata:
  1253            labels:
  1254              k8s-app: metrics-server
  1255          spec:
  1256            containers:
  1257            - args:
  1258              - --cert-dir=/tmp
  1259              - --secure-port=4443
  1260              - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
  1261              - --kubelet-use-node-status-port
  1262              - --metric-resolution=15s
  1263              - --kubelet-insecure-tls
  1264              image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
  1265              imagePullPolicy: IfNotPresent
  1266              livenessProbe:
  1267                failureThreshold: 3
  1268                httpGet:
  1269                  path: /livez
  1270                  port: https
  1271                  scheme: HTTPS
  1272                periodSeconds: 10
  1273              name: metrics-server
  1274              ports:
  1275              - containerPort: 4443
  1276                name: https
  1277                protocol: TCP
  1278              readinessProbe:
  1279                failureThreshold: 3
  1280                httpGet:
  1281                  path: /readyz
  1282                  port: https
  1283                  scheme: HTTPS
  1284                initialDelaySeconds: 20
  1285                periodSeconds: 10
  1286              resources:
  1287                requests:
  1288                  cpu: 100m
  1289                  memory: 200Mi
  1290              securityContext:
  1291                allowPrivilegeEscalation: false
  1292                readOnlyRootFilesystem: true
  1293                runAsNonRoot: true
  1294                runAsUser: 1000
  1295              volumeMounts:
  1296              - mountPath: /tmp
  1297                name: tmp-dir
  1298            nodeSelector:
  1299              kubernetes.io/os: linux
  1300            priorityClassName: system-cluster-critical
  1301            serviceAccountName: metrics-server
  1302            tolerations:
  1303            - effect: NoSchedule
  1304              key: node-role.kubernetes.io/master
  1305              operator: Exists
  1306            - effect: NoSchedule
  1307              key: node-role.kubernetes.io/control-plane
  1308              operator: Exists
  1309            volumes:
  1310            - emptyDir: {}
  1311              name: tmp-dir
  1312      ---
  1313      apiVersion: apiregistration.k8s.io/v1
  1314      kind: APIService
  1315      metadata:
  1316        labels:
  1317          k8s-app: metrics-server
  1318        name: v1beta1.metrics.k8s.io
  1319      spec:
  1320        group: metrics.k8s.io
  1321        groupPriorityMinimum: 100
  1322        insecureSkipTLSVerify: true
  1323        service:
  1324          name: metrics-server
  1325          namespace: kube-system
  1326        version: v1beta1
  1327        versionPriority: 100
  1328  kind: ConfigMap
  1329  metadata:
  1330    annotations:
  1331      note: generated
  1332    labels:
  1333      type: generated
  1334    name: metrics-server-${CLUSTER_NAME}
  1335    namespace: default