sigs.k8s.io/cluster-api-provider-azure@v1.14.3/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml (about)

     1  apiVersion: cluster.x-k8s.io/v1beta1
     2  kind: Cluster
     3  metadata:
     4    labels:
     5      cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure}
     6      cni: calico
     7      cni-windows: ${CLUSTER_NAME}-calico
     8      containerd-logger: enabled
     9      csi-proxy: enabled
    10      windows: enabled
    11    name: ${CLUSTER_NAME}
    12    namespace: default
    13  spec:
    14    clusterNetwork:
    15      pods:
    16        cidrBlocks:
    17        - 192.168.0.0/16
    18    controlPlaneRef:
    19      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    20      kind: KubeadmControlPlane
    21      name: ${CLUSTER_NAME}-control-plane
    22    infrastructureRef:
    23      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    24      kind: AzureCluster
    25      name: ${CLUSTER_NAME}
    26  ---
    27  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    28  kind: AzureCluster
    29  metadata:
    30    name: ${CLUSTER_NAME}
    31    namespace: default
    32  spec:
    33    additionalTags:
    34      buildProvenance: ${BUILD_PROVENANCE}
    35      creationTimestamp: ${TIMESTAMP}
    36      jobName: ${JOB_NAME}
    37    identityRef:
    38      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    39      kind: AzureClusterIdentity
    40      name: ${CLUSTER_IDENTITY_NAME}
    41    location: ${AZURE_LOCATION}
    42    networkSpec:
    43      subnets:
    44      - name: control-plane-subnet
    45        role: control-plane
    46      - name: node-subnet
    47        role: node
    48      vnet:
    49        name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
    50    resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
    51    subscriptionID: ${AZURE_SUBSCRIPTION_ID}
    52  ---
    53  apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    54  kind: KubeadmControlPlane
    55  metadata:
    56    annotations:
    57      controlplane.cluster.x-k8s.io/skip-kube-proxy: "true"
    58    name: ${CLUSTER_NAME}-control-plane
    59    namespace: default
    60  spec:
    61    kubeadmConfigSpec:
    62      clusterConfiguration:
    63        apiServer:
    64          extraArgs:
    65            cloud-provider: external
    66          timeoutForControlPlane: 20m
    67        controllerManager:
    68          extraArgs:
    69            allocate-node-cidrs: "false"
    70            cloud-provider: external
    71            cluster-name: ${CLUSTER_NAME}
    72            v: "4"
    73        etcd:
    74          local:
    75            dataDir: /var/lib/etcddisk/etcd
    76            extraArgs:
    77              quota-backend-bytes: "8589934592"
    78        kubernetesVersion: ci/${CI_VERSION}
    79      diskSetup:
    80        filesystems:
    81        - device: /dev/disk/azure/scsi1/lun0
    82          extraOpts:
    83          - -E
    84          - lazy_itable_init=1,lazy_journal_init=1
    85          filesystem: ext4
    86          label: etcd_disk
    87        - device: ephemeral0.1
    88          filesystem: ext4
    89          label: ephemeral0
    90          replaceFS: ntfs
    91        partitions:
    92        - device: /dev/disk/azure/scsi1/lun0
    93          layout: true
    94          overwrite: false
    95          tableType: gpt
    96      files:
    97      - content: |
    98          #!/bin/bash
    99  
   100          set -o nounset
   101          set -o pipefail
   102          set -o errexit
   103  
   104          systemctl stop kubelet
   105          az login --identity
   106          declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
   107          for BINARY in "$${BINARIES[@]}"; do
   108            echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}"
   109            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login
   110          done
   111          systemctl restart kubelet
   112  
   113          # prepull images from gcr.io/k8s-staging-ci-images and retag it to
   114          # registry.k8s.io so kubeadm can fetch correct images no matter what
   115          declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
   116          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   117          IMAGE_REGISTRY_PREFIX=registry.k8s.io
   118          for IMAGE in "$${IMAGES[@]}"; do
   119            $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}"
   120            $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}"
   121          done
   122  
   123          echo "kubeadm version: $(kubeadm version -o=short)"
   124          echo "kubectl version: $(kubectl version --client=true)"
   125          echo "kubelet version: $(kubelet --version)"
   126        owner: root:root
   127        path: /tmp/replace-k8s-binaries.sh
   128        permissions: "0744"
   129      - content: |
   130          #!/bin/bash
   131  
   132          set -o nounset
   133          set -o pipefail
   134          set -o errexit
   135  
   136          curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz
   137          tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq
   138          rm /tmp/yq_linux_amd64.tar.gz
   139  
   140          export KUBECONFIG=/etc/kubernetes/admin.conf
   141          kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}"
   142          systemctl stop kubelet
   143          yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
   144          yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
   145          yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
   146          systemctl restart kubelet
   147        owner: root:root
   148        path: /tmp/replace-k8s-components.sh
   149        permissions: "0744"
   150      - contentFrom:
   151          secret:
   152            key: control-plane-azure.json
   153            name: ${CLUSTER_NAME}-control-plane-azure-json
   154        owner: root:root
   155        path: /etc/kubernetes/azure.json
   156        permissions: "0644"
   157      - content: |
   158          #!/bin/bash
   159  
   160          set -o nounset
   161          set -o pipefail
   162          set -o errexit
   163          [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   164  
   165          az login --identity
   166          echo "Use OOT credential provider"
   167          mkdir -p /var/lib/kubelet/credential-provider
   168          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   169          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   170          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   171          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   172        owner: root:root
   173        path: /tmp/oot-cred-provider.sh
   174        permissions: "0744"
   175      initConfiguration:
   176        nodeRegistration:
   177          kubeletExtraArgs:
   178            cloud-provider: external
   179            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   180            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   181          name: '{{ ds.meta_data["local_hostname"] }}'
   182      joinConfiguration:
   183        nodeRegistration:
   184          kubeletExtraArgs:
   185            cloud-provider: external
   186            image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   187            image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   188          name: '{{ ds.meta_data["local_hostname"] }}'
   189      mounts:
   190      - - LABEL=etcd_disk
   191        - /var/lib/etcddisk
   192      postKubeadmCommands:
   193      - bash -c /tmp/replace-k8s-components.sh
   194      preKubeadmCommands:
   195      - bash -c /tmp/replace-k8s-binaries.sh
   196      - bash -c /tmp/oot-cred-provider.sh
   197      verbosity: 5
   198    machineTemplate:
   199      infrastructureRef:
   200        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   201        kind: AzureMachineTemplate
   202        name: ${CLUSTER_NAME}-control-plane
   203    replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1}
   204    version: ${KUBERNETES_VERSION}
   205  ---
   206  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   207  kind: AzureMachineTemplate
   208  metadata:
   209    name: ${CLUSTER_NAME}-control-plane
   210    namespace: default
   211  spec:
   212    template:
   213      spec:
   214        dataDisks:
   215        - diskSizeGB: 256
   216          lun: 0
   217          nameSuffix: etcddisk
   218        image:
   219          marketplace:
   220            offer: capi
   221            publisher: cncf-upstream
   222            sku: ubuntu-2204-gen1
   223            version: latest
   224        osDisk:
   225          diskSizeGB: 128
   226          osType: Linux
   227        sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   228        vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
   229  ---
   230  apiVersion: cluster.x-k8s.io/v1beta1
   231  kind: MachinePool
   232  metadata:
   233    name: ${CLUSTER_NAME}-mp-0
   234    namespace: default
   235  spec:
   236    clusterName: ${CLUSTER_NAME}
   237    replicas: ${WORKER_MACHINE_COUNT:=2}
   238    template:
   239      spec:
   240        bootstrap:
   241          configRef:
   242            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   243            kind: KubeadmConfig
   244            name: ${CLUSTER_NAME}-mp-0
   245        clusterName: ${CLUSTER_NAME}
   246        infrastructureRef:
   247          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   248          kind: AzureMachinePool
   249          name: ${CLUSTER_NAME}-mp-0
   250        version: ${KUBERNETES_VERSION}
   251  ---
   252  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   253  kind: AzureMachinePool
   254  metadata:
   255    name: ${CLUSTER_NAME}-mp-0
   256    namespace: default
   257  spec:
   258    location: ${AZURE_LOCATION}
   259    strategy:
   260      rollingUpdate:
   261        deletePolicy: Oldest
   262        maxSurge: 25%
   263        maxUnavailable: 1
   264      type: RollingUpdate
   265    template:
   266      image:
   267        marketplace:
   268          offer: capi
   269          publisher: cncf-upstream
   270          sku: ubuntu-2204-gen1
   271          version: latest
   272      osDisk:
   273        diskSizeGB: 30
   274        managedDisk:
   275          storageAccountType: Premium_LRS
   276        osType: Linux
   277      sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   278      vmExtensions:
   279      - name: CustomScript
   280        protectedSettings:
   281          commandToExecute: |
   282            #!/bin/sh
   283            echo "This script is a no-op used for extension testing purposes ..."
   284            touch test_file
   285        publisher: Microsoft.Azure.Extensions
   286        version: "2.1"
   287      vmSize: ${AZURE_NODE_MACHINE_TYPE}
   288  ---
   289  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   290  kind: KubeadmConfig
   291  metadata:
   292    name: ${CLUSTER_NAME}-mp-0
   293    namespace: default
   294  spec:
   295    files:
   296    - content: |
   297        #!/bin/bash
   298  
   299        set -o nounset
   300        set -o pipefail
   301        set -o errexit
   302        [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
   303  
   304        az login --identity
   305        echo "Use OOT credential provider"
   306        mkdir -p /var/lib/kubelet/credential-provider
   307        az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
   308        chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
   309        az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
   310        chmod 644 /var/lib/kubelet/credential-provider-config.yaml
   311      owner: root:root
   312      path: /tmp/oot-cred-provider.sh
   313      permissions: "0744"
   314    - content: |
   315        #!/bin/bash
   316  
   317        set -o nounset
   318        set -o pipefail
   319        set -o errexit
   320  
   321        systemctl stop kubelet
   322        az login --identity
   323        declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
   324        for BINARY in "$${BINARIES[@]}"; do
   325          echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}"
   326          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login
   327        done
   328        systemctl restart kubelet
   329  
   330        echo "kubeadm version: $(kubeadm version -o=short)"
   331        echo "kubectl version: $(kubectl version --client=true)"
   332        echo "kubelet version: $(kubelet --version)"
   333      owner: root:root
   334      path: /tmp/replace-k8s-binaries.sh
   335      permissions: "0744"
   336    - contentFrom:
   337        secret:
   338          key: control-plane-azure.json
   339          name: ${CLUSTER_NAME}-control-plane-azure-json
   340      owner: root:root
   341      path: /etc/kubernetes/azure.json
   342      permissions: "0644"
   343    joinConfiguration:
   344      nodeRegistration:
   345        kubeletExtraArgs:
   346          cloud-provider: external
   347          image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   348          image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   349        name: '{{ ds.meta_data["local_hostname"] }}'
   350    preKubeadmCommands:
   351    - bash -c /tmp/oot-cred-provider.sh
   352    - bash -c /tmp/replace-k8s-binaries.sh
   353  ---
   354  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   355  kind: AzureClusterIdentity
   356  metadata:
   357    labels:
   358      clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
   359    name: ${CLUSTER_IDENTITY_NAME}
   360    namespace: default
   361  spec:
   362    allowedNamespaces: {}
   363    clientID: ${AZURE_CLIENT_ID}
   364    clientSecret:
   365      name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME}
   366      namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
   367    tenantID: ${AZURE_TENANT_ID}
   368    type: ServicePrincipal
   369  ---
   370  apiVersion: cluster.x-k8s.io/v1beta1
   371  kind: MachinePool
   372  metadata:
   373    name: ${CLUSTER_NAME}-mp-win
   374    namespace: default
   375  spec:
   376    clusterName: ${CLUSTER_NAME}
   377    replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0}
   378    template:
   379      spec:
   380        bootstrap:
   381          configRef:
   382            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   383            kind: KubeadmConfig
   384            name: ${CLUSTER_NAME}-mp-win
   385        clusterName: ${CLUSTER_NAME}
   386        infrastructureRef:
   387          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   388          kind: AzureMachinePool
   389          name: ${CLUSTER_NAME}-mp-win
   390        version: ${KUBERNETES_VERSION}
   391  ---
   392  apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
   393  kind: AzureMachinePool
   394  metadata:
   395    annotations:
   396      runtime: containerd
   397      windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""}
   398    name: ${CLUSTER_NAME}-mp-win
   399    namespace: default
   400  spec:
   401    location: ${AZURE_LOCATION}
   402    template:
   403      image:
   404        marketplace:
   405          offer: capi-windows
   406          publisher: cncf-upstream
   407          sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1
   408          version: latest
   409      osDisk:
   410        diskSizeGB: 128
   411        managedDisk:
   412          storageAccountType: Premium_LRS
   413        osType: Windows
   414      sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
   415      vmSize: ${AZURE_NODE_MACHINE_TYPE}
   416  ---
   417  apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
   418  kind: KubeadmConfig
   419  metadata:
   420    name: ${CLUSTER_NAME}-mp-win
   421    namespace: default
   422  spec:
   423    files:
   424    - contentFrom:
   425        secret:
   426          key: worker-node-azure.json
   427          name: ${CLUSTER_NAME}-mp-win-azure-json
   428      owner: root:root
   429      path: c:/k/azure.json
   430      permissions: "0644"
   431    - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe
   432      path: C:/defender-exclude-calico.ps1
   433      permissions: "0744"
   434    - content: |
   435        $ErrorActionPreference = 'Stop'
   436  
   437        Stop-Service kubelet -Force
   438  
   439        $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}"
   440        if($$KUBE_GIT_VERSION -ne "")
   441        {
   442          $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy")
   443          $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64"
   444          foreach ( $$binary in $$binaries )
   445          {
   446            echo "downloading binary: $$ci_url/$$binary.exe"
   447            curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe"
   448          }
   449        }
   450  
   451        # Tag it to the ci version.  The image knows how to use the copy locally with the configmap
   452        # that is applied at at this stage (windows-kubeproxy-ci.yaml)
   453        ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess
   454        ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess"
   455  
   456        kubeadm.exe version -o=short
   457        kubectl.exe version --client=true
   458        kubelet.exe --version
   459        kube-proxy.exe --version
   460      path: C:/replace-pr-binaries.ps1
   461      permissions: "0744"
   462    - content: |
   463        $ErrorActionPreference = 'Stop'
   464  
   465        az login --identity
   466        echo "Use OOT credential provider"
   467        mkdir C:\var\lib\kubelet\credential-provider
   468        az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login
   469        cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider
   470        az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login
   471      path: C:/oot-cred-provider.ps1
   472      permissions: "0744"
   473    joinConfiguration:
   474      nodeRegistration:
   475        criSocket: npipe:////./pipe/containerd-containerd
   476        kubeletExtraArgs:
   477          cloud-provider: external
   478          image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
   479          image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
   480          pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9
   481        name: '{{ ds.meta_data["local_hostname"] }}'
   482    postKubeadmCommands:
   483    - nssm set kubelet start SERVICE_AUTO_START
   484    - powershell C:/defender-exclude-calico.ps1
   485    preKubeadmCommands:
   486    - powershell c:/create-external-network.ps1
   487    - powershell C:/replace-pr-binaries.ps1
   488    - powershell C:/oot-cred-provider.ps1
   489    users:
   490    - groups: Administrators
   491      name: capi
   492      sshAuthorizedKeys:
   493      - ${AZURE_SSH_PUBLIC_KEY:=""}
   494  ---
   495  apiVersion: addons.cluster.x-k8s.io/v1beta1
   496  kind: ClusterResourceSet
   497  metadata:
   498    name: ${CLUSTER_NAME}-calico-windows
   499    namespace: default
   500  spec:
   501    clusterSelector:
   502      matchLabels:
   503        cni-windows: ${CLUSTER_NAME}-calico
   504    resources:
   505    - kind: ConfigMap
   506      name: cni-${CLUSTER_NAME}-calico-windows
   507    strategy: ApplyOnce
   508  ---
   509  apiVersion: addons.cluster.x-k8s.io/v1beta1
   510  kind: ClusterResourceSet
   511  metadata:
   512    name: csi-proxy
   513    namespace: default
   514  spec:
   515    clusterSelector:
   516      matchLabels:
   517        csi-proxy: enabled
   518    resources:
   519    - kind: ConfigMap
   520      name: csi-proxy-addon
   521    strategy: ApplyOnce
   522  ---
   523  apiVersion: addons.cluster.x-k8s.io/v1beta1
   524  kind: ClusterResourceSet
   525  metadata:
   526    name: containerd-logger-${CLUSTER_NAME}
   527    namespace: default
   528  spec:
   529    clusterSelector:
   530      matchLabels:
   531        containerd-logger: enabled
   532    resources:
   533    - kind: ConfigMap
   534      name: containerd-logger-${CLUSTER_NAME}
   535    strategy: ApplyOnce
   536  ---
   537  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   538  kind: HelmChartProxy
   539  metadata:
   540    name: calico
   541    namespace: default
   542  spec:
   543    chartName: tigera-operator
   544    clusterSelector:
   545      matchLabels:
   546        cni: calico
   547    namespace: tigera-operator
   548    releaseName: projectcalico
   549    repoURL: https://docs.tigera.io/calico/charts
   550    valuesTemplate: |-
   551      installation:
   552        cni:
   553          type: Calico
   554        calicoNetwork:
   555          bgp: Disabled
   556          mtu: 1350
   557          ipPools:
   558          ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
   559          - cidr: {{ $cidr }}
   560            encapsulation: VXLAN{{end}}
   561        registry: mcr.microsoft.com/oss
   562      # Image and registry configuration for the tigera/operator pod.
   563      tigeraOperator:
   564        image: tigera/operator
   565        registry: mcr.microsoft.com/oss
   566      calicoctl:
   567        image: mcr.microsoft.com/oss/calico/ctl
   568    version: ${CALICO_VERSION}
   569  ---
   570  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   571  kind: HelmChartProxy
   572  metadata:
   573    name: azuredisk-csi-driver-chart
   574    namespace: default
   575  spec:
   576    chartName: azuredisk-csi-driver
   577    clusterSelector:
   578      matchLabels:
   579        azuredisk-csi: "true"
   580    namespace: kube-system
   581    releaseName: azuredisk-csi-driver-oot
   582    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
   583    valuesTemplate: |-
   584      controller:
   585        replicas: 1
   586        runOnControlPlane: true
   587      windows:
   588        useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}
   589  ---
   590  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   591  kind: HelmChartProxy
   592  metadata:
   593    name: cloud-provider-azure-chart
   594    namespace: default
   595  spec:
   596    chartName: cloud-provider-azure
   597    clusterSelector:
   598      matchLabels:
   599        cloud-provider: azure
   600    releaseName: cloud-provider-azure-oot
   601    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   602    valuesTemplate: |
   603      infra:
   604        clusterName: {{ .Cluster.metadata.name }}
   605      cloudControllerManager:
   606        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   607        logVerbosity: 4
   608  ---
   609  apiVersion: addons.cluster.x-k8s.io/v1alpha1
   610  kind: HelmChartProxy
   611  metadata:
   612    name: cloud-provider-azure-chart-ci
   613    namespace: default
   614  spec:
   615    chartName: cloud-provider-azure
   616    clusterSelector:
   617      matchLabels:
   618        cloud-provider: azure-ci
   619    releaseName: cloud-provider-azure-oot
   620    repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
   621    valuesTemplate: |
   622      infra:
   623        clusterName: {{ .Cluster.metadata.name }}
   624      cloudControllerManager:
   625        cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
   626        cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
   627        clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
   628        imageName: "${CCM_IMAGE_NAME:-""}"
   629        imageRepository: "${IMAGE_REGISTRY:-""}"
   630        imageTag: "${IMAGE_TAG_CCM:-""}"
   631        logVerbosity: ${CCM_LOG_VERBOSITY:-4}
   632        replicas: ${CCM_COUNT:-1}
   633        enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
   634      cloudNodeManager:
   635        imageName: "${CNM_IMAGE_NAME:-""}"
   636        imageRepository: "${IMAGE_REGISTRY:-""}"
   637        imageTag: "${IMAGE_TAG_CNM:-""}"
   638  ---
   639  apiVersion: v1
   640  data:
   641    kube-proxy-patch: |-
   642      apiVersion: v1
   643      kind: ConfigMap
   644      metadata:
   645        name: windows-kubeproxy-ci
   646        namespace: kube-system
   647      data:
   648        KUBEPROXY_PATH: "c:/k/kube-proxy.exe"
   649    proxy: |
   650      apiVersion: apps/v1
   651      kind: DaemonSet
   652      metadata:
   653        labels:
   654          k8s-app: kube-proxy
   655        name: kube-proxy-windows
   656        namespace: kube-system
   657      spec:
   658        selector:
   659          matchLabels:
   660            k8s-app: kube-proxy-windows
   661        template:
   662          metadata:
   663            labels:
   664              k8s-app: kube-proxy-windows
   665          spec:
   666            serviceAccountName: kube-proxy
   667            securityContext:
   668              windowsOptions:
   669                hostProcess: true
   670                runAsUserName: "NT AUTHORITY\\system"
   671            hostNetwork: true
   672            containers:
   673            - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess
   674              args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"]
   675              workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/"
   676              name: kube-proxy
   677              env:
   678              - name: NODE_NAME
   679                valueFrom:
   680                  fieldRef:
   681                    apiVersion: v1
   682                    fieldPath: spec.nodeName
   683              - name: POD_IP
   684                valueFrom:
   685                  fieldRef:
   686                    fieldPath: status.podIP
   687              - name: KUBEPROXY_PATH
   688                valueFrom:
   689                  configMapKeyRef:
   690                    name: windows-kubeproxy-ci
   691                    key: KUBEPROXY_PATH
   692                    optional: true
   693              volumeMounts:
   694              - mountPath: /var/lib/kube-proxy
   695                name: kube-proxy
   696            nodeSelector:
   697              kubernetes.io/os: windows
   698            tolerations:
   699            - key: CriticalAddonsOnly
   700              operator: Exists
   701            - operator: Exists
   702            volumes:
   703            - configMap:
   704                name: kube-proxy
   705              name: kube-proxy
   706        updateStrategy:
   707          type: RollingUpdate
   708    windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind:
   709      IPAMConfig\nmetadata:\n  name: default\nspec:\n  autoAllocateBlocks: true\n  strictAffinity:
   710      true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n  name: calico-static-rules\n
   711      \ namespace: calico-system\n  labels:\n    tier: node\n    app: calico\ndata:\n
   712      \ static-rules.json: |\n    {\n      \"Provider\": \"azure\",\n      \"Version\":
   713      \"0.1\",\n      \"Rules\": [\n        {\n          \"Name\": \"EndpointPolicy\",\n
   714      \         \"Rule\": {\n              \"Id\": \"wireserver\",\n              \"Type\":
   715      \"ACL\",\n              \"Protocol\": 6,\n              \"Action\": \"Block\",\n
   716      \             \"Direction\": \"Out\",\n              \"RemoteAddresses\": \"168.63.129.16/32\",\n
   717      \             \"RemotePorts\": \"80\",\n              \"Priority\": 200,\n              \"RuleType\":
   718      \"Switch\"\n            }\n          }\n      ]\n    } \n---\nkind: ConfigMap\napiVersion:
   719      v1\nmetadata:\n  name: calico-config-windows\n  namespace: calico-system\n  labels:\n
   720      \   tier: node\n    app: calico\ndata:\n  veth_mtu: \"1350\"\n  \n  cni_network_config:
   721      |\n    {\n      \"name\": \"Calico\",\n      \"cniVersion\": \"0.3.1\",\n      \"plugins\":
   722      [\n        {\n          \"windows_use_single_network\": true,\n          \"type\":
   723      \"calico\",\n          \"mode\": \"vxlan\",\n          \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n
   724      \         \"nodename_file_optional\": true,\n          \"log_file_path\": \"c:/cni.log\",\n
   725      \         \"log_level\": \"debug\",\n\n          \"vxlan_mac_prefix\": \"0E-2A\",\n
   726      \         \"vxlan_vni\": 4096,\n          \"mtu\": __CNI_MTU__,\n          \"policy\":
   727      {\n            \"type\": \"k8s\"\n          },\n\n          \"log_level\": \"info\",\n\n
   728      \         \"capabilities\": {\"dns\": true},\n          \"DNS\":  {\n            \"Search\":
   729      \ [\n              \"svc.cluster.local\"\n            ]\n          },\n\n          \"datastore_type\":
   730      \"kubernetes\",\n\n          \"kubernetes\": {\n            \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n
   731      \         },\n\n          \"ipam\": {\n            \"type\": \"calico-ipam\",\n
   732      \           \"subnet\": \"usePodCidr\"\n          },\n\n          \"policies\":
   733      \ [\n            {\n              \"Name\":  \"EndpointPolicy\",\n              \"Value\":
   734      \ {\n                \"Type\":  \"OutBoundNAT\",\n                \"ExceptionList\":
   735      \ [\n                  \"__K8S_SERVICE_CIDR__\"\n                ]\n              }\n
   736      \           },\n            {\n              \"Name\":  \"EndpointPolicy\",\n
   737      \             \"Value\":  {\n                \"Type\":  \"SDNROUTE\",\n                \"DestinationPrefix\":
   738      \ \"__K8S_SERVICE_CIDR__\",\n                \"NeedEncap\":  true\n              }\n
   739      \           }\n          ]\n        }\n      ]\n\n    }\n---\napiVersion: apps/v1\nkind:
   740      DaemonSet\nmetadata:\n  name: calico-node-windows\n  labels:\n    tier: node\n
   741      \   app: calico\n  namespace: calico-system\nspec:\n  selector:\n    matchLabels:\n
   742      \     app: calico\n  template:\n    metadata:\n      labels:\n        tier: node\n
   743      \       app: calico\n    spec:\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n
   744      \           nodeSelectorTerms:\n              - matchExpressions:\n                  -
   745      key: kubernetes.io/os\n                    operator: In\n                    values:\n
   746      \                     - windows\n                  - key: kubernetes.io/arch\n
   747      \                   operator: In\n                    values:\n                      -
   748      amd64\n      securityContext:\n        windowsOptions:\n          hostProcess:
   749      true\n          runAsUserName: \"NT AUTHORITY\\\\system\"\n      hostNetwork:
   750      true\n      serviceAccountName: calico-node\n      tolerations:\n      - operator:
   751      Exists\n        effect: NoSchedule\n        # Mark the pod as a critical add-on
   752      for rescheduling.\n      - key: CriticalAddonsOnly\n        operator: Exists\n
   753      \     - effect: NoExecute\n        operator: Exists\n      initContainers:\n        #
   754      This container installs the CNI binaries\n        # and CNI network config file
   755      on each node.\n        - name: install-cni\n          image: sigwindowstools/calico-install:v3.26.1-hostprocess\n
   756      \         args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n
   757      \         imagePullPolicy: Always\n          env:\n            # Name of the CNI
   758      config file to create.\n            - name: CNI_CONF_NAME\n              value:
   759      \"10-calico.conflist\"\n            # The CNI network config to install on each
   760      node.\n            - name: CNI_NETWORK_CONFIG\n              valueFrom:\n                configMapKeyRef:\n
   761      \                 name: calico-config-windows\n                  key: cni_network_config\n
   762      \           # Set the hostname based on the k8s node name.\n            - name:
   763      KUBERNETES_NODE_NAME\n              valueFrom:\n                fieldRef:\n                  fieldPath:
   764      spec.nodeName\n            # CNI MTU Config variable\n            - name: CNI_MTU\n
   765      \             valueFrom:\n                configMapKeyRef:\n                  name:
   766      calico-config-windows\n                  key: veth_mtu\n            # Prevents
   767      the container from sleeping forever.\n            - name: SLEEP\n              value:
   768      \"false\"\n            - name: K8S_SERVICE_CIDR\n              value: \"10.96.0.0/12\"\n
   769      \         volumeMounts:\n            - mountPath: /host/opt/cni/bin\n              name:
   770      cni-bin-dir\n            - mountPath: /host/etc/cni/net.d\n              name:
   771      cni-net-dir\n            - name: kubeadm-config\n              mountPath: /etc/kubeadm-config/\n
   772      \         securityContext:\n            windowsOptions:\n              hostProcess:
   773      true\n              runAsUserName: \"NT AUTHORITY\\\\system\"\n      containers:\n
   774      \     - name: calico-node-startup\n        image: sigwindowstools/calico-node:v3.26.1-hostprocess\n
   775      \       args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n
   776      \       workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        imagePullPolicy:
   777      Always\n        volumeMounts:\n        - name: calico-config-windows\n          mountPath:
   778      /etc/kube-calico-windows/\n        env:\n        - name: POD_NAME\n          valueFrom:\n
   779      \           fieldRef:\n              apiVersion: v1\n              fieldPath:
   780      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   781      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   782      name: CNI_IPAM_TYPE\n          value: \"calico-ipam\"\n        - name: CALICO_NETWORKING_BACKEND\n
   783      \         value: \"vxlan\"\n        - name: KUBECONFIG\n          value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n
   784      \       - name: VXLAN_VNI\n          value: \"4096\"\n      - name: calico-node-felix\n
   785      \       image: sigwindowstools/calico-node:v3.26.1-hostprocess\n        args:
   786      [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n        imagePullPolicy:
   787      Always\n        workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n        volumeMounts:\n
   788      \       - name: calico-config-windows\n          mountPath: /etc/kube-calico-windows/\n
   789      \       - name: calico-static-rules\n          mountPath: /calico/static-rules.json\n
   790      \         subPath: static-rules.json\n        env:\n        - name: POD_NAME\n
   791      \         valueFrom:\n            fieldRef:\n              apiVersion: v1\n              fieldPath:
   792      metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n
   793      \             apiVersion: v1\n              fieldPath: metadata.namespace\n        -
   794      name: VXLAN_VNI\n          value: \"4096\"\n        - name: KUBECONFIG\n          value:
   795      \"C:/etc/cni/net.d/calico-kubeconfig\"\n      volumes:\n      - name: calico-config-windows\n
   796      \       configMap:\n          name: calico-config-windows\n      - name: calico-static-rules\n
   797      \       configMap:\n          name: calico-static-rules\n      # Used to install
   798      CNI.\n      - name: cni-bin-dir\n        hostPath:\n          path: /opt/cni/bin\n
   799      \     - name: cni-net-dir\n        hostPath:\n          path: /etc/cni/net.d\n
   800      \     - name: kubeadm-config\n        configMap:\n          name: kubeadm-config\n---\napiVersion:
   801      apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: ipamconfigs.crd.projectcalico.org\nspec:\n
   802      \ group: crd.projectcalico.org\n  names:\n    kind: IPAMConfig\n    listKind:
   803      IPAMConfigList\n    plural: ipamconfigs\n    singular: ipamconfig\n  preserveUnknownFields:
   804      false\n  scope: Cluster\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n
   805      \       properties:\n          apiVersion:\n            description: 'APIVersion
   806      defines the versioned schema of this representation\n              of an object.
   807      Servers should convert recognized schemas to the latest\n              internal
   808      value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n
   809      \           type: string\n          kind:\n            description: 'Kind is a
   810      string value representing the REST resource this\n              object represents.
   811      Servers may infer this from the endpoint the client\n              submits requests
   812      to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n
   813      \           type: string\n          metadata:\n            type: object\n          spec:\n
   814      \           description: IPAMConfigSpec contains the specification for an IPAMConfig\n
   815      \             resource.\n            properties:\n              autoAllocateBlocks:\n
   816      \               type: boolean\n              maxBlocksPerHost:\n                description:
   817      MaxBlocksPerHost, if non-zero, is the max number of blocks\n                  that
   818      can be affine to each host.\n                maximum: 2147483647\n                minimum:
   819      0\n                type: integer\n              strictAffinity:\n                type:
   820      boolean\n            required:\n            - autoAllocateBlocks\n            -
   821      strictAffinity\n            type: object\n        type: object\n    served: true\n
   822      \   storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n
   823      \ conditions: []\n  storedVersions: []\n"
   824  kind: ConfigMap
   825  metadata:
   826    annotations:
   827      note: generated
   828    labels:
   829      type: generated
   830    name: cni-${CLUSTER_NAME}-calico-windows
   831    namespace: default
   832  ---
   833  apiVersion: v1
   834  data:
   835    csi-proxy: |
   836      apiVersion: apps/v1
   837      kind: DaemonSet
   838      metadata:
   839        labels:
   840          k8s-app: csi-proxy
   841        name: csi-proxy
   842        namespace: kube-system
   843      spec:
   844        selector:
   845          matchLabels:
   846            k8s-app: csi-proxy
   847        template:
   848          metadata:
   849            labels:
   850              k8s-app: csi-proxy
   851          spec:
   852            nodeSelector:
   853              "kubernetes.io/os": windows
   854            securityContext:
   855              windowsOptions:
   856                hostProcess: true
   857                runAsUserName: "NT AUTHORITY\\SYSTEM"
   858            hostNetwork: true
   859            containers:
   860              - name: csi-proxy
   861                image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2
   862  kind: ConfigMap
   863  metadata:
   864    annotations:
   865      note: generated
   866    labels:
   867      type: generated
   868    name: csi-proxy-addon
   869    namespace: default
   870  ---
   871  apiVersion: v1
   872  data:
   873    containerd-windows-logger: |
   874      apiVersion: apps/v1
   875      kind: DaemonSet
   876      metadata:
   877        labels:
   878          k8s-app: containerd-logger
   879        name: containerd-logger
   880        namespace: kube-system
   881      spec:
   882        selector:
   883          matchLabels:
   884            k8s-app: containerd-logger
   885        template:
   886          metadata:
   887            labels:
   888              k8s-app: containerd-logger
   889          spec:
   890            securityContext:
   891              windowsOptions:
   892                hostProcess: true
   893                runAsUserName: "NT AUTHORITY\\system"
   894            hostNetwork: true
   895            containers:
   896            - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0
   897              args: [ "config.json" ]
   898              name: containerd-logger
   899              imagePullPolicy: Always
   900              volumeMounts:
   901              - name: containerd-logger-config
   902                mountPath: /config.json
   903                subPath: config.json
   904            nodeSelector:
   905              kubernetes.io/os: windows
   906            tolerations:
   907            - key: CriticalAddonsOnly
   908              operator: Exists
   909            - operator: Exists
   910            volumes:
   911            - configMap:
   912                name: containerd-logger-config
   913              name: containerd-logger-config
   914        updateStrategy:
   915          type: RollingUpdate
   916      ---
   917      kind: ConfigMap
   918      apiVersion: v1
   919      metadata:
   920        name: containerd-logger-config
   921        namespace: kube-system
   922      data:
   923        config.json: |
   924          {
   925            "inputs": [
   926              {
   927                "type": "ETW",
   928                "sessionNamePrefix": "containerd",
   929                "cleanupOldSessions": true,
   930                "reuseExistingSession": true,
   931                "providers": [
   932                  {
   933                    "providerName": "Microsoft.Virtualization.RunHCS",
   934                    "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3",
   935                    "level": "Verbose"
   936                  },
   937                  {
   938                    "providerName": "ContainerD",
   939                    "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad",
   940                    "level": "Verbose"
   941                  }
   942                ]
   943              }
   944            ],
   945            "filters": [
   946              {
   947                  "type": "drop",
   948                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error"
   949              },
   950              {
   951                  "type": "drop",
   952                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error"
   953              },
   954              {
   955                  "type": "drop",
   956                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error"
   957              },
   958              {
   959                  "type": "drop",
   960                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error"
   961              },
   962              {
   963                  "type": "drop",
   964                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error"
   965              },
   966              {
   967                  "type": "drop",
   968                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error"
   969              },
   970              {
   971                  "type": "drop",
   972                  "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error"
   973              }
   974            ],
   975            "outputs": [
   976              {
   977                "type": "StdOutput"
   978              }
   979            ],
   980            "schemaVersion": "2016-08-11"
   981          }
   982  kind: ConfigMap
   983  metadata:
   984    annotations:
   985      note: generated
   986    labels:
   987      type: generated
   988    name: containerd-logger-${CLUSTER_NAME}
   989    namespace: default