sigs.k8s.io/cluster-api-provider-azure@v1.17.0/templates/test/dev/cluster-template-custom-builds.yaml (about) 1 apiVersion: cluster.x-k8s.io/v1beta1 2 kind: Cluster 3 metadata: 4 labels: 5 cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} 6 cni: calico 7 cni-windows: ${CLUSTER_NAME}-calico 8 containerd-logger: enabled 9 csi-proxy: enabled 10 metrics-server: enabled 11 name: ${CLUSTER_NAME} 12 namespace: default 13 spec: 14 clusterNetwork: 15 pods: 16 cidrBlocks: 17 - 192.168.0.0/16 18 controlPlaneRef: 19 apiVersion: controlplane.cluster.x-k8s.io/v1beta1 20 kind: KubeadmControlPlane 21 name: ${CLUSTER_NAME}-control-plane 22 infrastructureRef: 23 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 24 kind: AzureCluster 25 name: ${CLUSTER_NAME} 26 --- 27 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 28 kind: AzureCluster 29 metadata: 30 name: ${CLUSTER_NAME} 31 namespace: default 32 spec: 33 additionalTags: 34 buildProvenance: ${BUILD_PROVENANCE} 35 creationTimestamp: ${TIMESTAMP} 36 jobName: ${JOB_NAME} 37 identityRef: 38 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 39 kind: AzureClusterIdentity 40 name: ${CLUSTER_IDENTITY_NAME} 41 location: ${AZURE_LOCATION} 42 networkSpec: 43 subnets: 44 - name: control-plane-subnet 45 role: control-plane 46 - name: node-subnet 47 role: node 48 vnet: 49 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} 50 resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} 51 subscriptionID: ${AZURE_SUBSCRIPTION_ID} 52 --- 53 apiVersion: controlplane.cluster.x-k8s.io/v1beta1 54 kind: KubeadmControlPlane 55 metadata: 56 annotations: 57 controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" 58 name: ${CLUSTER_NAME}-control-plane 59 namespace: default 60 spec: 61 kubeadmConfigSpec: 62 clusterConfiguration: 63 apiServer: 64 extraArgs: 65 cloud-provider: external 66 feature-gates: ${K8S_FEATURE_GATES:-""} 67 timeoutForControlPlane: 20m 68 controllerManager: 69 extraArgs: 70 allocate-node-cidrs: "false" 71 cloud-provider: external 72 cluster-name: ${CLUSTER_NAME} 73 feature-gates: HPAContainerMetrics=true 74 v: "4" 75 etcd: 76 local: 77 dataDir: /var/lib/etcddisk/etcd 78 extraArgs: 79 quota-backend-bytes: "8589934592" 80 kubernetesVersion: ci/${CI_VERSION} 81 diskSetup: 82 filesystems: 83 - device: /dev/disk/azure/scsi1/lun0 84 extraOpts: 85 - -E 86 - lazy_itable_init=1,lazy_journal_init=1 87 filesystem: ext4 88 label: etcd_disk 89 - device: ephemeral0.1 90 filesystem: ext4 91 label: ephemeral0 92 replaceFS: ntfs 93 partitions: 94 - device: /dev/disk/azure/scsi1/lun0 95 layout: true 96 overwrite: false 97 tableType: gpt 98 files: 99 - contentFrom: 100 secret: 101 key: control-plane-azure.json 102 name: ${CLUSTER_NAME}-control-plane-azure-json 103 owner: root:root 104 path: /etc/kubernetes/azure.json 105 permissions: "0644" 106 - content: | 107 #!/bin/bash 108 109 set -o nounset 110 set -o pipefail 111 set -o errexit 112 [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" 113 114 echo "Use OOT credential provider" 115 mkdir -p /var/lib/kubelet/credential-provider 116 curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" 117 chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider 118 curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" 119 chmod 644 /var/lib/kubelet/credential-provider-config.yaml 120 owner: root:root 121 path: /tmp/oot-cred-provider.sh 122 permissions: "0744" 123 - content: | 124 #!/bin/bash 125 126 set -o nounset 127 set -o pipefail 128 set -o errexit 129 130 systemctl stop kubelet 131 declare -a BINARIES=("kubeadm" "kubectl" "kubelet") 132 az login --identity 133 for BINARY in "$${BINARIES[@]}"; do 134 echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" 135 az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login 136 done 137 systemctl restart kubelet 138 139 # prepull images from gcr.io/k8s-staging-ci-images and retag it to 140 # registry.k8s.io so kubeadm can fetch correct images no matter what 141 declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") 142 [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" 143 IMAGE_REGISTRY_PREFIX=registry.k8s.io 144 for IMAGE in "$${IMAGES[@]}"; do 145 $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}" 146 $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}" 147 done 148 149 echo "kubeadm version: $(kubeadm version -o=short)" 150 echo "kubectl version: $(kubectl version --client=true)" 151 echo "kubelet version: $(kubelet --version)" 152 owner: root:root 153 path: /tmp/replace-k8s-binaries.sh 154 permissions: "0744" 155 - content: | 156 #!/bin/bash 157 158 set -o nounset 159 set -o pipefail 160 set -o errexit 161 162 curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz 163 tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq 164 rm /tmp/yq_linux_amd64.tar.gz 165 166 export KUBECONFIG=/etc/kubernetes/admin.conf 167 kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}" 168 systemctl stop kubelet 169 yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml 170 yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml 171 yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml 172 systemctl restart kubelet 173 owner: root:root 174 path: /tmp/replace-k8s-components.sh 175 permissions: "0744" 176 initConfiguration: 177 nodeRegistration: 178 kubeletExtraArgs: 179 cloud-provider: external 180 image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider 181 image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml 182 name: '{{ ds.meta_data["local_hostname"] }}' 183 joinConfiguration: 184 nodeRegistration: 185 kubeletExtraArgs: 186 cloud-provider: external 187 image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider 188 image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml 189 name: '{{ ds.meta_data["local_hostname"] }}' 190 mounts: 191 - - LABEL=etcd_disk 192 - /var/lib/etcddisk 193 postKubeadmCommands: 194 - bash -c /tmp/replace-k8s-components.sh 195 preKubeadmCommands: 196 - bash -c /tmp/oot-cred-provider.sh 197 - bash -c /tmp/replace-k8s-binaries.sh 198 verbosity: 5 199 machineTemplate: 200 infrastructureRef: 201 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 202 kind: AzureMachineTemplate 203 name: ${CLUSTER_NAME}-control-plane 204 replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} 205 version: ${KUBERNETES_VERSION} 206 --- 207 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 208 kind: AzureMachineTemplate 209 metadata: 210 name: ${CLUSTER_NAME}-control-plane 211 namespace: default 212 spec: 213 template: 214 spec: 215 dataDisks: 216 - diskSizeGB: 256 217 lun: 0 218 nameSuffix: etcddisk 219 identity: UserAssigned 220 image: 221 marketplace: 222 offer: capi 223 publisher: cncf-upstream 224 sku: ubuntu-2204-gen1 225 version: latest 226 osDisk: 227 diskSizeGB: 128 228 osType: Linux 229 sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} 230 userAssignedIdentities: 231 - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} 232 vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} 233 --- 234 apiVersion: cluster.x-k8s.io/v1beta1 235 kind: MachineDeployment 236 metadata: 237 name: ${CLUSTER_NAME}-md-0 238 namespace: default 239 spec: 240 clusterName: ${CLUSTER_NAME} 241 replicas: ${WORKER_MACHINE_COUNT:=2} 242 selector: {} 243 template: 244 metadata: 245 labels: 246 nodepool: pool1 247 spec: 248 bootstrap: 249 configRef: 250 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 251 kind: KubeadmConfigTemplate 252 name: ${CLUSTER_NAME}-md-0 253 clusterName: ${CLUSTER_NAME} 254 infrastructureRef: 255 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 256 kind: AzureMachineTemplate 257 name: ${CLUSTER_NAME}-md-0 258 version: ${KUBERNETES_VERSION} 259 --- 260 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 261 kind: AzureMachineTemplate 262 metadata: 263 name: ${CLUSTER_NAME}-md-0 264 namespace: default 265 spec: 266 template: 267 spec: 268 identity: UserAssigned 269 image: 270 marketplace: 271 offer: capi 272 publisher: cncf-upstream 273 sku: ubuntu-2204-gen1 274 version: latest 275 osDisk: 276 diskSizeGB: 128 277 osType: Linux 278 sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} 279 userAssignedIdentities: 280 - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} 281 vmExtensions: 282 - name: CustomScript 283 protectedSettings: 284 commandToExecute: | 285 #!/bin/sh 286 echo "This script is a no-op used for extension testing purposes ..." 287 touch test_file 288 publisher: Microsoft.Azure.Extensions 289 version: "2.1" 290 vmSize: ${AZURE_NODE_MACHINE_TYPE} 291 --- 292 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 293 kind: KubeadmConfigTemplate 294 metadata: 295 name: ${CLUSTER_NAME}-md-0 296 namespace: default 297 spec: 298 template: 299 spec: 300 files: 301 - contentFrom: 302 secret: 303 key: worker-node-azure.json 304 name: ${CLUSTER_NAME}-md-0-azure-json 305 owner: root:root 306 path: /etc/kubernetes/azure.json 307 permissions: "0644" 308 - content: | 309 #!/bin/bash 310 311 set -o nounset 312 set -o pipefail 313 set -o errexit 314 [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" 315 316 echo "Use OOT credential provider" 317 mkdir -p /var/lib/kubelet/credential-provider 318 curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" 319 chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider 320 curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" 321 chmod 644 /var/lib/kubelet/credential-provider-config.yaml 322 owner: root:root 323 path: /tmp/oot-cred-provider.sh 324 permissions: "0744" 325 - content: | 326 #!/bin/bash 327 328 set -o nounset 329 set -o pipefail 330 set -o errexit 331 332 systemctl stop kubelet 333 declare -a BINARIES=("kubeadm" "kubectl" "kubelet") 334 az login --identity 335 for BINARY in "$${BINARIES[@]}"; do 336 echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" 337 az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login 338 done 339 systemctl restart kubelet 340 341 echo "kubeadm version: $(kubeadm version -o=short)" 342 echo "kubectl version: $(kubectl version --client=true)" 343 echo "kubelet version: $(kubelet --version)" 344 owner: root:root 345 path: /tmp/replace-k8s-binaries.sh 346 permissions: "0744" 347 joinConfiguration: 348 nodeRegistration: 349 kubeletExtraArgs: 350 cloud-provider: external 351 image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider 352 image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml 353 name: '{{ ds.meta_data["local_hostname"] }}' 354 preKubeadmCommands: 355 - bash -c /tmp/oot-cred-provider.sh 356 - bash -c /tmp/replace-k8s-binaries.sh 357 --- 358 apiVersion: cluster.x-k8s.io/v1beta1 359 kind: MachineDeployment 360 metadata: 361 name: ${CLUSTER_NAME}-md-win 362 namespace: default 363 spec: 364 clusterName: ${CLUSTER_NAME} 365 replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} 366 selector: {} 367 template: 368 spec: 369 bootstrap: 370 configRef: 371 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 372 kind: KubeadmConfigTemplate 373 name: ${CLUSTER_NAME}-md-win 374 clusterName: ${CLUSTER_NAME} 375 infrastructureRef: 376 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 377 kind: AzureMachineTemplate 378 name: ${CLUSTER_NAME}-md-win 379 version: ${KUBERNETES_VERSION} 380 --- 381 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 382 kind: AzureMachineTemplate 383 metadata: 384 annotations: 385 runtime: containerd 386 name: ${CLUSTER_NAME}-md-win 387 namespace: default 388 spec: 389 template: 390 metadata: 391 annotations: 392 runtime: containerd 393 windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} 394 spec: 395 identity: UserAssigned 396 image: 397 marketplace: 398 offer: capi-windows 399 publisher: cncf-upstream 400 sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 401 version: latest 402 osDisk: 403 diskSizeGB: 128 404 managedDisk: 405 storageAccountType: Premium_LRS 406 osType: Windows 407 sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} 408 userAssignedIdentities: 409 - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} 410 vmSize: ${AZURE_NODE_MACHINE_TYPE} 411 --- 412 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 413 kind: KubeadmConfigTemplate 414 metadata: 415 name: ${CLUSTER_NAME}-md-win 416 namespace: default 417 spec: 418 template: 419 spec: 420 files: 421 - contentFrom: 422 secret: 423 key: worker-node-azure.json 424 name: ${CLUSTER_NAME}-md-win-azure-json 425 owner: root:root 426 path: c:/k/azure.json 427 permissions: "0644" 428 - content: |- 429 Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe 430 Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe 431 path: C:/defender-exclude-calico.ps1 432 permissions: "0744" 433 - content: | 434 # /tmp is assumed created and required for upstream e2e tests to pass 435 New-Item -ItemType Directory -Force -Path C:\tmp\ 436 path: C:/create-temp-folder.ps1 437 permissions: "0744" 438 - content: | 439 $ErrorActionPreference = 'Stop' 440 441 $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" 442 if($$CONTAINERD_URL -ne ""){ 443 # Kubelet service depends on contianerd service so make a best effort attempt to stop it 444 Stop-Service kubelet -Force -ErrorAction SilentlyContinue 445 Stop-Service containerd -Force 446 echo "downloading containerd: $$CONTAINERD_URL" 447 curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" 448 tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 449 450 Start-Service containerd 451 } 452 453 containerd.exe --version 454 containerd-shim-runhcs-v1.exe --version 455 path: C:/replace-containerd.ps1 456 permissions: "0744" 457 - content: | 458 mkdir -Force c:/localdumps 459 reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f 460 reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f 461 reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f 462 # Enable sftp so we can copy crash dump files during log collection of stfp 463 $sshd_config = "$env:ProgramData\ssh\sshd_config" 464 if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } 465 Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" 466 sc.exe stop sshd 467 sc.exe start sshd 468 path: C:/collect-hns-crashes.ps1 469 permissions: "0744" 470 - content: | 471 $ErrorActionPreference = 'Stop' 472 473 echo "Use OOT credential provider" 474 mkdir C:\var\lib\kubelet\credential-provider 475 curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe 476 cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider 477 curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml 478 path: C:/oot-cred-provider.ps1 479 permissions: "0744" 480 - content: | 481 Write-Host "Installing Azure CLI" 482 $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi 483 # Need to add manually AZ to PATH as it is not added without a reset 484 Write-Host "Adding AZ to path:" 485 $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" 486 Write-Host "Version Azure CLI:" 487 az version 488 path: C:/install-az-cli.ps1 489 permissions: "0744" 490 - content: | 491 $ErrorActionPreference = 'Stop' 492 493 Stop-Service kubelet -Force 494 495 $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" 496 if($$KUBE_GIT_VERSION -ne "") 497 { 498 # Need to add manually AZ to PATH as it is not added without a reset 499 Write-Host "Adding AZ to path:" 500 $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" 501 # Install Binaries 502 $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") 503 $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" 504 echo "running az login via identity" 505 az login --identity 506 foreach ( $$binary in $$binaries ) 507 { 508 echo "downloading binary: $$ci_url/$$binary.exe" 509 az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login 510 } 511 } 512 513 # Tag it to the ci version. The image knows how to use the copy locally with the configmap 514 # that is applied at at this stage (windows-kubeproxy-ci.yaml) 515 ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess 516 ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" 517 518 kubeadm.exe version -o=short 519 kubectl.exe version --client=true 520 kubelet.exe --version 521 kube-proxy.exe --version 522 path: C:/replace-pr-binaries.ps1 523 permissions: "0744" 524 joinConfiguration: 525 nodeRegistration: 526 criSocket: npipe:////./pipe/containerd-containerd 527 kubeletExtraArgs: 528 cloud-provider: external 529 feature-gates: ${NODE_FEATURE_GATES:-""} 530 image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider 531 image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml 532 v: "2" 533 windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS 534 name: '{{ ds.meta_data["local_hostname"] }}' 535 postKubeadmCommands: 536 - nssm set kubelet start SERVICE_AUTO_START 537 - powershell C:/defender-exclude-calico.ps1 538 preKubeadmCommands: 539 - powershell C:/create-temp-folder.ps1 540 - powershell C:/replace-containerd.ps1 541 - powershell C:/collect-hns-crashes.ps1 542 - powershell C:/oot-cred-provider.ps1 543 - powershell C:/install-az-cli.ps1 544 - powershell C:/replace-pr-binaries.ps1 545 users: 546 - groups: Administrators 547 name: capi 548 sshAuthorizedKeys: 549 - ${AZURE_SSH_PUBLIC_KEY:=""} 550 --- 551 apiVersion: cluster.x-k8s.io/v1beta1 552 kind: MachineHealthCheck 553 metadata: 554 name: ${CLUSTER_NAME}-mhc-0 555 namespace: default 556 spec: 557 clusterName: ${CLUSTER_NAME} 558 maxUnhealthy: 100% 559 selector: 560 matchLabels: 561 nodepool: pool1 562 unhealthyConditions: 563 - status: "True" 564 timeout: 30s 565 type: E2ENodeUnhealthy 566 --- 567 apiVersion: addons.cluster.x-k8s.io/v1beta1 568 kind: ClusterResourceSet 569 metadata: 570 name: ${CLUSTER_NAME}-calico-windows 571 namespace: default 572 spec: 573 clusterSelector: 574 matchLabels: 575 cni-windows: ${CLUSTER_NAME}-calico 576 resources: 577 - kind: ConfigMap 578 name: cni-${CLUSTER_NAME}-calico-windows 579 strategy: ApplyOnce 580 --- 581 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 582 kind: AzureClusterIdentity 583 metadata: 584 labels: 585 clusterctl.cluster.x-k8s.io/move-hierarchy: "true" 586 name: ${CLUSTER_IDENTITY_NAME} 587 namespace: default 588 spec: 589 allowedNamespaces: {} 590 clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} 591 tenantID: ${AZURE_TENANT_ID} 592 type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} 593 --- 594 apiVersion: addons.cluster.x-k8s.io/v1beta1 595 kind: ClusterResourceSet 596 metadata: 597 name: csi-proxy 598 namespace: default 599 spec: 600 clusterSelector: 601 matchLabels: 602 csi-proxy: enabled 603 resources: 604 - kind: ConfigMap 605 name: csi-proxy-addon 606 strategy: ApplyOnce 607 --- 608 apiVersion: addons.cluster.x-k8s.io/v1beta1 609 kind: ClusterResourceSet 610 metadata: 611 name: containerd-logger-${CLUSTER_NAME} 612 namespace: default 613 spec: 614 clusterSelector: 615 matchLabels: 616 containerd-logger: enabled 617 resources: 618 - kind: ConfigMap 619 name: containerd-logger-${CLUSTER_NAME} 620 strategy: ApplyOnce 621 --- 622 apiVersion: addons.cluster.x-k8s.io/v1alpha1 623 kind: HelmChartProxy 624 metadata: 625 name: calico 626 namespace: default 627 spec: 628 chartName: tigera-operator 629 clusterSelector: 630 matchLabels: 631 cni: calico 632 namespace: tigera-operator 633 releaseName: projectcalico 634 repoURL: https://docs.tigera.io/calico/charts 635 valuesTemplate: |- 636 installation: 637 cni: 638 type: Calico 639 calicoNetwork: 640 bgp: Disabled 641 mtu: 1350 642 ipPools: 643 ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} 644 - cidr: {{ $cidr }} 645 encapsulation: VXLAN{{end}} 646 registry: mcr.microsoft.com/oss 647 # Image and registry configuration for the tigera/operator pod. 648 tigeraOperator: 649 image: tigera/operator 650 registry: mcr.microsoft.com/oss 651 calicoctl: 652 image: mcr.microsoft.com/oss/calico/ctl 653 version: ${CALICO_VERSION} 654 --- 655 apiVersion: addons.cluster.x-k8s.io/v1alpha1 656 kind: HelmChartProxy 657 metadata: 658 name: azuredisk-csi-driver-chart 659 namespace: default 660 spec: 661 chartName: azuredisk-csi-driver 662 clusterSelector: 663 matchLabels: 664 azuredisk-csi: "true" 665 namespace: kube-system 666 releaseName: azuredisk-csi-driver-oot 667 repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts 668 valuesTemplate: |- 669 controller: 670 replicas: 1 671 runOnControlPlane: true 672 windows: 673 useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} 674 --- 675 apiVersion: addons.cluster.x-k8s.io/v1alpha1 676 kind: HelmChartProxy 677 metadata: 678 name: cloud-provider-azure-chart 679 namespace: default 680 spec: 681 chartName: cloud-provider-azure 682 clusterSelector: 683 matchLabels: 684 cloud-provider: azure 685 releaseName: cloud-provider-azure-oot 686 repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo 687 valuesTemplate: | 688 infra: 689 clusterName: {{ .Cluster.metadata.name }} 690 cloudControllerManager: 691 clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} 692 logVerbosity: 4 693 --- 694 apiVersion: addons.cluster.x-k8s.io/v1alpha1 695 kind: HelmChartProxy 696 metadata: 697 name: cloud-provider-azure-chart-ci 698 namespace: default 699 spec: 700 chartName: cloud-provider-azure 701 clusterSelector: 702 matchLabels: 703 cloud-provider: azure-ci 704 releaseName: cloud-provider-azure-oot 705 repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo 706 valuesTemplate: | 707 infra: 708 clusterName: {{ .Cluster.metadata.name }} 709 cloudControllerManager: 710 cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} 711 cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} 712 clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} 713 imageName: "${CCM_IMAGE_NAME:-""}" 714 imageRepository: "${IMAGE_REGISTRY:-""}" 715 imageTag: "${IMAGE_TAG_CCM:-""}" 716 logVerbosity: ${CCM_LOG_VERBOSITY:-4} 717 replicas: ${CCM_COUNT:-1} 718 enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} 719 cloudNodeManager: 720 imageName: "${CNM_IMAGE_NAME:-""}" 721 imageRepository: "${IMAGE_REGISTRY:-""}" 722 imageTag: "${IMAGE_TAG_CNM:-""}" 723 --- 724 apiVersion: v1 725 data: 726 kube-proxy-patch: |- 727 apiVersion: v1 728 kind: ConfigMap 729 metadata: 730 name: windows-kubeproxy-ci 731 namespace: kube-system 732 data: 733 KUBEPROXY_PATH: "c:/k/kube-proxy.exe" 734 proxy: | 735 apiVersion: apps/v1 736 kind: DaemonSet 737 metadata: 738 labels: 739 k8s-app: kube-proxy 740 name: kube-proxy-windows 741 namespace: kube-system 742 spec: 743 selector: 744 matchLabels: 745 k8s-app: kube-proxy-windows 746 template: 747 metadata: 748 labels: 749 k8s-app: kube-proxy-windows 750 spec: 751 serviceAccountName: kube-proxy 752 securityContext: 753 windowsOptions: 754 hostProcess: true 755 runAsUserName: "NT AUTHORITY\\system" 756 hostNetwork: true 757 containers: 758 - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess 759 args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] 760 workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" 761 name: kube-proxy 762 env: 763 - name: NODE_NAME 764 valueFrom: 765 fieldRef: 766 apiVersion: v1 767 fieldPath: spec.nodeName 768 - name: POD_IP 769 valueFrom: 770 fieldRef: 771 fieldPath: status.podIP 772 - name: KUBEPROXY_PATH 773 valueFrom: 774 configMapKeyRef: 775 name: windows-kubeproxy-ci 776 key: KUBEPROXY_PATH 777 optional: true 778 volumeMounts: 779 - mountPath: /var/lib/kube-proxy 780 name: kube-proxy 781 nodeSelector: 782 kubernetes.io/os: windows 783 tolerations: 784 - key: CriticalAddonsOnly 785 operator: Exists 786 - operator: Exists 787 volumes: 788 - configMap: 789 name: kube-proxy 790 name: kube-proxy 791 updateStrategy: 792 type: RollingUpdate 793 windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: 794 IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: 795 true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n 796 \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n 797 \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": 798 \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n 799 \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": 800 \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n 801 \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n 802 \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": 803 \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: 804 v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n 805 \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: 806 |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": 807 [\n {\n \"windows_use_single_network\": true,\n \"type\": 808 \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n 809 \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n 810 \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n 811 \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": 812 {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n 813 \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": 814 \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": 815 \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n 816 \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n 817 \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": 818 \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": 819 \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": 820 \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n 821 \ },\n {\n \"Name\": \"EndpointPolicy\",\n 822 \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": 823 \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n 824 \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: 825 DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n 826 \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n 827 \ app: calico\n template:\n metadata:\n labels:\n tier: node\n 828 \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n 829 \ nodeSelectorTerms:\n - matchExpressions:\n - 830 key: kubernetes.io/os\n operator: In\n values:\n 831 \ - windows\n - key: kubernetes.io/arch\n 832 \ operator: In\n values:\n - 833 amd64\n securityContext:\n windowsOptions:\n hostProcess: 834 true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: 835 true\n serviceAccountName: calico-node\n tolerations:\n - operator: 836 Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on 837 for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n 838 \ - effect: NoExecute\n operator: Exists\n initContainers:\n # 839 This container installs the CNI binaries\n # and CNI network config file 840 on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.26.1-hostprocess\n 841 \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n 842 \ imagePullPolicy: Always\n env:\n # Name of the CNI 843 config file to create.\n - name: CNI_CONF_NAME\n value: 844 \"10-calico.conflist\"\n # The CNI network config to install on each 845 node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n 846 \ name: calico-config-windows\n key: cni_network_config\n 847 \ # Set the hostname based on the k8s node name.\n - name: 848 KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: 849 spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n 850 \ valueFrom:\n configMapKeyRef:\n name: 851 calico-config-windows\n key: veth_mtu\n # Prevents 852 the container from sleeping forever.\n - name: SLEEP\n value: 853 \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n 854 \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: 855 cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: 856 cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n 857 \ securityContext:\n windowsOptions:\n hostProcess: 858 true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n 859 \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.26.1-hostprocess\n 860 \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n 861 \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: 862 Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: 863 /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n 864 \ fieldRef:\n apiVersion: v1\n fieldPath: 865 metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n 866 \ apiVersion: v1\n fieldPath: metadata.namespace\n - 867 name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n 868 \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n 869 \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n 870 \ image: sigwindowstools/calico-node:v3.26.1-hostprocess\n args: 871 [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: 872 Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n 873 \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n 874 \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n 875 \ subPath: static-rules.json\n env:\n - name: POD_NAME\n 876 \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: 877 metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n 878 \ apiVersion: v1\n fieldPath: metadata.namespace\n - 879 name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: 880 \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n 881 \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n 882 \ configMap:\n name: calico-static-rules\n # Used to install 883 CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n 884 \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n 885 \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: 886 apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n 887 \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: 888 IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: 889 false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n 890 \ properties:\n apiVersion:\n description: 'APIVersion 891 defines the versioned schema of this representation\n of an object. 892 Servers should convert recognized schemas to the latest\n internal 893 value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n 894 \ type: string\n kind:\n description: 'Kind is a 895 string value representing the REST resource this\n object represents. 896 Servers may infer this from the endpoint the client\n submits requests 897 to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n 898 \ type: string\n metadata:\n type: object\n spec:\n 899 \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n 900 \ resource.\n properties:\n autoAllocateBlocks:\n 901 \ type: boolean\n maxBlocksPerHost:\n description: 902 MaxBlocksPerHost, if non-zero, is the max number of blocks\n that 903 can be affine to each host.\n maximum: 2147483647\n minimum: 904 0\n type: integer\n strictAffinity:\n type: 905 boolean\n required:\n - autoAllocateBlocks\n - 906 strictAffinity\n type: object\n type: object\n served: true\n 907 \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n 908 \ conditions: []\n storedVersions: []\n" 909 kind: ConfigMap 910 metadata: 911 annotations: 912 note: generated 913 labels: 914 type: generated 915 name: cni-${CLUSTER_NAME}-calico-windows 916 namespace: default 917 --- 918 apiVersion: v1 919 data: 920 csi-proxy: | 921 apiVersion: apps/v1 922 kind: DaemonSet 923 metadata: 924 labels: 925 k8s-app: csi-proxy 926 name: csi-proxy 927 namespace: kube-system 928 spec: 929 selector: 930 matchLabels: 931 k8s-app: csi-proxy 932 template: 933 metadata: 934 labels: 935 k8s-app: csi-proxy 936 spec: 937 nodeSelector: 938 "kubernetes.io/os": windows 939 securityContext: 940 windowsOptions: 941 hostProcess: true 942 runAsUserName: "NT AUTHORITY\\SYSTEM" 943 hostNetwork: true 944 containers: 945 - name: csi-proxy 946 image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 947 kind: ConfigMap 948 metadata: 949 annotations: 950 note: generated 951 labels: 952 type: generated 953 name: csi-proxy-addon 954 namespace: default 955 --- 956 apiVersion: v1 957 data: 958 containerd-windows-logger: | 959 apiVersion: apps/v1 960 kind: DaemonSet 961 metadata: 962 labels: 963 k8s-app: containerd-logger 964 name: containerd-logger 965 namespace: kube-system 966 spec: 967 selector: 968 matchLabels: 969 k8s-app: containerd-logger 970 template: 971 metadata: 972 labels: 973 k8s-app: containerd-logger 974 spec: 975 securityContext: 976 windowsOptions: 977 hostProcess: true 978 runAsUserName: "NT AUTHORITY\\system" 979 hostNetwork: true 980 containers: 981 - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 982 args: [ "config.json" ] 983 name: containerd-logger 984 imagePullPolicy: Always 985 volumeMounts: 986 - name: containerd-logger-config 987 mountPath: /config.json 988 subPath: config.json 989 nodeSelector: 990 kubernetes.io/os: windows 991 tolerations: 992 - key: CriticalAddonsOnly 993 operator: Exists 994 - operator: Exists 995 volumes: 996 - configMap: 997 name: containerd-logger-config 998 name: containerd-logger-config 999 updateStrategy: 1000 type: RollingUpdate 1001 --- 1002 kind: ConfigMap 1003 apiVersion: v1 1004 metadata: 1005 name: containerd-logger-config 1006 namespace: kube-system 1007 data: 1008 config.json: | 1009 { 1010 "inputs": [ 1011 { 1012 "type": "ETW", 1013 "sessionNamePrefix": "containerd", 1014 "cleanupOldSessions": true, 1015 "reuseExistingSession": true, 1016 "providers": [ 1017 { 1018 "providerName": "Microsoft.Virtualization.RunHCS", 1019 "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", 1020 "level": "Verbose" 1021 }, 1022 { 1023 "providerName": "ContainerD", 1024 "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", 1025 "level": "Verbose" 1026 } 1027 ] 1028 } 1029 ], 1030 "filters": [ 1031 { 1032 "type": "drop", 1033 "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" 1034 }, 1035 { 1036 "type": "drop", 1037 "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" 1038 }, 1039 { 1040 "type": "drop", 1041 "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" 1042 }, 1043 { 1044 "type": "drop", 1045 "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" 1046 }, 1047 { 1048 "type": "drop", 1049 "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" 1050 }, 1051 { 1052 "type": "drop", 1053 "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" 1054 }, 1055 { 1056 "type": "drop", 1057 "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" 1058 } 1059 ], 1060 "outputs": [ 1061 { 1062 "type": "StdOutput" 1063 } 1064 ], 1065 "schemaVersion": "2016-08-11" 1066 } 1067 kind: ConfigMap 1068 metadata: 1069 annotations: 1070 note: generated 1071 labels: 1072 type: generated 1073 name: containerd-logger-${CLUSTER_NAME} 1074 namespace: default 1075 --- 1076 apiVersion: addons.cluster.x-k8s.io/v1beta1 1077 kind: ClusterResourceSet 1078 metadata: 1079 name: metrics-server-${CLUSTER_NAME} 1080 namespace: default 1081 spec: 1082 clusterSelector: 1083 matchLabels: 1084 metrics-server: enabled 1085 resources: 1086 - kind: ConfigMap 1087 name: metrics-server-${CLUSTER_NAME} 1088 strategy: ApplyOnce 1089 --- 1090 apiVersion: v1 1091 data: 1092 metrics-server: | 1093 apiVersion: v1 1094 kind: ServiceAccount 1095 metadata: 1096 labels: 1097 k8s-app: metrics-server 1098 name: metrics-server 1099 namespace: kube-system 1100 --- 1101 apiVersion: rbac.authorization.k8s.io/v1 1102 kind: ClusterRole 1103 metadata: 1104 labels: 1105 k8s-app: metrics-server 1106 rbac.authorization.k8s.io/aggregate-to-admin: "true" 1107 rbac.authorization.k8s.io/aggregate-to-edit: "true" 1108 rbac.authorization.k8s.io/aggregate-to-view: "true" 1109 name: system:aggregated-metrics-reader 1110 rules: 1111 - apiGroups: 1112 - metrics.k8s.io 1113 resources: 1114 - pods 1115 - nodes 1116 verbs: 1117 - get 1118 - list 1119 - watch 1120 --- 1121 apiVersion: rbac.authorization.k8s.io/v1 1122 kind: ClusterRole 1123 metadata: 1124 labels: 1125 k8s-app: metrics-server 1126 name: system:metrics-server 1127 rules: 1128 - apiGroups: 1129 - "" 1130 resources: 1131 - nodes/metrics 1132 verbs: 1133 - get 1134 - apiGroups: 1135 - "" 1136 resources: 1137 - pods 1138 - nodes 1139 verbs: 1140 - get 1141 - list 1142 - watch 1143 --- 1144 apiVersion: rbac.authorization.k8s.io/v1 1145 kind: RoleBinding 1146 metadata: 1147 labels: 1148 k8s-app: metrics-server 1149 name: metrics-server-auth-reader 1150 namespace: kube-system 1151 roleRef: 1152 apiGroup: rbac.authorization.k8s.io 1153 kind: Role 1154 name: extension-apiserver-authentication-reader 1155 subjects: 1156 - kind: ServiceAccount 1157 name: metrics-server 1158 namespace: kube-system 1159 --- 1160 apiVersion: rbac.authorization.k8s.io/v1 1161 kind: ClusterRoleBinding 1162 metadata: 1163 labels: 1164 k8s-app: metrics-server 1165 name: metrics-server:system:auth-delegator 1166 roleRef: 1167 apiGroup: rbac.authorization.k8s.io 1168 kind: ClusterRole 1169 name: system:auth-delegator 1170 subjects: 1171 - kind: ServiceAccount 1172 name: metrics-server 1173 namespace: kube-system 1174 --- 1175 apiVersion: rbac.authorization.k8s.io/v1 1176 kind: ClusterRoleBinding 1177 metadata: 1178 labels: 1179 k8s-app: metrics-server 1180 name: system:metrics-server 1181 roleRef: 1182 apiGroup: rbac.authorization.k8s.io 1183 kind: ClusterRole 1184 name: system:metrics-server 1185 subjects: 1186 - kind: ServiceAccount 1187 name: metrics-server 1188 namespace: kube-system 1189 --- 1190 apiVersion: v1 1191 kind: Service 1192 metadata: 1193 labels: 1194 k8s-app: metrics-server 1195 name: metrics-server 1196 namespace: kube-system 1197 spec: 1198 ports: 1199 - name: https 1200 port: 443 1201 protocol: TCP 1202 targetPort: https 1203 selector: 1204 k8s-app: metrics-server 1205 --- 1206 apiVersion: apps/v1 1207 kind: Deployment 1208 metadata: 1209 labels: 1210 k8s-app: metrics-server 1211 name: metrics-server 1212 namespace: kube-system 1213 spec: 1214 selector: 1215 matchLabels: 1216 k8s-app: metrics-server 1217 strategy: 1218 rollingUpdate: 1219 maxUnavailable: 0 1220 template: 1221 metadata: 1222 labels: 1223 k8s-app: metrics-server 1224 spec: 1225 containers: 1226 - args: 1227 - --cert-dir=/tmp 1228 - --secure-port=4443 1229 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname 1230 - --kubelet-use-node-status-port 1231 - --metric-resolution=15s 1232 - --kubelet-insecure-tls 1233 image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 1234 imagePullPolicy: IfNotPresent 1235 livenessProbe: 1236 failureThreshold: 3 1237 httpGet: 1238 path: /livez 1239 port: https 1240 scheme: HTTPS 1241 periodSeconds: 10 1242 name: metrics-server 1243 ports: 1244 - containerPort: 4443 1245 name: https 1246 protocol: TCP 1247 readinessProbe: 1248 failureThreshold: 3 1249 httpGet: 1250 path: /readyz 1251 port: https 1252 scheme: HTTPS 1253 initialDelaySeconds: 20 1254 periodSeconds: 10 1255 resources: 1256 requests: 1257 cpu: 100m 1258 memory: 200Mi 1259 securityContext: 1260 allowPrivilegeEscalation: false 1261 readOnlyRootFilesystem: true 1262 runAsNonRoot: true 1263 runAsUser: 1000 1264 volumeMounts: 1265 - mountPath: /tmp 1266 name: tmp-dir 1267 nodeSelector: 1268 kubernetes.io/os: linux 1269 priorityClassName: system-cluster-critical 1270 serviceAccountName: metrics-server 1271 tolerations: 1272 - effect: NoSchedule 1273 key: node-role.kubernetes.io/master 1274 operator: Exists 1275 - effect: NoSchedule 1276 key: node-role.kubernetes.io/control-plane 1277 operator: Exists 1278 volumes: 1279 - emptyDir: {} 1280 name: tmp-dir 1281 --- 1282 apiVersion: apiregistration.k8s.io/v1 1283 kind: APIService 1284 metadata: 1285 labels: 1286 k8s-app: metrics-server 1287 name: v1beta1.metrics.k8s.io 1288 spec: 1289 group: metrics.k8s.io 1290 groupPriorityMinimum: 100 1291 insecureSkipTLSVerify: true 1292 service: 1293 name: metrics-server 1294 namespace: kube-system 1295 version: v1beta1 1296 versionPriority: 100 1297 kind: ConfigMap 1298 metadata: 1299 annotations: 1300 note: generated 1301 labels: 1302 type: generated 1303 name: metrics-server-${CLUSTER_NAME} 1304 namespace: default