k8s.io/kubernetes@v1.29.3/test/integration/scheduler/filters/filters_test.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package filters 18 19 import ( 20 "context" 21 "fmt" 22 "testing" 23 "time" 24 25 v1 "k8s.io/api/core/v1" 26 apierrors "k8s.io/apimachinery/pkg/api/errors" 27 "k8s.io/apimachinery/pkg/api/resource" 28 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 "k8s.io/apimachinery/pkg/util/wait" 30 utilfeature "k8s.io/apiserver/pkg/util/feature" 31 "k8s.io/client-go/kubernetes" 32 featuregatetesting "k8s.io/component-base/featuregate/testing" 33 "k8s.io/component-helpers/storage/volume" 34 "k8s.io/kubernetes/pkg/features" 35 st "k8s.io/kubernetes/pkg/scheduler/testing" 36 testutils "k8s.io/kubernetes/test/integration/util" 37 imageutils "k8s.io/kubernetes/test/utils/image" 38 "k8s.io/kubernetes/test/utils/ktesting" 39 "k8s.io/utils/pointer" 40 ) 41 42 var ( 43 createAndWaitForNodesInCache = testutils.CreateAndWaitForNodesInCache 44 createNamespacesWithLabels = testutils.CreateNamespacesWithLabels 45 createNode = testutils.CreateNode 46 createPausePod = testutils.CreatePausePod 47 deletePod = testutils.DeletePod 48 getPod = testutils.GetPod 49 initPausePod = testutils.InitPausePod 50 initTest = testutils.InitTestSchedulerWithNS 51 podScheduledIn = testutils.PodScheduledIn 52 podUnschedulable = testutils.PodUnschedulable 53 waitForPodUnschedulable = testutils.WaitForPodUnschedulable 54 ) 55 56 // This file tests the scheduler predicates functionality. 57 58 const pollInterval = 100 * time.Millisecond 59 60 var ( 61 ignorePolicy = v1.NodeInclusionPolicyIgnore 62 honorPolicy = v1.NodeInclusionPolicyHonor 63 taints = []v1.Taint{{Key: v1.TaintNodeUnschedulable, Value: "", Effect: v1.TaintEffectNoSchedule}} 64 ) 65 66 // TestInterPodAffinity verifies that scheduler's inter pod affinity and 67 // anti-affinity predicate functions works correctly. 68 func TestInterPodAffinity(t *testing.T) { 69 podLabel := map[string]string{"service": "securityscan"} 70 podLabel2 := map[string]string{"security": "S1"} 71 72 defaultNS := "ns1" 73 74 tests := []struct { 75 name string 76 pod *v1.Pod 77 pods []*v1.Pod 78 fits bool 79 enableMatchLabelKeysInAffinity bool 80 errorType string 81 }{ 82 { 83 name: "validates that a pod with an invalid podAffinity is rejected because of the LabelSelectorRequirement is invalid", 84 pod: &v1.Pod{ 85 ObjectMeta: metav1.ObjectMeta{ 86 Name: "fakename", 87 Labels: podLabel2, 88 }, 89 Spec: v1.PodSpec{ 90 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 91 Affinity: &v1.Affinity{ 92 PodAffinity: &v1.PodAffinity{ 93 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 94 { 95 LabelSelector: &metav1.LabelSelector{ 96 MatchExpressions: []metav1.LabelSelectorRequirement{ 97 { 98 Key: "security", 99 Operator: metav1.LabelSelectorOpDoesNotExist, 100 }, 101 }, 102 }, 103 TopologyKey: "region", 104 }, 105 }, 106 }, 107 }, 108 }, 109 }, 110 fits: false, 111 errorType: "invalidPod", 112 }, 113 { 114 name: "validates that Inter-pod-Affinity is respected if not matching", 115 pod: &v1.Pod{ 116 ObjectMeta: metav1.ObjectMeta{ 117 Name: "fakename", 118 Labels: podLabel2, 119 }, 120 Spec: v1.PodSpec{ 121 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 122 Affinity: &v1.Affinity{ 123 PodAffinity: &v1.PodAffinity{ 124 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 125 { 126 LabelSelector: &metav1.LabelSelector{ 127 MatchExpressions: []metav1.LabelSelectorRequirement{ 128 { 129 Key: "security", 130 Operator: metav1.LabelSelectorOpIn, 131 Values: []string{"securityscan"}, 132 }, 133 }, 134 }, 135 TopologyKey: "region", 136 }, 137 }, 138 }, 139 }, 140 }, 141 }, 142 fits: false, 143 }, 144 { 145 name: "validates that InterPodAffinity is respected if matching. requiredDuringSchedulingIgnoredDuringExecution in PodAffinity using In operator that matches the existing pod", 146 pod: &v1.Pod{ 147 ObjectMeta: metav1.ObjectMeta{ 148 Name: "fakename", 149 Labels: podLabel2, 150 }, 151 Spec: v1.PodSpec{ 152 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 153 Affinity: &v1.Affinity{ 154 PodAffinity: &v1.PodAffinity{ 155 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 156 { 157 LabelSelector: &metav1.LabelSelector{ 158 MatchExpressions: []metav1.LabelSelectorRequirement{ 159 { 160 Key: "service", 161 Operator: metav1.LabelSelectorOpIn, 162 Values: []string{"securityscan", "value2"}, 163 }, 164 }, 165 }, 166 TopologyKey: "region", 167 }, 168 }, 169 }, 170 }, 171 }, 172 }, 173 pods: []*v1.Pod{{ 174 ObjectMeta: metav1.ObjectMeta{ 175 Name: "fakename2", 176 Labels: podLabel, 177 }, 178 Spec: v1.PodSpec{ 179 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 180 NodeName: "testnode-0", 181 }, 182 }, 183 }, 184 fits: true, 185 }, 186 { 187 name: "validates that InterPodAffinity is respected if matching. requiredDuringSchedulingIgnoredDuringExecution in PodAffinity using not in operator in labelSelector that matches the existing pod", 188 pod: &v1.Pod{ 189 ObjectMeta: metav1.ObjectMeta{ 190 Name: "fakename", 191 Labels: podLabel2, 192 }, 193 Spec: v1.PodSpec{ 194 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 195 Affinity: &v1.Affinity{ 196 PodAffinity: &v1.PodAffinity{ 197 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 198 { 199 LabelSelector: &metav1.LabelSelector{ 200 MatchExpressions: []metav1.LabelSelectorRequirement{ 201 { 202 Key: "service", 203 Operator: metav1.LabelSelectorOpNotIn, 204 Values: []string{"securityscan3", "value3"}, 205 }, 206 }, 207 }, 208 TopologyKey: "region", 209 }, 210 }, 211 }, 212 }, 213 }, 214 }, 215 pods: []*v1.Pod{{Spec: v1.PodSpec{ 216 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 217 NodeName: "testnode-0"}, 218 ObjectMeta: metav1.ObjectMeta{ 219 Name: "fakename2", 220 Labels: podLabel}}}, 221 fits: true, 222 }, 223 { 224 name: "validates that inter-pod-affinity is respected when pods have different Namespaces", 225 pod: &v1.Pod{ 226 ObjectMeta: metav1.ObjectMeta{ 227 Name: "fakename", 228 Labels: podLabel2, 229 }, 230 Spec: v1.PodSpec{ 231 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 232 Affinity: &v1.Affinity{ 233 PodAffinity: &v1.PodAffinity{ 234 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 235 { 236 LabelSelector: &metav1.LabelSelector{ 237 MatchExpressions: []metav1.LabelSelectorRequirement{ 238 { 239 Key: "service", 240 Operator: metav1.LabelSelectorOpIn, 241 Values: []string{"securityscan", "value2"}, 242 }, 243 }, 244 }, 245 TopologyKey: "region", 246 Namespaces: []string{"diff-namespace"}, 247 }, 248 }, 249 }, 250 }, 251 }, 252 }, 253 pods: []*v1.Pod{{Spec: v1.PodSpec{ 254 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 255 NodeName: "testnode-0"}, 256 ObjectMeta: metav1.ObjectMeta{ 257 Name: "fakename2", 258 Labels: podLabel, Namespace: "ns2"}}}, 259 fits: false, 260 }, 261 { 262 name: "Doesn't satisfy the PodAffinity because of unmatching labelSelector with the existing pod", 263 pod: &v1.Pod{ 264 ObjectMeta: metav1.ObjectMeta{ 265 Name: "fakename", 266 Labels: podLabel, 267 }, 268 Spec: v1.PodSpec{ 269 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 270 Affinity: &v1.Affinity{ 271 PodAffinity: &v1.PodAffinity{ 272 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 273 { 274 LabelSelector: &metav1.LabelSelector{ 275 MatchExpressions: []metav1.LabelSelectorRequirement{ 276 { 277 Key: "service", 278 Operator: metav1.LabelSelectorOpIn, 279 Values: []string{"antivirusscan", "value2"}, 280 }, 281 }, 282 }, 283 TopologyKey: "region", 284 }, 285 }, 286 }, 287 }, 288 }, 289 }, 290 pods: []*v1.Pod{{Spec: v1.PodSpec{ 291 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 292 NodeName: "testnode-0"}, ObjectMeta: metav1.ObjectMeta{ 293 Name: "fakename2", 294 Labels: podLabel}}}, 295 fits: false, 296 }, 297 { 298 name: "validates that InterPodAffinity is respected if matching with multiple affinities in multiple RequiredDuringSchedulingIgnoredDuringExecution ", 299 pod: &v1.Pod{ 300 ObjectMeta: metav1.ObjectMeta{ 301 Name: "fakename", 302 Labels: podLabel2, 303 }, 304 Spec: v1.PodSpec{ 305 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 306 Affinity: &v1.Affinity{ 307 PodAffinity: &v1.PodAffinity{ 308 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 309 { 310 LabelSelector: &metav1.LabelSelector{ 311 MatchExpressions: []metav1.LabelSelectorRequirement{ 312 { 313 Key: "service", 314 Operator: metav1.LabelSelectorOpExists, 315 }, { 316 Key: "wrongkey", 317 Operator: metav1.LabelSelectorOpDoesNotExist, 318 }, 319 }, 320 }, 321 TopologyKey: "region", 322 }, { 323 LabelSelector: &metav1.LabelSelector{ 324 MatchExpressions: []metav1.LabelSelectorRequirement{ 325 { 326 Key: "service", 327 Operator: metav1.LabelSelectorOpIn, 328 Values: []string{"securityscan"}, 329 }, { 330 Key: "service", 331 Operator: metav1.LabelSelectorOpNotIn, 332 Values: []string{"WrongValue"}, 333 }, 334 }, 335 }, 336 TopologyKey: "region", 337 }, 338 }, 339 }, 340 }, 341 }, 342 }, 343 pods: []*v1.Pod{{Spec: v1.PodSpec{ 344 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 345 NodeName: "testnode-0"}, ObjectMeta: metav1.ObjectMeta{ 346 Name: "fakename2", 347 Labels: podLabel}}}, 348 fits: true, 349 }, 350 { 351 name: "The labelSelector requirements(items of matchExpressions) are ANDed, the pod cannot schedule onto the node because one of the matchExpression items doesn't match.", 352 pod: &v1.Pod{ 353 ObjectMeta: metav1.ObjectMeta{ 354 Labels: podLabel2, 355 Name: "fakename", 356 }, 357 Spec: v1.PodSpec{ 358 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 359 Affinity: &v1.Affinity{ 360 PodAffinity: &v1.PodAffinity{ 361 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 362 { 363 LabelSelector: &metav1.LabelSelector{ 364 MatchExpressions: []metav1.LabelSelectorRequirement{ 365 { 366 Key: "service", 367 Operator: metav1.LabelSelectorOpExists, 368 }, { 369 Key: "wrongkey", 370 Operator: metav1.LabelSelectorOpDoesNotExist, 371 }, 372 }, 373 }, 374 TopologyKey: "region", 375 }, { 376 LabelSelector: &metav1.LabelSelector{ 377 MatchExpressions: []metav1.LabelSelectorRequirement{ 378 { 379 Key: "service", 380 Operator: metav1.LabelSelectorOpIn, 381 Values: []string{"securityscan2"}, 382 }, { 383 Key: "service", 384 Operator: metav1.LabelSelectorOpNotIn, 385 Values: []string{"WrongValue"}, 386 }, 387 }, 388 }, 389 TopologyKey: "region", 390 }, 391 }, 392 }, 393 }, 394 }, 395 }, 396 pods: []*v1.Pod{{Spec: v1.PodSpec{ 397 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 398 NodeName: "testnode-0"}, ObjectMeta: metav1.ObjectMeta{ 399 Name: "fakename2", 400 Labels: podLabel}}}, 401 fits: false, 402 }, 403 { 404 name: "validates that InterPod Affinity and AntiAffinity is respected if matching", 405 pod: &v1.Pod{ 406 ObjectMeta: metav1.ObjectMeta{ 407 Name: "fakename", 408 Labels: podLabel2, 409 }, 410 Spec: v1.PodSpec{ 411 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 412 Affinity: &v1.Affinity{ 413 PodAffinity: &v1.PodAffinity{ 414 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 415 { 416 LabelSelector: &metav1.LabelSelector{ 417 MatchExpressions: []metav1.LabelSelectorRequirement{ 418 { 419 Key: "service", 420 Operator: metav1.LabelSelectorOpIn, 421 Values: []string{"securityscan", "value2"}, 422 }, 423 }, 424 }, 425 TopologyKey: "region", 426 }, 427 }, 428 }, 429 PodAntiAffinity: &v1.PodAntiAffinity{ 430 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 431 { 432 LabelSelector: &metav1.LabelSelector{ 433 MatchExpressions: []metav1.LabelSelectorRequirement{ 434 { 435 Key: "service", 436 Operator: metav1.LabelSelectorOpIn, 437 Values: []string{"antivirusscan", "value2"}, 438 }, 439 }, 440 }, 441 TopologyKey: "node", 442 }, 443 }, 444 }, 445 }, 446 }, 447 }, 448 pods: []*v1.Pod{{Spec: v1.PodSpec{ 449 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 450 NodeName: "testnode-0"}, ObjectMeta: metav1.ObjectMeta{ 451 Name: "fakename2", 452 Labels: podLabel}}}, 453 fits: true, 454 }, 455 { 456 name: "satisfies the PodAffinity and PodAntiAffinity and PodAntiAffinity symmetry with the existing pod", 457 pod: &v1.Pod{ 458 ObjectMeta: metav1.ObjectMeta{ 459 Name: "fakename", 460 Labels: podLabel2, 461 }, 462 Spec: v1.PodSpec{ 463 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 464 Affinity: &v1.Affinity{ 465 PodAffinity: &v1.PodAffinity{ 466 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 467 { 468 LabelSelector: &metav1.LabelSelector{ 469 MatchExpressions: []metav1.LabelSelectorRequirement{ 470 { 471 Key: "service", 472 Operator: metav1.LabelSelectorOpIn, 473 Values: []string{"securityscan", "value2"}, 474 }, 475 }, 476 }, 477 TopologyKey: "region", 478 }, 479 }, 480 }, 481 PodAntiAffinity: &v1.PodAntiAffinity{ 482 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 483 { 484 LabelSelector: &metav1.LabelSelector{ 485 MatchExpressions: []metav1.LabelSelectorRequirement{ 486 { 487 Key: "service", 488 Operator: metav1.LabelSelectorOpIn, 489 Values: []string{"antivirusscan", "value2"}, 490 }, 491 }, 492 }, 493 TopologyKey: "node", 494 }, 495 }, 496 }, 497 }, 498 }, 499 }, 500 pods: []*v1.Pod{ 501 { 502 Spec: v1.PodSpec{ 503 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 504 NodeName: "testnode-0", 505 Affinity: &v1.Affinity{ 506 PodAntiAffinity: &v1.PodAntiAffinity{ 507 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 508 { 509 LabelSelector: &metav1.LabelSelector{ 510 MatchExpressions: []metav1.LabelSelectorRequirement{ 511 { 512 Key: "service", 513 Operator: metav1.LabelSelectorOpIn, 514 Values: []string{"antivirusscan", "value2"}, 515 }, 516 }, 517 }, 518 TopologyKey: "node", 519 }, 520 }, 521 }, 522 }, 523 }, 524 ObjectMeta: metav1.ObjectMeta{ 525 Name: "fakename2", 526 Labels: podLabel}, 527 }, 528 }, 529 fits: true, 530 }, 531 { 532 name: "satisfies the PodAffinity but doesn't satisfies the PodAntiAffinity with the existing pod", 533 pod: &v1.Pod{ 534 ObjectMeta: metav1.ObjectMeta{ 535 Name: "fakename", 536 Labels: podLabel2, 537 }, 538 Spec: v1.PodSpec{ 539 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 540 Affinity: &v1.Affinity{ 541 PodAffinity: &v1.PodAffinity{ 542 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 543 { 544 LabelSelector: &metav1.LabelSelector{ 545 MatchExpressions: []metav1.LabelSelectorRequirement{ 546 { 547 Key: "service", 548 Operator: metav1.LabelSelectorOpIn, 549 Values: []string{"securityscan", "value2"}, 550 }, 551 }, 552 }, 553 TopologyKey: "region", 554 }, 555 }, 556 }, 557 PodAntiAffinity: &v1.PodAntiAffinity{ 558 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 559 { 560 LabelSelector: &metav1.LabelSelector{ 561 MatchExpressions: []metav1.LabelSelectorRequirement{ 562 { 563 Key: "service", 564 Operator: metav1.LabelSelectorOpIn, 565 Values: []string{"securityscan", "value2"}, 566 }, 567 }, 568 }, 569 TopologyKey: "zone", 570 }, 571 }, 572 }, 573 }, 574 }, 575 }, 576 pods: []*v1.Pod{{Spec: v1.PodSpec{ 577 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 578 NodeName: "testnode-0"}, ObjectMeta: metav1.ObjectMeta{ 579 Name: "fakename2", 580 Labels: podLabel}}}, 581 fits: false, 582 }, 583 { 584 name: "satisfies the PodAffinity and PodAntiAffinity but doesn't satisfies PodAntiAffinity symmetry with the existing pod", 585 pod: &v1.Pod{ 586 ObjectMeta: metav1.ObjectMeta{ 587 Name: "fakename", 588 Labels: podLabel, 589 }, 590 Spec: v1.PodSpec{ 591 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 592 Affinity: &v1.Affinity{ 593 PodAffinity: &v1.PodAffinity{ 594 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 595 { 596 LabelSelector: &metav1.LabelSelector{ 597 MatchExpressions: []metav1.LabelSelectorRequirement{ 598 { 599 Key: "service", 600 Operator: metav1.LabelSelectorOpIn, 601 Values: []string{"securityscan", "value2"}, 602 }, 603 }, 604 }, 605 TopologyKey: "region", 606 }, 607 }, 608 }, 609 PodAntiAffinity: &v1.PodAntiAffinity{ 610 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 611 { 612 LabelSelector: &metav1.LabelSelector{ 613 MatchExpressions: []metav1.LabelSelectorRequirement{ 614 { 615 Key: "service", 616 Operator: metav1.LabelSelectorOpIn, 617 Values: []string{"antivirusscan", "value2"}, 618 }, 619 }, 620 }, 621 TopologyKey: "node", 622 }, 623 }, 624 }, 625 }, 626 }, 627 }, 628 pods: []*v1.Pod{ 629 { 630 Spec: v1.PodSpec{ 631 NodeName: "testnode-0", 632 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 633 Affinity: &v1.Affinity{ 634 PodAntiAffinity: &v1.PodAntiAffinity{ 635 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 636 { 637 LabelSelector: &metav1.LabelSelector{ 638 MatchExpressions: []metav1.LabelSelectorRequirement{ 639 { 640 Key: "service", 641 Operator: metav1.LabelSelectorOpIn, 642 Values: []string{"securityscan", "value3"}, 643 }, 644 }, 645 }, 646 TopologyKey: "zone", 647 }, 648 }, 649 }, 650 }, 651 }, 652 ObjectMeta: metav1.ObjectMeta{ 653 Name: "fakename2", 654 Labels: podLabel}, 655 }, 656 }, 657 fits: false, 658 }, 659 { 660 name: "pod matches its own Label in PodAffinity and that matches the existing pod Labels", 661 pod: &v1.Pod{ 662 ObjectMeta: metav1.ObjectMeta{ 663 Name: "fakename", 664 Labels: podLabel, 665 }, 666 Spec: v1.PodSpec{ 667 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 668 Affinity: &v1.Affinity{ 669 PodAffinity: &v1.PodAffinity{ 670 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 671 { 672 LabelSelector: &metav1.LabelSelector{ 673 MatchExpressions: []metav1.LabelSelectorRequirement{ 674 { 675 Key: "service", 676 Operator: metav1.LabelSelectorOpNotIn, 677 Values: []string{"securityscan", "value2"}, 678 }, 679 }, 680 }, 681 TopologyKey: "region", 682 }, 683 }, 684 }, 685 }, 686 }, 687 }, 688 pods: []*v1.Pod{{Spec: v1.PodSpec{ 689 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 690 NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{ 691 Name: "fakename2", 692 Labels: podLabel}}}, 693 fits: false, 694 }, 695 { 696 name: "Verify that PodAntiAffinity of an existing pod is respected when PodAntiAffinity symmetry is not satisfied with the existing pod", 697 pod: &v1.Pod{ 698 ObjectMeta: metav1.ObjectMeta{ 699 Name: "fakename", 700 Labels: podLabel, 701 }, 702 Spec: v1.PodSpec{Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}}, 703 }, 704 pods: []*v1.Pod{ 705 { 706 Spec: v1.PodSpec{NodeName: "testnode-0", 707 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 708 Affinity: &v1.Affinity{ 709 PodAntiAffinity: &v1.PodAntiAffinity{ 710 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 711 { 712 LabelSelector: &metav1.LabelSelector{ 713 MatchExpressions: []metav1.LabelSelectorRequirement{ 714 { 715 Key: "service", 716 Operator: metav1.LabelSelectorOpIn, 717 Values: []string{"securityscan", "value2"}, 718 }, 719 }, 720 }, 721 TopologyKey: "zone", 722 }, 723 }, 724 }, 725 }, 726 }, 727 ObjectMeta: metav1.ObjectMeta{ 728 Name: "fakename2", 729 Labels: podLabel}, 730 }, 731 }, 732 fits: false, 733 }, 734 { 735 name: "Verify that PodAntiAffinity from existing pod is respected when pod statisfies PodAntiAffinity symmetry with the existing pod", 736 pod: &v1.Pod{ 737 ObjectMeta: metav1.ObjectMeta{ 738 Name: "fake-name", 739 Labels: podLabel, 740 }, 741 Spec: v1.PodSpec{Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}}, 742 }, 743 pods: []*v1.Pod{ 744 { 745 Spec: v1.PodSpec{NodeName: "testnode-0", 746 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 747 Affinity: &v1.Affinity{ 748 PodAntiAffinity: &v1.PodAntiAffinity{ 749 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 750 { 751 LabelSelector: &metav1.LabelSelector{ 752 MatchExpressions: []metav1.LabelSelectorRequirement{ 753 { 754 Key: "service", 755 Operator: metav1.LabelSelectorOpNotIn, 756 Values: []string{"securityscan", "value2"}, 757 }, 758 }, 759 }, 760 TopologyKey: "zone", 761 }, 762 }, 763 }, 764 }, 765 }, 766 ObjectMeta: metav1.ObjectMeta{ 767 Name: "fake-name2", 768 Labels: podLabel}, 769 }, 770 }, 771 fits: true, 772 }, 773 { 774 name: "nodes[0] and nodes[1] have same topologyKey and label value. nodes[0] has an existing pod that matches the inter pod affinity rule. The new pod can not be scheduled onto either of the two nodes.", 775 pod: &v1.Pod{ 776 ObjectMeta: metav1.ObjectMeta{Name: "fake-name2"}, 777 Spec: v1.PodSpec{ 778 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 779 NodeSelector: map[string]string{"region": "r1"}, 780 Affinity: &v1.Affinity{ 781 PodAntiAffinity: &v1.PodAntiAffinity{ 782 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 783 { 784 LabelSelector: &metav1.LabelSelector{ 785 MatchExpressions: []metav1.LabelSelectorRequirement{ 786 { 787 Key: "foo", 788 Operator: metav1.LabelSelectorOpIn, 789 Values: []string{"abc"}, 790 }, 791 }, 792 }, 793 TopologyKey: "region", 794 }, 795 }, 796 }, 797 }, 798 }, 799 }, 800 pods: []*v1.Pod{ 801 {Spec: v1.PodSpec{ 802 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 803 NodeName: "testnode-0"}, ObjectMeta: metav1.ObjectMeta{Name: "fakename", Labels: map[string]string{"foo": "abc"}}}, 804 }, 805 fits: false, 806 }, 807 { 808 name: "anti affinity: matchLabelKeys is merged into LabelSelector with In operator (feature flag: enabled)", 809 pod: &v1.Pod{ 810 Spec: v1.PodSpec{ 811 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 812 Affinity: &v1.Affinity{ 813 PodAntiAffinity: &v1.PodAntiAffinity{ 814 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 815 { 816 TopologyKey: "zone", 817 LabelSelector: &metav1.LabelSelector{ 818 MatchExpressions: []metav1.LabelSelectorRequirement{ 819 { 820 Key: "foo", 821 Operator: metav1.LabelSelectorOpExists, 822 }, 823 }, 824 }, 825 MatchLabelKeys: []string{"bar"}, 826 }, 827 }, 828 }, 829 }, 830 }, 831 ObjectMeta: metav1.ObjectMeta{ 832 Name: "incoming", 833 Labels: map[string]string{"foo": "", "bar": "a"}, 834 }, 835 }, 836 pods: []*v1.Pod{ 837 // It matches the incoming Pod's anti affinity's labelSelector. 838 // BUT, the matchLabelKeys make the existing Pod's anti affinity's labelSelector not match with this label. 839 { 840 Spec: v1.PodSpec{ 841 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 842 NodeName: "testnode-0", 843 }, 844 ObjectMeta: metav1.ObjectMeta{Name: "pod1", Labels: map[string]string{"foo": "", "bar": "fuga"}}, 845 }, 846 // It matches the incoming Pod's anti affinity's labelSelector. 847 // BUT, the matchLabelKeys make the existing Pod's anti affinity's labelSelector not match with this label. 848 { 849 Spec: v1.PodSpec{ 850 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 851 NodeName: "testnode-0", 852 }, 853 ObjectMeta: metav1.ObjectMeta{Name: "pod2", Labels: map[string]string{"foo": "", "bar": "hoge"}}, 854 }, 855 }, 856 enableMatchLabelKeysInAffinity: true, 857 fits: true, 858 }, 859 { 860 name: "anti affinity: mismatchLabelKeys is merged into LabelSelector with NotIn operator (feature flag: enabled)", 861 pod: &v1.Pod{ 862 Spec: v1.PodSpec{ 863 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 864 Affinity: &v1.Affinity{ 865 PodAntiAffinity: &v1.PodAntiAffinity{ 866 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 867 { 868 TopologyKey: "zone", 869 LabelSelector: &metav1.LabelSelector{ 870 MatchExpressions: []metav1.LabelSelectorRequirement{ 871 { 872 Key: "foo", 873 Operator: metav1.LabelSelectorOpExists, 874 }, 875 }, 876 }, 877 MismatchLabelKeys: []string{"bar"}, 878 }, 879 }, 880 }, 881 }, 882 }, 883 ObjectMeta: metav1.ObjectMeta{ 884 Name: "incoming", 885 Labels: map[string]string{"foo": "", "bar": "a"}, 886 }, 887 }, 888 pods: []*v1.Pod{ 889 // It matches the incoming Pod's anti affinity's labelSelector. 890 // BUT, the mismatchLabelKeys make the existing Pod's anti affinity's labelSelector not match with this label. 891 { 892 Spec: v1.PodSpec{ 893 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 894 NodeName: "testnode-0", 895 }, 896 ObjectMeta: metav1.ObjectMeta{Name: "pod1", Labels: map[string]string{"foo": "", "bar": "a"}}, 897 }, 898 // It matches the incoming Pod's anti affinity's labelSelector. 899 // BUT, the mismatchLabelKeys make the existing Pod's anti affinity's labelSelector not match with this label. 900 { 901 Spec: v1.PodSpec{ 902 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 903 NodeName: "testnode-0", 904 }, 905 ObjectMeta: metav1.ObjectMeta{Name: "pod2", Labels: map[string]string{"foo": "", "bar": "a"}}, 906 }, 907 }, 908 enableMatchLabelKeysInAffinity: true, 909 fits: true, 910 }, 911 { 912 name: "affinity: matchLabelKeys is merged into LabelSelector with In operator (feature flag: enabled)", 913 pod: &v1.Pod{ 914 Spec: v1.PodSpec{ 915 Containers: []v1.Container{ 916 { 917 Name: "container", 918 Image: imageutils.GetPauseImageName(), 919 Resources: v1.ResourceRequirements{ 920 Requests: v1.ResourceList{ 921 v1.ResourceMemory: resource.MustParse("1G"), 922 }, 923 }, 924 }, 925 }, 926 Affinity: &v1.Affinity{ 927 PodAffinity: &v1.PodAffinity{ 928 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 929 { 930 TopologyKey: "node", 931 LabelSelector: &metav1.LabelSelector{ 932 MatchExpressions: []metav1.LabelSelectorRequirement{ 933 { 934 Key: "foo", 935 Operator: metav1.LabelSelectorOpExists, 936 }, 937 }, 938 }, 939 MatchLabelKeys: []string{"bar"}, 940 }, 941 }, 942 }, 943 }, 944 }, 945 ObjectMeta: metav1.ObjectMeta{ 946 Name: "incoming", 947 Labels: map[string]string{"foo": "", "bar": "a"}, 948 }, 949 }, 950 pods: []*v1.Pod{ 951 { 952 // It matches the incoming affinity. But, it uses all resources on nodes[1]. 953 // So, the incoming Pod can no longer get scheduled on nodes[1]. 954 Spec: v1.PodSpec{ 955 Containers: []v1.Container{ 956 { 957 Name: "container", 958 Image: imageutils.GetPauseImageName(), 959 Resources: v1.ResourceRequirements{ 960 Requests: v1.ResourceList{ 961 v1.ResourceMemory: resource.MustParse("1G"), 962 }, 963 }, 964 }, 965 }, 966 NodeName: "anothernode-0", 967 }, 968 ObjectMeta: metav1.ObjectMeta{Name: "pod1", Labels: map[string]string{"foo": "", "bar": "a"}}, 969 }, 970 { 971 // It doesn't match the incoming affinity due to matchLabelKeys. 972 Spec: v1.PodSpec{ 973 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 974 NodeName: "testnode-0", 975 }, 976 ObjectMeta: metav1.ObjectMeta{Name: "pod2", Labels: map[string]string{"foo": "", "bar": "hoge"}}, 977 }, 978 { 979 // It doesn't match the incoming affinity due to matchLabelKeys. 980 Spec: v1.PodSpec{ 981 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 982 NodeName: "testnode-0", 983 }, 984 ObjectMeta: metav1.ObjectMeta{Name: "pod3", Labels: map[string]string{"foo": "", "bar": "fuga"}}, 985 }, 986 }, 987 enableMatchLabelKeysInAffinity: true, 988 fits: false, 989 }, 990 { 991 name: "affinity: mismatchLabelKeys is merged into LabelSelector with NotIn operator (feature flag: enabled)", 992 pod: &v1.Pod{ 993 Spec: v1.PodSpec{ 994 Containers: []v1.Container{ 995 { 996 Name: "container", 997 Image: imageutils.GetPauseImageName(), 998 Resources: v1.ResourceRequirements{ 999 Requests: v1.ResourceList{ 1000 v1.ResourceMemory: resource.MustParse("1G"), 1001 }, 1002 }, 1003 }, 1004 }, 1005 Affinity: &v1.Affinity{ 1006 PodAffinity: &v1.PodAffinity{ 1007 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 1008 { 1009 TopologyKey: "node", 1010 LabelSelector: &metav1.LabelSelector{ 1011 MatchExpressions: []metav1.LabelSelectorRequirement{ 1012 { 1013 Key: "foo", 1014 Operator: metav1.LabelSelectorOpExists, 1015 }, 1016 }, 1017 }, 1018 MismatchLabelKeys: []string{"bar"}, 1019 }, 1020 }, 1021 }, 1022 }, 1023 }, 1024 ObjectMeta: metav1.ObjectMeta{ 1025 Name: "incoming", 1026 Labels: map[string]string{"foo": "", "bar": "a"}, 1027 }, 1028 }, 1029 pods: []*v1.Pod{ 1030 { 1031 // It matches the incoming affinity. But, it uses all resources on nodes[1]. 1032 // So, the incoming Pod can no longer get scheduled on nodes[1]. 1033 Spec: v1.PodSpec{ 1034 Containers: []v1.Container{ 1035 { 1036 Name: "container", 1037 Image: imageutils.GetPauseImageName(), 1038 Resources: v1.ResourceRequirements{ 1039 Requests: v1.ResourceList{ 1040 v1.ResourceMemory: resource.MustParse("1G"), 1041 }, 1042 }, 1043 }, 1044 }, 1045 NodeName: "anothernode-0", 1046 }, 1047 ObjectMeta: metav1.ObjectMeta{Name: "pod1", Labels: map[string]string{"foo": "", "bar": "fuga"}}, 1048 }, 1049 { 1050 // It doesn't match the incoming affinity due to mismatchLabelKeys. 1051 Spec: v1.PodSpec{ 1052 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 1053 NodeName: "testnode-0", 1054 }, 1055 ObjectMeta: metav1.ObjectMeta{Name: "pod2", Labels: map[string]string{"foo": "", "bar": "a"}}, 1056 }, 1057 { 1058 // It doesn't match the incoming affinity due to mismatchLabelKeys. 1059 Spec: v1.PodSpec{ 1060 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 1061 NodeName: "testnode-0", 1062 }, 1063 ObjectMeta: metav1.ObjectMeta{Name: "pod3", Labels: map[string]string{"foo": "", "bar": "a"}}, 1064 }, 1065 }, 1066 enableMatchLabelKeysInAffinity: true, 1067 fits: false, 1068 }, 1069 } 1070 1071 for _, test := range tests { 1072 t.Run(test.name, func(t *testing.T) { 1073 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MatchLabelKeysInPodAffinity, test.enableMatchLabelKeysInAffinity)() 1074 _, ctx := ktesting.NewTestContext(t) 1075 1076 testCtx := initTest(t, "") 1077 cs := testCtx.ClientSet 1078 1079 if _, err := createNode(cs, st.MakeNode().Name("testnode-0").Label("region", "r1").Label("zone", "z11").Label("node", "n1").Capacity( 1080 map[v1.ResourceName]string{ 1081 v1.ResourceMemory: "1G", 1082 }, 1083 ).Obj()); err != nil { 1084 t.Fatalf("failed to create node: %v", err) 1085 } 1086 1087 // another test node has the same "region" and "zone" labels as testnode, but has a different "node" label. 1088 if _, err := createNode(cs, st.MakeNode().Name("anothernode-0").Label("region", "r1").Label("zone", "z11").Label("node", "n2").Capacity( 1089 map[v1.ResourceName]string{ 1090 v1.ResourceMemory: "1G", 1091 }, 1092 ).Obj()); err != nil { 1093 t.Fatalf("failed to create node: %v", err) 1094 } 1095 1096 if err := createNamespacesWithLabels(cs, []string{"ns1", "ns2"}, map[string]string{"team": "team1"}); err != nil { 1097 t.Fatal(err) 1098 } 1099 if err := createNamespacesWithLabels(cs, []string{"ns3"}, map[string]string{"team": "team2"}); err != nil { 1100 t.Fatal(err) 1101 } 1102 1103 for _, pod := range test.pods { 1104 if pod.Namespace == "" { 1105 pod.Namespace = defaultNS 1106 } 1107 createdPod, err := cs.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) 1108 if err != nil { 1109 t.Fatalf("Error while creating pod: %v", err) 1110 } 1111 err = wait.PollUntilContextTimeout(ctx, pollInterval, wait.ForeverTestTimeout, false, 1112 testutils.PodScheduled(cs, createdPod.Namespace, createdPod.Name)) 1113 if err != nil { 1114 t.Errorf("Error while creating pod: %v", err) 1115 } 1116 } 1117 if test.pod.Namespace == "" { 1118 test.pod.Namespace = defaultNS 1119 } 1120 1121 testPod, err := cs.CoreV1().Pods(test.pod.Namespace).Create(ctx, test.pod, metav1.CreateOptions{}) 1122 if err != nil { 1123 if !(test.errorType == "invalidPod" && apierrors.IsInvalid(err)) { 1124 t.Fatalf("Error while creating pod: %v", err) 1125 } 1126 } 1127 1128 if test.fits { 1129 err = wait.PollUntilContextTimeout(ctx, pollInterval, wait.ForeverTestTimeout, false, 1130 testutils.PodScheduled(cs, testPod.Namespace, testPod.Name)) 1131 } else { 1132 err = wait.PollUntilContextTimeout(ctx, pollInterval, wait.ForeverTestTimeout, false, 1133 podUnschedulable(cs, testPod.Namespace, testPod.Name)) 1134 } 1135 if err != nil { 1136 t.Errorf("Error while trying to fit a pod: %v", err) 1137 return 1138 } 1139 1140 err = cs.CoreV1().Pods(test.pod.Namespace).Delete(ctx, test.pod.Name, *metav1.NewDeleteOptions(0)) 1141 if err != nil { 1142 t.Errorf("Error while deleting pod: %v", err) 1143 } 1144 err = wait.PollUntilContextTimeout(ctx, pollInterval, wait.ForeverTestTimeout, true, 1145 testutils.PodDeleted(ctx, cs, testCtx.NS.Name, test.pod.Name)) 1146 if err != nil { 1147 t.Errorf("Error while waiting for pod to get deleted: %v", err) 1148 } 1149 for _, pod := range test.pods { 1150 err = cs.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1151 if err != nil { 1152 t.Errorf("Error while deleting pod: %v", err) 1153 } 1154 err = wait.PollUntilContextTimeout(ctx, pollInterval, wait.ForeverTestTimeout, true, 1155 testutils.PodDeleted(ctx, cs, pod.Namespace, pod.Name)) 1156 if err != nil { 1157 t.Errorf("Error while waiting for pod to get deleted: %v", err) 1158 } 1159 } 1160 }) 1161 } 1162 } 1163 1164 // TestInterPodAffinityWithNamespaceSelector verifies that inter pod affinity with NamespaceSelector works as expected. 1165 func TestInterPodAffinityWithNamespaceSelector(t *testing.T) { 1166 podLabel := map[string]string{"service": "securityscan"} 1167 tests := []struct { 1168 name string 1169 pod *v1.Pod 1170 existingPod *v1.Pod 1171 fits bool 1172 errorType string 1173 }{ 1174 { 1175 name: "MatchingNamespaces", 1176 pod: &v1.Pod{ 1177 ObjectMeta: metav1.ObjectMeta{ 1178 Name: "pod-ns-selector", 1179 }, 1180 Spec: v1.PodSpec{ 1181 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 1182 Affinity: &v1.Affinity{ 1183 PodAffinity: &v1.PodAffinity{ 1184 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 1185 { 1186 LabelSelector: &metav1.LabelSelector{ 1187 MatchExpressions: []metav1.LabelSelectorRequirement{ 1188 { 1189 Key: "service", 1190 Operator: metav1.LabelSelectorOpIn, 1191 Values: []string{"securityscan"}, 1192 }, 1193 }, 1194 }, 1195 NamespaceSelector: &metav1.LabelSelector{ 1196 MatchExpressions: []metav1.LabelSelectorRequirement{ 1197 { 1198 Key: "team", 1199 Operator: metav1.LabelSelectorOpIn, 1200 Values: []string{"team1"}, 1201 }, 1202 }, 1203 }, 1204 TopologyKey: "region", 1205 }, 1206 }, 1207 }, 1208 }, 1209 }, 1210 }, 1211 existingPod: &v1.Pod{ 1212 ObjectMeta: metav1.ObjectMeta{ 1213 Name: "fakename2", 1214 Labels: podLabel, 1215 Namespace: "ns2", 1216 }, 1217 Spec: v1.PodSpec{ 1218 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 1219 }, 1220 }, 1221 fits: true, 1222 }, 1223 { 1224 name: "MismatchingNamespaces", 1225 pod: &v1.Pod{ 1226 ObjectMeta: metav1.ObjectMeta{ 1227 Name: "pod-ns-selector", 1228 }, 1229 Spec: v1.PodSpec{ 1230 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 1231 Affinity: &v1.Affinity{ 1232 PodAffinity: &v1.PodAffinity{ 1233 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 1234 { 1235 LabelSelector: &metav1.LabelSelector{ 1236 MatchExpressions: []metav1.LabelSelectorRequirement{ 1237 { 1238 Key: "service", 1239 Operator: metav1.LabelSelectorOpIn, 1240 Values: []string{"securityscan"}, 1241 }, 1242 }, 1243 }, 1244 NamespaceSelector: &metav1.LabelSelector{ 1245 MatchExpressions: []metav1.LabelSelectorRequirement{ 1246 { 1247 Key: "team", 1248 Operator: metav1.LabelSelectorOpIn, 1249 Values: []string{"team1"}, 1250 }, 1251 }, 1252 }, 1253 TopologyKey: "region", 1254 }, 1255 }, 1256 }, 1257 }, 1258 }, 1259 }, 1260 existingPod: &v1.Pod{ 1261 ObjectMeta: metav1.ObjectMeta{ 1262 Name: "fakename2", 1263 Labels: podLabel, 1264 Namespace: "ns3", 1265 }, 1266 Spec: v1.PodSpec{ 1267 Containers: []v1.Container{{Name: "container", Image: imageutils.GetPauseImageName()}}, 1268 }, 1269 }, 1270 fits: false, 1271 }, 1272 } 1273 1274 for _, test := range tests { 1275 t.Run(test.name, func(t *testing.T) { 1276 testCtx := initTest(t, "") 1277 1278 // Add a few nodes with labels 1279 nodes, err := createAndWaitForNodesInCache(testCtx, "testnode", st.MakeNode().Label("region", "r1").Label("zone", "z11"), 2) 1280 if err != nil { 1281 t.Fatal(err) 1282 } 1283 test.existingPod.Spec.NodeName = nodes[0].Name 1284 1285 cs := testCtx.ClientSet 1286 1287 if err := createNamespacesWithLabels(cs, []string{"ns1", "ns2"}, map[string]string{"team": "team1"}); err != nil { 1288 t.Fatal(err) 1289 } 1290 if err := createNamespacesWithLabels(cs, []string{"ns3"}, map[string]string{"team": "team2"}); err != nil { 1291 t.Fatal(err) 1292 } 1293 defaultNS := "ns1" 1294 1295 createdPod, err := cs.CoreV1().Pods(test.existingPod.Namespace).Create(testCtx.Ctx, test.existingPod, metav1.CreateOptions{}) 1296 if err != nil { 1297 t.Fatalf("Error while creating pod: %v", err) 1298 } 1299 err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false, 1300 testutils.PodScheduled(cs, createdPod.Namespace, createdPod.Name)) 1301 if err != nil { 1302 t.Errorf("Error while creating pod: %v", err) 1303 } 1304 1305 if test.pod.Namespace == "" { 1306 test.pod.Namespace = defaultNS 1307 } 1308 1309 testPod, err := cs.CoreV1().Pods(test.pod.Namespace).Create(testCtx.Ctx, test.pod, metav1.CreateOptions{}) 1310 if err != nil { 1311 if !(test.errorType == "invalidPod" && apierrors.IsInvalid(err)) { 1312 t.Fatalf("Error while creating pod: %v", err) 1313 } 1314 } 1315 1316 if test.fits { 1317 err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false, 1318 testutils.PodScheduled(cs, testPod.Namespace, testPod.Name)) 1319 } else { 1320 err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false, 1321 podUnschedulable(cs, testPod.Namespace, testPod.Name)) 1322 } 1323 if err != nil { 1324 t.Errorf("Error while trying to fit a pod: %v", err) 1325 } 1326 err = cs.CoreV1().Pods(test.pod.Namespace).Delete(testCtx.Ctx, test.pod.Name, *metav1.NewDeleteOptions(0)) 1327 if err != nil { 1328 t.Errorf("Error while deleting pod: %v", err) 1329 } 1330 err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, true, 1331 testutils.PodDeleted(testCtx.Ctx, cs, testCtx.NS.Name, test.pod.Name)) 1332 if err != nil { 1333 t.Errorf("Error while waiting for pod to get deleted: %v", err) 1334 } 1335 err = cs.CoreV1().Pods(test.existingPod.Namespace).Delete(testCtx.Ctx, test.existingPod.Name, *metav1.NewDeleteOptions(0)) 1336 if err != nil { 1337 t.Errorf("Error while deleting pod: %v", err) 1338 } 1339 err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, true, 1340 testutils.PodDeleted(testCtx.Ctx, cs, test.existingPod.Namespace, test.existingPod.Name)) 1341 if err != nil { 1342 t.Errorf("Error while waiting for pod to get deleted: %v", err) 1343 } 1344 }) 1345 } 1346 } 1347 1348 // TestPodTopologySpreadFilter verifies that EvenPodsSpread predicate functions well. 1349 func TestPodTopologySpreadFilter(t *testing.T) { 1350 pause := imageutils.GetPauseImageName() 1351 // default nodes with labels "zone: zone-{0,1}" and "node: <node name>". 1352 defaultNodes := []*v1.Node{ 1353 st.MakeNode().Name("node-0").Label("node", "node-0").Label("zone", "zone-0").Obj(), 1354 st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-0").Obj(), 1355 st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Obj(), 1356 st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-1").Obj(), 1357 } 1358 1359 tests := []struct { 1360 name string 1361 incomingPod *v1.Pod 1362 existingPods []*v1.Pod 1363 fits bool 1364 nodes []*v1.Node 1365 candidateNodes []string // nodes expected to schedule onto 1366 enableMinDomains bool 1367 enableNodeInclusionPolicy bool 1368 enableMatchLabelKeys bool 1369 }{ 1370 // note: naming starts at index 0 1371 { 1372 name: "place pod on a 1/1/0/1 cluster with MaxSkew=1, node-2 is the only fit", 1373 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1374 SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1375 Obj(), 1376 existingPods: []*v1.Pod{ 1377 st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), 1378 st.MakePod().Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(), 1379 st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), 1380 }, 1381 fits: true, 1382 nodes: defaultNodes, 1383 candidateNodes: []string{"node-2"}, 1384 }, 1385 { 1386 name: "place pod on a 2/0/0/1 cluster with MaxSkew=2, node-{1,2,3} are good fits", 1387 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1388 SpreadConstraint(2, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1389 Obj(), 1390 existingPods: []*v1.Pod{ 1391 st.MakePod().Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(), 1392 st.MakePod().Name("p0b").Node("node-0").Label("foo", "").Container(pause).Obj(), 1393 st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), 1394 }, 1395 fits: true, 1396 nodes: defaultNodes, 1397 candidateNodes: []string{"node-1", "node-2", "node-3"}, 1398 }, 1399 { 1400 name: "pod is required to be placed on zone0, so only node-1 fits", 1401 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1402 NodeAffinityIn("zone", []string{"zone-0"}). 1403 SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1404 Obj(), 1405 existingPods: []*v1.Pod{ 1406 st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), 1407 st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), 1408 }, 1409 fits: true, 1410 nodes: defaultNodes, 1411 candidateNodes: []string{"node-1"}, 1412 }, 1413 { 1414 name: "two constraints: pod can only be placed to zone-1/node-2", 1415 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1416 SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1417 SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1418 Obj(), 1419 existingPods: []*v1.Pod{ 1420 st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), 1421 st.MakePod().Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(), 1422 st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), 1423 st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(), 1424 }, 1425 fits: true, 1426 nodes: defaultNodes, 1427 candidateNodes: []string{"node-2"}, 1428 }, 1429 { 1430 name: "pod cannot be placed onto any node", 1431 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1432 NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster 1433 SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1434 SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1435 Obj(), 1436 existingPods: []*v1.Pod{ 1437 st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1438 st.MakePod().Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), 1439 st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1440 st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), 1441 st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), 1442 }, 1443 fits: false, 1444 nodes: defaultNodes, 1445 }, 1446 { 1447 name: "high priority pod can preempt others", 1448 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).Priority(100). 1449 NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster 1450 SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1451 SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1452 Obj(), 1453 existingPods: []*v1.Pod{ 1454 st.MakePod().ZeroTerminationGracePeriod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1455 st.MakePod().ZeroTerminationGracePeriod().Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), 1456 st.MakePod().ZeroTerminationGracePeriod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1457 st.MakePod().ZeroTerminationGracePeriod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), 1458 st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), 1459 }, 1460 fits: true, 1461 nodes: defaultNodes, 1462 candidateNodes: []string{"node-1", "node-2", "node-3"}, 1463 }, 1464 { 1465 name: "pods spread across nodes as 2/2/1, maxSkew is 2, and the number of domains < minDomains, then the third node fits", 1466 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1467 NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster 1468 SpreadConstraint( 1469 2, 1470 "node", 1471 hardSpread, 1472 st.MakeLabelSelector().Exists("foo").Obj(), 1473 pointer.Int32(4), // larger than the number of domains (= 3) 1474 nil, 1475 nil, 1476 nil, 1477 ). 1478 Obj(), 1479 existingPods: []*v1.Pod{ 1480 st.MakePod().ZeroTerminationGracePeriod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1481 st.MakePod().ZeroTerminationGracePeriod().Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), 1482 st.MakePod().ZeroTerminationGracePeriod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1483 st.MakePod().ZeroTerminationGracePeriod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), 1484 st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), 1485 }, 1486 fits: true, 1487 nodes: defaultNodes, 1488 candidateNodes: []string{"node-3"}, 1489 enableMinDomains: true, 1490 }, 1491 { 1492 name: "pods spread across nodes as 2/2/1, maxSkew is 2, and the number of domains > minDomains, then the all nodes fit", 1493 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1494 NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster 1495 SpreadConstraint( 1496 2, 1497 "node", 1498 hardSpread, 1499 st.MakeLabelSelector().Exists("foo").Obj(), 1500 pointer.Int32(2), // smaller than the number of domains (= 3) 1501 nil, 1502 nil, 1503 nil, 1504 ). 1505 Obj(), 1506 existingPods: []*v1.Pod{ 1507 st.MakePod().ZeroTerminationGracePeriod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1508 st.MakePod().ZeroTerminationGracePeriod().Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), 1509 st.MakePod().ZeroTerminationGracePeriod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1510 st.MakePod().ZeroTerminationGracePeriod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), 1511 st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), 1512 }, 1513 fits: true, 1514 nodes: defaultNodes, 1515 candidateNodes: []string{"node-1", "node-2", "node-3"}, 1516 enableMinDomains: true, 1517 }, 1518 { 1519 name: "pods spread across zone as 2/1, maxSkew is 2 and the number of domains < minDomains, then the third and fourth nodes fit", 1520 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1521 SpreadConstraint( 1522 2, 1523 "zone", 1524 v1.DoNotSchedule, 1525 st.MakeLabelSelector().Exists("foo").Obj(), 1526 pointer.Int32(3), // larger than the number of domains(2) 1527 nil, 1528 nil, 1529 nil, 1530 ).Obj(), 1531 existingPods: []*v1.Pod{ 1532 st.MakePod().Name("p1a").Node("node-0").Label("foo", "").Container(pause).Obj(), 1533 st.MakePod().Name("p2a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1534 st.MakePod().Name("p3a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1535 }, 1536 fits: true, 1537 nodes: defaultNodes, 1538 candidateNodes: []string{"node-2", "node-3"}, 1539 enableMinDomains: true, 1540 }, 1541 { 1542 name: "pods spread across zone as 2/1, maxSkew is 2 and the number of domains < minDomains, then the all nodes fit", 1543 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1544 SpreadConstraint( 1545 2, 1546 "zone", 1547 v1.DoNotSchedule, 1548 st.MakeLabelSelector().Exists("foo").Obj(), 1549 pointer.Int32(1), // smaller than the number of domains(2) 1550 nil, 1551 nil, 1552 nil, 1553 ).Obj(), 1554 existingPods: []*v1.Pod{ 1555 st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1556 st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1557 st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), 1558 }, 1559 fits: true, 1560 nodes: defaultNodes, 1561 candidateNodes: []string{"node-0", "node-1", "node-2", "node-3"}, 1562 enableMinDomains: true, 1563 }, 1564 { 1565 name: "NodeAffinityPolicy honored with labelSelectors, pods spread across zone as 2/1", 1566 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1567 NodeSelector(map[string]string{"foo": ""}). 1568 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1569 Obj(), 1570 existingPods: []*v1.Pod{ 1571 st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1572 st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1573 st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), 1574 st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), 1575 }, 1576 fits: true, 1577 nodes: []*v1.Node{ 1578 st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), 1579 st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(), 1580 st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), 1581 st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), 1582 }, 1583 candidateNodes: []string{"node-4"}, // node-3 is filtered out by NodeAffinity plugin 1584 enableNodeInclusionPolicy: true, 1585 }, 1586 { 1587 name: "NodeAffinityPolicy ignored with nodeAffinity, pods spread across zone as 1/~2~", 1588 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1589 NodeAffinityIn("foo", []string{""}). 1590 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, &ignorePolicy, nil, nil). 1591 Obj(), 1592 existingPods: []*v1.Pod{ 1593 st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1594 st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), 1595 st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), 1596 }, 1597 fits: true, 1598 nodes: []*v1.Node{ 1599 st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), 1600 st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(), 1601 st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), 1602 st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), 1603 }, 1604 candidateNodes: []string{"node-1", "node-2"}, 1605 enableNodeInclusionPolicy: true, 1606 }, 1607 { 1608 name: "NodeTaintsPolicy honored, pods spread across zone as 2/1", 1609 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1610 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, &honorPolicy, nil). 1611 Obj(), 1612 existingPods: []*v1.Pod{ 1613 st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1614 st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1615 st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), 1616 st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), 1617 }, 1618 fits: true, 1619 nodes: []*v1.Node{ 1620 st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), 1621 st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(), 1622 st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Taints(taints).Obj(), 1623 st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), 1624 }, 1625 candidateNodes: []string{"node-4"}, // node-3 is filtered out by TaintToleration plugin 1626 enableNodeInclusionPolicy: true, 1627 }, 1628 { 1629 name: "NodeTaintsPolicy ignored, pods spread across zone as 2/2", 1630 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1631 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1632 Obj(), 1633 existingPods: []*v1.Pod{ 1634 st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1635 st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1636 st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), 1637 st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), 1638 }, 1639 fits: true, 1640 nodes: []*v1.Node{ 1641 st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), 1642 st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(), 1643 st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Taints(taints).Obj(), 1644 st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), 1645 }, 1646 candidateNodes: []string{"node-1", "node-2", "node-4"}, // node-3 is filtered out by TaintToleration plugin 1647 enableNodeInclusionPolicy: true, 1648 }, 1649 { 1650 // 1. to fulfil "zone" constraint, pods spread across zones as 2/1 1651 // 2. to fulfil "node" constraint, pods spread across zones as 1/1/~0~/1 1652 // intersection of (1) and (2) returns node-4 as node-3 is filtered out by NodeAffinity plugin. 1653 name: "two node inclusion Constraints, zone: honor/ignore, node: honor/ignore", 1654 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1655 NodeSelector(map[string]string{"foo": ""}). 1656 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1657 SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1658 Obj(), 1659 existingPods: []*v1.Pod{ 1660 st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1661 st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1662 st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), 1663 st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), 1664 }, 1665 fits: true, 1666 nodes: []*v1.Node{ 1667 st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), 1668 st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(), 1669 st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), 1670 st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), 1671 }, 1672 candidateNodes: []string{"node-4"}, 1673 enableNodeInclusionPolicy: true, 1674 }, 1675 { 1676 // 1. to fulfil "zone" constraint, pods spread across zones as 2/1 1677 // 2. to fulfil "node" constraint, pods spread across zones as 1/1/~0~/1 1678 // intersection of (1) and (2) returns node-4 as node-3 is filtered out by NodeAffinity plugin 1679 name: "feature gate disabled, two node inclusion Constraints, zone: honor/ignore, node: honor/ignore", 1680 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1681 NodeSelector(map[string]string{"foo": ""}). 1682 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1683 SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil). 1684 Obj(), 1685 existingPods: []*v1.Pod{ 1686 st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1687 st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1688 st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), 1689 st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), 1690 }, 1691 fits: true, 1692 nodes: []*v1.Node{ 1693 st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), 1694 st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(), 1695 st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), 1696 st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), 1697 }, 1698 candidateNodes: []string{"node-4"}, 1699 enableNodeInclusionPolicy: false, 1700 }, 1701 { 1702 // 1. to fulfil "zone" constraint, pods spread across zones as 2/2 1703 // 2. to fulfil "node" constraint, pods spread across zones as 1/~0~/~0~/1 1704 // intersection of (1) and (2) returns node-1 and node-4 as node-2, node-3 are filtered out by plugins 1705 name: "two node inclusion Constraints, zone: ignore/ignore, node: honor/honor", 1706 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1707 NodeSelector(map[string]string{"foo": ""}). 1708 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, &ignorePolicy, nil, nil). 1709 SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, &honorPolicy, nil). 1710 Obj(), 1711 existingPods: []*v1.Pod{ 1712 st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1713 st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1714 st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), 1715 st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), 1716 }, 1717 fits: true, 1718 nodes: []*v1.Node{ 1719 st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), 1720 st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(), 1721 st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), 1722 st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), 1723 }, 1724 candidateNodes: []string{"node-1", "node-4"}, 1725 enableNodeInclusionPolicy: true, 1726 }, 1727 { 1728 name: "matchLabelKeys ignored when feature gate disabled, pods spread across zone as 2/1", 1729 incomingPod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").Container(pause). 1730 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, []string{"bar"}). 1731 Obj(), 1732 existingPods: []*v1.Pod{ 1733 st.MakePod().Name("p1a").Node("node-0").Label("foo", "").Container(pause).Obj(), 1734 st.MakePod().Name("p2a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1735 st.MakePod().Name("p3a").Node("node-2").Label("foo", "").Label("bar", "").Container(pause).Obj(), 1736 }, 1737 fits: true, 1738 nodes: defaultNodes, 1739 candidateNodes: []string{"node-2", "node-3"}, 1740 enableMatchLabelKeys: false, 1741 }, 1742 { 1743 name: "matchLabelKeys ANDed with LabelSelector when LabelSelector isn't empty, pods spread across zone as 0/1", 1744 incomingPod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").Container(pause). 1745 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, []string{"bar"}). 1746 Obj(), 1747 existingPods: []*v1.Pod{ 1748 st.MakePod().Name("p1a").Node("node-0").Label("foo", "").Container(pause).Obj(), 1749 st.MakePod().Name("p2a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1750 st.MakePod().Name("p3a").Node("node-2").Label("foo", "").Label("bar", "").Container(pause).Obj(), 1751 }, 1752 fits: true, 1753 nodes: defaultNodes, 1754 candidateNodes: []string{"node-0", "node-1"}, 1755 enableMatchLabelKeys: true, 1756 }, 1757 { 1758 name: "matchLabelKeys ANDed with LabelSelector when LabelSelector is empty, pods spread across zone as 2/1", 1759 incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). 1760 SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Obj(), nil, nil, nil, []string{"foo"}). 1761 Obj(), 1762 existingPods: []*v1.Pod{ 1763 st.MakePod().Name("p1a").Node("node-0").Label("foo", "").Container(pause).Obj(), 1764 st.MakePod().Name("p2a").Node("node-1").Label("foo", "").Container(pause).Obj(), 1765 st.MakePod().Name("p3a").Node("node-2").Label("foo", "").Container(pause).Obj(), 1766 }, 1767 fits: true, 1768 nodes: defaultNodes, 1769 candidateNodes: []string{"node-2", "node-3"}, 1770 enableMatchLabelKeys: true, 1771 }, 1772 } 1773 for _, tt := range tests { 1774 t.Run(tt.name, func(t *testing.T) { 1775 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MinDomainsInPodTopologySpread, tt.enableMinDomains)() 1776 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeInclusionPolicyInPodTopologySpread, tt.enableNodeInclusionPolicy)() 1777 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MatchLabelKeysInPodTopologySpread, tt.enableMatchLabelKeys)() 1778 1779 testCtx := initTest(t, "pts-predicate") 1780 cs := testCtx.ClientSet 1781 ns := testCtx.NS.Name 1782 1783 for i := range tt.nodes { 1784 if _, err := createNode(cs, tt.nodes[i]); err != nil { 1785 t.Fatalf("Cannot create node: %v", err) 1786 } 1787 } 1788 1789 // set namespace to pods 1790 for i := range tt.existingPods { 1791 tt.existingPods[i].SetNamespace(ns) 1792 } 1793 tt.incomingPod.SetNamespace(ns) 1794 1795 allPods := append(tt.existingPods, tt.incomingPod) 1796 defer testutils.CleanupPods(testCtx.Ctx, cs, t, allPods) 1797 1798 for _, pod := range tt.existingPods { 1799 createdPod, err := cs.CoreV1().Pods(pod.Namespace).Create(testCtx.Ctx, pod, metav1.CreateOptions{}) 1800 if err != nil { 1801 t.Fatalf("Error while creating pod during test: %v", err) 1802 } 1803 err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false, 1804 testutils.PodScheduled(cs, createdPod.Namespace, createdPod.Name)) 1805 if err != nil { 1806 t.Errorf("Error while waiting for pod during test: %v", err) 1807 } 1808 } 1809 testPod, err := cs.CoreV1().Pods(tt.incomingPod.Namespace).Create(testCtx.Ctx, tt.incomingPod, metav1.CreateOptions{}) 1810 if err != nil { 1811 t.Fatalf("Error while creating pod during test: %v", err) 1812 } 1813 1814 if tt.fits { 1815 err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false, 1816 podScheduledIn(cs, testPod.Namespace, testPod.Name, tt.candidateNodes)) 1817 } else { 1818 err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false, 1819 podUnschedulable(cs, testPod.Namespace, testPod.Name)) 1820 } 1821 if err != nil { 1822 t.Errorf("Test Failed: %v", err) 1823 } 1824 }) 1825 } 1826 } 1827 1828 var ( 1829 hardSpread = v1.DoNotSchedule 1830 ) 1831 1832 func TestUnschedulablePodBecomesSchedulable(t *testing.T) { 1833 tests := []struct { 1834 name string 1835 init func(kubernetes.Interface, string) error 1836 pod *testutils.PausePodConfig 1837 update func(kubernetes.Interface, string) error 1838 }{ 1839 { 1840 name: "node gets added", 1841 pod: &testutils.PausePodConfig{ 1842 Name: "pod-1", 1843 }, 1844 update: func(cs kubernetes.Interface, _ string) error { 1845 _, err := createNode(cs, st.MakeNode().Name("node-added").Obj()) 1846 if err != nil { 1847 return fmt.Errorf("cannot create node: %v", err) 1848 } 1849 return nil 1850 }, 1851 }, 1852 { 1853 name: "node gets taint removed", 1854 init: func(cs kubernetes.Interface, _ string) error { 1855 node, err := createNode(cs, st.MakeNode().Name("node-tainted").Obj()) 1856 if err != nil { 1857 return fmt.Errorf("cannot create node: %v", err) 1858 } 1859 taint := v1.Taint{Key: "test", Value: "test", Effect: v1.TaintEffectNoSchedule} 1860 if err := testutils.AddTaintToNode(cs, node.Name, taint); err != nil { 1861 return fmt.Errorf("cannot add taint to node: %v", err) 1862 } 1863 return nil 1864 }, 1865 pod: &testutils.PausePodConfig{ 1866 Name: "pod-1", 1867 }, 1868 update: func(cs kubernetes.Interface, _ string) error { 1869 taint := v1.Taint{Key: "test", Value: "test", Effect: v1.TaintEffectNoSchedule} 1870 if err := testutils.RemoveTaintOffNode(cs, "node-tainted", taint); err != nil { 1871 return fmt.Errorf("cannot remove taint off node: %v", err) 1872 } 1873 return nil 1874 }, 1875 }, 1876 { 1877 name: "other pod gets deleted", 1878 init: func(cs kubernetes.Interface, ns string) error { 1879 nodeObject := st.MakeNode().Name("node-scheduler-integration-test").Capacity(map[v1.ResourceName]string{v1.ResourcePods: "1"}).Obj() 1880 _, err := createNode(cs, nodeObject) 1881 if err != nil { 1882 return fmt.Errorf("cannot create node: %v", err) 1883 } 1884 _, err = createPausePod(cs, initPausePod(&testutils.PausePodConfig{Name: "pod-to-be-deleted", Namespace: ns})) 1885 if err != nil { 1886 return fmt.Errorf("cannot create pod: %v", err) 1887 } 1888 return nil 1889 }, 1890 pod: &testutils.PausePodConfig{ 1891 Name: "pod-1", 1892 }, 1893 update: func(cs kubernetes.Interface, ns string) error { 1894 if err := deletePod(cs, "pod-to-be-deleted", ns); err != nil { 1895 return fmt.Errorf("cannot delete pod: %v", err) 1896 } 1897 return nil 1898 }, 1899 }, 1900 { 1901 name: "pod with pod-affinity gets added", 1902 init: func(cs kubernetes.Interface, _ string) error { 1903 _, err := createNode(cs, st.MakeNode().Name("node-1").Label("region", "test").Obj()) 1904 if err != nil { 1905 return fmt.Errorf("cannot create node: %v", err) 1906 } 1907 return nil 1908 }, 1909 pod: &testutils.PausePodConfig{ 1910 Name: "pod-1", 1911 Affinity: &v1.Affinity{ 1912 PodAffinity: &v1.PodAffinity{ 1913 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 1914 { 1915 LabelSelector: &metav1.LabelSelector{ 1916 MatchLabels: map[string]string{ 1917 "pod-with-affinity": "true", 1918 }, 1919 }, 1920 TopologyKey: "region", 1921 }, 1922 }, 1923 }, 1924 }, 1925 }, 1926 update: func(cs kubernetes.Interface, ns string) error { 1927 podConfig := &testutils.PausePodConfig{ 1928 Name: "pod-with-affinity", 1929 Namespace: ns, 1930 Labels: map[string]string{ 1931 "pod-with-affinity": "true", 1932 }, 1933 } 1934 if _, err := createPausePod(cs, initPausePod(podConfig)); err != nil { 1935 return fmt.Errorf("cannot create pod: %v", err) 1936 } 1937 return nil 1938 }, 1939 }, 1940 { 1941 name: "scheduled pod gets updated to match affinity", 1942 init: func(cs kubernetes.Interface, ns string) error { 1943 _, err := createNode(cs, st.MakeNode().Name("node-1").Label("region", "test").Obj()) 1944 if err != nil { 1945 return fmt.Errorf("cannot create node: %v", err) 1946 } 1947 if _, err := createPausePod(cs, initPausePod(&testutils.PausePodConfig{Name: "pod-to-be-updated", Namespace: ns})); err != nil { 1948 return fmt.Errorf("cannot create pod: %v", err) 1949 } 1950 return nil 1951 }, 1952 pod: &testutils.PausePodConfig{ 1953 Name: "pod-1", 1954 Affinity: &v1.Affinity{ 1955 PodAffinity: &v1.PodAffinity{ 1956 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ 1957 { 1958 LabelSelector: &metav1.LabelSelector{ 1959 MatchLabels: map[string]string{ 1960 "pod-with-affinity": "true", 1961 }, 1962 }, 1963 TopologyKey: "region", 1964 }, 1965 }, 1966 }, 1967 }, 1968 }, 1969 update: func(cs kubernetes.Interface, ns string) error { 1970 pod, err := getPod(cs, "pod-to-be-updated", ns) 1971 if err != nil { 1972 return fmt.Errorf("cannot get pod: %v", err) 1973 } 1974 pod.Labels = map[string]string{"pod-with-affinity": "true"} 1975 if _, err := cs.CoreV1().Pods(pod.Namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}); err != nil { 1976 return fmt.Errorf("cannot update pod: %v", err) 1977 } 1978 return nil 1979 }, 1980 }, 1981 { 1982 name: "scheduled pod uses read-write-once-pod pvc", 1983 init: func(cs kubernetes.Interface, ns string) error { 1984 _, err := createNode(cs, st.MakeNode().Name("node").Obj()) 1985 if err != nil { 1986 return fmt.Errorf("cannot create node: %v", err) 1987 } 1988 1989 storage := v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}} 1990 volType := v1.HostPathDirectoryOrCreate 1991 pv, err := testutils.CreatePV(cs, st.MakePersistentVolume(). 1992 Name("pv-with-read-write-once-pod"). 1993 AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). 1994 Capacity(storage.Requests). 1995 HostPathVolumeSource(&v1.HostPathVolumeSource{Path: "/mnt", Type: &volType}). 1996 Obj()) 1997 if err != nil { 1998 return fmt.Errorf("cannot create pv: %v", err) 1999 } 2000 pvc, err := testutils.CreatePVC(cs, st.MakePersistentVolumeClaim(). 2001 Name("pvc-with-read-write-once-pod"). 2002 Namespace(ns). 2003 // Annotation and volume name required for PVC to be considered bound. 2004 Annotation(volume.AnnBindCompleted, "true"). 2005 VolumeName(pv.Name). 2006 AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). 2007 Resources(storage). 2008 Obj()) 2009 if err != nil { 2010 return fmt.Errorf("cannot create pvc: %v", err) 2011 } 2012 2013 pod := initPausePod(&testutils.PausePodConfig{ 2014 Name: "pod-to-be-deleted", 2015 Namespace: ns, 2016 Volumes: []v1.Volume{{ 2017 Name: "volume", 2018 VolumeSource: v1.VolumeSource{ 2019 PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ 2020 ClaimName: pvc.Name, 2021 }, 2022 }, 2023 }}, 2024 }) 2025 if _, err := createPausePod(cs, pod); err != nil { 2026 return fmt.Errorf("cannot create pod: %v", err) 2027 } 2028 return nil 2029 }, 2030 pod: &testutils.PausePodConfig{ 2031 Name: "pod-to-take-over-pvc", 2032 Volumes: []v1.Volume{{ 2033 Name: "volume", 2034 VolumeSource: v1.VolumeSource{ 2035 PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ 2036 ClaimName: "pvc-with-read-write-once-pod", 2037 }, 2038 }, 2039 }}, 2040 }, 2041 update: func(cs kubernetes.Interface, ns string) error { 2042 return deletePod(cs, "pod-to-be-deleted", ns) 2043 }, 2044 }, 2045 } 2046 for _, tt := range tests { 2047 t.Run(tt.name, func(t *testing.T) { 2048 testCtx := initTest(t, "scheduler-informer") 2049 2050 if tt.init != nil { 2051 if err := tt.init(testCtx.ClientSet, testCtx.NS.Name); err != nil { 2052 t.Fatal(err) 2053 } 2054 } 2055 tt.pod.Namespace = testCtx.NS.Name 2056 pod, err := createPausePod(testCtx.ClientSet, initPausePod(tt.pod)) 2057 if err != nil { 2058 t.Fatal(err) 2059 } 2060 if err := waitForPodUnschedulable(testCtx.ClientSet, pod); err != nil { 2061 t.Errorf("Pod %v got scheduled: %v", pod.Name, err) 2062 } 2063 if err := tt.update(testCtx.ClientSet, testCtx.NS.Name); err != nil { 2064 t.Fatal(err) 2065 } 2066 if err := testutils.WaitForPodToSchedule(testCtx.ClientSet, pod); err != nil { 2067 t.Errorf("Pod %v was not scheduled: %v", pod.Name, err) 2068 } 2069 // Make sure pending queue is empty. 2070 pendingPods, s := testCtx.Scheduler.SchedulingQueue.PendingPods() 2071 if len(pendingPods) != 0 { 2072 t.Errorf("pending pods queue is not empty, size is: %d, summary is: %s", len(pendingPods), s) 2073 } 2074 }) 2075 } 2076 }