k8s.io/kubernetes@v1.29.3/test/integration/scheduler/queue_test.go (about) 1 /* 2 Copyright 2021 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package scheduler 18 19 import ( 20 "context" 21 "fmt" 22 "testing" 23 "time" 24 25 v1 "k8s.io/api/core/v1" 26 apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 27 apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 28 "k8s.io/apimachinery/pkg/api/errors" 29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 31 "k8s.io/apimachinery/pkg/runtime" 32 "k8s.io/apimachinery/pkg/runtime/schema" 33 "k8s.io/apimachinery/pkg/types" 34 "k8s.io/apimachinery/pkg/util/uuid" 35 "k8s.io/apimachinery/pkg/util/wait" 36 utilfeature "k8s.io/apiserver/pkg/util/feature" 37 "k8s.io/client-go/dynamic" 38 "k8s.io/client-go/kubernetes" 39 featuregatetesting "k8s.io/component-base/featuregate/testing" 40 "k8s.io/klog/v2" 41 configv1 "k8s.io/kube-scheduler/config/v1" 42 apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 43 "k8s.io/kubernetes/pkg/features" 44 "k8s.io/kubernetes/pkg/scheduler" 45 configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing" 46 "k8s.io/kubernetes/pkg/scheduler/framework" 47 "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" 48 "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" 49 frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 50 st "k8s.io/kubernetes/pkg/scheduler/testing" 51 testfwk "k8s.io/kubernetes/test/integration/framework" 52 testutils "k8s.io/kubernetes/test/integration/util" 53 imageutils "k8s.io/kubernetes/test/utils/image" 54 "k8s.io/utils/pointer" 55 ) 56 57 func TestSchedulingGates(t *testing.T) { 58 tests := []struct { 59 name string 60 pods []*v1.Pod 61 featureEnabled bool 62 want []string 63 rmPodsSchedulingGates []int 64 wantPostGatesRemoval []string 65 }{ 66 { 67 name: "feature disabled, regular pods", 68 pods: []*v1.Pod{ 69 st.MakePod().Name("p1").Container("pause").Obj(), 70 st.MakePod().Name("p2").Container("pause").Obj(), 71 }, 72 featureEnabled: false, 73 want: []string{"p1", "p2"}, 74 }, 75 { 76 name: "feature enabled, regular pods", 77 pods: []*v1.Pod{ 78 st.MakePod().Name("p1").Container("pause").Obj(), 79 st.MakePod().Name("p2").Container("pause").Obj(), 80 }, 81 featureEnabled: true, 82 want: []string{"p1", "p2"}, 83 }, 84 { 85 name: "feature disabled, one pod carrying scheduling gates", 86 pods: []*v1.Pod{ 87 st.MakePod().Name("p1").SchedulingGates([]string{"foo"}).Container("pause").Obj(), 88 st.MakePod().Name("p2").Container("pause").Obj(), 89 }, 90 featureEnabled: false, 91 want: []string{"p1", "p2"}, 92 }, 93 { 94 name: "feature enabled, one pod carrying scheduling gates", 95 pods: []*v1.Pod{ 96 st.MakePod().Name("p1").SchedulingGates([]string{"foo"}).Container("pause").Obj(), 97 st.MakePod().Name("p2").Container("pause").Obj(), 98 }, 99 featureEnabled: true, 100 want: []string{"p2"}, 101 }, 102 { 103 name: "feature enabled, two pod carrying scheduling gates, and remove gates of one pod", 104 pods: []*v1.Pod{ 105 st.MakePod().Name("p1").SchedulingGates([]string{"foo"}).Container("pause").Obj(), 106 st.MakePod().Name("p2").SchedulingGates([]string{"bar"}).Container("pause").Obj(), 107 st.MakePod().Name("p3").Container("pause").Obj(), 108 }, 109 featureEnabled: true, 110 want: []string{"p3"}, 111 rmPodsSchedulingGates: []int{1}, // remove gates of 'p2' 112 wantPostGatesRemoval: []string{"p2"}, 113 }, 114 } 115 116 for _, tt := range tests { 117 t.Run(tt.name, func(t *testing.T) { 118 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodSchedulingReadiness, tt.featureEnabled)() 119 120 // Use zero backoff seconds to bypass backoffQ. 121 // It's intended to not start the scheduler's queue, and hence to 122 // not start any flushing logic. We will pop and schedule the Pods manually later. 123 testCtx := testutils.InitTestSchedulerWithOptions( 124 t, 125 testutils.InitTestAPIServer(t, "pod-scheduling-gates", nil), 126 0, 127 scheduler.WithPodInitialBackoffSeconds(0), 128 scheduler.WithPodMaxBackoffSeconds(0), 129 ) 130 testutils.SyncSchedulerInformerFactory(testCtx) 131 132 cs, ns, ctx := testCtx.ClientSet, testCtx.NS.Name, testCtx.Ctx 133 for _, p := range tt.pods { 134 p.Namespace = ns 135 if _, err := cs.CoreV1().Pods(ns).Create(ctx, p, metav1.CreateOptions{}); err != nil { 136 t.Fatalf("Failed to create Pod %q: %v", p.Name, err) 137 } 138 } 139 140 // Wait for the pods to be present in the scheduling queue. 141 if err := wait.PollUntilContextTimeout(ctx, time.Millisecond*200, wait.ForeverTestTimeout, false, func(ctx context.Context) (bool, error) { 142 pendingPods, _ := testCtx.Scheduler.SchedulingQueue.PendingPods() 143 return len(pendingPods) == len(tt.pods), nil 144 }); err != nil { 145 t.Fatal(err) 146 } 147 148 // Pop the expected pods out. They should be de-queueable. 149 for _, wantPod := range tt.want { 150 podInfo := testutils.NextPodOrDie(t, testCtx) 151 if got := podInfo.Pod.Name; got != wantPod { 152 t.Errorf("Want %v to be popped out, but got %v", wantPod, got) 153 } 154 } 155 156 if len(tt.rmPodsSchedulingGates) == 0 { 157 return 158 } 159 // Remove scheduling gates from the pod spec. 160 for _, idx := range tt.rmPodsSchedulingGates { 161 patch := `{"spec": {"schedulingGates": null}}` 162 podName := tt.pods[idx].Name 163 if _, err := cs.CoreV1().Pods(ns).Patch(ctx, podName, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { 164 t.Fatalf("Failed to patch pod %v: %v", podName, err) 165 } 166 } 167 // Pop the expected pods out. They should be de-queueable. 168 for _, wantPod := range tt.wantPostGatesRemoval { 169 podInfo := testutils.NextPodOrDie(t, testCtx) 170 if got := podInfo.Pod.Name; got != wantPod { 171 t.Errorf("Want %v to be popped out, but got %v", wantPod, got) 172 } 173 } 174 }) 175 } 176 } 177 178 // TestCoreResourceEnqueue verify Pods failed by in-tree default plugins can be 179 // moved properly upon their registered events. 180 func TestCoreResourceEnqueue(t *testing.T) { 181 // Use zero backoff seconds to bypass backoffQ. 182 // It's intended to not start the scheduler's queue, and hence to 183 // not start any flushing logic. We will pop and schedule the Pods manually later. 184 testCtx := testutils.InitTestSchedulerWithOptions( 185 t, 186 testutils.InitTestAPIServer(t, "core-res-enqueue", nil), 187 0, 188 scheduler.WithPodInitialBackoffSeconds(0), 189 scheduler.WithPodMaxBackoffSeconds(0), 190 ) 191 testutils.SyncSchedulerInformerFactory(testCtx) 192 193 defer testCtx.Scheduler.SchedulingQueue.Close() 194 195 cs, ns, ctx := testCtx.ClientSet, testCtx.NS.Name, testCtx.Ctx 196 // Create one Node with a taint. 197 node := st.MakeNode().Name("fake-node").Capacity(map[v1.ResourceName]string{v1.ResourceCPU: "2"}).Obj() 198 node.Spec.Taints = []v1.Taint{{Key: "foo", Effect: v1.TaintEffectNoSchedule}} 199 if _, err := cs.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}); err != nil { 200 t.Fatalf("Failed to create Node %q: %v", node.Name, err) 201 } 202 203 // Create two Pods that are both unschedulable. 204 // - Pod1 is a best-effort Pod, but doesn't have the required toleration. 205 // - Pod2 requests a large amount of CPU resource that the node cannot fit. 206 // Note: Pod2 will fail the tainttoleration plugin b/c that's ordered prior to noderesources. 207 // - Pod3 has the required toleration, but requests a non-existing PVC. 208 pod1 := st.MakePod().Namespace(ns).Name("pod1").Container("image").Obj() 209 pod2 := st.MakePod().Namespace(ns).Name("pod2").Req(map[v1.ResourceName]string{v1.ResourceCPU: "4"}).Obj() 210 pod3 := st.MakePod().Namespace(ns).Name("pod3").Toleration("foo").PVC("pvc").Container("image").Obj() 211 for _, pod := range []*v1.Pod{pod1, pod2, pod3} { 212 if _, err := cs.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { 213 t.Fatalf("Failed to create Pod %q: %v", pod.Name, err) 214 } 215 } 216 217 // Wait for the three pods to be present in the scheduling queue. 218 if err := wait.PollUntilContextTimeout(ctx, time.Millisecond*200, wait.ForeverTestTimeout, false, func(ctx context.Context) (bool, error) { 219 pendingPods, _ := testCtx.Scheduler.SchedulingQueue.PendingPods() 220 return len(pendingPods) == 3, nil 221 }); err != nil { 222 t.Fatal(err) 223 } 224 225 // Pop the three pods out. They should be unschedulable. 226 for i := 0; i < 3; i++ { 227 podInfo := testutils.NextPodOrDie(t, testCtx) 228 fwk, ok := testCtx.Scheduler.Profiles[podInfo.Pod.Spec.SchedulerName] 229 if !ok { 230 t.Fatalf("Cannot find the profile for Pod %v", podInfo.Pod.Name) 231 } 232 // Schedule the Pod manually. 233 _, fitError := testCtx.Scheduler.SchedulePod(ctx, fwk, framework.NewCycleState(), podInfo.Pod) 234 if fitError == nil { 235 t.Fatalf("Expect Pod %v to fail at scheduling.", podInfo.Pod.Name) 236 } 237 testCtx.Scheduler.FailureHandler(ctx, fwk, podInfo, framework.NewStatus(framework.Unschedulable).WithError(fitError), nil, time.Now()) 238 } 239 240 // Trigger a NodeTaintChange event. 241 // We expect this event to trigger moving the test Pod from unschedulablePods to activeQ. 242 node.Spec.Taints = nil 243 if _, err := cs.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}); err != nil { 244 t.Fatalf("Failed to remove taints off the node: %v", err) 245 } 246 247 // Now we should be able to pop the Pod from activeQ again. 248 podInfo := testutils.NextPodOrDie(t, testCtx) 249 if podInfo.Attempts != 2 { 250 t.Fatalf("Expected the Pod to be attempted 2 times, but got %v", podInfo.Attempts) 251 } 252 if got := podInfo.Pod.Name; got != "pod1" { 253 t.Fatalf("Expected pod1 to be popped, but got %v", got) 254 } 255 256 // Pod2 and Pod3 are not expected to be popped out. 257 // - Although the failure reason has been lifted, Pod2 still won't be moved to active due to 258 // the node event's preCheckForNode(). 259 // - Regarding Pod3, the NodeTaintChange event is irrelevant with its scheduling failure. 260 podInfo = testutils.NextPod(t, testCtx) 261 if podInfo != nil { 262 t.Fatalf("Unexpected pod %v get popped out", podInfo.Pod.Name) 263 } 264 } 265 266 var _ framework.FilterPlugin = &fakeCRPlugin{} 267 var _ framework.EnqueueExtensions = &fakeCRPlugin{} 268 269 type fakeCRPlugin struct{} 270 271 func (f *fakeCRPlugin) Name() string { 272 return "fakeCRPlugin" 273 } 274 275 func (f *fakeCRPlugin) Filter(_ context.Context, _ *framework.CycleState, _ *v1.Pod, _ *framework.NodeInfo) *framework.Status { 276 return framework.NewStatus(framework.Unschedulable, "always fail") 277 } 278 279 // EventsToRegister returns the possible events that may make a Pod 280 // failed by this plugin schedulable. 281 func (f *fakeCRPlugin) EventsToRegister() []framework.ClusterEventWithHint { 282 return []framework.ClusterEventWithHint{ 283 {Event: framework.ClusterEvent{Resource: "foos.v1.example.com", ActionType: framework.All}}, 284 } 285 } 286 287 // TestCustomResourceEnqueue constructs a fake plugin that registers custom resources 288 // to verify Pods failed by this plugin can be moved properly upon CR events. 289 func TestCustomResourceEnqueue(t *testing.T) { 290 // Start API Server with apiextensions supported. 291 server := apiservertesting.StartTestServerOrDie( 292 t, apiservertesting.NewDefaultTestServerOptions(), 293 []string{"--disable-admission-plugins=ServiceAccount,TaintNodesByCondition", "--runtime-config=api/all=true"}, 294 testfwk.SharedEtcd(), 295 ) 296 testCtx := &testutils.TestContext{} 297 ctx, cancel := context.WithCancel(context.Background()) 298 testCtx.Ctx = ctx 299 testCtx.CloseFn = func() { 300 cancel() 301 server.TearDownFn() 302 } 303 304 apiExtensionClient := apiextensionsclient.NewForConfigOrDie(server.ClientConfig) 305 dynamicClient := dynamic.NewForConfigOrDie(server.ClientConfig) 306 307 // Create a Foo CRD. 308 fooCRD := &apiextensionsv1.CustomResourceDefinition{ 309 ObjectMeta: metav1.ObjectMeta{ 310 Name: "foos.example.com", 311 }, 312 Spec: apiextensionsv1.CustomResourceDefinitionSpec{ 313 Group: "example.com", 314 Scope: apiextensionsv1.NamespaceScoped, 315 Names: apiextensionsv1.CustomResourceDefinitionNames{ 316 Plural: "foos", 317 Kind: "Foo", 318 }, 319 Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ 320 { 321 Name: "v1", 322 Served: true, 323 Storage: true, 324 Schema: &apiextensionsv1.CustomResourceValidation{ 325 OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ 326 Type: "object", 327 Properties: map[string]apiextensionsv1.JSONSchemaProps{ 328 "field": {Type: "string"}, 329 }, 330 }, 331 }, 332 }, 333 }, 334 }, 335 } 336 var err error 337 fooCRD, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Create(testCtx.Ctx, fooCRD, metav1.CreateOptions{}) 338 if err != nil { 339 t.Fatal(err) 340 } 341 342 registry := frameworkruntime.Registry{ 343 "fakeCRPlugin": func(_ context.Context, _ runtime.Object, fh framework.Handle) (framework.Plugin, error) { 344 return &fakeCRPlugin{}, nil 345 }, 346 } 347 cfg := configtesting.V1ToInternalWithDefaults(t, configv1.KubeSchedulerConfiguration{ 348 Profiles: []configv1.KubeSchedulerProfile{{ 349 SchedulerName: pointer.String(v1.DefaultSchedulerName), 350 Plugins: &configv1.Plugins{ 351 Filter: configv1.PluginSet{ 352 Enabled: []configv1.Plugin{ 353 {Name: "fakeCRPlugin"}, 354 }, 355 }, 356 }, 357 }}}) 358 359 testCtx.KubeConfig = server.ClientConfig 360 testCtx.ClientSet = kubernetes.NewForConfigOrDie(server.ClientConfig) 361 testCtx.NS, err = testCtx.ClientSet.CoreV1().Namespaces().Create(testCtx.Ctx, &v1.Namespace{ 362 ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("cr-enqueue-%v", string(uuid.NewUUID()))}}, metav1.CreateOptions{}) 363 if err != nil && !errors.IsAlreadyExists(err) { 364 t.Fatalf("Failed to integration test ns: %v", err) 365 } 366 367 // Use zero backoff seconds to bypass backoffQ. 368 // It's intended to not start the scheduler's queue, and hence to 369 // not start any flushing logic. We will pop and schedule the Pods manually later. 370 testCtx = testutils.InitTestSchedulerWithOptions( 371 t, 372 testCtx, 373 0, 374 scheduler.WithProfiles(cfg.Profiles...), 375 scheduler.WithFrameworkOutOfTreeRegistry(registry), 376 scheduler.WithPodInitialBackoffSeconds(0), 377 scheduler.WithPodMaxBackoffSeconds(0), 378 ) 379 testutils.SyncSchedulerInformerFactory(testCtx) 380 381 defer testutils.CleanupTest(t, testCtx) 382 383 cs, ns, ctx := testCtx.ClientSet, testCtx.NS.Name, testCtx.Ctx 384 logger := klog.FromContext(ctx) 385 // Create one Node. 386 node := st.MakeNode().Name("fake-node").Obj() 387 if _, err := cs.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}); err != nil { 388 t.Fatalf("Failed to create Node %q: %v", node.Name, err) 389 } 390 391 // Create a testing Pod. 392 pause := imageutils.GetPauseImageName() 393 pod := st.MakePod().Namespace(ns).Name("fake-pod").Container(pause).Obj() 394 if _, err := cs.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { 395 t.Fatalf("Failed to create Pod %q: %v", pod.Name, err) 396 } 397 398 // Wait for the testing Pod to be present in the scheduling queue. 399 if err := wait.PollUntilContextTimeout(ctx, time.Millisecond*200, wait.ForeverTestTimeout, false, func(ctx context.Context) (bool, error) { 400 pendingPods, _ := testCtx.Scheduler.SchedulingQueue.PendingPods() 401 return len(pendingPods) == 1, nil 402 }); err != nil { 403 t.Fatal(err) 404 } 405 406 // Pop fake-pod out. It should be unschedulable. 407 podInfo := testutils.NextPodOrDie(t, testCtx) 408 fwk, ok := testCtx.Scheduler.Profiles[podInfo.Pod.Spec.SchedulerName] 409 if !ok { 410 t.Fatalf("Cannot find the profile for Pod %v", podInfo.Pod.Name) 411 } 412 // Schedule the Pod manually. 413 _, fitError := testCtx.Scheduler.SchedulePod(ctx, fwk, framework.NewCycleState(), podInfo.Pod) 414 // The fitError is expected to be non-nil as it failed the fakeCRPlugin plugin. 415 if fitError == nil { 416 t.Fatalf("Expect Pod %v to fail at scheduling.", podInfo.Pod.Name) 417 } 418 testCtx.Scheduler.FailureHandler(ctx, fwk, podInfo, framework.NewStatus(framework.Unschedulable).WithError(fitError), nil, time.Now()) 419 420 // Scheduling cycle is incremented from 0 to 1 after NextPod() is called, so 421 // pass a number larger than 1 to move Pod to unschedulablePods. 422 testCtx.Scheduler.SchedulingQueue.AddUnschedulableIfNotPresent(logger, podInfo, 10) 423 424 // Trigger a Custom Resource event. 425 // We expect this event to trigger moving the test Pod from unschedulablePods to activeQ. 426 crdGVR := schema.GroupVersionResource{Group: fooCRD.Spec.Group, Version: fooCRD.Spec.Versions[0].Name, Resource: "foos"} 427 crClient := dynamicClient.Resource(crdGVR).Namespace(ns) 428 if _, err := crClient.Create(ctx, &unstructured.Unstructured{ 429 Object: map[string]interface{}{ 430 "apiVersion": "example.com/v1", 431 "kind": "Foo", 432 "metadata": map[string]interface{}{"name": "foo1"}, 433 }, 434 }, metav1.CreateOptions{}); err != nil { 435 t.Fatalf("Unable to create cr: %v", err) 436 } 437 438 // Now we should be able to pop the Pod from activeQ again. 439 podInfo = testutils.NextPodOrDie(t, testCtx) 440 if podInfo.Attempts != 2 { 441 t.Errorf("Expected the Pod to be attempted 2 times, but got %v", podInfo.Attempts) 442 } 443 } 444 445 // TestRequeueByBindFailure verify Pods failed by bind plugin are 446 // put back to the queue regardless of whether event happens or not. 447 func TestRequeueByBindFailure(t *testing.T) { 448 fakeBind := &firstFailBindPlugin{} 449 registry := frameworkruntime.Registry{ 450 "firstFailBindPlugin": func(ctx context.Context, o runtime.Object, fh framework.Handle) (framework.Plugin, error) { 451 binder, err := defaultbinder.New(ctx, nil, fh) 452 if err != nil { 453 return nil, err 454 } 455 456 fakeBind.defaultBinderPlugin = binder.(framework.BindPlugin) 457 return fakeBind, nil 458 }, 459 } 460 461 cfg := configtesting.V1ToInternalWithDefaults(t, configv1.KubeSchedulerConfiguration{ 462 Profiles: []configv1.KubeSchedulerProfile{{ 463 SchedulerName: pointer.String(v1.DefaultSchedulerName), 464 Plugins: &configv1.Plugins{ 465 MultiPoint: configv1.PluginSet{ 466 Enabled: []configv1.Plugin{ 467 {Name: "firstFailBindPlugin"}, 468 }, 469 Disabled: []configv1.Plugin{ 470 {Name: names.DefaultBinder}, 471 }, 472 }, 473 }, 474 }}}) 475 476 // Use zero backoff seconds to bypass backoffQ. 477 testCtx := testutils.InitTestSchedulerWithOptions( 478 t, 479 testutils.InitTestAPIServer(t, "core-res-enqueue", nil), 480 0, 481 scheduler.WithPodInitialBackoffSeconds(0), 482 scheduler.WithPodMaxBackoffSeconds(0), 483 scheduler.WithProfiles(cfg.Profiles...), 484 scheduler.WithFrameworkOutOfTreeRegistry(registry), 485 ) 486 testutils.SyncSchedulerInformerFactory(testCtx) 487 488 go testCtx.Scheduler.Run(testCtx.Ctx) 489 490 cs, ns, ctx := testCtx.ClientSet, testCtx.NS.Name, testCtx.Ctx 491 node := st.MakeNode().Name("fake-node").Obj() 492 if _, err := cs.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}); err != nil { 493 t.Fatalf("Failed to create Node %q: %v", node.Name, err) 494 } 495 // create a pod. 496 pod := st.MakePod().Namespace(ns).Name("pod-1").Container(imageutils.GetPauseImageName()).Obj() 497 if _, err := cs.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { 498 t.Fatalf("Failed to create Pod %q: %v", pod.Name, err) 499 } 500 501 // 1. first binding try should fail. 502 // 2. The pod should be enqueued to activeQ/backoffQ without any event. 503 // 3. The pod should be scheduled in the second binding try. 504 // Here, waiting until (3). 505 err := wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, wait.ForeverTestTimeout, false, testutils.PodScheduled(cs, ns, pod.Name)) 506 if err != nil { 507 t.Fatalf("Expect pod-1 to be scheduled by the bind plugin: %v", err) 508 } 509 510 // Make sure the first binding trial was failed, and this pod is scheduled at the second trial. 511 if fakeBind.counter != 1 { 512 t.Fatalf("Expect pod-1 to be scheduled by the bind plugin in the second binding try: %v", err) 513 } 514 } 515 516 // firstFailBindPlugin rejects the Pod in the first Bind call. 517 type firstFailBindPlugin struct { 518 counter int 519 defaultBinderPlugin framework.BindPlugin 520 } 521 522 func (*firstFailBindPlugin) Name() string { 523 return "firstFailBindPlugin" 524 } 525 526 func (p *firstFailBindPlugin) Bind(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodename string) *framework.Status { 527 if p.counter == 0 { 528 // fail in the first Bind call. 529 p.counter++ 530 return framework.NewStatus(framework.Error, "firstFailBindPlugin rejects the Pod") 531 } 532 533 return p.defaultBinderPlugin.Bind(ctx, state, pod, nodename) 534 } 535 536 // TestRequeueByPermitRejection verify Pods failed by permit plugins in the binding cycle are 537 // put back to the queue, according to the correct scheduling cycle number. 538 func TestRequeueByPermitRejection(t *testing.T) { 539 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerQueueingHints, true)() 540 queueingHintCalledCounter := 0 541 fakePermit := &fakePermitPlugin{} 542 registry := frameworkruntime.Registry{ 543 fakePermitPluginName: func(ctx context.Context, o runtime.Object, fh framework.Handle) (framework.Plugin, error) { 544 fakePermit = &fakePermitPlugin{ 545 frameworkHandler: fh, 546 schedulingHint: func(logger klog.Logger, pod *v1.Pod, oldObj, newObj interface{}) (framework.QueueingHint, error) { 547 queueingHintCalledCounter++ 548 return framework.Queue, nil 549 }, 550 } 551 return fakePermit, nil 552 }, 553 } 554 cfg := configtesting.V1ToInternalWithDefaults(t, configv1.KubeSchedulerConfiguration{ 555 Profiles: []configv1.KubeSchedulerProfile{{ 556 SchedulerName: pointer.String(v1.DefaultSchedulerName), 557 Plugins: &configv1.Plugins{ 558 MultiPoint: configv1.PluginSet{ 559 Enabled: []configv1.Plugin{ 560 {Name: fakePermitPluginName}, 561 }, 562 }, 563 }, 564 }}}) 565 566 // Use zero backoff seconds to bypass backoffQ. 567 testCtx := testutils.InitTestSchedulerWithOptions( 568 t, 569 testutils.InitTestAPIServer(t, "core-res-enqueue", nil), 570 0, 571 scheduler.WithPodInitialBackoffSeconds(0), 572 scheduler.WithPodMaxBackoffSeconds(0), 573 scheduler.WithProfiles(cfg.Profiles...), 574 scheduler.WithFrameworkOutOfTreeRegistry(registry), 575 ) 576 testutils.SyncSchedulerInformerFactory(testCtx) 577 578 go testCtx.Scheduler.Run(testCtx.Ctx) 579 580 cs, ns, ctx := testCtx.ClientSet, testCtx.NS.Name, testCtx.Ctx 581 node := st.MakeNode().Name("fake-node").Obj() 582 if _, err := cs.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}); err != nil { 583 t.Fatalf("Failed to create Node %q: %v", node.Name, err) 584 } 585 // create a pod. 586 pod := st.MakePod().Namespace(ns).Name("pod-1").Container(imageutils.GetPauseImageName()).Obj() 587 if _, err := cs.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { 588 t.Fatalf("Failed to create Pod %q: %v", pod.Name, err) 589 } 590 591 // update node label. (causes the NodeUpdate event) 592 node.Labels = map[string]string{"updated": ""} 593 if _, err := cs.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}); err != nil { 594 t.Fatalf("Failed to add labels to the node: %v", err) 595 } 596 597 // create a pod to increment the scheduling cycle number in the scheduling queue. 598 // We can make sure NodeUpdate event, that has happened in the previous scheduling cycle, makes Pod to be enqueued to activeQ via the scheduling queue. 599 pod = st.MakePod().Namespace(ns).Name("pod-2").Container(imageutils.GetPauseImageName()).Obj() 600 if _, err := cs.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { 601 t.Fatalf("Failed to create Pod %q: %v", pod.Name, err) 602 } 603 604 // reject pod-1 to simulate the failure in Permit plugins. 605 // This pod-1 should be enqueued to activeQ because the NodeUpdate event has happened. 606 fakePermit.frameworkHandler.IterateOverWaitingPods(func(wp framework.WaitingPod) { 607 if wp.GetPod().Name == "pod-1" { 608 wp.Reject(fakePermitPluginName, "fakePermitPlugin rejects the Pod") 609 return 610 } 611 }) 612 613 // Wait for pod-2 to be scheduled. 614 err := wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, wait.ForeverTestTimeout, false, func(ctx context.Context) (done bool, err error) { 615 fakePermit.frameworkHandler.IterateOverWaitingPods(func(wp framework.WaitingPod) { 616 if wp.GetPod().Name == "pod-2" { 617 wp.Allow(fakePermitPluginName) 618 } 619 }) 620 621 return testutils.PodScheduled(cs, ns, "pod-2")(ctx) 622 }) 623 if err != nil { 624 t.Fatalf("Expect pod-2 to be scheduled") 625 } 626 627 err = wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, wait.ForeverTestTimeout, false, func(ctx context.Context) (done bool, err error) { 628 pod1Found := false 629 fakePermit.frameworkHandler.IterateOverWaitingPods(func(wp framework.WaitingPod) { 630 if wp.GetPod().Name == "pod-1" { 631 pod1Found = true 632 wp.Allow(fakePermitPluginName) 633 } 634 }) 635 return pod1Found, nil 636 }) 637 if err != nil { 638 t.Fatal("Expect pod-1 to be scheduled again") 639 } 640 641 if queueingHintCalledCounter != 1 { 642 t.Fatalf("Expected the scheduling hint to be called 1 time, but %v", queueingHintCalledCounter) 643 } 644 } 645 646 type fakePermitPlugin struct { 647 frameworkHandler framework.Handle 648 schedulingHint framework.QueueingHintFn 649 } 650 651 const fakePermitPluginName = "fakePermitPlugin" 652 653 func (p *fakePermitPlugin) Name() string { 654 return fakePermitPluginName 655 } 656 657 func (p *fakePermitPlugin) Permit(ctx context.Context, state *framework.CycleState, _ *v1.Pod, _ string) (*framework.Status, time.Duration) { 658 return framework.NewStatus(framework.Wait), wait.ForeverTestTimeout 659 } 660 661 func (p *fakePermitPlugin) EventsToRegister() []framework.ClusterEventWithHint { 662 return []framework.ClusterEventWithHint{ 663 {Event: framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeLabel}, QueueingHintFn: p.schedulingHint}, 664 } 665 }