knative.dev/pkg@v0.0.0-20260602142205-ac97e43f6622/webhook/resourcesemantics/validation/validation_admit_test.go (about) 1 /* 2 Copyright 2020 The Knative Authors 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package validation 18 19 import ( 20 "context" 21 "encoding/json" 22 "errors" 23 "testing" 24 "time" 25 26 // Injection stuff 27 _ "knative.dev/pkg/client/injection/kube/client/fake" 28 _ "knative.dev/pkg/client/injection/kube/informers/admissionregistration/v1/validatingwebhookconfiguration/fake" 29 _ "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/secret/fake" 30 pkgreconciler "knative.dev/pkg/reconciler" 31 32 admissionv1 "k8s.io/api/admission/v1" 33 admissionregistrationv1 "k8s.io/api/admissionregistration/v1beta1" 34 authenticationv1 "k8s.io/api/authentication/v1" 35 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 36 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 37 "k8s.io/apimachinery/pkg/runtime" 38 "k8s.io/apimachinery/pkg/runtime/schema" 39 "k8s.io/apimachinery/pkg/util/wait" 40 fakekubeclientset "k8s.io/client-go/kubernetes/fake" 41 42 "knative.dev/pkg/apis" 43 "knative.dev/pkg/system" 44 "knative.dev/pkg/webhook" 45 46 _ "knative.dev/pkg/system/testing" 47 48 . "knative.dev/pkg/logging/testing" 49 . "knative.dev/pkg/reconciler/testing" 50 . "knative.dev/pkg/testing" 51 "knative.dev/pkg/webhook/resourcesemantics" 52 . "knative.dev/pkg/webhook/testing" 53 ) 54 55 const ( 56 testResourceValidationPath = "/foo" 57 testResourceValidationName = "webhook.knative.dev" 58 user1 = "brutto@knative.dev" 59 user2 = "arrabbiato@knative.dev" 60 ) 61 62 var ( 63 handlers = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{ 64 { 65 Group: "pkg.knative.dev", 66 Version: "v1alpha1", 67 Kind: "Resource", 68 }: &Resource{}, 69 { 70 Group: "pkg.knative.dev", 71 Version: "v1beta1", 72 Kind: "Resource", 73 }: &Resource{}, 74 { 75 Group: "pkg.knative.dev", 76 Version: "v1alpha1", 77 Kind: "InnerDefaultResource", 78 }: &InnerDefaultResource{}, 79 { 80 Group: "pkg.knative.io", 81 Version: "v1alpha1", 82 Kind: "InnerDefaultResource", 83 }: &InnerDefaultResource{}, 84 } 85 86 callbacks = map[schema.GroupVersionKind]Callback{ 87 { 88 Group: "pkg.knative.dev", 89 Version: "v1alpha1", 90 Kind: "Resource", 91 }: NewCallback(resourceCallback, webhook.Create, webhook.Update, webhook.Delete), 92 { 93 Group: "pkg.knative.dev", 94 Version: "v1beta1", 95 Kind: "Resource", 96 }: NewCallback(resourceCallback, webhook.Create, webhook.Update, webhook.Delete), 97 { 98 Group: "pkg.knative.dev", 99 Version: "v1alpha1", 100 Kind: "CallbackResource", 101 }: NewCallback(resourceCallback, webhook.Create, webhook.Update, webhook.Delete), 102 } 103 initialResourceWebhook = &admissionregistrationv1.ValidatingWebhookConfiguration{ 104 ObjectMeta: metav1.ObjectMeta{ 105 Name: "webhook.knative.dev", 106 OwnerReferences: []metav1.OwnerReference{{ 107 Name: "asdf", 108 }}, 109 }, 110 Webhooks: []admissionregistrationv1.ValidatingWebhook{{ 111 Name: "webhook.knative.dev", 112 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 113 Service: &admissionregistrationv1.ServiceReference{ 114 Namespace: system.Namespace(), 115 Name: "webhook", 116 }, 117 }, 118 }}, 119 } 120 ) 121 122 func newNonRunningTestResourceAdmissionController(t *testing.T) ( 123 kubeClient *fakekubeclientset.Clientset, 124 ac webhook.AdmissionController, 125 ) { 126 t.Helper() 127 // Create fake clients 128 kubeClient = fakekubeclientset.NewSimpleClientset(initialResourceWebhook) 129 130 ac = newTestResourceAdmissionController(t) 131 return kubeClient, ac 132 } 133 134 func TestDeleteAllowed(t *testing.T) { 135 _, ac := newNonRunningTestResourceAdmissionController(t) 136 137 req := &admissionv1.AdmissionRequest{ 138 Operation: admissionv1.Delete, 139 Kind: metav1.GroupVersionKind{ 140 Group: "pkg.knative.dev", 141 Version: "v1alpha1", 142 Kind: "Resource", 143 }, 144 } 145 146 if resp := ac.Admit(TestContextWithLogger(t), req); !resp.Allowed { 147 t.Fatal("Unexpected denial of delete") 148 } 149 } 150 151 func TestConnectAllowed(t *testing.T) { 152 _, ac := newNonRunningTestResourceAdmissionController(t) 153 154 req := &admissionv1.AdmissionRequest{ 155 Operation: admissionv1.Connect, 156 Kind: metav1.GroupVersionKind{ 157 Group: "pkg.knative.dev", 158 Version: "v1alpha1", 159 Kind: "Resource", 160 }, 161 } 162 163 resp := ac.Admit(TestContextWithLogger(t), req) 164 if !resp.Allowed { 165 t.Fatalf("Unexpected denial of connect") 166 } 167 } 168 169 func TestUnknownKindFails(t *testing.T) { 170 _, ac := newNonRunningTestResourceAdmissionController(t) 171 172 req := &admissionv1.AdmissionRequest{ 173 Operation: admissionv1.Create, 174 Kind: metav1.GroupVersionKind{ 175 Group: "pkg.knative.dev", 176 Version: "v1alpha1", 177 Kind: "Garbage", 178 }, 179 } 180 181 ExpectFailsWith(t, ac.Admit(TestContextWithLogger(t), req), "unhandled kind") 182 } 183 184 func TestUnknownVersionFails(t *testing.T) { 185 _, ac := newNonRunningTestResourceAdmissionController(t) 186 req := &admissionv1.AdmissionRequest{ 187 Operation: admissionv1.Create, 188 Kind: metav1.GroupVersionKind{ 189 Group: "pkg.knative.dev", 190 Version: "v1beta2", 191 Kind: "Resource", 192 }, 193 } 194 ExpectFailsWith(t, ac.Admit(TestContextWithLogger(t), req), "unhandled kind") 195 } 196 197 func TestUnknownFieldFails(t *testing.T) { 198 _, ac := newNonRunningTestResourceAdmissionController(t) 199 req := &admissionv1.AdmissionRequest{ 200 Operation: admissionv1.Create, 201 Kind: metav1.GroupVersionKind{ 202 Group: "pkg.knative.dev", 203 Version: "v1alpha1", 204 Kind: "Resource", 205 }, 206 } 207 208 marshaled, err := json.Marshal(map[string]interface{}{ 209 "spec": map[string]interface{}{ 210 "foo": "bar", 211 }, 212 }) 213 if err != nil { 214 t.Fatal("Failed to marshal resource:", err) 215 } 216 req.Object.Raw = marshaled 217 218 ExpectFailsWith(t, ac.Admit(TestContextWithLogger(t), req), 219 `decoding request failed: cannot decode incoming new object: json: unknown field "foo"`) 220 } 221 222 func TestUnknownMetadataFieldSucceeds(t *testing.T) { 223 _, ac := newNonRunningTestResourceAdmissionController(t) 224 req := &admissionv1.AdmissionRequest{ 225 Operation: admissionv1.Create, 226 Kind: metav1.GroupVersionKind{ 227 Group: "pkg.knative.dev", 228 Version: "v1alpha1", 229 Kind: "Resource", 230 }, 231 } 232 233 marshaled, err := json.Marshal(map[string]interface{}{ 234 "apiVersion": "pkg.knative.dev/v1alpha1", 235 "kind": "Resource", 236 "metadata": map[string]string{ 237 "unknown": "property", 238 }, 239 "spec": map[string]string{ 240 "fieldWithValidation": "magic value", 241 }, 242 }) 243 if err != nil { 244 t.Fatal("Failed to marshal resource:", err) 245 } 246 req.Object.Raw = marshaled 247 248 ExpectAllowed(t, ac.Admit(TestContextWithLogger(t), req)) 249 } 250 251 func TestAdmitCreates(t *testing.T) { 252 tests := []struct { 253 name string 254 setup func(context.Context, *Resource) 255 rejection string 256 }{{ 257 name: "test simple creation (alpha, no diff)", 258 setup: func(ctx context.Context, r *Resource) { 259 r.TypeMeta.APIVersion = "v1alpha1" 260 r.SetDefaults(ctx) 261 r.Annotations = map[string]string{ 262 "pkg.knative.dev/creator": user1, 263 "pkg.knative.dev/lastModifier": user1, 264 } 265 }, 266 }, { 267 name: "test simple creation (beta, no diff)", 268 setup: func(ctx context.Context, r *Resource) { 269 r.TypeMeta.APIVersion = "v1beta1" 270 r.SetDefaults(ctx) 271 r.Annotations = map[string]string{ 272 "pkg.knative.dev/creator": user1, 273 "pkg.knative.dev/lastModifier": user1, 274 } 275 }, 276 }, { 277 name: "with bad field", 278 setup: func(ctx context.Context, r *Resource) { 279 // Put a bad value in. 280 r.Spec.FieldWithValidation = "not what's expected" 281 }, 282 rejection: "invalid value", 283 }} 284 285 for _, tc := range tests { 286 t.Run(tc.name, func(t *testing.T) { 287 r := CreateResource("a name") 288 ctx := apis.WithinCreate(apis.WithUserInfo( 289 TestContextWithLogger(t), 290 &authenticationv1.UserInfo{Username: user1})) 291 292 // Setup the resource. 293 tc.setup(ctx, r) 294 295 _, ac := newNonRunningTestResourceAdmissionController(t) 296 resp := ac.Admit(ctx, createCreateResource(ctx, t, r)) 297 298 if tc.rejection == "" { 299 ExpectAllowed(t, resp) 300 } else { 301 ExpectFailsWith(t, resp, tc.rejection) 302 } 303 }) 304 } 305 } 306 307 func resourceCallback(ctx context.Context, uns *unstructured.Unstructured) error { 308 var resource Resource 309 if err := runtime.DefaultUnstructuredConverter.FromUnstructured(uns.UnstructuredContent(), &resource); err != nil { 310 return err 311 } 312 313 if apis.IsInDelete(ctx) { 314 if resource.Spec.FieldForCallbackValidation != "magic delete" { 315 return errors.New("no magic delete") 316 } 317 return nil 318 } 319 320 if apis.IsDryRun(ctx) { 321 return errors.New("dryRun fail") 322 } 323 324 if resource.Spec.FieldForCallbackValidation != "" && 325 resource.Spec.FieldForCallbackValidation != "magic value" { 326 return errors.New(resource.Spec.FieldForCallbackValidation) 327 } 328 return nil 329 } 330 331 func TestValidationCreateCallback(t *testing.T) { 332 tests := []struct { 333 name string 334 dryRun bool 335 setup func(context.Context, *Resource) 336 rejection string 337 }{{ 338 name: "with dryRun reject", 339 dryRun: true, 340 setup: func(ctx context.Context, r *Resource) {}, 341 rejection: "validation callback failed: dryRun fail", 342 }, { 343 name: "with dryRun off", 344 dryRun: false, 345 setup: func(ctx context.Context, r *Resource) {}, 346 }, { 347 name: "with field magic value", 348 dryRun: false, 349 setup: func(ctx context.Context, r *Resource) { 350 // Put a good value in. 351 r.Spec.FieldForCallbackValidation = "magic value" 352 }, 353 }, { 354 name: "with field reject value", 355 dryRun: false, 356 setup: func(ctx context.Context, r *Resource) { 357 // Put a bad value in. 358 r.Spec.FieldForCallbackValidation = "callbacks hate this" 359 }, 360 rejection: "validation callback failed: callbacks hate this", 361 }} 362 363 for _, tc := range tests { 364 t.Run(tc.name, func(t *testing.T) { 365 r := CreateResource("a name") 366 ctx := apis.WithinCreate(apis.WithUserInfo( 367 TestContextWithLogger(t), 368 &authenticationv1.UserInfo{Username: user1})) 369 370 // Setup the resource. 371 tc.setup(ctx, r) 372 373 _, ac := newNonRunningTestResourceAdmissionController(t) 374 req := createCreateResource(ctx, t, r) 375 if tc.dryRun { 376 truePoint := true 377 req.DryRun = &truePoint 378 } 379 380 resp := ac.Admit(ctx, req) 381 382 if tc.rejection == "" { 383 ExpectAllowed(t, resp) 384 } else { 385 ExpectFailsWith(t, resp, tc.rejection) 386 } 387 }) 388 } 389 } 390 391 func TestValidationDeleteCallback(t *testing.T) { 392 tests := []struct { 393 name string 394 setup func(context.Context, *Resource) 395 rejection string 396 }{{ 397 name: "with field magic delete", 398 setup: func(ctx context.Context, r *Resource) { 399 // Put a good value in. 400 r.Spec.FieldForCallbackValidation = "magic delete" 401 }, 402 }, { 403 name: "with field reject value", 404 setup: func(ctx context.Context, r *Resource) { 405 // Put a bad value in. 406 r.Spec.FieldForCallbackValidation = "no magic delete" 407 }, 408 rejection: "validation callback failed: no magic delete", 409 }} 410 411 for _, tc := range tests { 412 t.Run(tc.name, func(t *testing.T) { 413 r := CreateResource("a name") 414 ctx := apis.WithinCreate(apis.WithUserInfo( 415 TestContextWithLogger(t), 416 &authenticationv1.UserInfo{Username: user1})) 417 418 // Setup the resource. 419 tc.setup(ctx, r) 420 421 _, ac := newNonRunningTestResourceAdmissionController(t) 422 req := createDeleteResource(ctx, t, r) 423 424 resp := ac.Admit(ctx, req) 425 426 if tc.rejection == "" { 427 ExpectAllowed(t, resp) 428 } else { 429 ExpectFailsWith(t, resp, tc.rejection) 430 } 431 }) 432 } 433 } 434 435 func createDeleteResource(ctx context.Context, t *testing.T, old *Resource) *admissionv1.AdmissionRequest { 436 t.Helper() 437 req := &admissionv1.AdmissionRequest{ 438 Operation: admissionv1.Delete, 439 Kind: metav1.GroupVersionKind{ 440 Group: "pkg.knative.dev", 441 Version: "v1alpha1", 442 Kind: "Resource", 443 }, 444 UserInfo: *apis.GetUserInfo(ctx), 445 } 446 marshaledOld, err := json.Marshal(old) 447 if err != nil { 448 t.Fatal("Failed to marshal resource:", err) 449 } 450 req.OldObject.Raw = marshaledOld 451 req.Resource.Group = "pkg.knative.dev" 452 return req 453 } 454 455 func createCreateResource(ctx context.Context, t *testing.T, r *Resource) *admissionv1.AdmissionRequest { 456 t.Helper() 457 req := &admissionv1.AdmissionRequest{ 458 Operation: admissionv1.Create, 459 Kind: metav1.GroupVersionKind{ 460 Group: "pkg.knative.dev", 461 Version: "v1alpha1", 462 Kind: "Resource", 463 }, 464 UserInfo: *apis.GetUserInfo(ctx), 465 } 466 marshaled, err := json.Marshal(r) 467 if err != nil { 468 t.Fatal("Failed to marshal resource:", err) 469 } 470 req.Object.Raw = marshaled 471 req.Resource.Group = "pkg.knative.dev" 472 return req 473 } 474 475 func TestAdmitUpdates(t *testing.T) { 476 tests := []struct { 477 name string 478 setup func(context.Context, *Resource) 479 mutate func(context.Context, *Resource) 480 subresource string 481 rejection string 482 }{{ 483 name: "test simple update (no diff)", 484 setup: func(ctx context.Context, r *Resource) { 485 r.SetDefaults(ctx) 486 }, 487 mutate: func(ctx context.Context, r *Resource) { 488 // If we don't change anything, the updater 489 // annotation doesn't change. 490 }, 491 }, { 492 name: "bad mutation (immutable)", 493 setup: func(ctx context.Context, r *Resource) { 494 r.SetDefaults(ctx) 495 }, 496 mutate: func(ctx context.Context, r *Resource) { 497 r.Spec.FieldThatsImmutableWithDefault = "something different" 498 }, 499 rejection: "Immutable field changed", 500 }, { 501 name: "bad mutation (validation)", 502 setup: func(ctx context.Context, r *Resource) { 503 r.SetDefaults(ctx) 504 }, 505 mutate: func(ctx context.Context, r *Resource) { 506 r.Spec.FieldWithValidation = "not what's expected" 507 }, 508 rejection: "invalid value", 509 }, { 510 name: "bad mutation (invalid subresource)", 511 setup: func(ctx context.Context, r *Resource) { 512 r.SetDefaults(ctx) 513 }, 514 mutate: func(ctx context.Context, r *Resource) { 515 }, 516 subresource: "badbadsubresource", 517 rejection: "validation failed: Disallowed subresource update: \nDisallowed subresource update: badbadsubresource", 518 }, { 519 name: "good mutation with valid subresource", 520 setup: func(ctx context.Context, r *Resource) { 521 r.SetDefaults(ctx) 522 }, 523 mutate: func(ctx context.Context, r *Resource) { 524 }, 525 subresource: "goodgoodsubresource", 526 }} 527 528 for _, tc := range tests { 529 t.Run(tc.name, func(t *testing.T) { 530 old := CreateResource("a name") 531 ctx := TestContextWithLogger(t) 532 533 old.Annotations = map[string]string{ 534 "pkg.knative.dev/creator": user1, 535 "pkg.knative.dev/lastModifier": user1, 536 } 537 538 tc.setup(ctx, old) 539 540 new := old.DeepCopy() 541 542 // Mutate the resource using the update context as user2 543 ctx = apis.WithUserInfo(apis.WithinUpdate(ctx, old), 544 &authenticationv1.UserInfo{Username: user2}) 545 tc.mutate(ctx, new) 546 547 _, ac := newNonRunningTestResourceAdmissionController(t) 548 resp := ac.Admit(ctx, createUpdateResource(ctx, t, old, new, tc.subresource)) 549 550 if tc.rejection == "" { 551 ExpectAllowed(t, resp) 552 } else { 553 ExpectFailsWith(t, resp, tc.rejection) 554 } 555 }) 556 } 557 } 558 559 func createUpdateResource(ctx context.Context, t *testing.T, old, new *Resource, subresource string) *admissionv1.AdmissionRequest { 560 t.Helper() 561 req := &admissionv1.AdmissionRequest{ 562 Operation: admissionv1.Update, 563 Kind: metav1.GroupVersionKind{ 564 Group: "pkg.knative.dev", 565 Version: "v1alpha1", 566 Kind: "Resource", 567 }, 568 UserInfo: *apis.GetUserInfo(ctx), 569 SubResource: subresource, 570 } 571 marshaled, err := json.Marshal(new) 572 if err != nil { 573 t.Fatal("Failed to marshal resource:", err) 574 } 575 req.Object.Raw = marshaled 576 marshaledOld, err := json.Marshal(old) 577 if err != nil { 578 t.Fatal("Failed to marshal resource:", err) 579 } 580 req.OldObject.Raw = marshaledOld 581 req.Resource.Group = "pkg.knative.dev" 582 return req 583 } 584 585 func createInnerDefaultResourceWithoutSpec(t *testing.T) []byte { 586 t.Helper() 587 r := InnerDefaultResource{ 588 TypeMeta: metav1.TypeMeta{ 589 Kind: "testKind", 590 APIVersion: "testAPIVersion", 591 }, 592 ObjectMeta: metav1.ObjectMeta{ 593 Namespace: system.Namespace(), 594 Name: "a name", 595 }, 596 } 597 // Remove the 'spec' field of the generated JSON by marshaling it to JSON, parsing that as a 598 // generic map[string]interface{}, removing 'spec', and marshaling it again. 599 origBytes, err := json.Marshal(r) 600 if err != nil { 601 t.Fatal("Error marshaling origBytes:", err) 602 } 603 var q map[string]interface{} 604 if err := json.Unmarshal(origBytes, &q); err != nil { 605 t.Fatal("Error unmarshaling origBytes:", err) 606 } 607 delete(q, "spec") 608 b, err := json.Marshal(q) 609 if err != nil { 610 t.Fatal("Error marshaling q:", err) 611 } 612 return b 613 } 614 615 func createInnerDefaultResourceWithSpecAndStatus(t *testing.T, spec *InnerDefaultSpec, status *InnerDefaultStatus) []byte { 616 t.Helper() 617 r := InnerDefaultResource{ 618 TypeMeta: metav1.TypeMeta{ 619 Kind: "testKind", 620 APIVersion: "testAPIVersion", 621 }, 622 ObjectMeta: metav1.ObjectMeta{ 623 Namespace: system.Namespace(), 624 Name: "a name", 625 }, 626 } 627 if spec != nil { 628 r.Spec = *spec 629 } 630 if status != nil { 631 r.Status = *status 632 } 633 634 b, err := json.Marshal(r) 635 if err != nil { 636 t.Fatal("Error marshaling bytes:", err) 637 } 638 return b 639 } 640 641 func TestNewResourceAdmissionController(t *testing.T) { 642 ctx, _ := SetupFakeContext(t) 643 644 defer func() { 645 if r := recover(); r == nil { 646 t.Errorf("Expected a second callback to panic") 647 } 648 }() 649 650 invalidSecondCallback := map[schema.GroupVersionKind]Callback{} 651 652 NewAdmissionController( 653 ctx, testResourceValidationName, testResourceValidationPath, 654 handlers, 655 func(ctx context.Context) context.Context { 656 return ctx 657 }, true, 658 callbacks, 659 invalidSecondCallback) 660 } 661 662 func TestNewResourceAdmissionControllerDuplicateVerb(t *testing.T) { 663 ctx, _ := SetupFakeContext(t) 664 665 defer func() { 666 if r := recover(); r == nil { 667 t.Errorf("Expected a second callback to panic") 668 } 669 }() 670 671 call := map[schema.GroupVersionKind]Callback{ 672 { 673 Group: "pkg.knative.dev", 674 Version: "v1alpha1", 675 Kind: "Resource", 676 }: NewCallback(resourceCallback, webhook.Create, webhook.Create), // Disallow duplicates under test 677 } 678 679 NewAdmissionController( 680 ctx, testResourceValidationName, testResourceValidationPath, 681 handlers, 682 func(ctx context.Context) context.Context { 683 return ctx 684 }, true, 685 call) 686 } 687 688 func newTestResourceAdmissionController(t *testing.T) webhook.AdmissionController { 689 ctx, _ := SetupFakeContext(t) 690 ctx = webhook.WithOptions(ctx, webhook.Options{ 691 SecretName: "webhook-secret", 692 }) 693 694 c := NewAdmissionControllerWithConfig( 695 ctx, testResourceValidationName, testResourceValidationPath, 696 handlers, 697 func(ctx context.Context) context.Context { 698 return ctx 699 }, true, callbacks) 700 if c == nil { 701 t.Fatal("Expected NewController to return a non-nil value") 702 } 703 704 if want, got := 0, c.WorkQueue().Len(); want != got { 705 t.Errorf("WorkQueue.Len() = %d, wanted %d", got, want) 706 } 707 708 la, ok := c.Reconciler.(pkgreconciler.LeaderAware) 709 if !ok { 710 t.Fatalf("%T is not leader aware", c.Reconciler) 711 } 712 713 if err := la.Promote(pkgreconciler.UniversalBucket(), c.MaybeEnqueueBucketKey); err != nil { 714 t.Error("Promote() =", err) 715 } 716 717 // Queue has async moving parts so if we check at the wrong moment, this might still be 0. 718 if wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, 250*time.Millisecond, true, func(ctx context.Context) (bool, error) { 719 return c.WorkQueue().Len() == 1, nil 720 }) != nil { 721 t.Error("Queue length was never 1") 722 } 723 724 return c.Reconciler.(*reconciler) 725 }