k8s.io/apiserver@v0.31.1/pkg/storage/cacher/cacher_whitebox_test.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package cacher 18 19 import ( 20 "context" 21 "crypto/rand" 22 "errors" 23 "fmt" 24 "reflect" 25 goruntime "runtime" 26 "strconv" 27 "strings" 28 "sync" 29 "testing" 30 "time" 31 32 "github.com/stretchr/testify/require" 33 "google.golang.org/grpc/metadata" 34 35 apiequality "k8s.io/apimachinery/pkg/api/equality" 36 apierrors "k8s.io/apimachinery/pkg/api/errors" 37 "k8s.io/apimachinery/pkg/api/meta" 38 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 39 "k8s.io/apimachinery/pkg/fields" 40 "k8s.io/apimachinery/pkg/labels" 41 "k8s.io/apimachinery/pkg/runtime" 42 "k8s.io/apimachinery/pkg/runtime/schema" 43 "k8s.io/apimachinery/pkg/util/wait" 44 "k8s.io/apimachinery/pkg/watch" 45 "k8s.io/apiserver/pkg/apis/example" 46 examplev1 "k8s.io/apiserver/pkg/apis/example/v1" 47 "k8s.io/apiserver/pkg/features" 48 "k8s.io/apiserver/pkg/storage" 49 "k8s.io/apiserver/pkg/storage/cacher/metrics" 50 etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" 51 etcdfeature "k8s.io/apiserver/pkg/storage/feature" 52 utilfeature "k8s.io/apiserver/pkg/util/feature" 53 featuregatetesting "k8s.io/component-base/featuregate/testing" 54 k8smetrics "k8s.io/component-base/metrics" 55 "k8s.io/component-base/metrics/testutil" 56 "k8s.io/utils/clock" 57 testingclock "k8s.io/utils/clock/testing" 58 "k8s.io/utils/pointer" 59 ) 60 61 func newTestCacherWithoutSyncing(s storage.Interface) (*Cacher, storage.Versioner, error) { 62 prefix := "pods" 63 config := Config{ 64 Storage: s, 65 Versioner: storage.APIObjectVersioner{}, 66 GroupResource: schema.GroupResource{Resource: "pods"}, 67 ResourcePrefix: prefix, 68 KeyFunc: func(obj runtime.Object) (string, error) { return storage.NamespaceKeyFunc(prefix, obj) }, 69 GetAttrsFunc: func(obj runtime.Object) (labels.Set, fields.Set, error) { 70 pod, ok := obj.(*example.Pod) 71 if !ok { 72 return storage.DefaultNamespaceScopedAttr(obj) 73 } 74 labelsSet, fieldsSet, err := storage.DefaultNamespaceScopedAttr(obj) 75 if err != nil { 76 return nil, nil, err 77 } 78 fieldsSet["spec.nodeName"] = pod.Spec.NodeName 79 return labelsSet, fieldsSet, nil 80 }, 81 NewFunc: func() runtime.Object { return &example.Pod{} }, 82 NewListFunc: func() runtime.Object { return &example.PodList{} }, 83 Codec: codecs.LegacyCodec(examplev1.SchemeGroupVersion), 84 Clock: clock.RealClock{}, 85 } 86 cacher, err := NewCacherFromConfig(config) 87 88 return cacher, storage.APIObjectVersioner{}, err 89 } 90 91 func newTestCacher(s storage.Interface) (*Cacher, storage.Versioner, error) { 92 cacher, versioner, err := newTestCacherWithoutSyncing(s) 93 if err != nil { 94 return nil, versioner, err 95 } 96 97 if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 98 // The tests assume that Get/GetList/Watch calls shouldn't fail. 99 // However, 429 error can now be returned if watchcache is under initialization. 100 // To avoid rewriting all tests, we wait for watcache to initialize. 101 if err := cacher.Wait(context.Background()); err != nil { 102 return nil, storage.APIObjectVersioner{}, err 103 } 104 } 105 return cacher, versioner, nil 106 } 107 108 type dummyStorage struct { 109 sync.RWMutex 110 err error 111 getListFn func(_ context.Context, _ string, _ storage.ListOptions, listObj runtime.Object) error 112 watchFn func(_ context.Context, _ string, _ storage.ListOptions) (watch.Interface, error) 113 114 // use getRequestWatchProgressCounter when reading 115 // the value of the counter 116 requestWatchProgressCounter int 117 } 118 119 func (d *dummyStorage) RequestWatchProgress(ctx context.Context) error { 120 d.Lock() 121 defer d.Unlock() 122 d.requestWatchProgressCounter++ 123 return nil 124 } 125 126 func (d *dummyStorage) getRequestWatchProgressCounter() int { 127 d.RLock() 128 defer d.RUnlock() 129 return d.requestWatchProgressCounter 130 } 131 132 type dummyWatch struct { 133 ch chan watch.Event 134 } 135 136 func (w *dummyWatch) ResultChan() <-chan watch.Event { 137 return w.ch 138 } 139 140 func (w *dummyWatch) Stop() { 141 close(w.ch) 142 } 143 144 func newDummyWatch() watch.Interface { 145 return &dummyWatch{ 146 ch: make(chan watch.Event), 147 } 148 } 149 150 func (d *dummyStorage) Versioner() storage.Versioner { return nil } 151 func (d *dummyStorage) Create(_ context.Context, _ string, _, _ runtime.Object, _ uint64) error { 152 return fmt.Errorf("unimplemented") 153 } 154 func (d *dummyStorage) Delete(_ context.Context, _ string, _ runtime.Object, _ *storage.Preconditions, _ storage.ValidateObjectFunc, _ runtime.Object) error { 155 return fmt.Errorf("unimplemented") 156 } 157 func (d *dummyStorage) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) { 158 if d.watchFn != nil { 159 return d.watchFn(ctx, key, opts) 160 } 161 d.RLock() 162 defer d.RUnlock() 163 164 return newDummyWatch(), d.err 165 } 166 func (d *dummyStorage) Get(_ context.Context, _ string, _ storage.GetOptions, _ runtime.Object) error { 167 d.RLock() 168 defer d.RUnlock() 169 170 return d.err 171 } 172 func (d *dummyStorage) GetList(ctx context.Context, resPrefix string, opts storage.ListOptions, listObj runtime.Object) error { 173 if d.getListFn != nil { 174 return d.getListFn(ctx, resPrefix, opts, listObj) 175 } 176 d.RLock() 177 defer d.RUnlock() 178 podList := listObj.(*example.PodList) 179 podList.ListMeta = metav1.ListMeta{ResourceVersion: "100"} 180 return d.err 181 } 182 func (d *dummyStorage) GuaranteedUpdate(_ context.Context, _ string, _ runtime.Object, _ bool, _ *storage.Preconditions, _ storage.UpdateFunc, _ runtime.Object) error { 183 return fmt.Errorf("unimplemented") 184 } 185 func (d *dummyStorage) Count(_ string) (int64, error) { 186 return 0, fmt.Errorf("unimplemented") 187 } 188 func (d *dummyStorage) ReadinessCheck() error { 189 return nil 190 } 191 func (d *dummyStorage) injectError(err error) { 192 d.Lock() 193 defer d.Unlock() 194 195 d.err = err 196 } 197 198 func TestGetListCacheBypass(t *testing.T) { 199 type testCase struct { 200 opts storage.ListOptions 201 expectBypass bool 202 } 203 commonTestCases := []testCase{ 204 {opts: storage.ListOptions{ResourceVersion: "0"}, expectBypass: false}, 205 {opts: storage.ListOptions{ResourceVersion: "1"}, expectBypass: false}, 206 207 {opts: storage.ListOptions{ResourceVersion: "", Predicate: storage.SelectionPredicate{Continue: "a"}}, expectBypass: true}, 208 {opts: storage.ListOptions{ResourceVersion: "0", Predicate: storage.SelectionPredicate{Continue: "a"}}, expectBypass: true}, 209 {opts: storage.ListOptions{ResourceVersion: "1", Predicate: storage.SelectionPredicate{Continue: "a"}}, expectBypass: true}, 210 211 {opts: storage.ListOptions{ResourceVersion: "0", Predicate: storage.SelectionPredicate{Limit: 500}}, expectBypass: false}, 212 {opts: storage.ListOptions{ResourceVersion: "1", Predicate: storage.SelectionPredicate{Limit: 500}}, expectBypass: true}, 213 214 {opts: storage.ListOptions{ResourceVersion: "", ResourceVersionMatch: metav1.ResourceVersionMatchExact}, expectBypass: true}, 215 {opts: storage.ListOptions{ResourceVersion: "0", ResourceVersionMatch: metav1.ResourceVersionMatchExact}, expectBypass: true}, 216 {opts: storage.ListOptions{ResourceVersion: "1", ResourceVersionMatch: metav1.ResourceVersionMatchExact}, expectBypass: true}, 217 } 218 219 t.Run("ConsistentListFromStorage", func(t *testing.T) { 220 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, false) 221 testCases := append(commonTestCases, 222 testCase{opts: storage.ListOptions{ResourceVersion: ""}, expectBypass: true}, 223 testCase{opts: storage.ListOptions{ResourceVersion: "", Predicate: storage.SelectionPredicate{Limit: 500}}, expectBypass: true}, 224 ) 225 for _, tc := range testCases { 226 testGetListCacheBypass(t, tc.opts, tc.expectBypass) 227 } 228 229 }) 230 t.Run("ConsistentListFromCache", func(t *testing.T) { 231 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true) 232 233 // TODO(p0lyn0mial): the following tests assume that etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress) 234 // evaluates to true. Otherwise the cache will be bypassed and the test will fail. 235 // 236 // If you were to run only TestGetListCacheBypass you would see that the test fail. 237 // However in CI all test are run and there must be a test(s) that properly 238 // initialize the storage layer so that the mentioned method evaluates to true 239 forceRequestWatchProgressSupport(t) 240 241 testCases := append(commonTestCases, 242 testCase{opts: storage.ListOptions{ResourceVersion: ""}, expectBypass: false}, 243 testCase{opts: storage.ListOptions{ResourceVersion: "", Predicate: storage.SelectionPredicate{Limit: 500}}, expectBypass: false}, 244 ) 245 for _, tc := range testCases { 246 testGetListCacheBypass(t, tc.opts, tc.expectBypass) 247 } 248 }) 249 } 250 251 func testGetListCacheBypass(t *testing.T, options storage.ListOptions, expectBypass bool) { 252 backingStorage := &dummyStorage{} 253 cacher, _, err := newTestCacher(backingStorage) 254 if err != nil { 255 t.Fatalf("Couldn't create cacher: %v", err) 256 } 257 defer cacher.Stop() 258 259 result := &example.PodList{} 260 261 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 262 if err := cacher.ready.wait(context.Background()); err != nil { 263 t.Fatalf("unexpected error waiting for the cache to be ready") 264 } 265 } 266 267 // Inject error to underlying layer and check if cacher is not bypassed. 268 backingStorage.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { 269 currentResourceVersion := "42" 270 switch { 271 // request made by getCurrentResourceVersionFromStorage by checking Limit 272 case key == cacher.resourcePrefix: 273 podList := listObj.(*example.PodList) 274 podList.ResourceVersion = currentResourceVersion 275 return nil 276 // request made by storage.GetList with revision from original request or 277 // returned by getCurrentResourceVersionFromStorage 278 case opts.ResourceVersion == options.ResourceVersion || opts.ResourceVersion == currentResourceVersion: 279 return errDummy 280 default: 281 t.Fatalf("Unexpected request %+v", opts) 282 return nil 283 } 284 } 285 err = cacher.GetList(context.TODO(), "pods/ns", options, result) 286 if err != nil && err != errDummy { 287 t.Fatalf("Unexpected error for List request with options: %v, err: %v", options, err) 288 } 289 gotBypass := err == errDummy 290 if gotBypass != expectBypass { 291 t.Errorf("Unexpected bypass result for List request with options %+v, bypass expected: %v, got: %v", options, expectBypass, gotBypass) 292 } 293 } 294 295 func TestConsistentReadFallback(t *testing.T) { 296 tcs := []struct { 297 name string 298 consistentReadsEnabled bool 299 watchCacheRV string 300 storageRV string 301 fallbackError bool 302 303 expectError bool 304 expectRV string 305 expectBlock bool 306 expectRequestsToStorage int 307 expectMetric string 308 }{ 309 { 310 name: "Success", 311 consistentReadsEnabled: true, 312 watchCacheRV: "42", 313 storageRV: "42", 314 expectRV: "42", 315 expectRequestsToStorage: 1, 316 expectMetric: ` 317 # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache. 318 # TYPE apiserver_watch_cache_consistent_read_total counter 319 apiserver_watch_cache_consistent_read_total{fallback="false", resource="pods", success="true"} 1 320 `, 321 }, 322 { 323 name: "Fallback", 324 consistentReadsEnabled: true, 325 watchCacheRV: "2", 326 storageRV: "42", 327 expectRV: "42", 328 expectBlock: true, 329 expectRequestsToStorage: 2, 330 expectMetric: ` 331 # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache. 332 # TYPE apiserver_watch_cache_consistent_read_total counter 333 apiserver_watch_cache_consistent_read_total{fallback="true", resource="pods", success="true"} 1 334 `, 335 }, 336 { 337 name: "Fallback Failure", 338 consistentReadsEnabled: true, 339 watchCacheRV: "2", 340 storageRV: "42", 341 fallbackError: true, 342 expectError: true, 343 expectBlock: true, 344 expectRequestsToStorage: 2, 345 expectMetric: ` 346 # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache. 347 # TYPE apiserver_watch_cache_consistent_read_total counter 348 apiserver_watch_cache_consistent_read_total{fallback="true", resource="pods", success="false"} 1 349 `, 350 }, 351 { 352 name: "Disabled", 353 watchCacheRV: "2", 354 storageRV: "42", 355 expectRV: "42", 356 expectRequestsToStorage: 1, 357 expectMetric: ``, 358 }, 359 } 360 for _, tc := range tcs { 361 t.Run(tc.name, func(t *testing.T) { 362 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, tc.consistentReadsEnabled) 363 if tc.consistentReadsEnabled { 364 forceRequestWatchProgressSupport(t) 365 } 366 367 registry := k8smetrics.NewKubeRegistry() 368 metrics.ConsistentReadTotal.Reset() 369 if err := registry.Register(metrics.ConsistentReadTotal); err != nil { 370 t.Errorf("unexpected error: %v", err) 371 } 372 backingStorage := &dummyStorage{} 373 backingStorage.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { 374 podList := listObj.(*example.PodList) 375 podList.ResourceVersion = tc.watchCacheRV 376 return nil 377 } 378 // TODO: Use fake clock for this test to reduce execution time. 379 cacher, _, err := newTestCacher(backingStorage) 380 if err != nil { 381 t.Fatalf("Couldn't create cacher: %v", err) 382 } 383 defer cacher.Stop() 384 385 if fmt.Sprintf("%d", cacher.watchCache.resourceVersion) != tc.watchCacheRV { 386 t.Fatalf("Expected watch cache RV to equal watchCacheRV, got: %d, want: %s", cacher.watchCache.resourceVersion, tc.watchCacheRV) 387 } 388 requestToStorageCount := 0 389 backingStorage.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { 390 requestToStorageCount += 1 391 podList := listObj.(*example.PodList) 392 if key == cacher.resourcePrefix { 393 podList.ResourceVersion = tc.storageRV 394 return nil 395 } 396 if tc.fallbackError { 397 return errDummy 398 } 399 podList.ResourceVersion = tc.storageRV 400 return nil 401 } 402 result := &example.PodList{} 403 start := cacher.clock.Now() 404 err = cacher.GetList(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: ""}, result) 405 duration := cacher.clock.Since(start) 406 if (err != nil) != tc.expectError { 407 t.Fatalf("Unexpected error err: %v", err) 408 } 409 if result.ResourceVersion != tc.expectRV { 410 t.Fatalf("Unexpected List response RV, got: %q, want: %q", result.ResourceVersion, tc.expectRV) 411 } 412 if requestToStorageCount != tc.expectRequestsToStorage { 413 t.Fatalf("Unexpected number of requests to storage, got: %d, want: %d", requestToStorageCount, tc.expectRequestsToStorage) 414 } 415 blocked := duration >= blockTimeout 416 if blocked != tc.expectBlock { 417 t.Fatalf("Unexpected block, got: %v, want: %v", blocked, tc.expectBlock) 418 } 419 420 if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectMetric), "apiserver_watch_cache_consistent_read_total"); err != nil { 421 t.Errorf("unexpected error: %v", err) 422 } 423 }) 424 } 425 } 426 427 func TestGetListNonRecursiveCacheBypass(t *testing.T) { 428 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, false) 429 backingStorage := &dummyStorage{} 430 cacher, _, err := newTestCacher(backingStorage) 431 if err != nil { 432 t.Fatalf("Couldn't create cacher: %v", err) 433 } 434 defer cacher.Stop() 435 436 pred := storage.SelectionPredicate{ 437 Limit: 500, 438 } 439 result := &example.PodList{} 440 441 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 442 if err := cacher.ready.wait(context.Background()); err != nil { 443 t.Fatalf("unexpected error waiting for the cache to be ready") 444 } 445 } 446 447 // Inject error to underlying layer and check if cacher is not bypassed. 448 backingStorage.injectError(errDummy) 449 err = cacher.GetList(context.TODO(), "pods/ns", storage.ListOptions{ 450 ResourceVersion: "0", 451 Predicate: pred, 452 }, result) 453 if err != nil { 454 t.Errorf("GetList with Limit and RV=0 should be served from cache: %v", err) 455 } 456 457 err = cacher.GetList(context.TODO(), "pods/ns", storage.ListOptions{ 458 ResourceVersion: "", 459 Predicate: pred, 460 }, result) 461 if err != errDummy { 462 t.Errorf("GetList with Limit without RV=0 should bypass cacher: %v", err) 463 } 464 } 465 466 func TestGetCacheBypass(t *testing.T) { 467 backingStorage := &dummyStorage{} 468 cacher, _, err := newTestCacher(backingStorage) 469 if err != nil { 470 t.Fatalf("Couldn't create cacher: %v", err) 471 } 472 defer cacher.Stop() 473 474 result := &example.Pod{} 475 476 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 477 if err := cacher.ready.wait(context.Background()); err != nil { 478 t.Fatalf("unexpected error waiting for the cache to be ready") 479 } 480 } 481 482 // Inject error to underlying layer and check if cacher is not bypassed. 483 backingStorage.injectError(errDummy) 484 err = cacher.Get(context.TODO(), "pods/ns/pod-0", storage.GetOptions{ 485 IgnoreNotFound: true, 486 ResourceVersion: "0", 487 }, result) 488 if err != nil { 489 t.Errorf("Get with RV=0 should be served from cache: %v", err) 490 } 491 492 err = cacher.Get(context.TODO(), "pods/ns/pod-0", storage.GetOptions{ 493 IgnoreNotFound: true, 494 ResourceVersion: "", 495 }, result) 496 if err != errDummy { 497 t.Errorf("Get without RV=0 should bypass cacher: %v", err) 498 } 499 } 500 501 func TestWatchCacheBypass(t *testing.T) { 502 backingStorage := &dummyStorage{} 503 cacher, _, err := newTestCacher(backingStorage) 504 if err != nil { 505 t.Fatalf("Couldn't create cacher: %v", err) 506 } 507 defer cacher.Stop() 508 509 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 510 if err := cacher.ready.wait(context.Background()); err != nil { 511 t.Fatalf("unexpected error waiting for the cache to be ready") 512 } 513 } 514 515 _, err = cacher.Watch(context.TODO(), "pod/ns", storage.ListOptions{ 516 ResourceVersion: "0", 517 Predicate: storage.Everything, 518 }) 519 if err != nil { 520 t.Errorf("Watch with RV=0 should be served from cache: %v", err) 521 } 522 523 _, err = cacher.Watch(context.TODO(), "pod/ns", storage.ListOptions{ 524 ResourceVersion: "", 525 Predicate: storage.Everything, 526 }) 527 if err != nil { 528 t.Errorf("Watch with RV=0 should be served from cache: %v", err) 529 } 530 531 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchFromStorageWithoutResourceVersion, false) 532 _, err = cacher.Watch(context.TODO(), "pod/ns", storage.ListOptions{ 533 ResourceVersion: "", 534 Predicate: storage.Everything, 535 }) 536 if err != nil { 537 t.Errorf("With WatchFromStorageWithoutResourceVersion disabled, watch with unset RV should be served from cache: %v", err) 538 } 539 540 // Inject error to underlying layer and check if cacher is not bypassed. 541 backingStorage.injectError(errDummy) 542 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchFromStorageWithoutResourceVersion, true) 543 _, err = cacher.Watch(context.TODO(), "pod/ns", storage.ListOptions{ 544 ResourceVersion: "", 545 Predicate: storage.Everything, 546 }) 547 if !errors.Is(err, errDummy) { 548 t.Errorf("With WatchFromStorageWithoutResourceVersion enabled, watch with unset RV should be served from storage: %v", err) 549 } 550 } 551 552 func TestTooManyRequestsNotReturned(t *testing.T) { 553 // Ensure that with ResilientWatchCacheInitialization feature disabled, we don't return 429 554 // errors when watchcache is not initialized. 555 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ResilientWatchCacheInitialization, false) 556 557 dummyErr := fmt.Errorf("dummy") 558 backingStorage := &dummyStorage{err: dummyErr} 559 cacher, _, err := newTestCacherWithoutSyncing(backingStorage) 560 if err != nil { 561 t.Fatalf("Couldn't create cacher: %v", err) 562 } 563 defer cacher.Stop() 564 565 opts := storage.ListOptions{ 566 ResourceVersion: "0", 567 Predicate: storage.Everything, 568 } 569 570 // Cancel the request so that it doesn't hang forever. 571 listCtx, listCancel := context.WithTimeout(context.Background(), 250*time.Millisecond) 572 defer listCancel() 573 574 result := &example.PodList{} 575 err = cacher.GetList(listCtx, "/pods/ns", opts, result) 576 if err != nil && apierrors.IsTooManyRequests(err) { 577 t.Errorf("Unexpected 429 error without ResilientWatchCacheInitialization feature for List") 578 } 579 580 watchCtx, watchCancel := context.WithTimeout(context.Background(), 250*time.Millisecond) 581 defer watchCancel() 582 583 _, err = cacher.Watch(watchCtx, "/pods/ns", opts) 584 if err != nil && apierrors.IsTooManyRequests(err) { 585 t.Errorf("Unexpected 429 error without ResilientWatchCacheInitialization feature for Watch") 586 } 587 } 588 589 func TestEmptyWatchEventCache(t *testing.T) { 590 server, etcdStorage := newEtcdTestStorage(t, etcd3testing.PathPrefix()) 591 defer server.Terminate(t) 592 593 // add a few objects 594 v := storage.APIObjectVersioner{} 595 lastRV := uint64(0) 596 for i := 0; i < 5; i++ { 597 pod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("foo-%d", i), Namespace: "test-ns"}} 598 out := &example.Pod{} 599 key := computePodKey(pod) 600 if err := etcdStorage.Create(context.Background(), key, pod, out, 0); err != nil { 601 t.Fatalf("Create failed: %v", err) 602 } 603 var err error 604 if lastRV, err = v.ParseResourceVersion(out.ResourceVersion); err != nil { 605 t.Fatalf("Unexpected error: %v", err) 606 } 607 } 608 609 cacher, _, err := newTestCacher(etcdStorage) 610 if err != nil { 611 t.Fatalf("Couldn't create cacher: %v", err) 612 } 613 defer cacher.Stop() 614 615 // Given that cacher is always initialized from the "current" version of etcd, 616 // we now have a cacher with an empty cache of watch events and a resourceVersion of rv. 617 // It should support establishing watches from rv and higher, but not older. 618 619 expectedResourceExpiredError := apierrors.NewResourceExpired("").ErrStatus 620 tests := []struct { 621 name string 622 resourceVersion uint64 623 expectedEvent *watch.Event 624 }{ 625 { 626 name: "RV-1", 627 resourceVersion: lastRV - 1, 628 expectedEvent: &watch.Event{Type: watch.Error, Object: &expectedResourceExpiredError}, 629 }, 630 { 631 name: "RV", 632 resourceVersion: lastRV, 633 }, 634 { 635 name: "RV+1", 636 resourceVersion: lastRV + 1, 637 }, 638 } 639 640 for _, tt := range tests { 641 t.Run(tt.name, func(t *testing.T) { 642 opts := storage.ListOptions{ 643 ResourceVersion: strconv.Itoa(int(tt.resourceVersion)), 644 Predicate: storage.Everything, 645 } 646 watcher, err := cacher.Watch(context.Background(), "/pods/test-ns", opts) 647 if err != nil { 648 t.Fatalf("Failed to create watch: %v", err) 649 } 650 defer watcher.Stop() 651 select { 652 case event := <-watcher.ResultChan(): 653 if tt.expectedEvent == nil { 654 t.Errorf("Unexpected event: type=%#v, object=%#v", event.Type, event.Object) 655 break 656 } 657 if e, a := tt.expectedEvent.Type, event.Type; e != a { 658 t.Errorf("Expected: %s, got: %s", e, a) 659 } 660 if e, a := tt.expectedEvent.Object, event.Object; !apiequality.Semantic.DeepDerivative(e, a) { 661 t.Errorf("Expected: %#v, got: %#v", e, a) 662 } 663 case <-time.After(1 * time.Second): 664 // the watch was established otherwise 665 // we would be blocking on cache.Watch(...) 666 // in addition to that, the tests are serial in nature, 667 // meaning that there is no other actor 668 // that could add items to the database, 669 // which could result in receiving new items. 670 // given that waiting 1s seems okay 671 if tt.expectedEvent != nil { 672 t.Errorf("Failed to get an event") 673 } 674 // watch remained established successfully 675 } 676 }) 677 } 678 } 679 680 func TestWatchNotHangingOnStartupFailure(t *testing.T) { 681 // Configure cacher so that it can't initialize, because of 682 // constantly failing lists to the underlying storage. 683 dummyErr := fmt.Errorf("dummy") 684 backingStorage := &dummyStorage{err: dummyErr} 685 cacher, _, err := newTestCacherWithoutSyncing(backingStorage) 686 if err != nil { 687 t.Fatalf("Couldn't create cacher: %v", err) 688 } 689 defer cacher.Stop() 690 691 ctx, cancel := context.WithCancel(context.Background()) 692 // Cancel the watch after some time to check if it will properly 693 // terminate instead of hanging forever. 694 go func() { 695 defer cancel() 696 cacher.clock.Sleep(1 * time.Second) 697 }() 698 699 // Watch hangs waiting on watchcache being initialized. 700 // Ensure that it terminates when its context is cancelled 701 // (e.g. the request is terminated for whatever reason). 702 _, err = cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "0"}) 703 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 704 if err == nil || err.Error() != apierrors.NewServiceUnavailable(context.Canceled.Error()).Error() { 705 t.Errorf("Unexpected error: %#v", err) 706 } 707 } else { 708 if err == nil || err.Error() != apierrors.NewTooManyRequests("storage is (re)initializing", 1).Error() { 709 t.Errorf("Unexpected error: %#v", err) 710 } 711 } 712 } 713 714 func TestWatcherNotGoingBackInTime(t *testing.T) { 715 backingStorage := &dummyStorage{} 716 cacher, v, err := newTestCacher(backingStorage) 717 if err != nil { 718 t.Fatalf("Couldn't create cacher: %v", err) 719 } 720 defer cacher.Stop() 721 722 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 723 if err := cacher.ready.wait(context.Background()); err != nil { 724 t.Fatalf("unexpected error waiting for the cache to be ready") 725 } 726 } 727 728 // Ensure there is some budget for slowing down processing. 729 cacher.dispatchTimeoutBudget.returnUnused(100 * time.Millisecond) 730 731 makePod := func(i int) *examplev1.Pod { 732 return &examplev1.Pod{ 733 ObjectMeta: metav1.ObjectMeta{ 734 Name: fmt.Sprintf("pod-%d", 1000+i), 735 Namespace: "ns", 736 ResourceVersion: fmt.Sprintf("%d", 1000+i), 737 }, 738 } 739 } 740 if err := cacher.watchCache.Add(makePod(0)); err != nil { 741 t.Errorf("error: %v", err) 742 } 743 744 totalPods := 100 745 746 // Create watcher that will be slowing down reading. 747 w1, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ 748 ResourceVersion: "999", 749 Predicate: storage.Everything, 750 }) 751 if err != nil { 752 t.Fatalf("Failed to create watch: %v", err) 753 } 754 defer w1.Stop() 755 go func() { 756 a := 0 757 for range w1.ResultChan() { 758 time.Sleep(time.Millisecond) 759 a++ 760 if a == 100 { 761 break 762 } 763 } 764 }() 765 766 // Now push a ton of object to cache. 767 for i := 1; i < totalPods; i++ { 768 cacher.watchCache.Add(makePod(i)) 769 } 770 771 // Create fast watcher and ensure it will get each object exactly once. 772 w2, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: storage.Everything}) 773 if err != nil { 774 t.Fatalf("Failed to create watch: %v", err) 775 } 776 defer w2.Stop() 777 778 shouldContinue := true 779 currentRV := uint64(0) 780 for shouldContinue { 781 select { 782 case event, ok := <-w2.ResultChan(): 783 if !ok { 784 shouldContinue = false 785 break 786 } 787 rv, err := v.ParseResourceVersion(event.Object.(metaRuntimeInterface).GetResourceVersion()) 788 if err != nil { 789 t.Errorf("unexpected parsing error: %v", err) 790 } else { 791 if rv < currentRV { 792 t.Errorf("watcher going back in time") 793 } 794 currentRV = rv 795 } 796 case <-time.After(time.Second): 797 w2.Stop() 798 } 799 } 800 } 801 802 func TestCacherDontAcceptRequestsStopped(t *testing.T) { 803 backingStorage := &dummyStorage{} 804 cacher, _, err := newTestCacher(backingStorage) 805 if err != nil { 806 t.Fatalf("Couldn't create cacher: %v", err) 807 } 808 809 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 810 if err := cacher.ready.wait(context.Background()); err != nil { 811 t.Fatalf("unexpected error waiting for the cache to be ready") 812 } 813 } 814 815 w, err := cacher.Watch(context.Background(), "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything}) 816 if err != nil { 817 t.Fatalf("Failed to create watch: %v", err) 818 } 819 820 watchClosed := make(chan struct{}) 821 go func() { 822 defer close(watchClosed) 823 for event := range w.ResultChan() { 824 switch event.Type { 825 case watch.Added, watch.Modified, watch.Deleted: 826 // ok 827 default: 828 t.Errorf("unexpected event %#v", event) 829 } 830 } 831 }() 832 833 cacher.Stop() 834 835 _, err = cacher.Watch(context.Background(), "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything}) 836 if err == nil { 837 t.Fatalf("Success to create Watch: %v", err) 838 } 839 840 result := &example.Pod{} 841 err = cacher.Get(context.TODO(), "pods/ns/pod-0", storage.GetOptions{ 842 IgnoreNotFound: true, 843 ResourceVersion: "1", 844 }, result) 845 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 846 if err == nil { 847 t.Fatalf("Success to create Get: %v", err) 848 } 849 } else { 850 if err != nil { 851 t.Fatalf("Failed to get object: %v:", err) 852 } 853 } 854 855 listResult := &example.PodList{} 856 err = cacher.GetList(context.TODO(), "pods/ns", storage.ListOptions{ 857 ResourceVersion: "1", 858 Recursive: true, 859 Predicate: storage.SelectionPredicate{ 860 Limit: 500, 861 }, 862 }, listResult) 863 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 864 if err == nil { 865 t.Fatalf("Success to create GetList: %v", err) 866 } 867 } else { 868 if err != nil { 869 t.Fatalf("Failed to list objects: %v", err) 870 } 871 } 872 873 select { 874 case <-watchClosed: 875 case <-time.After(wait.ForeverTestTimeout): 876 t.Errorf("timed out waiting for watch to close") 877 } 878 } 879 880 func TestCacherDontMissEventsOnReinitialization(t *testing.T) { 881 makePod := func(i int) *example.Pod { 882 return &example.Pod{ 883 ObjectMeta: metav1.ObjectMeta{ 884 Name: fmt.Sprintf("pod-%d", i), 885 Namespace: "ns", 886 ResourceVersion: fmt.Sprintf("%d", i), 887 }, 888 } 889 } 890 891 listCalls, watchCalls := 0, 0 892 backingStorage := &dummyStorage{ 893 getListFn: func(_ context.Context, _ string, _ storage.ListOptions, listObj runtime.Object) error { 894 podList := listObj.(*example.PodList) 895 var err error 896 switch listCalls { 897 case 0: 898 podList.ListMeta = metav1.ListMeta{ResourceVersion: "1"} 899 case 1: 900 podList.ListMeta = metav1.ListMeta{ResourceVersion: "10"} 901 default: 902 err = fmt.Errorf("unexpected list call") 903 } 904 listCalls++ 905 return err 906 }, 907 watchFn: func(_ context.Context, _ string, _ storage.ListOptions) (watch.Interface, error) { 908 var w *watch.FakeWatcher 909 var err error 910 switch watchCalls { 911 case 0: 912 w = watch.NewFakeWithChanSize(10, false) 913 for i := 2; i < 8; i++ { 914 w.Add(makePod(i)) 915 } 916 // Emit an error to force relisting. 917 w.Error(nil) 918 w.Stop() 919 case 1: 920 w = watch.NewFakeWithChanSize(10, false) 921 for i := 12; i < 18; i++ { 922 w.Add(makePod(i)) 923 } 924 w.Stop() 925 default: 926 err = fmt.Errorf("unexpected watch call") 927 } 928 watchCalls++ 929 return w, err 930 }, 931 } 932 cacher, _, err := newTestCacher(backingStorage) 933 if err != nil { 934 t.Fatalf("Couldn't create cacher: %v", err) 935 } 936 defer cacher.Stop() 937 938 concurrency := 1000 939 wg := sync.WaitGroup{} 940 wg.Add(concurrency) 941 942 // Ensure that test doesn't deadlock if cacher already processed everything 943 // and get back into Pending state before some watches get called. 944 ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 945 defer cancel() 946 947 errCh := make(chan error, concurrency) 948 for i := 0; i < concurrency; i++ { 949 go func() { 950 defer wg.Done() 951 w, err := cacher.Watch(ctx, "pods", storage.ListOptions{ResourceVersion: "1", Predicate: storage.Everything}) 952 if err != nil { 953 // Watch failed to initialize (this most probably means that cacher 954 // already moved back to Pending state before watch initialized. 955 // Ignore this case. 956 return 957 } 958 defer w.Stop() 959 960 prevRV := -1 961 for event := range w.ResultChan() { 962 if event.Type == watch.Error { 963 break 964 } 965 object := event.Object 966 if co, ok := object.(runtime.CacheableObject); ok { 967 object = co.GetObject() 968 } 969 rv, err := strconv.Atoi(object.(*example.Pod).ResourceVersion) 970 if err != nil { 971 errCh <- fmt.Errorf("incorrect resource version: %v", err) 972 return 973 } 974 if prevRV != -1 && prevRV+1 != rv { 975 errCh <- fmt.Errorf("unexpected event received, prevRV=%d, rv=%d", prevRV, rv) 976 return 977 } 978 prevRV = rv 979 } 980 981 }() 982 } 983 wg.Wait() 984 close(errCh) 985 986 for err := range errCh { 987 t.Error(err) 988 } 989 } 990 991 func TestCacherNoLeakWithMultipleWatchers(t *testing.T) { 992 backingStorage := &dummyStorage{} 993 cacher, _, err := newTestCacher(backingStorage) 994 if err != nil { 995 t.Fatalf("Couldn't create cacher: %v", err) 996 } 997 defer cacher.Stop() 998 999 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1000 if err := cacher.ready.wait(context.Background()); err != nil { 1001 t.Fatalf("unexpected error waiting for the cache to be ready") 1002 } 1003 } 1004 1005 pred := storage.Everything 1006 pred.AllowWatchBookmarks = true 1007 1008 // run the collision test for 3 seconds to let ~2 buckets expire 1009 stopCh := make(chan struct{}) 1010 var watchErr error 1011 time.AfterFunc(3*time.Second, func() { close(stopCh) }) 1012 1013 wg := &sync.WaitGroup{} 1014 1015 wg.Add(1) 1016 go func() { 1017 defer wg.Done() 1018 for { 1019 select { 1020 case <-stopCh: 1021 return 1022 default: 1023 ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 1024 defer cancel() 1025 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: pred}) 1026 if err != nil { 1027 watchErr = fmt.Errorf("Failed to create watch: %v", err) 1028 return 1029 } 1030 w.Stop() 1031 } 1032 } 1033 }() 1034 1035 wg.Add(1) 1036 go func() { 1037 defer wg.Done() 1038 for { 1039 select { 1040 case <-stopCh: 1041 return 1042 default: 1043 cacher.Lock() 1044 cacher.bookmarkWatchers.popExpiredWatchersThreadUnsafe() 1045 cacher.Unlock() 1046 } 1047 } 1048 }() 1049 1050 // wait for adding/removing watchers to end 1051 wg.Wait() 1052 1053 if watchErr != nil { 1054 t.Fatal(watchErr) 1055 } 1056 1057 // wait out the expiration period and pop expired watchers 1058 time.Sleep(2 * time.Second) 1059 cacher.Lock() 1060 defer cacher.Unlock() 1061 cacher.bookmarkWatchers.popExpiredWatchersThreadUnsafe() 1062 if len(cacher.bookmarkWatchers.watchersBuckets) != 0 { 1063 numWatchers := 0 1064 for bucketID, v := range cacher.bookmarkWatchers.watchersBuckets { 1065 numWatchers += len(v) 1066 t.Errorf("there are %v watchers at bucket Id %v with start Id %v", len(v), bucketID, cacher.bookmarkWatchers.startBucketID) 1067 } 1068 t.Errorf("unexpected bookmark watchers %v", numWatchers) 1069 } 1070 } 1071 1072 func testCacherSendBookmarkEvents(t *testing.T, allowWatchBookmarks, expectedBookmarks bool) { 1073 backingStorage := &dummyStorage{} 1074 cacher, _, err := newTestCacher(backingStorage) 1075 if err != nil { 1076 t.Fatalf("Couldn't create cacher: %v", err) 1077 } 1078 defer cacher.Stop() 1079 1080 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1081 if err := cacher.ready.wait(context.Background()); err != nil { 1082 t.Fatalf("unexpected error waiting for the cache to be ready") 1083 } 1084 } 1085 pred := storage.Everything 1086 pred.AllowWatchBookmarks = allowWatchBookmarks 1087 1088 ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 1089 defer cancel() 1090 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: pred}) 1091 if err != nil { 1092 t.Fatalf("Failed to create watch: %v", err) 1093 } 1094 1095 resourceVersion := uint64(1000) 1096 errc := make(chan error, 1) 1097 go func() { 1098 deadline := time.Now().Add(time.Second) 1099 for i := 0; time.Now().Before(deadline); i++ { 1100 err := cacher.watchCache.Add(&examplev1.Pod{ 1101 ObjectMeta: metav1.ObjectMeta{ 1102 Name: fmt.Sprintf("pod-%d", i), 1103 Namespace: "ns", 1104 ResourceVersion: fmt.Sprintf("%v", resourceVersion+uint64(i)), 1105 }}) 1106 if err != nil { 1107 errc <- fmt.Errorf("failed to add a pod: %v", err) 1108 return 1109 } 1110 time.Sleep(100 * time.Millisecond) 1111 } 1112 }() 1113 1114 timeoutCh := time.After(2 * time.Second) 1115 lastObservedRV := uint64(0) 1116 for { 1117 select { 1118 case err := <-errc: 1119 t.Fatal(err) 1120 return 1121 case event, ok := <-w.ResultChan(): 1122 if !ok { 1123 t.Fatal("Unexpected closed") 1124 } 1125 rv, err := cacher.versioner.ObjectResourceVersion(event.Object) 1126 if err != nil { 1127 t.Errorf("failed to parse resource version from %#v: %v", event.Object, err) 1128 } 1129 if event.Type == watch.Bookmark { 1130 if !expectedBookmarks { 1131 t.Fatalf("Unexpected bookmark events received") 1132 } 1133 1134 if rv < lastObservedRV { 1135 t.Errorf("Unexpected bookmark event resource version %v (last %v)", rv, lastObservedRV) 1136 } 1137 return 1138 } 1139 lastObservedRV = rv 1140 case <-timeoutCh: 1141 if expectedBookmarks { 1142 t.Fatal("Unexpected timeout to receive a bookmark event") 1143 } 1144 return 1145 } 1146 } 1147 } 1148 1149 func TestCacherSendBookmarkEvents(t *testing.T) { 1150 testCases := []struct { 1151 allowWatchBookmarks bool 1152 expectedBookmarks bool 1153 }{ 1154 { 1155 allowWatchBookmarks: true, 1156 expectedBookmarks: true, 1157 }, 1158 { 1159 allowWatchBookmarks: false, 1160 expectedBookmarks: false, 1161 }, 1162 } 1163 1164 for _, tc := range testCases { 1165 testCacherSendBookmarkEvents(t, tc.allowWatchBookmarks, tc.expectedBookmarks) 1166 } 1167 } 1168 1169 func TestCacherSendsMultipleWatchBookmarks(t *testing.T) { 1170 backingStorage := &dummyStorage{} 1171 cacher, _, err := newTestCacher(backingStorage) 1172 if err != nil { 1173 t.Fatalf("Couldn't create cacher: %v", err) 1174 } 1175 defer cacher.Stop() 1176 // Update bookmarkFrequency to speed up test. 1177 // Note that the frequency lower than 1s doesn't change much due to 1178 // resolution how frequency we recompute. 1179 cacher.bookmarkWatchers.bookmarkFrequency = time.Second 1180 1181 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1182 if err := cacher.ready.wait(context.Background()); err != nil { 1183 t.Fatalf("unexpected error waiting for the cache to be ready") 1184 } 1185 } 1186 pred := storage.Everything 1187 pred.AllowWatchBookmarks = true 1188 1189 makePod := func(index int) *examplev1.Pod { 1190 return &examplev1.Pod{ 1191 ObjectMeta: metav1.ObjectMeta{ 1192 Name: fmt.Sprintf("pod-%d", index), 1193 Namespace: "ns", 1194 ResourceVersion: fmt.Sprintf("%v", 100+index), 1195 }, 1196 } 1197 } 1198 1199 // Create pod to initialize watch cache. 1200 if err := cacher.watchCache.Add(makePod(0)); err != nil { 1201 t.Fatalf("failed to add a pod: %v", err) 1202 } 1203 1204 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 1205 defer cancel() 1206 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "100", Predicate: pred}) 1207 if err != nil { 1208 t.Fatalf("Failed to create watch: %v", err) 1209 } 1210 1211 // Create one more pod, to ensure that current RV is higher and thus 1212 // bookmarks will be delievere (events are delivered for RV higher 1213 // than the max from init events). 1214 if err := cacher.watchCache.Add(makePod(1)); err != nil { 1215 t.Fatalf("failed to add a pod: %v", err) 1216 } 1217 1218 timeoutCh := time.After(5 * time.Second) 1219 lastObservedRV := uint64(0) 1220 // Ensure that a watcher gets two bookmarks. 1221 for observedBookmarks := 0; observedBookmarks < 2; { 1222 select { 1223 case event, ok := <-w.ResultChan(): 1224 if !ok { 1225 t.Fatal("Unexpected closed") 1226 } 1227 rv, err := cacher.versioner.ObjectResourceVersion(event.Object) 1228 if err != nil { 1229 t.Errorf("failed to parse resource version from %#v: %v", event.Object, err) 1230 } 1231 if event.Type == watch.Bookmark { 1232 observedBookmarks++ 1233 if rv < lastObservedRV { 1234 t.Errorf("Unexpected bookmark event resource version %v (last %v)", rv, lastObservedRV) 1235 } 1236 } 1237 lastObservedRV = rv 1238 case <-timeoutCh: 1239 t.Fatal("Unexpected timeout to receive bookmark events") 1240 } 1241 } 1242 } 1243 1244 func TestDispatchingBookmarkEventsWithConcurrentStop(t *testing.T) { 1245 backingStorage := &dummyStorage{} 1246 cacher, _, err := newTestCacher(backingStorage) 1247 if err != nil { 1248 t.Fatalf("Couldn't create cacher: %v", err) 1249 } 1250 defer cacher.Stop() 1251 1252 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1253 if err := cacher.ready.wait(context.Background()); err != nil { 1254 t.Fatalf("unexpected error waiting for the cache to be ready") 1255 } 1256 } 1257 1258 // Ensure there is some budget for slowing down processing. 1259 cacher.dispatchTimeoutBudget.returnUnused(100 * time.Millisecond) 1260 1261 resourceVersion := uint64(1000) 1262 err = cacher.watchCache.Add(&examplev1.Pod{ 1263 ObjectMeta: metav1.ObjectMeta{ 1264 Name: "pod-0", 1265 Namespace: "ns", 1266 ResourceVersion: fmt.Sprintf("%v", resourceVersion), 1267 }}) 1268 if err != nil { 1269 t.Fatalf("failed to add a pod: %v", err) 1270 } 1271 1272 for i := 0; i < 1000; i++ { 1273 pred := storage.Everything 1274 pred.AllowWatchBookmarks = true 1275 ctx, cancel := context.WithTimeout(context.Background(), time.Second) 1276 defer cancel() 1277 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: pred}) 1278 if err != nil { 1279 t.Fatalf("Failed to create watch: %v", err) 1280 } 1281 bookmark := &watchCacheEvent{ 1282 Type: watch.Bookmark, 1283 ResourceVersion: uint64(i), 1284 Object: cacher.newFunc(), 1285 } 1286 err = cacher.versioner.UpdateObject(bookmark.Object, bookmark.ResourceVersion) 1287 if err != nil { 1288 t.Fatalf("failure to update version of object (%d) %#v", bookmark.ResourceVersion, bookmark.Object) 1289 } 1290 1291 wg := sync.WaitGroup{} 1292 wg.Add(2) 1293 go func() { 1294 cacher.processEvent(bookmark) 1295 wg.Done() 1296 }() 1297 1298 go func() { 1299 w.Stop() 1300 wg.Done() 1301 }() 1302 1303 done := make(chan struct{}) 1304 go func() { 1305 for range w.ResultChan() { 1306 } 1307 close(done) 1308 }() 1309 1310 select { 1311 case <-done: 1312 case <-time.After(time.Second): 1313 t.Fatal("receive result timeout") 1314 } 1315 w.Stop() 1316 wg.Wait() 1317 } 1318 } 1319 1320 func TestBookmarksOnResourceVersionUpdates(t *testing.T) { 1321 backingStorage := &dummyStorage{} 1322 cacher, _, err := newTestCacher(backingStorage) 1323 if err != nil { 1324 t.Fatalf("Couldn't create cacher: %v", err) 1325 } 1326 defer cacher.Stop() 1327 1328 // Ensure that bookmarks are sent more frequently than every 1m. 1329 cacher.bookmarkWatchers = newTimeBucketWatchers(clock.RealClock{}, 2*time.Second) 1330 1331 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1332 if err := cacher.ready.wait(context.Background()); err != nil { 1333 t.Fatalf("unexpected error waiting for the cache to be ready") 1334 } 1335 } 1336 1337 makePod := func(i int) *examplev1.Pod { 1338 return &examplev1.Pod{ 1339 ObjectMeta: metav1.ObjectMeta{ 1340 Name: fmt.Sprintf("pod-%d", i), 1341 Namespace: "ns", 1342 ResourceVersion: fmt.Sprintf("%d", i), 1343 }, 1344 } 1345 } 1346 if err := cacher.watchCache.Add(makePod(1000)); err != nil { 1347 t.Errorf("error: %v", err) 1348 } 1349 1350 pred := storage.Everything 1351 pred.AllowWatchBookmarks = true 1352 1353 w, err := cacher.Watch(context.TODO(), "/pods/ns", storage.ListOptions{ 1354 ResourceVersion: "1000", 1355 Predicate: pred, 1356 }) 1357 if err != nil { 1358 t.Fatalf("Failed to create watch: %v", err) 1359 } 1360 1361 expectedRV := 2000 1362 1363 var rcErr error 1364 1365 wg := sync.WaitGroup{} 1366 wg.Add(1) 1367 go func() { 1368 defer wg.Done() 1369 for { 1370 event, ok := <-w.ResultChan() 1371 if !ok { 1372 rcErr = errors.New("Unexpected closed channel") 1373 return 1374 } 1375 rv, err := cacher.versioner.ObjectResourceVersion(event.Object) 1376 if err != nil { 1377 t.Errorf("failed to parse resource version from %#v: %v", event.Object, err) 1378 } 1379 if event.Type == watch.Bookmark && rv == uint64(expectedRV) { 1380 return 1381 } 1382 } 1383 }() 1384 1385 // Simulate progress notify event. 1386 cacher.watchCache.UpdateResourceVersion(strconv.Itoa(expectedRV)) 1387 1388 wg.Wait() 1389 if rcErr != nil { 1390 t.Fatal(rcErr) 1391 } 1392 } 1393 1394 type fakeTimeBudget struct{} 1395 1396 func (f *fakeTimeBudget) takeAvailable() time.Duration { 1397 return 2 * time.Second 1398 } 1399 1400 func (f *fakeTimeBudget) returnUnused(_ time.Duration) {} 1401 1402 func TestStartingResourceVersion(t *testing.T) { 1403 backingStorage := &dummyStorage{} 1404 cacher, _, err := newTestCacher(backingStorage) 1405 if err != nil { 1406 t.Fatalf("Couldn't create cacher: %v", err) 1407 } 1408 defer cacher.Stop() 1409 1410 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1411 if err := cacher.ready.wait(context.Background()); err != nil { 1412 t.Fatalf("unexpected error waiting for the cache to be ready") 1413 } 1414 } 1415 1416 // Ensure there is some budget for slowing down processing. 1417 // We use the fakeTimeBudget to prevent this test from flaking under 1418 // the following conditions: 1419 // 1) in total we create 11 events that has to be processed by the watcher 1420 // 2) the size of the channels are set to 10 for the watcher 1421 // 3) if the test is cpu-starved and the internal goroutine is not picking 1422 // up these events from the channel, after consuming the whole time 1423 // budget (defaulted to 100ms) on waiting, we will simply close the watch, 1424 // which will cause the test failure 1425 // Using fakeTimeBudget gives us always a budget to wait and have a test 1426 // pick up something from ResultCh in the meantime. 1427 // 1428 // The same can potentially happen in production, but in that case a watch 1429 // can be resumed by the client. This doesn't work in the case of this test, 1430 // because we explicitly want to test the behavior that object changes are 1431 // happening after the watch was initiated. 1432 cacher.dispatchTimeoutBudget = &fakeTimeBudget{} 1433 1434 makePod := func(i int) *examplev1.Pod { 1435 return &examplev1.Pod{ 1436 ObjectMeta: metav1.ObjectMeta{ 1437 Name: "foo", 1438 Namespace: "ns", 1439 Labels: map[string]string{"foo": strconv.Itoa(i)}, 1440 ResourceVersion: fmt.Sprintf("%d", i), 1441 }, 1442 } 1443 } 1444 1445 if err := cacher.watchCache.Add(makePod(1000)); err != nil { 1446 t.Errorf("error: %v", err) 1447 } 1448 // Advance RV by 10. 1449 startVersion := uint64(1010) 1450 1451 watcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", storage.ListOptions{ResourceVersion: strconv.FormatUint(startVersion, 10), Predicate: storage.Everything}) 1452 if err != nil { 1453 t.Fatalf("Unexpected error: %v", err) 1454 } 1455 defer watcher.Stop() 1456 1457 for i := 1; i <= 11; i++ { 1458 if err := cacher.watchCache.Update(makePod(1000 + i)); err != nil { 1459 t.Errorf("error: %v", err) 1460 } 1461 } 1462 1463 e, ok := <-watcher.ResultChan() 1464 if !ok { 1465 t.Errorf("unexpectedly closed watch") 1466 } 1467 object := e.Object 1468 if co, ok := object.(runtime.CacheableObject); ok { 1469 object = co.GetObject() 1470 } 1471 pod := object.(*examplev1.Pod) 1472 podRV, err := cacher.versioner.ParseResourceVersion(pod.ResourceVersion) 1473 if err != nil { 1474 t.Fatalf("unexpected error: %v", err) 1475 } 1476 1477 // event should have at least rv + 1, since we're starting the watch at rv 1478 if podRV <= startVersion { 1479 t.Errorf("expected event with resourceVersion of at least %d, got %d", startVersion+1, podRV) 1480 } 1481 } 1482 1483 func TestDispatchEventWillNotBeBlockedByTimedOutWatcher(t *testing.T) { 1484 backingStorage := &dummyStorage{} 1485 cacher, _, err := newTestCacher(backingStorage) 1486 if err != nil { 1487 t.Fatalf("Couldn't create cacher: %v", err) 1488 } 1489 defer cacher.Stop() 1490 1491 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1492 if err := cacher.ready.wait(context.Background()); err != nil { 1493 t.Fatalf("unexpected error waiting for the cache to be ready") 1494 } 1495 } 1496 1497 // Ensure there is some budget for slowing down processing. 1498 // We use the fakeTimeBudget to prevent this test from flaking under 1499 // the following conditions: 1500 // 1) the watch w1 is blocked, so we were consuming the whole budget once 1501 // its buffer was filled in (10 items) 1502 // 2) the budget is refreshed once per second, so it basically wasn't 1503 // happening in the test at all 1504 // 3) if the test was cpu-starved and we weren't able to consume events 1505 // from w2 ResultCh it could have happened that its buffer was also 1506 // filling in and given we no longer had timeBudget (consumed in (1)) 1507 // trying to put next item was simply breaking the watch 1508 // Using fakeTimeBudget gives us always a budget to wait and have a test 1509 // pick up something from ResultCh in the meantime. 1510 cacher.dispatchTimeoutBudget = &fakeTimeBudget{} 1511 1512 makePod := func(i int) *examplev1.Pod { 1513 return &examplev1.Pod{ 1514 ObjectMeta: metav1.ObjectMeta{ 1515 Name: fmt.Sprintf("pod-%d", 1000+i), 1516 Namespace: "ns", 1517 ResourceVersion: fmt.Sprintf("%d", 1000+i), 1518 }, 1519 } 1520 } 1521 if err := cacher.watchCache.Add(makePod(0)); err != nil { 1522 t.Errorf("error: %v", err) 1523 } 1524 1525 totalPods := 50 1526 1527 // Create watcher that will be blocked. 1528 w1, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: storage.Everything}) 1529 if err != nil { 1530 t.Fatalf("Failed to create watch: %v", err) 1531 } 1532 defer w1.Stop() 1533 1534 // Create fast watcher and ensure it will get all objects. 1535 w2, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: storage.Everything}) 1536 if err != nil { 1537 t.Fatalf("Failed to create watch: %v", err) 1538 } 1539 defer w2.Stop() 1540 1541 // Now push a ton of object to cache. 1542 for i := 1; i < totalPods; i++ { 1543 cacher.watchCache.Add(makePod(i)) 1544 } 1545 1546 shouldContinue := true 1547 eventsCount := 0 1548 for shouldContinue { 1549 select { 1550 case event, ok := <-w2.ResultChan(): 1551 if !ok { 1552 shouldContinue = false 1553 break 1554 } 1555 if event.Type == watch.Added { 1556 eventsCount++ 1557 if eventsCount == totalPods { 1558 shouldContinue = false 1559 } 1560 } 1561 case <-time.After(wait.ForeverTestTimeout): 1562 shouldContinue = false 1563 w2.Stop() 1564 } 1565 } 1566 if eventsCount != totalPods { 1567 t.Errorf("watcher is blocked by slower one (count: %d)", eventsCount) 1568 } 1569 } 1570 1571 func verifyEvents(t *testing.T, w watch.Interface, events []watch.Event, strictOrder bool) { 1572 _, _, line, _ := goruntime.Caller(1) 1573 actualEvents := make([]watch.Event, len(events)) 1574 for idx := range events { 1575 select { 1576 case event := <-w.ResultChan(): 1577 actualEvents[idx] = event 1578 case <-time.After(wait.ForeverTestTimeout): 1579 t.Logf("(called from line %d)", line) 1580 t.Errorf("Timed out waiting for an event") 1581 } 1582 } 1583 validateEvents := func(expected, actual watch.Event) (bool, []string) { 1584 errors := []string{} 1585 if e, a := expected.Type, actual.Type; e != a { 1586 errors = append(errors, fmt.Sprintf("Expected: %s, got: %s", e, a)) 1587 } 1588 actualObject := actual.Object 1589 if co, ok := actualObject.(runtime.CacheableObject); ok { 1590 actualObject = co.GetObject() 1591 } 1592 if e, a := expected.Object, actualObject; !apiequality.Semantic.DeepEqual(e, a) { 1593 errors = append(errors, fmt.Sprintf("Expected: %#v, got: %#v", e, a)) 1594 } 1595 return len(errors) == 0, errors 1596 } 1597 1598 if len(events) != len(actualEvents) { 1599 t.Fatalf("unexpected number of events: %d, expected: %d, acutalEvents: %#v, expectedEvents:%#v", len(actualEvents), len(events), actualEvents, events) 1600 } 1601 1602 if strictOrder { 1603 for idx, expectedEvent := range events { 1604 valid, errors := validateEvents(expectedEvent, actualEvents[idx]) 1605 if !valid { 1606 t.Logf("(called from line %d)", line) 1607 for _, err := range errors { 1608 t.Errorf(err) 1609 } 1610 } 1611 } 1612 } 1613 for _, expectedEvent := range events { 1614 validated := false 1615 for _, actualEvent := range actualEvents { 1616 if validated, _ = validateEvents(expectedEvent, actualEvent); validated { 1617 break 1618 } 1619 } 1620 if !validated { 1621 t.Fatalf("Expected: %#v but didn't find", expectedEvent) 1622 } 1623 } 1624 } 1625 1626 func TestCachingDeleteEvents(t *testing.T) { 1627 backingStorage := &dummyStorage{} 1628 cacher, _, err := newTestCacher(backingStorage) 1629 if err != nil { 1630 t.Fatalf("Couldn't create cacher: %v", err) 1631 } 1632 defer cacher.Stop() 1633 1634 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1635 if err := cacher.ready.wait(context.Background()); err != nil { 1636 t.Fatalf("unexpected error waiting for the cache to be ready") 1637 } 1638 } 1639 1640 fooPredicate := storage.SelectionPredicate{ 1641 Label: labels.SelectorFromSet(map[string]string{"foo": "true"}), 1642 Field: fields.Everything(), 1643 } 1644 barPredicate := storage.SelectionPredicate{ 1645 Label: labels.SelectorFromSet(map[string]string{"bar": "true"}), 1646 Field: fields.Everything(), 1647 } 1648 1649 createWatch := func(pred storage.SelectionPredicate) watch.Interface { 1650 w, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: pred}) 1651 if err != nil { 1652 t.Fatalf("Failed to create watch: %v", err) 1653 } 1654 return w 1655 } 1656 1657 allEventsWatcher := createWatch(storage.Everything) 1658 defer allEventsWatcher.Stop() 1659 fooEventsWatcher := createWatch(fooPredicate) 1660 defer fooEventsWatcher.Stop() 1661 barEventsWatcher := createWatch(barPredicate) 1662 defer barEventsWatcher.Stop() 1663 1664 makePod := func(labels map[string]string, rv string) *examplev1.Pod { 1665 return &examplev1.Pod{ 1666 ObjectMeta: metav1.ObjectMeta{ 1667 Name: "pod", 1668 Namespace: "ns", 1669 Labels: labels, 1670 ResourceVersion: rv, 1671 }, 1672 } 1673 } 1674 pod1 := makePod(map[string]string{"foo": "true", "bar": "true"}, "1001") 1675 pod2 := makePod(map[string]string{"foo": "true"}, "1002") 1676 pod3 := makePod(map[string]string{}, "1003") 1677 pod4 := makePod(map[string]string{}, "1004") 1678 pod1DeletedAt2 := pod1.DeepCopyObject().(*examplev1.Pod) 1679 pod1DeletedAt2.ResourceVersion = "1002" 1680 pod2DeletedAt3 := pod2.DeepCopyObject().(*examplev1.Pod) 1681 pod2DeletedAt3.ResourceVersion = "1003" 1682 1683 allEvents := []watch.Event{ 1684 {Type: watch.Added, Object: pod1.DeepCopy()}, 1685 {Type: watch.Modified, Object: pod2.DeepCopy()}, 1686 {Type: watch.Modified, Object: pod3.DeepCopy()}, 1687 {Type: watch.Deleted, Object: pod4.DeepCopy()}, 1688 } 1689 fooEvents := []watch.Event{ 1690 {Type: watch.Added, Object: pod1.DeepCopy()}, 1691 {Type: watch.Modified, Object: pod2.DeepCopy()}, 1692 {Type: watch.Deleted, Object: pod2DeletedAt3.DeepCopy()}, 1693 } 1694 barEvents := []watch.Event{ 1695 {Type: watch.Added, Object: pod1.DeepCopy()}, 1696 {Type: watch.Deleted, Object: pod1DeletedAt2.DeepCopy()}, 1697 } 1698 1699 cacher.watchCache.Add(pod1) 1700 cacher.watchCache.Update(pod2) 1701 cacher.watchCache.Update(pod3) 1702 cacher.watchCache.Delete(pod4) 1703 1704 verifyEvents(t, allEventsWatcher, allEvents, true) 1705 verifyEvents(t, fooEventsWatcher, fooEvents, true) 1706 verifyEvents(t, barEventsWatcher, barEvents, true) 1707 } 1708 1709 func testCachingObjects(t *testing.T, watchersCount int) { 1710 backingStorage := &dummyStorage{} 1711 cacher, _, err := newTestCacher(backingStorage) 1712 if err != nil { 1713 t.Fatalf("Couldn't create cacher: %v", err) 1714 } 1715 defer cacher.Stop() 1716 1717 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1718 if err := cacher.ready.wait(context.Background()); err != nil { 1719 t.Fatalf("unexpected error waiting for the cache to be ready") 1720 } 1721 } 1722 1723 dispatchedEvents := []*watchCacheEvent{} 1724 cacher.watchCache.eventHandler = func(event *watchCacheEvent) { 1725 dispatchedEvents = append(dispatchedEvents, event) 1726 cacher.processEvent(event) 1727 } 1728 1729 watchers := make([]watch.Interface, 0, watchersCount) 1730 for i := 0; i < watchersCount; i++ { 1731 w, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "1000", Predicate: storage.Everything}) 1732 if err != nil { 1733 t.Fatalf("Failed to create watch: %v", err) 1734 } 1735 defer w.Stop() 1736 watchers = append(watchers, w) 1737 } 1738 1739 makePod := func(name, rv string) *examplev1.Pod { 1740 return &examplev1.Pod{ 1741 ObjectMeta: metav1.ObjectMeta{ 1742 Name: name, 1743 Namespace: "ns", 1744 ResourceVersion: rv, 1745 }, 1746 } 1747 } 1748 pod1 := makePod("pod", "1001") 1749 pod2 := makePod("pod", "1002") 1750 pod3 := makePod("pod", "1003") 1751 1752 cacher.watchCache.Add(pod1) 1753 cacher.watchCache.Update(pod2) 1754 cacher.watchCache.Delete(pod3) 1755 1756 // At this point, we already have dispatchedEvents fully propagated. 1757 1758 verifyEvents := func(w watch.Interface) { 1759 var event watch.Event 1760 for index := range dispatchedEvents { 1761 select { 1762 case event = <-w.ResultChan(): 1763 case <-time.After(wait.ForeverTestTimeout): 1764 t.Fatalf("timeout watiching for the event") 1765 } 1766 1767 var object runtime.Object 1768 if _, ok := event.Object.(runtime.CacheableObject); !ok { 1769 t.Fatalf("Object in %s event should support caching: %#v", event.Type, event.Object) 1770 } 1771 object = event.Object.(runtime.CacheableObject).GetObject() 1772 1773 if event.Type == watch.Deleted { 1774 resourceVersion, err := cacher.versioner.ObjectResourceVersion(cacher.watchCache.cache[index].PrevObject) 1775 if err != nil { 1776 t.Fatalf("Failed to parse resource version: %v", err) 1777 } 1778 updateResourceVersion(object, cacher.versioner, resourceVersion) 1779 } 1780 1781 var e runtime.Object 1782 switch event.Type { 1783 case watch.Added, watch.Modified: 1784 e = cacher.watchCache.cache[index].Object 1785 case watch.Deleted: 1786 e = cacher.watchCache.cache[index].PrevObject 1787 default: 1788 t.Errorf("unexpected watch event: %#v", event) 1789 } 1790 if a := object; !reflect.DeepEqual(a, e) { 1791 t.Errorf("event object messed up for %s: %#v, expected: %#v", event.Type, a, e) 1792 } 1793 } 1794 } 1795 1796 for i := range watchers { 1797 verifyEvents(watchers[i]) 1798 } 1799 } 1800 1801 func TestCachingObjects(t *testing.T) { 1802 t.Run("single watcher", func(t *testing.T) { testCachingObjects(t, 1) }) 1803 t.Run("many watcher", func(t *testing.T) { testCachingObjects(t, 3) }) 1804 } 1805 1806 func TestCacheIntervalInvalidationStopsWatch(t *testing.T) { 1807 backingStorage := &dummyStorage{} 1808 cacher, _, err := newTestCacher(backingStorage) 1809 if err != nil { 1810 t.Fatalf("Couldn't create cacher: %v", err) 1811 } 1812 defer cacher.Stop() 1813 1814 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 1815 if err := cacher.ready.wait(context.Background()); err != nil { 1816 t.Fatalf("unexpected error waiting for the cache to be ready") 1817 } 1818 } 1819 1820 // Ensure there is enough budget for slow processing since 1821 // the entire watch cache is going to be served through the 1822 // interval and events won't be popped from the cacheWatcher's 1823 // input channel until much later. 1824 cacher.dispatchTimeoutBudget.returnUnused(100 * time.Millisecond) 1825 1826 // We define a custom index validator such that the interval is 1827 // able to serve the first bufferSize elements successfully, but 1828 // on trying to fill it's buffer again, the indexValidator simulates 1829 // an invalidation leading to the watch being closed and the number 1830 // of events we actually process to be bufferSize, each event of 1831 // type watch.Added. 1832 valid := true 1833 invalidateCacheInterval := func() { 1834 valid = false 1835 } 1836 once := sync.Once{} 1837 indexValidator := func(index int) bool { 1838 isValid := valid && (index >= cacher.watchCache.startIndex) 1839 once.Do(invalidateCacheInterval) 1840 return isValid 1841 } 1842 cacher.watchCache.indexValidator = indexValidator 1843 1844 makePod := func(i int) *examplev1.Pod { 1845 return &examplev1.Pod{ 1846 ObjectMeta: metav1.ObjectMeta{ 1847 Name: fmt.Sprintf("pod-%d", 1000+i), 1848 Namespace: "ns", 1849 ResourceVersion: fmt.Sprintf("%d", 1000+i), 1850 }, 1851 } 1852 } 1853 1854 // 250 is arbitrary, point is to have enough elements such that 1855 // it generates more than bufferSize number of events allowing 1856 // us to simulate the invalidation of the cache interval. 1857 totalPods := 250 1858 for i := 0; i < totalPods; i++ { 1859 err := cacher.watchCache.Add(makePod(i)) 1860 if err != nil { 1861 t.Errorf("error: %v", err) 1862 } 1863 } 1864 ctx, cancel := context.WithCancel(context.Background()) 1865 defer cancel() 1866 1867 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ 1868 ResourceVersion: "999", 1869 Predicate: storage.Everything, 1870 }) 1871 if err != nil { 1872 t.Fatalf("Failed to create watch: %v", err) 1873 } 1874 defer w.Stop() 1875 1876 received := 0 1877 resChan := w.ResultChan() 1878 for event := range resChan { 1879 received++ 1880 t.Logf("event type: %v, events received so far: %d", event.Type, received) 1881 if event.Type != watch.Added { 1882 t.Errorf("unexpected event type, expected: %s, got: %s, event: %v", watch.Added, event.Type, event) 1883 } 1884 } 1885 // Since the watch is stopped after the interval is invalidated, 1886 // we should have processed exactly bufferSize number of elements. 1887 if received != bufferSize { 1888 t.Errorf("unexpected number of events received, expected: %d, got: %d", bufferSize+1, received) 1889 } 1890 } 1891 1892 func TestWaitUntilWatchCacheFreshAndForceAllEvents(t *testing.T) { 1893 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true) 1894 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true) 1895 forceRequestWatchProgressSupport(t) 1896 1897 scenarios := []struct { 1898 name string 1899 opts storage.ListOptions 1900 backingStorage *dummyStorage 1901 verifyBackingStore func(t *testing.T, s *dummyStorage) 1902 }{ 1903 { 1904 name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=105", 1905 opts: storage.ListOptions{ 1906 Predicate: func() storage.SelectionPredicate { 1907 p := storage.Everything 1908 p.AllowWatchBookmarks = true 1909 return p 1910 }(), 1911 SendInitialEvents: pointer.Bool(true), 1912 ResourceVersion: "105", 1913 }, 1914 verifyBackingStore: func(t *testing.T, s *dummyStorage) { 1915 require.NotEqual(t, 0, s.getRequestWatchProgressCounter(), "expected store.RequestWatchProgressCounter to be > 0. It looks like watch progress wasn't requested!") 1916 }, 1917 }, 1918 1919 { 1920 name: "legacy: allowWatchBookmarks=false, sendInitialEvents=true, RV=unset", 1921 opts: storage.ListOptions{ 1922 Predicate: func() storage.SelectionPredicate { 1923 p := storage.Everything 1924 p.AllowWatchBookmarks = false 1925 return p 1926 }(), 1927 SendInitialEvents: pointer.Bool(true), 1928 }, 1929 backingStorage: func() *dummyStorage { 1930 hasBeenPrimed := false 1931 s := &dummyStorage{} 1932 s.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { 1933 listAccessor, err := meta.ListAccessor(listObj) 1934 if err != nil { 1935 return err 1936 } 1937 // the first call to this function 1938 // primes the cacher 1939 if !hasBeenPrimed { 1940 listAccessor.SetResourceVersion("100") 1941 hasBeenPrimed = true 1942 return nil 1943 } 1944 listAccessor.SetResourceVersion("105") 1945 return nil 1946 } 1947 return s 1948 }(), 1949 verifyBackingStore: func(t *testing.T, s *dummyStorage) { 1950 require.NotEqual(t, 0, s.getRequestWatchProgressCounter(), "expected store.RequestWatchProgressCounter to be > 0. It looks like watch progress wasn't requested!") 1951 }, 1952 }, 1953 1954 { 1955 name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=unset", 1956 opts: storage.ListOptions{ 1957 Predicate: func() storage.SelectionPredicate { 1958 p := storage.Everything 1959 p.AllowWatchBookmarks = true 1960 return p 1961 }(), 1962 SendInitialEvents: pointer.Bool(true), 1963 }, 1964 backingStorage: func() *dummyStorage { 1965 hasBeenPrimed := false 1966 s := &dummyStorage{} 1967 s.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { 1968 listAccessor, err := meta.ListAccessor(listObj) 1969 if err != nil { 1970 return err 1971 } 1972 // the first call to this function 1973 // primes the cacher 1974 if !hasBeenPrimed { 1975 listAccessor.SetResourceVersion("100") 1976 hasBeenPrimed = true 1977 return nil 1978 } 1979 listAccessor.SetResourceVersion("105") 1980 return nil 1981 } 1982 return s 1983 }(), 1984 verifyBackingStore: func(t *testing.T, s *dummyStorage) { 1985 require.NotEqual(t, 0, s.getRequestWatchProgressCounter(), "expected store.RequestWatchProgressCounter to be > 0. It looks like watch progress wasn't requested!") 1986 }, 1987 }, 1988 } 1989 1990 for _, scenario := range scenarios { 1991 t.Run(scenario.name, func(t *testing.T) { 1992 t.Parallel() 1993 var backingStorage *dummyStorage 1994 if scenario.backingStorage != nil { 1995 backingStorage = scenario.backingStorage 1996 } else { 1997 backingStorage = &dummyStorage{} 1998 } 1999 cacher, _, err := newTestCacher(backingStorage) 2000 if err != nil { 2001 t.Fatalf("Couldn't create cacher: %v", err) 2002 } 2003 defer cacher.Stop() 2004 2005 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 2006 if err := cacher.ready.wait(context.Background()); err != nil { 2007 t.Fatalf("unexpected error waiting for the cache to be ready") 2008 } 2009 } 2010 2011 w, err := cacher.Watch(context.Background(), "pods/ns", scenario.opts) 2012 require.NoError(t, err, "failed to create watch: %v") 2013 defer w.Stop() 2014 var expectedErr *apierrors.StatusError 2015 if !errors.As(storage.NewTooLargeResourceVersionError(105, 100, resourceVersionTooHighRetrySeconds), &expectedErr) { 2016 t.Fatalf("Unable to convert NewTooLargeResourceVersionError to apierrors.StatusError") 2017 } 2018 verifyEvents(t, w, []watch.Event{ 2019 { 2020 Type: watch.Error, 2021 Object: &metav1.Status{ 2022 Status: metav1.StatusFailure, 2023 Message: expectedErr.Error(), 2024 Details: expectedErr.ErrStatus.Details, 2025 Reason: metav1.StatusReasonTimeout, 2026 Code: 504, 2027 }, 2028 }, 2029 }, true) 2030 2031 go func(t *testing.T) { 2032 err := cacher.watchCache.Add(makeTestPodDetails("pod1", 105, "node1", map[string]string{"label": "value1"})) 2033 require.NoError(t, err, "failed adding a pod to the watchCache") 2034 }(t) 2035 w, err = cacher.Watch(context.Background(), "pods/ns", scenario.opts) 2036 require.NoError(t, err, "failed to create watch: %v") 2037 defer w.Stop() 2038 verifyEvents(t, w, []watch.Event{ 2039 { 2040 Type: watch.Added, 2041 Object: makeTestPodDetails("pod1", 105, "node1", map[string]string{"label": "value1"}), 2042 }, 2043 }, true) 2044 if scenario.verifyBackingStore != nil { 2045 scenario.verifyBackingStore(t, backingStorage) 2046 } 2047 }) 2048 } 2049 } 2050 2051 type fakeStorage struct { 2052 pods []example.Pod 2053 storage.Interface 2054 } 2055 2056 func newObjectStorage(fakePods []example.Pod) *fakeStorage { 2057 return &fakeStorage{ 2058 pods: fakePods, 2059 } 2060 } 2061 2062 func (m fakeStorage) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { 2063 podList := listObj.(*example.PodList) 2064 podList.ListMeta = metav1.ListMeta{ResourceVersion: "12345"} 2065 podList.Items = m.pods 2066 return nil 2067 } 2068 func (m fakeStorage) Watch(_ context.Context, _ string, _ storage.ListOptions) (watch.Interface, error) { 2069 return newDummyWatch(), nil 2070 } 2071 2072 func BenchmarkCacher_GetList(b *testing.B) { 2073 testCases := []struct { 2074 totalObjectNum int 2075 expectObjectNum int 2076 }{ 2077 { 2078 totalObjectNum: 5000, 2079 expectObjectNum: 50, 2080 }, 2081 { 2082 totalObjectNum: 5000, 2083 expectObjectNum: 500, 2084 }, 2085 { 2086 totalObjectNum: 5000, 2087 expectObjectNum: 1000, 2088 }, 2089 { 2090 totalObjectNum: 5000, 2091 expectObjectNum: 2500, 2092 }, 2093 { 2094 totalObjectNum: 5000, 2095 expectObjectNum: 5000, 2096 }, 2097 } 2098 for _, tc := range testCases { 2099 b.Run( 2100 fmt.Sprintf("totalObjectNum=%d, expectObjectNum=%d", tc.totalObjectNum, tc.expectObjectNum), 2101 func(b *testing.B) { 2102 // create sample pods 2103 fakePods := make([]example.Pod, tc.totalObjectNum, tc.totalObjectNum) 2104 for i := range fakePods { 2105 fakePods[i].Namespace = "default" 2106 fakePods[i].Name = fmt.Sprintf("pod-%d", i) 2107 fakePods[i].ResourceVersion = strconv.Itoa(i) 2108 if i%(tc.totalObjectNum/tc.expectObjectNum) == 0 { 2109 fakePods[i].Spec.NodeName = "node-0" 2110 } 2111 data := make([]byte, 1024*2, 1024*2) // 2k labels 2112 rand.Read(data) 2113 fakePods[i].Spec.NodeSelector = map[string]string{ 2114 "key": string(data), 2115 } 2116 } 2117 2118 // build test cacher 2119 cacher, _, err := newTestCacher(newObjectStorage(fakePods)) 2120 if err != nil { 2121 b.Fatalf("new cacher: %v", err) 2122 } 2123 defer cacher.Stop() 2124 2125 // prepare result and pred 2126 parsedField, err := fields.ParseSelector("spec.nodeName=node-0") 2127 if err != nil { 2128 b.Fatalf("parse selector: %v", err) 2129 } 2130 pred := storage.SelectionPredicate{ 2131 Label: labels.Everything(), 2132 Field: parsedField, 2133 } 2134 2135 // now we start benchmarking 2136 b.ResetTimer() 2137 for i := 0; i < b.N; i++ { 2138 result := &example.PodList{} 2139 err = cacher.GetList(context.TODO(), "pods", storage.ListOptions{ 2140 Predicate: pred, 2141 Recursive: true, 2142 ResourceVersion: "12345", 2143 }, result) 2144 if err != nil { 2145 b.Fatalf("GetList cache: %v", err) 2146 } 2147 if len(result.Items) != tc.expectObjectNum { 2148 b.Fatalf("expect %d but got %d", tc.expectObjectNum, len(result.Items)) 2149 } 2150 } 2151 }) 2152 } 2153 } 2154 2155 // TestWatchListIsSynchronisedWhenNoEventsFromStoreReceived makes sure that 2156 // a bookmark event will be delivered even if the cacher has not received an event. 2157 func TestWatchListIsSynchronisedWhenNoEventsFromStoreReceived(t *testing.T) { 2158 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true) 2159 backingStorage := &dummyStorage{} 2160 cacher, _, err := newTestCacher(backingStorage) 2161 require.NoError(t, err, "failed to create cacher") 2162 defer cacher.Stop() 2163 2164 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 2165 if err := cacher.ready.wait(context.Background()); err != nil { 2166 t.Fatalf("unexpected error waiting for the cache to be ready") 2167 } 2168 } 2169 2170 pred := storage.Everything 2171 pred.AllowWatchBookmarks = true 2172 opts := storage.ListOptions{ 2173 Predicate: pred, 2174 SendInitialEvents: pointer.Bool(true), 2175 } 2176 w, err := cacher.Watch(context.Background(), "pods/ns", opts) 2177 require.NoError(t, err, "failed to create watch: %v") 2178 defer w.Stop() 2179 2180 verifyEvents(t, w, []watch.Event{ 2181 {Type: watch.Bookmark, Object: &example.Pod{ 2182 ObjectMeta: metav1.ObjectMeta{ 2183 ResourceVersion: "100", 2184 Annotations: map[string]string{metav1.InitialEventsAnnotationKey: "true"}, 2185 }, 2186 }}, 2187 }, true) 2188 } 2189 2190 func TestForgetWatcher(t *testing.T) { 2191 backingStorage := &dummyStorage{} 2192 cacher, _, err := newTestCacher(backingStorage) 2193 require.NoError(t, err) 2194 defer cacher.Stop() 2195 2196 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 2197 if err := cacher.ready.wait(context.Background()); err != nil { 2198 t.Fatalf("unexpected error waiting for the cache to be ready") 2199 } 2200 } 2201 2202 assertCacherInternalState := func(expectedWatchersCounter, expectedValueWatchersCounter int) { 2203 cacher.Lock() 2204 defer cacher.Unlock() 2205 2206 require.Len(t, cacher.watchers.allWatchers, expectedWatchersCounter) 2207 require.Len(t, cacher.watchers.valueWatchers, expectedValueWatchersCounter) 2208 } 2209 assertCacherInternalState(0, 0) 2210 2211 var forgetWatcherFn func(bool) 2212 var forgetCounter int 2213 forgetWatcherWrapped := func(drainWatcher bool) { 2214 forgetCounter++ 2215 forgetWatcherFn(drainWatcher) 2216 } 2217 w := newCacheWatcher( 2218 0, 2219 func(_ string, _ labels.Set, _ fields.Set) bool { return true }, 2220 nil, 2221 storage.APIObjectVersioner{}, 2222 testingclock.NewFakeClock(time.Now()).Now().Add(2*time.Minute), 2223 true, 2224 schema.GroupResource{Resource: "pods"}, 2225 "1", 2226 ) 2227 forgetWatcherFn = forgetWatcher(cacher, w, 0, namespacedName{}, "", false) 2228 addWatcher := func(w *cacheWatcher) { 2229 cacher.Lock() 2230 defer cacher.Unlock() 2231 2232 cacher.watchers.addWatcher(w, 0, namespacedName{}, "", false) 2233 } 2234 2235 addWatcher(w) 2236 assertCacherInternalState(1, 0) 2237 2238 forgetWatcherWrapped(false) 2239 assertCacherInternalState(0, 0) 2240 require.Equal(t, 1, forgetCounter) 2241 2242 forgetWatcherWrapped(false) 2243 assertCacherInternalState(0, 0) 2244 require.Equal(t, 2, forgetCounter) 2245 } 2246 2247 // TestGetWatchCacheResourceVersion test the following cases: 2248 // 2249 // +-----------------+---------------------+-----------------------+ 2250 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents | 2251 // +=================+=====================+=======================+ 2252 // | Unset | true/false | nil/true/false | 2253 // | 0 | true/false | nil/true/false | 2254 // | 95 | true/false | nil/true/false | 2255 // +-----------------+---------------------+-----------------------+ 2256 // where: 2257 // - false indicates the value of the param was set to "false" by a test case 2258 // - true indicates the value of the param was set to "true" by a test case 2259 func TestGetWatchCacheResourceVersion(t *testing.T) { 2260 listOptions := func(allowBookmarks bool, sendInitialEvents *bool, rv string) storage.ListOptions { 2261 p := storage.Everything 2262 p.AllowWatchBookmarks = allowBookmarks 2263 2264 opts := storage.ListOptions{} 2265 opts.Predicate = p 2266 opts.SendInitialEvents = sendInitialEvents 2267 opts.ResourceVersion = rv 2268 return opts 2269 } 2270 2271 scenarios := []struct { 2272 name string 2273 opts storage.ListOptions 2274 2275 expectedWatchResourceVersion int 2276 }{ 2277 // +-----------------+---------------------+-----------------------+ 2278 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents | 2279 // +=================+=====================+=======================+ 2280 // | Unset | true/false | nil/true/false | 2281 // +-----------------+---------------------+-----------------------+ 2282 { 2283 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=nil", 2284 opts: listOptions(true, nil, ""), 2285 // Expecting RV 0, due to https://github.com/kubernetes/kubernetes/pull/123935 reverted to serving those requests from watch cache. 2286 // Set to 100, when WatchFromStorageWithoutResourceVersion is set to true. 2287 expectedWatchResourceVersion: 0, 2288 }, 2289 { 2290 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=true", 2291 opts: listOptions(true, pointer.Bool(true), ""), 2292 expectedWatchResourceVersion: 100, 2293 }, 2294 { 2295 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=false", 2296 opts: listOptions(true, pointer.Bool(false), ""), 2297 expectedWatchResourceVersion: 100, 2298 }, 2299 { 2300 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=nil", 2301 opts: listOptions(false, nil, ""), 2302 // Expecting RV 0, due to https://github.com/kubernetes/kubernetes/pull/123935 reverted to serving those requests from watch cache. 2303 // Set to 100, when WatchFromStorageWithoutResourceVersion is set to true. 2304 expectedWatchResourceVersion: 0, 2305 }, 2306 { 2307 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=true, legacy", 2308 opts: listOptions(false, pointer.Bool(true), ""), 2309 expectedWatchResourceVersion: 100, 2310 }, 2311 { 2312 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=false", 2313 opts: listOptions(false, pointer.Bool(false), ""), 2314 expectedWatchResourceVersion: 100, 2315 }, 2316 // +-----------------+---------------------+-----------------------+ 2317 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents | 2318 // +=================+=====================+=======================+ 2319 // | 0 | true/false | nil/true/false | 2320 // +-----------------+---------------------+-----------------------+ 2321 { 2322 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=nil", 2323 opts: listOptions(true, nil, "0"), 2324 expectedWatchResourceVersion: 0, 2325 }, 2326 { 2327 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=true", 2328 opts: listOptions(true, pointer.Bool(true), "0"), 2329 expectedWatchResourceVersion: 0, 2330 }, 2331 { 2332 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=false", 2333 opts: listOptions(true, pointer.Bool(false), "0"), 2334 expectedWatchResourceVersion: 0, 2335 }, 2336 { 2337 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=nil", 2338 opts: listOptions(false, nil, "0"), 2339 expectedWatchResourceVersion: 0, 2340 }, 2341 { 2342 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=true", 2343 opts: listOptions(false, pointer.Bool(true), "0"), 2344 expectedWatchResourceVersion: 0, 2345 }, 2346 { 2347 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=false", 2348 opts: listOptions(false, pointer.Bool(false), "0"), 2349 expectedWatchResourceVersion: 0, 2350 }, 2351 // +-----------------+---------------------+-----------------------+ 2352 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents | 2353 // +=================+=====================+=======================+ 2354 // | 95 | true/false | nil/true/false | 2355 // +-----------------+---------------------+-----------------------+ 2356 { 2357 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=nil", 2358 opts: listOptions(true, nil, "95"), 2359 expectedWatchResourceVersion: 95, 2360 }, 2361 { 2362 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=true", 2363 opts: listOptions(true, pointer.Bool(true), "95"), 2364 expectedWatchResourceVersion: 95, 2365 }, 2366 { 2367 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=false", 2368 opts: listOptions(true, pointer.Bool(false), "95"), 2369 expectedWatchResourceVersion: 95, 2370 }, 2371 { 2372 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=nil", 2373 opts: listOptions(false, nil, "95"), 2374 expectedWatchResourceVersion: 95, 2375 }, 2376 { 2377 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=true", 2378 opts: listOptions(false, pointer.Bool(true), "95"), 2379 expectedWatchResourceVersion: 95, 2380 }, 2381 { 2382 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=false", 2383 opts: listOptions(false, pointer.Bool(false), "95"), 2384 expectedWatchResourceVersion: 95, 2385 }, 2386 } 2387 2388 for _, scenario := range scenarios { 2389 t.Run(scenario.name, func(t *testing.T) { 2390 backingStorage := &dummyStorage{} 2391 cacher, _, err := newTestCacher(backingStorage) 2392 require.NoError(t, err, "couldn't create cacher") 2393 defer cacher.Stop() 2394 2395 parsedResourceVersion := 0 2396 if len(scenario.opts.ResourceVersion) > 0 { 2397 parsedResourceVersion, err = strconv.Atoi(scenario.opts.ResourceVersion) 2398 require.NoError(t, err) 2399 } 2400 2401 actualResourceVersion, err := cacher.getWatchCacheResourceVersion(context.TODO(), uint64(parsedResourceVersion), scenario.opts) 2402 require.NoError(t, err) 2403 require.Equal(t, uint64(scenario.expectedWatchResourceVersion), actualResourceVersion, "received unexpected ResourceVersion") 2404 }) 2405 } 2406 } 2407 2408 // TestGetBookmarkAfterResourceVersionLockedFunc test the following cases: 2409 // 2410 // +-----------------+---------------------+-----------------------+ 2411 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents | 2412 // +=================+=====================+=======================+ 2413 // | Unset | true/false | nil/true/false | 2414 // | 0 | true/false | nil/true/false | 2415 // | 95 | true/false | nil/true/false | 2416 // +-----------------+---------------------+-----------------------+ 2417 // where: 2418 // - false indicates the value of the param was set to "false" by a test case 2419 // - true indicates the value of the param was set to "true" by a test case 2420 func TestGetBookmarkAfterResourceVersionLockedFunc(t *testing.T) { 2421 listOptions := func(allowBookmarks bool, sendInitialEvents *bool, rv string) storage.ListOptions { 2422 p := storage.Everything 2423 p.AllowWatchBookmarks = allowBookmarks 2424 2425 opts := storage.ListOptions{} 2426 opts.Predicate = p 2427 opts.SendInitialEvents = sendInitialEvents 2428 opts.ResourceVersion = rv 2429 return opts 2430 } 2431 2432 scenarios := []struct { 2433 name string 2434 opts storage.ListOptions 2435 requiredResourceVersion int 2436 watchCacheResourceVersion int 2437 2438 expectedBookmarkResourceVersion int 2439 }{ 2440 // +-----------------+---------------------+-----------------------+ 2441 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents | 2442 // +=================+=====================+=======================+ 2443 // | Unset | true/false | nil/true/false | 2444 // +-----------------+---------------------+-----------------------+ 2445 { 2446 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=nil", 2447 opts: listOptions(true, nil, ""), 2448 requiredResourceVersion: 100, 2449 watchCacheResourceVersion: 99, 2450 expectedBookmarkResourceVersion: 0, 2451 }, 2452 { 2453 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=true", 2454 opts: listOptions(true, pointer.Bool(true), ""), 2455 requiredResourceVersion: 100, 2456 watchCacheResourceVersion: 99, 2457 expectedBookmarkResourceVersion: 100, 2458 }, 2459 { 2460 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=false", 2461 opts: listOptions(true, pointer.Bool(false), ""), 2462 requiredResourceVersion: 100, 2463 watchCacheResourceVersion: 99, 2464 expectedBookmarkResourceVersion: 0, 2465 }, 2466 { 2467 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=nil", 2468 opts: listOptions(false, nil, ""), 2469 requiredResourceVersion: 100, 2470 watchCacheResourceVersion: 99, 2471 expectedBookmarkResourceVersion: 0, 2472 }, 2473 { 2474 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=true", 2475 opts: listOptions(false, pointer.Bool(true), ""), 2476 requiredResourceVersion: 100, 2477 watchCacheResourceVersion: 99, 2478 expectedBookmarkResourceVersion: 0, 2479 }, 2480 { 2481 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=false", 2482 opts: listOptions(false, pointer.Bool(false), ""), 2483 requiredResourceVersion: 100, 2484 watchCacheResourceVersion: 99, 2485 expectedBookmarkResourceVersion: 0, 2486 }, 2487 // +-----------------+---------------------+-----------------------+ 2488 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents | 2489 // +=================+=====================+=======================+ 2490 // | 0 | true/false | nil/true/false | 2491 // +-----------------+---------------------+-----------------------+ 2492 { 2493 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=nil", 2494 opts: listOptions(true, nil, "0"), 2495 requiredResourceVersion: 0, 2496 watchCacheResourceVersion: 99, 2497 expectedBookmarkResourceVersion: 0, 2498 }, 2499 { 2500 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=true", 2501 opts: listOptions(true, pointer.Bool(true), "0"), 2502 requiredResourceVersion: 0, 2503 watchCacheResourceVersion: 99, 2504 expectedBookmarkResourceVersion: 99, 2505 }, 2506 { 2507 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=false", 2508 opts: listOptions(true, pointer.Bool(false), "0"), 2509 requiredResourceVersion: 0, 2510 watchCacheResourceVersion: 99, 2511 expectedBookmarkResourceVersion: 0, 2512 }, 2513 { 2514 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=nil", 2515 opts: listOptions(false, nil, "0"), 2516 requiredResourceVersion: 0, 2517 watchCacheResourceVersion: 99, 2518 expectedBookmarkResourceVersion: 0, 2519 }, 2520 { 2521 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=true", 2522 opts: listOptions(false, pointer.Bool(true), "0"), 2523 requiredResourceVersion: 0, 2524 watchCacheResourceVersion: 99, 2525 expectedBookmarkResourceVersion: 0, 2526 }, 2527 { 2528 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=false", 2529 opts: listOptions(false, pointer.Bool(false), "0"), 2530 requiredResourceVersion: 0, 2531 watchCacheResourceVersion: 99, 2532 expectedBookmarkResourceVersion: 0, 2533 }, 2534 // +-----------------+---------------------+-----------------------+ 2535 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents | 2536 // +=================+=====================+=======================+ 2537 // | 95 | true/false | nil/true/false | 2538 // +-----------------+---------------------+-----------------------+ 2539 { 2540 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=nil", 2541 opts: listOptions(true, nil, "95"), 2542 requiredResourceVersion: 0, 2543 watchCacheResourceVersion: 99, 2544 expectedBookmarkResourceVersion: 0, 2545 }, 2546 { 2547 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=true", 2548 opts: listOptions(true, pointer.Bool(true), "95"), 2549 requiredResourceVersion: 0, 2550 watchCacheResourceVersion: 99, 2551 expectedBookmarkResourceVersion: 95, 2552 }, 2553 { 2554 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=false", 2555 opts: listOptions(true, pointer.Bool(false), "95"), 2556 requiredResourceVersion: 0, 2557 watchCacheResourceVersion: 99, 2558 expectedBookmarkResourceVersion: 0, 2559 }, 2560 { 2561 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=nil", 2562 opts: listOptions(false, nil, "95"), 2563 requiredResourceVersion: 100, 2564 watchCacheResourceVersion: 99, 2565 expectedBookmarkResourceVersion: 0, 2566 }, 2567 { 2568 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=true", 2569 opts: listOptions(false, pointer.Bool(true), "95"), 2570 requiredResourceVersion: 0, 2571 watchCacheResourceVersion: 99, 2572 expectedBookmarkResourceVersion: 0, 2573 }, 2574 { 2575 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=false", 2576 opts: listOptions(false, pointer.Bool(false), "95"), 2577 requiredResourceVersion: 0, 2578 watchCacheResourceVersion: 99, 2579 expectedBookmarkResourceVersion: 0, 2580 }, 2581 } 2582 for _, scenario := range scenarios { 2583 t.Run(scenario.name, func(t *testing.T) { 2584 backingStorage := &dummyStorage{} 2585 cacher, _, err := newTestCacher(backingStorage) 2586 require.NoError(t, err, "couldn't create cacher") 2587 2588 defer cacher.Stop() 2589 2590 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 2591 if err := cacher.ready.wait(context.Background()); err != nil { 2592 t.Fatalf("unexpected error waiting for the cache to be ready") 2593 } 2594 } 2595 2596 cacher.watchCache.UpdateResourceVersion(fmt.Sprintf("%d", scenario.watchCacheResourceVersion)) 2597 parsedResourceVersion := 0 2598 if len(scenario.opts.ResourceVersion) > 0 { 2599 parsedResourceVersion, err = strconv.Atoi(scenario.opts.ResourceVersion) 2600 require.NoError(t, err) 2601 } 2602 2603 getBookMarkFn, err := cacher.getBookmarkAfterResourceVersionLockedFunc(uint64(parsedResourceVersion), uint64(scenario.requiredResourceVersion), scenario.opts) 2604 require.NoError(t, err) 2605 cacher.watchCache.RLock() 2606 defer cacher.watchCache.RUnlock() 2607 getBookMarkResourceVersion := getBookMarkFn() 2608 require.Equal(t, uint64(scenario.expectedBookmarkResourceVersion), getBookMarkResourceVersion, "received unexpected ResourceVersion") 2609 }) 2610 } 2611 } 2612 2613 func TestWatchStreamSeparation(t *testing.T) { 2614 server, etcdStorage := newEtcdTestStorage(t, etcd3testing.PathPrefix()) 2615 t.Cleanup(func() { 2616 server.Terminate(t) 2617 }) 2618 setupOpts := &setupOptions{} 2619 withDefaults(setupOpts) 2620 config := Config{ 2621 Storage: etcdStorage, 2622 Versioner: storage.APIObjectVersioner{}, 2623 GroupResource: schema.GroupResource{Resource: "pods"}, 2624 ResourcePrefix: setupOpts.resourcePrefix, 2625 KeyFunc: setupOpts.keyFunc, 2626 GetAttrsFunc: GetPodAttrs, 2627 NewFunc: newPod, 2628 NewListFunc: newPodList, 2629 IndexerFuncs: setupOpts.indexerFuncs, 2630 Codec: codecs.LegacyCodec(examplev1.SchemeGroupVersion), 2631 Clock: setupOpts.clock, 2632 } 2633 tcs := []struct { 2634 name string 2635 separateCacheWatchRPC bool 2636 useWatchCacheContextMetadata bool 2637 expectBookmarkOnWatchCache bool 2638 expectBookmarkOnEtcd bool 2639 }{ 2640 { 2641 name: "common RPC > both get bookmarks", 2642 separateCacheWatchRPC: false, 2643 expectBookmarkOnEtcd: true, 2644 expectBookmarkOnWatchCache: true, 2645 }, 2646 { 2647 name: "common RPC & watch cache context > both get bookmarks", 2648 separateCacheWatchRPC: false, 2649 useWatchCacheContextMetadata: true, 2650 expectBookmarkOnEtcd: true, 2651 expectBookmarkOnWatchCache: true, 2652 }, 2653 { 2654 name: "separate RPC > only etcd gets bookmarks", 2655 separateCacheWatchRPC: true, 2656 expectBookmarkOnEtcd: true, 2657 expectBookmarkOnWatchCache: false, 2658 }, 2659 { 2660 name: "separate RPC & watch cache context > only watch cache gets bookmarks", 2661 separateCacheWatchRPC: true, 2662 useWatchCacheContextMetadata: true, 2663 expectBookmarkOnEtcd: false, 2664 expectBookmarkOnWatchCache: true, 2665 }, 2666 } 2667 for _, tc := range tcs { 2668 t.Run(tc.name, func(t *testing.T) { 2669 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SeparateCacheWatchRPC, tc.separateCacheWatchRPC) 2670 cacher, err := NewCacherFromConfig(config) 2671 if err != nil { 2672 t.Fatalf("Failed to initialize cacher: %v", err) 2673 } 2674 2675 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { 2676 if err := cacher.ready.wait(context.TODO()); err != nil { 2677 t.Fatalf("unexpected error waiting for the cache to be ready") 2678 } 2679 } 2680 2681 getCacherRV := func() uint64 { 2682 cacher.watchCache.RLock() 2683 defer cacher.watchCache.RUnlock() 2684 return cacher.watchCache.resourceVersion 2685 } 2686 waitContext, cancel := context.WithTimeout(context.Background(), 2*time.Second) 2687 defer cancel() 2688 waitForEtcdBookmark := watchAndWaitForBookmark(t, waitContext, cacher.storage) 2689 2690 var out example.Pod 2691 err = cacher.Create(context.Background(), "foo", &example.Pod{}, &out, 0) 2692 if err != nil { 2693 t.Fatal(err) 2694 } 2695 err = cacher.Delete(context.Background(), "foo", &out, nil, storage.ValidateAllObjectFunc, &example.Pod{}) 2696 if err != nil { 2697 t.Fatal(err) 2698 } 2699 versioner := storage.APIObjectVersioner{} 2700 var lastResourceVersion uint64 2701 lastResourceVersion, err = versioner.ObjectResourceVersion(&out) 2702 if err != nil { 2703 t.Fatal(err) 2704 } 2705 2706 var contextMetadata metadata.MD 2707 if tc.useWatchCacheContextMetadata { 2708 contextMetadata = cacher.watchCache.waitingUntilFresh.contextMetadata 2709 } 2710 // For the first 100ms from watch creation, watch progress requests are ignored. 2711 time.Sleep(200 * time.Millisecond) 2712 err = cacher.storage.RequestWatchProgress(metadata.NewOutgoingContext(context.Background(), contextMetadata)) 2713 if err != nil { 2714 t.Fatal(err) 2715 } 2716 // Give time for bookmark to arrive 2717 time.Sleep(time.Second) 2718 2719 etcdWatchResourceVersion := waitForEtcdBookmark() 2720 gotEtcdWatchBookmark := etcdWatchResourceVersion == lastResourceVersion 2721 if gotEtcdWatchBookmark != tc.expectBookmarkOnEtcd { 2722 t.Errorf("Unexpected etcd bookmark check result, rv: %d, lastRV: %d, wantMatching: %v", etcdWatchResourceVersion, lastResourceVersion, tc.expectBookmarkOnEtcd) 2723 } 2724 2725 watchCacheResourceVersion := getCacherRV() 2726 cacherGotBookmark := watchCacheResourceVersion == lastResourceVersion 2727 if cacherGotBookmark != tc.expectBookmarkOnWatchCache { 2728 t.Errorf("Unexpected watch cache bookmark check result, rv: %d, lastRV: %d, wantMatching: %v", watchCacheResourceVersion, lastResourceVersion, tc.expectBookmarkOnWatchCache) 2729 } 2730 }) 2731 } 2732 } 2733 2734 func TestComputeListLimit(t *testing.T) { 2735 scenarios := []struct { 2736 name string 2737 opts storage.ListOptions 2738 expectedLimit int64 2739 }{ 2740 { 2741 name: "limit is zero", 2742 opts: storage.ListOptions{ 2743 Predicate: storage.SelectionPredicate{ 2744 Limit: 0, 2745 }, 2746 }, 2747 expectedLimit: 0, 2748 }, 2749 { 2750 name: "limit is positive, RV is unset", 2751 opts: storage.ListOptions{ 2752 Predicate: storage.SelectionPredicate{ 2753 Limit: 1, 2754 }, 2755 ResourceVersion: "", 2756 }, 2757 expectedLimit: 1, 2758 }, 2759 { 2760 name: "limit is positive, RV = 100", 2761 opts: storage.ListOptions{ 2762 Predicate: storage.SelectionPredicate{ 2763 Limit: 1, 2764 }, 2765 ResourceVersion: "100", 2766 }, 2767 expectedLimit: 1, 2768 }, 2769 { 2770 name: "legacy case: limit is positive, RV = 0", 2771 opts: storage.ListOptions{ 2772 Predicate: storage.SelectionPredicate{ 2773 Limit: 1, 2774 }, 2775 ResourceVersion: "0", 2776 }, 2777 expectedLimit: 0, 2778 }, 2779 } 2780 2781 for _, scenario := range scenarios { 2782 t.Run(scenario.name, func(t *testing.T) { 2783 actualLimit := computeListLimit(scenario.opts) 2784 if actualLimit != scenario.expectedLimit { 2785 t.Errorf("computeListLimit returned = %v, expected %v", actualLimit, scenario.expectedLimit) 2786 } 2787 }) 2788 } 2789 } 2790 2791 func watchAndWaitForBookmark(t *testing.T, ctx context.Context, etcdStorage storage.Interface) func() (resourceVersion uint64) { 2792 opts := storage.ListOptions{ResourceVersion: "", Predicate: storage.Everything, Recursive: true} 2793 opts.Predicate.AllowWatchBookmarks = true 2794 w, err := etcdStorage.Watch(ctx, "/pods/", opts) 2795 if err != nil { 2796 t.Fatal(err) 2797 } 2798 2799 versioner := storage.APIObjectVersioner{} 2800 var rv uint64 2801 var wg sync.WaitGroup 2802 wg.Add(1) 2803 go func() { 2804 defer wg.Done() 2805 for event := range w.ResultChan() { 2806 if event.Type == watch.Bookmark { 2807 rv, err = versioner.ObjectResourceVersion(event.Object) 2808 break 2809 } 2810 } 2811 }() 2812 return func() (resourceVersion uint64) { 2813 defer w.Stop() 2814 wg.Wait() 2815 if err != nil { 2816 t.Fatal(err) 2817 } 2818 return rv 2819 } 2820 } 2821 2822 // TODO(p0lyn0mial): forceRequestWatchProgressSupport inits the storage layer 2823 // so that tests that require storage.RequestWatchProgress pass 2824 // 2825 // In the future we could have a function that would allow for setting the feature 2826 // only for duration of a test. 2827 func forceRequestWatchProgressSupport(t *testing.T) { 2828 if etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress) { 2829 return 2830 } 2831 2832 server, _ := newEtcdTestStorage(t, etcd3testing.PathPrefix()) 2833 defer server.Terminate(t) 2834 if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, wait.ForeverTestTimeout, true, func(_ context.Context) (bool, error) { 2835 return etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress), nil 2836 }); err != nil { 2837 t.Fatalf("failed to wait for required %v storage feature to initialize", storage.RequestWatchProgress) 2838 } 2839 }