k8s.io/apiserver@v0.31.1/pkg/storage/cacher/watch_cache_test.go (about) 1 /* 2 Copyright 2014 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package cacher 18 19 import ( 20 "context" 21 "fmt" 22 "strconv" 23 "strings" 24 "testing" 25 "time" 26 27 v1 "k8s.io/api/core/v1" 28 apiequality "k8s.io/apimachinery/pkg/api/equality" 29 "k8s.io/apimachinery/pkg/api/errors" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/fields" 32 "k8s.io/apimachinery/pkg/labels" 33 "k8s.io/apimachinery/pkg/runtime" 34 "k8s.io/apimachinery/pkg/runtime/schema" 35 "k8s.io/apimachinery/pkg/util/wait" 36 "k8s.io/apimachinery/pkg/watch" 37 "k8s.io/apiserver/pkg/features" 38 "k8s.io/apiserver/pkg/storage" 39 "k8s.io/apiserver/pkg/storage/cacher/metrics" 40 utilfeature "k8s.io/apiserver/pkg/util/feature" 41 "k8s.io/client-go/tools/cache" 42 featuregatetesting "k8s.io/component-base/featuregate/testing" 43 k8smetrics "k8s.io/component-base/metrics" 44 "k8s.io/component-base/metrics/testutil" 45 "k8s.io/utils/clock" 46 testingclock "k8s.io/utils/clock/testing" 47 ) 48 49 func makeTestPod(name string, resourceVersion uint64) *v1.Pod { 50 return makeTestPodDetails(name, resourceVersion, "some-node", map[string]string{"k8s-app": "my-app"}) 51 } 52 53 func makeTestPodDetails(name string, resourceVersion uint64, nodeName string, labels map[string]string) *v1.Pod { 54 return &v1.Pod{ 55 ObjectMeta: metav1.ObjectMeta{ 56 Namespace: "ns", 57 Name: name, 58 ResourceVersion: strconv.FormatUint(resourceVersion, 10), 59 Labels: labels, 60 }, 61 Spec: v1.PodSpec{ 62 NodeName: nodeName, 63 }, 64 } 65 } 66 67 func makeTestStoreElement(pod *v1.Pod) *storeElement { 68 return &storeElement{ 69 Key: "prefix/ns/" + pod.Name, 70 Object: pod, 71 Labels: labels.Set(pod.Labels), 72 Fields: fields.Set{"spec.nodeName": pod.Spec.NodeName}, 73 } 74 } 75 76 type testWatchCache struct { 77 *watchCache 78 79 bookmarkRevision chan int64 80 stopCh chan struct{} 81 } 82 83 func (w *testWatchCache) getAllEventsSince(resourceVersion uint64, opts storage.ListOptions) ([]*watchCacheEvent, error) { 84 cacheInterval, err := w.getCacheIntervalForEvents(resourceVersion, opts) 85 if err != nil { 86 return nil, err 87 } 88 89 result := []*watchCacheEvent{} 90 for { 91 event, err := cacheInterval.Next() 92 if err != nil { 93 return nil, err 94 } 95 if event == nil { 96 break 97 } 98 result = append(result, event) 99 } 100 101 return result, nil 102 } 103 104 func (w *testWatchCache) getCacheIntervalForEvents(resourceVersion uint64, opts storage.ListOptions) (*watchCacheInterval, error) { 105 w.RLock() 106 defer w.RUnlock() 107 108 return w.getAllEventsSinceLocked(resourceVersion, "", opts) 109 } 110 111 // newTestWatchCache just adds a fake clock. 112 func newTestWatchCache(capacity int, indexers *cache.Indexers) *testWatchCache { 113 keyFunc := func(obj runtime.Object) (string, error) { 114 return storage.NamespaceKeyFunc("prefix", obj) 115 } 116 getAttrsFunc := func(obj runtime.Object) (labels.Set, fields.Set, error) { 117 pod, ok := obj.(*v1.Pod) 118 if !ok { 119 return nil, nil, fmt.Errorf("not a pod") 120 } 121 return labels.Set(pod.Labels), fields.Set{"spec.nodeName": pod.Spec.NodeName}, nil 122 } 123 versioner := storage.APIObjectVersioner{} 124 mockHandler := func(*watchCacheEvent) {} 125 wc := &testWatchCache{} 126 wc.bookmarkRevision = make(chan int64, 1) 127 wc.stopCh = make(chan struct{}) 128 pr := newConditionalProgressRequester(wc.RequestWatchProgress, &immediateTickerFactory{}, nil) 129 go pr.Run(wc.stopCh) 130 wc.watchCache = newWatchCache(keyFunc, mockHandler, getAttrsFunc, versioner, indexers, testingclock.NewFakeClock(time.Now()), schema.GroupResource{Resource: "pods"}, pr) 131 // To preserve behavior of tests that assume a given capacity, 132 // resize it to th expected size. 133 wc.capacity = capacity 134 wc.cache = make([]*watchCacheEvent, capacity) 135 wc.lowerBoundCapacity = min(capacity, defaultLowerBoundCapacity) 136 wc.upperBoundCapacity = max(capacity, defaultUpperBoundCapacity) 137 138 return wc 139 } 140 141 type immediateTickerFactory struct{} 142 143 func (t *immediateTickerFactory) NewTimer(d time.Duration) clock.Timer { 144 timer := immediateTicker{ 145 c: make(chan time.Time), 146 } 147 timer.Reset(d) 148 return &timer 149 } 150 151 type immediateTicker struct { 152 c chan time.Time 153 } 154 155 func (t *immediateTicker) Reset(d time.Duration) (active bool) { 156 select { 157 case <-t.c: 158 active = true 159 default: 160 } 161 go func() { 162 t.c <- time.Now() 163 }() 164 return active 165 } 166 167 func (t *immediateTicker) C() <-chan time.Time { 168 return t.c 169 } 170 171 func (t *immediateTicker) Stop() bool { 172 select { 173 case <-t.c: 174 return true 175 default: 176 return false 177 } 178 } 179 180 func (w *testWatchCache) RequestWatchProgress(ctx context.Context) error { 181 go func() { 182 select { 183 case rev := <-w.bookmarkRevision: 184 w.UpdateResourceVersion(fmt.Sprintf("%d", rev)) 185 case <-ctx.Done(): 186 return 187 } 188 }() 189 return nil 190 } 191 192 func (w *testWatchCache) Stop() { 193 close(w.stopCh) 194 } 195 196 func TestWatchCacheBasic(t *testing.T) { 197 store := newTestWatchCache(2, &cache.Indexers{}) 198 defer store.Stop() 199 200 // Test Add/Update/Delete. 201 pod1 := makeTestPod("pod", 1) 202 if err := store.Add(pod1); err != nil { 203 t.Errorf("unexpected error: %v", err) 204 } 205 if item, ok, _ := store.Get(pod1); !ok { 206 t.Errorf("didn't find pod") 207 } else { 208 expected := makeTestStoreElement(makeTestPod("pod", 1)) 209 if !apiequality.Semantic.DeepEqual(expected, item) { 210 t.Errorf("expected %v, got %v", expected, item) 211 } 212 } 213 pod2 := makeTestPod("pod", 2) 214 if err := store.Update(pod2); err != nil { 215 t.Errorf("unexpected error: %v", err) 216 } 217 if item, ok, _ := store.Get(pod2); !ok { 218 t.Errorf("didn't find pod") 219 } else { 220 expected := makeTestStoreElement(makeTestPod("pod", 2)) 221 if !apiequality.Semantic.DeepEqual(expected, item) { 222 t.Errorf("expected %v, got %v", expected, item) 223 } 224 } 225 pod3 := makeTestPod("pod", 3) 226 if err := store.Delete(pod3); err != nil { 227 t.Errorf("unexpected error: %v", err) 228 } 229 if _, ok, _ := store.Get(pod3); ok { 230 t.Errorf("found pod") 231 } 232 233 // Test List. 234 store.Add(makeTestPod("pod1", 4)) 235 store.Add(makeTestPod("pod2", 5)) 236 store.Add(makeTestPod("pod3", 6)) 237 { 238 expected := map[string]storeElement{ 239 "prefix/ns/pod1": *makeTestStoreElement(makeTestPod("pod1", 4)), 240 "prefix/ns/pod2": *makeTestStoreElement(makeTestPod("pod2", 5)), 241 "prefix/ns/pod3": *makeTestStoreElement(makeTestPod("pod3", 6)), 242 } 243 items := make(map[string]storeElement) 244 for _, item := range store.List() { 245 elem := item.(*storeElement) 246 items[elem.Key] = *elem 247 } 248 if !apiequality.Semantic.DeepEqual(expected, items) { 249 t.Errorf("expected %v, got %v", expected, items) 250 } 251 } 252 253 // Test Replace. 254 store.Replace([]interface{}{ 255 makeTestPod("pod4", 7), 256 makeTestPod("pod5", 8), 257 }, "8") 258 { 259 expected := map[string]storeElement{ 260 "prefix/ns/pod4": *makeTestStoreElement(makeTestPod("pod4", 7)), 261 "prefix/ns/pod5": *makeTestStoreElement(makeTestPod("pod5", 8)), 262 } 263 items := make(map[string]storeElement) 264 for _, item := range store.List() { 265 elem := item.(*storeElement) 266 items[elem.Key] = *elem 267 } 268 if !apiequality.Semantic.DeepEqual(expected, items) { 269 t.Errorf("expected %v, got %v", expected, items) 270 } 271 } 272 } 273 274 func TestEvents(t *testing.T) { 275 store := newTestWatchCache(5, &cache.Indexers{}) 276 defer store.Stop() 277 278 // no dynamic-size cache to fit old tests. 279 store.lowerBoundCapacity = 5 280 store.upperBoundCapacity = 5 281 282 store.Add(makeTestPod("pod", 3)) 283 284 // Test for Added event. 285 { 286 _, err := store.getAllEventsSince(1, storage.ListOptions{}) 287 if err == nil { 288 t.Errorf("expected error too old") 289 } 290 if _, ok := err.(*errors.StatusError); !ok { 291 t.Errorf("expected error to be of type StatusError") 292 } 293 } 294 { 295 result, err := store.getAllEventsSince(2, storage.ListOptions{}) 296 if err != nil { 297 t.Errorf("unexpected error: %v", err) 298 } 299 if len(result) != 1 { 300 t.Fatalf("unexpected events: %v", result) 301 } 302 if result[0].Type != watch.Added { 303 t.Errorf("unexpected event type: %v", result[0].Type) 304 } 305 pod := makeTestPod("pod", uint64(3)) 306 if !apiequality.Semantic.DeepEqual(pod, result[0].Object) { 307 t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod) 308 } 309 if result[0].PrevObject != nil { 310 t.Errorf("unexpected item: %v", result[0].PrevObject) 311 } 312 } 313 314 store.Update(makeTestPod("pod", 4)) 315 store.Update(makeTestPod("pod", 5)) 316 317 // Test with not full cache. 318 { 319 _, err := store.getAllEventsSince(1, storage.ListOptions{}) 320 if err == nil { 321 t.Errorf("expected error too old") 322 } 323 } 324 { 325 result, err := store.getAllEventsSince(3, storage.ListOptions{}) 326 if err != nil { 327 t.Errorf("unexpected error: %v", err) 328 } 329 if len(result) != 2 { 330 t.Fatalf("unexpected events: %v", result) 331 } 332 for i := 0; i < 2; i++ { 333 if result[i].Type != watch.Modified { 334 t.Errorf("unexpected event type: %v", result[i].Type) 335 } 336 pod := makeTestPod("pod", uint64(i+4)) 337 if !apiequality.Semantic.DeepEqual(pod, result[i].Object) { 338 t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod) 339 } 340 prevPod := makeTestPod("pod", uint64(i+3)) 341 if !apiequality.Semantic.DeepEqual(prevPod, result[i].PrevObject) { 342 t.Errorf("unexpected item: %v, expected: %v", result[i].PrevObject, prevPod) 343 } 344 } 345 } 346 347 for i := 6; i < 10; i++ { 348 store.Update(makeTestPod("pod", uint64(i))) 349 } 350 351 // Test with full cache - there should be elements from 5 to 9. 352 { 353 _, err := store.getAllEventsSince(3, storage.ListOptions{}) 354 if err == nil { 355 t.Errorf("expected error too old") 356 } 357 } 358 { 359 result, err := store.getAllEventsSince(4, storage.ListOptions{}) 360 if err != nil { 361 t.Errorf("unexpected error: %v", err) 362 } 363 if len(result) != 5 { 364 t.Fatalf("unexpected events: %v", result) 365 } 366 for i := 0; i < 5; i++ { 367 pod := makeTestPod("pod", uint64(i+5)) 368 if !apiequality.Semantic.DeepEqual(pod, result[i].Object) { 369 t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod) 370 } 371 } 372 } 373 374 // Test for delete event. 375 store.Delete(makeTestPod("pod", uint64(10))) 376 377 { 378 result, err := store.getAllEventsSince(9, storage.ListOptions{}) 379 if err != nil { 380 t.Errorf("unexpected error: %v", err) 381 } 382 if len(result) != 1 { 383 t.Fatalf("unexpected events: %v", result) 384 } 385 if result[0].Type != watch.Deleted { 386 t.Errorf("unexpected event type: %v", result[0].Type) 387 } 388 pod := makeTestPod("pod", uint64(10)) 389 if !apiequality.Semantic.DeepEqual(pod, result[0].Object) { 390 t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod) 391 } 392 prevPod := makeTestPod("pod", uint64(9)) 393 if !apiequality.Semantic.DeepEqual(prevPod, result[0].PrevObject) { 394 t.Errorf("unexpected item: %v, expected: %v", result[0].PrevObject, prevPod) 395 } 396 } 397 } 398 399 func TestMarker(t *testing.T) { 400 store := newTestWatchCache(3, &cache.Indexers{}) 401 defer store.Stop() 402 403 // First thing that is called when propagated from storage is Replace. 404 store.Replace([]interface{}{ 405 makeTestPod("pod1", 5), 406 makeTestPod("pod2", 9), 407 }, "9") 408 409 _, err := store.getAllEventsSince(8, storage.ListOptions{}) 410 if err == nil || !strings.Contains(err.Error(), "too old resource version") { 411 t.Errorf("unexpected error: %v", err) 412 } 413 // Getting events from 8 should return no events, 414 // even though there is a marker there. 415 result, err := store.getAllEventsSince(9, storage.ListOptions{}) 416 if err != nil { 417 t.Fatalf("unexpected error: %v", err) 418 } 419 if len(result) != 0 { 420 t.Errorf("unexpected result: %#v, expected no events", result) 421 } 422 423 pod := makeTestPod("pods", 12) 424 store.Add(pod) 425 // Getting events from 8 should still work and return one event. 426 result, err = store.getAllEventsSince(9, storage.ListOptions{}) 427 if err != nil { 428 t.Fatalf("unexpected error: %v", err) 429 } 430 if len(result) != 1 || !apiequality.Semantic.DeepEqual(result[0].Object, pod) { 431 t.Errorf("unexpected result: %#v, expected %v", result, pod) 432 } 433 } 434 435 func TestWaitUntilFreshAndList(t *testing.T) { 436 ctx := context.Background() 437 store := newTestWatchCache(3, &cache.Indexers{ 438 "l:label": func(obj interface{}) ([]string, error) { 439 pod, ok := obj.(*v1.Pod) 440 if !ok { 441 return nil, fmt.Errorf("not a pod %#v", obj) 442 } 443 if value, ok := pod.Labels["label"]; ok { 444 return []string{value}, nil 445 } 446 return nil, nil 447 }, 448 "f:spec.nodeName": func(obj interface{}) ([]string, error) { 449 pod, ok := obj.(*v1.Pod) 450 if !ok { 451 return nil, fmt.Errorf("not a pod %#v", obj) 452 } 453 return []string{pod.Spec.NodeName}, nil 454 }, 455 }) 456 defer store.Stop() 457 // In background, update the store. 458 go func() { 459 store.Add(makeTestPodDetails("pod1", 2, "node1", map[string]string{"label": "value1"})) 460 store.Add(makeTestPodDetails("pod2", 3, "node1", map[string]string{"label": "value1"})) 461 store.Add(makeTestPodDetails("pod3", 5, "node2", map[string]string{"label": "value2"})) 462 }() 463 464 // list by empty MatchValues. 465 list, resourceVersion, indexUsed, err := store.WaitUntilFreshAndList(ctx, 5, "prefix/", nil) 466 if err != nil { 467 t.Fatalf("unexpected error: %v", err) 468 } 469 if resourceVersion != 5 { 470 t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion) 471 } 472 if len(list) != 3 { 473 t.Errorf("unexpected list returned: %#v", list) 474 } 475 if indexUsed != "" { 476 t.Errorf("Used index %q but expected none to be used", indexUsed) 477 } 478 479 // list by label index. 480 matchValues := []storage.MatchValue{ 481 {IndexName: "l:label", Value: "value1"}, 482 {IndexName: "f:spec.nodeName", Value: "node2"}, 483 } 484 list, resourceVersion, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", matchValues) 485 if err != nil { 486 t.Fatalf("unexpected error: %v", err) 487 } 488 if resourceVersion != 5 { 489 t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion) 490 } 491 if len(list) != 2 { 492 t.Errorf("unexpected list returned: %#v", list) 493 } 494 if indexUsed != "l:label" { 495 t.Errorf("Used index %q but expected %q", indexUsed, "l:label") 496 } 497 498 // list with spec.nodeName index. 499 matchValues = []storage.MatchValue{ 500 {IndexName: "l:not-exist-label", Value: "whatever"}, 501 {IndexName: "f:spec.nodeName", Value: "node2"}, 502 } 503 list, resourceVersion, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", matchValues) 504 if err != nil { 505 t.Fatalf("unexpected error: %v", err) 506 } 507 if resourceVersion != 5 { 508 t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion) 509 } 510 if len(list) != 1 { 511 t.Errorf("unexpected list returned: %#v", list) 512 } 513 if indexUsed != "f:spec.nodeName" { 514 t.Errorf("Used index %q but expected %q", indexUsed, "f:spec.nodeName") 515 } 516 517 // list with index not exists. 518 matchValues = []storage.MatchValue{ 519 {IndexName: "l:not-exist-label", Value: "whatever"}, 520 } 521 list, resourceVersion, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", matchValues) 522 if err != nil { 523 t.Fatalf("unexpected error: %v", err) 524 } 525 if resourceVersion != 5 { 526 t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion) 527 } 528 if len(list) != 3 { 529 t.Errorf("unexpected list returned: %#v", list) 530 } 531 if indexUsed != "" { 532 t.Errorf("Used index %q but expected none to be used", indexUsed) 533 } 534 } 535 536 func TestWaitUntilFreshAndListFromCache(t *testing.T) { 537 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true) 538 forceRequestWatchProgressSupport(t) 539 ctx := context.Background() 540 store := newTestWatchCache(3, &cache.Indexers{}) 541 defer store.Stop() 542 // In background, update the store. 543 go func() { 544 store.Add(makeTestPod("pod1", 2)) 545 store.bookmarkRevision <- 3 546 }() 547 548 // list from future revision. Requires watch cache to request bookmark to get it. 549 list, resourceVersion, indexUsed, err := store.WaitUntilFreshAndList(ctx, 3, "prefix/", nil) 550 if err != nil { 551 t.Fatalf("unexpected error: %v", err) 552 } 553 if resourceVersion != 3 { 554 t.Errorf("unexpected resourceVersion: %v, expected: 6", resourceVersion) 555 } 556 if len(list) != 1 { 557 t.Errorf("unexpected list returned: %#v", list) 558 } 559 if indexUsed != "" { 560 t.Errorf("Used index %q but expected none to be used", indexUsed) 561 } 562 } 563 564 func TestWaitUntilFreshAndGet(t *testing.T) { 565 ctx := context.Background() 566 store := newTestWatchCache(3, &cache.Indexers{}) 567 defer store.Stop() 568 569 // In background, update the store. 570 go func() { 571 store.Add(makeTestPod("foo", 2)) 572 store.Add(makeTestPod("bar", 5)) 573 }() 574 575 obj, exists, resourceVersion, err := store.WaitUntilFreshAndGet(ctx, 5, "prefix/ns/bar") 576 if err != nil { 577 t.Fatalf("unexpected error: %v", err) 578 } 579 if resourceVersion != 5 { 580 t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion) 581 } 582 if !exists { 583 t.Fatalf("no results returned: %#v", obj) 584 } 585 expected := makeTestStoreElement(makeTestPod("bar", 5)) 586 if !apiequality.Semantic.DeepEqual(expected, obj) { 587 t.Errorf("expected %v, got %v", expected, obj) 588 } 589 } 590 591 func TestWaitUntilFreshAndListTimeout(t *testing.T) { 592 tcs := []struct { 593 name string 594 ConsistentListFromCache bool 595 }{ 596 { 597 name: "FromStorage", 598 ConsistentListFromCache: false, 599 }, 600 { 601 name: "FromCache", 602 ConsistentListFromCache: true, 603 }, 604 } 605 for _, tc := range tcs { 606 t.Run(tc.name, func(t *testing.T) { 607 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, tc.ConsistentListFromCache) 608 ctx := context.Background() 609 store := newTestWatchCache(3, &cache.Indexers{}) 610 defer store.Stop() 611 fc := store.clock.(*testingclock.FakeClock) 612 613 // In background, step clock after the below call starts the timer. 614 go func() { 615 for !fc.HasWaiters() { 616 time.Sleep(time.Millisecond) 617 } 618 store.Add(makeTestPod("foo", 2)) 619 store.bookmarkRevision <- 3 620 fc.Step(blockTimeout) 621 622 // Add an object to make sure the test would 623 // eventually fail instead of just waiting 624 // forever. 625 time.Sleep(30 * time.Second) 626 store.Add(makeTestPod("bar", 4)) 627 }() 628 629 _, _, _, err := store.WaitUntilFreshAndList(ctx, 4, "", nil) 630 if !errors.IsTimeout(err) { 631 t.Errorf("expected timeout error but got: %v", err) 632 } 633 if !storage.IsTooLargeResourceVersion(err) { 634 t.Errorf("expected 'Too large resource version' cause in error but got: %v", err) 635 } 636 }) 637 } 638 } 639 640 type testLW struct { 641 ListFunc func(options metav1.ListOptions) (runtime.Object, error) 642 WatchFunc func(options metav1.ListOptions) (watch.Interface, error) 643 } 644 645 func (t *testLW) List(options metav1.ListOptions) (runtime.Object, error) { 646 return t.ListFunc(options) 647 } 648 func (t *testLW) Watch(options metav1.ListOptions) (watch.Interface, error) { 649 return t.WatchFunc(options) 650 } 651 652 func TestReflectorForWatchCache(t *testing.T) { 653 ctx := context.Background() 654 store := newTestWatchCache(5, &cache.Indexers{}) 655 defer store.Stop() 656 657 { 658 _, version, _, err := store.WaitUntilFreshAndList(ctx, 0, "", nil) 659 if err != nil { 660 t.Fatalf("unexpected error: %v", err) 661 } 662 if version != 0 { 663 t.Errorf("unexpected resource version: %d", version) 664 } 665 } 666 667 lw := &testLW{ 668 WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) { 669 fw := watch.NewFake() 670 go fw.Stop() 671 return fw, nil 672 }, 673 ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { 674 return &v1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "10"}}, nil 675 }, 676 } 677 r := cache.NewReflector(lw, &v1.Pod{}, store, 0) 678 r.ListAndWatch(wait.NeverStop) 679 680 { 681 _, version, _, err := store.WaitUntilFreshAndList(ctx, 10, "", nil) 682 if err != nil { 683 t.Fatalf("unexpected error: %v", err) 684 } 685 if version != 10 { 686 t.Errorf("unexpected resource version: %d", version) 687 } 688 } 689 } 690 691 func TestDynamicCache(t *testing.T) { 692 tests := []struct { 693 name string 694 eventCount int 695 cacheCapacity int 696 startIndex int 697 // interval is time duration between adjacent events. 698 lowerBoundCapacity int 699 upperBoundCapacity int 700 interval time.Duration 701 expectCapacity int 702 expectStartIndex int 703 }{ 704 { 705 name: "[capacity not equals 4*n] events inside eventFreshDuration cause cache expanding", 706 eventCount: 5, 707 cacheCapacity: 5, 708 lowerBoundCapacity: 5 / 2, 709 upperBoundCapacity: 5 * 2, 710 interval: eventFreshDuration / 6, 711 expectCapacity: 10, 712 expectStartIndex: 0, 713 }, 714 { 715 name: "[capacity not equals 4*n] events outside eventFreshDuration without change cache capacity", 716 eventCount: 5, 717 cacheCapacity: 5, 718 lowerBoundCapacity: 5 / 2, 719 upperBoundCapacity: 5 * 2, 720 interval: eventFreshDuration / 4, 721 expectCapacity: 5, 722 expectStartIndex: 0, 723 }, 724 { 725 name: "[capacity not equals 4*n] quarter of recent events outside eventFreshDuration cause cache shrinking", 726 eventCount: 5, 727 cacheCapacity: 5, 728 lowerBoundCapacity: 5 / 2, 729 upperBoundCapacity: 5 * 2, 730 interval: eventFreshDuration + time.Second, 731 expectCapacity: 2, 732 expectStartIndex: 3, 733 }, 734 { 735 name: "[capacity not equals 4*n] quarter of recent events outside eventFreshDuration cause cache shrinking with given lowerBoundCapacity", 736 eventCount: 5, 737 cacheCapacity: 5, 738 lowerBoundCapacity: 3, 739 upperBoundCapacity: 5 * 2, 740 interval: eventFreshDuration + time.Second, 741 expectCapacity: 3, 742 expectStartIndex: 2, 743 }, 744 { 745 name: "[capacity not equals 4*n] events inside eventFreshDuration cause cache expanding with given upperBoundCapacity", 746 eventCount: 5, 747 cacheCapacity: 5, 748 lowerBoundCapacity: 5 / 2, 749 upperBoundCapacity: 8, 750 interval: eventFreshDuration / 6, 751 expectCapacity: 8, 752 expectStartIndex: 0, 753 }, 754 { 755 name: "[capacity not equals 4*n] [startIndex not equal 0] events inside eventFreshDuration cause cache expanding", 756 eventCount: 5, 757 cacheCapacity: 5, 758 startIndex: 3, 759 lowerBoundCapacity: 5 / 2, 760 upperBoundCapacity: 5 * 2, 761 interval: eventFreshDuration / 6, 762 expectCapacity: 10, 763 expectStartIndex: 3, 764 }, 765 { 766 name: "[capacity not equals 4*n] [startIndex not equal 0] events outside eventFreshDuration without change cache capacity", 767 eventCount: 5, 768 cacheCapacity: 5, 769 startIndex: 3, 770 lowerBoundCapacity: 5 / 2, 771 upperBoundCapacity: 5 * 2, 772 interval: eventFreshDuration / 4, 773 expectCapacity: 5, 774 expectStartIndex: 3, 775 }, 776 { 777 name: "[capacity not equals 4*n] [startIndex not equal 0] quarter of recent events outside eventFreshDuration cause cache shrinking", 778 eventCount: 5, 779 cacheCapacity: 5, 780 startIndex: 3, 781 lowerBoundCapacity: 5 / 2, 782 upperBoundCapacity: 5 * 2, 783 interval: eventFreshDuration + time.Second, 784 expectCapacity: 2, 785 expectStartIndex: 6, 786 }, 787 { 788 name: "[capacity not equals 4*n] [startIndex not equal 0] quarter of recent events outside eventFreshDuration cause cache shrinking with given lowerBoundCapacity", 789 eventCount: 5, 790 cacheCapacity: 5, 791 startIndex: 3, 792 lowerBoundCapacity: 3, 793 upperBoundCapacity: 5 * 2, 794 interval: eventFreshDuration + time.Second, 795 expectCapacity: 3, 796 expectStartIndex: 5, 797 }, 798 { 799 name: "[capacity not equals 4*n] [startIndex not equal 0] events inside eventFreshDuration cause cache expanding with given upperBoundCapacity", 800 eventCount: 5, 801 cacheCapacity: 5, 802 startIndex: 3, 803 lowerBoundCapacity: 5 / 2, 804 upperBoundCapacity: 8, 805 interval: eventFreshDuration / 6, 806 expectCapacity: 8, 807 expectStartIndex: 3, 808 }, 809 { 810 name: "[capacity equals 4*n] events inside eventFreshDuration cause cache expanding", 811 eventCount: 8, 812 cacheCapacity: 8, 813 lowerBoundCapacity: 8 / 2, 814 upperBoundCapacity: 8 * 2, 815 interval: eventFreshDuration / 9, 816 expectCapacity: 16, 817 expectStartIndex: 0, 818 }, 819 { 820 name: "[capacity equals 4*n] events outside eventFreshDuration without change cache capacity", 821 eventCount: 8, 822 cacheCapacity: 8, 823 lowerBoundCapacity: 8 / 2, 824 upperBoundCapacity: 8 * 2, 825 interval: eventFreshDuration / 8, 826 expectCapacity: 8, 827 expectStartIndex: 0, 828 }, 829 { 830 name: "[capacity equals 4*n] quarter of recent events outside eventFreshDuration cause cache shrinking", 831 eventCount: 8, 832 cacheCapacity: 8, 833 lowerBoundCapacity: 8 / 2, 834 upperBoundCapacity: 8 * 2, 835 interval: eventFreshDuration/2 + time.Second, 836 expectCapacity: 4, 837 expectStartIndex: 4, 838 }, 839 { 840 name: "[capacity equals 4*n] quarter of recent events outside eventFreshDuration cause cache shrinking with given lowerBoundCapacity", 841 eventCount: 8, 842 cacheCapacity: 8, 843 lowerBoundCapacity: 7, 844 upperBoundCapacity: 8 * 2, 845 interval: eventFreshDuration/2 + time.Second, 846 expectCapacity: 7, 847 expectStartIndex: 1, 848 }, 849 { 850 name: "[capacity equals 4*n] events inside eventFreshDuration cause cache expanding with given upperBoundCapacity", 851 eventCount: 8, 852 cacheCapacity: 8, 853 lowerBoundCapacity: 8 / 2, 854 upperBoundCapacity: 10, 855 interval: eventFreshDuration / 9, 856 expectCapacity: 10, 857 expectStartIndex: 0, 858 }, 859 { 860 name: "[capacity equals 4*n] [startIndex not equal 0] events inside eventFreshDuration cause cache expanding", 861 eventCount: 8, 862 cacheCapacity: 8, 863 startIndex: 3, 864 lowerBoundCapacity: 8 / 2, 865 upperBoundCapacity: 8 * 2, 866 interval: eventFreshDuration / 9, 867 expectCapacity: 16, 868 expectStartIndex: 3, 869 }, 870 { 871 name: "[capacity equals 4*n] [startIndex not equal 0] events outside eventFreshDuration without change cache capacity", 872 eventCount: 8, 873 cacheCapacity: 8, 874 startIndex: 3, 875 lowerBoundCapacity: 8 / 2, 876 upperBoundCapacity: 8 * 2, 877 interval: eventFreshDuration / 8, 878 expectCapacity: 8, 879 expectStartIndex: 3, 880 }, 881 { 882 name: "[capacity equals 4*n] [startIndex not equal 0] quarter of recent events outside eventFreshDuration cause cache shrinking", 883 eventCount: 8, 884 cacheCapacity: 8, 885 startIndex: 3, 886 lowerBoundCapacity: 8 / 2, 887 upperBoundCapacity: 8 * 2, 888 interval: eventFreshDuration/2 + time.Second, 889 expectCapacity: 4, 890 expectStartIndex: 7, 891 }, 892 { 893 name: "[capacity equals 4*n] [startIndex not equal 0] quarter of recent events outside eventFreshDuration cause cache shrinking with given lowerBoundCapacity", 894 eventCount: 8, 895 cacheCapacity: 8, 896 startIndex: 3, 897 lowerBoundCapacity: 7, 898 upperBoundCapacity: 8 * 2, 899 interval: eventFreshDuration/2 + time.Second, 900 expectCapacity: 7, 901 expectStartIndex: 4, 902 }, 903 { 904 name: "[capacity equals 4*n] [startIndex not equal 0] events inside eventFreshDuration cause cache expanding with given upperBoundCapacity", 905 eventCount: 8, 906 cacheCapacity: 8, 907 startIndex: 3, 908 lowerBoundCapacity: 8 / 2, 909 upperBoundCapacity: 10, 910 interval: eventFreshDuration / 9, 911 expectCapacity: 10, 912 expectStartIndex: 3, 913 }, 914 } 915 916 for _, test := range tests { 917 t.Run(test.name, func(t *testing.T) { 918 store := newTestWatchCache(test.cacheCapacity, &cache.Indexers{}) 919 defer store.Stop() 920 store.cache = make([]*watchCacheEvent, test.cacheCapacity) 921 store.startIndex = test.startIndex 922 store.lowerBoundCapacity = test.lowerBoundCapacity 923 store.upperBoundCapacity = test.upperBoundCapacity 924 loadEventWithDuration(store, test.eventCount, test.interval) 925 nextInterval := store.clock.Now().Add(time.Duration(test.interval.Nanoseconds() * int64(test.eventCount))) 926 store.resizeCacheLocked(nextInterval) 927 if store.capacity != test.expectCapacity { 928 t.Errorf("expect capacity %d, but get %d", test.expectCapacity, store.capacity) 929 } 930 931 // check cache's startIndex, endIndex and all elements. 932 if store.startIndex != test.expectStartIndex { 933 t.Errorf("expect startIndex %d, but get %d", test.expectStartIndex, store.startIndex) 934 } 935 if store.endIndex != test.startIndex+test.eventCount { 936 t.Errorf("expect endIndex %d get %d", test.startIndex+test.eventCount, store.endIndex) 937 } 938 if !checkCacheElements(store) { 939 t.Errorf("some elements locations in cache is wrong") 940 } 941 }) 942 } 943 } 944 945 func loadEventWithDuration(cache *testWatchCache, count int, interval time.Duration) { 946 for i := 0; i < count; i++ { 947 event := &watchCacheEvent{ 948 Key: fmt.Sprintf("event-%d", i+cache.startIndex), 949 RecordTime: cache.clock.Now().Add(time.Duration(interval.Nanoseconds() * int64(i))), 950 } 951 cache.cache[(i+cache.startIndex)%cache.capacity] = event 952 } 953 cache.endIndex = cache.startIndex + count 954 } 955 956 func checkCacheElements(cache *testWatchCache) bool { 957 for i := cache.startIndex; i < cache.endIndex; i++ { 958 location := i % cache.capacity 959 if cache.cache[location].Key != fmt.Sprintf("event-%d", i) { 960 return false 961 } 962 } 963 return true 964 } 965 966 func TestCacheIncreaseDoesNotBreakWatch(t *testing.T) { 967 store := newTestWatchCache(2, &cache.Indexers{}) 968 defer store.Stop() 969 970 now := store.clock.Now() 971 addEvent := func(key string, rv uint64, t time.Time) { 972 event := &watchCacheEvent{ 973 Key: key, 974 ResourceVersion: rv, 975 RecordTime: t, 976 } 977 store.updateCache(event) 978 } 979 980 // Initial LIST comes from the moment of RV=10. 981 store.Replace(nil, "10") 982 983 addEvent("key1", 20, now) 984 985 // Force "key1" to rotate our of cache. 986 later := now.Add(2 * eventFreshDuration) 987 addEvent("key2", 30, later) 988 addEvent("key3", 40, later) 989 990 // Force cache resize. 991 addEvent("key4", 50, later.Add(time.Second)) 992 993 _, err := store.getAllEventsSince(15, storage.ListOptions{}) 994 if err == nil || !strings.Contains(err.Error(), "too old resource version") { 995 t.Errorf("unexpected error: %v", err) 996 } 997 } 998 999 func TestSuggestedWatchChannelSize(t *testing.T) { 1000 testCases := []struct { 1001 name string 1002 capacity int 1003 indexExists bool 1004 triggerUsed bool 1005 expected int 1006 }{ 1007 { 1008 name: "capacity=100, indexExists, triggerUsed", 1009 capacity: 100, 1010 indexExists: true, 1011 triggerUsed: true, 1012 expected: 10, 1013 }, 1014 { 1015 name: "capacity=100, indexExists, !triggerUsed", 1016 capacity: 100, 1017 indexExists: true, 1018 triggerUsed: false, 1019 expected: 10, 1020 }, 1021 { 1022 name: "capacity=100, !indexExists", 1023 capacity: 100, 1024 indexExists: false, 1025 triggerUsed: false, 1026 expected: 10, 1027 }, 1028 { 1029 name: "capacity=750, indexExists, triggerUsed", 1030 capacity: 750, 1031 indexExists: true, 1032 triggerUsed: true, 1033 expected: 10, 1034 }, 1035 { 1036 name: "capacity=750, indexExists, !triggerUsed", 1037 capacity: 750, 1038 indexExists: true, 1039 triggerUsed: false, 1040 expected: 10, 1041 }, 1042 { 1043 name: "capacity=750, !indexExists", 1044 capacity: 750, 1045 indexExists: false, 1046 triggerUsed: false, 1047 expected: 10, 1048 }, 1049 { 1050 name: "capacity=7500, indexExists, triggerUsed", 1051 capacity: 7500, 1052 indexExists: true, 1053 triggerUsed: true, 1054 expected: 10, 1055 }, 1056 { 1057 name: "capacity=7500, indexExists, !triggerUsed", 1058 capacity: 7500, 1059 indexExists: true, 1060 triggerUsed: false, 1061 expected: 100, 1062 }, 1063 { 1064 name: "capacity=7500, !indexExists", 1065 capacity: 7500, 1066 indexExists: false, 1067 triggerUsed: false, 1068 expected: 100, 1069 }, 1070 { 1071 name: "capacity=75000, indexExists, triggerUsed", 1072 capacity: 75000, 1073 indexExists: true, 1074 triggerUsed: true, 1075 expected: 10, 1076 }, 1077 { 1078 name: "capacity=75000, indexExists, !triggerUsed", 1079 capacity: 75000, 1080 indexExists: true, 1081 triggerUsed: false, 1082 expected: 1000, 1083 }, 1084 { 1085 name: "capacity=75000, !indexExists", 1086 capacity: 75000, 1087 indexExists: false, 1088 triggerUsed: false, 1089 expected: 100, 1090 }, 1091 { 1092 name: "capacity=750000, indexExists, triggerUsed", 1093 capacity: 750000, 1094 indexExists: true, 1095 triggerUsed: true, 1096 expected: 10, 1097 }, 1098 { 1099 name: "capacity=750000, indexExists, !triggerUsed", 1100 capacity: 750000, 1101 indexExists: true, 1102 triggerUsed: false, 1103 expected: 1000, 1104 }, 1105 { 1106 name: "capacity=750000, !indexExists", 1107 capacity: 750000, 1108 indexExists: false, 1109 triggerUsed: false, 1110 expected: 100, 1111 }, 1112 } 1113 1114 for _, test := range testCases { 1115 t.Run(test.name, func(t *testing.T) { 1116 store := newTestWatchCache(test.capacity, &cache.Indexers{}) 1117 defer store.Stop() 1118 got := store.suggestedWatchChannelSize(test.indexExists, test.triggerUsed) 1119 if got != test.expected { 1120 t.Errorf("unexpected channel size got: %v, expected: %v", got, test.expected) 1121 } 1122 }) 1123 } 1124 } 1125 1126 func BenchmarkWatchCache_updateCache(b *testing.B) { 1127 store := newTestWatchCache(defaultUpperBoundCapacity, &cache.Indexers{}) 1128 defer store.Stop() 1129 store.cache = store.cache[:0] 1130 store.upperBoundCapacity = defaultUpperBoundCapacity 1131 loadEventWithDuration(store, defaultUpperBoundCapacity, 0) 1132 add := &watchCacheEvent{ 1133 Key: fmt.Sprintf("event-%d", defaultUpperBoundCapacity), 1134 RecordTime: store.clock.Now(), 1135 } 1136 b.ResetTimer() 1137 for i := 0; i < b.N; i++ { 1138 store.updateCache(add) 1139 } 1140 } 1141 1142 func TestHistogramCacheReadWait(t *testing.T) { 1143 registry := k8smetrics.NewKubeRegistry() 1144 if err := registry.Register(metrics.WatchCacheReadWait); err != nil { 1145 t.Errorf("unexpected error: %v", err) 1146 } 1147 ctx := context.Background() 1148 testedMetrics := "apiserver_watch_cache_read_wait_seconds" 1149 store := newTestWatchCache(2, &cache.Indexers{}) 1150 defer store.Stop() 1151 1152 // In background, update the store. 1153 go func() { 1154 if err := store.Add(makeTestPod("foo", 2)); err != nil { 1155 t.Errorf("unexpected error: %v", err) 1156 } 1157 if err := store.Add(makeTestPod("bar", 5)); err != nil { 1158 t.Errorf("unexpected error: %v", err) 1159 } 1160 }() 1161 1162 testCases := []struct { 1163 desc string 1164 resourceVersion uint64 1165 want string 1166 }{ 1167 { 1168 desc: "resourceVersion is non-zero", 1169 resourceVersion: 5, 1170 want: ` 1171 # HELP apiserver_watch_cache_read_wait_seconds [ALPHA] Histogram of time spent waiting for a watch cache to become fresh. 1172 # TYPE apiserver_watch_cache_read_wait_seconds histogram 1173 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.005"} 1 1174 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.025"} 1 1175 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.05"} 1 1176 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.1"} 1 1177 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.2"} 1 1178 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.4"} 1 1179 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.6"} 1 1180 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.8"} 1 1181 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1"} 1 1182 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.25"} 1 1183 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.5"} 1 1184 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="2"} 1 1185 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="3"} 1 1186 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="+Inf"} 1 1187 apiserver_watch_cache_read_wait_seconds_sum{resource="pods"} 0 1188 apiserver_watch_cache_read_wait_seconds_count{resource="pods"} 1 1189 `, 1190 }, 1191 { 1192 desc: "resourceVersion is 0", 1193 resourceVersion: 0, 1194 want: ``, 1195 }, 1196 } 1197 1198 for _, test := range testCases { 1199 t.Run(test.desc, func(t *testing.T) { 1200 defer registry.Reset() 1201 if _, _, _, err := store.WaitUntilFreshAndGet(ctx, test.resourceVersion, "prefix/ns/bar"); err != nil { 1202 t.Errorf("unexpected error: %v", err) 1203 } 1204 if err := testutil.GatherAndCompare(registry, strings.NewReader(test.want), testedMetrics); err != nil { 1205 t.Errorf("unexpected error: %v", err) 1206 } 1207 }) 1208 } 1209 }