k8s.io/apiserver@v0.31.1/pkg/storage/cacher/watch_cache_test.go (about)

     1  /*
     2  Copyright 2014 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package cacher
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strconv"
    23  	"strings"
    24  	"testing"
    25  	"time"
    26  
    27  	v1 "k8s.io/api/core/v1"
    28  	apiequality "k8s.io/apimachinery/pkg/api/equality"
    29  	"k8s.io/apimachinery/pkg/api/errors"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/fields"
    32  	"k8s.io/apimachinery/pkg/labels"
    33  	"k8s.io/apimachinery/pkg/runtime"
    34  	"k8s.io/apimachinery/pkg/runtime/schema"
    35  	"k8s.io/apimachinery/pkg/util/wait"
    36  	"k8s.io/apimachinery/pkg/watch"
    37  	"k8s.io/apiserver/pkg/features"
    38  	"k8s.io/apiserver/pkg/storage"
    39  	"k8s.io/apiserver/pkg/storage/cacher/metrics"
    40  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    41  	"k8s.io/client-go/tools/cache"
    42  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    43  	k8smetrics "k8s.io/component-base/metrics"
    44  	"k8s.io/component-base/metrics/testutil"
    45  	"k8s.io/utils/clock"
    46  	testingclock "k8s.io/utils/clock/testing"
    47  )
    48  
    49  func makeTestPod(name string, resourceVersion uint64) *v1.Pod {
    50  	return makeTestPodDetails(name, resourceVersion, "some-node", map[string]string{"k8s-app": "my-app"})
    51  }
    52  
    53  func makeTestPodDetails(name string, resourceVersion uint64, nodeName string, labels map[string]string) *v1.Pod {
    54  	return &v1.Pod{
    55  		ObjectMeta: metav1.ObjectMeta{
    56  			Namespace:       "ns",
    57  			Name:            name,
    58  			ResourceVersion: strconv.FormatUint(resourceVersion, 10),
    59  			Labels:          labels,
    60  		},
    61  		Spec: v1.PodSpec{
    62  			NodeName: nodeName,
    63  		},
    64  	}
    65  }
    66  
    67  func makeTestStoreElement(pod *v1.Pod) *storeElement {
    68  	return &storeElement{
    69  		Key:    "prefix/ns/" + pod.Name,
    70  		Object: pod,
    71  		Labels: labels.Set(pod.Labels),
    72  		Fields: fields.Set{"spec.nodeName": pod.Spec.NodeName},
    73  	}
    74  }
    75  
    76  type testWatchCache struct {
    77  	*watchCache
    78  
    79  	bookmarkRevision chan int64
    80  	stopCh           chan struct{}
    81  }
    82  
    83  func (w *testWatchCache) getAllEventsSince(resourceVersion uint64, opts storage.ListOptions) ([]*watchCacheEvent, error) {
    84  	cacheInterval, err := w.getCacheIntervalForEvents(resourceVersion, opts)
    85  	if err != nil {
    86  		return nil, err
    87  	}
    88  
    89  	result := []*watchCacheEvent{}
    90  	for {
    91  		event, err := cacheInterval.Next()
    92  		if err != nil {
    93  			return nil, err
    94  		}
    95  		if event == nil {
    96  			break
    97  		}
    98  		result = append(result, event)
    99  	}
   100  
   101  	return result, nil
   102  }
   103  
   104  func (w *testWatchCache) getCacheIntervalForEvents(resourceVersion uint64, opts storage.ListOptions) (*watchCacheInterval, error) {
   105  	w.RLock()
   106  	defer w.RUnlock()
   107  
   108  	return w.getAllEventsSinceLocked(resourceVersion, "", opts)
   109  }
   110  
   111  // newTestWatchCache just adds a fake clock.
   112  func newTestWatchCache(capacity int, indexers *cache.Indexers) *testWatchCache {
   113  	keyFunc := func(obj runtime.Object) (string, error) {
   114  		return storage.NamespaceKeyFunc("prefix", obj)
   115  	}
   116  	getAttrsFunc := func(obj runtime.Object) (labels.Set, fields.Set, error) {
   117  		pod, ok := obj.(*v1.Pod)
   118  		if !ok {
   119  			return nil, nil, fmt.Errorf("not a pod")
   120  		}
   121  		return labels.Set(pod.Labels), fields.Set{"spec.nodeName": pod.Spec.NodeName}, nil
   122  	}
   123  	versioner := storage.APIObjectVersioner{}
   124  	mockHandler := func(*watchCacheEvent) {}
   125  	wc := &testWatchCache{}
   126  	wc.bookmarkRevision = make(chan int64, 1)
   127  	wc.stopCh = make(chan struct{})
   128  	pr := newConditionalProgressRequester(wc.RequestWatchProgress, &immediateTickerFactory{}, nil)
   129  	go pr.Run(wc.stopCh)
   130  	wc.watchCache = newWatchCache(keyFunc, mockHandler, getAttrsFunc, versioner, indexers, testingclock.NewFakeClock(time.Now()), schema.GroupResource{Resource: "pods"}, pr)
   131  	// To preserve behavior of tests that assume a given capacity,
   132  	// resize it to th expected size.
   133  	wc.capacity = capacity
   134  	wc.cache = make([]*watchCacheEvent, capacity)
   135  	wc.lowerBoundCapacity = min(capacity, defaultLowerBoundCapacity)
   136  	wc.upperBoundCapacity = max(capacity, defaultUpperBoundCapacity)
   137  
   138  	return wc
   139  }
   140  
   141  type immediateTickerFactory struct{}
   142  
   143  func (t *immediateTickerFactory) NewTimer(d time.Duration) clock.Timer {
   144  	timer := immediateTicker{
   145  		c: make(chan time.Time),
   146  	}
   147  	timer.Reset(d)
   148  	return &timer
   149  }
   150  
   151  type immediateTicker struct {
   152  	c chan time.Time
   153  }
   154  
   155  func (t *immediateTicker) Reset(d time.Duration) (active bool) {
   156  	select {
   157  	case <-t.c:
   158  		active = true
   159  	default:
   160  	}
   161  	go func() {
   162  		t.c <- time.Now()
   163  	}()
   164  	return active
   165  }
   166  
   167  func (t *immediateTicker) C() <-chan time.Time {
   168  	return t.c
   169  }
   170  
   171  func (t *immediateTicker) Stop() bool {
   172  	select {
   173  	case <-t.c:
   174  		return true
   175  	default:
   176  		return false
   177  	}
   178  }
   179  
   180  func (w *testWatchCache) RequestWatchProgress(ctx context.Context) error {
   181  	go func() {
   182  		select {
   183  		case rev := <-w.bookmarkRevision:
   184  			w.UpdateResourceVersion(fmt.Sprintf("%d", rev))
   185  		case <-ctx.Done():
   186  			return
   187  		}
   188  	}()
   189  	return nil
   190  }
   191  
   192  func (w *testWatchCache) Stop() {
   193  	close(w.stopCh)
   194  }
   195  
   196  func TestWatchCacheBasic(t *testing.T) {
   197  	store := newTestWatchCache(2, &cache.Indexers{})
   198  	defer store.Stop()
   199  
   200  	// Test Add/Update/Delete.
   201  	pod1 := makeTestPod("pod", 1)
   202  	if err := store.Add(pod1); err != nil {
   203  		t.Errorf("unexpected error: %v", err)
   204  	}
   205  	if item, ok, _ := store.Get(pod1); !ok {
   206  		t.Errorf("didn't find pod")
   207  	} else {
   208  		expected := makeTestStoreElement(makeTestPod("pod", 1))
   209  		if !apiequality.Semantic.DeepEqual(expected, item) {
   210  			t.Errorf("expected %v, got %v", expected, item)
   211  		}
   212  	}
   213  	pod2 := makeTestPod("pod", 2)
   214  	if err := store.Update(pod2); err != nil {
   215  		t.Errorf("unexpected error: %v", err)
   216  	}
   217  	if item, ok, _ := store.Get(pod2); !ok {
   218  		t.Errorf("didn't find pod")
   219  	} else {
   220  		expected := makeTestStoreElement(makeTestPod("pod", 2))
   221  		if !apiequality.Semantic.DeepEqual(expected, item) {
   222  			t.Errorf("expected %v, got %v", expected, item)
   223  		}
   224  	}
   225  	pod3 := makeTestPod("pod", 3)
   226  	if err := store.Delete(pod3); err != nil {
   227  		t.Errorf("unexpected error: %v", err)
   228  	}
   229  	if _, ok, _ := store.Get(pod3); ok {
   230  		t.Errorf("found pod")
   231  	}
   232  
   233  	// Test List.
   234  	store.Add(makeTestPod("pod1", 4))
   235  	store.Add(makeTestPod("pod2", 5))
   236  	store.Add(makeTestPod("pod3", 6))
   237  	{
   238  		expected := map[string]storeElement{
   239  			"prefix/ns/pod1": *makeTestStoreElement(makeTestPod("pod1", 4)),
   240  			"prefix/ns/pod2": *makeTestStoreElement(makeTestPod("pod2", 5)),
   241  			"prefix/ns/pod3": *makeTestStoreElement(makeTestPod("pod3", 6)),
   242  		}
   243  		items := make(map[string]storeElement)
   244  		for _, item := range store.List() {
   245  			elem := item.(*storeElement)
   246  			items[elem.Key] = *elem
   247  		}
   248  		if !apiequality.Semantic.DeepEqual(expected, items) {
   249  			t.Errorf("expected %v, got %v", expected, items)
   250  		}
   251  	}
   252  
   253  	// Test Replace.
   254  	store.Replace([]interface{}{
   255  		makeTestPod("pod4", 7),
   256  		makeTestPod("pod5", 8),
   257  	}, "8")
   258  	{
   259  		expected := map[string]storeElement{
   260  			"prefix/ns/pod4": *makeTestStoreElement(makeTestPod("pod4", 7)),
   261  			"prefix/ns/pod5": *makeTestStoreElement(makeTestPod("pod5", 8)),
   262  		}
   263  		items := make(map[string]storeElement)
   264  		for _, item := range store.List() {
   265  			elem := item.(*storeElement)
   266  			items[elem.Key] = *elem
   267  		}
   268  		if !apiequality.Semantic.DeepEqual(expected, items) {
   269  			t.Errorf("expected %v, got %v", expected, items)
   270  		}
   271  	}
   272  }
   273  
   274  func TestEvents(t *testing.T) {
   275  	store := newTestWatchCache(5, &cache.Indexers{})
   276  	defer store.Stop()
   277  
   278  	// no dynamic-size cache to fit old tests.
   279  	store.lowerBoundCapacity = 5
   280  	store.upperBoundCapacity = 5
   281  
   282  	store.Add(makeTestPod("pod", 3))
   283  
   284  	// Test for Added event.
   285  	{
   286  		_, err := store.getAllEventsSince(1, storage.ListOptions{})
   287  		if err == nil {
   288  			t.Errorf("expected error too old")
   289  		}
   290  		if _, ok := err.(*errors.StatusError); !ok {
   291  			t.Errorf("expected error to be of type StatusError")
   292  		}
   293  	}
   294  	{
   295  		result, err := store.getAllEventsSince(2, storage.ListOptions{})
   296  		if err != nil {
   297  			t.Errorf("unexpected error: %v", err)
   298  		}
   299  		if len(result) != 1 {
   300  			t.Fatalf("unexpected events: %v", result)
   301  		}
   302  		if result[0].Type != watch.Added {
   303  			t.Errorf("unexpected event type: %v", result[0].Type)
   304  		}
   305  		pod := makeTestPod("pod", uint64(3))
   306  		if !apiequality.Semantic.DeepEqual(pod, result[0].Object) {
   307  			t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod)
   308  		}
   309  		if result[0].PrevObject != nil {
   310  			t.Errorf("unexpected item: %v", result[0].PrevObject)
   311  		}
   312  	}
   313  
   314  	store.Update(makeTestPod("pod", 4))
   315  	store.Update(makeTestPod("pod", 5))
   316  
   317  	// Test with not full cache.
   318  	{
   319  		_, err := store.getAllEventsSince(1, storage.ListOptions{})
   320  		if err == nil {
   321  			t.Errorf("expected error too old")
   322  		}
   323  	}
   324  	{
   325  		result, err := store.getAllEventsSince(3, storage.ListOptions{})
   326  		if err != nil {
   327  			t.Errorf("unexpected error: %v", err)
   328  		}
   329  		if len(result) != 2 {
   330  			t.Fatalf("unexpected events: %v", result)
   331  		}
   332  		for i := 0; i < 2; i++ {
   333  			if result[i].Type != watch.Modified {
   334  				t.Errorf("unexpected event type: %v", result[i].Type)
   335  			}
   336  			pod := makeTestPod("pod", uint64(i+4))
   337  			if !apiequality.Semantic.DeepEqual(pod, result[i].Object) {
   338  				t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod)
   339  			}
   340  			prevPod := makeTestPod("pod", uint64(i+3))
   341  			if !apiequality.Semantic.DeepEqual(prevPod, result[i].PrevObject) {
   342  				t.Errorf("unexpected item: %v, expected: %v", result[i].PrevObject, prevPod)
   343  			}
   344  		}
   345  	}
   346  
   347  	for i := 6; i < 10; i++ {
   348  		store.Update(makeTestPod("pod", uint64(i)))
   349  	}
   350  
   351  	// Test with full cache - there should be elements from 5 to 9.
   352  	{
   353  		_, err := store.getAllEventsSince(3, storage.ListOptions{})
   354  		if err == nil {
   355  			t.Errorf("expected error too old")
   356  		}
   357  	}
   358  	{
   359  		result, err := store.getAllEventsSince(4, storage.ListOptions{})
   360  		if err != nil {
   361  			t.Errorf("unexpected error: %v", err)
   362  		}
   363  		if len(result) != 5 {
   364  			t.Fatalf("unexpected events: %v", result)
   365  		}
   366  		for i := 0; i < 5; i++ {
   367  			pod := makeTestPod("pod", uint64(i+5))
   368  			if !apiequality.Semantic.DeepEqual(pod, result[i].Object) {
   369  				t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod)
   370  			}
   371  		}
   372  	}
   373  
   374  	// Test for delete event.
   375  	store.Delete(makeTestPod("pod", uint64(10)))
   376  
   377  	{
   378  		result, err := store.getAllEventsSince(9, storage.ListOptions{})
   379  		if err != nil {
   380  			t.Errorf("unexpected error: %v", err)
   381  		}
   382  		if len(result) != 1 {
   383  			t.Fatalf("unexpected events: %v", result)
   384  		}
   385  		if result[0].Type != watch.Deleted {
   386  			t.Errorf("unexpected event type: %v", result[0].Type)
   387  		}
   388  		pod := makeTestPod("pod", uint64(10))
   389  		if !apiequality.Semantic.DeepEqual(pod, result[0].Object) {
   390  			t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod)
   391  		}
   392  		prevPod := makeTestPod("pod", uint64(9))
   393  		if !apiequality.Semantic.DeepEqual(prevPod, result[0].PrevObject) {
   394  			t.Errorf("unexpected item: %v, expected: %v", result[0].PrevObject, prevPod)
   395  		}
   396  	}
   397  }
   398  
   399  func TestMarker(t *testing.T) {
   400  	store := newTestWatchCache(3, &cache.Indexers{})
   401  	defer store.Stop()
   402  
   403  	// First thing that is called when propagated from storage is Replace.
   404  	store.Replace([]interface{}{
   405  		makeTestPod("pod1", 5),
   406  		makeTestPod("pod2", 9),
   407  	}, "9")
   408  
   409  	_, err := store.getAllEventsSince(8, storage.ListOptions{})
   410  	if err == nil || !strings.Contains(err.Error(), "too old resource version") {
   411  		t.Errorf("unexpected error: %v", err)
   412  	}
   413  	// Getting events from 8 should return no events,
   414  	// even though there is a marker there.
   415  	result, err := store.getAllEventsSince(9, storage.ListOptions{})
   416  	if err != nil {
   417  		t.Fatalf("unexpected error: %v", err)
   418  	}
   419  	if len(result) != 0 {
   420  		t.Errorf("unexpected result: %#v, expected no events", result)
   421  	}
   422  
   423  	pod := makeTestPod("pods", 12)
   424  	store.Add(pod)
   425  	// Getting events from 8 should still work and return one event.
   426  	result, err = store.getAllEventsSince(9, storage.ListOptions{})
   427  	if err != nil {
   428  		t.Fatalf("unexpected error: %v", err)
   429  	}
   430  	if len(result) != 1 || !apiequality.Semantic.DeepEqual(result[0].Object, pod) {
   431  		t.Errorf("unexpected result: %#v, expected %v", result, pod)
   432  	}
   433  }
   434  
   435  func TestWaitUntilFreshAndList(t *testing.T) {
   436  	ctx := context.Background()
   437  	store := newTestWatchCache(3, &cache.Indexers{
   438  		"l:label": func(obj interface{}) ([]string, error) {
   439  			pod, ok := obj.(*v1.Pod)
   440  			if !ok {
   441  				return nil, fmt.Errorf("not a pod %#v", obj)
   442  			}
   443  			if value, ok := pod.Labels["label"]; ok {
   444  				return []string{value}, nil
   445  			}
   446  			return nil, nil
   447  		},
   448  		"f:spec.nodeName": func(obj interface{}) ([]string, error) {
   449  			pod, ok := obj.(*v1.Pod)
   450  			if !ok {
   451  				return nil, fmt.Errorf("not a pod %#v", obj)
   452  			}
   453  			return []string{pod.Spec.NodeName}, nil
   454  		},
   455  	})
   456  	defer store.Stop()
   457  	// In background, update the store.
   458  	go func() {
   459  		store.Add(makeTestPodDetails("pod1", 2, "node1", map[string]string{"label": "value1"}))
   460  		store.Add(makeTestPodDetails("pod2", 3, "node1", map[string]string{"label": "value1"}))
   461  		store.Add(makeTestPodDetails("pod3", 5, "node2", map[string]string{"label": "value2"}))
   462  	}()
   463  
   464  	// list by empty MatchValues.
   465  	list, resourceVersion, indexUsed, err := store.WaitUntilFreshAndList(ctx, 5, "prefix/", nil)
   466  	if err != nil {
   467  		t.Fatalf("unexpected error: %v", err)
   468  	}
   469  	if resourceVersion != 5 {
   470  		t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion)
   471  	}
   472  	if len(list) != 3 {
   473  		t.Errorf("unexpected list returned: %#v", list)
   474  	}
   475  	if indexUsed != "" {
   476  		t.Errorf("Used index %q but expected none to be used", indexUsed)
   477  	}
   478  
   479  	// list by label index.
   480  	matchValues := []storage.MatchValue{
   481  		{IndexName: "l:label", Value: "value1"},
   482  		{IndexName: "f:spec.nodeName", Value: "node2"},
   483  	}
   484  	list, resourceVersion, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", matchValues)
   485  	if err != nil {
   486  		t.Fatalf("unexpected error: %v", err)
   487  	}
   488  	if resourceVersion != 5 {
   489  		t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion)
   490  	}
   491  	if len(list) != 2 {
   492  		t.Errorf("unexpected list returned: %#v", list)
   493  	}
   494  	if indexUsed != "l:label" {
   495  		t.Errorf("Used index %q but expected %q", indexUsed, "l:label")
   496  	}
   497  
   498  	// list with spec.nodeName index.
   499  	matchValues = []storage.MatchValue{
   500  		{IndexName: "l:not-exist-label", Value: "whatever"},
   501  		{IndexName: "f:spec.nodeName", Value: "node2"},
   502  	}
   503  	list, resourceVersion, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", matchValues)
   504  	if err != nil {
   505  		t.Fatalf("unexpected error: %v", err)
   506  	}
   507  	if resourceVersion != 5 {
   508  		t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion)
   509  	}
   510  	if len(list) != 1 {
   511  		t.Errorf("unexpected list returned: %#v", list)
   512  	}
   513  	if indexUsed != "f:spec.nodeName" {
   514  		t.Errorf("Used index %q but expected %q", indexUsed, "f:spec.nodeName")
   515  	}
   516  
   517  	// list with index not exists.
   518  	matchValues = []storage.MatchValue{
   519  		{IndexName: "l:not-exist-label", Value: "whatever"},
   520  	}
   521  	list, resourceVersion, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", matchValues)
   522  	if err != nil {
   523  		t.Fatalf("unexpected error: %v", err)
   524  	}
   525  	if resourceVersion != 5 {
   526  		t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion)
   527  	}
   528  	if len(list) != 3 {
   529  		t.Errorf("unexpected list returned: %#v", list)
   530  	}
   531  	if indexUsed != "" {
   532  		t.Errorf("Used index %q but expected none to be used", indexUsed)
   533  	}
   534  }
   535  
   536  func TestWaitUntilFreshAndListFromCache(t *testing.T) {
   537  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true)
   538  	forceRequestWatchProgressSupport(t)
   539  	ctx := context.Background()
   540  	store := newTestWatchCache(3, &cache.Indexers{})
   541  	defer store.Stop()
   542  	// In background, update the store.
   543  	go func() {
   544  		store.Add(makeTestPod("pod1", 2))
   545  		store.bookmarkRevision <- 3
   546  	}()
   547  
   548  	// list from future revision. Requires watch cache to request bookmark to get it.
   549  	list, resourceVersion, indexUsed, err := store.WaitUntilFreshAndList(ctx, 3, "prefix/", nil)
   550  	if err != nil {
   551  		t.Fatalf("unexpected error: %v", err)
   552  	}
   553  	if resourceVersion != 3 {
   554  		t.Errorf("unexpected resourceVersion: %v, expected: 6", resourceVersion)
   555  	}
   556  	if len(list) != 1 {
   557  		t.Errorf("unexpected list returned: %#v", list)
   558  	}
   559  	if indexUsed != "" {
   560  		t.Errorf("Used index %q but expected none to be used", indexUsed)
   561  	}
   562  }
   563  
   564  func TestWaitUntilFreshAndGet(t *testing.T) {
   565  	ctx := context.Background()
   566  	store := newTestWatchCache(3, &cache.Indexers{})
   567  	defer store.Stop()
   568  
   569  	// In background, update the store.
   570  	go func() {
   571  		store.Add(makeTestPod("foo", 2))
   572  		store.Add(makeTestPod("bar", 5))
   573  	}()
   574  
   575  	obj, exists, resourceVersion, err := store.WaitUntilFreshAndGet(ctx, 5, "prefix/ns/bar")
   576  	if err != nil {
   577  		t.Fatalf("unexpected error: %v", err)
   578  	}
   579  	if resourceVersion != 5 {
   580  		t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion)
   581  	}
   582  	if !exists {
   583  		t.Fatalf("no results returned: %#v", obj)
   584  	}
   585  	expected := makeTestStoreElement(makeTestPod("bar", 5))
   586  	if !apiequality.Semantic.DeepEqual(expected, obj) {
   587  		t.Errorf("expected %v, got %v", expected, obj)
   588  	}
   589  }
   590  
   591  func TestWaitUntilFreshAndListTimeout(t *testing.T) {
   592  	tcs := []struct {
   593  		name                    string
   594  		ConsistentListFromCache bool
   595  	}{
   596  		{
   597  			name:                    "FromStorage",
   598  			ConsistentListFromCache: false,
   599  		},
   600  		{
   601  			name:                    "FromCache",
   602  			ConsistentListFromCache: true,
   603  		},
   604  	}
   605  	for _, tc := range tcs {
   606  		t.Run(tc.name, func(t *testing.T) {
   607  			featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, tc.ConsistentListFromCache)
   608  			ctx := context.Background()
   609  			store := newTestWatchCache(3, &cache.Indexers{})
   610  			defer store.Stop()
   611  			fc := store.clock.(*testingclock.FakeClock)
   612  
   613  			// In background, step clock after the below call starts the timer.
   614  			go func() {
   615  				for !fc.HasWaiters() {
   616  					time.Sleep(time.Millisecond)
   617  				}
   618  				store.Add(makeTestPod("foo", 2))
   619  				store.bookmarkRevision <- 3
   620  				fc.Step(blockTimeout)
   621  
   622  				// Add an object to make sure the test would
   623  				// eventually fail instead of just waiting
   624  				// forever.
   625  				time.Sleep(30 * time.Second)
   626  				store.Add(makeTestPod("bar", 4))
   627  			}()
   628  
   629  			_, _, _, err := store.WaitUntilFreshAndList(ctx, 4, "", nil)
   630  			if !errors.IsTimeout(err) {
   631  				t.Errorf("expected timeout error but got: %v", err)
   632  			}
   633  			if !storage.IsTooLargeResourceVersion(err) {
   634  				t.Errorf("expected 'Too large resource version' cause in error but got: %v", err)
   635  			}
   636  		})
   637  	}
   638  }
   639  
   640  type testLW struct {
   641  	ListFunc  func(options metav1.ListOptions) (runtime.Object, error)
   642  	WatchFunc func(options metav1.ListOptions) (watch.Interface, error)
   643  }
   644  
   645  func (t *testLW) List(options metav1.ListOptions) (runtime.Object, error) {
   646  	return t.ListFunc(options)
   647  }
   648  func (t *testLW) Watch(options metav1.ListOptions) (watch.Interface, error) {
   649  	return t.WatchFunc(options)
   650  }
   651  
   652  func TestReflectorForWatchCache(t *testing.T) {
   653  	ctx := context.Background()
   654  	store := newTestWatchCache(5, &cache.Indexers{})
   655  	defer store.Stop()
   656  
   657  	{
   658  		_, version, _, err := store.WaitUntilFreshAndList(ctx, 0, "", nil)
   659  		if err != nil {
   660  			t.Fatalf("unexpected error: %v", err)
   661  		}
   662  		if version != 0 {
   663  			t.Errorf("unexpected resource version: %d", version)
   664  		}
   665  	}
   666  
   667  	lw := &testLW{
   668  		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
   669  			fw := watch.NewFake()
   670  			go fw.Stop()
   671  			return fw, nil
   672  		},
   673  		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
   674  			return &v1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "10"}}, nil
   675  		},
   676  	}
   677  	r := cache.NewReflector(lw, &v1.Pod{}, store, 0)
   678  	r.ListAndWatch(wait.NeverStop)
   679  
   680  	{
   681  		_, version, _, err := store.WaitUntilFreshAndList(ctx, 10, "", nil)
   682  		if err != nil {
   683  			t.Fatalf("unexpected error: %v", err)
   684  		}
   685  		if version != 10 {
   686  			t.Errorf("unexpected resource version: %d", version)
   687  		}
   688  	}
   689  }
   690  
   691  func TestDynamicCache(t *testing.T) {
   692  	tests := []struct {
   693  		name          string
   694  		eventCount    int
   695  		cacheCapacity int
   696  		startIndex    int
   697  		// interval is time duration between adjacent events.
   698  		lowerBoundCapacity int
   699  		upperBoundCapacity int
   700  		interval           time.Duration
   701  		expectCapacity     int
   702  		expectStartIndex   int
   703  	}{
   704  		{
   705  			name:               "[capacity not equals 4*n] events inside eventFreshDuration cause cache expanding",
   706  			eventCount:         5,
   707  			cacheCapacity:      5,
   708  			lowerBoundCapacity: 5 / 2,
   709  			upperBoundCapacity: 5 * 2,
   710  			interval:           eventFreshDuration / 6,
   711  			expectCapacity:     10,
   712  			expectStartIndex:   0,
   713  		},
   714  		{
   715  			name:               "[capacity not equals 4*n] events outside eventFreshDuration without change cache capacity",
   716  			eventCount:         5,
   717  			cacheCapacity:      5,
   718  			lowerBoundCapacity: 5 / 2,
   719  			upperBoundCapacity: 5 * 2,
   720  			interval:           eventFreshDuration / 4,
   721  			expectCapacity:     5,
   722  			expectStartIndex:   0,
   723  		},
   724  		{
   725  			name:               "[capacity not equals 4*n] quarter of recent events outside eventFreshDuration cause cache shrinking",
   726  			eventCount:         5,
   727  			cacheCapacity:      5,
   728  			lowerBoundCapacity: 5 / 2,
   729  			upperBoundCapacity: 5 * 2,
   730  			interval:           eventFreshDuration + time.Second,
   731  			expectCapacity:     2,
   732  			expectStartIndex:   3,
   733  		},
   734  		{
   735  			name:               "[capacity not equals 4*n] quarter of recent events outside eventFreshDuration cause cache shrinking with given lowerBoundCapacity",
   736  			eventCount:         5,
   737  			cacheCapacity:      5,
   738  			lowerBoundCapacity: 3,
   739  			upperBoundCapacity: 5 * 2,
   740  			interval:           eventFreshDuration + time.Second,
   741  			expectCapacity:     3,
   742  			expectStartIndex:   2,
   743  		},
   744  		{
   745  			name:               "[capacity not equals 4*n] events inside eventFreshDuration cause cache expanding with given upperBoundCapacity",
   746  			eventCount:         5,
   747  			cacheCapacity:      5,
   748  			lowerBoundCapacity: 5 / 2,
   749  			upperBoundCapacity: 8,
   750  			interval:           eventFreshDuration / 6,
   751  			expectCapacity:     8,
   752  			expectStartIndex:   0,
   753  		},
   754  		{
   755  			name:               "[capacity not equals 4*n] [startIndex not equal 0] events inside eventFreshDuration cause cache expanding",
   756  			eventCount:         5,
   757  			cacheCapacity:      5,
   758  			startIndex:         3,
   759  			lowerBoundCapacity: 5 / 2,
   760  			upperBoundCapacity: 5 * 2,
   761  			interval:           eventFreshDuration / 6,
   762  			expectCapacity:     10,
   763  			expectStartIndex:   3,
   764  		},
   765  		{
   766  			name:               "[capacity not equals 4*n] [startIndex not equal 0] events outside eventFreshDuration without change cache capacity",
   767  			eventCount:         5,
   768  			cacheCapacity:      5,
   769  			startIndex:         3,
   770  			lowerBoundCapacity: 5 / 2,
   771  			upperBoundCapacity: 5 * 2,
   772  			interval:           eventFreshDuration / 4,
   773  			expectCapacity:     5,
   774  			expectStartIndex:   3,
   775  		},
   776  		{
   777  			name:               "[capacity not equals 4*n] [startIndex not equal 0] quarter of recent events outside eventFreshDuration cause cache shrinking",
   778  			eventCount:         5,
   779  			cacheCapacity:      5,
   780  			startIndex:         3,
   781  			lowerBoundCapacity: 5 / 2,
   782  			upperBoundCapacity: 5 * 2,
   783  			interval:           eventFreshDuration + time.Second,
   784  			expectCapacity:     2,
   785  			expectStartIndex:   6,
   786  		},
   787  		{
   788  			name:               "[capacity not equals 4*n] [startIndex not equal 0] quarter of recent events outside eventFreshDuration cause cache shrinking with given lowerBoundCapacity",
   789  			eventCount:         5,
   790  			cacheCapacity:      5,
   791  			startIndex:         3,
   792  			lowerBoundCapacity: 3,
   793  			upperBoundCapacity: 5 * 2,
   794  			interval:           eventFreshDuration + time.Second,
   795  			expectCapacity:     3,
   796  			expectStartIndex:   5,
   797  		},
   798  		{
   799  			name:               "[capacity not equals 4*n] [startIndex not equal 0] events inside eventFreshDuration cause cache expanding with given upperBoundCapacity",
   800  			eventCount:         5,
   801  			cacheCapacity:      5,
   802  			startIndex:         3,
   803  			lowerBoundCapacity: 5 / 2,
   804  			upperBoundCapacity: 8,
   805  			interval:           eventFreshDuration / 6,
   806  			expectCapacity:     8,
   807  			expectStartIndex:   3,
   808  		},
   809  		{
   810  			name:               "[capacity equals 4*n] events inside eventFreshDuration cause cache expanding",
   811  			eventCount:         8,
   812  			cacheCapacity:      8,
   813  			lowerBoundCapacity: 8 / 2,
   814  			upperBoundCapacity: 8 * 2,
   815  			interval:           eventFreshDuration / 9,
   816  			expectCapacity:     16,
   817  			expectStartIndex:   0,
   818  		},
   819  		{
   820  			name:               "[capacity equals 4*n] events outside eventFreshDuration without change cache capacity",
   821  			eventCount:         8,
   822  			cacheCapacity:      8,
   823  			lowerBoundCapacity: 8 / 2,
   824  			upperBoundCapacity: 8 * 2,
   825  			interval:           eventFreshDuration / 8,
   826  			expectCapacity:     8,
   827  			expectStartIndex:   0,
   828  		},
   829  		{
   830  			name:               "[capacity equals 4*n] quarter of recent events outside eventFreshDuration cause cache shrinking",
   831  			eventCount:         8,
   832  			cacheCapacity:      8,
   833  			lowerBoundCapacity: 8 / 2,
   834  			upperBoundCapacity: 8 * 2,
   835  			interval:           eventFreshDuration/2 + time.Second,
   836  			expectCapacity:     4,
   837  			expectStartIndex:   4,
   838  		},
   839  		{
   840  			name:               "[capacity equals 4*n] quarter of recent events outside eventFreshDuration cause cache shrinking with given lowerBoundCapacity",
   841  			eventCount:         8,
   842  			cacheCapacity:      8,
   843  			lowerBoundCapacity: 7,
   844  			upperBoundCapacity: 8 * 2,
   845  			interval:           eventFreshDuration/2 + time.Second,
   846  			expectCapacity:     7,
   847  			expectStartIndex:   1,
   848  		},
   849  		{
   850  			name:               "[capacity equals 4*n] events inside eventFreshDuration cause cache expanding with given upperBoundCapacity",
   851  			eventCount:         8,
   852  			cacheCapacity:      8,
   853  			lowerBoundCapacity: 8 / 2,
   854  			upperBoundCapacity: 10,
   855  			interval:           eventFreshDuration / 9,
   856  			expectCapacity:     10,
   857  			expectStartIndex:   0,
   858  		},
   859  		{
   860  			name:               "[capacity equals 4*n] [startIndex not equal 0] events inside eventFreshDuration cause cache expanding",
   861  			eventCount:         8,
   862  			cacheCapacity:      8,
   863  			startIndex:         3,
   864  			lowerBoundCapacity: 8 / 2,
   865  			upperBoundCapacity: 8 * 2,
   866  			interval:           eventFreshDuration / 9,
   867  			expectCapacity:     16,
   868  			expectStartIndex:   3,
   869  		},
   870  		{
   871  			name:               "[capacity equals 4*n] [startIndex not equal 0] events outside eventFreshDuration without change cache capacity",
   872  			eventCount:         8,
   873  			cacheCapacity:      8,
   874  			startIndex:         3,
   875  			lowerBoundCapacity: 8 / 2,
   876  			upperBoundCapacity: 8 * 2,
   877  			interval:           eventFreshDuration / 8,
   878  			expectCapacity:     8,
   879  			expectStartIndex:   3,
   880  		},
   881  		{
   882  			name:               "[capacity equals 4*n] [startIndex not equal 0] quarter of recent events outside eventFreshDuration cause cache shrinking",
   883  			eventCount:         8,
   884  			cacheCapacity:      8,
   885  			startIndex:         3,
   886  			lowerBoundCapacity: 8 / 2,
   887  			upperBoundCapacity: 8 * 2,
   888  			interval:           eventFreshDuration/2 + time.Second,
   889  			expectCapacity:     4,
   890  			expectStartIndex:   7,
   891  		},
   892  		{
   893  			name:               "[capacity equals 4*n] [startIndex not equal 0] quarter of recent events outside eventFreshDuration cause cache shrinking with given lowerBoundCapacity",
   894  			eventCount:         8,
   895  			cacheCapacity:      8,
   896  			startIndex:         3,
   897  			lowerBoundCapacity: 7,
   898  			upperBoundCapacity: 8 * 2,
   899  			interval:           eventFreshDuration/2 + time.Second,
   900  			expectCapacity:     7,
   901  			expectStartIndex:   4,
   902  		},
   903  		{
   904  			name:               "[capacity equals 4*n] [startIndex not equal 0] events inside eventFreshDuration cause cache expanding with given upperBoundCapacity",
   905  			eventCount:         8,
   906  			cacheCapacity:      8,
   907  			startIndex:         3,
   908  			lowerBoundCapacity: 8 / 2,
   909  			upperBoundCapacity: 10,
   910  			interval:           eventFreshDuration / 9,
   911  			expectCapacity:     10,
   912  			expectStartIndex:   3,
   913  		},
   914  	}
   915  
   916  	for _, test := range tests {
   917  		t.Run(test.name, func(t *testing.T) {
   918  			store := newTestWatchCache(test.cacheCapacity, &cache.Indexers{})
   919  			defer store.Stop()
   920  			store.cache = make([]*watchCacheEvent, test.cacheCapacity)
   921  			store.startIndex = test.startIndex
   922  			store.lowerBoundCapacity = test.lowerBoundCapacity
   923  			store.upperBoundCapacity = test.upperBoundCapacity
   924  			loadEventWithDuration(store, test.eventCount, test.interval)
   925  			nextInterval := store.clock.Now().Add(time.Duration(test.interval.Nanoseconds() * int64(test.eventCount)))
   926  			store.resizeCacheLocked(nextInterval)
   927  			if store.capacity != test.expectCapacity {
   928  				t.Errorf("expect capacity %d, but get %d", test.expectCapacity, store.capacity)
   929  			}
   930  
   931  			// check cache's startIndex, endIndex and all elements.
   932  			if store.startIndex != test.expectStartIndex {
   933  				t.Errorf("expect startIndex %d, but get %d", test.expectStartIndex, store.startIndex)
   934  			}
   935  			if store.endIndex != test.startIndex+test.eventCount {
   936  				t.Errorf("expect endIndex %d get %d", test.startIndex+test.eventCount, store.endIndex)
   937  			}
   938  			if !checkCacheElements(store) {
   939  				t.Errorf("some elements locations in cache is wrong")
   940  			}
   941  		})
   942  	}
   943  }
   944  
   945  func loadEventWithDuration(cache *testWatchCache, count int, interval time.Duration) {
   946  	for i := 0; i < count; i++ {
   947  		event := &watchCacheEvent{
   948  			Key:        fmt.Sprintf("event-%d", i+cache.startIndex),
   949  			RecordTime: cache.clock.Now().Add(time.Duration(interval.Nanoseconds() * int64(i))),
   950  		}
   951  		cache.cache[(i+cache.startIndex)%cache.capacity] = event
   952  	}
   953  	cache.endIndex = cache.startIndex + count
   954  }
   955  
   956  func checkCacheElements(cache *testWatchCache) bool {
   957  	for i := cache.startIndex; i < cache.endIndex; i++ {
   958  		location := i % cache.capacity
   959  		if cache.cache[location].Key != fmt.Sprintf("event-%d", i) {
   960  			return false
   961  		}
   962  	}
   963  	return true
   964  }
   965  
   966  func TestCacheIncreaseDoesNotBreakWatch(t *testing.T) {
   967  	store := newTestWatchCache(2, &cache.Indexers{})
   968  	defer store.Stop()
   969  
   970  	now := store.clock.Now()
   971  	addEvent := func(key string, rv uint64, t time.Time) {
   972  		event := &watchCacheEvent{
   973  			Key:             key,
   974  			ResourceVersion: rv,
   975  			RecordTime:      t,
   976  		}
   977  		store.updateCache(event)
   978  	}
   979  
   980  	// Initial LIST comes from the moment of RV=10.
   981  	store.Replace(nil, "10")
   982  
   983  	addEvent("key1", 20, now)
   984  
   985  	// Force "key1" to rotate our of cache.
   986  	later := now.Add(2 * eventFreshDuration)
   987  	addEvent("key2", 30, later)
   988  	addEvent("key3", 40, later)
   989  
   990  	// Force cache resize.
   991  	addEvent("key4", 50, later.Add(time.Second))
   992  
   993  	_, err := store.getAllEventsSince(15, storage.ListOptions{})
   994  	if err == nil || !strings.Contains(err.Error(), "too old resource version") {
   995  		t.Errorf("unexpected error: %v", err)
   996  	}
   997  }
   998  
   999  func TestSuggestedWatchChannelSize(t *testing.T) {
  1000  	testCases := []struct {
  1001  		name        string
  1002  		capacity    int
  1003  		indexExists bool
  1004  		triggerUsed bool
  1005  		expected    int
  1006  	}{
  1007  		{
  1008  			name:        "capacity=100, indexExists, triggerUsed",
  1009  			capacity:    100,
  1010  			indexExists: true,
  1011  			triggerUsed: true,
  1012  			expected:    10,
  1013  		},
  1014  		{
  1015  			name:        "capacity=100, indexExists, !triggerUsed",
  1016  			capacity:    100,
  1017  			indexExists: true,
  1018  			triggerUsed: false,
  1019  			expected:    10,
  1020  		},
  1021  		{
  1022  			name:        "capacity=100, !indexExists",
  1023  			capacity:    100,
  1024  			indexExists: false,
  1025  			triggerUsed: false,
  1026  			expected:    10,
  1027  		},
  1028  		{
  1029  			name:        "capacity=750, indexExists, triggerUsed",
  1030  			capacity:    750,
  1031  			indexExists: true,
  1032  			triggerUsed: true,
  1033  			expected:    10,
  1034  		},
  1035  		{
  1036  			name:        "capacity=750, indexExists, !triggerUsed",
  1037  			capacity:    750,
  1038  			indexExists: true,
  1039  			triggerUsed: false,
  1040  			expected:    10,
  1041  		},
  1042  		{
  1043  			name:        "capacity=750, !indexExists",
  1044  			capacity:    750,
  1045  			indexExists: false,
  1046  			triggerUsed: false,
  1047  			expected:    10,
  1048  		},
  1049  		{
  1050  			name:        "capacity=7500, indexExists, triggerUsed",
  1051  			capacity:    7500,
  1052  			indexExists: true,
  1053  			triggerUsed: true,
  1054  			expected:    10,
  1055  		},
  1056  		{
  1057  			name:        "capacity=7500, indexExists, !triggerUsed",
  1058  			capacity:    7500,
  1059  			indexExists: true,
  1060  			triggerUsed: false,
  1061  			expected:    100,
  1062  		},
  1063  		{
  1064  			name:        "capacity=7500, !indexExists",
  1065  			capacity:    7500,
  1066  			indexExists: false,
  1067  			triggerUsed: false,
  1068  			expected:    100,
  1069  		},
  1070  		{
  1071  			name:        "capacity=75000, indexExists, triggerUsed",
  1072  			capacity:    75000,
  1073  			indexExists: true,
  1074  			triggerUsed: true,
  1075  			expected:    10,
  1076  		},
  1077  		{
  1078  			name:        "capacity=75000, indexExists, !triggerUsed",
  1079  			capacity:    75000,
  1080  			indexExists: true,
  1081  			triggerUsed: false,
  1082  			expected:    1000,
  1083  		},
  1084  		{
  1085  			name:        "capacity=75000, !indexExists",
  1086  			capacity:    75000,
  1087  			indexExists: false,
  1088  			triggerUsed: false,
  1089  			expected:    100,
  1090  		},
  1091  		{
  1092  			name:        "capacity=750000, indexExists, triggerUsed",
  1093  			capacity:    750000,
  1094  			indexExists: true,
  1095  			triggerUsed: true,
  1096  			expected:    10,
  1097  		},
  1098  		{
  1099  			name:        "capacity=750000, indexExists, !triggerUsed",
  1100  			capacity:    750000,
  1101  			indexExists: true,
  1102  			triggerUsed: false,
  1103  			expected:    1000,
  1104  		},
  1105  		{
  1106  			name:        "capacity=750000, !indexExists",
  1107  			capacity:    750000,
  1108  			indexExists: false,
  1109  			triggerUsed: false,
  1110  			expected:    100,
  1111  		},
  1112  	}
  1113  
  1114  	for _, test := range testCases {
  1115  		t.Run(test.name, func(t *testing.T) {
  1116  			store := newTestWatchCache(test.capacity, &cache.Indexers{})
  1117  			defer store.Stop()
  1118  			got := store.suggestedWatchChannelSize(test.indexExists, test.triggerUsed)
  1119  			if got != test.expected {
  1120  				t.Errorf("unexpected channel size got: %v, expected: %v", got, test.expected)
  1121  			}
  1122  		})
  1123  	}
  1124  }
  1125  
  1126  func BenchmarkWatchCache_updateCache(b *testing.B) {
  1127  	store := newTestWatchCache(defaultUpperBoundCapacity, &cache.Indexers{})
  1128  	defer store.Stop()
  1129  	store.cache = store.cache[:0]
  1130  	store.upperBoundCapacity = defaultUpperBoundCapacity
  1131  	loadEventWithDuration(store, defaultUpperBoundCapacity, 0)
  1132  	add := &watchCacheEvent{
  1133  		Key:        fmt.Sprintf("event-%d", defaultUpperBoundCapacity),
  1134  		RecordTime: store.clock.Now(),
  1135  	}
  1136  	b.ResetTimer()
  1137  	for i := 0; i < b.N; i++ {
  1138  		store.updateCache(add)
  1139  	}
  1140  }
  1141  
  1142  func TestHistogramCacheReadWait(t *testing.T) {
  1143  	registry := k8smetrics.NewKubeRegistry()
  1144  	if err := registry.Register(metrics.WatchCacheReadWait); err != nil {
  1145  		t.Errorf("unexpected error: %v", err)
  1146  	}
  1147  	ctx := context.Background()
  1148  	testedMetrics := "apiserver_watch_cache_read_wait_seconds"
  1149  	store := newTestWatchCache(2, &cache.Indexers{})
  1150  	defer store.Stop()
  1151  
  1152  	// In background, update the store.
  1153  	go func() {
  1154  		if err := store.Add(makeTestPod("foo", 2)); err != nil {
  1155  			t.Errorf("unexpected error: %v", err)
  1156  		}
  1157  		if err := store.Add(makeTestPod("bar", 5)); err != nil {
  1158  			t.Errorf("unexpected error: %v", err)
  1159  		}
  1160  	}()
  1161  
  1162  	testCases := []struct {
  1163  		desc            string
  1164  		resourceVersion uint64
  1165  		want            string
  1166  	}{
  1167  		{
  1168  			desc:            "resourceVersion is non-zero",
  1169  			resourceVersion: 5,
  1170  			want: `
  1171  		# HELP apiserver_watch_cache_read_wait_seconds [ALPHA] Histogram of time spent waiting for a watch cache to become fresh.
  1172      # TYPE apiserver_watch_cache_read_wait_seconds histogram
  1173        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.005"} 1
  1174        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.025"} 1
  1175        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.05"} 1
  1176        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.1"} 1
  1177        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.2"} 1
  1178        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.4"} 1
  1179        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.6"} 1
  1180        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.8"} 1
  1181        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1"} 1
  1182        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.25"} 1
  1183        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.5"} 1
  1184        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="2"} 1
  1185        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="3"} 1
  1186        apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="+Inf"} 1
  1187        apiserver_watch_cache_read_wait_seconds_sum{resource="pods"} 0
  1188        apiserver_watch_cache_read_wait_seconds_count{resource="pods"} 1
  1189  `,
  1190  		},
  1191  		{
  1192  			desc:            "resourceVersion is 0",
  1193  			resourceVersion: 0,
  1194  			want:            ``,
  1195  		},
  1196  	}
  1197  
  1198  	for _, test := range testCases {
  1199  		t.Run(test.desc, func(t *testing.T) {
  1200  			defer registry.Reset()
  1201  			if _, _, _, err := store.WaitUntilFreshAndGet(ctx, test.resourceVersion, "prefix/ns/bar"); err != nil {
  1202  				t.Errorf("unexpected error: %v", err)
  1203  			}
  1204  			if err := testutil.GatherAndCompare(registry, strings.NewReader(test.want), testedMetrics); err != nil {
  1205  				t.Errorf("unexpected error: %v", err)
  1206  			}
  1207  		})
  1208  	}
  1209  }