k8s.io/apiserver@v0.31.1/pkg/storage/cacher/cacher_whitebox_test.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package cacher
    18  
    19  import (
    20  	"context"
    21  	"crypto/rand"
    22  	"errors"
    23  	"fmt"
    24  	"reflect"
    25  	goruntime "runtime"
    26  	"strconv"
    27  	"strings"
    28  	"sync"
    29  	"testing"
    30  	"time"
    31  
    32  	"github.com/stretchr/testify/require"
    33  	"google.golang.org/grpc/metadata"
    34  
    35  	apiequality "k8s.io/apimachinery/pkg/api/equality"
    36  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    37  	"k8s.io/apimachinery/pkg/api/meta"
    38  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    39  	"k8s.io/apimachinery/pkg/fields"
    40  	"k8s.io/apimachinery/pkg/labels"
    41  	"k8s.io/apimachinery/pkg/runtime"
    42  	"k8s.io/apimachinery/pkg/runtime/schema"
    43  	"k8s.io/apimachinery/pkg/util/wait"
    44  	"k8s.io/apimachinery/pkg/watch"
    45  	"k8s.io/apiserver/pkg/apis/example"
    46  	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
    47  	"k8s.io/apiserver/pkg/features"
    48  	"k8s.io/apiserver/pkg/storage"
    49  	"k8s.io/apiserver/pkg/storage/cacher/metrics"
    50  	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
    51  	etcdfeature "k8s.io/apiserver/pkg/storage/feature"
    52  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    53  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    54  	k8smetrics "k8s.io/component-base/metrics"
    55  	"k8s.io/component-base/metrics/testutil"
    56  	"k8s.io/utils/clock"
    57  	testingclock "k8s.io/utils/clock/testing"
    58  	"k8s.io/utils/pointer"
    59  )
    60  
    61  func newTestCacherWithoutSyncing(s storage.Interface) (*Cacher, storage.Versioner, error) {
    62  	prefix := "pods"
    63  	config := Config{
    64  		Storage:        s,
    65  		Versioner:      storage.APIObjectVersioner{},
    66  		GroupResource:  schema.GroupResource{Resource: "pods"},
    67  		ResourcePrefix: prefix,
    68  		KeyFunc:        func(obj runtime.Object) (string, error) { return storage.NamespaceKeyFunc(prefix, obj) },
    69  		GetAttrsFunc: func(obj runtime.Object) (labels.Set, fields.Set, error) {
    70  			pod, ok := obj.(*example.Pod)
    71  			if !ok {
    72  				return storage.DefaultNamespaceScopedAttr(obj)
    73  			}
    74  			labelsSet, fieldsSet, err := storage.DefaultNamespaceScopedAttr(obj)
    75  			if err != nil {
    76  				return nil, nil, err
    77  			}
    78  			fieldsSet["spec.nodeName"] = pod.Spec.NodeName
    79  			return labelsSet, fieldsSet, nil
    80  		},
    81  		NewFunc:     func() runtime.Object { return &example.Pod{} },
    82  		NewListFunc: func() runtime.Object { return &example.PodList{} },
    83  		Codec:       codecs.LegacyCodec(examplev1.SchemeGroupVersion),
    84  		Clock:       clock.RealClock{},
    85  	}
    86  	cacher, err := NewCacherFromConfig(config)
    87  
    88  	return cacher, storage.APIObjectVersioner{}, err
    89  }
    90  
    91  func newTestCacher(s storage.Interface) (*Cacher, storage.Versioner, error) {
    92  	cacher, versioner, err := newTestCacherWithoutSyncing(s)
    93  	if err != nil {
    94  		return nil, versioner, err
    95  	}
    96  
    97  	if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
    98  		// The tests assume that Get/GetList/Watch calls shouldn't fail.
    99  		// However, 429 error can now be returned if watchcache is under initialization.
   100  		// To avoid rewriting all tests, we wait for watcache to initialize.
   101  		if err := cacher.Wait(context.Background()); err != nil {
   102  			return nil, storage.APIObjectVersioner{}, err
   103  		}
   104  	}
   105  	return cacher, versioner, nil
   106  }
   107  
   108  type dummyStorage struct {
   109  	sync.RWMutex
   110  	err       error
   111  	getListFn func(_ context.Context, _ string, _ storage.ListOptions, listObj runtime.Object) error
   112  	watchFn   func(_ context.Context, _ string, _ storage.ListOptions) (watch.Interface, error)
   113  
   114  	// use getRequestWatchProgressCounter when reading
   115  	// the value of the counter
   116  	requestWatchProgressCounter int
   117  }
   118  
   119  func (d *dummyStorage) RequestWatchProgress(ctx context.Context) error {
   120  	d.Lock()
   121  	defer d.Unlock()
   122  	d.requestWatchProgressCounter++
   123  	return nil
   124  }
   125  
   126  func (d *dummyStorage) getRequestWatchProgressCounter() int {
   127  	d.RLock()
   128  	defer d.RUnlock()
   129  	return d.requestWatchProgressCounter
   130  }
   131  
   132  type dummyWatch struct {
   133  	ch chan watch.Event
   134  }
   135  
   136  func (w *dummyWatch) ResultChan() <-chan watch.Event {
   137  	return w.ch
   138  }
   139  
   140  func (w *dummyWatch) Stop() {
   141  	close(w.ch)
   142  }
   143  
   144  func newDummyWatch() watch.Interface {
   145  	return &dummyWatch{
   146  		ch: make(chan watch.Event),
   147  	}
   148  }
   149  
   150  func (d *dummyStorage) Versioner() storage.Versioner { return nil }
   151  func (d *dummyStorage) Create(_ context.Context, _ string, _, _ runtime.Object, _ uint64) error {
   152  	return fmt.Errorf("unimplemented")
   153  }
   154  func (d *dummyStorage) Delete(_ context.Context, _ string, _ runtime.Object, _ *storage.Preconditions, _ storage.ValidateObjectFunc, _ runtime.Object) error {
   155  	return fmt.Errorf("unimplemented")
   156  }
   157  func (d *dummyStorage) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
   158  	if d.watchFn != nil {
   159  		return d.watchFn(ctx, key, opts)
   160  	}
   161  	d.RLock()
   162  	defer d.RUnlock()
   163  
   164  	return newDummyWatch(), d.err
   165  }
   166  func (d *dummyStorage) Get(_ context.Context, _ string, _ storage.GetOptions, _ runtime.Object) error {
   167  	d.RLock()
   168  	defer d.RUnlock()
   169  
   170  	return d.err
   171  }
   172  func (d *dummyStorage) GetList(ctx context.Context, resPrefix string, opts storage.ListOptions, listObj runtime.Object) error {
   173  	if d.getListFn != nil {
   174  		return d.getListFn(ctx, resPrefix, opts, listObj)
   175  	}
   176  	d.RLock()
   177  	defer d.RUnlock()
   178  	podList := listObj.(*example.PodList)
   179  	podList.ListMeta = metav1.ListMeta{ResourceVersion: "100"}
   180  	return d.err
   181  }
   182  func (d *dummyStorage) GuaranteedUpdate(_ context.Context, _ string, _ runtime.Object, _ bool, _ *storage.Preconditions, _ storage.UpdateFunc, _ runtime.Object) error {
   183  	return fmt.Errorf("unimplemented")
   184  }
   185  func (d *dummyStorage) Count(_ string) (int64, error) {
   186  	return 0, fmt.Errorf("unimplemented")
   187  }
   188  func (d *dummyStorage) ReadinessCheck() error {
   189  	return nil
   190  }
   191  func (d *dummyStorage) injectError(err error) {
   192  	d.Lock()
   193  	defer d.Unlock()
   194  
   195  	d.err = err
   196  }
   197  
   198  func TestGetListCacheBypass(t *testing.T) {
   199  	type testCase struct {
   200  		opts         storage.ListOptions
   201  		expectBypass bool
   202  	}
   203  	commonTestCases := []testCase{
   204  		{opts: storage.ListOptions{ResourceVersion: "0"}, expectBypass: false},
   205  		{opts: storage.ListOptions{ResourceVersion: "1"}, expectBypass: false},
   206  
   207  		{opts: storage.ListOptions{ResourceVersion: "", Predicate: storage.SelectionPredicate{Continue: "a"}}, expectBypass: true},
   208  		{opts: storage.ListOptions{ResourceVersion: "0", Predicate: storage.SelectionPredicate{Continue: "a"}}, expectBypass: true},
   209  		{opts: storage.ListOptions{ResourceVersion: "1", Predicate: storage.SelectionPredicate{Continue: "a"}}, expectBypass: true},
   210  
   211  		{opts: storage.ListOptions{ResourceVersion: "0", Predicate: storage.SelectionPredicate{Limit: 500}}, expectBypass: false},
   212  		{opts: storage.ListOptions{ResourceVersion: "1", Predicate: storage.SelectionPredicate{Limit: 500}}, expectBypass: true},
   213  
   214  		{opts: storage.ListOptions{ResourceVersion: "", ResourceVersionMatch: metav1.ResourceVersionMatchExact}, expectBypass: true},
   215  		{opts: storage.ListOptions{ResourceVersion: "0", ResourceVersionMatch: metav1.ResourceVersionMatchExact}, expectBypass: true},
   216  		{opts: storage.ListOptions{ResourceVersion: "1", ResourceVersionMatch: metav1.ResourceVersionMatchExact}, expectBypass: true},
   217  	}
   218  
   219  	t.Run("ConsistentListFromStorage", func(t *testing.T) {
   220  		featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, false)
   221  		testCases := append(commonTestCases,
   222  			testCase{opts: storage.ListOptions{ResourceVersion: ""}, expectBypass: true},
   223  			testCase{opts: storage.ListOptions{ResourceVersion: "", Predicate: storage.SelectionPredicate{Limit: 500}}, expectBypass: true},
   224  		)
   225  		for _, tc := range testCases {
   226  			testGetListCacheBypass(t, tc.opts, tc.expectBypass)
   227  		}
   228  
   229  	})
   230  	t.Run("ConsistentListFromCache", func(t *testing.T) {
   231  		featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true)
   232  
   233  		// TODO(p0lyn0mial): the following tests assume that etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress)
   234  		// evaluates to true. Otherwise the cache will be bypassed and the test will fail.
   235  		//
   236  		// If you were to run only TestGetListCacheBypass you would see that the test fail.
   237  		// However in CI all test are run and there must be a test(s) that properly
   238  		// initialize the storage layer so that the mentioned method evaluates to true
   239  		forceRequestWatchProgressSupport(t)
   240  
   241  		testCases := append(commonTestCases,
   242  			testCase{opts: storage.ListOptions{ResourceVersion: ""}, expectBypass: false},
   243  			testCase{opts: storage.ListOptions{ResourceVersion: "", Predicate: storage.SelectionPredicate{Limit: 500}}, expectBypass: false},
   244  		)
   245  		for _, tc := range testCases {
   246  			testGetListCacheBypass(t, tc.opts, tc.expectBypass)
   247  		}
   248  	})
   249  }
   250  
   251  func testGetListCacheBypass(t *testing.T, options storage.ListOptions, expectBypass bool) {
   252  	backingStorage := &dummyStorage{}
   253  	cacher, _, err := newTestCacher(backingStorage)
   254  	if err != nil {
   255  		t.Fatalf("Couldn't create cacher: %v", err)
   256  	}
   257  	defer cacher.Stop()
   258  
   259  	result := &example.PodList{}
   260  
   261  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
   262  		if err := cacher.ready.wait(context.Background()); err != nil {
   263  			t.Fatalf("unexpected error waiting for the cache to be ready")
   264  		}
   265  	}
   266  
   267  	// Inject error to underlying layer and check if cacher is not bypassed.
   268  	backingStorage.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
   269  		currentResourceVersion := "42"
   270  		switch {
   271  		// request made by getCurrentResourceVersionFromStorage by checking Limit
   272  		case key == cacher.resourcePrefix:
   273  			podList := listObj.(*example.PodList)
   274  			podList.ResourceVersion = currentResourceVersion
   275  			return nil
   276  		// request made by storage.GetList with revision from original request or
   277  		// returned by getCurrentResourceVersionFromStorage
   278  		case opts.ResourceVersion == options.ResourceVersion || opts.ResourceVersion == currentResourceVersion:
   279  			return errDummy
   280  		default:
   281  			t.Fatalf("Unexpected request %+v", opts)
   282  			return nil
   283  		}
   284  	}
   285  	err = cacher.GetList(context.TODO(), "pods/ns", options, result)
   286  	if err != nil && err != errDummy {
   287  		t.Fatalf("Unexpected error for List request with options: %v, err: %v", options, err)
   288  	}
   289  	gotBypass := err == errDummy
   290  	if gotBypass != expectBypass {
   291  		t.Errorf("Unexpected bypass result for List request with options %+v, bypass expected: %v, got: %v", options, expectBypass, gotBypass)
   292  	}
   293  }
   294  
   295  func TestConsistentReadFallback(t *testing.T) {
   296  	tcs := []struct {
   297  		name                   string
   298  		consistentReadsEnabled bool
   299  		watchCacheRV           string
   300  		storageRV              string
   301  		fallbackError          bool
   302  
   303  		expectError             bool
   304  		expectRV                string
   305  		expectBlock             bool
   306  		expectRequestsToStorage int
   307  		expectMetric            string
   308  	}{
   309  		{
   310  			name:                    "Success",
   311  			consistentReadsEnabled:  true,
   312  			watchCacheRV:            "42",
   313  			storageRV:               "42",
   314  			expectRV:                "42",
   315  			expectRequestsToStorage: 1,
   316  			expectMetric: `
   317  # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache.
   318  # TYPE apiserver_watch_cache_consistent_read_total counter
   319  apiserver_watch_cache_consistent_read_total{fallback="false", resource="pods", success="true"} 1
   320  `,
   321  		},
   322  		{
   323  			name:                    "Fallback",
   324  			consistentReadsEnabled:  true,
   325  			watchCacheRV:            "2",
   326  			storageRV:               "42",
   327  			expectRV:                "42",
   328  			expectBlock:             true,
   329  			expectRequestsToStorage: 2,
   330  			expectMetric: `
   331  # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache.
   332  # TYPE apiserver_watch_cache_consistent_read_total counter
   333  apiserver_watch_cache_consistent_read_total{fallback="true", resource="pods", success="true"} 1
   334  `,
   335  		},
   336  		{
   337  			name:                    "Fallback Failure",
   338  			consistentReadsEnabled:  true,
   339  			watchCacheRV:            "2",
   340  			storageRV:               "42",
   341  			fallbackError:           true,
   342  			expectError:             true,
   343  			expectBlock:             true,
   344  			expectRequestsToStorage: 2,
   345  			expectMetric: `
   346  # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache.
   347  # TYPE apiserver_watch_cache_consistent_read_total counter
   348  apiserver_watch_cache_consistent_read_total{fallback="true", resource="pods", success="false"} 1
   349  `,
   350  		},
   351  		{
   352  			name:                    "Disabled",
   353  			watchCacheRV:            "2",
   354  			storageRV:               "42",
   355  			expectRV:                "42",
   356  			expectRequestsToStorage: 1,
   357  			expectMetric:            ``,
   358  		},
   359  	}
   360  	for _, tc := range tcs {
   361  		t.Run(tc.name, func(t *testing.T) {
   362  			featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, tc.consistentReadsEnabled)
   363  			if tc.consistentReadsEnabled {
   364  				forceRequestWatchProgressSupport(t)
   365  			}
   366  
   367  			registry := k8smetrics.NewKubeRegistry()
   368  			metrics.ConsistentReadTotal.Reset()
   369  			if err := registry.Register(metrics.ConsistentReadTotal); err != nil {
   370  				t.Errorf("unexpected error: %v", err)
   371  			}
   372  			backingStorage := &dummyStorage{}
   373  			backingStorage.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
   374  				podList := listObj.(*example.PodList)
   375  				podList.ResourceVersion = tc.watchCacheRV
   376  				return nil
   377  			}
   378  			// TODO: Use fake clock for this test to reduce execution time.
   379  			cacher, _, err := newTestCacher(backingStorage)
   380  			if err != nil {
   381  				t.Fatalf("Couldn't create cacher: %v", err)
   382  			}
   383  			defer cacher.Stop()
   384  
   385  			if fmt.Sprintf("%d", cacher.watchCache.resourceVersion) != tc.watchCacheRV {
   386  				t.Fatalf("Expected watch cache RV to equal watchCacheRV, got: %d, want: %s", cacher.watchCache.resourceVersion, tc.watchCacheRV)
   387  			}
   388  			requestToStorageCount := 0
   389  			backingStorage.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
   390  				requestToStorageCount += 1
   391  				podList := listObj.(*example.PodList)
   392  				if key == cacher.resourcePrefix {
   393  					podList.ResourceVersion = tc.storageRV
   394  					return nil
   395  				}
   396  				if tc.fallbackError {
   397  					return errDummy
   398  				}
   399  				podList.ResourceVersion = tc.storageRV
   400  				return nil
   401  			}
   402  			result := &example.PodList{}
   403  			start := cacher.clock.Now()
   404  			err = cacher.GetList(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: ""}, result)
   405  			duration := cacher.clock.Since(start)
   406  			if (err != nil) != tc.expectError {
   407  				t.Fatalf("Unexpected error err: %v", err)
   408  			}
   409  			if result.ResourceVersion != tc.expectRV {
   410  				t.Fatalf("Unexpected List response RV, got: %q, want: %q", result.ResourceVersion, tc.expectRV)
   411  			}
   412  			if requestToStorageCount != tc.expectRequestsToStorage {
   413  				t.Fatalf("Unexpected number of requests to storage, got: %d, want: %d", requestToStorageCount, tc.expectRequestsToStorage)
   414  			}
   415  			blocked := duration >= blockTimeout
   416  			if blocked != tc.expectBlock {
   417  				t.Fatalf("Unexpected block, got: %v, want: %v", blocked, tc.expectBlock)
   418  			}
   419  
   420  			if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectMetric), "apiserver_watch_cache_consistent_read_total"); err != nil {
   421  				t.Errorf("unexpected error: %v", err)
   422  			}
   423  		})
   424  	}
   425  }
   426  
   427  func TestGetListNonRecursiveCacheBypass(t *testing.T) {
   428  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, false)
   429  	backingStorage := &dummyStorage{}
   430  	cacher, _, err := newTestCacher(backingStorage)
   431  	if err != nil {
   432  		t.Fatalf("Couldn't create cacher: %v", err)
   433  	}
   434  	defer cacher.Stop()
   435  
   436  	pred := storage.SelectionPredicate{
   437  		Limit: 500,
   438  	}
   439  	result := &example.PodList{}
   440  
   441  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
   442  		if err := cacher.ready.wait(context.Background()); err != nil {
   443  			t.Fatalf("unexpected error waiting for the cache to be ready")
   444  		}
   445  	}
   446  
   447  	// Inject error to underlying layer and check if cacher is not bypassed.
   448  	backingStorage.injectError(errDummy)
   449  	err = cacher.GetList(context.TODO(), "pods/ns", storage.ListOptions{
   450  		ResourceVersion: "0",
   451  		Predicate:       pred,
   452  	}, result)
   453  	if err != nil {
   454  		t.Errorf("GetList with Limit and RV=0 should be served from cache: %v", err)
   455  	}
   456  
   457  	err = cacher.GetList(context.TODO(), "pods/ns", storage.ListOptions{
   458  		ResourceVersion: "",
   459  		Predicate:       pred,
   460  	}, result)
   461  	if err != errDummy {
   462  		t.Errorf("GetList with Limit without RV=0 should bypass cacher: %v", err)
   463  	}
   464  }
   465  
   466  func TestGetCacheBypass(t *testing.T) {
   467  	backingStorage := &dummyStorage{}
   468  	cacher, _, err := newTestCacher(backingStorage)
   469  	if err != nil {
   470  		t.Fatalf("Couldn't create cacher: %v", err)
   471  	}
   472  	defer cacher.Stop()
   473  
   474  	result := &example.Pod{}
   475  
   476  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
   477  		if err := cacher.ready.wait(context.Background()); err != nil {
   478  			t.Fatalf("unexpected error waiting for the cache to be ready")
   479  		}
   480  	}
   481  
   482  	// Inject error to underlying layer and check if cacher is not bypassed.
   483  	backingStorage.injectError(errDummy)
   484  	err = cacher.Get(context.TODO(), "pods/ns/pod-0", storage.GetOptions{
   485  		IgnoreNotFound:  true,
   486  		ResourceVersion: "0",
   487  	}, result)
   488  	if err != nil {
   489  		t.Errorf("Get with RV=0 should be served from cache: %v", err)
   490  	}
   491  
   492  	err = cacher.Get(context.TODO(), "pods/ns/pod-0", storage.GetOptions{
   493  		IgnoreNotFound:  true,
   494  		ResourceVersion: "",
   495  	}, result)
   496  	if err != errDummy {
   497  		t.Errorf("Get without RV=0 should bypass cacher: %v", err)
   498  	}
   499  }
   500  
   501  func TestWatchCacheBypass(t *testing.T) {
   502  	backingStorage := &dummyStorage{}
   503  	cacher, _, err := newTestCacher(backingStorage)
   504  	if err != nil {
   505  		t.Fatalf("Couldn't create cacher: %v", err)
   506  	}
   507  	defer cacher.Stop()
   508  
   509  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
   510  		if err := cacher.ready.wait(context.Background()); err != nil {
   511  			t.Fatalf("unexpected error waiting for the cache to be ready")
   512  		}
   513  	}
   514  
   515  	_, err = cacher.Watch(context.TODO(), "pod/ns", storage.ListOptions{
   516  		ResourceVersion: "0",
   517  		Predicate:       storage.Everything,
   518  	})
   519  	if err != nil {
   520  		t.Errorf("Watch with RV=0 should be served from cache: %v", err)
   521  	}
   522  
   523  	_, err = cacher.Watch(context.TODO(), "pod/ns", storage.ListOptions{
   524  		ResourceVersion: "",
   525  		Predicate:       storage.Everything,
   526  	})
   527  	if err != nil {
   528  		t.Errorf("Watch with RV=0 should be served from cache: %v", err)
   529  	}
   530  
   531  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchFromStorageWithoutResourceVersion, false)
   532  	_, err = cacher.Watch(context.TODO(), "pod/ns", storage.ListOptions{
   533  		ResourceVersion: "",
   534  		Predicate:       storage.Everything,
   535  	})
   536  	if err != nil {
   537  		t.Errorf("With WatchFromStorageWithoutResourceVersion disabled, watch with unset RV should be served from cache: %v", err)
   538  	}
   539  
   540  	// Inject error to underlying layer and check if cacher is not bypassed.
   541  	backingStorage.injectError(errDummy)
   542  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchFromStorageWithoutResourceVersion, true)
   543  	_, err = cacher.Watch(context.TODO(), "pod/ns", storage.ListOptions{
   544  		ResourceVersion: "",
   545  		Predicate:       storage.Everything,
   546  	})
   547  	if !errors.Is(err, errDummy) {
   548  		t.Errorf("With WatchFromStorageWithoutResourceVersion enabled, watch with unset RV should be served from storage: %v", err)
   549  	}
   550  }
   551  
   552  func TestTooManyRequestsNotReturned(t *testing.T) {
   553  	// Ensure that with ResilientWatchCacheInitialization feature disabled, we don't return 429
   554  	// errors when watchcache is not initialized.
   555  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ResilientWatchCacheInitialization, false)
   556  
   557  	dummyErr := fmt.Errorf("dummy")
   558  	backingStorage := &dummyStorage{err: dummyErr}
   559  	cacher, _, err := newTestCacherWithoutSyncing(backingStorage)
   560  	if err != nil {
   561  		t.Fatalf("Couldn't create cacher: %v", err)
   562  	}
   563  	defer cacher.Stop()
   564  
   565  	opts := storage.ListOptions{
   566  		ResourceVersion: "0",
   567  		Predicate:       storage.Everything,
   568  	}
   569  
   570  	// Cancel the request so that it doesn't hang forever.
   571  	listCtx, listCancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
   572  	defer listCancel()
   573  
   574  	result := &example.PodList{}
   575  	err = cacher.GetList(listCtx, "/pods/ns", opts, result)
   576  	if err != nil && apierrors.IsTooManyRequests(err) {
   577  		t.Errorf("Unexpected 429 error without ResilientWatchCacheInitialization feature for List")
   578  	}
   579  
   580  	watchCtx, watchCancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
   581  	defer watchCancel()
   582  
   583  	_, err = cacher.Watch(watchCtx, "/pods/ns", opts)
   584  	if err != nil && apierrors.IsTooManyRequests(err) {
   585  		t.Errorf("Unexpected 429 error without ResilientWatchCacheInitialization feature for Watch")
   586  	}
   587  }
   588  
   589  func TestEmptyWatchEventCache(t *testing.T) {
   590  	server, etcdStorage := newEtcdTestStorage(t, etcd3testing.PathPrefix())
   591  	defer server.Terminate(t)
   592  
   593  	// add a few objects
   594  	v := storage.APIObjectVersioner{}
   595  	lastRV := uint64(0)
   596  	for i := 0; i < 5; i++ {
   597  		pod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("foo-%d", i), Namespace: "test-ns"}}
   598  		out := &example.Pod{}
   599  		key := computePodKey(pod)
   600  		if err := etcdStorage.Create(context.Background(), key, pod, out, 0); err != nil {
   601  			t.Fatalf("Create failed: %v", err)
   602  		}
   603  		var err error
   604  		if lastRV, err = v.ParseResourceVersion(out.ResourceVersion); err != nil {
   605  			t.Fatalf("Unexpected error: %v", err)
   606  		}
   607  	}
   608  
   609  	cacher, _, err := newTestCacher(etcdStorage)
   610  	if err != nil {
   611  		t.Fatalf("Couldn't create cacher: %v", err)
   612  	}
   613  	defer cacher.Stop()
   614  
   615  	// Given that cacher is always initialized from the "current" version of etcd,
   616  	// we now have a cacher with an empty cache of watch events and a resourceVersion of rv.
   617  	// It should support establishing watches from rv and higher, but not older.
   618  
   619  	expectedResourceExpiredError := apierrors.NewResourceExpired("").ErrStatus
   620  	tests := []struct {
   621  		name            string
   622  		resourceVersion uint64
   623  		expectedEvent   *watch.Event
   624  	}{
   625  		{
   626  			name:            "RV-1",
   627  			resourceVersion: lastRV - 1,
   628  			expectedEvent:   &watch.Event{Type: watch.Error, Object: &expectedResourceExpiredError},
   629  		},
   630  		{
   631  			name:            "RV",
   632  			resourceVersion: lastRV,
   633  		},
   634  		{
   635  			name:            "RV+1",
   636  			resourceVersion: lastRV + 1,
   637  		},
   638  	}
   639  
   640  	for _, tt := range tests {
   641  		t.Run(tt.name, func(t *testing.T) {
   642  			opts := storage.ListOptions{
   643  				ResourceVersion: strconv.Itoa(int(tt.resourceVersion)),
   644  				Predicate:       storage.Everything,
   645  			}
   646  			watcher, err := cacher.Watch(context.Background(), "/pods/test-ns", opts)
   647  			if err != nil {
   648  				t.Fatalf("Failed to create watch: %v", err)
   649  			}
   650  			defer watcher.Stop()
   651  			select {
   652  			case event := <-watcher.ResultChan():
   653  				if tt.expectedEvent == nil {
   654  					t.Errorf("Unexpected event: type=%#v, object=%#v", event.Type, event.Object)
   655  					break
   656  				}
   657  				if e, a := tt.expectedEvent.Type, event.Type; e != a {
   658  					t.Errorf("Expected: %s, got: %s", e, a)
   659  				}
   660  				if e, a := tt.expectedEvent.Object, event.Object; !apiequality.Semantic.DeepDerivative(e, a) {
   661  					t.Errorf("Expected: %#v, got: %#v", e, a)
   662  				}
   663  			case <-time.After(1 * time.Second):
   664  				// the watch was established otherwise
   665  				// we would be blocking on cache.Watch(...)
   666  				// in addition to that, the tests are serial in nature,
   667  				// meaning that there is no other actor
   668  				// that could add items to the database,
   669  				// which could result in receiving new items.
   670  				// given that waiting 1s seems okay
   671  				if tt.expectedEvent != nil {
   672  					t.Errorf("Failed to get an event")
   673  				}
   674  				// watch remained established successfully
   675  			}
   676  		})
   677  	}
   678  }
   679  
   680  func TestWatchNotHangingOnStartupFailure(t *testing.T) {
   681  	// Configure cacher so that it can't initialize, because of
   682  	// constantly failing lists to the underlying storage.
   683  	dummyErr := fmt.Errorf("dummy")
   684  	backingStorage := &dummyStorage{err: dummyErr}
   685  	cacher, _, err := newTestCacherWithoutSyncing(backingStorage)
   686  	if err != nil {
   687  		t.Fatalf("Couldn't create cacher: %v", err)
   688  	}
   689  	defer cacher.Stop()
   690  
   691  	ctx, cancel := context.WithCancel(context.Background())
   692  	// Cancel the watch after some time to check if it will properly
   693  	// terminate instead of hanging forever.
   694  	go func() {
   695  		defer cancel()
   696  		cacher.clock.Sleep(1 * time.Second)
   697  	}()
   698  
   699  	// Watch hangs waiting on watchcache being initialized.
   700  	// Ensure that it terminates when its context is cancelled
   701  	// (e.g. the request is terminated for whatever reason).
   702  	_, err = cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "0"})
   703  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
   704  		if err == nil || err.Error() != apierrors.NewServiceUnavailable(context.Canceled.Error()).Error() {
   705  			t.Errorf("Unexpected error: %#v", err)
   706  		}
   707  	} else {
   708  		if err == nil || err.Error() != apierrors.NewTooManyRequests("storage is (re)initializing", 1).Error() {
   709  			t.Errorf("Unexpected error: %#v", err)
   710  		}
   711  	}
   712  }
   713  
   714  func TestWatcherNotGoingBackInTime(t *testing.T) {
   715  	backingStorage := &dummyStorage{}
   716  	cacher, v, err := newTestCacher(backingStorage)
   717  	if err != nil {
   718  		t.Fatalf("Couldn't create cacher: %v", err)
   719  	}
   720  	defer cacher.Stop()
   721  
   722  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
   723  		if err := cacher.ready.wait(context.Background()); err != nil {
   724  			t.Fatalf("unexpected error waiting for the cache to be ready")
   725  		}
   726  	}
   727  
   728  	// Ensure there is some budget for slowing down processing.
   729  	cacher.dispatchTimeoutBudget.returnUnused(100 * time.Millisecond)
   730  
   731  	makePod := func(i int) *examplev1.Pod {
   732  		return &examplev1.Pod{
   733  			ObjectMeta: metav1.ObjectMeta{
   734  				Name:            fmt.Sprintf("pod-%d", 1000+i),
   735  				Namespace:       "ns",
   736  				ResourceVersion: fmt.Sprintf("%d", 1000+i),
   737  			},
   738  		}
   739  	}
   740  	if err := cacher.watchCache.Add(makePod(0)); err != nil {
   741  		t.Errorf("error: %v", err)
   742  	}
   743  
   744  	totalPods := 100
   745  
   746  	// Create watcher that will be slowing down reading.
   747  	w1, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{
   748  		ResourceVersion: "999",
   749  		Predicate:       storage.Everything,
   750  	})
   751  	if err != nil {
   752  		t.Fatalf("Failed to create watch: %v", err)
   753  	}
   754  	defer w1.Stop()
   755  	go func() {
   756  		a := 0
   757  		for range w1.ResultChan() {
   758  			time.Sleep(time.Millisecond)
   759  			a++
   760  			if a == 100 {
   761  				break
   762  			}
   763  		}
   764  	}()
   765  
   766  	// Now push a ton of object to cache.
   767  	for i := 1; i < totalPods; i++ {
   768  		cacher.watchCache.Add(makePod(i))
   769  	}
   770  
   771  	// Create fast watcher and ensure it will get each object exactly once.
   772  	w2, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: storage.Everything})
   773  	if err != nil {
   774  		t.Fatalf("Failed to create watch: %v", err)
   775  	}
   776  	defer w2.Stop()
   777  
   778  	shouldContinue := true
   779  	currentRV := uint64(0)
   780  	for shouldContinue {
   781  		select {
   782  		case event, ok := <-w2.ResultChan():
   783  			if !ok {
   784  				shouldContinue = false
   785  				break
   786  			}
   787  			rv, err := v.ParseResourceVersion(event.Object.(metaRuntimeInterface).GetResourceVersion())
   788  			if err != nil {
   789  				t.Errorf("unexpected parsing error: %v", err)
   790  			} else {
   791  				if rv < currentRV {
   792  					t.Errorf("watcher going back in time")
   793  				}
   794  				currentRV = rv
   795  			}
   796  		case <-time.After(time.Second):
   797  			w2.Stop()
   798  		}
   799  	}
   800  }
   801  
   802  func TestCacherDontAcceptRequestsStopped(t *testing.T) {
   803  	backingStorage := &dummyStorage{}
   804  	cacher, _, err := newTestCacher(backingStorage)
   805  	if err != nil {
   806  		t.Fatalf("Couldn't create cacher: %v", err)
   807  	}
   808  
   809  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
   810  		if err := cacher.ready.wait(context.Background()); err != nil {
   811  			t.Fatalf("unexpected error waiting for the cache to be ready")
   812  		}
   813  	}
   814  
   815  	w, err := cacher.Watch(context.Background(), "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything})
   816  	if err != nil {
   817  		t.Fatalf("Failed to create watch: %v", err)
   818  	}
   819  
   820  	watchClosed := make(chan struct{})
   821  	go func() {
   822  		defer close(watchClosed)
   823  		for event := range w.ResultChan() {
   824  			switch event.Type {
   825  			case watch.Added, watch.Modified, watch.Deleted:
   826  				// ok
   827  			default:
   828  				t.Errorf("unexpected event %#v", event)
   829  			}
   830  		}
   831  	}()
   832  
   833  	cacher.Stop()
   834  
   835  	_, err = cacher.Watch(context.Background(), "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything})
   836  	if err == nil {
   837  		t.Fatalf("Success to create Watch: %v", err)
   838  	}
   839  
   840  	result := &example.Pod{}
   841  	err = cacher.Get(context.TODO(), "pods/ns/pod-0", storage.GetOptions{
   842  		IgnoreNotFound:  true,
   843  		ResourceVersion: "1",
   844  	}, result)
   845  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
   846  		if err == nil {
   847  			t.Fatalf("Success to create Get: %v", err)
   848  		}
   849  	} else {
   850  		if err != nil {
   851  			t.Fatalf("Failed to get object: %v:", err)
   852  		}
   853  	}
   854  
   855  	listResult := &example.PodList{}
   856  	err = cacher.GetList(context.TODO(), "pods/ns", storage.ListOptions{
   857  		ResourceVersion: "1",
   858  		Recursive:       true,
   859  		Predicate: storage.SelectionPredicate{
   860  			Limit: 500,
   861  		},
   862  	}, listResult)
   863  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
   864  		if err == nil {
   865  			t.Fatalf("Success to create GetList: %v", err)
   866  		}
   867  	} else {
   868  		if err != nil {
   869  			t.Fatalf("Failed to list objects: %v", err)
   870  		}
   871  	}
   872  
   873  	select {
   874  	case <-watchClosed:
   875  	case <-time.After(wait.ForeverTestTimeout):
   876  		t.Errorf("timed out waiting for watch to close")
   877  	}
   878  }
   879  
   880  func TestCacherDontMissEventsOnReinitialization(t *testing.T) {
   881  	makePod := func(i int) *example.Pod {
   882  		return &example.Pod{
   883  			ObjectMeta: metav1.ObjectMeta{
   884  				Name:            fmt.Sprintf("pod-%d", i),
   885  				Namespace:       "ns",
   886  				ResourceVersion: fmt.Sprintf("%d", i),
   887  			},
   888  		}
   889  	}
   890  
   891  	listCalls, watchCalls := 0, 0
   892  	backingStorage := &dummyStorage{
   893  		getListFn: func(_ context.Context, _ string, _ storage.ListOptions, listObj runtime.Object) error {
   894  			podList := listObj.(*example.PodList)
   895  			var err error
   896  			switch listCalls {
   897  			case 0:
   898  				podList.ListMeta = metav1.ListMeta{ResourceVersion: "1"}
   899  			case 1:
   900  				podList.ListMeta = metav1.ListMeta{ResourceVersion: "10"}
   901  			default:
   902  				err = fmt.Errorf("unexpected list call")
   903  			}
   904  			listCalls++
   905  			return err
   906  		},
   907  		watchFn: func(_ context.Context, _ string, _ storage.ListOptions) (watch.Interface, error) {
   908  			var w *watch.FakeWatcher
   909  			var err error
   910  			switch watchCalls {
   911  			case 0:
   912  				w = watch.NewFakeWithChanSize(10, false)
   913  				for i := 2; i < 8; i++ {
   914  					w.Add(makePod(i))
   915  				}
   916  				// Emit an error to force relisting.
   917  				w.Error(nil)
   918  				w.Stop()
   919  			case 1:
   920  				w = watch.NewFakeWithChanSize(10, false)
   921  				for i := 12; i < 18; i++ {
   922  					w.Add(makePod(i))
   923  				}
   924  				w.Stop()
   925  			default:
   926  				err = fmt.Errorf("unexpected watch call")
   927  			}
   928  			watchCalls++
   929  			return w, err
   930  		},
   931  	}
   932  	cacher, _, err := newTestCacher(backingStorage)
   933  	if err != nil {
   934  		t.Fatalf("Couldn't create cacher: %v", err)
   935  	}
   936  	defer cacher.Stop()
   937  
   938  	concurrency := 1000
   939  	wg := sync.WaitGroup{}
   940  	wg.Add(concurrency)
   941  
   942  	// Ensure that test doesn't deadlock if cacher already processed everything
   943  	// and get back into Pending state before some watches get called.
   944  	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
   945  	defer cancel()
   946  
   947  	errCh := make(chan error, concurrency)
   948  	for i := 0; i < concurrency; i++ {
   949  		go func() {
   950  			defer wg.Done()
   951  			w, err := cacher.Watch(ctx, "pods", storage.ListOptions{ResourceVersion: "1", Predicate: storage.Everything})
   952  			if err != nil {
   953  				// Watch failed to initialize (this most probably means that cacher
   954  				// already moved back to Pending state before watch initialized.
   955  				// Ignore this case.
   956  				return
   957  			}
   958  			defer w.Stop()
   959  
   960  			prevRV := -1
   961  			for event := range w.ResultChan() {
   962  				if event.Type == watch.Error {
   963  					break
   964  				}
   965  				object := event.Object
   966  				if co, ok := object.(runtime.CacheableObject); ok {
   967  					object = co.GetObject()
   968  				}
   969  				rv, err := strconv.Atoi(object.(*example.Pod).ResourceVersion)
   970  				if err != nil {
   971  					errCh <- fmt.Errorf("incorrect resource version: %v", err)
   972  					return
   973  				}
   974  				if prevRV != -1 && prevRV+1 != rv {
   975  					errCh <- fmt.Errorf("unexpected event received, prevRV=%d, rv=%d", prevRV, rv)
   976  					return
   977  				}
   978  				prevRV = rv
   979  			}
   980  
   981  		}()
   982  	}
   983  	wg.Wait()
   984  	close(errCh)
   985  
   986  	for err := range errCh {
   987  		t.Error(err)
   988  	}
   989  }
   990  
   991  func TestCacherNoLeakWithMultipleWatchers(t *testing.T) {
   992  	backingStorage := &dummyStorage{}
   993  	cacher, _, err := newTestCacher(backingStorage)
   994  	if err != nil {
   995  		t.Fatalf("Couldn't create cacher: %v", err)
   996  	}
   997  	defer cacher.Stop()
   998  
   999  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1000  		if err := cacher.ready.wait(context.Background()); err != nil {
  1001  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1002  		}
  1003  	}
  1004  
  1005  	pred := storage.Everything
  1006  	pred.AllowWatchBookmarks = true
  1007  
  1008  	// run the collision test for 3 seconds to let ~2 buckets expire
  1009  	stopCh := make(chan struct{})
  1010  	var watchErr error
  1011  	time.AfterFunc(3*time.Second, func() { close(stopCh) })
  1012  
  1013  	wg := &sync.WaitGroup{}
  1014  
  1015  	wg.Add(1)
  1016  	go func() {
  1017  		defer wg.Done()
  1018  		for {
  1019  			select {
  1020  			case <-stopCh:
  1021  				return
  1022  			default:
  1023  				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
  1024  				defer cancel()
  1025  				w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: pred})
  1026  				if err != nil {
  1027  					watchErr = fmt.Errorf("Failed to create watch: %v", err)
  1028  					return
  1029  				}
  1030  				w.Stop()
  1031  			}
  1032  		}
  1033  	}()
  1034  
  1035  	wg.Add(1)
  1036  	go func() {
  1037  		defer wg.Done()
  1038  		for {
  1039  			select {
  1040  			case <-stopCh:
  1041  				return
  1042  			default:
  1043  				cacher.Lock()
  1044  				cacher.bookmarkWatchers.popExpiredWatchersThreadUnsafe()
  1045  				cacher.Unlock()
  1046  			}
  1047  		}
  1048  	}()
  1049  
  1050  	// wait for adding/removing watchers to end
  1051  	wg.Wait()
  1052  
  1053  	if watchErr != nil {
  1054  		t.Fatal(watchErr)
  1055  	}
  1056  
  1057  	// wait out the expiration period and pop expired watchers
  1058  	time.Sleep(2 * time.Second)
  1059  	cacher.Lock()
  1060  	defer cacher.Unlock()
  1061  	cacher.bookmarkWatchers.popExpiredWatchersThreadUnsafe()
  1062  	if len(cacher.bookmarkWatchers.watchersBuckets) != 0 {
  1063  		numWatchers := 0
  1064  		for bucketID, v := range cacher.bookmarkWatchers.watchersBuckets {
  1065  			numWatchers += len(v)
  1066  			t.Errorf("there are %v watchers at bucket Id %v with start Id %v", len(v), bucketID, cacher.bookmarkWatchers.startBucketID)
  1067  		}
  1068  		t.Errorf("unexpected bookmark watchers %v", numWatchers)
  1069  	}
  1070  }
  1071  
  1072  func testCacherSendBookmarkEvents(t *testing.T, allowWatchBookmarks, expectedBookmarks bool) {
  1073  	backingStorage := &dummyStorage{}
  1074  	cacher, _, err := newTestCacher(backingStorage)
  1075  	if err != nil {
  1076  		t.Fatalf("Couldn't create cacher: %v", err)
  1077  	}
  1078  	defer cacher.Stop()
  1079  
  1080  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1081  		if err := cacher.ready.wait(context.Background()); err != nil {
  1082  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1083  		}
  1084  	}
  1085  	pred := storage.Everything
  1086  	pred.AllowWatchBookmarks = allowWatchBookmarks
  1087  
  1088  	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
  1089  	defer cancel()
  1090  	w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: pred})
  1091  	if err != nil {
  1092  		t.Fatalf("Failed to create watch: %v", err)
  1093  	}
  1094  
  1095  	resourceVersion := uint64(1000)
  1096  	errc := make(chan error, 1)
  1097  	go func() {
  1098  		deadline := time.Now().Add(time.Second)
  1099  		for i := 0; time.Now().Before(deadline); i++ {
  1100  			err := cacher.watchCache.Add(&examplev1.Pod{
  1101  				ObjectMeta: metav1.ObjectMeta{
  1102  					Name:            fmt.Sprintf("pod-%d", i),
  1103  					Namespace:       "ns",
  1104  					ResourceVersion: fmt.Sprintf("%v", resourceVersion+uint64(i)),
  1105  				}})
  1106  			if err != nil {
  1107  				errc <- fmt.Errorf("failed to add a pod: %v", err)
  1108  				return
  1109  			}
  1110  			time.Sleep(100 * time.Millisecond)
  1111  		}
  1112  	}()
  1113  
  1114  	timeoutCh := time.After(2 * time.Second)
  1115  	lastObservedRV := uint64(0)
  1116  	for {
  1117  		select {
  1118  		case err := <-errc:
  1119  			t.Fatal(err)
  1120  			return
  1121  		case event, ok := <-w.ResultChan():
  1122  			if !ok {
  1123  				t.Fatal("Unexpected closed")
  1124  			}
  1125  			rv, err := cacher.versioner.ObjectResourceVersion(event.Object)
  1126  			if err != nil {
  1127  				t.Errorf("failed to parse resource version from %#v: %v", event.Object, err)
  1128  			}
  1129  			if event.Type == watch.Bookmark {
  1130  				if !expectedBookmarks {
  1131  					t.Fatalf("Unexpected bookmark events received")
  1132  				}
  1133  
  1134  				if rv < lastObservedRV {
  1135  					t.Errorf("Unexpected bookmark event resource version %v (last %v)", rv, lastObservedRV)
  1136  				}
  1137  				return
  1138  			}
  1139  			lastObservedRV = rv
  1140  		case <-timeoutCh:
  1141  			if expectedBookmarks {
  1142  				t.Fatal("Unexpected timeout to receive a bookmark event")
  1143  			}
  1144  			return
  1145  		}
  1146  	}
  1147  }
  1148  
  1149  func TestCacherSendBookmarkEvents(t *testing.T) {
  1150  	testCases := []struct {
  1151  		allowWatchBookmarks bool
  1152  		expectedBookmarks   bool
  1153  	}{
  1154  		{
  1155  			allowWatchBookmarks: true,
  1156  			expectedBookmarks:   true,
  1157  		},
  1158  		{
  1159  			allowWatchBookmarks: false,
  1160  			expectedBookmarks:   false,
  1161  		},
  1162  	}
  1163  
  1164  	for _, tc := range testCases {
  1165  		testCacherSendBookmarkEvents(t, tc.allowWatchBookmarks, tc.expectedBookmarks)
  1166  	}
  1167  }
  1168  
  1169  func TestCacherSendsMultipleWatchBookmarks(t *testing.T) {
  1170  	backingStorage := &dummyStorage{}
  1171  	cacher, _, err := newTestCacher(backingStorage)
  1172  	if err != nil {
  1173  		t.Fatalf("Couldn't create cacher: %v", err)
  1174  	}
  1175  	defer cacher.Stop()
  1176  	// Update bookmarkFrequency to speed up test.
  1177  	// Note that the frequency lower than 1s doesn't change much due to
  1178  	// resolution how frequency we recompute.
  1179  	cacher.bookmarkWatchers.bookmarkFrequency = time.Second
  1180  
  1181  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1182  		if err := cacher.ready.wait(context.Background()); err != nil {
  1183  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1184  		}
  1185  	}
  1186  	pred := storage.Everything
  1187  	pred.AllowWatchBookmarks = true
  1188  
  1189  	makePod := func(index int) *examplev1.Pod {
  1190  		return &examplev1.Pod{
  1191  			ObjectMeta: metav1.ObjectMeta{
  1192  				Name:            fmt.Sprintf("pod-%d", index),
  1193  				Namespace:       "ns",
  1194  				ResourceVersion: fmt.Sprintf("%v", 100+index),
  1195  			},
  1196  		}
  1197  	}
  1198  
  1199  	// Create pod to initialize watch cache.
  1200  	if err := cacher.watchCache.Add(makePod(0)); err != nil {
  1201  		t.Fatalf("failed to add a pod: %v", err)
  1202  	}
  1203  
  1204  	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
  1205  	defer cancel()
  1206  	w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "100", Predicate: pred})
  1207  	if err != nil {
  1208  		t.Fatalf("Failed to create watch: %v", err)
  1209  	}
  1210  
  1211  	// Create one more pod, to ensure that current RV is higher and thus
  1212  	// bookmarks will be delievere (events are delivered for RV higher
  1213  	// than the max from init events).
  1214  	if err := cacher.watchCache.Add(makePod(1)); err != nil {
  1215  		t.Fatalf("failed to add a pod: %v", err)
  1216  	}
  1217  
  1218  	timeoutCh := time.After(5 * time.Second)
  1219  	lastObservedRV := uint64(0)
  1220  	// Ensure that a watcher gets two bookmarks.
  1221  	for observedBookmarks := 0; observedBookmarks < 2; {
  1222  		select {
  1223  		case event, ok := <-w.ResultChan():
  1224  			if !ok {
  1225  				t.Fatal("Unexpected closed")
  1226  			}
  1227  			rv, err := cacher.versioner.ObjectResourceVersion(event.Object)
  1228  			if err != nil {
  1229  				t.Errorf("failed to parse resource version from %#v: %v", event.Object, err)
  1230  			}
  1231  			if event.Type == watch.Bookmark {
  1232  				observedBookmarks++
  1233  				if rv < lastObservedRV {
  1234  					t.Errorf("Unexpected bookmark event resource version %v (last %v)", rv, lastObservedRV)
  1235  				}
  1236  			}
  1237  			lastObservedRV = rv
  1238  		case <-timeoutCh:
  1239  			t.Fatal("Unexpected timeout to receive bookmark events")
  1240  		}
  1241  	}
  1242  }
  1243  
  1244  func TestDispatchingBookmarkEventsWithConcurrentStop(t *testing.T) {
  1245  	backingStorage := &dummyStorage{}
  1246  	cacher, _, err := newTestCacher(backingStorage)
  1247  	if err != nil {
  1248  		t.Fatalf("Couldn't create cacher: %v", err)
  1249  	}
  1250  	defer cacher.Stop()
  1251  
  1252  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1253  		if err := cacher.ready.wait(context.Background()); err != nil {
  1254  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1255  		}
  1256  	}
  1257  
  1258  	// Ensure there is some budget for slowing down processing.
  1259  	cacher.dispatchTimeoutBudget.returnUnused(100 * time.Millisecond)
  1260  
  1261  	resourceVersion := uint64(1000)
  1262  	err = cacher.watchCache.Add(&examplev1.Pod{
  1263  		ObjectMeta: metav1.ObjectMeta{
  1264  			Name:            "pod-0",
  1265  			Namespace:       "ns",
  1266  			ResourceVersion: fmt.Sprintf("%v", resourceVersion),
  1267  		}})
  1268  	if err != nil {
  1269  		t.Fatalf("failed to add a pod: %v", err)
  1270  	}
  1271  
  1272  	for i := 0; i < 1000; i++ {
  1273  		pred := storage.Everything
  1274  		pred.AllowWatchBookmarks = true
  1275  		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
  1276  		defer cancel()
  1277  		w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: pred})
  1278  		if err != nil {
  1279  			t.Fatalf("Failed to create watch: %v", err)
  1280  		}
  1281  		bookmark := &watchCacheEvent{
  1282  			Type:            watch.Bookmark,
  1283  			ResourceVersion: uint64(i),
  1284  			Object:          cacher.newFunc(),
  1285  		}
  1286  		err = cacher.versioner.UpdateObject(bookmark.Object, bookmark.ResourceVersion)
  1287  		if err != nil {
  1288  			t.Fatalf("failure to update version of object (%d) %#v", bookmark.ResourceVersion, bookmark.Object)
  1289  		}
  1290  
  1291  		wg := sync.WaitGroup{}
  1292  		wg.Add(2)
  1293  		go func() {
  1294  			cacher.processEvent(bookmark)
  1295  			wg.Done()
  1296  		}()
  1297  
  1298  		go func() {
  1299  			w.Stop()
  1300  			wg.Done()
  1301  		}()
  1302  
  1303  		done := make(chan struct{})
  1304  		go func() {
  1305  			for range w.ResultChan() {
  1306  			}
  1307  			close(done)
  1308  		}()
  1309  
  1310  		select {
  1311  		case <-done:
  1312  		case <-time.After(time.Second):
  1313  			t.Fatal("receive result timeout")
  1314  		}
  1315  		w.Stop()
  1316  		wg.Wait()
  1317  	}
  1318  }
  1319  
  1320  func TestBookmarksOnResourceVersionUpdates(t *testing.T) {
  1321  	backingStorage := &dummyStorage{}
  1322  	cacher, _, err := newTestCacher(backingStorage)
  1323  	if err != nil {
  1324  		t.Fatalf("Couldn't create cacher: %v", err)
  1325  	}
  1326  	defer cacher.Stop()
  1327  
  1328  	// Ensure that bookmarks are sent more frequently than every 1m.
  1329  	cacher.bookmarkWatchers = newTimeBucketWatchers(clock.RealClock{}, 2*time.Second)
  1330  
  1331  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1332  		if err := cacher.ready.wait(context.Background()); err != nil {
  1333  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1334  		}
  1335  	}
  1336  
  1337  	makePod := func(i int) *examplev1.Pod {
  1338  		return &examplev1.Pod{
  1339  			ObjectMeta: metav1.ObjectMeta{
  1340  				Name:            fmt.Sprintf("pod-%d", i),
  1341  				Namespace:       "ns",
  1342  				ResourceVersion: fmt.Sprintf("%d", i),
  1343  			},
  1344  		}
  1345  	}
  1346  	if err := cacher.watchCache.Add(makePod(1000)); err != nil {
  1347  		t.Errorf("error: %v", err)
  1348  	}
  1349  
  1350  	pred := storage.Everything
  1351  	pred.AllowWatchBookmarks = true
  1352  
  1353  	w, err := cacher.Watch(context.TODO(), "/pods/ns", storage.ListOptions{
  1354  		ResourceVersion: "1000",
  1355  		Predicate:       pred,
  1356  	})
  1357  	if err != nil {
  1358  		t.Fatalf("Failed to create watch: %v", err)
  1359  	}
  1360  
  1361  	expectedRV := 2000
  1362  
  1363  	var rcErr error
  1364  
  1365  	wg := sync.WaitGroup{}
  1366  	wg.Add(1)
  1367  	go func() {
  1368  		defer wg.Done()
  1369  		for {
  1370  			event, ok := <-w.ResultChan()
  1371  			if !ok {
  1372  				rcErr = errors.New("Unexpected closed channel")
  1373  				return
  1374  			}
  1375  			rv, err := cacher.versioner.ObjectResourceVersion(event.Object)
  1376  			if err != nil {
  1377  				t.Errorf("failed to parse resource version from %#v: %v", event.Object, err)
  1378  			}
  1379  			if event.Type == watch.Bookmark && rv == uint64(expectedRV) {
  1380  				return
  1381  			}
  1382  		}
  1383  	}()
  1384  
  1385  	// Simulate progress notify event.
  1386  	cacher.watchCache.UpdateResourceVersion(strconv.Itoa(expectedRV))
  1387  
  1388  	wg.Wait()
  1389  	if rcErr != nil {
  1390  		t.Fatal(rcErr)
  1391  	}
  1392  }
  1393  
  1394  type fakeTimeBudget struct{}
  1395  
  1396  func (f *fakeTimeBudget) takeAvailable() time.Duration {
  1397  	return 2 * time.Second
  1398  }
  1399  
  1400  func (f *fakeTimeBudget) returnUnused(_ time.Duration) {}
  1401  
  1402  func TestStartingResourceVersion(t *testing.T) {
  1403  	backingStorage := &dummyStorage{}
  1404  	cacher, _, err := newTestCacher(backingStorage)
  1405  	if err != nil {
  1406  		t.Fatalf("Couldn't create cacher: %v", err)
  1407  	}
  1408  	defer cacher.Stop()
  1409  
  1410  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1411  		if err := cacher.ready.wait(context.Background()); err != nil {
  1412  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1413  		}
  1414  	}
  1415  
  1416  	// Ensure there is some budget for slowing down processing.
  1417  	// We use the fakeTimeBudget to prevent this test from flaking under
  1418  	// the following conditions:
  1419  	// 1) in total we create 11 events that has to be processed by the watcher
  1420  	// 2) the size of the channels are set to 10 for the watcher
  1421  	// 3) if the test is cpu-starved and the internal goroutine is not picking
  1422  	//    up these events from the channel, after consuming the whole time
  1423  	//    budget (defaulted to 100ms) on waiting, we will simply close the watch,
  1424  	//    which will cause the test failure
  1425  	// Using fakeTimeBudget gives us always a budget to wait and have a test
  1426  	// pick up something from ResultCh in the meantime.
  1427  	//
  1428  	// The same can potentially happen in production, but in that case a watch
  1429  	// can be resumed by the client. This doesn't work in the case of this test,
  1430  	// because we explicitly want to test the behavior that object changes are
  1431  	// happening after the watch was initiated.
  1432  	cacher.dispatchTimeoutBudget = &fakeTimeBudget{}
  1433  
  1434  	makePod := func(i int) *examplev1.Pod {
  1435  		return &examplev1.Pod{
  1436  			ObjectMeta: metav1.ObjectMeta{
  1437  				Name:            "foo",
  1438  				Namespace:       "ns",
  1439  				Labels:          map[string]string{"foo": strconv.Itoa(i)},
  1440  				ResourceVersion: fmt.Sprintf("%d", i),
  1441  			},
  1442  		}
  1443  	}
  1444  
  1445  	if err := cacher.watchCache.Add(makePod(1000)); err != nil {
  1446  		t.Errorf("error: %v", err)
  1447  	}
  1448  	// Advance RV by 10.
  1449  	startVersion := uint64(1010)
  1450  
  1451  	watcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", storage.ListOptions{ResourceVersion: strconv.FormatUint(startVersion, 10), Predicate: storage.Everything})
  1452  	if err != nil {
  1453  		t.Fatalf("Unexpected error: %v", err)
  1454  	}
  1455  	defer watcher.Stop()
  1456  
  1457  	for i := 1; i <= 11; i++ {
  1458  		if err := cacher.watchCache.Update(makePod(1000 + i)); err != nil {
  1459  			t.Errorf("error: %v", err)
  1460  		}
  1461  	}
  1462  
  1463  	e, ok := <-watcher.ResultChan()
  1464  	if !ok {
  1465  		t.Errorf("unexpectedly closed watch")
  1466  	}
  1467  	object := e.Object
  1468  	if co, ok := object.(runtime.CacheableObject); ok {
  1469  		object = co.GetObject()
  1470  	}
  1471  	pod := object.(*examplev1.Pod)
  1472  	podRV, err := cacher.versioner.ParseResourceVersion(pod.ResourceVersion)
  1473  	if err != nil {
  1474  		t.Fatalf("unexpected error: %v", err)
  1475  	}
  1476  
  1477  	// event should have at least rv + 1, since we're starting the watch at rv
  1478  	if podRV <= startVersion {
  1479  		t.Errorf("expected event with resourceVersion of at least %d, got %d", startVersion+1, podRV)
  1480  	}
  1481  }
  1482  
  1483  func TestDispatchEventWillNotBeBlockedByTimedOutWatcher(t *testing.T) {
  1484  	backingStorage := &dummyStorage{}
  1485  	cacher, _, err := newTestCacher(backingStorage)
  1486  	if err != nil {
  1487  		t.Fatalf("Couldn't create cacher: %v", err)
  1488  	}
  1489  	defer cacher.Stop()
  1490  
  1491  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1492  		if err := cacher.ready.wait(context.Background()); err != nil {
  1493  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1494  		}
  1495  	}
  1496  
  1497  	// Ensure there is some budget for slowing down processing.
  1498  	// We use the fakeTimeBudget to prevent this test from flaking under
  1499  	// the following conditions:
  1500  	// 1) the watch w1 is blocked, so we were consuming the whole budget once
  1501  	//    its buffer was filled in (10 items)
  1502  	// 2) the budget is refreshed once per second, so it basically wasn't
  1503  	//    happening in the test at all
  1504  	// 3) if the test was cpu-starved and we weren't able to consume events
  1505  	//    from w2 ResultCh it could have happened that its buffer was also
  1506  	//    filling in and given we no longer had timeBudget (consumed in (1))
  1507  	//    trying to put next item was simply breaking the watch
  1508  	// Using fakeTimeBudget gives us always a budget to wait and have a test
  1509  	// pick up something from ResultCh in the meantime.
  1510  	cacher.dispatchTimeoutBudget = &fakeTimeBudget{}
  1511  
  1512  	makePod := func(i int) *examplev1.Pod {
  1513  		return &examplev1.Pod{
  1514  			ObjectMeta: metav1.ObjectMeta{
  1515  				Name:            fmt.Sprintf("pod-%d", 1000+i),
  1516  				Namespace:       "ns",
  1517  				ResourceVersion: fmt.Sprintf("%d", 1000+i),
  1518  			},
  1519  		}
  1520  	}
  1521  	if err := cacher.watchCache.Add(makePod(0)); err != nil {
  1522  		t.Errorf("error: %v", err)
  1523  	}
  1524  
  1525  	totalPods := 50
  1526  
  1527  	// Create watcher that will be blocked.
  1528  	w1, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: storage.Everything})
  1529  	if err != nil {
  1530  		t.Fatalf("Failed to create watch: %v", err)
  1531  	}
  1532  	defer w1.Stop()
  1533  
  1534  	// Create fast watcher and ensure it will get all objects.
  1535  	w2, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: storage.Everything})
  1536  	if err != nil {
  1537  		t.Fatalf("Failed to create watch: %v", err)
  1538  	}
  1539  	defer w2.Stop()
  1540  
  1541  	// Now push a ton of object to cache.
  1542  	for i := 1; i < totalPods; i++ {
  1543  		cacher.watchCache.Add(makePod(i))
  1544  	}
  1545  
  1546  	shouldContinue := true
  1547  	eventsCount := 0
  1548  	for shouldContinue {
  1549  		select {
  1550  		case event, ok := <-w2.ResultChan():
  1551  			if !ok {
  1552  				shouldContinue = false
  1553  				break
  1554  			}
  1555  			if event.Type == watch.Added {
  1556  				eventsCount++
  1557  				if eventsCount == totalPods {
  1558  					shouldContinue = false
  1559  				}
  1560  			}
  1561  		case <-time.After(wait.ForeverTestTimeout):
  1562  			shouldContinue = false
  1563  			w2.Stop()
  1564  		}
  1565  	}
  1566  	if eventsCount != totalPods {
  1567  		t.Errorf("watcher is blocked by slower one (count: %d)", eventsCount)
  1568  	}
  1569  }
  1570  
  1571  func verifyEvents(t *testing.T, w watch.Interface, events []watch.Event, strictOrder bool) {
  1572  	_, _, line, _ := goruntime.Caller(1)
  1573  	actualEvents := make([]watch.Event, len(events))
  1574  	for idx := range events {
  1575  		select {
  1576  		case event := <-w.ResultChan():
  1577  			actualEvents[idx] = event
  1578  		case <-time.After(wait.ForeverTestTimeout):
  1579  			t.Logf("(called from line %d)", line)
  1580  			t.Errorf("Timed out waiting for an event")
  1581  		}
  1582  	}
  1583  	validateEvents := func(expected, actual watch.Event) (bool, []string) {
  1584  		errors := []string{}
  1585  		if e, a := expected.Type, actual.Type; e != a {
  1586  			errors = append(errors, fmt.Sprintf("Expected: %s, got: %s", e, a))
  1587  		}
  1588  		actualObject := actual.Object
  1589  		if co, ok := actualObject.(runtime.CacheableObject); ok {
  1590  			actualObject = co.GetObject()
  1591  		}
  1592  		if e, a := expected.Object, actualObject; !apiequality.Semantic.DeepEqual(e, a) {
  1593  			errors = append(errors, fmt.Sprintf("Expected: %#v, got: %#v", e, a))
  1594  		}
  1595  		return len(errors) == 0, errors
  1596  	}
  1597  
  1598  	if len(events) != len(actualEvents) {
  1599  		t.Fatalf("unexpected number of events: %d, expected: %d, acutalEvents: %#v, expectedEvents:%#v", len(actualEvents), len(events), actualEvents, events)
  1600  	}
  1601  
  1602  	if strictOrder {
  1603  		for idx, expectedEvent := range events {
  1604  			valid, errors := validateEvents(expectedEvent, actualEvents[idx])
  1605  			if !valid {
  1606  				t.Logf("(called from line %d)", line)
  1607  				for _, err := range errors {
  1608  					t.Errorf(err)
  1609  				}
  1610  			}
  1611  		}
  1612  	}
  1613  	for _, expectedEvent := range events {
  1614  		validated := false
  1615  		for _, actualEvent := range actualEvents {
  1616  			if validated, _ = validateEvents(expectedEvent, actualEvent); validated {
  1617  				break
  1618  			}
  1619  		}
  1620  		if !validated {
  1621  			t.Fatalf("Expected: %#v but didn't find", expectedEvent)
  1622  		}
  1623  	}
  1624  }
  1625  
  1626  func TestCachingDeleteEvents(t *testing.T) {
  1627  	backingStorage := &dummyStorage{}
  1628  	cacher, _, err := newTestCacher(backingStorage)
  1629  	if err != nil {
  1630  		t.Fatalf("Couldn't create cacher: %v", err)
  1631  	}
  1632  	defer cacher.Stop()
  1633  
  1634  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1635  		if err := cacher.ready.wait(context.Background()); err != nil {
  1636  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1637  		}
  1638  	}
  1639  
  1640  	fooPredicate := storage.SelectionPredicate{
  1641  		Label: labels.SelectorFromSet(map[string]string{"foo": "true"}),
  1642  		Field: fields.Everything(),
  1643  	}
  1644  	barPredicate := storage.SelectionPredicate{
  1645  		Label: labels.SelectorFromSet(map[string]string{"bar": "true"}),
  1646  		Field: fields.Everything(),
  1647  	}
  1648  
  1649  	createWatch := func(pred storage.SelectionPredicate) watch.Interface {
  1650  		w, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: pred})
  1651  		if err != nil {
  1652  			t.Fatalf("Failed to create watch: %v", err)
  1653  		}
  1654  		return w
  1655  	}
  1656  
  1657  	allEventsWatcher := createWatch(storage.Everything)
  1658  	defer allEventsWatcher.Stop()
  1659  	fooEventsWatcher := createWatch(fooPredicate)
  1660  	defer fooEventsWatcher.Stop()
  1661  	barEventsWatcher := createWatch(barPredicate)
  1662  	defer barEventsWatcher.Stop()
  1663  
  1664  	makePod := func(labels map[string]string, rv string) *examplev1.Pod {
  1665  		return &examplev1.Pod{
  1666  			ObjectMeta: metav1.ObjectMeta{
  1667  				Name:            "pod",
  1668  				Namespace:       "ns",
  1669  				Labels:          labels,
  1670  				ResourceVersion: rv,
  1671  			},
  1672  		}
  1673  	}
  1674  	pod1 := makePod(map[string]string{"foo": "true", "bar": "true"}, "1001")
  1675  	pod2 := makePod(map[string]string{"foo": "true"}, "1002")
  1676  	pod3 := makePod(map[string]string{}, "1003")
  1677  	pod4 := makePod(map[string]string{}, "1004")
  1678  	pod1DeletedAt2 := pod1.DeepCopyObject().(*examplev1.Pod)
  1679  	pod1DeletedAt2.ResourceVersion = "1002"
  1680  	pod2DeletedAt3 := pod2.DeepCopyObject().(*examplev1.Pod)
  1681  	pod2DeletedAt3.ResourceVersion = "1003"
  1682  
  1683  	allEvents := []watch.Event{
  1684  		{Type: watch.Added, Object: pod1.DeepCopy()},
  1685  		{Type: watch.Modified, Object: pod2.DeepCopy()},
  1686  		{Type: watch.Modified, Object: pod3.DeepCopy()},
  1687  		{Type: watch.Deleted, Object: pod4.DeepCopy()},
  1688  	}
  1689  	fooEvents := []watch.Event{
  1690  		{Type: watch.Added, Object: pod1.DeepCopy()},
  1691  		{Type: watch.Modified, Object: pod2.DeepCopy()},
  1692  		{Type: watch.Deleted, Object: pod2DeletedAt3.DeepCopy()},
  1693  	}
  1694  	barEvents := []watch.Event{
  1695  		{Type: watch.Added, Object: pod1.DeepCopy()},
  1696  		{Type: watch.Deleted, Object: pod1DeletedAt2.DeepCopy()},
  1697  	}
  1698  
  1699  	cacher.watchCache.Add(pod1)
  1700  	cacher.watchCache.Update(pod2)
  1701  	cacher.watchCache.Update(pod3)
  1702  	cacher.watchCache.Delete(pod4)
  1703  
  1704  	verifyEvents(t, allEventsWatcher, allEvents, true)
  1705  	verifyEvents(t, fooEventsWatcher, fooEvents, true)
  1706  	verifyEvents(t, barEventsWatcher, barEvents, true)
  1707  }
  1708  
  1709  func testCachingObjects(t *testing.T, watchersCount int) {
  1710  	backingStorage := &dummyStorage{}
  1711  	cacher, _, err := newTestCacher(backingStorage)
  1712  	if err != nil {
  1713  		t.Fatalf("Couldn't create cacher: %v", err)
  1714  	}
  1715  	defer cacher.Stop()
  1716  
  1717  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1718  		if err := cacher.ready.wait(context.Background()); err != nil {
  1719  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1720  		}
  1721  	}
  1722  
  1723  	dispatchedEvents := []*watchCacheEvent{}
  1724  	cacher.watchCache.eventHandler = func(event *watchCacheEvent) {
  1725  		dispatchedEvents = append(dispatchedEvents, event)
  1726  		cacher.processEvent(event)
  1727  	}
  1728  
  1729  	watchers := make([]watch.Interface, 0, watchersCount)
  1730  	for i := 0; i < watchersCount; i++ {
  1731  		w, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "1000", Predicate: storage.Everything})
  1732  		if err != nil {
  1733  			t.Fatalf("Failed to create watch: %v", err)
  1734  		}
  1735  		defer w.Stop()
  1736  		watchers = append(watchers, w)
  1737  	}
  1738  
  1739  	makePod := func(name, rv string) *examplev1.Pod {
  1740  		return &examplev1.Pod{
  1741  			ObjectMeta: metav1.ObjectMeta{
  1742  				Name:            name,
  1743  				Namespace:       "ns",
  1744  				ResourceVersion: rv,
  1745  			},
  1746  		}
  1747  	}
  1748  	pod1 := makePod("pod", "1001")
  1749  	pod2 := makePod("pod", "1002")
  1750  	pod3 := makePod("pod", "1003")
  1751  
  1752  	cacher.watchCache.Add(pod1)
  1753  	cacher.watchCache.Update(pod2)
  1754  	cacher.watchCache.Delete(pod3)
  1755  
  1756  	// At this point, we already have dispatchedEvents fully propagated.
  1757  
  1758  	verifyEvents := func(w watch.Interface) {
  1759  		var event watch.Event
  1760  		for index := range dispatchedEvents {
  1761  			select {
  1762  			case event = <-w.ResultChan():
  1763  			case <-time.After(wait.ForeverTestTimeout):
  1764  				t.Fatalf("timeout watiching for the event")
  1765  			}
  1766  
  1767  			var object runtime.Object
  1768  			if _, ok := event.Object.(runtime.CacheableObject); !ok {
  1769  				t.Fatalf("Object in %s event should support caching: %#v", event.Type, event.Object)
  1770  			}
  1771  			object = event.Object.(runtime.CacheableObject).GetObject()
  1772  
  1773  			if event.Type == watch.Deleted {
  1774  				resourceVersion, err := cacher.versioner.ObjectResourceVersion(cacher.watchCache.cache[index].PrevObject)
  1775  				if err != nil {
  1776  					t.Fatalf("Failed to parse resource version: %v", err)
  1777  				}
  1778  				updateResourceVersion(object, cacher.versioner, resourceVersion)
  1779  			}
  1780  
  1781  			var e runtime.Object
  1782  			switch event.Type {
  1783  			case watch.Added, watch.Modified:
  1784  				e = cacher.watchCache.cache[index].Object
  1785  			case watch.Deleted:
  1786  				e = cacher.watchCache.cache[index].PrevObject
  1787  			default:
  1788  				t.Errorf("unexpected watch event: %#v", event)
  1789  			}
  1790  			if a := object; !reflect.DeepEqual(a, e) {
  1791  				t.Errorf("event object messed up for %s: %#v, expected: %#v", event.Type, a, e)
  1792  			}
  1793  		}
  1794  	}
  1795  
  1796  	for i := range watchers {
  1797  		verifyEvents(watchers[i])
  1798  	}
  1799  }
  1800  
  1801  func TestCachingObjects(t *testing.T) {
  1802  	t.Run("single watcher", func(t *testing.T) { testCachingObjects(t, 1) })
  1803  	t.Run("many watcher", func(t *testing.T) { testCachingObjects(t, 3) })
  1804  }
  1805  
  1806  func TestCacheIntervalInvalidationStopsWatch(t *testing.T) {
  1807  	backingStorage := &dummyStorage{}
  1808  	cacher, _, err := newTestCacher(backingStorage)
  1809  	if err != nil {
  1810  		t.Fatalf("Couldn't create cacher: %v", err)
  1811  	}
  1812  	defer cacher.Stop()
  1813  
  1814  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  1815  		if err := cacher.ready.wait(context.Background()); err != nil {
  1816  			t.Fatalf("unexpected error waiting for the cache to be ready")
  1817  		}
  1818  	}
  1819  
  1820  	// Ensure there is enough budget for slow processing since
  1821  	// the entire watch cache is going to be served through the
  1822  	// interval and events won't be popped from the cacheWatcher's
  1823  	// input channel until much later.
  1824  	cacher.dispatchTimeoutBudget.returnUnused(100 * time.Millisecond)
  1825  
  1826  	// We define a custom index validator such that the interval is
  1827  	// able to serve the first bufferSize elements successfully, but
  1828  	// on trying to fill it's buffer again, the indexValidator simulates
  1829  	// an invalidation leading to the watch being closed and the number
  1830  	// of events we actually process to be bufferSize, each event of
  1831  	// type watch.Added.
  1832  	valid := true
  1833  	invalidateCacheInterval := func() {
  1834  		valid = false
  1835  	}
  1836  	once := sync.Once{}
  1837  	indexValidator := func(index int) bool {
  1838  		isValid := valid && (index >= cacher.watchCache.startIndex)
  1839  		once.Do(invalidateCacheInterval)
  1840  		return isValid
  1841  	}
  1842  	cacher.watchCache.indexValidator = indexValidator
  1843  
  1844  	makePod := func(i int) *examplev1.Pod {
  1845  		return &examplev1.Pod{
  1846  			ObjectMeta: metav1.ObjectMeta{
  1847  				Name:            fmt.Sprintf("pod-%d", 1000+i),
  1848  				Namespace:       "ns",
  1849  				ResourceVersion: fmt.Sprintf("%d", 1000+i),
  1850  			},
  1851  		}
  1852  	}
  1853  
  1854  	// 250 is arbitrary, point is to have enough elements such that
  1855  	// it generates more than bufferSize number of events allowing
  1856  	// us to simulate the invalidation of the cache interval.
  1857  	totalPods := 250
  1858  	for i := 0; i < totalPods; i++ {
  1859  		err := cacher.watchCache.Add(makePod(i))
  1860  		if err != nil {
  1861  			t.Errorf("error: %v", err)
  1862  		}
  1863  	}
  1864  	ctx, cancel := context.WithCancel(context.Background())
  1865  	defer cancel()
  1866  
  1867  	w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{
  1868  		ResourceVersion: "999",
  1869  		Predicate:       storage.Everything,
  1870  	})
  1871  	if err != nil {
  1872  		t.Fatalf("Failed to create watch: %v", err)
  1873  	}
  1874  	defer w.Stop()
  1875  
  1876  	received := 0
  1877  	resChan := w.ResultChan()
  1878  	for event := range resChan {
  1879  		received++
  1880  		t.Logf("event type: %v, events received so far: %d", event.Type, received)
  1881  		if event.Type != watch.Added {
  1882  			t.Errorf("unexpected event type, expected: %s, got: %s, event: %v", watch.Added, event.Type, event)
  1883  		}
  1884  	}
  1885  	// Since the watch is stopped after the interval is invalidated,
  1886  	// we should have processed exactly bufferSize number of elements.
  1887  	if received != bufferSize {
  1888  		t.Errorf("unexpected number of events received, expected: %d, got: %d", bufferSize+1, received)
  1889  	}
  1890  }
  1891  
  1892  func TestWaitUntilWatchCacheFreshAndForceAllEvents(t *testing.T) {
  1893  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
  1894  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true)
  1895  	forceRequestWatchProgressSupport(t)
  1896  
  1897  	scenarios := []struct {
  1898  		name               string
  1899  		opts               storage.ListOptions
  1900  		backingStorage     *dummyStorage
  1901  		verifyBackingStore func(t *testing.T, s *dummyStorage)
  1902  	}{
  1903  		{
  1904  			name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=105",
  1905  			opts: storage.ListOptions{
  1906  				Predicate: func() storage.SelectionPredicate {
  1907  					p := storage.Everything
  1908  					p.AllowWatchBookmarks = true
  1909  					return p
  1910  				}(),
  1911  				SendInitialEvents: pointer.Bool(true),
  1912  				ResourceVersion:   "105",
  1913  			},
  1914  			verifyBackingStore: func(t *testing.T, s *dummyStorage) {
  1915  				require.NotEqual(t, 0, s.getRequestWatchProgressCounter(), "expected store.RequestWatchProgressCounter to be > 0. It looks like watch progress wasn't requested!")
  1916  			},
  1917  		},
  1918  
  1919  		{
  1920  			name: "legacy: allowWatchBookmarks=false, sendInitialEvents=true, RV=unset",
  1921  			opts: storage.ListOptions{
  1922  				Predicate: func() storage.SelectionPredicate {
  1923  					p := storage.Everything
  1924  					p.AllowWatchBookmarks = false
  1925  					return p
  1926  				}(),
  1927  				SendInitialEvents: pointer.Bool(true),
  1928  			},
  1929  			backingStorage: func() *dummyStorage {
  1930  				hasBeenPrimed := false
  1931  				s := &dummyStorage{}
  1932  				s.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
  1933  					listAccessor, err := meta.ListAccessor(listObj)
  1934  					if err != nil {
  1935  						return err
  1936  					}
  1937  					// the first call to this function
  1938  					// primes the cacher
  1939  					if !hasBeenPrimed {
  1940  						listAccessor.SetResourceVersion("100")
  1941  						hasBeenPrimed = true
  1942  						return nil
  1943  					}
  1944  					listAccessor.SetResourceVersion("105")
  1945  					return nil
  1946  				}
  1947  				return s
  1948  			}(),
  1949  			verifyBackingStore: func(t *testing.T, s *dummyStorage) {
  1950  				require.NotEqual(t, 0, s.getRequestWatchProgressCounter(), "expected store.RequestWatchProgressCounter to be > 0. It looks like watch progress wasn't requested!")
  1951  			},
  1952  		},
  1953  
  1954  		{
  1955  			name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=unset",
  1956  			opts: storage.ListOptions{
  1957  				Predicate: func() storage.SelectionPredicate {
  1958  					p := storage.Everything
  1959  					p.AllowWatchBookmarks = true
  1960  					return p
  1961  				}(),
  1962  				SendInitialEvents: pointer.Bool(true),
  1963  			},
  1964  			backingStorage: func() *dummyStorage {
  1965  				hasBeenPrimed := false
  1966  				s := &dummyStorage{}
  1967  				s.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
  1968  					listAccessor, err := meta.ListAccessor(listObj)
  1969  					if err != nil {
  1970  						return err
  1971  					}
  1972  					// the first call to this function
  1973  					// primes the cacher
  1974  					if !hasBeenPrimed {
  1975  						listAccessor.SetResourceVersion("100")
  1976  						hasBeenPrimed = true
  1977  						return nil
  1978  					}
  1979  					listAccessor.SetResourceVersion("105")
  1980  					return nil
  1981  				}
  1982  				return s
  1983  			}(),
  1984  			verifyBackingStore: func(t *testing.T, s *dummyStorage) {
  1985  				require.NotEqual(t, 0, s.getRequestWatchProgressCounter(), "expected store.RequestWatchProgressCounter to be > 0. It looks like watch progress wasn't requested!")
  1986  			},
  1987  		},
  1988  	}
  1989  
  1990  	for _, scenario := range scenarios {
  1991  		t.Run(scenario.name, func(t *testing.T) {
  1992  			t.Parallel()
  1993  			var backingStorage *dummyStorage
  1994  			if scenario.backingStorage != nil {
  1995  				backingStorage = scenario.backingStorage
  1996  			} else {
  1997  				backingStorage = &dummyStorage{}
  1998  			}
  1999  			cacher, _, err := newTestCacher(backingStorage)
  2000  			if err != nil {
  2001  				t.Fatalf("Couldn't create cacher: %v", err)
  2002  			}
  2003  			defer cacher.Stop()
  2004  
  2005  			if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  2006  				if err := cacher.ready.wait(context.Background()); err != nil {
  2007  					t.Fatalf("unexpected error waiting for the cache to be ready")
  2008  				}
  2009  			}
  2010  
  2011  			w, err := cacher.Watch(context.Background(), "pods/ns", scenario.opts)
  2012  			require.NoError(t, err, "failed to create watch: %v")
  2013  			defer w.Stop()
  2014  			var expectedErr *apierrors.StatusError
  2015  			if !errors.As(storage.NewTooLargeResourceVersionError(105, 100, resourceVersionTooHighRetrySeconds), &expectedErr) {
  2016  				t.Fatalf("Unable to convert NewTooLargeResourceVersionError to apierrors.StatusError")
  2017  			}
  2018  			verifyEvents(t, w, []watch.Event{
  2019  				{
  2020  					Type: watch.Error,
  2021  					Object: &metav1.Status{
  2022  						Status:  metav1.StatusFailure,
  2023  						Message: expectedErr.Error(),
  2024  						Details: expectedErr.ErrStatus.Details,
  2025  						Reason:  metav1.StatusReasonTimeout,
  2026  						Code:    504,
  2027  					},
  2028  				},
  2029  			}, true)
  2030  
  2031  			go func(t *testing.T) {
  2032  				err := cacher.watchCache.Add(makeTestPodDetails("pod1", 105, "node1", map[string]string{"label": "value1"}))
  2033  				require.NoError(t, err, "failed adding a pod to the watchCache")
  2034  			}(t)
  2035  			w, err = cacher.Watch(context.Background(), "pods/ns", scenario.opts)
  2036  			require.NoError(t, err, "failed to create watch: %v")
  2037  			defer w.Stop()
  2038  			verifyEvents(t, w, []watch.Event{
  2039  				{
  2040  					Type:   watch.Added,
  2041  					Object: makeTestPodDetails("pod1", 105, "node1", map[string]string{"label": "value1"}),
  2042  				},
  2043  			}, true)
  2044  			if scenario.verifyBackingStore != nil {
  2045  				scenario.verifyBackingStore(t, backingStorage)
  2046  			}
  2047  		})
  2048  	}
  2049  }
  2050  
  2051  type fakeStorage struct {
  2052  	pods []example.Pod
  2053  	storage.Interface
  2054  }
  2055  
  2056  func newObjectStorage(fakePods []example.Pod) *fakeStorage {
  2057  	return &fakeStorage{
  2058  		pods: fakePods,
  2059  	}
  2060  }
  2061  
  2062  func (m fakeStorage) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
  2063  	podList := listObj.(*example.PodList)
  2064  	podList.ListMeta = metav1.ListMeta{ResourceVersion: "12345"}
  2065  	podList.Items = m.pods
  2066  	return nil
  2067  }
  2068  func (m fakeStorage) Watch(_ context.Context, _ string, _ storage.ListOptions) (watch.Interface, error) {
  2069  	return newDummyWatch(), nil
  2070  }
  2071  
  2072  func BenchmarkCacher_GetList(b *testing.B) {
  2073  	testCases := []struct {
  2074  		totalObjectNum  int
  2075  		expectObjectNum int
  2076  	}{
  2077  		{
  2078  			totalObjectNum:  5000,
  2079  			expectObjectNum: 50,
  2080  		},
  2081  		{
  2082  			totalObjectNum:  5000,
  2083  			expectObjectNum: 500,
  2084  		},
  2085  		{
  2086  			totalObjectNum:  5000,
  2087  			expectObjectNum: 1000,
  2088  		},
  2089  		{
  2090  			totalObjectNum:  5000,
  2091  			expectObjectNum: 2500,
  2092  		},
  2093  		{
  2094  			totalObjectNum:  5000,
  2095  			expectObjectNum: 5000,
  2096  		},
  2097  	}
  2098  	for _, tc := range testCases {
  2099  		b.Run(
  2100  			fmt.Sprintf("totalObjectNum=%d, expectObjectNum=%d", tc.totalObjectNum, tc.expectObjectNum),
  2101  			func(b *testing.B) {
  2102  				// create sample pods
  2103  				fakePods := make([]example.Pod, tc.totalObjectNum, tc.totalObjectNum)
  2104  				for i := range fakePods {
  2105  					fakePods[i].Namespace = "default"
  2106  					fakePods[i].Name = fmt.Sprintf("pod-%d", i)
  2107  					fakePods[i].ResourceVersion = strconv.Itoa(i)
  2108  					if i%(tc.totalObjectNum/tc.expectObjectNum) == 0 {
  2109  						fakePods[i].Spec.NodeName = "node-0"
  2110  					}
  2111  					data := make([]byte, 1024*2, 1024*2) // 2k labels
  2112  					rand.Read(data)
  2113  					fakePods[i].Spec.NodeSelector = map[string]string{
  2114  						"key": string(data),
  2115  					}
  2116  				}
  2117  
  2118  				// build test cacher
  2119  				cacher, _, err := newTestCacher(newObjectStorage(fakePods))
  2120  				if err != nil {
  2121  					b.Fatalf("new cacher: %v", err)
  2122  				}
  2123  				defer cacher.Stop()
  2124  
  2125  				// prepare result and pred
  2126  				parsedField, err := fields.ParseSelector("spec.nodeName=node-0")
  2127  				if err != nil {
  2128  					b.Fatalf("parse selector: %v", err)
  2129  				}
  2130  				pred := storage.SelectionPredicate{
  2131  					Label: labels.Everything(),
  2132  					Field: parsedField,
  2133  				}
  2134  
  2135  				// now we start benchmarking
  2136  				b.ResetTimer()
  2137  				for i := 0; i < b.N; i++ {
  2138  					result := &example.PodList{}
  2139  					err = cacher.GetList(context.TODO(), "pods", storage.ListOptions{
  2140  						Predicate:       pred,
  2141  						Recursive:       true,
  2142  						ResourceVersion: "12345",
  2143  					}, result)
  2144  					if err != nil {
  2145  						b.Fatalf("GetList cache: %v", err)
  2146  					}
  2147  					if len(result.Items) != tc.expectObjectNum {
  2148  						b.Fatalf("expect %d but got %d", tc.expectObjectNum, len(result.Items))
  2149  					}
  2150  				}
  2151  			})
  2152  	}
  2153  }
  2154  
  2155  // TestWatchListIsSynchronisedWhenNoEventsFromStoreReceived makes sure that
  2156  // a bookmark event will be delivered even if the cacher has not received an event.
  2157  func TestWatchListIsSynchronisedWhenNoEventsFromStoreReceived(t *testing.T) {
  2158  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
  2159  	backingStorage := &dummyStorage{}
  2160  	cacher, _, err := newTestCacher(backingStorage)
  2161  	require.NoError(t, err, "failed to create cacher")
  2162  	defer cacher.Stop()
  2163  
  2164  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  2165  		if err := cacher.ready.wait(context.Background()); err != nil {
  2166  			t.Fatalf("unexpected error waiting for the cache to be ready")
  2167  		}
  2168  	}
  2169  
  2170  	pred := storage.Everything
  2171  	pred.AllowWatchBookmarks = true
  2172  	opts := storage.ListOptions{
  2173  		Predicate:         pred,
  2174  		SendInitialEvents: pointer.Bool(true),
  2175  	}
  2176  	w, err := cacher.Watch(context.Background(), "pods/ns", opts)
  2177  	require.NoError(t, err, "failed to create watch: %v")
  2178  	defer w.Stop()
  2179  
  2180  	verifyEvents(t, w, []watch.Event{
  2181  		{Type: watch.Bookmark, Object: &example.Pod{
  2182  			ObjectMeta: metav1.ObjectMeta{
  2183  				ResourceVersion: "100",
  2184  				Annotations:     map[string]string{metav1.InitialEventsAnnotationKey: "true"},
  2185  			},
  2186  		}},
  2187  	}, true)
  2188  }
  2189  
  2190  func TestForgetWatcher(t *testing.T) {
  2191  	backingStorage := &dummyStorage{}
  2192  	cacher, _, err := newTestCacher(backingStorage)
  2193  	require.NoError(t, err)
  2194  	defer cacher.Stop()
  2195  
  2196  	if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  2197  		if err := cacher.ready.wait(context.Background()); err != nil {
  2198  			t.Fatalf("unexpected error waiting for the cache to be ready")
  2199  		}
  2200  	}
  2201  
  2202  	assertCacherInternalState := func(expectedWatchersCounter, expectedValueWatchersCounter int) {
  2203  		cacher.Lock()
  2204  		defer cacher.Unlock()
  2205  
  2206  		require.Len(t, cacher.watchers.allWatchers, expectedWatchersCounter)
  2207  		require.Len(t, cacher.watchers.valueWatchers, expectedValueWatchersCounter)
  2208  	}
  2209  	assertCacherInternalState(0, 0)
  2210  
  2211  	var forgetWatcherFn func(bool)
  2212  	var forgetCounter int
  2213  	forgetWatcherWrapped := func(drainWatcher bool) {
  2214  		forgetCounter++
  2215  		forgetWatcherFn(drainWatcher)
  2216  	}
  2217  	w := newCacheWatcher(
  2218  		0,
  2219  		func(_ string, _ labels.Set, _ fields.Set) bool { return true },
  2220  		nil,
  2221  		storage.APIObjectVersioner{},
  2222  		testingclock.NewFakeClock(time.Now()).Now().Add(2*time.Minute),
  2223  		true,
  2224  		schema.GroupResource{Resource: "pods"},
  2225  		"1",
  2226  	)
  2227  	forgetWatcherFn = forgetWatcher(cacher, w, 0, namespacedName{}, "", false)
  2228  	addWatcher := func(w *cacheWatcher) {
  2229  		cacher.Lock()
  2230  		defer cacher.Unlock()
  2231  
  2232  		cacher.watchers.addWatcher(w, 0, namespacedName{}, "", false)
  2233  	}
  2234  
  2235  	addWatcher(w)
  2236  	assertCacherInternalState(1, 0)
  2237  
  2238  	forgetWatcherWrapped(false)
  2239  	assertCacherInternalState(0, 0)
  2240  	require.Equal(t, 1, forgetCounter)
  2241  
  2242  	forgetWatcherWrapped(false)
  2243  	assertCacherInternalState(0, 0)
  2244  	require.Equal(t, 2, forgetCounter)
  2245  }
  2246  
  2247  // TestGetWatchCacheResourceVersion test the following cases:
  2248  //
  2249  // +-----------------+---------------------+-----------------------+
  2250  // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents     |
  2251  // +=================+=====================+=======================+
  2252  // | Unset           | true/false          | nil/true/false        |
  2253  // | 0               | true/false          | nil/true/false        |
  2254  // | 95             | true/false          | nil/true/false         |
  2255  // +-----------------+---------------------+-----------------------+
  2256  // where:
  2257  // - false indicates the value of the param was set to "false" by a test case
  2258  // - true  indicates the value of the param was set to "true" by a test case
  2259  func TestGetWatchCacheResourceVersion(t *testing.T) {
  2260  	listOptions := func(allowBookmarks bool, sendInitialEvents *bool, rv string) storage.ListOptions {
  2261  		p := storage.Everything
  2262  		p.AllowWatchBookmarks = allowBookmarks
  2263  
  2264  		opts := storage.ListOptions{}
  2265  		opts.Predicate = p
  2266  		opts.SendInitialEvents = sendInitialEvents
  2267  		opts.ResourceVersion = rv
  2268  		return opts
  2269  	}
  2270  
  2271  	scenarios := []struct {
  2272  		name string
  2273  		opts storage.ListOptions
  2274  
  2275  		expectedWatchResourceVersion int
  2276  	}{
  2277  		// +-----------------+---------------------+-----------------------+
  2278  		// | ResourceVersion | AllowWatchBookmarks | SendInitialEvents     |
  2279  		// +=================+=====================+=======================+
  2280  		// | Unset           | true/false          | nil/true/false        |
  2281  		// +-----------------+---------------------+-----------------------+
  2282  		{
  2283  			name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=nil",
  2284  			opts: listOptions(true, nil, ""),
  2285  			// Expecting RV 0, due to https://github.com/kubernetes/kubernetes/pull/123935 reverted to serving those requests from watch cache.
  2286  			// Set to 100, when WatchFromStorageWithoutResourceVersion is set to true.
  2287  			expectedWatchResourceVersion: 0,
  2288  		},
  2289  		{
  2290  			name:                         "RV=unset, allowWatchBookmarks=true, sendInitialEvents=true",
  2291  			opts:                         listOptions(true, pointer.Bool(true), ""),
  2292  			expectedWatchResourceVersion: 100,
  2293  		},
  2294  		{
  2295  			name:                         "RV=unset, allowWatchBookmarks=true, sendInitialEvents=false",
  2296  			opts:                         listOptions(true, pointer.Bool(false), ""),
  2297  			expectedWatchResourceVersion: 100,
  2298  		},
  2299  		{
  2300  			name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=nil",
  2301  			opts: listOptions(false, nil, ""),
  2302  			// Expecting RV 0, due to https://github.com/kubernetes/kubernetes/pull/123935 reverted to serving those requests from watch cache.
  2303  			// Set to 100, when WatchFromStorageWithoutResourceVersion is set to true.
  2304  			expectedWatchResourceVersion: 0,
  2305  		},
  2306  		{
  2307  			name:                         "RV=unset, allowWatchBookmarks=false, sendInitialEvents=true, legacy",
  2308  			opts:                         listOptions(false, pointer.Bool(true), ""),
  2309  			expectedWatchResourceVersion: 100,
  2310  		},
  2311  		{
  2312  			name:                         "RV=unset, allowWatchBookmarks=false, sendInitialEvents=false",
  2313  			opts:                         listOptions(false, pointer.Bool(false), ""),
  2314  			expectedWatchResourceVersion: 100,
  2315  		},
  2316  		// +-----------------+---------------------+-----------------------+
  2317  		// | ResourceVersion | AllowWatchBookmarks | SendInitialEvents     |
  2318  		// +=================+=====================+=======================+
  2319  		// | 0               | true/false          | nil/true/false        |
  2320  		// +-----------------+---------------------+-----------------------+
  2321  		{
  2322  			name:                         "RV=0, allowWatchBookmarks=true, sendInitialEvents=nil",
  2323  			opts:                         listOptions(true, nil, "0"),
  2324  			expectedWatchResourceVersion: 0,
  2325  		},
  2326  		{
  2327  			name:                         "RV=0, allowWatchBookmarks=true, sendInitialEvents=true",
  2328  			opts:                         listOptions(true, pointer.Bool(true), "0"),
  2329  			expectedWatchResourceVersion: 0,
  2330  		},
  2331  		{
  2332  			name:                         "RV=0, allowWatchBookmarks=true, sendInitialEvents=false",
  2333  			opts:                         listOptions(true, pointer.Bool(false), "0"),
  2334  			expectedWatchResourceVersion: 0,
  2335  		},
  2336  		{
  2337  			name:                         "RV=0, allowWatchBookmarks=false, sendInitialEvents=nil",
  2338  			opts:                         listOptions(false, nil, "0"),
  2339  			expectedWatchResourceVersion: 0,
  2340  		},
  2341  		{
  2342  			name:                         "RV=0, allowWatchBookmarks=false, sendInitialEvents=true",
  2343  			opts:                         listOptions(false, pointer.Bool(true), "0"),
  2344  			expectedWatchResourceVersion: 0,
  2345  		},
  2346  		{
  2347  			name:                         "RV=0, allowWatchBookmarks=false, sendInitialEvents=false",
  2348  			opts:                         listOptions(false, pointer.Bool(false), "0"),
  2349  			expectedWatchResourceVersion: 0,
  2350  		},
  2351  		// +-----------------+---------------------+-----------------------+
  2352  		// | ResourceVersion | AllowWatchBookmarks | SendInitialEvents     |
  2353  		// +=================+=====================+=======================+
  2354  		// | 95             | true/false          | nil/true/false         |
  2355  		// +-----------------+---------------------+-----------------------+
  2356  		{
  2357  			name:                         "RV=95, allowWatchBookmarks=true, sendInitialEvents=nil",
  2358  			opts:                         listOptions(true, nil, "95"),
  2359  			expectedWatchResourceVersion: 95,
  2360  		},
  2361  		{
  2362  			name:                         "RV=95, allowWatchBookmarks=true, sendInitialEvents=true",
  2363  			opts:                         listOptions(true, pointer.Bool(true), "95"),
  2364  			expectedWatchResourceVersion: 95,
  2365  		},
  2366  		{
  2367  			name:                         "RV=95, allowWatchBookmarks=true, sendInitialEvents=false",
  2368  			opts:                         listOptions(true, pointer.Bool(false), "95"),
  2369  			expectedWatchResourceVersion: 95,
  2370  		},
  2371  		{
  2372  			name:                         "RV=95, allowWatchBookmarks=false, sendInitialEvents=nil",
  2373  			opts:                         listOptions(false, nil, "95"),
  2374  			expectedWatchResourceVersion: 95,
  2375  		},
  2376  		{
  2377  			name:                         "RV=95, allowWatchBookmarks=false, sendInitialEvents=true",
  2378  			opts:                         listOptions(false, pointer.Bool(true), "95"),
  2379  			expectedWatchResourceVersion: 95,
  2380  		},
  2381  		{
  2382  			name:                         "RV=95, allowWatchBookmarks=false, sendInitialEvents=false",
  2383  			opts:                         listOptions(false, pointer.Bool(false), "95"),
  2384  			expectedWatchResourceVersion: 95,
  2385  		},
  2386  	}
  2387  
  2388  	for _, scenario := range scenarios {
  2389  		t.Run(scenario.name, func(t *testing.T) {
  2390  			backingStorage := &dummyStorage{}
  2391  			cacher, _, err := newTestCacher(backingStorage)
  2392  			require.NoError(t, err, "couldn't create cacher")
  2393  			defer cacher.Stop()
  2394  
  2395  			parsedResourceVersion := 0
  2396  			if len(scenario.opts.ResourceVersion) > 0 {
  2397  				parsedResourceVersion, err = strconv.Atoi(scenario.opts.ResourceVersion)
  2398  				require.NoError(t, err)
  2399  			}
  2400  
  2401  			actualResourceVersion, err := cacher.getWatchCacheResourceVersion(context.TODO(), uint64(parsedResourceVersion), scenario.opts)
  2402  			require.NoError(t, err)
  2403  			require.Equal(t, uint64(scenario.expectedWatchResourceVersion), actualResourceVersion, "received unexpected ResourceVersion")
  2404  		})
  2405  	}
  2406  }
  2407  
  2408  // TestGetBookmarkAfterResourceVersionLockedFunc test the following cases:
  2409  //
  2410  // +-----------------+---------------------+-----------------------+
  2411  // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents     |
  2412  // +=================+=====================+=======================+
  2413  // | Unset           | true/false          | nil/true/false        |
  2414  // | 0               | true/false          | nil/true/false        |
  2415  // | 95             | true/false          | nil/true/false         |
  2416  // +-----------------+---------------------+-----------------------+
  2417  // where:
  2418  // - false indicates the value of the param was set to "false" by a test case
  2419  // - true  indicates the value of the param was set to "true" by a test case
  2420  func TestGetBookmarkAfterResourceVersionLockedFunc(t *testing.T) {
  2421  	listOptions := func(allowBookmarks bool, sendInitialEvents *bool, rv string) storage.ListOptions {
  2422  		p := storage.Everything
  2423  		p.AllowWatchBookmarks = allowBookmarks
  2424  
  2425  		opts := storage.ListOptions{}
  2426  		opts.Predicate = p
  2427  		opts.SendInitialEvents = sendInitialEvents
  2428  		opts.ResourceVersion = rv
  2429  		return opts
  2430  	}
  2431  
  2432  	scenarios := []struct {
  2433  		name                      string
  2434  		opts                      storage.ListOptions
  2435  		requiredResourceVersion   int
  2436  		watchCacheResourceVersion int
  2437  
  2438  		expectedBookmarkResourceVersion int
  2439  	}{
  2440  		// +-----------------+---------------------+-----------------------+
  2441  		// | ResourceVersion | AllowWatchBookmarks | SendInitialEvents     |
  2442  		// +=================+=====================+=======================+
  2443  		// | Unset           | true/false          | nil/true/false        |
  2444  		// +-----------------+---------------------+-----------------------+
  2445  		{
  2446  			name:                            "RV=unset, allowWatchBookmarks=true, sendInitialEvents=nil",
  2447  			opts:                            listOptions(true, nil, ""),
  2448  			requiredResourceVersion:         100,
  2449  			watchCacheResourceVersion:       99,
  2450  			expectedBookmarkResourceVersion: 0,
  2451  		},
  2452  		{
  2453  			name:                            "RV=unset, allowWatchBookmarks=true, sendInitialEvents=true",
  2454  			opts:                            listOptions(true, pointer.Bool(true), ""),
  2455  			requiredResourceVersion:         100,
  2456  			watchCacheResourceVersion:       99,
  2457  			expectedBookmarkResourceVersion: 100,
  2458  		},
  2459  		{
  2460  			name:                            "RV=unset, allowWatchBookmarks=true, sendInitialEvents=false",
  2461  			opts:                            listOptions(true, pointer.Bool(false), ""),
  2462  			requiredResourceVersion:         100,
  2463  			watchCacheResourceVersion:       99,
  2464  			expectedBookmarkResourceVersion: 0,
  2465  		},
  2466  		{
  2467  			name:                            "RV=unset, allowWatchBookmarks=false, sendInitialEvents=nil",
  2468  			opts:                            listOptions(false, nil, ""),
  2469  			requiredResourceVersion:         100,
  2470  			watchCacheResourceVersion:       99,
  2471  			expectedBookmarkResourceVersion: 0,
  2472  		},
  2473  		{
  2474  			name:                            "RV=unset, allowWatchBookmarks=false, sendInitialEvents=true",
  2475  			opts:                            listOptions(false, pointer.Bool(true), ""),
  2476  			requiredResourceVersion:         100,
  2477  			watchCacheResourceVersion:       99,
  2478  			expectedBookmarkResourceVersion: 0,
  2479  		},
  2480  		{
  2481  			name:                            "RV=unset, allowWatchBookmarks=false, sendInitialEvents=false",
  2482  			opts:                            listOptions(false, pointer.Bool(false), ""),
  2483  			requiredResourceVersion:         100,
  2484  			watchCacheResourceVersion:       99,
  2485  			expectedBookmarkResourceVersion: 0,
  2486  		},
  2487  		// +-----------------+---------------------+-----------------------+
  2488  		// | ResourceVersion | AllowWatchBookmarks | SendInitialEvents     |
  2489  		// +=================+=====================+=======================+
  2490  		// | 0               | true/false          | nil/true/false        |
  2491  		// +-----------------+---------------------+-----------------------+
  2492  		{
  2493  			name:                            "RV=0, allowWatchBookmarks=true, sendInitialEvents=nil",
  2494  			opts:                            listOptions(true, nil, "0"),
  2495  			requiredResourceVersion:         0,
  2496  			watchCacheResourceVersion:       99,
  2497  			expectedBookmarkResourceVersion: 0,
  2498  		},
  2499  		{
  2500  			name:                            "RV=0, allowWatchBookmarks=true, sendInitialEvents=true",
  2501  			opts:                            listOptions(true, pointer.Bool(true), "0"),
  2502  			requiredResourceVersion:         0,
  2503  			watchCacheResourceVersion:       99,
  2504  			expectedBookmarkResourceVersion: 99,
  2505  		},
  2506  		{
  2507  			name:                            "RV=0, allowWatchBookmarks=true, sendInitialEvents=false",
  2508  			opts:                            listOptions(true, pointer.Bool(false), "0"),
  2509  			requiredResourceVersion:         0,
  2510  			watchCacheResourceVersion:       99,
  2511  			expectedBookmarkResourceVersion: 0,
  2512  		},
  2513  		{
  2514  			name:                            "RV=0, allowWatchBookmarks=false, sendInitialEvents=nil",
  2515  			opts:                            listOptions(false, nil, "0"),
  2516  			requiredResourceVersion:         0,
  2517  			watchCacheResourceVersion:       99,
  2518  			expectedBookmarkResourceVersion: 0,
  2519  		},
  2520  		{
  2521  			name:                            "RV=0, allowWatchBookmarks=false, sendInitialEvents=true",
  2522  			opts:                            listOptions(false, pointer.Bool(true), "0"),
  2523  			requiredResourceVersion:         0,
  2524  			watchCacheResourceVersion:       99,
  2525  			expectedBookmarkResourceVersion: 0,
  2526  		},
  2527  		{
  2528  			name:                            "RV=0, allowWatchBookmarks=false, sendInitialEvents=false",
  2529  			opts:                            listOptions(false, pointer.Bool(false), "0"),
  2530  			requiredResourceVersion:         0,
  2531  			watchCacheResourceVersion:       99,
  2532  			expectedBookmarkResourceVersion: 0,
  2533  		},
  2534  		// +-----------------+---------------------+-----------------------+
  2535  		// | ResourceVersion | AllowWatchBookmarks | SendInitialEvents     |
  2536  		// +=================+=====================+=======================+
  2537  		// | 95             | true/false          | nil/true/false         |
  2538  		// +-----------------+---------------------+-----------------------+
  2539  		{
  2540  			name:                            "RV=95, allowWatchBookmarks=true, sendInitialEvents=nil",
  2541  			opts:                            listOptions(true, nil, "95"),
  2542  			requiredResourceVersion:         0,
  2543  			watchCacheResourceVersion:       99,
  2544  			expectedBookmarkResourceVersion: 0,
  2545  		},
  2546  		{
  2547  			name:                            "RV=95, allowWatchBookmarks=true, sendInitialEvents=true",
  2548  			opts:                            listOptions(true, pointer.Bool(true), "95"),
  2549  			requiredResourceVersion:         0,
  2550  			watchCacheResourceVersion:       99,
  2551  			expectedBookmarkResourceVersion: 95,
  2552  		},
  2553  		{
  2554  			name:                            "RV=95, allowWatchBookmarks=true, sendInitialEvents=false",
  2555  			opts:                            listOptions(true, pointer.Bool(false), "95"),
  2556  			requiredResourceVersion:         0,
  2557  			watchCacheResourceVersion:       99,
  2558  			expectedBookmarkResourceVersion: 0,
  2559  		},
  2560  		{
  2561  			name:                            "RV=95, allowWatchBookmarks=false, sendInitialEvents=nil",
  2562  			opts:                            listOptions(false, nil, "95"),
  2563  			requiredResourceVersion:         100,
  2564  			watchCacheResourceVersion:       99,
  2565  			expectedBookmarkResourceVersion: 0,
  2566  		},
  2567  		{
  2568  			name:                            "RV=95, allowWatchBookmarks=false, sendInitialEvents=true",
  2569  			opts:                            listOptions(false, pointer.Bool(true), "95"),
  2570  			requiredResourceVersion:         0,
  2571  			watchCacheResourceVersion:       99,
  2572  			expectedBookmarkResourceVersion: 0,
  2573  		},
  2574  		{
  2575  			name:                            "RV=95, allowWatchBookmarks=false, sendInitialEvents=false",
  2576  			opts:                            listOptions(false, pointer.Bool(false), "95"),
  2577  			requiredResourceVersion:         0,
  2578  			watchCacheResourceVersion:       99,
  2579  			expectedBookmarkResourceVersion: 0,
  2580  		},
  2581  	}
  2582  	for _, scenario := range scenarios {
  2583  		t.Run(scenario.name, func(t *testing.T) {
  2584  			backingStorage := &dummyStorage{}
  2585  			cacher, _, err := newTestCacher(backingStorage)
  2586  			require.NoError(t, err, "couldn't create cacher")
  2587  
  2588  			defer cacher.Stop()
  2589  
  2590  			if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  2591  				if err := cacher.ready.wait(context.Background()); err != nil {
  2592  					t.Fatalf("unexpected error waiting for the cache to be ready")
  2593  				}
  2594  			}
  2595  
  2596  			cacher.watchCache.UpdateResourceVersion(fmt.Sprintf("%d", scenario.watchCacheResourceVersion))
  2597  			parsedResourceVersion := 0
  2598  			if len(scenario.opts.ResourceVersion) > 0 {
  2599  				parsedResourceVersion, err = strconv.Atoi(scenario.opts.ResourceVersion)
  2600  				require.NoError(t, err)
  2601  			}
  2602  
  2603  			getBookMarkFn, err := cacher.getBookmarkAfterResourceVersionLockedFunc(uint64(parsedResourceVersion), uint64(scenario.requiredResourceVersion), scenario.opts)
  2604  			require.NoError(t, err)
  2605  			cacher.watchCache.RLock()
  2606  			defer cacher.watchCache.RUnlock()
  2607  			getBookMarkResourceVersion := getBookMarkFn()
  2608  			require.Equal(t, uint64(scenario.expectedBookmarkResourceVersion), getBookMarkResourceVersion, "received unexpected ResourceVersion")
  2609  		})
  2610  	}
  2611  }
  2612  
  2613  func TestWatchStreamSeparation(t *testing.T) {
  2614  	server, etcdStorage := newEtcdTestStorage(t, etcd3testing.PathPrefix())
  2615  	t.Cleanup(func() {
  2616  		server.Terminate(t)
  2617  	})
  2618  	setupOpts := &setupOptions{}
  2619  	withDefaults(setupOpts)
  2620  	config := Config{
  2621  		Storage:        etcdStorage,
  2622  		Versioner:      storage.APIObjectVersioner{},
  2623  		GroupResource:  schema.GroupResource{Resource: "pods"},
  2624  		ResourcePrefix: setupOpts.resourcePrefix,
  2625  		KeyFunc:        setupOpts.keyFunc,
  2626  		GetAttrsFunc:   GetPodAttrs,
  2627  		NewFunc:        newPod,
  2628  		NewListFunc:    newPodList,
  2629  		IndexerFuncs:   setupOpts.indexerFuncs,
  2630  		Codec:          codecs.LegacyCodec(examplev1.SchemeGroupVersion),
  2631  		Clock:          setupOpts.clock,
  2632  	}
  2633  	tcs := []struct {
  2634  		name                         string
  2635  		separateCacheWatchRPC        bool
  2636  		useWatchCacheContextMetadata bool
  2637  		expectBookmarkOnWatchCache   bool
  2638  		expectBookmarkOnEtcd         bool
  2639  	}{
  2640  		{
  2641  			name:                       "common RPC > both get bookmarks",
  2642  			separateCacheWatchRPC:      false,
  2643  			expectBookmarkOnEtcd:       true,
  2644  			expectBookmarkOnWatchCache: true,
  2645  		},
  2646  		{
  2647  			name:                         "common RPC & watch cache context > both get bookmarks",
  2648  			separateCacheWatchRPC:        false,
  2649  			useWatchCacheContextMetadata: true,
  2650  			expectBookmarkOnEtcd:         true,
  2651  			expectBookmarkOnWatchCache:   true,
  2652  		},
  2653  		{
  2654  			name:                       "separate RPC > only etcd gets bookmarks",
  2655  			separateCacheWatchRPC:      true,
  2656  			expectBookmarkOnEtcd:       true,
  2657  			expectBookmarkOnWatchCache: false,
  2658  		},
  2659  		{
  2660  			name:                         "separate RPC & watch cache context > only watch cache gets bookmarks",
  2661  			separateCacheWatchRPC:        true,
  2662  			useWatchCacheContextMetadata: true,
  2663  			expectBookmarkOnEtcd:         false,
  2664  			expectBookmarkOnWatchCache:   true,
  2665  		},
  2666  	}
  2667  	for _, tc := range tcs {
  2668  		t.Run(tc.name, func(t *testing.T) {
  2669  			featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SeparateCacheWatchRPC, tc.separateCacheWatchRPC)
  2670  			cacher, err := NewCacherFromConfig(config)
  2671  			if err != nil {
  2672  				t.Fatalf("Failed to initialize cacher: %v", err)
  2673  			}
  2674  
  2675  			if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
  2676  				if err := cacher.ready.wait(context.TODO()); err != nil {
  2677  					t.Fatalf("unexpected error waiting for the cache to be ready")
  2678  				}
  2679  			}
  2680  
  2681  			getCacherRV := func() uint64 {
  2682  				cacher.watchCache.RLock()
  2683  				defer cacher.watchCache.RUnlock()
  2684  				return cacher.watchCache.resourceVersion
  2685  			}
  2686  			waitContext, cancel := context.WithTimeout(context.Background(), 2*time.Second)
  2687  			defer cancel()
  2688  			waitForEtcdBookmark := watchAndWaitForBookmark(t, waitContext, cacher.storage)
  2689  
  2690  			var out example.Pod
  2691  			err = cacher.Create(context.Background(), "foo", &example.Pod{}, &out, 0)
  2692  			if err != nil {
  2693  				t.Fatal(err)
  2694  			}
  2695  			err = cacher.Delete(context.Background(), "foo", &out, nil, storage.ValidateAllObjectFunc, &example.Pod{})
  2696  			if err != nil {
  2697  				t.Fatal(err)
  2698  			}
  2699  			versioner := storage.APIObjectVersioner{}
  2700  			var lastResourceVersion uint64
  2701  			lastResourceVersion, err = versioner.ObjectResourceVersion(&out)
  2702  			if err != nil {
  2703  				t.Fatal(err)
  2704  			}
  2705  
  2706  			var contextMetadata metadata.MD
  2707  			if tc.useWatchCacheContextMetadata {
  2708  				contextMetadata = cacher.watchCache.waitingUntilFresh.contextMetadata
  2709  			}
  2710  			// For the first 100ms from watch creation, watch progress requests are ignored.
  2711  			time.Sleep(200 * time.Millisecond)
  2712  			err = cacher.storage.RequestWatchProgress(metadata.NewOutgoingContext(context.Background(), contextMetadata))
  2713  			if err != nil {
  2714  				t.Fatal(err)
  2715  			}
  2716  			// Give time for bookmark to arrive
  2717  			time.Sleep(time.Second)
  2718  
  2719  			etcdWatchResourceVersion := waitForEtcdBookmark()
  2720  			gotEtcdWatchBookmark := etcdWatchResourceVersion == lastResourceVersion
  2721  			if gotEtcdWatchBookmark != tc.expectBookmarkOnEtcd {
  2722  				t.Errorf("Unexpected etcd bookmark check result, rv: %d, lastRV: %d, wantMatching: %v", etcdWatchResourceVersion, lastResourceVersion, tc.expectBookmarkOnEtcd)
  2723  			}
  2724  
  2725  			watchCacheResourceVersion := getCacherRV()
  2726  			cacherGotBookmark := watchCacheResourceVersion == lastResourceVersion
  2727  			if cacherGotBookmark != tc.expectBookmarkOnWatchCache {
  2728  				t.Errorf("Unexpected watch cache bookmark check result, rv: %d, lastRV: %d, wantMatching: %v", watchCacheResourceVersion, lastResourceVersion, tc.expectBookmarkOnWatchCache)
  2729  			}
  2730  		})
  2731  	}
  2732  }
  2733  
  2734  func TestComputeListLimit(t *testing.T) {
  2735  	scenarios := []struct {
  2736  		name          string
  2737  		opts          storage.ListOptions
  2738  		expectedLimit int64
  2739  	}{
  2740  		{
  2741  			name: "limit is zero",
  2742  			opts: storage.ListOptions{
  2743  				Predicate: storage.SelectionPredicate{
  2744  					Limit: 0,
  2745  				},
  2746  			},
  2747  			expectedLimit: 0,
  2748  		},
  2749  		{
  2750  			name: "limit is positive, RV is unset",
  2751  			opts: storage.ListOptions{
  2752  				Predicate: storage.SelectionPredicate{
  2753  					Limit: 1,
  2754  				},
  2755  				ResourceVersion: "",
  2756  			},
  2757  			expectedLimit: 1,
  2758  		},
  2759  		{
  2760  			name: "limit is positive, RV = 100",
  2761  			opts: storage.ListOptions{
  2762  				Predicate: storage.SelectionPredicate{
  2763  					Limit: 1,
  2764  				},
  2765  				ResourceVersion: "100",
  2766  			},
  2767  			expectedLimit: 1,
  2768  		},
  2769  		{
  2770  			name: "legacy case: limit is positive, RV = 0",
  2771  			opts: storage.ListOptions{
  2772  				Predicate: storage.SelectionPredicate{
  2773  					Limit: 1,
  2774  				},
  2775  				ResourceVersion: "0",
  2776  			},
  2777  			expectedLimit: 0,
  2778  		},
  2779  	}
  2780  
  2781  	for _, scenario := range scenarios {
  2782  		t.Run(scenario.name, func(t *testing.T) {
  2783  			actualLimit := computeListLimit(scenario.opts)
  2784  			if actualLimit != scenario.expectedLimit {
  2785  				t.Errorf("computeListLimit returned = %v, expected %v", actualLimit, scenario.expectedLimit)
  2786  			}
  2787  		})
  2788  	}
  2789  }
  2790  
  2791  func watchAndWaitForBookmark(t *testing.T, ctx context.Context, etcdStorage storage.Interface) func() (resourceVersion uint64) {
  2792  	opts := storage.ListOptions{ResourceVersion: "", Predicate: storage.Everything, Recursive: true}
  2793  	opts.Predicate.AllowWatchBookmarks = true
  2794  	w, err := etcdStorage.Watch(ctx, "/pods/", opts)
  2795  	if err != nil {
  2796  		t.Fatal(err)
  2797  	}
  2798  
  2799  	versioner := storage.APIObjectVersioner{}
  2800  	var rv uint64
  2801  	var wg sync.WaitGroup
  2802  	wg.Add(1)
  2803  	go func() {
  2804  		defer wg.Done()
  2805  		for event := range w.ResultChan() {
  2806  			if event.Type == watch.Bookmark {
  2807  				rv, err = versioner.ObjectResourceVersion(event.Object)
  2808  				break
  2809  			}
  2810  		}
  2811  	}()
  2812  	return func() (resourceVersion uint64) {
  2813  		defer w.Stop()
  2814  		wg.Wait()
  2815  		if err != nil {
  2816  			t.Fatal(err)
  2817  		}
  2818  		return rv
  2819  	}
  2820  }
  2821  
  2822  // TODO(p0lyn0mial): forceRequestWatchProgressSupport inits the storage layer
  2823  // so that tests that require storage.RequestWatchProgress pass
  2824  //
  2825  // In the future we could have a function that would allow for setting the feature
  2826  // only for duration of a test.
  2827  func forceRequestWatchProgressSupport(t *testing.T) {
  2828  	if etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress) {
  2829  		return
  2830  	}
  2831  
  2832  	server, _ := newEtcdTestStorage(t, etcd3testing.PathPrefix())
  2833  	defer server.Terminate(t)
  2834  	if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, wait.ForeverTestTimeout, true, func(_ context.Context) (bool, error) {
  2835  		return etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress), nil
  2836  	}); err != nil {
  2837  		t.Fatalf("failed to wait for required %v storage feature to initialize", storage.RequestWatchProgress)
  2838  	}
  2839  }