k8s.io/apiserver@v0.31.1/pkg/storage/etcd3/watcher_test.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package etcd3
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"sync"
    24  	"testing"
    25  	"time"
    26  
    27  	"github.com/google/go-cmp/cmp"
    28  	clientv3 "go.etcd.io/etcd/client/v3"
    29  
    30  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/fields"
    33  	"k8s.io/apimachinery/pkg/labels"
    34  	"k8s.io/apimachinery/pkg/watch"
    35  	"k8s.io/apiserver/pkg/apis/example"
    36  	"k8s.io/apiserver/pkg/features"
    37  	"k8s.io/apiserver/pkg/storage"
    38  	"k8s.io/apiserver/pkg/storage/etcd3/testserver"
    39  	storagetesting "k8s.io/apiserver/pkg/storage/testing"
    40  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    41  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    42  	"k8s.io/utils/ptr"
    43  )
    44  
    45  func TestWatch(t *testing.T) {
    46  	ctx, store, _ := testSetup(t)
    47  	storagetesting.RunTestWatch(ctx, t, store)
    48  }
    49  
    50  func TestClusterScopedWatch(t *testing.T) {
    51  	ctx, store, _ := testSetup(t)
    52  	storagetesting.RunTestClusterScopedWatch(ctx, t, store)
    53  }
    54  
    55  func TestNamespaceScopedWatch(t *testing.T) {
    56  	ctx, store, _ := testSetup(t)
    57  	storagetesting.RunTestNamespaceScopedWatch(ctx, t, store)
    58  }
    59  
    60  func TestDeleteTriggerWatch(t *testing.T) {
    61  	ctx, store, _ := testSetup(t)
    62  	storagetesting.RunTestDeleteTriggerWatch(ctx, t, store)
    63  }
    64  
    65  func TestWatchFromZero(t *testing.T) {
    66  	ctx, store, client := testSetup(t)
    67  	storagetesting.RunTestWatchFromZero(ctx, t, store, compactStorage(client))
    68  }
    69  
    70  // TestWatchFromNonZero tests that
    71  // - watch from non-0 should just watch changes after given version
    72  func TestWatchFromNoneZero(t *testing.T) {
    73  	ctx, store, _ := testSetup(t)
    74  	storagetesting.RunTestWatchFromNonZero(ctx, t, store)
    75  }
    76  
    77  func TestDelayedWatchDelivery(t *testing.T) {
    78  	ctx, store, _ := testSetup(t)
    79  	storagetesting.RunTestDelayedWatchDelivery(ctx, t, store)
    80  }
    81  
    82  func TestWatchError(t *testing.T) {
    83  	ctx, store, _ := testSetup(t)
    84  	storagetesting.RunTestWatchError(ctx, t, &storeWithPrefixTransformer{store})
    85  }
    86  
    87  func TestWatchContextCancel(t *testing.T) {
    88  	ctx, store, _ := testSetup(t)
    89  	storagetesting.RunTestWatchContextCancel(ctx, t, store)
    90  }
    91  
    92  func TestWatcherTimeout(t *testing.T) {
    93  	ctx, store, _ := testSetup(t)
    94  	storagetesting.RunTestWatcherTimeout(ctx, t, store)
    95  }
    96  
    97  func TestWatchDeleteEventObjectHaveLatestRV(t *testing.T) {
    98  	ctx, store, _ := testSetup(t)
    99  	storagetesting.RunTestWatchDeleteEventObjectHaveLatestRV(ctx, t, store)
   100  }
   101  
   102  func TestWatchInitializationSignal(t *testing.T) {
   103  	ctx, store, _ := testSetup(t)
   104  	storagetesting.RunTestWatchInitializationSignal(ctx, t, store)
   105  }
   106  
   107  func TestProgressNotify(t *testing.T) {
   108  	clusterConfig := testserver.NewTestConfig(t)
   109  	clusterConfig.ExperimentalWatchProgressNotifyInterval = time.Second
   110  	ctx, store, _ := testSetup(t, withClientConfig(clusterConfig))
   111  
   112  	storagetesting.RunOptionalTestProgressNotify(ctx, t, store)
   113  }
   114  
   115  // TestWatchDispatchBookmarkEvents makes sure that
   116  // setting allowWatchBookmarks query param against
   117  // etcd implementation doesn't have any effect.
   118  func TestWatchDispatchBookmarkEvents(t *testing.T) {
   119  	clusterConfig := testserver.NewTestConfig(t)
   120  	clusterConfig.ExperimentalWatchProgressNotifyInterval = time.Second
   121  	ctx, store, _ := testSetup(t, withClientConfig(clusterConfig))
   122  
   123  	storagetesting.RunTestWatchDispatchBookmarkEvents(ctx, t, store, false)
   124  }
   125  
   126  func TestSendInitialEventsBackwardCompatibility(t *testing.T) {
   127  	ctx, store, _ := testSetup(t)
   128  	storagetesting.RunSendInitialEventsBackwardCompatibility(ctx, t, store)
   129  }
   130  
   131  func TestEtcdWatchSemantics(t *testing.T) {
   132  	ctx, store, _ := testSetup(t)
   133  	storagetesting.RunWatchSemantics(ctx, t, store)
   134  }
   135  
   136  func TestEtcdWatchSemanticsWithConcurrentDecode(t *testing.T) {
   137  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConcurrentWatchObjectDecode, true)
   138  	ctx, store, _ := testSetup(t)
   139  	storagetesting.RunWatchSemantics(ctx, t, store)
   140  }
   141  
   142  func TestEtcdWatchSemanticInitialEventsExtended(t *testing.T) {
   143  	ctx, store, _ := testSetup(t)
   144  	storagetesting.RunWatchSemanticInitialEventsExtended(ctx, t, store)
   145  }
   146  
   147  // =======================================================================
   148  // Implementation-specific tests are following.
   149  // The following tests are exercising the details of the implementation
   150  // not the actual user-facing contract of storage interface.
   151  // As such, they may focus e.g. on non-functional aspects like performance
   152  // impact.
   153  // =======================================================================
   154  
   155  func TestWatchErrResultNotBlockAfterCancel(t *testing.T) {
   156  	origCtx, store, _ := testSetup(t)
   157  	ctx, cancel := context.WithCancel(origCtx)
   158  	w := store.watcher.createWatchChan(ctx, "/abc", 0, false, false, storage.Everything)
   159  	// make resultChan and errChan blocking to ensure ordering.
   160  	w.resultChan = make(chan watch.Event)
   161  	w.errChan = make(chan error)
   162  	// The event flow goes like:
   163  	// - first we send an error, it should block on resultChan.
   164  	// - Then we cancel ctx. The blocking on resultChan should be freed up
   165  	//   and run() goroutine should return.
   166  	var wg sync.WaitGroup
   167  	wg.Add(1)
   168  	go func() {
   169  		w.run(false, true)
   170  		wg.Done()
   171  	}()
   172  	w.errChan <- fmt.Errorf("some error")
   173  	cancel()
   174  	wg.Wait()
   175  }
   176  
   177  // TestWatchErrorIncorrectConfiguration checks if an error
   178  // will be returned when the storage hasn't been properly
   179  // initialised for watch requests
   180  func TestWatchErrorIncorrectConfiguration(t *testing.T) {
   181  	scenarios := []struct {
   182  		name            string
   183  		setupFn         func(opts *setupOptions)
   184  		requestOpts     storage.ListOptions
   185  		enableWatchList bool
   186  		expectedErr     error
   187  	}{
   188  		{
   189  			name:        "no newFunc provided",
   190  			setupFn:     func(opts *setupOptions) { opts.newFunc = nil },
   191  			requestOpts: storage.ListOptions{ProgressNotify: true},
   192  			expectedErr: apierrors.NewInternalError(errors.New("progressNotify for watch is unsupported by the etcd storage because no newFunc was provided")),
   193  		},
   194  	}
   195  	for _, scenario := range scenarios {
   196  		t.Run(scenario.name, func(t *testing.T) {
   197  			if scenario.enableWatchList {
   198  				featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
   199  			}
   200  			origCtx, store, _ := testSetup(t, scenario.setupFn)
   201  			ctx, cancel := context.WithCancel(origCtx)
   202  			defer cancel()
   203  
   204  			w, err := store.watcher.Watch(ctx, "/abc", 0, scenario.requestOpts)
   205  			if err == nil {
   206  				t.Fatalf("expected an error but got none")
   207  			}
   208  			if w != nil {
   209  				t.Fatalf("didn't expect a watcher because the test assumes incorrect store initialisation")
   210  			}
   211  			if err.Error() != scenario.expectedErr.Error() {
   212  				t.Fatalf("unexpected err = %v, expected = %v", err, scenario.expectedErr)
   213  			}
   214  		})
   215  	}
   216  }
   217  
   218  func TestTooLargeResourceVersionErrorForWatchList(t *testing.T) {
   219  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
   220  	origCtx, store, _ := testSetup(t)
   221  	ctx, cancel := context.WithCancel(origCtx)
   222  	defer cancel()
   223  	requestOpts := storage.ListOptions{
   224  		SendInitialEvents: ptr.To(true),
   225  		Recursive:         true,
   226  		Predicate: storage.SelectionPredicate{
   227  			Field:               fields.Everything(),
   228  			Label:               labels.Everything(),
   229  			AllowWatchBookmarks: true,
   230  		},
   231  	}
   232  	var expectedErr *apierrors.StatusError
   233  	if !errors.As(storage.NewTooLargeResourceVersionError(uint64(102), 1, 0), &expectedErr) {
   234  		t.Fatalf("Unable to convert NewTooLargeResourceVersionError to apierrors.StatusError")
   235  	}
   236  
   237  	w, err := store.watcher.Watch(ctx, "/abc", int64(102), requestOpts)
   238  	if err != nil {
   239  		t.Fatal(err)
   240  	}
   241  	defer w.Stop()
   242  
   243  	actualEvent := <-w.ResultChan()
   244  	if actualEvent.Type != watch.Error {
   245  		t.Fatalf("Unexpected type of the event: %v, expected: %v", actualEvent.Type, watch.Error)
   246  	}
   247  	actualErr, ok := actualEvent.Object.(*metav1.Status)
   248  	if !ok {
   249  		t.Fatalf("Expected *apierrors.StatusError, got: %#v", actualEvent.Object)
   250  	}
   251  
   252  	if actualErr.Details.RetryAfterSeconds <= 0 {
   253  		t.Fatalf("RetryAfterSeconds must be > 0, actual value: %v", actualErr.Details.RetryAfterSeconds)
   254  	}
   255  	// rewrite the Details as it contains retry seconds
   256  	// and validate the whole struct
   257  	expectedErr.ErrStatus.Details = actualErr.Details
   258  	if diff := cmp.Diff(*actualErr, expectedErr.ErrStatus); diff != "" {
   259  		t.Fatalf("Unexpected error returned, diff: %v", diff)
   260  	}
   261  }
   262  
   263  func TestWatchChanSync(t *testing.T) {
   264  	testCases := []struct {
   265  		name             string
   266  		watchKey         string
   267  		watcherMaxLimit  int64
   268  		expectEventCount int
   269  		expectGetCount   int
   270  	}{
   271  		{
   272  			name:            "None of the current objects match watchKey: sync with empty page",
   273  			watchKey:        "/pods/test/",
   274  			watcherMaxLimit: 1,
   275  			expectGetCount:  1,
   276  		},
   277  		{
   278  			name:             "The number of current objects is less than defaultWatcherMaxLimit: sync with one page",
   279  			watchKey:         "/pods/",
   280  			watcherMaxLimit:  3,
   281  			expectEventCount: 2,
   282  			expectGetCount:   1,
   283  		},
   284  		{
   285  			name:             "a new item added to etcd before returning a second page is not returned: sync with two page",
   286  			watchKey:         "/pods/",
   287  			watcherMaxLimit:  1,
   288  			expectEventCount: 2,
   289  			expectGetCount:   2,
   290  		},
   291  	}
   292  
   293  	for _, testCase := range testCases {
   294  		t.Run(testCase.name, func(t *testing.T) {
   295  			defaultWatcherMaxLimit = testCase.watcherMaxLimit
   296  
   297  			origCtx, store, _ := testSetup(t)
   298  			initList, err := initStoreData(origCtx, store)
   299  			if err != nil {
   300  				t.Fatal(err)
   301  			}
   302  
   303  			kvWrapper := newEtcdClientKVWrapper(store.client.KV)
   304  			kvWrapper.getReactors = append(kvWrapper.getReactors, func() {
   305  				barThird := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "third", Name: "bar"}}
   306  				podKey := fmt.Sprintf("/pods/%s/%s", barThird.Namespace, barThird.Name)
   307  				storedObj := &example.Pod{}
   308  
   309  				err := store.Create(context.Background(), podKey, barThird, storedObj, 0)
   310  				if err != nil {
   311  					t.Errorf("failed to create object: %v", err)
   312  				}
   313  			})
   314  
   315  			store.client.KV = kvWrapper
   316  
   317  			w := store.watcher.createWatchChan(
   318  				origCtx,
   319  				testCase.watchKey,
   320  				0,
   321  				true,
   322  				false,
   323  				storage.Everything)
   324  
   325  			err = w.sync()
   326  			if err != nil {
   327  				t.Fatal(err)
   328  			}
   329  
   330  			// close incomingEventChan so we can read incomingEventChan non-blocking
   331  			close(w.incomingEventChan)
   332  
   333  			eventsReceived := 0
   334  			for event := range w.incomingEventChan {
   335  				eventsReceived++
   336  				storagetesting.ExpectContains(t, "incorrect list pods", initList, event.key)
   337  			}
   338  
   339  			if eventsReceived != testCase.expectEventCount {
   340  				t.Errorf("Unexpected number of events: %v, expected: %v", eventsReceived, testCase.expectEventCount)
   341  			}
   342  
   343  			if kvWrapper.getCallCounter != testCase.expectGetCount {
   344  				t.Errorf("Unexpected called times of client.KV.Get() : %v, expected: %v", kvWrapper.getCallCounter, testCase.expectGetCount)
   345  			}
   346  		})
   347  	}
   348  }
   349  
   350  // NOTE: it's not thread-safe
   351  type etcdClientKVWrapper struct {
   352  	clientv3.KV
   353  	// keeps track of the number of times Get method is called
   354  	getCallCounter int
   355  	// getReactors is called after the etcd KV's get function is executed.
   356  	getReactors []func()
   357  }
   358  
   359  func newEtcdClientKVWrapper(kv clientv3.KV) *etcdClientKVWrapper {
   360  	return &etcdClientKVWrapper{
   361  		KV:             kv,
   362  		getCallCounter: 0,
   363  	}
   364  }
   365  
   366  func (ecw *etcdClientKVWrapper) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
   367  	resp, err := ecw.KV.Get(ctx, key, opts...)
   368  	ecw.getCallCounter++
   369  	if err != nil {
   370  		return nil, err
   371  	}
   372  
   373  	if len(ecw.getReactors) > 0 {
   374  		reactor := ecw.getReactors[0]
   375  		ecw.getReactors = ecw.getReactors[1:]
   376  		reactor()
   377  	}
   378  
   379  	return resp, nil
   380  }
   381  
   382  func initStoreData(ctx context.Context, store storage.Interface) ([]interface{}, error) {
   383  	barFirst := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "first", Name: "bar"}}
   384  	barSecond := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "second", Name: "bar"}}
   385  
   386  	preset := []struct {
   387  		key       string
   388  		obj       *example.Pod
   389  		storedObj *example.Pod
   390  	}{
   391  		{
   392  			key: fmt.Sprintf("/pods/%s/%s", barFirst.Namespace, barFirst.Name),
   393  			obj: barFirst,
   394  		},
   395  		{
   396  			key: fmt.Sprintf("/pods/%s/%s", barSecond.Namespace, barSecond.Name),
   397  			obj: barSecond,
   398  		},
   399  	}
   400  
   401  	for i, ps := range preset {
   402  		preset[i].storedObj = &example.Pod{}
   403  		err := store.Create(ctx, ps.key, ps.obj, preset[i].storedObj, 0)
   404  		if err != nil {
   405  			return nil, fmt.Errorf("failed to create object: %w", err)
   406  		}
   407  	}
   408  
   409  	var created []interface{}
   410  	for _, item := range preset {
   411  		created = append(created, item.key)
   412  	}
   413  	return created, nil
   414  }