k8s.io/apiserver@v0.31.1/pkg/storage/etcd3/watcher_test.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package etcd3 18 19 import ( 20 "context" 21 "errors" 22 "fmt" 23 "sync" 24 "testing" 25 "time" 26 27 "github.com/google/go-cmp/cmp" 28 clientv3 "go.etcd.io/etcd/client/v3" 29 30 apierrors "k8s.io/apimachinery/pkg/api/errors" 31 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 32 "k8s.io/apimachinery/pkg/fields" 33 "k8s.io/apimachinery/pkg/labels" 34 "k8s.io/apimachinery/pkg/watch" 35 "k8s.io/apiserver/pkg/apis/example" 36 "k8s.io/apiserver/pkg/features" 37 "k8s.io/apiserver/pkg/storage" 38 "k8s.io/apiserver/pkg/storage/etcd3/testserver" 39 storagetesting "k8s.io/apiserver/pkg/storage/testing" 40 utilfeature "k8s.io/apiserver/pkg/util/feature" 41 featuregatetesting "k8s.io/component-base/featuregate/testing" 42 "k8s.io/utils/ptr" 43 ) 44 45 func TestWatch(t *testing.T) { 46 ctx, store, _ := testSetup(t) 47 storagetesting.RunTestWatch(ctx, t, store) 48 } 49 50 func TestClusterScopedWatch(t *testing.T) { 51 ctx, store, _ := testSetup(t) 52 storagetesting.RunTestClusterScopedWatch(ctx, t, store) 53 } 54 55 func TestNamespaceScopedWatch(t *testing.T) { 56 ctx, store, _ := testSetup(t) 57 storagetesting.RunTestNamespaceScopedWatch(ctx, t, store) 58 } 59 60 func TestDeleteTriggerWatch(t *testing.T) { 61 ctx, store, _ := testSetup(t) 62 storagetesting.RunTestDeleteTriggerWatch(ctx, t, store) 63 } 64 65 func TestWatchFromZero(t *testing.T) { 66 ctx, store, client := testSetup(t) 67 storagetesting.RunTestWatchFromZero(ctx, t, store, compactStorage(client)) 68 } 69 70 // TestWatchFromNonZero tests that 71 // - watch from non-0 should just watch changes after given version 72 func TestWatchFromNoneZero(t *testing.T) { 73 ctx, store, _ := testSetup(t) 74 storagetesting.RunTestWatchFromNonZero(ctx, t, store) 75 } 76 77 func TestDelayedWatchDelivery(t *testing.T) { 78 ctx, store, _ := testSetup(t) 79 storagetesting.RunTestDelayedWatchDelivery(ctx, t, store) 80 } 81 82 func TestWatchError(t *testing.T) { 83 ctx, store, _ := testSetup(t) 84 storagetesting.RunTestWatchError(ctx, t, &storeWithPrefixTransformer{store}) 85 } 86 87 func TestWatchContextCancel(t *testing.T) { 88 ctx, store, _ := testSetup(t) 89 storagetesting.RunTestWatchContextCancel(ctx, t, store) 90 } 91 92 func TestWatcherTimeout(t *testing.T) { 93 ctx, store, _ := testSetup(t) 94 storagetesting.RunTestWatcherTimeout(ctx, t, store) 95 } 96 97 func TestWatchDeleteEventObjectHaveLatestRV(t *testing.T) { 98 ctx, store, _ := testSetup(t) 99 storagetesting.RunTestWatchDeleteEventObjectHaveLatestRV(ctx, t, store) 100 } 101 102 func TestWatchInitializationSignal(t *testing.T) { 103 ctx, store, _ := testSetup(t) 104 storagetesting.RunTestWatchInitializationSignal(ctx, t, store) 105 } 106 107 func TestProgressNotify(t *testing.T) { 108 clusterConfig := testserver.NewTestConfig(t) 109 clusterConfig.ExperimentalWatchProgressNotifyInterval = time.Second 110 ctx, store, _ := testSetup(t, withClientConfig(clusterConfig)) 111 112 storagetesting.RunOptionalTestProgressNotify(ctx, t, store) 113 } 114 115 // TestWatchDispatchBookmarkEvents makes sure that 116 // setting allowWatchBookmarks query param against 117 // etcd implementation doesn't have any effect. 118 func TestWatchDispatchBookmarkEvents(t *testing.T) { 119 clusterConfig := testserver.NewTestConfig(t) 120 clusterConfig.ExperimentalWatchProgressNotifyInterval = time.Second 121 ctx, store, _ := testSetup(t, withClientConfig(clusterConfig)) 122 123 storagetesting.RunTestWatchDispatchBookmarkEvents(ctx, t, store, false) 124 } 125 126 func TestSendInitialEventsBackwardCompatibility(t *testing.T) { 127 ctx, store, _ := testSetup(t) 128 storagetesting.RunSendInitialEventsBackwardCompatibility(ctx, t, store) 129 } 130 131 func TestEtcdWatchSemantics(t *testing.T) { 132 ctx, store, _ := testSetup(t) 133 storagetesting.RunWatchSemantics(ctx, t, store) 134 } 135 136 func TestEtcdWatchSemanticsWithConcurrentDecode(t *testing.T) { 137 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConcurrentWatchObjectDecode, true) 138 ctx, store, _ := testSetup(t) 139 storagetesting.RunWatchSemantics(ctx, t, store) 140 } 141 142 func TestEtcdWatchSemanticInitialEventsExtended(t *testing.T) { 143 ctx, store, _ := testSetup(t) 144 storagetesting.RunWatchSemanticInitialEventsExtended(ctx, t, store) 145 } 146 147 // ======================================================================= 148 // Implementation-specific tests are following. 149 // The following tests are exercising the details of the implementation 150 // not the actual user-facing contract of storage interface. 151 // As such, they may focus e.g. on non-functional aspects like performance 152 // impact. 153 // ======================================================================= 154 155 func TestWatchErrResultNotBlockAfterCancel(t *testing.T) { 156 origCtx, store, _ := testSetup(t) 157 ctx, cancel := context.WithCancel(origCtx) 158 w := store.watcher.createWatchChan(ctx, "/abc", 0, false, false, storage.Everything) 159 // make resultChan and errChan blocking to ensure ordering. 160 w.resultChan = make(chan watch.Event) 161 w.errChan = make(chan error) 162 // The event flow goes like: 163 // - first we send an error, it should block on resultChan. 164 // - Then we cancel ctx. The blocking on resultChan should be freed up 165 // and run() goroutine should return. 166 var wg sync.WaitGroup 167 wg.Add(1) 168 go func() { 169 w.run(false, true) 170 wg.Done() 171 }() 172 w.errChan <- fmt.Errorf("some error") 173 cancel() 174 wg.Wait() 175 } 176 177 // TestWatchErrorIncorrectConfiguration checks if an error 178 // will be returned when the storage hasn't been properly 179 // initialised for watch requests 180 func TestWatchErrorIncorrectConfiguration(t *testing.T) { 181 scenarios := []struct { 182 name string 183 setupFn func(opts *setupOptions) 184 requestOpts storage.ListOptions 185 enableWatchList bool 186 expectedErr error 187 }{ 188 { 189 name: "no newFunc provided", 190 setupFn: func(opts *setupOptions) { opts.newFunc = nil }, 191 requestOpts: storage.ListOptions{ProgressNotify: true}, 192 expectedErr: apierrors.NewInternalError(errors.New("progressNotify for watch is unsupported by the etcd storage because no newFunc was provided")), 193 }, 194 } 195 for _, scenario := range scenarios { 196 t.Run(scenario.name, func(t *testing.T) { 197 if scenario.enableWatchList { 198 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true) 199 } 200 origCtx, store, _ := testSetup(t, scenario.setupFn) 201 ctx, cancel := context.WithCancel(origCtx) 202 defer cancel() 203 204 w, err := store.watcher.Watch(ctx, "/abc", 0, scenario.requestOpts) 205 if err == nil { 206 t.Fatalf("expected an error but got none") 207 } 208 if w != nil { 209 t.Fatalf("didn't expect a watcher because the test assumes incorrect store initialisation") 210 } 211 if err.Error() != scenario.expectedErr.Error() { 212 t.Fatalf("unexpected err = %v, expected = %v", err, scenario.expectedErr) 213 } 214 }) 215 } 216 } 217 218 func TestTooLargeResourceVersionErrorForWatchList(t *testing.T) { 219 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true) 220 origCtx, store, _ := testSetup(t) 221 ctx, cancel := context.WithCancel(origCtx) 222 defer cancel() 223 requestOpts := storage.ListOptions{ 224 SendInitialEvents: ptr.To(true), 225 Recursive: true, 226 Predicate: storage.SelectionPredicate{ 227 Field: fields.Everything(), 228 Label: labels.Everything(), 229 AllowWatchBookmarks: true, 230 }, 231 } 232 var expectedErr *apierrors.StatusError 233 if !errors.As(storage.NewTooLargeResourceVersionError(uint64(102), 1, 0), &expectedErr) { 234 t.Fatalf("Unable to convert NewTooLargeResourceVersionError to apierrors.StatusError") 235 } 236 237 w, err := store.watcher.Watch(ctx, "/abc", int64(102), requestOpts) 238 if err != nil { 239 t.Fatal(err) 240 } 241 defer w.Stop() 242 243 actualEvent := <-w.ResultChan() 244 if actualEvent.Type != watch.Error { 245 t.Fatalf("Unexpected type of the event: %v, expected: %v", actualEvent.Type, watch.Error) 246 } 247 actualErr, ok := actualEvent.Object.(*metav1.Status) 248 if !ok { 249 t.Fatalf("Expected *apierrors.StatusError, got: %#v", actualEvent.Object) 250 } 251 252 if actualErr.Details.RetryAfterSeconds <= 0 { 253 t.Fatalf("RetryAfterSeconds must be > 0, actual value: %v", actualErr.Details.RetryAfterSeconds) 254 } 255 // rewrite the Details as it contains retry seconds 256 // and validate the whole struct 257 expectedErr.ErrStatus.Details = actualErr.Details 258 if diff := cmp.Diff(*actualErr, expectedErr.ErrStatus); diff != "" { 259 t.Fatalf("Unexpected error returned, diff: %v", diff) 260 } 261 } 262 263 func TestWatchChanSync(t *testing.T) { 264 testCases := []struct { 265 name string 266 watchKey string 267 watcherMaxLimit int64 268 expectEventCount int 269 expectGetCount int 270 }{ 271 { 272 name: "None of the current objects match watchKey: sync with empty page", 273 watchKey: "/pods/test/", 274 watcherMaxLimit: 1, 275 expectGetCount: 1, 276 }, 277 { 278 name: "The number of current objects is less than defaultWatcherMaxLimit: sync with one page", 279 watchKey: "/pods/", 280 watcherMaxLimit: 3, 281 expectEventCount: 2, 282 expectGetCount: 1, 283 }, 284 { 285 name: "a new item added to etcd before returning a second page is not returned: sync with two page", 286 watchKey: "/pods/", 287 watcherMaxLimit: 1, 288 expectEventCount: 2, 289 expectGetCount: 2, 290 }, 291 } 292 293 for _, testCase := range testCases { 294 t.Run(testCase.name, func(t *testing.T) { 295 defaultWatcherMaxLimit = testCase.watcherMaxLimit 296 297 origCtx, store, _ := testSetup(t) 298 initList, err := initStoreData(origCtx, store) 299 if err != nil { 300 t.Fatal(err) 301 } 302 303 kvWrapper := newEtcdClientKVWrapper(store.client.KV) 304 kvWrapper.getReactors = append(kvWrapper.getReactors, func() { 305 barThird := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "third", Name: "bar"}} 306 podKey := fmt.Sprintf("/pods/%s/%s", barThird.Namespace, barThird.Name) 307 storedObj := &example.Pod{} 308 309 err := store.Create(context.Background(), podKey, barThird, storedObj, 0) 310 if err != nil { 311 t.Errorf("failed to create object: %v", err) 312 } 313 }) 314 315 store.client.KV = kvWrapper 316 317 w := store.watcher.createWatchChan( 318 origCtx, 319 testCase.watchKey, 320 0, 321 true, 322 false, 323 storage.Everything) 324 325 err = w.sync() 326 if err != nil { 327 t.Fatal(err) 328 } 329 330 // close incomingEventChan so we can read incomingEventChan non-blocking 331 close(w.incomingEventChan) 332 333 eventsReceived := 0 334 for event := range w.incomingEventChan { 335 eventsReceived++ 336 storagetesting.ExpectContains(t, "incorrect list pods", initList, event.key) 337 } 338 339 if eventsReceived != testCase.expectEventCount { 340 t.Errorf("Unexpected number of events: %v, expected: %v", eventsReceived, testCase.expectEventCount) 341 } 342 343 if kvWrapper.getCallCounter != testCase.expectGetCount { 344 t.Errorf("Unexpected called times of client.KV.Get() : %v, expected: %v", kvWrapper.getCallCounter, testCase.expectGetCount) 345 } 346 }) 347 } 348 } 349 350 // NOTE: it's not thread-safe 351 type etcdClientKVWrapper struct { 352 clientv3.KV 353 // keeps track of the number of times Get method is called 354 getCallCounter int 355 // getReactors is called after the etcd KV's get function is executed. 356 getReactors []func() 357 } 358 359 func newEtcdClientKVWrapper(kv clientv3.KV) *etcdClientKVWrapper { 360 return &etcdClientKVWrapper{ 361 KV: kv, 362 getCallCounter: 0, 363 } 364 } 365 366 func (ecw *etcdClientKVWrapper) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { 367 resp, err := ecw.KV.Get(ctx, key, opts...) 368 ecw.getCallCounter++ 369 if err != nil { 370 return nil, err 371 } 372 373 if len(ecw.getReactors) > 0 { 374 reactor := ecw.getReactors[0] 375 ecw.getReactors = ecw.getReactors[1:] 376 reactor() 377 } 378 379 return resp, nil 380 } 381 382 func initStoreData(ctx context.Context, store storage.Interface) ([]interface{}, error) { 383 barFirst := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "first", Name: "bar"}} 384 barSecond := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "second", Name: "bar"}} 385 386 preset := []struct { 387 key string 388 obj *example.Pod 389 storedObj *example.Pod 390 }{ 391 { 392 key: fmt.Sprintf("/pods/%s/%s", barFirst.Namespace, barFirst.Name), 393 obj: barFirst, 394 }, 395 { 396 key: fmt.Sprintf("/pods/%s/%s", barSecond.Namespace, barSecond.Name), 397 obj: barSecond, 398 }, 399 } 400 401 for i, ps := range preset { 402 preset[i].storedObj = &example.Pod{} 403 err := store.Create(ctx, ps.key, ps.obj, preset[i].storedObj, 0) 404 if err != nil { 405 return nil, fmt.Errorf("failed to create object: %w", err) 406 } 407 } 408 409 var created []interface{} 410 for _, item := range preset { 411 created = append(created, item.key) 412 } 413 return created, nil 414 }