github.com/cilium/cilium@v1.16.2/pkg/kvstore/store/watchstore_test.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package store 5 6 import ( 7 "context" 8 "path" 9 "sync" 10 "testing" 11 "time" 12 13 "github.com/prometheus/client_golang/prometheus" 14 "github.com/prometheus/client_golang/prometheus/testutil" 15 "github.com/stretchr/testify/require" 16 17 "github.com/cilium/cilium/pkg/kvstore" 18 "github.com/cilium/cilium/pkg/metrics" 19 ) 20 21 type fakeLWBackend struct { 22 t *testing.T 23 prefix string 24 events []kvstore.KeyValueEvent 25 } 26 27 func NewFakeLWBackend(t *testing.T, prefix string, events []kvstore.KeyValueEvent) *fakeLWBackend { 28 return &fakeLWBackend{ 29 t: t, 30 prefix: prefix, 31 events: events, 32 } 33 } 34 35 func (fb *fakeLWBackend) ListAndWatch(ctx context.Context, prefix string, _ int) *kvstore.Watcher { 36 ch := make(kvstore.EventChan) 37 38 go func() { 39 defer close(ch) 40 require.Equal(fb.t, fb.prefix, prefix) 41 42 for _, event := range fb.events { 43 event.Key = path.Join(fb.prefix, event.Key) 44 select { 45 case ch <- event: 46 case <-ctx.Done(): 47 require.Fail(fb.t, "Context closed before propagating all events", "pending: %#v", event) 48 } 49 } 50 51 <-ctx.Done() 52 }() 53 54 return &kvstore.Watcher{Events: ch} 55 } 56 57 type fakeObserver struct { 58 t *testing.T 59 updated chan *KVPair 60 deleted chan *KVPair 61 } 62 63 func NewFakeObserver(t *testing.T) *fakeObserver { 64 return &fakeObserver{ 65 t: t, 66 updated: make(chan *KVPair), 67 deleted: make(chan *KVPair), 68 } 69 } 70 71 func (fo *fakeObserver) OnUpdate(k Key) { 72 select { 73 case fo.updated <- k.(*KVPair): 74 case <-time.After(timeout): 75 require.Failf(fo.t, "Failed observing update event", "key: %s", k.GetKeyName()) 76 } 77 } 78 func (fo *fakeObserver) OnDelete(k NamedKey) { 79 select { 80 case fo.deleted <- k.(*KVPair): 81 case <-time.After(timeout): 82 require.Failf(fo.t, "Failed observing delete event", "key: %s", k.GetKeyName()) 83 } 84 } 85 86 func rwsRun(store WatchStore, prefix string, body func(), backend WatchStoreBackend) { 87 ctx, cancel := context.WithCancel(context.Background()) 88 89 var wg sync.WaitGroup 90 wg.Add(1) 91 go func() { 92 defer wg.Done() 93 store.Watch(ctx, backend, prefix) 94 }() 95 96 defer func() { 97 cancel() 98 wg.Wait() 99 }() 100 101 body() 102 } 103 104 func rwsDrain(t *testing.T, store WatchStore, observer *fakeObserver, expected []*KVPair) { 105 drainDone := make(chan struct{}) 106 go func() { 107 store.Drain() 108 close(drainDone) 109 }() 110 111 var actual []*KVPair 112 for range expected { 113 actual = append(actual, eventually(observer.deleted)) 114 } 115 116 // Since the drained elements are spilled out of a map, there's no ordering guarantee. 117 require.ElementsMatch(t, expected, actual) 118 119 select { 120 case <-drainDone: 121 case <-time.After(timeout): 122 require.Fail(t, "The drain operation did not complete when expected") 123 } 124 } 125 126 func TestRestartableWatchStore(t *testing.T) { 127 observer := NewFakeObserver(t) 128 f, _ := GetFactory(t) 129 store := f.NewWatchStore("qux", KVPairCreator, observer) 130 require.Equal(t, uint64(0), store.NumEntries()) 131 require.False(t, store.Synced()) 132 133 // Watch the kvstore once, and assert that the expected events are propagated 134 rwsRun(store, "foo/bar", func() { 135 require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated)) 136 require.False(t, store.Synced(), "The store should not yet be synced") 137 require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.updated)) 138 require.Eventually(t, store.Synced, timeout, tick, "The store should now be synced") 139 require.Equal(t, NewKVPair("key2", "value2B"), eventually(observer.updated)) 140 require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated)) 141 require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.deleted)) 142 require.Equal(t, uint64(2), store.NumEntries()) 143 }, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{ 144 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")}, 145 {Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2A")}, 146 {Typ: kvstore.EventTypeListDone}, 147 {Typ: kvstore.EventTypeModify, Key: "key2", Value: []byte("value2B")}, 148 {Typ: kvstore.EventTypeCreate, Key: "key3", Value: []byte("value3A")}, 149 {Typ: kvstore.EventTypeDelete, Key: "key4"}, // The key is not known locally -> no event 150 {Typ: kvstore.EventTypeDelete, Key: "key3"}, 151 })) 152 153 require.False(t, store.Synced(), "The store should no longer be synced, as stopped") 154 155 // Watch the kvstore a second time, and assert that the expected events (including 156 // stale keys deletions) are propagated, even though the watcher prefix changed. 157 rwsRun(store, "foo/baz", func() { 158 require.Equal(t, NewKVPair("key1", "value1C"), eventually(observer.updated)) 159 require.Equal(t, NewKVPair("key4", "value4A"), eventually(observer.updated)) 160 require.Equal(t, NewKVPair("key2", "value2B"), eventually(observer.deleted)) 161 require.Equal(t, NewKVPair("key2", "value2C"), eventually(observer.updated)) 162 require.True(t, store.Synced(), "The store should be synced again") 163 require.Equal(t, uint64(3), store.NumEntries()) 164 }, NewFakeLWBackend(t, "foo/baz/", []kvstore.KeyValueEvent{ 165 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1C")}, 166 {Typ: kvstore.EventTypeCreate, Key: "key4", Value: []byte("value4A")}, 167 {Typ: kvstore.EventTypeListDone}, 168 {Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2C")}, 169 })) 170 } 171 172 func TestRestartableWatchStoreDrain(t *testing.T) { 173 observer := NewFakeObserver(t) 174 f, _ := GetFactory(t) 175 store := f.NewWatchStore("qux", KVPairCreator, observer) 176 177 // Watch a few keys through the watch store 178 rwsRun(store, "foo/bar", func() { 179 require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated)) 180 require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.updated)) 181 require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated)) 182 require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.deleted)) 183 }, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{ 184 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")}, 185 {Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2A")}, 186 {Typ: kvstore.EventTypeListDone}, 187 {Typ: kvstore.EventTypeModify, Key: "key3", Value: []byte("value3A")}, 188 {Typ: kvstore.EventTypeDelete, Key: "key2"}, 189 })) 190 191 // Drain the store, and assert that a deletion event is emitted for all keys 192 rwsDrain(t, store, observer, []*KVPair{ 193 NewKVPair("key1", "value1A"), 194 NewKVPair("key3", "value3A"), 195 }) 196 197 // Make sure that it is possible to restart the watch store 198 rwsRun(store, "foo/bar", func() { 199 require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated)) 200 }, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{ 201 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")}, 202 })) 203 204 // And to drain it again 205 rwsDrain(t, store, observer, []*KVPair{ 206 NewKVPair("key1", "value1A"), 207 }) 208 } 209 210 func TestRestartableWatchStoreSyncCallback(t *testing.T) { 211 observer := NewFakeObserver(t) 212 callback := func(value string) func(context.Context) { 213 return func(context.Context) { 214 observer.OnUpdate(NewKVPair("callback/executed", value)) 215 } 216 } 217 f, _ := GetFactory(t) 218 store := f.NewWatchStore("qux", KVPairCreator, observer, 219 RWSWithOnSyncCallback(callback("1")), RWSWithOnSyncCallback(callback("2"))) 220 221 // The watcher is closed before receiving the list done event, the sync callbacks should not be executed 222 rwsRun(store, "foo/bar", func() { 223 require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated)) 224 }, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{ 225 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")}, 226 })) 227 228 // Assert that the callback are executed when the list done event is received 229 rwsRun(store, "foo/bar", func() { 230 require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated)) 231 require.Equal(t, NewKVPair("callback/executed", "1"), eventually(observer.updated)) 232 require.Equal(t, NewKVPair("callback/executed", "2"), eventually(observer.updated)) 233 require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.updated)) 234 }, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{ 235 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")}, 236 {Typ: kvstore.EventTypeListDone}, 237 {Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2A")}, 238 })) 239 240 // Assert that the callbacks are not executed a second time 241 rwsRun(store, "foo/bar", func() { 242 require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated)) 243 require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.deleted)) 244 require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated)) 245 }, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{ 246 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")}, 247 {Typ: kvstore.EventTypeListDone}, 248 {Typ: kvstore.EventTypeCreate, Key: "key3", Value: []byte("value3A")}, 249 })) 250 } 251 252 func TestRestartableWatchStoreConcurrent(t *testing.T) { 253 var wg sync.WaitGroup 254 ctx, cancel := context.WithCancel(context.Background()) 255 256 defer func() { 257 cancel() 258 wg.Wait() 259 }() 260 261 backend := NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{ 262 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1")}, 263 }) 264 observer := NewFakeObserver(t) 265 f, _ := GetFactory(t) 266 store := f.NewWatchStore("qux", KVPairCreator, observer) 267 268 wg.Add(1) 269 go func() { 270 store.Watch(ctx, backend, "foo/bar/") 271 wg.Done() 272 }() 273 274 // Ensure that the Watch operation running in the goroutine has started 275 require.Equal(t, NewKVPair("key1", "value1"), eventually(observer.updated)) 276 277 require.Panics(t, func() { store.Watch(ctx, backend, "foo/bar/") }, "store.Watch should panic when already running") 278 require.Panics(t, store.Drain, "store.Drain should panic when store.Watch is running") 279 } 280 281 func TestRestartableWatchStoreMetrics(t *testing.T) { 282 f, m := GetFactory(t) 283 metrics.NewLegacyMetrics() 284 require.True(t, m.KVStoreInitialSyncCompleted.IsEnabled()) 285 286 entries := prometheus.NewGauge(prometheus.GaugeOpts{Name: "test_elements_metric"}) 287 synced := m.KVStoreInitialSyncCompleted.WithLabelValues("nodes/v1", "qux", "read") 288 289 observer := NewFakeObserver(t) 290 store := f.NewWatchStore("qux", KVPairCreator, observer, RWSWithEntriesMetric(entries)) 291 292 require.Equal(t, float64(0), testutil.ToFloat64(entries)) 293 require.Equal(t, metrics.BoolToFloat64(false), testutil.ToFloat64(synced)) 294 295 rwsRun(store, "cilium/state/nodes/v1", func() { 296 require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated)) 297 require.Equal(t, metrics.BoolToFloat64(false), testutil.ToFloat64(synced)) 298 require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.updated)) 299 300 require.Eventually(t, func() bool { 301 return metrics.BoolToFloat64(true) == testutil.ToFloat64(synced) 302 }, timeout, tick) 303 304 require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.deleted)) 305 require.Equal(t, NewKVPair("key2", "value2B"), eventually(observer.updated)) 306 require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated)) 307 }, NewFakeLWBackend(t, "cilium/state/nodes/v1/", []kvstore.KeyValueEvent{ 308 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")}, 309 {Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2A")}, 310 {Typ: kvstore.EventTypeListDone}, 311 {Typ: kvstore.EventTypeDelete, Key: "key1"}, 312 {Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2B")}, 313 {Typ: kvstore.EventTypeCreate, Key: "key3", Value: []byte("value3A")}, 314 })) 315 316 // The metric should reflect the number of elements. 317 require.Equal(t, float64(2), testutil.ToFloat64(entries)) 318 require.Equal(t, metrics.BoolToFloat64(false), testutil.ToFloat64(synced)) 319 320 rwsRun(store, "cilium/state/nodes/v1", func() { 321 require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated)) 322 require.Equal(t, metrics.BoolToFloat64(false), testutil.ToFloat64(synced)) 323 require.Equal(t, NewKVPair("key2", "value2B"), eventually(observer.deleted)) 324 325 require.Eventually(t, func() bool { 326 return metrics.BoolToFloat64(true) == testutil.ToFloat64(synced) 327 }, timeout, tick) 328 329 require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated)) 330 }, NewFakeLWBackend(t, "cilium/state/nodes/v1/", []kvstore.KeyValueEvent{ 331 {Typ: kvstore.EventTypeCreate, Key: "key3", Value: []byte("value3A")}, 332 {Typ: kvstore.EventTypeListDone}, 333 {Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")}, 334 })) 335 }