github.com/cilium/cilium@v1.16.2/pkg/kvstore/store/store_test.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package store 5 6 import ( 7 "context" 8 "encoding/json" 9 "testing" 10 "time" 11 12 "github.com/stretchr/testify/assert" 13 "github.com/stretchr/testify/require" 14 "k8s.io/apimachinery/pkg/util/rand" 15 16 "github.com/cilium/cilium/pkg/defaults" 17 "github.com/cilium/cilium/pkg/kvstore" 18 "github.com/cilium/cilium/pkg/lock" 19 "github.com/cilium/cilium/pkg/option" 20 "github.com/cilium/cilium/pkg/testutils" 21 ) 22 23 const ( 24 testPrefix = "store-tests" 25 sharedKeyDeleteDelay = time.Second 26 ) 27 28 type TestType struct { 29 Name string 30 } 31 32 var _ = TestType{} 33 34 func (t *TestType) GetKeyName() string { return t.Name } 35 func (t *TestType) DeepKeyCopy() LocalKey { return &TestType{Name: t.Name} } 36 func (t *TestType) Marshal() ([]byte, error) { return json.Marshal(t) } 37 func (t *TestType) Unmarshal(_ string, data []byte) error { return json.Unmarshal(data, t) } 38 39 type opCounter struct { 40 deleted int 41 updated int 42 } 43 44 var ( 45 counter = map[string]*opCounter{} 46 counterLock lock.RWMutex 47 ) 48 49 func (t *TestType) deleted() int { 50 counterLock.RLock() 51 defer counterLock.RUnlock() 52 return counter[t.Name].deleted 53 } 54 55 func (t *TestType) updated() int { 56 counterLock.RLock() 57 defer counterLock.RUnlock() 58 return counter[t.Name].updated 59 } 60 61 func initTestType(name string) TestType { 62 t := TestType{} 63 t.Name = name 64 counterLock.Lock() 65 counter[name] = &opCounter{} 66 counterLock.Unlock() 67 return t 68 } 69 70 type observer struct{} 71 72 func (o *observer) OnUpdate(k Key) { 73 counterLock.Lock() 74 if c, ok := counter[k.(*TestType).Name]; ok { 75 c.updated++ 76 } 77 counterLock.Unlock() 78 } 79 func (o *observer) OnDelete(k NamedKey) { 80 counterLock.Lock() 81 counter[k.(*TestType).Name].deleted++ 82 counterLock.Unlock() 83 } 84 85 func newTestType() Key { 86 t := TestType{} 87 return &t 88 } 89 90 func TestStoreCreation(t *testing.T) { 91 testutils.IntegrationTest(t) 92 for _, backendName := range []string{"etcd", "consul"} { 93 t.Run(backendName, func(t *testing.T) { 94 kvstore.SetupDummy(t, backendName) 95 testStoreCreation(t) 96 }) 97 } 98 } 99 100 func testStoreCreation(t *testing.T) { 101 // Missing Prefix must result in error 102 store, err := JoinSharedStore(Configuration{}) 103 require.ErrorContains(t, err, "prefix must be specified") 104 require.Nil(t, store) 105 106 // Missing KeyCreator must result in error 107 store, err = JoinSharedStore(Configuration{Prefix: rand.String(12)}) 108 require.ErrorContains(t, err, "KeyCreator must be specified") 109 require.Nil(t, store) 110 111 // Basic creation should result in default values 112 store, err = JoinSharedStore(Configuration{Prefix: rand.String(12), KeyCreator: newTestType}) 113 require.NoError(t, err) 114 require.NotNil(t, store) 115 require.Equal(t, option.Config.KVstorePeriodicSync, store.conf.SynchronizationInterval) 116 store.Close(context.TODO()) 117 118 // Test with kvstore client specified 119 store, err = JoinSharedStore(Configuration{Prefix: rand.String(12), KeyCreator: newTestType, Backend: kvstore.Client()}) 120 require.NoError(t, err) 121 require.NotNil(t, store) 122 require.Equal(t, option.Config.KVstorePeriodicSync, store.conf.SynchronizationInterval) 123 store.Close(context.TODO()) 124 } 125 126 func TestStoreOperations(t *testing.T) { 127 testutils.IntegrationTest(t) 128 for _, backendName := range []string{"etcd", "consul"} { 129 t.Run(backendName, func(t *testing.T) { 130 kvstore.SetupDummy(t, backendName) 131 testStoreOperations(t) 132 }) 133 } 134 } 135 136 func testStoreOperations(t *testing.T) { 137 // Basic creation should result in default values 138 store, err := JoinSharedStore(Configuration{ 139 Prefix: rand.String(12), 140 KeyCreator: newTestType, 141 Observer: &observer{}, 142 SharedKeyDeleteDelay: sharedKeyDeleteDelay, 143 }) 144 require.NoError(t, err) 145 require.NotNil(t, store) 146 defer store.Close(context.TODO()) 147 148 localKey1 := initTestType("local1") 149 localKey2 := initTestType("local2") 150 localKey3 := initTestType("local3") 151 152 err = store.UpdateLocalKeySync(context.TODO(), &localKey1) 153 require.NoError(t, err) 154 err = store.UpdateLocalKeySync(context.TODO(), &localKey2) 155 require.NoError(t, err) 156 157 // due to the short sync interval, it is possible that multiple updates 158 // have occurred, make the test reliable by succeeding on at lest one 159 // update 160 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey1.updated(), 1) }, timeout, tick) 161 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey2.updated(), 1) }, timeout, tick) 162 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey3.updated()) }, timeout, tick) 163 164 store.DeleteLocalKey(context.TODO(), &localKey1) 165 // localKey1 will be deleted 2 times, one from local key and other from 166 // the kvstore watcher 167 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 2, localKey1.deleted()) }, timeout, tick) 168 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey2.deleted()) }, timeout, tick) 169 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey3.deleted()) }, timeout, tick) 170 171 store.DeleteLocalKey(context.TODO(), &localKey3) 172 // localKey3 won't be deleted because it was never added 173 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey3.deleted()) }, timeout, tick) 174 175 store.DeleteLocalKey(context.TODO(), &localKey2) 176 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 2, localKey1.deleted()) }, timeout, tick) 177 // localKey2 will be deleted 2 times, one from local key and other from 178 // the kvstore watcher 179 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 2, localKey2.deleted()) }, timeout, tick) 180 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey3.deleted()) }, timeout, tick) 181 } 182 183 func TestStorePeriodicSync(t *testing.T) { 184 testutils.IntegrationTest(t) 185 for _, backendName := range []string{"etcd", "consul"} { 186 t.Run(backendName, func(t *testing.T) { 187 kvstore.SetupDummy(t, backendName) 188 testStorePeriodicSync(t) 189 }) 190 } 191 } 192 193 func testStorePeriodicSync(t *testing.T) { 194 // Create a store with a very short periodic sync interval 195 store, err := JoinSharedStore(Configuration{ 196 Prefix: rand.String(12), 197 KeyCreator: newTestType, 198 SynchronizationInterval: 10 * time.Millisecond, 199 SharedKeyDeleteDelay: defaults.NodeDeleteDelay, 200 Observer: &observer{}, 201 }) 202 require.NoError(t, err) 203 require.NotNil(t, store) 204 defer store.Close(context.TODO()) 205 206 localKey1 := initTestType("local1") 207 localKey2 := initTestType("local2") 208 209 err = store.UpdateLocalKeySync(context.TODO(), &localKey1) 210 require.NoError(t, err) 211 err = store.UpdateLocalKeySync(context.TODO(), &localKey2) 212 require.NoError(t, err) 213 214 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey1.updated(), 1) }, timeout, tick) 215 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey2.updated(), 1) }, timeout, tick) 216 217 store.DeleteLocalKey(context.TODO(), &localKey1) 218 store.DeleteLocalKey(context.TODO(), &localKey2) 219 220 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 1, localKey1.deleted()) }, timeout, tick) 221 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 1, localKey2.deleted()) }, timeout, tick) 222 } 223 224 func TestStoreLocalKeyProtection(t *testing.T) { 225 testutils.IntegrationTest(t) 226 for _, backendName := range []string{"etcd", "consul"} { 227 t.Run(backendName, func(t *testing.T) { 228 kvstore.SetupDummy(t, backendName) 229 testStoreLocalKeyProtection(t) 230 }) 231 } 232 } 233 234 func testStoreLocalKeyProtection(t *testing.T) { 235 store, err := JoinSharedStore(Configuration{ 236 Prefix: rand.String(12), 237 KeyCreator: newTestType, 238 SynchronizationInterval: time.Hour, // ensure that periodic sync does not interfer 239 Observer: &observer{}, 240 }) 241 require.NoError(t, err) 242 require.NotNil(t, store) 243 defer store.Close(context.TODO()) 244 245 localKey1 := initTestType("local1") 246 247 err = store.UpdateLocalKeySync(context.TODO(), &localKey1) 248 require.NoError(t, err) 249 250 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey1.updated(), 1) }, timeout, tick) 251 // delete all keys 252 kvstore.Client().DeletePrefix(context.TODO(), store.conf.Prefix) 253 require.EventuallyWithT(t, func(c *assert.CollectT) { 254 v, err := kvstore.Client().Get(context.TODO(), store.keyPath(&localKey1)) 255 assert.NoError(c, err) 256 assert.NotNil(c, v) 257 }, timeout, tick) 258 } 259 260 func setupStoreCollaboration(t *testing.T, storePrefix, keyPrefix string) *SharedStore { 261 store, err := JoinSharedStore(Configuration{ 262 Prefix: storePrefix, 263 KeyCreator: newTestType, 264 SynchronizationInterval: time.Second, 265 Observer: &observer{}, 266 }) 267 require.NoError(t, err) 268 require.NotNil(t, store) 269 270 localKey1 := initTestType(keyPrefix + "-local1") 271 err = store.UpdateLocalKeySync(context.TODO(), &localKey1) 272 require.NoError(t, err) 273 274 localKey2 := initTestType(keyPrefix + "-local2") 275 err = store.UpdateLocalKeySync(context.TODO(), &localKey2) 276 require.NoError(t, err) 277 278 // wait until local keys was inserted and until the kvstore has confirmed the 279 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey1.updated(), 1) }, timeout, tick) 280 require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey2.updated(), 1) }, timeout, tick) 281 282 require.Len(t, store.getLocalKeys(), 2) 283 284 return store 285 } 286 287 func TestStoreCollaboration(t *testing.T) { 288 testutils.IntegrationTest(t) 289 for _, backendName := range []string{"etcd", "consul"} { 290 t.Run(backendName, func(t *testing.T) { 291 kvstore.SetupDummy(t, backendName) 292 testStoreCollaboration(t) 293 }) 294 } 295 } 296 297 func testStoreCollaboration(t *testing.T) { 298 storePrefix := rand.String(12) 299 300 collab1 := setupStoreCollaboration(t, storePrefix, rand.String(12)) 301 defer collab1.Close(context.TODO()) 302 303 collab2 := setupStoreCollaboration(t, storePrefix, rand.String(12)) 304 defer collab2.Close(context.TODO()) 305 306 require.EventuallyWithT(t, func(c *assert.CollectT) { 307 all := append(collab1.getLocalKeys(), collab2.getLocalKeys()...) 308 assert.ElementsMatch(c, collab1.getSharedKeys(), all) 309 assert.ElementsMatch(c, collab2.getSharedKeys(), all) 310 }, timeout, tick) 311 } 312 313 // getLocalKeys returns all local keys 314 func (s *SharedStore) getLocalKeys() []Key { 315 s.mutex.RLock() 316 defer s.mutex.RUnlock() 317 318 keys := make([]Key, len(s.localKeys)) 319 idx := 0 320 for _, key := range s.localKeys { 321 keys[idx] = key 322 idx++ 323 } 324 325 return keys 326 } 327 328 // getSharedKeys returns all shared keys 329 func (s *SharedStore) getSharedKeys() []Key { 330 s.mutex.RLock() 331 defer s.mutex.RUnlock() 332 333 keys := make([]Key, len(s.sharedKeys)) 334 idx := 0 335 for _, key := range s.sharedKeys { 336 keys[idx] = key 337 idx++ 338 } 339 340 return keys 341 }