github.com/cilium/cilium@v1.16.2/pkg/kvstore/store/store_test.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package store
     5  
     6  import (
     7  	"context"
     8  	"encoding/json"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/stretchr/testify/assert"
    13  	"github.com/stretchr/testify/require"
    14  	"k8s.io/apimachinery/pkg/util/rand"
    15  
    16  	"github.com/cilium/cilium/pkg/defaults"
    17  	"github.com/cilium/cilium/pkg/kvstore"
    18  	"github.com/cilium/cilium/pkg/lock"
    19  	"github.com/cilium/cilium/pkg/option"
    20  	"github.com/cilium/cilium/pkg/testutils"
    21  )
    22  
    23  const (
    24  	testPrefix           = "store-tests"
    25  	sharedKeyDeleteDelay = time.Second
    26  )
    27  
    28  type TestType struct {
    29  	Name string
    30  }
    31  
    32  var _ = TestType{}
    33  
    34  func (t *TestType) GetKeyName() string                    { return t.Name }
    35  func (t *TestType) DeepKeyCopy() LocalKey                 { return &TestType{Name: t.Name} }
    36  func (t *TestType) Marshal() ([]byte, error)              { return json.Marshal(t) }
    37  func (t *TestType) Unmarshal(_ string, data []byte) error { return json.Unmarshal(data, t) }
    38  
    39  type opCounter struct {
    40  	deleted int
    41  	updated int
    42  }
    43  
    44  var (
    45  	counter     = map[string]*opCounter{}
    46  	counterLock lock.RWMutex
    47  )
    48  
    49  func (t *TestType) deleted() int {
    50  	counterLock.RLock()
    51  	defer counterLock.RUnlock()
    52  	return counter[t.Name].deleted
    53  }
    54  
    55  func (t *TestType) updated() int {
    56  	counterLock.RLock()
    57  	defer counterLock.RUnlock()
    58  	return counter[t.Name].updated
    59  }
    60  
    61  func initTestType(name string) TestType {
    62  	t := TestType{}
    63  	t.Name = name
    64  	counterLock.Lock()
    65  	counter[name] = &opCounter{}
    66  	counterLock.Unlock()
    67  	return t
    68  }
    69  
    70  type observer struct{}
    71  
    72  func (o *observer) OnUpdate(k Key) {
    73  	counterLock.Lock()
    74  	if c, ok := counter[k.(*TestType).Name]; ok {
    75  		c.updated++
    76  	}
    77  	counterLock.Unlock()
    78  }
    79  func (o *observer) OnDelete(k NamedKey) {
    80  	counterLock.Lock()
    81  	counter[k.(*TestType).Name].deleted++
    82  	counterLock.Unlock()
    83  }
    84  
    85  func newTestType() Key {
    86  	t := TestType{}
    87  	return &t
    88  }
    89  
    90  func TestStoreCreation(t *testing.T) {
    91  	testutils.IntegrationTest(t)
    92  	for _, backendName := range []string{"etcd", "consul"} {
    93  		t.Run(backendName, func(t *testing.T) {
    94  			kvstore.SetupDummy(t, backendName)
    95  			testStoreCreation(t)
    96  		})
    97  	}
    98  }
    99  
   100  func testStoreCreation(t *testing.T) {
   101  	// Missing Prefix must result in error
   102  	store, err := JoinSharedStore(Configuration{})
   103  	require.ErrorContains(t, err, "prefix must be specified")
   104  	require.Nil(t, store)
   105  
   106  	// Missing KeyCreator must result in error
   107  	store, err = JoinSharedStore(Configuration{Prefix: rand.String(12)})
   108  	require.ErrorContains(t, err, "KeyCreator must be specified")
   109  	require.Nil(t, store)
   110  
   111  	// Basic creation should result in default values
   112  	store, err = JoinSharedStore(Configuration{Prefix: rand.String(12), KeyCreator: newTestType})
   113  	require.NoError(t, err)
   114  	require.NotNil(t, store)
   115  	require.Equal(t, option.Config.KVstorePeriodicSync, store.conf.SynchronizationInterval)
   116  	store.Close(context.TODO())
   117  
   118  	// Test with kvstore client specified
   119  	store, err = JoinSharedStore(Configuration{Prefix: rand.String(12), KeyCreator: newTestType, Backend: kvstore.Client()})
   120  	require.NoError(t, err)
   121  	require.NotNil(t, store)
   122  	require.Equal(t, option.Config.KVstorePeriodicSync, store.conf.SynchronizationInterval)
   123  	store.Close(context.TODO())
   124  }
   125  
   126  func TestStoreOperations(t *testing.T) {
   127  	testutils.IntegrationTest(t)
   128  	for _, backendName := range []string{"etcd", "consul"} {
   129  		t.Run(backendName, func(t *testing.T) {
   130  			kvstore.SetupDummy(t, backendName)
   131  			testStoreOperations(t)
   132  		})
   133  	}
   134  }
   135  
   136  func testStoreOperations(t *testing.T) {
   137  	// Basic creation should result in default values
   138  	store, err := JoinSharedStore(Configuration{
   139  		Prefix:               rand.String(12),
   140  		KeyCreator:           newTestType,
   141  		Observer:             &observer{},
   142  		SharedKeyDeleteDelay: sharedKeyDeleteDelay,
   143  	})
   144  	require.NoError(t, err)
   145  	require.NotNil(t, store)
   146  	defer store.Close(context.TODO())
   147  
   148  	localKey1 := initTestType("local1")
   149  	localKey2 := initTestType("local2")
   150  	localKey3 := initTestType("local3")
   151  
   152  	err = store.UpdateLocalKeySync(context.TODO(), &localKey1)
   153  	require.NoError(t, err)
   154  	err = store.UpdateLocalKeySync(context.TODO(), &localKey2)
   155  	require.NoError(t, err)
   156  
   157  	// due to the short sync interval, it is possible that multiple updates
   158  	// have occurred, make the test reliable by succeeding on at lest one
   159  	// update
   160  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey1.updated(), 1) }, timeout, tick)
   161  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey2.updated(), 1) }, timeout, tick)
   162  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey3.updated()) }, timeout, tick)
   163  
   164  	store.DeleteLocalKey(context.TODO(), &localKey1)
   165  	// localKey1 will be deleted 2 times, one from local key and other from
   166  	// the kvstore watcher
   167  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 2, localKey1.deleted()) }, timeout, tick)
   168  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey2.deleted()) }, timeout, tick)
   169  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey3.deleted()) }, timeout, tick)
   170  
   171  	store.DeleteLocalKey(context.TODO(), &localKey3)
   172  	// localKey3 won't be deleted because it was never added
   173  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey3.deleted()) }, timeout, tick)
   174  
   175  	store.DeleteLocalKey(context.TODO(), &localKey2)
   176  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 2, localKey1.deleted()) }, timeout, tick)
   177  	// localKey2 will be deleted 2 times, one from local key and other from
   178  	// the kvstore watcher
   179  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 2, localKey2.deleted()) }, timeout, tick)
   180  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 0, localKey3.deleted()) }, timeout, tick)
   181  }
   182  
   183  func TestStorePeriodicSync(t *testing.T) {
   184  	testutils.IntegrationTest(t)
   185  	for _, backendName := range []string{"etcd", "consul"} {
   186  		t.Run(backendName, func(t *testing.T) {
   187  			kvstore.SetupDummy(t, backendName)
   188  			testStorePeriodicSync(t)
   189  		})
   190  	}
   191  }
   192  
   193  func testStorePeriodicSync(t *testing.T) {
   194  	// Create a store with a very short periodic sync interval
   195  	store, err := JoinSharedStore(Configuration{
   196  		Prefix:                  rand.String(12),
   197  		KeyCreator:              newTestType,
   198  		SynchronizationInterval: 10 * time.Millisecond,
   199  		SharedKeyDeleteDelay:    defaults.NodeDeleteDelay,
   200  		Observer:                &observer{},
   201  	})
   202  	require.NoError(t, err)
   203  	require.NotNil(t, store)
   204  	defer store.Close(context.TODO())
   205  
   206  	localKey1 := initTestType("local1")
   207  	localKey2 := initTestType("local2")
   208  
   209  	err = store.UpdateLocalKeySync(context.TODO(), &localKey1)
   210  	require.NoError(t, err)
   211  	err = store.UpdateLocalKeySync(context.TODO(), &localKey2)
   212  	require.NoError(t, err)
   213  
   214  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey1.updated(), 1) }, timeout, tick)
   215  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey2.updated(), 1) }, timeout, tick)
   216  
   217  	store.DeleteLocalKey(context.TODO(), &localKey1)
   218  	store.DeleteLocalKey(context.TODO(), &localKey2)
   219  
   220  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 1, localKey1.deleted()) }, timeout, tick)
   221  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.EqualValues(c, 1, localKey2.deleted()) }, timeout, tick)
   222  }
   223  
   224  func TestStoreLocalKeyProtection(t *testing.T) {
   225  	testutils.IntegrationTest(t)
   226  	for _, backendName := range []string{"etcd", "consul"} {
   227  		t.Run(backendName, func(t *testing.T) {
   228  			kvstore.SetupDummy(t, backendName)
   229  			testStoreLocalKeyProtection(t)
   230  		})
   231  	}
   232  }
   233  
   234  func testStoreLocalKeyProtection(t *testing.T) {
   235  	store, err := JoinSharedStore(Configuration{
   236  		Prefix:                  rand.String(12),
   237  		KeyCreator:              newTestType,
   238  		SynchronizationInterval: time.Hour, // ensure that periodic sync does not interfer
   239  		Observer:                &observer{},
   240  	})
   241  	require.NoError(t, err)
   242  	require.NotNil(t, store)
   243  	defer store.Close(context.TODO())
   244  
   245  	localKey1 := initTestType("local1")
   246  
   247  	err = store.UpdateLocalKeySync(context.TODO(), &localKey1)
   248  	require.NoError(t, err)
   249  
   250  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey1.updated(), 1) }, timeout, tick)
   251  	// delete all keys
   252  	kvstore.Client().DeletePrefix(context.TODO(), store.conf.Prefix)
   253  	require.EventuallyWithT(t, func(c *assert.CollectT) {
   254  		v, err := kvstore.Client().Get(context.TODO(), store.keyPath(&localKey1))
   255  		assert.NoError(c, err)
   256  		assert.NotNil(c, v)
   257  	}, timeout, tick)
   258  }
   259  
   260  func setupStoreCollaboration(t *testing.T, storePrefix, keyPrefix string) *SharedStore {
   261  	store, err := JoinSharedStore(Configuration{
   262  		Prefix:                  storePrefix,
   263  		KeyCreator:              newTestType,
   264  		SynchronizationInterval: time.Second,
   265  		Observer:                &observer{},
   266  	})
   267  	require.NoError(t, err)
   268  	require.NotNil(t, store)
   269  
   270  	localKey1 := initTestType(keyPrefix + "-local1")
   271  	err = store.UpdateLocalKeySync(context.TODO(), &localKey1)
   272  	require.NoError(t, err)
   273  
   274  	localKey2 := initTestType(keyPrefix + "-local2")
   275  	err = store.UpdateLocalKeySync(context.TODO(), &localKey2)
   276  	require.NoError(t, err)
   277  
   278  	// wait until local keys was inserted and until the kvstore has confirmed the
   279  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey1.updated(), 1) }, timeout, tick)
   280  	require.EventuallyWithT(t, func(c *assert.CollectT) { assert.GreaterOrEqual(c, localKey2.updated(), 1) }, timeout, tick)
   281  
   282  	require.Len(t, store.getLocalKeys(), 2)
   283  
   284  	return store
   285  }
   286  
   287  func TestStoreCollaboration(t *testing.T) {
   288  	testutils.IntegrationTest(t)
   289  	for _, backendName := range []string{"etcd", "consul"} {
   290  		t.Run(backendName, func(t *testing.T) {
   291  			kvstore.SetupDummy(t, backendName)
   292  			testStoreCollaboration(t)
   293  		})
   294  	}
   295  }
   296  
   297  func testStoreCollaboration(t *testing.T) {
   298  	storePrefix := rand.String(12)
   299  
   300  	collab1 := setupStoreCollaboration(t, storePrefix, rand.String(12))
   301  	defer collab1.Close(context.TODO())
   302  
   303  	collab2 := setupStoreCollaboration(t, storePrefix, rand.String(12))
   304  	defer collab2.Close(context.TODO())
   305  
   306  	require.EventuallyWithT(t, func(c *assert.CollectT) {
   307  		all := append(collab1.getLocalKeys(), collab2.getLocalKeys()...)
   308  		assert.ElementsMatch(c, collab1.getSharedKeys(), all)
   309  		assert.ElementsMatch(c, collab2.getSharedKeys(), all)
   310  	}, timeout, tick)
   311  }
   312  
   313  // getLocalKeys returns all local keys
   314  func (s *SharedStore) getLocalKeys() []Key {
   315  	s.mutex.RLock()
   316  	defer s.mutex.RUnlock()
   317  
   318  	keys := make([]Key, len(s.localKeys))
   319  	idx := 0
   320  	for _, key := range s.localKeys {
   321  		keys[idx] = key
   322  		idx++
   323  	}
   324  
   325  	return keys
   326  }
   327  
   328  // getSharedKeys returns all shared keys
   329  func (s *SharedStore) getSharedKeys() []Key {
   330  	s.mutex.RLock()
   331  	defer s.mutex.RUnlock()
   332  
   333  	keys := make([]Key, len(s.sharedKeys))
   334  	idx := 0
   335  	for _, key := range s.sharedKeys {
   336  		keys[idx] = key
   337  		idx++
   338  	}
   339  
   340  	return keys
   341  }