github.com/cilium/cilium@v1.16.2/pkg/kvstore/store/watchstore_test.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package store
     5  
     6  import (
     7  	"context"
     8  	"path"
     9  	"sync"
    10  	"testing"
    11  	"time"
    12  
    13  	"github.com/prometheus/client_golang/prometheus"
    14  	"github.com/prometheus/client_golang/prometheus/testutil"
    15  	"github.com/stretchr/testify/require"
    16  
    17  	"github.com/cilium/cilium/pkg/kvstore"
    18  	"github.com/cilium/cilium/pkg/metrics"
    19  )
    20  
    21  type fakeLWBackend struct {
    22  	t      *testing.T
    23  	prefix string
    24  	events []kvstore.KeyValueEvent
    25  }
    26  
    27  func NewFakeLWBackend(t *testing.T, prefix string, events []kvstore.KeyValueEvent) *fakeLWBackend {
    28  	return &fakeLWBackend{
    29  		t:      t,
    30  		prefix: prefix,
    31  		events: events,
    32  	}
    33  }
    34  
    35  func (fb *fakeLWBackend) ListAndWatch(ctx context.Context, prefix string, _ int) *kvstore.Watcher {
    36  	ch := make(kvstore.EventChan)
    37  
    38  	go func() {
    39  		defer close(ch)
    40  		require.Equal(fb.t, fb.prefix, prefix)
    41  
    42  		for _, event := range fb.events {
    43  			event.Key = path.Join(fb.prefix, event.Key)
    44  			select {
    45  			case ch <- event:
    46  			case <-ctx.Done():
    47  				require.Fail(fb.t, "Context closed before propagating all events", "pending: %#v", event)
    48  			}
    49  		}
    50  
    51  		<-ctx.Done()
    52  	}()
    53  
    54  	return &kvstore.Watcher{Events: ch}
    55  }
    56  
    57  type fakeObserver struct {
    58  	t       *testing.T
    59  	updated chan *KVPair
    60  	deleted chan *KVPair
    61  }
    62  
    63  func NewFakeObserver(t *testing.T) *fakeObserver {
    64  	return &fakeObserver{
    65  		t:       t,
    66  		updated: make(chan *KVPair),
    67  		deleted: make(chan *KVPair),
    68  	}
    69  }
    70  
    71  func (fo *fakeObserver) OnUpdate(k Key) {
    72  	select {
    73  	case fo.updated <- k.(*KVPair):
    74  	case <-time.After(timeout):
    75  		require.Failf(fo.t, "Failed observing update event", "key: %s", k.GetKeyName())
    76  	}
    77  }
    78  func (fo *fakeObserver) OnDelete(k NamedKey) {
    79  	select {
    80  	case fo.deleted <- k.(*KVPair):
    81  	case <-time.After(timeout):
    82  		require.Failf(fo.t, "Failed observing delete event", "key: %s", k.GetKeyName())
    83  	}
    84  }
    85  
    86  func rwsRun(store WatchStore, prefix string, body func(), backend WatchStoreBackend) {
    87  	ctx, cancel := context.WithCancel(context.Background())
    88  
    89  	var wg sync.WaitGroup
    90  	wg.Add(1)
    91  	go func() {
    92  		defer wg.Done()
    93  		store.Watch(ctx, backend, prefix)
    94  	}()
    95  
    96  	defer func() {
    97  		cancel()
    98  		wg.Wait()
    99  	}()
   100  
   101  	body()
   102  }
   103  
   104  func rwsDrain(t *testing.T, store WatchStore, observer *fakeObserver, expected []*KVPair) {
   105  	drainDone := make(chan struct{})
   106  	go func() {
   107  		store.Drain()
   108  		close(drainDone)
   109  	}()
   110  
   111  	var actual []*KVPair
   112  	for range expected {
   113  		actual = append(actual, eventually(observer.deleted))
   114  	}
   115  
   116  	// Since the drained elements are spilled out of a map, there's no ordering guarantee.
   117  	require.ElementsMatch(t, expected, actual)
   118  
   119  	select {
   120  	case <-drainDone:
   121  	case <-time.After(timeout):
   122  		require.Fail(t, "The drain operation did not complete when expected")
   123  	}
   124  }
   125  
   126  func TestRestartableWatchStore(t *testing.T) {
   127  	observer := NewFakeObserver(t)
   128  	f, _ := GetFactory(t)
   129  	store := f.NewWatchStore("qux", KVPairCreator, observer)
   130  	require.Equal(t, uint64(0), store.NumEntries())
   131  	require.False(t, store.Synced())
   132  
   133  	// Watch the kvstore once, and assert that the expected events are propagated
   134  	rwsRun(store, "foo/bar", func() {
   135  		require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated))
   136  		require.False(t, store.Synced(), "The store should not yet be synced")
   137  		require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.updated))
   138  		require.Eventually(t, store.Synced, timeout, tick, "The store should now be synced")
   139  		require.Equal(t, NewKVPair("key2", "value2B"), eventually(observer.updated))
   140  		require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated))
   141  		require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.deleted))
   142  		require.Equal(t, uint64(2), store.NumEntries())
   143  	}, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{
   144  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")},
   145  		{Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2A")},
   146  		{Typ: kvstore.EventTypeListDone},
   147  		{Typ: kvstore.EventTypeModify, Key: "key2", Value: []byte("value2B")},
   148  		{Typ: kvstore.EventTypeCreate, Key: "key3", Value: []byte("value3A")},
   149  		{Typ: kvstore.EventTypeDelete, Key: "key4"}, // The key is not known locally -> no event
   150  		{Typ: kvstore.EventTypeDelete, Key: "key3"},
   151  	}))
   152  
   153  	require.False(t, store.Synced(), "The store should no longer be synced, as stopped")
   154  
   155  	// Watch the kvstore a second time, and assert that the expected events (including
   156  	// stale keys deletions) are propagated, even though the watcher prefix changed.
   157  	rwsRun(store, "foo/baz", func() {
   158  		require.Equal(t, NewKVPair("key1", "value1C"), eventually(observer.updated))
   159  		require.Equal(t, NewKVPair("key4", "value4A"), eventually(observer.updated))
   160  		require.Equal(t, NewKVPair("key2", "value2B"), eventually(observer.deleted))
   161  		require.Equal(t, NewKVPair("key2", "value2C"), eventually(observer.updated))
   162  		require.True(t, store.Synced(), "The store should be synced again")
   163  		require.Equal(t, uint64(3), store.NumEntries())
   164  	}, NewFakeLWBackend(t, "foo/baz/", []kvstore.KeyValueEvent{
   165  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1C")},
   166  		{Typ: kvstore.EventTypeCreate, Key: "key4", Value: []byte("value4A")},
   167  		{Typ: kvstore.EventTypeListDone},
   168  		{Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2C")},
   169  	}))
   170  }
   171  
   172  func TestRestartableWatchStoreDrain(t *testing.T) {
   173  	observer := NewFakeObserver(t)
   174  	f, _ := GetFactory(t)
   175  	store := f.NewWatchStore("qux", KVPairCreator, observer)
   176  
   177  	// Watch a few keys through the watch store
   178  	rwsRun(store, "foo/bar", func() {
   179  		require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated))
   180  		require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.updated))
   181  		require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated))
   182  		require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.deleted))
   183  	}, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{
   184  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")},
   185  		{Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2A")},
   186  		{Typ: kvstore.EventTypeListDone},
   187  		{Typ: kvstore.EventTypeModify, Key: "key3", Value: []byte("value3A")},
   188  		{Typ: kvstore.EventTypeDelete, Key: "key2"},
   189  	}))
   190  
   191  	// Drain the store, and assert that a deletion event is emitted for all keys
   192  	rwsDrain(t, store, observer, []*KVPair{
   193  		NewKVPair("key1", "value1A"),
   194  		NewKVPair("key3", "value3A"),
   195  	})
   196  
   197  	// Make sure that it is possible to restart the watch store
   198  	rwsRun(store, "foo/bar", func() {
   199  		require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated))
   200  	}, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{
   201  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")},
   202  	}))
   203  
   204  	// And to drain it again
   205  	rwsDrain(t, store, observer, []*KVPair{
   206  		NewKVPair("key1", "value1A"),
   207  	})
   208  }
   209  
   210  func TestRestartableWatchStoreSyncCallback(t *testing.T) {
   211  	observer := NewFakeObserver(t)
   212  	callback := func(value string) func(context.Context) {
   213  		return func(context.Context) {
   214  			observer.OnUpdate(NewKVPair("callback/executed", value))
   215  		}
   216  	}
   217  	f, _ := GetFactory(t)
   218  	store := f.NewWatchStore("qux", KVPairCreator, observer,
   219  		RWSWithOnSyncCallback(callback("1")), RWSWithOnSyncCallback(callback("2")))
   220  
   221  	// The watcher is closed before receiving the list done event, the sync callbacks should not be executed
   222  	rwsRun(store, "foo/bar", func() {
   223  		require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated))
   224  	}, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{
   225  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")},
   226  	}))
   227  
   228  	// Assert that the callback are executed when the list done event is received
   229  	rwsRun(store, "foo/bar", func() {
   230  		require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated))
   231  		require.Equal(t, NewKVPair("callback/executed", "1"), eventually(observer.updated))
   232  		require.Equal(t, NewKVPair("callback/executed", "2"), eventually(observer.updated))
   233  		require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.updated))
   234  	}, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{
   235  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")},
   236  		{Typ: kvstore.EventTypeListDone},
   237  		{Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2A")},
   238  	}))
   239  
   240  	// Assert that the callbacks are not executed a second time
   241  	rwsRun(store, "foo/bar", func() {
   242  		require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated))
   243  		require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.deleted))
   244  		require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated))
   245  	}, NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{
   246  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")},
   247  		{Typ: kvstore.EventTypeListDone},
   248  		{Typ: kvstore.EventTypeCreate, Key: "key3", Value: []byte("value3A")},
   249  	}))
   250  }
   251  
   252  func TestRestartableWatchStoreConcurrent(t *testing.T) {
   253  	var wg sync.WaitGroup
   254  	ctx, cancel := context.WithCancel(context.Background())
   255  
   256  	defer func() {
   257  		cancel()
   258  		wg.Wait()
   259  	}()
   260  
   261  	backend := NewFakeLWBackend(t, "foo/bar/", []kvstore.KeyValueEvent{
   262  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1")},
   263  	})
   264  	observer := NewFakeObserver(t)
   265  	f, _ := GetFactory(t)
   266  	store := f.NewWatchStore("qux", KVPairCreator, observer)
   267  
   268  	wg.Add(1)
   269  	go func() {
   270  		store.Watch(ctx, backend, "foo/bar/")
   271  		wg.Done()
   272  	}()
   273  
   274  	// Ensure that the Watch operation running in the goroutine has started
   275  	require.Equal(t, NewKVPair("key1", "value1"), eventually(observer.updated))
   276  
   277  	require.Panics(t, func() { store.Watch(ctx, backend, "foo/bar/") }, "store.Watch should panic when already running")
   278  	require.Panics(t, store.Drain, "store.Drain should panic when store.Watch is running")
   279  }
   280  
   281  func TestRestartableWatchStoreMetrics(t *testing.T) {
   282  	f, m := GetFactory(t)
   283  	metrics.NewLegacyMetrics()
   284  	require.True(t, m.KVStoreInitialSyncCompleted.IsEnabled())
   285  
   286  	entries := prometheus.NewGauge(prometheus.GaugeOpts{Name: "test_elements_metric"})
   287  	synced := m.KVStoreInitialSyncCompleted.WithLabelValues("nodes/v1", "qux", "read")
   288  
   289  	observer := NewFakeObserver(t)
   290  	store := f.NewWatchStore("qux", KVPairCreator, observer, RWSWithEntriesMetric(entries))
   291  
   292  	require.Equal(t, float64(0), testutil.ToFloat64(entries))
   293  	require.Equal(t, metrics.BoolToFloat64(false), testutil.ToFloat64(synced))
   294  
   295  	rwsRun(store, "cilium/state/nodes/v1", func() {
   296  		require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated))
   297  		require.Equal(t, metrics.BoolToFloat64(false), testutil.ToFloat64(synced))
   298  		require.Equal(t, NewKVPair("key2", "value2A"), eventually(observer.updated))
   299  
   300  		require.Eventually(t, func() bool {
   301  			return metrics.BoolToFloat64(true) == testutil.ToFloat64(synced)
   302  		}, timeout, tick)
   303  
   304  		require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.deleted))
   305  		require.Equal(t, NewKVPair("key2", "value2B"), eventually(observer.updated))
   306  		require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated))
   307  	}, NewFakeLWBackend(t, "cilium/state/nodes/v1/", []kvstore.KeyValueEvent{
   308  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")},
   309  		{Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2A")},
   310  		{Typ: kvstore.EventTypeListDone},
   311  		{Typ: kvstore.EventTypeDelete, Key: "key1"},
   312  		{Typ: kvstore.EventTypeCreate, Key: "key2", Value: []byte("value2B")},
   313  		{Typ: kvstore.EventTypeCreate, Key: "key3", Value: []byte("value3A")},
   314  	}))
   315  
   316  	// The metric should reflect the number of elements.
   317  	require.Equal(t, float64(2), testutil.ToFloat64(entries))
   318  	require.Equal(t, metrics.BoolToFloat64(false), testutil.ToFloat64(synced))
   319  
   320  	rwsRun(store, "cilium/state/nodes/v1", func() {
   321  		require.Equal(t, NewKVPair("key3", "value3A"), eventually(observer.updated))
   322  		require.Equal(t, metrics.BoolToFloat64(false), testutil.ToFloat64(synced))
   323  		require.Equal(t, NewKVPair("key2", "value2B"), eventually(observer.deleted))
   324  
   325  		require.Eventually(t, func() bool {
   326  			return metrics.BoolToFloat64(true) == testutil.ToFloat64(synced)
   327  		}, timeout, tick)
   328  
   329  		require.Equal(t, NewKVPair("key1", "value1A"), eventually(observer.updated))
   330  	}, NewFakeLWBackend(t, "cilium/state/nodes/v1/", []kvstore.KeyValueEvent{
   331  		{Typ: kvstore.EventTypeCreate, Key: "key3", Value: []byte("value3A")},
   332  		{Typ: kvstore.EventTypeListDone},
   333  		{Typ: kvstore.EventTypeCreate, Key: "key1", Value: []byte("value1A")},
   334  	}))
   335  }