github.com/sequix/cortex@v1.1.6/pkg/chunk/cache/cache_test.go (about)

     1  package cache_test
     2  
     3  import (
     4  	"context"
     5  	"math/rand"
     6  	"sort"
     7  	"strconv"
     8  	"testing"
     9  	"time"
    10  
    11  	"github.com/sequix/cortex/pkg/chunk"
    12  	"github.com/sequix/cortex/pkg/chunk/cache"
    13  	prom_chunk "github.com/sequix/cortex/pkg/chunk/encoding"
    14  	"github.com/prometheus/common/model"
    15  	"github.com/prometheus/prometheus/pkg/labels"
    16  	"github.com/stretchr/testify/require"
    17  )
    18  
    19  const userID = "1"
    20  
    21  func fillCache(t *testing.T, cache cache.Cache) ([]string, []chunk.Chunk) {
    22  	const chunkLen = 13 * 3600 // in seconds
    23  
    24  	// put 100 chunks from 0 to 99
    25  	keys := []string{}
    26  	bufs := [][]byte{}
    27  	chunks := []chunk.Chunk{}
    28  	for i := 0; i < 100; i++ {
    29  		ts := model.TimeFromUnix(int64(i * chunkLen))
    30  		promChunk, _ := prom_chunk.New().Add(model.SamplePair{
    31  			Timestamp: ts,
    32  			Value:     model.SampleValue(i),
    33  		})
    34  		c := chunk.NewChunk(
    35  			userID,
    36  			model.Fingerprint(1),
    37  			labels.Labels{
    38  				{Name: model.MetricNameLabel, Value: "foo"},
    39  				{Name: "bar", Value: "baz"},
    40  			},
    41  			promChunk[0],
    42  			ts,
    43  			ts.Add(chunkLen),
    44  		)
    45  
    46  		err := c.Encode()
    47  		require.NoError(t, err)
    48  		buf, err := c.Encoded()
    49  		require.NoError(t, err)
    50  
    51  		keys = append(keys, c.ExternalKey())
    52  		bufs = append(bufs, buf)
    53  		chunks = append(chunks, c)
    54  	}
    55  
    56  	cache.Store(context.Background(), keys, bufs)
    57  	return keys, chunks
    58  }
    59  
    60  func testCacheSingle(t *testing.T, cache cache.Cache, keys []string, chunks []chunk.Chunk) {
    61  	for i := 0; i < 100; i++ {
    62  		index := rand.Intn(len(keys))
    63  		key := keys[index]
    64  
    65  		found, bufs, missingKeys := cache.Fetch(context.Background(), []string{key})
    66  		require.Len(t, found, 1)
    67  		require.Len(t, bufs, 1)
    68  		require.Len(t, missingKeys, 0)
    69  
    70  		c, err := chunk.ParseExternalKey(userID, found[0])
    71  		require.NoError(t, err)
    72  		err = c.Decode(chunk.NewDecodeContext(), bufs[0])
    73  		require.NoError(t, err)
    74  		require.Equal(t, c, chunks[index])
    75  	}
    76  }
    77  
    78  func testCacheMultiple(t *testing.T, cache cache.Cache, keys []string, chunks []chunk.Chunk) {
    79  	// test getting them all
    80  	found, bufs, missingKeys := cache.Fetch(context.Background(), keys)
    81  	require.Len(t, found, len(keys))
    82  	require.Len(t, bufs, len(keys))
    83  	require.Len(t, missingKeys, 0)
    84  
    85  	result := []chunk.Chunk{}
    86  	for i := range found {
    87  		c, err := chunk.ParseExternalKey(userID, found[i])
    88  		require.NoError(t, err)
    89  		err = c.Decode(chunk.NewDecodeContext(), bufs[i])
    90  		require.NoError(t, err)
    91  		result = append(result, c)
    92  	}
    93  	require.Equal(t, chunks, result)
    94  }
    95  
    96  func testChunkFetcher(t *testing.T, c cache.Cache, keys []string, chunks []chunk.Chunk) {
    97  	fetcher, err := chunk.NewChunkFetcher(cache.Config{
    98  		Cache: c,
    99  	}, false, nil)
   100  	require.NoError(t, err)
   101  	defer fetcher.Stop()
   102  
   103  	found, err := fetcher.FetchChunks(context.Background(), chunks, keys)
   104  	require.NoError(t, err)
   105  	sort.Sort(byExternalKey(found))
   106  	sort.Sort(byExternalKey(chunks))
   107  	require.Equal(t, chunks, found)
   108  }
   109  
   110  type byExternalKey []chunk.Chunk
   111  
   112  func (a byExternalKey) Len() int           { return len(a) }
   113  func (a byExternalKey) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
   114  func (a byExternalKey) Less(i, j int) bool { return a[i].ExternalKey() < a[j].ExternalKey() }
   115  
   116  func testCacheMiss(t *testing.T, cache cache.Cache) {
   117  	for i := 0; i < 100; i++ {
   118  		key := strconv.Itoa(rand.Int())
   119  		found, bufs, missing := cache.Fetch(context.Background(), []string{key})
   120  		require.Empty(t, found)
   121  		require.Empty(t, bufs)
   122  		require.Len(t, missing, 1)
   123  	}
   124  }
   125  
   126  func testCache(t *testing.T, cache cache.Cache) {
   127  	keys, chunks := fillCache(t, cache)
   128  	t.Run("Single", func(t *testing.T) {
   129  		testCacheSingle(t, cache, keys, chunks)
   130  	})
   131  	t.Run("Multiple", func(t *testing.T) {
   132  		testCacheMultiple(t, cache, keys, chunks)
   133  	})
   134  	t.Run("Miss", func(t *testing.T) {
   135  		testCacheMiss(t, cache)
   136  	})
   137  	t.Run("Fetcher", func(t *testing.T) {
   138  		testChunkFetcher(t, cache, keys, chunks)
   139  	})
   140  }
   141  
   142  func TestMemcache(t *testing.T) {
   143  	t.Run("Unbatched", func(t *testing.T) {
   144  		cache := cache.NewMemcached(cache.MemcachedConfig{}, newMockMemcache(), "test")
   145  		testCache(t, cache)
   146  	})
   147  
   148  	t.Run("Batched", func(t *testing.T) {
   149  		cache := cache.NewMemcached(cache.MemcachedConfig{
   150  			BatchSize:   10,
   151  			Parallelism: 3,
   152  		}, newMockMemcache(), "test")
   153  		testCache(t, cache)
   154  	})
   155  }
   156  
   157  func TestFifoCache(t *testing.T) {
   158  	cache := cache.NewFifoCache("test", cache.FifoCacheConfig{Size: 1e3, Validity: 1 * time.Hour})
   159  	testCache(t, cache)
   160  }
   161  
   162  func TestSnappyCache(t *testing.T) {
   163  	cache := cache.NewSnappy(cache.NewMockCache())
   164  	testCache(t, cache)
   165  }