github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/ingester/index/multi_test.go (about)

     1  package index
     2  
     3  import (
     4  	"sort"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/prometheus/common/model"
     9  	"github.com/prometheus/prometheus/model/labels"
    10  	"github.com/stretchr/testify/require"
    11  
    12  	"github.com/grafana/loki/pkg/logproto"
    13  	"github.com/grafana/loki/pkg/querier/astmapper"
    14  	"github.com/grafana/loki/pkg/storage/config"
    15  	"github.com/grafana/loki/pkg/storage/stores/tsdb/index"
    16  )
    17  
    18  func MustParseDayTime(s string) config.DayTime {
    19  	t, err := time.Parse("2006-01-02", s)
    20  	if err != nil {
    21  		panic(err)
    22  	}
    23  	return config.DayTime{Time: model.TimeFromUnix(t.Unix())}
    24  }
    25  
    26  var testPeriodConfigs = []config.PeriodConfig{
    27  	{
    28  		From:      MustParseDayTime("2020-01-01"),
    29  		IndexType: config.StorageTypeBigTable,
    30  	},
    31  	{
    32  		From:      MustParseDayTime("2021-01-01"),
    33  		IndexType: config.TSDBType,
    34  	},
    35  	{
    36  		From:      MustParseDayTime("2022-01-01"),
    37  		IndexType: config.BoltDBShipperType,
    38  	},
    39  	{
    40  		From:      MustParseDayTime("2023-01-01"),
    41  		IndexType: config.TSDBType,
    42  	},
    43  }
    44  
    45  // Only run the specific shard factor validation logic if a period config using
    46  // tsdb exists
    47  func TestIgnoresInvalidShardFactorWhenTSDBNotPresent(t *testing.T) {
    48  	factor := uint32(6)
    49  	_, err := NewMultiInvertedIndex(
    50  		[]config.PeriodConfig{
    51  			{
    52  				From:      MustParseDayTime("2020-01-01"),
    53  				IndexType: config.StorageTypeBigTable,
    54  			},
    55  		},
    56  		factor,
    57  	)
    58  	require.Nil(t, err)
    59  
    60  	_, err = NewMultiInvertedIndex(
    61  		[]config.PeriodConfig{
    62  			{
    63  				From:      MustParseDayTime("2020-01-01"),
    64  				IndexType: config.StorageTypeBigTable,
    65  			},
    66  			{
    67  				From:      MustParseDayTime("2021-01-01"),
    68  				IndexType: config.TSDBType,
    69  			},
    70  		},
    71  		factor,
    72  	)
    73  	require.Error(t, err)
    74  
    75  }
    76  
    77  func TestMultiIndexCreation(t *testing.T) {
    78  	multi, err := NewMultiInvertedIndex(testPeriodConfigs, uint32(2))
    79  	require.Nil(t, err)
    80  
    81  	x, _ := NewBitPrefixWithShards(2)
    82  	expected := &Multi{
    83  		periods: []periodIndex{
    84  			{
    85  				Time: testPeriodConfigs[0].From.Time.Time(),
    86  				idx:  0,
    87  			},
    88  			{
    89  				Time: testPeriodConfigs[1].From.Time.Time(),
    90  				idx:  1,
    91  			},
    92  			{
    93  				Time: testPeriodConfigs[2].From.Time.Time(),
    94  				idx:  0,
    95  			},
    96  			{
    97  				Time: testPeriodConfigs[3].From.Time.Time(),
    98  				idx:  1,
    99  			},
   100  		},
   101  		indices: []Interface{
   102  			NewWithShards(2),
   103  			x,
   104  		},
   105  	}
   106  	require.Equal(t, expected, multi)
   107  }
   108  
   109  func TestMultiIndex(t *testing.T) {
   110  	factor := uint32(32)
   111  	multi, err := NewMultiInvertedIndex(testPeriodConfigs, factor)
   112  	require.Nil(t, err)
   113  
   114  	lbs := []logproto.LabelAdapter{
   115  		{Name: "foo", Value: "foo"},
   116  		{Name: "bar", Value: "bar"},
   117  		{Name: "buzz", Value: "buzz"},
   118  	}
   119  	sort.Sort(logproto.FromLabelAdaptersToLabels(lbs))
   120  	fp := model.Fingerprint((logproto.FromLabelAdaptersToLabels(lbs).Hash()))
   121  
   122  	ls := multi.Add(lbs, fp)
   123  
   124  	// Lookup at a time corresponding to a non-tsdb periodconfig
   125  	// and ensure we use modulo hashing
   126  	expShard := labelsSeriesIDHash(logproto.FromLabelAdaptersToLabels(lbs)) % factor
   127  	ids, err := multi.Lookup(
   128  		testPeriodConfigs[0].From.Time.Time(),
   129  		[]*labels.Matcher{
   130  			labels.MustNewMatcher(labels.MatchEqual, "foo", "foo"),
   131  		},
   132  		&astmapper.ShardAnnotation{Shard: int(expShard), Of: int(factor)},
   133  	)
   134  
   135  	require.Nil(t, err)
   136  	require.Equal(t, []model.Fingerprint{fp}, ids)
   137  
   138  	// Lookup at a time corresponding to a tsdb periodconfig
   139  	// and ensure we use bit prefix hashing
   140  	requiredBits := index.NewShard(0, factor).RequiredBits()
   141  	expShard = uint32(fp >> (64 - requiredBits))
   142  	ids, err = multi.Lookup(
   143  		testPeriodConfigs[1].From.Time.Time(),
   144  		[]*labels.Matcher{
   145  			labels.MustNewMatcher(labels.MatchEqual, "foo", "foo"),
   146  		},
   147  		&astmapper.ShardAnnotation{Shard: int(expShard), Of: int(factor)},
   148  	)
   149  
   150  	require.Nil(t, err)
   151  	require.Equal(t, []model.Fingerprint{fp}, ids)
   152  
   153  	// Delete the entry
   154  	multi.Delete(ls, fp)
   155  
   156  	// Ensure deleted entry is not in modulo variant
   157  	ids, err = multi.Lookup(
   158  		testPeriodConfigs[0].From.Time.Time(),
   159  		[]*labels.Matcher{
   160  			labels.MustNewMatcher(labels.MatchEqual, "foo", "foo"),
   161  		},
   162  		nil,
   163  	)
   164  
   165  	require.Nil(t, err)
   166  	require.Equal(t, 0, len(ids))
   167  
   168  	// Ensure deleted entry is not in bit prefix variant
   169  	ids, err = multi.Lookup(
   170  		testPeriodConfigs[1].From.Time.Time(),
   171  		[]*labels.Matcher{
   172  			labels.MustNewMatcher(labels.MatchEqual, "foo", "foo"),
   173  		},
   174  		nil,
   175  	)
   176  
   177  	require.Nil(t, err)
   178  	require.Equal(t, 0, len(ids))
   179  }