github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/storage/tsdb/bucketindex/index_test.go (about)

     1  package bucketindex
     2  
     3  import (
     4  	"testing"
     5  
     6  	"github.com/oklog/ulid"
     7  	"github.com/prometheus/prometheus/tsdb"
     8  	"github.com/stretchr/testify/assert"
     9  	"github.com/thanos-io/thanos/pkg/block/metadata"
    10  )
    11  
    12  func TestIndex_RemoveBlock(t *testing.T) {
    13  	block1 := ulid.MustNew(1, nil)
    14  	block2 := ulid.MustNew(2, nil)
    15  	block3 := ulid.MustNew(3, nil)
    16  	idx := &Index{
    17  		Blocks:             Blocks{{ID: block1}, {ID: block2}, {ID: block3}},
    18  		BlockDeletionMarks: BlockDeletionMarks{{ID: block2}, {ID: block3}},
    19  	}
    20  
    21  	idx.RemoveBlock(block2)
    22  	assert.ElementsMatch(t, []ulid.ULID{block1, block3}, idx.Blocks.GetULIDs())
    23  	assert.ElementsMatch(t, []ulid.ULID{block3}, idx.BlockDeletionMarks.GetULIDs())
    24  }
    25  
    26  func TestDetectBlockSegmentsFormat(t *testing.T) {
    27  	tests := map[string]struct {
    28  		meta           metadata.Meta
    29  		expectedFormat string
    30  		expectedNum    int
    31  	}{
    32  		"meta.json without SegmentFiles and Files": {
    33  			meta:           metadata.Meta{},
    34  			expectedFormat: SegmentsFormatUnknown,
    35  			expectedNum:    0,
    36  		},
    37  		"meta.json with SegmentFiles, 0 based 6 digits": {
    38  			meta: metadata.Meta{
    39  				Thanos: metadata.Thanos{
    40  					SegmentFiles: []string{
    41  						"000000",
    42  						"000001",
    43  						"000002",
    44  					},
    45  				},
    46  			},
    47  			expectedFormat: SegmentsFormatUnknown,
    48  			expectedNum:    0,
    49  		},
    50  		"meta.json with SegmentFiles, 1 based 6 digits": {
    51  			meta: metadata.Meta{
    52  				Thanos: metadata.Thanos{
    53  					SegmentFiles: []string{
    54  						"000001",
    55  						"000002",
    56  						"000003",
    57  					},
    58  				},
    59  			},
    60  			expectedFormat: SegmentsFormat1Based6Digits,
    61  			expectedNum:    3,
    62  		},
    63  		"meta.json with SegmentFiles, 1 based 6 digits but non consecutive": {
    64  			meta: metadata.Meta{
    65  				Thanos: metadata.Thanos{
    66  					SegmentFiles: []string{
    67  						"000001",
    68  						"000003",
    69  						"000004",
    70  					},
    71  				},
    72  			},
    73  			expectedFormat: SegmentsFormatUnknown,
    74  			expectedNum:    0,
    75  		},
    76  		"meta.json with Files, 0 based 6 digits": {
    77  			meta: metadata.Meta{
    78  				Thanos: metadata.Thanos{
    79  					Files: []metadata.File{
    80  						{RelPath: "index"},
    81  						{RelPath: "chunks/000000"},
    82  						{RelPath: "chunks/000001"},
    83  						{RelPath: "chunks/000002"},
    84  						{RelPath: "tombstone"},
    85  					},
    86  				},
    87  			},
    88  			expectedFormat: SegmentsFormatUnknown,
    89  			expectedNum:    0,
    90  		},
    91  		"meta.json with Files, 1 based 6 digits": {
    92  			meta: metadata.Meta{
    93  				Thanos: metadata.Thanos{
    94  					Files: []metadata.File{
    95  						{RelPath: "index"},
    96  						{RelPath: "chunks/000001"},
    97  						{RelPath: "chunks/000002"},
    98  						{RelPath: "chunks/000003"},
    99  						{RelPath: "tombstone"},
   100  					},
   101  				},
   102  			},
   103  			expectedFormat: SegmentsFormat1Based6Digits,
   104  			expectedNum:    3,
   105  		},
   106  		"meta.json with Files, 1 based 6 digits but non consecutive": {
   107  			meta: metadata.Meta{
   108  				Thanos: metadata.Thanos{
   109  					Files: []metadata.File{
   110  						{RelPath: "index"},
   111  						{RelPath: "chunks/000001"},
   112  						{RelPath: "chunks/000003"},
   113  						{RelPath: "chunks/000004"},
   114  						{RelPath: "tombstone"},
   115  					},
   116  				},
   117  			},
   118  			expectedFormat: SegmentsFormatUnknown,
   119  			expectedNum:    0,
   120  		},
   121  	}
   122  
   123  	for testName, testData := range tests {
   124  		t.Run(testName, func(t *testing.T) {
   125  			actualFormat, actualNum := detectBlockSegmentsFormat(testData.meta)
   126  			assert.Equal(t, testData.expectedFormat, actualFormat)
   127  			assert.Equal(t, testData.expectedNum, actualNum)
   128  		})
   129  	}
   130  }
   131  
   132  func TestBlockFromThanosMeta(t *testing.T) {
   133  	blockID := ulid.MustNew(1, nil)
   134  
   135  	tests := map[string]struct {
   136  		meta     metadata.Meta
   137  		expected Block
   138  	}{
   139  		"meta.json without SegmentFiles and Files": {
   140  			meta: metadata.Meta{
   141  				BlockMeta: tsdb.BlockMeta{
   142  					ULID:    blockID,
   143  					MinTime: 10,
   144  					MaxTime: 20,
   145  				},
   146  				Thanos: metadata.Thanos{},
   147  			},
   148  			expected: Block{
   149  				ID:             blockID,
   150  				MinTime:        10,
   151  				MaxTime:        20,
   152  				SegmentsFormat: SegmentsFormatUnknown,
   153  				SegmentsNum:    0,
   154  			},
   155  		},
   156  		"meta.json with SegmentFiles": {
   157  			meta: metadata.Meta{
   158  				BlockMeta: tsdb.BlockMeta{
   159  					ULID:    blockID,
   160  					MinTime: 10,
   161  					MaxTime: 20,
   162  				},
   163  				Thanos: metadata.Thanos{
   164  					SegmentFiles: []string{
   165  						"000001",
   166  						"000002",
   167  						"000003",
   168  					},
   169  				},
   170  			},
   171  			expected: Block{
   172  				ID:             blockID,
   173  				MinTime:        10,
   174  				MaxTime:        20,
   175  				SegmentsFormat: SegmentsFormat1Based6Digits,
   176  				SegmentsNum:    3,
   177  			},
   178  		},
   179  		"meta.json with Files": {
   180  			meta: metadata.Meta{
   181  				BlockMeta: tsdb.BlockMeta{
   182  					ULID:    blockID,
   183  					MinTime: 10,
   184  					MaxTime: 20,
   185  				},
   186  				Thanos: metadata.Thanos{
   187  					Files: []metadata.File{
   188  						{RelPath: "index"},
   189  						{RelPath: "chunks/000001"},
   190  						{RelPath: "chunks/000002"},
   191  						{RelPath: "chunks/000003"},
   192  						{RelPath: "tombstone"},
   193  					},
   194  				},
   195  			},
   196  			expected: Block{
   197  				ID:             blockID,
   198  				MinTime:        10,
   199  				MaxTime:        20,
   200  				SegmentsFormat: SegmentsFormat1Based6Digits,
   201  				SegmentsNum:    3,
   202  			},
   203  		},
   204  	}
   205  
   206  	for testName, testData := range tests {
   207  		t.Run(testName, func(t *testing.T) {
   208  			assert.Equal(t, testData.expected, *BlockFromThanosMeta(testData.meta))
   209  		})
   210  	}
   211  }
   212  
   213  func TestBlock_Within(t *testing.T) {
   214  	tests := []struct {
   215  		block    *Block
   216  		minT     int64
   217  		maxT     int64
   218  		expected bool
   219  	}{
   220  		{
   221  			block:    &Block{MinTime: 10, MaxTime: 20},
   222  			minT:     5,
   223  			maxT:     9,
   224  			expected: false,
   225  		}, {
   226  			block:    &Block{MinTime: 10, MaxTime: 20},
   227  			minT:     5,
   228  			maxT:     10,
   229  			expected: true,
   230  		}, {
   231  			block:    &Block{MinTime: 10, MaxTime: 20},
   232  			minT:     5,
   233  			maxT:     10,
   234  			expected: true,
   235  		}, {
   236  			block:    &Block{MinTime: 10, MaxTime: 20},
   237  			minT:     11,
   238  			maxT:     13,
   239  			expected: true,
   240  		}, {
   241  			block:    &Block{MinTime: 10, MaxTime: 20},
   242  			minT:     19,
   243  			maxT:     21,
   244  			expected: true,
   245  		}, {
   246  			block:    &Block{MinTime: 10, MaxTime: 20},
   247  			minT:     20,
   248  			maxT:     21,
   249  			expected: false,
   250  		},
   251  	}
   252  
   253  	for _, tc := range tests {
   254  		assert.Equal(t, tc.expected, tc.block.Within(tc.minT, tc.maxT))
   255  	}
   256  }
   257  
   258  func TestBlock_ThanosMeta(t *testing.T) {
   259  	blockID := ulid.MustNew(1, nil)
   260  	userID := "user-1"
   261  
   262  	tests := map[string]struct {
   263  		block    Block
   264  		expected *metadata.Meta
   265  	}{
   266  		"block with segment files format 1 based 6 digits": {
   267  			block: Block{
   268  				ID:             blockID,
   269  				MinTime:        10,
   270  				MaxTime:        20,
   271  				SegmentsFormat: SegmentsFormat1Based6Digits,
   272  				SegmentsNum:    3,
   273  			},
   274  			expected: &metadata.Meta{
   275  				BlockMeta: tsdb.BlockMeta{
   276  					ULID:    blockID,
   277  					MinTime: 10,
   278  					MaxTime: 20,
   279  					Version: metadata.TSDBVersion1,
   280  				},
   281  				Thanos: metadata.Thanos{
   282  					Version: metadata.ThanosVersion1,
   283  					Labels: map[string]string{
   284  						"__org_id__": userID,
   285  					},
   286  					SegmentFiles: []string{
   287  						"000001",
   288  						"000002",
   289  						"000003",
   290  					},
   291  				},
   292  			},
   293  		},
   294  		"block with unknown segment files format": {
   295  			block: Block{
   296  				ID:             blockID,
   297  				MinTime:        10,
   298  				MaxTime:        20,
   299  				SegmentsFormat: SegmentsFormatUnknown,
   300  				SegmentsNum:    0,
   301  			},
   302  			expected: &metadata.Meta{
   303  				BlockMeta: tsdb.BlockMeta{
   304  					ULID:    blockID,
   305  					MinTime: 10,
   306  					MaxTime: 20,
   307  					Version: metadata.TSDBVersion1,
   308  				},
   309  				Thanos: metadata.Thanos{
   310  					Version: metadata.ThanosVersion1,
   311  					Labels: map[string]string{
   312  						"__org_id__": userID,
   313  					},
   314  				},
   315  			},
   316  		},
   317  	}
   318  
   319  	for testName, testData := range tests {
   320  		t.Run(testName, func(t *testing.T) {
   321  			assert.Equal(t, testData.expected, testData.block.ThanosMeta(userID))
   322  		})
   323  	}
   324  }
   325  
   326  func TestBlockDeletionMark_ThanosDeletionMark(t *testing.T) {
   327  	block1 := ulid.MustNew(1, nil)
   328  	mark := &BlockDeletionMark{ID: block1, DeletionTime: 1}
   329  
   330  	assert.Equal(t, &metadata.DeletionMark{
   331  		ID:           block1,
   332  		Version:      metadata.DeletionMarkVersion1,
   333  		DeletionTime: 1,
   334  	}, mark.ThanosDeletionMark())
   335  }
   336  
   337  func TestBlockDeletionMarks_Clone(t *testing.T) {
   338  	block1 := ulid.MustNew(1, nil)
   339  	block2 := ulid.MustNew(2, nil)
   340  	orig := BlockDeletionMarks{{ID: block1, DeletionTime: 1}, {ID: block2, DeletionTime: 2}}
   341  
   342  	// The clone must be identical.
   343  	clone := orig.Clone()
   344  	assert.Equal(t, orig, clone)
   345  
   346  	// Changes to the original shouldn't be reflected to the clone.
   347  	orig[0].DeletionTime = -1
   348  	assert.Equal(t, int64(1), clone[0].DeletionTime)
   349  }