github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/storage/tsdb/bucketindex/updater_test.go (about)

     1  package bucketindex
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"path"
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/go-kit/log"
    11  	"github.com/oklog/ulid"
    12  	"github.com/pkg/errors"
    13  	"github.com/prometheus/prometheus/tsdb"
    14  	"github.com/stretchr/testify/assert"
    15  	"github.com/stretchr/testify/require"
    16  	"github.com/thanos-io/thanos/pkg/block"
    17  	"github.com/thanos-io/thanos/pkg/block/metadata"
    18  	"github.com/thanos-io/thanos/pkg/objstore"
    19  
    20  	"github.com/cortexproject/cortex/pkg/storage/bucket"
    21  	"github.com/cortexproject/cortex/pkg/storage/tsdb/testutil"
    22  )
    23  
    24  func TestUpdater_UpdateIndex(t *testing.T) {
    25  	const userID = "user-1"
    26  
    27  	bkt, _ := testutil.PrepareFilesystemBucket(t)
    28  
    29  	ctx := context.Background()
    30  	logger := log.NewNopLogger()
    31  
    32  	// Generate the initial index.
    33  	bkt = BucketWithGlobalMarkers(bkt)
    34  	block1 := testutil.MockStorageBlock(t, bkt, userID, 10, 20)
    35  	block2 := testutil.MockStorageBlock(t, bkt, userID, 20, 30)
    36  	block2Mark := testutil.MockStorageDeletionMark(t, bkt, userID, block2)
    37  
    38  	w := NewUpdater(bkt, userID, nil, logger)
    39  	returnedIdx, _, err := w.UpdateIndex(ctx, nil)
    40  	require.NoError(t, err)
    41  	assertBucketIndexEqual(t, returnedIdx, bkt, userID,
    42  		[]tsdb.BlockMeta{block1, block2},
    43  		[]*metadata.DeletionMark{block2Mark})
    44  
    45  	// Create new blocks, and update the index.
    46  	block3 := testutil.MockStorageBlock(t, bkt, userID, 30, 40)
    47  	block4 := testutil.MockStorageBlock(t, bkt, userID, 40, 50)
    48  	block4Mark := testutil.MockStorageDeletionMark(t, bkt, userID, block4)
    49  
    50  	returnedIdx, _, err = w.UpdateIndex(ctx, returnedIdx)
    51  	require.NoError(t, err)
    52  	assertBucketIndexEqual(t, returnedIdx, bkt, userID,
    53  		[]tsdb.BlockMeta{block1, block2, block3, block4},
    54  		[]*metadata.DeletionMark{block2Mark, block4Mark})
    55  
    56  	// Hard delete a block and update the index.
    57  	require.NoError(t, block.Delete(ctx, log.NewNopLogger(), bucket.NewUserBucketClient(userID, bkt, nil), block2.ULID))
    58  
    59  	returnedIdx, _, err = w.UpdateIndex(ctx, returnedIdx)
    60  	require.NoError(t, err)
    61  	assertBucketIndexEqual(t, returnedIdx, bkt, userID,
    62  		[]tsdb.BlockMeta{block1, block3, block4},
    63  		[]*metadata.DeletionMark{block4Mark})
    64  }
    65  
    66  func TestUpdater_UpdateIndex_ShouldSkipPartialBlocks(t *testing.T) {
    67  	const userID = "user-1"
    68  
    69  	bkt, _ := testutil.PrepareFilesystemBucket(t)
    70  
    71  	ctx := context.Background()
    72  	logger := log.NewNopLogger()
    73  
    74  	// Mock some blocks in the storage.
    75  	bkt = BucketWithGlobalMarkers(bkt)
    76  	block1 := testutil.MockStorageBlock(t, bkt, userID, 10, 20)
    77  	block2 := testutil.MockStorageBlock(t, bkt, userID, 20, 30)
    78  	block3 := testutil.MockStorageBlock(t, bkt, userID, 30, 40)
    79  	block2Mark := testutil.MockStorageDeletionMark(t, bkt, userID, block2)
    80  
    81  	// Delete a block's meta.json to simulate a partial block.
    82  	require.NoError(t, bkt.Delete(ctx, path.Join(userID, block3.ULID.String(), metadata.MetaFilename)))
    83  
    84  	w := NewUpdater(bkt, userID, nil, logger)
    85  	idx, partials, err := w.UpdateIndex(ctx, nil)
    86  	require.NoError(t, err)
    87  	assertBucketIndexEqual(t, idx, bkt, userID,
    88  		[]tsdb.BlockMeta{block1, block2},
    89  		[]*metadata.DeletionMark{block2Mark})
    90  
    91  	assert.Len(t, partials, 1)
    92  	assert.True(t, errors.Is(partials[block3.ULID], ErrBlockMetaNotFound))
    93  }
    94  
    95  func TestUpdater_UpdateIndex_ShouldSkipBlocksWithCorruptedMeta(t *testing.T) {
    96  	const userID = "user-1"
    97  
    98  	bkt, _ := testutil.PrepareFilesystemBucket(t)
    99  
   100  	ctx := context.Background()
   101  	logger := log.NewNopLogger()
   102  
   103  	// Mock some blocks in the storage.
   104  	bkt = BucketWithGlobalMarkers(bkt)
   105  	block1 := testutil.MockStorageBlock(t, bkt, userID, 10, 20)
   106  	block2 := testutil.MockStorageBlock(t, bkt, userID, 20, 30)
   107  	block3 := testutil.MockStorageBlock(t, bkt, userID, 30, 40)
   108  	block2Mark := testutil.MockStorageDeletionMark(t, bkt, userID, block2)
   109  
   110  	// Overwrite a block's meta.json with invalid data.
   111  	require.NoError(t, bkt.Upload(ctx, path.Join(userID, block3.ULID.String(), metadata.MetaFilename), bytes.NewReader([]byte("invalid!}"))))
   112  
   113  	w := NewUpdater(bkt, userID, nil, logger)
   114  	idx, partials, err := w.UpdateIndex(ctx, nil)
   115  	require.NoError(t, err)
   116  	assertBucketIndexEqual(t, idx, bkt, userID,
   117  		[]tsdb.BlockMeta{block1, block2},
   118  		[]*metadata.DeletionMark{block2Mark})
   119  
   120  	assert.Len(t, partials, 1)
   121  	assert.True(t, errors.Is(partials[block3.ULID], ErrBlockMetaCorrupted))
   122  }
   123  
   124  func TestUpdater_UpdateIndex_ShouldSkipCorruptedDeletionMarks(t *testing.T) {
   125  	const userID = "user-1"
   126  
   127  	bkt, _ := testutil.PrepareFilesystemBucket(t)
   128  
   129  	ctx := context.Background()
   130  	logger := log.NewNopLogger()
   131  
   132  	// Mock some blocks in the storage.
   133  	bkt = BucketWithGlobalMarkers(bkt)
   134  	block1 := testutil.MockStorageBlock(t, bkt, userID, 10, 20)
   135  	block2 := testutil.MockStorageBlock(t, bkt, userID, 20, 30)
   136  	block3 := testutil.MockStorageBlock(t, bkt, userID, 30, 40)
   137  	block2Mark := testutil.MockStorageDeletionMark(t, bkt, userID, block2)
   138  
   139  	// Overwrite a block's deletion-mark.json with invalid data.
   140  	require.NoError(t, bkt.Upload(ctx, path.Join(userID, block2Mark.ID.String(), metadata.DeletionMarkFilename), bytes.NewReader([]byte("invalid!}"))))
   141  
   142  	w := NewUpdater(bkt, userID, nil, logger)
   143  	idx, partials, err := w.UpdateIndex(ctx, nil)
   144  	require.NoError(t, err)
   145  	assertBucketIndexEqual(t, idx, bkt, userID,
   146  		[]tsdb.BlockMeta{block1, block2, block3},
   147  		[]*metadata.DeletionMark{})
   148  	assert.Empty(t, partials)
   149  }
   150  
   151  func TestUpdater_UpdateIndex_NoTenantInTheBucket(t *testing.T) {
   152  	const userID = "user-1"
   153  
   154  	ctx := context.Background()
   155  	bkt, _ := testutil.PrepareFilesystemBucket(t)
   156  
   157  	for _, oldIdx := range []*Index{nil, {}} {
   158  		w := NewUpdater(bkt, userID, nil, log.NewNopLogger())
   159  		idx, partials, err := w.UpdateIndex(ctx, oldIdx)
   160  
   161  		require.NoError(t, err)
   162  		assert.Equal(t, IndexVersion1, idx.Version)
   163  		assert.InDelta(t, time.Now().Unix(), idx.UpdatedAt, 2)
   164  		assert.Len(t, idx.Blocks, 0)
   165  		assert.Len(t, idx.BlockDeletionMarks, 0)
   166  		assert.Empty(t, partials)
   167  	}
   168  }
   169  
   170  func getBlockUploadedAt(t testing.TB, bkt objstore.Bucket, userID string, blockID ulid.ULID) int64 {
   171  	metaFile := path.Join(userID, blockID.String(), block.MetaFilename)
   172  
   173  	attrs, err := bkt.Attributes(context.Background(), metaFile)
   174  	require.NoError(t, err)
   175  
   176  	return attrs.LastModified.Unix()
   177  }
   178  
   179  func assertBucketIndexEqual(t testing.TB, idx *Index, bkt objstore.Bucket, userID string, expectedBlocks []tsdb.BlockMeta, expectedDeletionMarks []*metadata.DeletionMark) {
   180  	assert.Equal(t, IndexVersion1, idx.Version)
   181  	assert.InDelta(t, time.Now().Unix(), idx.UpdatedAt, 2)
   182  
   183  	// Build the list of expected block index entries.
   184  	var expectedBlockEntries []*Block
   185  	for _, b := range expectedBlocks {
   186  		expectedBlockEntries = append(expectedBlockEntries, &Block{
   187  			ID:         b.ULID,
   188  			MinTime:    b.MinTime,
   189  			MaxTime:    b.MaxTime,
   190  			UploadedAt: getBlockUploadedAt(t, bkt, userID, b.ULID),
   191  		})
   192  	}
   193  
   194  	assert.ElementsMatch(t, expectedBlockEntries, idx.Blocks)
   195  
   196  	// Build the list of expected block deletion mark index entries.
   197  	var expectedMarkEntries []*BlockDeletionMark
   198  	for _, m := range expectedDeletionMarks {
   199  		expectedMarkEntries = append(expectedMarkEntries, &BlockDeletionMark{
   200  			ID:           m.ID,
   201  			DeletionTime: m.DeletionTime,
   202  		})
   203  	}
   204  
   205  	assert.ElementsMatch(t, expectedMarkEntries, idx.BlockDeletionMarks)
   206  }