github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/query/metadata/metadata_test.go (about)

     1  /*
     2  Copyright 2023.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package metadata
    18  
    19  import (
    20  	"os"
    21  	"sort"
    22  	"strconv"
    23  	"sync"
    24  	"testing"
    25  
    26  	localstorage "github.com/siglens/siglens/pkg/blob/local"
    27  	"github.com/siglens/siglens/pkg/segment/structs"
    28  	log "github.com/sirupsen/logrus"
    29  	"github.com/stretchr/testify/assert"
    30  )
    31  
    32  func Test_AddSementInfo(t *testing.T) {
    33  	_ = localstorage.InitLocalStorage()
    34  	globalMetadata = &allSegmentMetadata{
    35  		allSegmentMicroIndex:        make([]*SegmentMicroIndex, 0),
    36  		segmentMetadataReverseIndex: make(map[string]*SegmentMicroIndex),
    37  		tableSortedMetadata:         make(map[string][]*SegmentMicroIndex),
    38  		updateLock:                  &sync.RWMutex{},
    39  	}
    40  	for i := uint64(0); i < 10; i++ {
    41  		currInfo := &SegmentMicroIndex{
    42  			SegMeta: structs.SegMeta{
    43  				LatestEpochMS:    i,
    44  				VirtualTableName: "test-1",
    45  				SegmentKey:       strconv.FormatUint(i, 10),
    46  			},
    47  		}
    48  		BulkAddSegmentMicroIndex([]*SegmentMicroIndex{currInfo})
    49  	}
    50  
    51  	for i := int64(0); i < 10; i++ {
    52  		assert.Equal(t, uint64(i), globalMetadata.segmentMetadataReverseIndex[strconv.FormatInt(i, 10)].LatestEpochMS)
    53  	}
    54  	nextVal := uint64(9)
    55  	for i := 0; i < 10; i++ {
    56  		assert.Equal(t, globalMetadata.allSegmentMicroIndex[i].LatestEpochMS, nextVal)
    57  		log.Infof("i %+v latest: %+v", i, globalMetadata.allSegmentMicroIndex[i].LatestEpochMS)
    58  		nextVal--
    59  	}
    60  	assert.Contains(t, globalMetadata.tableSortedMetadata, "test-1")
    61  	tableSorted := globalMetadata.tableSortedMetadata["test-1"]
    62  	nextVal = uint64(9)
    63  	for i := 0; i < 10; i++ {
    64  		assert.Equal(t, tableSorted[i].LatestEpochMS, nextVal)
    65  		log.Infof("i %+v latest: %+v", i, tableSorted[i].LatestEpochMS)
    66  		nextVal--
    67  	}
    68  	isTableSorted := sort.SliceIsSorted(tableSorted, func(i, j int) bool {
    69  		return tableSorted[i].LatestEpochMS > tableSorted[j].LatestEpochMS
    70  	})
    71  	assert.True(t, isTableSorted)
    72  
    73  	isAllDataSorted := sort.SliceIsSorted(globalMetadata.allSegmentMicroIndex, func(i, j int) bool {
    74  		return globalMetadata.allSegmentMicroIndex[i].LatestEpochMS > globalMetadata.allSegmentMicroIndex[j].LatestEpochMS
    75  	})
    76  	assert.True(t, isAllDataSorted)
    77  }
    78  
    79  func testBlockRebalance(t *testing.T, fileCount int) {
    80  	blockSizeToLoad := globalMetadata.allSegmentMicroIndex[0].MicroIndexSize + globalMetadata.allSegmentMicroIndex[0].SearchMetadataSize
    81  	RebalanceInMemoryCmi(blockSizeToLoad)
    82  
    83  	assert.True(t, globalMetadata.allSegmentMicroIndex[0].loadedMicroIndices)
    84  	loadedBlockFile := globalMetadata.allSegmentMicroIndex[0].SegmentKey
    85  	loadedBlockLatestTime := globalMetadata.allSegmentMicroIndex[0].LatestEpochMS
    86  	assert.True(t, globalMetadata.segmentMetadataReverseIndex[loadedBlockFile].loadedMicroIndices)
    87  	assert.Same(t, globalMetadata.allSegmentMicroIndex[0], globalMetadata.segmentMetadataReverseIndex[loadedBlockFile])
    88  
    89  	for i := 1; i < fileCount; i++ {
    90  		assert.False(t, globalMetadata.allSegmentMicroIndex[i].loadedMicroIndices)
    91  		currFile := globalMetadata.allSegmentMicroIndex[i].SegmentKey
    92  		assert.Same(t, globalMetadata.allSegmentMicroIndex[i], globalMetadata.segmentMetadataReverseIndex[currFile])
    93  		assert.LessOrEqual(t, globalMetadata.allSegmentMicroIndex[i].LatestEpochMS, loadedBlockLatestTime, "we loaded the lastest timestamp")
    94  	}
    95  
    96  	// load just one more search metadata
    97  	blockSizeToLoad += globalMetadata.allSegmentMicroIndex[1].SearchMetadataSize
    98  	RebalanceInMemoryCmi(blockSizeToLoad)
    99  	assert.True(t, globalMetadata.allSegmentMicroIndex[0].loadedMicroIndices)
   100  	assert.False(t, globalMetadata.allSegmentMicroIndex[1].loadedMicroIndices, "only load search metadata for idx 1")
   101  	for i := 2; i < fileCount; i++ {
   102  		assert.False(t, globalMetadata.allSegmentMicroIndex[i].loadedMicroIndices)
   103  		assert.False(t, globalMetadata.allSegmentMicroIndex[i].loadedSearchMetadata)
   104  		currFile := globalMetadata.allSegmentMicroIndex[i].SegmentKey
   105  		assert.Same(t, globalMetadata.allSegmentMicroIndex[i], globalMetadata.segmentMetadataReverseIndex[currFile])
   106  		assert.LessOrEqual(t, globalMetadata.allSegmentMicroIndex[i].LatestEpochMS, loadedBlockLatestTime, "we loaded the lastest timestamp")
   107  	}
   108  }
   109  
   110  func Test_RebalanceMetadata(t *testing.T) {
   111  	_ = localstorage.InitLocalStorage()
   112  	globalMetadata = &allSegmentMetadata{
   113  		allSegmentMicroIndex:        make([]*SegmentMicroIndex, 0),
   114  		segmentMetadataReverseIndex: make(map[string]*SegmentMicroIndex),
   115  		tableSortedMetadata:         make(map[string][]*SegmentMicroIndex),
   116  		updateLock:                  &sync.RWMutex{},
   117  	}
   118  
   119  	fileCount := 3
   120  	InitMockColumnarMetadataStore("data/", fileCount, 10, 10)
   121  	assert.Len(t, globalMetadata.allSegmentMicroIndex, fileCount)
   122  	assert.Len(t, globalMetadata.segmentMetadataReverseIndex, fileCount)
   123  
   124  	RebalanceInMemoryCmi(0)
   125  	for i := 0; i < fileCount; i++ {
   126  		assert.False(t, globalMetadata.allSegmentMicroIndex[i].loadedMicroIndices)
   127  	}
   128  
   129  	testBlockRebalance(t, fileCount)
   130  
   131  	err := os.RemoveAll("data/")
   132  	if err != nil {
   133  		assert.FailNow(t, "failed to remove data %+v", err)
   134  	}
   135  }