github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/concurrent_reading_benchmark_test.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package lsmkv
    13  
    14  import (
    15  	"context"
    16  	"crypto/rand"
    17  	"fmt"
    18  	"os"
    19  	"sync"
    20  	"testing"
    21  	"time"
    22  
    23  	"github.com/google/uuid"
    24  	"github.com/sirupsen/logrus"
    25  	"github.com/sirupsen/logrus/hooks/test"
    26  	"github.com/stretchr/testify/assert"
    27  	"github.com/stretchr/testify/require"
    28  	"github.com/weaviate/weaviate/entities/cyclemanager"
    29  )
    30  
    31  func BenchmarkConcurrentReading(b *testing.B) {
    32  	bucket, cleanup := prepareBucket(b)
    33  	defer cleanup()
    34  	keys := populateBucket(b, bucket)
    35  
    36  	b.ReportAllocs()
    37  	b.ResetTimer()
    38  
    39  	routines := 500
    40  
    41  	for i := 0; i < b.N; i++ {
    42  		wg := sync.WaitGroup{}
    43  		for r := 0; r < routines; r++ {
    44  			wg.Add(1)
    45  			go func() {
    46  				defer wg.Done()
    47  				for _, key := range keys {
    48  					_, err := bucket.MapList(key)
    49  					assert.Nil(b, err)
    50  				}
    51  			}()
    52  		}
    53  		wg.Wait()
    54  	}
    55  }
    56  
    57  func prepareBucket(b *testing.B) (bucket *Bucket, cleanup func()) {
    58  	dirName := fmt.Sprintf("./testdata/%d", mustRandIntn(10000000))
    59  	os.MkdirAll(dirName, 0o777)
    60  	defer func() {
    61  		err := os.RemoveAll(dirName)
    62  		fmt.Println(err)
    63  	}()
    64  
    65  	bucket, err := NewBucket(testCtxB(), dirName, "", nullLoggerB(), nil,
    66  		cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(),
    67  		WithStrategy(StrategyMapCollection),
    68  		WithMemtableThreshold(5000))
    69  	require.Nil(b, err)
    70  
    71  	return bucket, func() {
    72  		err := os.RemoveAll(dirName)
    73  		fmt.Println(err)
    74  	}
    75  }
    76  
    77  func populateBucket(b *testing.B, bucket *Bucket) (keys [][]byte) {
    78  	amount := 2000
    79  	valuesPerKey := 4
    80  	sizePerKey := 8
    81  	sizePerValue := 32
    82  
    83  	keys = make([][]byte, amount)
    84  	values := make([][]MapPair, amount)
    85  
    86  	for i := range keys {
    87  		uuid, err := uuid.New().MarshalBinary()
    88  		require.Nil(b, err)
    89  		keys[i] = uuid
    90  
    91  		values[i] = make([]MapPair, valuesPerKey)
    92  		for j := range values[i] {
    93  			values[i][j] = MapPair{
    94  				Key:   make([]byte, sizePerKey),
    95  				Value: make([]byte, sizePerValue),
    96  			}
    97  			rand.Read(values[i][j].Key)
    98  			rand.Read(values[i][j].Value)
    99  		}
   100  	}
   101  
   102  	wg := sync.WaitGroup{}
   103  	for i := range keys {
   104  		for j := 0; j < valuesPerKey; j++ {
   105  			time.Sleep(50 * time.Microsecond)
   106  			wg.Add(1)
   107  			go func(rowIndex, valueIndex int) {
   108  				defer wg.Done()
   109  				err := bucket.MapSet(keys[rowIndex], values[rowIndex][valueIndex])
   110  				assert.Nil(b, err)
   111  			}(i, j)
   112  		}
   113  	}
   114  	wg.Wait()
   115  
   116  	return
   117  }
   118  
   119  func testCtxB() context.Context {
   120  	return context.Background()
   121  }
   122  
   123  func nullLoggerB() logrus.FieldLogger {
   124  	log, _ := test.NewNullLogger()
   125  	return log
   126  }