github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/vector/hnsw/vector_cache_prefiller_test.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package hnsw
    13  
    14  import (
    15  	"context"
    16  	"testing"
    17  
    18  	"github.com/sirupsen/logrus/hooks/test"
    19  	"github.com/stretchr/testify/assert"
    20  	"github.com/weaviate/weaviate/adapters/repos/db/vector/common"
    21  )
    22  
    23  func TestVectorCachePrefilling(t *testing.T) {
    24  	cache := newFakeCache()
    25  	index := &hnsw{
    26  		nodes:               generateDummyVertices(100),
    27  		currentMaximumLayer: 3,
    28  		shardedNodeLocks:    common.NewDefaultShardedRWLocks(),
    29  	}
    30  
    31  	logger, _ := test.NewNullLogger()
    32  
    33  	pf := newVectorCachePrefiller[float32](cache, index, logger)
    34  
    35  	t.Run("prefill with limit >= graph size", func(t *testing.T) {
    36  		cache.Reset()
    37  		pf.Prefill(context.Background(), 100)
    38  		assert.Equal(t, allNumbersUpTo(100), cache.store)
    39  	})
    40  
    41  	t.Run("prefill with small limit so only the upper layer fits", func(t *testing.T) {
    42  		cache.Reset()
    43  		pf.Prefill(context.Background(), 7)
    44  		assert.Equal(t, map[uint64]struct{}{
    45  			0:  {},
    46  			15: {},
    47  			30: {},
    48  			45: {},
    49  			60: {},
    50  			75: {},
    51  			90: {},
    52  		}, cache.store)
    53  	})
    54  
    55  	t.Run("limit where a layer partially fits", func(t *testing.T) {
    56  		cache.Reset()
    57  		pf.Prefill(context.Background(), 10)
    58  		assert.Equal(t, map[uint64]struct{}{
    59  			// layer 3
    60  			0:  {},
    61  			15: {},
    62  			30: {},
    63  			45: {},
    64  			60: {},
    65  			75: {},
    66  			90: {},
    67  
    68  			// additional layer 2
    69  			5:  {},
    70  			10: {},
    71  			20: {},
    72  		}, cache.store)
    73  	})
    74  }
    75  
    76  func newFakeCache() *fakeCache {
    77  	return &fakeCache{
    78  		store: map[uint64]struct{}{},
    79  	}
    80  }
    81  
    82  type fakeCache struct {
    83  	store map[uint64]struct{}
    84  }
    85  
    86  func (f *fakeCache) MultiGet(ctx context.Context, id []uint64) ([][]float32, []error) {
    87  	panic("not implemented")
    88  }
    89  
    90  func (f *fakeCache) Get(ctx context.Context, id uint64) ([]float32, error) {
    91  	f.store[id] = struct{}{}
    92  	return nil, nil
    93  }
    94  
    95  func (f *fakeCache) Delete(ctx context.Context, id uint64) {
    96  	panic("not implemented")
    97  }
    98  
    99  func (f *fakeCache) Preload(id uint64, vec []float32) {
   100  	panic("not implemented")
   101  }
   102  
   103  func (f *fakeCache) Prefetch(id uint64) {
   104  	panic("not implemented")
   105  }
   106  
   107  func (f *fakeCache) Grow(id uint64) {
   108  	panic("not implemented")
   109  }
   110  
   111  func (f *fakeCache) UpdateMaxSize(size int64) {
   112  	panic("not implemented")
   113  }
   114  
   115  func (f *fakeCache) All() [][]float32 {
   116  	panic("not implemented")
   117  }
   118  
   119  func (f *fakeCache) Drop() {
   120  	panic("not implemented")
   121  }
   122  
   123  func (f *fakeCache) CopyMaxSize() int64 {
   124  	return 1e6
   125  }
   126  
   127  func (f *fakeCache) Reset() {
   128  	f.store = map[uint64]struct{}{}
   129  }
   130  
   131  func (f *fakeCache) Len() int32 {
   132  	return int32(len(f.store))
   133  }
   134  
   135  func (f *fakeCache) CountVectors() int64 {
   136  	panic("not implemented")
   137  }
   138  
   139  func generateDummyVertices(amount int) []*vertex {
   140  	out := make([]*vertex, amount)
   141  	for i := range out {
   142  		out[i] = &vertex{
   143  			id:    uint64(i),
   144  			level: levelForDummyVertex(i),
   145  		}
   146  	}
   147  
   148  	return out
   149  }
   150  
   151  // maximum of 3 layers
   152  // if id % 15 == 0 -> layer 3
   153  // if id % 5 == 0 -> layer 2
   154  // if id % 3 == 0 -> layer 1
   155  // remainder -> layer 0
   156  func levelForDummyVertex(id int) int {
   157  	if id%15 == 0 {
   158  		return 3
   159  	}
   160  
   161  	if id%5 == 0 {
   162  		return 2
   163  	}
   164  
   165  	if id%3 == 0 {
   166  		return 1
   167  	}
   168  
   169  	return 0
   170  }
   171  
   172  func allNumbersUpTo(size int) map[uint64]struct{} {
   173  	out := map[uint64]struct{}{}
   174  	for i := 0; i < size; i++ {
   175  		out[uint64(i)] = struct{}{}
   176  	}
   177  
   178  	return out
   179  }