github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/memtable_roaring_set_test.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package lsmkv
    13  
    14  import (
    15  	"path"
    16  	"testing"
    17  
    18  	"github.com/stretchr/testify/assert"
    19  	"github.com/stretchr/testify/require"
    20  	"github.com/weaviate/weaviate/adapters/repos/db/roaringset"
    21  )
    22  
    23  func TestMemtableRoaringSet(t *testing.T) {
    24  	memPath := func() string {
    25  		return path.Join(t.TempDir(), "fake")
    26  	}
    27  
    28  	t.Run("inserting individual entries", func(t *testing.T) {
    29  		cl, err := newCommitLogger(memPath())
    30  		require.NoError(t, err)
    31  
    32  		m, err := newMemtable(memPath(), StrategyRoaringSet, 0, cl, nil)
    33  		require.Nil(t, err)
    34  
    35  		key1, key2 := []byte("key1"), []byte("key2")
    36  
    37  		assert.Nil(t, m.roaringSetAddOne(key1, 1))
    38  		assert.Nil(t, m.roaringSetAddOne(key1, 2))
    39  		assert.Nil(t, m.roaringSetAddOne(key2, 3))
    40  		assert.Nil(t, m.roaringSetAddOne(key2, 4))
    41  		assert.Greater(t, m.Size(), uint64(0))
    42  
    43  		setKey1, err := m.roaringSetGet(key1)
    44  		require.Nil(t, err)
    45  		assert.True(t, setKey1.Additions.Contains(1))
    46  		assert.True(t, setKey1.Additions.Contains(2))
    47  		assert.False(t, setKey1.Additions.Contains(3))
    48  		assert.False(t, setKey1.Additions.Contains(4))
    49  
    50  		setKey2, err := m.roaringSetGet(key2)
    51  		require.Nil(t, err)
    52  		assert.False(t, setKey2.Additions.Contains(1))
    53  		assert.False(t, setKey2.Additions.Contains(2))
    54  		assert.True(t, setKey2.Additions.Contains(3))
    55  		assert.True(t, setKey2.Additions.Contains(4))
    56  
    57  		require.Nil(t, m.commitlog.close())
    58  	})
    59  
    60  	t.Run("inserting lists", func(t *testing.T) {
    61  		cl, err := newCommitLogger(memPath())
    62  		require.NoError(t, err)
    63  
    64  		m, err := newMemtable(memPath(), StrategyRoaringSet, 0, cl, nil)
    65  		require.Nil(t, err)
    66  
    67  		key1, key2 := []byte("key1"), []byte("key2")
    68  
    69  		assert.Nil(t, m.roaringSetAddList(key1, []uint64{1, 2}))
    70  		assert.Nil(t, m.roaringSetAddList(key2, []uint64{3, 4}))
    71  		assert.Greater(t, m.Size(), uint64(0))
    72  
    73  		setKey1, err := m.roaringSetGet(key1)
    74  		require.Nil(t, err)
    75  		assert.True(t, setKey1.Additions.Contains(1))
    76  		assert.True(t, setKey1.Additions.Contains(2))
    77  		assert.False(t, setKey1.Additions.Contains(3))
    78  		assert.False(t, setKey1.Additions.Contains(4))
    79  
    80  		setKey2, err := m.roaringSetGet(key2)
    81  		require.Nil(t, err)
    82  		assert.False(t, setKey2.Additions.Contains(1))
    83  		assert.False(t, setKey2.Additions.Contains(2))
    84  		assert.True(t, setKey2.Additions.Contains(3))
    85  		assert.True(t, setKey2.Additions.Contains(4))
    86  
    87  		require.Nil(t, m.commitlog.close())
    88  	})
    89  
    90  	t.Run("inserting bitmaps", func(t *testing.T) {
    91  		cl, err := newCommitLogger(memPath())
    92  		require.NoError(t, err)
    93  
    94  		m, err := newMemtable(memPath(), StrategyRoaringSet, 0, cl, nil)
    95  		require.Nil(t, err)
    96  
    97  		key1, key2 := []byte("key1"), []byte("key2")
    98  
    99  		bm1 := roaringset.NewBitmap(1, 2)
   100  		assert.Nil(t, m.roaringSetAddBitmap(key1, bm1))
   101  		bm2 := roaringset.NewBitmap(3, 4)
   102  		assert.Nil(t, m.roaringSetAddBitmap(key2, bm2))
   103  		assert.Greater(t, m.Size(), uint64(0))
   104  
   105  		setKey1, err := m.roaringSetGet(key1)
   106  		require.Nil(t, err)
   107  		assert.True(t, setKey1.Additions.Contains(1))
   108  		assert.True(t, setKey1.Additions.Contains(2))
   109  		assert.False(t, setKey1.Additions.Contains(3))
   110  		assert.False(t, setKey1.Additions.Contains(4))
   111  
   112  		setKey2, err := m.roaringSetGet(key2)
   113  		require.Nil(t, err)
   114  		assert.False(t, setKey2.Additions.Contains(1))
   115  		assert.False(t, setKey2.Additions.Contains(2))
   116  		assert.True(t, setKey2.Additions.Contains(3))
   117  		assert.True(t, setKey2.Additions.Contains(4))
   118  
   119  		require.Nil(t, m.commitlog.close())
   120  	})
   121  
   122  	t.Run("removing individual entries", func(t *testing.T) {
   123  		cl, err := newCommitLogger(memPath())
   124  		require.NoError(t, err)
   125  
   126  		m, err := newMemtable(memPath(), StrategyRoaringSet, 0, cl, nil)
   127  		require.Nil(t, err)
   128  
   129  		key1, key2 := []byte("key1"), []byte("key2")
   130  
   131  		assert.Nil(t, m.roaringSetRemoveOne(key1, 7))
   132  		assert.Nil(t, m.roaringSetRemoveOne(key2, 8))
   133  		assert.Greater(t, m.Size(), uint64(0))
   134  
   135  		setKey1, err := m.roaringSetGet(key1)
   136  		require.Nil(t, err)
   137  		assert.False(t, setKey1.Additions.Contains(7))
   138  		assert.True(t, setKey1.Deletions.Contains(7))
   139  
   140  		setKey2, err := m.roaringSetGet(key2)
   141  		require.Nil(t, err)
   142  		assert.False(t, setKey2.Additions.Contains(8))
   143  		assert.True(t, setKey2.Deletions.Contains(8))
   144  
   145  		require.Nil(t, m.commitlog.close())
   146  	})
   147  
   148  	t.Run("removing lists", func(t *testing.T) {
   149  		cl, err := newCommitLogger(memPath())
   150  		require.NoError(t, err)
   151  
   152  		m, err := newMemtable(memPath(), StrategyRoaringSet, 0, cl, nil)
   153  		require.Nil(t, err)
   154  
   155  		key1, key2 := []byte("key1"), []byte("key2")
   156  
   157  		assert.Nil(t, m.roaringSetRemoveList(key1, []uint64{7, 8}))
   158  		assert.Nil(t, m.roaringSetRemoveList(key2, []uint64{9, 10}))
   159  		assert.Greater(t, m.Size(), uint64(0))
   160  
   161  		setKey1, err := m.roaringSetGet(key1)
   162  		require.Nil(t, err)
   163  		assert.Equal(t, 0, setKey1.Additions.GetCardinality())
   164  		assert.Equal(t, 2, setKey1.Deletions.GetCardinality())
   165  		assert.True(t, setKey1.Deletions.Contains(7))
   166  		assert.True(t, setKey1.Deletions.Contains(8))
   167  
   168  		setKey2, err := m.roaringSetGet(key2)
   169  		require.Nil(t, err)
   170  		assert.Equal(t, 0, setKey2.Additions.GetCardinality())
   171  		assert.Equal(t, 2, setKey2.Deletions.GetCardinality())
   172  		assert.True(t, setKey2.Deletions.Contains(9))
   173  		assert.True(t, setKey2.Deletions.Contains(10))
   174  
   175  		require.Nil(t, m.commitlog.close())
   176  	})
   177  
   178  	t.Run("removing bitmaps", func(t *testing.T) {
   179  		cl, err := newCommitLogger(memPath())
   180  		require.NoError(t, err)
   181  
   182  		m, err := newMemtable(memPath(), StrategyRoaringSet, 0, cl, nil)
   183  		require.Nil(t, err)
   184  
   185  		key1, key2 := []byte("key1"), []byte("key2")
   186  
   187  		assert.Nil(t, m.roaringSetRemoveBitmap(key1, roaringset.NewBitmap(7, 8)))
   188  		assert.Nil(t, m.roaringSetRemoveBitmap(key2, roaringset.NewBitmap(9, 10)))
   189  		assert.Greater(t, m.Size(), uint64(0))
   190  
   191  		setKey1, err := m.roaringSetGet(key1)
   192  		require.Nil(t, err)
   193  		assert.Equal(t, 0, setKey1.Additions.GetCardinality())
   194  		assert.Equal(t, 2, setKey1.Deletions.GetCardinality())
   195  		assert.True(t, setKey1.Deletions.Contains(7))
   196  		assert.True(t, setKey1.Deletions.Contains(8))
   197  
   198  		setKey2, err := m.roaringSetGet(key2)
   199  		require.Nil(t, err)
   200  		assert.Equal(t, 0, setKey2.Additions.GetCardinality())
   201  		assert.Equal(t, 2, setKey2.Deletions.GetCardinality())
   202  		assert.True(t, setKey2.Deletions.Contains(9))
   203  		assert.True(t, setKey2.Deletions.Contains(10))
   204  
   205  		require.Nil(t, m.commitlog.close())
   206  	})
   207  
   208  	t.Run("adding/removing bitmaps", func(t *testing.T) {
   209  		cl, err := newCommitLogger(memPath())
   210  		require.NoError(t, err)
   211  
   212  		m, err := newMemtable(memPath(), StrategyRoaringSet, 0, cl, nil)
   213  		require.Nil(t, err)
   214  
   215  		key1, key2 := []byte("key1"), []byte("key2")
   216  
   217  		assert.Nil(t, m.roaringSetAddRemoveBitmaps(key1,
   218  			roaringset.NewBitmap(1, 2), roaringset.NewBitmap(7, 8)))
   219  		assert.Nil(t, m.roaringSetAddRemoveBitmaps(key2,
   220  			roaringset.NewBitmap(3, 4), roaringset.NewBitmap(9, 10)))
   221  		assert.Greater(t, m.Size(), uint64(0))
   222  
   223  		setKey1, err := m.roaringSetGet(key1)
   224  		require.Nil(t, err)
   225  		assert.Equal(t, 2, setKey1.Additions.GetCardinality())
   226  		assert.True(t, setKey1.Additions.Contains(1))
   227  		assert.True(t, setKey1.Additions.Contains(2))
   228  		assert.Equal(t, 2, setKey1.Deletions.GetCardinality())
   229  		assert.True(t, setKey1.Deletions.Contains(7))
   230  		assert.True(t, setKey1.Deletions.Contains(8))
   231  
   232  		setKey2, err := m.roaringSetGet(key2)
   233  		require.Nil(t, err)
   234  		assert.Equal(t, 2, setKey2.Additions.GetCardinality())
   235  		assert.True(t, setKey2.Additions.Contains(3))
   236  		assert.True(t, setKey2.Additions.Contains(4))
   237  		assert.Equal(t, 2, setKey2.Deletions.GetCardinality())
   238  		assert.True(t, setKey2.Deletions.Contains(9))
   239  		assert.True(t, setKey2.Deletions.Contains(10))
   240  
   241  		require.Nil(t, m.commitlog.close())
   242  	})
   243  }