github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/vector/hnsw/periodic_tombstone_removal_test.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package hnsw
    13  
    14  import (
    15  	"context"
    16  	"testing"
    17  	"time"
    18  
    19  	"github.com/sirupsen/logrus/hooks/test"
    20  	"github.com/stretchr/testify/assert"
    21  	"github.com/stretchr/testify/require"
    22  	"github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer"
    23  	"github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers"
    24  	"github.com/weaviate/weaviate/entities/cyclemanager"
    25  	ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw"
    26  	testhelper "github.com/weaviate/weaviate/test/helper"
    27  )
    28  
    29  func TestPeriodicTombstoneRemoval(t *testing.T) {
    30  	logger, _ := test.NewNullLogger()
    31  	cleanupIntervalSeconds := 1
    32  	tombstoneCallbacks := cyclemanager.NewCallbackGroup("tombstone", logger, 1)
    33  	tombstoneCleanupCycle := cyclemanager.NewManager(
    34  		cyclemanager.NewFixedTicker(time.Duration(cleanupIntervalSeconds)*time.Second),
    35  		tombstoneCallbacks.CycleCallback, logger)
    36  	tombstoneCleanupCycle.Start()
    37  
    38  	index, err := New(Config{
    39  		RootPath:              "doesnt-matter-as-committlogger-is-mocked-out",
    40  		ID:                    "automatic-tombstone-removal",
    41  		MakeCommitLoggerThunk: MakeNoopCommitLogger,
    42  		DistanceProvider:      distancer.NewCosineDistanceProvider(),
    43  		VectorForIDThunk:      testVectorForID,
    44  	}, ent.UserConfig{
    45  		CleanupIntervalSeconds: cleanupIntervalSeconds,
    46  		MaxConnections:         30,
    47  		EFConstruction:         128,
    48  	}, tombstoneCallbacks, cyclemanager.NewCallbackGroupNoop(),
    49  		cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t))
    50  	index.PostStartup()
    51  
    52  	require.Nil(t, err)
    53  
    54  	for i, vec := range testVectors {
    55  		err := index.Add(uint64(i), vec)
    56  		require.Nil(t, err)
    57  	}
    58  
    59  	t.Run("delete an entry and verify there is a tombstone", func(t *testing.T) {
    60  		for i := range testVectors {
    61  			if i%2 != 0 {
    62  				continue
    63  			}
    64  
    65  			err := index.Delete(uint64(i))
    66  			require.Nil(t, err)
    67  		}
    68  	})
    69  
    70  	t.Run("verify there are now tombstones", func(t *testing.T) {
    71  		index.tombstoneLock.RLock()
    72  		ts := len(index.tombstones)
    73  		index.tombstoneLock.RUnlock()
    74  		assert.True(t, ts > 0)
    75  	})
    76  
    77  	t.Run("wait for tombstones to disappear", func(t *testing.T) {
    78  		testhelper.AssertEventuallyEqual(t, true, func() interface{} {
    79  			index.tombstoneLock.RLock()
    80  			ts := len(index.tombstones)
    81  			index.tombstoneLock.RUnlock()
    82  			return ts == 0
    83  		}, "wait until tombstones have been cleaned up")
    84  	})
    85  
    86  	if err := index.Shutdown(context.Background()); err != nil {
    87  		t.Fatal(err)
    88  	}
    89  	if err := tombstoneCleanupCycle.StopAndWait(context.Background()); err != nil {
    90  		t.Fatal(err)
    91  	}
    92  }