github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/strategies_replace_integration_test.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  //go:build integrationTest
    13  // +build integrationTest
    14  
    15  package lsmkv
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"testing"
    21  
    22  	"github.com/stretchr/testify/assert"
    23  	"github.com/stretchr/testify/require"
    24  	"github.com/weaviate/weaviate/entities/cyclemanager"
    25  )
    26  
    27  func TestReplaceStrategy(t *testing.T) {
    28  	ctx := testCtx()
    29  	tests := bucketIntegrationTests{
    30  		{
    31  			name: "replaceInsertAndUpdate",
    32  			f:    replaceInsertAndUpdate,
    33  			opts: []BucketOption{
    34  				WithStrategy(StrategyReplace),
    35  			},
    36  		},
    37  		{
    38  			name: "replaceInsertAndUpdate_WithSecondaryKeys",
    39  			f:    replaceInsertAndUpdate_WithSecondaryKeys,
    40  			opts: []BucketOption{
    41  				WithStrategy(StrategyReplace),
    42  				WithSecondaryIndices(1),
    43  			},
    44  		},
    45  		{
    46  			name: "replaceInsertAndDelete",
    47  			f:    replaceInsertAndDelete,
    48  			opts: []BucketOption{
    49  				WithStrategy(StrategyReplace),
    50  			},
    51  		},
    52  		{
    53  			name: "replaceCursors",
    54  			f:    replaceCursors,
    55  			opts: []BucketOption{
    56  				WithStrategy(StrategyReplace),
    57  			},
    58  		},
    59  	}
    60  	tests.run(ctx, t)
    61  }
    62  
    63  func replaceInsertAndUpdate(ctx context.Context, t *testing.T, opts []BucketOption) {
    64  	dirName := t.TempDir()
    65  
    66  	t.Run("memtable-only", func(t *testing.T) {
    67  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
    68  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
    69  		require.Nil(t, err)
    70  
    71  		// so big it effectively never triggers as part of this test
    72  		b.SetMemtableThreshold(1e9)
    73  
    74  		t.Run("set original values and verify", func(t *testing.T) {
    75  			key1 := []byte("key-1")
    76  			key2 := []byte("key-2")
    77  			key3 := []byte("key-3")
    78  			orig1 := []byte("original value for key1")
    79  			orig2 := []byte("original value for key2")
    80  			orig3 := []byte("original value for key3")
    81  
    82  			err = b.Put(key1, orig1)
    83  			require.Nil(t, err)
    84  			err = b.Put(key2, orig2)
    85  			require.Nil(t, err)
    86  			err = b.Put(key3, orig3)
    87  			require.Nil(t, err)
    88  
    89  			assert.Equal(t, 3, b.Count())
    90  			assert.Equal(t, 0, b.CountAsync())
    91  
    92  			res, err := b.Get(key1)
    93  			require.Nil(t, err)
    94  			assert.Equal(t, res, orig1)
    95  			res, err = b.Get(key2)
    96  			require.Nil(t, err)
    97  			assert.Equal(t, res, orig2)
    98  			res, err = b.Get(key3)
    99  			require.Nil(t, err)
   100  			assert.Equal(t, res, orig3)
   101  		})
   102  
   103  		t.Run("replace some, keep one", func(t *testing.T) {
   104  			key1 := []byte("key-1")
   105  			key2 := []byte("key-2")
   106  			key3 := []byte("key-3")
   107  			orig1 := []byte("original value for key1")
   108  			replaced2 := []byte("updated value for key2")
   109  			replaced3 := []byte("updated value for key3")
   110  
   111  			err = b.Put(key2, replaced2)
   112  			require.Nil(t, err)
   113  			err = b.Put(key3, replaced3)
   114  			require.Nil(t, err)
   115  
   116  			assert.Equal(t, 3, b.Count())
   117  			assert.Equal(t, 0, b.CountAsync())
   118  
   119  			res, err := b.Get(key1)
   120  			require.Nil(t, err)
   121  			assert.Equal(t, res, orig1)
   122  			res, err = b.Get(key2)
   123  			require.Nil(t, err)
   124  			assert.Equal(t, res, replaced2)
   125  			res, err = b.Get(key3)
   126  			require.Nil(t, err)
   127  			assert.Equal(t, res, replaced3)
   128  		})
   129  	})
   130  
   131  	t.Run("with single flush in between updates", func(t *testing.T) {
   132  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   133  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   134  		require.Nil(t, err)
   135  
   136  		// so big it effectively never triggers as part of this test
   137  		b.SetMemtableThreshold(1e9)
   138  
   139  		t.Run("set original values and verify", func(t *testing.T) {
   140  			key1 := []byte("key-1")
   141  			key2 := []byte("key-2")
   142  			key3 := []byte("key-3")
   143  			orig1 := []byte("original value for key1")
   144  			orig2 := []byte("original value for key2")
   145  			orig3 := []byte("original value for key3")
   146  
   147  			err = b.Put(key1, orig1)
   148  			require.Nil(t, err)
   149  			err = b.Put(key2, orig2)
   150  			require.Nil(t, err)
   151  			err = b.Put(key3, orig3)
   152  			require.Nil(t, err)
   153  
   154  			res, err := b.Get(key1)
   155  			require.Nil(t, err)
   156  			assert.Equal(t, res, orig1)
   157  			res, err = b.Get(key2)
   158  			require.Nil(t, err)
   159  			assert.Equal(t, res, orig2)
   160  			res, err = b.Get(key3)
   161  			require.Nil(t, err)
   162  			assert.Equal(t, res, orig3)
   163  		})
   164  
   165  		t.Run("flush memtable to disk", func(t *testing.T) {
   166  			require.Nil(t, b.FlushAndSwitch())
   167  		})
   168  
   169  		t.Run("count only objects on disk segment", func(t *testing.T) {
   170  			assert.Equal(t, 3, b.Count())
   171  			assert.Equal(t, 3, b.CountAsync())
   172  		})
   173  
   174  		t.Run("replace some, keep one", func(t *testing.T) {
   175  			key1 := []byte("key-1")
   176  			key2 := []byte("key-2")
   177  			key3 := []byte("key-3")
   178  			orig1 := []byte("original value for key1")
   179  			replaced2 := []byte("updated value for key2")
   180  			replaced3 := []byte("updated value for key3")
   181  
   182  			err = b.Put(key2, replaced2)
   183  			require.Nil(t, err)
   184  			err = b.Put(key3, replaced3)
   185  			require.Nil(t, err)
   186  
   187  			// make sure that the updates aren't counted as additions
   188  			assert.Equal(t, 3, b.Count())
   189  
   190  			// happens to be the same value, but that's just a coincidence, async
   191  			// ignores the memtable
   192  			assert.Equal(t, 3, b.CountAsync())
   193  
   194  			res, err := b.Get(key1)
   195  			require.Nil(t, err)
   196  			assert.Equal(t, orig1, res)
   197  			res, err = b.Get(key2)
   198  			require.Nil(t, err)
   199  			assert.Equal(t, replaced2, res)
   200  			res, err = b.Get(key3)
   201  			require.Nil(t, err)
   202  			assert.Equal(t, replaced3, res)
   203  		})
   204  	})
   205  
   206  	t.Run("with a flush after the initial write and after the update", func(t *testing.T) {
   207  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   208  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   209  		require.Nil(t, err)
   210  
   211  		// so big it effectively never triggers as part of this test
   212  		b.SetMemtableThreshold(1e9)
   213  
   214  		t.Run("set original values and verify", func(t *testing.T) {
   215  			key1 := []byte("key-1")
   216  			key2 := []byte("key-2")
   217  			key3 := []byte("key-3")
   218  			orig1 := []byte("original value for key1")
   219  			orig2 := []byte("original value for key2")
   220  			orig3 := []byte("original value for key3")
   221  
   222  			err = b.Put(key1, orig1)
   223  			require.Nil(t, err)
   224  			err = b.Put(key2, orig2)
   225  			require.Nil(t, err)
   226  			err = b.Put(key3, orig3)
   227  			require.Nil(t, err)
   228  
   229  			res, err := b.Get(key1)
   230  			require.Nil(t, err)
   231  			assert.Equal(t, res, orig1)
   232  			res, err = b.Get(key2)
   233  			require.Nil(t, err)
   234  			assert.Equal(t, res, orig2)
   235  			res, err = b.Get(key3)
   236  			require.Nil(t, err)
   237  			assert.Equal(t, res, orig3)
   238  		})
   239  
   240  		t.Run("flush memtable to disk", func(t *testing.T) {
   241  			require.Nil(t, b.FlushAndSwitch())
   242  		})
   243  
   244  		t.Run("replace some, keep one", func(t *testing.T) {
   245  			key1 := []byte("key-1")
   246  			key2 := []byte("key-2")
   247  			key3 := []byte("key-3")
   248  			orig1 := []byte("original value for key1")
   249  			replaced2 := []byte("updated value for key2")
   250  			replaced3 := []byte("updated value for key3")
   251  
   252  			err = b.Put(key2, replaced2)
   253  			require.Nil(t, err)
   254  			err = b.Put(key3, replaced3)
   255  			require.Nil(t, err)
   256  
   257  			// Flush before verifying!
   258  			require.Nil(t, b.FlushAndSwitch())
   259  
   260  			res, err := b.Get(key1)
   261  			require.Nil(t, err)
   262  			assert.Equal(t, res, orig1)
   263  			res, err = b.Get(key2)
   264  			require.Nil(t, err)
   265  			assert.Equal(t, res, replaced2)
   266  			res, err = b.Get(key3)
   267  			require.Nil(t, err)
   268  			assert.Equal(t, res, replaced3)
   269  		})
   270  
   271  		t.Run("count objects over several segments", func(t *testing.T) {
   272  			assert.Equal(t, 3, b.Count())
   273  			assert.Equal(t, 3, b.CountAsync())
   274  		})
   275  	})
   276  
   277  	t.Run("update in memtable, then do an orderly shutdown, and re-init", func(t *testing.T) {
   278  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   279  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   280  		require.Nil(t, err)
   281  
   282  		// so big it effectively never triggers as part of this test
   283  		b.SetMemtableThreshold(1e9)
   284  
   285  		t.Run("set original values and verify", func(t *testing.T) {
   286  			key1 := []byte("key-1")
   287  			key2 := []byte("key-2")
   288  			key3 := []byte("key-3")
   289  			orig1 := []byte("original value for key1")
   290  			orig2 := []byte("original value for key2")
   291  			orig3 := []byte("original value for key3")
   292  
   293  			err = b.Put(key1, orig1)
   294  			require.Nil(t, err)
   295  			err = b.Put(key2, orig2)
   296  			require.Nil(t, err)
   297  			err = b.Put(key3, orig3)
   298  			require.Nil(t, err)
   299  		})
   300  
   301  		t.Run("replace some, keep one", func(t *testing.T) {
   302  			key1 := []byte("key-1")
   303  			key2 := []byte("key-2")
   304  			key3 := []byte("key-3")
   305  			orig1 := []byte("original value for key1")
   306  			replaced2 := []byte("updated value for key2")
   307  			replaced3 := []byte("updated value for key3")
   308  
   309  			err = b.Put(key2, replaced2)
   310  			require.Nil(t, err)
   311  			err = b.Put(key3, replaced3)
   312  			require.Nil(t, err)
   313  
   314  			res, err := b.Get(key1)
   315  			require.Nil(t, err)
   316  			assert.Equal(t, res, orig1)
   317  			res, err = b.Get(key2)
   318  			require.Nil(t, err)
   319  			assert.Equal(t, res, replaced2)
   320  			res, err = b.Get(key3)
   321  			require.Nil(t, err)
   322  			assert.Equal(t, res, replaced3)
   323  		})
   324  
   325  		t.Run("orderly shutdown", func(t *testing.T) {
   326  			b.Shutdown(context.Background())
   327  		})
   328  
   329  		t.Run("init another bucket on the same files", func(t *testing.T) {
   330  			b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   331  				cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   332  			require.Nil(t, err)
   333  
   334  			key1 := []byte("key-1")
   335  			key2 := []byte("key-2")
   336  			key3 := []byte("key-3")
   337  			orig1 := []byte("original value for key1")
   338  			replaced2 := []byte("updated value for key2")
   339  			replaced3 := []byte("updated value for key3")
   340  
   341  			res, err := b2.Get(key1)
   342  			require.Nil(t, err)
   343  			assert.Equal(t, res, orig1)
   344  			res, err = b2.Get(key2)
   345  			require.Nil(t, err)
   346  			assert.Equal(t, res, replaced2)
   347  			res, err = b2.Get(key3)
   348  			require.Nil(t, err)
   349  			assert.Equal(t, res, replaced3)
   350  
   351  			// count objects over several segments after disk read
   352  			assert.Equal(t, 3, b2.Count())
   353  			assert.Equal(t, 3, b2.CountAsync())
   354  		})
   355  	})
   356  }
   357  
   358  func replaceInsertAndUpdate_WithSecondaryKeys(ctx context.Context, t *testing.T, opts []BucketOption) {
   359  	dirName := t.TempDir()
   360  
   361  	t.Run("memtable-only", func(t *testing.T) {
   362  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   363  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   364  		require.Nil(t, err)
   365  
   366  		// so big it effectively never triggers as part of this test
   367  		b.SetMemtableThreshold(1e9)
   368  
   369  		t.Run("set original values and verify", func(t *testing.T) {
   370  			key1 := []byte("key-1")
   371  			key2 := []byte("key-2")
   372  			key3 := []byte("key-3")
   373  			secondaryKey1 := []byte("secondary-key-1")
   374  			secondaryKey2 := []byte("secondary-key-2")
   375  			secondaryKey3 := []byte("secondary-key-3")
   376  			orig1 := []byte("original value for key1")
   377  			orig2 := []byte("original value for key2")
   378  			orig3 := []byte("original value for key3")
   379  
   380  			err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1))
   381  			require.Nil(t, err)
   382  			err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2))
   383  			require.Nil(t, err)
   384  			err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3))
   385  			require.Nil(t, err)
   386  
   387  			res, err := b.GetBySecondary(0, secondaryKey1)
   388  			require.Nil(t, err)
   389  			assert.Equal(t, res, orig1)
   390  			res, err = b.GetBySecondary(0, secondaryKey2)
   391  			require.Nil(t, err)
   392  			assert.Equal(t, res, orig2)
   393  			res, err = b.GetBySecondary(0, secondaryKey3)
   394  			require.Nil(t, err)
   395  			assert.Equal(t, res, orig3)
   396  		})
   397  
   398  		t.Run("replace some values, keep one - secondary keys not changed", func(t *testing.T) {
   399  			key2 := []byte("key-2")
   400  			key3 := []byte("key-3")
   401  			secondaryKey1 := []byte("secondary-key-1")
   402  			secondaryKey2 := []byte("secondary-key-2")
   403  			secondaryKey3 := []byte("secondary-key-3")
   404  			orig1 := []byte("original value for key1")
   405  			replaced2 := []byte("updated value for key2")
   406  			replaced3 := []byte("updated value for key3")
   407  
   408  			err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2))
   409  			require.Nil(t, err)
   410  			err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3))
   411  			require.Nil(t, err)
   412  
   413  			res, err := b.GetBySecondary(0, secondaryKey1)
   414  			require.Nil(t, err)
   415  			assert.Equal(t, res, orig1)
   416  			res, err = b.GetBySecondary(0, secondaryKey2)
   417  			require.Nil(t, err)
   418  			assert.Equal(t, res, replaced2)
   419  			res, err = b.GetBySecondary(0, secondaryKey3)
   420  			require.Nil(t, err)
   421  			assert.Equal(t, res, replaced3)
   422  		})
   423  
   424  		t.Run("replace the secondary keys on an update", func(t *testing.T) {
   425  			key2 := []byte("key-2")
   426  			key3 := []byte("key-3")
   427  			secondaryKey1 := []byte("secondary-key-1")
   428  			secondaryKey2 := []byte("secondary-key-2-updated")
   429  			secondaryKey3 := []byte("secondary-key-3-updated")
   430  			orig1 := []byte("original value for key1")
   431  			replaced2 := []byte("twice updated value for key2")
   432  			replaced3 := []byte("twice updated value for key3")
   433  
   434  			err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2))
   435  			require.Nil(t, err)
   436  			err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3))
   437  			require.Nil(t, err)
   438  
   439  			// verify you can find by updated secondary keys
   440  			res, err := b.GetBySecondary(0, secondaryKey1)
   441  			require.Nil(t, err)
   442  			assert.Equal(t, res, orig1)
   443  			res, err = b.GetBySecondary(0, secondaryKey2)
   444  			require.Nil(t, err)
   445  			assert.Equal(t, res, replaced2)
   446  			res, err = b.GetBySecondary(0, secondaryKey3)
   447  			require.Nil(t, err)
   448  			assert.Equal(t, res, replaced3)
   449  		})
   450  	})
   451  
   452  	t.Run("with single flush in between updates", func(t *testing.T) {
   453  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   454  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   455  		require.Nil(t, err)
   456  
   457  		// so big it effectively never triggers as part of this test
   458  		b.SetMemtableThreshold(1e9)
   459  
   460  		t.Run("set original values", func(t *testing.T) {
   461  			key1 := []byte("key-1")
   462  			key2 := []byte("key-2")
   463  			key3 := []byte("key-3")
   464  			secondaryKey1 := []byte("secondary-key-1")
   465  			secondaryKey2 := []byte("secondary-key-2")
   466  			secondaryKey3 := []byte("secondary-key-3")
   467  			orig1 := []byte("original value for key1")
   468  			orig2 := []byte("original value for key2")
   469  			orig3 := []byte("original value for key3")
   470  
   471  			err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1))
   472  			require.Nil(t, err)
   473  			err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2))
   474  			require.Nil(t, err)
   475  			err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3))
   476  			require.Nil(t, err)
   477  		})
   478  
   479  		t.Run("flush memtable to disk", func(t *testing.T) {
   480  			require.Nil(t, b.FlushAndSwitch())
   481  		})
   482  
   483  		t.Run("replace the secondary keys on an update", func(t *testing.T) {
   484  			key2 := []byte("key-2")
   485  			key3 := []byte("key-3")
   486  			secondaryKey1 := []byte("secondary-key-1")
   487  			secondaryKey2 := []byte("secondary-key-2-updated")
   488  			secondaryKey3 := []byte("secondary-key-3-updated")
   489  			orig1 := []byte("original value for key1")
   490  			replaced2 := []byte("twice updated value for key2")
   491  			replaced3 := []byte("twice updated value for key3")
   492  
   493  			err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2))
   494  			require.Nil(t, err)
   495  			err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3))
   496  			require.Nil(t, err)
   497  
   498  			// verify you can find by updated secondary keys
   499  			res, err := b.GetBySecondary(0, secondaryKey1)
   500  			require.Nil(t, err)
   501  			assert.Equal(t, res, orig1)
   502  			res, err = b.GetBySecondary(0, secondaryKey2)
   503  			require.Nil(t, err)
   504  			assert.Equal(t, res, replaced2)
   505  			res, err = b.GetBySecondary(0, secondaryKey3)
   506  			require.Nil(t, err)
   507  			assert.Equal(t, res, replaced3)
   508  		})
   509  	})
   510  
   511  	t.Run("with a flush after initial write and update", func(t *testing.T) {
   512  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   513  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   514  		require.Nil(t, err)
   515  
   516  		// so big it effectively never triggers as part of this test
   517  		b.SetMemtableThreshold(1e9)
   518  
   519  		t.Run("set original values", func(t *testing.T) {
   520  			key1 := []byte("key-1")
   521  			key2 := []byte("key-2")
   522  			key3 := []byte("key-3")
   523  			secondaryKey1 := []byte("secondary-key-1")
   524  			secondaryKey2 := []byte("secondary-key-2")
   525  			secondaryKey3 := []byte("secondary-key-3")
   526  			orig1 := []byte("original value for key1")
   527  			orig2 := []byte("original value for key2")
   528  			orig3 := []byte("original value for key3")
   529  
   530  			err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1))
   531  			require.Nil(t, err)
   532  			err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2))
   533  			require.Nil(t, err)
   534  			err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3))
   535  			require.Nil(t, err)
   536  		})
   537  
   538  		t.Run("flush memtable to disk", func(t *testing.T) {
   539  			require.Nil(t, b.FlushAndSwitch())
   540  		})
   541  
   542  		t.Run("replace the secondary keys on an update", func(t *testing.T) {
   543  			key2 := []byte("key-2")
   544  			key3 := []byte("key-3")
   545  			secondaryKey2 := []byte("secondary-key-2-updated")
   546  			secondaryKey3 := []byte("secondary-key-3-updated")
   547  			replaced2 := []byte("twice updated value for key2")
   548  			replaced3 := []byte("twice updated value for key3")
   549  
   550  			err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2))
   551  			require.Nil(t, err)
   552  			err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3))
   553  			require.Nil(t, err)
   554  		})
   555  
   556  		t.Run("flush memtable to disk", func(t *testing.T) {
   557  			require.Nil(t, b.FlushAndSwitch())
   558  		})
   559  
   560  		t.Run("verify again", func(t *testing.T) {
   561  			secondaryKey1 := []byte("secondary-key-1")
   562  			secondaryKey2 := []byte("secondary-key-2-updated")
   563  			secondaryKey3 := []byte("secondary-key-3-updated")
   564  			orig1 := []byte("original value for key1")
   565  			replaced2 := []byte("twice updated value for key2")
   566  			replaced3 := []byte("twice updated value for key3")
   567  
   568  			// verify you can find by updated secondary keys
   569  			res, err := b.GetBySecondary(0, secondaryKey1)
   570  			require.Nil(t, err)
   571  			assert.Equal(t, res, orig1)
   572  			res, err = b.GetBySecondary(0, secondaryKey2)
   573  			require.Nil(t, err)
   574  			assert.Equal(t, res, replaced2)
   575  			res, err = b.GetBySecondary(0, secondaryKey3)
   576  			require.Nil(t, err)
   577  			assert.Equal(t, res, replaced3)
   578  		})
   579  	})
   580  
   581  	t.Run("update in memtable then do an orderly shutdown and reinit", func(t *testing.T) {
   582  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   583  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   584  		require.Nil(t, err)
   585  
   586  		// so big it effectively never triggers as part of this test
   587  		b.SetMemtableThreshold(1e9)
   588  
   589  		t.Run("set original values", func(t *testing.T) {
   590  			key1 := []byte("key-1")
   591  			key2 := []byte("key-2")
   592  			key3 := []byte("key-3")
   593  			secondaryKey1 := []byte("secondary-key-1")
   594  			secondaryKey2 := []byte("secondary-key-2")
   595  			secondaryKey3 := []byte("secondary-key-3")
   596  			orig1 := []byte("original value for key1")
   597  			orig2 := []byte("original value for key2")
   598  			orig3 := []byte("original value for key3")
   599  
   600  			err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1))
   601  			require.Nil(t, err)
   602  			err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2))
   603  			require.Nil(t, err)
   604  			err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3))
   605  			require.Nil(t, err)
   606  		})
   607  
   608  		t.Run("replace the secondary keys on an update", func(t *testing.T) {
   609  			key2 := []byte("key-2")
   610  			key3 := []byte("key-3")
   611  			secondaryKey2 := []byte("secondary-key-2-updated")
   612  			secondaryKey3 := []byte("secondary-key-3-updated")
   613  			replaced2 := []byte("twice updated value for key2")
   614  			replaced3 := []byte("twice updated value for key3")
   615  
   616  			err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2))
   617  			require.Nil(t, err)
   618  			err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3))
   619  			require.Nil(t, err)
   620  		})
   621  
   622  		t.Run("flush memtable to disk", func(t *testing.T) {
   623  			require.Nil(t, b.Shutdown(context.Background()))
   624  		})
   625  
   626  		t.Run("init a new one and verify", func(t *testing.T) {
   627  			b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   628  				cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   629  			require.Nil(t, err)
   630  
   631  			secondaryKey1 := []byte("secondary-key-1")
   632  			secondaryKey2 := []byte("secondary-key-2-updated")
   633  			secondaryKey3 := []byte("secondary-key-3-updated")
   634  			orig1 := []byte("original value for key1")
   635  			replaced2 := []byte("twice updated value for key2")
   636  			replaced3 := []byte("twice updated value for key3")
   637  
   638  			// verify you can find by updated secondary keys
   639  			res, err := b2.GetBySecondary(0, secondaryKey1)
   640  			require.Nil(t, err)
   641  			assert.Equal(t, res, orig1)
   642  			res, err = b2.GetBySecondary(0, secondaryKey2)
   643  			require.Nil(t, err)
   644  			assert.Equal(t, res, replaced2)
   645  			res, err = b2.GetBySecondary(0, secondaryKey3)
   646  			require.Nil(t, err)
   647  			assert.Equal(t, res, replaced3)
   648  		})
   649  	})
   650  }
   651  
   652  func replaceInsertAndDelete(ctx context.Context, t *testing.T, opts []BucketOption) {
   653  	dirName := t.TempDir()
   654  
   655  	t.Run("memtable-only", func(t *testing.T) {
   656  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   657  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   658  		require.Nil(t, err)
   659  
   660  		// so big it effectively never triggers as part of this test
   661  		b.SetMemtableThreshold(1e9)
   662  
   663  		t.Run("set original values", func(t *testing.T) {
   664  			key1 := []byte("key-1")
   665  			key2 := []byte("key-2")
   666  			key3 := []byte("key-3")
   667  			orig1 := []byte("original value for key1")
   668  			orig2 := []byte("original value for key2")
   669  			orig3 := []byte("original value for key3")
   670  
   671  			err = b.Put(key1, orig1)
   672  			require.Nil(t, err)
   673  			err = b.Put(key2, orig2)
   674  			require.Nil(t, err)
   675  			err = b.Put(key3, orig3)
   676  			require.Nil(t, err)
   677  		})
   678  
   679  		t.Run("delete some, keep one", func(t *testing.T) {
   680  			key1 := []byte("key-1")
   681  			key2 := []byte("key-2")
   682  			key3 := []byte("key-3")
   683  			orig1 := []byte("original value for key1")
   684  
   685  			err = b.Delete(key2)
   686  			require.Nil(t, err)
   687  			err = b.Delete(key3)
   688  			require.Nil(t, err)
   689  
   690  			res, err := b.Get(key1)
   691  			require.Nil(t, err)
   692  			assert.Equal(t, res, orig1)
   693  			res, err = b.Get(key2)
   694  			require.Nil(t, err)
   695  			assert.Nil(t, res)
   696  			res, err = b.Get(key3)
   697  			require.Nil(t, err)
   698  			assert.Nil(t, res)
   699  		})
   700  
   701  		t.Run("count objects", func(t *testing.T) {
   702  			assert.Equal(t, 1, b.Count())
   703  			// all happenin in the memtable so far, async does not know of any
   704  			// objects yet
   705  			assert.Equal(t, 0, b.CountAsync())
   706  		})
   707  	})
   708  
   709  	t.Run("with single flush in between updates", func(t *testing.T) {
   710  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   711  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   712  		require.Nil(t, err)
   713  
   714  		// so big it effectively never triggers as part of this test
   715  		b.SetMemtableThreshold(1e9)
   716  
   717  		t.Run("set original values", func(t *testing.T) {
   718  			key1 := []byte("key-1")
   719  			key2 := []byte("key-2")
   720  			key3 := []byte("key-3")
   721  			orig1 := []byte("original value for key1")
   722  			orig2 := []byte("original value for key2")
   723  			orig3 := []byte("original value for key3")
   724  
   725  			err = b.Put(key1, orig1)
   726  			require.Nil(t, err)
   727  			err = b.Put(key2, orig2)
   728  			require.Nil(t, err)
   729  			err = b.Put(key3, orig3)
   730  			require.Nil(t, err)
   731  		})
   732  
   733  		t.Run("flush to disk", func(t *testing.T) {
   734  			require.Nil(t, b.FlushAndSwitch())
   735  		})
   736  
   737  		t.Run("delete some, keep one", func(t *testing.T) {
   738  			key1 := []byte("key-1")
   739  			key2 := []byte("key-2")
   740  			key3 := []byte("key-3")
   741  			orig1 := []byte("original value for key1")
   742  
   743  			err = b.Delete(key2)
   744  			require.Nil(t, err)
   745  			err = b.Delete(key3)
   746  			require.Nil(t, err)
   747  
   748  			res, err := b.Get(key1)
   749  			require.Nil(t, err)
   750  			assert.Equal(t, res, orig1)
   751  			res, err = b.Get(key2)
   752  			require.Nil(t, err)
   753  			assert.Nil(t, res)
   754  			res, err = b.Get(key3)
   755  			require.Nil(t, err)
   756  			assert.Nil(t, res)
   757  		})
   758  
   759  		t.Run("count objects", func(t *testing.T) {
   760  			assert.Equal(t, 1, b.Count())
   761  			// async still looks at the objects in the segment, ignores deletes in
   762  			// the memtable
   763  			assert.Equal(t, 3, b.CountAsync())
   764  		})
   765  	})
   766  
   767  	t.Run("with flushes after initial write and delete", func(t *testing.T) {
   768  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   769  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   770  		require.Nil(t, err)
   771  
   772  		// so big it effectively never triggers as part of this test
   773  		b.SetMemtableThreshold(1e9)
   774  
   775  		t.Run("set original values", func(t *testing.T) {
   776  			key1 := []byte("key-1")
   777  			key2 := []byte("key-2")
   778  			key3 := []byte("key-3")
   779  			orig1 := []byte("original value for key1")
   780  			orig2 := []byte("original value for key2")
   781  			orig3 := []byte("original value for key3")
   782  
   783  			err = b.Put(key1, orig1)
   784  			require.Nil(t, err)
   785  			err = b.Put(key2, orig2)
   786  			require.Nil(t, err)
   787  			err = b.Put(key3, orig3)
   788  			require.Nil(t, err)
   789  		})
   790  
   791  		t.Run("flush to disk", func(t *testing.T) {
   792  			require.Nil(t, b.FlushAndSwitch())
   793  		})
   794  
   795  		t.Run("delete some, keep one", func(t *testing.T) {
   796  			key1 := []byte("key-1")
   797  			key2 := []byte("key-2")
   798  			key3 := []byte("key-3")
   799  			orig1 := []byte("original value for key1")
   800  
   801  			err = b.Delete(key2)
   802  			require.Nil(t, err)
   803  			err = b.Delete(key3)
   804  			require.Nil(t, err)
   805  
   806  			// Flush again!
   807  			require.Nil(t, b.FlushAndSwitch())
   808  
   809  			res, err := b.Get(key1)
   810  			require.Nil(t, err)
   811  			assert.Equal(t, res, orig1)
   812  			res, err = b.Get(key2)
   813  			require.Nil(t, err)
   814  			assert.Nil(t, res)
   815  			res, err = b.Get(key3)
   816  			require.Nil(t, err)
   817  			assert.Nil(t, res)
   818  		})
   819  
   820  		t.Run("count objects", func(t *testing.T) {
   821  			assert.Equal(t, 1, b.Count())
   822  			assert.Equal(t, 1, b.CountAsync())
   823  		})
   824  	})
   825  }
   826  
   827  func replaceCursors(ctx context.Context, t *testing.T, opts []BucketOption) {
   828  	t.Run("memtable-only", func(t *testing.T) {
   829  		r := getRandomSeed()
   830  		dirName := t.TempDir()
   831  
   832  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
   833  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
   834  		require.Nil(t, err)
   835  
   836  		// so big it effectively never triggers as part of this test
   837  		b.SetMemtableThreshold(1e9)
   838  
   839  		t.Run("set original values", func(t *testing.T) {
   840  			pairs := 20
   841  			keys := make([][]byte, pairs)
   842  			values := make([][]byte, pairs)
   843  
   844  			for i := range keys {
   845  				keys[i] = []byte(fmt.Sprintf("key-%03d", i))
   846  				values[i] = []byte(fmt.Sprintf("value-%03d", i))
   847  			}
   848  
   849  			// shuffle to make sure the BST isn't accidentally in order
   850  			r.Shuffle(len(keys), func(i, j int) {
   851  				keys[i], keys[j] = keys[j], keys[i]
   852  				values[i], values[j] = values[j], values[i]
   853  			})
   854  
   855  			for i := range keys {
   856  				err = b.Put(keys[i], values[i])
   857  				require.Nil(t, err)
   858  			}
   859  		})
   860  
   861  		t.Run("seek from somewhere in the middle", func(t *testing.T) {
   862  			expectedKeys := [][]byte{
   863  				[]byte("key-016"),
   864  				[]byte("key-017"),
   865  				[]byte("key-018"),
   866  				[]byte("key-019"),
   867  			}
   868  			expectedValues := [][]byte{
   869  				[]byte("value-016"),
   870  				[]byte("value-017"),
   871  				[]byte("value-018"),
   872  				[]byte("value-019"),
   873  			}
   874  
   875  			var retrievedKeys [][]byte
   876  			var retrievedValues [][]byte
   877  			c := b.Cursor()
   878  			defer c.Close()
   879  			for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() {
   880  				retrievedKeys = copyAndAppend(retrievedKeys, k)
   881  				retrievedValues = copyAndAppend(retrievedValues, v)
   882  			}
   883  
   884  			assert.Equal(t, expectedKeys, retrievedKeys)
   885  			assert.Equal(t, expectedValues, retrievedValues)
   886  		})
   887  
   888  		t.Run("start from the beginning", func(t *testing.T) {
   889  			expectedKeys := [][]byte{
   890  				[]byte("key-000"),
   891  				[]byte("key-001"),
   892  				[]byte("key-002"),
   893  			}
   894  			expectedValues := [][]byte{
   895  				[]byte("value-000"),
   896  				[]byte("value-001"),
   897  				[]byte("value-002"),
   898  			}
   899  
   900  			var retrievedKeys [][]byte
   901  			var retrievedValues [][]byte
   902  			c := b.Cursor()
   903  			defer c.Close()
   904  			retrieved := 0
   905  			for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() {
   906  				retrieved++
   907  				retrievedKeys = copyAndAppend(retrievedKeys, k)
   908  				retrievedValues = copyAndAppend(retrievedValues, v)
   909  			}
   910  
   911  			assert.Equal(t, expectedKeys, retrievedKeys)
   912  			assert.Equal(t, expectedValues, retrievedValues)
   913  		})
   914  
   915  		t.Run("replace a key", func(t *testing.T) {
   916  			key := []byte("key-002")
   917  			value := []byte("value-002-updated")
   918  
   919  			err = b.Put(key, value)
   920  			require.Nil(t, err)
   921  
   922  			expectedKeys := [][]byte{
   923  				[]byte("key-001"),
   924  				[]byte("key-002"),
   925  			}
   926  			expectedValues := [][]byte{
   927  				[]byte("value-001"),
   928  				[]byte("value-002-updated"),
   929  			}
   930  
   931  			var retrievedKeys [][]byte
   932  			var retrievedValues [][]byte
   933  			c := b.Cursor()
   934  			defer c.Close()
   935  			retrieved := 0
   936  			for k, v := c.Seek([]byte("key-001")); k != nil && retrieved < 2; k, v = c.Next() {
   937  				retrieved++
   938  				retrievedKeys = copyAndAppend(retrievedKeys, k)
   939  				retrievedValues = copyAndAppend(retrievedValues, v)
   940  			}
   941  
   942  			assert.Equal(t, expectedKeys, retrievedKeys)
   943  			assert.Equal(t, expectedValues, retrievedValues)
   944  		})
   945  
   946  		t.Run("delete a key", func(t *testing.T) {
   947  			key := []byte("key-002")
   948  
   949  			err = b.Delete(key)
   950  			require.Nil(t, err)
   951  
   952  			t.Run("seek to a specific key", func(t *testing.T) {
   953  				expectedKeys := [][]byte{
   954  					[]byte("key-001"),
   955  					[]byte("key-003"),
   956  				}
   957  				expectedValues := [][]byte{
   958  					[]byte("value-001"),
   959  					[]byte("value-003"),
   960  				}
   961  				var retrievedKeys [][]byte
   962  				var retrievedValues [][]byte
   963  				c := b.Cursor()
   964  				defer c.Close()
   965  				retrieved := 0
   966  				for k, v := c.Seek([]byte("key-001")); k != nil && retrieved < 2; k, v = c.Next() {
   967  					retrieved++
   968  					retrievedKeys = copyAndAppend(retrievedKeys, k)
   969  					retrievedValues = copyAndAppend(retrievedValues, v)
   970  				}
   971  
   972  				assert.Equal(t, expectedKeys, retrievedKeys)
   973  				assert.Equal(t, expectedValues, retrievedValues)
   974  			})
   975  
   976  			t.Run("seek to first key", func(t *testing.T) {
   977  				expectedKeys := [][]byte{
   978  					[]byte("key-000"),
   979  					[]byte("key-001"),
   980  					[]byte("key-003"),
   981  				}
   982  				expectedValues := [][]byte{
   983  					[]byte("value-000"),
   984  					[]byte("value-001"),
   985  					[]byte("value-003"),
   986  				}
   987  
   988  				var retrievedKeys [][]byte
   989  				var retrievedValues [][]byte
   990  				c := b.Cursor()
   991  				defer c.Close()
   992  				retrieved := 0
   993  				for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() {
   994  					retrieved++
   995  					retrievedKeys = copyAndAppend(retrievedKeys, k)
   996  					retrievedValues = copyAndAppend(retrievedValues, v)
   997  				}
   998  
   999  				assert.Equal(t, expectedKeys, retrievedKeys)
  1000  				assert.Equal(t, expectedValues, retrievedValues)
  1001  			})
  1002  		})
  1003  
  1004  		t.Run("delete the first key", func(t *testing.T) {
  1005  			key := []byte("key-000")
  1006  
  1007  			err = b.Delete(key)
  1008  			require.Nil(t, err)
  1009  
  1010  			t.Run("seek to a specific key", func(t *testing.T) {
  1011  				expectedKeys := [][]byte{
  1012  					[]byte("key-001"),
  1013  					[]byte("key-003"),
  1014  				}
  1015  				expectedValues := [][]byte{
  1016  					[]byte("value-001"),
  1017  					[]byte("value-003"),
  1018  				}
  1019  				var retrievedKeys [][]byte
  1020  				var retrievedValues [][]byte
  1021  				c := b.Cursor()
  1022  				defer c.Close()
  1023  				retrieved := 0
  1024  				for k, v := c.Seek([]byte("key-000")); k != nil && retrieved < 2; k, v = c.Next() {
  1025  					retrieved++
  1026  					retrievedKeys = copyAndAppend(retrievedKeys, k)
  1027  					retrievedValues = copyAndAppend(retrievedValues, v)
  1028  				}
  1029  
  1030  				assert.Equal(t, expectedKeys, retrievedKeys)
  1031  				assert.Equal(t, expectedValues, retrievedValues)
  1032  			})
  1033  
  1034  			t.Run("seek to first key", func(t *testing.T) {
  1035  				expectedKeys := [][]byte{
  1036  					[]byte("key-001"),
  1037  					[]byte("key-003"),
  1038  				}
  1039  				expectedValues := [][]byte{
  1040  					[]byte("value-001"),
  1041  					[]byte("value-003"),
  1042  				}
  1043  
  1044  				var retrievedKeys [][]byte
  1045  				var retrievedValues [][]byte
  1046  				c := b.Cursor()
  1047  				defer c.Close()
  1048  				retrieved := 0
  1049  				for k, v := c.First(); k != nil && retrieved < 2; k, v = c.Next() {
  1050  					retrieved++
  1051  					retrievedKeys = copyAndAppend(retrievedKeys, k)
  1052  					retrievedValues = copyAndAppend(retrievedValues, v)
  1053  				}
  1054  
  1055  				assert.Equal(t, expectedKeys, retrievedKeys)
  1056  				assert.Equal(t, expectedValues, retrievedValues)
  1057  			})
  1058  		})
  1059  	})
  1060  
  1061  	t.Run("with a single flush", func(t *testing.T) {
  1062  		r := getRandomSeed()
  1063  		dirName := t.TempDir()
  1064  
  1065  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
  1066  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
  1067  		require.Nil(t, err)
  1068  
  1069  		// so big it effectively never triggers as part of this test
  1070  		b.SetMemtableThreshold(1e9)
  1071  
  1072  		t.Run("set original values", func(t *testing.T) {
  1073  			pairs := 20
  1074  			keys := make([][]byte, pairs)
  1075  			values := make([][]byte, pairs)
  1076  
  1077  			for i := range keys {
  1078  				keys[i] = []byte(fmt.Sprintf("key-%03d", i))
  1079  				values[i] = []byte(fmt.Sprintf("value-%03d", i))
  1080  			}
  1081  
  1082  			// shuffle to make sure the BST isn't accidentally in order
  1083  			r.Shuffle(len(keys), func(i, j int) {
  1084  				keys[i], keys[j] = keys[j], keys[i]
  1085  				values[i], values[j] = values[j], values[i]
  1086  			})
  1087  
  1088  			for i := range keys {
  1089  				err = b.Put(keys[i], values[i])
  1090  				require.Nil(t, err)
  1091  			}
  1092  		})
  1093  
  1094  		t.Run("flush to disk", func(t *testing.T) {
  1095  			require.Nil(t, b.FlushAndSwitch())
  1096  		})
  1097  
  1098  		t.Run("seek from somewhere in the middle", func(t *testing.T) {
  1099  			expectedKeys := [][]byte{
  1100  				[]byte("key-016"),
  1101  				[]byte("key-017"),
  1102  				[]byte("key-018"),
  1103  				[]byte("key-019"),
  1104  			}
  1105  			expectedValues := [][]byte{
  1106  				[]byte("value-016"),
  1107  				[]byte("value-017"),
  1108  				[]byte("value-018"),
  1109  				[]byte("value-019"),
  1110  			}
  1111  
  1112  			var retrievedKeys [][]byte
  1113  			var retrievedValues [][]byte
  1114  			c := b.Cursor()
  1115  			defer c.Close()
  1116  			for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() {
  1117  				retrievedKeys = copyAndAppend(retrievedKeys, k)
  1118  				retrievedValues = copyAndAppend(retrievedValues, v)
  1119  			}
  1120  
  1121  			assert.Equal(t, expectedKeys, retrievedKeys)
  1122  			assert.Equal(t, expectedValues, retrievedValues)
  1123  		})
  1124  
  1125  		t.Run("start from the beginning", func(t *testing.T) {
  1126  			expectedKeys := [][]byte{
  1127  				[]byte("key-000"),
  1128  				[]byte("key-001"),
  1129  				[]byte("key-002"),
  1130  			}
  1131  			expectedValues := [][]byte{
  1132  				[]byte("value-000"),
  1133  				[]byte("value-001"),
  1134  				[]byte("value-002"),
  1135  			}
  1136  
  1137  			var retrievedKeys [][]byte
  1138  			var retrievedValues [][]byte
  1139  			c := b.Cursor()
  1140  			defer c.Close()
  1141  			retrieved := 0
  1142  			for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() {
  1143  				retrieved++
  1144  				retrievedKeys = copyAndAppend(retrievedKeys, k)
  1145  				retrievedValues = copyAndAppend(retrievedValues, v)
  1146  			}
  1147  
  1148  			assert.Equal(t, expectedKeys, retrievedKeys)
  1149  			assert.Equal(t, expectedValues, retrievedValues)
  1150  		})
  1151  	})
  1152  
  1153  	t.Run("mixing several disk segments and memtable - with updates", func(t *testing.T) {
  1154  		r := getRandomSeed()
  1155  		dirName := t.TempDir()
  1156  
  1157  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
  1158  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
  1159  		require.Nil(t, err)
  1160  
  1161  		// so big it effectively never triggers as part of this test
  1162  		b.SetMemtableThreshold(1e9)
  1163  
  1164  		t.Run("first third (%3==0)", func(t *testing.T) {
  1165  			pairs := 20
  1166  			var keys [][]byte
  1167  			var values [][]byte
  1168  
  1169  			for i := 0; i < pairs; i++ {
  1170  				if i%3 == 0 {
  1171  					keys = copyAndAppend(keys, []byte(fmt.Sprintf("key-%03d", i)))
  1172  					values = copyAndAppend(values, []byte(fmt.Sprintf("value-%03d", i)))
  1173  				}
  1174  			}
  1175  
  1176  			// shuffle to make sure the BST isn't accidentally in order
  1177  			r.Shuffle(len(keys), func(i, j int) {
  1178  				keys[i], keys[j] = keys[j], keys[i]
  1179  				values[i], values[j] = values[j], values[i]
  1180  			})
  1181  
  1182  			for i := range keys {
  1183  				err = b.Put(keys[i], values[i])
  1184  				require.Nil(t, err)
  1185  			}
  1186  		})
  1187  
  1188  		t.Run("flush to disk", func(t *testing.T) {
  1189  			require.Nil(t, b.FlushAndSwitch())
  1190  		})
  1191  
  1192  		t.Run("second third (%3==1)", func(t *testing.T) {
  1193  			pairs := 20
  1194  			var keys [][]byte
  1195  			var values [][]byte
  1196  
  1197  			for i := 0; i < pairs; i++ {
  1198  				if i%3 == 1 {
  1199  					keys = copyAndAppend(keys, []byte(fmt.Sprintf("key-%03d", i)))
  1200  					values = copyAndAppend(values, []byte(fmt.Sprintf("value-%03d", i)))
  1201  				}
  1202  			}
  1203  
  1204  			// shuffle to make sure the BST isn't accidentally in order
  1205  			r.Shuffle(len(keys), func(i, j int) {
  1206  				keys[i], keys[j] = keys[j], keys[i]
  1207  				values[i], values[j] = values[j], values[i]
  1208  			})
  1209  
  1210  			for i := range keys {
  1211  				err = b.Put(keys[i], values[i])
  1212  				require.Nil(t, err)
  1213  			}
  1214  		})
  1215  
  1216  		t.Run("update something that was already written in segment 1", func(t *testing.T) {
  1217  			require.Nil(t, b.Put([]byte("key-000"), []byte("updated-value-000")))
  1218  			require.Nil(t, b.Delete([]byte("key-003")))
  1219  		})
  1220  
  1221  		t.Run("flush to disk", func(t *testing.T) {
  1222  			require.Nil(t, b.FlushAndSwitch())
  1223  		})
  1224  
  1225  		t.Run("third third (%3==2) memtable only", func(t *testing.T) {
  1226  			pairs := 20
  1227  			var keys [][]byte
  1228  			var values [][]byte
  1229  
  1230  			for i := 0; i < pairs; i++ {
  1231  				if i%3 == 2 {
  1232  					keys = copyAndAppend(keys, []byte(fmt.Sprintf("key-%03d", i)))
  1233  					values = copyAndAppend(values, []byte(fmt.Sprintf("value-%03d", i)))
  1234  				}
  1235  			}
  1236  
  1237  			// shuffle to make sure the BST isn't accidentally in order
  1238  			r.Shuffle(len(keys), func(i, j int) {
  1239  				keys[i], keys[j] = keys[j], keys[i]
  1240  				values[i], values[j] = values[j], values[i]
  1241  			})
  1242  
  1243  			for i := range keys {
  1244  				err = b.Put(keys[i], values[i])
  1245  				require.Nil(t, err)
  1246  			}
  1247  
  1248  			// no flush for this one, so this segment stays in the memtable
  1249  		})
  1250  
  1251  		t.Run("update something that was already written previously", func(t *testing.T) {
  1252  			require.Nil(t, b.Put([]byte("key-000"), []byte("twice-updated-value-000")))
  1253  			require.Nil(t, b.Put([]byte("key-001"), []byte("once-updated-value-001")))
  1254  			require.Nil(t, b.Put([]byte("key-019"), []byte("once-updated-value-019")))
  1255  			require.Nil(t, b.Delete([]byte("key-018")))
  1256  		})
  1257  
  1258  		t.Run("seek from somewhere in the middle", func(t *testing.T) {
  1259  			expectedKeys := [][]byte{
  1260  				[]byte("key-016"),
  1261  				[]byte("key-017"),
  1262  				// key-018 deleted
  1263  				[]byte("key-019"),
  1264  			}
  1265  			expectedValues := [][]byte{
  1266  				[]byte("value-016"),
  1267  				[]byte("value-017"),
  1268  				[]byte("once-updated-value-019"),
  1269  			}
  1270  
  1271  			var retrievedKeys [][]byte
  1272  			var retrievedValues [][]byte
  1273  			c := b.Cursor()
  1274  			defer c.Close()
  1275  			for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() {
  1276  				retrievedKeys = copyAndAppend(retrievedKeys, k)
  1277  				retrievedValues = copyAndAppend(retrievedValues, v)
  1278  			}
  1279  
  1280  			assert.Equal(t, expectedKeys, retrievedKeys)
  1281  			assert.Equal(t, expectedValues, retrievedValues)
  1282  		})
  1283  
  1284  		t.Run("start from the beginning", func(t *testing.T) {
  1285  			expectedKeys := [][]byte{
  1286  				[]byte("key-000"),
  1287  				[]byte("key-001"),
  1288  				[]byte("key-002"),
  1289  				// key-003 was deleted
  1290  				[]byte("key-004"),
  1291  			}
  1292  			expectedValues := [][]byte{
  1293  				[]byte("twice-updated-value-000"),
  1294  				[]byte("once-updated-value-001"),
  1295  				[]byte("value-002"),
  1296  				[]byte("value-004"),
  1297  			}
  1298  
  1299  			var retrievedKeys [][]byte
  1300  			var retrievedValues [][]byte
  1301  			c := b.Cursor()
  1302  			defer c.Close()
  1303  			retrieved := 0
  1304  			for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() {
  1305  				retrieved++
  1306  				retrievedKeys = copyAndAppend(retrievedKeys, k)
  1307  				retrievedValues = copyAndAppend(retrievedValues, v)
  1308  			}
  1309  
  1310  			assert.Equal(t, expectedKeys, retrievedKeys)
  1311  			assert.Equal(t, expectedValues, retrievedValues)
  1312  		})
  1313  
  1314  		t.Run("re-add the deleted keys", func(t *testing.T) {
  1315  			require.Nil(t, b.Put([]byte("key-003"), []byte("readded-003")))
  1316  			require.Nil(t, b.Put([]byte("key-018"), []byte("readded-018")))
  1317  			// tombstones are now only in memtable
  1318  		})
  1319  
  1320  		t.Run("seek from somewhere in the middle", func(t *testing.T) {
  1321  			expectedKeys := [][]byte{
  1322  				[]byte("key-016"),
  1323  				[]byte("key-017"),
  1324  				[]byte("key-018"),
  1325  				[]byte("key-019"),
  1326  			}
  1327  			expectedValues := [][]byte{
  1328  				[]byte("value-016"),
  1329  				[]byte("value-017"),
  1330  				[]byte("readded-018"),
  1331  				[]byte("once-updated-value-019"),
  1332  			}
  1333  
  1334  			var retrievedKeys [][]byte
  1335  			var retrievedValues [][]byte
  1336  			c := b.Cursor()
  1337  			defer c.Close()
  1338  			for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() {
  1339  				retrievedKeys = copyAndAppend(retrievedKeys, k)
  1340  				retrievedValues = copyAndAppend(retrievedValues, v)
  1341  			}
  1342  
  1343  			assert.Equal(t, expectedKeys, retrievedKeys)
  1344  			assert.Equal(t, expectedValues, retrievedValues)
  1345  		})
  1346  
  1347  		t.Run("start from the beginning", func(t *testing.T) {
  1348  			expectedKeys := [][]byte{
  1349  				[]byte("key-000"),
  1350  				[]byte("key-001"),
  1351  				[]byte("key-002"),
  1352  				[]byte("key-003"),
  1353  			}
  1354  			expectedValues := [][]byte{
  1355  				[]byte("twice-updated-value-000"),
  1356  				[]byte("once-updated-value-001"),
  1357  				[]byte("value-002"),
  1358  				[]byte("readded-003"),
  1359  			}
  1360  
  1361  			var retrievedKeys [][]byte
  1362  			var retrievedValues [][]byte
  1363  			c := b.Cursor()
  1364  			defer c.Close()
  1365  			retrieved := 0
  1366  			for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() {
  1367  				retrieved++
  1368  				retrievedKeys = copyAndAppend(retrievedKeys, k)
  1369  				retrievedValues = copyAndAppend(retrievedValues, v)
  1370  			}
  1371  
  1372  			assert.Equal(t, expectedKeys, retrievedKeys)
  1373  			assert.Equal(t, expectedValues, retrievedValues)
  1374  		})
  1375  
  1376  		t.Run("perform a final flush to disk", func(t *testing.T) {
  1377  			require.Nil(t, b.FlushAndSwitch())
  1378  		})
  1379  
  1380  		t.Run("seek from somewhere in the middle", func(t *testing.T) {
  1381  			expectedKeys := [][]byte{
  1382  				[]byte("key-016"),
  1383  				[]byte("key-017"),
  1384  				[]byte("key-018"),
  1385  				[]byte("key-019"),
  1386  			}
  1387  			expectedValues := [][]byte{
  1388  				[]byte("value-016"),
  1389  				[]byte("value-017"),
  1390  				[]byte("readded-018"),
  1391  				[]byte("once-updated-value-019"),
  1392  			}
  1393  
  1394  			var retrievedKeys [][]byte
  1395  			var retrievedValues [][]byte
  1396  			c := b.Cursor()
  1397  			defer c.Close()
  1398  			for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() {
  1399  				retrievedKeys = copyAndAppend(retrievedKeys, k)
  1400  				retrievedValues = copyAndAppend(retrievedValues, v)
  1401  			}
  1402  
  1403  			assert.Equal(t, expectedKeys, retrievedKeys)
  1404  			assert.Equal(t, expectedValues, retrievedValues)
  1405  		})
  1406  
  1407  		t.Run("start from the beginning", func(t *testing.T) {
  1408  			expectedKeys := [][]byte{
  1409  				[]byte("key-000"),
  1410  				[]byte("key-001"),
  1411  				[]byte("key-002"),
  1412  				[]byte("key-003"),
  1413  			}
  1414  			expectedValues := [][]byte{
  1415  				[]byte("twice-updated-value-000"),
  1416  				[]byte("once-updated-value-001"),
  1417  				[]byte("value-002"),
  1418  				[]byte("readded-003"),
  1419  			}
  1420  
  1421  			var retrievedKeys [][]byte
  1422  			var retrievedValues [][]byte
  1423  			c := b.Cursor()
  1424  			defer c.Close()
  1425  			retrieved := 0
  1426  			for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() {
  1427  				retrieved++
  1428  				retrievedKeys = copyAndAppend(retrievedKeys, k)
  1429  				retrievedValues = copyAndAppend(retrievedValues, v)
  1430  			}
  1431  
  1432  			assert.Equal(t, expectedKeys, retrievedKeys)
  1433  			assert.Equal(t, expectedValues, retrievedValues)
  1434  		})
  1435  	})
  1436  
  1437  	// This test is inspired by unusual behavior encountered as part of the
  1438  	// evaluation of gh-1569 where a delete could sometimes lead to no data after
  1439  	// a restart which was caused by the disk segment cursor's .first() method
  1440  	// not returning the correct key. Thus we'd have a null-key with a tombstone
  1441  	// which would override whatever is the real "first" key, since null is
  1442  	// always smaller
  1443  	t.Run("with deletes as latest in some segments", func(t *testing.T) {
  1444  		dirName := t.TempDir()
  1445  
  1446  		b, err := NewBucket(ctx, dirName, "", nullLogger(), nil,
  1447  			cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...)
  1448  		require.Nil(t, err)
  1449  
  1450  		// so big it effectively never triggers as part of this test
  1451  		b.SetMemtableThreshold(1e9)
  1452  
  1453  		t.Run("add new datapoint", func(t *testing.T) {
  1454  			err := b.Put([]byte("key-1"), []byte("value-1"))
  1455  			require.Nil(t, err)
  1456  		})
  1457  
  1458  		t.Run("add datapoint and flush", func(t *testing.T) {
  1459  			err := b.Put([]byte("key-8"), []byte("value-8"))
  1460  			require.Nil(t, err)
  1461  
  1462  			require.Nil(t, b.FlushAndSwitch())
  1463  		})
  1464  
  1465  		t.Run("delete datapoint and flush", func(t *testing.T) {
  1466  			err := b.Delete([]byte("key-8"))
  1467  			// note that we are deleting the key with the 'higher' key, so a missing
  1468  			// key on the delete would definitely be mismatched. If we had instead
  1469  			// the deleted the first key, the incorrect tombstone would have been
  1470  			// correct by coincidence
  1471  			require.Nil(t, err)
  1472  
  1473  			require.Nil(t, b.FlushAndSwitch())
  1474  		})
  1475  
  1476  		t.Run("verify", func(t *testing.T) {
  1477  			expectedKeys := [][]byte{
  1478  				[]byte("key-1"),
  1479  			}
  1480  			expectedValues := [][]byte{
  1481  				[]byte("value-1"),
  1482  			}
  1483  
  1484  			var retrievedKeys [][]byte
  1485  			var retrievedValues [][]byte
  1486  			c := b.Cursor()
  1487  			defer c.Close()
  1488  			retrieved := 0
  1489  			for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() {
  1490  				retrieved++
  1491  				retrievedKeys = copyAndAppend(retrievedKeys, k)
  1492  				retrievedValues = copyAndAppend(retrievedValues, v)
  1493  			}
  1494  
  1495  			assert.Equal(t, expectedKeys, retrievedKeys)
  1496  			assert.Equal(t, expectedValues, retrievedValues)
  1497  		})
  1498  	})
  1499  }
  1500  
  1501  func copyAndAppend(list [][]byte, elem []byte) [][]byte {
  1502  	elemCopy := make([]byte, len(elem))
  1503  	copy(elemCopy, elem)
  1504  	return append(list, elemCopy)
  1505  }