github.com/zuoyebang/bitalosdb@v1.1.1-0.20240516111551-79a8c4d8ce20/bitalos_test.go (about)

     1  // Copyright 2021 The Bitalosdb author(hustxrb@163.com) and other contributors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bitalosdb
    16  
    17  import (
    18  	"bytes"
    19  	"encoding/binary"
    20  	"fmt"
    21  	"os"
    22  	"testing"
    23  	"time"
    24  
    25  	"github.com/cockroachdb/errors"
    26  	"github.com/cockroachdb/errors/oserror"
    27  	"github.com/stretchr/testify/require"
    28  	"github.com/zuoyebang/bitalosdb/internal/base"
    29  	"github.com/zuoyebang/bitalosdb/internal/compress"
    30  	"github.com/zuoyebang/bitalosdb/internal/consts"
    31  	"github.com/zuoyebang/bitalosdb/internal/options"
    32  	"github.com/zuoyebang/bitalosdb/internal/sortedkv"
    33  	"github.com/zuoyebang/bitalosdb/internal/unsafe2"
    34  	"github.com/zuoyebang/bitalosdb/internal/utils"
    35  )
    36  
    37  const (
    38  	testDirname                     = "./test-data"
    39  	testMaxWriteBufferNumber        = 8
    40  	testMemTableSize                = 64 << 20
    41  	testCacheSize                   = 128 << 20
    42  	testSlotId               uint16 = 1
    43  )
    44  
    45  var (
    46  	testOptsCompressType      int   = compress.CompressTypeNo
    47  	testOptsDisableWAL        bool  = false
    48  	testOptsUseMapIndex       bool  = true
    49  	testOptsUsePrefixCompress bool  = false
    50  	testOptsUseBlockCompress  bool  = false
    51  	testOptsCacheType         int   = consts.CacheTypeLru
    52  	testOptsCacheSize         int64 = 0
    53  	testOptsMemtableSize      int   = testMemTableSize
    54  	testOptsUseBitable        bool  = false
    55  )
    56  
    57  func resetTestOptsVal() {
    58  	testOptsCompressType = compress.CompressTypeNo
    59  	testOptsDisableWAL = false
    60  	testOptsUseMapIndex = true
    61  	testOptsUsePrefixCompress = false
    62  	testOptsUseBlockCompress = false
    63  	testOptsCacheType = consts.CacheTypeLru
    64  	testOptsCacheSize = 0
    65  	testOptsMemtableSize = testMemTableSize
    66  	testOptsUseBitable = false
    67  }
    68  
    69  var defaultLargeValBytes []byte
    70  
    71  func openTestDB(dir string, optspool *options.OptionsPool) *DB {
    72  	defer func() {
    73  		resetTestOptsVal()
    74  	}()
    75  	compactInfo := CompactEnv{
    76  		StartHour:     0,
    77  		EndHour:       23,
    78  		DeletePercent: 0.2,
    79  		BitreeMaxSize: 2 << 30,
    80  		Interval:      60,
    81  	}
    82  	opts := &Options{
    83  		BytesPerSync:                consts.DefaultBytesPerSync,
    84  		MemTableSize:                testOptsMemtableSize,
    85  		MemTableStopWritesThreshold: testMaxWriteBufferNumber,
    86  		CacheType:                   testOptsCacheType,
    87  		CacheSize:                   testOptsCacheSize,
    88  		CacheHashSize:               10000,
    89  		Verbose:                     true,
    90  		Comparer:                    DefaultComparer,
    91  		CompactInfo:                 compactInfo,
    92  		LogTag:                      "[bitalosdb/test]",
    93  		Logger:                      DefaultLogger,
    94  		DataType:                    "string",
    95  		CompressionType:             testOptsCompressType,
    96  		UseBithash:                  true,
    97  		UseBitable:                  testOptsUseBitable,
    98  		UseMapIndex:                 testOptsUseMapIndex,
    99  		UsePrefixCompress:           testOptsUsePrefixCompress,
   100  		UseBlockCompress:            testOptsUseBlockCompress,
   101  		DisableWAL:                  testOptsDisableWAL,
   102  		KeyHashFunc:                 options.TestKeyHashFunc,
   103  		KeyPrefixDeleteFunc:         options.TestKeyPrefixDeleteFunc,
   104  		KvCheckExpireFunc:           options.TestKvCheckExpireFunc,
   105  		KvTimestampFunc:             options.TestKvTimestampFunc,
   106  	}
   107  	_, err := os.Stat(dir)
   108  	if oserror.IsNotExist(err) {
   109  		if err = os.MkdirAll(dir, 0775); err != nil {
   110  			panic(err)
   111  		}
   112  	}
   113  
   114  	if optspool != nil {
   115  		opts = opts.Clone().EnsureDefaults()
   116  		opts.private.optspool = opts.ensureOptionsPool(optspool)
   117  	}
   118  
   119  	db, err := Open(dir, opts)
   120  	if err != nil {
   121  		panic(err)
   122  	}
   123  	return db
   124  }
   125  
   126  func testRandBytes(n int) []byte {
   127  	return utils.FuncRandBytes(n)
   128  }
   129  
   130  func makeTestKey(key []byte) []byte {
   131  	return sortedkv.MakeKey(key)
   132  }
   133  
   134  func makeTestVersionKey(key []byte, slotId uint16, version uint64) []byte {
   135  	return sortedkv.MakeKey2(key, slotId, version)
   136  }
   137  
   138  func makeTestIntKey(i int) []byte {
   139  	return sortedkv.MakeSlotKey([]byte(fmt.Sprintf("testkey_%d", i)), uint16(i&consts.DefaultBitowerNumMask))
   140  }
   141  
   142  func makeTestSlotIntKey(i int) []byte {
   143  	return sortedkv.MakeSlotKey([]byte(fmt.Sprintf("testkey_%d", i)), testSlotId)
   144  }
   145  
   146  func makeTestSlotKey(key []byte) []byte {
   147  	return sortedkv.MakeSlotKey(key, testSlotId)
   148  }
   149  
   150  func verifyGet(r Reader, key, val []byte) error {
   151  	v, closer, err := r.Get(key)
   152  	if err != nil {
   153  		return err
   154  	} else if !bytes.Equal(val, v) {
   155  		return errors.Errorf("key %s expected %s, but got %s", string(key), val, v)
   156  	}
   157  	if closer != nil {
   158  		closer()
   159  	}
   160  	return nil
   161  }
   162  
   163  func verifyGetNotFound(r Reader, key []byte) error {
   164  	val, _, err := r.Get(key)
   165  	if err != base.ErrNotFound {
   166  		return errors.Errorf("key %s expected nil, but got %s", string(key), val)
   167  	}
   168  	return nil
   169  }
   170  
   171  func testBitalosdbWrite(t *testing.T, num int, isSame bool) {
   172  	db := openTestDB(testDirname, nil)
   173  	keyIndex := 0
   174  	defaultLargeValBytes = testRandBytes(2048)
   175  	val := defaultLargeValBytes
   176  
   177  	for keyIndex < num {
   178  		keyIndex++
   179  		var key []byte
   180  		if isSame {
   181  			key = makeTestSlotKey([]byte(fmt.Sprintf("key_%d", keyIndex)))
   182  		} else {
   183  			key = makeTestIntKey(keyIndex)
   184  		}
   185  		require.NoError(t, db.Set(key, val, NoSync))
   186  	}
   187  
   188  	require.NoError(t, db.Flush())
   189  	require.NoError(t, db.Close())
   190  }
   191  
   192  func TestWriteMemtable(t *testing.T) {
   193  	defer os.RemoveAll(testDirname)
   194  	os.RemoveAll(testDirname)
   195  
   196  	db := openTestDB(testDirname, nil)
   197  	defer func() {
   198  		require.NoError(t, db.Close())
   199  	}()
   200  
   201  	num := 100
   202  	val := testRandBytes(2048)
   203  
   204  	for i := 0; i < num; i++ {
   205  		key := makeTestIntKey(i)
   206  		require.NoError(t, db.Set(key, val, NoSync))
   207  	}
   208  
   209  	for i := 0; i < num; i++ {
   210  		key := makeTestIntKey(i)
   211  		require.NoError(t, verifyGet(db, key, val))
   212  	}
   213  }
   214  
   215  func TestFlushDeletePercent(t *testing.T) {
   216  	defer os.RemoveAll(testDirname)
   217  	os.RemoveAll(testDirname)
   218  
   219  	testOptsMemtableSize = 3 << 20
   220  	db := openTestDB(testDirname, nil)
   221  	defer func() {
   222  		require.NoError(t, db.Close())
   223  	}()
   224  
   225  	val := testRandBytes(2048)
   226  	for i := 0; i < 1000; i++ {
   227  		key := makeTestSlotIntKey(i)
   228  		require.NoError(t, db.Set(key, val, NoSync))
   229  		if i > 400 {
   230  			require.NoError(t, db.Delete(key, NoSync))
   231  			require.NoError(t, db.Delete(key, NoSync))
   232  		}
   233  	}
   234  
   235  	time.Sleep(2 * time.Second)
   236  
   237  	for i := 0; i < 1000; i++ {
   238  		key := makeTestSlotIntKey(i)
   239  		err := verifyGet(db, key, val)
   240  		if i > 400 {
   241  			require.Equal(t, ErrNotFound, err)
   242  		} else {
   243  			require.NoError(t, err)
   244  		}
   245  	}
   246  }
   247  
   248  func TestWriteBitower(t *testing.T) {
   249  	for _, useCache := range []bool{false, true} {
   250  		t.Run(fmt.Sprintf("useCache=%t", useCache), func(t *testing.T) {
   251  			dirname := testDirname
   252  			defer os.RemoveAll(dirname)
   253  			os.RemoveAll(dirname)
   254  
   255  			if useCache {
   256  				testOptsCacheSize = 20 << 20
   257  			} else {
   258  				testOptsCacheSize = 0
   259  			}
   260  
   261  			val := testRandBytes(1024)
   262  			stepNum := 1000
   263  			startNum := 0
   264  			endNum := stepNum
   265  			seqNum := uint64(1)
   266  			hasWal := false
   267  
   268  			var bitowerMeta [consts.DefaultBitowerNum]bitowerMetaEditor
   269  			for i := 0; i < consts.DefaultBitowerNum; i++ {
   270  				bitowerMeta[i].NextFileNum = FileNum(2)
   271  				bitowerMeta[i].MinUnflushedLogNum = FileNum(1)
   272  			}
   273  
   274  			var keyList [][]byte
   275  
   276  			writeBitower := func(bdb *DB, index int) {
   277  				if hasWal {
   278  					bitowerMeta[index].MinUnflushedLogNum++
   279  					bitowerMeta[index].NextFileNum++
   280  				}
   281  
   282  				writeData := func(start, end int) {
   283  					for i := start; i < end; i++ {
   284  						key := makeTestIntKey(i)
   285  						keyList = append(keyList, key)
   286  						require.NoError(t, bdb.Set(key, val, NoSync))
   287  						seqNum++
   288  					}
   289  					bitower := bdb.bitowers[index]
   290  					require.NoError(t, bitower.Flush())
   291  
   292  					bitowerMeta[index].MinUnflushedLogNum++
   293  					bitowerMeta[index].NextFileNum++
   294  
   295  					require.Equal(t, seqNum, bdb.meta.atomic.logSeqNum)
   296  					require.Equal(t, bitowerMeta[index].MinUnflushedLogNum, bitower.mu.metaEdit.MinUnflushedLogNum)
   297  					require.Equal(t, bitowerMeta[index].NextFileNum, bitower.mu.metaEdit.NextFileNum)
   298  					require.Equal(t, seqNum, bdb.meta.atomic.logSeqNum)
   299  					require.Equal(t, false, bdb.isFlushedBitable())
   300  				}
   301  
   302  				for i := 1; i <= 3; i++ {
   303  					writeData(startNum, endNum)
   304  					startNum = endNum
   305  					endNum += stepNum
   306  				}
   307  			}
   308  
   309  			for i := 0; i < 3; i++ {
   310  				db := openTestDB(dirname, nil)
   311  				for i2 := range db.bitowers {
   312  					writeBitower(db, i2)
   313  				}
   314  
   315  				require.NoError(t, db.Close())
   316  
   317  				if i == 0 {
   318  					hasWal = true
   319  				}
   320  			}
   321  
   322  			db := openTestDB(dirname, nil)
   323  			for i := range keyList {
   324  				require.NoError(t, verifyGet(db, keyList[i], val))
   325  			}
   326  			require.NoError(t, db.Close())
   327  		})
   328  	}
   329  }
   330  
   331  func TestWriteCloseRead(t *testing.T) {
   332  	defer os.RemoveAll(testDirname)
   333  	os.RemoveAll(testDirname)
   334  
   335  	num := 1000
   336  	testBitalosdbWrite(t, num, false)
   337  
   338  	db := openTestDB(testDirname, nil)
   339  	defer func() {
   340  		require.NoError(t, db.Close())
   341  	}()
   342  
   343  	keyIndex := 0
   344  	defaultVal := defaultLargeValBytes
   345  	for keyIndex < num {
   346  		keyIndex++
   347  		key := makeTestIntKey(keyIndex)
   348  		require.NoError(t, verifyGet(db, key, defaultVal))
   349  	}
   350  }
   351  
   352  func TestWriteDelete(t *testing.T) {
   353  	defer os.RemoveAll(testDirname)
   354  	os.RemoveAll(testDirname)
   355  	db := openTestDB(testDirname, nil)
   356  	defer func() {
   357  		require.NoError(t, db.Close())
   358  	}()
   359  
   360  	val := testRandBytes(1024)
   361  
   362  	for i := 0; i < 10; i++ {
   363  		key := makeTestIntKey(i)
   364  		require.NoError(t, db.Set(key, val, NoSync))
   365  	}
   366  
   367  	for i := 0; i < 10; i++ {
   368  		key := makeTestIntKey(i)
   369  		require.NoError(t, verifyGet(db, key, val))
   370  	}
   371  
   372  	for i := 0; i < 5; i++ {
   373  		key := makeTestIntKey(i)
   374  		require.NoError(t, db.Delete(key, NoSync))
   375  	}
   376  
   377  	for i := 0; i < 5; i++ {
   378  		key := makeTestIntKey(i)
   379  		_, _, err := db.Get(key)
   380  		require.Equal(t, ErrNotFound, err)
   381  	}
   382  
   383  	for i := 5; i < 10; i++ {
   384  		key := makeTestIntKey(i)
   385  		require.NoError(t, verifyGet(db, key, val))
   386  	}
   387  }
   388  
   389  func TestLogSeqNum(t *testing.T) {
   390  	defer os.RemoveAll(testDirname)
   391  	os.RemoveAll(testDirname)
   392  	db := openTestDB(testDirname, nil)
   393  	defer func() {
   394  		require.NoError(t, db.Close())
   395  	}()
   396  
   397  	batch := db.NewBatchBitower()
   398  	key0 := makeTestSlotKey([]byte("key_0"))
   399  	key1 := makeTestSlotKey([]byte("key_1"))
   400  	val1 := makeTestSlotKey([]byte("val_1"))
   401  	key2 := makeTestSlotKey([]byte("key_2"))
   402  	val2 := makeTestSlotKey([]byte("val_2"))
   403  	require.NoError(t, batch.Set(key1, val1, NoSync))
   404  	require.NoError(t, batch.Set(key2, val2, NoSync))
   405  	require.NoError(t, batch.Commit(NoSync))
   406  	require.NoError(t, batch.Close())
   407  	require.NoError(t, db.Flush())
   408  
   409  	require.NoError(t, verifyGetNotFound(db, key0))
   410  	require.NoError(t, verifyGet(db, key1, val1))
   411  
   412  	batch = db.NewBatchBitower()
   413  	val1_1 := makeTestSlotKey([]byte("val_1_1"))
   414  	require.NoError(t, batch.Set(key1, val1_1, NoSync))
   415  	require.NoError(t, batch.Delete(key2, NoSync))
   416  	require.NoError(t, batch.Commit(NoSync))
   417  	require.NoError(t, batch.Close())
   418  
   419  	require.NoError(t, verifyGet(db, key1, val1_1))
   420  	require.NoError(t, verifyGetNotFound(db, key2))
   421  
   422  	iterOpts := &IterOptions{
   423  		SlotId: uint32(testSlotId),
   424  	}
   425  	it := db.NewIter(iterOpts)
   426  	it.First()
   427  	require.Equal(t, true, it.Valid())
   428  	require.Equal(t, key1, it.Key())
   429  	require.Equal(t, val1_1, it.Value())
   430  	it.Next()
   431  	require.Equal(t, false, it.Valid())
   432  	require.NoError(t, it.Close())
   433  }
   434  
   435  func TestSetKeyMultiValue(t *testing.T) {
   436  	defer os.RemoveAll(testDirname)
   437  	os.RemoveAll(testDirname)
   438  	db := openTestDB(testDirname, nil)
   439  	defer func() {
   440  		require.NoError(t, db.Close())
   441  	}()
   442  
   443  	key := makeTestSlotKey([]byte(fmt.Sprintf("key_1")))
   444  	for j := 0; j < 100; j++ {
   445  		v := []byte(fmt.Sprintf("%d", j))
   446  		require.NoError(t, db.Set(key, v, NoSync))
   447  	}
   448  
   449  	require.NoError(t, verifyGet(db, key, []byte("99")))
   450  	require.NoError(t, db.Flush())
   451  	require.NoError(t, verifyGet(db, key, []byte("99")))
   452  }
   453  
   454  func TestBatchSetMulti(t *testing.T) {
   455  	defer os.RemoveAll(testDirname)
   456  	os.RemoveAll(testDirname)
   457  
   458  	db := openTestDB(testDirname, nil)
   459  	defer func() {
   460  		require.NoError(t, db.Close())
   461  	}()
   462  
   463  	num := 100
   464  	kvList := make(map[string][]byte, num)
   465  	for i := 0; i < num; i++ {
   466  		b := db.NewBatchBitower()
   467  		key := makeTestSlotIntKey(i)
   468  		if i%2 == 0 {
   469  			val := testRandBytes(100)
   470  			_ = b.Set(key, val, NoSync)
   471  			kvList[unsafe2.String(key)] = val
   472  		} else {
   473  			val1 := testRandBytes(100)
   474  			val2 := testRandBytes(100)
   475  			_ = b.SetMultiValue(key, val1, val2)
   476  			var val []byte
   477  			val = append(val, val1...)
   478  			val = append(val, val2...)
   479  			kvList[unsafe2.String(key)] = val
   480  		}
   481  		require.NoError(t, b.Commit(NoSync))
   482  		require.NoError(t, b.Close())
   483  	}
   484  
   485  	for i := 0; i < 100; i++ {
   486  		key := makeTestSlotIntKey(i)
   487  		require.NoError(t, verifyGet(db, key, kvList[string(key)]))
   488  	}
   489  }
   490  
   491  func TestBithashWriteRead(t *testing.T) {
   492  	defer os.RemoveAll(testDirname)
   493  	os.RemoveAll(testDirname)
   494  
   495  	db := openTestDB(testDirname, nil)
   496  	defer func() {
   497  		require.NoError(t, db.Close())
   498  	}()
   499  
   500  	runNum := 2
   501  	type kvPair struct {
   502  		k, v []byte
   503  	}
   504  	kvList := make([][]kvPair, runNum)
   505  
   506  	wrFunc := func(i int) {
   507  		num := 50000
   508  		keyIndex := 0
   509  		var valueSize int
   510  		var isExist bool
   511  		kvList[i] = make([]kvPair, num)
   512  		for keyIndex < num {
   513  			key := makeTestIntKey(keyIndex)
   514  			if keyIndex%2 == 0 {
   515  				valueSize = 1200
   516  			} else {
   517  				valueSize = 5
   518  			}
   519  			val := testRandBytes(valueSize)
   520  			require.NoError(t, db.Set(key, val, NoSync))
   521  			kvList[i][keyIndex] = kvPair{
   522  				k: key,
   523  				v: val,
   524  			}
   525  			keyIndex++
   526  		}
   527  
   528  		require.NoError(t, db.Flush())
   529  
   530  		keyIndex = 0
   531  		for keyIndex < 100 {
   532  			key := makeTestIntKey(keyIndex)
   533  			require.NoError(t, db.Delete(key, NoSync))
   534  			keyIndex++
   535  		}
   536  
   537  		require.NoError(t, db.Flush())
   538  
   539  		for j, item := range kvList[i] {
   540  			err := verifyGet(db, item.k, item.v)
   541  			if j < 100 {
   542  				require.Equal(t, ErrNotFound, err)
   543  			} else {
   544  				require.NoError(t, err)
   545  			}
   546  
   547  			isExist, err = db.Exist(item.k)
   548  			if j < 100 {
   549  				require.Equal(t, ErrNotFound, err)
   550  				require.Equal(t, false, isExist)
   551  			} else {
   552  				require.NoError(t, err)
   553  				require.Equal(t, true, isExist)
   554  			}
   555  		}
   556  	}
   557  
   558  	wrFunc(0)
   559  	require.Equal(t, 25000, db.MetricsInfo().BithashKeyTotal)
   560  	require.Equal(t, 50, db.MetricsInfo().BithashDelKeyTotal)
   561  	time.Sleep(2 * time.Second)
   562  	wrFunc(1)
   563  	require.Equal(t, 50000, db.MetricsInfo().BithashKeyTotal)
   564  	require.Equal(t, 25050, db.MetricsInfo().BithashDelKeyTotal)
   565  	db.compactBithash(db.opts.CompactInfo.DeletePercent)
   566  }
   567  
   568  func TestBitpageWriteCloseRead(t *testing.T) {
   569  	defer os.RemoveAll(testDirname)
   570  	os.RemoveAll(testDirname)
   571  
   572  	val := testRandBytes(2048)
   573  
   574  	writeData := func(d *DB) {
   575  		keyIndex := 0
   576  		for keyIndex < 1000 {
   577  			keyIndex++
   578  			newKey := makeTestIntKey(keyIndex)
   579  			require.NoError(t, d.Set(newKey, val, NoSync))
   580  		}
   581  		require.NoError(t, d.Flush())
   582  	}
   583  
   584  	readData := func(d *DB) {
   585  		keyIndex := 0
   586  		for keyIndex < 1000 {
   587  			keyIndex++
   588  			key := makeTestIntKey(keyIndex)
   589  			require.NoError(t, verifyGet(d, key, val))
   590  
   591  			isExist, err := d.Exist(key)
   592  			require.NoError(t, err)
   593  			require.Equal(t, true, isExist)
   594  		}
   595  	}
   596  
   597  	testOptsUsePrefixCompress = false
   598  	testOptsUseBlockCompress = false
   599  	db := openTestDB(testDirname, nil)
   600  	writeData(db)
   601  	time.Sleep(2 * time.Second)
   602  	readData(db)
   603  	require.NoError(t, db.Close())
   604  
   605  	testOptsUsePrefixCompress = true
   606  	testOptsUseBlockCompress = true
   607  	db = openTestDB(testDirname, nil)
   608  	writeData(db)
   609  	time.Sleep(2 * time.Second)
   610  	readData(db)
   611  	require.NoError(t, db.Close())
   612  }
   613  
   614  func TestCacheSetGet(t *testing.T) {
   615  	for _, cacheType := range []int{consts.CacheTypeLru, consts.CacheTypeLfu} {
   616  		t.Run(fmt.Sprintf("cacheType:%d", cacheType), func(t *testing.T) {
   617  			defer os.RemoveAll(testDirname)
   618  			os.RemoveAll(testDirname)
   619  			testOptsCacheType = cacheType
   620  			testOptsCacheSize = testCacheSize
   621  			db := openTestDB(testDirname, nil)
   622  			num := 1000
   623  			keyIndex := 0
   624  			val := testRandBytes(2048)
   625  			for keyIndex < num {
   626  				key := makeTestIntKey(keyIndex)
   627  				require.NoError(t, db.Set(key, val, NoSync))
   628  				keyIndex++
   629  			}
   630  			require.NoError(t, db.Flush())
   631  
   632  			keyIndex = 0
   633  			for keyIndex < 10 {
   634  				key := makeTestIntKey(keyIndex)
   635  				require.NoError(t, db.Delete(key, NoSync))
   636  				keyIndex++
   637  			}
   638  			require.NoError(t, db.Flush())
   639  
   640  			keyIndex = 0
   641  			for keyIndex < num {
   642  				key := makeTestIntKey(keyIndex)
   643  				err := verifyGet(db, key, val)
   644  				cacheVal, cacheCloser, cacheFound := db.cache.Get(key, db.cache.GetKeyHash(key))
   645  				if keyIndex < 10 {
   646  					require.Equal(t, ErrNotFound, err)
   647  					require.Equal(t, false, cacheFound)
   648  				} else {
   649  					require.NoError(t, err)
   650  					require.Equal(t, true, cacheFound)
   651  					require.Equal(t, cacheVal, val)
   652  				}
   653  				keyIndex++
   654  				if cacheCloser != nil {
   655  					cacheCloser()
   656  				}
   657  			}
   658  			require.NoError(t, db.Close())
   659  		})
   660  	}
   661  }
   662  
   663  func TestMemFlushCheckExpire(t *testing.T) {
   664  	defer os.RemoveAll(testDirname)
   665  	os.RemoveAll(testDirname)
   666  
   667  	db := openTestDB(testDirname, nil)
   668  	defer func() {
   669  		require.NoError(t, db.Close())
   670  	}()
   671  
   672  	const randLen = 20
   673  	const valLen = randLen + 9
   674  	valBytes := testRandBytes(valLen)
   675  	now := uint64(time.Now().UnixMilli())
   676  	makeValue := func(i int) []byte {
   677  		val := make([]byte, valLen, valLen)
   678  		ttl := now
   679  		if i%2 == 0 {
   680  			ttl += 20000
   681  		}
   682  		val[0] = 1
   683  		binary.BigEndian.PutUint64(val[1:9], ttl)
   684  		copy(val[9:], valBytes)
   685  		return val[:]
   686  	}
   687  	makeKey := func(i int) []byte {
   688  		keyVersion := uint64(i/100 + 100)
   689  		key := []byte(fmt.Sprintf("key_%d", i))
   690  		return makeTestVersionKey(key, uint16(keyVersion), keyVersion)
   691  	}
   692  
   693  	count := 10000
   694  
   695  	for i := 0; i < count; i++ {
   696  		key := makeKey(i)
   697  		value := makeValue(i)
   698  		require.NoError(t, db.Set(key, value, NoSync))
   699  	}
   700  
   701  	time.Sleep(time.Second)
   702  	require.NoError(t, db.Flush())
   703  
   704  	for i := 0; i < count; i++ {
   705  		key := makeKey(i)
   706  		value := makeValue(i)
   707  		err := verifyGet(db, key, value)
   708  		if i%2 == 0 {
   709  			require.NoError(t, err)
   710  		} else {
   711  			require.Equal(t, ErrNotFound, err)
   712  		}
   713  	}
   714  }
   715  
   716  func TestMemFlushPrefixDeleteKey(t *testing.T) {
   717  	defer os.RemoveAll(testDirname)
   718  	os.RemoveAll(testDirname)
   719  
   720  	db := openTestDB(testDirname, nil)
   721  	defer func() {
   722  		require.NoError(t, db.Close())
   723  	}()
   724  
   725  	value := testRandBytes(100)
   726  	makeKey := func(i int) []byte {
   727  		keyVersion := uint64(i/100 + 100)
   728  		key := []byte(fmt.Sprintf("key_%d", i))
   729  		return makeTestVersionKey(key, uint16(keyVersion), keyVersion)
   730  	}
   731  
   732  	count := 1000
   733  	pdVersions := []uint64{102, 104, 106, 108, 110}
   734  	isPdVersion := func(n uint64) bool {
   735  		for i := range pdVersions {
   736  			if pdVersions[i] == n {
   737  				return true
   738  			}
   739  		}
   740  		return false
   741  	}
   742  
   743  	for i := 0; i < count; i++ {
   744  		key := makeKey(i)
   745  		require.NoError(t, db.Set(key, value, NoSync))
   746  	}
   747  
   748  	for _, ver := range pdVersions {
   749  		key := makeTestVersionKey(nil, uint16(ver), ver)
   750  		require.NoError(t, db.PrefixDeleteKeySet(key, NoSync))
   751  	}
   752  
   753  	require.NoError(t, db.Flush())
   754  
   755  	for i := 0; i < count; i++ {
   756  		key := makeKey(i)
   757  		err := verifyGet(db, key, value)
   758  		version := db.opts.KeyPrefixDeleteFunc(key)
   759  		if isPdVersion(version) {
   760  			require.Equal(t, ErrNotFound, err)
   761  		} else {
   762  			require.NoError(t, err)
   763  		}
   764  	}
   765  
   766  	for _, ver := range pdVersions {
   767  		key := makeTestVersionKey(nil, uint16(ver), ver)
   768  		_, vcloser, err := db.Get(key)
   769  		require.NoError(t, err)
   770  		if vcloser == nil {
   771  			t.Fatalf("delete version key vcloser is nil version:%d key:%s", ver, string(key))
   772  		}
   773  	}
   774  }
   775  
   776  func TestBitableCheckExpire(t *testing.T) {
   777  	defer os.RemoveAll(testDirname)
   778  	os.RemoveAll(testDirname)
   779  
   780  	testOptsUseBitable = true
   781  	db := openTestDB(testDirname, nil)
   782  	defer func() {
   783  		require.NoError(t, db.Close())
   784  	}()
   785  
   786  	valLen := 20 + 9
   787  	valBytes := testRandBytes(valLen)
   788  	now := uint64(time.Now().UnixMilli())
   789  	count := 10000
   790  	makeKey := func(i int) []byte {
   791  		keyVersion := uint64(i/100 + 100)
   792  		key := []byte(fmt.Sprintf("key_%d", i))
   793  		return makeTestVersionKey(key, uint16(keyVersion), keyVersion)
   794  	}
   795  	makeValue := func(i int) []byte {
   796  		val := make([]byte, valLen)
   797  		ttl := now
   798  		if i%5 == 0 {
   799  			ttl = now + 3000
   800  		} else {
   801  			ttl = now + 30000
   802  		}
   803  		val[0] = 1
   804  		binary.BigEndian.PutUint64(val[1:9], ttl)
   805  		copy(val[9:], valBytes)
   806  		return val[:]
   807  	}
   808  
   809  	writeData := func() {
   810  		for i := 0; i < count; i++ {
   811  			key := makeKey(i)
   812  			value := makeValue(i)
   813  			require.NoError(t, db.Set(key, value, NoSync))
   814  		}
   815  		require.NoError(t, db.Flush())
   816  	}
   817  
   818  	readData := func() {
   819  		for i := 0; i < count; i++ {
   820  			key := makeKey(i)
   821  			value := makeValue(i)
   822  			require.NoError(t, verifyGet(db, key, value))
   823  		}
   824  	}
   825  
   826  	readDeleteKV := func(jobId int) {
   827  		fmt.Println("readDeleteKV", jobId)
   828  		for i := 0; i < count; i++ {
   829  			key := makeKey(i)
   830  			value := makeValue(i)
   831  
   832  			isExist := true
   833  			if jobId == 2 && i%5 == 0 {
   834  				isExist = false
   835  			}
   836  
   837  			v, vcloser, err := db.Get(key)
   838  			if isExist {
   839  				require.NoError(t, err)
   840  				require.Equal(t, value, v)
   841  				vcloser()
   842  			} else {
   843  				if err != ErrNotFound || len(v) > 0 {
   844  					t.Fatal("find expire key", string(key))
   845  				}
   846  			}
   847  		}
   848  	}
   849  
   850  	writeData()
   851  	readData()
   852  	db.compactToBitable(1)
   853  	readData()
   854  	readDeleteKV(1)
   855  	writeData()
   856  	time.Sleep(3 * time.Second)
   857  	db.compactToBitable(1)
   858  	readDeleteKV(2)
   859  }