github.com/zuoyebang/bitalosdb@v1.1.1-0.20240516111551-79a8c4d8ce20/bitree_test.go (about)

     1  // Copyright 2021 The Bitalosdb author(hustxrb@163.com) and other contributors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bitalosdb
    16  
    17  import (
    18  	"encoding/binary"
    19  	"fmt"
    20  	"math/rand"
    21  	"os"
    22  	"sort"
    23  	"sync"
    24  	"sync/atomic"
    25  	"testing"
    26  	"time"
    27  
    28  	"github.com/stretchr/testify/require"
    29  	"github.com/zuoyebang/bitalosdb/internal/base"
    30  	"github.com/zuoyebang/bitalosdb/internal/consts"
    31  	"github.com/zuoyebang/bitalosdb/internal/options"
    32  	"github.com/zuoyebang/bitalosdb/internal/sortedkv"
    33  	"github.com/zuoyebang/bitalosdb/internal/utils"
    34  )
    35  
    36  var testOptsParams = [][]bool{
    37  	{true, false, false},
    38  	{true, false, true},
    39  	{true, true, false},
    40  	{true, true, true},
    41  }
    42  
    43  func testOpenDB(useBitable bool) *DB {
    44  	optspool := options.InitDefaultsOptionsPool()
    45  	testOptsUseBitable = useBitable
    46  	optspool.BaseOptions.BitpageFlushSize = 10 << 20
    47  	optspool.BaseOptions.BitpageSplitSize = 15 << 20
    48  	optspool.BithashOptions.TableMaxSize = 10 << 20
    49  	return openTestDB(testDirname, optspool)
    50  }
    51  
    52  func testOpenDB0(useBitable bool) *DB {
    53  	optspool := options.InitDefaultsOptionsPool()
    54  	testOptsUseBitable = useBitable
    55  	optspool.BaseOptions.KvSeparateSize = 2000
    56  	optspool.BaseOptions.BitpageFlushSize = 1 << 20
    57  	optspool.BaseOptions.BitpageSplitSize = 2 << 20
    58  	return openTestDB(testDirname, optspool)
    59  }
    60  
    61  func testOpenDB1(params []bool) *DB {
    62  	optspool := options.InitDefaultsOptionsPool()
    63  	testOptsUseBitable = true
    64  	testOptsUseMapIndex = params[0]
    65  	testOptsUsePrefixCompress = params[1]
    66  	testOptsUseBlockCompress = params[2]
    67  	optspool.BaseOptions.KvSeparateSize = 2000
    68  	optspool.BaseOptions.BitpageFlushSize = 1 << 20
    69  	optspool.BaseOptions.BitpageSplitSize = 2 << 20
    70  	return openTestDB(testDirname, optspool)
    71  }
    72  
    73  func testOpenDB2(params []bool) *DB {
    74  	optspool := options.InitDefaultsOptionsPool()
    75  	testOptsUseBitable = true
    76  	testOptsUseMapIndex = params[0]
    77  	testOptsUsePrefixCompress = params[1]
    78  	testOptsUseBlockCompress = params[2]
    79  	optspool.BaseOptions.UseBitable = true
    80  	optspool.BaseOptions.BitpageFlushSize = 10 << 20
    81  	optspool.BaseOptions.BitpageSplitSize = 15 << 20
    82  	optspool.BithashOptions.TableMaxSize = 10 << 20
    83  	return openTestDB(testDirname, optspool)
    84  }
    85  
    86  func testMakeSortedKey(i int) []byte {
    87  	return sortedkv.MakeSortedKey(i)
    88  }
    89  
    90  func testMakeSortedKVList(start, end int, seqNum uint64, vsize int) sortedkv.SortedKVList {
    91  	return sortedkv.MakeSortedKVList(start, end, seqNum, vsize)
    92  }
    93  
    94  func testMakeSortedKV2List(start, end int, seqNum uint64, vsize int) sortedkv.SortedKVList {
    95  	return sortedkv.MakeSortedKV2List(start, end, seqNum, vsize)
    96  }
    97  
    98  func testMakeSlotSortedKVList(start, end int, seqNum uint64, vsize int, slotId uint16) sortedkv.SortedKVList {
    99  	return sortedkv.MakeSlotSortedKVList(start, end, seqNum, vsize, slotId)
   100  }
   101  
   102  func TestBitowerWriterFlush(t *testing.T) {
   103  	defer os.RemoveAll(testDirname)
   104  	os.RemoveAll(testDirname)
   105  
   106  	db := testOpenDB(false)
   107  	defer func() {
   108  		require.NoError(t, db.Close())
   109  	}()
   110  	seqNum := uint64(0)
   111  
   112  	for i := range db.bitowers {
   113  		largeValue := utils.FuncRandBytes(520)
   114  		smallValue := utils.FuncRandBytes(500)
   115  		keyCount := 100
   116  		kvList := testMakeSlotSortedKVList(0, keyCount, seqNum, 1, uint16(i))
   117  		bw, err := db.bitowers[i].newFlushWriter()
   118  		require.NoError(t, err)
   119  		seqNum += uint64(keyCount)
   120  
   121  		for j := 0; j < keyCount; j++ {
   122  			if j%2 == 0 {
   123  				kvList[j].Value = smallValue
   124  			} else {
   125  				kvList[j].Value = largeValue
   126  			}
   127  			require.NoError(t, bw.Set(*kvList[j].Key, kvList[j].Value))
   128  		}
   129  		require.NoError(t, bw.Finish())
   130  
   131  		for j := 0; j < keyCount; j++ {
   132  			require.NoError(t, verifyGet(db, kvList[i].Key.UserKey, kvList[i].Value))
   133  		}
   134  	}
   135  }
   136  
   137  func TestDbWriterFlush(t *testing.T) {
   138  	defer os.RemoveAll(testDirname)
   139  	os.RemoveAll(testDirname)
   140  
   141  	db := testOpenDB(false)
   142  	w, err := db.newFlushWriter()
   143  	require.NoError(t, err)
   144  	largeValue := utils.FuncRandBytes(520)
   145  	smallValue := utils.FuncRandBytes(500)
   146  	keyCount := 100
   147  	seqNum := uint64(0)
   148  	kvList := testMakeSortedKVList(0, keyCount, seqNum, 1)
   149  	seqNum += uint64(keyCount)
   150  
   151  	for i := 0; i < keyCount; i++ {
   152  		if i%2 == 0 {
   153  			kvList[i].Value = smallValue
   154  		} else {
   155  			kvList[i].Value = largeValue
   156  		}
   157  		require.NoError(t, w.Set(*kvList[i].Key, kvList[i].Value))
   158  	}
   159  	require.NoError(t, w.Finish())
   160  
   161  	for i := 0; i < keyCount; i++ {
   162  		require.NoError(t, verifyGet(db, kvList[i].Key.UserKey, kvList[i].Value))
   163  	}
   164  
   165  	require.NoError(t, db.Close())
   166  }
   167  
   168  func testcase(caseFunc func([]bool)) {
   169  	for _, params := range testOptsParams {
   170  		fmt.Printf("testcase params:%v\n", params)
   171  		caseFunc(params)
   172  	}
   173  }
   174  
   175  func TestBitreeWrite(t *testing.T) {
   176  	testcase(func(params []bool) {
   177  		defer os.RemoveAll(testDirname)
   178  		os.RemoveAll(testDirname)
   179  		db := testOpenDB1(params)
   180  		loop := 6
   181  		step := 3000
   182  		count := loop * step
   183  		seqNum := uint64(0)
   184  		kvList := testMakeSlotSortedKVList(0, count, seqNum, 1024, testSlotId)
   185  		index := base.GetBitowerIndex(int(testSlotId))
   186  		bitower := db.bitowers[index]
   187  		seqNum += uint64(count)
   188  
   189  		writeData := func(index int) {
   190  			w, err := bitower.newFlushWriter()
   191  			require.NoError(t, err)
   192  			start := index * step
   193  			end := start + step
   194  			for i := start; i < end; i++ {
   195  				require.NoError(t, w.Set(*kvList[i].Key, kvList[i].Value))
   196  			}
   197  			require.NoError(t, w.Finish())
   198  
   199  			time.Sleep(2 * time.Second)
   200  			for i := start; i < end; i++ {
   201  				require.NoError(t, verifyGet(db, kvList[i].Key.UserKey, kvList[i].Value))
   202  			}
   203  		}
   204  
   205  		for i := 0; i < loop; i++ {
   206  			writeData(i)
   207  		}
   208  
   209  		bdbIter := bitower.btree.NewBdbIter()
   210  		i := 0
   211  		for ik, iv := bdbIter.First(); ik != nil; ik, iv = bdbIter.Next() {
   212  			fmt.Println("bdbIter", i, ik.String(), utils.BytesToUint32(iv))
   213  			i++
   214  		}
   215  		require.Equal(t, 13, i)
   216  		require.NoError(t, bdbIter.Close())
   217  
   218  		require.Equal(t, uint64(6), db.dbState.GetBitpageFlushCount())
   219  		require.Equal(t, uint64(6), db.dbState.GetBitpageSplitCount())
   220  		require.NoError(t, db.Close())
   221  	})
   222  }
   223  
   224  func TestBitreeSetPrefixDeleteKey(t *testing.T) {
   225  	defer os.RemoveAll(testDirname)
   226  	os.RemoveAll(testDirname)
   227  	db := testOpenDB0(false)
   228  
   229  	loop := 20
   230  	step := 2000
   231  	kvNum := loop * step
   232  	seqNum := uint64(1)
   233  	kvList := sortedkv.MakeSortedSamePrefixDeleteKVList2(0, kvNum, seqNum, 1024, testSlotId, 1500)
   234  	seqNum += uint64(kvNum + 1)
   235  	kvList2 := sortedkv.MakeSortedSameVersionKVList(0, 2000, seqNum, 1000000, 1024, testSlotId)
   236  	index := base.GetBitowerIndex(int(testSlotId))
   237  	bitower := db.bitowers[index]
   238  
   239  	writeData := func(index int) {
   240  		w, err := bitower.newFlushWriter()
   241  		require.NoError(t, err)
   242  		start := index * step
   243  		end := start + step
   244  		for i := start; i < end; i++ {
   245  			require.NoError(t, w.Set(*kvList[i].Key, kvList[i].Value))
   246  		}
   247  		require.NoError(t, w.Finish())
   248  		time.Sleep(1 * time.Second)
   249  	}
   250  
   251  	for i := 0; i < loop; i++ {
   252  		writeData(i)
   253  	}
   254  
   255  	pdVersions := []uint64{102, 108, 112, 120, 126, 130}
   256  	isPdVersion := func(n uint64) bool {
   257  		for i := range pdVersions {
   258  			if pdVersions[i] == n {
   259  				return true
   260  			}
   261  		}
   262  		return false
   263  	}
   264  
   265  	for _, v := range pdVersions {
   266  		seqNum++
   267  		pdKey := sortedkv.MakeKey2(nil, testSlotId, v)
   268  		pdIkey := base.MakeInternalKey(pdKey, seqNum, base.InternalKeyKindPrefixDelete)
   269  		kvList2 = append(kvList2, sortedkv.SortedKVItem{
   270  			Key:   &pdIkey,
   271  			Value: nil,
   272  		})
   273  	}
   274  	sort.Sort(kvList2)
   275  	w, err := bitower.newFlushWriter()
   276  	require.NoError(t, err)
   277  	for i := range kvList2 {
   278  		require.NoError(t, w.Set(*kvList2[i].Key, kvList2[i].Value))
   279  	}
   280  	require.NoError(t, w.Finish())
   281  	time.Sleep(1 * time.Second)
   282  
   283  	bitower.btree.ManualFlushBitpage()
   284  	time.Sleep(3 * time.Second)
   285  
   286  	for i := 0; i < kvNum; i++ {
   287  		key := kvList[i].Key.UserKey
   288  		v, vcloser, err := db.Get(key)
   289  		ver := db.opts.KeyPrefixDeleteFunc(key)
   290  		if isPdVersion(ver) {
   291  			require.Equal(t, ErrNotFound, err)
   292  		} else {
   293  			require.Equal(t, nil, err)
   294  			require.Equal(t, kvList[i].Value, v)
   295  			vcloser()
   296  		}
   297  	}
   298  
   299  	require.NoError(t, db.Close())
   300  }
   301  
   302  func TestBitreeKeyExpire(t *testing.T) {
   303  	defer os.RemoveAll(testDirname)
   304  	os.RemoveAll(testDirname)
   305  
   306  	db := testOpenDB0(true)
   307  	defer func() {
   308  		require.NoError(t, db.Close())
   309  	}()
   310  	count := 30000
   311  	seqNum := uint64(0)
   312  	kvList := testMakeSortedKV2List(0, count, seqNum, 1)
   313  	seqNum += uint64(count)
   314  	bithashCount := 0
   315  	bithashDelCount1 := 0
   316  	bithashDelCount := 0
   317  	smallValBytes := utils.FuncRandBytes(1900)
   318  	largeValBytes := utils.FuncRandBytes(2100)
   319  	now := uint64(time.Now().UnixMilli())
   320  
   321  	makeValue := func(i int, valBytes []byte) []byte {
   322  		var val []byte
   323  		var ttl uint64
   324  		if i%5 == 0 {
   325  			if i%15 == 0 {
   326  				ttl = now + 5000
   327  			} else {
   328  				ttl = now - 1000
   329  			}
   330  			val = make([]byte, len(valBytes)+9)
   331  			val[0] = 1
   332  			binary.BigEndian.PutUint64(val[1:9], ttl)
   333  			copy(val[9:], valBytes)
   334  		} else {
   335  			ttl = now + 100000
   336  			if i%3 == 0 {
   337  				val = make([]byte, len(valBytes)+1)
   338  				val[0] = 2
   339  				copy(val[1:], valBytes)
   340  			} else {
   341  				val = make([]byte, len(valBytes)+9)
   342  				val[0] = 1
   343  				if i%7 == 0 {
   344  					ttl = 0
   345  				}
   346  				binary.BigEndian.PutUint64(val[1:9], ttl)
   347  				copy(val[9:], valBytes)
   348  			}
   349  		}
   350  
   351  		return val
   352  	}
   353  
   354  	w, err := db.newFlushWriter()
   355  	require.NoError(t, err)
   356  
   357  	var value []byte
   358  	for i := 0; i < count; i++ {
   359  		if i%2 == 0 {
   360  			value = makeValue(i, largeValBytes)
   361  			bithashCount++
   362  			if i%5 == 0 {
   363  				bithashDelCount++
   364  				if i%15 != 0 {
   365  					bithashDelCount1++
   366  				}
   367  			}
   368  		} else {
   369  			value = makeValue(i, smallValBytes)
   370  		}
   371  		kvList[i].Value = value
   372  		require.NoError(t, w.Set(*kvList[i].Key, kvList[i].Value))
   373  	}
   374  	require.NoError(t, w.Finish())
   375  	time.Sleep(4 * time.Second)
   376  
   377  	for i := 0; i < count; i++ {
   378  		k := kvList[i].Key.UserKey
   379  		v, vcloser, err := db.Get(k)
   380  		if i%5 == 0 && i%15 != 0 {
   381  			require.Equal(t, ErrNotFound, err)
   382  		} else {
   383  			require.Equal(t, nil, err)
   384  			require.Equal(t, kvList[i].Value, v)
   385  			vcloser()
   386  		}
   387  	}
   388  
   389  	stats := db.MetricsInfo()
   390  	require.Equal(t, bithashCount, stats.BithashKeyTotal)
   391  	require.Equal(t, bithashDelCount1, stats.BithashDelKeyTotal)
   392  
   393  	time.Sleep(5 * time.Second)
   394  	db.compactToBitable(1)
   395  
   396  	stats = db.MetricsInfo()
   397  	require.Equal(t, bithashCount, stats.BithashKeyTotal)
   398  	require.Equal(t, bithashDelCount, stats.BithashDelKeyTotal)
   399  
   400  	for i := 0; i < count; i++ {
   401  		k := kvList[i].Key.UserKey
   402  		v, vcloser, err := db.Get(k)
   403  		if i%5 == 0 {
   404  			require.Equal(t, ErrNotFound, err)
   405  			if v != nil || vcloser != nil {
   406  				t.Fatal("find expire key return not nil", string(k), i, v)
   407  			}
   408  		} else {
   409  			require.Equal(t, nil, err)
   410  			require.Equal(t, kvList[i].Value, v)
   411  			vcloser()
   412  		}
   413  	}
   414  }
   415  
   416  func TestBitreeCheckpointLock(t *testing.T) {
   417  	defer os.RemoveAll(testDirname)
   418  	os.RemoveAll(testDirname)
   419  	db := testOpenDB0(true)
   420  	defer func() {
   421  		require.NoError(t, db.Close())
   422  	}()
   423  	seqNum := uint64(0)
   424  	loop := 6
   425  	step := 3000
   426  	count := (loop + 1) * step
   427  	kvList := testMakeSlotSortedKVList(0, count, seqNum, 1024, testSlotId)
   428  	index := base.GetBitowerIndex(int(testSlotId))
   429  	bitower := db.bitowers[index]
   430  	seqNum += uint64(count)
   431  
   432  	writeData := func(index int) {
   433  		w, err := bitower.newFlushWriter()
   434  		require.NoError(t, err)
   435  		start := index * step
   436  		end := start + step
   437  		for i := start; i < end; i++ {
   438  			require.NoError(t, w.Set(*kvList[i].Key, kvList[i].Value))
   439  		}
   440  		require.NoError(t, w.Finish())
   441  
   442  		for i := start; i < end; i++ {
   443  			require.NoError(t, verifyGet(db, kvList[i].Key.UserKey, kvList[i].Value))
   444  		}
   445  
   446  		time.Sleep(2 * time.Second)
   447  
   448  		if index == 2 {
   449  			db.dbState.SetDbHighPriority(true)
   450  		} else if index > 2 && index <= 5 {
   451  			require.Equal(t, uint64(3), db.dbState.GetBitpageFlushCount())
   452  			require.Equal(t, uint64(3), db.dbState.GetBitpageSplitCount())
   453  			if index == 5 {
   454  				db.dbState.SetDbHighPriority(false)
   455  			}
   456  		}
   457  	}
   458  
   459  	for i := 0; i < loop; i++ {
   460  		writeData(i)
   461  	}
   462  
   463  	time.Sleep(3 * time.Second)
   464  	require.Equal(t, uint64(4), db.dbState.GetBitpageFlushCount())
   465  	require.Equal(t, uint64(4), db.dbState.GetBitpageSplitCount())
   466  
   467  	writeData(loop)
   468  	time.Sleep(2 * time.Second)
   469  	require.Equal(t, uint64(5), db.dbState.GetBitpageFlushCount())
   470  	require.Equal(t, uint64(5), db.dbState.GetBitpageSplitCount())
   471  
   472  	for i := 0; i < count; i++ {
   473  		require.NoError(t, verifyGet(db, kvList[i].Key.UserKey, kvList[i].Value))
   474  	}
   475  }
   476  
   477  func TestBitableDeleteExpire(t *testing.T) {
   478  	defer os.RemoveAll(testDirname)
   479  	os.RemoveAll(testDirname)
   480  
   481  	for _, vlen := range []int{200, 2048} {
   482  		fmt.Println("vlen=", vlen)
   483  		db := testOpenDB(true)
   484  		now := uint64(time.Now().UnixMilli())
   485  		makeValue := func(i int, valBytes []byte) []byte {
   486  			var val []byte
   487  			var ttl uint64
   488  			if i%5 == 0 {
   489  				ttl = now + 3000
   490  			} else {
   491  				ttl = now + 30000
   492  			}
   493  			val = make([]byte, len(valBytes)+9)
   494  			val[0] = 1
   495  			binary.BigEndian.PutUint64(val[1:9], ttl)
   496  			copy(val[9:], valBytes)
   497  			return val
   498  		}
   499  
   500  		seqNum := uint64(0)
   501  		count := 100
   502  		kvList := testMakeSortedKVList(0, count, seqNum, vlen)
   503  		seqNum += uint64(count)
   504  
   505  		bw, err := db.newFlushWriter()
   506  		require.NoError(t, err)
   507  		for i := 0; i < count; i++ {
   508  			kvList[i].Value = makeValue(i, kvList[i].Value)
   509  			require.NoError(t, bw.Set(*kvList[i].Key, kvList[i].Value))
   510  		}
   511  		require.NoError(t, bw.Finish())
   512  
   513  		readKV := func() {
   514  			for i := 0; i < count; i++ {
   515  				require.NoError(t, verifyGet(db, kvList[i].Key.UserKey, kvList[i].Value))
   516  			}
   517  		}
   518  
   519  		readKV()
   520  		db.compactToBitable(1)
   521  		readKV()
   522  
   523  		bw, err = db.newFlushWriter()
   524  		require.NoError(t, err)
   525  		for i := 0; i < count; i++ {
   526  			if i%2 == 0 {
   527  				kvList[i].Key.SetKind(base.InternalKeyKindDelete)
   528  				kvList[i].Value = []byte(nil)
   529  			} else {
   530  				kvList[i].Key.SetKind(base.InternalKeyKindSet)
   531  				kvList[i].Value = makeValue(i, kvList[i].Value)
   532  			}
   533  			kvList[i].Key.SetSeqNum(seqNum)
   534  			seqNum++
   535  			require.NoError(t, bw.Set(*kvList[i].Key, kvList[i].Value))
   536  		}
   537  		require.NoError(t, bw.Finish())
   538  
   539  		readDeleteKV := func(jobId int) {
   540  			fmt.Println("readDeleteKV", jobId)
   541  			for i := 0; i < count; i++ {
   542  				v, vcloser, err := db.Get(kvList[i].Key.UserKey)
   543  				if jobId == 2 {
   544  					if i%2 == 0 || i%5 == 0 {
   545  						if err != ErrNotFound {
   546  							t.Fatalf("find expire key:%s", kvList[i].Key.String())
   547  						}
   548  					} else {
   549  						require.Equal(t, nil, err)
   550  						require.Equal(t, kvList[i].Value, v)
   551  						vcloser()
   552  					}
   553  				} else {
   554  					if i%2 == 0 {
   555  						require.Equal(t, ErrNotFound, err)
   556  					} else {
   557  						require.Equal(t, nil, err)
   558  						require.Equal(t, kvList[i].Value, v)
   559  						vcloser()
   560  					}
   561  				}
   562  			}
   563  		}
   564  
   565  		readDeleteKV(1)
   566  		time.Sleep(4 * time.Second)
   567  		db.compactToBitable(2)
   568  		readDeleteKV(2)
   569  
   570  		require.NoError(t, db.Close())
   571  	}
   572  }
   573  
   574  func TestBitableExpire(t *testing.T) {
   575  	defer os.RemoveAll(testDirname)
   576  	os.RemoveAll(testDirname)
   577  
   578  	for _, vlen := range []int{200, 2048} {
   579  		isBithash := vlen > consts.KvSeparateSize
   580  		bithashDelCount := 0
   581  		db := testOpenDB(true)
   582  
   583  		seqNum := uint64(0)
   584  		count := 1000
   585  		kvList := testMakeSortedKV2List(0, count, seqNum, vlen)
   586  		seqNum += uint64(count)
   587  		now := uint64(time.Now().UnixMilli())
   588  
   589  		makeValue := func(i int, valBytes []byte) []byte {
   590  			var val []byte
   591  			var ttl uint64
   592  			if i%5 == 0 {
   593  				ttl = now + 4000
   594  			} else {
   595  				ttl = now + 30000
   596  			}
   597  			val = make([]byte, len(valBytes)+9)
   598  			val[0] = 1
   599  			binary.BigEndian.PutUint64(val[1:9], ttl)
   600  			copy(val[9:], valBytes)
   601  			return val
   602  		}
   603  
   604  		writeData := func(stat bool) {
   605  			w, err := db.newFlushWriter()
   606  			require.NoError(t, err)
   607  			for i := 0; i < count; i++ {
   608  				kvList[i].Key.SetKind(base.InternalKeyKindSet)
   609  				kvList[i].Value = makeValue(i, kvList[i].Value)
   610  				require.NoError(t, w.Set(*kvList[i].Key, kvList[i].Value))
   611  
   612  				if isBithash && i%5 == 0 {
   613  					bithashDelCount++
   614  				}
   615  			}
   616  			require.NoError(t, w.Finish())
   617  		}
   618  
   619  		readKV := func(deleted bool) {
   620  			for i := 0; i < count; i++ {
   621  				v, vcloser, err := db.Get(kvList[i].Key.UserKey)
   622  				if deleted && i%5 == 0 {
   623  					if err != ErrNotFound {
   624  						t.Fatalf("find expire key:%s", kvList[i].Key.String())
   625  					}
   626  				} else {
   627  					require.Equal(t, nil, err)
   628  					require.Equal(t, kvList[i].Value, v)
   629  					vcloser()
   630  				}
   631  			}
   632  		}
   633  
   634  		writeData(false)
   635  		readKV(false)
   636  		db.compactToBitable(1)
   637  		readKV(false)
   638  
   639  		time.Sleep(5 * time.Second)
   640  
   641  		w, err := db.newFlushWriter()
   642  		require.NoError(t, err)
   643  		seqNum += uint64(count)
   644  		for i := 0; i < count; i++ {
   645  			if i%5 == 0 {
   646  				continue
   647  			}
   648  			kvList[i].Value = makeValue(i, kvList[i].Value)
   649  			require.NoError(t, w.Set(*kvList[i].Key, kvList[i].Value))
   650  		}
   651  		require.NoError(t, w.Finish())
   652  		db.compactToBitable(2)
   653  		if isBithash {
   654  			require.Equal(t, bithashDelCount, db.MetricsInfo().BithashDelKeyTotal)
   655  		}
   656  		readKV(true)
   657  
   658  		require.NoError(t, db.Close())
   659  	}
   660  }
   661  
   662  func TestBitpageSplitEmptyPage(t *testing.T) {
   663  	defer os.RemoveAll(testDirname)
   664  	os.RemoveAll(testDirname)
   665  
   666  	db := testOpenDB(false)
   667  	defer func() {
   668  		require.NoError(t, db.Close())
   669  	}()
   670  	seqNum := uint64(1)
   671  	largeValue := utils.FuncRandBytes(600)
   672  	smallValue := utils.FuncRandBytes(500)
   673  	count := uint64(1000000)
   674  	var writeIndex atomic.Uint64
   675  	step := uint64(100000)
   676  
   677  	write := func() bool {
   678  		db.dbState.LockDbWrite()
   679  		defer db.dbState.UnlockDbWrite()
   680  
   681  		i := writeIndex.Load()
   682  		if i >= count {
   683  			return true
   684  		}
   685  		w, err := db.newFlushWriter()
   686  		require.NoError(t, err)
   687  		start := i
   688  		end := start + step
   689  		kvList := testMakeSortedKVList(int(start), int(end), seqNum, 1)
   690  		seqNum += step
   691  		for j := 0; j < int(step); j++ {
   692  			n := sortedkv.ParseSortedKeyInt(kvList[j].Key.UserKey)
   693  			if n%100 == 0 {
   694  				kvList[j].Key.SetKind(base.InternalKeyKindDelete)
   695  				kvList[j].Value = []byte(nil)
   696  			} else {
   697  				if n%2 == 0 {
   698  					kvList[j].Value = smallValue
   699  				} else {
   700  					kvList[j].Value = largeValue
   701  				}
   702  			}
   703  			require.NoError(t, w.Set(*kvList[j].Key, kvList[j].Value))
   704  		}
   705  		require.NoError(t, w.Finish())
   706  		writeIndex.Store(end)
   707  		return false
   708  	}
   709  
   710  	for {
   711  		if write() {
   712  			break
   713  		}
   714  	}
   715  
   716  	read := func(ri int) {
   717  		key := testMakeSortedKey(ri)
   718  		v, vcloser, err := db.Get(key)
   719  		if ri%100 == 0 {
   720  			if err == nil {
   721  				t.Fatalf("read del key found key:%s ri:%d", string(key), ri)
   722  			}
   723  			if vcloser != nil {
   724  				t.Fatalf("vcloser is not nil key:%s", string(key))
   725  			}
   726  		} else {
   727  			require.Equal(t, nil, err)
   728  			if ri%2 == 0 {
   729  				require.Equal(t, smallValue, v)
   730  			} else {
   731  				require.Equal(t, largeValue, v)
   732  			}
   733  			vcloser()
   734  		}
   735  	}
   736  
   737  	jend := int(writeIndex.Load())
   738  	for j := 0; j < jend; j++ {
   739  		read(j)
   740  	}
   741  }
   742  
   743  func TestBitreeWriteConcurrentRead(t *testing.T) {
   744  	testcase(func(params []bool) {
   745  		defer os.RemoveAll(testDirname)
   746  		os.RemoveAll(testDirname)
   747  
   748  		db := testOpenDB2(params)
   749  
   750  		wd := func() {
   751  			seqNum := uint64(1)
   752  			largeValue := utils.FuncRandBytes(consts.KvSeparateSize * 2)
   753  			smallValue := utils.FuncRandBytes(consts.KvSeparateSize / 2)
   754  			count := uint64(5000000)
   755  			var writeIndex atomic.Uint64
   756  			closeCh := make(chan struct{})
   757  			step := uint64(100000)
   758  
   759  			write := func() bool {
   760  				db.dbState.LockDbWrite()
   761  				defer db.dbState.UnlockDbWrite()
   762  
   763  				i := writeIndex.Load()
   764  				if i >= count {
   765  					return true
   766  				}
   767  				w, err := db.newFlushWriter()
   768  				require.NoError(t, err)
   769  				start := i
   770  				end := start + step
   771  				kvList := testMakeSortedKVList(int(start), int(end), seqNum, 1)
   772  				seqNum += step
   773  				for j := 0; j < int(step); j++ {
   774  					n := sortedkv.ParseSortedKeyInt(kvList[j].Key.UserKey)
   775  					if n%100 == 0 {
   776  						kvList[j].Key.SetKind(base.InternalKeyKindDelete)
   777  						kvList[j].Value = []byte(nil)
   778  					} else {
   779  						if n%2 == 0 {
   780  							kvList[j].Value = smallValue
   781  						} else {
   782  							kvList[j].Value = largeValue
   783  						}
   784  					}
   785  					require.NoError(t, w.Set(*kvList[j].Key, kvList[j].Value))
   786  				}
   787  				require.NoError(t, w.Finish())
   788  				writeIndex.Store(end)
   789  				return false
   790  			}
   791  
   792  			read := func(ri int) {
   793  				key := testMakeSortedKey(ri)
   794  				v, vcloser, err := db.Get(key)
   795  				if ri%100 == 0 {
   796  					if err == nil {
   797  						t.Fatalf("read del key found key:%s ri:%d", string(key), ri)
   798  					}
   799  					if vcloser != nil {
   800  						t.Fatalf("vcloser is not nil key:%s", string(key))
   801  					}
   802  				} else {
   803  					if err == ErrNotFound {
   804  						t.Fatalf("read exist key not found key:%s ri:%d wi:%d", string(key), ri, writeIndex.Load())
   805  					}
   806  					require.Equal(t, nil, err)
   807  					if ri%2 == 0 {
   808  						require.Equal(t, smallValue, v)
   809  					} else {
   810  						require.Equal(t, largeValue, v)
   811  					}
   812  					vcloser()
   813  				}
   814  			}
   815  
   816  			wg := sync.WaitGroup{}
   817  			wg.Add(2)
   818  			go func() {
   819  				defer wg.Done()
   820  
   821  				for {
   822  					select {
   823  					case <-closeCh:
   824  						return
   825  					default:
   826  						if write() {
   827  							return
   828  						}
   829  					}
   830  				}
   831  			}()
   832  
   833  			go func() {
   834  				defer wg.Done()
   835  				rwg := sync.WaitGroup{}
   836  
   837  				for i := 0; i < 3; i++ {
   838  					rwg.Add(1)
   839  					go func(index int) {
   840  						defer rwg.Done()
   841  						rn := 0
   842  						for {
   843  							select {
   844  							case <-closeCh:
   845  								return
   846  							default:
   847  								wi := int(writeIndex.Load())
   848  								if wi < 10 {
   849  									time.Sleep(2 * time.Second)
   850  									continue
   851  								}
   852  								ri := rand.Intn(wi - 2)
   853  								read(ri)
   854  								rn++
   855  								if rn%500000 == 0 {
   856  									fmt.Println("read ok", index, rn)
   857  								}
   858  							}
   859  						}
   860  					}(i)
   861  				}
   862  				rwg.Wait()
   863  			}()
   864  
   865  			time.Sleep(60 * time.Second)
   866  			close(closeCh)
   867  			wg.Wait()
   868  
   869  			jend := int(writeIndex.Load())
   870  			for j := 0; j < jend; j++ {
   871  				read(j)
   872  			}
   873  		}
   874  
   875  		wd()
   876  
   877  		require.NoError(t, db.Close())
   878  	})
   879  }