github.com/zuoyebang/bitalosdb@v1.1.1-0.20240516111551-79a8c4d8ce20/checkpoint_test.go (about)

     1  // Copyright 2021 The Bitalosdb author(hustxrb@163.com) and other contributors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bitalosdb
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"os"
    21  	"strings"
    22  	"sync"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/zuoyebang/bitalosdb/internal/consts"
    27  	"github.com/zuoyebang/bitalosdb/internal/options"
    28  	"github.com/zuoyebang/bitalosdb/internal/vfs"
    29  
    30  	"github.com/stretchr/testify/require"
    31  )
    32  
    33  func TestCheckpointFlush(t *testing.T) {
    34  	defer func() {
    35  		os.RemoveAll(testDirname)
    36  		os.RemoveAll("./checkpoints")
    37  	}()
    38  	os.RemoveAll(testDirname)
    39  	os.RemoveAll("./checkpoints")
    40  
    41  	testOptsCacheType = consts.CacheTypeLru
    42  	testOptsCacheSize = testCacheSize
    43  	d1 := openTestDB(testDirname, nil)
    44  
    45  	ctx, cancel := context.WithCancel(context.Background())
    46  
    47  	var wg sync.WaitGroup
    48  	wg.Add(3)
    49  	go func() {
    50  		defer cancel()
    51  		defer wg.Done()
    52  		for i := 0; ctx.Err() == nil; i++ {
    53  			if err := d1.Set(makeTestIntKey(i), nil, nil); err != nil {
    54  				t.Error(err)
    55  				return
    56  			}
    57  		}
    58  	}()
    59  	go func() {
    60  		defer cancel()
    61  		defer wg.Done()
    62  		for ctx.Err() == nil {
    63  			if err := d1.Flush(); err != nil {
    64  				t.Error(err)
    65  				return
    66  			}
    67  		}
    68  	}()
    69  
    70  	loopNum := 5
    71  	check := make([]string, loopNum)
    72  	go func() {
    73  		defer ctx.Done()
    74  		defer cancel()
    75  		defer wg.Done()
    76  		for i := 0; ctx.Err() == nil && i < loopNum; i++ {
    77  			dir := fmt.Sprintf("checkpoints/checkpoint%d", i)
    78  			if err := d1.Checkpoint(dir); err != nil {
    79  				t.Error(err)
    80  				return
    81  			}
    82  			check[i] = dir
    83  		}
    84  	}()
    85  	<-ctx.Done()
    86  	wg.Wait()
    87  	require.NoError(t, d1.Close())
    88  
    89  	for _, dir := range check {
    90  		testOptsCacheType = consts.CacheTypeLru
    91  		testOptsCacheSize = testCacheSize
    92  		d2 := openTestDB(dir, nil)
    93  		if err := d2.Close(); err != nil {
    94  			t.Error(err)
    95  		}
    96  	}
    97  }
    98  
    99  func TestCheckpointFlushWAL(t *testing.T) {
   100  	const checkpointPath = "./checkpoints/checkpoint"
   101  	defer func() {
   102  		os.RemoveAll(testDirname)
   103  		os.RemoveAll("./checkpoints")
   104  	}()
   105  	os.RemoveAll(testDirname)
   106  	os.RemoveAll("./checkpoints")
   107  
   108  	fs := vfs.Default
   109  	key := makeTestSlotKey([]byte("key"))
   110  	value := []byte("value")
   111  
   112  	{
   113  		d := openTestDB(testDirname, nil)
   114  		wb := d.NewBatch()
   115  		err := wb.Set(key, value, nil)
   116  		require.NoError(t, err)
   117  		err = d.Apply(wb, NoSync)
   118  		require.NoError(t, err)
   119  		err = d.Checkpoint(checkpointPath, WithFlushedWAL())
   120  		require.NoError(t, err)
   121  		require.NoError(t, d.Close())
   122  	}
   123  
   124  	{
   125  		files, err := fs.List(checkpointPath)
   126  		require.NoError(t, err)
   127  		hasLogFile := false
   128  		for _, f := range files {
   129  			info, err := fs.Stat(fs.PathJoin(checkpointPath, f))
   130  			require.NoError(t, err)
   131  			if strings.HasSuffix(f, ".log") {
   132  				hasLogFile = true
   133  				require.NotZero(t, info.Size())
   134  			}
   135  		}
   136  		require.True(t, hasLogFile)
   137  	}
   138  
   139  	{
   140  		d := openTestDB(testDirname, nil)
   141  		iter := d.NewIter(&options.IterOptions{SlotId: uint32(testSlotId)})
   142  		require.True(t, iter.First())
   143  		require.Equal(t, key, iter.Key())
   144  		require.Equal(t, value, iter.Value())
   145  		require.False(t, iter.Next())
   146  		require.NoError(t, iter.Close())
   147  		require.NoError(t, d.Close())
   148  	}
   149  }
   150  
   151  func TestBtreeCheckpoint(t *testing.T) {
   152  	defer os.RemoveAll(testDirname)
   153  	os.RemoveAll(testDirname)
   154  
   155  	for _, vlen := range []int{200, 2048} {
   156  		repeatLoop := 4
   157  		step := 3000
   158  		count := repeatLoop * step
   159  		seqNum := uint64(0)
   160  		kvList := testMakeSortedKVList(0, count, seqNum, vlen)
   161  		seqNum += uint64(count)
   162  
   163  		writeData := func(loop int) {
   164  			db := testOpenDB0(true)
   165  			w, err := db.newFlushWriter()
   166  			require.NoError(t, err)
   167  			start := loop * step
   168  			end := start + step
   169  			for i := start; i < end; i++ {
   170  				require.NoError(t, w.Set(*kvList[i].Key, kvList[i].Value))
   171  			}
   172  			require.NoError(t, w.Finish())
   173  
   174  			time.Sleep(2 * time.Second)
   175  
   176  			if loop == 1 {
   177  				db.CompactBitree(loop)
   178  			}
   179  
   180  			destDir := fmt.Sprintf("test_checkpoint_%d", loop)
   181  			require.NoError(t, os.RemoveAll(destDir))
   182  
   183  			require.NoError(t, os.Mkdir(destDir, 0755))
   184  
   185  			for i := range db.bitowers {
   186  				require.NoError(t, db.bitowers[i].checkpointBitree(db.opts.FS, destDir))
   187  			}
   188  
   189  			readKV := func() {
   190  				for i := start; i < end; i++ {
   191  					require.NoError(t, verifyGet(db, kvList[i].Key.UserKey, kvList[i].Value))
   192  				}
   193  			}
   194  
   195  			readKV()
   196  			require.NoError(t, db.Close())
   197  
   198  			db = testOpenDB0(true)
   199  			readKV()
   200  			require.NoError(t, db.Close())
   201  			require.NoError(t, os.RemoveAll(destDir))
   202  		}
   203  
   204  		for i := 0; i < repeatLoop; i++ {
   205  			writeData(i)
   206  		}
   207  	}
   208  }