github.com/zuoyebang/bitalosdb@v1.1.1-0.20240516111551-79a8c4d8ce20/open_test.go (about)

     1  // Copyright 2021 The Bitalosdb author(hustxrb@163.com) and other contributors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bitalosdb
    16  
    17  import (
    18  	"fmt"
    19  	"os"
    20  	"sort"
    21  	"strconv"
    22  	"strings"
    23  	"testing"
    24  
    25  	"github.com/stretchr/testify/require"
    26  	"github.com/zuoyebang/bitalosdb/internal/base"
    27  	"github.com/zuoyebang/bitalosdb/internal/utils"
    28  	"github.com/zuoyebang/bitalosdb/internal/vfs"
    29  )
    30  
    31  func TestOpenCloseOpenClose(t *testing.T) {
    32  	dir := testDirname
    33  	defer os.RemoveAll(dir)
    34  	os.RemoveAll(dir)
    35  
    36  	fs := vfs.Default
    37  	opts := &Options{FS: fs}
    38  
    39  	for _, startFromEmpty := range []bool{false, true} {
    40  		for _, walDirname := range []string{"", "wal"} {
    41  			for _, length := range []int{-1, 0, 1, 1000, 10000, 100000} {
    42  				dirname := "/sharedDatabase" + walDirname
    43  				if startFromEmpty {
    44  					dirname = "/startFromEmpty" + walDirname + strconv.Itoa(length)
    45  				}
    46  				dirname = fs.PathJoin(dir, dirname)
    47  				if walDirname == "" {
    48  					opts.WALDir = ""
    49  				} else {
    50  					opts.WALDir = fs.PathJoin(dirname, walDirname)
    51  				}
    52  
    53  				got, xxx := []byte(nil), ""
    54  				if length >= 0 {
    55  					xxx = strings.Repeat("x", length)
    56  				}
    57  
    58  				d0, err := Open(dirname, opts)
    59  				if err != nil {
    60  					t.Fatalf("sfe=%t, length=%d: Open #0: %v", startFromEmpty, length, err)
    61  					continue
    62  				}
    63  				if length >= 0 {
    64  					err = d0.Set(makeTestKey([]byte("key")), []byte(xxx), nil)
    65  					if err != nil {
    66  						t.Errorf("sfe=%t, length=%d: Set: %v", startFromEmpty, length, err)
    67  						continue
    68  					}
    69  				}
    70  				err = d0.Close()
    71  				if err != nil {
    72  					t.Errorf("sfe=%t, length=%d: Close #0: %v",
    73  						startFromEmpty, length, err)
    74  					continue
    75  				}
    76  
    77  				d1, err := Open(dirname, opts)
    78  				if err != nil {
    79  					t.Errorf("sfe=%t, length=%d: Open #1: %v", startFromEmpty, length, err)
    80  					continue
    81  				}
    82  				if length >= 0 {
    83  					var closer func()
    84  					got, closer, err = d1.Get(makeTestKey([]byte("key")))
    85  					if err != nil {
    86  						t.Errorf("sfe=%t, length=%d: Get: %v", startFromEmpty, length, err)
    87  						continue
    88  					}
    89  					got = append([]byte(nil), got...)
    90  					closer()
    91  				}
    92  				err = d1.Close()
    93  				if err != nil {
    94  					t.Errorf("sfe=%t, length=%d: Close #1: %v", startFromEmpty, length, err)
    95  					continue
    96  				}
    97  
    98  				if length >= 0 && string(got) != xxx {
    99  					t.Errorf("sfe=%t, length=%d: got value differs from set value", startFromEmpty, length)
   100  					continue
   101  				}
   102  			}
   103  		}
   104  	}
   105  }
   106  
   107  func TestOpenWALReplay(t *testing.T) {
   108  	dir := testDirname
   109  	defer os.RemoveAll(dir)
   110  	os.RemoveAll(dir)
   111  
   112  	largeValue := []byte(strings.Repeat("a", 100<<10))
   113  	hugeValue := []byte(strings.Repeat("b", 1<<20))
   114  	keys := make([][]byte, 5)
   115  	for i := 0; i < 5; i++ {
   116  		keys[i] = makeTestSlotKey([]byte(fmt.Sprintf("%d", i)))
   117  	}
   118  
   119  	checkIter := func(iter *Iterator) {
   120  		t.Helper()
   121  
   122  		i := 0
   123  		for valid := iter.First(); valid; valid = iter.Next() {
   124  			require.Equal(t, keys[i], iter.Key())
   125  			i++
   126  		}
   127  		require.NoError(t, iter.Close())
   128  	}
   129  
   130  	fs := vfs.Default
   131  	d, err := Open(dir, &Options{
   132  		FS:           fs,
   133  		MemTableSize: 32 << 20,
   134  	})
   135  	require.NoError(t, err)
   136  	require.NoError(t, d.Set(keys[0], largeValue, nil))
   137  	require.NoError(t, d.Set(keys[1], largeValue, nil))
   138  	require.NoError(t, d.Set(keys[2], largeValue, nil))
   139  	require.NoError(t, d.Set(keys[3], hugeValue, nil))
   140  	require.NoError(t, d.Set(keys[4], largeValue, nil))
   141  	iterOpts := &IterOptions{SlotId: uint32(testSlotId)}
   142  	checkIter(d.NewIter(iterOpts))
   143  	require.NoError(t, d.Close())
   144  
   145  	index := base.GetBitowerIndex(int(testSlotId))
   146  	bitower := d.bitowers[index]
   147  	files, err := d.opts.FS.List(bitower.walDirname)
   148  	require.NoError(t, err)
   149  	sort.Strings(files)
   150  	logCount := 0
   151  	for _, fname := range files {
   152  		if strings.HasSuffix(fname, ".log") {
   153  			logCount++
   154  		}
   155  	}
   156  
   157  	require.Equal(t, 1, logCount)
   158  
   159  	d, err = Open(dir, &Options{
   160  		FS:           fs,
   161  		MemTableSize: 32 << 20,
   162  	})
   163  	require.NoError(t, err)
   164  
   165  	checkIter(d.NewIter(iterOpts))
   166  	require.NoError(t, d.Close())
   167  }
   168  
   169  func TestOpenWALReplay2(t *testing.T) {
   170  	dir := testDirname
   171  	defer os.RemoveAll(dir)
   172  	os.RemoveAll(dir)
   173  
   174  	for _, reason := range []string{"forced", "size"} {
   175  		t.Run(reason, func(t *testing.T) {
   176  			fs := vfs.Default
   177  			d, err := Open(dir, &Options{
   178  				FS:           fs,
   179  				MemTableSize: 1 << 20,
   180  			})
   181  			require.NoError(t, err)
   182  
   183  			switch reason {
   184  			case "forced":
   185  				require.NoError(t, d.Set(makeTestKey([]byte("1")), nil, nil))
   186  				require.NoError(t, d.Flush())
   187  				require.NoError(t, d.Set(makeTestKey([]byte("2")), nil, nil))
   188  			case "size":
   189  				largeValue := []byte(strings.Repeat("a", 100<<10))
   190  				require.NoError(t, d.Set(makeTestKey([]byte("1")), largeValue, nil))
   191  				require.NoError(t, d.Set(makeTestKey([]byte("2")), largeValue, nil))
   192  				require.NoError(t, d.Set(makeTestKey([]byte("3")), largeValue, nil))
   193  			}
   194  			require.NoError(t, d.Close())
   195  			d, err = Open(dir, &Options{
   196  				FS:           fs,
   197  				MemTableSize: 1 << 20,
   198  			})
   199  			require.NoError(t, err)
   200  			require.NoError(t, d.Close())
   201  		})
   202  	}
   203  }
   204  
   205  func TestOpenWALReplay3(t *testing.T) {
   206  	dir := testDirname
   207  	defer os.RemoveAll(dir)
   208  	os.RemoveAll(dir)
   209  
   210  	fs := vfs.Default
   211  	d, err := Open(dir, &Options{
   212  		FS:           fs,
   213  		MemTableSize: 1 << 20,
   214  	})
   215  	require.NoError(t, err)
   216  
   217  	bitower := d.bitowers[testSlotId]
   218  	bitowerWalDir := bitower.walDirname
   219  	bitower.mu.compact.flushing = true
   220  
   221  	num := 500
   222  	val := testRandBytes(1024)
   223  	for i := 0; i < num; i++ {
   224  		key := makeTestSlotIntKey(i)
   225  		require.NoError(t, d.Set(key, val, nil))
   226  	}
   227  
   228  	bitower.mu.compact.flushing = false
   229  	memNum := len(bitower.mu.mem.queue)
   230  	require.Equal(t, 5, memNum)
   231  	require.NoError(t, d.Close())
   232  
   233  	var expFns, actFns []FileNum
   234  	for i := 1; i <= memNum; i++ {
   235  		expFns = append(expFns, FileNum(i))
   236  	}
   237  	ls, _ := fs.List(bitowerWalDir)
   238  	sort.Strings(ls)
   239  	for _, filename := range ls {
   240  		if _, fn, ok := base.ParseFilename(fs, filename); ok {
   241  			actFns = append(actFns, fn)
   242  		}
   243  	}
   244  	require.Equal(t, expFns, actFns)
   245  
   246  	d, err = Open(dir, &Options{
   247  		FS:           fs,
   248  		MemTableSize: 1 << 20,
   249  	})
   250  	require.NoError(t, err)
   251  
   252  	bitower = d.bitowers[testSlotId]
   253  	require.Equal(t, FileNum(6), bitower.getMinUnflushedLogNum())
   254  	d.optspool.BaseOptions.DeleteFilePacer.Flush()
   255  
   256  	for _, fn := range actFns {
   257  		walFile := bitower.makeWalFilename(fn)
   258  		if utils.IsFileExist(walFile) {
   259  			t.Fatalf("obsolete wal exist file:%s", walFile)
   260  		}
   261  	}
   262  
   263  	require.NoError(t, d.Close())
   264  }