github.com/zuoyebang/bitalostable@v1.0.1-0.20240229032404-e3b99a834294/checkpoint_test.go (about)

     1  // Copyright 2019 The LevelDB-Go and Pebble and Bitalostored Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package bitalostable
     6  
     7  import (
     8  	"context"
     9  	"fmt"
    10  	"sort"
    11  	"strings"
    12  	"sync"
    13  	"testing"
    14  
    15  	"github.com/stretchr/testify/require"
    16  	"github.com/zuoyebang/bitalostable/internal/base"
    17  	"github.com/zuoyebang/bitalostable/internal/datadriven"
    18  	"github.com/zuoyebang/bitalostable/vfs"
    19  )
    20  
    21  func TestCheckpoint(t *testing.T) {
    22  	dbs := make(map[string]*DB)
    23  	defer func() {
    24  		for _, db := range dbs {
    25  			require.NoError(t, db.Close())
    26  		}
    27  	}()
    28  
    29  	var buf syncedBuffer
    30  	mem := vfs.NewMem()
    31  	opts := &Options{
    32  		FS:                    loggingFS{mem, &buf},
    33  		FormatMajorVersion:    FormatNewest,
    34  		L0CompactionThreshold: 10,
    35  	}
    36  
    37  	datadriven.RunTest(t, "testdata/checkpoint", func(td *datadriven.TestData) string {
    38  		switch td.Cmd {
    39  		case "batch":
    40  			if len(td.CmdArgs) != 1 {
    41  				return "batch <db>"
    42  			}
    43  			buf.Reset()
    44  			d := dbs[td.CmdArgs[0].String()]
    45  			b := d.NewBatch()
    46  			if err := runBatchDefineCmd(td, b); err != nil {
    47  				return err.Error()
    48  			}
    49  			if err := b.Commit(Sync); err != nil {
    50  				return err.Error()
    51  			}
    52  			return buf.String()
    53  
    54  		case "checkpoint":
    55  			if len(td.CmdArgs) != 2 {
    56  				return "checkpoint <db> <dir>"
    57  			}
    58  			buf.Reset()
    59  			d := dbs[td.CmdArgs[0].String()]
    60  			if err := d.Checkpoint(td.CmdArgs[1].String()); err != nil {
    61  				return err.Error()
    62  			}
    63  			return buf.String()
    64  
    65  		case "compact":
    66  			if len(td.CmdArgs) != 1 {
    67  				return "compact <db>"
    68  			}
    69  			buf.Reset()
    70  			d := dbs[td.CmdArgs[0].String()]
    71  			if err := d.Compact(nil, []byte("\xff"), false); err != nil {
    72  				return err.Error()
    73  			}
    74  			return buf.String()
    75  
    76  		case "flush":
    77  			if len(td.CmdArgs) != 1 {
    78  				return "flush <db>"
    79  			}
    80  			buf.Reset()
    81  			d := dbs[td.CmdArgs[0].String()]
    82  			if err := d.Flush(); err != nil {
    83  				return err.Error()
    84  			}
    85  			return buf.String()
    86  
    87  		case "list":
    88  			if len(td.CmdArgs) != 1 {
    89  				return "list <dir>"
    90  			}
    91  			paths, err := mem.List(td.CmdArgs[0].String())
    92  			if err != nil {
    93  				return err.Error()
    94  			}
    95  			sort.Strings(paths)
    96  			buf.Reset()
    97  			fmt.Fprintf(&buf, "%s\n", strings.Join(paths, "\n"))
    98  			return buf.String()
    99  
   100  		case "open":
   101  			if len(td.CmdArgs) != 1 && len(td.CmdArgs) != 2 {
   102  				return "open <dir> [readonly]"
   103  			}
   104  			opts.ReadOnly = false
   105  			if len(td.CmdArgs) == 2 {
   106  				if td.CmdArgs[1].String() != "readonly" {
   107  					return "open <dir> [readonly]"
   108  				}
   109  				opts.ReadOnly = true
   110  			}
   111  
   112  			buf.Reset()
   113  			dir := td.CmdArgs[0].String()
   114  			d, err := Open(dir, opts)
   115  			if err != nil {
   116  				return err.Error()
   117  			}
   118  			dbs[dir] = d
   119  			return buf.String()
   120  
   121  		case "scan":
   122  			if len(td.CmdArgs) != 1 {
   123  				return "scan <db>"
   124  			}
   125  			buf.Reset()
   126  			d := dbs[td.CmdArgs[0].String()]
   127  			iter := d.NewIter(nil)
   128  			for valid := iter.First(); valid; valid = iter.Next() {
   129  				fmt.Fprintf(&buf, "%s %s\n", iter.Key(), iter.Value())
   130  			}
   131  			fmt.Fprintf(&buf, ".\n")
   132  			if err := iter.Close(); err != nil {
   133  				fmt.Fprintf(&buf, "%v\n", err)
   134  			}
   135  			return buf.String()
   136  
   137  		default:
   138  			return fmt.Sprintf("unknown command: %s", td.Cmd)
   139  		}
   140  	})
   141  }
   142  
   143  func TestCheckpointCompaction(t *testing.T) {
   144  	fs := vfs.NewMem()
   145  	d, err := Open("", &Options{FS: fs})
   146  	require.NoError(t, err)
   147  
   148  	ctx, cancel := context.WithCancel(context.Background())
   149  
   150  	var wg sync.WaitGroup
   151  	wg.Add(4)
   152  	go func() {
   153  		defer cancel()
   154  		defer wg.Done()
   155  		for i := 0; ctx.Err() == nil; i++ {
   156  			if err := d.Set([]byte(fmt.Sprintf("key%06d", i)), nil, nil); err != nil {
   157  				t.Error(err)
   158  				return
   159  			}
   160  		}
   161  	}()
   162  	go func() {
   163  		defer cancel()
   164  		defer wg.Done()
   165  		for ctx.Err() == nil {
   166  			if err := d.Compact([]byte("key"), []byte("key999999"), false); err != nil {
   167  				t.Error(err)
   168  				return
   169  			}
   170  		}
   171  	}()
   172  	check := make(chan string, 100)
   173  	go func() {
   174  		defer cancel()
   175  		defer close(check)
   176  		defer wg.Done()
   177  		for i := 0; ctx.Err() == nil && i < 200; i++ {
   178  			dir := fmt.Sprintf("checkpoint%6d", i)
   179  			if err := d.Checkpoint(dir); err != nil {
   180  				t.Error(err)
   181  				return
   182  			}
   183  			select {
   184  			case <-ctx.Done():
   185  				return
   186  			case check <- dir:
   187  			}
   188  		}
   189  	}()
   190  	go func() {
   191  		opts := &Options{FS: fs}
   192  		defer cancel()
   193  		defer wg.Done()
   194  		for dir := range check {
   195  			d2, err := Open(dir, opts)
   196  			if err != nil {
   197  				t.Error(err)
   198  				return
   199  			}
   200  			// Check the checkpoint has all the sstables that the manifest
   201  			// claims it has.
   202  			tableInfos, _ := d2.SSTables()
   203  			for _, tables := range tableInfos {
   204  				for _, tbl := range tables {
   205  					if _, err := fs.Stat(base.MakeFilepath(fs, dir, base.FileTypeTable, tbl.FileNum)); err != nil {
   206  						t.Error(err)
   207  						return
   208  					}
   209  				}
   210  			}
   211  			if err := d2.Close(); err != nil {
   212  				t.Error(err)
   213  				return
   214  			}
   215  		}
   216  	}()
   217  	<-ctx.Done()
   218  	wg.Wait()
   219  	require.NoError(t, d.Close())
   220  }
   221  
   222  func TestCheckpointFlushWAL(t *testing.T) {
   223  	const checkpointPath = "checkpoints/checkpoint"
   224  	fs := vfs.NewStrictMem()
   225  	opts := &Options{FS: fs}
   226  	key, value := []byte("key"), []byte("value")
   227  
   228  	// Create a checkpoint from an unsynced DB.
   229  	{
   230  		d, err := Open("", opts)
   231  		require.NoError(t, err)
   232  		{
   233  			wb := d.NewBatch()
   234  			err = wb.Set(key, value, nil)
   235  			require.NoError(t, err)
   236  			err = d.Apply(wb, NoSync)
   237  			require.NoError(t, err)
   238  		}
   239  		err = d.Checkpoint(checkpointPath, WithFlushedWAL())
   240  		require.NoError(t, err)
   241  		require.NoError(t, d.Close())
   242  		fs.ResetToSyncedState()
   243  	}
   244  
   245  	// Check that the WAL has been flushed in the checkpoint.
   246  	{
   247  		files, err := fs.List(checkpointPath)
   248  		require.NoError(t, err)
   249  		hasLogFile := false
   250  		for _, f := range files {
   251  			info, err := fs.Stat(fs.PathJoin(checkpointPath, f))
   252  			require.NoError(t, err)
   253  			if strings.HasSuffix(f, ".log") {
   254  				hasLogFile = true
   255  				require.NotZero(t, info.Size())
   256  			}
   257  		}
   258  		require.True(t, hasLogFile)
   259  	}
   260  
   261  	// Check that the checkpoint contains the expected data.
   262  	{
   263  		d, err := Open(checkpointPath, opts)
   264  		require.NoError(t, err)
   265  		iter := d.NewIter(nil)
   266  		require.True(t, iter.First())
   267  		require.Equal(t, key, iter.Key())
   268  		require.Equal(t, value, iter.Value())
   269  		require.False(t, iter.Next())
   270  		require.NoError(t, iter.Close())
   271  		require.NoError(t, d.Close())
   272  	}
   273  }