github.com/zuoyebang/bitalostable@v1.0.1-0.20240229032404-e3b99a834294/level_checker_test.go (about)

     1  // Copyright 2019 The LevelDB-Go and Pebble and Bitalostored Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package bitalostable
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"io"
    11  	"path/filepath"
    12  	"strings"
    13  	"testing"
    14  
    15  	"github.com/cockroachdb/errors"
    16  	"github.com/stretchr/testify/require"
    17  	"github.com/zuoyebang/bitalostable/internal/base"
    18  	"github.com/zuoyebang/bitalostable/internal/datadriven"
    19  	"github.com/zuoyebang/bitalostable/internal/keyspan"
    20  	"github.com/zuoyebang/bitalostable/internal/manifest"
    21  	"github.com/zuoyebang/bitalostable/internal/private"
    22  	"github.com/zuoyebang/bitalostable/internal/rangedel"
    23  	"github.com/zuoyebang/bitalostable/sstable"
    24  	"github.com/zuoyebang/bitalostable/vfs"
    25  )
    26  
    27  func TestCheckLevelsBasics(t *testing.T) {
    28  	testCases := []string{"db-stage-1", "db-stage-2", "db-stage-3", "db-stage-4"}
    29  	for _, tc := range testCases {
    30  		t.Run(tc, func(t *testing.T) {
    31  			t.Logf("%s", t.Name())
    32  			fs := vfs.NewMem()
    33  			_, err := vfs.Clone(vfs.Default, fs, filepath.Join("testdata", tc), tc)
    34  			if err != nil {
    35  				t.Fatalf("%s: cloneFileSystem failed: %v", tc, err)
    36  			}
    37  			d, err := Open(tc, &Options{
    38  				FS: fs,
    39  			})
    40  			if err != nil {
    41  				t.Fatalf("%s: Open failed: %v", tc, err)
    42  			}
    43  			require.NoError(t, d.CheckLevels(nil))
    44  			require.NoError(t, d.Close())
    45  		})
    46  	}
    47  }
    48  
    49  type failMerger struct {
    50  	lastBuf    []byte
    51  	closeCount int
    52  }
    53  
    54  func (f *failMerger) MergeNewer(value []byte) error {
    55  	return nil
    56  }
    57  
    58  func (f *failMerger) MergeOlder(value []byte) error {
    59  	if string(value) == "fail-merge" {
    60  		f.lastBuf = nil
    61  		return errors.New("merge failed")
    62  	}
    63  	f.lastBuf = append(f.lastBuf[:0], value...)
    64  	return nil
    65  }
    66  
    67  func (f *failMerger) Finish(includesBase bool) ([]byte, io.Closer, error) {
    68  	if string(f.lastBuf) == "fail-finish" {
    69  		f.lastBuf = nil
    70  		return nil, nil, errors.New("finish failed")
    71  	}
    72  	f.closeCount++
    73  	return nil, f, nil
    74  }
    75  
    76  func (f *failMerger) Close() error {
    77  	f.closeCount--
    78  	f.lastBuf = nil
    79  	return nil
    80  }
    81  
    82  func TestCheckLevelsCornerCases(t *testing.T) {
    83  	memFS := vfs.NewMem()
    84  	cmp := DefaultComparer.Compare
    85  	var levels [][]*fileMetadata
    86  	formatKey := DefaultComparer.FormatKey
    87  	// Indexed by fileNum
    88  	var readers []*sstable.Reader
    89  	defer func() {
    90  		for _, r := range readers {
    91  			r.Close()
    92  		}
    93  	}()
    94  
    95  	var fileNum FileNum
    96  	newIters :=
    97  		func(file *manifest.FileMetadata, _ *IterOptions, _ internalIterOpts) (internalIterator, keyspan.FragmentIterator, error) {
    98  			r := readers[file.FileNum]
    99  			rangeDelIter, err := r.NewRawRangeDelIter()
   100  			if err != nil {
   101  				return nil, nil, err
   102  			}
   103  			iter, err := r.NewIter(nil /* lower */, nil /* upper */)
   104  			if err != nil {
   105  				return nil, nil, err
   106  			}
   107  			return iter, rangeDelIter, nil
   108  		}
   109  
   110  	fm := &failMerger{}
   111  	defer require.Equal(t, 0, fm.closeCount)
   112  
   113  	failMerger := &Merger{
   114  		Merge: func(key, value []byte) (ValueMerger, error) {
   115  			fm.lastBuf = append(fm.lastBuf[:0], value...)
   116  			return fm, nil
   117  		},
   118  
   119  		Name: "fail-merger",
   120  	}
   121  
   122  	datadriven.RunTest(t, "testdata/level_checker", func(d *datadriven.TestData) string {
   123  		switch d.Cmd {
   124  		case "define":
   125  			lines := strings.Split(d.Input, "\n")
   126  			levels = levels[:0]
   127  			for i := 0; i < len(lines); i++ {
   128  				line := lines[i]
   129  				line = strings.TrimSpace(line)
   130  				if line == "L" {
   131  					// start next level
   132  					levels = append(levels, nil)
   133  					continue
   134  				}
   135  				li := &levels[len(levels)-1]
   136  				keys := strings.Fields(line)
   137  				smallestKey := base.ParseInternalKey(keys[0])
   138  				largestKey := base.ParseInternalKey(keys[1])
   139  				m := (&fileMetadata{
   140  					FileNum: fileNum,
   141  				}).ExtendPointKeyBounds(cmp, smallestKey, largestKey)
   142  				*li = append(*li, m)
   143  
   144  				i++
   145  				line = lines[i]
   146  				line = strings.TrimSpace(line)
   147  				name := fmt.Sprint(fileNum)
   148  				fileNum++
   149  				f, err := memFS.Create(name)
   150  				if err != nil {
   151  					return err.Error()
   152  				}
   153  				writeUnfragmented := false
   154  				w := sstable.NewWriter(f, sstable.WriterOptions{})
   155  				for _, arg := range d.CmdArgs {
   156  					switch arg.Key {
   157  					case "disable-key-order-checks":
   158  						private.SSTableWriterDisableKeyOrderChecks(w)
   159  					case "write-unfragmented":
   160  						writeUnfragmented = true
   161  					default:
   162  						return fmt.Sprintf("unknown arg: %s", arg.Key)
   163  					}
   164  				}
   165  				var tombstones []keyspan.Span
   166  				frag := keyspan.Fragmenter{
   167  					Cmp:    cmp,
   168  					Format: formatKey,
   169  					Emit: func(fragmented keyspan.Span) {
   170  						tombstones = append(tombstones, fragmented)
   171  					},
   172  				}
   173  				keyvalues := strings.Fields(line)
   174  				for _, kv := range keyvalues {
   175  					j := strings.Index(kv, ":")
   176  					ikey := base.ParseInternalKey(kv[:j])
   177  					value := []byte(kv[j+1:])
   178  					var err error
   179  					switch ikey.Kind() {
   180  					case InternalKeyKindRangeDelete:
   181  						if writeUnfragmented {
   182  							err = w.Add(ikey, value)
   183  							break
   184  						}
   185  						frag.Add(rangedel.Decode(ikey, value, nil))
   186  					default:
   187  						err = w.Add(ikey, value)
   188  					}
   189  					if err != nil {
   190  						return err.Error()
   191  					}
   192  				}
   193  				frag.Finish()
   194  				for _, v := range tombstones {
   195  					if err := rangedel.Encode(&v, w.Add); err != nil {
   196  						return err.Error()
   197  					}
   198  				}
   199  				if err := w.Close(); err != nil {
   200  					return err.Error()
   201  				}
   202  				f, err = memFS.Open(name)
   203  				if err != nil {
   204  					return err.Error()
   205  				}
   206  				cacheOpts := private.SSTableCacheOpts(0, fileNum-1).(sstable.ReaderOption)
   207  				r, err := sstable.NewReader(f, sstable.ReaderOptions{}, cacheOpts)
   208  				if err != nil {
   209  					return err.Error()
   210  				}
   211  				readers = append(readers, r)
   212  			}
   213  			// TODO(sbhola): clean this up by wrapping levels in a Version and using
   214  			// Version.DebugString().
   215  			var buf bytes.Buffer
   216  			for i, l := range levels {
   217  				fmt.Fprintf(&buf, "Level %d\n", i+1)
   218  				for j, f := range l {
   219  					fmt.Fprintf(&buf, "  file %d: [%s-%s]\n", j, f.Smallest.String(), f.Largest.String())
   220  				}
   221  			}
   222  			return buf.String()
   223  		case "check":
   224  			merge := DefaultMerger.Merge
   225  			for _, arg := range d.CmdArgs {
   226  				switch arg.Key {
   227  				case "merger":
   228  					if len(arg.Vals) != 1 {
   229  						return fmt.Sprintf("expected one arg value, got %d", len(arg.Vals))
   230  					}
   231  					if arg.Vals[0] != failMerger.Name {
   232  						return "unsupported merger"
   233  					}
   234  					merge = failMerger.Merge
   235  				default:
   236  					return fmt.Sprintf("unknown arg: %s", arg.Key)
   237  				}
   238  			}
   239  
   240  			var files [numLevels][]*fileMetadata
   241  			for i := range levels {
   242  				// Start from level 1 in this test.
   243  				files[i+1] = levels[i]
   244  			}
   245  			version := manifest.NewVersion(
   246  				base.DefaultComparer.Compare,
   247  				base.DefaultFormatter,
   248  				0,
   249  				files)
   250  			readState := &readState{current: version}
   251  			c := &checkConfig{
   252  				cmp:       cmp,
   253  				readState: readState,
   254  				newIters:  newIters,
   255  				seqNum:    InternalKeySeqNumMax,
   256  				merge:     merge,
   257  				formatKey: formatKey,
   258  			}
   259  			if err := checkLevelsInternal(c); err != nil {
   260  				return err.Error()
   261  			}
   262  			return ""
   263  		default:
   264  			return fmt.Sprintf("unknown command: %s", d.Cmd)
   265  		}
   266  	})
   267  }