github.com/zuoyebang/bitalostable@v1.0.1-0.20240229032404-e3b99a834294/compaction_iter_test.go (about)

     1  // Copyright 2018 The LevelDB-Go and Pebble and Bitalostored Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package bitalostable
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"io"
    11  	"sort"
    12  	"strconv"
    13  	"strings"
    14  	"testing"
    15  
    16  	"github.com/zuoyebang/bitalostable/internal/base"
    17  	"github.com/zuoyebang/bitalostable/internal/datadriven"
    18  	"github.com/zuoyebang/bitalostable/internal/keyspan"
    19  	"github.com/zuoyebang/bitalostable/internal/rangekey"
    20  )
    21  
    22  func TestSnapshotIndex(t *testing.T) {
    23  	testCases := []struct {
    24  		snapshots      []uint64
    25  		seq            uint64
    26  		expectedIndex  int
    27  		expectedSeqNum uint64
    28  	}{
    29  		{[]uint64{}, 1, 0, InternalKeySeqNumMax},
    30  		{[]uint64{1}, 0, 0, 1},
    31  		{[]uint64{1}, 1, 1, InternalKeySeqNumMax},
    32  		{[]uint64{1}, 2, 1, InternalKeySeqNumMax},
    33  		{[]uint64{1, 3}, 1, 1, 3},
    34  		{[]uint64{1, 3}, 2, 1, 3},
    35  		{[]uint64{1, 3}, 3, 2, InternalKeySeqNumMax},
    36  		{[]uint64{1, 3}, 4, 2, InternalKeySeqNumMax},
    37  		{[]uint64{1, 3, 3}, 2, 1, 3},
    38  	}
    39  	for _, c := range testCases {
    40  		t.Run("", func(t *testing.T) {
    41  			idx, seqNum := snapshotIndex(c.seq, c.snapshots)
    42  			if c.expectedIndex != idx {
    43  				t.Fatalf("expected %d, but got %d", c.expectedIndex, idx)
    44  			}
    45  			if c.expectedSeqNum != seqNum {
    46  				t.Fatalf("expected %d, but got %d", c.expectedSeqNum, seqNum)
    47  			}
    48  		})
    49  	}
    50  }
    51  
    52  type debugMerger struct {
    53  	buf []byte
    54  }
    55  
    56  func (m *debugMerger) MergeNewer(value []byte) error {
    57  	m.buf = append(m.buf, value...)
    58  	return nil
    59  }
    60  
    61  func (m *debugMerger) MergeOlder(value []byte) error {
    62  	buf := make([]byte, 0, len(m.buf)+len(value))
    63  	buf = append(buf, value...)
    64  	buf = append(buf, m.buf...)
    65  	m.buf = buf
    66  	return nil
    67  }
    68  
    69  func (m *debugMerger) Finish(includesBase bool) ([]byte, io.Closer, error) {
    70  	if includesBase {
    71  		m.buf = append(m.buf, []byte("[base]")...)
    72  	}
    73  	return m.buf, nil, nil
    74  }
    75  
    76  func TestCompactionIter(t *testing.T) {
    77  	var merge Merge
    78  	var keys []InternalKey
    79  	var rangeKeys []keyspan.Span
    80  	var vals [][]byte
    81  	var snapshots []uint64
    82  	var elideTombstones bool
    83  	var allowZeroSeqnum bool
    84  	var interleavingIter *keyspan.InterleavingIter
    85  
    86  	// The input to the data-driven test is dependent on the format major
    87  	// version we are testing against.
    88  	fileFunc := func(formatVersion FormatMajorVersion) string {
    89  		if formatVersion < FormatSetWithDelete {
    90  			return "testdata/compaction_iter"
    91  		}
    92  		return "testdata/compaction_iter_set_with_del"
    93  	}
    94  
    95  	newIter := func(formatVersion FormatMajorVersion) *compactionIter {
    96  		// To adhere to the existing assumption that range deletion blocks in
    97  		// SSTables are not released while iterating, and therefore not
    98  		// susceptible to use-after-free bugs, we skip the zeroing of
    99  		// RangeDelete keys.
   100  		fi := &fakeIter{keys: keys, vals: vals}
   101  		interleavingIter = &keyspan.InterleavingIter{}
   102  		interleavingIter.Init(
   103  			base.DefaultComparer,
   104  			fi,
   105  			keyspan.NewIter(base.DefaultComparer.Compare, rangeKeys),
   106  			nil, nil, nil)
   107  		iter := newInvalidatingIter(interleavingIter)
   108  		iter.ignoreKind(InternalKeyKindRangeDelete)
   109  		if merge == nil {
   110  			merge = func(key, value []byte) (base.ValueMerger, error) {
   111  				m := &debugMerger{}
   112  				m.buf = append(m.buf, value...)
   113  				return m, nil
   114  			}
   115  		}
   116  
   117  		return newCompactionIter(
   118  			DefaultComparer.Compare,
   119  			DefaultComparer.Equal,
   120  			DefaultComparer.FormatKey,
   121  			merge,
   122  			iter,
   123  			snapshots,
   124  			&keyspan.Fragmenter{},
   125  			&keyspan.Fragmenter{},
   126  			allowZeroSeqnum,
   127  			func([]byte) bool {
   128  				return elideTombstones
   129  			},
   130  			func(_, _ []byte) bool {
   131  				return elideTombstones
   132  			},
   133  			formatVersion,
   134  		)
   135  	}
   136  
   137  	runTest := func(t *testing.T, formatVersion FormatMajorVersion) {
   138  		datadriven.RunTest(t, fileFunc(formatVersion), func(d *datadriven.TestData) string {
   139  			switch d.Cmd {
   140  			case "define":
   141  				merge = nil
   142  				if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "merger" &&
   143  					len(d.CmdArgs[0].Vals) > 0 && d.CmdArgs[0].Vals[0] == "deletable" {
   144  					merge = newDeletableSumValueMerger
   145  				}
   146  				keys = keys[:0]
   147  				vals = vals[:0]
   148  				rangeKeys = rangeKeys[:0]
   149  				for _, key := range strings.Split(d.Input, "\n") {
   150  					j := strings.Index(key, ":")
   151  					keys = append(keys, base.ParseInternalKey(key[:j]))
   152  					vals = append(vals, []byte(key[j+1:]))
   153  				}
   154  				return ""
   155  
   156  			case "define-range-keys":
   157  				for _, key := range strings.Split(d.Input, "\n") {
   158  					s := keyspan.ParseSpan(strings.TrimSpace(key))
   159  					rangeKeys = append(rangeKeys, s)
   160  				}
   161  				return ""
   162  
   163  			case "iter":
   164  				snapshots = snapshots[:0]
   165  				elideTombstones = false
   166  				allowZeroSeqnum = false
   167  				for _, arg := range d.CmdArgs {
   168  					switch arg.Key {
   169  					case "snapshots":
   170  						for _, val := range arg.Vals {
   171  							seqNum, err := strconv.Atoi(val)
   172  							if err != nil {
   173  								return err.Error()
   174  							}
   175  							snapshots = append(snapshots, uint64(seqNum))
   176  						}
   177  					case "elide-tombstones":
   178  						var err error
   179  						elideTombstones, err = strconv.ParseBool(arg.Vals[0])
   180  						if err != nil {
   181  							return err.Error()
   182  						}
   183  					case "allow-zero-seqnum":
   184  						var err error
   185  						allowZeroSeqnum, err = strconv.ParseBool(arg.Vals[0])
   186  						if err != nil {
   187  							return err.Error()
   188  						}
   189  					default:
   190  						return fmt.Sprintf("%s: unknown arg: %s", d.Cmd, arg.Key)
   191  					}
   192  				}
   193  				sort.Slice(snapshots, func(i, j int) bool {
   194  					return snapshots[i] < snapshots[j]
   195  				})
   196  
   197  				iter := newIter(formatVersion)
   198  				var b bytes.Buffer
   199  				for _, line := range strings.Split(d.Input, "\n") {
   200  					parts := strings.Fields(line)
   201  					if len(parts) == 0 {
   202  						continue
   203  					}
   204  					switch parts[0] {
   205  					case "first":
   206  						iter.First()
   207  					case "next":
   208  						iter.Next()
   209  					case "tombstones":
   210  						var key []byte
   211  						if len(parts) == 2 {
   212  							key = []byte(parts[1])
   213  						}
   214  						for _, v := range iter.Tombstones(key) {
   215  							for _, k := range v.Keys {
   216  								fmt.Fprintf(&b, "%s-%s#%d\n", v.Start, v.End, k.SeqNum())
   217  							}
   218  						}
   219  						fmt.Fprintf(&b, ".\n")
   220  						continue
   221  					case "range-keys":
   222  						var key []byte
   223  						if len(parts) == 2 {
   224  							key = []byte(parts[1])
   225  						}
   226  						for _, v := range iter.RangeKeys(key) {
   227  							fmt.Fprintf(&b, "%s\n", v)
   228  						}
   229  						fmt.Fprintf(&b, ".\n")
   230  						continue
   231  					default:
   232  						return fmt.Sprintf("unknown op: %s", parts[0])
   233  					}
   234  					if iter.Valid() {
   235  						fmt.Fprintf(&b, "%s:%s\n", iter.Key(), iter.Value())
   236  						if iter.Key().Kind() == InternalKeyKindRangeDelete {
   237  							iter.rangeDelFrag.Add(keyspan.Span{
   238  								Start: append([]byte{}, iter.Key().UserKey...),
   239  								End:   append([]byte{}, iter.Value()...),
   240  								Keys: []keyspan.Key{
   241  									{Trailer: iter.Key().Trailer},
   242  								},
   243  							})
   244  						}
   245  						if rangekey.IsRangeKey(iter.Key().Kind()) {
   246  							iter.rangeKeyFrag.Add(*interleavingIter.Span())
   247  						}
   248  					} else if err := iter.Error(); err != nil {
   249  						fmt.Fprintf(&b, "err=%v\n", err)
   250  					} else {
   251  						fmt.Fprintf(&b, ".\n")
   252  					}
   253  				}
   254  				return b.String()
   255  
   256  			default:
   257  				return fmt.Sprintf("unknown command: %s", d.Cmd)
   258  			}
   259  		})
   260  	}
   261  
   262  	// Rather than testing against all format version, we test against the
   263  	// significant boundaries.
   264  	formatVersions := []FormatMajorVersion{
   265  		FormatMostCompatible,
   266  		FormatSetWithDelete - 1,
   267  		FormatSetWithDelete,
   268  		FormatNewest,
   269  	}
   270  	for _, formatVersion := range formatVersions {
   271  		t.Run(fmt.Sprintf("version-%s", formatVersion), func(t *testing.T) {
   272  			runTest(t, formatVersion)
   273  		})
   274  	}
   275  }