github.com/petermattis/pebble@v0.0.0-20190905164901-ab51a2166067/range_del_test.go (about)

     1  // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package pebble
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"strconv"
    11  	"strings"
    12  	"testing"
    13  
    14  	"github.com/petermattis/pebble/cache"
    15  	"github.com/petermattis/pebble/internal/base"
    16  	"github.com/petermattis/pebble/internal/datadriven"
    17  	"github.com/petermattis/pebble/sstable"
    18  	"github.com/petermattis/pebble/vfs"
    19  )
    20  
    21  func TestRangeDel(t *testing.T) {
    22  	var d *DB
    23  
    24  	datadriven.RunTest(t, "testdata/range_del", func(td *datadriven.TestData) string {
    25  		switch td.Cmd {
    26  		case "define":
    27  			var err error
    28  			if d, err = runDBDefineCmd(td, nil /* options */); err != nil {
    29  				return err.Error()
    30  			}
    31  
    32  			d.mu.Lock()
    33  			// Disable the "dynamic base level" code for this test.
    34  			d.mu.versions.picker.baseLevel = 1
    35  			s := fmt.Sprintf("mem: %d\n%s", len(d.mu.mem.queue), d.mu.versions.currentVersion())
    36  			d.mu.Unlock()
    37  			return s
    38  
    39  		case "compact":
    40  			if err := runCompactCommand(td, d); err != nil {
    41  				return err.Error()
    42  			}
    43  			d.mu.Lock()
    44  			// Disable the "dynamic base level" code for this test.
    45  			d.mu.versions.picker.baseLevel = 1
    46  			s := d.mu.versions.currentVersion().String()
    47  			d.mu.Unlock()
    48  			return s
    49  
    50  		case "get":
    51  			snap := Snapshot{
    52  				db:     d,
    53  				seqNum: InternalKeySeqNumMax,
    54  			}
    55  
    56  			for _, arg := range td.CmdArgs {
    57  				if len(arg.Vals) != 1 {
    58  					return fmt.Sprintf("%s: %s=<value>", td.Cmd, arg.Key)
    59  				}
    60  				switch arg.Key {
    61  				case "seq":
    62  					var err error
    63  					snap.seqNum, err = strconv.ParseUint(arg.Vals[0], 10, 64)
    64  					if err != nil {
    65  						return err.Error()
    66  					}
    67  				default:
    68  					return fmt.Sprintf("%s: unknown arg: %s", td.Cmd, arg.Key)
    69  				}
    70  			}
    71  
    72  			var buf bytes.Buffer
    73  			for _, data := range strings.Split(td.Input, "\n") {
    74  				v, err := snap.Get([]byte(data))
    75  				if err != nil {
    76  					fmt.Fprintf(&buf, "%s\n", err)
    77  				} else {
    78  					fmt.Fprintf(&buf, "%s\n", v)
    79  				}
    80  			}
    81  			return buf.String()
    82  
    83  		case "iter":
    84  			snap := Snapshot{
    85  				db:     d,
    86  				seqNum: InternalKeySeqNumMax,
    87  			}
    88  
    89  			for _, arg := range td.CmdArgs {
    90  				if len(arg.Vals) != 1 {
    91  					return fmt.Sprintf("%s: %s=<value>", td.Cmd, arg.Key)
    92  				}
    93  				switch arg.Key {
    94  				case "seq":
    95  					var err error
    96  					snap.seqNum, err = strconv.ParseUint(arg.Vals[0], 10, 64)
    97  					if err != nil {
    98  						return err.Error()
    99  					}
   100  				default:
   101  					return fmt.Sprintf("%s: unknown arg: %s", td.Cmd, arg.Key)
   102  				}
   103  			}
   104  
   105  			iter := snap.NewIter(nil)
   106  			defer iter.Close()
   107  			return runIterCmd(td, iter)
   108  
   109  		default:
   110  			return fmt.Sprintf("unknown command: %s", td.Cmd)
   111  		}
   112  	})
   113  }
   114  
   115  // Verify that range tombstones at higher levels do not unintentionally delete
   116  // newer keys at lower levels. This test sets up one such scenario. The base
   117  // problem is that range tombstones are not truncated to sstable boundaries on
   118  // disk, only in memory.
   119  func TestRangeDelCompactionTruncation(t *testing.T) {
   120  	// Use a small target file size so that there is a single key per sstable.
   121  	d, err := Open("", &Options{
   122  		FS: vfs.NewMem(),
   123  		Levels: []LevelOptions{
   124  			{TargetFileSize: 100},
   125  			{TargetFileSize: 100},
   126  			{TargetFileSize: 1},
   127  		},
   128  	})
   129  	if err != nil {
   130  		t.Fatal(err)
   131  	}
   132  	defer d.Close()
   133  
   134  	d.mu.Lock()
   135  	d.mu.versions.dynamicBaseLevel = false
   136  	d.mu.Unlock()
   137  
   138  	lsm := func() string {
   139  		d.mu.Lock()
   140  		s := d.mu.versions.currentVersion().DebugString()
   141  		d.mu.Unlock()
   142  		return s
   143  	}
   144  	expectLSM := func(expected string) {
   145  		t.Helper()
   146  		expected = strings.TrimSpace(expected)
   147  		actual := strings.TrimSpace(lsm())
   148  		if expected != actual {
   149  			t.Fatalf("expected\n%sbut found\n%s", expected, actual)
   150  		}
   151  	}
   152  
   153  	if err := d.Set([]byte("a"), bytes.Repeat([]byte("b"), 100), nil); err != nil {
   154  		t.Fatal(err)
   155  	}
   156  	snap1 := d.NewSnapshot()
   157  	defer snap1.Close()
   158  	// Flush so that each version of "a" ends up in its own L0 table. If we
   159  	// allowed both versions in the same L0 table, compaction could trivially
   160  	// move the single L0 table to L1.
   161  	if err := d.Flush(); err != nil {
   162  		t.Fatal(err)
   163  	}
   164  	if err := d.Set([]byte("b"), bytes.Repeat([]byte("c"), 100), nil); err != nil {
   165  		t.Fatal(err)
   166  	}
   167  	snap2 := d.NewSnapshot()
   168  	defer snap2.Close()
   169  	if err := d.DeleteRange([]byte("a"), []byte("d"), nil); err != nil {
   170  		t.Fatal(err)
   171  	}
   172  
   173  	// Compact to produce the L1 tables.
   174  	if err := d.Compact([]byte("c"), []byte("c")); err != nil {
   175  		t.Fatal(err)
   176  	}
   177  	expectLSM(`
   178  1: a#2,15-b#72057594037927935,15 b#1,1-d#72057594037927935,15
   179  `)
   180  
   181  	// Compact again to move one of the tables to L2.
   182  	if err := d.Compact([]byte("c"), []byte("c")); err != nil {
   183  		t.Fatal(err)
   184  	}
   185  	expectLSM(`
   186  1: a#2,15-b#72057594037927935,15
   187  2: b#1,1-d#72057594037927935,15
   188  `)
   189  
   190  	// Write "b" and "c" to a new table.
   191  	if err := d.Set([]byte("b"), []byte("d"), nil); err != nil {
   192  		t.Fatal(err)
   193  	}
   194  	if err := d.Set([]byte("c"), []byte("e"), nil); err != nil {
   195  		t.Fatal(err)
   196  	}
   197  	if err := d.Flush(); err != nil {
   198  		t.Fatal(err)
   199  	}
   200  	expectLSM(`
   201  0: b#3,1-c#4,1
   202  1: a#2,15-b#72057594037927935,15
   203  2: b#1,1-d#72057594037927935,15
   204  `)
   205  
   206  	// "b" is still visible at this point as it should be.
   207  	if _, err := d.Get([]byte("b")); err != nil {
   208  		t.Fatalf("expected success, but found %v", err)
   209  	}
   210  
   211  	keys := func() string {
   212  		iter := d.NewIter(nil)
   213  		defer iter.Close()
   214  		var buf bytes.Buffer
   215  		var sep string
   216  		for iter.First(); iter.Valid(); iter.Next() {
   217  			fmt.Fprintf(&buf, "%s%s", sep, iter.Key())
   218  			sep = " "
   219  		}
   220  		return buf.String()
   221  	}
   222  
   223  	if expected, actual := `b c`, keys(); expected != actual {
   224  		t.Fatalf("expected %q, but found %q", expected, actual)
   225  	}
   226  
   227  	// Compact the L0 table. This will compact the L0 table into L1 and do to the
   228  	// sstable target size settings will create 2 tables in L1. Then L1 table
   229  	// containing "c" will be compacted again with the L2 table creating two
   230  	// tables in L2. Lastly, the L2 table containing "c" will be compacted
   231  	// creating the L3 table.
   232  	if err := d.Compact([]byte("c"), []byte("c")); err != nil {
   233  		t.Fatal(err)
   234  	}
   235  	expectLSM(`
   236  1: a#2,15-b#72057594037927935,15
   237  2: b#3,1-b#3,1
   238  3: b#2,15-c#72057594037927935,15 c#4,1-d#72057594037927935,15
   239  `)
   240  
   241  	// The L1 table still contains a tombstone from [a,d) which will improperly
   242  	// delete the newer version of "b" in L2.
   243  	if _, err := d.Get([]byte("b")); err != nil {
   244  		t.Errorf("expected success, but found %v", err)
   245  	}
   246  
   247  	if expected, actual := `b c`, keys(); expected != actual {
   248  		t.Errorf("expected %q, but found %q", expected, actual)
   249  	}
   250  }
   251  
   252  func BenchmarkRangeDelIterate(b *testing.B) {
   253  	for _, entries := range []int{10, 1000, 100000} {
   254  		b.Run(fmt.Sprintf("entries=%d", entries), func(b *testing.B) {
   255  			for _, deleted := range []int{entries, entries - 1} {
   256  				b.Run(fmt.Sprintf("deleted=%d", deleted), func(b *testing.B) {
   257  					mem := vfs.NewMem()
   258  					d, err := Open("", &Options{
   259  						Cache: cache.New(128 << 20), // 128 MB
   260  						FS:    mem,
   261  					})
   262  					if err != nil {
   263  						b.Fatal(err)
   264  					}
   265  					defer d.Close()
   266  
   267  					makeKey := func(i int) []byte {
   268  						return []byte(fmt.Sprintf("%09d", i))
   269  					}
   270  
   271  					// Create an sstable with N entries and ingest it. This is a fast way
   272  					// to get a lot of entries into pebble.
   273  					f, err := mem.Create("ext")
   274  					if err != nil {
   275  						b.Fatal(err)
   276  					}
   277  					w := sstable.NewWriter(f, nil, LevelOptions{
   278  						BlockSize: 32 << 10, // 32 KB
   279  					})
   280  					for i := 0; i < entries; i++ {
   281  						key := base.MakeInternalKey(makeKey(i), 0, InternalKeyKindSet)
   282  						if err := w.Add(key, nil); err != nil {
   283  							b.Fatal(err)
   284  						}
   285  					}
   286  					if err := w.Close(); err != nil {
   287  						b.Fatal(err)
   288  					}
   289  					if err := d.Ingest([]string{"ext"}); err != nil {
   290  						b.Fatal(err)
   291  					}
   292  					if err := mem.Remove("ext"); err != nil {
   293  						b.Fatal(err)
   294  					}
   295  
   296  					// Create a range tombstone that deletes most (or all) of those entries.
   297  					from := makeKey(0)
   298  					to := makeKey(deleted)
   299  					if err := d.DeleteRange(from, to, nil); err != nil {
   300  						b.Fatal(err)
   301  					}
   302  
   303  					b.ResetTimer()
   304  					for i := 0; i < b.N; i++ {
   305  						iter := d.NewIter(nil)
   306  						iter.SeekGE(from)
   307  						if deleted < entries {
   308  							if !iter.Valid() {
   309  								b.Fatal("key not found")
   310  							}
   311  						} else if iter.Valid() {
   312  							b.Fatal("unexpected key found")
   313  						}
   314  						if err := iter.Close(); err != nil {
   315  							b.Fatal(err)
   316  						}
   317  					}
   318  				})
   319  			}
   320  		})
   321  	}
   322  }