github.com/petermattis/pebble@v0.0.0-20190905164901-ab51a2166067/ingest_test.go (about)

     1  // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package pebble
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"io/ioutil"
    11  	"sort"
    12  	"strconv"
    13  	"strings"
    14  	"testing"
    15  	"time"
    16  
    17  	"github.com/kr/pretty"
    18  	"github.com/petermattis/pebble/internal/base"
    19  	"github.com/petermattis/pebble/internal/datadriven"
    20  	"github.com/petermattis/pebble/internal/manifest"
    21  	"github.com/petermattis/pebble/sstable"
    22  	"github.com/petermattis/pebble/vfs"
    23  	"golang.org/x/exp/rand"
    24  )
    25  
    26  func TestIngestLoad(t *testing.T) {
    27  	mem := vfs.NewMem()
    28  
    29  	datadriven.RunTest(t, "testdata/ingest_load", func(td *datadriven.TestData) string {
    30  		switch td.Cmd {
    31  		case "load":
    32  			f, err := mem.Create("ext")
    33  			if err != nil {
    34  				return err.Error()
    35  			}
    36  			w := sstable.NewWriter(f, nil, LevelOptions{})
    37  			for _, data := range strings.Split(td.Input, "\n") {
    38  				j := strings.Index(data, ":")
    39  				if j < 0 {
    40  					return fmt.Sprintf("malformed input: %s\n", data)
    41  				}
    42  				key := base.ParseInternalKey(data[:j])
    43  				value := []byte(data[j+1:])
    44  				if err := w.Add(key, value); err != nil {
    45  					return err.Error()
    46  				}
    47  			}
    48  			w.Close()
    49  
    50  			opts := &Options{
    51  				Comparer: DefaultComparer,
    52  				FS:       mem,
    53  			}
    54  			meta, err := ingestLoad(opts, []string{"ext"}, 0, []uint64{1})
    55  			if err != nil {
    56  				return err.Error()
    57  			}
    58  			var buf bytes.Buffer
    59  			for _, m := range meta {
    60  				fmt.Fprintf(&buf, "%d: %s-%s\n", m.FileNum, m.Smallest, m.Largest)
    61  			}
    62  			return buf.String()
    63  
    64  		default:
    65  			return fmt.Sprintf("unknown command: %s", td.Cmd)
    66  		}
    67  	})
    68  }
    69  
    70  func TestIngestLoadRand(t *testing.T) {
    71  	mem := vfs.NewMem()
    72  	rng := rand.New(rand.NewSource(uint64(time.Now().UnixNano())))
    73  	cmp := DefaultComparer.Compare
    74  
    75  	randBytes := func(size int) []byte {
    76  		data := make([]byte, size)
    77  		for i := range data {
    78  			data[i] = byte(rng.Int() & 0xff)
    79  		}
    80  		return data
    81  	}
    82  
    83  	paths := make([]string, 1+rng.Intn(10))
    84  	pending := make([]uint64, len(paths))
    85  	expected := make([]*fileMetadata, len(paths))
    86  	for i := range paths {
    87  		paths[i] = fmt.Sprint(i)
    88  		pending[i] = uint64(rng.Int63())
    89  		expected[i] = &fileMetadata{
    90  			FileNum: pending[i],
    91  		}
    92  
    93  		func() {
    94  			f, err := mem.Create(paths[i])
    95  			if err != nil {
    96  				t.Fatal(err)
    97  			}
    98  			defer f.Close()
    99  
   100  			keys := make([]InternalKey, 1+rng.Intn(100))
   101  			for i := range keys {
   102  				keys[i] = base.MakeInternalKey(
   103  					randBytes(1+rng.Intn(10)),
   104  					uint64(rng.Int63n(int64(InternalKeySeqNumMax))),
   105  					InternalKeyKindSet)
   106  			}
   107  			sort.Slice(keys, func(i, j int) bool {
   108  				return base.InternalCompare(cmp, keys[i], keys[j]) < 0
   109  			})
   110  
   111  			expected[i].Smallest = keys[0]
   112  			expected[i].Largest = keys[len(keys)-1]
   113  
   114  			w := sstable.NewWriter(f, nil, LevelOptions{})
   115  			for i := range keys {
   116  				w.Add(keys[i], nil)
   117  			}
   118  			if err := w.Close(); err != nil {
   119  				t.Fatal(err)
   120  			}
   121  
   122  			meta, err := w.Metadata()
   123  			if err != nil {
   124  				t.Fatal(err)
   125  			}
   126  			expected[i].Size = meta.Size
   127  		}()
   128  	}
   129  
   130  	opts := &Options{
   131  		Comparer: DefaultComparer,
   132  		FS:       mem,
   133  	}
   134  	meta, err := ingestLoad(opts, paths, 0, pending)
   135  	if err != nil {
   136  		t.Fatal(err)
   137  	}
   138  	if diff := pretty.Diff(expected, meta); diff != nil {
   139  		t.Fatalf("%s", strings.Join(diff, "\n"))
   140  	}
   141  }
   142  
   143  func TestIngestLoadNonExistent(t *testing.T) {
   144  	opts := &Options{
   145  		Comparer: DefaultComparer,
   146  		FS:       vfs.NewMem(),
   147  	}
   148  	if _, err := ingestLoad(opts, []string{"non-existent"}, 0, []uint64{1}); err == nil {
   149  		t.Fatalf("expected error, but found success")
   150  	}
   151  }
   152  
   153  func TestIngestLoadEmpty(t *testing.T) {
   154  	mem := vfs.NewMem()
   155  	f, err := mem.Create("empty")
   156  	if err != nil {
   157  		t.Fatal(err)
   158  	}
   159  	f.Close()
   160  
   161  	opts := &Options{
   162  		Comparer: DefaultComparer,
   163  		FS:       mem,
   164  	}
   165  	if _, err := ingestLoad(opts, []string{"empty"}, 0, []uint64{1}); err == nil {
   166  		t.Fatalf("expected error, but found success")
   167  	}
   168  }
   169  
   170  func TestIngestSortAndVerify(t *testing.T) {
   171  	comparers := map[string]Compare{
   172  		"default": DefaultComparer.Compare,
   173  		"reverse": func(a, b []byte) int {
   174  			return DefaultComparer.Compare(b, a)
   175  		},
   176  	}
   177  
   178  	t.Run("", func(t *testing.T) {
   179  		datadriven.RunTest(t, "testdata/ingest_sort_and_verify", func(d *datadriven.TestData) string {
   180  			switch d.Cmd {
   181  			case "ingest":
   182  				var buf bytes.Buffer
   183  				var meta []*fileMetadata
   184  				var cmpName string
   185  				d.ScanArgs(t, "cmp", &cmpName)
   186  				cmp := comparers[cmpName]
   187  				if cmp == nil {
   188  					return fmt.Sprintf("%s unknown comparer: %s", d.Cmd, cmpName)
   189  				}
   190  				for _, data := range strings.Split(d.Input, "\n") {
   191  					parts := strings.Split(data, "-")
   192  					if len(parts) != 2 {
   193  						return fmt.Sprintf("malformed test case: %s", d.Input)
   194  					}
   195  					smallest := base.ParseInternalKey(parts[0])
   196  					largest := base.ParseInternalKey(parts[1])
   197  					if cmp(smallest.UserKey, largest.UserKey) > 0 {
   198  						return fmt.Sprintf("range %v-%v is not valid", smallest, largest)
   199  					}
   200  					meta = append(meta, &fileMetadata{
   201  						Smallest: smallest,
   202  						Largest:  largest,
   203  					})
   204  				}
   205  				err := ingestSortAndVerify(cmp, meta)
   206  				if err != nil {
   207  					return fmt.Sprintf("%v\n", err)
   208  				}
   209  				for i := range meta {
   210  					fmt.Fprintf(&buf, "%v-%v\n", meta[i].Smallest, meta[i].Largest)
   211  				}
   212  				return buf.String()
   213  
   214  			default:
   215  				return fmt.Sprintf("unknown command: %s", d.Cmd)
   216  			}
   217  		})
   218  	})
   219  }
   220  
   221  func TestIngestLink(t *testing.T) {
   222  	// Test linking of tables into the DB directory. Test cleanup when one of the
   223  	// tables cannot be linked.
   224  
   225  	const dir = "db"
   226  	const count = 10
   227  	for i := 0; i <= count; i++ {
   228  		t.Run("", func(t *testing.T) {
   229  			mem := vfs.NewMem()
   230  			opts := &Options{FS: mem}
   231  			opts.EnsureDefaults()
   232  			if err := mem.MkdirAll(dir, 0755); err != nil {
   233  				t.Fatal(err)
   234  			}
   235  
   236  			paths := make([]string, 10)
   237  			meta := make([]*fileMetadata, len(paths))
   238  			contents := make([][]byte, len(paths))
   239  			for j := range paths {
   240  				paths[j] = fmt.Sprintf("external%d", j)
   241  				meta[j] = &fileMetadata{}
   242  				meta[j].FileNum = uint64(j)
   243  				f, err := mem.Create(paths[j])
   244  				if err != nil {
   245  					t.Fatal(err)
   246  				}
   247  				contents[j] = []byte(fmt.Sprintf("data%d", j))
   248  				if _, err := f.Write(contents[j]); err != nil {
   249  					t.Fatal(err)
   250  				}
   251  				f.Close()
   252  			}
   253  
   254  			if i < count {
   255  				mem.Remove(paths[i])
   256  			}
   257  
   258  			err := ingestLink(opts, dir, paths, meta)
   259  			if i < count {
   260  				if err == nil {
   261  					t.Fatalf("expected error, but found success")
   262  				}
   263  			} else if err != nil {
   264  				t.Fatal(err)
   265  			}
   266  
   267  			files, err := mem.List(dir)
   268  			if err != nil {
   269  				t.Fatal(err)
   270  			}
   271  			sort.Strings(files)
   272  
   273  			if i < count {
   274  				if len(files) > 0 {
   275  					t.Fatalf("expected all of the files to be cleaned up, but found:\n%s",
   276  						strings.Join(files, "\n"))
   277  				}
   278  			} else {
   279  				if len(files) != count {
   280  					t.Fatalf("expected %d files, but found:\n%s", count, strings.Join(files, "\n"))
   281  				}
   282  				for j := range files {
   283  					ftype, fileNum, ok := base.ParseFilename(files[j])
   284  					if !ok {
   285  						t.Fatalf("unable to parse filename: %s", files[j])
   286  					}
   287  					if fileTypeTable != ftype {
   288  						t.Fatalf("expected table, but found %d", ftype)
   289  					}
   290  					if uint64(j) != fileNum {
   291  						t.Fatalf("expected table %d, but found %d", j, fileNum)
   292  					}
   293  					f, err := mem.Open(dir + "/" + files[j])
   294  					if err != nil {
   295  						t.Fatal(err)
   296  					}
   297  					data, err := ioutil.ReadAll(f)
   298  					if err != nil {
   299  						t.Fatal(err)
   300  					}
   301  					f.Close()
   302  					if !bytes.Equal(contents[j], data) {
   303  						t.Fatalf("expected %s, but found %s", contents[j], data)
   304  					}
   305  				}
   306  			}
   307  		})
   308  	}
   309  }
   310  
   311  func TestIngestMemtableOverlaps(t *testing.T) {
   312  	comparers := []Comparer{
   313  		{Name: "default", Compare: DefaultComparer.Compare},
   314  		{Name: "reverse", Compare: func(a, b []byte) int {
   315  			return DefaultComparer.Compare(b, a)
   316  		}},
   317  	}
   318  	m := make(map[string]*Comparer)
   319  	for i := range comparers {
   320  		c := &comparers[i]
   321  		m[c.Name] = c
   322  	}
   323  
   324  	for _, comparer := range comparers {
   325  		t.Run(comparer.Name, func(t *testing.T) {
   326  			var mem *memTable
   327  
   328  			parseMeta := func(s string) *fileMetadata {
   329  				parts := strings.Split(s, "-")
   330  				if len(parts) != 2 {
   331  					t.Fatalf("malformed table spec: %s", s)
   332  				}
   333  				if mem.cmp([]byte(parts[0]), []byte(parts[1])) > 0 {
   334  					parts[0], parts[1] = parts[1], parts[0]
   335  				}
   336  				return &fileMetadata{
   337  					Smallest: InternalKey{UserKey: []byte(parts[0])},
   338  					Largest:  InternalKey{UserKey: []byte(parts[1])},
   339  				}
   340  			}
   341  
   342  			datadriven.RunTest(t, "testdata/ingest_memtable_overlaps", func(d *datadriven.TestData) string {
   343  				switch d.Cmd {
   344  				case "define":
   345  					b := newBatch(nil)
   346  					if err := runBatchDefineCmd(d, b); err != nil {
   347  						return err.Error()
   348  					}
   349  
   350  					opts := &Options{
   351  						Comparer: &comparer,
   352  					}
   353  					opts.EnsureDefaults()
   354  					if len(d.CmdArgs) > 1 {
   355  						return fmt.Sprintf("%s expects at most 1 argument", d.Cmd)
   356  					}
   357  					if len(d.CmdArgs) == 1 {
   358  						opts.Comparer = m[d.CmdArgs[0].String()]
   359  						if opts.Comparer == nil {
   360  							return fmt.Sprintf("%s unknown comparer: %s", d.Cmd, d.CmdArgs[0].String())
   361  						}
   362  					}
   363  
   364  					mem = newMemTable(opts)
   365  					if err := mem.apply(b, 0); err != nil {
   366  						return err.Error()
   367  					}
   368  					return ""
   369  
   370  				case "overlaps":
   371  					var buf bytes.Buffer
   372  					for _, data := range strings.Split(d.Input, "\n") {
   373  						var meta []*fileMetadata
   374  						for _, part := range strings.Fields(data) {
   375  							meta = append(meta, parseMeta(part))
   376  						}
   377  						fmt.Fprintf(&buf, "%t\n", ingestMemtableOverlaps(mem.cmp, mem, meta))
   378  					}
   379  					return buf.String()
   380  
   381  				default:
   382  					return fmt.Sprintf("unknown command: %s", d.Cmd)
   383  				}
   384  			})
   385  		})
   386  	}
   387  }
   388  
   389  func TestIngestTargetLevel(t *testing.T) {
   390  	cmp := DefaultComparer.Compare
   391  	var vers *version
   392  
   393  	parseMeta := func(s string) fileMetadata {
   394  		parts := strings.Split(s, "-")
   395  		if len(parts) != 2 {
   396  			t.Fatalf("malformed table spec: %s", s)
   397  		}
   398  		return fileMetadata{
   399  			Smallest: InternalKey{UserKey: []byte(parts[0])},
   400  			Largest:  InternalKey{UserKey: []byte(parts[1])},
   401  		}
   402  	}
   403  
   404  	datadriven.RunTest(t, "testdata/ingest_target_level", func(d *datadriven.TestData) string {
   405  		switch d.Cmd {
   406  		case "define":
   407  			vers = &version{}
   408  			if len(d.Input) == 0 {
   409  				return ""
   410  			}
   411  			for _, data := range strings.Split(d.Input, "\n") {
   412  				parts := strings.Split(data, ":")
   413  				if len(parts) != 2 {
   414  					return fmt.Sprintf("malformed test:\n%s", d.Input)
   415  				}
   416  				level, err := strconv.Atoi(parts[0])
   417  				if err != nil {
   418  					return err.Error()
   419  				}
   420  				if vers.Files[level] != nil {
   421  					return fmt.Sprintf("level %d already filled", level)
   422  				}
   423  				for _, table := range strings.Fields(parts[1]) {
   424  					vers.Files[level] = append(vers.Files[level], parseMeta(table))
   425  				}
   426  
   427  				if level == 0 {
   428  					manifest.SortBySeqNum(vers.Files[level])
   429  				} else {
   430  					manifest.SortBySmallest(vers.Files[level], cmp)
   431  				}
   432  			}
   433  			return ""
   434  
   435  		case "target":
   436  			var buf bytes.Buffer
   437  			for _, target := range strings.Split(d.Input, "\n") {
   438  				meta := parseMeta(target)
   439  				level := ingestTargetLevel(cmp, vers, &meta)
   440  				fmt.Fprintf(&buf, "%d\n", level)
   441  			}
   442  			return buf.String()
   443  
   444  		default:
   445  			return fmt.Sprintf("unknown command: %s", d.Cmd)
   446  		}
   447  	})
   448  }
   449  
   450  func TestIngest(t *testing.T) {
   451  	mem := vfs.NewMem()
   452  	err := mem.MkdirAll("ext", 0755)
   453  	if err != nil {
   454  		t.Fatal(err)
   455  	}
   456  
   457  	d, err := Open("", &Options{
   458  		FS:                    mem,
   459  		L0CompactionThreshold: 100,
   460  	})
   461  	if err != nil {
   462  		t.Fatal(err)
   463  	}
   464  
   465  	datadriven.RunTest(t, "testdata/ingest", func(td *datadriven.TestData) string {
   466  		switch td.Cmd {
   467  		case "build", "batch":
   468  			b := d.NewIndexedBatch()
   469  			if err := runBatchDefineCmd(td, b); err != nil {
   470  				return err.Error()
   471  			}
   472  
   473  			switch td.Cmd {
   474  			case "build":
   475  				if len(td.CmdArgs) != 1 {
   476  					return "build <path>: argument missing\n"
   477  				}
   478  				path := td.CmdArgs[0].String()
   479  
   480  				f, err := mem.Create(path)
   481  				if err != nil {
   482  					return err.Error()
   483  				}
   484  				w := sstable.NewWriter(f, nil, LevelOptions{})
   485  				iters := []internalIterator{
   486  					b.newInternalIter(nil),
   487  					b.newRangeDelIter(nil),
   488  				}
   489  				for _, iter := range iters {
   490  					if iter == nil {
   491  						continue
   492  					}
   493  					for key, val := iter.First(); key != nil; key, val = iter.Next() {
   494  						tmp := *key
   495  						tmp.SetSeqNum(10000)
   496  						if err := w.Add(tmp, val); err != nil {
   497  							return err.Error()
   498  						}
   499  					}
   500  					if err := iter.Close(); err != nil {
   501  						return err.Error()
   502  					}
   503  				}
   504  				if err := w.Close(); err != nil {
   505  					return err.Error()
   506  				}
   507  
   508  			case "batch":
   509  				if err := b.Commit(nil); err != nil {
   510  					return err.Error()
   511  				}
   512  			}
   513  			return ""
   514  
   515  		case "ingest":
   516  			if len(td.CmdArgs) == 0 {
   517  				return "ingest <paths>: path arguments missing\n"
   518  			}
   519  			var paths []string
   520  			for _, arg := range td.CmdArgs {
   521  				paths = append(paths, arg.String())
   522  			}
   523  
   524  			if err := d.Ingest(paths); err != nil {
   525  				return err.Error()
   526  			}
   527  			for _, path := range paths {
   528  				if err := mem.Remove(path); err != nil {
   529  					return err.Error()
   530  				}
   531  			}
   532  			return ""
   533  
   534  		case "get":
   535  			var buf bytes.Buffer
   536  			for _, data := range strings.Split(td.Input, "\n") {
   537  				v, err := d.Get([]byte(data))
   538  				if err != nil {
   539  					fmt.Fprintf(&buf, "%s: %s\n", data, err)
   540  				} else {
   541  					fmt.Fprintf(&buf, "%s:%s\n", data, v)
   542  				}
   543  			}
   544  			return buf.String()
   545  
   546  		case "iter":
   547  			iter := d.NewIter(nil)
   548  			defer iter.Close()
   549  			var b bytes.Buffer
   550  			for _, line := range strings.Split(td.Input, "\n") {
   551  				parts := strings.Fields(line)
   552  				if len(parts) == 0 {
   553  					continue
   554  				}
   555  				switch parts[0] {
   556  				case "seek-ge":
   557  					if len(parts) != 2 {
   558  						return fmt.Sprintf("seek-ge <key>\n")
   559  					}
   560  					iter.SeekGE([]byte(strings.TrimSpace(parts[1])))
   561  				case "seek-lt":
   562  					if len(parts) != 2 {
   563  						return fmt.Sprintf("seek-lt <key>\n")
   564  					}
   565  					iter.SeekLT([]byte(strings.TrimSpace(parts[1])))
   566  				case "next":
   567  					iter.Next()
   568  				case "prev":
   569  					iter.Prev()
   570  				default:
   571  					return fmt.Sprintf("unknown op: %s", parts[0])
   572  				}
   573  				if iter.Valid() {
   574  					fmt.Fprintf(&b, "%s:%s\n", iter.Key(), iter.Value())
   575  				} else if err := iter.Error(); err != nil {
   576  					fmt.Fprintf(&b, "err=%v\n", err)
   577  				} else {
   578  					fmt.Fprintf(&b, ".\n")
   579  				}
   580  			}
   581  			return b.String()
   582  
   583  		case "lsm":
   584  			d.mu.Lock()
   585  			s := d.mu.versions.currentVersion().String()
   586  			d.mu.Unlock()
   587  			return s
   588  
   589  		default:
   590  			return fmt.Sprintf("unknown command: %s", td.Cmd)
   591  		}
   592  	})
   593  }