github.com/petermattis/pebble@v0.0.0-20190905164901-ab51a2166067/event_listener_test.go (about)

     1  // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package pebble
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"io"
    11  	"os"
    12  	"strings"
    13  	"sync"
    14  	"testing"
    15  
    16  	"github.com/petermattis/pebble/internal/base"
    17  	"github.com/petermattis/pebble/internal/datadriven"
    18  	"github.com/petermattis/pebble/sstable"
    19  	"github.com/petermattis/pebble/vfs"
    20  	"github.com/stretchr/testify/require"
    21  )
    22  
    23  type syncedBuffer struct {
    24  	mu  sync.Mutex
    25  	buf bytes.Buffer
    26  }
    27  
    28  func (b *syncedBuffer) Reset() {
    29  	b.mu.Lock()
    30  	defer b.mu.Unlock()
    31  	b.buf.Reset()
    32  }
    33  
    34  func (b *syncedBuffer) Write(p []byte) (n int, err error) {
    35  	b.mu.Lock()
    36  	defer b.mu.Unlock()
    37  	return b.buf.Write(p)
    38  }
    39  
    40  func (b *syncedBuffer) Infof(format string, args ...interface{}) {
    41  	s := fmt.Sprintf(format, args...)
    42  	b.mu.Lock()
    43  	defer b.mu.Unlock()
    44  	b.buf.Write([]byte(s))
    45  	if n := len(s); n == 0 || s[n-1] != '\n' {
    46  		b.buf.Write([]byte("\n"))
    47  	}
    48  }
    49  
    50  func (b *syncedBuffer) Fatalf(format string, args ...interface{}) {
    51  	panic(fmt.Sprintf(format, args...))
    52  }
    53  
    54  func (b *syncedBuffer) String() string {
    55  	b.mu.Lock()
    56  	defer b.mu.Unlock()
    57  	return b.buf.String()
    58  }
    59  
    60  type loggingFS struct {
    61  	vfs.FS
    62  	w io.Writer
    63  }
    64  
    65  func (fs loggingFS) Create(name string) (vfs.File, error) {
    66  	fmt.Fprintf(fs.w, "create: %s\n", name)
    67  	f, err := fs.FS.Create(name)
    68  	if err != nil {
    69  		return nil, err
    70  	}
    71  	return loggingFile{f, name, fs.w}, nil
    72  }
    73  
    74  func (fs loggingFS) Link(oldname, newname string) error {
    75  	fmt.Fprintf(fs.w, "link: %s -> %s\n", oldname, newname)
    76  	return fs.FS.Link(oldname, newname)
    77  }
    78  
    79  func (fs loggingFS) OpenDir(name string) (vfs.File, error) {
    80  	fmt.Fprintf(fs.w, "open-dir: %s\n", name)
    81  	f, err := fs.FS.OpenDir(name)
    82  	if err != nil {
    83  		return nil, err
    84  	}
    85  	return loggingFile{f, name, fs.w}, nil
    86  }
    87  
    88  func (fs loggingFS) Rename(oldname, newname string) error {
    89  	fmt.Fprintf(fs.w, "rename: %s -> %s\n", oldname, newname)
    90  	return fs.FS.Rename(oldname, newname)
    91  }
    92  
    93  func (fs loggingFS) MkdirAll(dir string, perm os.FileMode) error {
    94  	fmt.Fprintf(fs.w, "mkdir-all: %s %#o\n", dir, perm)
    95  	return fs.FS.MkdirAll(dir, perm)
    96  }
    97  
    98  type loggingFile struct {
    99  	vfs.File
   100  	name string
   101  	w    io.Writer
   102  }
   103  
   104  func (f loggingFile) Sync() error {
   105  	fmt.Fprintf(f.w, "sync: %s\n", f.name)
   106  	return f.File.Sync()
   107  }
   108  
   109  // Verify event listener actions, as well as expected filesystem operations.
   110  func TestEventListener(t *testing.T) {
   111  	var d *DB
   112  	var buf syncedBuffer
   113  	mem := vfs.NewMem()
   114  	err := mem.MkdirAll("ext", 0755)
   115  	if err != nil {
   116  		t.Fatal(err)
   117  	}
   118  
   119  	datadriven.RunTest(t, "testdata/event_listener", func(td *datadriven.TestData) string {
   120  		switch td.Cmd {
   121  		case "open":
   122  			buf.Reset()
   123  			var err error
   124  			d, err = Open("db", &Options{
   125  				FS:                  loggingFS{mem, &buf},
   126  				EventListener:       MakeLoggingEventListener(&buf),
   127  				MaxManifestFileSize: 1,
   128  				WALDir:              "wal",
   129  			})
   130  			if err != nil {
   131  				return err.Error()
   132  			}
   133  			return buf.String()
   134  
   135  		case "flush":
   136  			buf.Reset()
   137  			if err := d.Set([]byte("a"), nil, nil); err != nil {
   138  				return err.Error()
   139  			}
   140  			if err := d.Flush(); err != nil {
   141  				return err.Error()
   142  			}
   143  			return buf.String()
   144  
   145  		case "compact":
   146  			buf.Reset()
   147  			if err := d.Set([]byte("a"), nil, nil); err != nil {
   148  				return err.Error()
   149  			}
   150  			if err := d.Compact([]byte("a"), []byte("b")); err != nil {
   151  				return err.Error()
   152  			}
   153  			return buf.String()
   154  
   155  		case "ingest":
   156  			buf.Reset()
   157  			f, err := mem.Create("ext/0")
   158  			if err != nil {
   159  				return err.Error()
   160  			}
   161  			w := sstable.NewWriter(f, nil, LevelOptions{})
   162  			if err := w.Add(base.MakeInternalKey([]byte("a"), 0, InternalKeyKindSet), nil); err != nil {
   163  				return err.Error()
   164  			}
   165  			if err := w.Close(); err != nil {
   166  				return err.Error()
   167  			}
   168  			if err := d.Ingest([]string{"ext/0"}); err != nil {
   169  				return err.Error()
   170  			}
   171  			if err := mem.Remove("ext/0"); err != nil {
   172  				return err.Error()
   173  			}
   174  			return buf.String()
   175  
   176  		case "metrics":
   177  			return d.Metrics().String()
   178  
   179  		default:
   180  			return fmt.Sprintf("unknown command: %s", td.Cmd)
   181  		}
   182  	})
   183  }
   184  
   185  func TestWriteStallEvents(t *testing.T) {
   186  	const flushCount = 10
   187  	const writeStallEnd = "write stall ending"
   188  
   189  	testCases := []struct {
   190  		delayFlush bool
   191  		expected   string
   192  	}{
   193  		{true, "memtable count limit reached"},
   194  		{false, "L0 file count limit exceeded"},
   195  	}
   196  
   197  	for _, c := range testCases {
   198  		t.Run("", func(t *testing.T) {
   199  			stallEnded := make(chan struct{}, 1)
   200  			createReleased := make(chan struct{}, flushCount)
   201  			var buf syncedBuffer
   202  			var delayOnce sync.Once
   203  			listener := EventListener{
   204  				TableCreated: func(info TableCreateInfo) {
   205  					if c.delayFlush == (info.Reason == "flushing") {
   206  						delayOnce.Do(func() {
   207  							<-createReleased
   208  						})
   209  					}
   210  				},
   211  				WriteStallBegin: func(info WriteStallBeginInfo) {
   212  					fmt.Fprintln(&buf, info.String())
   213  					createReleased <- struct{}{}
   214  				},
   215  				WriteStallEnd: func() {
   216  					fmt.Fprintln(&buf, writeStallEnd)
   217  					select {
   218  					case stallEnded <- struct{}{}:
   219  					default:
   220  					}
   221  				},
   222  			}
   223  			d, err := Open("db", &Options{
   224  				EventListener:               listener,
   225  				FS:                          vfs.NewMem(),
   226  				MemTableStopWritesThreshold: 2,
   227  				L0CompactionThreshold:       2,
   228  				L0StopWritesThreshold:       2,
   229  			})
   230  			if err != nil {
   231  				t.Fatal(err)
   232  			}
   233  			defer d.Close()
   234  
   235  			for i := 0; i < flushCount; i++ {
   236  				if err := d.Set([]byte("a"), nil, NoSync); err != nil {
   237  					t.Fatal(err)
   238  				}
   239  				ch, err := d.AsyncFlush()
   240  				if err != nil {
   241  					t.Fatal(err)
   242  				}
   243  				// If we're delaying the flush (because we're testing for memtable
   244  				// write stalls), we can't wait for the flush to finish as doing so
   245  				// would deadlock. If we're not delaying the flush (because we're
   246  				// testing for L0 write stals), we wait for the flush to finish so we
   247  				// don't create too many memtables which would trigger a memtable write
   248  				// stall.
   249  				if !c.delayFlush {
   250  					<-ch
   251  				}
   252  				if strings.Contains(buf.String(), c.expected) {
   253  					break
   254  				}
   255  			}
   256  			<-stallEnded
   257  
   258  			events := buf.String()
   259  			require.Contains(t, events, c.expected)
   260  			require.Contains(t, events, writeStallEnd)
   261  			if testing.Verbose() {
   262  				t.Logf("\n%s", events)
   263  			}
   264  		})
   265  	}
   266  }