github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/metrics_test.go (about)

     1  // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package pebble
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"math/rand"
    11  	"strconv"
    12  	"strings"
    13  	"sync"
    14  	"testing"
    15  	"time"
    16  
    17  	"github.com/cockroachdb/datadriven"
    18  	"github.com/cockroachdb/pebble/internal/humanize"
    19  	"github.com/cockroachdb/pebble/internal/testkeys"
    20  	"github.com/cockroachdb/pebble/vfs"
    21  	"github.com/cockroachdb/redact"
    22  	"github.com/stretchr/testify/require"
    23  )
    24  
    25  func exampleMetrics() Metrics {
    26  	var m Metrics
    27  	m.BlockCache.Size = 1
    28  	m.BlockCache.Count = 2
    29  	m.BlockCache.Hits = 3
    30  	m.BlockCache.Misses = 4
    31  	m.Compact.Count = 5
    32  	m.Compact.DefaultCount = 27
    33  	m.Compact.DeleteOnlyCount = 28
    34  	m.Compact.ElisionOnlyCount = 29
    35  	m.Compact.MoveCount = 30
    36  	m.Compact.ReadCount = 31
    37  	m.Compact.RewriteCount = 32
    38  	m.Compact.MultiLevelCount = 33
    39  	m.Compact.EstimatedDebt = 6
    40  	m.Compact.InProgressBytes = 7
    41  	m.Compact.NumInProgress = 2
    42  	m.Flush.Count = 8
    43  	m.Flush.AsIngestBytes = 34
    44  	m.Flush.AsIngestTableCount = 35
    45  	m.Flush.AsIngestCount = 36
    46  	m.Filter.Hits = 9
    47  	m.Filter.Misses = 10
    48  	m.MemTable.Size = 11
    49  	m.MemTable.Count = 12
    50  	m.MemTable.ZombieSize = 13
    51  	m.MemTable.ZombieCount = 14
    52  	m.Snapshots.Count = 4
    53  	m.Snapshots.EarliestSeqNum = 1024
    54  	m.Table.ZombieSize = 15
    55  	m.Table.BackingTableCount = 1
    56  	m.Table.BackingTableSize = 2 << 20
    57  	m.Table.ZombieCount = 16
    58  	m.TableCache.Size = 17
    59  	m.TableCache.Count = 18
    60  	m.TableCache.Hits = 19
    61  	m.TableCache.Misses = 20
    62  	m.TableIters = 21
    63  	m.WAL.Files = 22
    64  	m.WAL.ObsoleteFiles = 23
    65  	m.WAL.Size = 24
    66  	m.WAL.BytesIn = 25
    67  	m.WAL.BytesWritten = 26
    68  	m.Ingest.Count = 27
    69  
    70  	for i := range m.Levels {
    71  		l := &m.Levels[i]
    72  		base := uint64((i + 1) * 100)
    73  		l.Sublevels = int32(i + 1)
    74  		l.NumFiles = int64(base) + 1
    75  		l.NumVirtualFiles = uint64(base) + 1
    76  		l.VirtualSize = base + 3
    77  		l.Size = int64(base) + 2
    78  		l.Score = float64(base) + 3
    79  		l.BytesIn = base + 4
    80  		l.BytesIngested = base + 4
    81  		l.BytesMoved = base + 6
    82  		l.BytesRead = base + 7
    83  		l.BytesCompacted = base + 8
    84  		l.BytesFlushed = base + 9
    85  		l.TablesCompacted = base + 10
    86  		l.TablesFlushed = base + 11
    87  		l.TablesIngested = base + 12
    88  		l.TablesMoved = base + 13
    89  		l.MultiLevel.BytesInTop = base + 4
    90  		l.MultiLevel.BytesIn = base + 4
    91  		l.MultiLevel.BytesRead = base + 4
    92  	}
    93  	return m
    94  }
    95  
    96  func TestMetrics(t *testing.T) {
    97  	opts := &Options{
    98  		Comparer:              testkeys.Comparer,
    99  		FormatMajorVersion:    FormatNewest,
   100  		FS:                    vfs.NewMem(),
   101  		L0CompactionThreshold: 8,
   102  	}
   103  	opts.Experimental.EnableValueBlocks = func() bool { return true }
   104  	opts.Levels = append(opts.Levels, LevelOptions{TargetFileSize: 50})
   105  
   106  	// Prevent foreground flushes and compactions from triggering asynchronous
   107  	// follow-up compactions. This avoids asynchronously-scheduled work from
   108  	// interfering with the expected metrics output and reduces test flakiness.
   109  	opts.DisableAutomaticCompactions = true
   110  
   111  	// Increase the threshold for memtable stalls to allow for more flushable
   112  	// ingests.
   113  	opts.MemTableStopWritesThreshold = 4
   114  
   115  	d, err := Open("", opts)
   116  	require.NoError(t, err)
   117  	defer func() {
   118  		require.NoError(t, d.Close())
   119  	}()
   120  
   121  	iters := make(map[string]*Iterator)
   122  	defer func() {
   123  		for _, i := range iters {
   124  			require.NoError(t, i.Close())
   125  		}
   126  	}()
   127  
   128  	datadriven.RunTest(t, "testdata/metrics", func(t *testing.T, td *datadriven.TestData) string {
   129  		switch td.Cmd {
   130  		case "example":
   131  			m := exampleMetrics()
   132  			res := m.String()
   133  
   134  			// Nothing in the metrics should be redacted.
   135  			redacted := string(redact.Sprintf("%s", &m).Redact())
   136  			if redacted != res {
   137  				td.Fatalf(t, "redacted metrics don't match\nunredacted:\n%s\nredacted:%s\n", res, redacted)
   138  			}
   139  			return res
   140  
   141  		case "batch":
   142  			b := d.NewBatch()
   143  			if err := runBatchDefineCmd(td, b); err != nil {
   144  				return err.Error()
   145  			}
   146  			b.Commit(nil)
   147  			return ""
   148  
   149  		case "build":
   150  			if err := runBuildCmd(td, d, d.opts.FS); err != nil {
   151  				return err.Error()
   152  			}
   153  			return ""
   154  
   155  		case "compact":
   156  			if err := runCompactCmd(td, d); err != nil {
   157  				return err.Error()
   158  			}
   159  
   160  			d.mu.Lock()
   161  			s := d.mu.versions.currentVersion().String()
   162  			d.mu.Unlock()
   163  			return s
   164  
   165  		case "delay-flush":
   166  			d.mu.Lock()
   167  			defer d.mu.Unlock()
   168  			switch td.Input {
   169  			case "enable":
   170  				d.mu.compact.flushing = true
   171  			case "disable":
   172  				d.mu.compact.flushing = false
   173  			default:
   174  				return fmt.Sprintf("unknown directive %q (expected 'enable'/'disable')", td.Input)
   175  			}
   176  			return ""
   177  
   178  		case "flush":
   179  			if err := d.Flush(); err != nil {
   180  				return err.Error()
   181  			}
   182  
   183  			d.mu.Lock()
   184  			s := d.mu.versions.currentVersion().String()
   185  			d.mu.Unlock()
   186  			return s
   187  
   188  		case "ingest":
   189  			if err := runIngestCmd(td, d, d.opts.FS); err != nil {
   190  				return err.Error()
   191  			}
   192  			return ""
   193  
   194  		case "lsm":
   195  			d.mu.Lock()
   196  			s := d.mu.versions.currentVersion().String()
   197  			d.mu.Unlock()
   198  			return s
   199  
   200  		case "ingest-and-excise":
   201  			if err := runIngestAndExciseCmd(td, d, d.opts.FS); err != nil {
   202  				return err.Error()
   203  			}
   204  			return ""
   205  
   206  		case "iter-close":
   207  			if len(td.CmdArgs) != 1 {
   208  				return "iter-close <name>"
   209  			}
   210  			name := td.CmdArgs[0].String()
   211  			if iter := iters[name]; iter != nil {
   212  				if err := iter.Close(); err != nil {
   213  					return err.Error()
   214  				}
   215  				delete(iters, name)
   216  			} else {
   217  				return fmt.Sprintf("%s: not found", name)
   218  			}
   219  
   220  			// The deletion of obsolete files happens asynchronously when an iterator
   221  			// is closed. Wait for the obsolete tables to be deleted.
   222  			d.cleanupManager.Wait()
   223  			return ""
   224  
   225  		case "iter-new":
   226  			if len(td.CmdArgs) != 1 {
   227  				return "iter-new <name>"
   228  			}
   229  			name := td.CmdArgs[0].String()
   230  			if iter := iters[name]; iter != nil {
   231  				if err := iter.Close(); err != nil {
   232  					return err.Error()
   233  				}
   234  			}
   235  			iter, _ := d.NewIter(nil)
   236  			// Some iterators (eg. levelIter) do not instantiate the underlying
   237  			// iterator until the first positioning call. Position the iterator
   238  			// so that levelIters will have loaded an sstable.
   239  			iter.First()
   240  			iters[name] = iter
   241  			return ""
   242  
   243  		case "metrics":
   244  			// The asynchronous loading of table stats can change metrics, so
   245  			// wait for all the tables' stats to be loaded.
   246  			d.mu.Lock()
   247  			d.waitTableStats()
   248  			d.mu.Unlock()
   249  
   250  			return d.Metrics().StringForTests()
   251  
   252  		case "metrics-value":
   253  			// metrics-value confirms the value of a given metric. Note that there
   254  			// are some metrics which aren't deterministic and behave differently
   255  			// for invariant/non-invariant builds. An example of this is cache
   256  			// hit rates. Under invariant builds, the excising code will try
   257  			// to create iterators and confirm that the virtual sstable bounds
   258  			// are accurate. Reads on these iterators will change the cache hit
   259  			// rates.
   260  			lines := strings.Split(td.Input, "\n")
   261  			m := d.Metrics()
   262  			// TODO(bananabrick): Use reflection to pull the values associated
   263  			// with the metrics fields.
   264  			var buf bytes.Buffer
   265  			for i := range lines {
   266  				line := lines[i]
   267  				if line == "num-backing" {
   268  					buf.WriteString(fmt.Sprintf("%d\n", m.Table.BackingTableCount))
   269  				} else if line == "backing-size" {
   270  					buf.WriteString(fmt.Sprintf("%s\n", humanize.Bytes.Uint64(m.Table.BackingTableSize)))
   271  				} else if line == "virtual-size" {
   272  					buf.WriteString(fmt.Sprintf("%s\n", humanize.Bytes.Uint64(m.VirtualSize())))
   273  				} else if strings.HasPrefix(line, "num-virtual") {
   274  					splits := strings.Split(line, " ")
   275  					if len(splits) == 1 {
   276  						buf.WriteString(fmt.Sprintf("%d\n", m.NumVirtual()))
   277  						continue
   278  					}
   279  					// Level is specified.
   280  					l, err := strconv.Atoi(splits[1])
   281  					if err != nil {
   282  						panic(err)
   283  					}
   284  					if l >= numLevels {
   285  						panic(fmt.Sprintf("invalid level %d", l))
   286  					}
   287  					buf.WriteString(fmt.Sprintf("%d\n", m.Levels[l].NumVirtualFiles))
   288  				} else {
   289  					panic(fmt.Sprintf("invalid field: %s", line))
   290  				}
   291  			}
   292  			return buf.String()
   293  
   294  		case "disk-usage":
   295  			return humanize.Bytes.Uint64(d.Metrics().DiskSpaceUsage()).String()
   296  
   297  		case "additional-metrics":
   298  			// The asynchronous loading of table stats can change metrics, so
   299  			// wait for all the tables' stats to be loaded.
   300  			d.mu.Lock()
   301  			d.waitTableStats()
   302  			d.mu.Unlock()
   303  
   304  			m := d.Metrics()
   305  			var b strings.Builder
   306  			fmt.Fprintf(&b, "block bytes written:\n")
   307  			fmt.Fprintf(&b, " __level___data-block__value-block\n")
   308  			for i := range m.Levels {
   309  				fmt.Fprintf(&b, "%7d ", i)
   310  				fmt.Fprintf(&b, "%12s %12s\n",
   311  					humanize.Bytes.Uint64(m.Levels[i].Additional.BytesWrittenDataBlocks),
   312  					humanize.Bytes.Uint64(m.Levels[i].Additional.BytesWrittenValueBlocks))
   313  			}
   314  			return b.String()
   315  
   316  		default:
   317  			return fmt.Sprintf("unknown command: %s", td.Cmd)
   318  		}
   319  	})
   320  }
   321  
   322  func TestMetricsWAmpDisableWAL(t *testing.T) {
   323  	d, err := Open("", &Options{FS: vfs.NewMem(), DisableWAL: true})
   324  	require.NoError(t, err)
   325  	ks := testkeys.Alpha(2)
   326  	wo := WriteOptions{Sync: false}
   327  	for i := 0; i < 5; i++ {
   328  		v := []byte(strconv.Itoa(i))
   329  		for j := int64(0); j < ks.Count(); j++ {
   330  			require.NoError(t, d.Set(testkeys.Key(ks, j), v, &wo))
   331  		}
   332  		require.NoError(t, d.Flush())
   333  		require.NoError(t, d.Compact([]byte("a"), []byte("z"), false /* parallelize */))
   334  	}
   335  	m := d.Metrics()
   336  	tot := m.Total()
   337  	require.Greater(t, tot.WriteAmp(), 1.0)
   338  	require.NoError(t, d.Close())
   339  }
   340  
   341  // TestMetricsWALBytesWrittenMonotonicity tests that the
   342  // Metrics.WAL.BytesWritten metric is always nondecreasing.
   343  // It's a regression test for issue #3505.
   344  func TestMetricsWALBytesWrittenMonotonicity(t *testing.T) {
   345  	fs := vfs.NewMem()
   346  	d, err := Open("", &Options{
   347  		FS: fs,
   348  		// Use a tiny memtable size so that we get frequent flushes. While a
   349  		// flush is in-progress or completing, the WAL bytes written should
   350  		// remain nondecreasing.
   351  		MemTableSize: 1 << 20, /* 20 KiB */
   352  	})
   353  	require.NoError(t, err)
   354  
   355  	stopCh := make(chan struct{})
   356  
   357  	ks := testkeys.Alpha(3)
   358  	var wg sync.WaitGroup
   359  	const concurrentWriters = 5
   360  	wg.Add(concurrentWriters)
   361  	for w := 0; w < concurrentWriters; w++ {
   362  		go func() {
   363  			defer wg.Done()
   364  			data := make([]byte, 1<<10) // 1 KiB
   365  			rng := rand.New(rand.NewSource(time.Now().UnixNano()))
   366  			_, err := rng.Read(data)
   367  			require.NoError(t, err)
   368  
   369  			buf := make([]byte, ks.MaxLen())
   370  			for i := 0; ; i++ {
   371  				select {
   372  				case <-stopCh:
   373  					return
   374  				default:
   375  				}
   376  				n := testkeys.WriteKey(buf, ks, int64(i)%ks.Count())
   377  				require.NoError(t, d.Set(buf[:n], data, NoSync))
   378  			}
   379  		}()
   380  	}
   381  
   382  	func() {
   383  		defer func() { close(stopCh) }()
   384  		abort := time.After(time.Second)
   385  		var prevWALBytesWritten uint64
   386  		for {
   387  			select {
   388  			case <-abort:
   389  				return
   390  			default:
   391  			}
   392  
   393  			m := d.Metrics()
   394  			if m.WAL.BytesWritten < prevWALBytesWritten {
   395  				t.Fatalf("WAL bytes written decreased: %d -> %d", prevWALBytesWritten, m.WAL.BytesWritten)
   396  			}
   397  			prevWALBytesWritten = m.WAL.BytesWritten
   398  		}
   399  	}()
   400  	wg.Wait()
   401  }