github.com/cockroachdb/pebble@v1.1.2/cmd/pebble/tombstone.go (about)

     1  // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"fmt"
     9  	"log"
    10  	"sync"
    11  	"time"
    12  
    13  	"github.com/cockroachdb/errors"
    14  	"github.com/cockroachdb/pebble/internal/humanize"
    15  	"github.com/spf13/cobra"
    16  )
    17  
    18  func init() {
    19  	// NB: the tombstone workload piggybacks off the existing flags and
    20  	// configs for the queue and ycsb workloads.
    21  	initQueue(tombstoneCmd)
    22  	initYCSB(tombstoneCmd)
    23  }
    24  
    25  var tombstoneCmd = &cobra.Command{
    26  	Use:   "tombstone <dir>",
    27  	Short: "run the mixed-workload point tombstone benchmark",
    28  	Long: `
    29  Run a customizable YCSB workload, alongside a single-writer, fixed-sized queue
    30  workload. This command is intended for evaluating compaction heuristics
    31  surrounding point tombstones.
    32  
    33  The queue workload writes a point tombstone with every operation. A compaction
    34  strategy that does not account for point tombstones may accumulate many
    35  uncompacted tombstones, causing steady growth of the disk space consumed by
    36  the queue keyspace.
    37  
    38  The --queue-values flag controls the distribution of the queue value sizes.
    39  Larger values are more likely to exhibit problematic point tombstone behavior
    40  on a database using a min-overlapping ratio heuristic because the compact
    41  point tombstones may overlap many tables in the next level.
    42  
    43  The --queue-size flag controls the fixed number of live keys in the queue. Low
    44  queue sizes may not exercise problematic tombstone behavior if queue sets and
    45  deletes get written to the same sstable. The large-valued sets can serve as a
    46  counterweight to the point tombstones, narrowing the keyrange of the sstable
    47  inflating its size relative to its overlap with the next level.
    48  	`,
    49  	Args: cobra.ExactArgs(1),
    50  	RunE: runTombstoneCmd,
    51  }
    52  
    53  func runTombstoneCmd(cmd *cobra.Command, args []string) error {
    54  	if wipe && ycsbConfig.prepopulatedKeys > 0 {
    55  		return errors.New("--wipe and --prepopulated-keys both specified which is nonsensical")
    56  	}
    57  
    58  	weights, err := ycsbParseWorkload(ycsbConfig.workload)
    59  	if err != nil {
    60  		return err
    61  	}
    62  
    63  	keyDist, err := ycsbParseKeyDist(ycsbConfig.keys)
    64  	if err != nil {
    65  		return err
    66  	}
    67  
    68  	batchDist := ycsbConfig.batch
    69  	scanDist := ycsbConfig.scans
    70  	if err != nil {
    71  		return err
    72  	}
    73  
    74  	valueDist := ycsbConfig.values
    75  	y := newYcsb(weights, keyDist, batchDist, scanDist, valueDist)
    76  	q, queueOps := queueTest()
    77  
    78  	queueStart := []byte("queue-")
    79  	queueEnd := append(append([]byte{}, queueStart...), 0xFF)
    80  
    81  	var lastElapsed time.Duration
    82  	var lastQueueOps int64
    83  
    84  	var pdb pebbleDB
    85  	runTest(args[0], test{
    86  		init: func(d DB, wg *sync.WaitGroup) {
    87  			pdb = d.(pebbleDB)
    88  			y.init(d, wg)
    89  			q.init(d, wg)
    90  		},
    91  		tick: func(elapsed time.Duration, i int) {
    92  			if i%20 == 0 {
    93  				fmt.Println("                                             queue                         ycsb")
    94  				fmt.Println("________elapsed______queue_size__ops/sec(inst)___ops/sec(cum)__ops/sec(inst)___ops/sec(cum)")
    95  			}
    96  
    97  			curQueueOps := queueOps.Load()
    98  			dur := elapsed - lastElapsed
    99  			queueOpsPerSec := float64(curQueueOps-lastQueueOps) / dur.Seconds()
   100  			queueCumOpsPerSec := float64(curQueueOps) / elapsed.Seconds()
   101  
   102  			lastQueueOps = curQueueOps
   103  			lastElapsed = elapsed
   104  
   105  			var ycsbOpsPerSec, ycsbCumOpsPerSec float64
   106  			y.reg.Tick(func(tick histogramTick) {
   107  				h := tick.Hist
   108  				ycsbOpsPerSec = float64(h.TotalCount()) / tick.Elapsed.Seconds()
   109  				ycsbCumOpsPerSec = float64(tick.Cumulative.TotalCount()) / elapsed.Seconds()
   110  			})
   111  
   112  			queueSize, err := pdb.d.EstimateDiskUsage(queueStart, queueEnd)
   113  			if err != nil {
   114  				log.Fatal(err)
   115  			}
   116  			fmt.Printf("%15s %15s %14.1f %14.1f %14.1f %14.1f\n",
   117  				time.Duration(elapsed.Seconds()+0.5)*time.Second,
   118  				humanize.Bytes.Uint64(queueSize),
   119  				queueOpsPerSec,
   120  				queueCumOpsPerSec,
   121  				ycsbOpsPerSec,
   122  				ycsbCumOpsPerSec)
   123  		},
   124  		done: func(elapsed time.Duration) {
   125  			fmt.Println("________elapsed______queue_size")
   126  			queueSize, err := pdb.d.EstimateDiskUsage(queueStart, queueEnd)
   127  			if err != nil {
   128  				log.Fatal(err)
   129  			}
   130  			fmt.Printf("%15s %15s\n", elapsed.Truncate(time.Second), humanize.Bytes.Uint64(queueSize))
   131  		},
   132  	})
   133  	return nil
   134  }