github.com/petermattis/pebble@v0.0.0-20190905164901-ab51a2166067/cmd/pebble/sync.go (about)

     1  // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"context"
     9  	"fmt"
    10  	"log"
    11  	"sync"
    12  	"sync/atomic"
    13  	"time"
    14  
    15  	"github.com/petermattis/pebble"
    16  	"github.com/spf13/cobra"
    17  	"golang.org/x/exp/rand"
    18  )
    19  
    20  var syncConfig struct {
    21  	batch   string
    22  	walOnly bool
    23  	values  string
    24  }
    25  
    26  var syncCmd = &cobra.Command{
    27  	Use:   "sync <dir>",
    28  	Short: "run the sync benchmark",
    29  	Long:  ``,
    30  	Args:  cobra.ExactArgs(1),
    31  	Run:   runSync,
    32  }
    33  
    34  func init() {
    35  	syncCmd.Flags().StringVar(
    36  		&syncConfig.batch, "batch", "5",
    37  		"batch size distribution [{zipf,uniform}:]min[-max]")
    38  	syncCmd.Flags().BoolVar(
    39  		&syncConfig.walOnly, "wal-only", false, "write data only to the WAL")
    40  	syncCmd.Flags().StringVar(
    41  		&syncConfig.values, "values", "uniform:60-80/1.0",
    42  		"value size distribution [{zipf,uniform}:]min[-max][/<target-compression>]")
    43  }
    44  
    45  func runSync(cmd *cobra.Command, args []string) {
    46  	reg := newHistogramRegistry()
    47  	var bytes, lastBytes uint64
    48  
    49  	opts := pebble.Sync
    50  	if disableWAL {
    51  		opts = pebble.NoSync
    52  	}
    53  
    54  	batchDist, err := parseRandVarSpec(syncConfig.batch)
    55  	if err != nil {
    56  		fmt.Println(err)
    57  		return
    58  	}
    59  
    60  	valueDist, targetCompression, err := parseValuesSpec(syncConfig.values)
    61  	if err != nil {
    62  		fmt.Println(err)
    63  		return
    64  	}
    65  
    66  	runTest(args[0], test{
    67  		init: func(d DB, wg *sync.WaitGroup) {
    68  			limiter, err := newFluctuatingRateLimiter(maxOpsPerSec)
    69  			if err != nil {
    70  				fmt.Println(err)
    71  				return
    72  			}
    73  			wg.Add(concurrency)
    74  			for i := 0; i < concurrency; i++ {
    75  				latency := reg.Register("ops")
    76  				go func() {
    77  					defer wg.Done()
    78  
    79  					rand := rand.New(rand.NewSource(uint64(time.Now().UnixNano())))
    80  					var raw []byte
    81  					var buf []byte
    82  					for {
    83  						limiter.Wait(context.Background())
    84  						start := time.Now()
    85  						b := d.NewBatch()
    86  						var n uint64
    87  						count := int(batchDist.Uint64())
    88  						for j := 0; j < count; j++ {
    89  							length := int(valueDist.Uint64())
    90  							block := randomBlock(rand, length, targetCompression)
    91  
    92  							if syncConfig.walOnly {
    93  								if err := b.LogData(block, nil); err != nil {
    94  									log.Fatal(err)
    95  								}
    96  							} else {
    97  								raw = encodeUint32Ascending(raw[:0], rand.Uint32())
    98  								key := mvccEncode(buf[:0], raw, 0, 0)
    99  								buf = key[:0]
   100  								if err := b.Set(key, block, nil); err != nil {
   101  									log.Fatal(err)
   102  								}
   103  							}
   104  							n += uint64(len(block))
   105  						}
   106  						if err := b.Commit(opts); err != nil {
   107  							log.Fatal(err)
   108  						}
   109  						latency.Record(time.Since(start))
   110  						atomic.AddUint64(&bytes, n)
   111  					}
   112  				}()
   113  			}
   114  		},
   115  
   116  		tick: func(elapsed time.Duration, i int) {
   117  			if i%20 == 0 {
   118  				fmt.Println("_elapsed____ops/sec___mb/sec__p50(ms)__p95(ms)__p99(ms)_pMax(ms)")
   119  			}
   120  			reg.Tick(func(tick histogramTick) {
   121  				h := tick.Hist
   122  				n := atomic.LoadUint64(&bytes)
   123  				fmt.Printf("%8s %10.1f %8.1f %8.1f %8.1f %8.1f %8.1f\n",
   124  					time.Duration(elapsed.Seconds()+0.5)*time.Second,
   125  					float64(h.TotalCount())/tick.Elapsed.Seconds(),
   126  					float64(n-lastBytes)/(1024.0*1024.0)/tick.Elapsed.Seconds(),
   127  					time.Duration(h.ValueAtQuantile(50)).Seconds()*1000,
   128  					time.Duration(h.ValueAtQuantile(95)).Seconds()*1000,
   129  					time.Duration(h.ValueAtQuantile(99)).Seconds()*1000,
   130  					time.Duration(h.ValueAtQuantile(100)).Seconds()*1000,
   131  				)
   132  				lastBytes = n
   133  			})
   134  		},
   135  
   136  		done: func(elapsed time.Duration) {
   137  			fmt.Println("\n_elapsed___ops(total)_ops/sec(cum)_mb/sec(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)")
   138  			reg.Tick(func(tick histogramTick) {
   139  				h := tick.Cumulative
   140  				fmt.Printf("%7.1fs %12d %12.1f %11.1f %8.1f %8.1f %8.1f %8.1f %8.1f\n\n",
   141  					elapsed.Seconds(), h.TotalCount(),
   142  					float64(h.TotalCount())/elapsed.Seconds(),
   143  					float64(atomic.LoadUint64(&bytes)/(1024.0*1024.0))/elapsed.Seconds(),
   144  					time.Duration(h.Mean()).Seconds()*1000,
   145  					time.Duration(h.ValueAtQuantile(50)).Seconds()*1000,
   146  					time.Duration(h.ValueAtQuantile(95)).Seconds()*1000,
   147  					time.Duration(h.ValueAtQuantile(99)).Seconds()*1000,
   148  					time.Duration(h.ValueAtQuantile(100)).Seconds()*1000)
   149  			})
   150  		},
   151  	})
   152  }