github.com/petermattis/pebble@v0.0.0-20190905164901-ab51a2166067/cmd/pebble/scan.go (about)

     1  // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"context"
     9  	"fmt"
    10  	"log"
    11  	"math"
    12  	"sync"
    13  	"sync/atomic"
    14  	"time"
    15  
    16  	"github.com/petermattis/pebble"
    17  	"github.com/spf13/cobra"
    18  	"golang.org/x/exp/rand"
    19  )
    20  
    21  var scanConfig struct {
    22  	reverse bool
    23  	rows    string
    24  	values  string
    25  }
    26  
    27  var scanCmd = &cobra.Command{
    28  	Use:   "scan <dir>",
    29  	Short: "run the scan benchmark",
    30  	Long:  ``,
    31  	Args:  cobra.ExactArgs(1),
    32  	Run:   runScan,
    33  }
    34  
    35  func init() {
    36  	scanCmd.Flags().BoolVarP(
    37  		&scanConfig.reverse, "reverse", "r", false, "reverse scan")
    38  	scanCmd.Flags().StringVar(
    39  		&scanConfig.rows, "rows", "100", "number of rows to scan in each operation")
    40  	scanCmd.Flags().StringVar(
    41  		&scanConfig.values, "values", "8",
    42  		"value size distribution [{zipf,uniform}:]min[-max][/<target-compression>]")
    43  }
    44  
    45  func runScan(cmd *cobra.Command, args []string) {
    46  	var (
    47  		bytes       int64
    48  		scanned     int64
    49  		lastBytes   int64
    50  		lastScanned int64
    51  		lastElapsed time.Duration
    52  	)
    53  
    54  	opts := pebble.Sync
    55  	if disableWAL {
    56  		opts = pebble.NoSync
    57  	}
    58  
    59  	rowDist, err := parseRandVarSpec(scanConfig.rows)
    60  	if err != nil {
    61  		fmt.Println(err)
    62  		return
    63  	}
    64  
    65  	valueDist, targetCompression, err := parseValuesSpec(scanConfig.values)
    66  	if err != nil {
    67  		fmt.Println(err)
    68  		return
    69  	}
    70  
    71  	runTest(args[0], test{
    72  		init: func(d DB, wg *sync.WaitGroup) {
    73  			const count = 100000
    74  			const batch = 1000
    75  
    76  			rng := rand.New(rand.NewSource(1449168817))
    77  			keys := make([][]byte, count)
    78  
    79  			for i := 0; i < count; {
    80  				b := d.NewBatch()
    81  				for end := i + batch; i < end; i++ {
    82  					keys[i] = mvccEncode(nil, encodeUint32Ascending([]byte("key-"), uint32(i)), uint64(i+1), 0)
    83  					length := int(valueDist.Uint64())
    84  					value := randomBlock(rng, length, targetCompression)
    85  					if err := b.Set(keys[i], value, nil); err != nil {
    86  						log.Fatal(err)
    87  					}
    88  				}
    89  				if err := b.Commit(opts); err != nil {
    90  					log.Fatal(err)
    91  				}
    92  			}
    93  
    94  			if err := d.Flush(); err != nil {
    95  				log.Fatal(err)
    96  			}
    97  
    98  			limiter, err := newFluctuatingRateLimiter(maxOpsPerSec)
    99  			if err != nil {
   100  				fmt.Println(err)
   101  				return
   102  			}
   103  			wg.Add(concurrency)
   104  			for i := 0; i < concurrency; i++ {
   105  				go func(i int) {
   106  					defer wg.Done()
   107  
   108  					rng := rand.New(rand.NewSource(uint64(i)))
   109  					startKeyBuf := append(make([]byte, 0, 64), []byte("key-")...)
   110  					endKeyBuf := append(make([]byte, 0, 64), []byte("key-")...)
   111  					minTS := encodeUint64Ascending(nil, math.MaxUint64)
   112  
   113  					for {
   114  						limiter.Wait(context.Background())
   115  						rows := int(rowDist.Uint64())
   116  						startIdx := rng.Int31n(int32(len(keys) - rows))
   117  						startKey := encodeUint32Ascending(startKeyBuf[:4], uint32(startIdx))
   118  						endKey := encodeUint32Ascending(endKeyBuf[:4], uint32(startIdx+int32(rows)))
   119  
   120  						var count int
   121  						var nbytes int64
   122  						if scanConfig.reverse {
   123  							count, nbytes = mvccReverseScan(d, startKey, endKey, minTS)
   124  						} else {
   125  							count, nbytes = mvccForwardScan(d, startKey, endKey, minTS)
   126  						}
   127  
   128  						if count != rows {
   129  							log.Fatalf("scanned %d, expected %d\n", count, rows)
   130  						}
   131  
   132  						atomic.AddInt64(&bytes, nbytes)
   133  						atomic.AddInt64(&scanned, int64(count))
   134  					}
   135  				}(i)
   136  			}
   137  		},
   138  
   139  		tick: func(elapsed time.Duration, i int) {
   140  			if i%20 == 0 {
   141  				fmt.Println("_elapsed_______rows/sec_______MB/sec_______ns/row")
   142  			}
   143  
   144  			curBytes := atomic.LoadInt64(&bytes)
   145  			curScanned := atomic.LoadInt64(&scanned)
   146  			dur := elapsed - lastElapsed
   147  			fmt.Printf("%8s %14.1f %12.1f %12.1f\n",
   148  				time.Duration(elapsed.Seconds()+0.5)*time.Second,
   149  				float64(curScanned-lastScanned)/dur.Seconds(),
   150  				float64(curBytes-lastBytes)/(dur.Seconds()*(1<<20)),
   151  				float64(dur)/float64(curScanned-lastScanned),
   152  			)
   153  			lastBytes = curBytes
   154  			lastScanned = curScanned
   155  			lastElapsed = elapsed
   156  		},
   157  
   158  		done: func(elapsed time.Duration) {
   159  			curBytes := atomic.LoadInt64(&bytes)
   160  			curScanned := atomic.LoadInt64(&scanned)
   161  			fmt.Println("\n_elapsed___ops/sec(cum)__MB/sec(cum)__ns/row(avg)")
   162  			fmt.Printf("%7.1fs %14.1f %12.1f %12.1f\n\n",
   163  				elapsed.Seconds(),
   164  				float64(curScanned)/elapsed.Seconds(),
   165  				float64(curBytes)/(elapsed.Seconds()*(1<<20)),
   166  				float64(elapsed)/float64(curScanned),
   167  			)
   168  		},
   169  	})
   170  }