github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/perf/codec-perf-rig/main.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package main
    23  
    24  import (
    25  	"bytes"
    26  	"context"
    27  	"encoding/binary"
    28  	"fmt"
    29  	"time"
    30  
    31  	flag "github.com/juju/gnuflag"
    32  
    33  	"github.com/dolthub/dolt/go/store/chunks"
    34  	"github.com/dolthub/dolt/go/store/d"
    35  	"github.com/dolthub/dolt/go/store/datas"
    36  	"github.com/dolthub/dolt/go/store/types"
    37  	"github.com/dolthub/dolt/go/store/util/profile"
    38  )
    39  
    40  var (
    41  	count    = flag.Uint64("count", 100000, "number of elements")
    42  	blobSize = flag.Uint64("blobsize", 2<<24 /* 32MB */, "size of blob of create")
    43  )
    44  
    45  const numberSize = uint64(8)
    46  const strPrefix = "i am a 32 bytes.....%12d"
    47  const stringSize = uint64(32)
    48  const structSize = uint64(64)
    49  
    50  func main() {
    51  	profile.RegisterProfileFlags(flag.CommandLine)
    52  	flag.Parse(true)
    53  
    54  	buildCount := *count
    55  	insertCount := buildCount
    56  	defer profile.MaybeStartProfile().Stop()
    57  
    58  	collectionTypes := []string{"List", "Set", "Map"}
    59  	buildFns := []buildCollectionFn{buildList, buildSet, buildMap}
    60  	buildIncrFns := []buildCollectionFn{buildListIncrementally, buildSetIncrementally, buildMapIncrementally}
    61  	readFns := []readCollectionFn{readList, readSet, readMap}
    62  
    63  	elementTypes := []string{"numbers (8 B)", "strings (32 B)", "structs (64 B)"}
    64  	elementSizes := []uint64{numberSize, stringSize, structSize}
    65  	valueFns := []createValueFn{createNumber, createString, createStruct}
    66  
    67  	for i, colType := range collectionTypes {
    68  		fmt.Printf("Testing %s: \t\tbuild %d\t\t\tscan %d\t\t\tinsert %d\n", colType, buildCount, buildCount, insertCount)
    69  
    70  		for j, elementType := range elementTypes {
    71  			valueFn := valueFns[j]
    72  
    73  			// Build One-Time
    74  			storage := &chunks.MemoryStorage{}
    75  			db := datas.NewDatabase(storage.NewView())
    76  			ds, err := db.GetDataset(context.Background(), "test")
    77  			d.Chk.NoError(err)
    78  			t1 := time.Now()
    79  			col := buildFns[i](db, buildCount, valueFn)
    80  			ds, err = db.CommitValue(context.Background(), ds, col)
    81  			d.Chk.NoError(err)
    82  			buildDuration := time.Since(t1)
    83  
    84  			// Read
    85  			t1 = time.Now()
    86  			val, ok, err := ds.MaybeHeadValue()
    87  			d.Chk.NoError(err)
    88  			d.Chk.True(ok)
    89  			col = val.(types.Collection)
    90  			readFns[i](col)
    91  			readDuration := time.Since(t1)
    92  
    93  			// Build Incrementally
    94  			storage = &chunks.MemoryStorage{}
    95  			db = datas.NewDatabase(storage.NewView())
    96  			ds, err = db.GetDataset(context.Background(), "test")
    97  			d.Chk.NoError(err)
    98  			t1 = time.Now()
    99  			col = buildIncrFns[i](db, insertCount, valueFn)
   100  			ds, err = db.CommitValue(context.Background(), ds, col)
   101  			d.Chk.NoError(err)
   102  			incrDuration := time.Since(t1)
   103  
   104  			elementSize := elementSizes[j]
   105  			buildSize := elementSize * buildCount
   106  			incrSize := elementSize * insertCount
   107  
   108  			fmt.Printf("%s\t\t%s\t\t%s\t\t%s\n", elementType, rate(buildDuration, buildSize), rate(readDuration, buildSize), rate(incrDuration, incrSize))
   109  		}
   110  		fmt.Println()
   111  	}
   112  
   113  	fmt.Printf("Testing Blob: \t\tbuild %d MB\t\t\tscan %d MB\n", *blobSize/1000000, *blobSize/1000000)
   114  
   115  	storage := &chunks.MemoryStorage{}
   116  	db := datas.NewDatabase(storage.NewView())
   117  	ds, err := db.GetDataset(context.Background(), "test")
   118  	d.Chk.NoError(err)
   119  
   120  	blobBytes := makeBlobBytes(*blobSize)
   121  	t1 := time.Now()
   122  	blob, err := types.NewBlob(context.Background(), db, bytes.NewReader(blobBytes))
   123  	d.Chk.NoError(err)
   124  	_, err = db.CommitValue(context.Background(), ds, blob)
   125  	d.Chk.NoError(err)
   126  	buildDuration := time.Since(t1)
   127  
   128  	db = datas.NewDatabase(storage.NewView())
   129  	ds, err = db.GetDataset(context.Background(), "test")
   130  	d.Chk.NoError(err)
   131  	t1 = time.Now()
   132  	blobVal, ok, err := ds.MaybeHeadValue()
   133  	d.Chk.NoError(err)
   134  	d.Chk.True(ok)
   135  	blob = blobVal.(types.Blob)
   136  	buff := &bytes.Buffer{}
   137  	blob.Copy(context.Background(), buff)
   138  	outBytes := buff.Bytes()
   139  	readDuration := time.Since(t1)
   140  	d.PanicIfFalse(bytes.Equal(blobBytes, outBytes))
   141  	fmt.Printf("\t\t\t%s\t\t%s\n\n", rate(buildDuration, *blobSize), rate(readDuration, *blobSize))
   142  }
   143  
   144  func rate(d time.Duration, size uint64) string {
   145  	return fmt.Sprintf("%d ms (%.2f MB/s)", uint64(d)/1000000, float64(size)*1000/float64(d))
   146  }
   147  
   148  type createValueFn func(i uint64) types.Value
   149  type buildCollectionFn func(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection
   150  type readCollectionFn func(value types.Collection)
   151  
   152  func makeBlobBytes(byteLength uint64) []byte {
   153  	buff := &bytes.Buffer{}
   154  	counter := uint64(0)
   155  	for uint64(buff.Len()) < byteLength {
   156  		err := binary.Write(buff, binary.BigEndian, counter)
   157  		d.Chk.NoError(err)
   158  		counter++
   159  	}
   160  	return buff.Bytes()
   161  }
   162  
   163  func createString(i uint64) types.Value {
   164  	return types.String(fmt.Sprintf("%s%d", strPrefix, i))
   165  }
   166  
   167  func createNumber(i uint64) types.Value {
   168  	return types.Float(i)
   169  }
   170  
   171  var structTemplate = types.MakeStructTemplate("S1", []string{"bool", "num", "str"})
   172  
   173  func createStruct(i uint64) types.Value {
   174  	st, err := structTemplate.NewStruct(types.Format_7_18, []types.Value{
   175  		types.Bool(i%2 == 0), // "bool"
   176  		types.Float(i),       // "num"
   177  		types.String(fmt.Sprintf("i am a 55 bytes............................%12d", i)), // "str"
   178  	})
   179  
   180  	d.Chk.NoError(err)
   181  
   182  	return st
   183  }
   184  
   185  func buildList(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
   186  	values := make([]types.Value, count)
   187  	for i := uint64(0); i < count; i++ {
   188  		values[i] = createFn(i)
   189  	}
   190  
   191  	l, err := types.NewList(context.Background(), vrw, values...)
   192  
   193  	d.Chk.NoError(err)
   194  
   195  	return l
   196  }
   197  
   198  func buildListIncrementally(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
   199  	l, err := types.NewList(context.Background(), vrw)
   200  
   201  	d.Chk.NoError(err)
   202  
   203  	le := l.Edit()
   204  
   205  	for i := uint64(0); i < count; i++ {
   206  		le.Append(createFn(i))
   207  	}
   208  
   209  	l, err = le.List(context.Background())
   210  
   211  	d.Chk.NoError(err)
   212  
   213  	return l
   214  }
   215  
   216  func readList(c types.Collection) {
   217  	_ = c.(types.List).IterAll(context.Background(), func(v types.Value, idx uint64) error {
   218  		return nil
   219  	})
   220  }
   221  
   222  func buildSet(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
   223  	values := make([]types.Value, count)
   224  	for i := uint64(0); i < count; i++ {
   225  		values[i] = createFn(i)
   226  	}
   227  
   228  	s, err := types.NewSet(context.Background(), vrw, values...)
   229  
   230  	d.Chk.NoError(err)
   231  
   232  	return s
   233  }
   234  
   235  func buildSetIncrementally(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
   236  	s, err := types.NewSet(context.Background(), vrw)
   237  	d.Chk.NoError(err)
   238  
   239  	se := s.Edit()
   240  	for i := uint64(0); i < count; i++ {
   241  		se.Insert(createFn(i))
   242  	}
   243  
   244  	s, err = se.Set(context.Background())
   245  	d.Chk.NoError(err)
   246  
   247  	return s
   248  }
   249  
   250  func readSet(c types.Collection) {
   251  	_ = c.(types.Set).IterAll(context.Background(), func(v types.Value) error {
   252  		return nil
   253  	})
   254  }
   255  
   256  func buildMap(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
   257  	values := make([]types.Value, count*2)
   258  	for i := uint64(0); i < count*2; i++ {
   259  		values[i] = createFn(i)
   260  	}
   261  
   262  	m, err := types.NewMap(context.Background(), vrw, values...)
   263  
   264  	d.Chk.NoError(err)
   265  
   266  	return m
   267  }
   268  
   269  func buildMapIncrementally(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
   270  	m, err := types.NewMap(context.Background(), vrw)
   271  	d.Chk.NoError(err)
   272  
   273  	me := m.Edit()
   274  
   275  	for i := uint64(0); i < count*2; i += 2 {
   276  		me.Set(createFn(i), createFn(i+1))
   277  	}
   278  
   279  	m, err = me.Map(context.Background())
   280  	d.Chk.NoError(err)
   281  
   282  	return m
   283  }
   284  
   285  func readMap(c types.Collection) {
   286  	_ = c.(types.Map).IterAll(context.Background(), func(k types.Value, v types.Value) error {
   287  		return nil
   288  	})
   289  }