github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/perf/codec-perf-rig/main.go (about) 1 // Copyright 2019 Dolthub, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 // 15 // This file incorporates work covered by the following copyright and 16 // permission notice: 17 // 18 // Copyright 2016 Attic Labs, Inc. All rights reserved. 19 // Licensed under the Apache License, version 2.0: 20 // http://www.apache.org/licenses/LICENSE-2.0 21 22 package main 23 24 import ( 25 "bytes" 26 "context" 27 "encoding/binary" 28 "fmt" 29 "time" 30 31 flag "github.com/juju/gnuflag" 32 33 "github.com/dolthub/dolt/go/store/chunks" 34 "github.com/dolthub/dolt/go/store/d" 35 "github.com/dolthub/dolt/go/store/datas" 36 "github.com/dolthub/dolt/go/store/types" 37 "github.com/dolthub/dolt/go/store/util/profile" 38 ) 39 40 var ( 41 count = flag.Uint64("count", 100000, "number of elements") 42 blobSize = flag.Uint64("blobsize", 2<<24 /* 32MB */, "size of blob of create") 43 ) 44 45 const numberSize = uint64(8) 46 const strPrefix = "i am a 32 bytes.....%12d" 47 const stringSize = uint64(32) 48 const structSize = uint64(64) 49 50 func main() { 51 profile.RegisterProfileFlags(flag.CommandLine) 52 flag.Parse(true) 53 54 buildCount := *count 55 insertCount := buildCount 56 defer profile.MaybeStartProfile().Stop() 57 58 collectionTypes := []string{"List", "Set", "Map"} 59 buildFns := []buildCollectionFn{buildList, buildSet, buildMap} 60 buildIncrFns := []buildCollectionFn{buildListIncrementally, buildSetIncrementally, buildMapIncrementally} 61 readFns := []readCollectionFn{readList, readSet, readMap} 62 63 elementTypes := []string{"numbers (8 B)", "strings (32 B)", "structs (64 B)"} 64 elementSizes := []uint64{numberSize, stringSize, structSize} 65 valueFns := []createValueFn{createNumber, createString, createStruct} 66 67 for i, colType := range collectionTypes { 68 fmt.Printf("Testing %s: \t\tbuild %d\t\t\tscan %d\t\t\tinsert %d\n", colType, buildCount, buildCount, insertCount) 69 70 for j, elementType := range elementTypes { 71 valueFn := valueFns[j] 72 73 // Build One-Time 74 storage := &chunks.MemoryStorage{} 75 db := datas.NewDatabase(storage.NewView()) 76 ds, err := db.GetDataset(context.Background(), "test") 77 d.Chk.NoError(err) 78 t1 := time.Now() 79 col := buildFns[i](db, buildCount, valueFn) 80 ds, err = db.CommitValue(context.Background(), ds, col) 81 d.Chk.NoError(err) 82 buildDuration := time.Since(t1) 83 84 // Read 85 t1 = time.Now() 86 val, ok, err := ds.MaybeHeadValue() 87 d.Chk.NoError(err) 88 d.Chk.True(ok) 89 col = val.(types.Collection) 90 readFns[i](col) 91 readDuration := time.Since(t1) 92 93 // Build Incrementally 94 storage = &chunks.MemoryStorage{} 95 db = datas.NewDatabase(storage.NewView()) 96 ds, err = db.GetDataset(context.Background(), "test") 97 d.Chk.NoError(err) 98 t1 = time.Now() 99 col = buildIncrFns[i](db, insertCount, valueFn) 100 ds, err = db.CommitValue(context.Background(), ds, col) 101 d.Chk.NoError(err) 102 incrDuration := time.Since(t1) 103 104 elementSize := elementSizes[j] 105 buildSize := elementSize * buildCount 106 incrSize := elementSize * insertCount 107 108 fmt.Printf("%s\t\t%s\t\t%s\t\t%s\n", elementType, rate(buildDuration, buildSize), rate(readDuration, buildSize), rate(incrDuration, incrSize)) 109 } 110 fmt.Println() 111 } 112 113 fmt.Printf("Testing Blob: \t\tbuild %d MB\t\t\tscan %d MB\n", *blobSize/1000000, *blobSize/1000000) 114 115 storage := &chunks.MemoryStorage{} 116 db := datas.NewDatabase(storage.NewView()) 117 ds, err := db.GetDataset(context.Background(), "test") 118 d.Chk.NoError(err) 119 120 blobBytes := makeBlobBytes(*blobSize) 121 t1 := time.Now() 122 blob, err := types.NewBlob(context.Background(), db, bytes.NewReader(blobBytes)) 123 d.Chk.NoError(err) 124 _, err = db.CommitValue(context.Background(), ds, blob) 125 d.Chk.NoError(err) 126 buildDuration := time.Since(t1) 127 128 db = datas.NewDatabase(storage.NewView()) 129 ds, err = db.GetDataset(context.Background(), "test") 130 d.Chk.NoError(err) 131 t1 = time.Now() 132 blobVal, ok, err := ds.MaybeHeadValue() 133 d.Chk.NoError(err) 134 d.Chk.True(ok) 135 blob = blobVal.(types.Blob) 136 buff := &bytes.Buffer{} 137 blob.Copy(context.Background(), buff) 138 outBytes := buff.Bytes() 139 readDuration := time.Since(t1) 140 d.PanicIfFalse(bytes.Equal(blobBytes, outBytes)) 141 fmt.Printf("\t\t\t%s\t\t%s\n\n", rate(buildDuration, *blobSize), rate(readDuration, *blobSize)) 142 } 143 144 func rate(d time.Duration, size uint64) string { 145 return fmt.Sprintf("%d ms (%.2f MB/s)", uint64(d)/1000000, float64(size)*1000/float64(d)) 146 } 147 148 type createValueFn func(i uint64) types.Value 149 type buildCollectionFn func(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection 150 type readCollectionFn func(value types.Collection) 151 152 func makeBlobBytes(byteLength uint64) []byte { 153 buff := &bytes.Buffer{} 154 counter := uint64(0) 155 for uint64(buff.Len()) < byteLength { 156 err := binary.Write(buff, binary.BigEndian, counter) 157 d.Chk.NoError(err) 158 counter++ 159 } 160 return buff.Bytes() 161 } 162 163 func createString(i uint64) types.Value { 164 return types.String(fmt.Sprintf("%s%d", strPrefix, i)) 165 } 166 167 func createNumber(i uint64) types.Value { 168 return types.Float(i) 169 } 170 171 var structTemplate = types.MakeStructTemplate("S1", []string{"bool", "num", "str"}) 172 173 func createStruct(i uint64) types.Value { 174 st, err := structTemplate.NewStruct(types.Format_7_18, []types.Value{ 175 types.Bool(i%2 == 0), // "bool" 176 types.Float(i), // "num" 177 types.String(fmt.Sprintf("i am a 55 bytes............................%12d", i)), // "str" 178 }) 179 180 d.Chk.NoError(err) 181 182 return st 183 } 184 185 func buildList(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection { 186 values := make([]types.Value, count) 187 for i := uint64(0); i < count; i++ { 188 values[i] = createFn(i) 189 } 190 191 l, err := types.NewList(context.Background(), vrw, values...) 192 193 d.Chk.NoError(err) 194 195 return l 196 } 197 198 func buildListIncrementally(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection { 199 l, err := types.NewList(context.Background(), vrw) 200 201 d.Chk.NoError(err) 202 203 le := l.Edit() 204 205 for i := uint64(0); i < count; i++ { 206 le.Append(createFn(i)) 207 } 208 209 l, err = le.List(context.Background()) 210 211 d.Chk.NoError(err) 212 213 return l 214 } 215 216 func readList(c types.Collection) { 217 _ = c.(types.List).IterAll(context.Background(), func(v types.Value, idx uint64) error { 218 return nil 219 }) 220 } 221 222 func buildSet(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection { 223 values := make([]types.Value, count) 224 for i := uint64(0); i < count; i++ { 225 values[i] = createFn(i) 226 } 227 228 s, err := types.NewSet(context.Background(), vrw, values...) 229 230 d.Chk.NoError(err) 231 232 return s 233 } 234 235 func buildSetIncrementally(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection { 236 s, err := types.NewSet(context.Background(), vrw) 237 d.Chk.NoError(err) 238 239 se := s.Edit() 240 for i := uint64(0); i < count; i++ { 241 se.Insert(createFn(i)) 242 } 243 244 s, err = se.Set(context.Background()) 245 d.Chk.NoError(err) 246 247 return s 248 } 249 250 func readSet(c types.Collection) { 251 _ = c.(types.Set).IterAll(context.Background(), func(v types.Value) error { 252 return nil 253 }) 254 } 255 256 func buildMap(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection { 257 values := make([]types.Value, count*2) 258 for i := uint64(0); i < count*2; i++ { 259 values[i] = createFn(i) 260 } 261 262 m, err := types.NewMap(context.Background(), vrw, values...) 263 264 d.Chk.NoError(err) 265 266 return m 267 } 268 269 func buildMapIncrementally(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection { 270 m, err := types.NewMap(context.Background(), vrw) 271 d.Chk.NoError(err) 272 273 me := m.Edit() 274 275 for i := uint64(0); i < count*2; i += 2 { 276 me.Set(createFn(i), createFn(i+1)) 277 } 278 279 m, err = me.Map(context.Background()) 280 d.Chk.NoError(err) 281 282 return m 283 } 284 285 func readMap(c types.Collection) { 286 _ = c.(types.Map).IterAll(context.Background(), func(k types.Value, v types.Value) error { 287 return nil 288 }) 289 }