github.com/attic-labs/noms@v0.0.0-20210827224422-e5fa29d95e8b/go/nbs/benchmarks/block_store_benchmarks.go (about) 1 // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 // Licensed under the Apache License, version 2.0: 3 // http://www.apache.org/licenses/LICENSE-2.0 4 5 package main 6 7 import ( 8 "fmt" 9 "sync" 10 11 "github.com/attic-labs/noms/go/chunks" 12 "github.com/attic-labs/noms/go/hash" 13 "github.com/stretchr/testify/assert" 14 ) 15 16 type storeOpenFn func() chunks.ChunkStore 17 18 func benchmarkNovelWrite(refreshStore storeOpenFn, src *dataSource, t assert.TestingT) bool { 19 store := refreshStore() 20 writeToEmptyStore(store, src, t) 21 assert.NoError(t, store.Close()) 22 return true 23 } 24 25 func writeToEmptyStore(store chunks.ChunkStore, src *dataSource, t assert.TestingT) { 26 root := store.Root() 27 assert.Equal(t, hash.Hash{}, root) 28 29 chunx := goReadChunks(src) 30 for c := range chunx { 31 store.Put(*c) 32 } 33 newRoot := chunks.NewChunk([]byte("root")) 34 store.Put(newRoot) 35 assert.True(t, store.Commit(newRoot.Hash(), root)) 36 } 37 38 func goReadChunks(src *dataSource) <-chan *chunks.Chunk { 39 chunx := make(chan *chunks.Chunk, 1024) 40 go func() { 41 src.ReadChunks(chunx) 42 close(chunx) 43 }() 44 return chunx 45 } 46 47 func benchmarkNoRefreshWrite(openStore storeOpenFn, src *dataSource, t assert.TestingT) { 48 store := openStore() 49 chunx := goReadChunks(src) 50 for c := range chunx { 51 store.Put(*c) 52 } 53 assert.NoError(t, store.Close()) 54 } 55 56 func verifyChunk(h hash.Hash, c chunks.Chunk) { 57 if len(c.Data()) == 0 { 58 panic(fmt.Sprintf("Failed to fetch %s\n", h.String())) 59 } 60 } 61 62 func benchmarkRead(openStore storeOpenFn, hashes hashSlice, src *dataSource, t assert.TestingT) { 63 store := openStore() 64 for _, h := range hashes { 65 verifyChunk(h, store.Get(h)) 66 } 67 assert.NoError(t, store.Close()) 68 } 69 70 func verifyChunks(hashes hash.HashSlice, foundChunks chan *chunks.Chunk) { 71 requested := hashes.HashSet() 72 73 for c := range foundChunks { 74 if _, ok := requested[c.Hash()]; !ok { 75 panic(fmt.Sprintf("Got unexpected chunk: %s", c.Hash().String())) 76 } 77 78 delete(requested, c.Hash()) 79 } 80 81 if len(requested) > 0 { 82 for h := range requested { 83 fmt.Printf("Failed to fetch %s\n", h.String()) 84 } 85 panic("failed to fetch chunks") 86 } 87 } 88 89 func benchmarkReadMany(openStore storeOpenFn, hashes hashSlice, src *dataSource, batchSize, concurrency int, t assert.TestingT) { 90 store := openStore() 91 batch := make(hash.HashSlice, 0, batchSize) 92 93 wg := sync.WaitGroup{} 94 limit := make(chan struct{}, concurrency) 95 96 for _, h := range hashes { 97 batch = append(batch, h) 98 99 if len(batch) == batchSize { 100 limit <- struct{}{} 101 wg.Add(1) 102 go func(hashes hash.HashSlice) { 103 chunkChan := make(chan *chunks.Chunk, len(hashes)) 104 store.GetMany(hashes.HashSet(), chunkChan) 105 close(chunkChan) 106 verifyChunks(hashes, chunkChan) 107 wg.Done() 108 <-limit 109 }(batch) 110 111 batch = make([]hash.Hash, 0, batchSize) 112 } 113 } 114 115 if len(batch) > 0 { 116 chunkChan := make(chan *chunks.Chunk, len(batch)) 117 store.GetMany(batch.HashSet(), chunkChan) 118 close(chunkChan) 119 120 verifyChunks(batch, chunkChan) 121 } 122 123 wg.Wait() 124 125 assert.NoError(t, store.Close()) 126 } 127 128 func ensureNovelWrite(wrote bool, openStore storeOpenFn, src *dataSource, t assert.TestingT) bool { 129 if !wrote { 130 store := openStore() 131 defer store.Close() 132 writeToEmptyStore(store, src, t) 133 } 134 return true 135 }