github.com/attic-labs/noms@v0.0.0-20210827224422-e5fa29d95e8b/go/nbs/benchmarks/block_store_benchmarks.go (about)

     1  // Copyright 2016 Attic Labs, Inc. All rights reserved.
     2  // Licensed under the Apache License, version 2.0:
     3  // http://www.apache.org/licenses/LICENSE-2.0
     4  
     5  package main
     6  
     7  import (
     8  	"fmt"
     9  	"sync"
    10  
    11  	"github.com/attic-labs/noms/go/chunks"
    12  	"github.com/attic-labs/noms/go/hash"
    13  	"github.com/stretchr/testify/assert"
    14  )
    15  
    16  type storeOpenFn func() chunks.ChunkStore
    17  
    18  func benchmarkNovelWrite(refreshStore storeOpenFn, src *dataSource, t assert.TestingT) bool {
    19  	store := refreshStore()
    20  	writeToEmptyStore(store, src, t)
    21  	assert.NoError(t, store.Close())
    22  	return true
    23  }
    24  
    25  func writeToEmptyStore(store chunks.ChunkStore, src *dataSource, t assert.TestingT) {
    26  	root := store.Root()
    27  	assert.Equal(t, hash.Hash{}, root)
    28  
    29  	chunx := goReadChunks(src)
    30  	for c := range chunx {
    31  		store.Put(*c)
    32  	}
    33  	newRoot := chunks.NewChunk([]byte("root"))
    34  	store.Put(newRoot)
    35  	assert.True(t, store.Commit(newRoot.Hash(), root))
    36  }
    37  
    38  func goReadChunks(src *dataSource) <-chan *chunks.Chunk {
    39  	chunx := make(chan *chunks.Chunk, 1024)
    40  	go func() {
    41  		src.ReadChunks(chunx)
    42  		close(chunx)
    43  	}()
    44  	return chunx
    45  }
    46  
    47  func benchmarkNoRefreshWrite(openStore storeOpenFn, src *dataSource, t assert.TestingT) {
    48  	store := openStore()
    49  	chunx := goReadChunks(src)
    50  	for c := range chunx {
    51  		store.Put(*c)
    52  	}
    53  	assert.NoError(t, store.Close())
    54  }
    55  
    56  func verifyChunk(h hash.Hash, c chunks.Chunk) {
    57  	if len(c.Data()) == 0 {
    58  		panic(fmt.Sprintf("Failed to fetch %s\n", h.String()))
    59  	}
    60  }
    61  
    62  func benchmarkRead(openStore storeOpenFn, hashes hashSlice, src *dataSource, t assert.TestingT) {
    63  	store := openStore()
    64  	for _, h := range hashes {
    65  		verifyChunk(h, store.Get(h))
    66  	}
    67  	assert.NoError(t, store.Close())
    68  }
    69  
    70  func verifyChunks(hashes hash.HashSlice, foundChunks chan *chunks.Chunk) {
    71  	requested := hashes.HashSet()
    72  
    73  	for c := range foundChunks {
    74  		if _, ok := requested[c.Hash()]; !ok {
    75  			panic(fmt.Sprintf("Got unexpected chunk: %s", c.Hash().String()))
    76  		}
    77  
    78  		delete(requested, c.Hash())
    79  	}
    80  
    81  	if len(requested) > 0 {
    82  		for h := range requested {
    83  			fmt.Printf("Failed to fetch %s\n", h.String())
    84  		}
    85  		panic("failed to fetch chunks")
    86  	}
    87  }
    88  
    89  func benchmarkReadMany(openStore storeOpenFn, hashes hashSlice, src *dataSource, batchSize, concurrency int, t assert.TestingT) {
    90  	store := openStore()
    91  	batch := make(hash.HashSlice, 0, batchSize)
    92  
    93  	wg := sync.WaitGroup{}
    94  	limit := make(chan struct{}, concurrency)
    95  
    96  	for _, h := range hashes {
    97  		batch = append(batch, h)
    98  
    99  		if len(batch) == batchSize {
   100  			limit <- struct{}{}
   101  			wg.Add(1)
   102  			go func(hashes hash.HashSlice) {
   103  				chunkChan := make(chan *chunks.Chunk, len(hashes))
   104  				store.GetMany(hashes.HashSet(), chunkChan)
   105  				close(chunkChan)
   106  				verifyChunks(hashes, chunkChan)
   107  				wg.Done()
   108  				<-limit
   109  			}(batch)
   110  
   111  			batch = make([]hash.Hash, 0, batchSize)
   112  		}
   113  	}
   114  
   115  	if len(batch) > 0 {
   116  		chunkChan := make(chan *chunks.Chunk, len(batch))
   117  		store.GetMany(batch.HashSet(), chunkChan)
   118  		close(chunkChan)
   119  
   120  		verifyChunks(batch, chunkChan)
   121  	}
   122  
   123  	wg.Wait()
   124  
   125  	assert.NoError(t, store.Close())
   126  }
   127  
   128  func ensureNovelWrite(wrote bool, openStore storeOpenFn, src *dataSource, t assert.TestingT) bool {
   129  	if !wrote {
   130  		store := openStore()
   131  		defer store.Close()
   132  		writeToEmptyStore(store, src, t)
   133  	}
   134  	return true
   135  }