bitbucket.org/number571/tendermint@v0.8.14/test/e2e/runner/benchmark.go (about) 1 package main 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "math" 8 "path/filepath" 9 "time" 10 11 e2e "bitbucket.org/number571/tendermint/test/e2e/pkg" 12 "bitbucket.org/number571/tendermint/types" 13 ) 14 15 // Benchmark is a simple function for fetching, calculating and printing 16 // the following metrics: 17 // 1. Average block production time 18 // 2. Block interval standard deviation 19 // 3. Max block interval (slowest block) 20 // 4. Min block interval (fastest block) 21 // 22 // Metrics are based of the `benchmarkLength`, the amount of consecutive blocks 23 // sampled from in the testnet 24 func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error { 25 block, _, err := waitForHeight(testnet, 0) 26 if err != nil { 27 return err 28 } 29 30 logger.Info("Beginning benchmark period...", "height", block.Height) 31 startAt := time.Now() 32 // wait for the length of the benchmark period in blocks to pass. We allow 5 seconds for each block 33 // which should be sufficient. 34 waitingTime := time.Duration(benchmarkLength*5) * time.Second 35 endHeight, err := waitForAllNodes(testnet, block.Height+benchmarkLength, waitingTime) 36 if err != nil { 37 return err 38 } 39 dur := time.Since(startAt) 40 41 logger.Info("Ending benchmark period", "height", endHeight) 42 43 // fetch a sample of blocks 44 blocks, err := fetchBlockChainSample(testnet, benchmarkLength) 45 if err != nil { 46 return err 47 } 48 49 // slice into time intervals and collate data 50 timeIntervals := splitIntoBlockIntervals(blocks) 51 testnetStats := extractTestnetStats(timeIntervals) 52 53 // populate data 54 testnetStats.populateTxns(blocks) 55 testnetStats.totalTime = dur 56 testnetStats.benchmarkLength = benchmarkLength 57 testnetStats.startHeight = blocks[0].Header.Height 58 testnetStats.endHeight = blocks[len(blocks)-1].Header.Height 59 60 // print and return 61 logger.Info(testnetStats.String()) 62 logger.Info(testnetStats.getReportJSON(testnet)) 63 return nil 64 } 65 66 func (t *testnetStats) populateTxns(blocks []*types.BlockMeta) { 67 t.numtxns = 0 68 for _, b := range blocks { 69 t.numtxns += int64(b.NumTxs) 70 } 71 } 72 73 type testnetStats struct { 74 startHeight int64 75 endHeight int64 76 77 benchmarkLength int64 78 numtxns int64 79 totalTime time.Duration 80 // average time to produce a block 81 mean time.Duration 82 // standard deviation of block production 83 std float64 84 // longest time to produce a block 85 max time.Duration 86 // shortest time to produce a block 87 min time.Duration 88 } 89 90 func (t *testnetStats) getReportJSON(net *e2e.Testnet) string { 91 jsn, err := json.Marshal(map[string]interface{}{ 92 "case": filepath.Base(net.File), 93 "blocks": t.endHeight - t.startHeight, 94 "stddev": t.std, 95 "mean": t.mean.Seconds(), 96 "max": t.max.Seconds(), 97 "min": t.min.Seconds(), 98 "size": len(net.Nodes), 99 "txns": t.numtxns, 100 "dur": t.totalTime.Seconds(), 101 "length": t.benchmarkLength, 102 }) 103 104 if err != nil { 105 return "" 106 } 107 108 return string(jsn) 109 } 110 111 func (t *testnetStats) String() string { 112 return fmt.Sprintf(`Benchmarked from height %v to %v 113 Mean Block Interval: %v 114 Standard Deviation: %f 115 Max Block Interval: %v 116 Min Block Interval: %v 117 `, 118 t.startHeight, 119 t.endHeight, 120 t.mean, 121 t.std, 122 t.max, 123 t.min, 124 ) 125 } 126 127 // fetchBlockChainSample waits for `benchmarkLength` amount of blocks to pass, fetching 128 // all of the headers for these blocks from an archive node and returning it. 129 func fetchBlockChainSample(testnet *e2e.Testnet, benchmarkLength int64) ([]*types.BlockMeta, error) { 130 var blocks []*types.BlockMeta 131 132 // Find the first archive node 133 archiveNode := testnet.ArchiveNodes()[0] 134 c, err := archiveNode.Client() 135 if err != nil { 136 return nil, err 137 } 138 139 // find the latest height 140 ctx := context.Background() 141 s, err := c.Status(ctx) 142 if err != nil { 143 return nil, err 144 } 145 146 to := s.SyncInfo.LatestBlockHeight 147 from := to - benchmarkLength + 1 148 if from <= testnet.InitialHeight { 149 return nil, fmt.Errorf("tesnet was unable to reach required height for benchmarking (latest height %d)", to) 150 } 151 152 // Fetch blocks 153 for from < to { 154 // fetch the blockchain metas. Currently we can only fetch 20 at a time 155 resp, err := c.BlockchainInfo(ctx, from, min(from+19, to)) 156 if err != nil { 157 return nil, err 158 } 159 160 blockMetas := resp.BlockMetas 161 // we receive blocks in descending order so we have to add them in reverse 162 for i := len(blockMetas) - 1; i >= 0; i-- { 163 if blockMetas[i].Header.Height != from { 164 return nil, fmt.Errorf("node gave us another header. Wanted %d, got %d", 165 from, 166 blockMetas[i].Header.Height, 167 ) 168 } 169 from++ 170 blocks = append(blocks, blockMetas[i]) 171 } 172 } 173 174 return blocks, nil 175 } 176 177 func splitIntoBlockIntervals(blocks []*types.BlockMeta) []time.Duration { 178 intervals := make([]time.Duration, len(blocks)-1) 179 lastTime := blocks[0].Header.Time 180 for i, block := range blocks { 181 // skip the first block 182 if i == 0 { 183 continue 184 } 185 186 intervals[i-1] = block.Header.Time.Sub(lastTime) 187 lastTime = block.Header.Time 188 } 189 return intervals 190 } 191 192 func extractTestnetStats(intervals []time.Duration) testnetStats { 193 var ( 194 sum, mean time.Duration 195 std float64 196 max = intervals[0] 197 min = intervals[0] 198 ) 199 200 for _, interval := range intervals { 201 sum += interval 202 203 if interval > max { 204 max = interval 205 } 206 207 if interval < min { 208 min = interval 209 } 210 } 211 mean = sum / time.Duration(len(intervals)) 212 213 for _, interval := range intervals { 214 diff := (interval - mean).Seconds() 215 std += math.Pow(diff, 2) 216 } 217 std = math.Sqrt(std / float64(len(intervals))) 218 219 return testnetStats{ 220 mean: mean, 221 std: std, 222 max: max, 223 min: min, 224 } 225 } 226 227 func min(a, b int64) int64 { 228 if a > b { 229 return b 230 } 231 return a 232 }