github.com/jgbaldwinbrown/perf@v0.1.1/benchstat/data.go (about) 1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package benchstat 6 7 import ( 8 "bytes" 9 "fmt" 10 "io" 11 "strconv" 12 "strings" 13 14 "golang.org/x/perf/internal/stats" 15 "golang.org/x/perf/storage/benchfmt" 16 ) 17 18 // A Collection is a collection of benchmark results. 19 type Collection struct { 20 // Configs, Groups, and Units give the set of configs, 21 // groups, and units from the keys in Stats in an order 22 // meant to match the order the benchmarks were read in. 23 Configs, Groups, Units []string 24 25 // Benchmarks gives the set of benchmarks from the keys in 26 // Stats by group in an order meant to match the order 27 // benchmarks were read in. 28 Benchmarks map[string][]string 29 30 // Metrics holds the accumulated metrics for each key. 31 Metrics map[Key]*Metrics 32 33 // DeltaTest is the test to use to decide if a change is significant. 34 // If nil, it defaults to UTest. 35 DeltaTest DeltaTest 36 37 // Alpha is the p-value cutoff to report a change as significant. 38 // If zero, it defaults to 0.05. 39 Alpha float64 40 41 // AddGeoMean specifies whether to add a line to the table 42 // showing the geometric mean of all the benchmark results. 43 AddGeoMean bool 44 45 // SplitBy specifies the labels to split results by. 46 // By default, results will only be split by full name. 47 SplitBy []string 48 49 // Order specifies the row display order for this table. 50 // If Order is nil, the table rows are printed in order of 51 // first appearance in the input. 52 Order Order 53 } 54 55 // A Key identifies one metric (e.g., "ns/op", "B/op") from one 56 // benchmark (function name sans "Benchmark" prefix) and optional 57 // group in one configuration (input file name). 58 type Key struct { 59 Config, Group, Benchmark, Unit string 60 } 61 62 // A Metrics holds the measurements of a single metric 63 // (for example, ns/op or MB/s) 64 // for all runs of a particular benchmark. 65 type Metrics struct { 66 Unit string // unit being measured 67 Values []float64 // measured values 68 RValues []float64 // Values with outliers removed 69 Min float64 // min of RValues 70 Mean float64 // mean of RValues 71 Max float64 // max of RValues 72 } 73 74 // FormatMean formats m.Mean using scaler. 75 func (m *Metrics) FormatMean(scaler Scaler) string { 76 var s string 77 if scaler != nil { 78 s = scaler(m.Mean) 79 } else { 80 s = fmt.Sprint(m.Mean) 81 } 82 return s 83 } 84 85 // FormatDiff computes and formats the percent variation of max and min compared to mean. 86 // If b.Mean or b.Max is zero, FormatDiff returns an empty string. 87 func (m *Metrics) FormatDiff() string { 88 if m.Mean == 0 || m.Max == 0 { 89 return "" 90 } 91 diff := 1 - m.Min/m.Mean 92 if d := m.Max/m.Mean - 1; d > diff { 93 diff = d 94 } 95 return fmt.Sprintf("%.0f%%", diff*100.0) 96 } 97 98 // Format returns a textual formatting of "Mean ±Diff" using scaler. 99 func (m *Metrics) Format(scaler Scaler) string { 100 if m.Unit == "" { 101 return "" 102 } 103 mean := m.FormatMean(scaler) 104 diff := m.FormatDiff() 105 if diff == "" { 106 return mean + " " 107 } 108 return fmt.Sprintf("%s ±%3s", mean, diff) 109 } 110 111 // computeStats updates the derived statistics in m from the raw 112 // samples in m.Values. 113 func (m *Metrics) computeStats() { 114 // Discard outliers. 115 values := stats.Sample{Xs: m.Values} 116 q1, q3 := values.Percentile(0.25), values.Percentile(0.75) 117 lo, hi := q1-1.5*(q3-q1), q3+1.5*(q3-q1) 118 for _, value := range m.Values { 119 if lo <= value && value <= hi { 120 m.RValues = append(m.RValues, value) 121 } 122 } 123 124 // Compute statistics of remaining data. 125 m.Min, m.Max = stats.Bounds(m.RValues) 126 m.Mean = stats.Mean(m.RValues) 127 } 128 129 // addMetrics returns the metrics with the given key from c, 130 // creating a new one if needed. 131 func (c *Collection) addMetrics(key Key) *Metrics { 132 if c.Metrics == nil { 133 c.Metrics = make(map[Key]*Metrics) 134 } 135 if stat, ok := c.Metrics[key]; ok { 136 return stat 137 } 138 139 addString := func(strings *[]string, add string) { 140 for _, s := range *strings { 141 if s == add { 142 return 143 } 144 } 145 *strings = append(*strings, add) 146 } 147 addString(&c.Configs, key.Config) 148 addString(&c.Groups, key.Group) 149 if c.Benchmarks == nil { 150 c.Benchmarks = make(map[string][]string) 151 } 152 benchmarks := c.Benchmarks[key.Group] 153 addString(&benchmarks, key.Benchmark) 154 c.Benchmarks[key.Group] = benchmarks 155 addString(&c.Units, key.Unit) 156 m := &Metrics{Unit: key.Unit} 157 c.Metrics[key] = m 158 return m 159 } 160 161 // AddConfig adds the benchmark results in the formatted data 162 // to the named configuration. 163 // If the input is large, AddFile should be preferred, 164 // since it avoids the need to read a copy of the entire raw input 165 // into memory. 166 func (c *Collection) AddConfig(config string, data []byte) { 167 err := c.AddFile(config, bytes.NewReader(data)) 168 if err != nil { 169 // bytes.Reader never returns errors 170 panic(err) 171 } 172 } 173 174 // AddFile adds the benchmark results in the formatted data 175 // (read from the reader r) to the named configuration. 176 func (c *Collection) AddFile(config string, r io.Reader) error { 177 c.Configs = append(c.Configs, config) 178 key := Key{Config: config} 179 br := benchfmt.NewReader(r) 180 for br.Next() { 181 c.addResult(key, br.Result()) 182 } 183 return br.Err() 184 } 185 186 // AddResults adds the benchmark results to the named configuration. 187 func (c *Collection) AddResults(config string, results []*benchfmt.Result) { 188 c.Configs = append(c.Configs, config) 189 key := Key{Config: config} 190 for _, r := range results { 191 c.addResult(key, r) 192 } 193 } 194 195 func (c *Collection) addResult(key Key, r *benchfmt.Result) { 196 f := strings.Fields(r.Content) 197 if len(f) < 4 { 198 return 199 } 200 name := f[0] 201 if !strings.HasPrefix(name, "Benchmark") { 202 return 203 } 204 name = strings.TrimPrefix(name, "Benchmark") 205 n, _ := strconv.Atoi(f[1]) 206 if n == 0 { 207 return 208 } 209 key.Group = c.makeGroup(r) 210 key.Benchmark = name 211 for i := 2; i+2 <= len(f); i += 2 { 212 val, err := strconv.ParseFloat(f[i], 64) 213 if err != nil { 214 continue 215 } 216 key.Unit = f[i+1] 217 m := c.addMetrics(key) 218 m.Values = append(m.Values, val) 219 } 220 } 221 222 func (c *Collection) makeGroup(r *benchfmt.Result) string { 223 var out string 224 for _, s := range c.SplitBy { 225 v := r.NameLabels[s] 226 if v == "" { 227 v = r.Labels[s] 228 } 229 if v != "" { 230 if out != "" { 231 out = out + " " 232 } 233 out += fmt.Sprintf("%s:%s", s, v) 234 } 235 } 236 return out 237 }