github.com/jgbaldwinbrown/perf@v0.1.1/benchproc/projection_test.go (about) 1 // Copyright 2022 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package benchproc 6 7 import ( 8 "fmt" 9 "strings" 10 "testing" 11 12 "golang.org/x/perf/benchfmt" 13 "golang.org/x/perf/benchproc/internal/parse" 14 ) 15 16 // mustParse parses a single projection. 17 func mustParse(t *testing.T, proj string) (*Projection, *Filter) { 18 f, err := NewFilter("*") 19 if err != nil { 20 t.Fatalf("unexpected error: %v", err) 21 } 22 s, err := (&ProjectionParser{}).Parse(proj, f) 23 if err != nil { 24 t.Fatalf("unexpected error parsing %q: %v", proj, err) 25 } 26 return s, f 27 } 28 29 // r constructs a benchfmt.Result with the given full name and file 30 // config, which is specified as alternating key/value pairs. The 31 // result has 1 iteration and no values. 32 func r(t *testing.T, fullName string, fileConfig ...string) *benchfmt.Result { 33 res := &benchfmt.Result{ 34 Name: benchfmt.Name(fullName), 35 Iters: 1, 36 } 37 38 if len(fileConfig)%2 != 0 { 39 t.Fatal("fileConfig must be alternating key/value pairs") 40 } 41 for i := 0; i < len(fileConfig); i += 2 { 42 cfg := benchfmt.Config{Key: fileConfig[i], Value: []byte(fileConfig[i+1]), File: true} 43 res.Config = append(res.Config, cfg) 44 } 45 46 return res 47 } 48 49 // p constructs a benchfmt.Result like r, then projects it using p. 50 func p(t *testing.T, p *Projection, fullName string, fileConfig ...string) Key { 51 res := r(t, fullName, fileConfig...) 52 return p.Project(res) 53 } 54 55 func TestProjectionBasic(t *testing.T) { 56 check := func(key Key, want string) { 57 t.Helper() 58 got := key.String() 59 if got != want { 60 t.Errorf("got %s, want %s", got, want) 61 } 62 } 63 64 var s *Projection 65 66 // Sub-name config. 67 s, _ = mustParse(t, ".fullname") 68 check(p(t, s, "Name/a=1"), ".fullname:Name/a=1") 69 s, _ = mustParse(t, "/a") 70 check(p(t, s, "Name/a=1"), "/a:1") 71 s, _ = mustParse(t, ".name") 72 check(p(t, s, "Name/a=1"), ".name:Name") 73 74 // Fixed file config. 75 s, _ = mustParse(t, "a") 76 check(p(t, s, "", "a", "1", "b", "2"), "a:1") 77 check(p(t, s, "", "b", "2"), "") // Missing values are omitted 78 check(p(t, s, "", "a", "", "b", "2"), "") 79 80 // Variable file config. 81 s, _ = mustParse(t, ".config") 82 check(p(t, s, "", "a", "1", "b", "2"), "a:1 b:2") 83 check(p(t, s, "", "c", "3"), "c:3") 84 check(p(t, s, "", "c", "3", "a", "2"), "a:2 c:3") 85 } 86 87 func TestProjectionIntern(t *testing.T) { 88 s, _ := mustParse(t, "a,b") 89 90 c12 := p(t, s, "", "a", "1", "b", "2") 91 92 if c12 != p(t, s, "", "a", "1", "b", "2") { 93 t.Errorf("Keys should be equal") 94 } 95 96 if c12 == p(t, s, "", "a", "1", "b", "3") { 97 t.Errorf("Keys should not be equal") 98 } 99 100 if c12 != p(t, s, "", "a", "1", "b", "2", "c", "3") { 101 t.Errorf("Keys should be equal") 102 } 103 } 104 105 func fieldNames(fields []*Field) string { 106 names := new(strings.Builder) 107 for i, f := range fields { 108 if i > 0 { 109 names.WriteByte(' ') 110 } 111 names.WriteString(f.Name) 112 } 113 return names.String() 114 } 115 116 func TestProjectionParsing(t *testing.T) { 117 // Basic parsing is tested by the syntax package. Here we test 118 // additional processing done by this package. 119 120 check := func(proj string, want, wantFlat string) { 121 t.Helper() 122 s, _ := mustParse(t, proj) 123 got := fieldNames(s.Fields()) 124 if got != want { 125 t.Errorf("%s: got fields %v, want %v", proj, got, want) 126 } 127 gotFlat := fieldNames(s.FlattenedFields()) 128 if gotFlat != wantFlat { 129 t.Errorf("%s: got flat fields %v, want %v", proj, gotFlat, wantFlat) 130 } 131 } 132 checkErr := func(proj, error string, pos int) { 133 t.Helper() 134 f, _ := NewFilter("*") 135 _, err := (&ProjectionParser{}).Parse(proj, f) 136 if se, _ := err.(*parse.SyntaxError); se == nil || se.Msg != error || se.Off != pos { 137 t.Errorf("%s: want error %s at %d; got %s", proj, error, pos, err) 138 } 139 } 140 141 check("a,b,c", "a b c", "a b c") 142 check("a,.config,c", "a .config c", "a c") // .config hasn't been populated with anything yet 143 check("a,.fullname,c", "a .fullname c", "a .fullname c") 144 check("a,.name,c", "a .name c", "a .name c") 145 check("a,/b,c", "a /b c", "a /b c") 146 147 checkErr("a@foo", "unknown order \"foo\"", 2) 148 149 checkErr(".config@(1 2)", "fixed order not allowed for .config", 8) 150 } 151 152 func TestProjectionFiltering(t *testing.T) { 153 _, f := mustParse(t, "a@(a b c)") 154 check := func(val string, want bool) { 155 t.Helper() 156 res := r(t, "", "a", val) 157 got, _ := f.Apply(res) 158 if want != got { 159 t.Errorf("%s: want %v, got %v", val, want, got) 160 } 161 } 162 check("a", true) 163 check("b", true) 164 check("aa", false) 165 check("z", false) 166 } 167 168 func TestProjectionExclusion(t *testing.T) { 169 // The underlying name normalization has already been tested 170 // thoroughly in benchfmt/extract_test.go, so here we just 171 // have to test that it's being plumbed right. 172 173 check := func(key Key, want string) { 174 t.Helper() 175 got := key.String() 176 if got != want { 177 t.Errorf("got %s, want %s", got, want) 178 } 179 } 180 181 // Create the main projection. 182 var pp ProjectionParser 183 f, _ := NewFilter("*") 184 s, err := pp.Parse(".fullname,.config", f) 185 if err != nil { 186 t.Fatalf("unexpected error %v", err) 187 } 188 // Parse specific keys that should be excluded from fullname 189 // and config. 190 _, err = pp.Parse(".name,/a,exc", f) 191 if err != nil { 192 t.Fatalf("unexpected error %v", err) 193 } 194 195 check(p(t, s, "Name"), ".fullname:*") 196 check(p(t, s, "Name/a=1"), ".fullname:*") 197 check(p(t, s, "Name/a=1/b=2"), ".fullname:*/b=2") 198 199 check(p(t, s, "Name", "exc", "1"), ".fullname:*") 200 check(p(t, s, "Name", "exc", "1", "abc", "2"), ".fullname:* abc:2") 201 check(p(t, s, "Name", "abc", "2"), ".fullname:* abc:2") 202 } 203 204 func TestProjectionResidue(t *testing.T) { 205 check := func(mainProj string, want string) { 206 t.Helper() 207 208 // Get the residue of mainProj. 209 var pp ProjectionParser 210 f, _ := NewFilter("*") 211 _, err := pp.Parse(mainProj, f) 212 if err != nil { 213 t.Fatalf("unexpected error %v", err) 214 } 215 s := pp.Residue() 216 217 // Project a test result. 218 key := p(t, s, "Name/a=1/b=2", "x", "3", "y", "4") 219 got := key.String() 220 if got != want { 221 t.Errorf("got %s, want %s", got, want) 222 } 223 } 224 225 // Full residue. 226 check("", "x:3 y:4 .fullname:Name/a=1/b=2") 227 // Empty residue. 228 check(".config,.fullname", "") 229 // Partial residues. 230 check("x,/a", "y:4 .fullname:Name/b=2") 231 check(".config", ".fullname:Name/a=1/b=2") 232 check(".fullname", "x:3 y:4") 233 check(".name", "x:3 y:4 .fullname:*/a=1/b=2") 234 } 235 236 // ExampleProjectionParser_Residue demonstrates residue projections. 237 // 238 // This example groups a set of results by .fullname and goos, but the 239 // two results for goos:linux have two different goarch values, 240 // indicating the user probably unintentionally grouped uncomparable 241 // results together. The example uses ProjectionParser.Residue and 242 // NonSingularFields to warn the user about this. 243 func ExampleProjectionParser_Residue() { 244 var pp ProjectionParser 245 p, _ := pp.Parse(".fullname,goos", nil) 246 residue := pp.Residue() 247 248 // Aggregate each result by p and track the residue of each group. 249 type group struct { 250 values []float64 251 residues []Key 252 } 253 groups := make(map[Key]*group) 254 var keys []Key 255 256 for _, result := range results(` 257 goos: linux 258 goarch: amd64 259 BenchmarkAlloc 1 128 ns/op 260 261 goos: linux 262 goarch: arm64 263 BenchmarkAlloc 1 137 ns/op 264 265 goos: darwin 266 goarch: amd64 267 BenchmarkAlloc 1 130 ns/op`) { 268 // Map result to a group. 269 key := p.Project(result) 270 g, ok := groups[key] 271 if !ok { 272 g = new(group) 273 groups[key] = g 274 keys = append(keys, key) 275 } 276 277 // Add value to the group. 278 speed, _ := result.Value("sec/op") 279 g.values = append(g.values, speed) 280 281 // Add residue to the group. 282 g.residues = append(g.residues, residue.Project(result)) 283 } 284 285 // Report aggregated results. 286 SortKeys(keys) 287 for _, k := range keys { 288 g := groups[k] 289 // Report the result. 290 fmt.Println(k, mean(g.values), "sec/op") 291 // Check if the grouped results vary in some unexpected way. 292 nonsingular := NonSingularFields(g.residues) 293 if len(nonsingular) > 0 { 294 // Report a potential issue. 295 fmt.Printf("warning: results vary in %s and may be uncomparable\n", nonsingular) 296 } 297 } 298 299 // Output: 300 // .fullname:Alloc goos:linux 1.325e-07 sec/op 301 // warning: results vary in [goarch] and may be uncomparable 302 // .fullname:Alloc goos:darwin 1.3e-07 sec/op 303 } 304 305 func results(data string) []*benchfmt.Result { 306 var out []*benchfmt.Result 307 r := benchfmt.NewReader(strings.NewReader(data), "<string>") 308 for r.Scan() { 309 switch rec := r.Result(); rec := rec.(type) { 310 case *benchfmt.Result: 311 out = append(out, rec.Clone()) 312 case *benchfmt.SyntaxError: 313 panic("unexpected error in test data: " + rec.Error()) 314 } 315 } 316 return out 317 } 318 319 func mean(xs []float64) float64 { 320 var sum float64 321 for _, x := range xs { 322 sum += x 323 } 324 return sum / float64(len(xs)) 325 } 326 327 func TestProjectionValues(t *testing.T) { 328 s, unit, err := (&ProjectionParser{}).ParseWithUnit("x", nil) 329 if err != nil { 330 t.Fatalf("unexpected error parsing %q: %v", "x", err) 331 } 332 333 check := func(key Key, want, wantUnit string) { 334 t.Helper() 335 got := key.String() 336 if got != want { 337 t.Errorf("got %s, want %s", got, want) 338 } 339 gotUnit := key.Get(unit) 340 if gotUnit != wantUnit { 341 t.Errorf("got unit %s, want %s", gotUnit, wantUnit) 342 } 343 } 344 345 res := r(t, "Name", "x", "1") 346 res.Values = []benchfmt.Value{{Value: 100, Unit: "ns/op"}, {Value: 1.21, Unit: "gigawatts"}} 347 keys := s.ProjectValues(res) 348 if len(keys) != len(res.Values) { 349 t.Fatalf("got %d Keys, want %d", len(keys), len(res.Values)) 350 } 351 352 check(keys[0], "x:1 .unit:ns/op", "ns/op") 353 check(keys[1], "x:1 .unit:gigawatts", "gigawatts") 354 }