github.com/jgbaldwinbrown/perf@v0.1.1/benchseries/chart.go (about) 1 // Copyright 2022 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package benchseries 6 7 import ( 8 "fmt" 9 "image/color" 10 "math" 11 "os" 12 "path/filepath" 13 "sort" 14 "strings" 15 16 "gonum.org/v1/plot/vg/draw" 17 "gonum.org/v1/plot/vg/vgimg" 18 19 // "gonum.org/v1/plot/vg/vgpdf" 20 // "gonum.org/v1/plot/vg/vgsvg" 21 22 "gonum.org/v1/plot" 23 "gonum.org/v1/plot/plotter" 24 "gonum.org/v1/plot/vg" 25 ) 26 27 type ChartOptions int 28 29 type Point struct { 30 numHash, denHash string 31 values plotter.Values 32 changes []float64 33 changeBVID []benchValID 34 } 35 36 // because there are holes in that data, the benchmark index can be larger than the valueIndex 37 type benchValID struct { 38 benchIndex, valueIndex int 39 } 40 41 const pointRad = 5 42 43 func Chart(cs []*ComparisonSeries, pngDir, pdfDir, svgDir string, logScale bool, threshold float64, boring bool) { 44 45 doDir := func(s string) { 46 if s != "" { 47 os.MkdirAll(s, 0777) 48 } 49 } 50 doDir(pngDir) 51 doDir(pdfDir) 52 doDir(svgDir) 53 54 for _, g := range cs { 55 56 var allValues []float64 57 58 // When boring runs are omitted, this is where the interesting parts go. 59 selectedPoints := []*Point{} 60 61 // select a subset of the points, inserting "..." whenever there is a gap. 62 for i, s := range g.Series { 63 64 values := make(plotter.Values, 0, len(g.Benchmarks)) 65 changes := make([]float64, 0, len(g.Benchmarks)) 66 changeBenches := make([]benchValID, 0, len(g.Benchmarks)) // which benchmarks changed? 67 for j := range g.Benchmarks { 68 sum := g.Summaries[i][j] 69 if sum.Defined() { 70 ch := math.NaN() 71 v := sum.Center 72 if math.IsInf(v, 0) { 73 continue 74 } 75 if i > 0 { 76 psum := g.Summaries[i-1][j] 77 if psum.Defined() { 78 ch = psum.HeurOverlap(sum, threshold) 79 } 80 } 81 changes = append(changes, ch) 82 changeBenches = append(changeBenches, benchValID{j, len(values)}) 83 values = append(values, v) 84 allValues = append(allValues, v) 85 } 86 } 87 hp := g.HashPairs[s] 88 selectedPoints = append(selectedPoints, &Point{numHash: hp.NumHash, denHash: hp.DenHash, values: values, changes: changes, changeBVID: changeBenches}) 89 } 90 91 // Want lines that grab most of the data; it's outliers we worry about 92 sort.Float64s(allValues) 93 lav := len(allValues) 94 const Nth = 25 95 lapNth := (lav + (Nth / 2)) / Nth 96 minLine := allValues[lapNth] 97 maxLine := allValues[lav-lapNth-1] 98 99 pl := plot.New() 100 101 pl.Title.Text = g.Unit 102 pl.Title.TextStyle.Font.Size = 40 103 pl.Y.Label.Text = "tip measure / baseline measure" 104 105 if logScale { 106 pl.Y.Scale = plot.LogScale{} 107 // TODO perhaps these are not the best lines for a log scale. 108 pl.Y.Tick.Marker = ratioLines(minLine, maxLine, allValues[0], allValues[lav-1]) 109 110 } else { 111 pl.Y.Tick.Marker = ratioLines(minLine, maxLine, allValues[0], allValues[lav-1]) 112 } 113 pl.Y.Tick.Label.Font.Size = 20 114 115 grid := plotter.NewGrid() 116 grid.Vertical.Color = nil 117 pl.Add(grid) 118 119 w := vg.Points(10) 120 121 var nominalX []string 122 var boxes []plot.Plotter 123 for i, sp := range selectedPoints { 124 125 perm := absSortedPermFor(sp.changes) 126 rmsChange := norm(sp.changes, 2) 127 sp.changes = permute(sp.changes, perm) 128 l := len(sp.changes) 129 movers := make(map[int]bool) 130 moves := []directedColor{} 131 for k := 1; k <= 5; k++ { 132 if l-k < 0 { 133 break 134 } 135 ch := sp.changes[l-k] 136 c := math.Abs(ch) 137 noteMove := func(clr color.Color) { 138 index := sp.changeBVID[perm[l-k]].valueIndex 139 bi := sp.changeBVID[perm[l-k]].benchIndex 140 prevIndex := -1 141 for _, psp := range selectedPoints[i-1].changeBVID { 142 if psp.benchIndex == bi { 143 prevIndex = psp.valueIndex 144 } 145 } 146 // it should not be the case that a move was recorded, yet there was no match for prevIndex 147 movers[index] = true 148 moves = append(moves, directedColor{prev: selectedPoints[i-1].values.Value(prevIndex), index: index, change: ch, clr: clr}) 149 } 150 if c >= 100 { 151 noteMove(green(0xff)) 152 } else if c > 5 { 153 noteMove(red(0xff)) 154 } else if c > 4 { 155 noteMove(purple(0xff)) 156 } else if c > 3 { 157 noteMove(blue(0xff)) 158 } else { 159 break 160 } 161 162 } 163 164 p := sp.values 165 b, err := MyNewBoxPlot(w, float64(i), p, moves, movers) 166 if err != nil { 167 panic(err) 168 } 169 b.bp.BoxStyle.Color = color.Black 170 b.bp.GlyphStyle.Radius = pointRad 171 172 boxes = append(boxes, b) 173 174 if rmsChange > 6 { 175 b.bp.BoxStyle.Color = red(0xff) 176 b.bp.FillColor = red(0x50) 177 } else if rmsChange > 4 { 178 b.bp.BoxStyle.Color = purple(0xFF) 179 b.bp.FillColor = purple(0x50) 180 } else if percentile(sp.changes, 1) > 4 { 181 b.bp.BoxStyle.Color = blue(0xff) 182 b.bp.FillColor = blue(0x50) 183 } 184 label := sp.numHash + "/" + sp.denHash 185 186 nominalX = append(nominalX, label) 187 } 188 pl.Add(boxes...) 189 pl.NominalX(nominalX...) 190 191 pl.X.Tick.Width = vg.Points(0.5) 192 pl.X.Tick.Length = vg.Points(8) 193 194 pl.X.Tick.Label.Rotation = -math.Pi / 8 195 pl.X.Tick.Label.YAlign = draw.YTop 196 pl.X.Tick.Label.XAlign = draw.XLeft 197 pl.X.Tick.Label.Font.Size = 15 198 199 // Force the unit ratio onto the graph to ensure there is a scale. 200 if pl.Y.Min > 1 { 201 pl.Y.Min = 1 202 } 203 if pl.Y.Max < 1 { 204 pl.Y.Max = 1 205 } 206 207 // Heuristic width and height 208 width := 1.5 * float64(2+len(selectedPoints)) 209 height := width / 3 210 if pl.Y.Max > 1 && pl.Y.Max-1 > 2*(math.Max(maxLine, minLine)-1) || 211 pl.Y.Min < 1 && 1-pl.Y.Min > 2*(1-math.Min(maxLine, minLine)) { 212 height = height * 1.5 213 } 214 if height < 5 { 215 height = 5 216 } 217 dpi := 300 218 219 // // Override heuristics if demanded 220 // if *flagWidth != 0 { 221 // width = *flagWidth 222 // } 223 // if *flagHeight != 0 { 224 // height = *flagHeight 225 // } 226 227 // Scale down dpi to conform to twitter limits 228 initialWidth := float64(dpi) * width / 2.54 229 if initialWidth > 8190 { 230 dpi = int(math.Trunc(float64(dpi) * 8190 / initialWidth)) 231 } 232 //fmt.Printf("%s: W=%f, H=%f, DPI=%d, PYM=%f, PYm=%f, Ml=%f, ml=%f\n", filename, width, height, dpi, 233 // p.Y.Max, p.Y.Min, maxLine, minLine) 234 235 filename := strings.ReplaceAll(g.Unit, "/", "-per-") 236 237 do := func(dir, sfx string, can vg.CanvasWriterTo) { 238 file := filepath.Join(dir, filename) + "." + sfx 239 f, err := os.Create(file) 240 if err != nil { 241 panic(err) 242 } 243 244 pl.Draw(draw.New(can)) 245 _, err = can.WriteTo(f) 246 if err != nil { 247 panic(err) 248 } 249 f.Close() 250 } 251 252 if pngDir != "" { 253 do(pngDir, "png", vgimg.PngCanvas{Canvas: vgimg.NewWith(vgimg.UseWH(vg.Length(width)*vg.Centimeter, vg.Length(height)*vg.Centimeter), 254 vgimg.UseDPI(dpi), vgimg.UseBackgroundColor(color.White))}) 255 } 256 // if pdfDir != "" { 257 // do(pdfDir, "pdf", vgpdf.Canvas{Canvas: vgimg.NewWith(vgimg.UseWH(vg.Length(width)*vg.Centimeter, vg.Length(height)*vg.Centimeter), 258 // vgimg.UseDPI(dpi), vgimg.UseBackgroundColor(color.White))}) 259 // } 260 // if svgDir != "" { 261 // do(svgDir, "svg", vgsvg.Canvas{Canvas: vgimg.NewWith(vgimg.UseWH(vg.Length(width)*vg.Centimeter, vg.Length(height)*vg.Centimeter), 262 // vgimg.UseDPI(dpi), vgimg.UseBackgroundColor(color.White))}) 263 // } 264 // Other formats, including default PNG 265 //if err := p.Save(20*vg.Inch, 5*vg.Inch, filepath.Join(dir, filename)+".png"); err != nil { 266 // panic(err) 267 //} 268 //if err := p.Save(20*vg.Inch, 5*vg.Inch, filepath.Join(dir, filename)+".svg"); err != nil { 269 // panic(err) 270 //} 271 //if err := p.Save(20*vg.Inch, 5*vg.Inch, filepath.Join(dir, filename)+".pdf"); err != nil { 272 // panic(err) 273 //} 274 } 275 } 276 277 func red(alpha uint8) color.Color { 278 return color.NRGBA{0xFF, 0, 0, alpha} 279 } 280 func green(alpha uint8) color.Color { 281 return color.NRGBA{0, 0xFF, 0, alpha} 282 } 283 func blue(alpha uint8) color.Color { 284 return color.NRGBA{0, 0, 0xFF, alpha} 285 } 286 func purple(alpha uint8) color.Color { 287 return color.NRGBA{0x99, 0, 0xFF, alpha} 288 } 289 290 type directedColor struct { 291 prev float64 292 index int 293 change float64 294 clr color.Color 295 } 296 297 type MyBoxPlot struct { 298 bp *plotter.BoxPlot 299 movers map[int]bool 300 moves []directedColor 301 } 302 303 func MyNewBoxPlot(w vg.Length, loc float64, values plotter.Valuer, moves []directedColor, movers map[int]bool) (*MyBoxPlot, error) { 304 b, err := plotter.NewBoxPlot(w, loc, values) 305 if err != nil { 306 return nil, err 307 } 308 return &MyBoxPlot{bp: b, moves: moves, movers: movers}, nil 309 } 310 311 // Plot draws the BoxPlot on Canvas c and Plot plt. 312 func (p *MyBoxPlot) Plot(c draw.Canvas, plt *plot.Plot) { 313 b := p.bp 314 315 trX, trY := plt.Transforms(&c) 316 x := trX(b.Location) 317 px := trX(b.Location - 1) 318 if !c.ContainsX(x) { 319 return 320 } 321 x += b.Offset 322 px += b.Offset 323 324 med := trY(b.Median) 325 q1 := trY(b.Quartile1) 326 q3 := trY(b.Quartile3) 327 aLow := trY(b.AdjLow) 328 aHigh := trY(b.AdjHigh) 329 330 pts := []vg.Point{ 331 {X: x - b.Width/2, Y: q1}, 332 {X: x - b.Width/2, Y: q3}, 333 {X: x + b.Width/2, Y: q3}, 334 {X: x + b.Width/2, Y: q1}, 335 {X: x - b.Width/2 - b.BoxStyle.Width/2, Y: q1}, 336 } 337 box := c.ClipLinesY(pts) 338 if b.FillColor != nil { 339 c.FillPolygon(b.FillColor, c.ClipPolygonY(pts)) 340 } 341 c.StrokeLines(b.BoxStyle, box...) 342 343 medLine := c.ClipLinesY([]vg.Point{ 344 {X: x - b.Width/2, Y: med}, 345 {X: x + b.Width/2, Y: med}, 346 }) 347 c.StrokeLines(b.MedianStyle, medLine...) 348 349 cap := b.CapWidth / 2 350 whisks := c.ClipLinesY( 351 []vg.Point{{X: x, Y: q3}, {X: x, Y: aHigh}}, 352 []vg.Point{{X: x - cap, Y: aHigh}, {X: x + cap, Y: aHigh}}, 353 []vg.Point{{X: x, Y: q1}, {X: x, Y: aLow}}, 354 []vg.Point{{X: x - cap, Y: aLow}, {X: x + cap, Y: aLow}}, 355 ) 356 c.StrokeLines(b.WhiskerStyle, whisks...) 357 358 for _, out := range b.Outside { 359 y := trY(b.Value(out)) 360 if c.ContainsY(y) { 361 c.DrawGlyphNoClip(b.GlyphStyle, vg.Point{X: x, Y: y}) 362 } 363 } 364 365 for _, dc := range p.moves { 366 clr := dc.clr 367 y := trY(b.Value(dc.index)) 368 py := trY(dc.prev) 369 if c.ContainsY(y) && c.ContainsY(py) { 370 c.SetLineStyle(draw.LineStyle{Color: clr, Width: vg.Points(1)}) 371 p := make(vg.Path, 0, 3) 372 p.Move(vg.Point{X: px, Y: py}) 373 p.Line(vg.Point{X: x, Y: y}) 374 c.Stroke(p) 375 } 376 } 377 } 378 379 func (b *MyBoxPlot) DataRange() (float64, float64, float64, float64) { return b.bp.DataRange() } 380 func (b *MyBoxPlot) GlyphBoxes(plt *plot.Plot) []plot.GlyphBox { return b.bp.GlyphBoxes(plt) } 381 func (b *MyBoxPlot) OutsideLabels(labels plotter.Labeller) (*plotter.Labels, error) { 382 return b.bp.OutsideLabels(labels) 383 } 384 385 const ( 386 cosπover4 = vg.Length(.707106781202420) 387 sinπover6 = vg.Length(.500000000025921) 388 cosπover6 = vg.Length(.866025403769473) 389 ) 390 391 // CrossGlyph is a glyph that draws a big X. 392 // this version draws a heavier X. 393 type CrossGlyph struct{} 394 395 // DrawGlyph implements the Glyph interface. 396 func (CrossGlyph) DrawGlyph(c *draw.Canvas, sty draw.GlyphStyle, pt vg.Point) { 397 c.SetLineStyle(draw.LineStyle{Color: sty.Color, Width: vg.Points(1)}) 398 r := sty.Radius * cosπover4 399 p := make(vg.Path, 0, 2) 400 p.Move(vg.Point{X: pt.X - r, Y: pt.Y - r}) 401 p.Line(vg.Point{X: pt.X + r, Y: pt.Y + r}) 402 c.Stroke(p) 403 p = p[:0] 404 p.Move(vg.Point{X: pt.X - r, Y: pt.Y + r}) 405 p.Line(vg.Point{X: pt.X + r, Y: pt.Y - r}) 406 c.Stroke(p) 407 } 408 409 type TriDown struct{} 410 411 // DrawGlyph implements the Glyph interface. 412 func (TriDown) DrawGlyph(c *draw.Canvas, sty draw.GlyphStyle, pt vg.Point) { 413 c.SetLineStyle(draw.LineStyle{Color: sty.Color, Width: vg.Points(1)}) 414 r := sty.Radius * cosπover4 415 p := make(vg.Path, 0, 3) 416 p.Move(vg.Point{X: pt.X - r, Y: pt.Y + r}) 417 p.Line(vg.Point{X: pt.X, Y: pt.Y - r}) 418 c.Stroke(p) 419 p = p[:0] 420 p.Move(vg.Point{X: pt.X + r, Y: pt.Y + r}) 421 p.Line(vg.Point{X: pt.X, Y: pt.Y - r}) 422 c.Stroke(p) 423 p = p[:0] 424 p.Move(vg.Point{X: pt.X - r, Y: pt.Y}) 425 p.Line(vg.Point{X: pt.X + r, Y: pt.Y}) 426 c.Stroke(p) 427 } 428 429 type TriUp struct{} 430 431 // DrawGlyph implements the Glyph interface. 432 func (TriUp) DrawGlyph(c *draw.Canvas, sty draw.GlyphStyle, pt vg.Point) { 433 c.SetLineStyle(draw.LineStyle{Color: sty.Color, Width: vg.Points(1)}) 434 r := sty.Radius * cosπover4 435 p := make(vg.Path, 0, 3) 436 p.Move(vg.Point{X: pt.X - r, Y: pt.Y - r}) 437 p.Line(vg.Point{X: pt.X, Y: pt.Y + r}) 438 c.Stroke(p) 439 p = p[:0] 440 p.Move(vg.Point{X: pt.X + r, Y: pt.Y - r}) 441 p.Line(vg.Point{X: pt.X, Y: pt.Y + r}) 442 c.Stroke(p) 443 p = p[:0] 444 p.Move(vg.Point{X: pt.X - r, Y: pt.Y}) 445 p.Line(vg.Point{X: pt.X + r, Y: pt.Y}) 446 c.Stroke(p) 447 } 448 449 type Lines struct { 450 ticks []plot.Tick 451 } 452 453 // roundish finds a roundish fraction less than x, and the number of digits for formatting. 454 // x is distance from 1.0, so 1 +/- roundish(x) gives a good location for a grid line. 455 func roundish(x float64) (float64, int) { 456 if !(x > 0) { // catch NaN also. 457 panic(fmt.Sprintf("Roundish(%.9g <= 0)", x)) 458 } 459 if x >= 1 { 460 return math.Trunc(x), 0 461 } 462 if x >= 0.5 { 463 return 0.5, 1 464 } 465 if x >= 0.25 { 466 return 0.25, 2 467 } 468 if x >= 0.2 { 469 return 0.2, 1 470 } 471 if x >= 0.1 { 472 return 0.1, 1 473 } 474 x, n := roundish(x * 10) 475 return x / 10, n + 1 476 } 477 478 func reverseTicks(ticks []plot.Tick) []plot.Tick { 479 l := len(ticks) 480 for i := 0; i < l/2; i++ { 481 ticks[i], ticks[l-i-1] = ticks[l-i-1], ticks[i] 482 } 483 return ticks 484 } 485 486 func ratioLines(low, high, min, max float64) Lines { 487 if high <= 1 { 488 if low == 1 { 489 // TODO this is a degenerate case 490 return Lines{ticks: []plot.Tick{one}} 491 } 492 493 step, k := roundish(1 - low) 494 var ticks []plot.Tick 495 for t := 1.0; t > min; t -= step { 496 ticks = append(ticks, tick(t, k)) 497 } 498 499 return Lines{ticks: reverseTicks(ticks)} 500 } else if low >= 1 { 501 502 step, k := roundish(high - 1) 503 k++ // for 1.frac 504 var ticks []plot.Tick 505 for t := 1.0; t < max; t += step { 506 ticks = append(ticks, tick(t, k)) 507 } 508 509 return Lines{ticks: ticks} 510 } 511 rmin, kmin := roundish(1 - low) 512 rmax, k := roundish(high - 1) 513 if rmax < rmin { 514 rmax = rmin 515 k = kmin 516 } 517 k++ // for 1.frac 518 519 step := rmax 520 var ticks []plot.Tick 521 for t := 1.0; t > min; t -= step { 522 ticks = append(ticks, tick(t, k)) 523 } 524 ticks = reverseTicks(ticks) 525 for t := 1.0 + step; t < max; t += step { 526 ticks = append(ticks, tick(t, k)) 527 } 528 529 return Lines{ticks: ticks} 530 } 531 532 func tick(x float64, k int) plot.Tick { 533 return plot.Tick{Value: x, Label: fmt.Sprintf("%.[2]*[1]g", x, k)} 534 } 535 536 var one = plot.Tick{ 537 Value: 1.0, Label: "1.0", 538 } 539 540 func (u Lines) Ticks(min, max float64) []plot.Tick { 541 return u.ticks 542 }