github.com/jgbaldwinbrown/perf@v0.1.1/benchstat/delta.go (about) 1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Significance tests. 6 7 package benchstat 8 9 import ( 10 "errors" 11 12 "golang.org/x/perf/internal/stats" 13 ) 14 15 // A DeltaTest compares the old and new metrics and returns the 16 // expected probability that they are drawn from the same distribution. 17 // 18 // If a probability cannot be computed, the DeltaTest returns an 19 // error explaining why. Common errors include ErrSamplesEqual 20 // (all samples are equal), ErrSampleSize (there aren't enough samples), 21 // and ErrZeroVariance (the sample has zero variance). 22 // 23 // As a special case, the missing test NoDeltaTest returns -1, nil. 24 type DeltaTest func(old, new *Metrics) (float64, error) 25 26 // Errors returned by DeltaTest. 27 var ( 28 ErrSamplesEqual = errors.New("all equal") 29 ErrSampleSize = errors.New("too few samples") 30 ErrZeroVariance = errors.New("zero variance") 31 ) 32 33 // NoDeltaTest applies no delta test; it returns -1, nil. 34 func NoDeltaTest(old, new *Metrics) (pval float64, err error) { 35 return -1, nil 36 } 37 38 // TTest is a DeltaTest using the two-sample Welch t-test. 39 func TTest(old, new *Metrics) (pval float64, err error) { 40 t, err := stats.TwoSampleWelchTTest(stats.Sample{Xs: old.RValues}, stats.Sample{Xs: new.RValues}, stats.LocationDiffers) 41 if err != nil { 42 return -1, convertErr(err) 43 } 44 return t.P, nil 45 } 46 47 // UTest is a DeltaTest using the Mann-Whitney U test. 48 func UTest(old, new *Metrics) (pval float64, err error) { 49 u, err := stats.MannWhitneyUTest(old.RValues, new.RValues, stats.LocationDiffers) 50 if err != nil { 51 return -1, convertErr(err) 52 } 53 return u.P, nil 54 } 55 56 // convertErr converts from the stats package's internal errors 57 // to errors exported by this package and expected from 58 // a DeltaTest. 59 // Using different errors makes it possible for clients to use 60 // package benchstat without access to the internal stats package, 61 // and it also gives us a chance to use shorter error messages. 62 func convertErr(err error) error { 63 switch err { 64 case stats.ErrZeroVariance: 65 return ErrZeroVariance 66 case stats.ErrSampleSize: 67 return ErrSampleSize 68 case stats.ErrSamplesEqual: 69 return ErrSamplesEqual 70 } 71 return err 72 }