github.com/jgbaldwinbrown/perf@v0.1.1/benchstat/delta.go (about)

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Significance tests.
     6  
     7  package benchstat
     8  
     9  import (
    10  	"errors"
    11  
    12  	"golang.org/x/perf/internal/stats"
    13  )
    14  
    15  // A DeltaTest compares the old and new metrics and returns the
    16  // expected probability that they are drawn from the same distribution.
    17  //
    18  // If a probability cannot be computed, the DeltaTest returns an
    19  // error explaining why. Common errors include ErrSamplesEqual
    20  // (all samples are equal), ErrSampleSize (there aren't enough samples),
    21  // and ErrZeroVariance (the sample has zero variance).
    22  //
    23  // As a special case, the missing test NoDeltaTest returns -1, nil.
    24  type DeltaTest func(old, new *Metrics) (float64, error)
    25  
    26  // Errors returned by DeltaTest.
    27  var (
    28  	ErrSamplesEqual = errors.New("all equal")
    29  	ErrSampleSize   = errors.New("too few samples")
    30  	ErrZeroVariance = errors.New("zero variance")
    31  )
    32  
    33  // NoDeltaTest applies no delta test; it returns -1, nil.
    34  func NoDeltaTest(old, new *Metrics) (pval float64, err error) {
    35  	return -1, nil
    36  }
    37  
    38  // TTest is a DeltaTest using the two-sample Welch t-test.
    39  func TTest(old, new *Metrics) (pval float64, err error) {
    40  	t, err := stats.TwoSampleWelchTTest(stats.Sample{Xs: old.RValues}, stats.Sample{Xs: new.RValues}, stats.LocationDiffers)
    41  	if err != nil {
    42  		return -1, convertErr(err)
    43  	}
    44  	return t.P, nil
    45  }
    46  
    47  // UTest is a DeltaTest using the Mann-Whitney U test.
    48  func UTest(old, new *Metrics) (pval float64, err error) {
    49  	u, err := stats.MannWhitneyUTest(old.RValues, new.RValues, stats.LocationDiffers)
    50  	if err != nil {
    51  		return -1, convertErr(err)
    52  	}
    53  	return u.P, nil
    54  }
    55  
    56  // convertErr converts from the stats package's internal errors
    57  // to errors exported by this package and expected from
    58  // a DeltaTest.
    59  // Using different errors makes it possible for clients to use
    60  // package benchstat without access to the internal stats package,
    61  // and it also gives us a chance to use shorter error messages.
    62  func convertErr(err error) error {
    63  	switch err {
    64  	case stats.ErrZeroVariance:
    65  		return ErrZeroVariance
    66  	case stats.ErrSampleSize:
    67  		return ErrSampleSize
    68  	case stats.ErrSamplesEqual:
    69  		return ErrSamplesEqual
    70  	}
    71  	return err
    72  }