github.com/jgbaldwinbrown/perf@v0.1.1/cmd/benchstat/main_test.go (about)

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"bytes"
     9  	"flag"
    10  	"io"
    11  	"os"
    12  	"os/exec"
    13  	"path/filepath"
    14  	"testing"
    15  )
    16  
    17  func dotSlash(s string) string {
    18  	return "." + string(filepath.Separator) + s
    19  }
    20  
    21  func TestGolden(t *testing.T) {
    22  	if err := os.Chdir("testdata"); err != nil {
    23  		t.Fatal(err)
    24  	}
    25  	defer os.Chdir("..")
    26  	check(t, "exampleold", "exampleold.txt")
    27  	check(t, "example", "exampleold.txt", "examplenew.txt")
    28  	if t.Failed() {
    29  		t.Fatal("skipping other tests")
    30  	}
    31  	check(t, "exampleoldhtml", "-html", "exampleold.txt")
    32  	check(t, "examplehtml", "-html", "exampleold.txt", "examplenew.txt")
    33  	if t.Failed() {
    34  		t.Fatal("skipping other tests")
    35  	}
    36  	check(t, "all", "new.txt", "old.txt", "slashslash4.txt", "x386.txt")
    37  	check(t, "allnosplit", "-split", "", "new.txt", "old.txt", "slashslash4.txt", "x386.txt")
    38  	check(t, "oldnew", "old.txt", "new.txt")
    39  	check(t, "oldnewgeo", "-geomean", "old.txt", "new.txt")
    40  	check(t, "new4", "new.txt", "slashslash4.txt")
    41  	check(t, "oldnewhtml", "-html", "old.txt", "new.txt")
    42  	check(t, "oldnew4html", "-html", "old.txt", "new.txt", "slashslash4.txt")
    43  	check(t, "oldnewttest", "-delta-test=ttest", "old.txt", "new.txt")
    44  	check(t, "packagesold", "packagesold.txt")
    45  	check(t, "packages", "packagesold.txt", "packagesnew.txt")
    46  	check(t, "units", "units-old.txt", "units-new.txt")
    47  	check(t, "zero", "-delta-test=none", "zero-old.txt", "zero-new.txt")
    48  	check(t, "namesort", "-sort=name", "old.txt", "new.txt")
    49  	check(t, "deltasort", "-sort=delta", "old.txt", "new.txt")
    50  	check(t, "rdeltasort", "-sort=-delta", "old.txt", "new.txt")
    51  
    52  	check(t, "oldcsv", "-geomean", "-csv", "old.txt")
    53  	check(t, "allcsv", "-geomean", "-csv", "old.txt", "new.txt", "slashslash4.txt", "x386.txt")
    54  	check(t, "allnosplitcsv", "-geomean", "-csv", "-split", "", dotSlash("new.txt"), dotSlash("old.txt"), dotSlash("slashslash4.txt"), dotSlash("x386.txt")) // note order: new old slashslash4 x386; dotSlash tests common prefix removal
    55  
    56  	if dotSlash("x") == "./x" { // Golden files have hardcoded "/" path separators in them
    57  		check(t, "allnorangecsv", "-geomean", "-csv", "-norange", dotSlash("old.txt"), dotSlash("new.txt"), "slashslash4.txt", "x386.txt") // Mixed ./ tests common prefix non-removal
    58  	}
    59  
    60  }
    61  
    62  func check(t *testing.T, name string, files ...string) {
    63  	t.Run(name, func(t *testing.T) {
    64  		os.Args = append([]string{"benchstat"}, files...)
    65  		t.Logf("running %v", os.Args)
    66  		r, w, err := os.Pipe()
    67  		if err != nil {
    68  			t.Fatal(err)
    69  		}
    70  		c := make(chan []byte)
    71  		go func() {
    72  			data, err := io.ReadAll(r)
    73  			if err != nil {
    74  				t.Error(err)
    75  			}
    76  			c <- data
    77  		}()
    78  		stdout := os.Stdout
    79  		stderr := os.Stderr
    80  		os.Stdout = w
    81  		os.Stderr = w
    82  		exit = func(code int) { t.Fatalf("exit %d during main", code) }
    83  		*flagGeomean = false
    84  		*flagHTML = false
    85  		*flagNoRange = false
    86  		*flagDeltaTest = "utest"
    87  		*flagSplit = flag.Lookup("split").DefValue
    88  
    89  		main()
    90  
    91  		w.Close()
    92  		os.Stdout = stdout
    93  		os.Stderr = stderr
    94  		exit = os.Exit
    95  
    96  		data := <-c
    97  		golden, err := os.ReadFile(name + ".golden")
    98  		if err != nil {
    99  			t.Fatal(err)
   100  		}
   101  		if !bytes.Equal(data, golden) {
   102  			t.Errorf("wrong output: diff have want:\n%s", diff(t, data, golden))
   103  		}
   104  	})
   105  }
   106  
   107  // diff returns the output of 'diff -u old new'.
   108  func diff(t *testing.T, old, new []byte) string {
   109  	data, err := exec.Command("diff", "-u", writeTemp(t, old), writeTemp(t, new)).CombinedOutput()
   110  	if len(data) > 0 {
   111  		return string(data)
   112  	}
   113  	// Most likely, "diff not found" so print the bad output so there is something.
   114  	return "ERROR: " + err.Error() + ": test output = \n" + string(old)
   115  }
   116  
   117  func writeTemp(t *testing.T, data []byte) string {
   118  	f, err := os.CreateTemp("", "benchstat_test")
   119  	if err != nil {
   120  		t.Fatal(err)
   121  	}
   122  	f.Write(data)
   123  	name := f.Name()
   124  	f.Close()
   125  	return name
   126  }