github.com/jgbaldwinbrown/perf@v0.1.1/cmd/benchstat/main_test.go (about) 1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package main 6 7 import ( 8 "bytes" 9 "flag" 10 "io" 11 "os" 12 "os/exec" 13 "path/filepath" 14 "testing" 15 ) 16 17 func dotSlash(s string) string { 18 return "." + string(filepath.Separator) + s 19 } 20 21 func TestGolden(t *testing.T) { 22 if err := os.Chdir("testdata"); err != nil { 23 t.Fatal(err) 24 } 25 defer os.Chdir("..") 26 check(t, "exampleold", "exampleold.txt") 27 check(t, "example", "exampleold.txt", "examplenew.txt") 28 if t.Failed() { 29 t.Fatal("skipping other tests") 30 } 31 check(t, "exampleoldhtml", "-html", "exampleold.txt") 32 check(t, "examplehtml", "-html", "exampleold.txt", "examplenew.txt") 33 if t.Failed() { 34 t.Fatal("skipping other tests") 35 } 36 check(t, "all", "new.txt", "old.txt", "slashslash4.txt", "x386.txt") 37 check(t, "allnosplit", "-split", "", "new.txt", "old.txt", "slashslash4.txt", "x386.txt") 38 check(t, "oldnew", "old.txt", "new.txt") 39 check(t, "oldnewgeo", "-geomean", "old.txt", "new.txt") 40 check(t, "new4", "new.txt", "slashslash4.txt") 41 check(t, "oldnewhtml", "-html", "old.txt", "new.txt") 42 check(t, "oldnew4html", "-html", "old.txt", "new.txt", "slashslash4.txt") 43 check(t, "oldnewttest", "-delta-test=ttest", "old.txt", "new.txt") 44 check(t, "packagesold", "packagesold.txt") 45 check(t, "packages", "packagesold.txt", "packagesnew.txt") 46 check(t, "units", "units-old.txt", "units-new.txt") 47 check(t, "zero", "-delta-test=none", "zero-old.txt", "zero-new.txt") 48 check(t, "namesort", "-sort=name", "old.txt", "new.txt") 49 check(t, "deltasort", "-sort=delta", "old.txt", "new.txt") 50 check(t, "rdeltasort", "-sort=-delta", "old.txt", "new.txt") 51 52 check(t, "oldcsv", "-geomean", "-csv", "old.txt") 53 check(t, "allcsv", "-geomean", "-csv", "old.txt", "new.txt", "slashslash4.txt", "x386.txt") 54 check(t, "allnosplitcsv", "-geomean", "-csv", "-split", "", dotSlash("new.txt"), dotSlash("old.txt"), dotSlash("slashslash4.txt"), dotSlash("x386.txt")) // note order: new old slashslash4 x386; dotSlash tests common prefix removal 55 56 if dotSlash("x") == "./x" { // Golden files have hardcoded "/" path separators in them 57 check(t, "allnorangecsv", "-geomean", "-csv", "-norange", dotSlash("old.txt"), dotSlash("new.txt"), "slashslash4.txt", "x386.txt") // Mixed ./ tests common prefix non-removal 58 } 59 60 } 61 62 func check(t *testing.T, name string, files ...string) { 63 t.Run(name, func(t *testing.T) { 64 os.Args = append([]string{"benchstat"}, files...) 65 t.Logf("running %v", os.Args) 66 r, w, err := os.Pipe() 67 if err != nil { 68 t.Fatal(err) 69 } 70 c := make(chan []byte) 71 go func() { 72 data, err := io.ReadAll(r) 73 if err != nil { 74 t.Error(err) 75 } 76 c <- data 77 }() 78 stdout := os.Stdout 79 stderr := os.Stderr 80 os.Stdout = w 81 os.Stderr = w 82 exit = func(code int) { t.Fatalf("exit %d during main", code) } 83 *flagGeomean = false 84 *flagHTML = false 85 *flagNoRange = false 86 *flagDeltaTest = "utest" 87 *flagSplit = flag.Lookup("split").DefValue 88 89 main() 90 91 w.Close() 92 os.Stdout = stdout 93 os.Stderr = stderr 94 exit = os.Exit 95 96 data := <-c 97 golden, err := os.ReadFile(name + ".golden") 98 if err != nil { 99 t.Fatal(err) 100 } 101 if !bytes.Equal(data, golden) { 102 t.Errorf("wrong output: diff have want:\n%s", diff(t, data, golden)) 103 } 104 }) 105 } 106 107 // diff returns the output of 'diff -u old new'. 108 func diff(t *testing.T, old, new []byte) string { 109 data, err := exec.Command("diff", "-u", writeTemp(t, old), writeTemp(t, new)).CombinedOutput() 110 if len(data) > 0 { 111 return string(data) 112 } 113 // Most likely, "diff not found" so print the bad output so there is something. 114 return "ERROR: " + err.Error() + ": test output = \n" + string(old) 115 } 116 117 func writeTemp(t *testing.T, data []byte) string { 118 f, err := os.CreateTemp("", "benchstat_test") 119 if err != nil { 120 t.Fatal(err) 121 } 122 f.Write(data) 123 name := f.Name() 124 f.Close() 125 return name 126 }