github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/perf/suite/suite.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  // Package suite implements a performance test suite for Noms, intended for
    23  // measuring and reporting long running tests.
    24  //
    25  // Usage is similar to testify's suite:
    26  //  1. Define a test suite struct which inherits from suite.PerfSuite.
    27  //  2. Define methods on that struct that start with the word "Test", optionally
    28  //     followed by digits, then followed a non-empty capitalized string.
    29  //  3. Call suite.Run with an instance of that struct.
    30  //  4. Run go test with the -perf <path to noms db> flag.
    31  //
    32  // Flags:
    33  //  -perf.mem      Backs the database by a memory store, instead of nbs.
    34  //  -perf.prefix   Gives the dataset IDs for test results a prefix.
    35  //  -perf.repeat   Sets how many times tests are repeated ("reps").
    36  //  -perf.run      Only run tests that match a regex (case insensitive).
    37  //  -perf.testdata Sets a custom path to the Noms testdata directory.
    38  //
    39  // PerfSuite also supports testify/suite style Setup/TearDown methods:
    40  //  Setup/TearDownSuite is called exactly once.
    41  //  Setup/TearDownRep   is called for each repetition of the test runs, i.e. -perf.repeat times.
    42  //  Setup/TearDownTest  is called for every test.
    43  //
    44  // Test results are written to Noms, along with a dump of the environment they were recorded in.
    45  //
    46  // Test names are derived from that "non-empty capitalized string": "Test" is omitted because it's
    47  // redundant, and leading digits are omitted to allow for manual test ordering. For example:
    48  //
    49  //  > cat ./samples/go/csv/csv-import/perf_test.go
    50  //  type perfSuite {
    51  //    suite.PerfSuite
    52  //  }
    53  //
    54  //  func (s *perfSuite) TestFoo() { ... }
    55  //  func (s *perfSuite) TestZoo() { ... }
    56  //  func (s *perfSuite) Test01Qux() { ... }
    57  //  func (s *perfSuite) Test02Bar() { ... }
    58  //
    59  //  func TestPerf(t *testing.T) {
    60  //    suite.Run("csv-import", t, &perfSuite{})
    61  //  }
    62  //
    63  //  > noms serve &
    64  //  > go test -v ./samples/go/csv/... -perf http://localhost:8000 -perf.repeat 3
    65  //  (perf) RUN(1/3) Test01Qux (recorded as "Qux")
    66  //  (perf) PASS:    Test01Qux (5s, paused 15s, total 20s)
    67  //  (perf) RUN(1/3) Test02Bar (recorded as "Bar")
    68  //  (perf) PASS:    Test02Bar (15s, paused 2s, total 17s)
    69  //  (perf) RUN(1/3) TestFoo (recorded as "Foo")
    70  //  (perf) PASS:    TestFoo (10s, paused 1s, total 11s)
    71  //  (perf) RUN(1/3) TestZoo (recorded as "Zoo")
    72  //  (perf) PASS:    TestZoo (1s, paused 42s, total 43s)
    73  //  ...
    74  //
    75  //  > noms show http://localhost:8000::csv-import
    76  //  {
    77  //    environment: ...
    78  //    tests: [{
    79  //      "Bar": {elapsed: 15s, paused: 2s,  total: 17s},
    80  //      "Foo": {elapsed: 10s, paused: 1s,  total: 11s},
    81  //      "Qux": {elapsed: 5s,  paused: 15s, total: 20s},
    82  //      "Zoo": {elapsed: 1s,  paused: 42s, total: 43s},
    83  //    }, ...]
    84  //    ...
    85  //  }
    86  package suite
    87  
    88  import (
    89  	"bytes"
    90  	"context"
    91  	"flag"
    92  	"fmt"
    93  	"io"
    94  	"io/ioutil"
    95  	"os"
    96  	"os/exec"
    97  	"path"
    98  	"path/filepath"
    99  	"reflect"
   100  	"regexp"
   101  	"strings"
   102  	"testing"
   103  	"time"
   104  
   105  	"github.com/google/uuid"
   106  	"github.com/shirou/gopsutil/cpu"
   107  	"github.com/shirou/gopsutil/disk"
   108  	"github.com/shirou/gopsutil/host"
   109  	"github.com/shirou/gopsutil/mem"
   110  	"github.com/stretchr/testify/assert"
   111  	"github.com/stretchr/testify/require"
   112  	testifySuite "github.com/stretchr/testify/suite"
   113  
   114  	"github.com/dolthub/dolt/go/libraries/utils/osutil"
   115  	"github.com/dolthub/dolt/go/store/chunks"
   116  	"github.com/dolthub/dolt/go/store/datas"
   117  	"github.com/dolthub/dolt/go/store/marshal"
   118  	"github.com/dolthub/dolt/go/store/spec"
   119  	"github.com/dolthub/dolt/go/store/types"
   120  )
   121  
   122  var (
   123  	perfFlag         = flag.String("perf", "", "The database to write perf tests to. If this isn't specified, perf tests are skipped. If you want a dry run, use \"mem\" as a database")
   124  	perfMemFlag      = flag.Bool("perf.mem", false, "Back the test database by a memory store, not nbs. This will affect test timing, but it's provided in case you're low on disk space")
   125  	perfPrefixFlag   = flag.String("perf.prefix", "", `Prefix for the dataset IDs where results are written. For example, a prefix of "foo/" will write test datasets like "foo/csv-import" instead of just "csv-import"`)
   126  	perfRepeatFlag   = flag.Int("perf.repeat", 1, "The number of times to repeat each perf test")
   127  	perfRunFlag      = flag.String("perf.run", "", "Only run perf tests that match a regular expression")
   128  	perfTestdataFlag = flag.String("perf.testdata", "", "Path to the noms testdata directory. By default this is ../testdata relative to the noms directory")
   129  	testNamePattern  = regexp.MustCompile("^Test[0-9]*([A-Z].*$)")
   130  )
   131  
   132  // PerfSuite is the core of the perf testing suite. See package documentation for details.
   133  type PerfSuite struct {
   134  	// T is the testing.T instance set when the suite is passed into Run.
   135  	T *testing.T
   136  
   137  	// W is the io.Writer to write test output, which only outputs if the verbose flag is set.
   138  	W io.Writer
   139  
   140  	// AtticLabs is the path to the attic-labs directory (e.g. /path/to/go/src/github.com/attic-labs).
   141  	AtticLabs string
   142  
   143  	// Testdata is the path to the testdata directory - typically /path/to/go/src/github.com/attic-labs, but it can be overridden with the -perf.testdata flag.
   144  	Testdata string
   145  
   146  	// Database is a Noms database that tests can use for reading and writing. State is persisted across a single Run of a suite.
   147  	Database datas.Database
   148  
   149  	// DatabaseSpec is the Noms spec of Database (typically a localhost URL).
   150  	DatabaseSpec string
   151  
   152  	tempFiles []*os.File
   153  	tempDirs  []string
   154  	paused    time.Duration
   155  	datasetID string
   156  }
   157  
   158  // SetupRepSuite has a SetupRep method, which runs every repetition of the test, i.e. -perf.repeat times in total.
   159  type SetupRepSuite interface {
   160  	SetupRep()
   161  }
   162  
   163  // TearDownRepSuite has a TearDownRep method, which runs every repetition of the test, i.e. -perf.repeat times in total.
   164  type TearDownRepSuite interface {
   165  	TearDownRep()
   166  }
   167  
   168  type perfSuiteT interface {
   169  	Suite() *PerfSuite
   170  }
   171  
   172  type environment struct {
   173  	DiskUsages map[string]disk.UsageStat
   174  	Cpus       map[int]cpu.InfoStat
   175  	Mem        mem.VirtualMemoryStat
   176  	Host       host.InfoStat
   177  	Partitions map[string]disk.PartitionStat
   178  }
   179  
   180  type timeInfo struct {
   181  	elapsed, paused, total time.Duration
   182  }
   183  
   184  type testRep map[string]timeInfo
   185  
   186  type nopWriter struct{}
   187  
   188  func (r nopWriter) Write(p []byte) (int, error) {
   189  	return len(p), nil
   190  }
   191  
   192  // Run runs suiteT and writes results to dataset datasetID in the database given by the -perf command line flag.
   193  func Run(datasetID string, t *testing.T, suiteT perfSuiteT) {
   194  	t.Skip()
   195  	assert := assert.New(t)
   196  
   197  	if !assert.NotEqual("", datasetID) {
   198  		return
   199  	}
   200  
   201  	// Piggy-back off the go test -v flag.
   202  	verboseFlag := flag.Lookup("test.v")
   203  	assert.NotNil(verboseFlag)
   204  	verbose := verboseFlag.Value.(flag.Getter).Get().(bool)
   205  
   206  	if *perfFlag == "" {
   207  		if verbose {
   208  			fmt.Printf("(perf) Skipping %s, -perf flag not set\n", datasetID)
   209  		}
   210  		return
   211  	}
   212  
   213  	suite := suiteT.Suite()
   214  	suite.T = t
   215  	if verbose {
   216  		suite.W = os.Stdout
   217  	} else {
   218  		suite.W = nopWriter{}
   219  	}
   220  
   221  	id, _ := uuid.NewUUID()
   222  	suite.AtticLabs = filepath.Join(os.TempDir(), "attic-labs", "noms", "suite", id.String())
   223  	suite.Testdata = *perfTestdataFlag
   224  	if suite.Testdata == "" {
   225  		suite.Testdata = filepath.Join(suite.AtticLabs, "testdata")
   226  	}
   227  
   228  	// Clean up temporary directories/files last.
   229  	defer func() {
   230  		for _, f := range suite.tempFiles {
   231  			f.Close()
   232  			os.Remove(f.Name())
   233  		}
   234  		for _, d := range suite.tempDirs {
   235  			os.RemoveAll(d)
   236  		}
   237  	}()
   238  
   239  	suite.datasetID = datasetID
   240  
   241  	// This is the database the perf test results are written to.
   242  	sp, err := spec.ForDatabase(*perfFlag)
   243  	if !assert.NoError(err) {
   244  		return
   245  	}
   246  	defer sp.Close()
   247  
   248  	// List of test runs, each a map of test name => timing info.
   249  	testReps := make([]testRep, *perfRepeatFlag)
   250  
   251  	// Note: the default value of perfRunFlag is "", which is actually a valid
   252  	// regular expression that matches everything.
   253  	perfRunRe, err := regexp.Compile("(?i)" + *perfRunFlag)
   254  	if !assert.NoError(err, `Invalid regular expression "%s"`, *perfRunFlag) {
   255  		return
   256  	}
   257  
   258  	defer func() {
   259  		db := sp.GetDatabase(context.Background())
   260  
   261  		reps := make([]types.Value, *perfRepeatFlag)
   262  		for i, rep := range testReps {
   263  			timesSlice := types.ValueSlice{}
   264  			for name, info := range rep {
   265  				st, err := types.NewStruct(db.Format(), "", types.StructData{
   266  					"elapsed": types.Float(info.elapsed.Nanoseconds()),
   267  					"paused":  types.Float(info.paused.Nanoseconds()),
   268  					"total":   types.Float(info.total.Nanoseconds()),
   269  				})
   270  
   271  				require.NoError(t, err)
   272  				timesSlice = append(timesSlice, types.String(name), st)
   273  			}
   274  			reps[i], err = types.NewMap(context.Background(), db, timesSlice...)
   275  		}
   276  
   277  		l, err := types.NewList(context.Background(), db, reps...)
   278  		require.NoError(t, err)
   279  		record, err := types.NewStruct(db.Format(), "", map[string]types.Value{
   280  			"environment":      suite.getEnvironment(db),
   281  			"nomsRevision":     types.String(suite.getGitHead(path.Join(suite.AtticLabs, "noms"))),
   282  			"testdataRevision": types.String(suite.getGitHead(suite.Testdata)),
   283  			"reps":             l,
   284  		})
   285  		require.NoError(t, err)
   286  
   287  		ds, err := db.GetDataset(context.Background(), *perfPrefixFlag+datasetID)
   288  		require.NoError(t, err)
   289  		_, err = db.CommitValue(context.Background(), ds, record)
   290  		require.NoError(t, err)
   291  	}()
   292  
   293  	if t, ok := suiteT.(testifySuite.SetupAllSuite); ok {
   294  		t.SetupSuite()
   295  	}
   296  
   297  	for repIdx := 0; repIdx < *perfRepeatFlag; repIdx++ {
   298  		testReps[repIdx] = testRep{}
   299  
   300  		storage := &chunks.MemoryStorage{}
   301  		memCS := storage.NewView()
   302  		suite.DatabaseSpec = "mem://"
   303  		suite.Database = datas.NewDatabase(memCS)
   304  		defer suite.Database.Close()
   305  
   306  		if t, ok := suiteT.(SetupRepSuite); ok {
   307  			t.SetupRep()
   308  		}
   309  
   310  		for t, mIdx := reflect.TypeOf(suiteT), 0; mIdx < t.NumMethod(); mIdx++ {
   311  			m := t.Method(mIdx)
   312  
   313  			parts := testNamePattern.FindStringSubmatch(m.Name)
   314  			if parts == nil {
   315  				continue
   316  			}
   317  
   318  			recordName := parts[1]
   319  			if !perfRunRe.MatchString(recordName) && !perfRunRe.MatchString(m.Name) {
   320  				continue
   321  			}
   322  
   323  			if _, ok := testReps[repIdx][recordName]; ok {
   324  				assert.Fail(`Multiple tests are named "%s"`, recordName)
   325  				continue
   326  			}
   327  
   328  			if verbose {
   329  				fmt.Printf("(perf) RUN(%d/%d) %s (as \"%s\")\n", repIdx+1, *perfRepeatFlag, m.Name, recordName)
   330  			}
   331  
   332  			if t, ok := suiteT.(testifySuite.SetupTestSuite); ok {
   333  				t.SetupTest()
   334  			}
   335  
   336  			start := time.Now()
   337  			suite.paused = 0
   338  
   339  			err := callSafe(m.Name, m.Func, suiteT)
   340  
   341  			total := time.Since(start)
   342  			elapsed := total - suite.paused
   343  
   344  			if verbose && err == nil {
   345  				fmt.Printf("(perf) PASS:    %s (%s, paused for %s, total %s)\n", m.Name, elapsed, suite.paused, total)
   346  			} else if err != nil {
   347  				fmt.Printf("(perf) FAIL:    %s (%s, paused for %s, total %s)\n", m.Name, elapsed, suite.paused, total)
   348  				fmt.Println(err)
   349  			}
   350  
   351  			if osutil.IsWindows && elapsed == 0 {
   352  				elapsed = 1
   353  				total = 1
   354  			}
   355  			testReps[repIdx][recordName] = timeInfo{elapsed, suite.paused, total}
   356  
   357  			if t, ok := suiteT.(testifySuite.TearDownTestSuite); ok {
   358  				t.TearDownTest()
   359  			}
   360  		}
   361  
   362  		if t, ok := suiteT.(TearDownRepSuite); ok {
   363  			t.TearDownRep()
   364  		}
   365  	}
   366  
   367  	if t, ok := suiteT.(testifySuite.TearDownAllSuite); ok {
   368  		t.TearDownSuite()
   369  	}
   370  }
   371  
   372  func (suite *PerfSuite) Suite() *PerfSuite {
   373  	return suite
   374  }
   375  
   376  // NewAssert returns the assert.Assertions instance for this test.
   377  func (suite *PerfSuite) NewAssert() *assert.Assertions {
   378  	return assert.New(suite.T)
   379  }
   380  
   381  // TempFile creates a temporary file, which will be automatically cleaned up by
   382  // the perf test suite. Files will be prefixed with the test's dataset ID
   383  func (suite *PerfSuite) TempFile() *os.File {
   384  	f, err := ioutil.TempFile("", suite.tempPrefix())
   385  	require.NoError(suite.T, err)
   386  	suite.tempFiles = append(suite.tempFiles, f)
   387  	return f
   388  }
   389  
   390  // TempDir creates a temporary directory, which will be automatically cleaned
   391  // up by the perf test suite. Directories will be prefixed with the test's
   392  // dataset ID.
   393  func (suite *PerfSuite) TempDir() string {
   394  	d, err := ioutil.TempDir("", suite.tempPrefix())
   395  	require.NoError(suite.T, err)
   396  	suite.tempDirs = append(suite.tempDirs, d)
   397  	return d
   398  }
   399  
   400  func (suite *PerfSuite) tempPrefix() string {
   401  	sep := fmt.Sprintf("%c", os.PathSeparator)
   402  	return strings.Replace(fmt.Sprintf("perf.%s.", suite.datasetID), sep, ".", -1)
   403  }
   404  
   405  // Pause pauses the test timer while fn is executing. Useful for omitting long setup code (e.g. copying files) from the test elapsed time.
   406  func (suite *PerfSuite) Pause(fn func()) {
   407  	start := time.Now()
   408  	fn()
   409  	suite.paused += time.Since(start)
   410  }
   411  
   412  // OpenGlob opens the concatenation of all files that match pattern, returned
   413  // as []io.Reader so it can be used immediately with io.MultiReader.
   414  //
   415  // Large CSV files in testdata are broken up into foo.a, foo.b, etc to get
   416  // around GitHub file size restrictions.
   417  func (suite *PerfSuite) OpenGlob(pattern ...string) []io.Reader {
   418  	glob, err := filepath.Glob(path.Join(pattern...))
   419  	require.NoError(suite.T, err)
   420  
   421  	files := make([]io.Reader, len(glob))
   422  	for i, m := range glob {
   423  		f, err := os.Open(m)
   424  		require.NoError(suite.T, err)
   425  		files[i] = f
   426  	}
   427  
   428  	return files
   429  }
   430  
   431  // CloseGlob closes all of the files, designed to be used with OpenGlob.
   432  func (suite *PerfSuite) CloseGlob(files []io.Reader) {
   433  	for _, f := range files {
   434  		require.NoError(suite.T, f.(*os.File).Close())
   435  	}
   436  }
   437  
   438  func callSafe(name string, fun reflect.Value, args ...interface{}) (err error) {
   439  	defer func() {
   440  		if r := recover(); r != nil {
   441  			err = r.(error)
   442  		}
   443  	}()
   444  
   445  	funArgs := make([]reflect.Value, len(args))
   446  	for i, arg := range args {
   447  		funArgs[i] = reflect.ValueOf(arg)
   448  	}
   449  
   450  	fun.Call(funArgs)
   451  	return
   452  }
   453  
   454  func (suite *PerfSuite) getEnvironment(vrw types.ValueReadWriter) types.Value {
   455  	env := environment{
   456  		DiskUsages: map[string]disk.UsageStat{},
   457  		Cpus:       map[int]cpu.InfoStat{},
   458  		Partitions: map[string]disk.PartitionStat{},
   459  	}
   460  
   461  	partitions, err := disk.Partitions(false)
   462  	require.NoError(suite.T, err)
   463  	for _, p := range partitions {
   464  		usage, err := disk.Usage(p.Mountpoint)
   465  		require.NoError(suite.T, err)
   466  		env.DiskUsages[p.Mountpoint] = *usage
   467  		env.Partitions[p.Device] = p
   468  	}
   469  
   470  	cpus, err := cpu.Info()
   471  	require.NoError(suite.T, err)
   472  	for i, c := range cpus {
   473  		env.Cpus[i] = c
   474  	}
   475  
   476  	mem, err := mem.VirtualMemory()
   477  	require.NoError(suite.T, err)
   478  	env.Mem = *mem
   479  
   480  	hostInfo, err := host.Info()
   481  	require.NoError(suite.T, err)
   482  	env.Host = *hostInfo
   483  
   484  	envStruct, err := marshal.Marshal(context.Background(), vrw, env)
   485  	require.NoError(suite.T, err)
   486  	return envStruct
   487  }
   488  
   489  func (suite *PerfSuite) getGitHead(dir string) string {
   490  	stdout := &bytes.Buffer{}
   491  	cmd := exec.Command("git", "rev-parse", "HEAD")
   492  	cmd.Stdout = stdout
   493  	cmd.Dir = dir
   494  	if err := cmd.Run(); err != nil {
   495  		return ""
   496  	}
   497  	return strings.TrimSpace(stdout.String())
   498  }