github.com/ncw/rclone@v1.48.1-0.20190724201158-a35aa1360e3e/fstest/test_all/run.go (about)

     1  // Run a test
     2  
     3  // +build go1.11
     4  
     5  package main
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"go/build"
    11  	"io"
    12  	"log"
    13  	"os"
    14  	"os/exec"
    15  	"path"
    16  	"regexp"
    17  	"runtime"
    18  	"sort"
    19  	"strings"
    20  	"sync"
    21  	"time"
    22  
    23  	"github.com/ncw/rclone/fs"
    24  )
    25  
    26  // Control concurrency per backend if required
    27  var (
    28  	oneOnlyMu sync.Mutex
    29  	oneOnly   = map[string]*sync.Mutex{}
    30  )
    31  
    32  // Run holds info about a running test
    33  //
    34  // A run just runs one command line, but it can be run multiple times
    35  // if retries are needed.
    36  type Run struct {
    37  	// Config
    38  	Remote    string // name of the test remote
    39  	Backend   string // name of the backend
    40  	Path      string // path to the source directory
    41  	SubDir    bool   // add -sub-dir to tests
    42  	FastList  bool   // add -fast-list to tests
    43  	NoRetries bool   // don't retry if set
    44  	OneOnly   bool   // only run test for this backend at once
    45  	NoBinary  bool   // set to not build a binary
    46  	Ignore    map[string]struct{}
    47  	// Internals
    48  	cmdLine     []string
    49  	cmdString   string
    50  	try         int
    51  	err         error
    52  	output      []byte
    53  	failedTests []string
    54  	runFlag     string
    55  	logDir      string   // directory to place the logs
    56  	trialName   string   // name/log file name of current trial
    57  	trialNames  []string // list of all the trials
    58  }
    59  
    60  // Runs records multiple Run objects
    61  type Runs []*Run
    62  
    63  // Sort interface
    64  func (rs Runs) Len() int      { return len(rs) }
    65  func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] }
    66  func (rs Runs) Less(i, j int) bool {
    67  	a, b := rs[i], rs[j]
    68  	if a.Backend < b.Backend {
    69  		return true
    70  	} else if a.Backend > b.Backend {
    71  		return false
    72  	}
    73  	if a.Remote < b.Remote {
    74  		return true
    75  	} else if a.Remote > b.Remote {
    76  		return false
    77  	}
    78  	if a.Path < b.Path {
    79  		return true
    80  	} else if a.Path > b.Path {
    81  		return false
    82  	}
    83  	if !a.SubDir && b.SubDir {
    84  		return true
    85  	} else if a.SubDir && !b.SubDir {
    86  		return false
    87  	}
    88  	if !a.FastList && b.FastList {
    89  		return true
    90  	} else if a.FastList && !b.FastList {
    91  		return false
    92  	}
    93  	return false
    94  }
    95  
    96  // dumpOutput prints the error output
    97  func (r *Run) dumpOutput() {
    98  	log.Println("------------------------------------------------------------")
    99  	log.Printf("---- %q ----", r.cmdString)
   100  	log.Println(string(r.output))
   101  	log.Println("------------------------------------------------------------")
   102  }
   103  
   104  // This converts a slice of test names into a regexp which matches
   105  // them.
   106  func testsToRegexp(tests []string) string {
   107  	var split []map[string]struct{}
   108  	// Make a slice with maps of the used parts at each level
   109  	for _, test := range tests {
   110  		for i, name := range strings.Split(test, "/") {
   111  			if i >= len(split) {
   112  				split = append(split, make(map[string]struct{}))
   113  			}
   114  			split[i][name] = struct{}{}
   115  		}
   116  	}
   117  	var out []string
   118  	for _, level := range split {
   119  		var testsInLevel = []string{}
   120  		for name := range level {
   121  			testsInLevel = append(testsInLevel, name)
   122  		}
   123  		sort.Strings(testsInLevel)
   124  		if len(testsInLevel) > 1 {
   125  			out = append(out, "^("+strings.Join(testsInLevel, "|")+")$")
   126  		} else {
   127  			out = append(out, "^"+testsInLevel[0]+"$")
   128  		}
   129  	}
   130  	return strings.Join(out, "/")
   131  }
   132  
   133  var failRe = regexp.MustCompile(`(?m)^\s*--- FAIL: (Test.*?) \(`)
   134  
   135  // findFailures looks for all the tests which failed
   136  func (r *Run) findFailures() {
   137  	oldFailedTests := r.failedTests
   138  	r.failedTests = nil
   139  	excludeParents := map[string]struct{}{}
   140  	ignored := 0
   141  	for _, matches := range failRe.FindAllSubmatch(r.output, -1) {
   142  		failedTest := string(matches[1])
   143  		// Skip any ignored failures
   144  		if _, found := r.Ignore[failedTest]; found {
   145  			ignored++
   146  		} else {
   147  			r.failedTests = append(r.failedTests, failedTest)
   148  		}
   149  		// Find all the parents of this test
   150  		parts := strings.Split(failedTest, "/")
   151  		for i := len(parts) - 1; i >= 1; i-- {
   152  			excludeParents[strings.Join(parts[:i], "/")] = struct{}{}
   153  		}
   154  	}
   155  	// Exclude the parents
   156  	var newTests = r.failedTests[:0]
   157  	for _, failedTest := range r.failedTests {
   158  		if _, excluded := excludeParents[failedTest]; !excluded {
   159  			newTests = append(newTests, failedTest)
   160  		}
   161  	}
   162  	r.failedTests = newTests
   163  	if len(r.failedTests) == 0 && ignored > 0 {
   164  		log.Printf("%q - Found %d ignored errors only - marking as good", r.cmdString, ignored)
   165  		r.err = nil
   166  		r.dumpOutput()
   167  		return
   168  	}
   169  	if len(r.failedTests) != 0 {
   170  		r.runFlag = testsToRegexp(r.failedTests)
   171  	} else {
   172  		r.runFlag = ""
   173  	}
   174  	if r.passed() && len(r.failedTests) != 0 {
   175  		log.Printf("%q - Expecting no errors but got: %v", r.cmdString, r.failedTests)
   176  		r.dumpOutput()
   177  	} else if !r.passed() && len(r.failedTests) == 0 {
   178  		log.Printf("%q - Expecting errors but got none: %v", r.cmdString, r.failedTests)
   179  		r.dumpOutput()
   180  		r.failedTests = oldFailedTests
   181  	}
   182  }
   183  
   184  // nextCmdLine returns the next command line
   185  func (r *Run) nextCmdLine() []string {
   186  	cmdLine := r.cmdLine
   187  	if r.runFlag != "" {
   188  		cmdLine = append(cmdLine, "-test.run", r.runFlag)
   189  	}
   190  	return cmdLine
   191  }
   192  
   193  // trial runs a single test
   194  func (r *Run) trial() {
   195  	cmdLine := r.nextCmdLine()
   196  	cmdString := toShell(cmdLine)
   197  	msg := fmt.Sprintf("%q - Starting (try %d/%d)", cmdString, r.try, *maxTries)
   198  	log.Println(msg)
   199  	logName := path.Join(r.logDir, r.trialName)
   200  	out, err := os.Create(logName)
   201  	if err != nil {
   202  		log.Fatalf("Couldn't create log file: %v", err)
   203  	}
   204  	defer func() {
   205  		err := out.Close()
   206  		if err != nil {
   207  			log.Fatalf("Failed to close log file: %v", err)
   208  		}
   209  	}()
   210  	_, _ = fmt.Fprintln(out, msg)
   211  
   212  	// Early exit if --try-run
   213  	if *dryRun {
   214  		log.Printf("Not executing as --dry-run: %v", cmdLine)
   215  		_, _ = fmt.Fprintln(out, "--dry-run is set - not running")
   216  		return
   217  	}
   218  
   219  	// Internal buffer
   220  	var b bytes.Buffer
   221  	multiOut := io.MultiWriter(out, &b)
   222  
   223  	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
   224  	cmd.Stderr = multiOut
   225  	cmd.Stdout = multiOut
   226  	cmd.Dir = r.Path
   227  	start := time.Now()
   228  	r.err = cmd.Run()
   229  	r.output = b.Bytes()
   230  	duration := time.Since(start)
   231  	r.findFailures()
   232  	if r.passed() {
   233  		msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, r.try, *maxTries)
   234  	} else {
   235  		msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, r.try, *maxTries, r.err, r.failedTests)
   236  	}
   237  	log.Println(msg)
   238  	_, _ = fmt.Fprintln(out, msg)
   239  }
   240  
   241  // passed returns true if the test passed
   242  func (r *Run) passed() bool {
   243  	return r.err == nil
   244  }
   245  
   246  // GOPATH returns the current GOPATH
   247  func GOPATH() string {
   248  	gopath := os.Getenv("GOPATH")
   249  	if gopath == "" {
   250  		gopath = build.Default.GOPATH
   251  	}
   252  	return gopath
   253  }
   254  
   255  // BinaryName turns a package name into a binary name
   256  func (r *Run) BinaryName() string {
   257  	binary := path.Base(r.Path) + ".test"
   258  	if runtime.GOOS == "windows" {
   259  		binary += ".exe"
   260  	}
   261  	return binary
   262  }
   263  
   264  // BinaryPath turns a package name into a binary path
   265  func (r *Run) BinaryPath() string {
   266  	return path.Join(r.Path, r.BinaryName())
   267  }
   268  
   269  // PackagePath returns the path to the package
   270  func (r *Run) PackagePath() string {
   271  	return path.Join(GOPATH(), "src", r.Path)
   272  }
   273  
   274  // MakeTestBinary makes the binary we will run
   275  func (r *Run) MakeTestBinary() {
   276  	binary := r.BinaryPath()
   277  	binaryName := r.BinaryName()
   278  	log.Printf("%s: Making test binary %q", r.Path, binaryName)
   279  	cmdLine := []string{"go", "test", "-c"}
   280  	if *dryRun {
   281  		log.Printf("Not executing: %v", cmdLine)
   282  		return
   283  	}
   284  	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
   285  	cmd.Dir = r.Path
   286  	err := cmd.Run()
   287  	if err != nil {
   288  		log.Fatalf("Failed to make test binary: %v", err)
   289  	}
   290  	if _, err := os.Stat(binary); err != nil {
   291  		log.Fatalf("Couldn't find test binary %q", binary)
   292  	}
   293  }
   294  
   295  // RemoveTestBinary removes the binary made in makeTestBinary
   296  func (r *Run) RemoveTestBinary() {
   297  	if *dryRun {
   298  		return
   299  	}
   300  	binary := r.BinaryPath()
   301  	err := os.Remove(binary) // Delete the binary when finished
   302  	if err != nil {
   303  		log.Printf("Error removing test binary %q: %v", binary, err)
   304  	}
   305  }
   306  
   307  // Name returns the run name as a file name friendly string
   308  func (r *Run) Name() string {
   309  	ns := []string{
   310  		r.Backend,
   311  		strings.Replace(r.Path, "/", ".", -1),
   312  		r.Remote,
   313  	}
   314  	if r.SubDir {
   315  		ns = append(ns, "subdir")
   316  	}
   317  	if r.FastList {
   318  		ns = append(ns, "fastlist")
   319  	}
   320  	ns = append(ns, fmt.Sprintf("%d", r.try))
   321  	s := strings.Join(ns, "-")
   322  	s = strings.Replace(s, ":", "", -1)
   323  	return s
   324  }
   325  
   326  // Init the Run
   327  func (r *Run) Init() {
   328  	prefix := "-test."
   329  	if r.NoBinary {
   330  		prefix = "-"
   331  		r.cmdLine = []string{"go", "test"}
   332  	} else {
   333  		r.cmdLine = []string{"./" + r.BinaryName()}
   334  	}
   335  	r.cmdLine = append(r.cmdLine, prefix+"v", prefix+"timeout", timeout.String(), "-remote", r.Remote)
   336  	r.try = 1
   337  	if *verbose {
   338  		r.cmdLine = append(r.cmdLine, "-verbose")
   339  		fs.Config.LogLevel = fs.LogLevelDebug
   340  	}
   341  	if *runOnly != "" {
   342  		r.cmdLine = append(r.cmdLine, prefix+"run", *runOnly)
   343  	}
   344  	if r.SubDir {
   345  		r.cmdLine = append(r.cmdLine, "-subdir")
   346  	}
   347  	if r.FastList {
   348  		r.cmdLine = append(r.cmdLine, "-fast-list")
   349  	}
   350  	r.cmdString = toShell(r.cmdLine)
   351  }
   352  
   353  // Logs returns all the log names
   354  func (r *Run) Logs() []string {
   355  	return r.trialNames
   356  }
   357  
   358  // FailedTests returns the failed tests as a comma separated string, limiting the number
   359  func (r *Run) FailedTests() string {
   360  	const maxTests = 5
   361  	ts := r.failedTests
   362  	if len(ts) > maxTests {
   363  		ts = ts[:maxTests:maxTests]
   364  		ts = append(ts, fmt.Sprintf("… (%d more)", len(r.failedTests)-maxTests))
   365  	}
   366  	return strings.Join(ts, ", ")
   367  }
   368  
   369  // Run runs all the trials for this test
   370  func (r *Run) Run(logDir string, result chan<- *Run) {
   371  	if r.OneOnly {
   372  		oneOnlyMu.Lock()
   373  		mu := oneOnly[r.Backend]
   374  		if mu == nil {
   375  			mu = new(sync.Mutex)
   376  			oneOnly[r.Backend] = mu
   377  		}
   378  		oneOnlyMu.Unlock()
   379  		mu.Lock()
   380  		defer mu.Unlock()
   381  	}
   382  	r.Init()
   383  	r.logDir = logDir
   384  	for r.try = 1; r.try <= *maxTries; r.try++ {
   385  		r.trialName = r.Name() + ".txt"
   386  		r.trialNames = append(r.trialNames, r.trialName)
   387  		log.Printf("Starting run with log %q", r.trialName)
   388  		r.trial()
   389  		if r.passed() || r.NoRetries {
   390  			break
   391  		}
   392  	}
   393  	if !r.passed() {
   394  		r.dumpOutput()
   395  	}
   396  	result <- r
   397  }