github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/fstest/test_all/run.go (about)

     1  // Run a test
     2  
     3  package main
     4  
     5  import (
     6  	"bytes"
     7  	"fmt"
     8  	"go/build"
     9  	"io"
    10  	"log"
    11  	"os"
    12  	"os/exec"
    13  	"path"
    14  	"regexp"
    15  	"runtime"
    16  	"sort"
    17  	"strconv"
    18  	"strings"
    19  	"sync"
    20  	"time"
    21  
    22  	"github.com/rclone/rclone/fs"
    23  	"github.com/rclone/rclone/fstest/testserver"
    24  )
    25  
    26  // Control concurrency per backend if required
    27  var (
    28  	oneOnlyMu sync.Mutex
    29  	oneOnly   = map[string]*sync.Mutex{}
    30  )
    31  
    32  // Run holds info about a running test
    33  //
    34  // A run just runs one command line, but it can be run multiple times
    35  // if retries are needed.
    36  type Run struct {
    37  	// Config
    38  	Remote    string // name of the test remote
    39  	Backend   string // name of the backend
    40  	Path      string // path to the source directory
    41  	FastList  bool   // add -fast-list to tests
    42  	Short     bool   // add -short
    43  	NoRetries bool   // don't retry if set
    44  	OneOnly   bool   // only run test for this backend at once
    45  	NoBinary  bool   // set to not build a binary
    46  	SizeLimit int64  // maximum test file size
    47  	Ignore    map[string]struct{}
    48  	// Internals
    49  	cmdLine     []string
    50  	cmdString   string
    51  	try         int
    52  	err         error
    53  	output      []byte
    54  	failedTests []string
    55  	runFlag     string
    56  	logDir      string   // directory to place the logs
    57  	trialName   string   // name/log file name of current trial
    58  	trialNames  []string // list of all the trials
    59  }
    60  
    61  // Runs records multiple Run objects
    62  type Runs []*Run
    63  
    64  // Sort interface
    65  func (rs Runs) Len() int      { return len(rs) }
    66  func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] }
    67  func (rs Runs) Less(i, j int) bool {
    68  	a, b := rs[i], rs[j]
    69  	if a.Backend < b.Backend {
    70  		return true
    71  	} else if a.Backend > b.Backend {
    72  		return false
    73  	}
    74  	if a.Remote < b.Remote {
    75  		return true
    76  	} else if a.Remote > b.Remote {
    77  		return false
    78  	}
    79  	if a.Path < b.Path {
    80  		return true
    81  	} else if a.Path > b.Path {
    82  		return false
    83  	}
    84  	if !a.FastList && b.FastList {
    85  		return true
    86  	} else if a.FastList && !b.FastList {
    87  		return false
    88  	}
    89  	return false
    90  }
    91  
    92  // dumpOutput prints the error output
    93  func (r *Run) dumpOutput() {
    94  	log.Println("------------------------------------------------------------")
    95  	log.Printf("---- %q ----", r.cmdString)
    96  	log.Println(string(r.output))
    97  	log.Println("------------------------------------------------------------")
    98  }
    99  
   100  // This converts a slice of test names into a regexp which matches
   101  // them.
   102  func testsToRegexp(tests []string) string {
   103  	var split []map[string]struct{}
   104  	// Make a slice with maps of the used parts at each level
   105  	for _, test := range tests {
   106  		for i, name := range strings.Split(test, "/") {
   107  			if i >= len(split) {
   108  				split = append(split, make(map[string]struct{}))
   109  			}
   110  			split[i][name] = struct{}{}
   111  		}
   112  	}
   113  	var out []string
   114  	for _, level := range split {
   115  		var testsInLevel = []string{}
   116  		for name := range level {
   117  			testsInLevel = append(testsInLevel, name)
   118  		}
   119  		sort.Strings(testsInLevel)
   120  		if len(testsInLevel) > 1 {
   121  			out = append(out, "^("+strings.Join(testsInLevel, "|")+")$")
   122  		} else {
   123  			out = append(out, "^"+testsInLevel[0]+"$")
   124  		}
   125  	}
   126  	return strings.Join(out, "/")
   127  }
   128  
   129  var failRe = regexp.MustCompile(`(?m)^\s*--- FAIL: (Test.*?) \(`)
   130  
   131  // findFailures looks for all the tests which failed
   132  func (r *Run) findFailures() {
   133  	oldFailedTests := r.failedTests
   134  	r.failedTests = nil
   135  	excludeParents := map[string]struct{}{}
   136  	ignored := 0
   137  	for _, matches := range failRe.FindAllSubmatch(r.output, -1) {
   138  		failedTest := string(matches[1])
   139  		// Skip any ignored failures
   140  		if _, found := r.Ignore[failedTest]; found {
   141  			ignored++
   142  		} else {
   143  			r.failedTests = append(r.failedTests, failedTest)
   144  		}
   145  		// Find all the parents of this test
   146  		parts := strings.Split(failedTest, "/")
   147  		for i := len(parts) - 1; i >= 1; i-- {
   148  			excludeParents[strings.Join(parts[:i], "/")] = struct{}{}
   149  		}
   150  	}
   151  	// Exclude the parents
   152  	var newTests = r.failedTests[:0]
   153  	for _, failedTest := range r.failedTests {
   154  		if _, excluded := excludeParents[failedTest]; !excluded {
   155  			newTests = append(newTests, failedTest)
   156  		}
   157  	}
   158  	r.failedTests = newTests
   159  	if len(r.failedTests) == 0 && ignored > 0 {
   160  		log.Printf("%q - Found %d ignored errors only - marking as good", r.cmdString, ignored)
   161  		r.err = nil
   162  		r.dumpOutput()
   163  		return
   164  	}
   165  	if len(r.failedTests) != 0 {
   166  		r.runFlag = testsToRegexp(r.failedTests)
   167  	} else {
   168  		r.runFlag = ""
   169  	}
   170  	if r.passed() && len(r.failedTests) != 0 {
   171  		log.Printf("%q - Expecting no errors but got: %v", r.cmdString, r.failedTests)
   172  		r.dumpOutput()
   173  	} else if !r.passed() && len(r.failedTests) == 0 {
   174  		log.Printf("%q - Expecting errors but got none: %v", r.cmdString, r.failedTests)
   175  		r.dumpOutput()
   176  		r.failedTests = oldFailedTests
   177  	}
   178  }
   179  
   180  // nextCmdLine returns the next command line
   181  func (r *Run) nextCmdLine() []string {
   182  	cmdLine := r.cmdLine
   183  	if r.runFlag != "" {
   184  		cmdLine = append(cmdLine, "-test.run", r.runFlag)
   185  	}
   186  	return cmdLine
   187  }
   188  
   189  // trial runs a single test
   190  func (r *Run) trial() {
   191  	cmdLine := r.nextCmdLine()
   192  	cmdString := toShell(cmdLine)
   193  	msg := fmt.Sprintf("%q - Starting (try %d/%d)", cmdString, r.try, *maxTries)
   194  	log.Println(msg)
   195  	logName := path.Join(r.logDir, r.trialName)
   196  	out, err := os.Create(logName)
   197  	if err != nil {
   198  		log.Fatalf("Couldn't create log file: %v", err)
   199  	}
   200  	defer func() {
   201  		err := out.Close()
   202  		if err != nil {
   203  			log.Fatalf("Failed to close log file: %v", err)
   204  		}
   205  	}()
   206  	_, _ = fmt.Fprintln(out, msg)
   207  
   208  	// Early exit if --try-run
   209  	if *dryRun {
   210  		log.Printf("Not executing as --dry-run: %v", cmdLine)
   211  		_, _ = fmt.Fprintln(out, "--dry-run is set - not running")
   212  		return
   213  	}
   214  
   215  	// Start the test server if required
   216  	finish, err := testserver.Start(r.Remote)
   217  	if err != nil {
   218  		log.Printf("%s: Failed to start test server: %v", r.Remote, err)
   219  		_, _ = fmt.Fprintf(out, "%s: Failed to start test server: %v\n", r.Remote, err)
   220  		r.err = err
   221  		return
   222  	}
   223  	defer finish()
   224  
   225  	// Internal buffer
   226  	var b bytes.Buffer
   227  	multiOut := io.MultiWriter(out, &b)
   228  
   229  	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
   230  	cmd.Stderr = multiOut
   231  	cmd.Stdout = multiOut
   232  	cmd.Dir = r.Path
   233  	start := time.Now()
   234  	r.err = cmd.Run()
   235  	r.output = b.Bytes()
   236  	duration := time.Since(start)
   237  	r.findFailures()
   238  	if r.passed() {
   239  		msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, r.try, *maxTries)
   240  	} else {
   241  		msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, r.try, *maxTries, r.err, r.failedTests)
   242  	}
   243  	log.Println(msg)
   244  	_, _ = fmt.Fprintln(out, msg)
   245  }
   246  
   247  // passed returns true if the test passed
   248  func (r *Run) passed() bool {
   249  	return r.err == nil
   250  }
   251  
   252  // GOPATH returns the current GOPATH
   253  func GOPATH() string {
   254  	gopath := os.Getenv("GOPATH")
   255  	if gopath == "" {
   256  		gopath = build.Default.GOPATH
   257  	}
   258  	return gopath
   259  }
   260  
   261  // BinaryName turns a package name into a binary name
   262  func (r *Run) BinaryName() string {
   263  	binary := path.Base(r.Path) + ".test"
   264  	if runtime.GOOS == "windows" {
   265  		binary += ".exe"
   266  	}
   267  	return binary
   268  }
   269  
   270  // BinaryPath turns a package name into a binary path
   271  func (r *Run) BinaryPath() string {
   272  	return path.Join(r.Path, r.BinaryName())
   273  }
   274  
   275  // PackagePath returns the path to the package
   276  func (r *Run) PackagePath() string {
   277  	return path.Join(GOPATH(), "src", r.Path)
   278  }
   279  
   280  // MakeTestBinary makes the binary we will run
   281  func (r *Run) MakeTestBinary() {
   282  	binary := r.BinaryPath()
   283  	binaryName := r.BinaryName()
   284  	log.Printf("%s: Making test binary %q", r.Path, binaryName)
   285  	cmdLine := []string{"go", "test", "-c"}
   286  	if *dryRun {
   287  		log.Printf("Not executing: %v", cmdLine)
   288  		return
   289  	}
   290  	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
   291  	cmd.Dir = r.Path
   292  	err := cmd.Run()
   293  	if err != nil {
   294  		log.Fatalf("Failed to make test binary: %v", err)
   295  	}
   296  	if _, err := os.Stat(binary); err != nil {
   297  		log.Fatalf("Couldn't find test binary %q", binary)
   298  	}
   299  }
   300  
   301  // RemoveTestBinary removes the binary made in makeTestBinary
   302  func (r *Run) RemoveTestBinary() {
   303  	if *dryRun {
   304  		return
   305  	}
   306  	binary := r.BinaryPath()
   307  	err := os.Remove(binary) // Delete the binary when finished
   308  	if err != nil {
   309  		log.Printf("Error removing test binary %q: %v", binary, err)
   310  	}
   311  }
   312  
   313  // Name returns the run name as a file name friendly string
   314  func (r *Run) Name() string {
   315  	ns := []string{
   316  		r.Backend,
   317  		strings.Replace(r.Path, "/", ".", -1),
   318  		r.Remote,
   319  	}
   320  	if r.FastList {
   321  		ns = append(ns, "fastlist")
   322  	}
   323  	ns = append(ns, fmt.Sprintf("%d", r.try))
   324  	s := strings.Join(ns, "-")
   325  	s = strings.Replace(s, ":", "", -1)
   326  	return s
   327  }
   328  
   329  // Init the Run
   330  func (r *Run) Init() {
   331  	prefix := "-test."
   332  	if r.NoBinary {
   333  		prefix = "-"
   334  		r.cmdLine = []string{"go", "test"}
   335  	} else {
   336  		r.cmdLine = []string{"./" + r.BinaryName()}
   337  	}
   338  	r.cmdLine = append(r.cmdLine, prefix+"v", prefix+"timeout", timeout.String(), "-remote", r.Remote)
   339  	if *listRetries > 0 {
   340  		r.cmdLine = append(r.cmdLine, "-list-retries", fmt.Sprint(*listRetries))
   341  	}
   342  	r.try = 1
   343  	if *verbose {
   344  		r.cmdLine = append(r.cmdLine, "-verbose")
   345  		fs.Config.LogLevel = fs.LogLevelDebug
   346  	}
   347  	if *runOnly != "" {
   348  		r.cmdLine = append(r.cmdLine, prefix+"run", *runOnly)
   349  	}
   350  	if r.FastList {
   351  		r.cmdLine = append(r.cmdLine, "-fast-list")
   352  	}
   353  	if r.Short {
   354  		r.cmdLine = append(r.cmdLine, "-short")
   355  	}
   356  	if r.SizeLimit > 0 {
   357  		r.cmdLine = append(r.cmdLine, "-size-limit", strconv.FormatInt(r.SizeLimit, 10))
   358  	}
   359  	r.cmdString = toShell(r.cmdLine)
   360  }
   361  
   362  // Logs returns all the log names
   363  func (r *Run) Logs() []string {
   364  	return r.trialNames
   365  }
   366  
   367  // FailedTests returns the failed tests as a comma separated string, limiting the number
   368  func (r *Run) FailedTests() string {
   369  	const maxTests = 5
   370  	ts := r.failedTests
   371  	if len(ts) > maxTests {
   372  		ts = ts[:maxTests:maxTests]
   373  		ts = append(ts, fmt.Sprintf("… (%d more)", len(r.failedTests)-maxTests))
   374  	}
   375  	return strings.Join(ts, ", ")
   376  }
   377  
   378  // Run runs all the trials for this test
   379  func (r *Run) Run(logDir string, result chan<- *Run) {
   380  	if r.OneOnly {
   381  		oneOnlyMu.Lock()
   382  		mu := oneOnly[r.Backend]
   383  		if mu == nil {
   384  			mu = new(sync.Mutex)
   385  			oneOnly[r.Backend] = mu
   386  		}
   387  		oneOnlyMu.Unlock()
   388  		mu.Lock()
   389  		defer mu.Unlock()
   390  	}
   391  	r.Init()
   392  	r.logDir = logDir
   393  	for r.try = 1; r.try <= *maxTries; r.try++ {
   394  		r.trialName = r.Name() + ".txt"
   395  		r.trialNames = append(r.trialNames, r.trialName)
   396  		log.Printf("Starting run with log %q", r.trialName)
   397  		r.trial()
   398  		if r.passed() || r.NoRetries {
   399  			break
   400  		}
   401  	}
   402  	if !r.passed() {
   403  		r.dumpOutput()
   404  	}
   405  	result <- r
   406  }