github.com/ncw/rclone@v1.48.1-0.20190724201158-a35aa1360e3e/fstest/test_all/run.go (about) 1 // Run a test 2 3 // +build go1.11 4 5 package main 6 7 import ( 8 "bytes" 9 "fmt" 10 "go/build" 11 "io" 12 "log" 13 "os" 14 "os/exec" 15 "path" 16 "regexp" 17 "runtime" 18 "sort" 19 "strings" 20 "sync" 21 "time" 22 23 "github.com/ncw/rclone/fs" 24 ) 25 26 // Control concurrency per backend if required 27 var ( 28 oneOnlyMu sync.Mutex 29 oneOnly = map[string]*sync.Mutex{} 30 ) 31 32 // Run holds info about a running test 33 // 34 // A run just runs one command line, but it can be run multiple times 35 // if retries are needed. 36 type Run struct { 37 // Config 38 Remote string // name of the test remote 39 Backend string // name of the backend 40 Path string // path to the source directory 41 SubDir bool // add -sub-dir to tests 42 FastList bool // add -fast-list to tests 43 NoRetries bool // don't retry if set 44 OneOnly bool // only run test for this backend at once 45 NoBinary bool // set to not build a binary 46 Ignore map[string]struct{} 47 // Internals 48 cmdLine []string 49 cmdString string 50 try int 51 err error 52 output []byte 53 failedTests []string 54 runFlag string 55 logDir string // directory to place the logs 56 trialName string // name/log file name of current trial 57 trialNames []string // list of all the trials 58 } 59 60 // Runs records multiple Run objects 61 type Runs []*Run 62 63 // Sort interface 64 func (rs Runs) Len() int { return len(rs) } 65 func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] } 66 func (rs Runs) Less(i, j int) bool { 67 a, b := rs[i], rs[j] 68 if a.Backend < b.Backend { 69 return true 70 } else if a.Backend > b.Backend { 71 return false 72 } 73 if a.Remote < b.Remote { 74 return true 75 } else if a.Remote > b.Remote { 76 return false 77 } 78 if a.Path < b.Path { 79 return true 80 } else if a.Path > b.Path { 81 return false 82 } 83 if !a.SubDir && b.SubDir { 84 return true 85 } else if a.SubDir && !b.SubDir { 86 return false 87 } 88 if !a.FastList && b.FastList { 89 return true 90 } else if a.FastList && !b.FastList { 91 return false 92 } 93 return false 94 } 95 96 // dumpOutput prints the error output 97 func (r *Run) dumpOutput() { 98 log.Println("------------------------------------------------------------") 99 log.Printf("---- %q ----", r.cmdString) 100 log.Println(string(r.output)) 101 log.Println("------------------------------------------------------------") 102 } 103 104 // This converts a slice of test names into a regexp which matches 105 // them. 106 func testsToRegexp(tests []string) string { 107 var split []map[string]struct{} 108 // Make a slice with maps of the used parts at each level 109 for _, test := range tests { 110 for i, name := range strings.Split(test, "/") { 111 if i >= len(split) { 112 split = append(split, make(map[string]struct{})) 113 } 114 split[i][name] = struct{}{} 115 } 116 } 117 var out []string 118 for _, level := range split { 119 var testsInLevel = []string{} 120 for name := range level { 121 testsInLevel = append(testsInLevel, name) 122 } 123 sort.Strings(testsInLevel) 124 if len(testsInLevel) > 1 { 125 out = append(out, "^("+strings.Join(testsInLevel, "|")+")$") 126 } else { 127 out = append(out, "^"+testsInLevel[0]+"$") 128 } 129 } 130 return strings.Join(out, "/") 131 } 132 133 var failRe = regexp.MustCompile(`(?m)^\s*--- FAIL: (Test.*?) \(`) 134 135 // findFailures looks for all the tests which failed 136 func (r *Run) findFailures() { 137 oldFailedTests := r.failedTests 138 r.failedTests = nil 139 excludeParents := map[string]struct{}{} 140 ignored := 0 141 for _, matches := range failRe.FindAllSubmatch(r.output, -1) { 142 failedTest := string(matches[1]) 143 // Skip any ignored failures 144 if _, found := r.Ignore[failedTest]; found { 145 ignored++ 146 } else { 147 r.failedTests = append(r.failedTests, failedTest) 148 } 149 // Find all the parents of this test 150 parts := strings.Split(failedTest, "/") 151 for i := len(parts) - 1; i >= 1; i-- { 152 excludeParents[strings.Join(parts[:i], "/")] = struct{}{} 153 } 154 } 155 // Exclude the parents 156 var newTests = r.failedTests[:0] 157 for _, failedTest := range r.failedTests { 158 if _, excluded := excludeParents[failedTest]; !excluded { 159 newTests = append(newTests, failedTest) 160 } 161 } 162 r.failedTests = newTests 163 if len(r.failedTests) == 0 && ignored > 0 { 164 log.Printf("%q - Found %d ignored errors only - marking as good", r.cmdString, ignored) 165 r.err = nil 166 r.dumpOutput() 167 return 168 } 169 if len(r.failedTests) != 0 { 170 r.runFlag = testsToRegexp(r.failedTests) 171 } else { 172 r.runFlag = "" 173 } 174 if r.passed() && len(r.failedTests) != 0 { 175 log.Printf("%q - Expecting no errors but got: %v", r.cmdString, r.failedTests) 176 r.dumpOutput() 177 } else if !r.passed() && len(r.failedTests) == 0 { 178 log.Printf("%q - Expecting errors but got none: %v", r.cmdString, r.failedTests) 179 r.dumpOutput() 180 r.failedTests = oldFailedTests 181 } 182 } 183 184 // nextCmdLine returns the next command line 185 func (r *Run) nextCmdLine() []string { 186 cmdLine := r.cmdLine 187 if r.runFlag != "" { 188 cmdLine = append(cmdLine, "-test.run", r.runFlag) 189 } 190 return cmdLine 191 } 192 193 // trial runs a single test 194 func (r *Run) trial() { 195 cmdLine := r.nextCmdLine() 196 cmdString := toShell(cmdLine) 197 msg := fmt.Sprintf("%q - Starting (try %d/%d)", cmdString, r.try, *maxTries) 198 log.Println(msg) 199 logName := path.Join(r.logDir, r.trialName) 200 out, err := os.Create(logName) 201 if err != nil { 202 log.Fatalf("Couldn't create log file: %v", err) 203 } 204 defer func() { 205 err := out.Close() 206 if err != nil { 207 log.Fatalf("Failed to close log file: %v", err) 208 } 209 }() 210 _, _ = fmt.Fprintln(out, msg) 211 212 // Early exit if --try-run 213 if *dryRun { 214 log.Printf("Not executing as --dry-run: %v", cmdLine) 215 _, _ = fmt.Fprintln(out, "--dry-run is set - not running") 216 return 217 } 218 219 // Internal buffer 220 var b bytes.Buffer 221 multiOut := io.MultiWriter(out, &b) 222 223 cmd := exec.Command(cmdLine[0], cmdLine[1:]...) 224 cmd.Stderr = multiOut 225 cmd.Stdout = multiOut 226 cmd.Dir = r.Path 227 start := time.Now() 228 r.err = cmd.Run() 229 r.output = b.Bytes() 230 duration := time.Since(start) 231 r.findFailures() 232 if r.passed() { 233 msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, r.try, *maxTries) 234 } else { 235 msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, r.try, *maxTries, r.err, r.failedTests) 236 } 237 log.Println(msg) 238 _, _ = fmt.Fprintln(out, msg) 239 } 240 241 // passed returns true if the test passed 242 func (r *Run) passed() bool { 243 return r.err == nil 244 } 245 246 // GOPATH returns the current GOPATH 247 func GOPATH() string { 248 gopath := os.Getenv("GOPATH") 249 if gopath == "" { 250 gopath = build.Default.GOPATH 251 } 252 return gopath 253 } 254 255 // BinaryName turns a package name into a binary name 256 func (r *Run) BinaryName() string { 257 binary := path.Base(r.Path) + ".test" 258 if runtime.GOOS == "windows" { 259 binary += ".exe" 260 } 261 return binary 262 } 263 264 // BinaryPath turns a package name into a binary path 265 func (r *Run) BinaryPath() string { 266 return path.Join(r.Path, r.BinaryName()) 267 } 268 269 // PackagePath returns the path to the package 270 func (r *Run) PackagePath() string { 271 return path.Join(GOPATH(), "src", r.Path) 272 } 273 274 // MakeTestBinary makes the binary we will run 275 func (r *Run) MakeTestBinary() { 276 binary := r.BinaryPath() 277 binaryName := r.BinaryName() 278 log.Printf("%s: Making test binary %q", r.Path, binaryName) 279 cmdLine := []string{"go", "test", "-c"} 280 if *dryRun { 281 log.Printf("Not executing: %v", cmdLine) 282 return 283 } 284 cmd := exec.Command(cmdLine[0], cmdLine[1:]...) 285 cmd.Dir = r.Path 286 err := cmd.Run() 287 if err != nil { 288 log.Fatalf("Failed to make test binary: %v", err) 289 } 290 if _, err := os.Stat(binary); err != nil { 291 log.Fatalf("Couldn't find test binary %q", binary) 292 } 293 } 294 295 // RemoveTestBinary removes the binary made in makeTestBinary 296 func (r *Run) RemoveTestBinary() { 297 if *dryRun { 298 return 299 } 300 binary := r.BinaryPath() 301 err := os.Remove(binary) // Delete the binary when finished 302 if err != nil { 303 log.Printf("Error removing test binary %q: %v", binary, err) 304 } 305 } 306 307 // Name returns the run name as a file name friendly string 308 func (r *Run) Name() string { 309 ns := []string{ 310 r.Backend, 311 strings.Replace(r.Path, "/", ".", -1), 312 r.Remote, 313 } 314 if r.SubDir { 315 ns = append(ns, "subdir") 316 } 317 if r.FastList { 318 ns = append(ns, "fastlist") 319 } 320 ns = append(ns, fmt.Sprintf("%d", r.try)) 321 s := strings.Join(ns, "-") 322 s = strings.Replace(s, ":", "", -1) 323 return s 324 } 325 326 // Init the Run 327 func (r *Run) Init() { 328 prefix := "-test." 329 if r.NoBinary { 330 prefix = "-" 331 r.cmdLine = []string{"go", "test"} 332 } else { 333 r.cmdLine = []string{"./" + r.BinaryName()} 334 } 335 r.cmdLine = append(r.cmdLine, prefix+"v", prefix+"timeout", timeout.String(), "-remote", r.Remote) 336 r.try = 1 337 if *verbose { 338 r.cmdLine = append(r.cmdLine, "-verbose") 339 fs.Config.LogLevel = fs.LogLevelDebug 340 } 341 if *runOnly != "" { 342 r.cmdLine = append(r.cmdLine, prefix+"run", *runOnly) 343 } 344 if r.SubDir { 345 r.cmdLine = append(r.cmdLine, "-subdir") 346 } 347 if r.FastList { 348 r.cmdLine = append(r.cmdLine, "-fast-list") 349 } 350 r.cmdString = toShell(r.cmdLine) 351 } 352 353 // Logs returns all the log names 354 func (r *Run) Logs() []string { 355 return r.trialNames 356 } 357 358 // FailedTests returns the failed tests as a comma separated string, limiting the number 359 func (r *Run) FailedTests() string { 360 const maxTests = 5 361 ts := r.failedTests 362 if len(ts) > maxTests { 363 ts = ts[:maxTests:maxTests] 364 ts = append(ts, fmt.Sprintf("… (%d more)", len(r.failedTests)-maxTests)) 365 } 366 return strings.Join(ts, ", ") 367 } 368 369 // Run runs all the trials for this test 370 func (r *Run) Run(logDir string, result chan<- *Run) { 371 if r.OneOnly { 372 oneOnlyMu.Lock() 373 mu := oneOnly[r.Backend] 374 if mu == nil { 375 mu = new(sync.Mutex) 376 oneOnly[r.Backend] = mu 377 } 378 oneOnlyMu.Unlock() 379 mu.Lock() 380 defer mu.Unlock() 381 } 382 r.Init() 383 r.logDir = logDir 384 for r.try = 1; r.try <= *maxTries; r.try++ { 385 r.trialName = r.Name() + ".txt" 386 r.trialNames = append(r.trialNames, r.trialName) 387 log.Printf("Starting run with log %q", r.trialName) 388 r.trial() 389 if r.passed() || r.NoRetries { 390 break 391 } 392 } 393 if !r.passed() { 394 r.dumpOutput() 395 } 396 result <- r 397 }