github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/fstest/test_all/run.go (about) 1 // Run a test 2 3 package main 4 5 import ( 6 "bytes" 7 "fmt" 8 "go/build" 9 "io" 10 "log" 11 "os" 12 "os/exec" 13 "path" 14 "regexp" 15 "runtime" 16 "sort" 17 "strconv" 18 "strings" 19 "sync" 20 "time" 21 22 "github.com/rclone/rclone/fs" 23 "github.com/rclone/rclone/fstest/testserver" 24 ) 25 26 // Control concurrency per backend if required 27 var ( 28 oneOnlyMu sync.Mutex 29 oneOnly = map[string]*sync.Mutex{} 30 ) 31 32 // Run holds info about a running test 33 // 34 // A run just runs one command line, but it can be run multiple times 35 // if retries are needed. 36 type Run struct { 37 // Config 38 Remote string // name of the test remote 39 Backend string // name of the backend 40 Path string // path to the source directory 41 FastList bool // add -fast-list to tests 42 Short bool // add -short 43 NoRetries bool // don't retry if set 44 OneOnly bool // only run test for this backend at once 45 NoBinary bool // set to not build a binary 46 SizeLimit int64 // maximum test file size 47 Ignore map[string]struct{} 48 // Internals 49 cmdLine []string 50 cmdString string 51 try int 52 err error 53 output []byte 54 failedTests []string 55 runFlag string 56 logDir string // directory to place the logs 57 trialName string // name/log file name of current trial 58 trialNames []string // list of all the trials 59 } 60 61 // Runs records multiple Run objects 62 type Runs []*Run 63 64 // Sort interface 65 func (rs Runs) Len() int { return len(rs) } 66 func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] } 67 func (rs Runs) Less(i, j int) bool { 68 a, b := rs[i], rs[j] 69 if a.Backend < b.Backend { 70 return true 71 } else if a.Backend > b.Backend { 72 return false 73 } 74 if a.Remote < b.Remote { 75 return true 76 } else if a.Remote > b.Remote { 77 return false 78 } 79 if a.Path < b.Path { 80 return true 81 } else if a.Path > b.Path { 82 return false 83 } 84 if !a.FastList && b.FastList { 85 return true 86 } else if a.FastList && !b.FastList { 87 return false 88 } 89 return false 90 } 91 92 // dumpOutput prints the error output 93 func (r *Run) dumpOutput() { 94 log.Println("------------------------------------------------------------") 95 log.Printf("---- %q ----", r.cmdString) 96 log.Println(string(r.output)) 97 log.Println("------------------------------------------------------------") 98 } 99 100 // This converts a slice of test names into a regexp which matches 101 // them. 102 func testsToRegexp(tests []string) string { 103 var split []map[string]struct{} 104 // Make a slice with maps of the used parts at each level 105 for _, test := range tests { 106 for i, name := range strings.Split(test, "/") { 107 if i >= len(split) { 108 split = append(split, make(map[string]struct{})) 109 } 110 split[i][name] = struct{}{} 111 } 112 } 113 var out []string 114 for _, level := range split { 115 var testsInLevel = []string{} 116 for name := range level { 117 testsInLevel = append(testsInLevel, name) 118 } 119 sort.Strings(testsInLevel) 120 if len(testsInLevel) > 1 { 121 out = append(out, "^("+strings.Join(testsInLevel, "|")+")$") 122 } else { 123 out = append(out, "^"+testsInLevel[0]+"$") 124 } 125 } 126 return strings.Join(out, "/") 127 } 128 129 var failRe = regexp.MustCompile(`(?m)^\s*--- FAIL: (Test.*?) \(`) 130 131 // findFailures looks for all the tests which failed 132 func (r *Run) findFailures() { 133 oldFailedTests := r.failedTests 134 r.failedTests = nil 135 excludeParents := map[string]struct{}{} 136 ignored := 0 137 for _, matches := range failRe.FindAllSubmatch(r.output, -1) { 138 failedTest := string(matches[1]) 139 // Skip any ignored failures 140 if _, found := r.Ignore[failedTest]; found { 141 ignored++ 142 } else { 143 r.failedTests = append(r.failedTests, failedTest) 144 } 145 // Find all the parents of this test 146 parts := strings.Split(failedTest, "/") 147 for i := len(parts) - 1; i >= 1; i-- { 148 excludeParents[strings.Join(parts[:i], "/")] = struct{}{} 149 } 150 } 151 // Exclude the parents 152 var newTests = r.failedTests[:0] 153 for _, failedTest := range r.failedTests { 154 if _, excluded := excludeParents[failedTest]; !excluded { 155 newTests = append(newTests, failedTest) 156 } 157 } 158 r.failedTests = newTests 159 if len(r.failedTests) == 0 && ignored > 0 { 160 log.Printf("%q - Found %d ignored errors only - marking as good", r.cmdString, ignored) 161 r.err = nil 162 r.dumpOutput() 163 return 164 } 165 if len(r.failedTests) != 0 { 166 r.runFlag = testsToRegexp(r.failedTests) 167 } else { 168 r.runFlag = "" 169 } 170 if r.passed() && len(r.failedTests) != 0 { 171 log.Printf("%q - Expecting no errors but got: %v", r.cmdString, r.failedTests) 172 r.dumpOutput() 173 } else if !r.passed() && len(r.failedTests) == 0 { 174 log.Printf("%q - Expecting errors but got none: %v", r.cmdString, r.failedTests) 175 r.dumpOutput() 176 r.failedTests = oldFailedTests 177 } 178 } 179 180 // nextCmdLine returns the next command line 181 func (r *Run) nextCmdLine() []string { 182 cmdLine := r.cmdLine 183 if r.runFlag != "" { 184 cmdLine = append(cmdLine, "-test.run", r.runFlag) 185 } 186 return cmdLine 187 } 188 189 // trial runs a single test 190 func (r *Run) trial() { 191 cmdLine := r.nextCmdLine() 192 cmdString := toShell(cmdLine) 193 msg := fmt.Sprintf("%q - Starting (try %d/%d)", cmdString, r.try, *maxTries) 194 log.Println(msg) 195 logName := path.Join(r.logDir, r.trialName) 196 out, err := os.Create(logName) 197 if err != nil { 198 log.Fatalf("Couldn't create log file: %v", err) 199 } 200 defer func() { 201 err := out.Close() 202 if err != nil { 203 log.Fatalf("Failed to close log file: %v", err) 204 } 205 }() 206 _, _ = fmt.Fprintln(out, msg) 207 208 // Early exit if --try-run 209 if *dryRun { 210 log.Printf("Not executing as --dry-run: %v", cmdLine) 211 _, _ = fmt.Fprintln(out, "--dry-run is set - not running") 212 return 213 } 214 215 // Start the test server if required 216 finish, err := testserver.Start(r.Remote) 217 if err != nil { 218 log.Printf("%s: Failed to start test server: %v", r.Remote, err) 219 _, _ = fmt.Fprintf(out, "%s: Failed to start test server: %v\n", r.Remote, err) 220 r.err = err 221 return 222 } 223 defer finish() 224 225 // Internal buffer 226 var b bytes.Buffer 227 multiOut := io.MultiWriter(out, &b) 228 229 cmd := exec.Command(cmdLine[0], cmdLine[1:]...) 230 cmd.Stderr = multiOut 231 cmd.Stdout = multiOut 232 cmd.Dir = r.Path 233 start := time.Now() 234 r.err = cmd.Run() 235 r.output = b.Bytes() 236 duration := time.Since(start) 237 r.findFailures() 238 if r.passed() { 239 msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, r.try, *maxTries) 240 } else { 241 msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, r.try, *maxTries, r.err, r.failedTests) 242 } 243 log.Println(msg) 244 _, _ = fmt.Fprintln(out, msg) 245 } 246 247 // passed returns true if the test passed 248 func (r *Run) passed() bool { 249 return r.err == nil 250 } 251 252 // GOPATH returns the current GOPATH 253 func GOPATH() string { 254 gopath := os.Getenv("GOPATH") 255 if gopath == "" { 256 gopath = build.Default.GOPATH 257 } 258 return gopath 259 } 260 261 // BinaryName turns a package name into a binary name 262 func (r *Run) BinaryName() string { 263 binary := path.Base(r.Path) + ".test" 264 if runtime.GOOS == "windows" { 265 binary += ".exe" 266 } 267 return binary 268 } 269 270 // BinaryPath turns a package name into a binary path 271 func (r *Run) BinaryPath() string { 272 return path.Join(r.Path, r.BinaryName()) 273 } 274 275 // PackagePath returns the path to the package 276 func (r *Run) PackagePath() string { 277 return path.Join(GOPATH(), "src", r.Path) 278 } 279 280 // MakeTestBinary makes the binary we will run 281 func (r *Run) MakeTestBinary() { 282 binary := r.BinaryPath() 283 binaryName := r.BinaryName() 284 log.Printf("%s: Making test binary %q", r.Path, binaryName) 285 cmdLine := []string{"go", "test", "-c"} 286 if *dryRun { 287 log.Printf("Not executing: %v", cmdLine) 288 return 289 } 290 cmd := exec.Command(cmdLine[0], cmdLine[1:]...) 291 cmd.Dir = r.Path 292 err := cmd.Run() 293 if err != nil { 294 log.Fatalf("Failed to make test binary: %v", err) 295 } 296 if _, err := os.Stat(binary); err != nil { 297 log.Fatalf("Couldn't find test binary %q", binary) 298 } 299 } 300 301 // RemoveTestBinary removes the binary made in makeTestBinary 302 func (r *Run) RemoveTestBinary() { 303 if *dryRun { 304 return 305 } 306 binary := r.BinaryPath() 307 err := os.Remove(binary) // Delete the binary when finished 308 if err != nil { 309 log.Printf("Error removing test binary %q: %v", binary, err) 310 } 311 } 312 313 // Name returns the run name as a file name friendly string 314 func (r *Run) Name() string { 315 ns := []string{ 316 r.Backend, 317 strings.Replace(r.Path, "/", ".", -1), 318 r.Remote, 319 } 320 if r.FastList { 321 ns = append(ns, "fastlist") 322 } 323 ns = append(ns, fmt.Sprintf("%d", r.try)) 324 s := strings.Join(ns, "-") 325 s = strings.Replace(s, ":", "", -1) 326 return s 327 } 328 329 // Init the Run 330 func (r *Run) Init() { 331 prefix := "-test." 332 if r.NoBinary { 333 prefix = "-" 334 r.cmdLine = []string{"go", "test"} 335 } else { 336 r.cmdLine = []string{"./" + r.BinaryName()} 337 } 338 r.cmdLine = append(r.cmdLine, prefix+"v", prefix+"timeout", timeout.String(), "-remote", r.Remote) 339 if *listRetries > 0 { 340 r.cmdLine = append(r.cmdLine, "-list-retries", fmt.Sprint(*listRetries)) 341 } 342 r.try = 1 343 if *verbose { 344 r.cmdLine = append(r.cmdLine, "-verbose") 345 fs.Config.LogLevel = fs.LogLevelDebug 346 } 347 if *runOnly != "" { 348 r.cmdLine = append(r.cmdLine, prefix+"run", *runOnly) 349 } 350 if r.FastList { 351 r.cmdLine = append(r.cmdLine, "-fast-list") 352 } 353 if r.Short { 354 r.cmdLine = append(r.cmdLine, "-short") 355 } 356 if r.SizeLimit > 0 { 357 r.cmdLine = append(r.cmdLine, "-size-limit", strconv.FormatInt(r.SizeLimit, 10)) 358 } 359 r.cmdString = toShell(r.cmdLine) 360 } 361 362 // Logs returns all the log names 363 func (r *Run) Logs() []string { 364 return r.trialNames 365 } 366 367 // FailedTests returns the failed tests as a comma separated string, limiting the number 368 func (r *Run) FailedTests() string { 369 const maxTests = 5 370 ts := r.failedTests 371 if len(ts) > maxTests { 372 ts = ts[:maxTests:maxTests] 373 ts = append(ts, fmt.Sprintf("… (%d more)", len(r.failedTests)-maxTests)) 374 } 375 return strings.Join(ts, ", ") 376 } 377 378 // Run runs all the trials for this test 379 func (r *Run) Run(logDir string, result chan<- *Run) { 380 if r.OneOnly { 381 oneOnlyMu.Lock() 382 mu := oneOnly[r.Backend] 383 if mu == nil { 384 mu = new(sync.Mutex) 385 oneOnly[r.Backend] = mu 386 } 387 oneOnlyMu.Unlock() 388 mu.Lock() 389 defer mu.Unlock() 390 } 391 r.Init() 392 r.logDir = logDir 393 for r.try = 1; r.try <= *maxTries; r.try++ { 394 r.trialName = r.Name() + ".txt" 395 r.trialNames = append(r.trialNames, r.trialName) 396 log.Printf("Starting run with log %q", r.trialName) 397 r.trial() 398 if r.passed() || r.NoRetries { 399 break 400 } 401 } 402 if !r.passed() { 403 r.dumpOutput() 404 } 405 result <- r 406 }