github.com/slayercat/go@v0.0.0-20170428012452-c51559813f61/src/testing/sub_test.go (about) 1 // Copyright 2016 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package testing 6 7 import ( 8 "bytes" 9 "fmt" 10 "os" 11 "regexp" 12 "runtime" 13 "strings" 14 "sync" 15 "sync/atomic" 16 "time" 17 ) 18 19 func init() { 20 // Make benchmark tests run 10* faster. 21 *benchTime = 100 * time.Millisecond 22 } 23 24 func TestTestContext(t *T) { 25 const ( 26 add1 = 0 27 done = 1 28 ) 29 // After each of the calls are applied to the context, the 30 type call struct { 31 typ int // run or done 32 // result from applying the call 33 running int 34 waiting int 35 started bool 36 } 37 testCases := []struct { 38 max int 39 run []call 40 }{{ 41 max: 1, 42 run: []call{ 43 {typ: add1, running: 1, waiting: 0, started: true}, 44 {typ: done, running: 0, waiting: 0, started: false}, 45 }, 46 }, { 47 max: 1, 48 run: []call{ 49 {typ: add1, running: 1, waiting: 0, started: true}, 50 {typ: add1, running: 1, waiting: 1, started: false}, 51 {typ: done, running: 1, waiting: 0, started: true}, 52 {typ: done, running: 0, waiting: 0, started: false}, 53 {typ: add1, running: 1, waiting: 0, started: true}, 54 }, 55 }, { 56 max: 3, 57 run: []call{ 58 {typ: add1, running: 1, waiting: 0, started: true}, 59 {typ: add1, running: 2, waiting: 0, started: true}, 60 {typ: add1, running: 3, waiting: 0, started: true}, 61 {typ: add1, running: 3, waiting: 1, started: false}, 62 {typ: add1, running: 3, waiting: 2, started: false}, 63 {typ: add1, running: 3, waiting: 3, started: false}, 64 {typ: done, running: 3, waiting: 2, started: true}, 65 {typ: add1, running: 3, waiting: 3, started: false}, 66 {typ: done, running: 3, waiting: 2, started: true}, 67 {typ: done, running: 3, waiting: 1, started: true}, 68 {typ: done, running: 3, waiting: 0, started: true}, 69 {typ: done, running: 2, waiting: 0, started: false}, 70 {typ: done, running: 1, waiting: 0, started: false}, 71 {typ: done, running: 0, waiting: 0, started: false}, 72 }, 73 }} 74 for i, tc := range testCases { 75 ctx := &testContext{ 76 startParallel: make(chan bool), 77 maxParallel: tc.max, 78 } 79 for j, call := range tc.run { 80 doCall := func(f func()) chan bool { 81 done := make(chan bool) 82 go func() { 83 f() 84 done <- true 85 }() 86 return done 87 } 88 started := false 89 switch call.typ { 90 case add1: 91 signal := doCall(ctx.waitParallel) 92 select { 93 case <-signal: 94 started = true 95 case ctx.startParallel <- true: 96 <-signal 97 } 98 case done: 99 signal := doCall(ctx.release) 100 select { 101 case <-signal: 102 case <-ctx.startParallel: 103 started = true 104 <-signal 105 } 106 } 107 if started != call.started { 108 t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started) 109 } 110 if ctx.running != call.running { 111 t.Errorf("%d:%d:running: got %v; want %v", i, j, ctx.running, call.running) 112 } 113 if ctx.numWaiting != call.waiting { 114 t.Errorf("%d:%d:waiting: got %v; want %v", i, j, ctx.numWaiting, call.waiting) 115 } 116 } 117 } 118 } 119 120 func TestTRun(t *T) { 121 realTest := t 122 testCases := []struct { 123 desc string 124 ok bool 125 maxPar int 126 chatty bool 127 output string 128 f func(*T) 129 }{{ 130 desc: "failnow skips future sequential and parallel tests at same level", 131 ok: false, 132 maxPar: 1, 133 output: ` 134 --- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs) 135 --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs) 136 `, 137 f: func(t *T) { 138 ranSeq := false 139 ranPar := false 140 t.Run("", func(t *T) { 141 t.Run("par", func(t *T) { 142 t.Parallel() 143 ranPar = true 144 }) 145 t.Run("seq", func(t *T) { 146 ranSeq = true 147 }) 148 t.FailNow() 149 t.Run("seq", func(t *T) { 150 realTest.Error("test must be skipped") 151 }) 152 t.Run("par", func(t *T) { 153 t.Parallel() 154 realTest.Error("test must be skipped.") 155 }) 156 }) 157 if !ranPar { 158 realTest.Error("parallel test was not run") 159 } 160 if !ranSeq { 161 realTest.Error("sequential test was not run") 162 } 163 }, 164 }, { 165 desc: "failure in parallel test propagates upwards", 166 ok: false, 167 maxPar: 1, 168 output: ` 169 --- FAIL: failure in parallel test propagates upwards (N.NNs) 170 --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs) 171 --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs) 172 `, 173 f: func(t *T) { 174 t.Run("", func(t *T) { 175 t.Parallel() 176 t.Run("par", func(t *T) { 177 t.Parallel() 178 t.Fail() 179 }) 180 }) 181 }, 182 }, { 183 desc: "skipping without message, chatty", 184 ok: true, 185 chatty: true, 186 output: ` 187 === RUN skipping without message, chatty 188 --- SKIP: skipping without message, chatty (N.NNs)`, 189 f: func(t *T) { t.SkipNow() }, 190 }, { 191 desc: "chatty with recursion", 192 ok: true, 193 chatty: true, 194 output: ` 195 === RUN chatty with recursion 196 === RUN chatty with recursion/#00 197 === RUN chatty with recursion/#00/#00 198 --- PASS: chatty with recursion (N.NNs) 199 --- PASS: chatty with recursion/#00 (N.NNs) 200 --- PASS: chatty with recursion/#00/#00 (N.NNs)`, 201 f: func(t *T) { 202 t.Run("", func(t *T) { 203 t.Run("", func(t *T) {}) 204 }) 205 }, 206 }, { 207 desc: "skipping without message, not chatty", 208 ok: true, 209 f: func(t *T) { t.SkipNow() }, 210 }, { 211 desc: "skipping after error", 212 output: ` 213 --- FAIL: skipping after error (N.NNs) 214 sub_test.go:NNN: an error 215 sub_test.go:NNN: skipped`, 216 f: func(t *T) { 217 t.Error("an error") 218 t.Skip("skipped") 219 }, 220 }, { 221 desc: "use Run to locally synchronize parallelism", 222 ok: true, 223 maxPar: 1, 224 f: func(t *T) { 225 var count uint32 226 t.Run("waitGroup", func(t *T) { 227 for i := 0; i < 4; i++ { 228 t.Run("par", func(t *T) { 229 t.Parallel() 230 atomic.AddUint32(&count, 1) 231 }) 232 } 233 }) 234 if count != 4 { 235 t.Errorf("count was %d; want 4", count) 236 } 237 }, 238 }, { 239 desc: "alternate sequential and parallel", 240 // Sequential tests should partake in the counting of running threads. 241 // Otherwise, if one runs parallel subtests in sequential tests that are 242 // itself subtests of parallel tests, the counts can get askew. 243 ok: true, 244 maxPar: 1, 245 f: func(t *T) { 246 t.Run("a", func(t *T) { 247 t.Parallel() 248 t.Run("b", func(t *T) { 249 // Sequential: ensure running count is decremented. 250 t.Run("c", func(t *T) { 251 t.Parallel() 252 }) 253 254 }) 255 }) 256 }, 257 }, { 258 desc: "alternate sequential and parallel 2", 259 // Sequential tests should partake in the counting of running threads. 260 // Otherwise, if one runs parallel subtests in sequential tests that are 261 // itself subtests of parallel tests, the counts can get askew. 262 ok: true, 263 maxPar: 2, 264 f: func(t *T) { 265 for i := 0; i < 2; i++ { 266 t.Run("a", func(t *T) { 267 t.Parallel() 268 time.Sleep(time.Nanosecond) 269 for i := 0; i < 2; i++ { 270 t.Run("b", func(t *T) { 271 time.Sleep(time.Nanosecond) 272 for i := 0; i < 2; i++ { 273 t.Run("c", func(t *T) { 274 t.Parallel() 275 time.Sleep(time.Nanosecond) 276 }) 277 } 278 279 }) 280 } 281 }) 282 } 283 }, 284 }, { 285 desc: "stress test", 286 ok: true, 287 maxPar: 4, 288 f: func(t *T) { 289 t.Parallel() 290 for i := 0; i < 12; i++ { 291 t.Run("a", func(t *T) { 292 t.Parallel() 293 time.Sleep(time.Nanosecond) 294 for i := 0; i < 12; i++ { 295 t.Run("b", func(t *T) { 296 time.Sleep(time.Nanosecond) 297 for i := 0; i < 12; i++ { 298 t.Run("c", func(t *T) { 299 t.Parallel() 300 time.Sleep(time.Nanosecond) 301 t.Run("d1", func(t *T) {}) 302 t.Run("d2", func(t *T) {}) 303 t.Run("d3", func(t *T) {}) 304 t.Run("d4", func(t *T) {}) 305 }) 306 } 307 }) 308 } 309 }) 310 } 311 }, 312 }, { 313 desc: "skip output", 314 ok: true, 315 maxPar: 4, 316 f: func(t *T) { 317 t.Skip() 318 }, 319 }, { 320 desc: "panic on goroutine fail after test exit", 321 ok: false, 322 maxPar: 4, 323 f: func(t *T) { 324 ch := make(chan bool) 325 t.Run("", func(t *T) { 326 go func() { 327 <-ch 328 defer func() { 329 if r := recover(); r == nil { 330 realTest.Errorf("expected panic") 331 } 332 ch <- true 333 }() 334 t.Errorf("failed after success") 335 }() 336 }) 337 ch <- true 338 <-ch 339 }, 340 }} 341 for _, tc := range testCases { 342 ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", "")) 343 buf := &bytes.Buffer{} 344 root := &T{ 345 common: common{ 346 signal: make(chan bool), 347 name: "Test", 348 w: buf, 349 chatty: tc.chatty, 350 }, 351 context: ctx, 352 } 353 ok := root.Run(tc.desc, tc.f) 354 ctx.release() 355 356 if ok != tc.ok { 357 t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok) 358 } 359 if ok != !root.Failed() { 360 t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed()) 361 } 362 if ctx.running != 0 || ctx.numWaiting != 0 { 363 t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting) 364 } 365 got := strings.TrimSpace(buf.String()) 366 want := strings.TrimSpace(tc.output) 367 re := makeRegexp(want) 368 if ok, err := regexp.MatchString(re, got); !ok || err != nil { 369 t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) 370 } 371 } 372 } 373 374 func TestBRun(t *T) { 375 work := func(b *B) { 376 for i := 0; i < b.N; i++ { 377 time.Sleep(time.Nanosecond) 378 } 379 } 380 testCases := []struct { 381 desc string 382 failed bool 383 chatty bool 384 output string 385 f func(*B) 386 }{{ 387 desc: "simulate sequential run of subbenchmarks.", 388 f: func(b *B) { 389 b.Run("", func(b *B) { work(b) }) 390 time1 := b.result.NsPerOp() 391 b.Run("", func(b *B) { work(b) }) 392 time2 := b.result.NsPerOp() 393 if time1 >= time2 { 394 t.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1, time2) 395 } 396 }, 397 }, { 398 desc: "bytes set by all benchmarks", 399 f: func(b *B) { 400 b.Run("", func(b *B) { b.SetBytes(10); work(b) }) 401 b.Run("", func(b *B) { b.SetBytes(10); work(b) }) 402 if b.result.Bytes != 20 { 403 t.Errorf("bytes: got: %d; want 20", b.result.Bytes) 404 } 405 }, 406 }, { 407 desc: "bytes set by some benchmarks", 408 // In this case the bytes result is meaningless, so it must be 0. 409 f: func(b *B) { 410 b.Run("", func(b *B) { b.SetBytes(10); work(b) }) 411 b.Run("", func(b *B) { work(b) }) 412 b.Run("", func(b *B) { b.SetBytes(10); work(b) }) 413 if b.result.Bytes != 0 { 414 t.Errorf("bytes: got: %d; want 0", b.result.Bytes) 415 } 416 }, 417 }, { 418 desc: "failure carried over to root", 419 failed: true, 420 output: "--- FAIL: root", 421 f: func(b *B) { b.Fail() }, 422 }, { 423 desc: "skipping without message, chatty", 424 chatty: true, 425 output: "--- SKIP: root", 426 f: func(b *B) { b.SkipNow() }, 427 }, { 428 desc: "skipping with message, chatty", 429 chatty: true, 430 output: ` 431 --- SKIP: root 432 sub_test.go:NNN: skipping`, 433 f: func(b *B) { b.Skip("skipping") }, 434 }, { 435 desc: "chatty with recursion", 436 chatty: true, 437 f: func(b *B) { 438 b.Run("", func(b *B) { 439 b.Run("", func(b *B) {}) 440 }) 441 }, 442 }, { 443 desc: "skipping without message, not chatty", 444 f: func(b *B) { b.SkipNow() }, 445 }, { 446 desc: "skipping after error", 447 failed: true, 448 output: ` 449 --- FAIL: root 450 sub_test.go:NNN: an error 451 sub_test.go:NNN: skipped`, 452 f: func(b *B) { 453 b.Error("an error") 454 b.Skip("skipped") 455 }, 456 }, { 457 desc: "memory allocation", 458 f: func(b *B) { 459 const bufSize = 256 460 alloc := func(b *B) { 461 var buf [bufSize]byte 462 for i := 0; i < b.N; i++ { 463 _ = append([]byte(nil), buf[:]...) 464 } 465 } 466 b.Run("", func(b *B) { 467 alloc(b) 468 b.ReportAllocs() 469 }) 470 b.Run("", func(b *B) { 471 alloc(b) 472 b.ReportAllocs() 473 }) 474 // runtime.MemStats sometimes reports more allocations than the 475 // benchmark is responsible for. Luckily the point of this test is 476 // to ensure that the results are not underreported, so we can 477 // simply verify the lower bound. 478 if got := b.result.MemAllocs; got < 2 { 479 t.Errorf("MemAllocs was %v; want 2", got) 480 } 481 if got := b.result.MemBytes; got < 2*bufSize { 482 t.Errorf("MemBytes was %v; want %v", got, 2*bufSize) 483 } 484 }, 485 }} 486 for _, tc := range testCases { 487 var ok bool 488 buf := &bytes.Buffer{} 489 // This is almost like the Benchmark function, except that we override 490 // the benchtime and catch the failure result of the subbenchmark. 491 root := &B{ 492 common: common{ 493 signal: make(chan bool), 494 name: "root", 495 w: buf, 496 chatty: tc.chatty, 497 }, 498 benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure. 499 benchTime: time.Microsecond, 500 } 501 root.runN(1) 502 if ok != !tc.failed { 503 t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, !tc.failed) 504 } 505 if !ok != root.Failed() { 506 t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed()) 507 } 508 // All tests are run as subtests 509 if root.result.N != 1 { 510 t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N) 511 } 512 got := strings.TrimSpace(buf.String()) 513 want := strings.TrimSpace(tc.output) 514 re := makeRegexp(want) 515 if ok, err := regexp.MatchString(re, got); !ok || err != nil { 516 t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) 517 } 518 } 519 } 520 521 func makeRegexp(s string) string { 522 s = strings.Replace(s, ":NNN:", `:\d\d\d:`, -1) 523 s = strings.Replace(s, "(N.NNs)", `\(\d*\.\d*s\)`, -1) 524 return s 525 } 526 527 func TestBenchmarkOutput(t *T) { 528 // Ensure Benchmark initialized common.w by invoking it with an error and 529 // normal case. 530 Benchmark(func(b *B) { b.Error("do not print this output") }) 531 Benchmark(func(b *B) {}) 532 } 533 534 func TestBenchmarkStartsFrom1(t *T) { 535 var first = true 536 Benchmark(func(b *B) { 537 if first && b.N != 1 { 538 panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N)) 539 } 540 first = false 541 }) 542 } 543 544 func TestParallelSub(t *T) { 545 c := make(chan int) 546 block := make(chan int) 547 for i := 0; i < 10; i++ { 548 go func(i int) { 549 <-block 550 t.Run(fmt.Sprint(i), func(t *T) {}) 551 c <- 1 552 }(i) 553 } 554 close(block) 555 for i := 0; i < 10; i++ { 556 <-c 557 } 558 } 559 560 type funcWriter func([]byte) (int, error) 561 562 func (fw funcWriter) Write(b []byte) (int, error) { return fw(b) } 563 564 func TestRacyOutput(t *T) { 565 var runs int32 // The number of running Writes 566 var races int32 // Incremented for each race detected 567 raceDetector := func(b []byte) (int, error) { 568 // Check if some other goroutine is concurrently calling Write. 569 if atomic.LoadInt32(&runs) > 0 { 570 atomic.AddInt32(&races, 1) // Race detected! 571 } 572 atomic.AddInt32(&runs, 1) 573 defer atomic.AddInt32(&runs, -1) 574 runtime.Gosched() // Increase probability of a race 575 return len(b), nil 576 } 577 578 var wg sync.WaitGroup 579 root := &T{ 580 common: common{w: funcWriter(raceDetector), chatty: true}, 581 context: newTestContext(1, newMatcher(regexp.MatchString, "", "")), 582 } 583 root.Run("", func(t *T) { 584 for i := 0; i < 100; i++ { 585 wg.Add(1) 586 go func(i int) { 587 defer wg.Done() 588 t.Run(fmt.Sprint(i), func(t *T) { 589 t.Logf("testing run %d", i) 590 }) 591 }(i) 592 } 593 }) 594 wg.Wait() 595 596 if races > 0 { 597 t.Errorf("detected %d racy Writes", races) 598 } 599 } 600 601 func TestBenchmark(t *T) { 602 res := Benchmark(func(b *B) { 603 for i := 0; i < 5; i++ { 604 b.Run("", func(b *B) { 605 fmt.Fprintf(os.Stderr, "b.N: %v\n", b.N) 606 for i := 0; i < b.N; i++ { 607 time.Sleep(time.Millisecond) 608 } 609 }) 610 } 611 }) 612 if res.NsPerOp() < 4000000 { 613 t.Errorf("want >5ms; got %v", time.Duration(res.NsPerOp())) 614 } 615 }