github.com/mckael/restic@v0.8.3/internal/pipe/pipe_test.go (about) 1 package pipe_test 2 3 import ( 4 "context" 5 "io/ioutil" 6 "os" 7 "path/filepath" 8 "runtime" 9 "sync" 10 "testing" 11 "time" 12 13 "github.com/restic/restic/internal/debug" 14 "github.com/restic/restic/internal/pipe" 15 rtest "github.com/restic/restic/internal/test" 16 ) 17 18 type stats struct { 19 dirs, files int 20 } 21 22 func acceptAll(string, os.FileInfo) bool { 23 return true 24 } 25 26 func statPath(path string) (stats, error) { 27 var s stats 28 29 // count files and directories with filepath.Walk() 30 err := filepath.Walk(rtest.TestWalkerPath, func(p string, fi os.FileInfo, err error) error { 31 if fi == nil { 32 return err 33 } 34 35 if fi.IsDir() { 36 s.dirs++ 37 } else { 38 s.files++ 39 } 40 41 return err 42 }) 43 44 return s, err 45 } 46 47 const maxWorkers = 100 48 49 func TestPipelineWalkerWithSplit(t *testing.T) { 50 if rtest.TestWalkerPath == "" { 51 t.Skipf("walkerpath not set, skipping TestPipelineWalker") 52 } 53 54 var err error 55 if !filepath.IsAbs(rtest.TestWalkerPath) { 56 rtest.TestWalkerPath, err = filepath.Abs(rtest.TestWalkerPath) 57 rtest.OK(t, err) 58 } 59 60 before, err := statPath(rtest.TestWalkerPath) 61 rtest.OK(t, err) 62 63 t.Logf("walking path %s with %d dirs, %d files", rtest.TestWalkerPath, 64 before.dirs, before.files) 65 66 // account for top level dir 67 before.dirs++ 68 69 after := stats{} 70 m := sync.Mutex{} 71 72 worker := func(wg *sync.WaitGroup, done <-chan struct{}, entCh <-chan pipe.Entry, dirCh <-chan pipe.Dir) { 73 defer wg.Done() 74 for { 75 select { 76 case e, ok := <-entCh: 77 if !ok { 78 // channel is closed 79 return 80 } 81 82 m.Lock() 83 after.files++ 84 m.Unlock() 85 86 e.Result() <- true 87 88 case dir, ok := <-dirCh: 89 if !ok { 90 // channel is closed 91 return 92 } 93 94 // wait for all content 95 for _, ch := range dir.Entries { 96 <-ch 97 } 98 99 m.Lock() 100 after.dirs++ 101 m.Unlock() 102 103 dir.Result() <- true 104 case <-done: 105 // pipeline was cancelled 106 return 107 } 108 } 109 } 110 111 var wg sync.WaitGroup 112 done := make(chan struct{}) 113 entCh := make(chan pipe.Entry) 114 dirCh := make(chan pipe.Dir) 115 116 for i := 0; i < maxWorkers; i++ { 117 wg.Add(1) 118 go worker(&wg, done, entCh, dirCh) 119 } 120 121 jobs := make(chan pipe.Job, 200) 122 wg.Add(1) 123 go func() { 124 pipe.Split(jobs, dirCh, entCh) 125 close(entCh) 126 close(dirCh) 127 wg.Done() 128 }() 129 130 resCh := make(chan pipe.Result, 1) 131 pipe.Walk(context.TODO(), []string{rtest.TestWalkerPath}, acceptAll, jobs, resCh) 132 133 // wait for all workers to terminate 134 wg.Wait() 135 136 // wait for top-level blob 137 <-resCh 138 139 t.Logf("walked path %s with %d dirs, %d files", rtest.TestWalkerPath, 140 after.dirs, after.files) 141 142 rtest.Assert(t, before == after, "stats do not match, expected %v, got %v", before, after) 143 } 144 145 func TestPipelineWalker(t *testing.T) { 146 if rtest.TestWalkerPath == "" { 147 t.Skipf("walkerpath not set, skipping TestPipelineWalker") 148 } 149 150 ctx, cancel := context.WithCancel(context.TODO()) 151 defer cancel() 152 153 var err error 154 if !filepath.IsAbs(rtest.TestWalkerPath) { 155 rtest.TestWalkerPath, err = filepath.Abs(rtest.TestWalkerPath) 156 rtest.OK(t, err) 157 } 158 159 before, err := statPath(rtest.TestWalkerPath) 160 rtest.OK(t, err) 161 162 t.Logf("walking path %s with %d dirs, %d files", rtest.TestWalkerPath, 163 before.dirs, before.files) 164 165 // account for top level dir 166 before.dirs++ 167 168 after := stats{} 169 m := sync.Mutex{} 170 171 worker := func(ctx context.Context, wg *sync.WaitGroup, jobs <-chan pipe.Job) { 172 defer wg.Done() 173 for { 174 select { 175 case job, ok := <-jobs: 176 if !ok { 177 // channel is closed 178 return 179 } 180 rtest.Assert(t, job != nil, "job is nil") 181 182 switch j := job.(type) { 183 case pipe.Dir: 184 // wait for all content 185 for _, ch := range j.Entries { 186 <-ch 187 } 188 189 m.Lock() 190 after.dirs++ 191 m.Unlock() 192 193 j.Result() <- true 194 case pipe.Entry: 195 m.Lock() 196 after.files++ 197 m.Unlock() 198 199 j.Result() <- true 200 } 201 202 case <-ctx.Done(): 203 // pipeline was cancelled 204 return 205 } 206 } 207 } 208 209 var wg sync.WaitGroup 210 jobs := make(chan pipe.Job) 211 212 for i := 0; i < maxWorkers; i++ { 213 wg.Add(1) 214 go worker(ctx, &wg, jobs) 215 } 216 217 resCh := make(chan pipe.Result, 1) 218 pipe.Walk(ctx, []string{rtest.TestWalkerPath}, acceptAll, jobs, resCh) 219 220 // wait for all workers to terminate 221 wg.Wait() 222 223 // wait for top-level blob 224 <-resCh 225 226 t.Logf("walked path %s with %d dirs, %d files", rtest.TestWalkerPath, 227 after.dirs, after.files) 228 229 rtest.Assert(t, before == after, "stats do not match, expected %v, got %v", before, after) 230 } 231 232 func createFile(filename, data string) error { 233 f, err := os.Create(filename) 234 if err != nil { 235 return err 236 } 237 238 defer f.Close() 239 240 _, err = f.Write([]byte(data)) 241 if err != nil { 242 return err 243 } 244 245 return nil 246 } 247 248 func TestPipeWalkerError(t *testing.T) { 249 dir, err := ioutil.TempDir("", "restic-test-") 250 rtest.OK(t, err) 251 252 base := filepath.Base(dir) 253 254 var testjobs = []struct { 255 path []string 256 err bool 257 }{ 258 {[]string{base, "a", "file_a"}, false}, 259 {[]string{base, "a"}, false}, 260 {[]string{base, "b"}, true}, 261 {[]string{base, "c", "file_c"}, false}, 262 {[]string{base, "c"}, false}, 263 {[]string{base}, false}, 264 {[]string{}, false}, 265 } 266 267 rtest.OK(t, os.Mkdir(filepath.Join(dir, "a"), 0755)) 268 rtest.OK(t, os.Mkdir(filepath.Join(dir, "b"), 0755)) 269 rtest.OK(t, os.Mkdir(filepath.Join(dir, "c"), 0755)) 270 271 rtest.OK(t, createFile(filepath.Join(dir, "a", "file_a"), "file a")) 272 rtest.OK(t, createFile(filepath.Join(dir, "b", "file_b"), "file b")) 273 rtest.OK(t, createFile(filepath.Join(dir, "c", "file_c"), "file c")) 274 275 ranHook := false 276 testdir := filepath.Join(dir, "b") 277 278 // install hook that removes the dir right before readdirnames() 279 debug.Hook("pipe.readdirnames", func(context interface{}) { 280 path := context.(string) 281 282 if path != testdir { 283 return 284 } 285 286 t.Logf("in hook, removing test file %v", testdir) 287 ranHook = true 288 289 rtest.OK(t, os.RemoveAll(testdir)) 290 }) 291 292 ctx, cancel := context.WithCancel(context.TODO()) 293 294 ch := make(chan pipe.Job) 295 resCh := make(chan pipe.Result, 1) 296 297 go pipe.Walk(ctx, []string{dir}, acceptAll, ch, resCh) 298 299 i := 0 300 for job := range ch { 301 if i == len(testjobs) { 302 t.Errorf("too many jobs received") 303 break 304 } 305 306 p := filepath.Join(testjobs[i].path...) 307 if p != job.Path() { 308 t.Errorf("job %d has wrong path: expected %q, got %q", i, p, job.Path()) 309 } 310 311 if testjobs[i].err { 312 if job.Error() == nil { 313 t.Errorf("job %d expected error but got nil", i) 314 } 315 } else { 316 if job.Error() != nil { 317 t.Errorf("job %d expected no error but got %v", i, job.Error()) 318 } 319 } 320 321 i++ 322 } 323 324 if i != len(testjobs) { 325 t.Errorf("expected %d jobs, got %d", len(testjobs), i) 326 } 327 328 cancel() 329 330 rtest.Assert(t, ranHook, "hook did not run") 331 rtest.OK(t, os.RemoveAll(dir)) 332 } 333 334 func BenchmarkPipelineWalker(b *testing.B) { 335 if rtest.TestWalkerPath == "" { 336 b.Skipf("walkerpath not set, skipping BenchPipelineWalker") 337 } 338 339 var max time.Duration 340 m := sync.Mutex{} 341 342 fileWorker := func(ctx context.Context, wg *sync.WaitGroup, ch <-chan pipe.Entry) { 343 defer wg.Done() 344 for { 345 select { 346 case e, ok := <-ch: 347 if !ok { 348 // channel is closed 349 return 350 } 351 352 // simulate backup 353 //time.Sleep(10 * time.Millisecond) 354 355 e.Result() <- true 356 case <-ctx.Done(): 357 // pipeline was cancelled 358 return 359 } 360 } 361 } 362 363 dirWorker := func(ctx context.Context, wg *sync.WaitGroup, ch <-chan pipe.Dir) { 364 defer wg.Done() 365 for { 366 select { 367 case dir, ok := <-ch: 368 if !ok { 369 // channel is closed 370 return 371 } 372 373 start := time.Now() 374 375 // wait for all content 376 for _, ch := range dir.Entries { 377 <-ch 378 } 379 380 d := time.Since(start) 381 m.Lock() 382 if d > max { 383 max = d 384 } 385 m.Unlock() 386 387 dir.Result() <- true 388 case <-ctx.Done(): 389 // pipeline was cancelled 390 return 391 } 392 } 393 } 394 395 ctx, cancel := context.WithCancel(context.TODO()) 396 defer cancel() 397 398 for i := 0; i < b.N; i++ { 399 max = 0 400 entCh := make(chan pipe.Entry, 200) 401 dirCh := make(chan pipe.Dir, 200) 402 403 var wg sync.WaitGroup 404 b.Logf("starting %d workers", maxWorkers) 405 for i := 0; i < maxWorkers; i++ { 406 wg.Add(2) 407 go dirWorker(ctx, &wg, dirCh) 408 go fileWorker(ctx, &wg, entCh) 409 } 410 411 jobs := make(chan pipe.Job, 200) 412 wg.Add(1) 413 go func() { 414 pipe.Split(jobs, dirCh, entCh) 415 close(entCh) 416 close(dirCh) 417 wg.Done() 418 }() 419 420 resCh := make(chan pipe.Result, 1) 421 pipe.Walk(ctx, []string{rtest.TestWalkerPath}, acceptAll, jobs, resCh) 422 423 // wait for all workers to terminate 424 wg.Wait() 425 426 // wait for final result 427 <-resCh 428 429 b.Logf("max duration for a dir: %v", max) 430 } 431 } 432 433 func TestPipelineWalkerMultiple(t *testing.T) { 434 if rtest.TestWalkerPath == "" { 435 t.Skipf("walkerpath not set, skipping TestPipelineWalker") 436 } 437 438 ctx, cancel := context.WithCancel(context.TODO()) 439 defer cancel() 440 441 paths, err := filepath.Glob(filepath.Join(rtest.TestWalkerPath, "*")) 442 rtest.OK(t, err) 443 444 before, err := statPath(rtest.TestWalkerPath) 445 rtest.OK(t, err) 446 447 t.Logf("walking paths %v with %d dirs, %d files", paths, 448 before.dirs, before.files) 449 450 after := stats{} 451 m := sync.Mutex{} 452 453 worker := func(ctx context.Context, wg *sync.WaitGroup, jobs <-chan pipe.Job) { 454 defer wg.Done() 455 for { 456 select { 457 case job, ok := <-jobs: 458 if !ok { 459 // channel is closed 460 return 461 } 462 rtest.Assert(t, job != nil, "job is nil") 463 464 switch j := job.(type) { 465 case pipe.Dir: 466 // wait for all content 467 for _, ch := range j.Entries { 468 <-ch 469 } 470 471 m.Lock() 472 after.dirs++ 473 m.Unlock() 474 475 j.Result() <- true 476 case pipe.Entry: 477 m.Lock() 478 after.files++ 479 m.Unlock() 480 481 j.Result() <- true 482 } 483 484 case <-ctx.Done(): 485 // pipeline was cancelled 486 return 487 } 488 } 489 } 490 491 var wg sync.WaitGroup 492 jobs := make(chan pipe.Job) 493 494 for i := 0; i < maxWorkers; i++ { 495 wg.Add(1) 496 go worker(ctx, &wg, jobs) 497 } 498 499 resCh := make(chan pipe.Result, 1) 500 pipe.Walk(ctx, paths, acceptAll, jobs, resCh) 501 502 // wait for all workers to terminate 503 wg.Wait() 504 505 // wait for top-level blob 506 <-resCh 507 508 t.Logf("walked %d paths with %d dirs, %d files", len(paths), after.dirs, after.files) 509 510 rtest.Assert(t, before == after, "stats do not match, expected %v, got %v", before, after) 511 } 512 513 func dirsInPath(path string) int { 514 if path == "/" || path == "." || path == "" { 515 return 0 516 } 517 518 n := 0 519 for dir := path; dir != "/" && dir != "."; dir = filepath.Dir(dir) { 520 n++ 521 } 522 523 return n 524 } 525 526 func TestPipeWalkerRoot(t *testing.T) { 527 if runtime.GOOS == "windows" { 528 t.Skipf("not running TestPipeWalkerRoot on %s", runtime.GOOS) 529 return 530 } 531 532 cwd, err := os.Getwd() 533 rtest.OK(t, err) 534 535 testPaths := []string{ 536 string(filepath.Separator), 537 ".", 538 cwd, 539 } 540 541 for _, path := range testPaths { 542 testPipeWalkerRootWithPath(path, t) 543 } 544 } 545 546 func testPipeWalkerRootWithPath(path string, t *testing.T) { 547 pattern := filepath.Join(path, "*") 548 rootPaths, err := filepath.Glob(pattern) 549 rtest.OK(t, err) 550 551 for i, p := range rootPaths { 552 rootPaths[i], err = filepath.Rel(path, p) 553 rtest.OK(t, err) 554 } 555 556 t.Logf("paths in %v (pattern %q) expanded to %v items", path, pattern, len(rootPaths)) 557 558 jobCh := make(chan pipe.Job) 559 var jobs []pipe.Job 560 561 worker := func(wg *sync.WaitGroup) { 562 defer wg.Done() 563 for job := range jobCh { 564 jobs = append(jobs, job) 565 } 566 } 567 568 var wg sync.WaitGroup 569 wg.Add(1) 570 go worker(&wg) 571 572 filter := func(p string, fi os.FileInfo) bool { 573 p, err := filepath.Rel(path, p) 574 rtest.OK(t, err) 575 return dirsInPath(p) <= 1 576 } 577 578 resCh := make(chan pipe.Result, 1) 579 pipe.Walk(context.TODO(), []string{path}, filter, jobCh, resCh) 580 581 wg.Wait() 582 583 t.Logf("received %d jobs", len(jobs)) 584 585 for i, job := range jobs[:len(jobs)-1] { 586 path := job.Path() 587 if path == "." || path == ".." || path == string(filepath.Separator) { 588 t.Errorf("job %v has invalid path %q", i, path) 589 } 590 } 591 592 lastPath := jobs[len(jobs)-1].Path() 593 if lastPath != "" { 594 t.Errorf("last job has non-empty path %q", lastPath) 595 } 596 597 if len(jobs) < len(rootPaths) { 598 t.Errorf("want at least %v jobs, got %v for path %v\n", len(rootPaths), len(jobs), path) 599 } 600 }