github.com/advanderveer/restic@v0.8.1-0.20171209104529-42a8c19aaea6/internal/checker/checker.go (about) 1 package checker 2 3 import ( 4 "context" 5 "crypto/sha256" 6 "fmt" 7 "io" 8 "os" 9 "sync" 10 11 "github.com/restic/restic/internal/errors" 12 "github.com/restic/restic/internal/fs" 13 "github.com/restic/restic/internal/hashing" 14 "github.com/restic/restic/internal/restic" 15 16 "github.com/restic/restic/internal/debug" 17 "github.com/restic/restic/internal/pack" 18 "github.com/restic/restic/internal/repository" 19 ) 20 21 // Checker runs various checks on a repository. It is advisable to create an 22 // exclusive Lock in the repository before running any checks. 23 // 24 // A Checker only tests for internal errors within the data structures of the 25 // repository (e.g. missing blobs), and needs a valid Repository to work on. 26 type Checker struct { 27 packs restic.IDSet 28 blobs restic.IDSet 29 blobRefs struct { 30 sync.Mutex 31 M map[restic.ID]uint 32 } 33 indexes map[restic.ID]*repository.Index 34 orphanedPacks restic.IDs 35 36 masterIndex *repository.MasterIndex 37 38 repo restic.Repository 39 } 40 41 // New returns a new checker which runs on repo. 42 func New(repo restic.Repository) *Checker { 43 c := &Checker{ 44 packs: restic.NewIDSet(), 45 blobs: restic.NewIDSet(), 46 masterIndex: repository.NewMasterIndex(), 47 indexes: make(map[restic.ID]*repository.Index), 48 repo: repo, 49 } 50 51 c.blobRefs.M = make(map[restic.ID]uint) 52 53 return c 54 } 55 56 const defaultParallelism = 40 57 58 // ErrDuplicatePacks is returned when a pack is found in more than one index. 59 type ErrDuplicatePacks struct { 60 PackID restic.ID 61 Indexes restic.IDSet 62 } 63 64 func (e ErrDuplicatePacks) Error() string { 65 return fmt.Sprintf("pack %v contained in several indexes: %v", e.PackID.Str(), e.Indexes) 66 } 67 68 // ErrOldIndexFormat is returned when an index with the old format is 69 // found. 70 type ErrOldIndexFormat struct { 71 restic.ID 72 } 73 74 func (err ErrOldIndexFormat) Error() string { 75 return fmt.Sprintf("index %v has old format", err.ID.Str()) 76 } 77 78 // LoadIndex loads all index files. 79 func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { 80 debug.Log("Start") 81 type indexRes struct { 82 Index *repository.Index 83 err error 84 ID string 85 } 86 87 indexCh := make(chan indexRes) 88 89 worker := func(ctx context.Context, id restic.ID) error { 90 debug.Log("worker got index %v", id) 91 idx, err := repository.LoadIndexWithDecoder(ctx, c.repo, id, repository.DecodeIndex) 92 if errors.Cause(err) == repository.ErrOldIndexFormat { 93 debug.Log("index %v has old format", id.Str()) 94 hints = append(hints, ErrOldIndexFormat{id}) 95 96 idx, err = repository.LoadIndexWithDecoder(ctx, c.repo, id, repository.DecodeOldIndex) 97 } 98 99 err = errors.Wrapf(err, "error loading index %v", id.Str()) 100 101 select { 102 case indexCh <- indexRes{Index: idx, ID: id.String(), err: err}: 103 case <-ctx.Done(): 104 } 105 106 return nil 107 } 108 109 go func() { 110 defer close(indexCh) 111 debug.Log("start loading indexes in parallel") 112 err := repository.FilesInParallel(ctx, c.repo.Backend(), restic.IndexFile, defaultParallelism, 113 repository.ParallelWorkFuncParseID(worker)) 114 debug.Log("loading indexes finished, error: %v", err) 115 if err != nil { 116 panic(err) 117 } 118 }() 119 120 done := make(chan struct{}) 121 defer close(done) 122 123 packToIndex := make(map[restic.ID]restic.IDSet) 124 125 for res := range indexCh { 126 debug.Log("process index %v, err %v", res.ID, res.err) 127 128 if res.err != nil { 129 errs = append(errs, res.err) 130 continue 131 } 132 133 idxID, err := restic.ParseID(res.ID) 134 if err != nil { 135 errs = append(errs, errors.Errorf("unable to parse as index ID: %v", res.ID)) 136 continue 137 } 138 139 c.indexes[idxID] = res.Index 140 c.masterIndex.Insert(res.Index) 141 142 debug.Log("process blobs") 143 cnt := 0 144 for blob := range res.Index.Each(ctx) { 145 c.packs.Insert(blob.PackID) 146 c.blobs.Insert(blob.ID) 147 c.blobRefs.M[blob.ID] = 0 148 cnt++ 149 150 if _, ok := packToIndex[blob.PackID]; !ok { 151 packToIndex[blob.PackID] = restic.NewIDSet() 152 } 153 packToIndex[blob.PackID].Insert(idxID) 154 } 155 156 debug.Log("%d blobs processed", cnt) 157 } 158 159 debug.Log("checking for duplicate packs") 160 for packID := range c.packs { 161 debug.Log(" check pack %v: contained in %d indexes", packID.Str(), len(packToIndex[packID])) 162 if len(packToIndex[packID]) > 1 { 163 hints = append(hints, ErrDuplicatePacks{ 164 PackID: packID, 165 Indexes: packToIndex[packID], 166 }) 167 } 168 } 169 170 c.repo.SetIndex(c.masterIndex) 171 172 return hints, errs 173 } 174 175 // PackError describes an error with a specific pack. 176 type PackError struct { 177 ID restic.ID 178 Orphaned bool 179 Err error 180 } 181 182 func (e PackError) Error() string { 183 return "pack " + e.ID.String() + ": " + e.Err.Error() 184 } 185 186 func packIDTester(ctx context.Context, repo restic.Repository, inChan <-chan restic.ID, errChan chan<- error, wg *sync.WaitGroup) { 187 debug.Log("worker start") 188 defer debug.Log("worker done") 189 190 defer wg.Done() 191 192 for id := range inChan { 193 h := restic.Handle{Type: restic.DataFile, Name: id.String()} 194 ok, err := repo.Backend().Test(ctx, h) 195 if err != nil { 196 err = PackError{ID: id, Err: err} 197 } else { 198 if !ok { 199 err = PackError{ID: id, Err: errors.New("does not exist")} 200 } 201 } 202 203 if err != nil { 204 debug.Log("error checking for pack %s: %v", id.Str(), err) 205 select { 206 case <-ctx.Done(): 207 return 208 case errChan <- err: 209 } 210 211 continue 212 } 213 214 debug.Log("pack %s exists", id.Str()) 215 } 216 } 217 218 // Packs checks that all packs referenced in the index are still available and 219 // there are no packs that aren't in an index. errChan is closed after all 220 // packs have been checked. 221 func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { 222 defer close(errChan) 223 224 debug.Log("checking for %d packs", len(c.packs)) 225 seenPacks := restic.NewIDSet() 226 227 var workerWG sync.WaitGroup 228 229 IDChan := make(chan restic.ID) 230 for i := 0; i < defaultParallelism; i++ { 231 workerWG.Add(1) 232 go packIDTester(ctx, c.repo, IDChan, errChan, &workerWG) 233 } 234 235 for id := range c.packs { 236 seenPacks.Insert(id) 237 IDChan <- id 238 } 239 close(IDChan) 240 241 debug.Log("waiting for %d workers to terminate", defaultParallelism) 242 workerWG.Wait() 243 debug.Log("workers terminated") 244 245 for id := range c.repo.List(ctx, restic.DataFile) { 246 debug.Log("check data blob %v", id.Str()) 247 if !seenPacks.Has(id) { 248 c.orphanedPacks = append(c.orphanedPacks, id) 249 select { 250 case <-ctx.Done(): 251 return 252 case errChan <- PackError{ID: id, Orphaned: true, Err: errors.New("not referenced in any index")}: 253 } 254 } 255 } 256 } 257 258 // Error is an error that occurred while checking a repository. 259 type Error struct { 260 TreeID restic.ID 261 BlobID restic.ID 262 Err error 263 } 264 265 func (e Error) Error() string { 266 if !e.BlobID.IsNull() && !e.TreeID.IsNull() { 267 msg := "tree " + e.TreeID.Str() 268 msg += ", blob " + e.BlobID.Str() 269 msg += ": " + e.Err.Error() 270 return msg 271 } 272 273 if !e.TreeID.IsNull() { 274 return "tree " + e.TreeID.Str() + ": " + e.Err.Error() 275 } 276 277 return e.Err.Error() 278 } 279 280 func loadTreeFromSnapshot(ctx context.Context, repo restic.Repository, id restic.ID) (restic.ID, error) { 281 sn, err := restic.LoadSnapshot(ctx, repo, id) 282 if err != nil { 283 debug.Log("error loading snapshot %v: %v", id.Str(), err) 284 return restic.ID{}, err 285 } 286 287 if sn.Tree == nil { 288 debug.Log("snapshot %v has no tree", id.Str()) 289 return restic.ID{}, errors.Errorf("snapshot %v has no tree", id) 290 } 291 292 return *sn.Tree, nil 293 } 294 295 // loadSnapshotTreeIDs loads all snapshots from backend and returns the tree IDs. 296 func loadSnapshotTreeIDs(ctx context.Context, repo restic.Repository) (restic.IDs, []error) { 297 var trees struct { 298 IDs restic.IDs 299 sync.Mutex 300 } 301 302 var errs struct { 303 errs []error 304 sync.Mutex 305 } 306 307 snapshotWorker := func(ctx context.Context, strID string) error { 308 id, err := restic.ParseID(strID) 309 if err != nil { 310 return err 311 } 312 313 debug.Log("load snapshot %v", id.Str()) 314 315 treeID, err := loadTreeFromSnapshot(ctx, repo, id) 316 if err != nil { 317 errs.Lock() 318 errs.errs = append(errs.errs, err) 319 errs.Unlock() 320 return nil 321 } 322 323 debug.Log("snapshot %v has tree %v", id.Str(), treeID.Str()) 324 trees.Lock() 325 trees.IDs = append(trees.IDs, treeID) 326 trees.Unlock() 327 328 return nil 329 } 330 331 err := repository.FilesInParallel(ctx, repo.Backend(), restic.SnapshotFile, defaultParallelism, snapshotWorker) 332 if err != nil { 333 errs.errs = append(errs.errs, err) 334 } 335 336 return trees.IDs, errs.errs 337 } 338 339 // TreeError collects several errors that occurred while processing a tree. 340 type TreeError struct { 341 ID restic.ID 342 Errors []error 343 } 344 345 func (e TreeError) Error() string { 346 return fmt.Sprintf("tree %v: %v", e.ID.Str(), e.Errors) 347 } 348 349 type treeJob struct { 350 restic.ID 351 error 352 *restic.Tree 353 } 354 355 // loadTreeWorker loads trees from repo and sends them to out. 356 func loadTreeWorker(ctx context.Context, repo restic.Repository, 357 in <-chan restic.ID, out chan<- treeJob, 358 wg *sync.WaitGroup) { 359 360 defer func() { 361 debug.Log("exiting") 362 wg.Done() 363 }() 364 365 var ( 366 inCh = in 367 outCh = out 368 job treeJob 369 ) 370 371 outCh = nil 372 for { 373 select { 374 case <-ctx.Done(): 375 return 376 377 case treeID, ok := <-inCh: 378 if !ok { 379 return 380 } 381 debug.Log("load tree %v", treeID.Str()) 382 383 tree, err := repo.LoadTree(ctx, treeID) 384 debug.Log("load tree %v (%v) returned err: %v", tree, treeID.Str(), err) 385 job = treeJob{ID: treeID, error: err, Tree: tree} 386 outCh = out 387 inCh = nil 388 389 case outCh <- job: 390 debug.Log("sent tree %v", job.ID.Str()) 391 outCh = nil 392 inCh = in 393 } 394 } 395 } 396 397 // checkTreeWorker checks the trees received and sends out errors to errChan. 398 func (c *Checker) checkTreeWorker(ctx context.Context, in <-chan treeJob, out chan<- error, wg *sync.WaitGroup) { 399 defer func() { 400 debug.Log("exiting") 401 wg.Done() 402 }() 403 404 var ( 405 inCh = in 406 outCh = out 407 treeError TreeError 408 ) 409 410 outCh = nil 411 for { 412 select { 413 case <-ctx.Done(): 414 debug.Log("done channel closed, exiting") 415 return 416 417 case job, ok := <-inCh: 418 if !ok { 419 debug.Log("input channel closed, exiting") 420 return 421 } 422 423 id := job.ID 424 alreadyChecked := false 425 c.blobRefs.Lock() 426 if c.blobRefs.M[id] > 0 { 427 alreadyChecked = true 428 } 429 c.blobRefs.M[id]++ 430 debug.Log("tree %v refcount %d", job.ID.Str(), c.blobRefs.M[id]) 431 c.blobRefs.Unlock() 432 433 if alreadyChecked { 434 continue 435 } 436 437 debug.Log("check tree %v (tree %v, err %v)", job.ID.Str(), job.Tree, job.error) 438 439 var errs []error 440 if job.error != nil { 441 errs = append(errs, job.error) 442 } else { 443 errs = c.checkTree(job.ID, job.Tree) 444 } 445 446 if len(errs) > 0 { 447 debug.Log("checked tree %v: %v errors", job.ID.Str(), len(errs)) 448 treeError = TreeError{ID: job.ID, Errors: errs} 449 outCh = out 450 inCh = nil 451 } 452 453 case outCh <- treeError: 454 debug.Log("tree %v: sent %d errors", treeError.ID, len(treeError.Errors)) 455 outCh = nil 456 inCh = in 457 } 458 } 459 } 460 461 func filterTrees(ctx context.Context, backlog restic.IDs, loaderChan chan<- restic.ID, in <-chan treeJob, out chan<- treeJob) { 462 defer func() { 463 debug.Log("closing output channels") 464 close(loaderChan) 465 close(out) 466 }() 467 468 var ( 469 inCh = in 470 outCh = out 471 loadCh = loaderChan 472 job treeJob 473 nextTreeID restic.ID 474 outstandingLoadTreeJobs = 0 475 ) 476 477 outCh = nil 478 loadCh = nil 479 480 for { 481 if loadCh == nil && len(backlog) > 0 { 482 loadCh = loaderChan 483 nextTreeID, backlog = backlog[0], backlog[1:] 484 } 485 486 if loadCh == nil && outCh == nil && outstandingLoadTreeJobs == 0 { 487 debug.Log("backlog is empty, all channels nil, exiting") 488 return 489 } 490 491 select { 492 case <-ctx.Done(): 493 return 494 495 case loadCh <- nextTreeID: 496 outstandingLoadTreeJobs++ 497 loadCh = nil 498 499 case j, ok := <-inCh: 500 if !ok { 501 debug.Log("input channel closed") 502 inCh = nil 503 in = nil 504 continue 505 } 506 507 outstandingLoadTreeJobs-- 508 509 debug.Log("input job tree %v", j.ID.Str()) 510 511 var err error 512 513 if j.error != nil { 514 debug.Log("received job with error: %v (tree %v, ID %v)", j.error, j.Tree, j.ID.Str()) 515 } else if j.Tree == nil { 516 debug.Log("received job with nil tree pointer: %v (ID %v)", j.error, j.ID.Str()) 517 err = errors.New("tree is nil and error is nil") 518 } else { 519 debug.Log("subtrees for tree %v: %v", j.ID.Str(), j.Tree.Subtrees()) 520 for _, id := range j.Tree.Subtrees() { 521 if id.IsNull() { 522 // We do not need to raise this error here, it is 523 // checked when the tree is checked. Just make sure 524 // that we do not add any null IDs to the backlog. 525 debug.Log("tree %v has nil subtree", j.ID.Str()) 526 continue 527 } 528 backlog = append(backlog, id) 529 } 530 } 531 532 if err != nil { 533 // send a new job with the new error instead of the old one 534 j = treeJob{ID: j.ID, error: err} 535 } 536 537 job = j 538 outCh = out 539 inCh = nil 540 541 case outCh <- job: 542 debug.Log("tree sent to check: %v", job.ID.Str()) 543 outCh = nil 544 inCh = in 545 } 546 } 547 } 548 549 // Structure checks that for all snapshots all referenced data blobs and 550 // subtrees are available in the index. errChan is closed after all trees have 551 // been traversed. 552 func (c *Checker) Structure(ctx context.Context, errChan chan<- error) { 553 defer close(errChan) 554 555 trees, errs := loadSnapshotTreeIDs(ctx, c.repo) 556 debug.Log("need to check %d trees from snapshots, %d errs returned", len(trees), len(errs)) 557 558 for _, err := range errs { 559 select { 560 case <-ctx.Done(): 561 return 562 case errChan <- err: 563 } 564 } 565 566 treeIDChan := make(chan restic.ID) 567 treeJobChan1 := make(chan treeJob) 568 treeJobChan2 := make(chan treeJob) 569 570 var wg sync.WaitGroup 571 for i := 0; i < defaultParallelism; i++ { 572 wg.Add(2) 573 go loadTreeWorker(ctx, c.repo, treeIDChan, treeJobChan1, &wg) 574 go c.checkTreeWorker(ctx, treeJobChan2, errChan, &wg) 575 } 576 577 filterTrees(ctx, trees, treeIDChan, treeJobChan1, treeJobChan2) 578 579 wg.Wait() 580 } 581 582 func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) { 583 debug.Log("checking tree %v", id.Str()) 584 585 var blobs []restic.ID 586 587 for _, node := range tree.Nodes { 588 switch node.Type { 589 case "file": 590 if node.Content == nil { 591 errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q has nil blob list", node.Name)}) 592 } 593 594 for b, blobID := range node.Content { 595 if blobID.IsNull() { 596 errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q blob %d has null ID", node.Name, b)}) 597 continue 598 } 599 blobs = append(blobs, blobID) 600 } 601 case "dir": 602 if node.Subtree == nil { 603 errs = append(errs, Error{TreeID: id, Err: errors.Errorf("dir node %q has no subtree", node.Name)}) 604 continue 605 } 606 607 if node.Subtree.IsNull() { 608 errs = append(errs, Error{TreeID: id, Err: errors.Errorf("dir node %q subtree id is null", node.Name)}) 609 continue 610 } 611 612 case "symlink", "socket", "chardev", "dev", "fifo": 613 // nothing to check 614 615 default: 616 errs = append(errs, Error{TreeID: id, Err: errors.Errorf("node %q with invalid type %q", node.Name, node.Type)}) 617 } 618 619 if node.Name == "" { 620 errs = append(errs, Error{TreeID: id, Err: errors.New("node with empty name")}) 621 } 622 } 623 624 for _, blobID := range blobs { 625 c.blobRefs.Lock() 626 c.blobRefs.M[blobID]++ 627 debug.Log("blob %v refcount %d", blobID.Str(), c.blobRefs.M[blobID]) 628 c.blobRefs.Unlock() 629 630 if !c.blobs.Has(blobID) { 631 debug.Log("tree %v references blob %v which isn't contained in index", id.Str(), blobID.Str()) 632 633 errs = append(errs, Error{TreeID: id, BlobID: blobID, Err: errors.New("not found in index")}) 634 } 635 } 636 637 return errs 638 } 639 640 // UnusedBlobs returns all blobs that have never been referenced. 641 func (c *Checker) UnusedBlobs() (blobs restic.IDs) { 642 c.blobRefs.Lock() 643 defer c.blobRefs.Unlock() 644 645 debug.Log("checking %d blobs", len(c.blobs)) 646 for id := range c.blobs { 647 if c.blobRefs.M[id] == 0 { 648 debug.Log("blob %v not referenced", id.Str()) 649 blobs = append(blobs, id) 650 } 651 } 652 653 return blobs 654 } 655 656 // CountPacks returns the number of packs in the repository. 657 func (c *Checker) CountPacks() uint64 { 658 return uint64(len(c.packs)) 659 } 660 661 // checkPack reads a pack and checks the integrity of all blobs. 662 func checkPack(ctx context.Context, r restic.Repository, id restic.ID) error { 663 debug.Log("checking pack %v", id.Str()) 664 h := restic.Handle{Type: restic.DataFile, Name: id.String()} 665 666 rd, err := r.Backend().Load(ctx, h, 0, 0) 667 if err != nil { 668 return err 669 } 670 671 packfile, err := fs.TempFile("", "restic-temp-check-") 672 if err != nil { 673 return errors.Wrap(err, "TempFile") 674 } 675 676 defer func() { 677 _ = packfile.Close() 678 _ = os.Remove(packfile.Name()) 679 }() 680 681 hrd := hashing.NewReader(rd, sha256.New()) 682 size, err := io.Copy(packfile, hrd) 683 if err != nil { 684 return errors.Wrap(err, "Copy") 685 } 686 687 if err = rd.Close(); err != nil { 688 return err 689 } 690 691 hash := restic.IDFromHash(hrd.Sum(nil)) 692 debug.Log("hash for pack %v is %v", id.Str(), hash.Str()) 693 694 if !hash.Equal(id) { 695 debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str()) 696 return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str()) 697 } 698 699 blobs, err := pack.List(r.Key(), packfile, size) 700 if err != nil { 701 return err 702 } 703 704 var errs []error 705 var buf []byte 706 for i, blob := range blobs { 707 debug.Log(" check blob %d: %v", i, blob) 708 709 buf = buf[:cap(buf)] 710 if uint(len(buf)) < blob.Length { 711 buf = make([]byte, blob.Length) 712 } 713 buf = buf[:blob.Length] 714 715 _, err := packfile.Seek(int64(blob.Offset), 0) 716 if err != nil { 717 return errors.Errorf("Seek(%v): %v", blob.Offset, err) 718 } 719 720 _, err = io.ReadFull(packfile, buf) 721 if err != nil { 722 debug.Log(" error loading blob %v: %v", blob.ID.Str(), err) 723 errs = append(errs, errors.Errorf("blob %v: %v", i, err)) 724 continue 725 } 726 727 nonce, ciphertext := buf[:r.Key().NonceSize()], buf[r.Key().NonceSize():] 728 plaintext, err := r.Key().Open(ciphertext[:0], nonce, ciphertext, nil) 729 if err != nil { 730 debug.Log(" error decrypting blob %v: %v", blob.ID.Str(), err) 731 errs = append(errs, errors.Errorf("blob %v: %v", i, err)) 732 continue 733 } 734 735 hash := restic.Hash(plaintext) 736 if !hash.Equal(blob.ID) { 737 debug.Log(" Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()) 738 errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())) 739 continue 740 } 741 } 742 743 if len(errs) > 0 { 744 return errors.Errorf("pack %v contains %v errors: %v", id.Str(), len(errs), errs) 745 } 746 747 return nil 748 } 749 750 // ReadData loads all data from the repository and checks the integrity. 751 func (c *Checker) ReadData(ctx context.Context, p *restic.Progress, errChan chan<- error) { 752 defer close(errChan) 753 754 p.Start() 755 defer p.Done() 756 757 worker := func(wg *sync.WaitGroup, in <-chan restic.ID) { 758 defer wg.Done() 759 for { 760 var id restic.ID 761 var ok bool 762 763 select { 764 case <-ctx.Done(): 765 return 766 case id, ok = <-in: 767 if !ok { 768 return 769 } 770 } 771 772 err := checkPack(ctx, c.repo, id) 773 p.Report(restic.Stat{Blobs: 1}) 774 if err == nil { 775 continue 776 } 777 778 select { 779 case <-ctx.Done(): 780 return 781 case errChan <- err: 782 } 783 } 784 } 785 786 ch := c.repo.List(ctx, restic.DataFile) 787 788 var wg sync.WaitGroup 789 for i := 0; i < defaultParallelism; i++ { 790 wg.Add(1) 791 go worker(&wg, ch) 792 } 793 794 wg.Wait() 795 }