github.com/fawick/restic@v0.1.1-0.20171126184616-c02923fbfc79/internal/repository/repack.go (about)

     1  package repository
     2  
     3  import (
     4  	"context"
     5  	"crypto/sha256"
     6  	"fmt"
     7  	"io"
     8  	"os"
     9  
    10  	"github.com/restic/restic/internal/debug"
    11  	"github.com/restic/restic/internal/fs"
    12  	"github.com/restic/restic/internal/hashing"
    13  	"github.com/restic/restic/internal/pack"
    14  	"github.com/restic/restic/internal/restic"
    15  
    16  	"github.com/restic/restic/internal/errors"
    17  )
    18  
    19  // Repack takes a list of packs together with a list of blobs contained in
    20  // these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
    21  // into a new pack. Returned is the list of obsolete packs which can then
    22  // be removed.
    23  func Repack(ctx context.Context, repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *restic.Progress) (obsoletePacks restic.IDSet, err error) {
    24  	debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
    25  
    26  	for packID := range packs {
    27  		// load the complete pack into a temp file
    28  		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
    29  
    30  		tempfile, err := fs.TempFile("", "restic-temp-repack-")
    31  		if err != nil {
    32  			return nil, errors.Wrap(err, "TempFile")
    33  		}
    34  
    35  		beRd, err := repo.Backend().Load(ctx, h, 0, 0)
    36  		if err != nil {
    37  			return nil, err
    38  		}
    39  
    40  		hrd := hashing.NewReader(beRd, sha256.New())
    41  		packLength, err := io.Copy(tempfile, hrd)
    42  		if err != nil {
    43  			return nil, errors.Wrap(err, "Copy")
    44  		}
    45  
    46  		if err = beRd.Close(); err != nil {
    47  			return nil, errors.Wrap(err, "Close")
    48  		}
    49  
    50  		hash := restic.IDFromHash(hrd.Sum(nil))
    51  		debug.Log("pack %v loaded (%d bytes), hash %v", packID.Str(), packLength, hash.Str())
    52  
    53  		if !packID.Equal(hash) {
    54  			return nil, errors.Errorf("hash does not match id: want %v, got %v", packID, hash)
    55  		}
    56  
    57  		_, err = tempfile.Seek(0, 0)
    58  		if err != nil {
    59  			return nil, errors.Wrap(err, "Seek")
    60  		}
    61  
    62  		blobs, err := pack.List(repo.Key(), tempfile, packLength)
    63  		if err != nil {
    64  			return nil, err
    65  		}
    66  
    67  		debug.Log("processing pack %v, blobs: %v", packID.Str(), len(blobs))
    68  		var buf []byte
    69  		for _, entry := range blobs {
    70  			h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
    71  			if !keepBlobs.Has(h) {
    72  				continue
    73  			}
    74  
    75  			debug.Log("  process blob %v", h)
    76  
    77  			buf = buf[:]
    78  			if uint(len(buf)) < entry.Length {
    79  				buf = make([]byte, entry.Length)
    80  			}
    81  			buf = buf[:entry.Length]
    82  
    83  			n, err := tempfile.ReadAt(buf, int64(entry.Offset))
    84  			if err != nil {
    85  				return nil, errors.Wrap(err, "ReadAt")
    86  			}
    87  
    88  			if n != len(buf) {
    89  				return nil, errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v",
    90  					h, tempfile.Name(), len(buf), n)
    91  			}
    92  
    93  			nonce, ciphertext := buf[:repo.Key().NonceSize()], buf[repo.Key().NonceSize():]
    94  			plaintext, err := repo.Key().Open(ciphertext[:0], nonce, ciphertext, nil)
    95  			if err != nil {
    96  				return nil, err
    97  			}
    98  
    99  			id := restic.Hash(plaintext)
   100  			if !id.Equal(entry.ID) {
   101  				debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v",
   102  					h.Type, h.ID, tempfile.Name(), id)
   103  				fmt.Fprintf(os.Stderr, "read blob %v from %v: wrong data returned, hash is %v",
   104  					h, tempfile.Name(), id)
   105  			}
   106  
   107  			_, err = repo.SaveBlob(ctx, entry.Type, plaintext, entry.ID)
   108  			if err != nil {
   109  				return nil, err
   110  			}
   111  
   112  			debug.Log("  saved blob %v", entry.ID.Str())
   113  
   114  			keepBlobs.Delete(h)
   115  		}
   116  
   117  		if err = tempfile.Close(); err != nil {
   118  			return nil, errors.Wrap(err, "Close")
   119  		}
   120  
   121  		if err = fs.RemoveIfExists(tempfile.Name()); err != nil {
   122  			return nil, errors.Wrap(err, "Remove")
   123  		}
   124  		if p != nil {
   125  			p.Report(restic.Stat{Blobs: 1})
   126  		}
   127  	}
   128  
   129  	if err := repo.Flush(); err != nil {
   130  		return nil, err
   131  	}
   132  
   133  	return packs, nil
   134  }