github.com/mckael/restic@v0.8.3/internal/repository/packer_manager.go (about) 1 package repository 2 3 import ( 4 "context" 5 "crypto/sha256" 6 "io" 7 "os" 8 "sync" 9 10 "github.com/restic/restic/internal/errors" 11 "github.com/restic/restic/internal/hashing" 12 "github.com/restic/restic/internal/restic" 13 14 "github.com/restic/restic/internal/crypto" 15 "github.com/restic/restic/internal/debug" 16 "github.com/restic/restic/internal/fs" 17 "github.com/restic/restic/internal/pack" 18 ) 19 20 // Saver implements saving data in a backend. 21 type Saver interface { 22 Save(context.Context, restic.Handle, io.Reader) error 23 } 24 25 // Packer holds a pack.Packer together with a hash writer. 26 type Packer struct { 27 *pack.Packer 28 hw *hashing.Writer 29 tmpfile *os.File 30 } 31 32 // packerManager keeps a list of open packs and creates new on demand. 33 type packerManager struct { 34 be Saver 35 key *crypto.Key 36 pm sync.Mutex 37 packers []*Packer 38 } 39 40 const minPackSize = 4 * 1024 * 1024 41 42 // newPackerManager returns an new packer manager which writes temporary files 43 // to a temporary directory 44 func newPackerManager(be Saver, key *crypto.Key) *packerManager { 45 return &packerManager{ 46 be: be, 47 key: key, 48 } 49 } 50 51 // findPacker returns a packer for a new blob of size bytes. Either a new one is 52 // created or one is returned that already has some blobs. 53 func (r *packerManager) findPacker() (packer *Packer, err error) { 54 r.pm.Lock() 55 defer r.pm.Unlock() 56 57 // search for a suitable packer 58 if len(r.packers) > 0 { 59 p := r.packers[0] 60 r.packers = r.packers[1:] 61 return p, nil 62 } 63 64 // no suitable packer found, return new 65 debug.Log("create new pack") 66 tmpfile, err := fs.TempFile("", "restic-temp-pack-") 67 if err != nil { 68 return nil, errors.Wrap(err, "fs.TempFile") 69 } 70 71 hw := hashing.NewWriter(tmpfile, sha256.New()) 72 p := pack.NewPacker(r.key, hw) 73 packer = &Packer{ 74 Packer: p, 75 hw: hw, 76 tmpfile: tmpfile, 77 } 78 79 return packer, nil 80 } 81 82 // insertPacker appends p to s.packs. 83 func (r *packerManager) insertPacker(p *Packer) { 84 r.pm.Lock() 85 defer r.pm.Unlock() 86 87 r.packers = append(r.packers, p) 88 debug.Log("%d packers\n", len(r.packers)) 89 } 90 91 // savePacker stores p in the backend. 92 func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packer) error { 93 debug.Log("save packer for %v with %d blobs (%d bytes)\n", t, p.Packer.Count(), p.Packer.Size()) 94 _, err := p.Packer.Finalize() 95 if err != nil { 96 return err 97 } 98 99 _, err = p.tmpfile.Seek(0, 0) 100 if err != nil { 101 return errors.Wrap(err, "Seek") 102 } 103 104 id := restic.IDFromHash(p.hw.Sum(nil)) 105 h := restic.Handle{Type: restic.DataFile, Name: id.String()} 106 107 err = r.be.Save(ctx, h, p.tmpfile) 108 if err != nil { 109 debug.Log("Save(%v) error: %v", h, err) 110 return err 111 } 112 113 debug.Log("saved as %v", h) 114 115 if t == restic.TreeBlob && r.Cache != nil { 116 debug.Log("saving tree pack file in cache") 117 118 _, err = p.tmpfile.Seek(0, 0) 119 if err != nil { 120 return errors.Wrap(err, "Seek") 121 } 122 123 err := r.Cache.Save(h, p.tmpfile) 124 if err != nil { 125 return err 126 } 127 } 128 129 err = p.tmpfile.Close() 130 if err != nil { 131 return errors.Wrap(err, "close tempfile") 132 } 133 134 err = fs.RemoveIfExists(p.tmpfile.Name()) 135 if err != nil { 136 return errors.Wrap(err, "Remove") 137 } 138 139 // update blobs in the index 140 for _, b := range p.Packer.Blobs() { 141 debug.Log(" updating blob %v to pack %v", b.ID, id) 142 r.idx.Store(restic.PackedBlob{ 143 Blob: restic.Blob{ 144 Type: b.Type, 145 ID: b.ID, 146 Offset: b.Offset, 147 Length: uint(b.Length), 148 }, 149 PackID: id, 150 }) 151 } 152 153 return nil 154 } 155 156 // countPacker returns the number of open (unfinished) packers. 157 func (r *packerManager) countPacker() int { 158 r.pm.Lock() 159 defer r.pm.Unlock() 160 161 return len(r.packers) 162 }