github.com/mckael/restic@v0.8.3/internal/cache/backend.go (about) 1 package cache 2 3 import ( 4 "context" 5 "io" 6 "sync" 7 8 "github.com/pkg/errors" 9 "github.com/restic/restic/internal/debug" 10 "github.com/restic/restic/internal/restic" 11 ) 12 13 // Backend wraps a restic.Backend and adds a cache. 14 type Backend struct { 15 restic.Backend 16 *Cache 17 18 // inProgress contains the handle for all files that are currently 19 // downloaded. The channel in the value is closed as soon as the download 20 // is finished. 21 inProgressMutex sync.Mutex 22 inProgress map[restic.Handle]chan struct{} 23 } 24 25 // ensure cachedBackend implements restic.Backend 26 var _ restic.Backend = &Backend{} 27 28 func newBackend(be restic.Backend, c *Cache) *Backend { 29 return &Backend{ 30 Backend: be, 31 Cache: c, 32 inProgress: make(map[restic.Handle]chan struct{}), 33 } 34 } 35 36 // Remove deletes a file from the backend and the cache if it has been cached. 37 func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { 38 debug.Log("cache Remove(%v)", h) 39 err := b.Backend.Remove(ctx, h) 40 if err != nil { 41 return err 42 } 43 44 return b.Cache.Remove(h) 45 } 46 47 var autoCacheTypes = map[restic.FileType]struct{}{ 48 restic.IndexFile: struct{}{}, 49 restic.SnapshotFile: struct{}{}, 50 } 51 52 // Save stores a new file in the backend and the cache. 53 func (b *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { 54 if _, ok := autoCacheTypes[h.Type]; !ok { 55 return b.Backend.Save(ctx, h, rd) 56 } 57 58 debug.Log("Save(%v): auto-store in the cache", h) 59 60 seeker, ok := rd.(io.Seeker) 61 if !ok { 62 return errors.New("reader is not a seeker") 63 } 64 65 pos, err := seeker.Seek(0, io.SeekCurrent) 66 if err != nil { 67 return errors.Wrapf(err, "Seek") 68 } 69 70 if pos != 0 { 71 return errors.Errorf("reader is not rewind (pos %d)", pos) 72 } 73 74 err = b.Backend.Save(ctx, h, rd) 75 if err != nil { 76 return err 77 } 78 79 _, err = seeker.Seek(pos, io.SeekStart) 80 if err != nil { 81 return errors.Wrapf(err, "Seek") 82 } 83 84 err = b.Cache.Save(h, rd) 85 if err != nil { 86 debug.Log("unable to save %v to cache: %v", h, err) 87 _ = b.Cache.Remove(h) 88 return nil 89 } 90 91 return nil 92 } 93 94 var autoCacheFiles = map[restic.FileType]bool{ 95 restic.IndexFile: true, 96 restic.SnapshotFile: true, 97 } 98 99 func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { 100 finish := make(chan struct{}) 101 defer func() { 102 close(finish) 103 104 // remove the finish channel from the map 105 b.inProgressMutex.Lock() 106 delete(b.inProgress, h) 107 b.inProgressMutex.Unlock() 108 }() 109 110 b.inProgressMutex.Lock() 111 other, alreadyDownloading := b.inProgress[h] 112 if !alreadyDownloading { 113 b.inProgress[h] = finish 114 } 115 b.inProgressMutex.Unlock() 116 117 if alreadyDownloading { 118 debug.Log("readahead %v is already performed by somebody else, delegating...", h) 119 <-other 120 debug.Log("download %v finished", h) 121 return nil 122 } 123 124 err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error { 125 return b.Cache.Save(h, rd) 126 }) 127 if err != nil { 128 // try to remove from the cache, ignore errors 129 _ = b.Cache.Remove(h) 130 return err 131 } 132 133 return nil 134 } 135 136 // loadFromCacheOrDelegate will try to load the file from the cache, and fall 137 // back to the backend if that fails. 138 func (b *Backend) loadFromCacheOrDelegate(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { 139 rd, err := b.Cache.Load(h, length, offset) 140 if err != nil { 141 return b.Backend.Load(ctx, h, length, offset, consumer) 142 } 143 144 err = consumer(rd) 145 if err != nil { 146 rd.Close() // ignore secondary errors 147 return err 148 } 149 return rd.Close() 150 } 151 152 // Load loads a file from the cache or the backend. 153 func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { 154 b.inProgressMutex.Lock() 155 waitForFinish, inProgress := b.inProgress[h] 156 b.inProgressMutex.Unlock() 157 158 if inProgress { 159 debug.Log("downloading %v is already in progress, waiting for finish", h) 160 <-waitForFinish 161 debug.Log("downloading %v finished", h) 162 } 163 164 if b.Cache.Has(h) { 165 debug.Log("Load(%v, %v, %v) from cache", h, length, offset) 166 rd, err := b.Cache.Load(h, length, offset) 167 if err == nil { 168 err = consumer(rd) 169 if err != nil { 170 rd.Close() // ignore secondary errors 171 return err 172 } 173 return rd.Close() 174 } 175 debug.Log("error loading %v from cache: %v", h, err) 176 } 177 178 // partial file requested 179 if offset != 0 || length != 0 { 180 if b.Cache.PerformReadahead(h) { 181 debug.Log("performing readahead for %v", h) 182 183 err := b.cacheFile(ctx, h) 184 if err == nil { 185 return b.loadFromCacheOrDelegate(ctx, h, length, offset, consumer) 186 } 187 188 debug.Log("error caching %v: %v", h, err) 189 } 190 191 debug.Log("Load(%v, %v, %v): partial file requested, delegating to backend", h, length, offset) 192 return b.Backend.Load(ctx, h, length, offset, consumer) 193 } 194 195 // if we don't automatically cache this file type, fall back to the backend 196 if _, ok := autoCacheFiles[h.Type]; !ok { 197 debug.Log("Load(%v, %v, %v): delegating to backend", h, length, offset) 198 return b.Backend.Load(ctx, h, length, offset, consumer) 199 } 200 201 debug.Log("auto-store %v in the cache", h) 202 err := b.cacheFile(ctx, h) 203 204 if err == nil { 205 // load the cached version 206 rd, err := b.Cache.Load(h, 0, 0) 207 if err != nil { 208 return err 209 } 210 err = consumer(rd) 211 if err != nil { 212 rd.Close() // ignore secondary errors 213 return err 214 } 215 return rd.Close() 216 } 217 218 debug.Log("error caching %v: %v, falling back to backend", h, err) 219 return b.Backend.Load(ctx, h, length, offset, consumer) 220 } 221 222 // Stat tests whether the backend has a file. If it does not exist but still 223 // exists in the cache, it is removed from the cache. 224 func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { 225 debug.Log("cache Stat(%v)", h) 226 227 fi, err := b.Backend.Stat(ctx, h) 228 if err != nil { 229 if b.Backend.IsNotExist(err) { 230 // try to remove from the cache, ignore errors 231 _ = b.Cache.Remove(h) 232 } 233 234 return fi, err 235 } 236 237 return fi, err 238 } 239 240 // IsNotExist returns true if the error is caused by a non-existing file. 241 func (b *Backend) IsNotExist(err error) bool { 242 return b.Backend.IsNotExist(err) 243 }