github.com/fawick/restic@v0.1.1-0.20171126184616-c02923fbfc79/internal/repository/master_index.go (about) 1 package repository 2 3 import ( 4 "context" 5 "sync" 6 7 "github.com/restic/restic/internal/errors" 8 "github.com/restic/restic/internal/restic" 9 10 "github.com/restic/restic/internal/debug" 11 ) 12 13 // MasterIndex is a collection of indexes and IDs of chunks that are in the process of being saved. 14 type MasterIndex struct { 15 idx []*Index 16 idxMutex sync.RWMutex 17 } 18 19 // NewMasterIndex creates a new master index. 20 func NewMasterIndex() *MasterIndex { 21 return &MasterIndex{} 22 } 23 24 // Lookup queries all known Indexes for the ID and returns the first match. 25 func (mi *MasterIndex) Lookup(id restic.ID, tpe restic.BlobType) (blobs []restic.PackedBlob, err error) { 26 mi.idxMutex.RLock() 27 defer mi.idxMutex.RUnlock() 28 29 debug.Log("looking up id %v, tpe %v", id.Str(), tpe) 30 31 for _, idx := range mi.idx { 32 blobs, err = idx.Lookup(id, tpe) 33 if err == nil { 34 debug.Log("found id %v: %v", id.Str(), blobs) 35 return 36 } 37 } 38 39 debug.Log("id %v not found in any index", id.Str()) 40 return nil, errors.Errorf("id %v not found in any index", id) 41 } 42 43 // LookupSize queries all known Indexes for the ID and returns the first match. 44 func (mi *MasterIndex) LookupSize(id restic.ID, tpe restic.BlobType) (uint, error) { 45 mi.idxMutex.RLock() 46 defer mi.idxMutex.RUnlock() 47 48 for _, idx := range mi.idx { 49 if idx.Has(id, tpe) { 50 return idx.LookupSize(id, tpe) 51 } 52 } 53 54 return 0, errors.Errorf("id %v not found in any index", id) 55 } 56 57 // ListPack returns the list of blobs in a pack. The first matching index is 58 // returned, or nil if no index contains information about the pack id. 59 func (mi *MasterIndex) ListPack(id restic.ID) (list []restic.PackedBlob) { 60 mi.idxMutex.RLock() 61 defer mi.idxMutex.RUnlock() 62 63 for _, idx := range mi.idx { 64 list := idx.ListPack(id) 65 if len(list) > 0 { 66 return list 67 } 68 } 69 70 return nil 71 } 72 73 // Has queries all known Indexes for the ID and returns the first match. 74 func (mi *MasterIndex) Has(id restic.ID, tpe restic.BlobType) bool { 75 mi.idxMutex.RLock() 76 defer mi.idxMutex.RUnlock() 77 78 for _, idx := range mi.idx { 79 if idx.Has(id, tpe) { 80 return true 81 } 82 } 83 84 return false 85 } 86 87 // Count returns the number of blobs of type t in the index. 88 func (mi *MasterIndex) Count(t restic.BlobType) (n uint) { 89 mi.idxMutex.RLock() 90 defer mi.idxMutex.RUnlock() 91 92 var sum uint 93 for _, idx := range mi.idx { 94 sum += idx.Count(t) 95 } 96 97 return sum 98 } 99 100 // Insert adds a new index to the MasterIndex. 101 func (mi *MasterIndex) Insert(idx *Index) { 102 mi.idxMutex.Lock() 103 defer mi.idxMutex.Unlock() 104 105 mi.idx = append(mi.idx, idx) 106 } 107 108 // Remove deletes an index from the MasterIndex. 109 func (mi *MasterIndex) Remove(index *Index) { 110 mi.idxMutex.Lock() 111 defer mi.idxMutex.Unlock() 112 113 for i, idx := range mi.idx { 114 if idx == index { 115 mi.idx = append(mi.idx[:i], mi.idx[i+1:]...) 116 return 117 } 118 } 119 } 120 121 // Store remembers the id and pack in the index. 122 func (mi *MasterIndex) Store(pb restic.PackedBlob) { 123 mi.idxMutex.Lock() 124 defer mi.idxMutex.Unlock() 125 126 for _, idx := range mi.idx { 127 if !idx.Final() { 128 idx.Store(pb) 129 return 130 } 131 } 132 133 newIdx := NewIndex() 134 newIdx.Store(pb) 135 mi.idx = append(mi.idx, newIdx) 136 } 137 138 // NotFinalIndexes returns all indexes that have not yet been saved. 139 func (mi *MasterIndex) NotFinalIndexes() []*Index { 140 mi.idxMutex.Lock() 141 defer mi.idxMutex.Unlock() 142 143 var list []*Index 144 145 for _, idx := range mi.idx { 146 if !idx.Final() { 147 list = append(list, idx) 148 } 149 } 150 151 debug.Log("return %d indexes", len(list)) 152 return list 153 } 154 155 // FullIndexes returns all indexes that are full. 156 func (mi *MasterIndex) FullIndexes() []*Index { 157 mi.idxMutex.Lock() 158 defer mi.idxMutex.Unlock() 159 160 var list []*Index 161 162 debug.Log("checking %d indexes", len(mi.idx)) 163 for _, idx := range mi.idx { 164 if idx.Final() { 165 debug.Log("index %p is final", idx) 166 continue 167 } 168 169 if IndexFull(idx) { 170 debug.Log("index %p is full", idx) 171 list = append(list, idx) 172 } else { 173 debug.Log("index %p not full", idx) 174 } 175 } 176 177 debug.Log("return %d indexes", len(list)) 178 return list 179 } 180 181 // All returns all indexes. 182 func (mi *MasterIndex) All() []*Index { 183 mi.idxMutex.Lock() 184 defer mi.idxMutex.Unlock() 185 186 return mi.idx 187 } 188 189 // Each returns a channel that yields all blobs known to the index. When the 190 // context is cancelled, the background goroutine terminates. This blocks any 191 // modification of the index. 192 func (mi *MasterIndex) Each(ctx context.Context) <-chan restic.PackedBlob { 193 mi.idxMutex.RLock() 194 195 ch := make(chan restic.PackedBlob) 196 197 go func() { 198 defer mi.idxMutex.RUnlock() 199 defer func() { 200 close(ch) 201 }() 202 203 for _, idx := range mi.idx { 204 idxCh := idx.Each(ctx) 205 for pb := range idxCh { 206 select { 207 case <-ctx.Done(): 208 return 209 case ch <- pb: 210 } 211 } 212 } 213 }() 214 215 return ch 216 } 217 218 // RebuildIndex combines all known indexes to a new index, leaving out any 219 // packs whose ID is contained in packBlacklist. The new index contains the IDs 220 // of all known indexes in the "supersedes" field. 221 func (mi *MasterIndex) RebuildIndex(packBlacklist restic.IDSet) (*Index, error) { 222 mi.idxMutex.Lock() 223 defer mi.idxMutex.Unlock() 224 225 debug.Log("start rebuilding index of %d indexes, pack blacklist: %v", len(mi.idx), packBlacklist) 226 227 newIndex := NewIndex() 228 229 ctx, cancel := context.WithCancel(context.TODO()) 230 defer cancel() 231 232 for i, idx := range mi.idx { 233 debug.Log("adding index %d", i) 234 235 for pb := range idx.Each(ctx) { 236 if packBlacklist.Has(pb.PackID) { 237 continue 238 } 239 240 newIndex.Store(pb) 241 } 242 243 if !idx.Final() { 244 debug.Log("index %d isn't final, don't add to supersedes field", i) 245 continue 246 } 247 248 id, err := idx.ID() 249 if err != nil { 250 debug.Log("index %d does not have an ID: %v", err) 251 return nil, err 252 } 253 254 debug.Log("adding index id %v to supersedes field", id.Str()) 255 256 err = newIndex.AddToSupersedes(id) 257 if err != nil { 258 return nil, err 259 } 260 } 261 262 return newIndex, nil 263 }