github.com/fawick/restic@v0.1.1-0.20171126184616-c02923fbfc79/internal/repository/master_index.go (about)

     1  package repository
     2  
     3  import (
     4  	"context"
     5  	"sync"
     6  
     7  	"github.com/restic/restic/internal/errors"
     8  	"github.com/restic/restic/internal/restic"
     9  
    10  	"github.com/restic/restic/internal/debug"
    11  )
    12  
    13  // MasterIndex is a collection of indexes and IDs of chunks that are in the process of being saved.
    14  type MasterIndex struct {
    15  	idx      []*Index
    16  	idxMutex sync.RWMutex
    17  }
    18  
    19  // NewMasterIndex creates a new master index.
    20  func NewMasterIndex() *MasterIndex {
    21  	return &MasterIndex{}
    22  }
    23  
    24  // Lookup queries all known Indexes for the ID and returns the first match.
    25  func (mi *MasterIndex) Lookup(id restic.ID, tpe restic.BlobType) (blobs []restic.PackedBlob, err error) {
    26  	mi.idxMutex.RLock()
    27  	defer mi.idxMutex.RUnlock()
    28  
    29  	debug.Log("looking up id %v, tpe %v", id.Str(), tpe)
    30  
    31  	for _, idx := range mi.idx {
    32  		blobs, err = idx.Lookup(id, tpe)
    33  		if err == nil {
    34  			debug.Log("found id %v: %v", id.Str(), blobs)
    35  			return
    36  		}
    37  	}
    38  
    39  	debug.Log("id %v not found in any index", id.Str())
    40  	return nil, errors.Errorf("id %v not found in any index", id)
    41  }
    42  
    43  // LookupSize queries all known Indexes for the ID and returns the first match.
    44  func (mi *MasterIndex) LookupSize(id restic.ID, tpe restic.BlobType) (uint, error) {
    45  	mi.idxMutex.RLock()
    46  	defer mi.idxMutex.RUnlock()
    47  
    48  	for _, idx := range mi.idx {
    49  		if idx.Has(id, tpe) {
    50  			return idx.LookupSize(id, tpe)
    51  		}
    52  	}
    53  
    54  	return 0, errors.Errorf("id %v not found in any index", id)
    55  }
    56  
    57  // ListPack returns the list of blobs in a pack. The first matching index is
    58  // returned, or nil if no index contains information about the pack id.
    59  func (mi *MasterIndex) ListPack(id restic.ID) (list []restic.PackedBlob) {
    60  	mi.idxMutex.RLock()
    61  	defer mi.idxMutex.RUnlock()
    62  
    63  	for _, idx := range mi.idx {
    64  		list := idx.ListPack(id)
    65  		if len(list) > 0 {
    66  			return list
    67  		}
    68  	}
    69  
    70  	return nil
    71  }
    72  
    73  // Has queries all known Indexes for the ID and returns the first match.
    74  func (mi *MasterIndex) Has(id restic.ID, tpe restic.BlobType) bool {
    75  	mi.idxMutex.RLock()
    76  	defer mi.idxMutex.RUnlock()
    77  
    78  	for _, idx := range mi.idx {
    79  		if idx.Has(id, tpe) {
    80  			return true
    81  		}
    82  	}
    83  
    84  	return false
    85  }
    86  
    87  // Count returns the number of blobs of type t in the index.
    88  func (mi *MasterIndex) Count(t restic.BlobType) (n uint) {
    89  	mi.idxMutex.RLock()
    90  	defer mi.idxMutex.RUnlock()
    91  
    92  	var sum uint
    93  	for _, idx := range mi.idx {
    94  		sum += idx.Count(t)
    95  	}
    96  
    97  	return sum
    98  }
    99  
   100  // Insert adds a new index to the MasterIndex.
   101  func (mi *MasterIndex) Insert(idx *Index) {
   102  	mi.idxMutex.Lock()
   103  	defer mi.idxMutex.Unlock()
   104  
   105  	mi.idx = append(mi.idx, idx)
   106  }
   107  
   108  // Remove deletes an index from the MasterIndex.
   109  func (mi *MasterIndex) Remove(index *Index) {
   110  	mi.idxMutex.Lock()
   111  	defer mi.idxMutex.Unlock()
   112  
   113  	for i, idx := range mi.idx {
   114  		if idx == index {
   115  			mi.idx = append(mi.idx[:i], mi.idx[i+1:]...)
   116  			return
   117  		}
   118  	}
   119  }
   120  
   121  // Store remembers the id and pack in the index.
   122  func (mi *MasterIndex) Store(pb restic.PackedBlob) {
   123  	mi.idxMutex.Lock()
   124  	defer mi.idxMutex.Unlock()
   125  
   126  	for _, idx := range mi.idx {
   127  		if !idx.Final() {
   128  			idx.Store(pb)
   129  			return
   130  		}
   131  	}
   132  
   133  	newIdx := NewIndex()
   134  	newIdx.Store(pb)
   135  	mi.idx = append(mi.idx, newIdx)
   136  }
   137  
   138  // NotFinalIndexes returns all indexes that have not yet been saved.
   139  func (mi *MasterIndex) NotFinalIndexes() []*Index {
   140  	mi.idxMutex.Lock()
   141  	defer mi.idxMutex.Unlock()
   142  
   143  	var list []*Index
   144  
   145  	for _, idx := range mi.idx {
   146  		if !idx.Final() {
   147  			list = append(list, idx)
   148  		}
   149  	}
   150  
   151  	debug.Log("return %d indexes", len(list))
   152  	return list
   153  }
   154  
   155  // FullIndexes returns all indexes that are full.
   156  func (mi *MasterIndex) FullIndexes() []*Index {
   157  	mi.idxMutex.Lock()
   158  	defer mi.idxMutex.Unlock()
   159  
   160  	var list []*Index
   161  
   162  	debug.Log("checking %d indexes", len(mi.idx))
   163  	for _, idx := range mi.idx {
   164  		if idx.Final() {
   165  			debug.Log("index %p is final", idx)
   166  			continue
   167  		}
   168  
   169  		if IndexFull(idx) {
   170  			debug.Log("index %p is full", idx)
   171  			list = append(list, idx)
   172  		} else {
   173  			debug.Log("index %p not full", idx)
   174  		}
   175  	}
   176  
   177  	debug.Log("return %d indexes", len(list))
   178  	return list
   179  }
   180  
   181  // All returns all indexes.
   182  func (mi *MasterIndex) All() []*Index {
   183  	mi.idxMutex.Lock()
   184  	defer mi.idxMutex.Unlock()
   185  
   186  	return mi.idx
   187  }
   188  
   189  // Each returns a channel that yields all blobs known to the index. When the
   190  // context is cancelled, the background goroutine terminates. This blocks any
   191  // modification of the index.
   192  func (mi *MasterIndex) Each(ctx context.Context) <-chan restic.PackedBlob {
   193  	mi.idxMutex.RLock()
   194  
   195  	ch := make(chan restic.PackedBlob)
   196  
   197  	go func() {
   198  		defer mi.idxMutex.RUnlock()
   199  		defer func() {
   200  			close(ch)
   201  		}()
   202  
   203  		for _, idx := range mi.idx {
   204  			idxCh := idx.Each(ctx)
   205  			for pb := range idxCh {
   206  				select {
   207  				case <-ctx.Done():
   208  					return
   209  				case ch <- pb:
   210  				}
   211  			}
   212  		}
   213  	}()
   214  
   215  	return ch
   216  }
   217  
   218  // RebuildIndex combines all known indexes to a new index, leaving out any
   219  // packs whose ID is contained in packBlacklist. The new index contains the IDs
   220  // of all known indexes in the "supersedes" field.
   221  func (mi *MasterIndex) RebuildIndex(packBlacklist restic.IDSet) (*Index, error) {
   222  	mi.idxMutex.Lock()
   223  	defer mi.idxMutex.Unlock()
   224  
   225  	debug.Log("start rebuilding index of %d indexes, pack blacklist: %v", len(mi.idx), packBlacklist)
   226  
   227  	newIndex := NewIndex()
   228  
   229  	ctx, cancel := context.WithCancel(context.TODO())
   230  	defer cancel()
   231  
   232  	for i, idx := range mi.idx {
   233  		debug.Log("adding index %d", i)
   234  
   235  		for pb := range idx.Each(ctx) {
   236  			if packBlacklist.Has(pb.PackID) {
   237  				continue
   238  			}
   239  
   240  			newIndex.Store(pb)
   241  		}
   242  
   243  		if !idx.Final() {
   244  			debug.Log("index %d isn't final, don't add to supersedes field", i)
   245  			continue
   246  		}
   247  
   248  		id, err := idx.ID()
   249  		if err != nil {
   250  			debug.Log("index %d does not have an ID: %v", err)
   251  			return nil, err
   252  		}
   253  
   254  		debug.Log("adding index id %v to supersedes field", id.Str())
   255  
   256  		err = newIndex.AddToSupersedes(id)
   257  		if err != nil {
   258  			return nil, err
   259  		}
   260  	}
   261  
   262  	return newIndex, nil
   263  }