gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/persistedlru.go (about)

     1  package renter
     2  
     3  import (
     4  	"container/list"
     5  	"encoding/hex"
     6  	"fmt"
     7  	"io"
     8  	"math"
     9  	"os"
    10  	"path/filepath"
    11  	"sync"
    12  	"time"
    13  
    14  	"gitlab.com/NebulousLabs/errors"
    15  	"gitlab.com/NebulousLabs/fastrand"
    16  	"gitlab.com/SkynetLabs/skyd/build"
    17  	"gitlab.com/SkynetLabs/skyd/skymodules"
    18  	"go.sia.tech/siad/crypto"
    19  	"golang.org/x/crypto/chacha20"
    20  	"golang.org/x/crypto/chacha20poly1305"
    21  )
    22  
    23  // baseSectorSectionIndex is the sentinel value for the index we use to cache
    24  // the base sector of a skylink.
    25  const baseSectorSectionIndex = math.MaxUint64
    26  
    27  // PersistedLRU is the interface for a persistedLRU. This is useful for mocking
    28  // the lru during testing.
    29  type PersistedLRU interface {
    30  	// Get tries to fetch data from the cache. If the data is not cached, false is
    31  	// returned.
    32  	Get(dsid skymodules.DataSourceID, sectionIndex uint64) (*downloadedData, bool, error)
    33  	// Put adds a new section to the cache.
    34  	Put(dsid skymodules.DataSourceID, sectionIndex uint64, data *downloadedData) error
    35  
    36  	// NumCachedSections returns the number of sections currently cached for
    37  	// a data source with a given id.
    38  	NumCachedSections(dsid skymodules.DataSourceID) uint64
    39  }
    40  
    41  type (
    42  	// cachedDataSource describes a cached datasource which can contain multiple
    43  	// sections.
    44  	cachedDataSource struct {
    45  		staticID  skymodules.DataSourceID
    46  		staticKey []byte
    47  		staticLRU *persistedLRU
    48  
    49  		sections map[uint64]struct{}
    50  		mu       sync.Mutex
    51  	}
    52  
    53  	// lruElement describes an element within the LRU.
    54  	lruElement struct {
    55  		staticDSID         skymodules.DataSourceID
    56  		staticSectionIndex uint64
    57  	}
    58  
    59  	// cacheHit is a single hit to the cache at a certain time.
    60  	cacheHit struct {
    61  		staticTime time.Time
    62  	}
    63  
    64  	// cacheHitTracker tracks how many times a cached section gets accessed
    65  	// within a certain period of time.
    66  	cacheHitTracker struct {
    67  		staticMinHits  uint
    68  		staticDuration time.Duration
    69  
    70  		pruning bool
    71  		hits    map[skymodules.DataSourceID]map[uint64][]cacheHit
    72  		mu      sync.Mutex
    73  	}
    74  
    75  	// persistedLRU is the LRU itself. It stores cached elements in a tree
    76  	// structure on disk.
    77  	persistedLRU struct {
    78  		staticSalt []byte
    79  
    80  		staticPath string
    81  
    82  		staticHitTracker *cacheHitTracker
    83  		staticLRU        *list.List
    84  		lruElements      map[skymodules.DataSourceID]map[uint64]*list.Element
    85  
    86  		cachedSize         int64
    87  		staticMaxCacheSize int64
    88  
    89  		dataSources map[skymodules.DataSourceID]*cachedDataSource
    90  		staticWG    sync.WaitGroup
    91  		staticDone  chan struct{}
    92  		mu          sync.Mutex
    93  	}
    94  )
    95  
    96  // newCacheHitTracker creates a new cacheHitTracker.
    97  func newCacheHitTracker(minHits uint, duration time.Duration) *cacheHitTracker {
    98  	return &cacheHitTracker{
    99  		staticMinHits:  minHits,
   100  		staticDuration: duration,
   101  		hits:           make(map[skymodules.DataSourceID]map[uint64][]cacheHit),
   102  	}
   103  }
   104  
   105  // pruneHits prunes hits older than the cutoff from a cacheHit slice.
   106  func pruneHits(hits []cacheHit, cutoff time.Time) []cacheHit {
   107  	toRemove := 0
   108  	for i := 0; i < len(hits); i++ {
   109  		if hits[i].staticTime.Before(cutoff) {
   110  			toRemove++
   111  		} else {
   112  			break // no more older than this
   113  		}
   114  	}
   115  	return hits[toRemove:]
   116  }
   117  
   118  // Prune prunes the whole hit tracker of hits which are already too far in the
   119  // past. Specific Hits are usually pruned when they are added but this makes
   120  // sure we also prune those hits for datasources which are hit infrequently.  To
   121  // make sure this doesn't block downloads, we set a flag to indicate that a
   122  // pruning process is going on. In that case, ReportHit won't return true and
   123  // the cache will therefore not cache any new entries while the pruning is
   124  // happening. Assuming that we serve 100,000 unique sectors per hour and we look
   125  // back 24 hours to decide whether to cache an entry, we end up with 2.4 million
   126  // entries in the tracker. Empirical testing showed that this takes around 5
   127  // seconds to prune.
   128  func (ht *cacheHitTracker) Prune() {
   129  	// Set the flag to indicate that we are currently pruning.
   130  	ht.mu.Lock()
   131  	if ht.pruning {
   132  		ht.mu.Unlock()
   133  		return // Already pruning
   134  	}
   135  	ht.pruning = true
   136  	ht.mu.Unlock()
   137  
   138  	cutoff := time.Now().Add(-ht.staticDuration)
   139  	for dsid, hitsPerSection := range ht.hits {
   140  		for section, hits := range hitsPerSection {
   141  			hits = pruneHits(hits, cutoff)
   142  			if len(hits) == 0 {
   143  				delete(hitsPerSection, section)
   144  			}
   145  		}
   146  		if len(hitsPerSection) == 0 {
   147  			delete(ht.hits, dsid)
   148  		}
   149  	}
   150  
   151  	// Unset the flag again.
   152  	ht.mu.Lock()
   153  	ht.pruning = false
   154  	ht.mu.Unlock()
   155  }
   156  
   157  // ReportHit adds a cache hit to the tracker. The return value indicates whether
   158  // enough hits happened within a specified timeframe for the data to be cached.
   159  func (ht *cacheHitTracker) ReportHit(dsid skymodules.DataSourceID, sectionID uint64) bool {
   160  	ht.mu.Lock()
   161  	defer ht.mu.Unlock()
   162  
   163  	// Always return 'false' while we prune the hit tracker. This should
   164  	// only take a few seconds for multiple million datasources.
   165  	if ht.pruning {
   166  		return false
   167  	}
   168  
   169  	// Get past hits.
   170  	hitsPerSection, exists := ht.hits[dsid]
   171  	if !exists {
   172  		// Init
   173  		hitsPerSection = make(map[uint64][]cacheHit)
   174  		ht.hits[dsid] = hitsPerSection
   175  	}
   176  
   177  	// Get hits for this section.
   178  	hits := hitsPerSection[sectionID]
   179  
   180  	// Append new one.
   181  	now := time.Now()
   182  	hits = append(hits, cacheHit{
   183  		staticTime: now,
   184  	})
   185  
   186  	// We only need to keep staticMinHits so we remove any additional old
   187  	// hits we have first.
   188  	if uint(len(hits)) > ht.staticMinHits {
   189  		hits = hits[uint(len(hits))-ht.staticMinHits:]
   190  	}
   191  
   192  	// Remove all hits that happened more than the specified duration ago.
   193  	cutoff := now.Add(-ht.staticDuration)
   194  	hits = pruneHits(hits, cutoff)
   195  
   196  	// If non remain, clear the map.
   197  	if len(hits) == 0 {
   198  		delete(ht.hits[dsid], sectionID)
   199  		if len(ht.hits[dsid]) == 0 {
   200  			delete(ht.hits, dsid)
   201  		}
   202  		return false
   203  	}
   204  
   205  	// Otherwise update the map.
   206  	ht.hits[dsid][sectionID] = hits
   207  
   208  	// If enough remain, return true.
   209  	return uint(len(hits)) >= ht.staticMinHits
   210  }
   211  
   212  // freeSection removes a section from the datasource and deletes it from disk.
   213  // It also returns the deleted files length and whether it was the last section.
   214  func (ds *cachedDataSource) freeSection(index uint64) (int64, bool, error) {
   215  	_, exists := ds.sections[index]
   216  	if !exists {
   217  		// already freed.
   218  		return 0, true, nil
   219  	}
   220  	delete(ds.sections, index)
   221  
   222  	// Remove the section from disk.
   223  	length, err := ds.staticLRU.staticRemoveCacheFile(ds.staticID, index)
   224  	return length, len(ds.sections) == 0, err
   225  }
   226  
   227  // get returns some cached data from a datasource. If the data isn't available,
   228  // it returns false.
   229  func (ds *cachedDataSource) get(dsid skymodules.DataSourceID, sectionIndex uint64) (_ []byte, _ bool, err error) {
   230  	_, exists := ds.sections[sectionIndex]
   231  	if !exists {
   232  		return nil, false, nil
   233  	}
   234  
   235  	// Open the cache file.
   236  	cacheFile, err := ds.staticLRU.staticOpenCacheFile(dsid, sectionIndex)
   237  	if err != nil {
   238  		return nil, false, err
   239  	}
   240  	defer func() {
   241  		err = errors.Compose(err, cacheFile.Close())
   242  	}()
   243  
   244  	// Read the section. Do so by fetching the size of the file and then
   245  	// pre-allocating the memory instead of using io.ReadAll to reduce
   246  	// allocations.
   247  	fi, err := cacheFile.Stat()
   248  	if err != nil {
   249  		return nil, false, err
   250  	}
   251  	data := make([]byte, fi.Size())
   252  	_, err = io.ReadFull(cacheFile, data)
   253  	if err != nil {
   254  		return nil, false, err
   255  	}
   256  
   257  	// Decrypt the data.
   258  	cc, err := chacha20.NewUnauthenticatedCipher(ds.staticKey, dsid[:chacha20.NonceSizeX])
   259  	if err != nil {
   260  		return nil, false, err
   261  	}
   262  	cc.XORKeyStream(data, data)
   263  	return data, true, nil
   264  }
   265  
   266  // newSection creates a new section within the datasource.
   267  func (ds *cachedDataSource) newSection(index uint64) {
   268  	_, exists := ds.sections[index]
   269  	if exists {
   270  		build.Critical("adding duplicate section to data source")
   271  	}
   272  	ds.sections[index] = struct{}{}
   273  }
   274  
   275  // put places some data to be cached into a data source at the given section
   276  // index.
   277  func (ds *cachedDataSource) put(dsid skymodules.DataSourceID, sectionIndex uint64, dd *downloadedData) (_ bool, _ int, err error) {
   278  	// Check if the section is already cached.
   279  	_, exists := ds.sections[sectionIndex]
   280  	if exists {
   281  		return false, 0, nil
   282  	}
   283  
   284  	// Grab a buffer from the pool and return it when done.
   285  	buf := staticLRUBufferPool.Get()
   286  	defer staticLRUBufferPool.Put(buf)
   287  
   288  	// Marshal the data.
   289  	err = dd.Marshal(buf)
   290  	if err != nil {
   291  		return false, 0, err
   292  	}
   293  	data := buf.Bytes()
   294  
   295  	// Encrypt the data.
   296  	cc, err := chacha20.NewUnauthenticatedCipher(ds.staticKey, dsid[:chacha20.NonceSizeX])
   297  	if err != nil {
   298  		return false, 0, err
   299  	}
   300  	data = append([]byte{}, data...) // deep copy to not modify input
   301  	cc.XORKeyStream(data, data)
   302  
   303  	// Open the cache file.
   304  	cacheFile, err := ds.staticLRU.staticOpenCacheFile(dsid, sectionIndex)
   305  	if err != nil {
   306  		return false, 0, err
   307  	}
   308  	// Cleanup.
   309  	defer func() {
   310  		err = errors.Compose(err, cacheFile.Close())
   311  		if err != nil {
   312  			ds.freeSection(sectionIndex)
   313  		}
   314  	}()
   315  	// Create the section and write it to the file.
   316  	ds.newSection(sectionIndex)
   317  	_, err = cacheFile.Write(data)
   318  	if err != nil {
   319  		return false, 0, err
   320  	}
   321  	return true, len(data), nil
   322  }
   323  
   324  // newPersistedLRU creates a new LRU at the given root path with the given max
   325  // size.
   326  func newPersistedLRU(path string, maxSize uint64, hitsBeforeCache uint, duration time.Duration) (*persistedLRU, error) {
   327  	// Remove root dir to prune any existing cached elements.
   328  	if err := os.RemoveAll(path); err != nil {
   329  		return nil, err
   330  	}
   331  	// Create root dir.
   332  	if err := os.MkdirAll(path, skymodules.DefaultDirPerm); err != nil {
   333  		return nil, err
   334  	}
   335  	lru := &persistedLRU{
   336  		dataSources:        make(map[skymodules.DataSourceID]*cachedDataSource),
   337  		lruElements:        make(map[skymodules.DataSourceID]map[uint64]*list.Element),
   338  		staticHitTracker:   newCacheHitTracker(hitsBeforeCache, duration),
   339  		staticMaxCacheSize: int64(maxSize),
   340  		staticLRU:          list.New(),
   341  		staticPath:         path,
   342  		staticDone:         make(chan struct{}),
   343  		staticSalt:         fastrand.Bytes(crypto.HashSize),
   344  	}
   345  	// Spin up pruning thread.
   346  	lru.staticWG.Add(1)
   347  	go func() {
   348  		defer lru.staticWG.Done()
   349  		t := time.NewTicker(duration / 2)
   350  		for {
   351  			select {
   352  			case <-t.C:
   353  			case <-lru.staticDone:
   354  				return
   355  			}
   356  			lru.staticHitTracker.Prune()
   357  		}
   358  	}()
   359  	return lru, nil
   360  }
   361  
   362  // NumCachedSections returns the number of sections currently in the cache for a
   363  // given datasource.
   364  func (lru *persistedLRU) NumCachedSections(dsid skymodules.DataSourceID) uint64 {
   365  	ds, exists := lru.managedAcquireDataSource(dsid)
   366  	if !exists {
   367  		return 0
   368  	}
   369  	defer lru.managedReturnDataSource(ds)
   370  	return uint64(len(ds.sections))
   371  }
   372  
   373  // Close closes the persisted LRU and stops the background pruning thread.
   374  func (lru *persistedLRU) Close() error {
   375  	close(lru.staticDone)
   376  	lru.staticWG.Wait()
   377  	return nil
   378  }
   379  
   380  // managedAcquireCreateDataSource is a helper method to correctly acquire or
   381  // create and acquire a datasource.
   382  func (lru *persistedLRU) managedAcquireCreateDataSource(dsid skymodules.DataSourceID) *cachedDataSource {
   383  	for {
   384  		// Acquire LRU lock and get the datasource or create it if it
   385  		// doesn't exist.
   386  		lru.mu.Lock()
   387  		ds, exists := lru.dataSources[dsid]
   388  		if !exists {
   389  			ds = lru.staticNewCachedDataSource(dsid)
   390  			lru.dataSources[dsid] = ds
   391  		}
   392  		lru.mu.Unlock()
   393  
   394  		// Lock the datasource.
   395  		ds.mu.Lock()
   396  
   397  		// Lock the LRU again to see if the datasource is still in the
   398  		// map. It might have been removed since we unlocked the LRU
   399  		// lock. This is unlikely but possible.
   400  		lru.mu.Lock()
   401  		ds2, exists := lru.dataSources[dsid]
   402  		lru.mu.Unlock()
   403  		if !exists || ds2 != ds {
   404  			// If the datasource in the map doesn't match the one we
   405  			// locked, we unlock it and try again. Either it was
   406  			// deleted or some other thread added one in the
   407  			// meantime.
   408  			ds.mu.Unlock()
   409  			continue
   410  		}
   411  		// Successfully locked the data source.
   412  		return ds
   413  	}
   414  }
   415  
   416  // managedAcquireDataSource is a helper method to correctly lock a datasource.
   417  func (lru *persistedLRU) managedAcquireDataSource(dsid skymodules.DataSourceID) (*cachedDataSource, bool) {
   418  	// Acquire LRU lock and check if the data source exists.
   419  	lru.mu.Lock()
   420  	ds, exists := lru.dataSources[dsid]
   421  	lru.mu.Unlock()
   422  	if !exists {
   423  		// Nothing to do if it doesn't.
   424  		return nil, false
   425  	}
   426  
   427  	// Lock the data source.
   428  	ds.mu.Lock()
   429  
   430  	// Lock the LRU again to see if the datasource is still in the
   431  	// map. It might have been removed since we unlocked the LRU
   432  	// lock. This is unlikely but possible.
   433  	lru.mu.Lock()
   434  	ds2, exists := lru.dataSources[dsid]
   435  	lru.mu.Unlock()
   436  	if !exists || ds2 != ds {
   437  		// If the datasource in the map doesn't match the one we
   438  		// locked, we unlock it and try again. Either it was
   439  		// deleted or some other thread added one in the
   440  		// meantime.
   441  		ds.mu.Unlock()
   442  		return nil, false
   443  	}
   444  	// Successfully locked the data source.
   445  	return ds, true
   446  }
   447  
   448  // managedDeleteDataSource is a helper method to correctly unlock and delete a
   449  // datasoure from the LRU.
   450  func (lru *persistedLRU) managedDeleteDataSource(ds *cachedDataSource) {
   451  	lru.mu.Lock()
   452  	ds, exists := lru.dataSources[ds.staticID]
   453  	if !exists {
   454  		lru.mu.Unlock()
   455  		build.Critical("trying to delete already deleted data source")
   456  		return
   457  	}
   458  	delete(lru.dataSources, ds.staticID)
   459  	lru.mu.Unlock()
   460  	ds.mu.Unlock()
   461  }
   462  
   463  // managedReturnDataSource is a helper method to correctly unlock a datasource.
   464  func (lru *persistedLRU) managedReturnDataSource(ds *cachedDataSource) {
   465  	lru.mu.Lock()
   466  	ds, exists := lru.dataSources[ds.staticID]
   467  	lru.mu.Unlock()
   468  	if !exists {
   469  		build.Critical("no data source with that id")
   470  	}
   471  	ds.mu.Unlock()
   472  }
   473  
   474  // managedPruneLRU prunes the least recently used element from the persistedLRU.
   475  func (lru *persistedLRU) managedPruneLRU() (int64, bool, error) {
   476  	lru.mu.Lock()
   477  	ele := lru.staticLRU.Back()
   478  	if ele == nil {
   479  		lru.mu.Unlock()
   480  		return 0, false, nil
   481  	}
   482  	lru.staticLRU.Remove(ele)
   483  
   484  	toPrune := ele.Value.(lruElement)
   485  
   486  	// Cleanup the lru's maps first.
   487  	sections, exists := lru.lruElements[toPrune.staticDSID]
   488  	if exists {
   489  		// Delete the element in the inner map.
   490  		delete(sections, toPrune.staticSectionIndex)
   491  	}
   492  	if len(sections) == 0 {
   493  		delete(lru.lruElements, toPrune.staticDSID)
   494  	}
   495  
   496  	// Then find the datasource.
   497  	lru.mu.Unlock()
   498  	ds, exists := lru.managedAcquireDataSource(toPrune.staticDSID)
   499  	if !exists {
   500  		// no ds
   501  		return 0, true, nil
   502  	}
   503  	length, deleted, err := ds.freeSection(toPrune.staticSectionIndex)
   504  
   505  	// Delete the datasource if it was marked as deleted.
   506  	if deleted {
   507  		lru.managedDeleteDataSource(ds)
   508  	} else {
   509  		lru.managedReturnDataSource(ds)
   510  	}
   511  	return length, true, err
   512  }
   513  
   514  // staticNewCachedDataSource creates a new cachedDataSource object.
   515  func (lru *persistedLRU) staticNewCachedDataSource(id skymodules.DataSourceID) *cachedDataSource {
   516  	return &cachedDataSource{
   517  		staticID:  id,
   518  		staticKey: fastrand.Bytes(chacha20poly1305.KeySize),
   519  		staticLRU: lru,
   520  		sections:  make(map[uint64]struct{}),
   521  	}
   522  }
   523  
   524  // dataSourceIDToPath returns the datasource's cache folder for a given cache
   525  // root.
   526  func dataSourceIDToPath(root string, dsid skymodules.DataSourceID, salt []byte) string {
   527  	// Apply salt. The salt will protect us from malicious actors trying to
   528  	// download skylinks in a way that the directory tree on disk becomes
   529  	// unbalanced. By applying a salt to the path, they can't predict where
   530  	// the cache for the skylink is going to leave, therefore randomising
   531  	// the location.
   532  	dsid = skymodules.DataSourceID(crypto.HashAll(dsid, salt))
   533  
   534  	// Encode to hex.
   535  	s := hex.EncodeToString(dsid[:])
   536  
   537  	// Using a depth of 2 - approach will result in 65536 folders on the
   538  	// bottom layer of the tree and twice that in total. Assuming a 4kib
   539  	// block size of the filesystem, that's and approximately 500 mib folder
   540  	// overhead if all the folders exist. If we decide to increase the depth
   541  	// we might want to add support for deleting empty folders again but
   542  	// that would add some locking complexity.
   543  	return filepath.Join(root, s[0:2], s[2:4], s[4:])
   544  }
   545  
   546  // staticDataSourceIDToPath is a helper method to get the path for a given
   547  // datasource and section.
   548  func (lru *persistedLRU) staticDataSourceIDToPath(dsid skymodules.DataSourceID, sectionIndex uint64) string {
   549  	path := dataSourceIDToPath(lru.staticPath, dsid, lru.staticSalt)
   550  	return filepath.Join(path, fmt.Sprint(sectionIndex)+".dat")
   551  }
   552  
   553  // staticOpenCacheFile is a helper method to open a cache file for a given
   554  // datasource and section.
   555  func (lru *persistedLRU) staticOpenCacheFile(dsid skymodules.DataSourceID, sectionIndex uint64) (*os.File, error) {
   556  	path := lru.staticDataSourceIDToPath(dsid, sectionIndex)
   557  	dir := filepath.Dir(path)
   558  	if err := os.MkdirAll(dir, skymodules.DefaultDirPerm); err != nil {
   559  		return nil, err
   560  	}
   561  	return os.OpenFile(path, os.O_RDWR|os.O_CREATE, skymodules.DefaultFilePerm)
   562  }
   563  
   564  // staticRemoveCacheFile removes a cache file from disk and returns its size.
   565  func (lru *persistedLRU) staticRemoveCacheFile(dsid skymodules.DataSourceID, sectionIndex uint64) (int64, error) {
   566  	path := lru.staticDataSourceIDToPath(dsid, sectionIndex)
   567  	fi, err := os.Stat(path)
   568  	if err != nil {
   569  		return 0, err
   570  	}
   571  	return fi.Size(), os.Remove(path)
   572  }
   573  
   574  // Get tries to fetch data from the cache. If the data is not cached, false is
   575  // returned.
   576  func (lru *persistedLRU) Get(dsid skymodules.DataSourceID, sectionIndex uint64) (*downloadedData, bool, error) {
   577  	ds, exists := lru.managedAcquireDataSource(dsid)
   578  	if !exists {
   579  		return nil, false, nil
   580  	}
   581  	data, found, err := ds.get(dsid, sectionIndex)
   582  	if err != nil {
   583  		lru.managedReturnDataSource(ds)
   584  		return nil, false, err
   585  	}
   586  	lru.managedReturnDataSource(ds)
   587  
   588  	if found {
   589  		// Refresh the cache if we got the data cached.
   590  		lru.managedRefreshCachedEntry(dsid, sectionIndex)
   591  
   592  		// Unmarshal it.
   593  		var dd downloadedData
   594  		return &dd, true, dd.Unmarshal(data)
   595  	}
   596  	return nil, false, nil
   597  }
   598  
   599  // Put adds a new section to the cache.
   600  func (lru *persistedLRU) Put(dsid skymodules.DataSourceID, sectionIndex uint64, dd *downloadedData) error {
   601  	// Check the hit tracker to see if we should actually cache the section.
   602  	if cache := lru.staticHitTracker.ReportHit(dsid, sectionIndex); !cache {
   603  		return nil // don't cache yet
   604  	}
   605  
   606  	// Get the cached datasource or create if possible.
   607  	ds := lru.managedAcquireCreateDataSource(dsid)
   608  
   609  	// Add the section to the source.
   610  	added, n, err := ds.put(dsid, sectionIndex, dd)
   611  	if err != nil {
   612  		if added {
   613  			lru.managedDeleteDataSource(ds)
   614  		} else {
   615  			lru.managedReturnDataSource(ds)
   616  		}
   617  		return err
   618  	}
   619  
   620  	// Unlock ds.
   621  	lru.managedReturnDataSource(ds)
   622  
   623  	// If it was added, we add the length of the added data to the sum.
   624  	if added {
   625  		lru.managedAddCachedEntry(dsid, sectionIndex, int64(n))
   626  	}
   627  
   628  	// Update the lru.
   629  	lru.managedTryPruneData()
   630  	return nil
   631  }
   632  
   633  // managedAddCachedEntry adds a new entry to cache to the LRU.
   634  func (lru *persistedLRU) managedAddCachedEntry(dsid skymodules.DataSourceID, sectionIndex uint64, size int64) {
   635  	lru.mu.Lock()
   636  	defer lru.mu.Unlock()
   637  
   638  	elements, exists := lru.lruElements[dsid]
   639  	if !exists {
   640  		elements = make(map[uint64]*list.Element)
   641  		lru.lruElements[dsid] = elements
   642  	}
   643  	_, exists = elements[sectionIndex]
   644  	if !exists {
   645  		// Push a new element.
   646  		elements[sectionIndex] = lru.staticLRU.PushFront(lruElement{
   647  			staticDSID:         dsid,
   648  			staticSectionIndex: sectionIndex,
   649  		})
   650  		// Increment the cachedSize.
   651  		lru.cachedSize += size
   652  	}
   653  }
   654  
   655  // managedRefreshCachedEntry moves a cached element to the front of the LRU.
   656  func (lru *persistedLRU) managedRefreshCachedEntry(dsid skymodules.DataSourceID, sectionIndex uint64) {
   657  	lru.mu.Lock()
   658  	defer lru.mu.Unlock()
   659  
   660  	elements, exists := lru.lruElements[dsid]
   661  	if !exists {
   662  		return
   663  	}
   664  	el, exists := elements[sectionIndex]
   665  	if exists {
   666  		// Move element to the front.
   667  		lru.staticLRU.MoveToFront(el)
   668  	}
   669  }
   670  
   671  // managedTryPruneData checks the current cache size and if necessary, prunes
   672  // it. To avoid holding a lock while doing disk i/o, it will assume that the
   673  // pruning is successful by subtracting the amount of data to prune from the
   674  // cache size right away. After the pruning it will adjust the cache size again
   675  // using the actual amount of pruned data.
   676  func (lru *persistedLRU) managedTryPruneData() error {
   677  	// Figure out how much data we need to prune and assume that it is
   678  	// pruned.
   679  	lru.mu.Lock()
   680  	toPrune := lru.cachedSize - lru.staticMaxCacheSize
   681  	if toPrune <= 0 {
   682  		lru.mu.Unlock()
   683  		return nil
   684  	}
   685  	lru.cachedSize -= toPrune
   686  	lru.mu.Unlock()
   687  
   688  	// Prune at least toPrune data. If we encounter an error we break but we
   689  	// can't return right away since we still need to adjust the cache size.
   690  	var err error
   691  	for toPrune > 0 {
   692  		var pruned int64
   693  		var more bool
   694  		pruned, more, err = lru.managedPruneLRU()
   695  		if err != nil {
   696  			break
   697  		}
   698  		if pruned == 0 && !more {
   699  			break
   700  		}
   701  		toPrune -= pruned
   702  	}
   703  
   704  	// Adjust the cachedSize now that we know how much data we pruned
   705  	// exactly.
   706  	lru.mu.Lock()
   707  	defer lru.mu.Unlock()
   708  	if toPrune != 0 {
   709  		lru.cachedSize += toPrune
   710  		if lru.cachedSize < 0 {
   711  			lru.cachedSize = 0
   712  			build.Critical("managedAddCachedData: negative cachedSize after prune")
   713  		}
   714  	}
   715  	return err
   716  }
   717  
   718  // noopLRU implements the PersistedLRU interface with only no-ops.
   719  type noopLRU struct{}
   720  
   721  // newNoOpLRU creates a new noopLRU.
   722  func newNoOpLRU() PersistedLRU {
   723  	return &noopLRU{}
   724  }
   725  
   726  // NumCachedSections always returns 0.
   727  func (lru *noopLRU) NumCachedSections(dsid skymodules.DataSourceID) uint64 {
   728  	return 0
   729  }
   730  
   731  // Get is a no-op.
   732  func (lru *noopLRU) Get(dsid skymodules.DataSourceID, sectionIndex uint64) (*downloadedData, bool, error) {
   733  	return nil, false, nil
   734  }
   735  
   736  // Put is a no-op.
   737  func (lru *noopLRU) Put(dsid skymodules.DataSourceID, sectionIndex uint64, data *downloadedData) error {
   738  	return nil
   739  }