github.com/keybase/client/go@v0.0.0-20240309051027-028f7c731f8b/kbfs/libgit/on_demand_storer.go (about)

     1  // Copyright 2017 Keybase Inc. All rights reserved.
     2  // Use of this source code is governed by a BSD
     3  // license that can be found in the LICENSE file.
     4  
     5  package libgit
     6  
     7  import (
     8  	lru "github.com/hashicorp/golang-lru"
     9  	"github.com/pkg/errors"
    10  	"gopkg.in/src-d/go-git.v4/plumbing"
    11  	"gopkg.in/src-d/go-git.v4/plumbing/storer"
    12  	"gopkg.in/src-d/go-git.v4/storage"
    13  )
    14  
    15  // OnDemandStorer is a wrapper around a storage.Storer that reads
    16  // encoded objects from disk only when the data is needed, to avoid
    17  // pulling too much data into memory.
    18  type OnDemandStorer struct {
    19  	storage.Storer
    20  	recentCache *lru.Cache
    21  }
    22  
    23  var _ storage.Storer = (*OnDemandStorer)(nil)
    24  var _ storer.DeltaObjectStorer = (*OnDemandStorer)(nil)
    25  
    26  // NewOnDemandStorer constructs an on-demand storage layer on top of
    27  // an existing `Storer`.
    28  func NewOnDemandStorer(s storage.Storer) (*OnDemandStorer, error) {
    29  	// Track a small number of recent in-memory objects, to improve
    30  	// performance without impacting memory too much.
    31  	//
    32  	// LRU is very helpful here because of the way delta compression
    33  	// works. It first sorts the objects by type and descending size,
    34  	// and then compares each object to a sliding window of previous
    35  	// objects to find a good match. By default in git, the sliding
    36  	// window for compression is 10, and it's good to have a
    37  	// slightly larger cache size than that to avoid thrashing.
    38  	//
    39  	// To avoid memory pressure, it might be nice to additionally
    40  	// consider capping the total size of this cache (e.g., with
    41  	// github.com/keybase/cache). However, since the set in use is
    42  	// based on the sliding window, it seems like that should be the
    43  	// limiting factor. Eventually we might hit some repo where
    44  	// there's a set of large objects that can overrun memory, but at
    45  	// the limit that could be any two objects, and then the
    46  	// compression algorithm in go-git is worthless.  So for now,
    47  	// let's just limit by number of entries, and add size-limits
    48  	// later if needed.
    49  	recentCache, err := lru.New(25)
    50  	if err != nil {
    51  		return nil, err
    52  	}
    53  	return &OnDemandStorer{s, recentCache}, nil
    54  }
    55  
    56  // EncodedObject implements the storage.Storer interface for OnDemandStorer.
    57  func (ods *OnDemandStorer) EncodedObject(
    58  	ot plumbing.ObjectType, hash plumbing.Hash) (
    59  	plumbing.EncodedObject, error) {
    60  	o := &onDemandObject{
    61  		s:           ods.Storer,
    62  		hash:        hash,
    63  		objType:     ot,
    64  		size:        -1,
    65  		recentCache: ods.recentCache,
    66  	}
    67  	// If the object is missing, we need to return an error for that
    68  	// here.  But don't read all the object data from disk by calling
    69  	// `Storer.EncodedObject()` or `o.cache()`.  Instead use a
    70  	// KBFS-specific `HasEncodedObject()` method that just tells us
    71  	// whether or not the object exists.
    72  	err := ods.Storer.HasEncodedObject(hash)
    73  	if err != nil {
    74  		return nil, err
    75  	}
    76  
    77  	return o, nil
    78  }
    79  
    80  // DeltaObject implements the storer.DeltaObjectStorer interface for
    81  // OnDemandStorer.
    82  func (ods *OnDemandStorer) DeltaObject(
    83  	ot plumbing.ObjectType, hash plumbing.Hash) (
    84  	plumbing.EncodedObject, error) {
    85  	edos, ok := ods.Storer.(storer.DeltaObjectStorer)
    86  	if !ok {
    87  		return nil, errors.New("Not a delta storer")
    88  	}
    89  	o := &onDemandDeltaObject{
    90  		s:           edos,
    91  		hash:        hash,
    92  		objType:     ot,
    93  		size:        -1,
    94  		recentCache: ods.recentCache,
    95  	}
    96  	// Need to see if this is a delta object, which means reading all
    97  	// the data.
    98  	_, err := o.cache()
    99  	_, notDelta := err.(notDeltaError)
   100  	if notDelta {
   101  		return ods.EncodedObject(ot, hash)
   102  	} else if err != nil {
   103  		return nil, err
   104  	}
   105  	return o, nil
   106  }