github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/graveler/committed/meta_range_manager.go (about)

     1  package committed
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"errors"
     7  	"fmt"
     8  
     9  	"github.com/treeverse/lakefs/pkg/graveler"
    10  )
    11  
    12  type Params struct {
    13  	// MinRangeSizeBytes is the smallest size for splitting a range partition as a result
    14  	// of adding a record.  Smaller ranges are still possible due to re-using an existing
    15  	MinRangeSizeBytes uint64
    16  	// MaxRangeSizeBytes is the largest size of a range partition.  In practice the range
    17  	// is split only after an additional record.
    18  	MaxRangeSizeBytes uint64
    19  	// RangeSizeEntriesRaggedness allows raggedness in splitting range partitions.  It is
    20  	// the expected number of records after MinRangeSizeBytes at which to split the range
    21  	// -- ranges are split at the first key with hash divisible by this raggedness.
    22  	RangeSizeEntriesRaggedness float64
    23  	// MaxUploaders is the maximal number of uploaders to use in a single metarange writer.
    24  	MaxUploaders int
    25  }
    26  
    27  type metaRangeManager struct {
    28  	params       Params
    29  	metaManager  RangeManager // For metaranges
    30  	rangeManager RangeManager // For ranges
    31  }
    32  
    33  var ErrNeedBatchClosers = errors.New("need at least 1 batch uploaded")
    34  
    35  func NewMetaRangeManager(params Params, metaManager, rangeManager RangeManager) (MetaRangeManager, error) {
    36  	if params.MaxUploaders < 1 {
    37  		return nil, fmt.Errorf("only %d async closers: %w", params.MaxUploaders, ErrNeedBatchClosers)
    38  	}
    39  	return &metaRangeManager{
    40  		params:       params,
    41  		metaManager:  metaManager,
    42  		rangeManager: rangeManager,
    43  	}, nil
    44  }
    45  
    46  func (m *metaRangeManager) Exists(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (bool, error) {
    47  	return m.metaManager.Exists(ctx, Namespace(ns), ID(id))
    48  }
    49  
    50  // GetValue finds the matching graveler.ValueRecord in the MetaRange with the rangeID
    51  func (m *metaRangeManager) GetValue(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID, key graveler.Key) (*graveler.ValueRecord, error) {
    52  	// Fetch range containing key.
    53  	rng, err := m.GetRangeByKey(ctx, ns, id, key)
    54  	if err != nil {
    55  		return nil, err
    56  	}
    57  
    58  	r, err := m.rangeManager.GetValue(ctx, Namespace(ns), rng.ID, Key(key))
    59  	if err != nil {
    60  		return nil, fmt.Errorf("get value in range %s of %s for %s: %w", rng.ID, id, key, err)
    61  	}
    62  	value, err := UnmarshalValue(r.Value)
    63  	if err != nil {
    64  		return nil, err
    65  	}
    66  	return &graveler.ValueRecord{
    67  		Key:   key,
    68  		Value: value,
    69  	}, nil
    70  }
    71  
    72  func (m *metaRangeManager) GetRangeByKey(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID, key graveler.Key) (*Range, error) {
    73  	v, err := m.metaManager.GetValueGE(ctx, Namespace(ns), ID(id), Key(key))
    74  	if errors.Is(err, ErrNotFound) {
    75  		return nil, err
    76  	}
    77  	if err != nil {
    78  		return nil, fmt.Errorf("find range in %s: %w", id, err)
    79  	}
    80  
    81  	gv, err := UnmarshalValue(v.Value)
    82  	if err != nil {
    83  		return nil, fmt.Errorf("unmarshal value for %s: %w", string(v.Key), err)
    84  	}
    85  
    86  	rng, err := UnmarshalRange(gv.Data)
    87  	if err != nil {
    88  		return nil, fmt.Errorf("unmarshal %s: %w", string(v.Key), err)
    89  	}
    90  
    91  	if !(bytes.Compare(rng.MinKey, key) <= 0 && bytes.Compare(key, rng.MaxKey) <= 0) {
    92  		return nil, ErrNotFound
    93  	}
    94  
    95  	rng.ID = ID(gv.Identity)
    96  	return &rng, nil
    97  }
    98  
    99  func (m *metaRangeManager) NewWriter(ctx context.Context, ns graveler.StorageNamespace, metadata graveler.Metadata) MetaRangeWriter {
   100  	return NewGeneralMetaRangeWriter(ctx, m.rangeManager, m.metaManager, &m.params, Namespace(ns), metadata)
   101  }
   102  
   103  func (m *metaRangeManager) NewMetaRangeIterator(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (Iterator, error) {
   104  	if id == "" {
   105  		return NewEmptyIterator(), nil
   106  	}
   107  	rangesIt, err := m.metaManager.NewRangeIterator(ctx, Namespace(ns), ID(id))
   108  	if err != nil {
   109  		return nil, fmt.Errorf("manage metarange %s: %w", id, err)
   110  	}
   111  	return NewIterator(ctx, m.rangeManager, Namespace(ns), rangesIt), nil
   112  }
   113  
   114  func (m *metaRangeManager) GetMetaRangeURI(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (string, error) {
   115  	return m.metaManager.GetURI(ctx, Namespace(ns), ID(id))
   116  }
   117  
   118  func (m *metaRangeManager) GetRangeURI(ctx context.Context, ns graveler.StorageNamespace, id graveler.RangeID) (string, error) {
   119  	return m.rangeManager.GetURI(ctx, Namespace(ns), ID(id))
   120  }