github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/storage/pebble/config.go (about)

     1  package pebble
     2  
     3  import (
     4  	"github.com/cockroachdb/pebble"
     5  	"github.com/cockroachdb/pebble/bloom"
     6  )
     7  
     8  // DefaultPebbleOptions returns an optimized set of pebble options.
     9  // This is mostly copied form pebble's nightly performance benchmark.
    10  func DefaultPebbleOptions(cache *pebble.Cache, comparer *pebble.Comparer) *pebble.Options {
    11  	opts := &pebble.Options{
    12  		Cache:              cache,
    13  		Comparer:           comparer,
    14  		FormatMajorVersion: pebble.FormatNewest,
    15  
    16  		// Soft and hard limits on read amplificaction of L0 respectfully.
    17  		L0CompactionThreshold: 2,
    18  		L0StopWritesThreshold: 1000,
    19  
    20  		// When the maximum number of bytes for a level is exceeded, compaction is requested.
    21  		LBaseMaxBytes: 64 << 20, // 64 MB
    22  		Levels:        make([]pebble.LevelOptions, 7),
    23  		MaxOpenFiles:  16384,
    24  
    25  		// Writes are stopped when the sum of the queued memtable sizes exceeds MemTableStopWritesThreshold*MemTableSize.
    26  		MemTableSize:                64 << 20,
    27  		MemTableStopWritesThreshold: 4,
    28  
    29  		// The default is 1.
    30  		MaxConcurrentCompactions: func() int { return 4 },
    31  	}
    32  
    33  	for i := 0; i < len(opts.Levels); i++ {
    34  		l := &opts.Levels[i]
    35  		// The default is 4KiB (uncompressed), which is too small
    36  		// for good performance (esp. on stripped storage).
    37  		l.BlockSize = 32 << 10       // 32 KB
    38  		l.IndexBlockSize = 256 << 10 // 256 KB
    39  
    40  		// The bloom filter speedsup our SeekPrefixGE by skipping
    41  		// sstables that do not contain the prefix
    42  		l.FilterPolicy = bloom.FilterPolicy(MinLookupKeyLen)
    43  		l.FilterType = pebble.TableFilter
    44  
    45  		if i > 0 {
    46  			// L0 starts at 2MiB, each level is 2x the previous.
    47  			l.TargetFileSize = opts.Levels[i-1].TargetFileSize * 2
    48  		}
    49  		l.EnsureDefaults()
    50  	}
    51  
    52  	// TODO(rbtz): benchmark with and without bloom filters on L6
    53  	// opts.Levels[6].FilterPolicy = nil
    54  
    55  	// Splitting sstables during flush allows increased compaction flexibility and concurrency when those
    56  	// tables are compacted to lower levels.
    57  	opts.FlushSplitBytes = opts.Levels[0].TargetFileSize
    58  	opts.EnsureDefaults()
    59  
    60  	return opts
    61  }