github.com/coocood/badger@v1.5.1-0.20200528065104-c02ac3616d04/options.go (about)

     1  /*
     2   * Copyright 2017 Dgraph Labs, Inc. and Contributors
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package badger
    18  
    19  import (
    20  	"github.com/coocood/badger/options"
    21  )
    22  
    23  // NOTE: Keep the comments in the following to 75 chars width, so they
    24  // format nicely in godoc.
    25  
    26  // Options are params for creating DB object.
    27  //
    28  // This package provides DefaultOptions which contains options that should
    29  // work for most applications. Consider using that as a starting point before
    30  // customizing it for your own needs.
    31  type Options struct {
    32  	// 1. Mandatory flags
    33  	// -------------------
    34  	// Directory to store the data in. Should exist and be writable.
    35  	Dir string
    36  	// Directory to store the value log in. Can be the same as Dir. Should
    37  	// exist and be writable.
    38  	ValueDir string
    39  
    40  	// 2. Frequently modified flags
    41  	// -----------------------------
    42  	// Sync all writes to disk. Setting this to true would slow down data
    43  	// loading significantly.
    44  	SyncWrites bool
    45  
    46  	// 3. Flags that user might want to review
    47  	// ----------------------------------------
    48  	// The following affect all levels of LSM tree.
    49  	MaxMemTableSize int64 // Each mem table is at most this size.
    50  	MaxTableSize    int64 // Each table file is at most this size.
    51  	// If value size >= this threshold, only store value offsets in tree.
    52  	// If set to 0, all values are stored in SST.
    53  	ValueThreshold int
    54  	// Maximum number of tables to keep in memory, before stalling.
    55  	NumMemtables int
    56  	// The following affect how we handle LSM tree L0.
    57  	// Maximum number of Level 0 tables before we start compacting.
    58  	NumLevelZeroTables int
    59  
    60  	// If we hit this number of Level 0 tables, we will stall until L0 is
    61  	// compacted away.
    62  	NumLevelZeroTablesStall int
    63  
    64  	MaxBlockCacheSize int64
    65  	MaxIndexCacheSize int64
    66  
    67  	// Maximum total size for L1.
    68  	LevelOneSize int64
    69  
    70  	// Size of single value log file.
    71  	ValueLogFileSize int64
    72  
    73  	// Max number of entries a value log file can hold (approximately). A value log file would be
    74  	// determined by the smaller of its file size and max entries.
    75  	ValueLogMaxEntries uint32
    76  
    77  	// Max number of value log files to keep before safely remove.
    78  	ValueLogMaxNumFiles int
    79  
    80  	// Number of compaction workers to run concurrently.
    81  	NumCompactors int
    82  
    83  	// Transaction start and commit timestamps are managed by end-user.
    84  	// A managed transaction can only set values by SetEntry with a non-zero version key.
    85  	ManagedTxns bool
    86  
    87  	// 4. Flags for testing purposes
    88  	// ------------------------------
    89  	VolatileMode bool
    90  	DoNotCompact bool // Stops LSM tree from compactions.
    91  
    92  	maxBatchCount int64 // max entries in batch
    93  	maxBatchSize  int64 // max batch size in bytes
    94  
    95  	// Open the DB as read-only. With this set, multiple processes can
    96  	// open the same Badger DB. Note: if the DB being opened had crashed
    97  	// before and has vlog data to be replayed, ReadOnly will cause Open
    98  	// to fail with an appropriate message.
    99  	ReadOnly bool
   100  
   101  	// Truncate value log to delete corrupt data, if any. Would not truncate if ReadOnly is set.
   102  	Truncate bool
   103  
   104  	TableBuilderOptions options.TableBuilderOptions
   105  
   106  	ValueLogWriteOptions options.ValueLogWriterOptions
   107  
   108  	CompactionFilterFactory func(targetLevel int, smallest, biggest []byte) CompactionFilter
   109  
   110  	CompactL0WhenClose bool
   111  }
   112  
   113  // CompactionFilter is an interface that user can implement to remove certain keys.
   114  type CompactionFilter interface {
   115  	// Filter is the method the compaction process invokes for kv that is being compacted. The returned decision
   116  	// indicates that the kv should be preserved, deleted or dropped in the output of this compaction run.
   117  	Filter(key, val, userMeta []byte) Decision
   118  
   119  	// Guards returns specifications that may splits the SST files
   120  	// A key is associated to a guard that has the longest matched Prefix.
   121  	Guards() []Guard
   122  }
   123  
   124  // Guard specifies when to finish a SST file during compaction. The rule is the following:
   125  // 1. The key must match the Prefix of the Guard, otherwise the SST should finish.
   126  // 2. If the key up to MatchLen is the different than the previous key and MinSize is reached, the SST should finish.
   127  type Guard struct {
   128  	Prefix   []byte
   129  	MatchLen int
   130  	MinSize  int64
   131  }
   132  
   133  // Decision is the type for compaction filter decision.
   134  type Decision int
   135  
   136  const (
   137  	// DecisionKeep indicates the entry should be reserved.
   138  	DecisionKeep Decision = 0
   139  	// DecisionMarkTombstone converts the entry to a delete tombstone.
   140  	DecisionMarkTombstone Decision = 1
   141  	// DecisionDrop simply drops the entry, doesn't leave a delete tombstone.
   142  	DecisionDrop Decision = 2
   143  )
   144  
   145  // DefaultOptions sets a list of recommended options for good performance.
   146  // Feel free to modify these to suit your needs.
   147  var DefaultOptions = Options{
   148  	DoNotCompact:            false,
   149  	LevelOneSize:            256 << 20,
   150  	MaxMemTableSize:         64 << 20,
   151  	MaxTableSize:            8 << 20,
   152  	NumCompactors:           3,
   153  	NumLevelZeroTables:      5,
   154  	NumLevelZeroTablesStall: 10,
   155  	NumMemtables:            5,
   156  	SyncWrites:              true,
   157  	ValueLogFileSize:        256 << 20,
   158  	ValueLogMaxEntries:      1000000,
   159  	ValueLogMaxNumFiles:     1,
   160  	ValueThreshold:          32,
   161  	Truncate:                false,
   162  	MaxBlockCacheSize:       1 << 30,
   163  	MaxIndexCacheSize:       1 << 30,
   164  	TableBuilderOptions: options.TableBuilderOptions{
   165  		SuRFStartLevel:      8,
   166  		HashUtilRatio:       0.75,
   167  		WriteBufferSize:     2 * 1024 * 1024,
   168  		BytesPerSecond:      -1,
   169  		MaxLevels:           7,
   170  		LevelSizeMultiplier: 10,
   171  		BlockSize:           64 * 1024,
   172  		// TODO: use lz4 instead of snappy for better (de)compress performance.
   173  		CompressionPerLevel: []options.CompressionType{options.None, options.None, options.Snappy, options.Snappy, options.Snappy, options.ZSTD, options.ZSTD},
   174  		LogicalBloomFPR:     0.01,
   175  		SuRFOptions: options.SuRFOptions{
   176  			HashSuffixLen:  8,
   177  			RealSuffixLen:  8,
   178  			BitsPerKeyHint: 40,
   179  		},
   180  	},
   181  	ValueLogWriteOptions: options.ValueLogWriterOptions{
   182  		WriteBufferSize: 2 * 1024 * 1024,
   183  	},
   184  	CompactL0WhenClose: true,
   185  }
   186  
   187  // LSMOnlyOptions follows from DefaultOptions, but sets a higher ValueThreshold so values would
   188  // be colocated with the LSM tree, with value log largely acting as a write-ahead log only. These
   189  // options would reduce the disk usage of value log, and make Badger act like a typical LSM tree.
   190  var LSMOnlyOptions = Options{}
   191  
   192  func init() {
   193  	LSMOnlyOptions = DefaultOptions
   194  
   195  	LSMOnlyOptions.ValueThreshold = 65500      // Max value length which fits in uint16.
   196  	LSMOnlyOptions.ValueLogFileSize = 64 << 20 // Allow easy space reclamation.
   197  }