github.com/cockroachdb/pebble@v1.1.2/cmd/pebble/db.go (about)

     1  // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"log"
     9  
    10  	"github.com/cockroachdb/pebble"
    11  	"github.com/cockroachdb/pebble/bloom"
    12  	"github.com/cockroachdb/pebble/internal/bytealloc"
    13  	"github.com/cockroachdb/pebble/objstorage/remote"
    14  	"github.com/cockroachdb/pebble/vfs"
    15  )
    16  
    17  // DB specifies the minimal interfaces that need to be implemented to support
    18  // the pebble command.
    19  type DB interface {
    20  	NewIter(*pebble.IterOptions) iterator
    21  	NewBatch() batch
    22  	Scan(iter iterator, key []byte, count int64, reverse bool) error
    23  	Metrics() *pebble.Metrics
    24  	Flush() error
    25  }
    26  
    27  type iterator interface {
    28  	SeekLT(key []byte) bool
    29  	SeekGE(key []byte) bool
    30  	Valid() bool
    31  	Key() []byte
    32  	Value() []byte
    33  	First() bool
    34  	Next() bool
    35  	Last() bool
    36  	Prev() bool
    37  	Close() error
    38  }
    39  
    40  type batch interface {
    41  	Close() error
    42  	Commit(opts *pebble.WriteOptions) error
    43  	Set(key, value []byte, opts *pebble.WriteOptions) error
    44  	Delete(key []byte, opts *pebble.WriteOptions) error
    45  	LogData(data []byte, opts *pebble.WriteOptions) error
    46  }
    47  
    48  // Adapters for Pebble. Since the interfaces above are based on Pebble's
    49  // interfaces, it can simply forward calls for everything.
    50  type pebbleDB struct {
    51  	d       *pebble.DB
    52  	ballast []byte
    53  }
    54  
    55  func newPebbleDB(dir string) DB {
    56  	cache := pebble.NewCache(cacheSize)
    57  	defer cache.Unref()
    58  	opts := &pebble.Options{
    59  		Cache:                       cache,
    60  		Comparer:                    mvccComparer,
    61  		DisableWAL:                  disableWAL,
    62  		FormatMajorVersion:          pebble.FormatNewest,
    63  		L0CompactionThreshold:       2,
    64  		L0StopWritesThreshold:       1000,
    65  		LBaseMaxBytes:               64 << 20, // 64 MB
    66  		Levels:                      make([]pebble.LevelOptions, 7),
    67  		MaxOpenFiles:                16384,
    68  		MemTableSize:                64 << 20,
    69  		MemTableStopWritesThreshold: 4,
    70  		Merger: &pebble.Merger{
    71  			Name: "cockroach_merge_operator",
    72  		},
    73  		MaxConcurrentCompactions: func() int {
    74  			return 3
    75  		},
    76  	}
    77  
    78  	for i := 0; i < len(opts.Levels); i++ {
    79  		l := &opts.Levels[i]
    80  		l.BlockSize = 32 << 10       // 32 KB
    81  		l.IndexBlockSize = 256 << 10 // 256 KB
    82  		l.FilterPolicy = bloom.FilterPolicy(10)
    83  		l.FilterType = pebble.TableFilter
    84  		if i > 0 {
    85  			l.TargetFileSize = opts.Levels[i-1].TargetFileSize * 2
    86  		}
    87  		l.EnsureDefaults()
    88  	}
    89  	opts.Levels[6].FilterPolicy = nil
    90  	opts.FlushSplitBytes = opts.Levels[0].TargetFileSize
    91  
    92  	opts.EnsureDefaults()
    93  
    94  	if verbose {
    95  		lel := pebble.MakeLoggingEventListener(nil)
    96  		opts.EventListener = &lel
    97  		opts.EventListener.TableDeleted = nil
    98  		opts.EventListener.TableIngested = nil
    99  		opts.EventListener.WALCreated = nil
   100  		opts.EventListener.WALDeleted = nil
   101  	}
   102  
   103  	if pathToLocalSharedStorage != "" {
   104  		opts.Experimental.RemoteStorage = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
   105  			// Store all shared objects on local disk, for convenience.
   106  			"": remote.NewLocalFS(pathToLocalSharedStorage, vfs.Default),
   107  		})
   108  		opts.Experimental.CreateOnShared = remote.CreateOnSharedAll
   109  		if secondaryCacheSize != 0 {
   110  			opts.Experimental.SecondaryCacheSizeBytes = secondaryCacheSize
   111  		}
   112  	}
   113  
   114  	p, err := pebble.Open(dir, opts)
   115  	if err != nil {
   116  		log.Fatal(err)
   117  	}
   118  	if pathToLocalSharedStorage != "" {
   119  		if err := p.SetCreatorID(1); err != nil {
   120  			log.Fatal(err)
   121  		}
   122  	}
   123  	return pebbleDB{
   124  		d:       p,
   125  		ballast: make([]byte, 1<<30),
   126  	}
   127  }
   128  
   129  func (p pebbleDB) Flush() error {
   130  	return p.d.Flush()
   131  }
   132  
   133  func (p pebbleDB) NewIter(opts *pebble.IterOptions) iterator {
   134  	iter, _ := p.d.NewIter(opts)
   135  	return iter
   136  }
   137  
   138  func (p pebbleDB) NewBatch() batch {
   139  	return p.d.NewBatch()
   140  }
   141  
   142  func (p pebbleDB) Scan(iter iterator, key []byte, count int64, reverse bool) error {
   143  	var data bytealloc.A
   144  	if reverse {
   145  		for i, valid := 0, iter.SeekLT(key); valid; valid = iter.Prev() {
   146  			data, _ = data.Copy(iter.Key())
   147  			data, _ = data.Copy(iter.Value())
   148  			i++
   149  			if i >= int(count) {
   150  				break
   151  			}
   152  		}
   153  	} else {
   154  		for i, valid := 0, iter.SeekGE(key); valid; valid = iter.Next() {
   155  			data, _ = data.Copy(iter.Key())
   156  			data, _ = data.Copy(iter.Value())
   157  			i++
   158  			if i >= int(count) {
   159  				break
   160  			}
   161  		}
   162  	}
   163  	return nil
   164  }
   165  
   166  func (p pebbleDB) Metrics() *pebble.Metrics {
   167  	return p.d.Metrics()
   168  }