github.com/MetalBlockchain/subnet-evm@v0.4.9/ethdb/leveldb/leveldb.go (about)

     1  // (c) 2021-2022, Ava Labs, Inc.
     2  //
     3  // This file is a derived work, based on the go-ethereum library whose original
     4  // notices appear below.
     5  //
     6  // It is distributed under a license compatible with the licensing terms of the
     7  // original code from which it is derived.
     8  //
     9  // Much love to the original authors for their work.
    10  // **********
    11  // Copyright 2018 The go-ethereum Authors
    12  // This file is part of the go-ethereum library.
    13  //
    14  // The go-ethereum library is free software: you can redistribute it and/or modify
    15  // it under the terms of the GNU Lesser General Public License as published by
    16  // the Free Software Foundation, either version 3 of the License, or
    17  // (at your option) any later version.
    18  //
    19  // The go-ethereum library is distributed in the hope that it will be useful,
    20  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    21  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    22  // GNU Lesser General Public License for more details.
    23  //
    24  // You should have received a copy of the GNU Lesser General Public License
    25  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    26  
    27  //go:build !js
    28  // +build !js
    29  
    30  // Package leveldb implements the key-value database layer based on LevelDB.
    31  package leveldb
    32  
    33  import (
    34  	"fmt"
    35  	"strconv"
    36  	"strings"
    37  	"sync"
    38  	"time"
    39  
    40  	"github.com/MetalBlockchain/subnet-evm/ethdb"
    41  	"github.com/MetalBlockchain/subnet-evm/metrics"
    42  	"github.com/ethereum/go-ethereum/common"
    43  	"github.com/ethereum/go-ethereum/log"
    44  	"github.com/syndtr/goleveldb/leveldb"
    45  	"github.com/syndtr/goleveldb/leveldb/errors"
    46  	"github.com/syndtr/goleveldb/leveldb/filter"
    47  	"github.com/syndtr/goleveldb/leveldb/opt"
    48  	"github.com/syndtr/goleveldb/leveldb/util"
    49  )
    50  
    51  const (
    52  	// degradationWarnInterval specifies how often warning should be printed if the
    53  	// leveldb database cannot keep up with requested writes.
    54  	degradationWarnInterval = time.Minute
    55  
    56  	// minCache is the minimum amount of memory in megabytes to allocate to leveldb
    57  	// read and write caching, split half and half.
    58  	minCache = 16
    59  
    60  	// minHandles is the minimum number of files handles to allocate to the open
    61  	// database files.
    62  	minHandles = 16
    63  
    64  	// metricsGatheringInterval specifies the interval to retrieve leveldb database
    65  	// compaction, io and pause stats to report to the user.
    66  	metricsGatheringInterval = 3 * time.Second
    67  )
    68  
    69  // Database is a persistent key-value store. Apart from basic data storage
    70  // functionality it also supports batch writes and iterating over the keyspace in
    71  // binary-alphabetical order.
    72  type Database struct {
    73  	fn string      // filename for reporting
    74  	db *leveldb.DB // LevelDB instance
    75  
    76  	compTimeMeter      metrics.Meter // Meter for measuring the total time spent in database compaction
    77  	compReadMeter      metrics.Meter // Meter for measuring the data read during compaction
    78  	compWriteMeter     metrics.Meter // Meter for measuring the data written during compaction
    79  	writeDelayNMeter   metrics.Meter // Meter for measuring the write delay number due to database compaction
    80  	writeDelayMeter    metrics.Meter // Meter for measuring the write delay duration due to database compaction
    81  	diskSizeGauge      metrics.Gauge // Gauge for tracking the size of all the levels in the database
    82  	diskReadMeter      metrics.Meter // Meter for measuring the effective amount of data read
    83  	diskWriteMeter     metrics.Meter // Meter for measuring the effective amount of data written
    84  	memCompGauge       metrics.Gauge // Gauge for tracking the number of memory compaction
    85  	level0CompGauge    metrics.Gauge // Gauge for tracking the number of table compaction in level0
    86  	nonlevel0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in non0 level
    87  	seekCompGauge      metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt
    88  
    89  	quitLock sync.Mutex      // Mutex protecting the quit channel access
    90  	quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
    91  
    92  	log log.Logger // Contextual logger tracking the database path
    93  }
    94  
    95  // New returns a wrapped LevelDB object. The namespace is the prefix that the
    96  // metrics reporting should use for surfacing internal stats.
    97  func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) {
    98  	return NewCustom(file, namespace, func(options *opt.Options) {
    99  		// Ensure we have some minimal caching and file guarantees
   100  		if cache < minCache {
   101  			cache = minCache
   102  		}
   103  		if handles < minHandles {
   104  			handles = minHandles
   105  		}
   106  		// Set default options
   107  		options.OpenFilesCacheCapacity = handles
   108  		options.BlockCacheCapacity = cache / 2 * opt.MiB
   109  		options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally
   110  		if readonly {
   111  			options.ReadOnly = true
   112  		}
   113  	})
   114  }
   115  
   116  // NewCustom returns a wrapped LevelDB object. The namespace is the prefix that the
   117  // metrics reporting should use for surfacing internal stats.
   118  // The customize function allows the caller to modify the leveldb options.
   119  func NewCustom(file string, namespace string, customize func(options *opt.Options)) (*Database, error) {
   120  	options := configureOptions(customize)
   121  	logger := log.New("database", file)
   122  	usedCache := options.GetBlockCacheCapacity() + options.GetWriteBuffer()*2
   123  	logCtx := []interface{}{"cache", common.StorageSize(usedCache), "handles", options.GetOpenFilesCacheCapacity()}
   124  	if options.ReadOnly {
   125  		logCtx = append(logCtx, "readonly", "true")
   126  	}
   127  	logger.Info("Allocated cache and file handles", logCtx...)
   128  
   129  	// Open the db and recover any potential corruptions
   130  	db, err := leveldb.OpenFile(file, options)
   131  	if _, corrupted := err.(*errors.ErrCorrupted); corrupted {
   132  		db, err = leveldb.RecoverFile(file, nil)
   133  	}
   134  	if err != nil {
   135  		return nil, err
   136  	}
   137  	// Assemble the wrapper with all the registered metrics
   138  	ldb := &Database{
   139  		fn:       file,
   140  		db:       db,
   141  		log:      logger,
   142  		quitChan: make(chan chan error),
   143  	}
   144  	ldb.compTimeMeter = metrics.NewRegisteredMeter(namespace+"compact/time", nil)
   145  	ldb.compReadMeter = metrics.NewRegisteredMeter(namespace+"compact/input", nil)
   146  	ldb.compWriteMeter = metrics.NewRegisteredMeter(namespace+"compact/output", nil)
   147  	ldb.diskSizeGauge = metrics.NewRegisteredGauge(namespace+"disk/size", nil)
   148  	ldb.diskReadMeter = metrics.NewRegisteredMeter(namespace+"disk/read", nil)
   149  	ldb.diskWriteMeter = metrics.NewRegisteredMeter(namespace+"disk/write", nil)
   150  	ldb.writeDelayMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/duration", nil)
   151  	ldb.writeDelayNMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/counter", nil)
   152  	ldb.memCompGauge = metrics.NewRegisteredGauge(namespace+"compact/memory", nil)
   153  	ldb.level0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/level0", nil)
   154  	ldb.nonlevel0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/nonlevel0", nil)
   155  	ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil)
   156  
   157  	// Start up the metrics gathering and return
   158  	go ldb.meter(metricsGatheringInterval)
   159  	return ldb, nil
   160  }
   161  
   162  // configureOptions sets some default options, then runs the provided setter.
   163  func configureOptions(customizeFn func(*opt.Options)) *opt.Options {
   164  	// Set default options
   165  	options := &opt.Options{
   166  		Filter:                 filter.NewBloomFilter(10),
   167  		DisableSeeksCompaction: true,
   168  	}
   169  	// Allow caller to make custom modifications to the options
   170  	if customizeFn != nil {
   171  		customizeFn(options)
   172  	}
   173  	return options
   174  }
   175  
   176  // Close stops the metrics collection, flushes any pending data to disk and closes
   177  // all io accesses to the underlying key-value store.
   178  func (db *Database) Close() error {
   179  	db.quitLock.Lock()
   180  	defer db.quitLock.Unlock()
   181  
   182  	if db.quitChan != nil {
   183  		errc := make(chan error)
   184  		db.quitChan <- errc
   185  		if err := <-errc; err != nil {
   186  			db.log.Error("Metrics collection failed", "err", err)
   187  		}
   188  		db.quitChan = nil
   189  	}
   190  	return db.db.Close()
   191  }
   192  
   193  // Has retrieves if a key is present in the key-value store.
   194  func (db *Database) Has(key []byte) (bool, error) {
   195  	return db.db.Has(key, nil)
   196  }
   197  
   198  // Get retrieves the given key if it's present in the key-value store.
   199  func (db *Database) Get(key []byte) ([]byte, error) {
   200  	dat, err := db.db.Get(key, nil)
   201  	if err != nil {
   202  		return nil, err
   203  	}
   204  	return dat, nil
   205  }
   206  
   207  // Put inserts the given value into the key-value store.
   208  func (db *Database) Put(key []byte, value []byte) error {
   209  	return db.db.Put(key, value, nil)
   210  }
   211  
   212  // Delete removes the key from the key-value store.
   213  func (db *Database) Delete(key []byte) error {
   214  	return db.db.Delete(key, nil)
   215  }
   216  
   217  // NewBatch creates a write-only key-value store that buffers changes to its host
   218  // database until a final write is called.
   219  func (db *Database) NewBatch() ethdb.Batch {
   220  	return &batch{
   221  		db: db.db,
   222  		b:  new(leveldb.Batch),
   223  	}
   224  }
   225  
   226  // NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
   227  func (db *Database) NewBatchWithSize(size int) ethdb.Batch {
   228  	return &batch{
   229  		db: db.db,
   230  		b:  leveldb.MakeBatch(size),
   231  	}
   232  }
   233  
   234  // NewIterator creates a binary-alphabetical iterator over a subset
   235  // of database content with a particular key prefix, starting at a particular
   236  // initial key (or after, if it does not exist).
   237  func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
   238  	return db.db.NewIterator(bytesPrefixRange(prefix, start), nil)
   239  }
   240  
   241  // Stat returns a particular internal stat of the database.
   242  func (db *Database) Stat(property string) (string, error) {
   243  	return db.db.GetProperty(property)
   244  }
   245  
   246  // Compact flattens the underlying data store for the given key range. In essence,
   247  // deleted and overwritten versions are discarded, and the data is rearranged to
   248  // reduce the cost of operations needed to access them.
   249  //
   250  // A nil start is treated as a key before all keys in the data store; a nil limit
   251  // is treated as a key after all keys in the data store. If both is nil then it
   252  // will compact entire data store.
   253  func (db *Database) Compact(start []byte, limit []byte) error {
   254  	return db.db.CompactRange(util.Range{Start: start, Limit: limit})
   255  }
   256  
   257  // Path returns the path to the database directory.
   258  func (db *Database) Path() string {
   259  	return db.fn
   260  }
   261  
   262  // meter periodically retrieves internal leveldb counters and reports them to
   263  // the metrics subsystem.
   264  //
   265  // This is how a LevelDB stats table looks like (currently):
   266  //   Compactions
   267  //    Level |   Tables   |    Size(MB)   |    Time(sec)  |    Read(MB)   |   Write(MB)
   268  //   -------+------------+---------------+---------------+---------------+---------------
   269  //      0   |          0 |       0.00000 |       1.27969 |       0.00000 |      12.31098
   270  //      1   |         85 |     109.27913 |      28.09293 |     213.92493 |     214.26294
   271  //      2   |        523 |    1000.37159 |       7.26059 |      66.86342 |      66.77884
   272  //      3   |        570 |    1113.18458 |       0.00000 |       0.00000 |       0.00000
   273  //
   274  // This is how the write delay look like (currently):
   275  // DelayN:5 Delay:406.604657ms Paused: false
   276  //
   277  // This is how the iostats look like (currently):
   278  // Read(MB):3895.04860 Write(MB):3654.64712
   279  func (db *Database) meter(refresh time.Duration) {
   280  	// Create the counters to store current and previous compaction values
   281  	compactions := make([][]float64, 2)
   282  	for i := 0; i < 2; i++ {
   283  		compactions[i] = make([]float64, 4)
   284  	}
   285  	// Create storage for iostats.
   286  	var iostats [2]float64
   287  
   288  	// Create storage and warning log tracer for write delay.
   289  	var (
   290  		delaystats      [2]int64
   291  		lastWritePaused time.Time
   292  	)
   293  
   294  	var (
   295  		errc chan error
   296  		merr error
   297  	)
   298  
   299  	timer := time.NewTimer(refresh)
   300  	defer timer.Stop()
   301  
   302  	// Iterate ad infinitum and collect the stats
   303  	for i := 1; errc == nil && merr == nil; i++ {
   304  		// Retrieve the database stats
   305  		stats, err := db.db.GetProperty("leveldb.stats")
   306  		if err != nil {
   307  			db.log.Error("Failed to read database stats", "err", err)
   308  			merr = err
   309  			continue
   310  		}
   311  		// Find the compaction table, skip the header
   312  		lines := strings.Split(stats, "\n")
   313  		for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" {
   314  			lines = lines[1:]
   315  		}
   316  		if len(lines) <= 3 {
   317  			db.log.Error("Compaction leveldbTable not found")
   318  			merr = errors.New("compaction leveldbTable not found")
   319  			continue
   320  		}
   321  		lines = lines[3:]
   322  
   323  		// Iterate over all the leveldbTable rows, and accumulate the entries
   324  		for j := 0; j < len(compactions[i%2]); j++ {
   325  			compactions[i%2][j] = 0
   326  		}
   327  		for _, line := range lines {
   328  			parts := strings.Split(line, "|")
   329  			if len(parts) != 6 {
   330  				break
   331  			}
   332  			for idx, counter := range parts[2:] {
   333  				value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64)
   334  				if err != nil {
   335  					db.log.Error("Compaction entry parsing failed", "err", err)
   336  					merr = err
   337  					continue
   338  				}
   339  				compactions[i%2][idx] += value
   340  			}
   341  		}
   342  		// Update all the requested meters
   343  		if db.diskSizeGauge != nil {
   344  			db.diskSizeGauge.Update(int64(compactions[i%2][0] * 1024 * 1024))
   345  		}
   346  		if db.compTimeMeter != nil {
   347  			db.compTimeMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1000 * 1000 * 1000))
   348  		}
   349  		if db.compReadMeter != nil {
   350  			db.compReadMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024))
   351  		}
   352  		if db.compWriteMeter != nil {
   353  			db.compWriteMeter.Mark(int64((compactions[i%2][3] - compactions[(i-1)%2][3]) * 1024 * 1024))
   354  		}
   355  		// Retrieve the write delay statistic
   356  		writedelay, err := db.db.GetProperty("leveldb.writedelay")
   357  		if err != nil {
   358  			db.log.Error("Failed to read database write delay statistic", "err", err)
   359  			merr = err
   360  			continue
   361  		}
   362  		var (
   363  			delayN        int64
   364  			delayDuration string
   365  			duration      time.Duration
   366  			paused        bool
   367  		)
   368  		if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil {
   369  			db.log.Error("Write delay statistic not found")
   370  			merr = err
   371  			continue
   372  		}
   373  		duration, err = time.ParseDuration(delayDuration)
   374  		if err != nil {
   375  			db.log.Error("Failed to parse delay duration", "err", err)
   376  			merr = err
   377  			continue
   378  		}
   379  		if db.writeDelayNMeter != nil {
   380  			db.writeDelayNMeter.Mark(delayN - delaystats[0])
   381  		}
   382  		if db.writeDelayMeter != nil {
   383  			db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1])
   384  		}
   385  		// If a warning that db is performing compaction has been displayed, any subsequent
   386  		// warnings will be withheld for one minute not to overwhelm the user.
   387  		if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 &&
   388  			time.Now().After(lastWritePaused.Add(degradationWarnInterval)) {
   389  			db.log.Warn("Database compacting, degraded performance")
   390  			lastWritePaused = time.Now()
   391  		}
   392  		delaystats[0], delaystats[1] = delayN, duration.Nanoseconds()
   393  
   394  		// Retrieve the database iostats.
   395  		ioStats, err := db.db.GetProperty("leveldb.iostats")
   396  		if err != nil {
   397  			db.log.Error("Failed to read database iostats", "err", err)
   398  			merr = err
   399  			continue
   400  		}
   401  		var nRead, nWrite float64
   402  		parts := strings.Split(ioStats, " ")
   403  		if len(parts) < 2 {
   404  			db.log.Error("Bad syntax of ioStats", "ioStats", ioStats)
   405  			merr = fmt.Errorf("bad syntax of ioStats %s", ioStats)
   406  			continue
   407  		}
   408  		if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil {
   409  			db.log.Error("Bad syntax of read entry", "entry", parts[0])
   410  			merr = err
   411  			continue
   412  		}
   413  		if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil {
   414  			db.log.Error("Bad syntax of write entry", "entry", parts[1])
   415  			merr = err
   416  			continue
   417  		}
   418  		if db.diskReadMeter != nil {
   419  			db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024))
   420  		}
   421  		if db.diskWriteMeter != nil {
   422  			db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024))
   423  		}
   424  		iostats[0], iostats[1] = nRead, nWrite
   425  
   426  		compCount, err := db.db.GetProperty("leveldb.compcount")
   427  		if err != nil {
   428  			db.log.Error("Failed to read database iostats", "err", err)
   429  			merr = err
   430  			continue
   431  		}
   432  
   433  		var (
   434  			memComp       uint32
   435  			level0Comp    uint32
   436  			nonLevel0Comp uint32
   437  			seekComp      uint32
   438  		)
   439  		if n, err := fmt.Sscanf(compCount, "MemComp:%d Level0Comp:%d NonLevel0Comp:%d SeekComp:%d", &memComp, &level0Comp, &nonLevel0Comp, &seekComp); n != 4 || err != nil {
   440  			db.log.Error("Compaction count statistic not found")
   441  			merr = err
   442  			continue
   443  		}
   444  		db.memCompGauge.Update(int64(memComp))
   445  		db.level0CompGauge.Update(int64(level0Comp))
   446  		db.nonlevel0CompGauge.Update(int64(nonLevel0Comp))
   447  		db.seekCompGauge.Update(int64(seekComp))
   448  
   449  		// Sleep a bit, then repeat the stats collection
   450  		select {
   451  		case errc = <-db.quitChan:
   452  			// Quit requesting, stop hammering the database
   453  		case <-timer.C:
   454  			timer.Reset(refresh)
   455  			// Timeout, gather a new set of stats
   456  		}
   457  	}
   458  
   459  	if errc == nil {
   460  		errc = <-db.quitChan
   461  	}
   462  	errc <- merr
   463  }
   464  
   465  // batch is a write-only leveldb batch that commits changes to its host database
   466  // when Write is called. A batch cannot be used concurrently.
   467  type batch struct {
   468  	db   *leveldb.DB
   469  	b    *leveldb.Batch
   470  	size int
   471  }
   472  
   473  // Put inserts the given value into the batch for later committing.
   474  func (b *batch) Put(key, value []byte) error {
   475  	b.b.Put(key, value)
   476  	b.size += len(value)
   477  	return nil
   478  }
   479  
   480  // Delete inserts the a key removal into the batch for later committing.
   481  func (b *batch) Delete(key []byte) error {
   482  	b.b.Delete(key)
   483  	b.size += len(key)
   484  	return nil
   485  }
   486  
   487  // ValueSize retrieves the amount of data queued up for writing.
   488  func (b *batch) ValueSize() int {
   489  	return b.size
   490  }
   491  
   492  // Write flushes any accumulated data to disk.
   493  func (b *batch) Write() error {
   494  	return b.db.Write(b.b, nil)
   495  }
   496  
   497  // Reset resets the batch for reuse.
   498  func (b *batch) Reset() {
   499  	b.b.Reset()
   500  	b.size = 0
   501  }
   502  
   503  // Replay replays the batch contents.
   504  func (b *batch) Replay(w ethdb.KeyValueWriter) error {
   505  	return b.b.Replay(&replayer{writer: w})
   506  }
   507  
   508  // replayer is a small wrapper to implement the correct replay methods.
   509  type replayer struct {
   510  	writer  ethdb.KeyValueWriter
   511  	failure error
   512  }
   513  
   514  // Put inserts the given value into the key-value data store.
   515  func (r *replayer) Put(key, value []byte) {
   516  	// If the replay already failed, stop executing ops
   517  	if r.failure != nil {
   518  		return
   519  	}
   520  	r.failure = r.writer.Put(key, value)
   521  }
   522  
   523  // Delete removes the key from the key-value data store.
   524  func (r *replayer) Delete(key []byte) {
   525  	// If the replay already failed, stop executing ops
   526  	if r.failure != nil {
   527  		return
   528  	}
   529  	r.failure = r.writer.Delete(key)
   530  }
   531  
   532  // bytesPrefixRange returns key range that satisfy
   533  // - the given prefix, and
   534  // - the given seek position
   535  func bytesPrefixRange(prefix, start []byte) *util.Range {
   536  	r := util.BytesPrefix(prefix)
   537  	r.Start = append(r.Start, start...)
   538  	return r
   539  }