github.com/MetalBlockchain/subnet-evm@v0.4.9/ethdb/leveldb/leveldb.go (about) 1 // (c) 2021-2022, Ava Labs, Inc. 2 // 3 // This file is a derived work, based on the go-ethereum library whose original 4 // notices appear below. 5 // 6 // It is distributed under a license compatible with the licensing terms of the 7 // original code from which it is derived. 8 // 9 // Much love to the original authors for their work. 10 // ********** 11 // Copyright 2018 The go-ethereum Authors 12 // This file is part of the go-ethereum library. 13 // 14 // The go-ethereum library is free software: you can redistribute it and/or modify 15 // it under the terms of the GNU Lesser General Public License as published by 16 // the Free Software Foundation, either version 3 of the License, or 17 // (at your option) any later version. 18 // 19 // The go-ethereum library is distributed in the hope that it will be useful, 20 // but WITHOUT ANY WARRANTY; without even the implied warranty of 21 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 // GNU Lesser General Public License for more details. 23 // 24 // You should have received a copy of the GNU Lesser General Public License 25 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 26 27 //go:build !js 28 // +build !js 29 30 // Package leveldb implements the key-value database layer based on LevelDB. 31 package leveldb 32 33 import ( 34 "fmt" 35 "strconv" 36 "strings" 37 "sync" 38 "time" 39 40 "github.com/MetalBlockchain/subnet-evm/ethdb" 41 "github.com/MetalBlockchain/subnet-evm/metrics" 42 "github.com/ethereum/go-ethereum/common" 43 "github.com/ethereum/go-ethereum/log" 44 "github.com/syndtr/goleveldb/leveldb" 45 "github.com/syndtr/goleveldb/leveldb/errors" 46 "github.com/syndtr/goleveldb/leveldb/filter" 47 "github.com/syndtr/goleveldb/leveldb/opt" 48 "github.com/syndtr/goleveldb/leveldb/util" 49 ) 50 51 const ( 52 // degradationWarnInterval specifies how often warning should be printed if the 53 // leveldb database cannot keep up with requested writes. 54 degradationWarnInterval = time.Minute 55 56 // minCache is the minimum amount of memory in megabytes to allocate to leveldb 57 // read and write caching, split half and half. 58 minCache = 16 59 60 // minHandles is the minimum number of files handles to allocate to the open 61 // database files. 62 minHandles = 16 63 64 // metricsGatheringInterval specifies the interval to retrieve leveldb database 65 // compaction, io and pause stats to report to the user. 66 metricsGatheringInterval = 3 * time.Second 67 ) 68 69 // Database is a persistent key-value store. Apart from basic data storage 70 // functionality it also supports batch writes and iterating over the keyspace in 71 // binary-alphabetical order. 72 type Database struct { 73 fn string // filename for reporting 74 db *leveldb.DB // LevelDB instance 75 76 compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction 77 compReadMeter metrics.Meter // Meter for measuring the data read during compaction 78 compWriteMeter metrics.Meter // Meter for measuring the data written during compaction 79 writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction 80 writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction 81 diskSizeGauge metrics.Gauge // Gauge for tracking the size of all the levels in the database 82 diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read 83 diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written 84 memCompGauge metrics.Gauge // Gauge for tracking the number of memory compaction 85 level0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in level0 86 nonlevel0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in non0 level 87 seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt 88 89 quitLock sync.Mutex // Mutex protecting the quit channel access 90 quitChan chan chan error // Quit channel to stop the metrics collection before closing the database 91 92 log log.Logger // Contextual logger tracking the database path 93 } 94 95 // New returns a wrapped LevelDB object. The namespace is the prefix that the 96 // metrics reporting should use for surfacing internal stats. 97 func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) { 98 return NewCustom(file, namespace, func(options *opt.Options) { 99 // Ensure we have some minimal caching and file guarantees 100 if cache < minCache { 101 cache = minCache 102 } 103 if handles < minHandles { 104 handles = minHandles 105 } 106 // Set default options 107 options.OpenFilesCacheCapacity = handles 108 options.BlockCacheCapacity = cache / 2 * opt.MiB 109 options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally 110 if readonly { 111 options.ReadOnly = true 112 } 113 }) 114 } 115 116 // NewCustom returns a wrapped LevelDB object. The namespace is the prefix that the 117 // metrics reporting should use for surfacing internal stats. 118 // The customize function allows the caller to modify the leveldb options. 119 func NewCustom(file string, namespace string, customize func(options *opt.Options)) (*Database, error) { 120 options := configureOptions(customize) 121 logger := log.New("database", file) 122 usedCache := options.GetBlockCacheCapacity() + options.GetWriteBuffer()*2 123 logCtx := []interface{}{"cache", common.StorageSize(usedCache), "handles", options.GetOpenFilesCacheCapacity()} 124 if options.ReadOnly { 125 logCtx = append(logCtx, "readonly", "true") 126 } 127 logger.Info("Allocated cache and file handles", logCtx...) 128 129 // Open the db and recover any potential corruptions 130 db, err := leveldb.OpenFile(file, options) 131 if _, corrupted := err.(*errors.ErrCorrupted); corrupted { 132 db, err = leveldb.RecoverFile(file, nil) 133 } 134 if err != nil { 135 return nil, err 136 } 137 // Assemble the wrapper with all the registered metrics 138 ldb := &Database{ 139 fn: file, 140 db: db, 141 log: logger, 142 quitChan: make(chan chan error), 143 } 144 ldb.compTimeMeter = metrics.NewRegisteredMeter(namespace+"compact/time", nil) 145 ldb.compReadMeter = metrics.NewRegisteredMeter(namespace+"compact/input", nil) 146 ldb.compWriteMeter = metrics.NewRegisteredMeter(namespace+"compact/output", nil) 147 ldb.diskSizeGauge = metrics.NewRegisteredGauge(namespace+"disk/size", nil) 148 ldb.diskReadMeter = metrics.NewRegisteredMeter(namespace+"disk/read", nil) 149 ldb.diskWriteMeter = metrics.NewRegisteredMeter(namespace+"disk/write", nil) 150 ldb.writeDelayMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/duration", nil) 151 ldb.writeDelayNMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/counter", nil) 152 ldb.memCompGauge = metrics.NewRegisteredGauge(namespace+"compact/memory", nil) 153 ldb.level0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/level0", nil) 154 ldb.nonlevel0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/nonlevel0", nil) 155 ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil) 156 157 // Start up the metrics gathering and return 158 go ldb.meter(metricsGatheringInterval) 159 return ldb, nil 160 } 161 162 // configureOptions sets some default options, then runs the provided setter. 163 func configureOptions(customizeFn func(*opt.Options)) *opt.Options { 164 // Set default options 165 options := &opt.Options{ 166 Filter: filter.NewBloomFilter(10), 167 DisableSeeksCompaction: true, 168 } 169 // Allow caller to make custom modifications to the options 170 if customizeFn != nil { 171 customizeFn(options) 172 } 173 return options 174 } 175 176 // Close stops the metrics collection, flushes any pending data to disk and closes 177 // all io accesses to the underlying key-value store. 178 func (db *Database) Close() error { 179 db.quitLock.Lock() 180 defer db.quitLock.Unlock() 181 182 if db.quitChan != nil { 183 errc := make(chan error) 184 db.quitChan <- errc 185 if err := <-errc; err != nil { 186 db.log.Error("Metrics collection failed", "err", err) 187 } 188 db.quitChan = nil 189 } 190 return db.db.Close() 191 } 192 193 // Has retrieves if a key is present in the key-value store. 194 func (db *Database) Has(key []byte) (bool, error) { 195 return db.db.Has(key, nil) 196 } 197 198 // Get retrieves the given key if it's present in the key-value store. 199 func (db *Database) Get(key []byte) ([]byte, error) { 200 dat, err := db.db.Get(key, nil) 201 if err != nil { 202 return nil, err 203 } 204 return dat, nil 205 } 206 207 // Put inserts the given value into the key-value store. 208 func (db *Database) Put(key []byte, value []byte) error { 209 return db.db.Put(key, value, nil) 210 } 211 212 // Delete removes the key from the key-value store. 213 func (db *Database) Delete(key []byte) error { 214 return db.db.Delete(key, nil) 215 } 216 217 // NewBatch creates a write-only key-value store that buffers changes to its host 218 // database until a final write is called. 219 func (db *Database) NewBatch() ethdb.Batch { 220 return &batch{ 221 db: db.db, 222 b: new(leveldb.Batch), 223 } 224 } 225 226 // NewBatchWithSize creates a write-only database batch with pre-allocated buffer. 227 func (db *Database) NewBatchWithSize(size int) ethdb.Batch { 228 return &batch{ 229 db: db.db, 230 b: leveldb.MakeBatch(size), 231 } 232 } 233 234 // NewIterator creates a binary-alphabetical iterator over a subset 235 // of database content with a particular key prefix, starting at a particular 236 // initial key (or after, if it does not exist). 237 func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { 238 return db.db.NewIterator(bytesPrefixRange(prefix, start), nil) 239 } 240 241 // Stat returns a particular internal stat of the database. 242 func (db *Database) Stat(property string) (string, error) { 243 return db.db.GetProperty(property) 244 } 245 246 // Compact flattens the underlying data store for the given key range. In essence, 247 // deleted and overwritten versions are discarded, and the data is rearranged to 248 // reduce the cost of operations needed to access them. 249 // 250 // A nil start is treated as a key before all keys in the data store; a nil limit 251 // is treated as a key after all keys in the data store. If both is nil then it 252 // will compact entire data store. 253 func (db *Database) Compact(start []byte, limit []byte) error { 254 return db.db.CompactRange(util.Range{Start: start, Limit: limit}) 255 } 256 257 // Path returns the path to the database directory. 258 func (db *Database) Path() string { 259 return db.fn 260 } 261 262 // meter periodically retrieves internal leveldb counters and reports them to 263 // the metrics subsystem. 264 // 265 // This is how a LevelDB stats table looks like (currently): 266 // Compactions 267 // Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) 268 // -------+------------+---------------+---------------+---------------+--------------- 269 // 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 270 // 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 271 // 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 272 // 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 273 // 274 // This is how the write delay look like (currently): 275 // DelayN:5 Delay:406.604657ms Paused: false 276 // 277 // This is how the iostats look like (currently): 278 // Read(MB):3895.04860 Write(MB):3654.64712 279 func (db *Database) meter(refresh time.Duration) { 280 // Create the counters to store current and previous compaction values 281 compactions := make([][]float64, 2) 282 for i := 0; i < 2; i++ { 283 compactions[i] = make([]float64, 4) 284 } 285 // Create storage for iostats. 286 var iostats [2]float64 287 288 // Create storage and warning log tracer for write delay. 289 var ( 290 delaystats [2]int64 291 lastWritePaused time.Time 292 ) 293 294 var ( 295 errc chan error 296 merr error 297 ) 298 299 timer := time.NewTimer(refresh) 300 defer timer.Stop() 301 302 // Iterate ad infinitum and collect the stats 303 for i := 1; errc == nil && merr == nil; i++ { 304 // Retrieve the database stats 305 stats, err := db.db.GetProperty("leveldb.stats") 306 if err != nil { 307 db.log.Error("Failed to read database stats", "err", err) 308 merr = err 309 continue 310 } 311 // Find the compaction table, skip the header 312 lines := strings.Split(stats, "\n") 313 for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { 314 lines = lines[1:] 315 } 316 if len(lines) <= 3 { 317 db.log.Error("Compaction leveldbTable not found") 318 merr = errors.New("compaction leveldbTable not found") 319 continue 320 } 321 lines = lines[3:] 322 323 // Iterate over all the leveldbTable rows, and accumulate the entries 324 for j := 0; j < len(compactions[i%2]); j++ { 325 compactions[i%2][j] = 0 326 } 327 for _, line := range lines { 328 parts := strings.Split(line, "|") 329 if len(parts) != 6 { 330 break 331 } 332 for idx, counter := range parts[2:] { 333 value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) 334 if err != nil { 335 db.log.Error("Compaction entry parsing failed", "err", err) 336 merr = err 337 continue 338 } 339 compactions[i%2][idx] += value 340 } 341 } 342 // Update all the requested meters 343 if db.diskSizeGauge != nil { 344 db.diskSizeGauge.Update(int64(compactions[i%2][0] * 1024 * 1024)) 345 } 346 if db.compTimeMeter != nil { 347 db.compTimeMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1000 * 1000 * 1000)) 348 } 349 if db.compReadMeter != nil { 350 db.compReadMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) 351 } 352 if db.compWriteMeter != nil { 353 db.compWriteMeter.Mark(int64((compactions[i%2][3] - compactions[(i-1)%2][3]) * 1024 * 1024)) 354 } 355 // Retrieve the write delay statistic 356 writedelay, err := db.db.GetProperty("leveldb.writedelay") 357 if err != nil { 358 db.log.Error("Failed to read database write delay statistic", "err", err) 359 merr = err 360 continue 361 } 362 var ( 363 delayN int64 364 delayDuration string 365 duration time.Duration 366 paused bool 367 ) 368 if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { 369 db.log.Error("Write delay statistic not found") 370 merr = err 371 continue 372 } 373 duration, err = time.ParseDuration(delayDuration) 374 if err != nil { 375 db.log.Error("Failed to parse delay duration", "err", err) 376 merr = err 377 continue 378 } 379 if db.writeDelayNMeter != nil { 380 db.writeDelayNMeter.Mark(delayN - delaystats[0]) 381 } 382 if db.writeDelayMeter != nil { 383 db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1]) 384 } 385 // If a warning that db is performing compaction has been displayed, any subsequent 386 // warnings will be withheld for one minute not to overwhelm the user. 387 if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && 388 time.Now().After(lastWritePaused.Add(degradationWarnInterval)) { 389 db.log.Warn("Database compacting, degraded performance") 390 lastWritePaused = time.Now() 391 } 392 delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() 393 394 // Retrieve the database iostats. 395 ioStats, err := db.db.GetProperty("leveldb.iostats") 396 if err != nil { 397 db.log.Error("Failed to read database iostats", "err", err) 398 merr = err 399 continue 400 } 401 var nRead, nWrite float64 402 parts := strings.Split(ioStats, " ") 403 if len(parts) < 2 { 404 db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) 405 merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) 406 continue 407 } 408 if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { 409 db.log.Error("Bad syntax of read entry", "entry", parts[0]) 410 merr = err 411 continue 412 } 413 if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { 414 db.log.Error("Bad syntax of write entry", "entry", parts[1]) 415 merr = err 416 continue 417 } 418 if db.diskReadMeter != nil { 419 db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) 420 } 421 if db.diskWriteMeter != nil { 422 db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) 423 } 424 iostats[0], iostats[1] = nRead, nWrite 425 426 compCount, err := db.db.GetProperty("leveldb.compcount") 427 if err != nil { 428 db.log.Error("Failed to read database iostats", "err", err) 429 merr = err 430 continue 431 } 432 433 var ( 434 memComp uint32 435 level0Comp uint32 436 nonLevel0Comp uint32 437 seekComp uint32 438 ) 439 if n, err := fmt.Sscanf(compCount, "MemComp:%d Level0Comp:%d NonLevel0Comp:%d SeekComp:%d", &memComp, &level0Comp, &nonLevel0Comp, &seekComp); n != 4 || err != nil { 440 db.log.Error("Compaction count statistic not found") 441 merr = err 442 continue 443 } 444 db.memCompGauge.Update(int64(memComp)) 445 db.level0CompGauge.Update(int64(level0Comp)) 446 db.nonlevel0CompGauge.Update(int64(nonLevel0Comp)) 447 db.seekCompGauge.Update(int64(seekComp)) 448 449 // Sleep a bit, then repeat the stats collection 450 select { 451 case errc = <-db.quitChan: 452 // Quit requesting, stop hammering the database 453 case <-timer.C: 454 timer.Reset(refresh) 455 // Timeout, gather a new set of stats 456 } 457 } 458 459 if errc == nil { 460 errc = <-db.quitChan 461 } 462 errc <- merr 463 } 464 465 // batch is a write-only leveldb batch that commits changes to its host database 466 // when Write is called. A batch cannot be used concurrently. 467 type batch struct { 468 db *leveldb.DB 469 b *leveldb.Batch 470 size int 471 } 472 473 // Put inserts the given value into the batch for later committing. 474 func (b *batch) Put(key, value []byte) error { 475 b.b.Put(key, value) 476 b.size += len(value) 477 return nil 478 } 479 480 // Delete inserts the a key removal into the batch for later committing. 481 func (b *batch) Delete(key []byte) error { 482 b.b.Delete(key) 483 b.size += len(key) 484 return nil 485 } 486 487 // ValueSize retrieves the amount of data queued up for writing. 488 func (b *batch) ValueSize() int { 489 return b.size 490 } 491 492 // Write flushes any accumulated data to disk. 493 func (b *batch) Write() error { 494 return b.db.Write(b.b, nil) 495 } 496 497 // Reset resets the batch for reuse. 498 func (b *batch) Reset() { 499 b.b.Reset() 500 b.size = 0 501 } 502 503 // Replay replays the batch contents. 504 func (b *batch) Replay(w ethdb.KeyValueWriter) error { 505 return b.b.Replay(&replayer{writer: w}) 506 } 507 508 // replayer is a small wrapper to implement the correct replay methods. 509 type replayer struct { 510 writer ethdb.KeyValueWriter 511 failure error 512 } 513 514 // Put inserts the given value into the key-value data store. 515 func (r *replayer) Put(key, value []byte) { 516 // If the replay already failed, stop executing ops 517 if r.failure != nil { 518 return 519 } 520 r.failure = r.writer.Put(key, value) 521 } 522 523 // Delete removes the key from the key-value data store. 524 func (r *replayer) Delete(key []byte) { 525 // If the replay already failed, stop executing ops 526 if r.failure != nil { 527 return 528 } 529 r.failure = r.writer.Delete(key) 530 } 531 532 // bytesPrefixRange returns key range that satisfy 533 // - the given prefix, and 534 // - the given seek position 535 func bytesPrefixRange(prefix, start []byte) *util.Range { 536 r := util.BytesPrefix(prefix) 537 r.Start = append(r.Start, start...) 538 return r 539 }