github.com/JFJun/bsc@v1.0.0/ethdb/leveldb/leveldb.go (about) 1 // Copyright 2018 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // +build !js 18 19 // Package leveldb implements the key-value database layer based on LevelDB. 20 package leveldb 21 22 import ( 23 "fmt" 24 "strconv" 25 "strings" 26 "sync" 27 "time" 28 29 "github.com/JFJun/bsc/common" 30 "github.com/JFJun/bsc/ethdb" 31 "github.com/JFJun/bsc/log" 32 "github.com/JFJun/bsc/metrics" 33 "github.com/syndtr/goleveldb/leveldb" 34 "github.com/syndtr/goleveldb/leveldb/errors" 35 "github.com/syndtr/goleveldb/leveldb/filter" 36 "github.com/syndtr/goleveldb/leveldb/opt" 37 "github.com/syndtr/goleveldb/leveldb/util" 38 ) 39 40 const ( 41 // degradationWarnInterval specifies how often warning should be printed if the 42 // leveldb database cannot keep up with requested writes. 43 degradationWarnInterval = time.Minute 44 45 // minCache is the minimum amount of memory in megabytes to allocate to leveldb 46 // read and write caching, split half and half. 47 minCache = 16 48 49 // minHandles is the minimum number of files handles to allocate to the open 50 // database files. 51 minHandles = 16 52 53 // metricsGatheringInterval specifies the interval to retrieve leveldb database 54 // compaction, io and pause stats to report to the user. 55 metricsGatheringInterval = 3 * time.Second 56 ) 57 58 // Database is a persistent key-value store. Apart from basic data storage 59 // functionality it also supports batch writes and iterating over the keyspace in 60 // binary-alphabetical order. 61 type Database struct { 62 fn string // filename for reporting 63 db *leveldb.DB // LevelDB instance 64 65 compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction 66 compReadMeter metrics.Meter // Meter for measuring the data read during compaction 67 compWriteMeter metrics.Meter // Meter for measuring the data written during compaction 68 writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction 69 writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction 70 diskSizeGauge metrics.Gauge // Gauge for tracking the size of all the levels in the database 71 diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read 72 diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written 73 memCompGauge metrics.Gauge // Gauge for tracking the number of memory compaction 74 level0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in level0 75 nonlevel0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in non0 level 76 seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt 77 78 quitLock sync.Mutex // Mutex protecting the quit channel access 79 quitChan chan chan error // Quit channel to stop the metrics collection before closing the database 80 81 log log.Logger // Contextual logger tracking the database path 82 } 83 84 // New returns a wrapped LevelDB object. The namespace is the prefix that the 85 // metrics reporting should use for surfacing internal stats. 86 func New(file string, cache int, handles int, namespace string) (*Database, error) { 87 // Ensure we have some minimal caching and file guarantees 88 if cache < minCache { 89 cache = minCache 90 } 91 if handles < minHandles { 92 handles = minHandles 93 } 94 logger := log.New("database", file) 95 logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles) 96 97 // Open the db and recover any potential corruptions 98 db, err := leveldb.OpenFile(file, &opt.Options{ 99 OpenFilesCacheCapacity: handles, 100 BlockCacheCapacity: cache / 2 * opt.MiB, 101 WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally 102 Filter: filter.NewBloomFilter(10), 103 DisableSeeksCompaction: true, 104 }) 105 if _, corrupted := err.(*errors.ErrCorrupted); corrupted { 106 db, err = leveldb.RecoverFile(file, nil) 107 } 108 if err != nil { 109 return nil, err 110 } 111 // Assemble the wrapper with all the registered metrics 112 ldb := &Database{ 113 fn: file, 114 db: db, 115 log: logger, 116 quitChan: make(chan chan error), 117 } 118 ldb.compTimeMeter = metrics.NewRegisteredMeter(namespace+"compact/time", nil) 119 ldb.compReadMeter = metrics.NewRegisteredMeter(namespace+"compact/input", nil) 120 ldb.compWriteMeter = metrics.NewRegisteredMeter(namespace+"compact/output", nil) 121 ldb.diskSizeGauge = metrics.NewRegisteredGauge(namespace+"disk/size", nil) 122 ldb.diskReadMeter = metrics.NewRegisteredMeter(namespace+"disk/read", nil) 123 ldb.diskWriteMeter = metrics.NewRegisteredMeter(namespace+"disk/write", nil) 124 ldb.writeDelayMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/duration", nil) 125 ldb.writeDelayNMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/counter", nil) 126 ldb.memCompGauge = metrics.NewRegisteredGauge(namespace+"compact/memory", nil) 127 ldb.level0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/level0", nil) 128 ldb.nonlevel0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/nonlevel0", nil) 129 ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil) 130 131 // Start up the metrics gathering and return 132 go ldb.meter(metricsGatheringInterval) 133 return ldb, nil 134 } 135 136 // Close stops the metrics collection, flushes any pending data to disk and closes 137 // all io accesses to the underlying key-value store. 138 func (db *Database) Close() error { 139 db.quitLock.Lock() 140 defer db.quitLock.Unlock() 141 142 if db.quitChan != nil { 143 errc := make(chan error) 144 db.quitChan <- errc 145 if err := <-errc; err != nil { 146 db.log.Error("Metrics collection failed", "err", err) 147 } 148 db.quitChan = nil 149 } 150 return db.db.Close() 151 } 152 153 // Has retrieves if a key is present in the key-value store. 154 func (db *Database) Has(key []byte) (bool, error) { 155 return db.db.Has(key, nil) 156 } 157 158 // Get retrieves the given key if it's present in the key-value store. 159 func (db *Database) Get(key []byte) ([]byte, error) { 160 dat, err := db.db.Get(key, nil) 161 if err != nil { 162 return nil, err 163 } 164 return dat, nil 165 } 166 167 // Put inserts the given value into the key-value store. 168 func (db *Database) Put(key []byte, value []byte) error { 169 return db.db.Put(key, value, nil) 170 } 171 172 // Delete removes the key from the key-value store. 173 func (db *Database) Delete(key []byte) error { 174 return db.db.Delete(key, nil) 175 } 176 177 // NewBatch creates a write-only key-value store that buffers changes to its host 178 // database until a final write is called. 179 func (db *Database) NewBatch() ethdb.Batch { 180 return &batch{ 181 db: db.db, 182 b: new(leveldb.Batch), 183 } 184 } 185 186 // NewIterator creates a binary-alphabetical iterator over a subset 187 // of database content with a particular key prefix, starting at a particular 188 // initial key (or after, if it does not exist). 189 func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { 190 return db.db.NewIterator(bytesPrefixRange(prefix, start), nil) 191 } 192 193 // Stat returns a particular internal stat of the database. 194 func (db *Database) Stat(property string) (string, error) { 195 return db.db.GetProperty(property) 196 } 197 198 // Compact flattens the underlying data store for the given key range. In essence, 199 // deleted and overwritten versions are discarded, and the data is rearranged to 200 // reduce the cost of operations needed to access them. 201 // 202 // A nil start is treated as a key before all keys in the data store; a nil limit 203 // is treated as a key after all keys in the data store. If both is nil then it 204 // will compact entire data store. 205 func (db *Database) Compact(start []byte, limit []byte) error { 206 return db.db.CompactRange(util.Range{Start: start, Limit: limit}) 207 } 208 209 // Path returns the path to the database directory. 210 func (db *Database) Path() string { 211 return db.fn 212 } 213 214 // meter periodically retrieves internal leveldb counters and reports them to 215 // the metrics subsystem. 216 // 217 // This is how a LevelDB stats table looks like (currently): 218 // Compactions 219 // Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) 220 // -------+------------+---------------+---------------+---------------+--------------- 221 // 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 222 // 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 223 // 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 224 // 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 225 // 226 // This is how the write delay look like (currently): 227 // DelayN:5 Delay:406.604657ms Paused: false 228 // 229 // This is how the iostats look like (currently): 230 // Read(MB):3895.04860 Write(MB):3654.64712 231 func (db *Database) meter(refresh time.Duration) { 232 // Create the counters to store current and previous compaction values 233 compactions := make([][]float64, 2) 234 for i := 0; i < 2; i++ { 235 compactions[i] = make([]float64, 4) 236 } 237 // Create storage for iostats. 238 var iostats [2]float64 239 240 // Create storage and warning log tracer for write delay. 241 var ( 242 delaystats [2]int64 243 lastWritePaused time.Time 244 ) 245 246 var ( 247 errc chan error 248 merr error 249 ) 250 251 // Iterate ad infinitum and collect the stats 252 for i := 1; errc == nil && merr == nil; i++ { 253 // Retrieve the database stats 254 stats, err := db.db.GetProperty("leveldb.stats") 255 if err != nil { 256 db.log.Error("Failed to read database stats", "err", err) 257 merr = err 258 continue 259 } 260 // Find the compaction table, skip the header 261 lines := strings.Split(stats, "\n") 262 for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { 263 lines = lines[1:] 264 } 265 if len(lines) <= 3 { 266 db.log.Error("Compaction leveldbTable not found") 267 merr = errors.New("compaction leveldbTable not found") 268 continue 269 } 270 lines = lines[3:] 271 272 // Iterate over all the leveldbTable rows, and accumulate the entries 273 for j := 0; j < len(compactions[i%2]); j++ { 274 compactions[i%2][j] = 0 275 } 276 for _, line := range lines { 277 parts := strings.Split(line, "|") 278 if len(parts) != 6 { 279 break 280 } 281 for idx, counter := range parts[2:] { 282 value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) 283 if err != nil { 284 db.log.Error("Compaction entry parsing failed", "err", err) 285 merr = err 286 continue 287 } 288 compactions[i%2][idx] += value 289 } 290 } 291 // Update all the requested meters 292 if db.diskSizeGauge != nil { 293 db.diskSizeGauge.Update(int64(compactions[i%2][0] * 1024 * 1024)) 294 } 295 if db.compTimeMeter != nil { 296 db.compTimeMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1000 * 1000 * 1000)) 297 } 298 if db.compReadMeter != nil { 299 db.compReadMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) 300 } 301 if db.compWriteMeter != nil { 302 db.compWriteMeter.Mark(int64((compactions[i%2][3] - compactions[(i-1)%2][3]) * 1024 * 1024)) 303 } 304 // Retrieve the write delay statistic 305 writedelay, err := db.db.GetProperty("leveldb.writedelay") 306 if err != nil { 307 db.log.Error("Failed to read database write delay statistic", "err", err) 308 merr = err 309 continue 310 } 311 var ( 312 delayN int64 313 delayDuration string 314 duration time.Duration 315 paused bool 316 ) 317 if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { 318 db.log.Error("Write delay statistic not found") 319 merr = err 320 continue 321 } 322 duration, err = time.ParseDuration(delayDuration) 323 if err != nil { 324 db.log.Error("Failed to parse delay duration", "err", err) 325 merr = err 326 continue 327 } 328 if db.writeDelayNMeter != nil { 329 db.writeDelayNMeter.Mark(delayN - delaystats[0]) 330 } 331 if db.writeDelayMeter != nil { 332 db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1]) 333 } 334 // If a warning that db is performing compaction has been displayed, any subsequent 335 // warnings will be withheld for one minute not to overwhelm the user. 336 if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && 337 time.Now().After(lastWritePaused.Add(degradationWarnInterval)) { 338 db.log.Warn("Database compacting, degraded performance") 339 lastWritePaused = time.Now() 340 } 341 delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() 342 343 // Retrieve the database iostats. 344 ioStats, err := db.db.GetProperty("leveldb.iostats") 345 if err != nil { 346 db.log.Error("Failed to read database iostats", "err", err) 347 merr = err 348 continue 349 } 350 var nRead, nWrite float64 351 parts := strings.Split(ioStats, " ") 352 if len(parts) < 2 { 353 db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) 354 merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) 355 continue 356 } 357 if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { 358 db.log.Error("Bad syntax of read entry", "entry", parts[0]) 359 merr = err 360 continue 361 } 362 if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { 363 db.log.Error("Bad syntax of write entry", "entry", parts[1]) 364 merr = err 365 continue 366 } 367 if db.diskReadMeter != nil { 368 db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) 369 } 370 if db.diskWriteMeter != nil { 371 db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) 372 } 373 iostats[0], iostats[1] = nRead, nWrite 374 375 compCount, err := db.db.GetProperty("leveldb.compcount") 376 if err != nil { 377 db.log.Error("Failed to read database iostats", "err", err) 378 merr = err 379 continue 380 } 381 382 var ( 383 memComp uint32 384 level0Comp uint32 385 nonLevel0Comp uint32 386 seekComp uint32 387 ) 388 if n, err := fmt.Sscanf(compCount, "MemComp:%d Level0Comp:%d NonLevel0Comp:%d SeekComp:%d", &memComp, &level0Comp, &nonLevel0Comp, &seekComp); n != 4 || err != nil { 389 db.log.Error("Compaction count statistic not found") 390 merr = err 391 continue 392 } 393 db.memCompGauge.Update(int64(memComp)) 394 db.level0CompGauge.Update(int64(level0Comp)) 395 db.nonlevel0CompGauge.Update(int64(nonLevel0Comp)) 396 db.seekCompGauge.Update(int64(seekComp)) 397 398 // Sleep a bit, then repeat the stats collection 399 select { 400 case errc = <-db.quitChan: 401 // Quit requesting, stop hammering the database 402 case <-time.After(refresh): 403 // Timeout, gather a new set of stats 404 } 405 } 406 407 if errc == nil { 408 errc = <-db.quitChan 409 } 410 errc <- merr 411 } 412 413 // batch is a write-only leveldb batch that commits changes to its host database 414 // when Write is called. A batch cannot be used concurrently. 415 type batch struct { 416 db *leveldb.DB 417 b *leveldb.Batch 418 size int 419 } 420 421 // Put inserts the given value into the batch for later committing. 422 func (b *batch) Put(key, value []byte) error { 423 b.b.Put(key, value) 424 b.size += len(value) 425 return nil 426 } 427 428 // Delete inserts the a key removal into the batch for later committing. 429 func (b *batch) Delete(key []byte) error { 430 b.b.Delete(key) 431 b.size++ 432 return nil 433 } 434 435 // ValueSize retrieves the amount of data queued up for writing. 436 func (b *batch) ValueSize() int { 437 return b.size 438 } 439 440 // Write flushes any accumulated data to disk. 441 func (b *batch) Write() error { 442 return b.db.Write(b.b, nil) 443 } 444 445 // Reset resets the batch for reuse. 446 func (b *batch) Reset() { 447 b.b.Reset() 448 b.size = 0 449 } 450 451 // Replay replays the batch contents. 452 func (b *batch) Replay(w ethdb.KeyValueWriter) error { 453 return b.b.Replay(&replayer{writer: w}) 454 } 455 456 // replayer is a small wrapper to implement the correct replay methods. 457 type replayer struct { 458 writer ethdb.KeyValueWriter 459 failure error 460 } 461 462 // Put inserts the given value into the key-value data store. 463 func (r *replayer) Put(key, value []byte) { 464 // If the replay already failed, stop executing ops 465 if r.failure != nil { 466 return 467 } 468 r.failure = r.writer.Put(key, value) 469 } 470 471 // Delete removes the key from the key-value data store. 472 func (r *replayer) Delete(key []byte) { 473 // If the replay already failed, stop executing ops 474 if r.failure != nil { 475 return 476 } 477 r.failure = r.writer.Delete(key) 478 } 479 480 // bytesPrefixRange returns key range that satisfy 481 // - the given prefix, and 482 // - the given seek position 483 func bytesPrefixRange(prefix, start []byte) *util.Range { 484 r := util.BytesPrefix(prefix) 485 r.Start = append(r.Start, start...) 486 return r 487 }