github.com/abschain-develop/go-abs@v2.0.3+incompatible/ethdb/database.go (about) 1 // Copyright 2014 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // +build !js 18 19 package ethdb 20 21 import ( 22 "fmt" 23 "strconv" 24 "strings" 25 "sync" 26 "time" 27 28 "github.com/abschain-develop/go-abs/log" 29 "github.com/abschain-develop/go-abs/metrics" 30 "github.com/syndtr/goleveldb/leveldb" 31 "github.com/syndtr/goleveldb/leveldb/errors" 32 "github.com/syndtr/goleveldb/leveldb/filter" 33 "github.com/syndtr/goleveldb/leveldb/iterator" 34 "github.com/syndtr/goleveldb/leveldb/opt" 35 "github.com/syndtr/goleveldb/leveldb/util" 36 ) 37 38 const ( 39 writePauseWarningThrottler = 1 * time.Minute 40 ) 41 42 var OpenFileLimit = 64 43 44 type LDBDatabase struct { 45 fn string // filename for reporting 46 db *leveldb.DB // LevelDB instance 47 48 compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction 49 compReadMeter metrics.Meter // Meter for measuring the data read during compaction 50 compWriteMeter metrics.Meter // Meter for measuring the data written during compaction 51 writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction 52 writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction 53 diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read 54 diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written 55 56 quitLock sync.Mutex // Mutex protecting the quit channel access 57 quitChan chan chan error // Quit channel to stop the metrics collection before closing the database 58 59 log log.Logger // Contextual logger tracking the database path 60 } 61 62 // NewLDBDatabase returns a LevelDB wrapped object. 63 func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { 64 logger := log.New("database", file) 65 66 // Ensure we have some minimal caching and file guarantees 67 if cache < 16 { 68 cache = 16 69 } 70 if handles < 16 { 71 handles = 16 72 } 73 logger.Info("Allocated cache and file handles", "cache", cache, "handles", handles) 74 75 // Open the db and recover any potential corruptions 76 db, err := leveldb.OpenFile(file, &opt.Options{ 77 OpenFilesCacheCapacity: handles, 78 BlockCacheCapacity: cache / 2 * opt.MiB, 79 WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally 80 Filter: filter.NewBloomFilter(10), 81 }) 82 if _, corrupted := err.(*errors.ErrCorrupted); corrupted { 83 db, err = leveldb.RecoverFile(file, nil) 84 } 85 // (Re)check for errors and abort if opening of the db failed 86 if err != nil { 87 return nil, err 88 } 89 return &LDBDatabase{ 90 fn: file, 91 db: db, 92 log: logger, 93 }, nil 94 } 95 96 // Path returns the path to the database directory. 97 func (db *LDBDatabase) Path() string { 98 return db.fn 99 } 100 101 // Put puts the given key / value to the queue 102 func (db *LDBDatabase) Put(key []byte, value []byte) error { 103 return db.db.Put(key, value, nil) 104 } 105 106 func (db *LDBDatabase) Has(key []byte) (bool, error) { 107 return db.db.Has(key, nil) 108 } 109 110 // Get returns the given key if it's present. 111 func (db *LDBDatabase) Get(key []byte) ([]byte, error) { 112 dat, err := db.db.Get(key, nil) 113 if err != nil { 114 return nil, err 115 } 116 return dat, nil 117 } 118 119 // Delete deletes the key from the queue and database 120 func (db *LDBDatabase) Delete(key []byte) error { 121 return db.db.Delete(key, nil) 122 } 123 124 func (db *LDBDatabase) NewIterator() iterator.Iterator { 125 return db.db.NewIterator(nil, nil) 126 } 127 128 // NewIteratorWithPrefix returns a iterator to iterate over subset of database content with a particular prefix. 129 func (db *LDBDatabase) NewIteratorWithPrefix(prefix []byte) iterator.Iterator { 130 return db.db.NewIterator(util.BytesPrefix(prefix), nil) 131 } 132 133 func (db *LDBDatabase) Close() { 134 // Stop the metrics collection to avoid internal database races 135 db.quitLock.Lock() 136 defer db.quitLock.Unlock() 137 138 if db.quitChan != nil { 139 errc := make(chan error) 140 db.quitChan <- errc 141 if err := <-errc; err != nil { 142 db.log.Error("Metrics collection failed", "err", err) 143 } 144 db.quitChan = nil 145 } 146 err := db.db.Close() 147 if err == nil { 148 db.log.Info("Database closed") 149 } else { 150 db.log.Error("Failed to close database", "err", err) 151 } 152 } 153 154 func (db *LDBDatabase) LDB() *leveldb.DB { 155 return db.db 156 } 157 158 // Meter configures the database metrics collectors and 159 func (db *LDBDatabase) Meter(prefix string) { 160 // Initialize all the metrics collector at the requested prefix 161 db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) 162 db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) 163 db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) 164 db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil) 165 db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil) 166 db.writeDelayMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/duration", nil) 167 db.writeDelayNMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/counter", nil) 168 169 // Create a quit channel for the periodic collector and run it 170 db.quitLock.Lock() 171 db.quitChan = make(chan chan error) 172 db.quitLock.Unlock() 173 174 go db.meter(3 * time.Second) 175 } 176 177 // meter periodically retrieves internal leveldb counters and reports them to 178 // the metrics subsystem. 179 // 180 // This is how a stats table look like (currently): 181 // Compactions 182 // Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) 183 // -------+------------+---------------+---------------+---------------+--------------- 184 // 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 185 // 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 186 // 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 187 // 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 188 // 189 // This is how the write delay look like (currently): 190 // DelayN:5 Delay:406.604657ms Paused: false 191 // 192 // This is how the iostats look like (currently): 193 // Read(MB):3895.04860 Write(MB):3654.64712 194 func (db *LDBDatabase) meter(refresh time.Duration) { 195 // Create the counters to store current and previous compaction values 196 compactions := make([][]float64, 2) 197 for i := 0; i < 2; i++ { 198 compactions[i] = make([]float64, 3) 199 } 200 // Create storage for iostats. 201 var iostats [2]float64 202 203 // Create storage and warning log tracer for write delay. 204 var ( 205 delaystats [2]int64 206 lastWritePaused time.Time 207 ) 208 209 var ( 210 errc chan error 211 merr error 212 ) 213 214 // Iterate ad infinitum and collect the stats 215 for i := 1; errc == nil && merr == nil; i++ { 216 // Retrieve the database stats 217 stats, err := db.db.GetProperty("leveldb.stats") 218 if err != nil { 219 db.log.Error("Failed to read database stats", "err", err) 220 merr = err 221 continue 222 } 223 // Find the compaction table, skip the header 224 lines := strings.Split(stats, "\n") 225 for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { 226 lines = lines[1:] 227 } 228 if len(lines) <= 3 { 229 db.log.Error("Compaction table not found") 230 merr = errors.New("compaction table not found") 231 continue 232 } 233 lines = lines[3:] 234 235 // Iterate over all the table rows, and accumulate the entries 236 for j := 0; j < len(compactions[i%2]); j++ { 237 compactions[i%2][j] = 0 238 } 239 for _, line := range lines { 240 parts := strings.Split(line, "|") 241 if len(parts) != 6 { 242 break 243 } 244 for idx, counter := range parts[3:] { 245 value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) 246 if err != nil { 247 db.log.Error("Compaction entry parsing failed", "err", err) 248 merr = err 249 continue 250 } 251 compactions[i%2][idx] += value 252 } 253 } 254 // Update all the requested meters 255 if db.compTimeMeter != nil { 256 db.compTimeMeter.Mark(int64((compactions[i%2][0] - compactions[(i-1)%2][0]) * 1000 * 1000 * 1000)) 257 } 258 if db.compReadMeter != nil { 259 db.compReadMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1024 * 1024)) 260 } 261 if db.compWriteMeter != nil { 262 db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) 263 } 264 265 // Retrieve the write delay statistic 266 writedelay, err := db.db.GetProperty("leveldb.writedelay") 267 if err != nil { 268 db.log.Error("Failed to read database write delay statistic", "err", err) 269 merr = err 270 continue 271 } 272 var ( 273 delayN int64 274 delayDuration string 275 duration time.Duration 276 paused bool 277 ) 278 if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { 279 db.log.Error("Write delay statistic not found") 280 merr = err 281 continue 282 } 283 duration, err = time.ParseDuration(delayDuration) 284 if err != nil { 285 db.log.Error("Failed to parse delay duration", "err", err) 286 merr = err 287 continue 288 } 289 if db.writeDelayNMeter != nil { 290 db.writeDelayNMeter.Mark(delayN - delaystats[0]) 291 } 292 if db.writeDelayMeter != nil { 293 db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1]) 294 } 295 // If a warning that db is performing compaction has been displayed, any subsequent 296 // warnings will be withheld for one minute not to overwhelm the user. 297 if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && 298 time.Now().After(lastWritePaused.Add(writePauseWarningThrottler)) { 299 db.log.Warn("Database compacting, degraded performance") 300 lastWritePaused = time.Now() 301 } 302 delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() 303 304 // Retrieve the database iostats. 305 ioStats, err := db.db.GetProperty("leveldb.iostats") 306 if err != nil { 307 db.log.Error("Failed to read database iostats", "err", err) 308 merr = err 309 continue 310 } 311 var nRead, nWrite float64 312 parts := strings.Split(ioStats, " ") 313 if len(parts) < 2 { 314 db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) 315 merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) 316 continue 317 } 318 if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { 319 db.log.Error("Bad syntax of read entry", "entry", parts[0]) 320 merr = err 321 continue 322 } 323 if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { 324 db.log.Error("Bad syntax of write entry", "entry", parts[1]) 325 merr = err 326 continue 327 } 328 if db.diskReadMeter != nil { 329 db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) 330 } 331 if db.diskWriteMeter != nil { 332 db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) 333 } 334 iostats[0], iostats[1] = nRead, nWrite 335 336 // Sleep a bit, then repeat the stats collection 337 select { 338 case errc = <-db.quitChan: 339 // Quit requesting, stop hammering the database 340 case <-time.After(refresh): 341 // Timeout, gather a new set of stats 342 } 343 } 344 345 if errc == nil { 346 errc = <-db.quitChan 347 } 348 errc <- merr 349 } 350 351 func (db *LDBDatabase) NewBatch() Batch { 352 return &ldbBatch{db: db.db, b: new(leveldb.Batch)} 353 } 354 355 type ldbBatch struct { 356 db *leveldb.DB 357 b *leveldb.Batch 358 size int 359 } 360 361 func (b *ldbBatch) Put(key, value []byte) error { 362 b.b.Put(key, value) 363 b.size += len(value) 364 return nil 365 } 366 367 func (b *ldbBatch) Delete(key []byte) error { 368 b.b.Delete(key) 369 b.size += 1 370 return nil 371 } 372 373 func (b *ldbBatch) Write() error { 374 return b.db.Write(b.b, nil) 375 } 376 377 func (b *ldbBatch) ValueSize() int { 378 return b.size 379 } 380 381 func (b *ldbBatch) Reset() { 382 b.b.Reset() 383 b.size = 0 384 }