github.com/ylsGit/go-ethereum@v1.6.5/ethdb/database.go (about) 1 // Copyright 2014 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package ethdb 18 19 import ( 20 "strconv" 21 "strings" 22 "sync" 23 "time" 24 25 "github.com/ethereum/go-ethereum/log" 26 "github.com/ethereum/go-ethereum/metrics" 27 "github.com/syndtr/goleveldb/leveldb" 28 "github.com/syndtr/goleveldb/leveldb/errors" 29 "github.com/syndtr/goleveldb/leveldb/filter" 30 "github.com/syndtr/goleveldb/leveldb/iterator" 31 "github.com/syndtr/goleveldb/leveldb/opt" 32 33 gometrics "github.com/rcrowley/go-metrics" 34 ) 35 36 var OpenFileLimit = 64 37 38 type LDBDatabase struct { 39 fn string // filename for reporting 40 db *leveldb.DB // LevelDB instance 41 42 getTimer gometrics.Timer // Timer for measuring the database get request counts and latencies 43 putTimer gometrics.Timer // Timer for measuring the database put request counts and latencies 44 delTimer gometrics.Timer // Timer for measuring the database delete request counts and latencies 45 missMeter gometrics.Meter // Meter for measuring the missed database get requests 46 readMeter gometrics.Meter // Meter for measuring the database get request data usage 47 writeMeter gometrics.Meter // Meter for measuring the database put request data usage 48 compTimeMeter gometrics.Meter // Meter for measuring the total time spent in database compaction 49 compReadMeter gometrics.Meter // Meter for measuring the data read during compaction 50 compWriteMeter gometrics.Meter // Meter for measuring the data written during compaction 51 52 quitLock sync.Mutex // Mutex protecting the quit channel access 53 quitChan chan chan error // Quit channel to stop the metrics collection before closing the database 54 55 log log.Logger // Contextual logger tracking the database path 56 } 57 58 // NewLDBDatabase returns a LevelDB wrapped object. 59 func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { 60 logger := log.New("database", file) 61 62 // Ensure we have some minimal caching and file guarantees 63 if cache < 16 { 64 cache = 16 65 } 66 if handles < 16 { 67 handles = 16 68 } 69 logger.Info("Allocated cache and file handles", "cache", cache, "handles", handles) 70 71 // Open the db and recover any potential corruptions 72 db, err := leveldb.OpenFile(file, &opt.Options{ 73 OpenFilesCacheCapacity: handles, 74 BlockCacheCapacity: cache / 2 * opt.MiB, 75 WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally 76 Filter: filter.NewBloomFilter(10), 77 }) 78 if _, corrupted := err.(*errors.ErrCorrupted); corrupted { 79 db, err = leveldb.RecoverFile(file, nil) 80 } 81 // (Re)check for errors and abort if opening of the db failed 82 if err != nil { 83 return nil, err 84 } 85 return &LDBDatabase{ 86 fn: file, 87 db: db, 88 log: logger, 89 }, nil 90 } 91 92 // Path returns the path to the database directory. 93 func (db *LDBDatabase) Path() string { 94 return db.fn 95 } 96 97 // Put puts the given key / value to the queue 98 func (db *LDBDatabase) Put(key []byte, value []byte) error { 99 // Measure the database put latency, if requested 100 if db.putTimer != nil { 101 defer db.putTimer.UpdateSince(time.Now()) 102 } 103 // Generate the data to write to disk, update the meter and write 104 //value = rle.Compress(value) 105 106 if db.writeMeter != nil { 107 db.writeMeter.Mark(int64(len(value))) 108 } 109 return db.db.Put(key, value, nil) 110 } 111 112 // Get returns the given key if it's present. 113 func (db *LDBDatabase) Get(key []byte) ([]byte, error) { 114 // Measure the database get latency, if requested 115 if db.getTimer != nil { 116 defer db.getTimer.UpdateSince(time.Now()) 117 } 118 // Retrieve the key and increment the miss counter if not found 119 dat, err := db.db.Get(key, nil) 120 if err != nil { 121 if db.missMeter != nil { 122 db.missMeter.Mark(1) 123 } 124 return nil, err 125 } 126 // Otherwise update the actually retrieved amount of data 127 if db.readMeter != nil { 128 db.readMeter.Mark(int64(len(dat))) 129 } 130 return dat, nil 131 //return rle.Decompress(dat) 132 } 133 134 // Delete deletes the key from the queue and database 135 func (db *LDBDatabase) Delete(key []byte) error { 136 // Measure the database delete latency, if requested 137 if db.delTimer != nil { 138 defer db.delTimer.UpdateSince(time.Now()) 139 } 140 // Execute the actual operation 141 return db.db.Delete(key, nil) 142 } 143 144 func (db *LDBDatabase) NewIterator() iterator.Iterator { 145 return db.db.NewIterator(nil, nil) 146 } 147 148 func (db *LDBDatabase) Close() { 149 // Stop the metrics collection to avoid internal database races 150 db.quitLock.Lock() 151 defer db.quitLock.Unlock() 152 153 if db.quitChan != nil { 154 errc := make(chan error) 155 db.quitChan <- errc 156 if err := <-errc; err != nil { 157 db.log.Error("Metrics collection failed", "err", err) 158 } 159 } 160 err := db.db.Close() 161 if err == nil { 162 db.log.Info("Database closed") 163 } else { 164 db.log.Error("Failed to close database", "err", err) 165 } 166 } 167 168 func (db *LDBDatabase) LDB() *leveldb.DB { 169 return db.db 170 } 171 172 // Meter configures the database metrics collectors and 173 func (db *LDBDatabase) Meter(prefix string) { 174 // Short circuit metering if the metrics system is disabled 175 if !metrics.Enabled { 176 return 177 } 178 // Initialize all the metrics collector at the requested prefix 179 db.getTimer = metrics.NewTimer(prefix + "user/gets") 180 db.putTimer = metrics.NewTimer(prefix + "user/puts") 181 db.delTimer = metrics.NewTimer(prefix + "user/dels") 182 db.missMeter = metrics.NewMeter(prefix + "user/misses") 183 db.readMeter = metrics.NewMeter(prefix + "user/reads") 184 db.writeMeter = metrics.NewMeter(prefix + "user/writes") 185 db.compTimeMeter = metrics.NewMeter(prefix + "compact/time") 186 db.compReadMeter = metrics.NewMeter(prefix + "compact/input") 187 db.compWriteMeter = metrics.NewMeter(prefix + "compact/output") 188 189 // Create a quit channel for the periodic collector and run it 190 db.quitLock.Lock() 191 db.quitChan = make(chan chan error) 192 db.quitLock.Unlock() 193 194 go db.meter(3 * time.Second) 195 } 196 197 // meter periodically retrieves internal leveldb counters and reports them to 198 // the metrics subsystem. 199 // 200 // This is how a stats table look like (currently): 201 // Compactions 202 // Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) 203 // -------+------------+---------------+---------------+---------------+--------------- 204 // 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 205 // 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 206 // 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 207 // 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 208 func (db *LDBDatabase) meter(refresh time.Duration) { 209 // Create the counters to store current and previous values 210 counters := make([][]float64, 2) 211 for i := 0; i < 2; i++ { 212 counters[i] = make([]float64, 3) 213 } 214 // Iterate ad infinitum and collect the stats 215 for i := 1; ; i++ { 216 // Retrieve the database stats 217 stats, err := db.db.GetProperty("leveldb.stats") 218 if err != nil { 219 db.log.Error("Failed to read database stats", "err", err) 220 return 221 } 222 // Find the compaction table, skip the header 223 lines := strings.Split(stats, "\n") 224 for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { 225 lines = lines[1:] 226 } 227 if len(lines) <= 3 { 228 db.log.Error("Compaction table not found") 229 return 230 } 231 lines = lines[3:] 232 233 // Iterate over all the table rows, and accumulate the entries 234 for j := 0; j < len(counters[i%2]); j++ { 235 counters[i%2][j] = 0 236 } 237 for _, line := range lines { 238 parts := strings.Split(line, "|") 239 if len(parts) != 6 { 240 break 241 } 242 for idx, counter := range parts[3:] { 243 value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) 244 if err != nil { 245 db.log.Error("Compaction entry parsing failed", "err", err) 246 return 247 } 248 counters[i%2][idx] += value 249 } 250 } 251 // Update all the requested meters 252 if db.compTimeMeter != nil { 253 db.compTimeMeter.Mark(int64((counters[i%2][0] - counters[(i-1)%2][0]) * 1000 * 1000 * 1000)) 254 } 255 if db.compReadMeter != nil { 256 db.compReadMeter.Mark(int64((counters[i%2][1] - counters[(i-1)%2][1]) * 1024 * 1024)) 257 } 258 if db.compWriteMeter != nil { 259 db.compWriteMeter.Mark(int64((counters[i%2][2] - counters[(i-1)%2][2]) * 1024 * 1024)) 260 } 261 // Sleep a bit, then repeat the stats collection 262 select { 263 case errc := <-db.quitChan: 264 // Quit requesting, stop hammering the database 265 errc <- nil 266 return 267 268 case <-time.After(refresh): 269 // Timeout, gather a new set of stats 270 } 271 } 272 } 273 274 // TODO: remove this stuff and expose leveldb directly 275 276 func (db *LDBDatabase) NewBatch() Batch { 277 return &ldbBatch{db: db.db, b: new(leveldb.Batch)} 278 } 279 280 type ldbBatch struct { 281 db *leveldb.DB 282 b *leveldb.Batch 283 } 284 285 func (b *ldbBatch) Put(key, value []byte) error { 286 b.b.Put(key, value) 287 return nil 288 } 289 290 func (b *ldbBatch) Write() error { 291 return b.db.Write(b.b, nil) 292 } 293 294 type table struct { 295 db Database 296 prefix string 297 } 298 299 // NewTable returns a Database object that prefixes all keys with a given 300 // string. 301 func NewTable(db Database, prefix string) Database { 302 return &table{ 303 db: db, 304 prefix: prefix, 305 } 306 } 307 308 func (dt *table) Put(key []byte, value []byte) error { 309 return dt.db.Put(append([]byte(dt.prefix), key...), value) 310 } 311 312 func (dt *table) Get(key []byte) ([]byte, error) { 313 return dt.db.Get(append([]byte(dt.prefix), key...)) 314 } 315 316 func (dt *table) Delete(key []byte) error { 317 return dt.db.Delete(append([]byte(dt.prefix), key...)) 318 } 319 320 func (dt *table) Close() { 321 // Do nothing; don't close the underlying DB. 322 } 323 324 type tableBatch struct { 325 batch Batch 326 prefix string 327 } 328 329 // NewTableBatch returns a Batch object which prefixes all keys with a given string. 330 func NewTableBatch(db Database, prefix string) Batch { 331 return &tableBatch{db.NewBatch(), prefix} 332 } 333 334 func (dt *table) NewBatch() Batch { 335 return &tableBatch{dt.db.NewBatch(), dt.prefix} 336 } 337 338 func (tb *tableBatch) Put(key, value []byte) error { 339 return tb.batch.Put(append([]byte(tb.prefix), key...), value) 340 } 341 342 func (tb *tableBatch) Write() error { 343 return tb.batch.Write() 344 }