github.com/waltonchain/waltonchain_gwtc_src@v1.1.4-0.20201225072101-8a298c95a819/wtcdb/database.go (about) 1 // Copyright 2014 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-wtc library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-wtc library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package wtcdb 18 19 import ( 20 "strconv" 21 "strings" 22 "sync" 23 "time" 24 25 "github.com/wtc/go-wtc/log" 26 "github.com/wtc/go-wtc/metrics" 27 "github.com/syndtr/goleveldb/leveldb" 28 "github.com/syndtr/goleveldb/leveldb/errors" 29 "github.com/syndtr/goleveldb/leveldb/filter" 30 "github.com/syndtr/goleveldb/leveldb/iterator" 31 "github.com/syndtr/goleveldb/leveldb/opt" 32 33 gometrics "github.com/rcrowley/go-metrics" 34 ) 35 36 var OpenFileLimit = 64 37 38 type LDBDatabase struct { 39 fn string // filename for reporting 40 db *leveldb.DB // LevelDB instance 41 42 getTimer gometrics.Timer // Timer for measuring the database get request counts and latencies 43 putTimer gometrics.Timer // Timer for measuring the database put request counts and latencies 44 delTimer gometrics.Timer // Timer for measuring the database delete request counts and latencies 45 missMeter gometrics.Meter // Meter for measuring the missed database get requests 46 readMeter gometrics.Meter // Meter for measuring the database get request data usage 47 writeMeter gometrics.Meter // Meter for measuring the database put request data usage 48 compTimeMeter gometrics.Meter // Meter for measuring the total time spent in database compaction 49 compReadMeter gometrics.Meter // Meter for measuring the data read during compaction 50 compWriteMeter gometrics.Meter // Meter for measuring the data written during compaction 51 52 quitLock sync.Mutex // Mutex protecting the quit channel access 53 quitChan chan chan error // Quit channel to stop the metrics collection before closing the database 54 55 log log.Logger // Contextual logger tracking the database path 56 } 57 58 // NewLDBDatabase returns a LevelDB wrapped object. 59 func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { 60 logger := log.New("database", file) 61 62 // Ensure we have some minimal caching and file guarantees 63 if cache < 16 { 64 cache = 16 65 } 66 if handles < 16 { 67 handles = 16 68 } 69 // logger.Info("Allocated cache and file handles", "cache", cache, "handles", handles) 70 71 // Open the db and recover any potential corruptions 72 db, err := leveldb.OpenFile(file, &opt.Options{ 73 OpenFilesCacheCapacity: handles, 74 BlockCacheCapacity: cache / 2 * opt.MiB, 75 WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally 76 Filter: filter.NewBloomFilter(10), 77 }) 78 if _, corrupted := err.(*errors.ErrCorrupted); corrupted { 79 db, err = leveldb.RecoverFile(file, nil) 80 } 81 // (Re)check for errors and abort if opening of the db failed 82 if err != nil { 83 return nil, err 84 } 85 return &LDBDatabase{ 86 fn: file, 87 db: db, 88 log: logger, 89 }, nil 90 } 91 92 // Path returns the path to the database directory. 93 func (db *LDBDatabase) Path() string { 94 return db.fn 95 } 96 97 // Put puts the given key / value to the queue 98 func (db *LDBDatabase) Put(key []byte, value []byte) error { 99 // Measure the database put latency, if requested 100 if db.putTimer != nil { 101 defer db.putTimer.UpdateSince(time.Now()) 102 } 103 // Generate the data to write to disk, update the meter and write 104 //value = rle.Compress(value) 105 106 if db.writeMeter != nil { 107 db.writeMeter.Mark(int64(len(value))) 108 } 109 return db.db.Put(key, value, nil) 110 } 111 112 func (db *LDBDatabase) Has(key []byte) (bool, error) { 113 return db.db.Has(key, nil) 114 } 115 116 // Get returns the given key if it's present. 117 func (db *LDBDatabase) Get(key []byte) ([]byte, error) { 118 // Measure the database get latency, if requested 119 if db.getTimer != nil { 120 defer db.getTimer.UpdateSince(time.Now()) 121 } 122 // Retrieve the key and increment the miss counter if not found 123 dat, err := db.db.Get(key, nil) 124 if err != nil { 125 if db.missMeter != nil { 126 db.missMeter.Mark(1) 127 } 128 return nil, err 129 } 130 // Otherwise update the actually retrieved amount of data 131 if db.readMeter != nil { 132 db.readMeter.Mark(int64(len(dat))) 133 } 134 return dat, nil 135 //return rle.Decompress(dat) 136 } 137 138 // Delete deletes the key from the queue and database 139 func (db *LDBDatabase) Delete(key []byte) error { 140 // Measure the database delete latency, if requested 141 if db.delTimer != nil { 142 defer db.delTimer.UpdateSince(time.Now()) 143 } 144 // Execute the actual operation 145 return db.db.Delete(key, nil) 146 } 147 148 func (db *LDBDatabase) NewIterator() iterator.Iterator { 149 return db.db.NewIterator(nil, nil) 150 } 151 152 func (db *LDBDatabase) Close() { 153 // Stop the metrics collection to avoid internal database races 154 db.quitLock.Lock() 155 defer db.quitLock.Unlock() 156 157 if db.quitChan != nil { 158 errc := make(chan error) 159 db.quitChan <- errc 160 if err := <-errc; err != nil { 161 db.log.Error("Metrics collection failed", "err", err) 162 } 163 } 164 err := db.db.Close() 165 if err == nil { 166 // db.log.Info("Database closed") 167 } else { 168 db.log.Error("Failed to close database", "err", err) 169 } 170 } 171 172 func (db *LDBDatabase) LDB() *leveldb.DB { 173 return db.db 174 } 175 176 // Meter configures the database metrics collectors and 177 func (db *LDBDatabase) Meter(prefix string) { 178 // Short circuit metering if the metrics system is disabled 179 if !metrics.Enabled { 180 return 181 } 182 // Initialize all the metrics collector at the requested prefix 183 db.getTimer = metrics.NewTimer(prefix + "user/gets") 184 db.putTimer = metrics.NewTimer(prefix + "user/puts") 185 db.delTimer = metrics.NewTimer(prefix + "user/dels") 186 db.missMeter = metrics.NewMeter(prefix + "user/misses") 187 db.readMeter = metrics.NewMeter(prefix + "user/reads") 188 db.writeMeter = metrics.NewMeter(prefix + "user/writes") 189 db.compTimeMeter = metrics.NewMeter(prefix + "compact/time") 190 db.compReadMeter = metrics.NewMeter(prefix + "compact/input") 191 db.compWriteMeter = metrics.NewMeter(prefix + "compact/output") 192 193 // Create a quit channel for the periodic collector and run it 194 db.quitLock.Lock() 195 db.quitChan = make(chan chan error) 196 db.quitLock.Unlock() 197 198 go db.meter(3 * time.Second) 199 } 200 201 // meter periodically retrieves internal leveldb counters and reports them to 202 // the metrics subsystem. 203 // 204 // This is how a stats table look like (currently): 205 // Compactions 206 // Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) 207 // -------+------------+---------------+---------------+---------------+--------------- 208 // 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 209 // 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 210 // 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 211 // 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 212 func (db *LDBDatabase) meter(refresh time.Duration) { 213 // Create the counters to store current and previous values 214 counters := make([][]float64, 2) 215 for i := 0; i < 2; i++ { 216 counters[i] = make([]float64, 3) 217 } 218 // Iterate ad infinitum and collect the stats 219 for i := 1; ; i++ { 220 // Retrieve the database stats 221 stats, err := db.db.GetProperty("leveldb.stats") 222 if err != nil { 223 db.log.Error("Failed to read database stats", "err", err) 224 return 225 } 226 // Find the compaction table, skip the header 227 lines := strings.Split(stats, "\n") 228 for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { 229 lines = lines[1:] 230 } 231 if len(lines) <= 3 { 232 db.log.Error("Compaction table not found") 233 return 234 } 235 lines = lines[3:] 236 237 // Iterate over all the table rows, and accumulate the entries 238 for j := 0; j < len(counters[i%2]); j++ { 239 counters[i%2][j] = 0 240 } 241 for _, line := range lines { 242 parts := strings.Split(line, "|") 243 if len(parts) != 6 { 244 break 245 } 246 for idx, counter := range parts[3:] { 247 value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) 248 if err != nil { 249 db.log.Error("Compaction entry parsing failed", "err", err) 250 return 251 } 252 counters[i%2][idx] += value 253 } 254 } 255 // Update all the requested meters 256 if db.compTimeMeter != nil { 257 db.compTimeMeter.Mark(int64((counters[i%2][0] - counters[(i-1)%2][0]) * 1000 * 1000 * 1000)) 258 } 259 if db.compReadMeter != nil { 260 db.compReadMeter.Mark(int64((counters[i%2][1] - counters[(i-1)%2][1]) * 1024 * 1024)) 261 } 262 if db.compWriteMeter != nil { 263 db.compWriteMeter.Mark(int64((counters[i%2][2] - counters[(i-1)%2][2]) * 1024 * 1024)) 264 } 265 // Sleep a bit, then repeat the stats collection 266 select { 267 case errc := <-db.quitChan: 268 // Quit requesting, stop hammering the database 269 errc <- nil 270 return 271 272 case <-time.After(refresh): 273 // Timeout, gather a new set of stats 274 } 275 } 276 } 277 278 func (db *LDBDatabase) NewBatch() Batch { 279 return &ldbBatch{db: db.db, b: new(leveldb.Batch)} 280 } 281 282 type ldbBatch struct { 283 db *leveldb.DB 284 b *leveldb.Batch 285 size int 286 } 287 288 func (b *ldbBatch) Put(key, value []byte) error { 289 b.b.Put(key, value) 290 b.size += len(value) 291 return nil 292 } 293 294 func (b *ldbBatch) Write() error { 295 return b.db.Write(b.b, nil) 296 } 297 298 func (b *ldbBatch) ValueSize() int { 299 return b.size 300 } 301 302 type table struct { 303 db Database 304 prefix string 305 } 306 307 // NewTable returns a Database object that prefixes all keys with a given 308 // string. 309 func NewTable(db Database, prefix string) Database { 310 return &table{ 311 db: db, 312 prefix: prefix, 313 } 314 } 315 316 func (dt *table) Put(key []byte, value []byte) error { 317 return dt.db.Put(append([]byte(dt.prefix), key...), value) 318 } 319 320 func (dt *table) Has(key []byte) (bool, error) { 321 return dt.db.Has(append([]byte(dt.prefix), key...)) 322 } 323 324 func (dt *table) Get(key []byte) ([]byte, error) { 325 return dt.db.Get(append([]byte(dt.prefix), key...)) 326 } 327 328 func (dt *table) Delete(key []byte) error { 329 return dt.db.Delete(append([]byte(dt.prefix), key...)) 330 } 331 332 func (dt *table) Close() { 333 // Do nothing; don't close the underlying DB. 334 } 335 336 type tableBatch struct { 337 batch Batch 338 prefix string 339 } 340 341 // NewTableBatch returns a Batch object which prefixes all keys with a given string. 342 func NewTableBatch(db Database, prefix string) Batch { 343 return &tableBatch{db.NewBatch(), prefix} 344 } 345 346 func (dt *table) NewBatch() Batch { 347 return &tableBatch{dt.db.NewBatch(), dt.prefix} 348 } 349 350 func (tb *tableBatch) Put(key, value []byte) error { 351 return tb.batch.Put(append([]byte(tb.prefix), key...), value) 352 } 353 354 func (tb *tableBatch) Write() error { 355 return tb.batch.Write() 356 } 357 358 func (tb *tableBatch) ValueSize() int { 359 return tb.batch.ValueSize() 360 }